Introduction
Introduction Statistics Contact Development Disclaimer Help
First round of updates to support new signatures - warvox - VoIP based wardiali…
Log
Files
Refs
README
---
commit 39e637f7729852d500d1037f0b2d6071a3a1f36c
parent 8b1275dfbbc99341bfbacb181a71a75799bf3dee
Author: HD Moore <[email protected]>
Date: Sat, 23 May 2009 07:41:08 +0000
First round of updates to support new signatures
Diffstat:
M bin/create_sig.rb | 26 +++++++++++++++-----------
M etc/sigs/01.default.rb | 98 +++++++++++++++++++----------…
M etc/sigs/99.default.rb | 4 ++--
M lib/warvox/audio/raw.rb | 103 +++++++++++++++++++++++++++++…
M lib/warvox/jobs/analysis.rb | 73 ++++++++++++++++++++++++-----…
M web/app/controllers/analyze_contro… | 2 +-
M web/app/controllers/dial_results_c… | 2 +-
A web/db/migrate/20090522202032_add_… | 9 +++++++++
M web/db/schema.rb | 3 ++-
9 files changed, 248 insertions(+), 72 deletions(-)
---
diff --git a/bin/create_sig.rb b/bin/create_sig.rb
@@ -10,24 +10,28 @@ while File.symlink?(base)
end
$:.unshift(File.join(File.expand_path(File.dirname(base)), '..', 'lib'))
require 'warvox'
+require 'yaml'
#
# Script
#
def usage
- $stderr.puts "#{$0} [warvox.db] [num1] [num2] <fuzz-factor> <db-thresh…
- exit
+ $stderr.puts "#{$0} [raw-file] <skip-count> <length-count>"
+ exit(1)
end
-inp = ARGV.shift() || usage
-num1 = ARGV.shift() || usage
-num2 = ARGV.shift() || usage
-fuzz = (ARGV.shift() || 100).to_i
-thresh = (ARGV.shift() || 800).to_i
+inp = ARGV.shift() || usage()
+skp = (ARGV.shift() || 0).to_i
+len = (ARGV.shift() || 0).to_i
-info1 = []
-info2 = []
+raw = WarVOX::Audio::Raw.from_file(inp)
+raw.samples = (raw.samples[skp, raw.samples.length]||[]) if skp > 0
+raw.samples = (raw.samples[0, len]||[]) if len > 0
-wdb = WarVOX::DB.new(inp, thresh)
-puts wdb.find_sig(num1, num2, { :fuzz => fuzz }).inspect
+if(raw.samples.length == 0)
+ $stderr.puts "Error: the sample length is too short to create a signat…
+ exit(1)
+end
+
+$stdout.puts raw.to_freq.inspect.gsub(/\s+/,'')
diff --git a/etc/sigs/01.default.rb b/etc/sigs/01.default.rb
@@ -3,13 +3,6 @@
#
#
-# Variables:
-# pks = peak frequencies
-# ppz = top 10 frequencies per sample
-# flow = flow signature
-#
-
-#
# These signatures are used first and catch the majority of common
# systems. If you want to force a different type of detection, add
# your signatures to a file starting with "00." and place it in
@@ -19,42 +12,66 @@
#
-# Look for silence by checking for any significant noise
+# Initialize some local variables out of data
+#
+freq = data[:freq]
+fcnt = data[:fcnt]
+maxf = data[:maxf]
+
+#
+# Look for silence by checking the frequency signature
#
-if(flow.split(/\s+/).grep(/^H,/).length == 0)
- line_type = 'silence'
+if(freq.map{|f| f.length}.inject(:+) == 0)
+ @line_type = 'silence'
break
end
+#
+# Look for silence by checking for a strong frequency in each sample
+#
+scnt = 0
+ecnt = 0
+freq.each do |fsec|
+ scnt += 1
+ if(fsec.length == 0)
+ ecnt += 1
+ next
+ end
+ sump = 0
+ fsec.map {|x| sump += x[1] }
+ savg = sump / fsec.length
+ ecnt += 1 if (savg < 100)
+end
+if(ecnt == scnt)
+ @line_type = 'silence'
+ break
+end
+
+# Store these into data for use later on
+data[:scnt] = scnt
+data[:ecnt] = ecnt
#
-# Summarize detection of a whole bunch of frequencies (used below)
+# Look for modems by detecting a 2100hz answer + 2250hz tone
#
-f_2250 = 0
-f_440 = f_350 = 0
-f_1625 = f_1660 = f_1825 = f_2100 = f_1100 = 0
-f_600 = f_1855 = 0
+if( (fcnt[2100] > 1.0 or fcnt[2230] > 1.0) and fcnt[2250] > 0.5)
+ @line_type = 'modem'
+ break
+end
-pkz.each do |fb|
- fb.each do |f|
- f_2250 += 0.1 if(f[0] > 2240 and f[0] < 2260)
- f_440 += 0.1 if(f[0] > 437 and f[0] < 444)
- f_350 += 0.1 if(f[0] > 345 and f[0] < 355)
- f_1625 += 0.1 if(f[0] > 1620 and f[0] < 1630)
- f_1660 += 0.1 if(f[0] > 1655 and f[0] < 1665)
- f_1825 += 0.1 if(f[0] > 1820 and f[0] < 1830)
- f_1855 += 0.1 if(f[0] > 1850 and f[0] < 1860)
- f_2100 += 0.1 if(f[0] > 2090 and f[0] < 2110)
- f_1100 += 0.1 if(f[0] > 1090 and f[0] < 1110)
- f_600 += 0.1 if(f[0] > 595 and f[0] < 605) …
- end
+#
+# Look for modems by detecting a peak frequency of 2250hz
+#
+if(fcnt[2100] > 1.0 and (maxf > 2245.0 and maxf < 2255.0))
+ @line_type = 'modem'
+ break
end
#
-# Look for modems by detecting a 2250hz tone
+# Look for modems by detecting a peak frequency of 3000hz
#
-if(f_2250 > 1.0)
- line_type = 'modem'
+if(fcnt[2100] > 1.0 and (maxf > 2995.0 and maxf < 3005.0))
+ @line_type = 'modem'
break
end
@@ -62,17 +79,22 @@ end
# Look for faxes by checking for a handful of tones (min two)
#
fax_sum = 0
-[ f_1625, f_1660, f_1825, f_2100, f_600, f_1855, f_1100].map{|x| fax_sum += [x…
+[
+ fcnt[1625], fcnt[1660], fcnt[1825], fcnt[2100],
+ fcnt[600], fcnt[1855], fcnt[1100], fcnt[2250],
+ fcnt[2230], fcnt[2220], fcnt[1800], fcnt[2095],
+ fcnt[2105]
+].map{|x| fax_sum += [x,1.0].min }
if(fax_sum >= 2.0)
- line_type = 'fax'
+ @line_type = 'fax'
break
end
#
# Dial tone detection (440hz + 350hz)
#
-if(f_440 > 1.0 and f_350 > 1.0)
- line_type = 'dialtone'
+if(fcnt[440] > 1.0 and fcnt[350] > 1.0)
+ @line_type = 'dialtone'
break
end
@@ -83,10 +105,8 @@ end
# this signature can fail. For non-US numbers, the beep
# is often a different frequency entirely.
#
-f_1000 = 0
-pks.each{|f| f_1000 += 1 if(f[0] > 990 and f[0] < 1010) }
-if(f_1000 > 0)
- line_type = 'voicemail'
+if(fcnt[1000] >= 1.0)
+ @line_type = 'voicemail'
break
end
diff --git a/etc/sigs/99.default.rb b/etc/sigs/99.default.rb
@@ -4,7 +4,7 @@
#
# Fall back to 'voice' if nothing else has been matched
-# This should be last signature file processed
+# This should be the last signature file processed
#
-line_type = 'voice'
+@line_type = 'voice'
diff --git a/lib/warvox/audio/raw.rb b/lib/warvox/audio/raw.rb
@@ -2,7 +2,13 @@ module WarVOX
module Audio
class Raw
-
+ @@kissfft_loaded = false
+ begin
+ require 'kissfft'
+ @@kissfft_loaded = true
+ rescue ::LoadError
+ end
+
require 'zlib'
##
@@ -131,6 +137,101 @@ class Raw
return sig
end
+ def to_freq(opts={})
+
+ if(not @@kissfft_loaded)
+ raise RuntimeError, "The KissFFT module is not availab…
+ end
+
+ freq_cnt = opts[:frequency_count] || 20
+
+ # Perform a DFT on the samples
+ ffts = KissFFT.fftr(8192, 8000, 1, self.samples)
+
+ self.class.fft_to_freq_sig(ffts, freq_cnt)
+ end
+
+ def self.fft_to_freq_sig(ffts, freq_cnt)
+ sig = []
+ ffts.each do |s|
+ res = []
+ maxp = 0
+ maxf = 0
+ s.each do |f|
+ if( f[1] > maxp )
+ maxf,maxp = f
+ end
+
+ if(maxf > 0 and f[1] < maxp and (maxf + 4.5 < …
+ res << [maxf, maxp]
+ maxf,maxp = [0,0]
+ end
+ end
+
+ sig << res.sort{ |a,b| # …
+ a[1] <=> b[1]
+ }.reverse[0,freq_cnt].sort { |a,b| # t…
+ a[0] <=> b[0] …
+ }.map {|a| [a[0].round, a[1].round ] } # …
+ end
+
+ sig
+ end
+
+ # Find pattern inside of sample
+ def self.compare_freq_sig(pat, zam, opts)
+
+ fuzz_f = opts[:fuzz_f] || 7
+ fuzz_p = opts[:fuzz_p] || 10
+ final = []
+
+ 0.upto(zam.length - 1) do |si|
+ res = []
+ sam = zam[si, zam.length]
+
+ 0.upto(pat.length - 1) do |pi|
+ diff = []
+ next if not pat[pi]
+ next if pat[pi].length == 0
+ pat[pi].each do |x|
+ next if not sam[pi]
+ next if sam[pi].length == 0
+ sam[pi].each do |y|
+ if(
+ (x[0] - fuzz_f) < y[0]…
+ (x[0] + fuzz_f) > y[0]…
+ (x[1] - fuzz_p) < y[1]…
+ (x[1] + fuzz_p) > y[1]
+ )
+ diff << x
+ break
+ end
+ end
+ end
+ res << diff
+ end
+ next if res.length == 0
+
+ prev = 0
+ rsum = 0
+ ridx = 0
+ res.each_index do |xi|
+ len = res[xi].length
+ if(xi == 0)
+ rsum += (len < 2) ? -40 : +20
+ else
+ rsum += 20 if(prev > 11 and len > 11)
+ rsum += len
+ end
+ prev = len
+ end
+
+ final << [ (rsum / res.length.to_f), res.map {|x| x.le…
+ end
+
+ final
+ end
+
end
end
end
diff --git a/lib/warvox/jobs/analysis.rb b/lib/warvox/jobs/analysis.rb
@@ -14,6 +14,24 @@ class Analysis < Base
rescue ::LoadError
end
+ class SignalProcessor
+ attr_accessor :line_type
+ attr_accessor :signatures
+ attr_accessor :data
+
+ def initialize
+ @signatures = []
+ @data = {}
+ end
+
+ def proc(str)
+ while(true);
+ eval(str)
+ break
+ end
+ end
+ end
+
def type
'analysis'
end
@@ -113,9 +131,11 @@ class Analysis < Base
#
# Create the signature database
- #
+ #
raw = WarVOX::Audio::Raw.from_file(input)
- flow = raw.to_flow
+ fft = KissFFT.fftr(8192, 8000, 1, raw.samples)
+ freq = WarVOX::Audio::Raw.fft_to_freq_sig(fft, 20)
+ flow = freq.inspect.gsub(/\s+/, '')
fd = File.new("#{bname}.sig", "wb")
fd.write "#{num} #{flow}\n"
fd.close
@@ -141,9 +161,6 @@ class Analysis < Base
# Data files for spectrum plotting
frefile = Tempfile.new("frefile")
- # Perform a DFT on the samples
- fft = KissFFT.fftr(8192, 8000, 1, raw.samples)
-
# Calculate the peak frequencies for the sample
maxf = 0
maxp = 0
@@ -174,9 +191,9 @@ class Analysis < Base
fft.each do |slot|
pks << slot.sort{|a,b| a[1] <=> b[1] }.reverse[0]
pkz << slot.sort{|a,b| a[1] <=> b[1] }.reverse[0..9]
- slot.each do |freq|
- avg[ freq[0] ] ||= 0
- avg[ freq[0] ] += freq[1]
+ slot.each do |f|
+ avg[ f[0] ] ||= 0
+ avg[ f[0] ] += f[1]
end
end
@@ -190,25 +207,49 @@ class Analysis < Base
end
frefile.flush
- # Make a guess as to what kind of phone number we found
- line_type = nil
+ # Count significant frequencies across the sample
+ fcnt = {}
+ 0.step(4000, 5) {|f| fcnt[f] = 0 }
+ pkz.each do |fb|
+ fb.each do |f|
+ fdx = ((f[0] / 5.0).round * 5.0).to_i
+ fcnt[fdx] += 0.1
+ end
+ end
+
+ #
+ # Signature processing
+ #
+
+ sproc = SignalProcessor.new
+ sproc.data =
+ {
+ :raw => raw,
+ :freq => freq,
+ :fcnt => fcnt,
+ :fft => fft,
+ :pks => pks,
+ :pkz => pkz,
+ :maxf => maxf,
+ :maxp => maxp
+ }
WarVOX::Config.signatures_load.each do |sigfile|
begin
str = File.read(sigfile, File.size(sigfile))
- while(true)
- eval(str, binding)
- break
- end
+ sproc.proc(str)
rescue ::Exception => e
$stderr.puts "DEBUG: Caught exception in #{sig…
end
- break if line_type
+ break if sproc.line_type
end
# Save the guessed line type
- res[:line_type] = line_type
+ res[:line_type] = sproc.line_type
+ # Save any matched signatures
+ res[:signatures] = sproc.signatures.map{|s| "#{s[0]}:#{s[1]}:#…
+
# Plot samples to a graph
plotter = Tempfile.new("gnuplot")
diff --git a/web/app/controllers/analyze_controller.rb b/web/app/controllers/an…
@@ -24,7 +24,7 @@ class AnalyzeController < ApplicationController
@g1 = Ezgraphix::Graphic.new(:c_type => 'col3d', :div_name => 'calls_p…
@g1.render_options(:caption => 'Detected Lines by Type', :y_name => 'L…
- ltypes = DialResult.find( :all, :select => 'DISTINCT line_type' ).map{…
+ ltypes = DialResult.find( :all, :select => 'DISTINCT line_type', :cond…
res_types = {}
ltypes.each do |k|
diff --git a/web/app/controllers/dial_results_controller.rb b/web/app/controlle…
@@ -50,7 +50,7 @@ class DialResultsController < ApplicationController
@g1 = Ezgraphix::Graphic.new(:c_type => 'col3d', :div_name => 'calls_p…
@g1.render_options(:caption => 'Detected Lines by Type', :y_name => 'L…
- ltypes = DialResult.find( :all, :select => 'DISTINCT line_type' ).map{…
+ ltypes = DialResult.find( :all, :select => 'DISTINCT line_type', :cond…
res_types = {}
ltypes.each do |k|
diff --git a/web/db/migrate/20090522202032_add_signatures_to_dial_results.rb b/…
@@ -0,0 +1,9 @@
+class AddSignaturesToDialResults < ActiveRecord::Migration
+ def self.up
+ add_column :dial_results, :signatures, :string
+ end
+
+ def self.down
+ remove_column :dial_results, :signatures
+ end
+end
diff --git a/web/db/schema.rb b/web/db/schema.rb
@@ -9,7 +9,7 @@
#
# It's strongly recommended to check this file into your version control syste…
-ActiveRecord::Schema.define(:version => 20090304014033) do
+ActiveRecord::Schema.define(:version => 20090522202032) do
create_table "dial_jobs", :force => true do |t|
t.string "range"
@@ -44,6 +44,7 @@ ActiveRecord::Schema.define(:version => 20090304014033) do
t.string "sig_data"
t.string "line_type"
t.string "notes"
+ t.string "signatures"
end
create_table "providers", :force => true do |t|
You are viewing proxied material from jay.scot. The copyright of proxied material belongs to its original authors. Any comments or complaints in relation to proxied material should be directed to the original authors of the content concerned. Please see the disclaimer for more details.