GopherProxy

	Note that the x64 segfault is fixed - warvox - VoIP based wardialing tool, fork…
	Log
	Files
	Refs
	README
	---
	commit e72bc147d3962ac7cbf11cf067115838cdc22c89
	parent b638c150a0aaba8a71813689e92dc00b4c9aaf95
	Author: HD Moore <[email protected]>
	Date: Fri, 6 Mar 2009 02:28:14 +0000

	Note that the x64 segfault is fixed


	Diffstat:
	M BUGS \| 5 ++---
	M etc/warvox.conf \| 7 +++++++
	M lib/warvox/config.rb \| 10 +++++++++-
	M lib/warvox/jobs/analysis.rb \| 89 ++++++++++++++++++++++-------…
	M lib/warvox/jobs/base.rb \| 5 +++--

	5 files changed, 86 insertions(+), 30 deletions(-)
	---
	diff --git a/BUGS b/BUGS
	@@ -1,7 +1,6 @@
	KNOWN BUGS

	- * The ruby-kissfft module segfaults on 64-bit Linux systems
	-
	- * Stopping WarVOX in mid-scan will leave a hung scan job
	+ * Stopping WarVOX in mid-scan will leave a hung scan job, the job will
	+ need to be removed and restarted.


	diff --git a/etc/warvox.conf b/etc/warvox.conf
	@@ -26,3 +26,10 @@ tools:
	lame: lame
	iaxrecord: %BASE%/bin/iaxrecord

	+#
	+# Concurrent processing jobs, change this to
	+# increase processing performance if your
	+# system has multiple CPU cores. Recommended
	+# value is twice your core count.
	+#
	+analysis_threads: 2
	diff --git a/lib/warvox/config.rb b/lib/warvox/config.rb
	@@ -38,7 +38,15 @@ module Config
	return nil if not info['data_path']
	File.expand_path(info['data_path'].gsub('%BASE%', WarVOX::Base…
	end
	-
	+
	+ def self.analysis_threads
	+ info = YAML.load_file(WarVOX::Conf)
	+ return 1 if not info
	+ return 1 if not info['analysis_threads']
	+ [ info['analysis_threads'].to_i, 1 ].max
	+ end
	+
	+
	# This method searches the PATH environment variable for
	# a fully qualified path to the supplied file name.
	# Stolen from Rex
	diff --git a/lib/warvox/jobs/analysis.rb b/lib/warvox/jobs/analysis.rb
	@@ -3,6 +3,9 @@ module Jobs
	class Analysis < Base

	require 'fileutils'
	+ require 'tempfile'
	+ require 'yaml'
	+ require 'open3'

	@@kissfft_loaded = false
	begin
	@@ -49,33 +52,76 @@ class Analysis < Base

	def start_processing
	todo = ::DialResult.find_all_by_dial_job_id(@name)
	+ jobs = []
	todo.each do \|r\|
	next if r.processed
	next if not r.completed
	next if r.busy
	- analyze_call(r)
	+ jobs << r
	end
	+
	+ max_threads = WarVOX::Config.analysis_threads
	+
	+ while(not jobs.empty?)
	+ threads = []
	+ output = []
	+ 1.upto(max_threads) do
	+ j = jobs.shift \|\| break
	+ output << j
	+ threads << Thread.new { run_analyze_call(j) }
	+ end
	+
	+ # Wait for the threads to complete
	+ threads.each {\|t\| t.join}
	+
	+ # Save the results to the database
	+ output.each {\|r\| db_save(r) if r.processed }
	+ end
	+ end
	+
	+ def run_analyze_call(r)
	+ $stderr.puts "DEBUG: Processing audio for #{r.number}..."
	+
	+ bin = File.join(WarVOX::Base, 'bin', 'analyze_result.rb')
	+ pfd = IO.popen("#{bin} '#{r.rawfile}'")
	+ out = YAML.load(pfd.read)
	+ pfd.close
	+
	+ return if not out
	+
	+ out.each_key do \|k\|
	+ setter = "#{k.to_s}="
	+ if(r.respond_to?(setter))
	+ r.send(setter, out[k])
	+ end
	+ end
	+
	+ r.processed_at = Time.now
	+ r.processed = true
	+ true
	end

	- def analyze_call(r)
	+ # Takes the raw file path as an argument, returns a hash
	+ def analyze_call(input)

	- return if not r.rawfile
	- return if not File.exist?(r.rawfile)
	+ return if not input
	+ return if not File.exist?(input)

	- bname = r.rawfile.gsub(/\..*/, '')
	- num = r.number
	+ bname = input.gsub(/\..*/, '')
	+ num = File.basename(bname)
	+ res = {}

	#
	# Create the signature database
	#
	- raw = WarVOX::Audio::Raw.from_file(r.rawfile)
	+ raw = WarVOX::Audio::Raw.from_file(input)
	flow = raw.to_flow
	fd = File.new("#{bname}.sig", "wb")
	fd.write "#{num} #{flow}\n"
	fd.close

	# Save the signature data
	- r.sig_data = flow
	+ res[:sig_data] = flow

	#
	# Create a raw decompressed file
	@@ -96,13 +142,13 @@ class Analysis < Base
	frefile = Tempfile.new("frefile")

	# Perform a DFT on the samples
	- res = KissFFT.fftr(8192, 8000, 1, raw.samples)
	+ fft = KissFFT.fftr(8192, 8000, 1, raw.samples)

	# Calculate the peak frequencies for the sample
	maxf = 0
	maxp = 0
	tones = {}
	- res.each do \|x\|
	+ fft.each do \|x\|
	rank = x.sort{\|a,b\| a[1].to_i <=> b[1].to_i }.reverse
	rank[0..10].each do \|t\|
	f = t[0].round
	@@ -119,12 +165,12 @@ class Analysis < Base
	end

	# Save the peak frequency
	- r.peak_freq = maxf
	+ res[:peak_freq] = maxf

	# Calculate average frequency and peaks over time
	avg = {}
	pks = []
	- res.each do \|slot\|
	+ fft.each do \|slot\|
	pks << slot.sort{\|a,b\| a[1] <=> b[1] }.reverse[0]
	slot.each do \|freq\|
	avg[ freq[0] ] \|\|= 0
	@@ -133,11 +179,11 @@ class Analysis < Base
	end

	# Save the peak frequencies over time
	- r.peak_freq_data = pks.map{\|f\| "#{f[0]}-#{f[1]}" }.join(" ")
	+ res[:peak_freq_data] = pks.map{\|f\| "#{f[0]}-#{f[1]}" }.join(" …

	# Generate the frequency file
	avg.keys.sort.each do \|k\|
	- avg[k] = avg[k] / res.length
	+ avg[k] = avg[k] / fft.length
	frefile.write("#{k} #{avg[k]}\n")
	end
	frefile.flush
	@@ -165,7 +211,7 @@ class Analysis < Base
	end

	# Look for the 1000hz voicemail BEEP
	- if(r.peak_freq > 990 and r.peak_freq < 1010)
	+ if(res[:peak_freq] > 990 and res[:peak_freq] < 1010)
	line_type = 'voicemail'
	break
	end
	@@ -203,7 +249,7 @@ class Analysis < Base
	end

	# Save the guessed line type
	- r.line_type = line_type
	+ res[:line_type] = line_type

	# Plot samples to a graph
	plotter = Tempfile.new("gnuplot")
	@@ -254,19 +300,14 @@ class Analysis < Base
	File.unlink(rawfile.path)
	rawfile.close

	- # Save the changes
	- r.processed = true
	- r.processed_at = Time.now
	- db_save(r)
	-
	clear_zombies()
	+
	+ res
	end
	end


	class CallAnalysis < Analysis
	-
	- require 'fileutils'

	@@kissfft_loaded = false
	begin
	@@ -282,7 +323,7 @@ class CallAnalysis < Analysis
	def initialize(result_id)
	@name = result_id
	if(not @@kissfft_loaded)
	- raise RuntimeError, "The KissFFT module is not availab…
	+ raise RuntimeError, "The KissFFT module is not availab…
	end
	end

	diff --git a/lib/warvox/jobs/base.rb b/lib/warvox/jobs/base.rb
	@@ -16,17 +16,18 @@ class Base
	end

	def db_save(obj)
	- max_tries = 10
	+ max_tries = 100
	cur_tries = 0
	begin
	obj.save
	rescue ::SQLite3::BusyException => e
	cur_tries += 1
	if(cur_tries > max_tries)
	+ $stderr.puts "ERROR: Database is still locked …
	raise e
	return
	end
	- Kernel.select(nil, nil, nil, 0.25)
	+ Kernel.select(nil, nil, nil, rand(10) * 0.25 )
	retry
	end
	end