Automatic matching improvements, additional of a maltego transform for finding … | |
Log | |
Files | |
Refs | |
README | |
--- | |
commit 3a47479fdb9dc463945b69e36fe5a1a71a3f4bc8 | |
parent 2af809ed64362553e2fd7b309fd8da3490351bb6 | |
Author: HD Moore <[email protected]> | |
Date: Sat, 14 Feb 2009 22:04:43 +0000 | |
Automatic matching improvements, additional of a maltego transform for finding … | |
Diffstat: | |
M bin/automatch.rb | 41 ++++++++++++++++++++---------… | |
A bin/link_maltego.rb | 159 +++++++++++++++++++++++++++++… | |
A docs/maltego/local_transform.png | 0 | |
A docs/maltego/sample_output.png | 0 | |
M lib/warvox/db.rb | 66 +++++++++++++++++++----------… | |
5 files changed, 226 insertions(+), 40 deletions(-) | |
--- | |
diff --git a/bin/automatch.rb b/bin/automatch.rb | |
@@ -16,13 +16,14 @@ require 'warvox' | |
# | |
def usage | |
- $stderr.puts "#{$0} [warvox.db] <db-threshold>" | |
+ $stderr.puts "#{$0} [warvox.db] <db-threshold> <fuzz>" | |
exit | |
end | |
threads = 2 | |
inp = ARGV.shift || usage | |
thresh = (ARGV.shift() || 800).to_i | |
+fuzz = (ARGV.shift() || 100).to_i | |
wdb = WarVOX::DB.new(inp, thresh) | |
# Scrub the carriers out of the pool first | |
@@ -33,45 +34,57 @@ end | |
groups = | |
{ | |
- "carriers" => car.keys | |
+ "carriers" => car.keys, | |
+ "unique" => [] | |
} | |
oset = wdb.keys.sort | |
iset = oset.dup | |
- | |
while(not oset.empty?) | |
+ s = Time.now | |
k = oset.shift | |
- found = [] | |
- best = nil | |
+ found = {} | |
next if not iset.include?(k) | |
iset.each do |n| | |
next if k == n | |
begin | |
- res = wdb.find_sig(k,n) | |
+ res = wdb.find_sig(k,n,{ :fuzz => fuzz }) | |
rescue ::WarVOX::DB::Error | |
end | |
next if not res | |
next if res[:len] < 5 | |
- found << res | |
+ if(not found[n] or found[n][:len] < res[:len]) | |
+ found[n] = res | |
+ end | |
+ end | |
+ | |
+ if(found.empty?) | |
+ next | |
end | |
- next if found.empty? | |
- | |
- groups[k] = [ ] | |
- found.each do |f| | |
- groups[k] << [ f[:num2], f[:len] ] | |
+ groups[k] = [ [k, 0] ] | |
+ found.keys.sort.each do |n| | |
+ groups[k] << [n, found[n][:len]] | |
end | |
- $stdout.puts "#{k} " + groups[k].map{|x| "#{x[0]}-#{x[1]}" }.join(" ") | |
+ $stdout.puts groups[k].map{|x| "#{x[0]}-#{x[1]}" }.join(" ") | |
$stdout.flush | |
groups[k].unshift(k) | |
-end | |
+ # Remove matches from the search listing | |
+ iset.delete(k) | |
+ found.keys.each do |k| | |
+ iset.delete(k) | |
+ end | |
+end | |
+iset.each do |k| | |
+ puts "#{k}-0" | |
+end | |
diff --git a/bin/link_maltego.rb b/bin/link_maltego.rb | |
@@ -0,0 +1,159 @@ | |
+#!/usr/bin/env ruby | |
+################### | |
+ | |
+# | |
+# Load the library path | |
+# | |
+base = __FILE__ | |
+while File.symlink?(base) | |
+ base = File.expand_path(File.readlink(base), File.dirname(base)) | |
+end | |
+$:.unshift(File.join(File.expand_path(File.dirname(base)), '..', 'lib')) | |
+require 'warvox' | |
+require 'rexml/document' | |
+ | |
+# | |
+# Script | |
+# | |
+ | |
+# | |
+# http://ctas.paterva.com/view/Specification | |
+# | |
+ | |
+def xml_results_empty | |
+ root = REXML::Element.new('MaltegoMessage') | |
+ xml2 = root.add_element('MaltegoTransformResponseMessage') | |
+ xml2.add_element('Entities') | |
+ root | |
+end | |
+ | |
+def xml_results_matches(res) | |
+ root = REXML::Element.new('MaltegoMessage') | |
+ xml2 = root.add_element('MaltegoTransformResponseMessage') | |
+ xml3 = xml2.add_element('Entities') | |
+ | |
+ res.each_key do |k| | |
+ num_area = k[0,3] | |
+ num_city = k[3,3] | |
+ num_last = k[6,4] | |
+ | |
+ num = num_area + " " + num_city + " " + num_last | |
+ | |
+ val = REXML::Element.new('Value') | |
+ val.add_text(num) | |
+ | |
+ adf = REXML::Element.new('AdditionalFields') | |
+ | |
+ adf_area = REXML::Element.new('Field') | |
+ adf_area.add_attribute('Name', 'areacode') | |
+ adf_area.add_text( REXML::Text.new( num_area.to_s ) ) | |
+ adf << adf_area | |
+ | |
+ adf_city = REXML::Element.new('Field') | |
+ adf_city.add_attribute('Name', 'citycode') | |
+ adf_city.add_text( REXML::Text.new( num_city.to_s ) ) | |
+ adf << adf_city | |
+ | |
+ adf_last = REXML::Element.new('Field') | |
+ adf_last.add_attribute('Name', 'lastnumbers') | |
+ adf_last.add_text( REXML::Text.new( num_last.to_s ) ) | |
+ adf << adf_last | |
+ | |
+ adf_info = REXML::Element.new('Field') | |
+ adf_info.add_attribute('Name', 'additional') | |
+ adf_info.add_text( REXML::Text.new( "Sig: " + res[k][:… | |
+ adf << adf_info | |
+ | |
+ | |
+ wgt = REXML::Element.new('Weight') | |
+ wgt.add_text( REXML::Text.new( [res[k][:len] * 10, 100].min.t… | |
+ | |
+ ent = REXML::Element.new('Entity') | |
+ ent.add_attribute('Type', 'PhoneNumber') | |
+ | |
+ ent << val | |
+ ent << wgt | |
+ ent << adf | |
+ | |
+ xml3 << ent | |
+ end | |
+ root | |
+end | |
+ | |
+ | |
+# Only report each percentage once | |
+@progress_done = {} | |
+ | |
+def report_progress(pct) | |
+ return if @progress_done[pct] | |
+ $stderr.puts "%#{pct}" | |
+ $stderr.flush | |
+ @progress_done[pct] = true | |
+end | |
+ | |
+def usage | |
+ $stderr.puts "#{$0} [target] [params]" | |
+ exit | |
+end | |
+ | |
+# | |
+# Parse input | |
+# | |
+ | |
+params = {} | |
+target = ARGV.shift || usage() | |
+(ARGV.shift || usage()).split('#').each do |param| | |
+ k,v = param.split('=', 2) | |
+ params[k] = v | |
+end | |
+ | |
+# XXX: Problematic right now | |
+# target_number = params['areacode'] + params['citycode'] + params['lastnumber… | |
+ | |
+target_number = target.scan(/\d+/).join | |
+if(target_number.length != 10) | |
+ $stderr.puts "D: Only 10 digit US numbers are currently supported" | |
+ $stdout.puts xml_results_empty().to_s | |
+ exit | |
+end | |
+ | |
+ | |
+# | |
+# Search database | |
+# | |
+ | |
+carriers = {} | |
+ | |
+data_root = File.join(File.dirname(base), '..', 'data') | |
+wdb = WarVOX::DB.new(nil) | |
+ | |
+Dir.new(data_root).entries.grep(/\.db$/).each do |db| | |
+ $stderr.puts "D: Loading #{db}..." | |
+ wdb.import(File.join(data_root, db)) | |
+end | |
+ | |
+# No matching number | |
+if(not wdb[target_number]) | |
+ $stderr.puts "D: Target #{target_number} (#{target}) is not in the War… | |
+ $stdout.puts xml_results_empty().to_s | |
+ exit | |
+end | |
+ | |
+found = {} | |
+cnt = 0 | |
+wdb.each_key do |n| | |
+ cnt += 1 | |
+ | |
+ report_progress(((cnt / wdb.keys.length.to_f) * 100).to_i.to_s) | |
+ | |
+ next if target_number == n | |
+ begin | |
+ res = wdb.find_sig(target_number, n, { :fuzz => 100 }) | |
+ rescue ::WarVOX::DB::Error | |
+ end | |
+ next if not res | |
+ next if res[:len] < 5 | |
+ found[n] = res | |
+end | |
+ | |
+$stdout.puts xml_results_matches(found).to_s | |
diff --git a/docs/maltego/local_transform.png b/docs/maltego/local_transform.png | |
Binary files differ. | |
diff --git a/docs/maltego/sample_output.png b/docs/maltego/sample_output.png | |
Binary files differ. | |
diff --git a/lib/warvox/db.rb b/lib/warvox/db.rb | |
@@ -12,7 +12,10 @@ class DB < ::Hash | |
self.path = path | |
self.threshold = threshold | |
self.version = VERSION | |
- | |
+ import(path) if path | |
+ end | |
+ | |
+ def import(path) | |
File.open(path, "r") do |fd| | |
fd.each_line do |line| | |
line.strip! | |
@@ -32,47 +35,36 @@ class DB < ::Hash | |
self[name] << [s, l.to_i, a.to_i] | |
end | |
end | |
- end | |
+ end | |
end | |
# | |
# Utility methods | |
# | |
- # Find the largest pattern shared between two samples | |
- def find_sig(num1, num2, opts={}) | |
- | |
- fuzz = opts[:fuzz] || 100 | |
- info1 = self[num1] | |
- info2 = self[num2] | |
- | |
- # Make sure both samples exist in the database | |
- if ( not (info1 and info2 and not (info1.empty? or info2.empty… | |
- raise Error, "The database must contain both numbers" | |
- end | |
- | |
- # Remove the silence prefix from both samples | |
- info1.shift if info1[0][0] == "L" | |
- info2.shift if info2[0][0] == "L" | |
+ # Find a signature within a sample | |
+ def find_match(pat, sam, opts={}) | |
+ | |
+ fuzz = opts[:fuzz] || 100 | |
+ min_sig = opts[:min_sig] || 2 | |
- min_sig = 2 | |
idx = 0 | |
fnd = nil | |
mat = nil | |
r = 0 | |
- while(idx < info1.length-min_sig) | |
- sig = info1[idx,info1.length] | |
+ while(idx < pat.length-min_sig) | |
+ sig = pat[idx,pat.length] | |
idx2 = 0 | |
- while (idx2 < info2.length) | |
+ while (idx2 < sam.length) | |
c = 0 | |
0.upto(sig.length-1) do |si| | |
- break if not info2[idx2+si] | |
+ break if not sam[idx2+si] | |
break if not ( | |
- sig[si][0] == info2[idx2+si][0… | |
- info2[idx2 + si][1] > sig[si][… | |
- info2[idx2 + si][1] < sig[si][… | |
+ sig[si][0] == sam[idx2+si][0] … | |
+ sam[idx2 + si][1] > sig[si][1]… | |
+ sam[idx2 + si][1] < sig[si][1]… | |
) | |
c += 1 | |
end | |
@@ -80,13 +72,35 @@ class DB < ::Hash | |
if (c > r) | |
r = c | |
fnd = sig[0, r] | |
- mat = info2[idx2, r] | |
+ mat = sam[idx2, r] | |
end | |
idx2 += 1 | |
end | |
idx += 1 | |
end | |
+ # Return the results | |
+ [fnd, mat, r] | |
+ end | |
+ | |
+ # Find the largest pattern shared between two samples | |
+ def find_sig(num1, num2, opts={}) | |
+ | |
+ fuzz = opts[:fuzz] || 100 | |
+ info1 = self[num1] | |
+ info2 = self[num2] | |
+ | |
+ # Make sure both samples exist in the database | |
+ if ( not (info1 and info2 and not (info1.empty? or info2.empty… | |
+ raise Error, "The database must contain both numbers" | |
+ end | |
+ | |
+ # Remove the silence prefix from both samples | |
+ info1.shift if info1[0][0] == "L" | |
+ info2.shift if info2[0][0] == "L" | |
+ | |
+ fnd,mat,r = find_match(info1, info2, { :fuzz => fuzz }) | |
+ | |
return nil if not fnd | |
sig = [] |