From 0bb34c94beb67abf0ced602d9701d5d0d3ed82c4 Mon Sep 17 00:00:00 2001 From: et <> Date: Sun, 21 Mar 2010 03:23:27 +0000 Subject: [PATCH] Excluding extensions git-svn-id: file:///home/svn/framework3/trunk@8863 4d416f70-5f16-0410-b530-b9f4589650da --- tools/msfcrawler.rb | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tools/msfcrawler.rb b/tools/msfcrawler.rb index 3ad71237d2..fa45422a05 100755 --- a/tools/msfcrawler.rb +++ b/tools/msfcrawler.rb @@ -47,6 +47,9 @@ $dbs = false # Thread number $threadnum = 1 +# Dont crawl +$dontcrawl = ".exe,.zip,.tar,.bz2,.run,.asc,.gz," + # Use proxy $useproxy = false @@ -151,6 +154,11 @@ class HttpCrawler if !@ViewedQueue.include?(hashsig(hashreq)) @ViewedQueue[hashsig(hashreq)] = Time.now + if !File.extname(hashreq['uri']).empty? and $dontcrawl.include? File.extname(hashreq['uri']) + puts "URI not crawled #{hashreq['uri']}" + else + + #if i < $threadnum # a.push(Thread.new { @@ -178,7 +186,8 @@ class HttpCrawler #else # sleep(0.01) and a.delete_if {|x| not x.alive?} while not a.empty? # i = 0 - #end + #end + end else #puts "#{hashreq} already visited at #{@ViewedQueue[hashsig(hashreq)]}" end @@ -242,9 +251,9 @@ class HttpCrawler # resp.transfer_chunked = false if resp['Set-Cookie'] - #puts "SET COOKIE: #{resp['Set-Cookie']}" + #puts "Set Cookie: #{resp['Set-Cookie']}" #puts "Storing in cookie jar for host:port #{reqopts['rhost']}:#{reqopts['rport']}" - $cookiejar["#{reqopts['rhost']}:#{reqopts['rport']}"] = resp['Set-Cookie'] + #$cookiejar["#{reqopts['rhost']}:#{reqopts['rport']}"] = resp['Set-Cookie'] end #puts ("#{resp.to_s}") @@ -272,8 +281,10 @@ class HttpCrawler puts "No response" end sleep($sleeptime) - rescue ::Rex::ConnectionRefused, ::Rex::HostUnreachable, ::Rex::ConnectionTimeout - rescue ::Timeout::Error, ::Errno::EPIPE + #rescue ::Rex::ConnectionRefused, ::Rex::HostUnreachable, ::Rex::ConnectionTimeout + #rescue ::Timeout::Error, ::Errno::EPIPE + rescue + "ERROR" end end