2020-02-09 04:59:32 +08:00
#!/usr/bin/env ruby
# -*- coding: binary -*-
#
# Check for data scripts to ensure they are up to date
#
# by h00die
#
require 'digest'
require 'open-uri'
require 'optparse'
2020-05-15 04:04:19 +08:00
require 'tempfile'
2020-02-09 04:59:32 +08:00
options = { }
optparse = OptionParser . new do | opts |
2020-08-06 00:48:55 +08:00
opts . banner = 'Usage: check_external_scripts.rb [options]'
2020-02-09 04:59:32 +08:00
opts . on ( '-u' , '--update' , 'Overwrite old scripts with newer ones.' ) do
options [ :update ] = true
end
opts . on ( '-h' , '--help' , 'Display this screen.' ) do
puts opts
exit
end
end
optparse . parse!
# colors and puts templates from msftidy.rb
class String
def red
" \e [1;31;40m #{ self } \e [0m "
end
def yellow
" \e [1;33;40m #{ self } \e [0m "
end
def green
" \e [1;32;40m #{ self } \e [0m "
end
def cyan
" \e [1;36;40m #{ self } \e [0m "
end
end
#
# Display an error message, given some text
#
def error ( txt )
puts " [ #{ 'ERROR' . red } ] #{ cleanup_text ( txt ) } "
end
#
# Display a warning message, given some text
#
2020-05-15 04:04:19 +08:00
def warning ( txt )
2020-02-09 04:59:32 +08:00
puts " [ #{ 'WARNING' . yellow } ] #{ cleanup_text ( txt ) } "
end
#
# Display a info message, given some text
#
def info ( txt )
puts " [ #{ 'INFO' . cyan } ] #{ cleanup_text ( txt ) } "
end
def cleanup_text ( txt )
# remove line breaks
txt = txt . gsub ( / [ \ r \ n] / , ' ' )
# replace multiple spaces by one space
txt . gsub ( / \ s{2,} / , ' ' )
end
2020-08-06 00:48:55 +08:00
def cleanup_sqlmap_decloak_dir
2020-08-06 00:10:39 +08:00
unless system ( 'rm -rf /tmp/sqlmap_decloak' )
2020-08-06 00:48:55 +08:00
error 'Could not remove existing /tmp/sqlmap_decloak directory'
2020-08-06 00:10:39 +08:00
end
2020-08-06 00:40:49 +08:00
end
2020-08-06 00:48:55 +08:00
def clone_sqlmap_decloak
2020-08-06 00:40:49 +08:00
cleanup_sqlmap_decloak_dir
2020-08-06 00:48:55 +08:00
unless system ( 'git clone -q --depth=1 https://github.com/sqlmapproject/sqlmap.git /tmp/sqlmap_decloak' )
2020-08-06 00:10:39 +08:00
error " Either 'git' is not installed, your internet is not connected, or /tmp/sqlmap_decloak could not be removed. "
end
2020-08-06 00:40:49 +08:00
end
# https://github.com/rapid7/metasploit-framework/pull/13191#issuecomment-626584689
def decloak ( file )
2020-08-06 00:10:39 +08:00
unless system ( " python /tmp/sqlmap_decloak/extra/cloak/cloak.py -d -i #{ file . path } -o #{ file . path } _decloak " )
unless system ( " python3 /tmp/sqlmap_decloak/extra/cloak/cloak.py -d -i #{ file . path } -o #{ file . path } _decloak " )
error " Either python is not installed, or the file at #{ file . path } could not be found! Please double check your computer's setup and check that the #{ file . path } file exists! "
end
end
2020-08-06 00:40:49 +08:00
File . binread ( " #{ file . path } _decloak " )
2020-05-15 04:04:19 +08:00
end
2020-02-09 04:59:32 +08:00
#
#
# Main
#
#
2020-11-22 00:52:18 +08:00
scripts = [ ]
2020-08-04 21:06:45 +08:00
###
# Bloodhound/Sharphound files
###
2020-11-22 00:52:18 +08:00
# https://github.com/BloodHoundAD/BloodHound/commit/b6ab5cd369c70219c6376d9f5c4fcd63f34fb4a0
2020-02-09 04:59:32 +08:00
scripts << {
2020-08-04 21:06:45 +08:00
name : 'Sharphound (Bloodhound) ps1' ,
2020-11-22 00:52:18 +08:00
addr : 'https://raw.githubusercontent.com/BloodHoundAD/BloodHound/master/Collectors/SharpHound.ps1' ,
2020-04-03 10:32:10 +08:00
dest : '/data/post/powershell/SharpHound.ps1' ,
subs : [
2020-05-15 04:04:19 +08:00
[ " \t " , ' ' ] , # tabs to spaces
[ / \ s+$ / , '' ] # trailing whitespace
2020-04-03 10:32:10 +08:00
]
}
2020-08-04 21:06:45 +08:00
scripts << {
name : 'Sharphound (Bloodhound) exe' ,
2020-11-22 00:52:18 +08:00
addr : 'https://raw.githubusercontent.com/BloodHoundAD/BloodHound/master/Collectors/SharpHound.exe' ,
2020-08-04 21:06:45 +08:00
dest : '/data/post/SharpHound.exe' ,
2020-08-06 00:10:39 +08:00
subs : [ ]
2020-08-04 21:06:45 +08:00
}
2020-04-03 23:05:48 +08:00
###
# JTR files
###
scripts << {
name : 'JTR - dumb16.conf' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/dumb16.conf' ,
dest : '/data/jtr/dumb16.conf' ,
subs : [ ]
}
scripts << {
name : 'JTR - alnumspace.chr' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/alnumspace.chr' ,
dest : '/data/jtr/alnumspace.chr' ,
subs : [ ]
}
scripts << {
name : 'JTR - regex_alphabets.conf' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/regex_alphabets.conf' ,
dest : '/data/jtr/regex_alphabets.conf' ,
subs : [ ]
}
scripts << {
name : 'JTR - latin1.chr' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/latin1.chr' ,
dest : '/data/jtr/latin1.chr' ,
subs : [ ]
}
scripts << {
name : 'JTR - lowerspace.chr' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/lowerspace.chr' ,
dest : '/data/jtr/lowerspace.chr' ,
subs : [ ]
}
scripts << {
name : 'JTR - utf8.chr' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/utf8.chr' ,
dest : '/data/jtr/utf8.chr' ,
subs : [ ]
}
scripts << {
name : 'JTR - john.conf' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/john.conf' ,
dest : '/data/jtr/john.conf' ,
subs : [ ]
}
scripts << {
name : 'JTR - dumb32.conf' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/dumb32.conf' ,
dest : '/data/jtr/dumb32.conf' ,
subs : [ ]
}
scripts << {
name : 'JTR - alpha.chr' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/alpha.chr' ,
dest : '/data/jtr/alpha.chr' ,
subs : [ ]
}
scripts << {
name : 'JTR - dynamic.conf' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/dynamic.conf' ,
dest : '/data/jtr/dynamic.conf' ,
subs : [ ]
}
scripts << {
name : 'JTR - repeats32.conf' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/repeats32.conf' ,
dest : '/data/jtr/repeats32.conf' ,
subs : [ ]
}
scripts << {
name : 'JTR - lm_ascii.chr' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/lm_ascii.chr' ,
dest : '/data/jtr/lm_ascii.chr' ,
subs : [ ]
}
scripts << {
name : 'JTR - upper.chr' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/upper.chr' ,
dest : '/data/jtr/upper.chr' ,
subs : [ ]
}
scripts << {
name : 'JTR - lowernum.chr' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/lowernum.chr' ,
dest : '/data/jtr/lowernum.chr' ,
subs : [ ]
}
scripts << {
name : 'JTR - ascii.chr' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/ascii.chr' ,
dest : '/data/jtr/ascii.chr' ,
subs : [ ]
}
scripts << {
name : 'JTR - dynamic_disabled.conf' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/dynamic_disabled.conf' ,
dest : '/data/jtr/dynamic_disabled.conf' ,
subs : [ ]
}
scripts << {
name : 'JTR - hybrid.conf' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/hybrid.conf' ,
dest : '/data/jtr/hybrid.conf' ,
subs : [ ]
}
scripts << {
name : 'JTR - repeats16.conf' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/repeats16.conf' ,
dest : '/data/jtr/repeats16.conf' ,
subs : [ ]
}
scripts << {
name : 'JTR - digits.chr' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/digits.chr' ,
dest : '/data/jtr/digits.chr' ,
subs : [ ]
}
scripts << {
name : 'JTR - uppernum.chr' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/uppernum.chr' ,
dest : '/data/jtr/uppernum.chr' ,
subs : [ ]
}
scripts << {
name : 'JTR - lanman.chr' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/lanman.chr' ,
dest : '/data/jtr/lanman.chr' ,
subs : [ ]
}
scripts << {
name : 'JTR - dynamic_flat_sse_formats.conf' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/dynamic_flat_sse_formats.conf' ,
dest : '/data/jtr/dynamic_flat_sse_formats.conf' ,
subs : [ ]
}
scripts << {
name : 'JTR - alnum.chr' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/alnum.chr' ,
dest : '/data/jtr/alnum.chr' ,
subs : [ ]
}
scripts << {
name : 'JTR - lower.chr' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/lower.chr' ,
dest : '/data/jtr/lower.chr' ,
subs : [ ]
}
scripts << {
name : 'JTR - korelogic.conf' ,
addr : 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/korelogic.conf' ,
dest : '/data/jtr/korelogic.conf' ,
subs : [ ]
}
2020-08-06 00:48:55 +08:00
2020-04-03 23:05:48 +08:00
###
# SQLMap UDF files
###
scripts << {
name : 'SQLMap UDF - lib_mysqludf_sys_32.so' ,
addr : 'https://github.com/sqlmapproject/sqlmap/blob/master/data/udf/mysql/linux/32/lib_mysqludf_sys.so_?raw=true' ,
dest : '/data/exploits/mysql/lib_mysqludf_sys_32.so' ,
subs : [ ]
}
scripts << {
name : 'SQLMap UDF - lib_mysqludf_sys_64.so' ,
addr : 'https://github.com/sqlmapproject/sqlmap/blob/master/data/udf/mysql/linux/64/lib_mysqludf_sys.so_?raw=true' ,
dest : '/data/exploits/mysql/lib_mysqludf_sys_64.so' ,
subs : [ ]
}
scripts << {
name : 'SQLMap UDF - lib_mysqludf_sys_32.dll' ,
addr : 'https://github.com/sqlmapproject/sqlmap/blob/master/data/udf/mysql/windows/32/lib_mysqludf_sys.dll_?raw=true' ,
dest : '/data/exploits/mysql/lib_mysqludf_sys_32.dll' ,
subs : [ ]
}
scripts << {
name : 'SQLMap UDF - lib_mysqludf_sys_64.dll' ,
addr : 'https://github.com/sqlmapproject/sqlmap/blob/master/data/udf/mysql/windows/64/lib_mysqludf_sys.dll_?raw=true' ,
dest : '/data/exploits/mysql/lib_mysqludf_sys_64.dll' ,
subs : [ ]
}
2020-02-09 04:59:32 +08:00
2020-11-22 00:52:18 +08:00
###
# CMS Files
###
# https://github.com/rapid7/metasploit-framework/pull/11862#issuecomment-496578367
scripts << {
name : 'WordPress - Plugins List' ,
addr : 'https://plugins.svn.wordpress.org' ,
dest : '/data/wordlists/wp-plugins.txt' ,
subs : [
[ / ^((?! <li>).)* / , '' ] , # remove all non-plugin lines
[ / <li><a href="[^"]+"> / , '' ] , # remove beginning
[ / \/ < \/ a>< \/ li> / , '' ] , # remove end
[ / ^ \ s* / , '' ] # remove empty lines
]
}
scripts << {
name : 'WordPress - Themes List' ,
addr : 'https://themes.svn.wordpress.org' ,
dest : '/data/wordlists/wp-themes.txt' ,
subs : [
[ / ^((?! <li>).)* / , '' ] , # remove all non-plugin lines
[ / <li><a href="[^"]+"> / , '' ] , # remove beginning
[ / \/ < \/ a>< \/ li> / , '' ] , # remove end
[ / ^ \ s* / , '' ] # remove empty lines
]
}
2020-11-22 00:56:33 +08:00
# Joomla's is more complicated. It looks for more than
# just components. Because of that, if you want the
# file updated, see:
# https://github.com/rapid7/metasploit-framework/pull/11199#issue-242415518
2020-11-22 01:43:30 +08:00
# python3 tools/dev/update_joomla_components.py
2020-11-22 00:52:18 +08:00
2020-02-09 04:59:32 +08:00
path = File . expand_path ( '../../' , File . dirname ( __FILE__ ) )
2020-08-06 00:40:49 +08:00
clone_sqlmap_decloak
2020-02-09 04:59:32 +08:00
scripts . each do | script |
2020-04-03 10:32:10 +08:00
puts " Downloading: #{ script [ :name ] } "
2020-02-09 04:59:32 +08:00
begin
2020-04-03 10:32:10 +08:00
old_content = File . binread ( path + script [ :dest ] )
old_hash = Digest :: SHA1 . hexdigest old_content
2020-02-09 04:59:32 +08:00
info " Old Hash: #{ old_hash } "
2020-04-03 10:32:10 +08:00
2020-05-15 04:04:19 +08:00
new_content = URI . open ( script [ :addr ] ) . read
if script . key? ( :subs )
2020-04-03 10:32:10 +08:00
script [ :subs ] . each do | sub |
new_content . gsub! ( sub [ 0 ] , sub [ 1 ] )
end
end
2020-05-15 04:04:19 +08:00
if script [ :name ] . start_with? ( 'SQLMap UDF' )
2020-08-06 00:10:39 +08:00
info ( 'Performing decloaking' )
2020-05-15 04:04:19 +08:00
f = Tempfile . new ( 'sqlmap_udf' )
f . write ( new_content )
f . close
new_content = decloak ( f )
end
2020-04-03 10:32:10 +08:00
new_hash = Digest :: SHA1 . hexdigest new_content
2020-02-09 04:59:32 +08:00
info " New Hash: #{ new_hash } "
2020-04-03 10:32:10 +08:00
2020-02-09 04:59:32 +08:00
unless old_hash == new_hash
2020-05-15 04:04:19 +08:00
warning ' New version identified!'
2020-02-09 04:59:32 +08:00
if options [ :update ] == true
2020-05-15 04:04:19 +08:00
warning " Updating MSF copy of #{ script [ :dest ] } "
2020-04-03 10:32:10 +08:00
File . binwrite ( path + script [ :dest ] , new_content )
2020-02-09 04:59:32 +08:00
end
end
rescue OpenURI :: HTTPError
2020-04-03 10:32:10 +08:00
error " Unable to download, check URL: #{ script [ :addr ] } "
2020-02-09 04:59:32 +08:00
rescue Errno :: ENOENT
2020-04-03 10:32:10 +08:00
error " Destination not found, check path: #{ path + script [ :dest ] } "
2020-02-09 04:59:32 +08:00
end
end
2020-08-06 00:40:49 +08:00
2020-08-06 00:48:55 +08:00
cleanup_sqlmap_decloak_dir