2021-11-17 23:59:17 +08:00
require 'fileutils'
require 'uri'
require 'open3'
require 'optparse'
2022-04-06 20:12:18 +08:00
require_relative './navigation'
2021-11-17 23:59:17 +08:00
2022-02-27 03:19:54 +08:00
# Temporary build module to help migrate and build the Metasploit wiki https://github.com/rapid7/metasploit-framework/wiki into a format
2021-11-17 23:59:17 +08:00
# supported by Jekyll, as well as creating a hierarchical folder structure for nested documentation
#
# For now the doc folder only contains the key files for building the docs site and no content. The content is created on demand
# from the metasploit-framework wiki on each build
#
# In the future, the markdown files will be committed directly to the metasploit-framework directory, the wiki history will be
# merged with metasploit-framework, and the old wiki will no longer be updated.
module Build
2022-02-27 03:19:54 +08:00
# The metasploit-framework.wiki files that are committed to Metasploit framework's repository
2021-11-17 23:59:17 +08:00
WIKI_PATH = 'metasploit-framework.wiki' . freeze
2022-02-27 03:19:54 +08:00
# A locally cloned version of https://github.com/rapid7/metasploit-framework/wiki
OLD_WIKI_PATH = 'metasploit-framework.wiki.old' . freeze
2022-01-26 22:29:36 +08:00
PRODUCTION_BUILD_ARTIFACTS = '_site' . freeze
2021-11-17 23:59:17 +08:00
# For now we Git clone the existing metasploit wiki and generate the Jekyll markdown files
# for each build. This allows changes to be made to the existing wiki until it's migrated
# into the main framework repo
module Git
def self . clone_wiki!
unless File . exist? ( WIKI_PATH )
2021-11-30 22:53:30 +08:00
Build . run_command " git clone https://github.com/rapid7/metasploit-framework.wiki.git #{ WIKI_PATH } " , exception : true
2021-11-17 23:59:17 +08:00
end
2022-02-27 03:19:54 +08:00
unless File . exist? ( OLD_WIKI_PATH )
Build . run_command " git clone https://github.com/rapid7/metasploit-framework.wiki.git #{ OLD_WIKI_PATH } " , exception : true
end
Build . run_command " cd #{ OLD_WIKI_PATH } ; git pull " , exception : true
2021-11-17 23:59:17 +08:00
end
end
2022-01-26 22:29:36 +08:00
# Configuration for generating the new website hierarchy, from the existing metasploit-framework wiki
2021-11-17 23:59:17 +08:00
class Config
include Enumerable
2022-02-27 03:19:54 +08:00
2022-04-06 20:12:18 +08:00
def initialize ( config )
@config = config
2021-11-17 23:59:17 +08:00
end
def validate!
configured_paths = all_file_paths
missing_paths = available_paths . map { | path | path . gsub ( " #{ WIKI_PATH } / " , '' ) } - ignored_paths - existing_docs - configured_paths
raise " Unhandled paths #{ missing_paths . join ( ', ' ) } " if missing_paths . any?
each do | page |
page_keys = page . keys
allowed_keys = % i [ path new_base_name nav_order title new_path folder children has_children parents ]
invalid_keys = page_keys - allowed_keys
raise " #{ page } had invalid keys #{ invalid_keys . join ( ', ' ) } " if invalid_keys . any?
end
# Ensure unique folder names
folder_titles = to_enum . select { | page | page [ :folder ] } . map { | page | page [ :title ] }
duplicate_folder = folder_titles . tally . select { | _name , count | count > 1 }
raise " Duplicate folder titles, will cause issues: #{ duplicate_folder } " if duplicate_folder . any?
# Ensure no folder titles match file titles
page_titles = to_enum . reject { | page | page [ :folder ] } . map { | page | page [ :title ] }
title_collisions = ( folder_titles & page_titles ) . tally
raise " Duplicate folder/page titles, will cause issues: #{ title_collisions } " if title_collisions . any?
# Ensure there are no files being migrated to multiple places
page_paths = to_enum . reject { | page | page [ :path ] } . map { | page | page [ :title ] }
duplicate_page_paths = page_paths . tally . select { | _name , count | count > 1 }
raise " Duplicate paths, will cause issues: #{ duplicate_page_paths } " if duplicate_page_paths . any?
2021-12-15 01:53:33 +08:00
# Ensure new file paths are only alphanumeric and hyphenated
new_paths = to_enum . map { | page | page [ :new_path ] }
2022-01-26 22:29:36 +08:00
invalid_new_paths = new_paths . reject { | path | File . basename ( path ) =~ / ^[a-zA-Z0-9_-]* \ .md$ / }
2021-12-15 01:53:33 +08:00
raise " Only alphanumeric and hyphenated file names required: #{ invalid_new_paths } " if invalid_new_paths . any?
2021-11-17 23:59:17 +08:00
end
def available_paths
Dir . glob ( " #{ WIKI_PATH } /**/*{.md,.textile} " , File :: FNM_DOTMATCH )
end
def ignored_paths
[
'_Sidebar.md' ,
'dev/_Sidebar.md' ,
]
end
def existing_docs
existing_docs = Dir . glob ( 'docs/**/*' , File :: FNM_DOTMATCH )
existing_docs
end
def each ( & block )
config . each do | parent |
recurse ( with_metadata ( parent ) , & block )
end
end
def all_file_paths
to_enum . map { | item | item [ :path ] } . to_a
end
protected
# depth first traversal
def recurse ( parent_with_metadata , & block )
block . call ( parent_with_metadata )
parent_with_metadata [ :children ] . to_a . each do | child |
child_with_metadata = with_metadata ( child , parents : parent_with_metadata [ :parents ] + [ parent_with_metadata ] )
recurse ( child_with_metadata , & block )
end
end
def with_metadata ( child , parents : [ ] )
child = child . clone
if child [ :folder ]
parent_folders = parents . map { | page | page [ :folder ] }
child [ :new_path ] = File . join ( * parent_folders , child [ :folder ] , 'index.md' )
else
path = child [ :path ]
base_name = child [ :new_base_name ] || File . basename ( path )
# title calculation
computed_title = File . basename ( base_name , '.md' ) . gsub ( '-' , ' ' )
if child [ :title ] . is_a? ( Proc )
child [ :title ] = child [ :title ] . call ( computed_title )
else
child [ :title ] || = computed_title
end
parent_folders = parents . map { | page | page [ :folder ] }
child [ :new_path ] = File . join ( * parent_folders , base_name . downcase )
end
child [ :parents ] = parents
child [ :has_children ] = true if child [ :children ] . to_a . any?
child
end
attr_reader :config
end
# Extracts markdown links from https://github.com/rapid7/metasploit-framework/wiki into a Jekyll format
2021-12-15 01:53:33 +08:00
# Additionally corrects links to Github
2021-11-17 23:59:17 +08:00
class LinkCorrector
def initialize ( config )
@config = config
@links = { }
end
def extract ( markdown )
extracted_absolute_wiki_links = extract_absolute_wiki_links ( markdown )
@links = @links . merge ( extracted_absolute_wiki_links )
extracted_relative_links = extract_relative_links ( markdown )
@links = @links . merge ( extracted_relative_links )
@links
end
def rerender ( markdown )
links || = @links
new_markdown = markdown . clone
links . each_value do | link |
new_markdown . gsub! ( link [ :full_match ] , link [ :replacement ] )
end
fix_github_username_links ( new_markdown )
end
attr_reader :links
protected
def pages
@config . enum_for ( :each ) . map { | page | page }
end
# scans for absolute links to the old wiki such as 'https://github.com/rapid7/metasploit-framework/wiki/Metasploit-Web-Service'
def extract_absolute_wiki_links ( markdown )
new_links = { }
markdown . scan ( %r{ (https?://github.com/rapid7/metasploit-framework/wiki/([ \ w().%_-]+)) } ) do | full_match , old_path |
full_match = full_match . gsub ( / [).]+$ / , '' )
old_path = URI . decode_www_form_component ( old_path . gsub ( / [).]+$ / , '' ) )
new_path = new_path_for ( old_path )
replacement = " {% link docs/ #{ new_path } %} "
link = {
full_match : full_match ,
type : :absolute ,
new_path : new_path ,
replacement : replacement
}
new_links [ full_match ] = link
end
new_links
end
# Scans for substrings such as '[[Reference Sites|Git Reference Sites]]'
def extract_relative_links ( markdown )
existing_links = @links
new_links = { }
markdown . scan ( / ( \ [ \ [([ \ w_ '().:,-]+)(?: \ |([ \ w_ '():,.-]+))? \ ] \ ]) / ) do | full_match , left , right |
old_path = ( right || left )
new_path = new_path_for ( old_path )
2021-12-15 01:53:33 +08:00
if existing_links [ full_match ] && existing_links [ full_match ] [ :new_path ] != new_path
raise " Link for #{ full_match } previously resolved to #{ existing_links [ full_match ] [ :new_path ] } , but now resolves to #{ new_path } "
end
2021-11-17 23:59:17 +08:00
link_text = left
replacement = " [ #{ link_text } ]({% link docs/ #{ new_path } %}) "
link = {
full_match : full_match ,
type : :relative ,
left : left ,
right : right ,
new_path : new_path ,
replacement : replacement
}
new_links [ full_match ] = link
end
new_links
end
def new_path_for ( old_path )
old_path = old_path . gsub ( ' ' , '-' )
matched_pages = pages . select do | page |
! page [ :folder ] &&
2022-04-22 06:06:03 +08:00
File . basename ( page [ :path ] ) . downcase == " #{ File . basename ( old_path ) } .md " . downcase
2021-11-17 23:59:17 +08:00
end
if matched_pages . empty?
2021-12-01 05:43:54 +08:00
raise " Missing path for #{ old_path } "
2021-11-17 23:59:17 +08:00
end
if matched_pages . count > 1
2021-12-01 05:43:54 +08:00
raise " Duplicate paths for #{ old_path } "
2021-11-17 23:59:17 +08:00
end
matched_pages . first . fetch ( :new_path )
end
def fix_github_username_links ( content )
known_github_names = [
'@0a2940' ,
'@ChrisTuncer' ,
'@TomSellers' ,
'@asoto-r7' ,
'@busterb' ,
'@bwatters-r7' ,
'@jbarnett-r7' ,
'@jlee-r7' ,
'@jmartin-r7' ,
'@mcfakepants' ,
2022-04-06 20:12:18 +08:00
'@Op3n4M3' ,
'@gwillcox-r7' ,
2021-11-17 23:59:17 +08:00
'@red0xff' ,
'@mkienow-r7' ,
'@pbarry-r7' ,
'@schierlm' ,
'@timwr' ,
'@zerosteiner' ,
2022-04-06 20:12:18 +08:00
'@zeroSteiner' ,
'@harmj0y' ,
2021-11-17 23:59:17 +08:00
]
ignored_tags = [
'@harmj0yDescription' ,
'@phpsessid' ,
'@http_client' ,
'@abstract' ,
'@accepts_all_logins' ,
'@addresses' ,
'@aliases' ,
'@channel' ,
'@client' ,
'@dep' ,
'@handle' ,
'@instance' ,
'@param' ,
'@pid' ,
'@process' ,
'@return' ,
'@scanner' ,
'@yieldparam' ,
'@yieldreturn' ,
]
# Replace any dangling github usernames, i.e. `@foo` - but not `[@foo](http://...)` or `email@example.com`
content . gsub ( / (?<![ \ [| \ w])@[ \ w-]+ / ) do | username |
if known_github_names . include? username
" [ #{ username } ](https://www.github.com/ #{ username . gsub ( '@' , '' ) } ) "
elsif ignored_tags . include? username
username
else
raise " Unexpected username: ' #{ username } ' "
end
end
end
end
2022-01-26 22:29:36 +08:00
# Parses a wiki page and can add/remove/update a deprecation notice
class WikiDeprecationText
2022-02-27 03:19:54 +08:00
MAINTAINER_MESSAGE_PREFIX = " <!-- Maintainers: "
private_constant :MAINTAINER_MESSAGE_PREFIX
USER_MESSAGE_PREFIX = '#### Documentation Update:' . freeze
private_constant :USER_MESSAGE_PREFIX
2022-01-26 22:29:36 +08:00
2022-02-27 03:19:54 +08:00
def self . upsert ( original_wiki_content , old_path : , new_url : )
history_link = old_path . include? ( " #{ WIKI_PATH } /Home.md " ) ? './Home/_history' : './_history'
maintainer_message = " #{ MAINTAINER_MESSAGE_PREFIX } Please do not modify this file directly, create a pull request instead --> \n \n "
user_message = " #{ USER_MESSAGE_PREFIX } This Wiki page should be viewable at [ #{ new_url } ]( #{ new_url } ). Or if it is no longer available, see this page's [previous history]( #{ history_link } ) \n \n "
deprecation_text = ( maintainer_message * 3 ) + user_message
" #{ deprecation_text } #{ WikiDeprecationText . remove ( original_wiki_content ) } "
2022-01-26 22:29:36 +08:00
end
def self . remove ( original_wiki_content )
2022-02-27 03:19:54 +08:00
original_wiki_content
. gsub ( / #{ MAINTAINER_MESSAGE_PREFIX } .*$ \ s+ / , '' )
. gsub ( / #{ USER_MESSAGE_PREFIX } .*$ \ s+ / , '' )
2022-01-26 22:29:36 +08:00
end
end
2021-11-17 23:59:17 +08:00
# Converts Wiki markdown pages into a valid Jekyll format
class WikiMigration
2022-01-26 22:29:36 +08:00
# Implements two core components:
# - Converts the existing Wiki markdown pages into a Jekyll format
# - Optionally updates the existing Wiki markdown pages with a link to the new website location
def run ( config , options = { } )
2021-11-17 23:59:17 +08:00
config . validate!
# Clean up new docs folder in preparation for regenerating it entirely from the latest wiki
result_folder = File . join ( '.' , 'docs' )
2021-12-15 01:53:33 +08:00
FileUtils . remove_dir ( result_folder , true )
2021-11-17 23:59:17 +08:00
FileUtils . mkdir ( result_folder )
link_corrector = link_corrector_for ( config )
2021-12-15 01:53:33 +08:00
config . each do | page |
2021-11-17 23:59:17 +08:00
page_config = {
layout : 'default' ,
** page . slice ( :title , :has_children , :nav_order ) ,
2022-02-27 03:30:51 +08:00
parent : ( page [ :parents ] [ - 1 ] || { } ) [ :title ] ,
warning : " Do not modify this file directly. Please modify metasploit-framework/docs/metasploit-framework.wiki instead " ,
2022-02-27 03:19:54 +08:00
old_path : page [ :path ] ? File . join ( WIKI_PATH , page [ :path ] ) : " none - folder automatically generated "
2021-11-17 23:59:17 +08:00
} . compact
page_config [ :has_children ] = true if page [ :has_children ]
preamble = << ~ PREAMBLE
- - -
2021-12-02 09:36:09 +08:00
#{page_config.map { |key, value| "#{key}: #{value.to_s.strip.inspect}" }.join("\n")}
2021-11-17 23:59:17 +08:00
- - -
PREAMBLE
new_path = File . join ( result_folder , page [ :new_path ] )
FileUtils . mkdir_p ( File . dirname ( new_path ) )
if page [ :folder ]
2022-01-26 22:29:36 +08:00
new_docs_content = preamble . rstrip + " \n "
2021-11-17 23:59:17 +08:00
else
2022-01-26 22:29:36 +08:00
old_path = File . join ( WIKI_PATH , page [ :path ] )
previous_content = File . read ( old_path , encoding : Encoding :: UTF_8 )
new_docs_content = preamble + WikiDeprecationText . remove ( previous_content )
new_docs_content = link_corrector . rerender ( new_docs_content )
2022-02-27 03:19:54 +08:00
# Update the old Wiki with links to the new website
if options [ :update_wiki_deprecation_notice ]
new_url = options [ :update_wiki_deprecation_notice ] [ :new_website_url ]
2022-01-26 22:29:36 +08:00
if page [ :new_path ] != 'home.md'
new_url += 'docs/' + page [ :new_path ] . gsub ( '.md' , '.html' )
end
2022-02-27 03:19:54 +08:00
updated_wiki_content = WikiDeprecationText . upsert ( previous_content , old_path : old_path , new_url : new_url )
old_wiki_path = File . join ( OLD_WIKI_PATH , page [ :path ] )
File . write ( old_wiki_path , updated_wiki_content , mode : 'w' , encoding : Encoding :: UTF_8 )
2022-01-26 22:29:36 +08:00
end
2021-11-17 23:59:17 +08:00
end
2022-01-26 22:29:36 +08:00
File . write ( new_path , new_docs_content , mode : 'w' , encoding : Encoding :: UTF_8 )
2021-11-17 23:59:17 +08:00
end
# Now that the docs folder is created, time to move the home.md file out
FileUtils . mv ( 'docs/home.md' , 'index.md' )
end
protected
def link_corrector_for ( config )
link_corrector = LinkCorrector . new ( config )
2021-12-15 01:53:33 +08:00
config . each do | page |
2021-11-17 23:59:17 +08:00
unless page [ :folder ]
2022-01-08 02:18:17 +08:00
content = File . read ( File . join ( WIKI_PATH , page [ :path ] ) , encoding : Encoding :: UTF_8 )
2021-11-17 23:59:17 +08:00
link_corrector . extract ( content )
end
end
link_corrector
end
end
2021-11-30 22:53:30 +08:00
# Serve the production build at http://127.0.0.1:4000/metasploit-framework/
class ProductionServer
2021-12-15 01:53:33 +08:00
autoload :WEBrick , 'webrick'
2021-11-30 22:53:30 +08:00
def self . run
server = WEBrick :: HTTPServer . new (
{
Port : 4000
}
)
2022-01-10 21:04:53 +08:00
server . mount ( '/' , WEBrick :: HTTPServlet :: FileHandler , PRODUCTION_BUILD_ARTIFACTS )
2021-11-30 22:53:30 +08:00
trap ( 'INT' ) do
server . shutdown
rescue StandardError
nil
end
server . start
ensure
server . shutdown
end
end
def self . run_command ( command , exception : true )
puts command
2022-01-26 22:29:36 +08:00
result = ''
2021-11-30 22:53:30 +08:00
:: Open3 . popen2e (
{ 'BUNDLE_GEMFILE' = > File . join ( Dir . pwd , 'Gemfile' ) } ,
'/bin/bash' , '--login' , '-c' , command
2021-12-15 01:53:33 +08:00
) do | stdin , stdout_and_stderr , wait_thread |
2021-11-30 22:53:30 +08:00
stdin . close_write
2021-12-15 01:53:33 +08:00
while wait_thread . alive?
2021-11-30 22:53:30 +08:00
ready = IO . select ( [ stdout_and_stderr ] , nil , nil , 1 )
2022-01-26 22:29:36 +08:00
next unless ready
reads , _writes , _errors = ready
2021-11-30 22:53:30 +08:00
2022-01-26 22:29:36 +08:00
reads . to_a . each do | io |
data = io . read_nonblock ( 1024 )
puts data
result += data
rescue EOFError , Errno :: EAGAIN
# noop
2021-11-30 22:53:30 +08:00
end
end
2021-12-15 01:53:33 +08:00
if ! wait_thread . value . success? && exception
raise " command did not succeed, exit status #{ wait_thread . value . exitstatus . inspect } "
2021-11-30 22:53:30 +08:00
end
end
result
end
2021-11-17 23:59:17 +08:00
def self . run ( options )
2022-02-27 03:19:54 +08:00
Git . clone_wiki! if options [ :wiki_pull ]
# Create a new branch based on the commits from https://github.com/rapid7/metasploit-framework/wiki to move
# Wiki files into the metasploit-framework repo
if options [ :create_wiki_to_framework_migration_branch ]
starting_branch = run_command ( " git rev-parse --abbrev-ref HEAD " ) . chomp
new_wiki_branch_name = " move-all-docs-into-folder "
new_framework_branch_name = " merge-metasploit-framework-wiki-into-metasploit-framework "
begin
# Create a new folder and branch in the old metasploit wiki for where we'd like it to be inside of the metasploit-framework repo
Dir . chdir ( OLD_WIKI_PATH ) do
FileUtils . mkdir_p ( " metasploit-framework.wiki " )
run_command ( " mv *[^metasploit-framework.wiki]* metasploit-framework.wiki " , exception : false )
run_command ( " git branch -d #{ new_wiki_branch_name } " , exception : false )
run_command ( " git checkout -b #{ new_wiki_branch_name } " )
run_command ( " git commit -am 'Move to folder' " )
end
# Create a new branch that can be used to create a pull request
run_command ( " git branch -D #{ new_framework_branch_name } " , exception : false )
run_command ( " git checkout -b #{ new_framework_branch_name } " )
run_command ( " git remote add -f wiki #{ OLD_WIKI_PATH } " , exception : false )
run_command ( " git remote update wiki " )
run_command ( " git merge -m 'Migrate docs from wiki to main repository' wiki/ #{ new_wiki_branch_name } --allow-unrelated-histories " )
puts " new branch #{ new_framework_branch_name } successfully created "
ensure
run_command ( " git checkout #{ starting_branch } " )
end
end
if options [ :copy_old_wiki ]
FileUtils . copy_entry ( OLD_WIKI_PATH , WIKI_PATH , preserve = false , dereference_root = false , remove_destination = true )
# Remove any deprecation text that might be present after copying the old wiki
Dir . glob ( File . join ( WIKI_PATH , '**' , '*.md' ) ) do | path |
previous_content = File . read ( path , encoding : Encoding :: UTF_8 )
new_content = WikiDeprecationText . remove ( previous_content )
File . write ( path , new_content , mode : 'w' , encoding : Encoding :: UTF_8 )
end
end
2021-11-17 23:59:17 +08:00
2022-02-27 03:19:54 +08:00
unless options [ :build_content ]
2022-04-06 20:12:18 +08:00
config = Config . new ( NAVIGATION_CONFIG )
2021-12-01 05:43:54 +08:00
migrator = WikiMigration . new
2022-01-26 22:29:36 +08:00
migrator . run ( config , options )
2021-12-01 05:43:54 +08:00
end
2021-11-30 22:53:30 +08:00
if options [ :production ]
2021-12-15 01:53:33 +08:00
FileUtils . remove_dir ( PRODUCTION_BUILD_ARTIFACTS , true )
2022-04-25 19:08:15 +08:00
run_command ( 'JEKYLL_ENV=production bundle exec jekyll build' )
2021-11-30 22:53:30 +08:00
if options [ :serve ]
ProductionServer . run
end
elsif options [ :serve ]
run_command ( 'bundle exec jekyll serve --config _config.yml,_config_development.yml --incremental' )
end
2021-11-17 23:59:17 +08:00
end
end
if $PROGRAM_NAME == __FILE__
2022-02-27 03:19:54 +08:00
options = {
copy_old_wiki : true ,
wiki_pull : true
}
2021-11-17 23:59:17 +08:00
options_parser = OptionParser . new do | opts |
opts . banner = " Usage: #{ File . basename ( __FILE__ ) } [options] "
opts . on '-h' , '--help' , 'Help banner.' do
return print ( opts . help )
end
2022-02-27 03:19:54 +08:00
opts . on ( '--production' , 'Run a production build' ) do | production |
options [ :production ] = production
end
opts . on ( '--serve' , 'serve the docs site' ) do | serve |
options [ :serve ] = serve
end
opts . on ( '--[no]-copy-old-wiki [FLAG]' , TrueClass , 'Copy the content from the old wiki to the new local wiki folder' ) do | copy_old_wiki |
options [ :copy_old_wiki ] = copy_old_wiki
2021-11-17 23:59:17 +08:00
end
2021-11-30 22:53:30 +08:00
2022-02-27 03:19:54 +08:00
opts . on ( '--[no-]-wiki-pull' , FalseClass , 'Pull the Metasploit Wiki' ) do | wiki_pull |
options [ :wiki_pull ] = wiki_pull
2021-12-01 05:43:54 +08:00
end
2022-02-27 03:19:54 +08:00
opts . on ( '--update-wiki-deprecation-notice [WEBSITE_URL]' , 'Updates the old wiki deprecation notes' ) do | new_website_url |
2022-01-26 22:29:36 +08:00
new_website_url || = 'https://docs.metasploit.com/'
2022-02-27 03:19:54 +08:00
options [ :update_wiki_deprecation_notice ] = {
2022-01-26 22:29:36 +08:00
new_website_url : new_website_url
}
end
2022-02-27 03:19:54 +08:00
opts . on ( '--create-wiki-to-framework-migration-branch' ) do
options [ :create_wiki_to_framework_migration_branch ] = true
2021-11-30 22:53:30 +08:00
end
2021-11-17 23:59:17 +08:00
end
options_parser . parse!
Build . run ( options )
end