From f4580317982f91bbce6dd0cf47f0ea0ae123d43b Mon Sep 17 00:00:00 2001
From: Matthew Kienow <matthew_kienow@rapid7.com>
Date: Wed, 1 Aug 2018 02:29:39 -0400
Subject: [PATCH 01/15] Add enhanced msfdb with web service support

Derived from the msfdb script in the metasploit-omnibus repo.
---
 msfdb | 767 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 767 insertions(+)
 create mode 100755 msfdb

diff --git a/msfdb b/msfdb
new file mode 100755
index 0000000000..df0258f09b
--- /dev/null
+++ b/msfdb
@@ -0,0 +1,767 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+require 'json'
+require 'net/http'
+require 'net/https'
+require 'open3'
+require 'optparse'
+require 'rex/socket'
+require 'rex/text'
+require 'sysrandom/securerandom'
+require 'uri'
+require 'yaml'
+
+@script_name = File.basename(__FILE__)
+@framework = File.expand_path(File.dirname(__FILE__))
+
+@localconf = "#{ENV['HOME']}/.msf4"
+@db = "#{@localconf}/db"
+@db_conf = "#{@localconf}/database.yml"
+
+@ws_tag = 'msf-ws'
+@ws_conf = "#{@localconf}/#{@ws_tag}-config.ru"
+@ws_ssl_key_default = "#{@localconf}/#{@ws_tag}-key.pem"
+@ws_ssl_cert_default = "#{@localconf}/#{@ws_tag}-cert.pem"
+@ws_log = "#{@localconf}/logs/#{@ws_tag}.log"
+@ws_pid = "#{@localconf}/#{@ws_tag}.pid"
+
+@current_user = ENV['LOGNAME'] || ENV['USERNAME'] || ENV['USER']
+@msf_ws_user = (@current_user || "msfadmin").to_s.strip
+@ws_generated_ssl = false
+@ws_api_token = nil
+
+@components = %w[database webservice]
+
+@options = {
+    component: :all,
+    debug: false,
+    msf_db_name: 'msf',
+    msf_db_user: 'msf',
+    msftest_db_name: 'msftest',
+    msftest_db_user: 'msftest',
+    db_port: 5433,
+    db_pool: 200,
+    address: 'localhost',
+    port: 8080,
+    ssl: true,
+    ssl_cert: @ws_ssl_cert_default,
+    ssl_key: @ws_ssl_key_default,
+    ssl_disable_verify: false,
+    ws_env: ENV['RACK_ENV'] || 'production',
+    retry_max: 10,
+    retry_delay: 5.0,
+    ws_user: nil
+}
+
+
+def run_cmd(cmd, input = nil)
+  exitstatus = 0
+  err = out = ""
+
+  puts "run_cmd: cmd=#{cmd}, input=#{input}" if @options[:debug]
+
+  Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr|
+    stdin.puts(input) if input
+    if @options[:debug]
+      err = stderr.read
+      out = stdout.read
+    end
+    exitstatus = wait_thr.value.exitstatus
+  end
+
+  if exitstatus != 0
+    if @options[:debug]
+      puts "'#{cmd}' returned #{exitstatus}"
+      puts out
+      puts err
+    end
+  end
+
+  exitstatus
+end
+
+def run_psql(cmd, db_name: 'postgres')
+  if @options[:debug]
+    puts "psql -p #{@options[:db_port]} -c \"#{cmd};\" #{db_name}"
+  end
+
+  run_cmd("psql -p #{@options[:db_port]} -c \"#{cmd};\" #{db_name}")
+end
+
+def pw_gen
+  SecureRandom.base64(32)
+end
+
+def tail(file)
+  begin
+    File.readlines(file).last.to_s.strip
+  rescue
+    nil
+  end
+end
+
+def status_db
+  update_db_port
+
+  if Dir.exist?(@db)
+    if run_cmd("pg_ctl -o \"-p #{@options[:db_port]}\" -D #{@db} status") == 0
+      puts "Database started at #{@db}"
+    else
+      puts "Database is not running at #{@db}"
+    end
+  else
+    puts "No database found at #{@db}"
+  end
+end
+
+def started_db
+  if run_cmd("pg_ctl -o \"-p #{@options[:db_port]}\" -D #{@db} status") == 0
+    puts "Database already started at #{@db}"
+    return true
+  end
+
+  print "Starting database at #{@db}..."
+  run_cmd("pg_ctl -o \"-p #{@options[:db_port]}\" -D #{@db} -l #{@db}/log start")
+  sleep(2)
+  if run_cmd("pg_ctl -o \"-p #{@options[:db_port]}\" -D #{@db} status") != 0
+    puts 'failed'
+    false
+  else
+    puts 'success'
+    true
+  end
+end
+
+def start_db
+  if !Dir.exist?(@db)
+    puts "No database found at #{@db}, not starting"
+    return
+  end
+
+  update_db_port
+
+  while !started_db
+    last_log = tail("#{@db}/log")
+    puts last_log
+    fixed = false
+    if last_log =~ /not compatible/
+      puts "Please attempt to upgrade the database manually using pg_upgrade."
+    end
+    if !fixed
+      if ask_yn('If your database is corrupt, would you to reinitialize it?')
+        fixed = reinit_db
+      end
+    end
+    if !fixed
+      if !ask_yn('Database not started, try again?')
+        return
+      end
+    end
+  end
+end
+
+def stop_db
+  update_db_port
+
+  if run_cmd("pg_ctl -o \"-p #{@options[:db_port]}\" -D #{@db} status") == 0
+    puts "Stopping database at #{@db}"
+    run_cmd("pg_ctl -o \"-p #{@options[:db_port]}\" -D #{@db} stop")
+  else
+    puts "Database is no longer running at #{@db}"
+  end
+end
+
+def restart_db
+  stop_db
+  start_db
+end
+
+def create_db
+  puts "Creating database at #{@db}"
+  Dir.mkdir(@db)
+  run_cmd("initdb --auth-host=trust --auth-local=trust -E UTF8 #{@db}")
+
+  File.open("#{@db}/pg_hba.conf", 'w') do |f|
+    f.puts "host    \"msf\"      \"#{@options[:msf_db_user]}\"      127.0.0.1/32           md5"
+    f.puts "host    \"msftest\"  \"#{@options[:msftest_db_user]}\"  127.0.0.1/32           md5"
+    f.puts "host    \"postgresql\"  \"#{@options[:msftest_db_user]}\"  127.0.0.1/32           md5"
+    f.puts "host    \"template1\"   all                127.0.0.1/32           trust"
+    if Gem.win_platform?
+      f.puts "host    all             all                127.0.0.1/32           trust"
+      f.puts "host    all             all                ::1/128                trust"
+    else
+      f.puts "local   all             all                                       trust"
+    end
+  end
+
+  File.open("#{@db}/postgresql.conf", 'a') do |f|
+    f.puts "port = #{@options[:db_port]}"
+  end
+end
+
+def init_db
+  if Dir.exist?(@db)
+    puts "Found a database at #{@db}, checking to see if it is started"
+    start_db
+    return
+  end
+
+  if File.exist?(@db_conf)
+    if !ask_yn("Found database config at #{@db_conf}, do you want to overwrite it?")
+      return
+    end
+  end
+
+  # Generate new database passwords
+  msf_pass = pw_gen
+  msftest_pass = pw_gen
+
+  # Write a default database config file
+  Dir.mkdir(@localconf) unless File.directory?(@localconf)
+  File.open(@db_conf, 'w') do |f|
+    f.puts <<~EOF
+      development: &pgsql
+        adapter: postgresql
+        database: #{@options[:msf_db_name]}
+        username: #{@options[:msf_db_user]}
+        password: #{msf_pass}
+        host: 127.0.0.1
+        port: #{@options[:db_port]}
+        pool: #{@options[:db_pool]}
+
+      production: &production
+        <<: *pgsql
+
+      test:
+        <<: *pgsql
+        database: #{@options[:msftest_db_name]}
+        username: #{@options[:msftest_db_user]}
+        password: #{msftest_pass}
+    EOF
+  end
+
+  File.chmod(0640, @db_conf)
+
+  create_db
+
+  start_db
+
+  puts 'Creating database users'
+  run_psql("create user #{@options[:msf_db_user]} with password '#{msf_pass}'")
+  run_psql("create user #{@options[:msftest_db_user]} with password '#{msftest_pass}'")
+  run_psql("alter role #{@options[:msf_db_user]} createdb")
+  run_psql("alter role #{@options[:msftest_db_user]} createdb")
+  run_psql("alter role #{@options[:msf_db_user]} with password '#{msf_pass}'")
+  run_psql("alter role #{@options[:msftest_db_user]} with password '#{msftest_pass}'")
+  run_cmd("createdb -p #{@options[:db_port]} -O #{@options[:msf_db_user]} -h 127.0.0.1 -U #{@options[:msf_db_user]} -E UTF-8 -T template0 #{@options[:msf_db_name]}",
+    "#{msf_pass}\n#{msf_pass}\n")
+  run_cmd("createdb -p #{@options[:db_port]} -O #{@options[:msftest_db_user]} -h 127.0.0.1 -U #{@options[:msftest_db_user]} -E UTF-8 -T template0 #{@options[:msftest_db_name]}",
+    "#{msftest_pass}\n#{msftest_pass}\n")
+
+  puts 'Creating initial database schema'
+  Dir.chdir(@framework) do
+    run_cmd("bundle exec rake db:migrate")
+  end
+end
+
+def update_db_port
+  if File.file?(@db_conf)
+    config = YAML.load(File.read(@db_conf))
+    if config["production"] && config["production"]["port"]
+      port = config["production"]["port"]
+      if port != @options[:db_port]
+        puts "Using database port #{port} found in #{@db_conf}"
+        @options[:db_port] = port
+      end
+    end
+  end
+end
+
+def ask_yn(question)
+  loop do
+    print "#{question}: "
+    yn = STDIN.gets
+    case yn
+    when /^[Yy]/
+      return true
+    when /^[Nn]/
+      return false
+    else
+      puts 'Please answer yes or no.'
+    end
+  end
+end
+
+def ask_value(question, default_value)
+  print "#{question}[#{default_value}]: "
+  input = STDIN.gets.strip
+  if input.nil? || input.empty?
+    return default_value
+  else
+    return input
+  end
+end
+
+
+def delete_db
+  if Dir.exist?(@db)
+    puts "Deleting all data at #{@db}"
+    stop_db
+    FileUtils.rm_rf(@db)
+    if File.exist?(@db_conf) && ask_yn("Delete database configuration at #{@db_conf}?")
+      File.delete(@db_conf)
+    end
+  else
+    puts "No data at #{@db}, doing nothing"
+  end
+end
+
+def reinit_db
+  delete_db
+  init_db
+end
+
+def status_web_service
+  if File.file?(@ws_pid)
+    ws_pid = tail(@ws_pid)
+    if ws_pid.nil? || !process_active?(ws_pid.to_i)
+      puts "MSF web service PID file found, but no active process running as PID #{ws_pid}"
+    else
+      puts "MSF web service is running as PID #{ws_pid}"
+    end
+  else
+    puts "No MSF web service PID file found at #{@ws_pid}"
+  end
+end
+
+def init_web_service
+  if File.file?(@ws_conf)
+    if !ask_yn("Found web service config at #{@ws_conf}, do you want to overwrite it?")
+      return
+    end
+  end
+
+  if @options[:ws_user].nil?
+    @msf_ws_user = ask_value("Initial MSF web service account username?", @msf_ws_user)
+  else
+    @msf_ws_user = @options[:ws_user]
+  end
+
+  # Write a default Rack config file for the web service
+  Dir.mkdir(@localconf) unless File.directory?(@localconf)
+  File.open(@ws_conf, 'w') do |f|
+    f.puts <<~EOF
+      # #{File.basename(@ws_conf)}
+      # created on: #{Time.now.utc}
+      lib_path = File.expand_path('./lib/', '#{@framework}')
+      $LOAD_PATH << lib_path unless $LOAD_PATH.include?(lib_path)
+      require 'msf/core/db_manager/http/metasploit_api_app'
+      run MetasploitApiApp
+    EOF
+  end
+  File.chmod(0640, @ws_conf)
+
+  if @options[:ssl] && ((!File.file?(@options[:ssl_key]) || !File.file?(@options[:ssl_cert])) ||
+      (@options[:ssl_key] == @ws_ssl_key_default && @options[:ssl_cert] == @ws_ssl_cert_default))
+    generate_web_service_ssl(key: @options[:ssl_key], cert: @options[:ssl_cert])
+  end
+
+  if !start_web_service
+    return
+  end
+
+  # wait until web service is online
+  retry_count = 0
+  is_online = web_service_online
+  while !is_online && retry_count < @options[:retry_max]
+    retry_count += 1
+    if @options[:debug]
+      puts "MSF web service doesn't appear to be online. Sleeping #{@options[:retry_delay]}s until check #{retry_count}/#{@options[:retry_max]}"
+    end
+    sleep(@options[:retry_delay])
+    is_online = web_service_online
+  end
+
+  if is_online
+    add_web_service_user
+  else
+    puts "MSF web service does not appear to be online; aborting initialize."
+  end
+end
+
+def start_web_service
+  unless File.file?(@ws_conf)
+    puts "No MSF web service configuration found at #{@ws_conf}, not starting"
+    return false
+  end
+
+  # daemonize MSF web service
+  puts "Starting MSF web service"
+  if run_cmd("#{thin_cmd} start") == 0
+    puts "MSF web service started"
+    return true
+  else
+    puts "MSF web service not started"
+    return false
+  end
+end
+
+def stop_web_service
+  ws_pid = tail(@ws_pid)
+  if ws_pid.nil? || !process_active?(ws_pid.to_i)
+    puts "MSF web service is no longer running"
+    if File.file?(@ws_pid)
+      puts "Deleting MSF web service PID file #{@ws_pid}"
+      File.delete(@ws_pid)
+    end
+  else
+    puts "Stopping MSF web service PID #{ws_pid}"
+    run_cmd("#{thin_cmd} stop")
+  end
+end
+
+def restart_web_service
+  stop_web_service
+  start_web_service
+end
+
+def delete_web_service
+  stop_web_service
+  if File.file?(@ws_conf) && ask_yn("Delete MSF web service configuration at #{@ws_conf}?")
+    File.delete(@ws_conf)
+  end
+end
+
+def reinit_web_service
+  delete_web_service
+  init_web_service
+end
+
+def generate_web_service_ssl(key:, cert:)
+  @ws_generated_ssl = true
+  if (File.file?(key) || File.file?(cert)) &&
+      !ask_yn("Either MSF web service SSL key #{key} or certificate #{cert} already exist, overwrite both?")
+    return
+  end
+
+  puts "Generating SSL key and certificate for MSF web service"
+  # @ssl_cert = Rex::Socket::SslTcpServer.ssl_generate_certificate
+  @ssl_key, @ssl_cert, @ssl_extra_chain_cert = Rex::Socket::Ssl.ssl_generate_certificate
+
+  # write PEM format key and certificate
+  mode = 'wb'
+  mode_int = 0600
+  File.open(key, mode) { |f| f.write(@ssl_key.to_pem) }
+  File.chmod(mode_int, key)
+
+  File.open(cert, mode) { |f| f.write(@ssl_cert.to_pem) }
+  File.chmod(mode_int, cert)
+end
+
+def web_service_online
+  msf_version_uri = get_web_service_uri(path: '/api/v1/msf/version')
+  response = http_request(uri: msf_version_uri, method: :get,
+                          skip_verify: skip_ssl_verify, cert: get_ssl_cert)
+  puts "web_service_online: response=#{response}" if @options[:debug]
+  !response.nil? && !response.dig(:data, :metasploit_version).nil?
+end
+
+def add_web_service_user
+  puts "Creating MSF web service user #{@msf_ws_user}"
+
+  # Generate new web service user password
+  msf_ws_pass = pw_gen
+  cred_data = { username: @msf_ws_user, password: msf_ws_pass }
+
+  # Send request to create new admin user
+  user_data = cred_data.merge({ admin: true })
+  user_uri = get_web_service_uri(path: '/api/v1/users')
+  response = http_request(uri: user_uri, data: user_data, method: :post,
+                          skip_verify: skip_ssl_verify, cert: get_ssl_cert)
+  puts "add_web_service_user: create user response=#{response}" if @options[:debug]
+  if response.nil? || response.dig(:data, :username) != @msf_ws_user
+    puts "Error creating MSF web service user #{@msf_ws_user}"
+    return false
+  end
+  puts "\nMSF web service user: #{@msf_ws_user}"
+  puts "MSF web service password: #{msf_ws_pass}"
+
+  # Send request to create new API token for the user
+  generate_token_uri = get_web_service_uri(path: '/api/v1/auth/generate-token')
+  response = http_request(uri: generate_token_uri, query: cred_data, method: :get,
+                          skip_verify: skip_ssl_verify, cert: get_ssl_cert)
+  puts "add_web_service_user: generate token response=#{response}" if @options[:debug]
+  if response.nil? || (@ws_api_token = response.dig(:data, :token)).nil?
+    puts "Error creating MSF web service user API token"
+    return false
+  end
+  puts "MSF web service user API token: #{@ws_api_token}"
+  puts "Please store these credentials securely."
+  return true
+end
+
+def get_web_service_uri(path: nil)
+  uri_class = @options[:ssl] ? URI::HTTPS : URI::HTTP
+  uri_class.build({host: get_web_service_host, port: @options[:port], path: path})
+end
+
+def get_web_service_host
+  # user specified any address INADDR_ANY (0.0.0.0), return a routable address
+  @options[:address] == '0.0.0.0' ? 'localhost' : @options[:address]
+end
+
+def skip_ssl_verify
+  @ws_generated_ssl || @options[:ssl_disable_verify]
+end
+
+def get_ssl_cert
+  @options[:ssl] ? @options[:ssl_cert] : nil
+end
+
+def thin_cmd
+  server_opts = "--rackup #{@ws_conf} --address #{@options[:address]} --port #{@options[:port]}"
+  ssl_opts = @options[:ssl] ? "--ssl --ssl-key-file #{@options[:ssl_key]} --ssl-cert-file #{@options[:ssl_cert]}" : ''
+  ssl_opts << ' --ssl-disable-verify' if skip_ssl_verify
+  adapter_opts = "--environment #{@options[:ws_env]}"
+  daemon_opts = "--daemonize --log #{@ws_log} --pid #{@ws_pid} --tag #{@ws_tag}"
+  all_opts = [server_opts, ssl_opts, adapter_opts, daemon_opts].reject(&:empty?).join(' ')
+
+  "thin #{all_opts}"
+end
+
+def process_active?(pid)
+  begin
+    # group_id = Process.getpgid(pid)
+    Process.kill(0, pid)
+    true
+  rescue Errno::ESRCH
+    false
+  end
+end
+
+def http_request(uri:, query: nil, data: nil, method: :get, skip_verify: false, cert: nil)
+  headers = { 'User-Agent': @script_name }
+  query_str = (!query.nil? && !query.empty?) ? URI.encode_www_form(query.compact) : nil
+  uri.query = query_str
+
+  http = Net::HTTP.new(uri.host, uri.port)
+  if uri.is_a?(URI::HTTPS)
+    http.use_ssl = true
+    if skip_verify
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    else
+      # https://stackoverflow.com/questions/22093042/implementing-https-certificate-pubkey-pinning-with-ruby
+      http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      user_passed_cert = OpenSSL::X509::Certificate.new(File.read(cert))
+
+      http.verify_callback = lambda do |preverify_ok, cert_store|
+        server_cert = cert_store.chain[0]
+        return true unless server_cert.to_der == cert_store.current_cert.to_der
+        same_public_key?(server_cert, user_passed_cert)
+      end
+    end
+  end
+
+  begin
+    case method
+      when :get
+        request = Net::HTTP::Get.new(uri.request_uri, initheader=headers)
+      when :post
+        request = Net::HTTP::Post.new(uri.request_uri, initheader=headers)
+      else
+        raise Exception, "Request method #{method} is not handled"
+    end
+
+    request.content_type = 'application/json'
+    unless data.nil?
+      json_body = data.to_json
+      request.body = json_body
+    end
+
+    response = http.request(request)
+    unless response.body.nil? || response.body.empty?
+      return JSON.parse(response.body, symbolize_names: true)
+    end
+  rescue EOFError => e
+    puts "No data was returned for HTTP #{method} request #{uri.request_uri}, message: #{e.message}" if @options[:debug]
+  rescue => e
+    puts "Problem with HTTP #{method} request #{uri.request_uri}, message: #{e.message}" if @options[:debug]
+  end
+end
+
+# Tells us whether the private keys on the passed certificates match
+# and use the same algo
+def same_public_key?(ref_cert, actual_cert)
+  pkr, pka = ref_cert.public_key, actual_cert.public_key
+
+  # First check if the public keys use the same crypto...
+  return false unless pkr.class == pka.class
+  # ...and then - that they have the same contents
+  return false unless pkr.to_pem == pka.to_pem
+
+  true
+end
+
+def parse_args(args)
+  subtext = <<~USAGE
+    Commands:
+      init     initialize the component
+      reinit   delete and reinitialize the component
+      delete   delete and stop the component
+      status   check component status
+      start    start the component
+      stop     stop the component
+      restart  restart the component
+  USAGE
+
+  parser = OptionParser.new do |opts|
+    opts.banner = "Usage: #{@script_name} [options] <command>"
+    opts.separator('Manage a Metasploit Framework database and web service')
+    opts.separator('')
+    opts.separator('General Options:')
+    opts.on("--component COMPONENT", @components + ['all'], "Component used with provided command (default: all)",
+            "  (#{@components.join(', ')})") { |component|
+      @options[:component] = component.to_sym
+    }
+
+    opts.on("-d", "--debug", "Enable debug output") { |d| @options[:debug] = d }
+
+    opts.separator('')
+    opts.separator('Database Options:')
+    opts.on("--msf-db-name NAME", "Database name (default: #{@options[:msf_db_name]})") { |n|
+      @options[:msf_db_name] = n
+    }
+
+    opts.on("--msf-db-user-name USER", "Database username (default: #{@options[:msf_db_user]})") { |u|
+      @options[:msf_db_user] = u
+    }
+
+    opts.on("--msf-test-db-name NAME", "Test database name (default: #{@options[:msftest_db_name]})") { |n|
+      @options[:msftest_db_name] = n
+    }
+
+    opts.on("--msf-test-db-user-name USER", "Test database username (default: #{@options[:msftest_db_user]})") { |u|
+      @options[:msftest_db_user] = u
+    }
+
+    opts.on("--db-port PORT", Integer, "Database port (default: #{@options[:db_port]})") { |p|
+      @options[:db_port] = p
+    }
+
+    opts.on("--db-pool MAX", Integer, "Database connection pool size (default: #{@options[:db_pool]})") { |m|
+      @options[:db_pool] = m
+    }
+
+    opts.separator('')
+    opts.separator('Web Service Options:')
+    opts.on("-a", "--address ADDRESS",
+            "Bind to host address (default: #{@options[:address]})") { |a|
+      @options[:address] = a
+    }
+
+    opts.on("-p", "--port PORT", Integer,
+            "Web service port (default: #{@options[:port]})") { |p|
+      @options[:port] = p
+    }
+
+    opts.on("--[no-]ssl", "Enable SSL (default: #{@options[:ssl]})") { |s| @options[:ssl] = s }
+
+    opts.on("--ssl-key-file PATH", "Path to private key (default: #{@options[:ssl_key]})") { |p|
+      @options[:ssl_key] = p
+    }
+
+    opts.on("--ssl-cert-file PATH", "Path to certificate (default: #{@options[:ssl_cert]})") { |p|
+      @options[:ssl_cert] = p
+    }
+
+    opts.on("--[no-]ssl-disable-verify",
+            "Disables (optional) client cert requests (default: #{@options[:ssl_disable_verify]})") { |v|
+      @options[:ssl_disable_verify] = v
+    }
+
+    opts.on("--environment ENV", ['production', 'development'],
+            "Web service framework environment (default: #{@options[:ws_env]})",
+            "  (production, development)") { |e|
+      @options[:ws_env] = e
+    }
+
+    opts.on("--retry-max MAX", Integer,
+            "Maximum number of web service connect attempts (default: #{@options[:retry_max]})") { |m|
+      @options[:retry_max] = m
+    }
+
+    opts.on("--retry-delay DELAY", Float,
+            "Delay in seconds between web service connect attempts (default: #{@options[:retry_delay]})") { |d|
+      @options[:retry_delay] = d
+    }
+
+    opts.on("--user USER", "Initial web service admin username (default: #{@options[:ws_user]})") { |u|
+      @options[:ws_user] = u
+    }
+
+    opts.separator('')
+    opts.separator(subtext)
+  end
+
+  parser.parse!(args)
+
+  if args.length != 1
+    puts parser
+    abort
+  end
+
+  @options
+end
+
+def invoke_command(commands, component, command)
+  method = commands[component][command]
+  if !method.nil?
+    send(method)
+  else
+    puts "Error: unrecognized command '#{command}' for #{component}"
+  end
+end
+
+
+
+if $PROGRAM_NAME == __FILE__
+  # Bomb out if we're root
+  if !Gem.win_platform? && Process.uid.zero?
+    puts "Please run #{@script_name} as a non-root user"
+    abort
+  end
+
+  # map component commands to methods
+  commands = {
+      database: {
+          init: :init_db,
+          reinit: :reinit_db,
+          delete: :delete_db,
+          status: :status_db,
+          start: :start_db,
+          stop: :stop_db,
+          restart: :restart_db
+      },
+      webservice: {
+          init: :init_web_service,
+          reinit: :reinit_web_service,
+          delete: :delete_web_service,
+          status: :status_web_service,
+          start: :start_web_service,
+          stop: :stop_web_service,
+          restart: :restart_web_service
+      }
+  }
+
+  parse_args(ARGV)
+
+  command = ARGV[0].to_sym
+  if @options[:component] == :all
+    @components.each { |component|
+      invoke_command(commands, component.to_sym, command)
+    }
+  else
+    invoke_command(commands, @options[:component], command)
+  end
+end

From 2c2af114c44b585b0cfa89885ea834b0106bc180 Mon Sep 17 00:00:00 2001
From: Matthew Kienow <matthew_kienow@rapid7.com>
Date: Wed, 1 Aug 2018 17:42:03 -0400
Subject: [PATCH 02/15] Fix database name typo in privs for development

Also updates authentication method to trust, but verify. Change made
to original script in parallel see rapid7/metasploit-omnibus/#73.
---
 msfdb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msfdb b/msfdb
index df0258f09b..47c4d7ad81 100755
--- a/msfdb
+++ b/msfdb
@@ -185,7 +185,7 @@ def create_db
   File.open("#{@db}/pg_hba.conf", 'w') do |f|
     f.puts "host    \"msf\"      \"#{@options[:msf_db_user]}\"      127.0.0.1/32           md5"
     f.puts "host    \"msftest\"  \"#{@options[:msftest_db_user]}\"  127.0.0.1/32           md5"
-    f.puts "host    \"postgresql\"  \"#{@options[:msftest_db_user]}\"  127.0.0.1/32           md5"
+    f.puts "host    \"postgres\"  \"#{@options[:msftest_db_user]}\"  127.0.0.1/32           trust"
     f.puts "host    \"template1\"   all                127.0.0.1/32           trust"
     if Gem.win_platform?
       f.puts "host    all             all                127.0.0.1/32           trust"

From ca1359bda3325703f3da93babf4ed3bc6581e10a Mon Sep 17 00:00:00 2001
From: Matthew Kienow <matthew_kienow@rapid7.com>
Date: Wed, 1 Aug 2018 17:44:45 -0400
Subject: [PATCH 03/15] Reword prompt from original script

---
 msfdb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msfdb b/msfdb
index 47c4d7ad81..92e472930c 100755
--- a/msfdb
+++ b/msfdb
@@ -149,7 +149,7 @@ def start_db
       puts "Please attempt to upgrade the database manually using pg_upgrade."
     end
     if !fixed
-      if ask_yn('If your database is corrupt, would you to reinitialize it?')
+      if ask_yn('Your database may be corrupt, would you like to reinitialize it?')
         fixed = reinit_db
       end
     end

From fbc9d3ee8321a5c6605df7c6406bc0e6586e9852 Mon Sep 17 00:00:00 2001
From: Matthew Kienow <matthew_kienow@rapid7.com>
Date: Thu, 2 Aug 2018 12:38:52 -0400
Subject: [PATCH 04/15] Add print methods from HttpDBManagerService

---
 .../core/db_manager/http/servlet_helper.rb    | 30 ++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/lib/msf/core/db_manager/http/servlet_helper.rb b/lib/msf/core/db_manager/http/servlet_helper.rb
index c86e13cc2c..f247a8a0d5 100644
--- a/lib/msf/core/db_manager/http/servlet_helper.rb
+++ b/lib/msf/core/db_manager/http/servlet_helper.rb
@@ -2,10 +2,13 @@ require 'json'
 require 'msf/core/db_manager/http/db_manager_proxy'
 require 'msf/core/db_manager/http/job_processor'
 require 'metasploit/framework/data_service/remote/http/response_data_helper'
+require 'rex/ui/text/output/stdio'
 
 module ServletHelper
   include ResponseDataHelper
 
+  @@console_printer = Rex::Ui::Text::Output::Stdio.new
+
   def set_error_on_response(error)
     print_error "Error handling request: #{error.message}", error
     headers = {'Content-Type' => 'text/plain'}
@@ -134,7 +137,32 @@ module ServletHelper
   def warden_options
     env['warden.options']
   end
-  
+
+  def print_line(msg)
+    @@console_printer.print_line(msg)
+  end
+
+  def print_warning(msg)
+    @@console_printer.print_warning(msg)
+  end
+
+  def print_good(msg)
+    @@console_printer.print_good(msg)
+  end
+
+  def print_error(msg, exception = nil)
+    unless exception.nil?
+      msg += "\n    Call Stack:"
+      exception.backtrace.each {|line|
+        msg += "\n"
+        msg += "\t #{line}"
+      }
+    end
+
+    @@console_printer.print_error(msg)
+  end
+
+
   #######
   private
   #######

From be2ad2b947063b7983b4074fad28db81a16fb7b7 Mon Sep 17 00:00:00 2001
From: Matthew Kienow <matthew_kienow@rapid7.com>
Date: Fri, 3 Aug 2018 00:29:47 -0400
Subject: [PATCH 05/15] Rework client authentication configuration file

Restore the md5 auth-method for the postgres DB, msftest user. Use the
default client authentication config to create roles and databases
before writing our own config.
---
 msfdb | 37 ++++++++++++++++++++++---------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/msfdb b/msfdb
index 92e472930c..ec102b47dc 100755
--- a/msfdb
+++ b/msfdb
@@ -182,19 +182,6 @@ def create_db
   Dir.mkdir(@db)
   run_cmd("initdb --auth-host=trust --auth-local=trust -E UTF8 #{@db}")
 
-  File.open("#{@db}/pg_hba.conf", 'w') do |f|
-    f.puts "host    \"msf\"      \"#{@options[:msf_db_user]}\"      127.0.0.1/32           md5"
-    f.puts "host    \"msftest\"  \"#{@options[:msftest_db_user]}\"  127.0.0.1/32           md5"
-    f.puts "host    \"postgres\"  \"#{@options[:msftest_db_user]}\"  127.0.0.1/32           trust"
-    f.puts "host    \"template1\"   all                127.0.0.1/32           trust"
-    if Gem.win_platform?
-      f.puts "host    all             all                127.0.0.1/32           trust"
-      f.puts "host    all             all                ::1/128                trust"
-    else
-      f.puts "local   all             all                                       trust"
-    end
-  end
-
   File.open("#{@db}/postgresql.conf", 'a') do |f|
     f.puts "port = #{@options[:db_port]}"
   end
@@ -255,9 +242,12 @@ def init_db
   run_psql("alter role #{@options[:msf_db_user]} with password '#{msf_pass}'")
   run_psql("alter role #{@options[:msftest_db_user]} with password '#{msftest_pass}'")
   run_cmd("createdb -p #{@options[:db_port]} -O #{@options[:msf_db_user]} -h 127.0.0.1 -U #{@options[:msf_db_user]} -E UTF-8 -T template0 #{@options[:msf_db_name]}",
-    "#{msf_pass}\n#{msf_pass}\n")
+          "#{msf_pass}\n#{msf_pass}\n")
   run_cmd("createdb -p #{@options[:db_port]} -O #{@options[:msftest_db_user]} -h 127.0.0.1 -U #{@options[:msftest_db_user]} -E UTF-8 -T template0 #{@options[:msftest_db_name]}",
-    "#{msftest_pass}\n#{msftest_pass}\n")
+          "#{msftest_pass}\n#{msftest_pass}\n")
+
+  write_db_client_auth_config
+  restart_db
 
   puts 'Creating initial database schema'
   Dir.chdir(@framework) do
@@ -265,6 +255,23 @@ def init_db
   end
 end
 
+def write_db_client_auth_config
+  client_auth_config = "#{@db}/pg_hba.conf"
+  puts "Writing client authentication configuration file #{client_auth_config}"
+  File.open(client_auth_config, 'w') do |f|
+    f.puts "host    \"#{@options[:msf_db_name]}\"      \"#{@options[:msf_db_user]}\"      127.0.0.1/32           md5"
+    f.puts "host    \"#{@options[:msftest_db_name]}\"  \"#{@options[:msftest_db_user]}\"  127.0.0.1/32           md5"
+    f.puts "host    \"postgres\"  \"#{@options[:msftest_db_user]}\"  127.0.0.1/32           md5"
+    f.puts "host    \"template1\"   all                127.0.0.1/32           trust"
+    if Gem.win_platform?
+      f.puts "host    all             all                127.0.0.1/32           trust"
+      f.puts "host    all             all                ::1/128                trust"
+    else
+      f.puts "local   all             all                                       trust"
+    end
+  end
+end
+
 def update_db_port
   if File.file?(@db_conf)
     config = YAML.load(File.read(@db_conf))

From 70b29824bc1639669ba058ee9b5cae845e0f8a78 Mon Sep 17 00:00:00 2001
From: Matthew Kienow <matthew_kienow@rapid7.com>
Date: Fri, 3 Aug 2018 12:30:03 -0400
Subject: [PATCH 06/15] Output web service information to help user

---
 msfdb | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/msfdb b/msfdb
index ec102b47dc..3bb999657d 100755
--- a/msfdb
+++ b/msfdb
@@ -391,7 +391,9 @@ def init_web_service
   end
 
   if is_online
-    add_web_service_user
+    if add_web_service_user
+      output_web_service_information
+    end
   else
     puts "MSF web service does not appear to be online; aborting initialize."
   end
@@ -491,7 +493,7 @@ def add_web_service_user
     puts "Error creating MSF web service user #{@msf_ws_user}"
     return false
   end
-  puts "\nMSF web service user: #{@msf_ws_user}"
+  puts "\nMSF web service username: #{@msf_ws_user}"
   puts "MSF web service password: #{msf_ws_pass}"
 
   # Send request to create new API token for the user
@@ -508,6 +510,21 @@ def add_web_service_user
   return true
 end
 
+def output_web_service_information
+  puts ''
+  puts 'MSF web service configuration complete'
+  puts 'Add the data service in msfconsole using the command:'
+  # build data services command based on install options
+  ds_cmd = "data_services --add --token #{@ws_api_token}"
+  ds_cmd << " --ssl --cert #{@options[:ssl_cert]}" if @options[:ssl]
+  ds_cmd << " --skip-verify" if skip_ssl_verify
+  ds_cmd << " #{get_web_service_host}"
+  puts "#{ds_cmd}"
+  puts ''
+  puts 'The username and password are credentials for the API account:'
+  puts "#{get_web_service_uri(path: '/api/v1/auth/account')}"
+end
+
 def get_web_service_uri(path: nil)
   uri_class = @options[:ssl] ? URI::HTTPS : URI::HTTP
   uri_class.build({host: get_web_service_host, port: @options[:port], path: path})

From b4085e6a5ff1e8e69fced60f68fdc6d2de072454 Mon Sep 17 00:00:00 2001
From: Matthew Kienow <matthew_kienow@rapid7.com>
Date: Fri, 3 Aug 2018 18:43:50 -0400
Subject: [PATCH 07/15] Add required env to fix unexpected behavior

---
 msfdb | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/msfdb b/msfdb
index 3bb999657d..ea4a201787 100755
--- a/msfdb
+++ b/msfdb
@@ -357,13 +357,38 @@ def init_web_service
 
   # Write a default Rack config file for the web service
   Dir.mkdir(@localconf) unless File.directory?(@localconf)
+  # TODO: free the REST API from all of these requirements
   File.open(@ws_conf, 'w') do |f|
     f.puts <<~EOF
       # #{File.basename(@ws_conf)}
       # created on: #{Time.now.utc}
-      lib_path = File.expand_path('./lib/', '#{@framework}')
-      $LOAD_PATH << lib_path unless $LOAD_PATH.include?(lib_path)
+      require 'pathname'
+      require File.expand_path('./config/boot', '#{@framework}')
+      require 'metasploit/framework/parsed_options/remote_db'
       require 'msf/core/db_manager/http/metasploit_api_app'
+
+      def require_environment!(parsed_options)
+        # RAILS_ENV must be set before requiring 'config/application.rb'
+        parsed_options.environment!
+        ARGV.replace(parsed_options.positional)
+
+        # allow other Rails::Applications to use this command
+        if !defined?(Rails) || Rails.application.nil?
+          # @see https://github.com/rails/rails/blob/v3.2.17/railties/lib/rails/commands.rb#L39-L40
+          require File.expand_path('./config/application', '#{@framework}')
+        end
+
+        # have to configure before requiring environment because
+        # config/environment.rb calls initialize! and the initializers will use
+        # the configuration from the parsed options.
+        parsed_options.configure(Rails.application)
+
+        Rails.application.require_environment!
+      end
+
+      parsed_options = Metasploit::Framework::ParsedOptions::RemoteDB.new
+      require_environment!(parsed_options)
+
       run MetasploitApiApp
     EOF
   end

From 8a5d396fdc51e827283363f2cb991bd94bc6b06c Mon Sep 17 00:00:00 2001
From: Matthew Kienow <matthew_kienow@rapid7.com>
Date: Mon, 6 Aug 2018 14:27:16 -0400
Subject: [PATCH 08/15] Fix dependency issue

---
 msfdb | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/msfdb b/msfdb
index ea4a201787..b9ad671413 100755
--- a/msfdb
+++ b/msfdb
@@ -362,8 +362,11 @@ def init_web_service
     f.puts <<~EOF
       # #{File.basename(@ws_conf)}
       # created on: #{Time.now.utc}
-      require 'pathname'
-      require File.expand_path('./config/boot', '#{@framework}')
+
+      @framework_path = '#{@framework}'
+      $LOAD_PATH << @framework_path unless $LOAD_PATH.include?(@framework_path)
+
+      require File.expand_path('./config/boot', @framework_path)
       require 'metasploit/framework/parsed_options/remote_db'
       require 'msf/core/db_manager/http/metasploit_api_app'
 
@@ -375,7 +378,7 @@ def init_web_service
         # allow other Rails::Applications to use this command
         if !defined?(Rails) || Rails.application.nil?
           # @see https://github.com/rails/rails/blob/v3.2.17/railties/lib/rails/commands.rb#L39-L40
-          require File.expand_path('./config/application', '#{@framework}')
+          require File.expand_path('./config/application', @framework_path)
         end
 
         # have to configure before requiring environment because

From 0844a205f6a93592ed0c44ccaa0e10f0e73f1fe3 Mon Sep 17 00:00:00 2001
From: Matthew Kienow <matthew_kienow@rapid7.com>
Date: Mon, 6 Aug 2018 16:16:47 -0400
Subject: [PATCH 09/15] Correct predicate method names per style guide

---
 msfdb | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/msfdb b/msfdb
index b9ad671413..309711ce24 100755
--- a/msfdb
+++ b/msfdb
@@ -408,14 +408,14 @@ def init_web_service
 
   # wait until web service is online
   retry_count = 0
-  is_online = web_service_online
+  is_online = web_service_online?
   while !is_online && retry_count < @options[:retry_max]
     retry_count += 1
     if @options[:debug]
       puts "MSF web service doesn't appear to be online. Sleeping #{@options[:retry_delay]}s until check #{retry_count}/#{@options[:retry_max]}"
     end
     sleep(@options[:retry_delay])
-    is_online = web_service_online
+    is_online = web_service_online?
   end
 
   if is_online
@@ -496,11 +496,11 @@ def generate_web_service_ssl(key:, cert:)
   File.chmod(mode_int, cert)
 end
 
-def web_service_online
+def web_service_online?
   msf_version_uri = get_web_service_uri(path: '/api/v1/msf/version')
   response = http_request(uri: msf_version_uri, method: :get,
-                          skip_verify: skip_ssl_verify, cert: get_ssl_cert)
-  puts "web_service_online: response=#{response}" if @options[:debug]
+                          skip_verify: skip_ssl_verify?, cert: get_ssl_cert)
+  puts "web_service_online?: response=#{response}" if @options[:debug]
   !response.nil? && !response.dig(:data, :metasploit_version).nil?
 end
 
@@ -515,7 +515,7 @@ def add_web_service_user
   user_data = cred_data.merge({ admin: true })
   user_uri = get_web_service_uri(path: '/api/v1/users')
   response = http_request(uri: user_uri, data: user_data, method: :post,
-                          skip_verify: skip_ssl_verify, cert: get_ssl_cert)
+                          skip_verify: skip_ssl_verify?, cert: get_ssl_cert)
   puts "add_web_service_user: create user response=#{response}" if @options[:debug]
   if response.nil? || response.dig(:data, :username) != @msf_ws_user
     puts "Error creating MSF web service user #{@msf_ws_user}"
@@ -527,7 +527,7 @@ def add_web_service_user
   # Send request to create new API token for the user
   generate_token_uri = get_web_service_uri(path: '/api/v1/auth/generate-token')
   response = http_request(uri: generate_token_uri, query: cred_data, method: :get,
-                          skip_verify: skip_ssl_verify, cert: get_ssl_cert)
+                          skip_verify: skip_ssl_verify?, cert: get_ssl_cert)
   puts "add_web_service_user: generate token response=#{response}" if @options[:debug]
   if response.nil? || (@ws_api_token = response.dig(:data, :token)).nil?
     puts "Error creating MSF web service user API token"
@@ -545,7 +545,7 @@ def output_web_service_information
   # build data services command based on install options
   ds_cmd = "data_services --add --token #{@ws_api_token}"
   ds_cmd << " --ssl --cert #{@options[:ssl_cert]}" if @options[:ssl]
-  ds_cmd << " --skip-verify" if skip_ssl_verify
+  ds_cmd << " --skip-verify" if skip_ssl_verify?
   ds_cmd << " #{get_web_service_host}"
   puts "#{ds_cmd}"
   puts ''
@@ -563,7 +563,7 @@ def get_web_service_host
   @options[:address] == '0.0.0.0' ? 'localhost' : @options[:address]
 end
 
-def skip_ssl_verify
+def skip_ssl_verify?
   @ws_generated_ssl || @options[:ssl_disable_verify]
 end
 
@@ -574,7 +574,7 @@ end
 def thin_cmd
   server_opts = "--rackup #{@ws_conf} --address #{@options[:address]} --port #{@options[:port]}"
   ssl_opts = @options[:ssl] ? "--ssl --ssl-key-file #{@options[:ssl_key]} --ssl-cert-file #{@options[:ssl_cert]}" : ''
-  ssl_opts << ' --ssl-disable-verify' if skip_ssl_verify
+  ssl_opts << ' --ssl-disable-verify' if skip_ssl_verify?
   adapter_opts = "--environment #{@options[:ws_env]}"
   daemon_opts = "--daemonize --log #{@ws_log} --pid #{@ws_pid} --tag #{@ws_tag}"
   all_opts = [server_opts, ssl_opts, adapter_opts, daemon_opts].reject(&:empty?).join(' ')

From c6a976820f090c74d965ea4c4be6a97467dbe25d Mon Sep 17 00:00:00 2001
From: Matthew Kienow <matthew_kienow@rapid7.com>
Date: Mon, 6 Aug 2018 16:26:36 -0400
Subject: [PATCH 10/15] Make web service init and start more robust

---
 msfdb | 51 ++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 46 insertions(+), 5 deletions(-)

diff --git a/msfdb b/msfdb
index 309711ce24..2cb429108e 100755
--- a/msfdb
+++ b/msfdb
@@ -343,6 +343,12 @@ def status_web_service
 end
 
 def init_web_service
+  if File.file?(@ws_conf)
+    puts "Found web service config at #{@ws_conf}, checking to see if it is started"
+    start_web_service
+    return
+  end
+
   if File.file?(@ws_conf)
     if !ask_yn("Found web service config at #{@ws_conf}, do you want to overwrite it?")
       return
@@ -433,15 +439,45 @@ def start_web_service
     return false
   end
 
+  # check if MSF web service is already started
+  if File.file?(@ws_pid)
+    ws_pid = tail(@ws_pid)
+    if ws_pid.nil? || !process_active?(ws_pid.to_i)
+      puts "MSF web service PID file found, but no active process running as PID #{ws_pid}"
+      puts "Deleting MSF web service PID file #{@ws_pid}"
+      File.delete(@ws_pid)
+    else
+      puts "MSF web service is already running as PID #{ws_pid}"
+      return false
+    end
+  end
+
   # daemonize MSF web service
   puts "Starting MSF web service"
   if run_cmd("#{thin_cmd} start") == 0
-    puts "MSF web service started"
-    return true
-  else
-    puts "MSF web service not started"
-    return false
+    # wait until web service is started
+    retry_count = 0
+    is_started = web_service_started?
+    while !is_started && retry_count < @options[:retry_max]
+      retry_count += 1
+      if @options[:debug]
+        puts "MSF web service doesn't appear to be started. Sleeping #{@options[:retry_delay]}s until check #{retry_count}/#{@options[:retry_max]}"
+      end
+      sleep(@options[:retry_delay])
+      is_started = web_service_started?
+    end
+
+    if is_started
+      puts "MSF web service started"
+      return true
+    else
+      puts "MSF web service does not appear to be started; aborting start."
+      return false
+    end
   end
+
+  puts "Failed to start MSF web service"
+  return false
 end
 
 def stop_web_service
@@ -475,6 +511,11 @@ def reinit_web_service
   init_web_service
 end
 
+def web_service_started?
+  ws_pid = tail(@ws_pid)
+  !ws_pid.nil? && process_active?(ws_pid.to_i)
+end
+
 def generate_web_service_ssl(key:, cert:)
   @ws_generated_ssl = true
   if (File.file?(key) || File.file?(cert)) &&

From 956bc2fa4f22be2f326c988a1b2e7fe7cee7a82d Mon Sep 17 00:00:00 2001
From: Matthew Kienow <matthew_kienow@rapid7.com>
Date: Mon, 6 Aug 2018 17:44:44 -0400
Subject: [PATCH 11/15] Prompt user before deleting all DB data

---
 msfdb | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/msfdb b/msfdb
index 2cb429108e..63d30d0e9d 100755
--- a/msfdb
+++ b/msfdb
@@ -313,10 +313,14 @@ end
 
 def delete_db
   if Dir.exist?(@db)
-    puts "Deleting all data at #{@db}"
     stop_db
-    FileUtils.rm_rf(@db)
-    if File.exist?(@db_conf) && ask_yn("Delete database configuration at #{@db_conf}?")
+
+    if ask_yn("Delete all data at #{@db}?")
+      puts "Deleting all data at #{@db}"
+      FileUtils.rm_rf(@db)
+    end
+
+    if File.file?(@db_conf) && ask_yn("Delete database configuration at #{@db_conf}?")
       File.delete(@db_conf)
     end
   else

From f7a45933929945f7e12dde9352b95c1390bb2f8a Mon Sep 17 00:00:00 2001
From: Matthew Kienow <matthew_kienow@rapid7.com>
Date: Tue, 7 Aug 2018 15:29:41 -0400
Subject: [PATCH 12/15] Make web service init and start more robust

Remove PID check code since thin will stop and cleanup the PID under
certain circumstances after it has started and a PID file is written.
Reuse web service online check for this purpose.
---
 msfdb | 110 +++++++++++++++++++++++++++++-----------------------------
 1 file changed, 55 insertions(+), 55 deletions(-)

diff --git a/msfdb b/msfdb
index 63d30d0e9d..67ed95512b 100755
--- a/msfdb
+++ b/msfdb
@@ -47,7 +47,7 @@ require 'yaml'
     ssl: true,
     ssl_cert: @ws_ssl_cert_default,
     ssl_key: @ws_ssl_key_default,
-    ssl_disable_verify: false,
+    ssl_disable_verify: true,
     ws_env: ENV['RACK_ENV'] || 'production',
     retry_max: 10,
     retry_delay: 5.0,
@@ -349,16 +349,10 @@ end
 def init_web_service
   if File.file?(@ws_conf)
     puts "Found web service config at #{@ws_conf}, checking to see if it is started"
-    start_web_service
+    start_web_service(expect_auth: true)
     return
   end
 
-  if File.file?(@ws_conf)
-    if !ask_yn("Found web service config at #{@ws_conf}, do you want to overwrite it?")
-      return
-    end
-  end
-
   if @options[:ws_user].nil?
     @msf_ws_user = ask_value("Initial MSF web service account username?", @msf_ws_user)
   else
@@ -412,32 +406,14 @@ def init_web_service
     generate_web_service_ssl(key: @options[:ssl_key], cert: @options[:ssl_cert])
   end
 
-  if !start_web_service
-    return
-  end
-
-  # wait until web service is online
-  retry_count = 0
-  is_online = web_service_online?
-  while !is_online && retry_count < @options[:retry_max]
-    retry_count += 1
-    if @options[:debug]
-      puts "MSF web service doesn't appear to be online. Sleeping #{@options[:retry_delay]}s until check #{retry_count}/#{@options[:retry_max]}"
-    end
-    sleep(@options[:retry_delay])
-    is_online = web_service_online?
-  end
-
-  if is_online
+  if start_web_service(expect_auth: false)
     if add_web_service_user
       output_web_service_information
     end
-  else
-    puts "MSF web service does not appear to be online; aborting initialize."
   end
 end
 
-def start_web_service
+def start_web_service(expect_auth: true)
   unless File.file?(@ws_conf)
     puts "No MSF web service configuration found at #{@ws_conf}, not starting"
     return false
@@ -457,31 +433,37 @@ def start_web_service
   end
 
   # daemonize MSF web service
-  puts "Starting MSF web service"
+  puts 'Attempting to start MSF web service...'
   if run_cmd("#{thin_cmd} start") == 0
-    # wait until web service is started
+    # wait until web service is online
     retry_count = 0
-    is_started = web_service_started?
-    while !is_started && retry_count < @options[:retry_max]
+    response_data = web_service_online_check(expect_auth: expect_auth)
+    is_online = response_data[:state] != :offline
+    while !is_online && retry_count < @options[:retry_max]
       retry_count += 1
       if @options[:debug]
-        puts "MSF web service doesn't appear to be started. Sleeping #{@options[:retry_delay]}s until check #{retry_count}/#{@options[:retry_max]}"
+        puts "MSF web service doesn't appear to be online. Sleeping #{@options[:retry_delay]}s until check #{retry_count}/#{@options[:retry_max]}"
       end
       sleep(@options[:retry_delay])
-      is_started = web_service_started?
+      response_data = web_service_online_check(expect_auth: expect_auth)
+      is_online = response_data[:state] != :offline
     end
 
-    if is_started
-      puts "MSF web service started"
+    if response_data[:state] == :online
+      puts 'MSF web service started and online'
       return true
+    elsif response_data[:state] == :error
+      puts 'MSF web service appears to be started, but may not operate as expected.'
+      puts "#{response_data[:message]}"
     else
-      puts "MSF web service does not appear to be started; aborting start."
-      return false
+      puts 'MSF web service does not appear to be started.'
     end
+    puts "Please see #{@ws_log} for additional details."
+    return false
+  else
+    puts 'Failed to start MSF web service'
+    return false
   end
-
-  puts "Failed to start MSF web service"
-  return false
 end
 
 def stop_web_service
@@ -515,11 +497,6 @@ def reinit_web_service
   init_web_service
 end
 
-def web_service_started?
-  ws_pid = tail(@ws_pid)
-  !ws_pid.nil? && process_active?(ws_pid.to_i)
-end
-
 def generate_web_service_ssl(key:, cert:)
   @ws_generated_ssl = true
   if (File.file?(key) || File.file?(cert)) &&
@@ -541,12 +518,31 @@ def generate_web_service_ssl(key:, cert:)
   File.chmod(mode_int, cert)
 end
 
-def web_service_online?
+def web_service_online_check(expect_auth:)
   msf_version_uri = get_web_service_uri(path: '/api/v1/msf/version')
-  response = http_request(uri: msf_version_uri, method: :get,
+  response_data = http_request(uri: msf_version_uri, method: :get,
                           skip_verify: skip_ssl_verify?, cert: get_ssl_cert)
-  puts "web_service_online?: response=#{response}" if @options[:debug]
-  !response.nil? && !response.dig(:data, :metasploit_version).nil?
+
+  if !response_data[:exception].nil? && response_data[:exception].is_a?(Errno::ECONNREFUSED)
+    response_data[:state] = :offline
+  elsif !response_data[:exception].nil? && response_data[:exception].is_a?(OpenSSL::OpenSSLError)
+    response_data[:state] = :error
+    response_data[:message] = 'Detected an SSL issue. Please set the same options used to initialize the web service or reinitialize.'
+  elsif !response_data[:response].nil? && response_data[:response].dig(:error, :code) == 401
+    if expect_auth
+      response_data[:state] = :online
+    else
+      response_data[:state] = :error
+      response_data[:message] = 'MSF web service expects authentication. If you wish to reinitialize the web service account you will need to reinitialize the database.'
+    end
+  elsif !response_data[:response].nil? && !response_data[:response].dig(:data, :metasploit_version).nil?
+    response_data[:state] = :online
+  else
+    response_data[:state] = :error
+  end
+
+  puts "web_service_online: expect_auth=#{expect_auth}, response_msg=#{response_data}" if @options[:debug]
+  response_data
 end
 
 def add_web_service_user
@@ -559,8 +555,9 @@ def add_web_service_user
   # Send request to create new admin user
   user_data = cred_data.merge({ admin: true })
   user_uri = get_web_service_uri(path: '/api/v1/users')
-  response = http_request(uri: user_uri, data: user_data, method: :post,
+  response_data = http_request(uri: user_uri, data: user_data, method: :post,
                           skip_verify: skip_ssl_verify?, cert: get_ssl_cert)
+  response = response_data[:response]
   puts "add_web_service_user: create user response=#{response}" if @options[:debug]
   if response.nil? || response.dig(:data, :username) != @msf_ws_user
     puts "Error creating MSF web service user #{@msf_ws_user}"
@@ -571,8 +568,9 @@ def add_web_service_user
 
   # Send request to create new API token for the user
   generate_token_uri = get_web_service_uri(path: '/api/v1/auth/generate-token')
-  response = http_request(uri: generate_token_uri, query: cred_data, method: :get,
+  response_data = http_request(uri: generate_token_uri, query: cred_data, method: :get,
                           skip_verify: skip_ssl_verify?, cert: get_ssl_cert)
+  response = response_data[:response]
   puts "add_web_service_user: generate token response=#{response}" if @options[:debug]
   if response.nil? || (@ws_api_token = response.dig(:data, :token)).nil?
     puts "Error creating MSF web service user API token"
@@ -661,6 +659,7 @@ def http_request(uri:, query: nil, data: nil, method: :get, skip_verify: false,
   end
 
   begin
+    response_data = { response: nil }
     case method
       when :get
         request = Net::HTTP::Get.new(uri.request_uri, initheader=headers)
@@ -678,13 +677,14 @@ def http_request(uri:, query: nil, data: nil, method: :get, skip_verify: false,
 
     response = http.request(request)
     unless response.body.nil? || response.body.empty?
-      return JSON.parse(response.body, symbolize_names: true)
+      response_data[:response] = JSON.parse(response.body, symbolize_names: true)
     end
-  rescue EOFError => e
-    puts "No data was returned for HTTP #{method} request #{uri.request_uri}, message: #{e.message}" if @options[:debug]
   rescue => e
+    response_data[:exception] = e
     puts "Problem with HTTP #{method} request #{uri.request_uri}, message: #{e.message}" if @options[:debug]
   end
+
+  response_data
 end
 
 # Tells us whether the private keys on the passed certificates match

From 9b93e0cfcdac95a88c215a465fa77c41d1b96b7c Mon Sep 17 00:00:00 2001
From: Matthew Kienow <matthew_kienow@rapid7.com>
Date: Tue, 7 Aug 2018 15:31:44 -0400
Subject: [PATCH 13/15] Style correction to prefer single-quoted strings

---
 msfdb | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/msfdb b/msfdb
index 67ed95512b..29a680954c 100755
--- a/msfdb
+++ b/msfdb
@@ -146,7 +146,7 @@ def start_db
     puts last_log
     fixed = false
     if last_log =~ /not compatible/
-      puts "Please attempt to upgrade the database manually using pg_upgrade."
+      puts 'Please attempt to upgrade the database manually using pg_upgrade.'
     end
     if !fixed
       if ask_yn('Your database may be corrupt, would you like to reinitialize it?')
@@ -251,7 +251,7 @@ def init_db
 
   puts 'Creating initial database schema'
   Dir.chdir(@framework) do
-    run_cmd("bundle exec rake db:migrate")
+    run_cmd('bundle exec rake db:migrate')
   end
 end
 
@@ -354,7 +354,7 @@ def init_web_service
   end
 
   if @options[:ws_user].nil?
-    @msf_ws_user = ask_value("Initial MSF web service account username?", @msf_ws_user)
+    @msf_ws_user = ask_value('Initial MSF web service account username?', @msf_ws_user)
   else
     @msf_ws_user = @options[:ws_user]
   end
@@ -469,7 +469,7 @@ end
 def stop_web_service
   ws_pid = tail(@ws_pid)
   if ws_pid.nil? || !process_active?(ws_pid.to_i)
-    puts "MSF web service is no longer running"
+    puts 'MSF web service is no longer running'
     if File.file?(@ws_pid)
       puts "Deleting MSF web service PID file #{@ws_pid}"
       File.delete(@ws_pid)
@@ -504,7 +504,7 @@ def generate_web_service_ssl(key:, cert:)
     return
   end
 
-  puts "Generating SSL key and certificate for MSF web service"
+  puts 'Generating SSL key and certificate for MSF web service'
   # @ssl_cert = Rex::Socket::SslTcpServer.ssl_generate_certificate
   @ssl_key, @ssl_cert, @ssl_extra_chain_cert = Rex::Socket::Ssl.ssl_generate_certificate
 
@@ -573,11 +573,11 @@ def add_web_service_user
   response = response_data[:response]
   puts "add_web_service_user: generate token response=#{response}" if @options[:debug]
   if response.nil? || (@ws_api_token = response.dig(:data, :token)).nil?
-    puts "Error creating MSF web service user API token"
+    puts 'Error creating MSF web service user API token'
     return false
   end
   puts "MSF web service user API token: #{@ws_api_token}"
-  puts "Please store these credentials securely."
+  puts 'Please store these credentials securely.'
   return true
 end
 
@@ -627,7 +627,6 @@ end
 
 def process_active?(pid)
   begin
-    # group_id = Process.getpgid(pid)
     Process.kill(0, pid)
     true
   rescue Errno::ESRCH

From d9eae8e11327eda088af85001de544f43919bf13 Mon Sep 17 00:00:00 2001
From: Matthew Kienow <matthew_kienow@rapid7.com>
Date: Tue, 7 Aug 2018 17:37:01 -0400
Subject: [PATCH 14/15] Enhance init handling of existing database config

---
 msfdb | 100 ++++++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 69 insertions(+), 31 deletions(-)

diff --git a/msfdb b/msfdb
index 29a680954c..31daab04b3 100755
--- a/msfdb
+++ b/msfdb
@@ -194,15 +194,77 @@ def init_db
     return
   end
 
-  if File.exist?(@db_conf)
-    if !ask_yn("Found database config at #{@db_conf}, do you want to overwrite it?")
+  if File.exist?(@db_conf) &&
+      !ask_yn("Found database config at #{@db_conf}, do you want to overwrite it?")
+    if !load_db_config
+      puts "Failed to load existing database config. Please reinit and overwrite the file."
       return
     end
+  else
+    write_db_config
   end
 
-  # Generate new database passwords
-  msf_pass = pw_gen
-  msftest_pass = pw_gen
+  create_db
+  start_db
+
+  puts 'Creating database users'
+  run_psql("create user #{@options[:msf_db_user]} with password '#{@msf_pass}'")
+  run_psql("create user #{@options[:msftest_db_user]} with password '#{@msftest_pass}'")
+  run_psql("alter role #{@options[:msf_db_user]} createdb")
+  run_psql("alter role #{@options[:msftest_db_user]} createdb")
+  run_psql("alter role #{@options[:msf_db_user]} with password '#{@msf_pass}'")
+  run_psql("alter role #{@options[:msftest_db_user]} with password '#{@msftest_pass}'")
+  run_cmd("createdb -p #{@options[:db_port]} -O #{@options[:msf_db_user]} -h 127.0.0.1 -U #{@options[:msf_db_user]} -E UTF-8 -T template0 #{@options[:msf_db_name]}",
+          "#{@msf_pass}\n#{@msf_pass}\n")
+  run_cmd("createdb -p #{@options[:db_port]} -O #{@options[:msftest_db_user]} -h 127.0.0.1 -U #{@options[:msftest_db_user]} -E UTF-8 -T template0 #{@options[:msftest_db_name]}",
+          "#{@msftest_pass}\n#{@msftest_pass}\n")
+
+  write_db_client_auth_config
+  restart_db
+
+  puts 'Creating initial database schema'
+  Dir.chdir(@framework) do
+    run_cmd('bundle exec rake db:migrate')
+  end
+end
+
+def load_db_config
+  if File.file?(@db_conf)
+    config = YAML.load(File.read(@db_conf))
+
+    production = config['production']
+    if production.nil?
+      puts "No production section found in database config #{@db_conf}."
+      return false
+    end
+
+    test = config['test']
+    if test.nil?
+      puts "No test section found in database config #{@db_conf}."
+      return false
+    end
+
+    # get values for development and production
+    @options[:msf_db_name] = production['database']
+    @options[:msf_db_user] = production['username']
+    @msf_pass = production['password']
+    @options[:db_port] = production['port']
+    @options[:db_pool] = production['pool']
+
+    # get values for test
+    @options[:msftest_db_name] = test['database']
+    @options[:msftest_db_user] = test['username']
+    @msftest_pass = test['password']
+    return true
+  end
+
+  return false
+end
+
+def write_db_config
+  # Generate new database passwords if not already assigned
+  @msf_pass ||= pw_gen
+  @msftest_pass ||= pw_gen
 
   # Write a default database config file
   Dir.mkdir(@localconf) unless File.directory?(@localconf)
@@ -212,7 +274,7 @@ def init_db
         adapter: postgresql
         database: #{@options[:msf_db_name]}
         username: #{@options[:msf_db_user]}
-        password: #{msf_pass}
+        password: #{@msf_pass}
         host: 127.0.0.1
         port: #{@options[:db_port]}
         pool: #{@options[:db_pool]}
@@ -224,35 +286,11 @@ def init_db
         <<: *pgsql
         database: #{@options[:msftest_db_name]}
         username: #{@options[:msftest_db_user]}
-        password: #{msftest_pass}
+        password: #{@msftest_pass}
     EOF
   end
 
   File.chmod(0640, @db_conf)
-
-  create_db
-
-  start_db
-
-  puts 'Creating database users'
-  run_psql("create user #{@options[:msf_db_user]} with password '#{msf_pass}'")
-  run_psql("create user #{@options[:msftest_db_user]} with password '#{msftest_pass}'")
-  run_psql("alter role #{@options[:msf_db_user]} createdb")
-  run_psql("alter role #{@options[:msftest_db_user]} createdb")
-  run_psql("alter role #{@options[:msf_db_user]} with password '#{msf_pass}'")
-  run_psql("alter role #{@options[:msftest_db_user]} with password '#{msftest_pass}'")
-  run_cmd("createdb -p #{@options[:db_port]} -O #{@options[:msf_db_user]} -h 127.0.0.1 -U #{@options[:msf_db_user]} -E UTF-8 -T template0 #{@options[:msf_db_name]}",
-          "#{msf_pass}\n#{msf_pass}\n")
-  run_cmd("createdb -p #{@options[:db_port]} -O #{@options[:msftest_db_user]} -h 127.0.0.1 -U #{@options[:msftest_db_user]} -E UTF-8 -T template0 #{@options[:msftest_db_name]}",
-          "#{msftest_pass}\n#{msftest_pass}\n")
-
-  write_db_client_auth_config
-  restart_db
-
-  puts 'Creating initial database schema'
-  Dir.chdir(@framework) do
-    run_cmd('bundle exec rake db:migrate')
-  end
 end
 
 def write_db_client_auth_config

From 75249d82d8bd38063d615ae37a9e21769cdb0063 Mon Sep 17 00:00:00 2001
From: Matthew Kienow <matthew_kienow@rapid7.com>
Date: Tue, 7 Aug 2018 18:37:18 -0400
Subject: [PATCH 15/15] Improve web service status messages

---
 msfdb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/msfdb b/msfdb
index 31daab04b3..c5318b9d71 100755
--- a/msfdb
+++ b/msfdb
@@ -375,12 +375,12 @@ def status_web_service
   if File.file?(@ws_pid)
     ws_pid = tail(@ws_pid)
     if ws_pid.nil? || !process_active?(ws_pid.to_i)
-      puts "MSF web service PID file found, but no active process running as PID #{ws_pid}"
+      puts "MSF web service is not running: PID file found at #{@ws_pid}, but no active process running as PID #{ws_pid}"
     else
       puts "MSF web service is running as PID #{ws_pid}"
     end
   else
-    puts "No MSF web service PID file found at #{@ws_pid}"
+    puts "MSF web service is not running: no PID file found at #{@ws_pid}"
   end
 end