diff --git a/autoupdater.rb b/autoupdater.rb index 0b09b64078..664cc10603 100755 --- a/autoupdater.rb +++ b/autoupdater.rb @@ -1,13 +1,13 @@ #!/usr/bin/ruby +require 'concurrent' require 'filelock' require 'set' require_relative 'verify' $stdout.sync = true -echo_set = Set[] -tested = Set[] -queue = Queue.new +echo_set = Concurrent::Set[] +tested = Concurrent::Set[] File.readlines("accelerated-domains.china.conf").each do |line| line.chomp! @@ -16,42 +16,37 @@ File.readlines("accelerated-domains.china.conf").each do |line| end end -threads = [] -ENV.fetch("JOBS", "1").to_i.times.each do - threads << Thread.new do - v = ChinaListVerify.new - while domain = queue.pop - begin - domain = PublicSuffix.domain(domain, ignore_private: true) - rescue PublicSuffix::DomainNotAllowed, PublicSuffix::DomainInvalid - next - end +v = ChinaListVerify.new +pool = Concurrent::FixedThreadPool.new(ENV.fetch("JOBS", Concurrent.processor_count).to_i) - next if tested.include? domain - tested << domain +ARGF.each do |domain| + pool.post do + domain.chomp!.downcase! + next if domain.empty? or domain.end_with?('.arpa', '.cn', '.top') - if v.check_domain_verbose(domain, enable_cdnlist: false, show_green: true) - Filelock '.git.lock' do - puts `./updater.rb -a #{domain}` - puts `git commit -S -am "accelerated-domains: add #{domain}"` if $?.success? - puts `./update-local` if $?.success? - end + if !echo_set.include? domain + puts "Trying to detect #{domain}" + echo_set << domain + end + + begin + domain = PublicSuffix.domain(domain, ignore_private: true) + rescue PublicSuffix::DomainNotAllowed, PublicSuffix::DomainInvalid + next + end + + next if tested.include? domain + tested << domain + + if v.check_domain_verbose(domain, enable_cdnlist: false, show_green: true) + Filelock '.git.lock' do + puts `./updater.rb -a #{domain}` + puts `git commit -S -am "accelerated-domains: add #{domain}"` if $?.success? + puts `./update-local` if $?.success? end end end end -ARGF.each do |domain| - domain.chomp!.downcase! - next if domain.empty? or domain.end_with?('.arpa', '.cn', '.top') - - if !echo_set.include? domain - puts "Trying to detect #{domain}" - echo_set << domain - end - - queue << domain -end - -queue.close -threads.each(&:join) +pool.shutdown +pool.wait_for_termination