Merge pull request #13 from Evolix/checks-processor

Checks processor

Capfile

@@ -40,5 +40,7 @@ install_plugin Capistrano::Puma  # Default puma tasks
 # install_plugin Capistrano::Puma::Monit  # if you need the monit tasks
 # install_plugin Capistrano::Puma::Nginx  # if you want to upload a nginx site template
 
+require "whenever/capistrano"
+
 # Load custom tasks from `lib/capistrano/tasks` if you have any defined
 Dir.glob("lib/capistrano/tasks/*.rake").each { |r| import r }

Gemfile

@@ -38,6 +38,7 @@ gem 'bcrypt', '~> 3.1.7'
 
 gem 'open4'
 gem 'naught'
+gem 'whenever', require: false
 
 
 gem 'octicons'

Gemfile.lock

@@ -91,6 +91,7 @@ GEM
     chromedriver-helper (1.2.0)
       archive-zip (~> 0.10)
       nokogiri (~> 1.8)
+    chronic (0.10.2)
     coderay (1.1.2)
     concurrent-ruby (1.0.5)
     crass (1.0.4)
@@ -308,6 +309,8 @@ GEM
     websocket-driver (0.7.0)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.3)
+    whenever (0.10.0)
+      chronic (>= 0.6.3)
     xpath (3.0.0)
       nokogiri (~> 1.8)
 
@@ -357,6 +360,7 @@ DEPENDENCIES
   uglifier (>= 1.3.0)
   web-console (>= 3.3.0)
   webpacker (~> 3.5)
+  whenever
 
 RUBY VERSION
    ruby 2.5.1p57

app/jobs/whois_sync_job.rb

@@ -1,25 +1,28 @@
 class WhoisSyncJob < ApplicationJob
   queue_as :default
 
+  rescue_from StandardError do |exception|
+    check_logger.log(:standard_error, exception) if check.present?
+    raise # rubocop:disable Style/SignalException
+  end
+
   rescue_from ActiveRecord::RecordNotFound do; end
 
+  # parser error are already logged
+  rescue_from Whois::Error do; end
+
   attr_reader :check
 
   def perform(check_id)
-    @check = Check.find(check_id)
-    check.update_attribute(:last_run_at, Time.now)
+    prepare_check(check_id)
 
-    response = Whois.ask(check.domain)
+    response = Whois.ask(check.domain, logger: check_logger)
     return unless response.valid?
 
     update_from_response(response)
-
-    check.save!
   rescue Whois::DomainNotFoundError
     check.active = false
     check.save!
-  rescue Whois::ParserError # rubocop:disable Lint/HandleExceptions
-    # already logged
   end
 
   def update_from_response(response)
@@ -27,5 +30,19 @@ class WhoisSyncJob < ApplicationJob
     check.domain_updated_at = response.updated_at
     check.domain_expires_at = response.expire_at
     check.last_success_at = Time.now
+
+    check.save!
+  end
+
+  private
+
+  def prepare_check(check_id)
+    @check = Check.find(check_id)
+    check.update_attribute(:last_run_at, Time.now)
+  end
+
+  # logger is a reserved ActiveJob method
+  def check_logger
+    @check_logger ||= CheckLogger.new(check)
   end
 end

app/models/check.rb

@@ -52,6 +52,12 @@ class Check < ApplicationRecord
   after_update :reset_notifications
   after_save :enqueue_sync
 
+  scope :active, -> { where(active: true) }
+  scope :last_run_failed, -> {
+    where("(last_success_at IS NULL AND last_run_at IS NOT NULL)
+      OR (last_success_at <= DATE_SUB(last_run_at, INTERVAL 5 MINUTE))")
+  }
+
   private
 
   def domain_created_at_past

app/models/notification.rb

@@ -31,6 +31,9 @@ class Notification < ApplicationRecord
   validates :delay, numericality: { only_integer: true, greater_than_or_equal_to: 1 }
   validates :recipient, presence: true
 
+  scope :active_check, -> { Check.active }
+  scope :check_last_run_failed, -> { Check.last_run_failed }
+
   def pending!
     self.sent_at = nil
     super

app/services/check_logger.rb

@@ -11,7 +11,7 @@ class CheckLogger
       log_command_result(message)
     when :parsed_response
       log_parsed_response(message)
-    when :parser_error, :service_error
+    when :parser_error, :service_error, :standard_error
       log_error(message)
     end
   end

app/services/check_processor.rb

@@ -0,0 +1,70 @@
+class CheckProcessor
+  attr_reader :configuration
+
+  def initialize(configuration = nil)
+    @configuration = configuration || default_configuration
+  end
+
+  def sync_dates # rubocop:disable Metrics/MethodLength
+    %i[
+      resolve_last_run_failed
+      resolve_expire_short_term
+      resolve_expire_long_term
+      resolve_unknown_expiry
+    ].each do |resolver|
+      public_send(resolver).find_each(batch_size: 100).each do |check|
+        process(check)
+
+        sleep configuration.interval
+      end
+    end
+  end
+
+  def resolve_last_run_failed
+    scope.last_run_failed
+  end
+
+  def resolve_expire_long_term
+    scope
+      .where("DATE(domain_expires_at) >= DATE_ADD(CURDATE(), INTERVAL ? DAY)",
+            configuration.long_term)
+      .where("DATEDIFF(domain_expires_at, CURDATE()) MOD ? = 0",
+            configuration.long_term_frequency)
+  end
+
+  def resolve_expire_short_term
+    scope.where("DATE(domain_expires_at) < DATE_ADD(CURDATE(), INTERVAL ? DAY)",
+               configuration.long_term)
+  end
+
+  def resolve_unknown_expiry
+    scope.where("domain_expires_at IS NULL")
+  end
+
+  private
+
+  def scope
+    Check
+      .active
+      .where("last_run_at IS NULL OR last_run_at < DATE_SUB(NOW(), INTERVAL 12 HOUR)")
+  end
+
+  def process(check)
+    case check.kind.to_sym
+    when :domain
+      WhoisSyncJob.perform_now(check.id)
+    else
+      fail ArgumentError, "Unsupported check kind `#{check.kind}`"
+    end
+  end
+
+  def default_configuration
+    config = Rails.configuration.chexpire.fetch("checks", {})
+
+    OpenStruct.new(
+      interval: config.fetch("interval") { 0.00 },
+      long_term: config.fetch("long_term") { 60 },
+      long_term_frequency: config.fetch("long_term_frequency") { 10 },
+    )
+  end
+end

app/services/notifier.rb

@@ -1,3 +1,5 @@
+require "notifier/processor"
+
 module Notifier
   class << self
     def process_all(configuration = nil)

app/services/notifier/resolver.rb

@@ -12,7 +12,7 @@ module Notifier
       # Logical rules are in plain ruby inside processor
       scope
         .includes(check: :logs)
-        .where("checks.last_success_at <= DATE_SUB(checks.last_run_at, INTERVAL 5 MINUTE)")
+        .merge(Check.last_run_failed)
     end
 
     private
@@ -20,7 +20,8 @@ module Notifier
     def scope
       Notification
         .includes(:check)
-        .where(status: [:pending, :failed], checks: { active: true })
+        .where(status: [:pending, :failed])
+        .merge(Check.active)
         .where.not(checks: { user: ignore_users })
     end
 

app/services/whois/errors.rb

@@ -1,10 +1,10 @@
 module Whois
-  class WhoisError < StandardError; end
+  class Error < StandardError; end
 
-  class WhoisCommandError < WhoisError; end
-  class UnsupportedDomainError < WhoisError; end
-  class DomainNotFoundError < WhoisError; end
-  class ParserError < WhoisError; end
+  class WhoisCommandError < Error; end
+  class UnsupportedDomainError < Error; end
+  class DomainNotFoundError < Error; end
+  class ParserError < Error; end
 
   class FieldNotFoundError < ParserError; end
   class MissingDateFormatError < ParserError; end

app/services/whois/parser/base.rb

@@ -29,7 +29,7 @@ module Whois
         logger.log :parsed_response, response
 
         response
-      rescue StandardError => ex
+      rescue ParserError => ex
         logger.log :parser_error, ex
         raise
       end

config/chexpire.example.yml

@@ -3,6 +3,10 @@ default: &default
   notifier:
     interval: 0.00
     failure_days: 3
+  checks:
+    interval: 0.5
+    long_term: 60
+    long_term_frequency: 10
 
 development:
   <<: *default

config/chexpire.test.yml

@@ -4,3 +4,7 @@ test:
   notifier:
     interval: 0.00
     failure_days: 3
+  checks:
+    interval: 0.00
+    long_term: 60
+    long_term_frequency: 10

config/deploy.rb

@@ -43,3 +43,5 @@ append :linked_files,
 
 # Uncomment the following to require manually verifying the host key before first deploy.
 # set :ssh_options, verify_host_key: :secure
+
+set :whenever_identifier, ->{ "#{fetch(:application)}_#{fetch(:stage)}" }

config/schedule.rb

@@ -0,0 +1,27 @@
+# Use this file to easily define all of your cron jobs.
+#
+# It's helpful, but not entirely necessary to understand cron before proceeding.
+# http://en.wikipedia.org/wiki/Cron
+
+set :output, standard: "log/cron.log"
+
+#
+# every 2.hours do
+#   command "/usr/bin/some_great_command"
+#   runner "MyModel.some_method"
+#   rake "some:great:rake:task"
+# end
+#
+# every 4.days do
+#   runner "AnotherModel.prune_old_records"
+# end
+
+# Learn more: http://github.com/javan/whenever
+
+every 1.day, at: '4:30 am', roles: [:app] do
+  rake "checks:sync_dates"
+end
+
+every 1.day, at: '10:30 am', roles: [:app] do
+  rake "notifications:send_all"
+end

lib/tasks/checks.rake

@@ -0,0 +1,7 @@
+namespace :checks do
+  desc "Refresh expiry dates for checks"
+  task sync_dates: :environment do
+    process = CheckProcessor.new
+    process.sync_dates
+  end
+end

lib/tasks/notifications.rake

@@ -0,0 +1,8 @@
+# require "services/notifier"
+
+namespace :notifications do
+  desc "Send all notifications after checks have been performend"
+  task send_all: :environment do
+    Notifier.process_all
+  end
+end

test/factories/checks.rb

@@ -54,14 +54,18 @@ FactoryBot.define do
       domain_expires_at 1.week.from_now
     end
 
+    trait :expires_next_year do
+      domain_expires_at 1.year.from_now
+    end
+
     trait :last_runs_failed do
-      last_run_at Time.now - 90.minutes
-      last_success_at 1.week.ago - 2.hours
+      last_run_at 3.days.ago - 90.minutes
+      last_success_at 7.days.ago - 2.hours
     end
 
     trait :last_run_succeed do
-      last_run_at 1.hour.ago
-      last_success_at 1.hour.ago
+      last_run_at 25.hour.ago
+      last_success_at 25.hour.ago
     end
 
     trait :inactive do

test/jobs/whois_sync_job_test.rb

@@ -6,7 +6,7 @@ class WhoisSyncJobTest < ActiveJob::TestCase
     check = create(:check, :nil_dates, domain: domain)
 
     mock_system_command("whois", domain, stdout: whois_response(domain)) do
-      WhoisSyncJob.new.perform(check.id)
+      WhoisSyncJob.perform_now(check.id)
     end
 
     check.reload
@@ -24,7 +24,7 @@ class WhoisSyncJobTest < ActiveJob::TestCase
     original_updated_at = check.updated_at
 
     mock_system_command("whois", "domain.fr", stdout: "not a response") do
-      WhoisSyncJob.new.perform(check.id)
+      WhoisSyncJob.perform_now(check.id)
     end
 
     check.reload
@@ -35,12 +35,32 @@ class WhoisSyncJobTest < ActiveJob::TestCase
     assert check.active?
   end
 
-  test "Disable check when whois responds domain not found" do
+  test "should ignore not found (removed) checks" do
+    assert_nothing_raised do
+      WhoisSyncJob.perform_now("9999999")
+    end
+  end
+
+  test "should log and re-raise StandardError" do
+    check = create(:check)
+
+    assert_raise StandardError do
+      Whois.stub :ask, nil do
+        WhoisSyncJob.perform_now(check.id)
+      end
+    end
+
+    assert_equal 1, check.logs.count
+    assert_match(/undefined method \W+valid\?/, check.logs.last.error)
+    assert check.logs.last.failed?
+  end
+
+  test "disable check when whois responds domain not found" do
     domain = "willneverexist.fr"
     check = create(:check, :nil_dates, domain: domain)
 
     mock_system_command("whois", domain, stdout: whois_response(domain)) do
-      WhoisSyncJob.new.perform(check.id)
+      WhoisSyncJob.perform_now(check.id)
     end
 
     check.reload
@@ -50,6 +70,16 @@ class WhoisSyncJobTest < ActiveJob::TestCase
     assert_nil check.last_success_at
   end
 
+  test "default logger is CheckLogger" do
+    check = create(:check)
+
+    mock_system_command("whois", check.domain) do
+      WhoisSyncJob.perform_now(check.id)
+    end
+
+    assert_equal 1, check.logs.count
+  end
+
   private
 
   def whois_response(domain)

test/services/check_logger_test.rb

@@ -39,7 +39,7 @@ class CheckLoggerTest < ActiveSupport::TestCase
     assert @logger.check_log.failed?
   end
 
-  test "should log a successful parsed command" do
+  test "should log a successful parsed response" do
     response = OpenStruct.new(
       domain: "example.fr",
       extracted: "some data",
@@ -52,6 +52,17 @@ class CheckLoggerTest < ActiveSupport::TestCase
     assert @logger.check_log.succeed?
   end
 
+  test "should log as failed a empty/error parsed response" do
+    response = OpenStruct.new(
+      domain: "example.fr",
+      valid?: false,
+    )
+    @logger.log :parsed_response, response
+
+    assert_equal response.to_json, @logger.check_log.parsed_response
+    assert @logger.check_log.failed?
+  end
+
   test "should log parser error with a backtrace" do
     @logger.log :parser_error, mock_exception
 
@@ -60,6 +71,14 @@ class CheckLoggerTest < ActiveSupport::TestCase
     assert @logger.check_log.failed?
   end
 
+  test "should log standard error with a backtrace" do
+    @logger.log :standard_error, mock_exception
+
+    assert_includes @logger.check_log.error, "my error occured"
+    assert_includes @logger.check_log.error, "minitest.rb"
+    assert @logger.check_log.failed?
+  end
+
   test "should log service error" do
     @logger.log :service_error, mock_exception
 

test/services/check_processor_test.rb

@@ -0,0 +1,133 @@
+require "test_helper"
+
+class CheckProcessorTest < ActiveSupport::TestCase
+  setup do
+    @processor = CheckProcessor.new
+  end
+
+  test "process WhoisSyncJob for domain checks" do
+    domain = "domain.fr"
+    check = create(:check, :domain, :nil_dates, domain: domain)
+
+    mock_system_command("whois", domain, stdout: file_fixture("whois/domain.fr.txt").read) do
+      @processor.send(:process, check)
+    end
+
+    check.reload
+
+    assert_equal Time.new(2019, 2, 17, 0, 0, 0, 0), check.domain_expires_at
+  end
+
+  test "raises an error for an unsupported check kind" do
+    check = build(:check)
+
+    check.stub :kind, :unknown do
+      assert_raises ArgumentError do
+        @processor.send(:process, check)
+      end
+    end
+  end
+
+  test "resolve_last_run_failed includes already and never succeeded" do
+    c1 = create(:check, :last_runs_failed)
+    c2 = create(:check, :last_run_succeed)
+    c3 = create(:check, last_run_at: 4.days.ago, last_success_at: nil)
+
+    checks = @processor.resolve_last_run_failed
+
+    assert_includes checks, c1
+    assert_not_includes checks, c2
+    assert_includes checks, c3
+  end
+
+  test "resolve_unknown_expiry" do
+    c1 = create(:check, :nil_dates)
+    c2 = create(:check)
+
+    checks = @processor.resolve_unknown_expiry
+
+    assert_includes checks, c1
+    assert_not_includes checks, c2
+  end
+
+  test "resolve_expire_short_term" do
+    c1 = create(:check, :expires_next_week)
+    c2 = create(:check, :expires_next_year)
+
+    checks = @processor.resolve_expire_short_term
+
+    assert_includes checks, c1
+    assert_not_includes checks, c2
+  end
+
+  test "resolve_expire_long_term returns checks with respect of frequency" do
+    c1 = create(:check, domain_expires_at: 380.days.from_now)
+    c2 = create(:check, domain_expires_at: 390.days.from_now)
+    c3 = create(:check, domain_expires_at: 391.days.from_now)
+    c4 = create(:check, domain_expires_at: 20.days.from_now)
+
+    checks = @processor.resolve_expire_long_term
+
+    assert_includes checks, c1
+    assert_includes checks, c2
+    assert_not_includes checks, c3
+    assert_not_includes checks, c4
+  end
+
+  test "resolvers does not include checks recently executed" do
+    c1 = create(:check, :expires_next_week)
+    c2 = create(:check, :expires_next_week, last_run_at: 4.hours.ago)
+
+    checks = @processor.resolve_expire_short_term
+
+    assert_includes checks, c1
+    assert_not_includes checks, c2
+  end
+
+  test "resolvers include checks never executed" do
+    c1 = create(:check, :expires_next_week, last_run_at: 4.days.ago)
+
+    checks = @processor.resolve_expire_short_term
+
+    assert_includes checks, c1
+  end
+
+  test "resolvers does not include inactive checks" do
+    c1 = create(:check, :expires_next_week)
+    c2 = create(:check, :expires_next_week, :inactive)
+
+    checks = @processor.resolve_expire_short_term
+
+    assert_includes checks, c1
+    assert_not_includes checks, c2
+  end
+
+  test "#sync_dates respects the interval configuration between sends" do
+    create_list(:check, 3, :expires_next_week)
+
+    configuration = Minitest::Mock.new
+    2.times do configuration.expect(:long_term, 60) end
+    configuration.expect(:long_term_frequency, 10)
+
+    3.times do
+      configuration.expect(:interval, 0.000001)
+    end
+
+    processor = CheckProcessor.new(configuration)
+
+    mock = Minitest::Mock.new
+    assert_stub = lambda { |actual_time|
+      assert_equal 0.000001, actual_time
+      mock
+    }
+
+    processor.stub :process, nil do
+      processor.stub :sleep, assert_stub do
+        processor.sync_dates
+      end
+    end
+
+    configuration.verify
+    mock.verify
+  end
+end