diff --git a/Capfile b/Capfile index 28e4a8e..7e9c9a1 100644 --- a/Capfile +++ b/Capfile @@ -40,5 +40,7 @@ install_plugin Capistrano::Puma # Default puma tasks # install_plugin Capistrano::Puma::Monit # if you need the monit tasks # install_plugin Capistrano::Puma::Nginx # if you want to upload a nginx site template +require "whenever/capistrano" + # Load custom tasks from `lib/capistrano/tasks` if you have any defined Dir.glob("lib/capistrano/tasks/*.rake").each { |r| import r } diff --git a/Gemfile b/Gemfile index ea40166..ea6eb71 100644 --- a/Gemfile +++ b/Gemfile @@ -38,6 +38,7 @@ gem 'bcrypt', '~> 3.1.7' gem 'open4' gem 'naught' +gem 'whenever', require: false gem 'octicons' diff --git a/Gemfile.lock b/Gemfile.lock index 646924f..6ce9113 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -91,6 +91,7 @@ GEM chromedriver-helper (1.2.0) archive-zip (~> 0.10) nokogiri (~> 1.8) + chronic (0.10.2) coderay (1.1.2) concurrent-ruby (1.0.5) crass (1.0.4) @@ -308,6 +309,8 @@ GEM websocket-driver (0.7.0) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.3) + whenever (0.10.0) + chronic (>= 0.6.3) xpath (3.0.0) nokogiri (~> 1.8) @@ -357,6 +360,7 @@ DEPENDENCIES uglifier (>= 1.3.0) web-console (>= 3.3.0) webpacker (~> 3.5) + whenever RUBY VERSION ruby 2.5.1p57 diff --git a/app/jobs/whois_sync_job.rb b/app/jobs/whois_sync_job.rb index 32cf69c..c0e1d17 100644 --- a/app/jobs/whois_sync_job.rb +++ b/app/jobs/whois_sync_job.rb @@ -1,25 +1,28 @@ class WhoisSyncJob < ApplicationJob queue_as :default + rescue_from StandardError do |exception| + check_logger.log(:standard_error, exception) if check.present? + raise # rubocop:disable Style/SignalException + end + rescue_from ActiveRecord::RecordNotFound do; end + # parser error are already logged + rescue_from Whois::Error do; end + attr_reader :check def perform(check_id) - @check = Check.find(check_id) - check.update_attribute(:last_run_at, Time.now) + prepare_check(check_id) - response = Whois.ask(check.domain) + response = Whois.ask(check.domain, logger: check_logger) return unless response.valid? update_from_response(response) - - check.save! rescue Whois::DomainNotFoundError check.active = false check.save! - rescue Whois::ParserError # rubocop:disable Lint/HandleExceptions - # already logged end def update_from_response(response) @@ -27,5 +30,19 @@ class WhoisSyncJob < ApplicationJob check.domain_updated_at = response.updated_at check.domain_expires_at = response.expire_at check.last_success_at = Time.now + + check.save! + end + + private + + def prepare_check(check_id) + @check = Check.find(check_id) + check.update_attribute(:last_run_at, Time.now) + end + + # logger is a reserved ActiveJob method + def check_logger + @check_logger ||= CheckLogger.new(check) end end diff --git a/app/models/check.rb b/app/models/check.rb index 7a02a34..fb6b0aa 100644 --- a/app/models/check.rb +++ b/app/models/check.rb @@ -52,6 +52,12 @@ class Check < ApplicationRecord after_update :reset_notifications after_save :enqueue_sync + scope :active, -> { where(active: true) } + scope :last_run_failed, -> { + where("(last_success_at IS NULL AND last_run_at IS NOT NULL) + OR (last_success_at <= DATE_SUB(last_run_at, INTERVAL 5 MINUTE))") + } + private def domain_created_at_past diff --git a/app/models/notification.rb b/app/models/notification.rb index f3c3bd1..27f6289 100644 --- a/app/models/notification.rb +++ b/app/models/notification.rb @@ -31,6 +31,9 @@ class Notification < ApplicationRecord validates :delay, numericality: { only_integer: true, greater_than_or_equal_to: 1 } validates :recipient, presence: true + scope :active_check, -> { Check.active } + scope :check_last_run_failed, -> { Check.last_run_failed } + def pending! self.sent_at = nil super diff --git a/app/services/check_logger.rb b/app/services/check_logger.rb index 829165d..bd0124e 100644 --- a/app/services/check_logger.rb +++ b/app/services/check_logger.rb @@ -11,7 +11,7 @@ class CheckLogger log_command_result(message) when :parsed_response log_parsed_response(message) - when :parser_error, :service_error + when :parser_error, :service_error, :standard_error log_error(message) end end diff --git a/app/services/check_processor.rb b/app/services/check_processor.rb new file mode 100644 index 0000000..31737c3 --- /dev/null +++ b/app/services/check_processor.rb @@ -0,0 +1,70 @@ +class CheckProcessor + attr_reader :configuration + + def initialize(configuration = nil) + @configuration = configuration || default_configuration + end + + def sync_dates # rubocop:disable Metrics/MethodLength + %i[ + resolve_last_run_failed + resolve_expire_short_term + resolve_expire_long_term + resolve_unknown_expiry + ].each do |resolver| + public_send(resolver).find_each(batch_size: 100).each do |check| + process(check) + + sleep configuration.interval + end + end + end + + def resolve_last_run_failed + scope.last_run_failed + end + + def resolve_expire_long_term + scope + .where("DATE(domain_expires_at) >= DATE_ADD(CURDATE(), INTERVAL ? DAY)", + configuration.long_term) + .where("DATEDIFF(domain_expires_at, CURDATE()) MOD ? = 0", + configuration.long_term_frequency) + end + + def resolve_expire_short_term + scope.where("DATE(domain_expires_at) < DATE_ADD(CURDATE(), INTERVAL ? DAY)", + configuration.long_term) + end + + def resolve_unknown_expiry + scope.where("domain_expires_at IS NULL") + end + + private + + def scope + Check + .active + .where("last_run_at IS NULL OR last_run_at < DATE_SUB(NOW(), INTERVAL 12 HOUR)") + end + + def process(check) + case check.kind.to_sym + when :domain + WhoisSyncJob.perform_now(check.id) + else + fail ArgumentError, "Unsupported check kind `#{check.kind}`" + end + end + + def default_configuration + config = Rails.configuration.chexpire.fetch("checks", {}) + + OpenStruct.new( + interval: config.fetch("interval") { 0.00 }, + long_term: config.fetch("long_term") { 60 }, + long_term_frequency: config.fetch("long_term_frequency") { 10 }, + ) + end +end diff --git a/app/services/notifier.rb b/app/services/notifier.rb index 5927cde..36836fd 100644 --- a/app/services/notifier.rb +++ b/app/services/notifier.rb @@ -1,3 +1,5 @@ +require "notifier/processor" + module Notifier class << self def process_all(configuration = nil) diff --git a/app/services/notifier/resolver.rb b/app/services/notifier/resolver.rb index 21e1fa8..a40c959 100644 --- a/app/services/notifier/resolver.rb +++ b/app/services/notifier/resolver.rb @@ -12,7 +12,7 @@ module Notifier # Logical rules are in plain ruby inside processor scope .includes(check: :logs) - .where("checks.last_success_at <= DATE_SUB(checks.last_run_at, INTERVAL 5 MINUTE)") + .merge(Check.last_run_failed) end private @@ -20,7 +20,8 @@ module Notifier def scope Notification .includes(:check) - .where(status: [:pending, :failed], checks: { active: true }) + .where(status: [:pending, :failed]) + .merge(Check.active) .where.not(checks: { user: ignore_users }) end diff --git a/app/services/whois/errors.rb b/app/services/whois/errors.rb index 219ed84..5813977 100644 --- a/app/services/whois/errors.rb +++ b/app/services/whois/errors.rb @@ -1,10 +1,10 @@ module Whois - class WhoisError < StandardError; end + class Error < StandardError; end - class WhoisCommandError < WhoisError; end - class UnsupportedDomainError < WhoisError; end - class DomainNotFoundError < WhoisError; end - class ParserError < WhoisError; end + class WhoisCommandError < Error; end + class UnsupportedDomainError < Error; end + class DomainNotFoundError < Error; end + class ParserError < Error; end class FieldNotFoundError < ParserError; end class MissingDateFormatError < ParserError; end diff --git a/app/services/whois/parser/base.rb b/app/services/whois/parser/base.rb index 64e132d..5d00f82 100644 --- a/app/services/whois/parser/base.rb +++ b/app/services/whois/parser/base.rb @@ -29,7 +29,7 @@ module Whois logger.log :parsed_response, response response - rescue StandardError => ex + rescue ParserError => ex logger.log :parser_error, ex raise end diff --git a/config/chexpire.example.yml b/config/chexpire.example.yml index 3543eaf..2e4ce13 100644 --- a/config/chexpire.example.yml +++ b/config/chexpire.example.yml @@ -3,6 +3,10 @@ default: &default notifier: interval: 0.00 failure_days: 3 + checks: + interval: 0.5 + long_term: 60 + long_term_frequency: 10 development: <<: *default diff --git a/config/chexpire.test.yml b/config/chexpire.test.yml index 371a542..db88e9a 100644 --- a/config/chexpire.test.yml +++ b/config/chexpire.test.yml @@ -4,3 +4,7 @@ test: notifier: interval: 0.00 failure_days: 3 + checks: + interval: 0.00 + long_term: 60 + long_term_frequency: 10 diff --git a/config/deploy.rb b/config/deploy.rb index cf3712d..e1cec8c 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -43,3 +43,5 @@ append :linked_files, # Uncomment the following to require manually verifying the host key before first deploy. # set :ssh_options, verify_host_key: :secure + +set :whenever_identifier, ->{ "#{fetch(:application)}_#{fetch(:stage)}" } diff --git a/config/schedule.rb b/config/schedule.rb new file mode 100644 index 0000000..709286e --- /dev/null +++ b/config/schedule.rb @@ -0,0 +1,27 @@ +# Use this file to easily define all of your cron jobs. +# +# It's helpful, but not entirely necessary to understand cron before proceeding. +# http://en.wikipedia.org/wiki/Cron + +set :output, standard: "log/cron.log" + +# +# every 2.hours do +# command "/usr/bin/some_great_command" +# runner "MyModel.some_method" +# rake "some:great:rake:task" +# end +# +# every 4.days do +# runner "AnotherModel.prune_old_records" +# end + +# Learn more: http://github.com/javan/whenever + +every 1.day, at: '4:30 am', roles: [:app] do + rake "checks:sync_dates" +end + +every 1.day, at: '10:30 am', roles: [:app] do + rake "notifications:send_all" +end diff --git a/lib/tasks/checks.rake b/lib/tasks/checks.rake new file mode 100644 index 0000000..5d82236 --- /dev/null +++ b/lib/tasks/checks.rake @@ -0,0 +1,7 @@ +namespace :checks do + desc "Refresh expiry dates for checks" + task sync_dates: :environment do + process = CheckProcessor.new + process.sync_dates + end +end diff --git a/lib/tasks/notifications.rake b/lib/tasks/notifications.rake new file mode 100644 index 0000000..1bb02f8 --- /dev/null +++ b/lib/tasks/notifications.rake @@ -0,0 +1,8 @@ +# require "services/notifier" + +namespace :notifications do + desc "Send all notifications after checks have been performend" + task send_all: :environment do + Notifier.process_all + end +end diff --git a/test/factories/checks.rb b/test/factories/checks.rb index 7621f96..62d95ec 100644 --- a/test/factories/checks.rb +++ b/test/factories/checks.rb @@ -54,14 +54,18 @@ FactoryBot.define do domain_expires_at 1.week.from_now end + trait :expires_next_year do + domain_expires_at 1.year.from_now + end + trait :last_runs_failed do - last_run_at Time.now - 90.minutes - last_success_at 1.week.ago - 2.hours + last_run_at 3.days.ago - 90.minutes + last_success_at 7.days.ago - 2.hours end trait :last_run_succeed do - last_run_at 1.hour.ago - last_success_at 1.hour.ago + last_run_at 25.hour.ago + last_success_at 25.hour.ago end trait :inactive do diff --git a/test/jobs/whois_sync_job_test.rb b/test/jobs/whois_sync_job_test.rb index a20089e..e96755d 100644 --- a/test/jobs/whois_sync_job_test.rb +++ b/test/jobs/whois_sync_job_test.rb @@ -6,7 +6,7 @@ class WhoisSyncJobTest < ActiveJob::TestCase check = create(:check, :nil_dates, domain: domain) mock_system_command("whois", domain, stdout: whois_response(domain)) do - WhoisSyncJob.new.perform(check.id) + WhoisSyncJob.perform_now(check.id) end check.reload @@ -24,7 +24,7 @@ class WhoisSyncJobTest < ActiveJob::TestCase original_updated_at = check.updated_at mock_system_command("whois", "domain.fr", stdout: "not a response") do - WhoisSyncJob.new.perform(check.id) + WhoisSyncJob.perform_now(check.id) end check.reload @@ -35,12 +35,32 @@ class WhoisSyncJobTest < ActiveJob::TestCase assert check.active? end - test "Disable check when whois responds domain not found" do + test "should ignore not found (removed) checks" do + assert_nothing_raised do + WhoisSyncJob.perform_now("9999999") + end + end + + test "should log and re-raise StandardError" do + check = create(:check) + + assert_raise StandardError do + Whois.stub :ask, nil do + WhoisSyncJob.perform_now(check.id) + end + end + + assert_equal 1, check.logs.count + assert_match(/undefined method \W+valid\?/, check.logs.last.error) + assert check.logs.last.failed? + end + + test "disable check when whois responds domain not found" do domain = "willneverexist.fr" check = create(:check, :nil_dates, domain: domain) mock_system_command("whois", domain, stdout: whois_response(domain)) do - WhoisSyncJob.new.perform(check.id) + WhoisSyncJob.perform_now(check.id) end check.reload @@ -50,6 +70,16 @@ class WhoisSyncJobTest < ActiveJob::TestCase assert_nil check.last_success_at end + test "default logger is CheckLogger" do + check = create(:check) + + mock_system_command("whois", check.domain) do + WhoisSyncJob.perform_now(check.id) + end + + assert_equal 1, check.logs.count + end + private def whois_response(domain) diff --git a/test/services/check_logger_test.rb b/test/services/check_logger_test.rb index 8b3b0e1..cbd1044 100644 --- a/test/services/check_logger_test.rb +++ b/test/services/check_logger_test.rb @@ -39,7 +39,7 @@ class CheckLoggerTest < ActiveSupport::TestCase assert @logger.check_log.failed? end - test "should log a successful parsed command" do + test "should log a successful parsed response" do response = OpenStruct.new( domain: "example.fr", extracted: "some data", @@ -52,6 +52,17 @@ class CheckLoggerTest < ActiveSupport::TestCase assert @logger.check_log.succeed? end + test "should log as failed a empty/error parsed response" do + response = OpenStruct.new( + domain: "example.fr", + valid?: false, + ) + @logger.log :parsed_response, response + + assert_equal response.to_json, @logger.check_log.parsed_response + assert @logger.check_log.failed? + end + test "should log parser error with a backtrace" do @logger.log :parser_error, mock_exception @@ -60,6 +71,14 @@ class CheckLoggerTest < ActiveSupport::TestCase assert @logger.check_log.failed? end + test "should log standard error with a backtrace" do + @logger.log :standard_error, mock_exception + + assert_includes @logger.check_log.error, "my error occured" + assert_includes @logger.check_log.error, "minitest.rb" + assert @logger.check_log.failed? + end + test "should log service error" do @logger.log :service_error, mock_exception diff --git a/test/services/check_processor_test.rb b/test/services/check_processor_test.rb new file mode 100644 index 0000000..66e1e53 --- /dev/null +++ b/test/services/check_processor_test.rb @@ -0,0 +1,133 @@ +require "test_helper" + +class CheckProcessorTest < ActiveSupport::TestCase + setup do + @processor = CheckProcessor.new + end + + test "process WhoisSyncJob for domain checks" do + domain = "domain.fr" + check = create(:check, :domain, :nil_dates, domain: domain) + + mock_system_command("whois", domain, stdout: file_fixture("whois/domain.fr.txt").read) do + @processor.send(:process, check) + end + + check.reload + + assert_equal Time.new(2019, 2, 17, 0, 0, 0, 0), check.domain_expires_at + end + + test "raises an error for an unsupported check kind" do + check = build(:check) + + check.stub :kind, :unknown do + assert_raises ArgumentError do + @processor.send(:process, check) + end + end + end + + test "resolve_last_run_failed includes already and never succeeded" do + c1 = create(:check, :last_runs_failed) + c2 = create(:check, :last_run_succeed) + c3 = create(:check, last_run_at: 4.days.ago, last_success_at: nil) + + checks = @processor.resolve_last_run_failed + + assert_includes checks, c1 + assert_not_includes checks, c2 + assert_includes checks, c3 + end + + test "resolve_unknown_expiry" do + c1 = create(:check, :nil_dates) + c2 = create(:check) + + checks = @processor.resolve_unknown_expiry + + assert_includes checks, c1 + assert_not_includes checks, c2 + end + + test "resolve_expire_short_term" do + c1 = create(:check, :expires_next_week) + c2 = create(:check, :expires_next_year) + + checks = @processor.resolve_expire_short_term + + assert_includes checks, c1 + assert_not_includes checks, c2 + end + + test "resolve_expire_long_term returns checks with respect of frequency" do + c1 = create(:check, domain_expires_at: 380.days.from_now) + c2 = create(:check, domain_expires_at: 390.days.from_now) + c3 = create(:check, domain_expires_at: 391.days.from_now) + c4 = create(:check, domain_expires_at: 20.days.from_now) + + checks = @processor.resolve_expire_long_term + + assert_includes checks, c1 + assert_includes checks, c2 + assert_not_includes checks, c3 + assert_not_includes checks, c4 + end + + test "resolvers does not include checks recently executed" do + c1 = create(:check, :expires_next_week) + c2 = create(:check, :expires_next_week, last_run_at: 4.hours.ago) + + checks = @processor.resolve_expire_short_term + + assert_includes checks, c1 + assert_not_includes checks, c2 + end + + test "resolvers include checks never executed" do + c1 = create(:check, :expires_next_week, last_run_at: 4.days.ago) + + checks = @processor.resolve_expire_short_term + + assert_includes checks, c1 + end + + test "resolvers does not include inactive checks" do + c1 = create(:check, :expires_next_week) + c2 = create(:check, :expires_next_week, :inactive) + + checks = @processor.resolve_expire_short_term + + assert_includes checks, c1 + assert_not_includes checks, c2 + end + + test "#sync_dates respects the interval configuration between sends" do + create_list(:check, 3, :expires_next_week) + + configuration = Minitest::Mock.new + 2.times do configuration.expect(:long_term, 60) end + configuration.expect(:long_term_frequency, 10) + + 3.times do + configuration.expect(:interval, 0.000001) + end + + processor = CheckProcessor.new(configuration) + + mock = Minitest::Mock.new + assert_stub = lambda { |actual_time| + assert_equal 0.000001, actual_time + mock + } + + processor.stub :process, nil do + processor.stub :sleep, assert_stub do + processor.sync_dates + end + end + + configuration.verify + mock.verify + end +end