From 4e6a3d2ed5bc71ccceb3f86d282827cebbf30d47 Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Tue, 29 May 2018 22:31:55 +0200 Subject: [PATCH 01/12] [GEM] +naught for NullLogger --- Gemfile | 2 ++ Gemfile.lock | 2 ++ lib/null_logger.rb | 3 +++ 3 files changed, 7 insertions(+) create mode 100644 lib/null_logger.rb diff --git a/Gemfile b/Gemfile index 2db3ac3..c1b10d7 100644 --- a/Gemfile +++ b/Gemfile @@ -36,6 +36,8 @@ gem 'bcrypt', '~> 3.1.7' # Use ActiveStorage variant # gem 'mini_magick', '~> 4.8' +gem 'naught' + # Reduces boot times through caching; required in config/boot.rb gem 'bootsnap', '>= 1.1.0', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 65be68a..a8385ca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -141,6 +141,7 @@ GEM msgpack (1.2.4) multi_json (1.13.1) mysql2 (0.5.1) + naught (1.1.0) net-scp (1.2.1) net-ssh (>= 2.6.5) net-ssh (4.2.0) @@ -295,6 +296,7 @@ DEPENDENCIES letter_opener_web listen (>= 3.0.5, < 3.2) mysql2 (>= 0.4.4, < 0.6.0) + naught pry-byebug pry-rails puma (~> 3.11) diff --git a/lib/null_logger.rb b/lib/null_logger.rb new file mode 100644 index 0000000..a15f514 --- /dev/null +++ b/lib/null_logger.rb @@ -0,0 +1,3 @@ +require "naught" + +NullLogger = Naught.build From 2a329ad934d5cf38cf2f789c451661d740e54d1a Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Tue, 29 May 2018 22:32:25 +0200 Subject: [PATCH 02/12] Test: prepare guard & mocks --- Gemfile | 4 ++++ Gemfile.lock | 22 ++++++++++++++++++++++ Guardfile | 29 +++++++++++++++++++++++++++++ test/test_helper.rb | 2 ++ 4 files changed, 57 insertions(+) create mode 100644 Guardfile diff --git a/Gemfile b/Gemfile index c1b10d7..f94c190 100644 --- a/Gemfile +++ b/Gemfile @@ -60,6 +60,10 @@ group :development do gem 'annotate', require: false gem 'letter_opener_web' + + gem "guard" + gem "guard-minitest" + gem 'capistrano-rails' gem "capistrano", "~> 3.10", require: false gem "capistrano-rbenv", require: false diff --git a/Gemfile.lock b/Gemfile.lock index a8385ca..03bbdcb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -106,8 +106,22 @@ GEM erubi (1.7.1) execjs (2.7.0) ffi (1.9.23) + formatador (0.2.5) globalid (0.4.1) activesupport (>= 4.2.0) + guard (2.14.2) + formatador (>= 0.2.4) + listen (>= 2.7, < 4.0) + lumberjack (>= 1.0.12, < 2.0) + nenv (~> 0.1) + notiffany (~> 0.0) + pry (>= 0.9.12) + shellany (~> 0.0) + thor (>= 0.18.1) + guard-compat (1.2.1) + guard-minitest (2.4.6) + guard-compat (~> 1.2) + minitest (>= 3.0) i18n (1.0.1) concurrent-ruby (~> 1.0) io-like (0.3.0) @@ -129,6 +143,7 @@ GEM loofah (2.2.2) crass (~> 1.0.2) nokogiri (>= 1.5.9) + lumberjack (1.0.13) mail (2.7.0) mini_mime (>= 0.1.1) marcel (0.3.2) @@ -142,12 +157,16 @@ GEM multi_json (1.13.1) mysql2 (0.5.1) naught (1.1.0) + nenv (0.3.0) net-scp (1.2.1) net-ssh (>= 2.6.5) net-ssh (4.2.0) nio4r (2.3.1) nokogiri (1.8.2) mini_portile2 (~> 2.3.0) + notiffany (0.1.1) + nenv (~> 0.1) + shellany (~> 0.0) orm_adapter (0.5.0) parallel (1.12.1) parser (2.5.1.0) @@ -229,6 +248,7 @@ GEM selenium-webdriver (3.12.0) childprocess (~> 0.5) rubyzip (~> 1.2) + shellany (0.0.1) simple_form (4.0.1) actionpack (>= 5.0) activemodel (>= 5.0) @@ -291,6 +311,8 @@ DEPENDENCIES chromedriver-helper devise (~> 4.4) devise-i18n (~> 1.6) + guard + guard-minitest jbuilder (~> 2.5) launchy letter_opener_web diff --git a/Guardfile b/Guardfile new file mode 100644 index 0000000..566e5dc --- /dev/null +++ b/Guardfile @@ -0,0 +1,29 @@ +# A sample Guardfile +# More info at https://github.com/guard/guard#readme + +## Uncomment and set this to only include directories you want to watch +# directories %w(app lib config test spec features) \ +# .select{|d| Dir.exists?(d) ? d : UI.warning("Directory #{d} does not exist")} + +## Note: if you are using the `directories` clause above and you are not +## watching the project directory ('.'), then you will want to move +## the Guardfile to a watched dir and symlink it back, e.g. +# +# $ mkdir config +# $ mv Guardfile config/ +# $ ln -s config/Guardfile . +# +# and, you'll have to watch "config/Guardfile" instead of "Guardfile" + +guard "minitest", spring: "bin/rails test" do + # Rails 5 + watch(%r{^app/(.+)\.rb$}) { |m| "test/#{m[1]}_test.rb" } + watch(%r{^app/controllers/application_controller\.rb$}) { "test/controllers" } + watch(%r{^app/controllers/(.+)_controller\.rb$}) { |m| "test/integration/#{m[1]}_test.rb" } + watch(%r{^app/views/(.+)_mailer/.+}) { |m| "test/mailers/#{m[1]}_mailer_test.rb" } + watch(%r{^app/services/whois/.+\.rb}) { |_m| "test/services/whois" } + watch(%r{^lib/(.*/)?([^/]+)\.rb$}) { |m| "test/#{m[1]}test_#{m[2]}.rb" } + watch(%r{^test/.+_test\.rb$}) + watch(%r{^test/test_helper\.rb$}) { "test" } + watch(%r{^test/fixtures/.+\.yml$}) { "test" } +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 04f5f23..2c91312 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -2,6 +2,8 @@ ENV["RAILS_ENV"] ||= "test" require_relative "../config/environment" require "rails/test_help" +require "minitest/mock" + class ActiveSupport::TestCase # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order. fixtures :all From 022d681c330bb88e56eeebf971beee203191373b Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Tue, 29 May 2018 22:33:12 +0200 Subject: [PATCH 03/12] SystemCommand wrapper --- app/services/system_command.rb | 37 ++++++++++++++++++++++++++++ test/services/system_command_test.rb | 21 ++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 app/services/system_command.rb create mode 100644 test/services/system_command_test.rb diff --git a/app/services/system_command.rb b/app/services/system_command.rb new file mode 100644 index 0000000..604f684 --- /dev/null +++ b/app/services/system_command.rb @@ -0,0 +1,37 @@ +require "null_logger" + +class SystemCommand + attr_reader :program + attr_reader :args + attr_reader :logger + + def initialize(program, args, logger: NullLogger.new) + @program = program + @args = Array.wrap(args) + @logger = logger + end + + def execute + logger.log :before_command, syscmd + + raw = `syscmd` + + logger.log :after_command, raw + + raw + end + + def syscmd + escaped_args = args.map { |arg| + '"' + escape_arg(arg) + '"' + } + + [program, escaped_args].join(" ") + end + + private + + def escape_arg(arg) + arg.to_s.gsub('"') { '\"' } + end +end diff --git a/test/services/system_command_test.rb b/test/services/system_command_test.rb new file mode 100644 index 0000000..84cd3c4 --- /dev/null +++ b/test/services/system_command_test.rb @@ -0,0 +1,21 @@ +require "test_helper" +require "system_command" + +class SystemCommandTest < ActiveSupport::TestCase + test "should execute and log a command" do + mock_logger = Minitest::Mock.new + expected = 'whois "example.org"' + + mock_logger.expect(:log, nil, [:before_command, expected]) + mock_logger.expect(:log, nil, [:after_command, "my result"]) + + command = SystemCommand.new("whois", "example.org", logger: mock_logger) + assert_equal expected, command.syscmd + + command.stub(:`, "my result") do + assert_equal "my result", command.execute + end + + mock_logger.verify + end +end From c6b1ac7162699905e0484ab8c7f289cbf928c504 Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Tue, 29 May 2018 22:33:35 +0200 Subject: [PATCH 04/12] Domain tld & normalize helpers --- app/helpers/domain_helper.rb | 12 ++++++++++++ test/helpers/domain_helper_test.rb | 22 ++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 app/helpers/domain_helper.rb create mode 100644 test/helpers/domain_helper_test.rb diff --git a/app/helpers/domain_helper.rb b/app/helpers/domain_helper.rb new file mode 100644 index 0000000..f291fbc --- /dev/null +++ b/app/helpers/domain_helper.rb @@ -0,0 +1,12 @@ +module DomainHelper + def normalize_domain(str) + str.strip.downcase + end + + def tld(str) + parts = normalize_domain(str).split(".") + fail ArgumentError unless parts.size >= 2 + + ".#{parts.last}" + end +end diff --git a/test/helpers/domain_helper_test.rb b/test/helpers/domain_helper_test.rb new file mode 100644 index 0000000..9b6cd59 --- /dev/null +++ b/test/helpers/domain_helper_test.rb @@ -0,0 +1,22 @@ +require "test_helper" +require "domain_helper" + +class DomainHelperTest < ActiveSupport::TestCase + include DomainHelper + + test "should normalize a domain name" do + assert_equal "example.org", normalize_domain(" example.org ") + assert_equal "example.org", normalize_domain("eXaMple.oRg") + end + + test "tld should return the domain tld" do + assert_equal ".org", tld("exaMple.ORG") + assert_equal ".fr", tld("www.example.fr") + assert_equal ".com", tld("www.example-dashed.com") + assert_equal ".uk", tld("www.example.co.uk") + + assert_raises(ArgumentError) do + tld("not a domain") + end + end +end From 00c85e7796f03a76e9362d20b58fd9bd5dd7b2d0 Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Wed, 30 May 2018 12:04:07 +0200 Subject: [PATCH 05/12] Whois parsing of .fr tld --- app/services/whois.rb | 33 +++++ app/services/whois/command.rb | 18 +++ app/services/whois/errors.rb | 8 ++ app/services/whois/parser.rb | 19 +++ app/services/whois/parser/base.rb | 74 ++++++++++ app/services/whois/parser/entry/base.rb | 30 +++++ app/services/whois/parser/entry/blank.rb | 13 ++ app/services/whois/parser/entry/field.rb | 23 ++++ app/services/whois/parser/entry/text.rb | 16 +++ app/services/whois/parser/entry_builder.rb | 55 ++++++++ app/services/whois/parser/fr.rb | 39 ++++++ app/services/whois/response.rb | 11 ++ test/fixtures/files/whois/domain.fr.txt | 150 +++++++++++++++++++++ test/services/whois/command_test.rb | 27 ++++ test/services/whois/parser/fr_test.rb | 21 +++ test/services/whois/parser_test.rb | 15 +++ test/services/whois_test.rb | 9 ++ 17 files changed, 561 insertions(+) create mode 100644 app/services/whois.rb create mode 100644 app/services/whois/command.rb create mode 100644 app/services/whois/errors.rb create mode 100644 app/services/whois/parser.rb create mode 100644 app/services/whois/parser/base.rb create mode 100644 app/services/whois/parser/entry/base.rb create mode 100644 app/services/whois/parser/entry/blank.rb create mode 100644 app/services/whois/parser/entry/field.rb create mode 100644 app/services/whois/parser/entry/text.rb create mode 100644 app/services/whois/parser/entry_builder.rb create mode 100644 app/services/whois/parser/fr.rb create mode 100644 app/services/whois/response.rb create mode 100644 test/fixtures/files/whois/domain.fr.txt create mode 100644 test/services/whois/command_test.rb create mode 100644 test/services/whois/parser/fr_test.rb create mode 100644 test/services/whois/parser_test.rb create mode 100644 test/services/whois_test.rb diff --git a/app/services/whois.rb b/app/services/whois.rb new file mode 100644 index 0000000..80bfa4b --- /dev/null +++ b/app/services/whois.rb @@ -0,0 +1,33 @@ +require "null_logger" +require "domain_helper" +require "whois/command" +require "whois/parser" +require "whois/response" + +module Whois + class << self + def ask(domain, logger: NullLogger.new) + Service.new(domain, logger).call + end + end + + class Service + attr_reader :domain + attr_reader :logger + + def initialize(domain, logger) + @domain = domain + @logger = logger + end + + def call + command = Command.new(domain, logger: logger) + raw_response = command.run + + parser = Parser.for(domain, logger: logger) + response = parser.parse(raw_response) + + response + end + end +end diff --git a/app/services/whois/command.rb b/app/services/whois/command.rb new file mode 100644 index 0000000..0daaebc --- /dev/null +++ b/app/services/whois/command.rb @@ -0,0 +1,18 @@ +require "null_logger" +require "system_command" + +module Whois + class Command + attr_reader :logger + attr_reader :domain + + def initialize(domain, logger: NullLogger.new) + @domain = domain + @logger = logger + end + + def run + SystemCommand.new("whois", domain, logger: logger).execute + end + end +end diff --git a/app/services/whois/errors.rb b/app/services/whois/errors.rb new file mode 100644 index 0000000..794d44b --- /dev/null +++ b/app/services/whois/errors.rb @@ -0,0 +1,8 @@ +module Whois + class UnsupportedDomainError < StandardError; end + class ParserError < StandardError; end + class CommentNotFoundError < ParserError; end + class FieldNotFoundError < ParserError; end + class MissingDateFormatError < ParserError; end + class InvalidDateError < ParserError; end +end diff --git a/app/services/whois/parser.rb b/app/services/whois/parser.rb new file mode 100644 index 0000000..bc80592 --- /dev/null +++ b/app/services/whois/parser.rb @@ -0,0 +1,19 @@ +require "null_logger" +require "whois/errors" +require "whois/parser/fr" + +module Whois + module Parser + PARSERS = [Fr].freeze + + class << self + def for(domain, logger: NullLogger.new) + parser_class = PARSERS.find { |k| k.supports?(domain) } + + fail UnsupportedDomainError, "Unsupported domain '#{domain}'" if parser_class.nil? + + parser_class.new(domain, logger: logger) + end + end + end +end diff --git a/app/services/whois/parser/base.rb b/app/services/whois/parser/base.rb new file mode 100644 index 0000000..ac0c1e5 --- /dev/null +++ b/app/services/whois/parser/base.rb @@ -0,0 +1,74 @@ +require "null_logger" +require_relative "../response" +require_relative "../errors" +require_relative "entry_builder" + +module Whois + module Parser + class Base + extend DomainHelper + + attr_reader :domain + attr_reader :logger + attr_reader :response + attr_reader :entries + attr_reader :date_format + + def initialize(domain, logger: NullLogger.new) + @domain = domain + @logger = logger + @response = Response.new(domain) + @date_format = nil + end + + def parse(raw) + @entries = build_entries(raw) + + do_parse + + response + end + + protected + + def get_field!(name, after: -1, value: nil) + fields.detect { |field| + field.index > after && + field.name == name && + (value.nil? || field.value == value) + } || fail(FieldNotFoundError, "Field `#{name}` not found, after index #{after}") + end + + def get_value!(name, after: -1) + get_field!(name, after: after).value + end + + def parse_date(str) + fail MissingDateFormatError, "Date format not set" if date_format.nil? + + begin + Date.strptime(str, date_format) + rescue ArgumentError + raise InvalidDateError, "Date `#{str}` does not match format #{date_format}" + end + end + + private + + def build_entries(raw) + builder = EntryBuilder.new( + field_regex: self.class::FIELD_REGEX, + comment_regex: self.class::COMMENT_REGEX, + ) + + raw.split("\n").map.each_with_index { |line, index| + builder.build_from_line(line, index) + }.sort_by(&:index) + end + + def fields + @fields ||= entries.select(&:field?) + end + end + end +end diff --git a/app/services/whois/parser/entry/base.rb b/app/services/whois/parser/entry/base.rb new file mode 100644 index 0000000..a7a4c0c --- /dev/null +++ b/app/services/whois/parser/entry/base.rb @@ -0,0 +1,30 @@ +module Whois + module Parser + module Entry + class Base + attr_reader :index + + def initialize(index) + @index = index + @comment = false + end + + def comment! + @comment = true + end + + def comment? + @comment == true + end + + def blank? + false + end + + def field? + false + end + end + end + end +end diff --git a/app/services/whois/parser/entry/blank.rb b/app/services/whois/parser/entry/blank.rb new file mode 100644 index 0000000..3933cbe --- /dev/null +++ b/app/services/whois/parser/entry/blank.rb @@ -0,0 +1,13 @@ +require_relative "base" + +module Whois + module Parser + module Entry + class Blank < Base + def blank? + true + end + end + end + end +end diff --git a/app/services/whois/parser/entry/field.rb b/app/services/whois/parser/entry/field.rb new file mode 100644 index 0000000..e4ccf91 --- /dev/null +++ b/app/services/whois/parser/entry/field.rb @@ -0,0 +1,23 @@ +require_relative "base" + +module Whois + module Parser + module Entry + class Field < Base + attr_reader :index + attr_reader :name + attr_reader :value + + def initialize(index, name, value) + super index + @name = name.strip + @value = value.strip + end + + def field? + true + end + end + end + end +end diff --git a/app/services/whois/parser/entry/text.rb b/app/services/whois/parser/entry/text.rb new file mode 100644 index 0000000..4bb8361 --- /dev/null +++ b/app/services/whois/parser/entry/text.rb @@ -0,0 +1,16 @@ +require_relative "base" + +module Whois + module Parser + module Entry + class Text < Base + attr_reader :text + + def initialize(index, text) + super index + @text = text.strip + end + end + end + end +end diff --git a/app/services/whois/parser/entry_builder.rb b/app/services/whois/parser/entry_builder.rb new file mode 100644 index 0000000..cedcdb2 --- /dev/null +++ b/app/services/whois/parser/entry_builder.rb @@ -0,0 +1,55 @@ +require_relative "entry/blank" +require_relative "entry/field" +require_relative "entry/text" + +module Whois + module Parser + class EntryBuilder + attr_reader :field_regex + attr_reader :comment_regex + + def initialize(field_regex:, comment_regex:) + @field_regex = field_regex + @comment_regex = comment_regex + end + + def build_from_line(line, index) + text = normalize_text(line) + + return Entry::Blank.new(index) if line.empty? + + build(index, text).tap do |entry| + entry.comment! if comment?(line) + end + end + + private + + def build(index, text) + parts = field_regex.match(text) + + if parts.nil? + Entry::Text.new(index, text) + else + Entry::Field.new(index, parts[:name], parts[:value]) + end + end + + def normalize_text(line) + line.strip! + + comment_data = comment_regex.match(line) + + if comment_data.nil? + line + else + comment_data[:text] + end + end + + def comment?(line) + comment_regex.match?(line) + end + end + end +end diff --git a/app/services/whois/parser/fr.rb b/app/services/whois/parser/fr.rb new file mode 100644 index 0000000..7bd8a58 --- /dev/null +++ b/app/services/whois/parser/fr.rb @@ -0,0 +1,39 @@ +require "domain_helper" +require_relative "base" + +module Whois::Parser + class Fr < Base + SUPPORTED_TLD = %w[.fr].freeze + COMMENT_REGEX = /^%+ +(?.+)$/ + FIELD_REGEX = /^(?[^:]+)\s*:\s+(?.+)$/ + + def self.supports?(domain) + SUPPORTED_TLD.include?(tld(domain)) + end + + protected + + def do_parse + set_date_format + + domain_index = get_field!("domain", value: domain).index + + created_date = get_value!("created", after: domain_index) + response.created_on = parse_date(created_date) + + expire_date = get_value!("Expiry Date", after: domain_index) + response.expire_on = parse_date(expire_date) + + updated_date = get_value!("last-update", after: domain_index) + response.updated_on = parse_date(updated_date) + end + + private + + def set_date_format + afnic_format = get_field!("complete date format").value + + @date_format = "%d/%m/%Y" if afnic_format == "DD/MM/YYYY" + end + end +end diff --git a/app/services/whois/response.rb b/app/services/whois/response.rb new file mode 100644 index 0000000..5438986 --- /dev/null +++ b/app/services/whois/response.rb @@ -0,0 +1,11 @@ +module Whois + class Response + attr_accessor :created_on + attr_accessor :updated_on + attr_accessor :expire_on + + def initialize(domain) + @domain = domain + end + end +end diff --git a/test/fixtures/files/whois/domain.fr.txt b/test/fixtures/files/whois/domain.fr.txt new file mode 100644 index 0000000..790e057 --- /dev/null +++ b/test/fixtures/files/whois/domain.fr.txt @@ -0,0 +1,150 @@ +% IANA WHOIS server +% for more information on IANA, visit http://www.iana.org +% This query returned 1 object + +refer: whois.nic.fr + +domain: FR + +organisation: Association Française pour le Nommage Internet en Coopération (A.F.N.I.C.) +address: Immeuble Le Stephenson +address: 1 rue Stephenson +address: 78180 Montigny-le-Bretonneux +address: France + +contact: administrative +name: TLD Admin Contact +organisation: Association Française pour le Nommage Internet en Coopération (A.F.N.I.C.) +address: Immeuble Le Stephenson +address: 1 rue Stephenson +address: 78180 Montigny-le-Bretonneux +address: France +phone: +33 1 39 30 83 05 +fax-no: +33 1 39 30 83 01 +e-mail: tld-admin@nic.fr + +contact: technical +name: TLD Tech Contact +organisation: Association Française pour le Nommage Internet en Coopération (A.F.N.I.C.) +address: Immeuble Le Stephenson +address: 1 rue Stephenson +address: 78180 Montigny-le-Bretonneux +address: France +phone: +33 1 39 30 83 81 +fax-no: +33 1 39 30 83 01 +e-mail: tld-tech@nic.fr + +nserver: D.EXT.NIC.FR 192.5.4.2 2001:500:2e:0:0:0:0:2 +nserver: D.NIC.FR 194.0.9.1 2001:678:c:0:0:0:0:1 +nserver: E.EXT.NIC.FR 193.176.144.22 2a00:d78:0:102:193:176:144:22 +nserver: F.EXT.NIC.FR 194.146.106.46 2001:67c:1010:11:0:0:0:53 +nserver: G.EXT.NIC.FR 194.0.36.1 2001:678:4c:0:0:0:0:1 +ds-rdata: 35095 8 2 23c6caadc9927ee98061f2b52c9b8da6b53f3f648f814a4a86a0faf9854bfa8e +ds-rdata: 42104 8 2 8D913A49C3FA2A39BA0065B4E18BA793E3AD128F7C6C8AA008AEFE0A17985DF5 + +whois: whois.nic.fr + +status: ACTIVE +remarks: Registration information: http://www.nic.fr/ + +created: 1986-09-02 +changed: 2018-01-22 +source: IANA + +%% +%% This is the AFNIC Whois server. +%% +%% complete date format : DD/MM/YYYY +%% short date format : DD/MM +%% version : FRNIC-2.5 +%% +%% Rights restricted by copyright. +%% See https://www.afnic.fr/en/products-and-services/services/whois/whois-special-notice/ +%% +%% Use '-h' option to obtain more information about this service. +%% +%% [5ca0:1e73:fedf:ed4:2af1:f4a6:56e6:8308 REQUEST] >> domain.fr +%% +%% RL Net [##########] - RL IP [#########.] +%% + +domain: domain.fr +status: ACTIVE +hold: NO +holder-c: E1768-FRNIC +admin-c: GC647-FRNIC +tech-c: OVH5-FRNIC +zone-c: NFC1-FRNIC +nsl-id: NSL60350-FRNIC +registrar: OVH +Expiry Date: 17/02/2019 +created: 18/02/2004 +last-update: 28/01/2017 +source: FRNIC + +ns-list: NSL60350-FRNIC +nserver: ns4.dnsserver.fr +nserver: ns0.dnsserver.com +source: FRNIC + +registrar: OVH +type: Isp Option 1 +address: 2 Rue Kellermann +address: 59100 ROUBAIX +country: FR +phone: +33 8 99 88 77 66 +fax-no: +33 3 20 20 20 20 +e-mail: support@registrar.fr +website: http://www.registrar.fr +anonymous: NO +registered: 21/10/1999 +source: FRNIC + +nic-hdl: SB999-FRNIC +type: PERSON +contact: Rex Lorne +address: Chexpire +address: Impasse Pastourelle +address: 13001 Marseille +country: FR +phone: +33 1 23 45 67 89 +registrar: OVH +changed: 18/02/2004 frnic-dbm-updates@nic.fr +anonymous: NO +obsoleted: NO +source: FRNIC + +nic-hdl: OVH5-FRNIC +type: ROLE +contact: OVH NET +address: OVH +address: 140, quai du Sartel +address: 59100 Roubaix +country: FR +phone: +33 8 99 88 77 66 +e-mail: tech@registrar.fr +trouble: Information: http://www.registrar.fr +trouble: Questions: mailto:tech@registrar.fr +trouble: Spam: mailto:abuse@registrar.fr +admin-c: OK217-FRNIC +tech-c: OK217-FRNIC +notify: tech@registrar.fr +registrar: OVH +changed: 11/10/2006 tech@registrar.fr +anonymous: NO +obsoleted: NO +source: FRNIC + +nic-hdl: E9999-FRNIC +type: ORGANIZATION +contact: Chexpire +address: Impasse Pastourelle +address: 13001 Marseille +country: FR +phone: +33 1 23 45 67 89 +e-mail: info@domain.fr +registrar: OVH +changed: 11/03/2012 nic@nic.fr +anonymous: NO +obsoleted: NO +source: FRNIC diff --git a/test/services/whois/command_test.rb b/test/services/whois/command_test.rb new file mode 100644 index 0000000..fb4f52b --- /dev/null +++ b/test/services/whois/command_test.rb @@ -0,0 +1,27 @@ +require "test_helper" +require "whois/command" + +module Whois + class CommandTest < ActiveSupport::TestCase + test "should return the result and log the command" do + result = "mocked whois result" + + mock = Minitest::Mock.new + mock.expect(:execute, result) + + stub = lambda do |program, args, _logger| + assert_equal "whois", program + assert_equal "example.org", args + + mock + end + + SystemCommand.stub(:new, stub) do + command = Command.new("example.org") + assert_equal result, command.run + end + + mock.verify + end + end +end diff --git a/test/services/whois/parser/fr_test.rb b/test/services/whois/parser/fr_test.rb new file mode 100644 index 0000000..2e90ce7 --- /dev/null +++ b/test/services/whois/parser/fr_test.rb @@ -0,0 +1,21 @@ +require "test_helper" +require "whois/parser/fr" +require "whois/response" + +module Whois + class FrTest < ActiveSupport::TestCase + setup do + @parser = Parser::Fr.new("domain.fr") + @domain_fr = file_fixture("whois/domain.fr.txt").read + end + + test "should parse a whois response" do + response = @parser.parse(@domain_fr) + assert_kind_of Response, response + + assert_equal Date.new(2004, 2, 18), response.created_on + assert_equal Date.new(2017, 1, 28), response.updated_on + assert_equal Date.new(2019, 2, 17), response.expire_on + end + end +end diff --git a/test/services/whois/parser_test.rb b/test/services/whois/parser_test.rb new file mode 100644 index 0000000..52a6676 --- /dev/null +++ b/test/services/whois/parser_test.rb @@ -0,0 +1,15 @@ +require "test_helper" +require "whois/parser" +require "whois/errors" + +module Whois + class ParserTest < ActiveSupport::TestCase + test "should instanciate a parser class matching the tld" do + assert_kind_of Parser::Fr, Parser.for("example.fr") + + assert_raises UnsupportedDomainError do + Parser.for("example.xyz") + end + end + end +end diff --git a/test/services/whois_test.rb b/test/services/whois_test.rb new file mode 100644 index 0000000..ea953b8 --- /dev/null +++ b/test/services/whois_test.rb @@ -0,0 +1,9 @@ +require "test_helper" +require "whois" + +class WhoisTest < ActiveSupport::TestCase + test "should instanciate a parser class matching the tld" do + # TODO: stub system command + # assert_kind_of Whois::Response, Whois.ask("example.fr") + end +end From 79165fb5b80a46bc0384a2861b256385b3bfbdd4 Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Wed, 30 May 2018 13:23:15 +0200 Subject: [PATCH 06/12] System calls with Open4 && more tests --- Gemfile | 1 + Gemfile.lock | 2 ++ app/services/system_command.rb | 21 +++++++++-- app/services/whois.rb | 36 +++++++++++++------ app/services/whois/command.rb | 18 ---------- app/services/whois/errors.rb | 8 +++-- test/services/system_command_test.rb | 19 ++++++---- test/services/whois/command_test.rb | 27 --------------- test/services/whois_test.rb | 52 +++++++++++++++++++++++++--- 9 files changed, 113 insertions(+), 71 deletions(-) delete mode 100644 app/services/whois/command.rb delete mode 100644 test/services/whois/command_test.rb diff --git a/Gemfile b/Gemfile index f94c190..14498f3 100644 --- a/Gemfile +++ b/Gemfile @@ -36,6 +36,7 @@ gem 'bcrypt', '~> 3.1.7' # Use ActiveStorage variant # gem 'mini_magick', '~> 4.8' +gem 'open4' gem 'naught' # Reduces boot times through caching; required in config/boot.rb diff --git a/Gemfile.lock b/Gemfile.lock index 03bbdcb..423758d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -167,6 +167,7 @@ GEM notiffany (0.1.1) nenv (~> 0.1) shellany (~> 0.0) + open4 (1.3.4) orm_adapter (0.5.0) parallel (1.12.1) parser (2.5.1.0) @@ -319,6 +320,7 @@ DEPENDENCIES listen (>= 3.0.5, < 3.2) mysql2 (>= 0.4.4, < 0.6.0) naught + open4 pry-byebug pry-rails puma (~> 3.11) diff --git a/app/services/system_command.rb b/app/services/system_command.rb index 604f684..c8f052c 100644 --- a/app/services/system_command.rb +++ b/app/services/system_command.rb @@ -1,5 +1,8 @@ +require "open4" require "null_logger" +SystemCommandResult = Struct.new(:command, :exit_status, :stdout, :stderr) + class SystemCommand attr_reader :program attr_reader :args @@ -14,11 +17,11 @@ class SystemCommand def execute logger.log :before_command, syscmd - raw = `syscmd` + result = call(syscmd) - logger.log :after_command, raw + logger.log :after_command, result - raw + result end def syscmd @@ -31,6 +34,18 @@ class SystemCommand private + def call(cmd) + pid, _, stdout, stderr = Open4.popen4 cmd + _, status = Process.waitpid2 pid + + SystemCommandResult.new( + syscmd, + status.exitstatus, + stdout.read.strip, + stderr.read.strip, + ) + end + def escape_arg(arg) arg.to_s.gsub('"') { '\"' } end diff --git a/app/services/whois.rb b/app/services/whois.rb index 80bfa4b..25a2973 100644 --- a/app/services/whois.rb +++ b/app/services/whois.rb @@ -1,33 +1,47 @@ require "null_logger" require "domain_helper" -require "whois/command" -require "whois/parser" -require "whois/response" +require "system_command" +require_relative "whois/parser" +require_relative "whois/response" +require_relative "whois/errors" module Whois class << self - def ask(domain, logger: NullLogger.new) - Service.new(domain, logger).call + def ask(domain, system_klass: SystemCommand, logger: NullLogger.new) + Service.new(domain, system_klass, logger: logger).call end end class Service attr_reader :domain attr_reader :logger + attr_reader :system_klass - def initialize(domain, logger) + def initialize(domain, system_klass: SystemCommand, logger: NullLogger.new) @domain = domain @logger = logger + @system_klass = system_klass end def call - command = Command.new(domain, logger: logger) - raw_response = command.run + result = run_command + parse(result) + end + def run_command + command = system_klass.new("whois", domain, logger: logger) + result = command.execute + + unless result.exit_status.zero? + fail WhoisCommandError, "Whois command failed with status #{result.exit_status}" + end + + result + end + + def parse(result) parser = Parser.for(domain, logger: logger) - response = parser.parse(raw_response) - - response + parser.parse(result.stdout) end end end diff --git a/app/services/whois/command.rb b/app/services/whois/command.rb deleted file mode 100644 index 0daaebc..0000000 --- a/app/services/whois/command.rb +++ /dev/null @@ -1,18 +0,0 @@ -require "null_logger" -require "system_command" - -module Whois - class Command - attr_reader :logger - attr_reader :domain - - def initialize(domain, logger: NullLogger.new) - @domain = domain - @logger = logger - end - - def run - SystemCommand.new("whois", domain, logger: logger).execute - end - end -end diff --git a/app/services/whois/errors.rb b/app/services/whois/errors.rb index 794d44b..8afc799 100644 --- a/app/services/whois/errors.rb +++ b/app/services/whois/errors.rb @@ -1,6 +1,10 @@ module Whois - class UnsupportedDomainError < StandardError; end - class ParserError < StandardError; end + class WhoisError < StandardError; end + + class WhoisCommandError < WhoisError; end + class UnsupportedDomainError < WhoisError; end + class ParserError < WhoisError; end + class CommentNotFoundError < ParserError; end class FieldNotFoundError < ParserError; end class MissingDateFormatError < ParserError; end diff --git a/test/services/system_command_test.rb b/test/services/system_command_test.rb index 84cd3c4..548a142 100644 --- a/test/services/system_command_test.rb +++ b/test/services/system_command_test.rb @@ -4,16 +4,23 @@ require "system_command" class SystemCommandTest < ActiveSupport::TestCase test "should execute and log a command" do mock_logger = Minitest::Mock.new - expected = 'whois "example.org"' + expected_cmd = 'whois "example.org"' - mock_logger.expect(:log, nil, [:before_command, expected]) - mock_logger.expect(:log, nil, [:after_command, "my result"]) + expected_result = SystemCommandResult.new( + expected_cmd, + 0, + "my result", + "", + ) + + mock_logger.expect(:log, nil, [:before_command, expected_cmd]) + mock_logger.expect(:log, nil, [:after_command, expected_result]) command = SystemCommand.new("whois", "example.org", logger: mock_logger) - assert_equal expected, command.syscmd + assert_equal expected_cmd, command.syscmd - command.stub(:`, "my result") do - assert_equal "my result", command.execute + command.stub(:call, expected_result) do + assert_equal expected_result, command.execute end mock_logger.verify diff --git a/test/services/whois/command_test.rb b/test/services/whois/command_test.rb deleted file mode 100644 index fb4f52b..0000000 --- a/test/services/whois/command_test.rb +++ /dev/null @@ -1,27 +0,0 @@ -require "test_helper" -require "whois/command" - -module Whois - class CommandTest < ActiveSupport::TestCase - test "should return the result and log the command" do - result = "mocked whois result" - - mock = Minitest::Mock.new - mock.expect(:execute, result) - - stub = lambda do |program, args, _logger| - assert_equal "whois", program - assert_equal "example.org", args - - mock - end - - SystemCommand.stub(:new, stub) do - command = Command.new("example.org") - assert_equal result, command.run - end - - mock.verify - end - end -end diff --git a/test/services/whois_test.rb b/test/services/whois_test.rb index ea953b8..3b33154 100644 --- a/test/services/whois_test.rb +++ b/test/services/whois_test.rb @@ -1,9 +1,53 @@ require "test_helper" require "whois" +require "system_command" -class WhoisTest < ActiveSupport::TestCase - test "should instanciate a parser class matching the tld" do - # TODO: stub system command - # assert_kind_of Whois::Response, Whois.ask("example.fr") +module Whois + class ServiceTest < ActiveSupport::TestCase + test "should run the command, return the result" do + result = OpenStruct.new(exit_status: 0) + + mock_system_klass("whois", "example.org", result) do |system_klass| + service = Service.new("example.org", system_klass: system_klass) + assert_equal result, service.run_command + end + end + + test "should raise an exception if exit status > 0" do + result = OpenStruct.new(exit_status: 1) + + mock_system_klass("whois", "example.org", result) do |system_klass| + service = Service.new("example.org", system_klass: system_klass) + + assert_raises WhoisCommandError do + service.run_command + end + end + end + + test "should parse from a command result" do + result = OpenStruct.new( + exit_status: 0, + stdout: file_fixture("whois/domain.fr.txt").read, + ) + + service = Service.new("domain.fr") + assert_kind_of Response, service.parse(result) + end + + def mock_system_klass(program, command_args, result) + system_klass = Minitest::Mock.new + system_command = Minitest::Mock.new.expect(:execute, result) + system_klass.expect(:new, system_command) do |arg1, arg2, logger:| + arg1 == program && + arg2 == command_args && + logger.class == NullLogger + end + + yield system_klass + + system_klass.verify + system_command.verify + end end end From 53fb0d38de5a0b07f41423f73e3491bab026deb2 Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Wed, 30 May 2018 16:48:25 +0200 Subject: [PATCH 07/12] Whois: Time in UTC instead of DateTime --- app/services/whois/parser/base.rb | 2 +- app/services/whois/parser/fr.rb | 12 ++++++++---- app/services/whois/response.rb | 6 +++--- test/services/whois/parser/fr_test.rb | 8 +++++--- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/app/services/whois/parser/base.rb b/app/services/whois/parser/base.rb index ac0c1e5..a0ef594 100644 --- a/app/services/whois/parser/base.rb +++ b/app/services/whois/parser/base.rb @@ -47,7 +47,7 @@ module Whois fail MissingDateFormatError, "Date format not set" if date_format.nil? begin - Date.strptime(str, date_format) + Time.strptime(str, date_format) rescue ArgumentError raise InvalidDateError, "Date `#{str}` does not match format #{date_format}" end diff --git a/app/services/whois/parser/fr.rb b/app/services/whois/parser/fr.rb index 7bd8a58..85e4b96 100644 --- a/app/services/whois/parser/fr.rb +++ b/app/services/whois/parser/fr.rb @@ -19,21 +19,25 @@ module Whois::Parser domain_index = get_field!("domain", value: domain).index created_date = get_value!("created", after: domain_index) - response.created_on = parse_date(created_date) + response.created_at = parse_date(created_date) expire_date = get_value!("Expiry Date", after: domain_index) - response.expire_on = parse_date(expire_date) + response.expire_at = parse_date(expire_date) updated_date = get_value!("last-update", after: domain_index) - response.updated_on = parse_date(updated_date) + response.updated_at = parse_date(updated_date) end private + def parse_date(str) + super "#{str} UTC" + end + def set_date_format afnic_format = get_field!("complete date format").value - @date_format = "%d/%m/%Y" if afnic_format == "DD/MM/YYYY" + @date_format = "%d/%m/%Y %Z" if afnic_format == "DD/MM/YYYY" end end end diff --git a/app/services/whois/response.rb b/app/services/whois/response.rb index 5438986..4360702 100644 --- a/app/services/whois/response.rb +++ b/app/services/whois/response.rb @@ -1,8 +1,8 @@ module Whois class Response - attr_accessor :created_on - attr_accessor :updated_on - attr_accessor :expire_on + attr_accessor :created_at + attr_accessor :updated_at + attr_accessor :expire_at def initialize(domain) @domain = domain diff --git a/test/services/whois/parser/fr_test.rb b/test/services/whois/parser/fr_test.rb index 2e90ce7..017805d 100644 --- a/test/services/whois/parser/fr_test.rb +++ b/test/services/whois/parser/fr_test.rb @@ -13,9 +13,11 @@ module Whois response = @parser.parse(@domain_fr) assert_kind_of Response, response - assert_equal Date.new(2004, 2, 18), response.created_on - assert_equal Date.new(2017, 1, 28), response.updated_on - assert_equal Date.new(2019, 2, 17), response.expire_on + assert_equal Time.new(2004, 2, 18, 0, 0, 0, 0), response.created_at + assert response.created_at.utc? + + assert_equal Time.new(2017, 1, 28, 0, 0, 0, 0), response.updated_at + assert_equal Time.new(2019, 2, 17, 0, 0, 0, 0), response.expire_at end end end From ec4dc321f69c6e5e06a84beadd169f90041476a0 Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Wed, 30 May 2018 16:50:41 +0200 Subject: [PATCH 08/12] Create CheckLog model --- app/models/check.rb | 1 + app/models/check_log.rb | 28 +++++++++++++++++ .../20180530123611_create_check_logs.rb | 14 +++++++++ db/schema.rb | 15 +++++++++- test/fixtures/check_logs.yml | 24 +++++++++++++++ test/models/check_log_test.rb | 30 +++++++++++++++++++ 6 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 app/models/check_log.rb create mode 100644 db/migrate/20180530123611_create_check_logs.rb create mode 100644 test/fixtures/check_logs.yml create mode 100644 test/models/check_log_test.rb diff --git a/app/models/check.rb b/app/models/check.rb index a409c75..2276dbe 100644 --- a/app/models/check.rb +++ b/app/models/check.rb @@ -28,6 +28,7 @@ class Check < ApplicationRecord belongs_to :user + has_many :logs, class_name: "CheckLog" enum kind: [:domain, :ssl] diff --git a/app/models/check_log.rb b/app/models/check_log.rb new file mode 100644 index 0000000..29ccb2c --- /dev/null +++ b/app/models/check_log.rb @@ -0,0 +1,28 @@ +# == Schema Information +# +# Table name: check_logs +# +# id :bigint(8) not null, primary key +# error :text(65535) +# exit_status :integer +# parsed_response :text(65535) +# raw_response :text(65535) +# status :integer +# created_at :datetime not null +# updated_at :datetime not null +# check_id :bigint(8) +# +# Indexes +# +# index_check_logs_on_check_id (check_id) +# +# Foreign Keys +# +# fk_rails_... (check_id => checks.id) +# + +class CheckLog < ApplicationRecord + belongs_to :check + + enum status: [:pending, :succeed, :failed] +end diff --git a/db/migrate/20180530123611_create_check_logs.rb b/db/migrate/20180530123611_create_check_logs.rb new file mode 100644 index 0000000..d5a2fc6 --- /dev/null +++ b/db/migrate/20180530123611_create_check_logs.rb @@ -0,0 +1,14 @@ +class CreateCheckLogs < ActiveRecord::Migration[5.2] + def change + create_table :check_logs do |t| + t.references :check, foreign_key: true + t.text :raw_response + t.integer :exit_status + t.text :parsed_response + t.text :error + t.integer :status + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 25fe422..d87ebf7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,19 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2018_05_29_092950) do +ActiveRecord::Schema.define(version: 2018_05_30_123611) do + + create_table "check_logs", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t| + t.bigint "check_id" + t.text "raw_response" + t.integer "exit_status" + t.text "parsed_response" + t.text "error" + t.integer "status" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["check_id"], name: "index_check_logs_on_check_id" + end create_table "checks", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t| t.bigint "user_id" @@ -53,5 +65,6 @@ ActiveRecord::Schema.define(version: 2018_05_29_092950) do t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true end + add_foreign_key "check_logs", "checks" add_foreign_key "checks", "users" end diff --git a/test/fixtures/check_logs.yml b/test/fixtures/check_logs.yml new file mode 100644 index 0000000..aa39702 --- /dev/null +++ b/test/fixtures/check_logs.yml @@ -0,0 +1,24 @@ +# == Schema Information +# +# Table name: check_logs +# +# id :bigint(8) not null, primary key +# error :text(65535) +# exit_status :integer +# parsed_response :text(65535) +# raw_response :text(65535) +# status :integer +# created_at :datetime not null +# updated_at :datetime not null +# check_id :bigint(8) +# +# Indexes +# +# index_check_logs_on_check_id (check_id) +# +# Foreign Keys +# +# fk_rails_... (check_id => checks.id) +# + +# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html diff --git a/test/models/check_log_test.rb b/test/models/check_log_test.rb new file mode 100644 index 0000000..52ec971 --- /dev/null +++ b/test/models/check_log_test.rb @@ -0,0 +1,30 @@ +# == Schema Information +# +# Table name: check_logs +# +# id :bigint(8) not null, primary key +# error :text(65535) +# exit_status :integer +# parsed_response :text(65535) +# raw_response :text(65535) +# status :integer +# created_at :datetime not null +# updated_at :datetime not null +# check_id :bigint(8) +# +# Indexes +# +# index_check_logs_on_check_id (check_id) +# +# Foreign Keys +# +# fk_rails_... (check_id => checks.id) +# + +require "test_helper" + +class CheckLogTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end From 123bec60af36a124fd5bcd5fc09579809d549f2f Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Wed, 30 May 2018 16:51:24 +0200 Subject: [PATCH 09/12] CheckLogger for command & parser --- app/services/check_logger.rb | 47 +++++++++++++++++ app/services/whois.rb | 3 ++ app/services/whois/errors.rb | 1 - app/services/whois/parser/base.rb | 4 ++ app/services/whois/response.rb | 4 ++ test/services/check_logger_test.rb | 81 ++++++++++++++++++++++++++++++ 6 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 app/services/check_logger.rb create mode 100644 test/services/check_logger_test.rb diff --git a/app/services/check_logger.rb b/app/services/check_logger.rb new file mode 100644 index 0000000..829165d --- /dev/null +++ b/app/services/check_logger.rb @@ -0,0 +1,47 @@ +class CheckLogger + attr_reader :check_log + + def initialize(check) + @check_log = CheckLog.create!(check: check, status: :pending) + end + + def log(event, message) + case event + when :after_command + log_command_result(message) + when :parsed_response + log_parsed_response(message) + when :parser_error, :service_error + log_error(message) + end + end + + private + + def log_command_result(result) + check_log.exit_status = result.exit_status + check_log.raw_response = result.stdout + + if result.exit_status > 0 # rubocop:disable Style/NumericPredicate + check_log.error = result.stderr + check_log.status = :failed + end + + check_log.save! + end + + def log_parsed_response(response) + check_log.parsed_response = response.to_json + + if response.valid? + check_log.succeed! + else + check_log.failed! + end + end + + def log_error(exception) + check_log.error = [exception.message, exception.backtrace].join("\n") + check_log.failed! + end +end diff --git a/app/services/whois.rb b/app/services/whois.rb index 25a2973..8ab3599 100644 --- a/app/services/whois.rb +++ b/app/services/whois.rb @@ -26,6 +26,9 @@ module Whois def call result = run_command parse(result) + rescue StandardError => ex + logger.log :service_error, ex + raise end def run_command diff --git a/app/services/whois/errors.rb b/app/services/whois/errors.rb index 8afc799..209de12 100644 --- a/app/services/whois/errors.rb +++ b/app/services/whois/errors.rb @@ -5,7 +5,6 @@ module Whois class UnsupportedDomainError < WhoisError; end class ParserError < WhoisError; end - class CommentNotFoundError < ParserError; end class FieldNotFoundError < ParserError; end class MissingDateFormatError < ParserError; end class InvalidDateError < ParserError; end diff --git a/app/services/whois/parser/base.rb b/app/services/whois/parser/base.rb index a0ef594..4597cb3 100644 --- a/app/services/whois/parser/base.rb +++ b/app/services/whois/parser/base.rb @@ -26,7 +26,11 @@ module Whois do_parse + logger.log :parsed_response, response + response + rescue StandardError => ex + logger.log :parser_error, ex end protected diff --git a/app/services/whois/response.rb b/app/services/whois/response.rb index 4360702..e4718fa 100644 --- a/app/services/whois/response.rb +++ b/app/services/whois/response.rb @@ -7,5 +7,9 @@ module Whois def initialize(domain) @domain = domain end + + def valid? + created_at.present? + end end end diff --git a/test/services/check_logger_test.rb b/test/services/check_logger_test.rb new file mode 100644 index 0000000..79a6e73 --- /dev/null +++ b/test/services/check_logger_test.rb @@ -0,0 +1,81 @@ +require "test_helper" +require "check_logger" +require "system_command" + +class CheckLoggerTest < ActiveSupport::TestCase + setup do + @check = checks(:domain_example_org) + @logger = CheckLogger.new(@check) + end + + test "should create a pending CheckLog" do + assert_difference -> { CheckLog.where(check: @check).count }, +1 do + CheckLogger.new(@check) + end + + assert last_log.pending? + end + + test "should log a success raw result command" do + result = SystemCommandResult.new("command", 0, "the result", "") + + assert_no_difference -> { CheckLog.where(check: @check).count } do + @logger.log :after_command, result + end + + assert_equal "the result", @logger.check_log.raw_response + assert_nil @logger.check_log.error + assert_equal 0, @logger.check_log.exit_status + assert @logger.check_log.pending? + end + + test "should log a raw result command with an error" do + result = SystemCommandResult.new("command", 1, "optional stdout", "an error occured") + @logger.log :after_command, result + + assert_equal "optional stdout", @logger.check_log.raw_response + assert_equal "an error occured", @logger.check_log.error + assert_equal 1, @logger.check_log.exit_status + assert @logger.check_log.failed? + end + + test "should log a successful parsed command" do + response = OpenStruct.new( + domain: "example.fr", + extracted: "some data", + valid?: true, + ) + @logger.log :parsed_response, response + + assert_equal response.to_json, @logger.check_log.parsed_response + assert_nil @logger.check_log.error + assert @logger.check_log.succeed? + end + + test "should log parser error with a backtrace" do + @logger.log :parser_error, mock_exception + + assert_includes @logger.check_log.error, "my error occured" + assert_includes @logger.check_log.error, "minitest.rb" + assert @logger.check_log.failed? + end + + test "should log service error" do + @logger.log :service_error, mock_exception + + assert_not_nil @logger.check_log.error + assert @logger.check_log.failed? + end + + private + + def last_log + CheckLog.where(check: @check).last + end + + def mock_exception + exception = ArgumentError.new("my error occured") + exception.set_backtrace(caller) + exception + end +end From 62a53314c5f82964a11c72b0dc24633739c10a2d Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Wed, 30 May 2018 18:15:20 +0200 Subject: [PATCH 10/12] Whois: intercept domain not found --- app/services/whois/errors.rb | 1 + app/services/whois/parser/base.rb | 11 +++ app/services/whois/parser/entry/base.rb | 4 ++ app/services/whois/parser/entry/text.rb | 4 ++ app/services/whois/parser/fr.rb | 61 +++++++++------- .../files/whois/willneverexist.fr.txt | 71 +++++++++++++++++++ test/services/whois/parser/fr_test.rb | 19 +++-- 7 files changed, 139 insertions(+), 32 deletions(-) create mode 100644 test/fixtures/files/whois/willneverexist.fr.txt diff --git a/app/services/whois/errors.rb b/app/services/whois/errors.rb index 209de12..219ed84 100644 --- a/app/services/whois/errors.rb +++ b/app/services/whois/errors.rb @@ -3,6 +3,7 @@ module Whois class WhoisCommandError < WhoisError; end class UnsupportedDomainError < WhoisError; end + class DomainNotFoundError < WhoisError; end class ParserError < WhoisError; end class FieldNotFoundError < ParserError; end diff --git a/app/services/whois/parser/base.rb b/app/services/whois/parser/base.rb index 4597cb3..64e132d 100644 --- a/app/services/whois/parser/base.rb +++ b/app/services/whois/parser/base.rb @@ -31,6 +31,7 @@ module Whois response rescue StandardError => ex logger.log :parser_error, ex + raise end protected @@ -57,6 +58,16 @@ module Whois end end + def comment_include?(str) + entries.any? { |e| + e.comment? && e.text? && e.text.include?(str) + } + end + + def raise_not_found + fail DomainNotFoundError, "Domain #{domain} not found in the registry database." + end + private def build_entries(raw) diff --git a/app/services/whois/parser/entry/base.rb b/app/services/whois/parser/entry/base.rb index a7a4c0c..5356772 100644 --- a/app/services/whois/parser/entry/base.rb +++ b/app/services/whois/parser/entry/base.rb @@ -24,6 +24,10 @@ module Whois def field? false end + + def text? + false + end end end end diff --git a/app/services/whois/parser/entry/text.rb b/app/services/whois/parser/entry/text.rb index 4bb8361..68ac771 100644 --- a/app/services/whois/parser/entry/text.rb +++ b/app/services/whois/parser/entry/text.rb @@ -10,6 +10,10 @@ module Whois super index @text = text.strip end + + def text? + true + end end end end diff --git a/app/services/whois/parser/fr.rb b/app/services/whois/parser/fr.rb index 85e4b96..4f05db2 100644 --- a/app/services/whois/parser/fr.rb +++ b/app/services/whois/parser/fr.rb @@ -1,43 +1,52 @@ require "domain_helper" +require "whois/errors" require_relative "base" -module Whois::Parser - class Fr < Base - SUPPORTED_TLD = %w[.fr].freeze - COMMENT_REGEX = /^%+ +(?.+)$/ - FIELD_REGEX = /^(?[^:]+)\s*:\s+(?.+)$/ +module Whois + module Parser + class Fr < Base + SUPPORTED_TLD = %w[.fr].freeze + COMMENT_REGEX = /^%+ +(?.+)$/ + FIELD_REGEX = /^(?[^:]+)\s*:\s+(?.+)$/ - def self.supports?(domain) - SUPPORTED_TLD.include?(tld(domain)) - end + def self.supports?(domain) + SUPPORTED_TLD.include?(tld(domain)) + end - protected + protected - def do_parse - set_date_format + def do_parse + raise_not_found if comment_include?("No entries found") - domain_index = get_field!("domain", value: domain).index + set_date_format - created_date = get_value!("created", after: domain_index) - response.created_at = parse_date(created_date) + extract_values + end - expire_date = get_value!("Expiry Date", after: domain_index) - response.expire_at = parse_date(expire_date) + private - updated_date = get_value!("last-update", after: domain_index) - response.updated_at = parse_date(updated_date) - end + def extract_values + domain_index = get_field!("domain", value: domain).index - private + created_date = get_value!("created", after: domain_index) + response.created_at = parse_date(created_date) - def parse_date(str) - super "#{str} UTC" - end + expire_date = get_value!("Expiry Date", after: domain_index) + response.expire_at = parse_date(expire_date) - def set_date_format - afnic_format = get_field!("complete date format").value + updated_date = get_value!("last-update", after: domain_index) + response.updated_at = parse_date(updated_date) + end - @date_format = "%d/%m/%Y %Z" if afnic_format == "DD/MM/YYYY" + def parse_date(str) + super "#{str} UTC" + end + + def set_date_format + afnic_format = get_field!("complete date format").value + + @date_format = "%d/%m/%Y %Z" if afnic_format == "DD/MM/YYYY" + end end end end diff --git a/test/fixtures/files/whois/willneverexist.fr.txt b/test/fixtures/files/whois/willneverexist.fr.txt new file mode 100644 index 0000000..ce2bbcf --- /dev/null +++ b/test/fixtures/files/whois/willneverexist.fr.txt @@ -0,0 +1,71 @@ +% IANA WHOIS server +% for more information on IANA, visit http://www.iana.org +% This query returned 1 object + +refer: whois.nic.fr + +domain: FR + +organisation: Association Française pour le Nommage Internet en Coopération (A.F.N.I.C.) +address: Immeuble Le Stephenson +address: 1 rue Stephenson +address: 78180 Montigny-le-Bretonneux +address: France + +contact: administrative +name: TLD Admin Contact +organisation: Association Française pour le Nommage Internet en Coopération (A.F.N.I.C.) +address: Immeuble Le Stephenson +address: 1 rue Stephenson +address: 78180 Montigny-le-Bretonneux +address: France +phone: +33 1 39 30 83 05 +fax-no: +33 1 39 30 83 01 +e-mail: tld-admin@nic.fr + +contact: technical +name: TLD Tech Contact +organisation: Association Française pour le Nommage Internet en Coopération (A.F.N.I.C.) +address: Immeuble Le Stephenson +address: 1 rue Stephenson +address: 78180 Montigny-le-Bretonneux +address: France +phone: +33 1 39 30 83 81 +fax-no: +33 1 39 30 83 01 +e-mail: tld-tech@nic.fr + +nserver: D.EXT.NIC.FR 192.5.4.2 2001:500:2e:0:0:0:0:2 +nserver: D.NIC.FR 194.0.9.1 2001:678:c:0:0:0:0:1 +nserver: E.EXT.NIC.FR 193.176.144.22 2a00:d78:0:102:193:176:144:22 +nserver: F.EXT.NIC.FR 194.146.106.46 2001:67c:1010:11:0:0:0:53 +nserver: G.EXT.NIC.FR 194.0.36.1 2001:678:4c:0:0:0:0:1 +ds-rdata: 35095 8 2 23c6caadc9927ee98061f2b52c9b8da6b53f3f648f814a4a86a0faf9843e2c4e +ds-rdata: 42104 8 2 8D913A49C3FA2A39BA0065B4E18BA793E3AD128F7C6C8AA008AEFE0A14435DD5 + +whois: whois.nic.fr + +status: ACTIVE +remarks: Registration information: http://www.nic.fr/ + +created: 1986-09-02 +changed: 2018-01-22 +source: IANA + +%% +%% This is the AFNIC Whois server. +%% +%% complete date format : DD/MM/YYYY +%% short date format : DD/MM +%% version : FRNIC-2.5 +%% +%% Rights restricted by copyright. +%% See https://www.afnic.fr/en/products-and-services/services/whois/whois-special-notice/ +%% +%% Use '-h' option to obtain more information about this service. +%% +%% [11.22.33.44 REQUEST] >> willneverexist.fr +%% +%% RL Net [##########] - RL IP [#########.] +%% + +%% No entries found in the AFNIC Database. diff --git a/test/services/whois/parser/fr_test.rb b/test/services/whois/parser/fr_test.rb index 017805d..14f8e77 100644 --- a/test/services/whois/parser/fr_test.rb +++ b/test/services/whois/parser/fr_test.rb @@ -1,16 +1,14 @@ require "test_helper" require "whois/parser/fr" require "whois/response" +require "whois/errors" module Whois class FrTest < ActiveSupport::TestCase - setup do - @parser = Parser::Fr.new("domain.fr") - @domain_fr = file_fixture("whois/domain.fr.txt").read - end - test "should parse a whois response" do - response = @parser.parse(@domain_fr) + parser = Parser::Fr.new("domain.fr") + domain_fr = file_fixture("whois/domain.fr.txt").read + response = parser.parse(domain_fr) assert_kind_of Response, response assert_equal Time.new(2004, 2, 18, 0, 0, 0, 0), response.created_at @@ -19,5 +17,14 @@ module Whois assert_equal Time.new(2017, 1, 28, 0, 0, 0, 0), response.updated_at assert_equal Time.new(2019, 2, 17, 0, 0, 0, 0), response.expire_at end + + test "should raises DomainNotFoundError when domain is not registered" do + parser = Parser::Fr.new("willneverexist.fr") + not_found_fr = file_fixture("whois/willneverexist.fr.txt").read + + assert_raises DomainNotFoundError do + parser.parse(not_found_fr) + end + end end end From 1179a10775b84f4c6c74f0d5a38ae6c9f202c132 Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Wed, 30 May 2018 18:15:56 +0200 Subject: [PATCH 11/12] Checks form: do not ask for created at anymore --- app/models/check.rb | 1 - app/views/checks/_form.html.erb | 1 - app/views/checks/_table.html.erb | 4 +++- app/views/checks/edit.html.erb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/models/check.rb b/app/models/check.rb index 2276dbe..f3bbbd4 100644 --- a/app/models/check.rb +++ b/app/models/check.rb @@ -40,7 +40,6 @@ class Check < ApplicationRecord validates :kind, presence: true validates :domain, presence: true - validates :domain_created_at, presence: true validate :domain_created_at_past validate :domain_updated_at_past validates :comment, length: { maximum: 255 } diff --git a/app/views/checks/_form.html.erb b/app/views/checks/_form.html.erb index 79be782..ffd7c39 100644 --- a/app/views/checks/_form.html.erb +++ b/app/views/checks/_form.html.erb @@ -5,7 +5,6 @@ <%= f.input :kind, as: :radio_buttons, collection: Check.kinds.keys if check.new_record? %> <% end %> - <%= f.input :domain_created_at, as: :date, start_year: 1990, end_year: Date.today.year %> <%= f.input :comment %> <%= f.input :vendor %> diff --git a/app/views/checks/_table.html.erb b/app/views/checks/_table.html.erb index 7fdf93c..a25efbd 100644 --- a/app/views/checks/_table.html.erb +++ b/app/views/checks/_table.html.erb @@ -2,7 +2,9 @@
Domain: <%= check.domain %>
Kind: <%= check.kind %>
-
Date de création: <%= check.domain_created_at.to_date %>
+
Created date: <%= l(check.domain_created_at.to_date) if check.domain_created_at.present? %>
+
Update date: <%= l(check.domain_updated_at.to_date) if check.domain_updated_at.present? %>
+
Expire date: <%= l(check.domain_expire_at.to_date) if check.domain_expire_at.present? %>
<% if check.comment? %>
Comment: <%= check.comment %>
diff --git a/app/views/checks/edit.html.erb b/app/views/checks/edit.html.erb index 694395b..86ae5d1 100644 --- a/app/views/checks/edit.html.erb +++ b/app/views/checks/edit.html.erb @@ -8,7 +8,7 @@
-
+
<%= button_to("Delete", check_path(@check), class: "btn btn-danger", method: :delete, data: { confirm: "Are you sure ?" }) %>
From 53ca6f1f7fb704a825c6b5175164de20b29a2fea Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Wed, 30 May 2018 18:16:43 +0200 Subject: [PATCH 12/12] Ask Job at whois creation (inline in dev) --- app/jobs/whois_sync_job.rb | 27 +++++++++++++++++++++++++++ app/models/check.rb | 9 +++++++++ app/services/whois.rb | 2 +- config/environments/development.rb | 2 ++ test/jobs/whois_sync_job_test.rb | 7 +++++++ 5 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 app/jobs/whois_sync_job.rb create mode 100644 test/jobs/whois_sync_job_test.rb diff --git a/app/jobs/whois_sync_job.rb b/app/jobs/whois_sync_job.rb new file mode 100644 index 0000000..45ecceb --- /dev/null +++ b/app/jobs/whois_sync_job.rb @@ -0,0 +1,27 @@ +class WhoisSyncJob < ApplicationJob + queue_as :default + + rescue_from ActiveRecord::RecordNotFound do; end + + attr_reader :check + + def perform(check_id) + @check = Check.find(check_id) + response = Whois.ask(check.domain) + + return unless response.valid? + + update_from_response(response) + + check.save! + rescue Whois::DomainNotFoundError + check.active = false + check.save! + end + + def update_from_response(response) + check.domain_created_at = response.created_at + check.domain_updated_at = response.updated_at + check.domain_expire_at = response.expire_at + end +end diff --git a/app/models/check.rb b/app/models/check.rb index f3bbbd4..f9be2f1 100644 --- a/app/models/check.rb +++ b/app/models/check.rb @@ -45,6 +45,8 @@ class Check < ApplicationRecord validates :comment, length: { maximum: 255 } validates :vendor, length: { maximum: 255 } + after_save :enqueue_sync + protected def domain_created_at_past @@ -54,4 +56,11 @@ class Check < ApplicationRecord def domain_updated_at_past errors.add(:domain_updated_at, :past) if domain_updated_at.present? && domain_updated_at.future? end + + def enqueue_sync + return unless active? + return unless saved_changes.key?("domain") + + WhoisSyncJob.perform_later(id) if domain? + end end diff --git a/app/services/whois.rb b/app/services/whois.rb index 8ab3599..68c9b27 100644 --- a/app/services/whois.rb +++ b/app/services/whois.rb @@ -8,7 +8,7 @@ require_relative "whois/errors" module Whois class << self def ask(domain, system_klass: SystemCommand, logger: NullLogger.new) - Service.new(domain, system_klass, logger: logger).call + Service.new(domain, system_klass: system_klass, logger: logger).call end end diff --git a/config/environments/development.rb b/config/environments/development.rb index 1888e36..0c7f9d2 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -67,4 +67,6 @@ Rails.application.configure do # Use an evented file watcher to asynchronously detect changes in source code, # routes, locales, etc. This feature depends on the listen gem. config.file_watcher = ActiveSupport::EventedFileUpdateChecker + + config.active_job.queue_adapter = :inline end diff --git a/test/jobs/whois_sync_job_test.rb b/test/jobs/whois_sync_job_test.rb new file mode 100644 index 0000000..56c5ee4 --- /dev/null +++ b/test/jobs/whois_sync_job_test.rb @@ -0,0 +1,7 @@ +require "test_helper" + +class WhoisSyncJobTest < ActiveJob::TestCase + # test "the truth" do + # assert true + # end +end