Merge pull request #187 from ninoseki/fix-issues

fix: fix issues

lib/miteru/crawler.rb

@@ -8,7 +8,7 @@ class Crawler < Service
     def call(website)
       Try[OpenSSL::SSL::SSLError, ::HTTP::Error, Addressable::URI::InvalidURIError] do
         Miteru.logger.info("Website:#{website.truncated_url} has #{website.kits.length} kit(s).")
-        return unless website.has_kits?
+        return unless website.kits?
 
         notify website
 
@@ -17,11 +17,10 @@ def call(website)
         website.kits.each do |kit|
           downloader = Downloader.new(kit)
           result = downloader.result
-
           if result.success?
-            Miteru.logger.info("Kit:#{kit.truncated_url} downloaded as #{result.value!}")
+            Miteru.logger.info("Kit:#{kit.truncated_url} downloaded as #{result.value!}.")
           else
-            Miteru.logger.warn("Kit:#{kit.truncated_url} failed to download - #{result.failure}")
+            Miteru.logger.warn("Kit:#{kit.truncated_url} failed to download - #{result.failure}.")
           end
         end
       end.recover { nil }.value!
@@ -33,8 +32,18 @@ def auto_download?
       Miteru.config.auto_download
     end
 
+    #
+    # @param [Miteru::Website] website
+    #
     def notify(website)
-      Parallel.each(notifiers) { |notifier| notifier.call(website) }
+      notifiers.each do |notifier|
+        result = notifier.result(website)
+        if result.success?
+          Miteru.logger.info("Notifier:#{notifier.name} succeeded.")
+        else
+          Miteru.logger.warn("Notifier:#{notifier.name} failed - #{result.failure}.")
+        end
+      end
     end
 
     #

lib/miteru/kit.rb

@@ -25,6 +25,8 @@ class Kit < Service
     # @param [String] source
     #
     def initialize(url, source:)
+      super()
+
       @url = url
       @source = source
 
@@ -37,7 +39,6 @@ def initialize(url, source:)
     def valid?
       # make a HEAD request for the validation
       before_validation
-
       valid_ext? && reachable? && valid_mime_type? && valid_content_length?
     end
 
@@ -110,7 +111,7 @@ def valid_ext?
     end
 
     def http
-      HTTP::Factory.build
+      HTTP::Factory.build(raise_exception: false)
     end
 
     def before_validation

lib/miteru/notifiers/base.rb

@@ -11,6 +11,10 @@ def callable?
         raise NotImplementedError
       end
 
+      def name
+        @name ||= self.class.to_s.split("::").last
+      end
+
       class << self
         def inherited(child)
           super

lib/miteru/notifiers/urlscan.rb

@@ -9,10 +9,7 @@ class UrlScan < Base
       def call(website)
         return unless callable?
 
-        kits = website.kits.select(&:downloaded?)
-        return unless kits.any?
-
-        kits.each { |kit| submit(kit.url) }
+        website.kits.each { |kit| submit(kit.url) }
       end
 
       def callable?
@@ -33,7 +30,7 @@ def headers
       end
 
       def timeout
-        Miteru.config.timeout
+        Miteru.config.api_timeout
       end
 
       def tags
@@ -45,7 +42,7 @@ def visibility
       end
 
       def submit(url)
-        http.post("/api/v1/scan/", json: {tags:, visibility:, url:})
+        http.post("https://urlscan.io/api/v1/scan/", json: {tags:, visibility:, url:})
       end
     end
   end

lib/miteru/orchestrator.rb

@@ -6,16 +6,21 @@ def call
       Miteru.logger.info("#{websites.length} websites loaded in total.") if verbose?
 
       if Miteru.sidekiq?
-
         websites.each do |website|
           Jobs::CrawleJob.perform_async(website.url, website.source)
-          Miteru.logger.info("Website:#{website.truncated_url} crawler job queued") if verbose?
+          Miteru.logger.info("Website:#{website.truncated_url} crawler job queued.") if verbose?
         end
       else
         Miteru.logger.info("Use #{threads} thread(s).") if verbose?
         Parallel.each(websites, in_threads: threads) do |website|
-          Miteru.logger.info("Website:#{website.truncated_url} crawling started") if verbose?
-          crawl(website)
+          Miteru.logger.info("Website:#{website.truncated_url} crawling started.") if verbose?
+
+          result = Crawler.result(website)
+          if result.success?
+            Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
+          else
+            Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
+          end
         end
       end
     end

lib/miteru/sidekiq/jobs.rb

@@ -14,7 +14,14 @@ class CrawleJob
       #
       def perform(url, source)
         website = Miteru::Website.new(url, source:)
-        with_db_connection { Crawler.call(website) }
+        with_db_connection do
+          result = Crawler.result(website)
+          if result.success?
+            Miteru.logger.info("Crawler:#{website.truncated_url} succeeded.")
+          else
+            Miteru.logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
+          end
+        end
       end
     end
   end

lib/miteru/website.rb

@@ -33,15 +33,8 @@ def index?
       title.to_s.start_with? "Index of"
     end
 
-    def has_kits?
-      @has_kits ||= lambda do
-        Try[Addressable::URI::InvalidURIError, Encoding::CompatibilityError, ::HTTP::Error, LL::ParserError,
-          OpenSSL::SSL::SSLError, StatusError, ArgumentError] do
-          !kits.empty?
-        end.recover do
-          false
-        end.value!
-      end.call
+    def kits?
+      kits.any?
     end
 
     def links
@@ -81,8 +74,9 @@ def href_links
       Try[Addressable::URI::InvalidURIError, Encoding::CompatibilityError, ::HTTP::Error, LL::ParserError,
         OpenSSL::SSL::SSLError, StatusError, ArgumentError] do
         doc.css("a").filter_map { |a| a.get("href") }.map do |href|
-          href = href.start_with?("/") ? href : "/#{href}"
-          url + href
+          normalized_href = href.start_with?("/") ? href : "/#{href}"
+          normalized_url = url.end_with?("/") ? url.delete_suffix("/") : url
+          normalized_url + normalized_href
         end
       end.recover { [] }.value!
     end

miteru.gemspec

@@ -40,6 +40,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'webmock', '~> 3.19'
 
   spec.add_dependency 'activerecord', '7.1.3'
+  spec.add_dependency 'addressable', '2.8.6'
   spec.add_dependency 'anyway_config', '2.6.2'
   spec.add_dependency 'colorize', '1.1.0'
   spec.add_dependency 'dotenv', '2.8.1'

spec/website_spec.rb

@@ -25,11 +25,9 @@
     end
   end
 
-  context "when giving a url which contains a phishing kit" do
-    describe "#has_kits?" do
-      it do
-        expect(website.has_kits?).to be(true)
-      end
+  describe "#kits?" do
+    it do
+      expect(website.kits?).to be(true)
     end
   end
 end