(FACT-2619) External facts cache

.rubocop.yml

@@ -80,6 +80,7 @@ Metrics/ClassLength:
     - 'lib/framework/core/options/option_store.rb'
     - 'lib/framework/cli/cli.rb'
     - 'lib/resolvers/networking_linux_resolver.rb'
+    - 'lib/framework/core/cache_manager.rb'
 
 Naming/AccessorMethodName:
   Exclude:

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config --exclude-limit 1000`
-# on 2020-05-11 09:20:37 +0300 using RuboCop version 0.74.0.
+# on 2020-06-10 16:06:37 +0300 using RuboCop version 0.81.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -110,7 +110,7 @@ RSpec/LeakyConstantDeclaration:
     - 'spec/facter/resolvers/macosx/mountpoints_resolver_spec.rb'
     - 'spec/facter/resolvers/utils/windows/network_utils_spec.rb'
 
-# Offense count: 99
+# Offense count: 96
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: have_received, receive
 RSpec/MessageSpies:
@@ -177,7 +177,7 @@ RSpec/SubjectStub:
     - 'spec/custom_facts/util/fact_spec.rb'
     - 'spec/custom_facts/util/resolution_spec.rb'
 
-# Offense count: 131
+# Offense count: 128
 # Configuration parameters: IgnoreNameless, IgnoreSymbolicNames.
 RSpec/VerifiedDoubles:
   Exclude:

lib/custom_facts/util/collection.rb

@@ -95,8 +95,7 @@ def external_facts
         return @external_facts unless @external_facts.nil?
 
         load_external_facts
-        external_facts = @facts.select { |_k, v| v.options[:fact_type] == :external }
-        @external_facts = Facter::Utils.deep_copy(external_facts.keys)
+        @external_facts = @facts.select { |_k, v| v.options[:fact_type] == :external }
       end
 
       def invalidate_custom_facts
@@ -148,11 +147,7 @@ def to_hash
 
       def value(name)
         fact = fact(name)
-        fact_value = fact&.value
-        return Facter.core_value(name) if fact_value.nil?
-
-        core_value = Facter.core_value(name) if fact.used_resolution_weight <= 0
-        core_value.nil? ? fact_value : core_value
+        fact&.value
       end
 
       private

lib/custom_facts/util/config.rb

@@ -8,7 +8,7 @@ module LegacyFacter
   module Util
     module Config
       def self.ext_fact_loader
-        @ext_fact_loader || LegacyFacter::Util::DirectoryLoader.default_loader
+        @ext_fact_loader || LegacyFacter::Util::DirectoryLoader.new
       end
 
       def self.ext_fact_loader=(loader)

lib/custom_facts/util/directory_loader.rb

@@ -30,55 +30,82 @@ class NoSuchDirectoryError < RuntimeError
       EXTERNAL_FACT_WEIGHT = 10_000
 
       # Directory for fact loading
-      attr_reader :directory
+      attr_reader :directories
 
-      def initialize(dir, weight = nil)
-        @directory = dir
-        @weight = weight || EXTERNAL_FACT_WEIGHT
+      def initialize(dir = LegacyFacter::Util::Config.external_facts_dirs, weight = EXTERNAL_FACT_WEIGHT)
+        @directories = [dir].flatten
+        @weight = weight
       end
 
-      def self.loader_for(dir)
-        raise NoSuchDirectoryError unless File.directory?(dir)
+      # Load facts from files in fact directory using the relevant parser classes to
+      # parse them.
+      def load(collection)
+        weight = @weight
+
+        searched_facts, cached_facts = load_directory_entries(collection)
+
+        load_cached_facts(collection, cached_facts, weight)
 
-        LegacyFacter::Util::DirectoryLoader.new(dir)
+        load_searched_facts(collection, searched_facts, weight)
       end
 
-      def self.default_loader
-        loaders = LegacyFacter::Util::Config.external_facts_dirs.collect do |dir|
-          LegacyFacter::Util::DirectoryLoader.new(dir)
+      private
+
+      def load_directory_entries(_collection)
+        cm = Facter::CacheManager.new
+        facts = []
+        entries.each do |file|
+          basename = File.basename(file)
+          if facts.find { |f| f.name == basename } && cm.group_cached?(basename)
+            Facter.log_exception(Exception.new("Caching is enabled for group \"#{basename}\" while "\
+              'there are at least two external facts files with the same filename'))
+          else
+            searched_fact = Facter::SearchedFact.new(basename, nil, [], nil, :file)
+            searched_fact.file = file
+            facts << searched_fact
+          end
         end
-        LegacyFacter::Util::CompositeLoader.new(loaders)
+
+        cm.resolve_facts(facts)
       end
 
-      # Load facts from files in fact directory using the relevant parser classes to
-      # parse them.
-      def load(collection)
-        weight = @weight
-        entries.each do |file|
-          parser = LegacyFacter::Util::Parser.parser_for(file)
+      def load_cached_facts(collection, cached_facts, weight)
+        cached_facts.each do |cached_fact|
+          collection.add(cached_fact.name, value: cached_fact.value, fact_type: :external,
+                                           file: cached_fact.file) { has_weight(weight) }
+        end
+      end
+
+      def load_searched_facts(collection, searched_facts, weight)
+        searched_facts.each do |fact|
+          parser = LegacyFacter::Util::Parser.parser_for(fact.file)
           next if parser.nil?
 
           data = parser.results
           if data == false
-            LegacyFacter.warn "Could not interpret fact file #{file}"
+            LegacyFacter.warn "Could not interpret fact file #{fact.file}"
           elsif (data == {}) || data.nil?
-            LegacyFacter.warn "Fact file #{file} was parsed but returned an empty data set"
+            LegacyFacter.warn "Fact file #{fact.file} was parsed but returned an empty data set"
           else
-            data.each { |p, v| collection.add(p, value: v, fact_type: :external) { has_weight(weight) } }
+            data.each do |p, v|
+              collection.add(p, value: v, fact_type: :external,
+                                file: fact.file) { has_weight(weight) }
+            end
           end
         end
       end
 
-      private
-
       def entries
-        Dir.entries(directory).find_all { |f| should_parse?(f) }.sort.map { |f| File.join(directory, f) }
+        dirs = @directories.select { |directory| File.directory?(directory) }.map do |directory|
+          Dir.entries(directory).map { |directory_entry| File.join(directory, directory_entry) }
+        end
+        dirs.flatten.select { |f| should_parse?(f) }
       rescue Errno::ENOENT
         []
       end
 
       def should_parse?(file)
-        file !~ /^\./
+        File.basename(file) !~ /^\./
       end
     end
   end

lib/custom_facts/util/fact.rb

@@ -123,8 +123,10 @@ def value
           announce_when_no_suitable_resolution(suitable_resolutions)
           announce_when_no_value_found(@value)
 
-          @value
+          @value = resolve_value
         end
+
+        @value
       end
 
       # @api private
@@ -138,6 +140,13 @@ def extract_ldapname_option!(options)
 
       private
 
+      def resolve_value
+        return Facter.core_value(name) if @value.nil?
+
+        core_value = Facter.core_value(name) if @used_resolution_weight <= 0
+        core_value.nil? ? @value : core_value
+      end
+
       # Are we in the midst of a search?
       def searching?
         @searching

lib/custom_facts/util/resolution.rb

@@ -86,7 +86,7 @@ def evaluate(&block)
       end
 
       def options(options)
-        accepted_options = %i[name value timeout weight fact_type]
+        accepted_options = %i[name value timeout weight fact_type file]
 
         accepted_options.each do |option_name|
           instance_variable_set("@#{option_name}", options.delete(option_name)) if options.key?(option_name)

lib/framework/core/cache_manager.rb

@@ -13,14 +13,17 @@ def resolve_facts(searched_facts)
       return searched_facts, [] if !File.directory?(@cache_dir) || !Options[:cache]
 
       facts = []
-      searched_facts.each do |fact|
+      searched_facts.delete_if do |fact|
         res = resolve_fact(fact)
-        facts << res if res
+        if res
+          facts << res
+          true
+        else
+          false
+        end
       end
-      facts.each do |fact|
-        searched_facts.delete_if { |f| f.name == fact.name }
-      end
-      [searched_facts, facts]
+
+      [searched_facts, facts.flatten]
     end
 
     def cache_facts(resolved_facts)
@@ -37,10 +40,21 @@ def cache_facts(resolved_facts)
       end
     end
 
+    def group_cached?(group_name)
+      cached = @fact_groups.get_group_ttls(group_name) ? true : false
+      delete_cache(group_name) unless cached
+      cached
+    end
+
     private
 
     def resolve_fact(searched_fact)
-      group_name = @fact_groups.get_fact_group(searched_fact.name)
+      group_name =  if searched_fact.file
+                      searched_fact.name
+                    else
+                      @fact_groups.get_fact_group(searched_fact.name)
+                    end
+
       return unless group_name
 
       return unless group_cached?(group_name)
@@ -51,16 +65,32 @@ def resolve_fact(searched_fact)
       return unless data
 
       @log.debug("loading cached values for #{group_name} facts")
-      create_fact(searched_fact, data[searched_fact.name])
+
+      create_facts(searched_fact, data)
     end
 
-    def create_fact(searched_fact, value)
-      Facter::ResolvedFact.new(searched_fact.name, value, searched_fact.type,
-                               searched_fact.user_query, searched_fact.filter_tokens)
+    def create_facts(searched_fact, data)
+      if searched_fact.type == :file
+        facts = []
+        data.each do |fact_name, fact_value|
+          fact = Facter::ResolvedFact.new(fact_name, fact_value, searched_fact.type,
+                                          searched_fact.user_query, searched_fact.filter_tokens)
+          fact.file = searched_fact.file
+          facts << fact
+        end
+        facts
+      else
+        [Facter::ResolvedFact.new(searched_fact.name, data[searched_fact.name], searched_fact.type,
+                                  searched_fact.user_query, searched_fact.filter_tokens)]
+      end
     end
 
     def cache_fact(fact)
-      group_name = @fact_groups.get_fact_group(fact.name)
+      group_name = if fact.file
+                     File.basename(fact.file)
+                   else
+                     @fact_groups.get_fact_group(fact.name)
+                   end
       return if !group_name || fact.value.nil?
 
       return unless group_cached?(group_name)
@@ -98,12 +128,6 @@ def read_group_json(group_name)
       @groups[group_name] = data
     end
 
-    def group_cached?(group_name)
-      cached = @fact_groups.get_group_ttls(group_name) ? true : false
-      delete_cache(group_name) unless cached
-      cached
-    end
-
     def check_ttls?(group_name)
       ttls = @fact_groups.get_group_ttls(group_name)
       return false unless ttls

lib/framework/core/fact/external/external_fact_manager.rb

@@ -17,10 +17,11 @@ def external_facts(custom_facts)
       resolved_custom_facts = []
 
       custom_facts.each do |custom_fact|
-        fact_value = LegacyFacter.value(custom_fact.name)
-        resolved_fact = ResolvedFact.new(custom_fact.name, fact_value, :custom)
+        fact = LegacyFacter[custom_fact.name]
+        resolved_fact = ResolvedFact.new(custom_fact.name, fact.value, :custom)
         resolved_fact.filter_tokens = []
         resolved_fact.user_query = custom_fact.user_query
+        resolved_fact.file = fact.options[:file]
 
         resolved_custom_facts << resolved_fact
       end

lib/framework/core/fact_loaders/external_fact_loader.rb

@@ -34,8 +34,9 @@ def load_external_facts
 
       external_facts_to_load = LegacyFacter.collection.external_facts
 
-      external_facts_to_load&.each do |external_fact_name|
-        loaded_fact = LoadedFact.new(external_fact_name.to_s, nil, :external)
+      external_facts_to_load&.each do |k, v|
+        loaded_fact = LoadedFact.new(k.to_s, nil, :external)
+        loaded_fact.file = v.options[:file]
         external_facts << loaded_fact
       end
 

lib/framework/parsers/query_parser.rb

@@ -26,7 +26,8 @@ def parse(query_list, loaded_fact)
         matched_facts = []
         @log.debug "User query is: #{query_list}"
         @query_list = query_list
-        query_list = loaded_fact.map(&:name) unless query_list.any?
+
+        return no_user_query(loaded_fact) unless query_list.any?
 
         query_list.each do |query|
           @log.debug "Query is #{query}"
@@ -37,6 +38,14 @@ def parse(query_list, loaded_fact)
         matched_facts.flatten(1)
       end
 
+      def no_user_query(loaded_facts)
+        searched_facts = []
+        loaded_facts.each do |loaded_fact|
+          searched_facts << SearchedFact.new(loaded_fact.name, loaded_fact.klass, [], '', loaded_fact.type)
+        end
+        searched_facts
+      end
+
       def search_for_facts(query, loaded_fact_hash)
         resolvable_fact_list = []
         query = query.to_s
@@ -90,7 +99,10 @@ def construct_loaded_fact(query_tokens, query_token_range, loaded_fact)
         fact_name = loaded_fact.name.to_s
         klass_name = loaded_fact.klass
         type = loaded_fact.type
-        SearchedFact.new(fact_name, klass_name, filter_tokens, user_query, type)
+        sf = SearchedFact.new(fact_name, klass_name, filter_tokens, user_query, type)
+        sf.file = loaded_fact.file
+
+        sf
       end
 
       def construct_filter_tokens(query_tokens, query_token_range)

lib/models/loaded_fact.rb

@@ -3,11 +3,13 @@
 module Facter
   class LoadedFact
     attr_reader :name, :klass, :type
+    attr_accessor :file
 
-    def initialize(name, klass, type = nil)
+    def initialize(name, klass, type = nil, file = nil)
       @name = name
       @klass = klass
       @type = type.nil? ? :core : type
+      @file = file
     end
   end
 end

lib/models/resolved_fact.rb

@@ -3,7 +3,7 @@
 module Facter
   class ResolvedFact
     attr_reader :name, :type
-    attr_accessor :user_query, :filter_tokens, :value
+    attr_accessor :user_query, :filter_tokens, :value, :file
 
     def initialize(name, value = '', type = :core, user_query = nil, filter_tokens = [])
       @name = name