perf(cache): Optimize cache by removing null attributes

groyoh · groyoh · commit 9db8ea45b1ef · 2025-11-05T10:17:45.000+01:00
diff --git a/app/services/subscriptions/charge_cache_middleware.rb b/app/services/subscriptions/charge_cache_middleware.rb
@@ -12,28 +12,90 @@ def initialize(subscription:, charge:, to_datetime:, cache: true)
     def call(charge_filter:)
       return yield unless cache
 
-      json = Subscriptions::ChargeCacheService.call(subscription:, charge:, charge_filter:, expires_in: cache_expiration) do
-        yield
-          .map { |fee| fee.attributes.merge("pricing_unit_usage" => fee.pricing_unit_usage&.attributes) }
-          .to_json
+      fees = nil
+      cached_fees = Subscriptions::ChargeCacheService.call(subscription:, charge:, charge_filter:, expires_in: cache_expiration) do
+        fees = yield
+        fees.map do |fee|
+          fee_attributes = fee.attributes
+          if (pricing_unit_usage = fee.pricing_unit_usage).present?
+            pricing_unit_usage_attributes = compact_hash(pricing_unit_usage.attributes, COMPACTABLE_PRICING_UNIT_USAGE_ATTRIBUTES)
+            fee_attributes["pricing_unit_usage"] = pricing_unit_usage_attributes
+          end
+          compact_fee(fee_attributes)
+        end.to_json
       end
 
-      JSON.parse(json).map do |j|
-        pricing_unit_usage = if j["pricing_unit_usage"].present?
-          PricingUnitUsage.new(j["pricing_unit_usage"].slice(*PricingUnitUsage.column_names))
-        end
+      return fees if fees # avoid parsing the JSON if we already have the fees
 
-        Fee.new(
-          **j.slice(*Fee.column_names),
-          pricing_unit_usage:
-        )
-      end
+      parse_cached_fees(cached_fees)
     end
 
     private
 
     attr_reader :subscription, :charge, :to_datetime, :cache
 
+    COMPACTABLE_PROPERTIES = [
+      "fixed_charges_duration",
+      "fixed_charges_from_datetime",
+      "fixed_charges_to_datetime"
+    ]
+
+    COMPACTABLE_ATTRIBUTES = [
+      "add_on_id",
+      "applied_add_on_id",
+      "charge_filter_id",
+      "created_at",
+      "deleted_at",
+      "description",
+      "failed_at",
+      "fixed_charge_id",
+      "group_id",
+      "invoice_display_name",
+      "invoice_id",
+      "pay_in_advance_event_id",
+      "pay_in_advance_event_transaction_id",
+      "pay_in_advance",
+      "pricing_unit_usage",
+      "refunded_at",
+      "succeeded_at",
+      "true_up_parent_fee_id",
+      "updated_at",
+      "id"
+    ]
+
+    COMPACTABLE_PRICING_UNIT_USAGE_ATTRIBUTES = [
+      "id",
+      "fee_id",
+      "created_at",
+      "updated_at"
+    ]
+
+    def parse_cached_fees(cached_fees)
+      JSON.parse(cached_fees).map do |fee_attributes|
+        pricing_unit_usage = fee_attributes["pricing_unit_usage"]
+        pricing_unit_usage = if pricing_unit_usage.present?
+          PricingUnitUsage.new(pricing_unit_usage.slice(*PricingUnitUsage.column_names))
+        end
+
+        Fee.new(**fee_attributes.slice(*Fee.column_names), pricing_unit_usage:)
+      end
+    end
+
+    # This method is used to compact the fee attributes to reduce the size of the cached fees. We explicitly define
+    # which attributes are compactable and which are not to avoid losing information. For instance, the `grouped_by`
+    # should keep the `nil` values.
+    def compact_fee(fee_attributes)
+      fee_attributes = compact_hash(fee_attributes, COMPACTABLE_ATTRIBUTES)
+      if (properties = fee_attributes["properties"]).present?
+        fee_attributes["properties"] = compact_hash(properties, COMPACTABLE_PROPERTIES)
+      end
+      fee_attributes
+    end
+
+    def compact_hash(object, compactable_keys)
+      object.reject { |key, value| compactable_keys.include?(key) && value.nil? }
+    end
+
     def cache_expiration
       [(to_datetime - Time.current).to_i.seconds, 0].max
     end
diff --git a/config/environments/production.rb b/config/environments/production.rb
@@ -61,7 +61,8 @@
         Rails.logger.error(exception.backtrace.join("\n"))
 
         Sentry.capture_exception(exception)
-      }
+      },
+      compress_threshold: 512 # 512 bytes
     }
 
     if ENV["LAGO_REDIS_CACHE_PASSWORD"].present? && !ENV["LAGO_REDIS_CACHE_PASSWORD"].empty?
diff --git a/spec/factories/charges.rb b/spec/factories/charges.rb
@@ -10,7 +10,7 @@
     factory :standard_charge do
       charge_model { "standard" }
       properties do
-        {amount: Faker::Number.between(from: 100, to: 500).to_s}
+        {amount: "10"}
       end
     end
 
diff --git a/spec/factories/fees.rb b/spec/factories/fees.rb
@@ -45,6 +45,7 @@
 
     invoiceable_type { "Charge" }
     invoiceable_id { charge.id }
+    events_count { 1 }
 
     properties do
       {
diff --git a/spec/scenarios/customer_usage_spec.rb b/spec/scenarios/customer_usage_spec.rb
@@ -2,53 +2,126 @@
 
 require "rails_helper"
 
-describe "Customer usage Scenario" do
+describe "Customer usage Scenario", cache: :redis do
   let(:organization) { create(:organization, webhook_url: nil) }
 
   let(:timezone) { "UTC" }
   let(:customer) { create(:customer, organization:, timezone:, currency: "EUR") }
 
-  let(:plan) { create(:plan, organization:, amount_cents: 700, pay_in_advance: false, interval: "yearly") }
+  let(:plan) { create(:plan, organization:, amount_cents: 5000, pay_in_advance: false, interval: "yearly") }
+  let(:billable_metric) { create(:billable_metric, organization:, code: "image_generation", name: "Image generation") }
+  let(:charge) { create(:standard_charge, plan:, billable_metric:, invoice_display_name: "Image generation") }
+
+  before { charge }
+
+  def customer_usage_json(units: 1.0, from_datetime: "2043-01-01T09:30:00Z", to_datetime: "2043-12-31T23:59:59Z")
+    total_amount_cents = 1000 * units
+    {
+      customer_usage: {
+        amount_cents: total_amount_cents,
+        charges_usage: [
+          {
+            amount_cents: total_amount_cents,
+            amount_currency: "EUR",
+            billable_metric: {
+              aggregation_type: "count_agg",
+              code: "image_generation",
+              lago_id: billable_metric.id,
+              name: billable_metric.name
+            },
+            charge: {
+              charge_model: "standard",
+              invoice_display_name: "Image generation",
+              lago_id: charge.id
+            },
+            events_count: units.to_i,
+            filters: [],
+            grouped_usage: [],
+            pricing_unit_details: nil,
+            total_aggregated_units: units.to_d.to_s,
+            units: units.to_d.to_s
+          }
+        ],
+        currency: "EUR",
+        from_datetime: from_datetime,
+        issuing_date: to_datetime.slice(0, 10),
+        lago_invoice_id: nil,
+        taxes_amount_cents: 0,
+        to_datetime: to_datetime,
+        total_amount_cents: total_amount_cents
+      }
+    }
+  end
+
+  def fetch_and_assert_current_usage(units: 1.0, from_datetime: "2043-01-01T09:30:00Z", to_datetime: "2043-12-31T23:59:59Z")
+    subscription = customer.subscriptions.first
+    fetch_current_usage(customer:, subscription:)
+
+    expect(json).to eq(customer_usage_json(units: units, from_datetime: from_datetime, to_datetime: to_datetime))
+  end
 
   context "with start date in the past" do
     it "retrieve the customer usage" do
-      travel_to(DateTime.new(2023, 8, 8, 9, 30)) do
-        create_subscription(
+      travel_to(DateTime.new(2043, 8, 8, 9, 30)) do
+        subscription = create_subscription(
           {
             external_customer_id: customer.external_id,
             external_id: customer.external_id,
             plan_code: plan.code,
-            subscription_at: DateTime.new(2023, 1, 1, 9, 30).iso8601
-          }
+            subscription_at: DateTime.new(2043, 1, 1, 9, 30).iso8601
+          },
+          as: :model
         )
 
-        subscription = customer.subscriptions.first
-        fetch_current_usage(customer:, subscription:)
+        fetch_and_assert_current_usage(units: 0)
+
+        create_event({
+          external_subscription_id: subscription.external_id,
+          timestamp: Time.now.to_f,
+          code: "image_generation",
+          properties: {}
+        })
 
-        expect(json[:customer_usage][:from_datetime]).to eq("2023-01-01T09:30:00Z")
-        expect(json[:customer_usage][:to_datetime]).to eq("2023-12-31T23:59:59Z")
+        fetch_and_assert_current_usage(units: 1)
+
+        # test cache
+        fetch_and_assert_current_usage(units: 1)
+
+        create_event({
+          external_subscription_id: subscription.external_id,
+          timestamp: Time.now.to_f,
+          code: "image_generation",
+          properties: {}
+        })
+
+        fetch_and_assert_current_usage(units: 2)
+
+        Event.last.destroy
+
+        # test cache
+        fetch_and_assert_current_usage(units: 2)
+
+        Rails.cache.clear
+
+        fetch_and_assert_current_usage(units: 1)
       end
     end
 
     context "with Europe/Berlin timezone" do
       let(:timezone) { "Europe/Berlin" }
 
       it "retrieve the customer usage" do
-        travel_to(DateTime.new(2023, 8, 8, 9, 30)) do
+        travel_to(DateTime.new(2043, 8, 8, 9, 30)) do
           create_subscription(
             {
               external_customer_id: customer.external_id,
               external_id: customer.external_id,
               plan_code: plan.code,
-              subscription_at: DateTime.new(2023, 1, 1, 9, 30).iso8601
+              subscription_at: DateTime.new(2043, 1, 1, 9, 30).iso8601
             }
           )
 
-          subscription = customer.subscriptions.first
-          fetch_current_usage(customer:, subscription:)
-
-          expect(json[:customer_usage][:from_datetime]).to eq("2023-01-01T09:30:00Z")
-          expect(json[:customer_usage][:to_datetime]).to eq("2023-12-31T22:59:59Z")
+          fetch_and_assert_current_usage(units: 0, from_datetime: "2043-01-01T09:30:00Z", to_datetime: "2043-12-31T22:59:59Z")
         end
       end
     end
@@ -57,21 +130,17 @@
       let(:timezone) { "America/Los_Angeles" }
 
       it "retrieve the customer usage" do
-        travel_to(DateTime.new(2023, 8, 8, 9, 30)) do
+        travel_to(DateTime.new(2043, 8, 8, 9, 30)) do
           create_subscription(
             {
               external_customer_id: customer.external_id,
               external_id: customer.external_id,
               plan_code: plan.code,
-              subscription_at: DateTime.new(2023, 1, 1, 9, 30).iso8601
+              subscription_at: DateTime.new(2043, 1, 1, 9, 30).iso8601
             }
           )
 
-          subscription = customer.subscriptions.first
-          fetch_current_usage(customer:, subscription:)
-
-          expect(json[:customer_usage][:from_datetime]).to eq("2023-01-01T09:30:00Z")
-          expect(json[:customer_usage][:to_datetime]).to eq("2024-01-01T07:59:59Z")
+          fetch_and_assert_current_usage(units: 0, from_datetime: "2043-01-01T09:30:00Z", to_datetime: "2044-01-01T07:59:59Z")
         end
       end
     end
diff --git a/spec/services/subscriptions/charge_cache_middleware_spec.rb b/spec/services/subscriptions/charge_cache_middleware_spec.rb

Original file line number	Diff line number	Diff line change
`@@ -61,7 +61,8 @@`
`61`	`61`	`Rails.logger.error(exception.backtrace.join("\n"))`
`62`	`62`
`63`	`63`	`Sentry.capture_exception(exception)`
`64`		`- }`
	`64`	`+ },`
	`65`	`+ compress_threshold: 512 # 512 bytes`
`65`	`66`	`}`
`66`	`67`
`67`	`68`	`if ENV["LAGO_REDIS_CACHE_PASSWORD"].present? && !ENV["LAGO_REDIS_CACHE_PASSWORD"].empty?`
Original file line number	Diff line number	Diff line change
`@@ -45,6 +45,7 @@`
`45`	`45`
`46`	`46`	`invoiceable_type { "Charge" }`
`47`	`47`	`invoiceable_id { charge.id }`
	`48`	`+ events_count { 1 }`
`48`	`49`
`49`	`50`	`properties do`
`50`	`51`	`{`