Merge pull request #391 from splitio/rb_segments_storage_standalone_mode

EmilianoSanchez · web-flow · commit bd0294e5c4ee · 2025-02-28T14:47:56.000-03:00
[Rule-based segments] Add implementations for InMemory and LocalStorage
diff --git a/src/storages/AbstractSplitsCacheSync.ts b/src/storages/AbstractSplitsCacheSync.ts
@@ -1,5 +1,5 @@
 import { ISplitsCacheSync } from './types';
-import { ISplit } from '../dtos/types';
+import { IRBSegment, ISplit } from '../dtos/types';
 import { objectAssign } from '../utils/lang/objectAssign';
 import { IN_SEGMENT, IN_LARGE_SEGMENT } from '../utils/constants';
 
@@ -80,7 +80,7 @@ export abstract class AbstractSplitsCacheSync implements ISplitsCacheSync {
  * Given a parsed split, it returns a boolean flagging if its conditions use segments matchers (rules & whitelists).
  * This util is intended to simplify the implementation of `splitsCache::usesSegments` method
  */
-export function usesSegments(split: ISplit) {
+export function usesSegments(split: ISplit | IRBSegment) {
   const conditions = split.conditions || [];
   for (let i = 0; i < conditions.length; i++) {
     const matchers = conditions[i].matcherGroup.matchers;
diff --git a/src/storages/KeyBuilderCS.ts b/src/storages/KeyBuilderCS.ts
@@ -15,7 +15,7 @@ export class KeyBuilderCS extends KeyBuilder implements MySegmentsKeyBuilder {
   constructor(prefix: string, matchingKey: string) {
     super(prefix);
     this.matchingKey = matchingKey;
-    this.regexSplitsCacheKey = new RegExp(`^${prefix}\\.(splits?|trafficType|flagSet)\\.`);
+    this.regexSplitsCacheKey = new RegExp(`^${prefix}\\.(splits?|trafficType|flagSet|rbsegment)\\.`);
   }
 
   /**
@@ -47,6 +47,10 @@ export class KeyBuilderCS extends KeyBuilder implements MySegmentsKeyBuilder {
     return startsWith(key, `${this.prefix}.split.`);
   }
 
+  isRBSegmentKey(key: string) {
+    return startsWith(key, `${this.prefix}.rbsegment.`);
+  }
+
   buildSplitsWithSegmentCountKey() {
     return `${this.prefix}.splits.usingSegments`;
   }
diff --git a/src/storages/__tests__/RBSegmentsCacheSync.spec.ts b/src/storages/__tests__/RBSegmentsCacheSync.spec.ts
@@ -0,0 +1,74 @@
+import { RBSegmentsCacheInMemory } from '../inMemory/RBSegmentsCacheInMemory';
+import { RBSegmentsCacheInLocal } from '../inLocalStorage/RBSegmentsCacheInLocal';
+import { KeyBuilderCS } from '../KeyBuilderCS';
+import { rbSegment, rbSegmentWithInSegmentMatcher } from '../__tests__/testUtils';
+import { IRBSegmentsCacheSync } from '../types';
+import { fullSettings } from '../../utils/settingsValidation/__tests__/settings.mocks';
+
+const cacheInMemory = new RBSegmentsCacheInMemory();
+const cacheInLocal = new RBSegmentsCacheInLocal(fullSettings, new KeyBuilderCS('SPLITIO', 'user'));
+
+describe.each([cacheInMemory, cacheInLocal])('Rule-based segments cache sync (Memory & LocalStorage)', (cache: IRBSegmentsCacheSync) => {
+
+  beforeEach(() => {
+    cache.clear();
+  });
+
+  test('clear should reset the cache state', () => {
+    cache.update([rbSegment], [], 1);
+    expect(cache.getChangeNumber()).toBe(1);
+    expect(cache.get(rbSegment.name)).not.toBeNull();
+
+    cache.clear();
+    expect(cache.getChangeNumber()).toBe(-1);
+    expect(cache.get(rbSegment.name)).toBeNull();
+  });
+
+  test('update should add and remove segments correctly', () => {
+    // Add segments
+    expect(cache.update([rbSegment, rbSegmentWithInSegmentMatcher], [], 1)).toBe(true);
+    expect(cache.get(rbSegment.name)).toEqual(rbSegment);
+    expect(cache.get(rbSegmentWithInSegmentMatcher.name)).toEqual(rbSegmentWithInSegmentMatcher);
+    expect(cache.getChangeNumber()).toBe(1);
+
+    // Remove a segment
+    expect(cache.update([], [rbSegment], 2)).toBe(true);
+    expect(cache.get(rbSegment.name)).toBeNull();
+    expect(cache.get(rbSegmentWithInSegmentMatcher.name)).toEqual(rbSegmentWithInSegmentMatcher);
+    expect(cache.getChangeNumber()).toBe(2);
+
+    // Remove remaining segment
+    expect(cache.update([], [rbSegmentWithInSegmentMatcher], 3)).toBe(true);
+    expect(cache.get(rbSegment.name)).toBeNull();
+    expect(cache.get(rbSegmentWithInSegmentMatcher.name)).toBeNull();
+    expect(cache.getChangeNumber()).toBe(3);
+
+    // No changes
+    expect(cache.update([], [rbSegmentWithInSegmentMatcher], 4)).toBe(false);
+    expect(cache.getChangeNumber()).toBe(4);
+  });
+
+  test('contains should check for segment existence correctly', () => {
+    cache.update([rbSegment, rbSegmentWithInSegmentMatcher], [], 1);
+
+    expect(cache.contains(new Set([rbSegment.name]))).toBe(true);
+    expect(cache.contains(new Set([rbSegment.name, rbSegmentWithInSegmentMatcher.name]))).toBe(true);
+    expect(cache.contains(new Set(['nonexistent']))).toBe(false);
+    expect(cache.contains(new Set([rbSegment.name, 'nonexistent']))).toBe(false);
+
+    cache.update([], [rbSegment, rbSegmentWithInSegmentMatcher], 2);
+  });
+
+  test('usesSegments should track segments usage correctly', () => {
+    expect(cache.usesSegments()).toBe(true); // Initially true when changeNumber is -1
+
+    cache.update([rbSegment], [], 1); // rbSegment doesn't have IN_SEGMENT matcher
+    expect(cache.usesSegments()).toBe(false);
+
+    cache.update([rbSegmentWithInSegmentMatcher], [], 2); // rbSegmentWithInSegmentMatcher has IN_SEGMENT matcher
+    expect(cache.usesSegments()).toBe(true);
+
+    cache.clear();
+    expect(cache.usesSegments()).toBe(true); // True after clear since changeNumber is -1
+  });
+});
diff --git a/src/storages/inLocalStorage/RBSegmentsCacheInLocal.ts b/src/storages/inLocalStorage/RBSegmentsCacheInLocal.ts
@@ -0,0 +1,144 @@
+import { IRBSegment } from '../../dtos/types';
+import { ILogger } from '../../logger/types';
+import { ISettings } from '../../types';
+import { isFiniteNumber, isNaNNumber, toNumber } from '../../utils/lang';
+import { setToArray } from '../../utils/lang/sets';
+import { usesSegments } from '../AbstractSplitsCacheSync';
+import { KeyBuilderCS } from '../KeyBuilderCS';
+import { IRBSegmentsCacheSync } from '../types';
+import { LOG_PREFIX } from './constants';
+
+export class RBSegmentsCacheInLocal implements IRBSegmentsCacheSync {
+
+  private readonly keys: KeyBuilderCS;
+  private readonly log: ILogger;
+  private hasSync?: boolean;
+
+  constructor(settings: ISettings, keys: KeyBuilderCS) {
+    this.keys = keys;
+    this.log = settings.log;
+  }
+
+  clear() {
+    this.getNames().forEach(name => this.remove(name));
+    localStorage.removeItem(this.keys.buildRBSegmentsTillKey());
+    this.hasSync = false;
+  }
+
+  update(toAdd: IRBSegment[], toRemove: IRBSegment[], changeNumber: number): boolean {
+    this.setChangeNumber(changeNumber);
+    const updated = toAdd.map(toAdd => this.add(toAdd)).some(result => result);
+    return toRemove.map(toRemove => this.remove(toRemove.name)).some(result => result) || updated;
+  }
+
+  private setChangeNumber(changeNumber: number) {
+    try {
+      localStorage.setItem(this.keys.buildRBSegmentsTillKey(), changeNumber + '');
+      localStorage.setItem(this.keys.buildLastUpdatedKey(), Date.now() + '');
+      this.hasSync = true;
+    } catch (e) {
+      this.log.error(LOG_PREFIX + e);
+    }
+  }
+
+  private updateSegmentCount(diff: number){
+    const segmentsCountKey = this.keys.buildSplitsWithSegmentCountKey();
+    const count = toNumber(localStorage.getItem(segmentsCountKey)) + diff;
+    // @ts-expect-error
+    if (count > 0) localStorage.setItem(segmentsCountKey, count);
+    else localStorage.removeItem(segmentsCountKey);
+  }
+
+  private add(rbSegment: IRBSegment): boolean {
+    try {
+      const name = rbSegment.name;
+      const rbSegmentKey = this.keys.buildRBSegmentKey(name);
+      const rbSegmentFromLocalStorage = localStorage.getItem(rbSegmentKey);
+      const previous = rbSegmentFromLocalStorage ? JSON.parse(rbSegmentFromLocalStorage) : null;
+
+      localStorage.setItem(rbSegmentKey, JSON.stringify(rbSegment));
+
+      let usesSegmentsDiff = 0;
+      if (previous && usesSegments(previous)) usesSegmentsDiff--;
+      if (usesSegments(rbSegment)) usesSegmentsDiff++;
+      if (usesSegmentsDiff !== 0) this.updateSegmentCount(usesSegmentsDiff);
+
+      return true;
+    } catch (e) {
+      this.log.error(LOG_PREFIX + e);
+      return false;
+    }
+  }
+
+  private remove(name: string): boolean {
+    try {
+      const rbSegment = this.get(name);
+      if (!rbSegment) return false;
+
+      localStorage.removeItem(this.keys.buildRBSegmentKey(name));
+
+      if (usesSegments(rbSegment)) this.updateSegmentCount(-1);
+
+      return true;
+    } catch (e) {
+      this.log.error(LOG_PREFIX + e);
+      return false;
+    }
+  }
+
+  private getNames(): string[] {
+    const len = localStorage.length;
+    const accum = [];
+
+    let cur = 0;
+
+    while (cur < len) {
+      const key = localStorage.key(cur);
+
+      if (key != null && this.keys.isRBSegmentKey(key)) accum.push(this.keys.extractKey(key));
+
+      cur++;
+    }
+
+    return accum;
+  }
+
+  get(name: string): IRBSegment | null {
+    const item = localStorage.getItem(this.keys.buildRBSegmentKey(name));
+    return item && JSON.parse(item);
+  }
+
+  contains(names: Set<string>): boolean {
+    const namesArray = setToArray(names);
+    const namesInStorage = this.getNames();
+    return namesArray.every(name => namesInStorage.indexOf(name) !== -1);
+  }
+
+  getChangeNumber(): number {
+    const n = -1;
+    let value: string | number | null = localStorage.getItem(this.keys.buildRBSegmentsTillKey());
+
+    if (value !== null) {
+      value = parseInt(value, 10);
+
+      return isNaNNumber(value) ? n : value;
+    }
+
+    return n;
+  }
+
+  usesSegments(): boolean {
+    // If cache hasn't been synchronized, assume we need segments
+    if (!this.hasSync) return true;
+
+    const storedCount = localStorage.getItem(this.keys.buildSplitsWithSegmentCountKey());
+    const splitsWithSegmentsCount = storedCount === null ? 0 : toNumber(storedCount);
+
+    if (isFiniteNumber(splitsWithSegmentsCount)) {
+      return splitsWithSegmentsCount > 0;
+    } else {
+      return true;
+    }
+  }
+
+}
diff --git a/src/storages/inLocalStorage/SplitsCacheInLocal.ts b/src/storages/inLocalStorage/SplitsCacheInLocal.ts
@@ -57,16 +57,14 @@ export class SplitsCacheInLocal extends AbstractSplitsCacheSync {
 
   private _incrementCounts(split: ISplit) {
     try {
-      if (split) {
-        const ttKey = this.keys.buildTrafficTypeKey(split.trafficTypeName);
-        // @ts-expect-error
-        localStorage.setItem(ttKey, toNumber(localStorage.getItem(ttKey)) + 1);
+      const ttKey = this.keys.buildTrafficTypeKey(split.trafficTypeName);
+      // @ts-expect-error
+      localStorage.setItem(ttKey, toNumber(localStorage.getItem(ttKey)) + 1);
 
-        if (usesSegments(split)) {
-          const segmentsCountKey = this.keys.buildSplitsWithSegmentCountKey();
-          // @ts-expect-error
-          localStorage.setItem(segmentsCountKey, toNumber(localStorage.getItem(segmentsCountKey)) + 1);
-        }
+      if (usesSegments(split)) {
+        const segmentsCountKey = this.keys.buildSplitsWithSegmentCountKey();
+        // @ts-expect-error
+        localStorage.setItem(segmentsCountKey, toNumber(localStorage.getItem(segmentsCountKey)) + 1);
       }
     } catch (e) {
       this.log.error(LOG_PREFIX + e);
diff --git a/src/storages/inLocalStorage/index.ts b/src/storages/inLocalStorage/index.ts
@@ -14,6 +14,7 @@ import { STORAGE_LOCALSTORAGE } from '../../utils/constants';
 import { shouldRecordTelemetry, TelemetryCacheInMemory } from '../inMemory/TelemetryCacheInMemory';
 import { UniqueKeysCacheInMemoryCS } from '../inMemory/UniqueKeysCacheInMemoryCS';
 import { getMatching } from '../../utils/key';
+import { RBSegmentsCacheInLocal } from './RBSegmentsCacheInLocal';
 
 export interface InLocalStorageOptions {
   prefix?: string
@@ -40,11 +41,13 @@ export function InLocalStorage(options: InLocalStorageOptions = {}): IStorageSyn
     const expirationTimestamp = Date.now() - DEFAULT_CACHE_EXPIRATION_IN_MILLIS;
 
     const splits = new SplitsCacheInLocal(settings, keys, expirationTimestamp);
+    const rbSegments = new RBSegmentsCacheInLocal(settings, keys);
     const segments = new MySegmentsCacheInLocal(log, keys);
     const largeSegments = new MySegmentsCacheInLocal(log, myLargeSegmentsKeyBuilder(prefix, matchingKey));
 
     return {
       splits,
+      rbSegments,
       segments,
       largeSegments,
       impressions: new ImpressionsCacheInMemory(impressionsQueueSize),
@@ -60,6 +63,7 @@ export function InLocalStorage(options: InLocalStorageOptions = {}): IStorageSyn
 
         return {
           splits: this.splits,
+          rbSegments: this.rbSegments,
           segments: new MySegmentsCacheInLocal(log, new KeyBuilderCS(prefix, matchingKey)),
           largeSegments: new MySegmentsCacheInLocal(log, myLargeSegmentsKeyBuilder(prefix, matchingKey)),
           impressions: this.impressions,
diff --git a/src/storages/inMemory/InMemoryStorage.ts b/src/storages/inMemory/InMemoryStorage.ts
@@ -7,6 +7,7 @@ import { ImpressionCountsCacheInMemory } from './ImpressionCountsCacheInMemory';
 import { LOCALHOST_MODE, STORAGE_MEMORY } from '../../utils/constants';
 import { shouldRecordTelemetry, TelemetryCacheInMemory } from './TelemetryCacheInMemory';
 import { UniqueKeysCacheInMemory } from './UniqueKeysCacheInMemory';
+import { RBSegmentsCacheInMemory } from './RBSegmentsCacheInMemory';
 
 /**
  * InMemory storage factory for standalone server-side SplitFactory
@@ -17,10 +18,12 @@ export function InMemoryStorageFactory(params: IStorageFactoryParams): IStorageS
   const { settings: { scheduler: { impressionsQueueSize, eventsQueueSize, }, sync: { __splitFiltersValidation } } } = params;
 
   const splits = new SplitsCacheInMemory(__splitFiltersValidation);
+  const rbSegments = new RBSegmentsCacheInMemory();
   const segments = new SegmentsCacheInMemory();
 
   const storage = {
     splits,
+    rbSegments,
     segments,
     impressions: new ImpressionsCacheInMemory(impressionsQueueSize),
     impressionCounts: new ImpressionCountsCacheInMemory(),
diff --git a/src/storages/inMemory/InMemoryStorageCS.ts b/src/storages/inMemory/InMemoryStorageCS.ts
@@ -7,6 +7,7 @@ import { ImpressionCountsCacheInMemory } from './ImpressionCountsCacheInMemory';
 import { LOCALHOST_MODE, STORAGE_MEMORY } from '../../utils/constants';
 import { shouldRecordTelemetry, TelemetryCacheInMemory } from './TelemetryCacheInMemory';
 import { UniqueKeysCacheInMemoryCS } from './UniqueKeysCacheInMemoryCS';
+import { RBSegmentsCacheInMemory } from './RBSegmentsCacheInMemory';
 
 /**
  * InMemory storage factory for standalone client-side SplitFactory
@@ -17,11 +18,13 @@ export function InMemoryStorageCSFactory(params: IStorageFactoryParams): IStorag
   const { settings: { scheduler: { impressionsQueueSize, eventsQueueSize }, sync: { __splitFiltersValidation } } } = params;
 
   const splits = new SplitsCacheInMemory(__splitFiltersValidation);
+  const rbSegments = new RBSegmentsCacheInMemory();
   const segments = new MySegmentsCacheInMemory();
   const largeSegments = new MySegmentsCacheInMemory();
 
   const storage = {
     splits,
+    rbSegments,
     segments,
     largeSegments,
     impressions: new ImpressionsCacheInMemory(impressionsQueueSize),
@@ -36,6 +39,7 @@ export function InMemoryStorageCSFactory(params: IStorageFactoryParams): IStorag
     shared() {
       return {
         splits: this.splits,
+        rbSegments: this.rbSegments,
         segments: new MySegmentsCacheInMemory(),
         largeSegments: new MySegmentsCacheInMemory(),
         impressions: this.impressions,
diff --git a/src/storages/inMemory/RBSegmentsCacheInMemory.ts b/src/storages/inMemory/RBSegmentsCacheInMemory.ts
diff --git a/src/storages/inRedis/index.ts b/src/storages/inRedis/index.ts
diff --git a/src/storages/pluggable/index.ts b/src/storages/pluggable/index.ts
diff --git a/src/storages/types.ts b/src/storages/types.ts

Original file line number	Diff line number	Diff line change
`@@ -15,7 +15,7 @@ export class KeyBuilderCS extends KeyBuilder implements MySegmentsKeyBuilder {`
`15`	`15`	`constructor(prefix: string, matchingKey: string) {`
`16`	`16`	`super(prefix);`
`17`	`17`	`this.matchingKey = matchingKey;`
`18`		- this.regexSplitsCacheKey = new RegExp(`^${prefix}\\.(splits?\|trafficType\|flagSet)\\.`);
	`18`	+ this.regexSplitsCacheKey = new RegExp(`^${prefix}\\.(splits?\|trafficType\|flagSet\|rbsegment)\\.`);
`19`	`19`	`}`
`20`	`20`
`21`	`21`	`/**`
`@@ -47,6 +47,10 @@ export class KeyBuilderCS extends KeyBuilder implements MySegmentsKeyBuilder {`
`47`	`47`	return startsWith(key, `${this.prefix}.split.`);
`48`	`48`	`}`
`49`	`49`
	`50`	`+ isRBSegmentKey(key: string) {`
	`51`	+ return startsWith(key, `${this.prefix}.rbsegment.`);
	`52`	`+ }`
	`53`	`+`
`50`	`54`	`buildSplitsWithSegmentCountKey() {`
`51`	`55`	return `${this.prefix}.splits.usingSegments`;
`52`	`56`	`}`