Skip to content

Updated evaluator #573

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 30, 2025
Merged

Updated evaluator #573

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 21 additions & 8 deletions splitio/engine/evaluator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from splitio.models.grammar.matchers.misc import DependencyMatcher
from splitio.models.grammar.matchers.keys import UserDefinedSegmentMatcher
from splitio.models.grammar.matchers import RuleBasedSegmentMatcher
from splitio.models.grammar.matchers.prerequisites import PrerequisitesMatcher
from splitio.models.rule_based_segments import SegmentType
from splitio.optional.loaders import asyncio

Expand Down Expand Up @@ -56,12 +57,22 @@ def eval_with_context(self, key, bucketing, feature_name, attrs, ctx):
label = Label.KILLED
_treatment = feature.default_treatment
else:
treatment, label = self._treatment_for_flag(feature, key, bucketing, attrs, ctx)
if treatment is None:
label = Label.NO_CONDITION_MATCHED
_treatment = feature.default_treatment
else:
_treatment = treatment
if feature.prerequisites is not None:
prerequisites_matcher = PrerequisitesMatcher(feature.prerequisites)
if not prerequisites_matcher.match(key, attrs, {
'evaluator': self,
'bucketing_key': bucketing,
'ec': ctx}):
label = Label.PREREQUISITES_NOT_MET
_treatment = feature.default_treatment

if _treatment == CONTROL:
treatment, label = self._treatment_for_flag(feature, key, bucketing, attrs, ctx)
if treatment is None:
label = Label.NO_CONDITION_MATCHED
_treatment = feature.default_treatment
else:
_treatment = treatment

return {
'treatment': _treatment,
Expand Down Expand Up @@ -133,7 +144,6 @@ def context_for(self, key, feature_names):
rb_segments
)


class AsyncEvaluationDataFactory:

def __init__(self, split_storage, segment_storage, rbs_segment_storage):
Expand Down Expand Up @@ -199,6 +209,7 @@ def get_pending_objects(features, splits, rbsegments, rb_segments, pending_membe
pending_rbs = set()
for feature in features.values():
cf, cs, crbs = get_dependencies(feature)
cf.extend(get_prerequisites(feature))
pending.update(filter(lambda f: f not in splits, cf))
pending_memberships.update(cs)
pending_rbs.update(filter(lambda f: f not in rb_segments, crbs))
Expand All @@ -223,4 +234,6 @@ def update_objects(fetched, fetched_rbs, splits, rb_segments):
rb_segments.update(rbsegments)

return features, rbsegments, splits, rb_segments


def get_prerequisites(feature):
return [prerequisite.feature_flag_name for prerequisite in feature.prerequisites]
5 changes: 5 additions & 0 deletions splitio/models/impressions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,8 @@ class Label(object): # pylint: disable=too-few-public-methods
# Treatment: control
# Label: not ready
NOT_READY = 'not ready'

# Condition: Prerequisites not met
# Treatment: Default treatment
# Label: prerequisites not met
PREREQUISITES_NOT_MET = "prerequisites not met"
126 changes: 116 additions & 10 deletions tests/engine/test_evaluator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import pytest
import copy

from splitio.models.splits import Split, Status
from splitio.models.splits import Split, Status, from_raw, Prerequisites
from splitio.models import segments
from splitio.models.grammar.condition import Condition, ConditionType
from splitio.models.impressions import Label
Expand Down Expand Up @@ -127,6 +127,7 @@ def test_evaluate_treatment_killed_split(self, mocker):
mocked_split.killed = True
mocked_split.change_number = 123
mocked_split.get_configurations_for.return_value = '{"some_property": 123}'
mocked_split.prerequisites = []

ctx = EvaluationContext(flags={'some': mocked_split}, segment_memberships=set(), rbs_segments={})
result = e.eval_with_context('some_key', 'some_bucketing_key', 'some', {}, ctx)
Expand All @@ -146,6 +147,8 @@ def test_evaluate_treatment_ok(self, mocker):
mocked_split.killed = False
mocked_split.change_number = 123
mocked_split.get_configurations_for.return_value = '{"some_property": 123}'
mocked_split.prerequisites = []

ctx = EvaluationContext(flags={'some': mocked_split}, segment_memberships=set(), rbs_segments={})
result = e.eval_with_context('some_key', 'some_bucketing_key', 'some', {}, ctx)
assert result['treatment'] == 'on'
Expand All @@ -165,6 +168,8 @@ def test_evaluate_treatment_ok_no_config(self, mocker):
mocked_split.killed = False
mocked_split.change_number = 123
mocked_split.get_configurations_for.return_value = None
mocked_split.prerequisites = []

ctx = EvaluationContext(flags={'some': mocked_split}, segment_memberships=set(), rbs_segments={})
result = e.eval_with_context('some_key', 'some_bucketing_key', 'some', {}, ctx)
assert result['treatment'] == 'on'
Expand All @@ -184,13 +189,15 @@ def test_evaluate_treatments(self, mocker):
mocked_split.killed = False
mocked_split.change_number = 123
mocked_split.get_configurations_for.return_value = '{"some_property": 123}'
mocked_split.prerequisites = []

mocked_split2 = mocker.Mock(spec=Split)
mocked_split2.name = 'feature4'
mocked_split2.default_treatment = 'on'
mocked_split2.killed = False
mocked_split2.change_number = 123
mocked_split2.get_configurations_for.return_value = None
mocked_split2.prerequisites = []

ctx = EvaluationContext(flags={'feature2': mocked_split, 'feature4': mocked_split2}, segment_memberships=set(), rbs_segments={})
results = e.eval_many_with_context('some_key', 'some_bucketing_key', ['feature2', 'feature4'], {}, ctx)
Expand All @@ -215,6 +222,8 @@ def test_get_gtreatment_for_split_no_condition_matches(self, mocker):
mocked_split.change_number = '123'
mocked_split.conditions = []
mocked_split.get_configurations_for = None
mocked_split.prerequisites = []

ctx = EvaluationContext(flags={'some': mocked_split}, segment_memberships=set(), rbs_segments={})
assert e._treatment_for_flag(mocked_split, 'some_key', 'some_bucketing', {}, ctx) == (
'off',
Expand All @@ -232,6 +241,8 @@ def test_get_gtreatment_for_split_non_rollout(self, mocker):
mocked_split = mocker.Mock(spec=Split)
mocked_split.killed = False
mocked_split.conditions = [mocked_condition_1]
mocked_split.prerequisites = []

treatment, label = e._treatment_for_flag(mocked_split, 'some_key', 'some_bucketing', {}, EvaluationContext(None, None, None))
assert treatment == 'on'
assert label == 'some_label'
Expand All @@ -240,7 +251,7 @@ def test_evaluate_treatment_with_rule_based_segment(self, mocker):
"""Test that a non-killed split returns the appropriate treatment."""
e = evaluator.Evaluator(splitters.Splitter())

mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False)
mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, [])

ctx = EvaluationContext(flags={'some': mocked_split}, segment_memberships=set(), rbs_segments={'sample_rule_based_segment': rule_based_segments.from_raw(rbs_raw)})
result = e.eval_with_context('bilal@split.io', 'bilal@split.io', 'some', {'email': 'bilal@split.io'}, ctx)
Expand All @@ -257,7 +268,7 @@ def test_evaluate_treatment_with_rbs_in_condition(self):
with open(rbs_segments, 'r') as flo:
data = json.loads(flo.read())

mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False)
mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, [])
rbs = rule_based_segments.from_raw(data["rbs"]["d"][0])
rbs2 = rule_based_segments.from_raw(data["rbs"]["d"][1])
rbs_storage.update([rbs, rbs2], [], 12)
Expand All @@ -279,7 +290,7 @@ def test_using_segment_in_excluded(self):
segment_storage = InMemorySegmentStorage()
evaluation_facctory = EvaluationDataFactory(splits_storage, segment_storage, rbs_storage)

mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False)
mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, [])
rbs = rule_based_segments.from_raw(data["rbs"]["d"][0])
rbs_storage.update([rbs], [], 12)
splits_storage.update([mocked_split], [], 12)
Expand All @@ -303,7 +314,7 @@ def test_using_rbs_in_excluded(self):
segment_storage = InMemorySegmentStorage()
evaluation_facctory = EvaluationDataFactory(splits_storage, segment_storage, rbs_storage)

mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False)
mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, [])
rbs = rule_based_segments.from_raw(data["rbs"]["d"][0])
rbs2 = rule_based_segments.from_raw(data["rbs"]["d"][1])
rbs_storage.update([rbs, rbs2], [], 12)
Expand All @@ -315,7 +326,52 @@ def test_using_rbs_in_excluded(self):
assert e.eval_with_context('bilal', 'bilal', 'some', {'email': 'bilal'}, ctx)['treatment'] == "on"
ctx = evaluation_facctory.context_for('bilal2@split.io', ['some'])
assert e.eval_with_context('bilal2@split.io', 'bilal2@split.io', 'some', {'email': 'bilal2@split.io'}, ctx)['treatment'] == "off"


def test_prerequisites(self):
splits_load = os.path.join(os.path.dirname(__file__), '../models/grammar/files', 'splits_prereq.json')
with open(splits_load, 'r') as flo:
data = json.loads(flo.read())
e = evaluator.Evaluator(splitters.Splitter())
splits_storage = InMemorySplitStorage()
rbs_storage = InMemoryRuleBasedSegmentStorage()
segment_storage = InMemorySegmentStorage()
evaluation_facctory = EvaluationDataFactory(splits_storage, segment_storage, rbs_storage)

rbs = rule_based_segments.from_raw(data["rbs"]["d"][0])
split1 = from_raw(data["ff"]["d"][0])
split2 = from_raw(data["ff"]["d"][1])
split3 = from_raw(data["ff"]["d"][2])
split4 = from_raw(data["ff"]["d"][3])
rbs_storage.update([rbs], [], 12)
splits_storage.update([split1, split2, split3, split4], [], 12)
segment = segments.from_raw({'name': 'segment-test', 'added': ['pato@split.io'], 'removed': [], 'till': 123})
segment_storage.put(segment)

ctx = evaluation_facctory.context_for('bilal@split.io', ['test_prereq'])
assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'test_prereq', {'email': 'bilal@split.io'}, ctx)['treatment'] == "on"
assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'test_prereq', {}, ctx)['treatment'] == "def_treatment"

ctx = evaluation_facctory.context_for('mauro@split.io', ['test_prereq'])
assert e.eval_with_context('mauro@split.io', 'mauro@split.io', 'test_prereq', {'email': 'mauro@split.io'}, ctx)['treatment'] == "def_treatment"

ctx = evaluation_facctory.context_for('pato@split.io', ['test_prereq'])
assert e.eval_with_context('pato@split.io', 'pato@split.io', 'test_prereq', {'email': 'pato@split.io'}, ctx)['treatment'] == "def_treatment"

ctx = evaluation_facctory.context_for('nico@split.io', ['test_prereq'])
assert e.eval_with_context('nico@split.io', 'nico@split.io', 'test_prereq', {'email': 'nico@split.io'}, ctx)['treatment'] == "on"

ctx = evaluation_facctory.context_for('bilal@split.io', ['prereq_chain'])
assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'prereq_chain', {'email': 'bilal@split.io'}, ctx)['treatment'] == "on_whitelist"

ctx = evaluation_facctory.context_for('nico@split.io', ['prereq_chain'])
assert e.eval_with_context('nico@split.io', 'nico@split.io', 'test_prereq', {'email': 'nico@split.io'}, ctx)['treatment'] == "on"

ctx = evaluation_facctory.context_for('pato@split.io', ['prereq_chain'])
assert e.eval_with_context('pato@split.io', 'pato@split.io', 'prereq_chain', {'email': 'pato@split.io'}, ctx)['treatment'] == "on_default"

ctx = evaluation_facctory.context_for('mauro@split.io', ['prereq_chain'])
assert e.eval_with_context('mauro@split.io', 'mauro@split.io', 'prereq_chain', {'email': 'mauro@split.io'}, ctx)['treatment'] == "on_default"

@pytest.mark.asyncio
async def test_evaluate_treatment_with_rbs_in_condition_async(self):
e = evaluator.Evaluator(splitters.Splitter())
Expand Down Expand Up @@ -388,16 +444,63 @@ async def test_using_rbs_in_excluded_async(self):
ctx = await evaluation_facctory.context_for('bilal2@split.io', ['some'])
assert e.eval_with_context('bilal2@split.io', 'bilal2@split.io', 'some', {'email': 'bilal2@split.io'}, ctx)['treatment'] == "off"

@pytest.mark.asyncio
async def test_prerequisites(self):
splits_load = os.path.join(os.path.dirname(__file__), '../models/grammar/files', 'splits_prereq.json')
with open(splits_load, 'r') as flo:
data = json.loads(flo.read())
e = evaluator.Evaluator(splitters.Splitter())
splits_storage = InMemorySplitStorageAsync()
rbs_storage = InMemoryRuleBasedSegmentStorageAsync()
segment_storage = InMemorySegmentStorageAsync()
evaluation_facctory = AsyncEvaluationDataFactory(splits_storage, segment_storage, rbs_storage)

rbs = rule_based_segments.from_raw(data["rbs"]["d"][0])
split1 = from_raw(data["ff"]["d"][0])
split2 = from_raw(data["ff"]["d"][1])
split3 = from_raw(data["ff"]["d"][2])
split4 = from_raw(data["ff"]["d"][3])
await rbs_storage.update([rbs], [], 12)
await splits_storage.update([split1, split2, split3, split4], [], 12)
segment = segments.from_raw({'name': 'segment-test', 'added': ['pato@split.io'], 'removed': [], 'till': 123})
await segment_storage.put(segment)

ctx = await evaluation_facctory.context_for('bilal@split.io', ['test_prereq'])
assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'test_prereq', {'email': 'bilal@split.io'}, ctx)['treatment'] == "on"
assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'test_prereq', {}, ctx)['treatment'] == "def_treatment"

ctx = await evaluation_facctory.context_for('mauro@split.io', ['test_prereq'])
assert e.eval_with_context('mauro@split.io', 'mauro@split.io', 'test_prereq', {'email': 'mauro@split.io'}, ctx)['treatment'] == "def_treatment"

ctx = await evaluation_facctory.context_for('pato@split.io', ['test_prereq'])
assert e.eval_with_context('pato@split.io', 'pato@split.io', 'test_prereq', {'email': 'pato@split.io'}, ctx)['treatment'] == "def_treatment"

ctx = await evaluation_facctory.context_for('nico@split.io', ['test_prereq'])
assert e.eval_with_context('nico@split.io', 'nico@split.io', 'test_prereq', {'email': 'nico@split.io'}, ctx)['treatment'] == "on"

ctx = await evaluation_facctory.context_for('bilal@split.io', ['prereq_chain'])
assert e.eval_with_context('bilal@split.io', 'bilal@split.io', 'prereq_chain', {'email': 'bilal@split.io'}, ctx)['treatment'] == "on_whitelist"

ctx = await evaluation_facctory.context_for('nico@split.io', ['prereq_chain'])
assert e.eval_with_context('nico@split.io', 'nico@split.io', 'test_prereq', {'email': 'nico@split.io'}, ctx)['treatment'] == "on"

ctx = await evaluation_facctory.context_for('pato@split.io', ['prereq_chain'])
assert e.eval_with_context('pato@split.io', 'pato@split.io', 'prereq_chain', {'email': 'pato@split.io'}, ctx)['treatment'] == "on_default"

ctx = await evaluation_facctory.context_for('mauro@split.io', ['prereq_chain'])
assert e.eval_with_context('mauro@split.io', 'mauro@split.io', 'prereq_chain', {'email': 'mauro@split.io'}, ctx)['treatment'] == "on_default"

class EvaluationDataFactoryTests(object):
"""Test evaluation factory class."""

def test_get_context(self):
"""Test context."""
mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False)
mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, [Prerequisites('split2', ['on'])])
split2 = Split('split2', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, [])
flag_storage = InMemorySplitStorage([])
segment_storage = InMemorySegmentStorage()
rbs_segment_storage = InMemoryRuleBasedSegmentStorage()
flag_storage.update([mocked_split], [], -1)
flag_storage.update([mocked_split, split2], [], -1)
rbs = copy.deepcopy(rbs_raw)
rbs['conditions'].append(
{"matcherGroup": {
Expand All @@ -421,6 +524,7 @@ def test_get_context(self):
ec = eval_factory.context_for('bilal@split.io', ['some'])
assert ec.rbs_segments == {'sample_rule_based_segment': rbs}
assert ec.segment_memberships == {"employees": False}
assert ec.flags.get("split2").name == "split2"

segment_storage.update("employees", {"mauro@split.io"}, {}, 1234)
ec = eval_factory.context_for('mauro@split.io', ['some'])
Expand All @@ -433,11 +537,12 @@ class EvaluationDataFactoryAsyncTests(object):
@pytest.mark.asyncio
async def test_get_context(self):
"""Test context."""
mocked_split = Split('some', 123, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False)
mocked_split = Split('some', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, [Prerequisites('split2', ['on'])])
split2 = Split('split2', 12345, False, 'off', 'user', Status.ACTIVE, 12, split_conditions, 1.2, 100, 1234, {}, None, False, [])
flag_storage = InMemorySplitStorageAsync([])
segment_storage = InMemorySegmentStorageAsync()
rbs_segment_storage = InMemoryRuleBasedSegmentStorageAsync()
await flag_storage.update([mocked_split], [], -1)
await flag_storage.update([mocked_split, split2], [], -1)
rbs = copy.deepcopy(rbs_raw)
rbs['conditions'].append(
{"matcherGroup": {
Expand All @@ -461,6 +566,7 @@ async def test_get_context(self):
ec = await eval_factory.context_for('bilal@split.io', ['some'])
assert ec.rbs_segments == {'sample_rule_based_segment': rbs}
assert ec.segment_memberships == {"employees": False}
assert ec.flags.get("split2").name == "split2"

await segment_storage.update("employees", {"mauro@split.io"}, {}, 1234)
ec = await eval_factory.context_for('mauro@split.io', ['some'])
Expand Down