From 8f6827256db232c4455bf979f9bf0eaba0c43d90 Mon Sep 17 00:00:00 2001 From: Simone Berni Date: Tue, 9 Jan 2024 16:16:20 +0100 Subject: [PATCH] Job bi (#2052) * job bi Signed-off-by: 0ssigeno * More Signed-off-by: 0ssigeno * Job bi Signed-off-by: 0ssigeno * Migration Signed-off-by: 0ssigeno * Fix Signed-off-by: 0ssigeno * Fix Signed-off-by: 0ssigeno * Added playbook in the elastic template Signed-off-by: 0ssigeno * Fix Signed-off-by: 0ssigeno * Missin migrations Signed-off-by: 0ssigeno --------- Signed-off-by: 0ssigeno --- ..._analyzerreport_analyzerreportsbisearch.py | 18 +++ api_app/analyzers_manager/models.py | 1 + api_app/analyzers_manager/queryset.py | 2 +- ...onnectorreport_connectorreportsbisearch.py | 18 +++ api_app/connectors_manager/models.py | 1 + api_app/connectors_manager/queryset.py | 2 +- ..._ingestorreport_ingestorreportsbisearch.py | 18 +++ api_app/ingestors_manager/models.py | 1 + api_app/ingestors_manager/queryset.py | 2 +- api_app/migrations/0053_job_sent_to_bi.py | 17 +++ api_app/migrations/0054_job_jobbisearch.py | 18 +++ api_app/models.py | 9 ++ .../0022_pivotreport_pivotreportsbisearch.py | 18 +++ api_app/pivots_manager/models.py | 1 + api_app/pivots_manager/queryset.py | 2 +- api_app/queryset.py | 112 ++++++++++-------- api_app/serializers.py | 84 ++++++++++--- ...ualizerreport_visualizerreportsbisearch.py | 18 +++ api_app/visualizers_manager/models.py | 1 + api_app/visualizers_manager/queryset.py | 2 +- .../elastic_search_mappings/intel_owl_bi.json | 3 + intel_owl/tasks.py | 13 +- 22 files changed, 283 insertions(+), 78 deletions(-) create mode 100644 api_app/analyzers_manager/migrations/0057_analyzerreport_analyzerreportsbisearch.py create mode 100644 api_app/connectors_manager/migrations/0028_connectorreport_connectorreportsbisearch.py create mode 100644 api_app/ingestors_manager/migrations/0015_ingestorreport_ingestorreportsbisearch.py create mode 100644 api_app/migrations/0053_job_sent_to_bi.py create mode 100644 api_app/migrations/0054_job_jobbisearch.py create mode 100644 api_app/pivots_manager/migrations/0022_pivotreport_pivotreportsbisearch.py create mode 100644 api_app/visualizers_manager/migrations/0035_visualizerreport_visualizerreportsbisearch.py diff --git a/api_app/analyzers_manager/migrations/0057_analyzerreport_analyzerreportsbisearch.py b/api_app/analyzers_manager/migrations/0057_analyzerreport_analyzerreportsbisearch.py new file mode 100644 index 0000000000..56b2d1c383 --- /dev/null +++ b/api_app/analyzers_manager/migrations/0057_analyzerreport_analyzerreportsbisearch.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.8 on 2024-01-08 15:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("analyzers_manager", "0056_alter_analyzer_config_dns0_rrsets_data"), + ] + + operations = [ + migrations.AddIndex( + model_name="analyzerreport", + index=models.Index( + fields=["sent_to_bi", "-start_time"], name="analyzerreportsBISearch" + ), + ), + ] diff --git a/api_app/analyzers_manager/models.py b/api_app/analyzers_manager/models.py index 42bc289cd8..a18bbdc2dc 100644 --- a/api_app/analyzers_manager/models.py +++ b/api_app/analyzers_manager/models.py @@ -29,6 +29,7 @@ class AnalyzerReport(AbstractReport): class Meta: unique_together = [("config", "job")] + indexes = AbstractReport.Meta.indexes class MimeTypes(models.TextChoices): diff --git a/api_app/analyzers_manager/queryset.py b/api_app/analyzers_manager/queryset.py index e9b0ec96d3..ce6da8fd50 100644 --- a/api_app/analyzers_manager/queryset.py +++ b/api_app/analyzers_manager/queryset.py @@ -8,7 +8,7 @@ class AnalyzerReportQuerySet(AbstractReportQuerySet): @classmethod - def _get_serializer_class(cls) -> Type["AnalyzerReportBISerializer"]: + def _get_bi_serializer_class(cls) -> Type["AnalyzerReportBISerializer"]: from api_app.analyzers_manager.serializers import AnalyzerReportBISerializer return AnalyzerReportBISerializer diff --git a/api_app/connectors_manager/migrations/0028_connectorreport_connectorreportsbisearch.py b/api_app/connectors_manager/migrations/0028_connectorreport_connectorreportsbisearch.py new file mode 100644 index 0000000000..811e7044e2 --- /dev/null +++ b/api_app/connectors_manager/migrations/0028_connectorreport_connectorreportsbisearch.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.8 on 2024-01-09 14:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("connectors_manager", "0027_connectorreport_sent_to_bi"), + ] + + operations = [ + migrations.AddIndex( + model_name="connectorreport", + index=models.Index( + fields=["sent_to_bi", "-start_time"], name="connectorreportsBISearch" + ), + ), + ] diff --git a/api_app/connectors_manager/models.py b/api_app/connectors_manager/models.py index a7e2b26597..0524f709ea 100644 --- a/api_app/connectors_manager/models.py +++ b/api_app/connectors_manager/models.py @@ -17,6 +17,7 @@ class ConnectorReport(AbstractReport): class Meta: unique_together = [("config", "job")] + indexes = AbstractReport.Meta.indexes class ConnectorConfig(PythonConfig): diff --git a/api_app/connectors_manager/queryset.py b/api_app/connectors_manager/queryset.py index 1450d22a1a..bcc969a890 100644 --- a/api_app/connectors_manager/queryset.py +++ b/api_app/connectors_manager/queryset.py @@ -8,7 +8,7 @@ class ConnectorReportQuerySet(AbstractReportQuerySet): @classmethod - def _get_serializer_class(cls) -> Type["ConnectorReportBISerializer"]: + def _get_bi_serializer_class(cls) -> Type["ConnectorReportBISerializer"]: from api_app.connectors_manager.serializers import ConnectorReportBISerializer return ConnectorReportBISerializer diff --git a/api_app/ingestors_manager/migrations/0015_ingestorreport_ingestorreportsbisearch.py b/api_app/ingestors_manager/migrations/0015_ingestorreport_ingestorreportsbisearch.py new file mode 100644 index 0000000000..9a5a73860b --- /dev/null +++ b/api_app/ingestors_manager/migrations/0015_ingestorreport_ingestorreportsbisearch.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.8 on 2024-01-09 14:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("ingestors_manager", "0014_ingestorreport_sent_to_bi"), + ] + + operations = [ + migrations.AddIndex( + model_name="ingestorreport", + index=models.Index( + fields=["sent_to_bi", "-start_time"], name="ingestorreportsBISearch" + ), + ), + ] diff --git a/api_app/ingestors_manager/models.py b/api_app/ingestors_manager/models.py index 70ef0cbbcc..ff5bfa907a 100644 --- a/api_app/ingestors_manager/models.py +++ b/api_app/ingestors_manager/models.py @@ -36,6 +36,7 @@ class IngestorReport(AbstractReport): class Meta: ordering = ["pk"] + indexes = AbstractReport.Meta.indexes def clean_report(self): if isinstance(self.report, list) and self.max_size_report is not None: diff --git a/api_app/ingestors_manager/queryset.py b/api_app/ingestors_manager/queryset.py index 386a7cdb48..5b1d2c385e 100644 --- a/api_app/ingestors_manager/queryset.py +++ b/api_app/ingestors_manager/queryset.py @@ -8,7 +8,7 @@ class IngestorReportQuerySet(AbstractReportQuerySet): @classmethod - def _get_serializer_class(cls) -> Type["IngestorReportBISerializer"]: + def _get_bi_serializer_class(cls) -> Type["IngestorReportBISerializer"]: from api_app.ingestors_manager.serializers import IngestorReportBISerializer return IngestorReportBISerializer diff --git a/api_app/migrations/0053_job_sent_to_bi.py b/api_app/migrations/0053_job_sent_to_bi.py new file mode 100644 index 0000000000..c39b8a4ed5 --- /dev/null +++ b/api_app/migrations/0053_job_sent_to_bi.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.8 on 2024-01-08 14:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api_app", "0052_periodic_task_bi"), + ] + + operations = [ + migrations.AddField( + model_name="job", + name="sent_to_bi", + field=models.BooleanField(default=False, editable=False), + ), + ] diff --git a/api_app/migrations/0054_job_jobbisearch.py b/api_app/migrations/0054_job_jobbisearch.py new file mode 100644 index 0000000000..b9ce218316 --- /dev/null +++ b/api_app/migrations/0054_job_jobbisearch.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.8 on 2024-01-08 15:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api_app", "0053_job_sent_to_bi"), + ] + + operations = [ + migrations.AddIndex( + model_name="job", + index=models.Index( + fields=["sent_to_bi", "-received_request_time"], name="JobBISearch" + ), + ), + ] diff --git a/api_app/models.py b/api_app/models.py index 4c9e8be34b..70fa493b2a 100644 --- a/api_app/models.py +++ b/api_app/models.py @@ -219,6 +219,9 @@ class Meta: fields=["playbook_to_execute", "finished_analysis_time", "user"], name="PlaybookConfigOrdering", ), + models.Index( + fields=["sent_to_bi", "-received_request_time"], name="JobBISearch" + ), ] # constants @@ -306,6 +309,7 @@ class Meta: scan_check_time = models.DurationField( null=True, blank=True, default=datetime.timedelta(hours=24) ) + sent_to_bi = models.BooleanField(editable=False, default=False) def __str__(self): return f'{self.__class__.__name__}(#{self.pk}, "{self.analyzed_object_name}")' @@ -914,6 +918,11 @@ class AbstractReport(models.Model): class Meta: abstract = True + indexes = [ + models.Index( + fields=["sent_to_bi", "-start_time"], name="%(class)ssBISearch" + ) + ] def __str__(self): return f"{self.__class__.__name__}(job:#{self.job_id}, {self.config.name})" diff --git a/api_app/pivots_manager/migrations/0022_pivotreport_pivotreportsbisearch.py b/api_app/pivots_manager/migrations/0022_pivotreport_pivotreportsbisearch.py new file mode 100644 index 0000000000..fc48691149 --- /dev/null +++ b/api_app/pivots_manager/migrations/0022_pivotreport_pivotreportsbisearch.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.8 on 2024-01-09 14:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("pivots_manager", "0021_pivotreport_sent_to_bi"), + ] + + operations = [ + migrations.AddIndex( + model_name="pivotreport", + index=models.Index( + fields=["sent_to_bi", "-start_time"], name="pivotreportsBISearch" + ), + ), + ] diff --git a/api_app/pivots_manager/models.py b/api_app/pivots_manager/models.py index acf6a81896..2ff7e1c6fb 100644 --- a/api_app/pivots_manager/models.py +++ b/api_app/pivots_manager/models.py @@ -28,6 +28,7 @@ class PivotReport(AbstractReport): class Meta: unique_together = [("config", "job")] + indexes = AbstractReport.Meta.indexes class PivotMap(models.Model): diff --git a/api_app/pivots_manager/queryset.py b/api_app/pivots_manager/queryset.py index d17b97f442..9ccf9ad1b9 100644 --- a/api_app/pivots_manager/queryset.py +++ b/api_app/pivots_manager/queryset.py @@ -36,7 +36,7 @@ def valid( class PivotReportQuerySet(AbstractReportQuerySet): @classmethod - def _get_serializer_class(cls) -> Type["PivotReportBISerializer"]: + def _get_bi_serializer_class(cls) -> Type["PivotReportBISerializer"]: from api_app.pivots_manager.serializers import PivotReportBISerializer return PivotReportBISerializer diff --git a/api_app/queryset.py b/api_app/queryset.py index 3ac141a7fc..2ad92e8302 100644 --- a/api_app/queryset.py +++ b/api_app/queryset.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: from api_app.models import PythonConfig - from api_app.serializers import AbstractReportBISerializer + from api_app.serializers import AbstractBIInterface from celery.canvas import Signature from django.db import models @@ -37,6 +37,57 @@ from certego_saas.apps.user.models import User +class SendToBiQuerySet(models.QuerySet): + @classmethod + def _get_bi_serializer_class(cls) -> Type["AbstractBIInterface"]: + raise NotImplementedError() + + @staticmethod + def _create_index_template(): + if not settings.ELASTICSEARCH_CLIENT.indices.exists_template( + name=settings.ELASTICSEARCH_BI_INDEX + ): + with open( + settings.CONFIG_ROOT / "elastic_search_mappings" / "intel_owl_bi.json" + ) as f: + body = json.load(f) + body["index_patterns"] = [f"{settings.ELASTICSEARCH_BI_INDEX}-*"] + settings.ELASTICSEARCH_CLIENT.indices.put_template( + name=settings.ELASTICSEARCH_BI_INDEX, body=body + ) + + def send_to_elastic_as_bi(self, max_timeout: int = 60) -> bool: + from elasticsearch.helpers import bulk + + BULK_MAX_SIZE = 1000 + found_errors = False + + p = Paginator(self, BULK_MAX_SIZE) + for i in p.page_range: + page = p.get_page(i) + objects = page.object_list + serializer = self._get_bi_serializer_class()(instance=objects, many=True) + objects_serialized = serializer.data + _, errors = bulk( + settings.ELASTICSEARCH_CLIENT, + objects_serialized, + request_timeout=max_timeout, + ) + if errors: + logging.error( + f"Errors on sending to elastic: {errors}." + " We are not marking objects as sent." + ) + found_errors |= errors + else: + logging.info("BI sent") + self.model.objects.filter( + pk__in=objects.values_list("pk", flat=True) + ).update(sent_to_bi=True) + self._create_index_template() + return found_errors + + class CleanOnCreateQuerySet(models.QuerySet): def create(self, **kwargs): obj = self.model(**kwargs) @@ -78,7 +129,13 @@ def annotate_runnable(self, user: User = None) -> QuerySet: return self.annotate(runnable=Exists(qs)) -class JobQuerySet(CleanOnCreateQuerySet): +class JobQuerySet(CleanOnCreateQuerySet, SendToBiQuerySet): + @classmethod + def _get_bi_serializer_class(cls): + from api_app.serializers import JobBISerializer + + return JobBISerializer + def visible_for_user(self, user: User) -> "JobQuerySet": """ User has access to: @@ -230,55 +287,8 @@ def annotate_value_for_user( ) -class AbstractReportQuerySet(QuerySet): - @classmethod - def _get_serializer_class(cls) -> Type["AbstractReportBISerializer"]: - raise NotImplementedError() - - @staticmethod - def _create_index_template(): - if not settings.ELASTICSEARCH_CLIENT.indices.exists_template( - name=settings.ELASTICSEARCH_BI_INDEX - ): - with open( - settings.CONFIG_ROOT / "elastic_search_mappings" / "intel_owl_bi.json" - ) as f: - body = json.load(f) - body["index_patterns"] = [f"{settings.ELASTICSEARCH_BI_INDEX}-*"] - settings.ELASTICSEARCH_CLIENT.indices.put_template( - name=settings.ELASTICSEARCH_BI_INDEX, body=body - ) - - def send_to_elastic_as_bi(self, max_timeout: int = 60) -> bool: - from elasticsearch.helpers import bulk - - BULK_MAX_SIZE = 1000 - found_errors = False - - p = Paginator(self.order_by("pk"), BULK_MAX_SIZE) - for i in p.page_range: - page = p.get_page(i) - objects: AbstractReportQuerySet = page.object_list - serializer = self._get_serializer_class()(instance=objects, many=True) - objects_serialized = serializer.data - _, errors = bulk( - settings.ELASTICSEARCH_CLIENT, - objects_serialized, - request_timeout=max_timeout, - ) - if errors: - logging.error( - f"Errors on sending to elastic: {errors}." - " We are not marking objects as sent." - ) - found_errors |= errors - else: - logging.info("BI sent") - self.model.objects.filter( - pk__in=objects.values_list("pk", flat=True) - ).update(sent_to_bi=True) - self._create_index_template() - return found_errors +class AbstractReportQuerySet(SendToBiQuerySet): + ... class ModelWithOwnershipQuerySet: diff --git a/api_app/serializers.py b/api_app/serializers.py index 29b0c4da81..5afff1a1ba 100644 --- a/api_app/serializers.py +++ b/api_app/serializers.py @@ -20,7 +20,8 @@ from durin.serializers import UserSerializer from rest_framework import serializers as rfs from rest_framework.exceptions import ValidationError -from rest_framework.fields import SerializerMethodField, empty +from rest_framework.fields import Field, SerializerMethodField, empty +from rest_framework.serializers import ModelSerializer from certego_saas.apps.organization.organization import Organization from certego_saas.apps.organization.permissions import IsObjectOwnerOrSameOrgPermission @@ -1207,17 +1208,13 @@ def to_representation(self, instance): return super(PythonConfigSerializer, self).to_representation(instance) -class AbstractReportListSerializer(rfs.ListSerializer): - ... - - class AbstractReportSerializerInterface(rfs.ModelSerializer): name = rfs.SlugRelatedField(read_only=True, source="config", slug_field="name") type = rfs.SerializerMethodField(read_only=True, method_name="get_type") class Meta: fields = ["name", "process_time", "status", "end_time", "parameters", "type"] - list_serializer_class = AbstractReportListSerializer + list_serializer_class = rfs.ListSerializer def get_type(self, instance: AbstractReport): return instance.__class__.__name__.replace("Report", "").lower() @@ -1226,24 +1223,32 @@ def to_internal_value(self, data): raise NotImplementedError() -class AbstractReportBISerializer(AbstractReportSerializerInterface): +class AbstractBIInterface(ModelSerializer): application = rfs.CharField(read_only=True, default="IntelOwl") - timestamp = rfs.DateTimeField(source="start_time") - username = rfs.CharField(source="job.user.username") environment = rfs.SerializerMethodField(method_name="get_environment") + timestamp: Field + username: Field + name: Field + type: Field + process_time: Field + status: Field + end_time: Field class Meta: - fields = AbstractReportSerializerInterface.Meta.fields + [ + fields = [ "application", + "environment", "timestamp", "username", - "environment", + "name", + "type", + "process_time", + "status", + "end_time", ] - list_serializer_class = ( - AbstractReportSerializerInterface.Meta.list_serializer_class - ) - def get_environment(self, instance: AbstractReport): + @staticmethod + def get_environment(instance): if settings.STAGE_PRODUCTION: return "prod" elif settings.STAGE_STAGING: @@ -1251,8 +1256,8 @@ def get_environment(self, instance: AbstractReport): else: return "test" - def to_representation(self, instance: AbstractReport): - data = super().to_representation(instance) + @staticmethod + def to_elastic_dict(data): return { "_source": data, "_type": "_doc", @@ -1261,6 +1266,51 @@ def to_representation(self, instance: AbstractReport): } +class AbstractReportBISerializer( + AbstractBIInterface, AbstractReportSerializerInterface +): + timestamp = rfs.DateTimeField(source="start_time") + username = rfs.CharField(source="job.user.username") + + class Meta: + fields = AbstractBIInterface.Meta.fields + [ + "parameters", + ] + list_serializer_class = ( + AbstractReportSerializerInterface.Meta.list_serializer_class + ) + + def to_representation(self, instance: AbstractReport): + data = super().to_representation(instance) + return self.to_elastic_dict(data) + + +class JobBISerializer(AbstractBIInterface, ModelSerializer): + timestamp = rfs.DateTimeField(source="received_request_time") + username = rfs.CharField(source="user.username") + name = rfs.CharField(source="pk") + type = rfs.SerializerMethodField(read_only=True, method_name="get_type") + end_time = rfs.DateTimeField(source="finished_analysis_time") + playbook = rfs.SerializerMethodField(source="get_playbook") + + class Meta: + model = Job + fields = AbstractBIInterface.Meta.fields + ["playbook", "runtime_configuration"] + list_serializer_class = ( + AbstractReportSerializerInterface.Meta.list_serializer_class + ) + + def to_representation(self, instance: Job): + data = super().to_representation(instance) + return self.to_elastic_dict(data) + + def get_type(self, instance: Job): + return instance.__class__.__name__.lower() + + def get_playbook(self, instance: Job): + return instance.playbook_to_execute.name if instance.playbook_to_execute else "" + + class AbstractReportSerializer(AbstractReportSerializerInterface): class Meta: fields = AbstractReportSerializerInterface.Meta.fields + [ diff --git a/api_app/visualizers_manager/migrations/0035_visualizerreport_visualizerreportsbisearch.py b/api_app/visualizers_manager/migrations/0035_visualizerreport_visualizerreportsbisearch.py new file mode 100644 index 0000000000..2f41ca13f7 --- /dev/null +++ b/api_app/visualizers_manager/migrations/0035_visualizerreport_visualizerreportsbisearch.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.8 on 2024-01-09 14:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("visualizers_manager", "0034_visualizerreport_sent_to_bi"), + ] + + operations = [ + migrations.AddIndex( + model_name="visualizerreport", + index=models.Index( + fields=["sent_to_bi", "-start_time"], name="visualizerreportsBISearch" + ), + ), + ] diff --git a/api_app/visualizers_manager/models.py b/api_app/visualizers_manager/models.py index be73c92854..e3ed1d9051 100644 --- a/api_app/visualizers_manager/models.py +++ b/api_app/visualizers_manager/models.py @@ -21,6 +21,7 @@ class VisualizerReport(AbstractReport): class Meta: ordering = ["pk"] + indexes = AbstractReport.Meta.indexes class VisualizerConfig(PythonConfig): diff --git a/api_app/visualizers_manager/queryset.py b/api_app/visualizers_manager/queryset.py index 6d5d0cccbb..8566102c04 100644 --- a/api_app/visualizers_manager/queryset.py +++ b/api_app/visualizers_manager/queryset.py @@ -8,7 +8,7 @@ class VisualizerReportQuerySet(AbstractReportQuerySet): @classmethod - def _get_serializer_class(cls) -> Type["VisualizerReportBISerializer"]: + def _get_bi_serializer_class(cls) -> Type["VisualizerReportBISerializer"]: from api_app.visualizers_manager.serializers import VisualizerReportBISerializer return VisualizerReportBISerializer diff --git a/configuration/elastic_search_mappings/intel_owl_bi.json b/configuration/elastic_search_mappings/intel_owl_bi.json index c345a3a4b7..8ae263db84 100644 --- a/configuration/elastic_search_mappings/intel_owl_bi.json +++ b/configuration/elastic_search_mappings/intel_owl_bi.json @@ -33,6 +33,9 @@ "parameters": { "type": "object", "dynamic": true + }, + "playbook": { + "type": "keyword" } } } diff --git a/intel_owl/tasks.py b/intel_owl/tasks.py index eac12c3304..516c3a7668 100644 --- a/intel_owl/tasks.py +++ b/intel_owl/tasks.py @@ -344,11 +344,11 @@ def beat_init_connect(*args, sender: Consumer = None, **kwargs): @shared_task(base=FailureLoggedTask, name="send_bi_to_elastic", soft_time_limit=300) -def send_bi_to_elastic(max_timeout: int = 60): +def send_bi_to_elastic(max_timeout: int = 60, max_objects: int = 10000): from api_app.analyzers_manager.models import AnalyzerReport from api_app.connectors_manager.models import ConnectorReport from api_app.ingestors_manager.models import IngestorReport - from api_app.models import AbstractReport + from api_app.models import AbstractReport, Job from api_app.pivots_manager.models import PivotReport from api_app.visualizers_manager.models import VisualizerReport @@ -361,9 +361,12 @@ def send_bi_to_elastic(max_timeout: int = 60): VisualizerReport, ]: report_class: typing.Type[AbstractReport] - report_class.objects.filter(sent_to_bi=False).send_to_elastic_as_bi( - max_timeout=max_timeout - ) + report_class.objects.filter(sent_to_bi=False).order_by("-start_time")[ + :max_objects + ].send_to_elastic_as_bi(max_timeout=max_timeout) + Job.objects.filter(sent_to_bi=False).order_by("-received_request_time")[ + :max_objects + ].send_to_elastic_as_bi(max_timeout=max_timeout) # set logger