plug threat intel feed into detector creation

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>
opensearch-project · Oct 12, 2023 · d84fa71 · d84fa71
1 parent c7d595b
commit d84fa71
Show file tree

Hide file tree

Showing 5 changed files with 77 additions and 23 deletions.
diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java
@@ -12,12 +12,9 @@
 import java.util.function.Supplier;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.opensearch.cluster.routing.Preference;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.action.ActionRequest;
 import org.opensearch.core.action.ActionResponse;
-import org.opensearch.action.search.SearchRequest;
-import org.opensearch.action.search.SearchResponse;
 import org.opensearch.client.Client;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.node.DiscoveryNode;
@@ -38,7 +35,6 @@
 import org.opensearch.index.codec.CodecServiceFactory;
 import org.opensearch.index.engine.EngineFactory;
 import org.opensearch.index.mapper.Mapper;
-import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.plugins.ActionPlugin;
 import org.opensearch.plugins.ClusterPlugin;
 import org.opensearch.plugins.EnginePlugin;
@@ -49,7 +45,6 @@
 import org.opensearch.rest.RestController;
 import org.opensearch.rest.RestHandler;
 import org.opensearch.script.ScriptService;
-import org.opensearch.search.builder.SearchSourceBuilder;
 import org.opensearch.securityanalytics.action.*;
 import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecService;
 import org.opensearch.securityanalytics.correlation.index.mapper.CorrelationVectorFieldMapper;
@@ -62,6 +57,8 @@
 import org.opensearch.securityanalytics.model.CustomLogType;
 import org.opensearch.securityanalytics.model.ThreatIntelFeedData;
 import org.opensearch.securityanalytics.resthandler.*;
+import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService;
+import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService;
 import org.opensearch.securityanalytics.transport.*;
 import org.opensearch.securityanalytics.model.Rule;
 import org.opensearch.securityanalytics.model.Detector;
@@ -129,6 +126,7 @@ public Collection<Object> createComponents(Client client,
                                                NamedWriteableRegistry namedWriteableRegistry,
                                                IndexNameExpressionResolver indexNameExpressionResolver,
                                                Supplier<RepositoriesService> repositoriesServiceSupplier) {
+
         builtinLogTypeLoader = new BuiltinLogTypeLoader();
         logTypeService = new LogTypeService(client, clusterService, xContentRegistry, builtinLogTypeLoader);
         detectorIndices = new DetectorIndices(client.admin(), clusterService, threadPool);
@@ -139,11 +137,13 @@ public Collection<Object> createComponents(Client client,
         mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager, logTypeService);
         ruleIndices = new RuleIndices(logTypeService, client, clusterService, threadPool);
         correlationRuleIndices = new CorrelationRuleIndices(client, clusterService);
+        ThreatIntelFeedDataService threatIntelFeedDataService = new ThreatIntelFeedDataService(clusterService.state(), client, indexNameExpressionResolver, xContentRegistry);
+        DetectorThreatIntelService detectorThreatIntelService = new DetectorThreatIntelService(threatIntelFeedDataService);
         this.client = client;
 
         return List.of(
                 detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices,
-                mapperService, indexTemplateManager, builtinLogTypeLoader
+                mapperService, indexTemplateManager, builtinLogTypeLoader, threatIntelFeedDataService, detectorThreatIntelService
         );
     }
 

diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java
@@ -1,7 +1,10 @@
 package org.opensearch.securityanalytics.threatIntel;
 
 import org.opensearch.commons.alerting.model.DocLevelQuery;
+import org.opensearch.core.rest.RestStatus;
+import org.opensearch.securityanalytics.model.Detector;
 import org.opensearch.securityanalytics.model.ThreatIntelFeedData;
+import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
 
 import java.util.Collections;
 import java.util.List;
@@ -11,8 +14,14 @@
 
 public class DetectorThreatIntelService {
 
+    private final ThreatIntelFeedDataService threatIntelFeedDataService;
+
+    public DetectorThreatIntelService(ThreatIntelFeedDataService threatIntelFeedDataService) {
+        this.threatIntelFeedDataService = threatIntelFeedDataService;
+    }
+
     /** Convert the feed data IOCs into query string query format to create doc level queries. */
-    public static DocLevelQuery createDocLevelQueryFromThreatIntelList(
+    public DocLevelQuery createDocLevelQueryFromThreatIntelList(
             List<ThreatIntelFeedData> tifdList, String docLevelQueryId
             ) {
         Set<String> iocs = tifdList.stream().map(ThreatIntelFeedData::getIocValue).collect(Collectors.toSet());
@@ -23,7 +32,7 @@ public static DocLevelQuery createDocLevelQueryFromThreatIntelList(
         );
     }
 
-    private static String buildQueryStringQueryWithIocList(Set<String> iocs) {
+    private String buildQueryStringQueryWithIocList(Set<String> iocs) {
         StringBuilder sb = new StringBuilder();
 
         for(String ioc : iocs) {
@@ -36,4 +45,17 @@ private static String buildQueryStringQueryWithIocList(Set<String> iocs) {
         }
         return sb.toString();
     }
+
+    public DocLevelQuery createDocLevelQueryFromThreatIntel(Detector detector) {
+        // for testing validation only.
+        if(detector.getThreatIntelEnabled() ==false) {
+            throw new SecurityAnalyticsException(
+                    "trying to create threat intel feed queries when flag to use threat intel is disabled.",
+                    RestStatus.FORBIDDEN, new IllegalArgumentException());
+
+        }
+        // TODO: plugin logic to run job for populating threat intel feed data
+        /*threatIntelFeedDataService.getThreatIntelFeedData("ip_address", );*/
+        return null;
+    }
 }
diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java
@@ -28,25 +28,45 @@
  */
 public class ThreatIntelFeedDataService {
     private static final Logger log = LogManager.getLogger(FindingsService.class);
+    private final ClusterState state;
+    private final Client client;
+    private final IndexNameExpressionResolver indexNameExpressionResolver;
 
-    public static void getThreatIntelFeedData(ClusterState state, Client client, IndexNameExpressionResolver indexNameExpressionResolver,
-                                       String feedName, String iocType,
-                                       ActionListener<List<ThreatIntelFeedData>> listener, NamedXContentRegistry xContentRegistry) {
-        String indexPattern = String.format(".opendsearch-sap-threatintel-%s*", feedName);
-        String tifdIndex = IndexUtils.getNewIndexByCreationDate(state, indexNameExpressionResolver, indexPattern);
+    public ThreatIntelFeedDataService(
+            ClusterState state,
+            Client client,
+            IndexNameExpressionResolver indexNameExpressionResolver,
+            NamedXContentRegistry xContentRegistry) {
+        this.state = state;
+        this.client = client;
+        this.indexNameExpressionResolver = indexNameExpressionResolver;
+        this.xContentRegistry = xContentRegistry;
+    }
+
+    private final NamedXContentRegistry xContentRegistry;
+
+    public void getThreatIntelFeedData(
+            String iocType,
+            ActionListener<List<ThreatIntelFeedData>> listener
+    ) {
+        String tifdIndex = IndexUtils.getNewIndexByCreationDate(
+                this.state,
+                this.indexNameExpressionResolver,
+                ".opendsearch-sap-threatintel*"
+        );
         SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
         sourceBuilder.query(QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("ioc_type", iocType)));
         SearchRequest searchRequest = new SearchRequest(tifdIndex);
         searchRequest.source().size(9999); //TODO: convert to scroll
         searchRequest.source(sourceBuilder);
-        client.search(searchRequest, ActionListener.wrap(r -> listener.onResponse(getTifdList(r, xContentRegistry)), e -> {
+        client.search(searchRequest, ActionListener.wrap(r -> listener.onResponse(getTifdList(r)), e -> {
             log.error(String.format(
-                    "Failed to fetch threat intel feed data %s from system index %s", feedName, tifdIndex), e);
+                    "Failed to fetch threat intel feed data from system index %s", tifdIndex), e);
             listener.onFailure(e);
         }));
     }
 
-    private static List<ThreatIntelFeedData> getTifdList(SearchResponse searchResponse, NamedXContentRegistry xContentRegistry) {
+    private List<ThreatIntelFeedData> getTifdList(SearchResponse searchResponse) {
         List<ThreatIntelFeedData> list = new ArrayList<>();
         if (searchResponse.getHits().getHits().length != 0) {
             Arrays.stream(searchResponse.getHits().getHits()).forEach(hit -> {
@@ -57,8 +77,10 @@ private static List<ThreatIntelFeedData> getTifdList(SearchResponse searchRespon
                     );
                     list.add(ThreatIntelFeedData.parse(xcp, hit.getId(), hit.getVersion()));
                 } catch (Exception e) {
-                    log.error(() ->
-                            new ParameterizedMessage("Failed to parse Threat intel feed data doc from hit {}", hit), e);
+                    log.error(() -> new ParameterizedMessage(
+                                    "Failed to parse Threat intel feed data doc from hit {}", hit),
+                            e
+                    );
                 }
 
             });

diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java
@@ -96,6 +96,7 @@
 import org.opensearch.securityanalytics.rules.backend.QueryBackend;
 import org.opensearch.securityanalytics.rules.exceptions.SigmaError;
 import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
+import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService;
 import org.opensearch.securityanalytics.util.DetectorIndices;
 import org.opensearch.securityanalytics.util.DetectorUtils;
 import org.opensearch.securityanalytics.util.IndexUtils;
@@ -155,6 +156,7 @@ public class TransportIndexDetectorAction extends HandledTransportAction<IndexDe
 
     private final MonitorService monitorService;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
+    private final DetectorThreatIntelService detectorThreatIntelService;
 
     private final TimeValue indexTimeout;
     @Inject
@@ -170,7 +172,8 @@ public TransportIndexDetectorAction(TransportService transportService,
                                         Settings settings,
                                         NamedWriteableRegistry namedWriteableRegistry,
                                         LogTypeService logTypeService,
-                                        IndexNameExpressionResolver indexNameExpressionResolver) {
+                                        IndexNameExpressionResolver indexNameExpressionResolver,
+                                        DetectorThreatIntelService detectorThreatIntelService) {
         super(IndexDetectorAction.NAME, transportService, actionFilters, IndexDetectorRequest::new);
         this.client = client;
         this.xContentRegistry = xContentRegistry;
@@ -183,6 +186,7 @@ public TransportIndexDetectorAction(TransportService transportService,
         this.namedWriteableRegistry = namedWriteableRegistry;
         this.logTypeService = logTypeService;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
+        this.detectorThreatIntelService = detectorThreatIntelService;
         this.threadPool = this.detectorIndices.getThreadPool();
         this.indexTimeout = SecurityAnalyticsSettings.INDEX_TIMEOUT.get(this.settings);
         this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings);
@@ -648,8 +652,14 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List<Pair<String, Rule>
             DocLevelQuery docLevelQuery = new DocLevelQuery(id, name, Collections.emptyList(), actualQuery, tags);
             docLevelQueries.add(docLevelQuery);
         }
-        if(detector.getThreatIntelEnabled()) {
-            DetectorThreatIntelService
+        try {
+            if (detector.getThreatIntelEnabled()) {
+                DocLevelQuery docLevelQueryFromThreatIntel = detectorThreatIntelService.createDocLevelQueryFromThreatIntel(detector);
+                docLevelQueries.add(docLevelQueryFromThreatIntel);
+            }
+        } catch (Exception e) {
+            // not failing detector creation if any fatal exception occurs during doc level query creation from threat intel feed data
+            log.error("Failed to convert threat intel feed to. Proceeding with detector creation", e);
         }
         DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput(detector.getName(), detector.getInputs().get(0).getIndices(), docLevelQueries);
         docLevelMonitorInputs.add(docLevelMonitorInput);

diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java
@@ -172,8 +172,8 @@ public static CustomLogType randomCustomLogType(String name, String description,
     public static ThreatIntelFeedData randomThreatIntelFeedData() {
         return new ThreatIntelFeedData(
                 "IP_ADDRESS",
-                "123.442.111.112",
-                OpenSearchRestTestCase.randomAlphaOfLength(10),
+                ip,
+                "alientVault",
                 Instant.now()
         );
     }