-
Notifications
You must be signed in to change notification settings - Fork 72
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Threat intel feeds job runner and unit tests (#654)
* fix doc level query constructor (#651) Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * add mapping for indices storing threat intel feed data * fix feed indices mapping * add threat intel feed data dao Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * add threatIntelEnabled field in detector. Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * add threat intel feed service and searching feeds Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * ti feed data to doc level query convertor logic added * plug threat intel feed into detector creation Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * Preliminary framework for jobscheduler and datasource (#626) Signed-off-by: Joanne Wang <jowg@amazon.com> * with listener and processor Signed-off-by: Joanne Wang <jowg@amazon.com> * removed actions Signed-off-by: Joanne Wang <jowg@amazon.com> * clean up Signed-off-by: Joanne Wang <jowg@amazon.com> * added parser Signed-off-by: Joanne Wang <jowg@amazon.com> * add unit tests Signed-off-by: Joanne Wang <jowg@amazon.com> * refactored class names Signed-off-by: Joanne Wang <jowg@amazon.com> * before moving db Signed-off-by: Joanne Wang <jowg@amazon.com> * after moving db Signed-off-by: Joanne Wang <jowg@amazon.com> * added actions to plugin and removed user schedule Signed-off-by: Joanne Wang <jowg@amazon.com> * unit tests Signed-off-by: Joanne Wang <jowg@amazon.com> * fix build error Signed-off-by: Joanne Wang <jowg@amazon.com> * changed transport naming Signed-off-by: Joanne Wang <jowg@amazon.com> --------- Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> Signed-off-by: Joanne Wang <jowg@amazon.com> Co-authored-by: Surya Sashank Nistala <snistala@amazon.com>
- Loading branch information
1 parent
cfd1bf0
commit 55d332d
Showing
62 changed files
with
4,341 additions
and
2,511 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
161 changes: 161 additions & 0 deletions
161
src/main/java/org/opensearch/securityanalytics/sampleextension/SampleExtensionPlugin.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,161 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
| package org.opensearch.securityanalytics.sampleextension; | ||
|
|
||
| import org.apache.logging.log4j.LogManager; | ||
| import org.apache.logging.log4j.Logger; | ||
| import org.opensearch.client.Client; | ||
| import org.opensearch.cluster.metadata.IndexNameExpressionResolver; | ||
| import org.opensearch.cluster.node.DiscoveryNodes; | ||
| import org.opensearch.cluster.service.ClusterService; | ||
| import org.opensearch.common.settings.ClusterSettings; | ||
| import org.opensearch.common.settings.IndexScopedSettings; | ||
| import org.opensearch.common.settings.Settings; | ||
| import org.opensearch.common.settings.SettingsFilter; | ||
| import org.opensearch.core.common.io.stream.NamedWriteableRegistry; | ||
| import org.opensearch.core.xcontent.NamedXContentRegistry; | ||
| import org.opensearch.core.xcontent.XContentParser; | ||
| import org.opensearch.core.xcontent.XContentParserUtils; | ||
| import org.opensearch.env.Environment; | ||
| import org.opensearch.env.NodeEnvironment; | ||
| import org.opensearch.jobscheduler.spi.JobSchedulerExtension; | ||
| import org.opensearch.jobscheduler.spi.ScheduledJobParser; | ||
| import org.opensearch.jobscheduler.spi.ScheduledJobRunner; | ||
| import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; | ||
| import org.opensearch.plugins.ActionPlugin; | ||
| import org.opensearch.plugins.Plugin; | ||
| import org.opensearch.repositories.RepositoriesService; | ||
| import org.opensearch.rest.RestController; | ||
| import org.opensearch.rest.RestHandler; | ||
| import org.opensearch.script.ScriptService; | ||
| import org.opensearch.threadpool.ThreadPool; | ||
| import org.opensearch.watcher.ResourceWatcherService; | ||
|
|
||
| import java.io.IOException; | ||
| import java.time.Instant; | ||
| import java.util.Collection; | ||
| import java.util.Collections; | ||
| import java.util.List; | ||
| import java.util.function.Supplier; | ||
|
|
||
| /** | ||
| * Sample JobScheduler extension plugin. | ||
| * | ||
| * It use ".scheduler_sample_extension" index to manage its scheduled jobs, and exposes a REST API | ||
| * endpoint using {@link SampleExtensionRestHandler}. | ||
| * | ||
| */ | ||
| public class SampleExtensionPlugin extends Plugin implements ActionPlugin, JobSchedulerExtension { | ||
| private static final Logger log = LogManager.getLogger(SampleExtensionPlugin.class); | ||
|
|
||
| static final String JOB_INDEX_NAME = ".scheduler_sample_extension"; | ||
|
|
||
| @Override | ||
| public Collection<Object> createComponents( | ||
| Client client, | ||
| ClusterService clusterService, | ||
| ThreadPool threadPool, | ||
| ResourceWatcherService resourceWatcherService, | ||
| ScriptService scriptService, | ||
| NamedXContentRegistry xContentRegistry, | ||
| Environment environment, | ||
| NodeEnvironment nodeEnvironment, | ||
| NamedWriteableRegistry namedWriteableRegistry, | ||
| IndexNameExpressionResolver indexNameExpressionResolver, | ||
| Supplier<RepositoriesService> repositoriesServiceSupplier | ||
| ) { | ||
| SampleJobRunner jobRunner = SampleJobRunner.getJobRunnerInstance(); | ||
| jobRunner.setClusterService(clusterService); | ||
| jobRunner.setThreadPool(threadPool); | ||
| jobRunner.setClient(client); | ||
|
|
||
| return Collections.emptyList(); | ||
| } | ||
|
|
||
| @Override | ||
| public String getJobType() { | ||
| return "scheduler_sample_extension"; | ||
| } | ||
|
|
||
| @Override | ||
| public String getJobIndex() { | ||
| return JOB_INDEX_NAME; | ||
| } | ||
|
|
||
| @Override | ||
| public ScheduledJobRunner getJobRunner() { | ||
| return SampleJobRunner.getJobRunnerInstance(); | ||
| } | ||
|
|
||
| @Override | ||
| public ScheduledJobParser getJobParser() { | ||
| return (parser, id, jobDocVersion) -> { | ||
| SampleJobParameter jobParameter = new SampleJobParameter(); | ||
| XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); | ||
|
|
||
| while (!parser.nextToken().equals(XContentParser.Token.END_OBJECT)) { | ||
| String fieldName = parser.currentName(); | ||
| parser.nextToken(); | ||
| switch (fieldName) { | ||
| case SampleJobParameter.NAME_FIELD: | ||
| jobParameter.setJobName(parser.text()); | ||
| break; | ||
| case SampleJobParameter.ENABLED_FILED: | ||
| jobParameter.setEnabled(parser.booleanValue()); | ||
| break; | ||
| case SampleJobParameter.ENABLED_TIME_FILED: | ||
| jobParameter.setEnabledTime(parseInstantValue(parser)); | ||
| break; | ||
| case SampleJobParameter.LAST_UPDATE_TIME_FIELD: | ||
| jobParameter.setLastUpdateTime(parseInstantValue(parser)); | ||
| break; | ||
| case SampleJobParameter.SCHEDULE_FIELD: | ||
| jobParameter.setSchedule(ScheduleParser.parse(parser)); | ||
| break; | ||
| case SampleJobParameter.INDEX_NAME_FIELD: | ||
| jobParameter.setIndexToWatch(parser.text()); | ||
| break; | ||
| case SampleJobParameter.LOCK_DURATION_SECONDS: | ||
| jobParameter.setLockDurationSeconds(parser.longValue()); | ||
| break; | ||
| case SampleJobParameter.JITTER: | ||
| jobParameter.setJitter(parser.doubleValue()); | ||
| break; | ||
| default: | ||
| XContentParserUtils.throwUnknownToken(parser.currentToken(), parser.getTokenLocation()); | ||
| } | ||
| } | ||
| return jobParameter; | ||
| }; | ||
| } | ||
|
|
||
| private Instant parseInstantValue(XContentParser parser) throws IOException { | ||
| if (XContentParser.Token.VALUE_NULL.equals(parser.currentToken())) { | ||
| return null; | ||
| } | ||
| if (parser.currentToken().isValue()) { | ||
| return Instant.ofEpochMilli(parser.longValue()); | ||
| } | ||
| XContentParserUtils.throwUnknownToken(parser.currentToken(), parser.getTokenLocation()); | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public List<RestHandler> getRestHandlers( | ||
| Settings settings, | ||
| RestController restController, | ||
| ClusterSettings clusterSettings, | ||
| IndexScopedSettings indexScopedSettings, | ||
| SettingsFilter settingsFilter, | ||
| IndexNameExpressionResolver indexNameExpressionResolver, | ||
| Supplier<DiscoveryNodes> nodesInCluster | ||
| ) { | ||
| return Collections.singletonList(new SampleExtensionRestHandler()); | ||
| } | ||
| } |
Oops, something went wrong.