-
Notifications
You must be signed in to change notification settings - Fork 194
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'opensearch-project:main' into geoip-processor-integrati…
…on-test
- Loading branch information
Showing
125 changed files
with
5,926 additions
and
1,005 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
69 changes: 69 additions & 0 deletions
69
data-prepper-api/src/test/java/org/opensearch/dataprepper/model/codec/OutputCodecTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
package org.opensearch.dataprepper.model.codec; | ||
|
||
import com.fasterxml.jackson.core.JsonProcessingException; | ||
import org.junit.jupiter.api.BeforeEach; | ||
import org.junit.jupiter.api.Test; | ||
import org.opensearch.dataprepper.model.event.DefaultEventMetadata; | ||
import org.opensearch.dataprepper.model.event.Event; | ||
import org.opensearch.dataprepper.model.event.EventMetadata; | ||
import org.opensearch.dataprepper.model.event.EventType; | ||
import org.opensearch.dataprepper.model.event.JacksonEvent; | ||
import org.opensearch.dataprepper.model.log.JacksonLog; | ||
|
||
import java.io.IOException; | ||
import java.io.OutputStream; | ||
import java.util.Arrays; | ||
import java.util.LinkedHashMap; | ||
import java.util.Map; | ||
import java.util.Set; | ||
import java.util.UUID; | ||
|
||
import static org.junit.Assert.assertNotEquals; | ||
|
||
public class OutputCodecTest { | ||
|
||
@BeforeEach | ||
public void setUp() { | ||
} | ||
|
||
@Test | ||
public void testWriteMetrics() throws JsonProcessingException { | ||
OutputCodec outputCodec = new OutputCodec() { | ||
@Override | ||
public void start(OutputStream outputStream) throws IOException { | ||
} | ||
|
||
@Override | ||
public void writeEvent(Event event, OutputStream outputStream, String tagsTargetKey) throws IOException { | ||
} | ||
|
||
@Override | ||
public void complete(OutputStream outputStream) throws IOException { | ||
} | ||
|
||
@Override | ||
public String getExtension() { | ||
return null; | ||
} | ||
}; | ||
|
||
final Set<String> testTags = Set.of("tag1"); | ||
final EventMetadata defaultEventMetadata = DefaultEventMetadata.builder(). | ||
withEventType(EventType.LOG.toString()). | ||
withTags(testTags).build(); | ||
Map<String, Object> json = generateJson(); | ||
final JacksonEvent event = JacksonLog.builder().withData(json).withEventMetadata(defaultEventMetadata).build(); | ||
Event tagsToEvent = outputCodec.addTagsToEvent(event, "Tag"); | ||
assertNotEquals(event.toJsonString(), tagsToEvent.toJsonString()); | ||
} | ||
|
||
private static Map<String, Object> generateJson() { | ||
final Map<String, Object> jsonObject = new LinkedHashMap<>(); | ||
for (int i = 0; i < 2; i++) { | ||
jsonObject.put(UUID.randomUUID().toString(), UUID.randomUUID().toString()); | ||
} | ||
jsonObject.put(UUID.randomUUID().toString(), Arrays.asList(UUID.randomUUID().toString(), | ||
UUID.randomUUID().toString(), UUID.randomUUID().toString())); | ||
return jsonObject; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
plugins { | ||
id 'java' | ||
id 'java-library' | ||
} | ||
|
||
repositories { | ||
mavenCentral() | ||
} | ||
|
||
dependencies { | ||
api project(':data-prepper-api') | ||
implementation project(':data-prepper-plugins:aws-plugin-api') | ||
implementation project(path: ':data-prepper-plugins:common') | ||
testImplementation 'org.junit.jupiter:junit-jupiter' | ||
implementation 'com.fasterxml.jackson.core:jackson-core' | ||
implementation 'com.fasterxml.jackson.core:jackson-databind' | ||
implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' | ||
implementation 'software.amazon.awssdk:cloudwatch' | ||
implementation 'software.amazon.awssdk:cloudwatchlogs' | ||
implementation 'org.apache.commons:commons-lang3:3.12.0' | ||
testImplementation project(path: ':data-prepper-test-common') | ||
testImplementation project(path: ':data-prepper-test-common') | ||
} | ||
|
||
jacocoTestCoverageVerification { | ||
dependsOn jacocoTestReport | ||
violationRules { | ||
rule { //in addition to core projects rule | ||
limit { | ||
minimum = 0.90 | ||
} | ||
} | ||
} | ||
} | ||
|
||
test { | ||
useJUnitPlatform() | ||
} |
48 changes: 48 additions & 0 deletions
48
...oudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/config/AwsConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
package org.opensearch.dataprepper.plugins.sink.config; | ||
|
||
import com.fasterxml.jackson.annotation.JsonProperty; | ||
import jakarta.validation.constraints.Size; | ||
import software.amazon.awssdk.regions.Region; | ||
|
||
import java.util.Map; | ||
|
||
/** | ||
* AwsConfig is based on the S3-Sink AwsAuthenticationOptions | ||
* where the configuration allows the sink to fetch Aws credentials | ||
* and resources. | ||
*/ | ||
public class AwsConfig { | ||
public static int DEFAULT_CONNECTION_ATTEMPTS = 5; | ||
|
||
@JsonProperty("region") | ||
@Size(min = 1, message = "Region cannot be empty string") | ||
private String awsRegion; | ||
|
||
@JsonProperty("sts_role_arn") | ||
@Size(min = 20, max = 2048, message = "awsStsRoleArn length should be between 1 and 2048 characters") | ||
private String awsStsRoleArn; | ||
|
||
@JsonProperty("sts_header_overrides") | ||
@Size(max = 5, message = "sts_header_overrides supports a maximum of 5 headers to override") | ||
private Map<String, String> awsStsHeaderOverrides; | ||
|
||
@JsonProperty("sts_external_id") | ||
@Size(min = 2, max = 1224, message = "awsStsExternalId length should be between 2 and 1224 characters") | ||
private String awsStsExternalId; | ||
|
||
public Region getAwsRegion() { | ||
return awsRegion != null ? Region.of(awsRegion) : null; | ||
} | ||
|
||
public String getAwsStsRoleArn() { | ||
return awsStsRoleArn; | ||
} | ||
|
||
public String getAwsStsExternalId() { | ||
return awsStsExternalId; | ||
} | ||
|
||
public Map<String, String> getAwsStsHeaderOverrides() { | ||
return awsStsHeaderOverrides; | ||
} | ||
} |
52 changes: 52 additions & 0 deletions
52
...atch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/config/CwlSinkConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
package org.opensearch.dataprepper.plugins.sink.config; | ||
|
||
import com.fasterxml.jackson.annotation.JsonProperty; | ||
import jakarta.validation.Valid; | ||
import jakarta.validation.constraints.NotEmpty; | ||
import jakarta.validation.constraints.NotNull; | ||
|
||
public class CwlSinkConfig { | ||
public static final String DEFAULT_BUFFER_TYPE = "in_memory"; | ||
|
||
@JsonProperty("aws") | ||
@NotNull | ||
@Valid | ||
private AwsConfig awsConfig; | ||
|
||
@JsonProperty("threshold") | ||
@NotNull | ||
private ThresholdConfig thresholdConfig; | ||
|
||
@JsonProperty("buffer_type") | ||
private String bufferType = DEFAULT_BUFFER_TYPE; | ||
|
||
@JsonProperty("log_group") | ||
@NotEmpty | ||
@NotNull | ||
private String logGroup; | ||
|
||
@JsonProperty("log_stream") | ||
@NotEmpty | ||
@NotNull | ||
private String logStream; | ||
|
||
public AwsConfig getAwsConfig() { | ||
return awsConfig; | ||
} | ||
|
||
public ThresholdConfig getThresholdConfig() { | ||
return thresholdConfig; | ||
} | ||
|
||
public String getBufferType() { | ||
return bufferType; | ||
} | ||
|
||
public String getLogGroup() { | ||
return logGroup; | ||
} | ||
|
||
public String getLogStream() { | ||
return logStream; | ||
} | ||
} |
Oops, something went wrong.