From 9fac032aba8783a4f03aa24a29a1456581648588 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 9 Aug 2023 13:44:26 -0700 Subject: [PATCH] Copy apache from main to 2.x (#877) Signed-off-by: Simeon Widdis (cherry picked from commit 0b73b42ab61fcdb4aa3181db8d9fc3cb9d6b6542) --- auto_sync_commit_metadata.json | 4 +- .../repository/apache/apache-1.0.0.json | 45 ++++ .../apache/assets/apache-1.0.0.ndjson | 11 + .../repository/apache/data/sample.json | 206 +++++++++++++++ .../repository/apache/info/INGESTION.md | 149 +++++++++++ .../__data__/repository/apache/info/README.md | 14 + .../schemas/communication-1.0.0.mapping.json | 134 ++++++++++ .../apache/schemas/http-1.0.0.mapping.json | 165 ++++++++++++ .../schemas/logs_apache-1.0.0.mapping.json | 242 ++++++++++++++++++ .../repository/apache/static/dashboard1.png | Bin 0 -> 217383 bytes .../repository/apache/static/logo.png | Bin 0 -> 32093 bytes 11 files changed, 968 insertions(+), 2 deletions(-) create mode 100644 server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json create mode 100644 server/adaptors/integrations/__data__/repository/apache/assets/apache-1.0.0.ndjson create mode 100644 server/adaptors/integrations/__data__/repository/apache/data/sample.json create mode 100644 server/adaptors/integrations/__data__/repository/apache/info/INGESTION.md create mode 100644 server/adaptors/integrations/__data__/repository/apache/info/README.md create mode 100644 server/adaptors/integrations/__data__/repository/apache/schemas/communication-1.0.0.mapping.json create mode 100644 server/adaptors/integrations/__data__/repository/apache/schemas/http-1.0.0.mapping.json create mode 100644 server/adaptors/integrations/__data__/repository/apache/schemas/logs_apache-1.0.0.mapping.json create mode 100644 server/adaptors/integrations/__data__/repository/apache/static/dashboard1.png create mode 100644 server/adaptors/integrations/__data__/repository/apache/static/logo.png diff --git a/auto_sync_commit_metadata.json b/auto_sync_commit_metadata.json index e34356e08..9530df3c3 100644 --- a/auto_sync_commit_metadata.json +++ b/auto_sync_commit_metadata.json @@ -1,4 +1,4 @@ { - "last_github_commit": "3ddada3a0d75f3db8eadc72d6973e7d148a556da", - "last_gitfarm_commit": "2b7262f8bdebe5938751121986aeab44775010b3" + "last_github_commit": "0b73b42ab61fcdb4aa3181db8d9fc3cb9d6b6542", + "last_gitfarm_commit": "f4ef15497b054e4fdf817e7d4a5663d101eea169" } \ No newline at end of file diff --git a/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json b/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json new file mode 100644 index 000000000..c8c90977c --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json @@ -0,0 +1,45 @@ +{ + "name": "apache", + "version": "1.0.0", + "displayName": "Apache Dashboard", + "description": "Apache web logs collector", + "license": "Apache-2.0", + "type": "logs_apache", + "author": "OpenSearch", + "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info", + "statics": { + "logo": { + "annotation": "Apache Logo", + "path": "logo.png" + }, + "gallery": [ + { + "annotation": "Apache Dashboard", + "path": "dashboard1.png" + } + ] + }, + "components": [ + { + "name": "communication", + "version": "1.0.0" + }, + { + "name": "http", + "version": "1.0.0" + }, + { + "name": "logs_apache", + "version": "1.0.0" + } + ], + "assets": { + "savedObjects": { + "name": "apache", + "version": "1.0.0" + } + }, + "sampleData": { + "path": "sample.json" + } +} diff --git a/server/adaptors/integrations/__data__/repository/apache/assets/apache-1.0.0.ndjson b/server/adaptors/integrations/__data__/repository/apache/assets/apache-1.0.0.ndjson new file mode 100644 index 000000000..6bee20b26 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/apache/assets/apache-1.0.0.ndjson @@ -0,0 +1,11 @@ +{"attributes": {"fields": "[{\"name\": \"_index\", \"type\": \"string\", \"esTypes\": [\"_index\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": false}, {\"name\": \"_score\", \"type\": \"number\", \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": false}, {\"name\": \"_source\", \"type\": \"_source\", \"esTypes\": [\"_source\"], \"count\": 0, \"scripted\": false, \"searchable\": false, \"aggregatable\": false, \"readFromDocValues\": false}, {\"name\": \"_type\", \"type\": \"string\", \"esTypes\": [\"_type\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": false}, {\"name\": \"severity.number\", \"type\": \"number\", \"esTypes\": [\"long\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"severity.text\", \"type\": \"string\", \"esTypes\": [\"text\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": false, \"readFromDocValues\": false}, {\"name\": \"severity.text.keyword\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true, \"subType\": {\"multi\": {\"parent\": \"severity.text\"}}}, {\"name\": \"attributes.data_stream.dataset\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"attributes.data_stream.namespace\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"attributes.data_stream.type\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"body\", \"type\": \"string\", \"esTypes\": [\"text\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"@message\", \"type\": \"string\", \"esTypes\": [\"alias\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"@timestamp\", \"type\": \"date\", \"esTypes\": [\"date\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"observedTimestamp\", \"type\": \"date\", \"esTypes\": [\"date\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"observerTime\", \"type\": \"string\", \"esTypes\": [\"alias\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"traceId\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"spanId\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"schemaUrl\", \"type\": \"string\", \"esTypes\": [\"text\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": false, \"readFromDocValues\": false}, {\"name\": \"schemaUrl.keyword\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true, \"subType\": {\"multi\": {\"parent\": \"schemaUrl\"}}}, {\"name\": \"instrumentationScope.name\", \"type\": \"string\", \"esTypes\": [\"text\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": false, \"readFromDocValues\": false}, {\"name\": \"instrumentationScope.name.keyword\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true, \"subType\": {\"multi\": {\"parent\": \"instrumentationScope.name\"}}}, {\"name\": \"instrumentationScope.version\", \"type\": \"string\", \"esTypes\": [\"text\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": false, \"readFromDocValues\": false}, {\"name\": \"instrumentationScope.version.keyword\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true, \"subType\": {\"multi\": {\"parent\": \"instrumentationScope.version\"}}}, {\"name\": \"instrumentationScope.dropped_attributes_count\", \"type\": \"number\", \"esTypes\": [\"integer\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"instrumentationScope.schemaUrl\", \"type\": \"string\", \"esTypes\": [\"text\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": false, \"readFromDocValues\": false}, {\"name\": \"instrumentationScope.schemaUrl.keyword\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true, \"subType\": {\"multi\": {\"parent\": \"instrumentationScope.schemaUrl\"}}}, {\"name\": \"event.domain\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"event.name\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"event.source\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"event.category\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"event.type\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"event.kind\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"event.result\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"event.exception.message\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"event.exception.type\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"event.exception.stacktrace\", \"type\": \"string\", \"esTypes\": [\"text\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.flavor\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.user_agent\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.url\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.schema\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.target\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.route\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.client.ip\", \"type\": \"ip\", \"esTypes\": [\"ip\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.resent_count\", \"type\": \"number\", \"esTypes\": [\"integer\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.request.id\", \"type\": \"string\", \"esTypes\": [\"text\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": false, \"readFromDocValues\": false}, {\"name\": \"http.request.id.keyword\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true, \"subType\": {\"multi\": {\"parent\": \"http.request.id\"}}}, {\"name\": \"http.request.body.content\", \"type\": \"string\", \"esTypes\": [\"text\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.request.bytes\", \"type\": \"number\", \"esTypes\": [\"long\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.request.method\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.request.referrer\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.request.mime_type\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.response.id\", \"type\": \"string\", \"esTypes\": [\"text\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": false, \"readFromDocValues\": false}, {\"name\": \"http.response.id.keyword\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true, \"subType\": {\"multi\": {\"parent\": \"http.response.id\"}}}, {\"name\": \"http.response.body.content\", \"type\": \"string\", \"esTypes\": [\"text\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.response.bytes\", \"type\": \"number\", \"esTypes\": [\"long\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.response.status_code\", \"type\": \"number\", \"esTypes\": [\"integer\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.sock.family\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.source.address\", \"type\": \"string\", \"esTypes\": [\"text\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": false, \"readFromDocValues\": false}, {\"name\": \"communication.source.address.keyword\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true, \"subType\": {\"multi\": {\"parent\": \"communication.source.address\"}}}, {\"name\": \"communication.source.domain\", \"type\": \"string\", \"esTypes\": [\"text\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": false, \"readFromDocValues\": false}, {\"name\": \"communication.source.domain.keyword\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true, \"subType\": {\"multi\": {\"parent\": \"communication.source.domain\"}}}, {\"name\": \"communication.source.bytes\", \"type\": \"number\", \"esTypes\": [\"long\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.source.ip\", \"type\": \"ip\", \"esTypes\": [\"ip\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.source.port\", \"type\": \"number\", \"esTypes\": [\"long\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.source.mac\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.source.packets\", \"type\": \"number\", \"esTypes\": [\"long\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.source.geo.city_name\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.source.geo.country_iso_code\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.source.geo.country_name\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.source.geo.location\", \"type\": \"geo_point\", \"esTypes\": [\"geo_point\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.destination.address\", \"type\": \"string\", \"esTypes\": [\"text\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": false, \"readFromDocValues\": false}, {\"name\": \"communication.destination.address.keyword\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true, \"subType\": {\"multi\": {\"parent\": \"communication.destination.address\"}}}, {\"name\": \"communication.destination.domain\", \"type\": \"string\", \"esTypes\": [\"text\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": false, \"readFromDocValues\": false}, {\"name\": \"communication.destination.domain.keyword\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true, \"subType\": {\"multi\": {\"parent\": \"communication.destination.domain\"}}}, {\"name\": \"communication.destination.bytes\", \"type\": \"number\", \"esTypes\": [\"long\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.destination.ip\", \"type\": \"ip\", \"esTypes\": [\"ip\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.destination.port\", \"type\": \"number\", \"esTypes\": [\"long\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.destination.mac\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.destination.packets\", \"type\": \"number\", \"esTypes\": [\"long\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.destination.geo.city_name\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.destination.geo.country_iso_code\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.destination.geo.country_name\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"communication.destination.geo.location\", \"type\": \"geo_point\", \"esTypes\": [\"geo_point\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.user_agent.original\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.user_agent.name\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.user_agent.version\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.user_agent.device.name\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.user_agent.os.type\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.user_agent.os.platform\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.user_agent.os.name\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.user_agent.os.full\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.user_agent.os.family\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.user_agent.os.version\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}, {\"name\": \"http.user_agent.os.kernel\", \"type\": \"string\", \"esTypes\": [\"keyword\"], \"count\": 0, \"scripted\": false, \"searchable\": true, \"aggregatable\": true, \"readFromDocValues\": true}]", "timeFieldName": "@timestamp", "title": "ss4o_logs-*-*"}, "id": "d8a559c0-04b9-11ee-a4b2-e1966e8247fd", "migrationVersion": {"index-pattern": "7.6.0"}, "references": [], "type": "index-pattern", "updated_at": "2023-07-21T16:00:22", "version": "WzE0LDFd"} +{ "attributes": { "columns": [ "communication.source.address.keyword", "http.request.method", "http.url", "http.response.status_code" ], "description": "", "hits": 0, "kibanaSavedObjectMeta": { "searchSourceJSON": "{ \"filter\": [], \"highlight\": { \"fields\": { \"*\": {} }, \"fragment_size\": 2147483647, \"post_tags\": [ \"@/kibana-highlighted-field@\" ], \"pre_tags\": [ \"@kibana-highlighted-field@\" ], \"require_field_match\": false }, \"indexRefName\": \"kibanaSavedObjectMeta.searchSourceJSON.index\" }" }, "sort": [ [ "@timestamp", "desc" ] ], "title": "Apache access logs ", "version": 1 }, "coreMigrationVersion": "8.0.0", "id": "Apache-access-logs-ecs", "migrationVersion": { "search": "7.9.3" }, "references": [ { "id": "d8a559c0-04b9-11ee-a4b2-e1966e8247fd", "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern" } ], "type": "search", "updated_at": "2021-08-04T16:33:55.372Z", "version": "WzQzNDQsMV0=" } +{ "attributes": { "columns": [ "communication.source.address.keyword", "severity.text", "apache2.error.module", "body" ], "description": "", "hits": 0, "kibanaSavedObjectMeta": { "searchSourceJSON": "{ \"filter\": [], \"highlight\": { \"fields\": { \"*\": {} }, \"fragment_size\": 2147483647, \"post_tags\": [ \"@/kibana-highlighted-field@\" ], \"pre_tags\": [ \"@kibana-highlighted-field@\" ], \"require_field_match\": false }, \"indexRefName\": \"kibanaSavedObjectMeta.searchSourceJSON.index\",\"query\": { \"language\": \"kuery\", \"query\": \"event.domain:apache.error\"}}" }, "sort": [ [ "@timestamp", "desc" ] ], "title": "Apache errors log ", "version": 1 }, "coreMigrationVersion": "8.0.0", "id": "Apache-errors-log-ecs", "migrationVersion": { "search": "7.9.3" }, "references": [ { "id": "d8a559c0-04b9-11ee-a4b2-e1966e8247fd", "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern" } ], "type": "search", "updated_at": "2021-08-04T16:33:55.372Z", "version": "WzQzNDMsMV0=" } +{ "attributes": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{ \"filter\": [] }" }, "savedSearchRefName": "search_0", "title": "Browsers breakdown ", "uiStateJSON": "{}", "version": 1, "visState": "{ \"aggs\": [ { \"enabled\": true, \"id\": \"1\", \"params\": { \"field\": \"communication.source.address.keyword\" }, \"schema\": \"metric\", \"type\": \"cardinality\" }, { \"enabled\": true, \"id\": \"2\", \"params\": { \"field\": \"http.user_agent.name\", \"order\": \"desc\", \"orderBy\": \"1\", \"size\": 5 }, \"schema\": \"segment\", \"type\": \"terms\" }, { \"enabled\": true, \"id\": \"3\", \"params\": { \"field\": \"http.user_agent.version\", \"order\": \"desc\", \"orderBy\": \"1\", \"size\": 5 }, \"schema\": \"segment\", \"type\": \"terms\" } ], \"listeners\": {}, \"params\": { \"addLegend\": true, \"addTooltip\": true, \"distinctColors\": true, \"isDonut\": true, \"legendPosition\": \"bottom\", \"palette\": { \"name\": \"kibana_palette\", \"type\": \"palette\" }, \"shareYAxis\": true }, \"title\": \"Apache browsers ECS\", \"type\": \"pie\" }" }, "coreMigrationVersion": "8.0.0", "id": "Apache-browsers-ecs", "migrationVersion": { "visualization": "7.10.0" }, "references": [ { "id": "Apache-access-logs-ecs", "name": "search_0", "type": "search" } ], "type": "visualization", "updated_at": "2021-08-04T16:33:55.372Z", "version": "WzQzMzksMV0=" } +{ "attributes": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{ \"filter\": [ ] }" }, "savedSearchRefName": "search_0", "title": "Unique IPs map ", "uiStateJSON": "{\"mapCenter\": [14.944784875088372,5.09765625 ]}", "version": 1, "visState":"{ \"title\":\"Hehe\", \"type\":\"region_map\", \"aggs\":[ { \"id\":\"1\", \"enabled\":true, \"type\":\"count\", \"params\":{ }, \"schema\":\"metric\" }, { \"id\":\"2\", \"enabled\":true, \"type\":\"terms\", \"params\":{ \"field\":\"communication.source.geo.country_iso_code\", \"orderBy\":\"1\", \"order\":\"desc\", \"size\":5, \"otherBucket\":false, \"otherBucketLabel\":\"Other\", \"missingBucket\":false, \"missingBucketLabel\":\"Missing\" }, \"schema\":\"segment\" } ], \"params\":{ \"addTooltip\":true, \"colorSchema\":\"Yellow to Red\", \"emsHotLink\":\"?locale=en#file/world_countries\", \"isDisplayWarning\":true, \"layerChosenByUser\":\"default\", \"legendPosition\":\"bottomright\", \"mapCenter\":[ 0, 0 ], \"mapZoom\":2, \"outlineWeight\":1, \"selectedCustomJoinField\":null, \"selectedJoinField\":{ \"description\":\"ISO 3166-1 alpha-2 Code\", \"name\":\"iso2\", \"type\":\"id\" }, \"selectedLayer\":{ \"attribution\":\"Made with NaturalEarth\", \"created_at\":\"2017-04-26T17:12:15.978370\", \"fields\":[ { \"description\":\"ISO 3166-1 alpha-2 Code\", \"name\":\"iso2\", \"type\":\"id\" }, { \"description\":\"ISO 3166-1 alpha-3 Code\", \"name\":\"iso3\", \"type\":\"id\" }, { \"description\":\"Name\", \"name\":\"name\", \"type\":\"name\" } ], \"format\":{ \"type\":\"geojson\" }, \"id\":\"world_countries\", \"isEMS\":true, \"layerId\":\"elastic_maps_service.World Countries\", \"name\":\"World Countries\", \"origin\":\"elastic_maps_service\" }, \"showAllShapes\":true, \"wms\":{ \"enabled\":false, \"options\":{ \"attribution\":\"\", \"format\":\"image/png\", \"layers\":\"\", \"styles\":\"\", \"transparent\":true, \"version\":\"\" }, \"selectedTmsLayer\":{ \"attribution\":\"Map data © OpenStreetMap contributors\", \"id\":\"road_map\", \"maxZoom\":14, \"minZoom\":0, \"origin\":\"elastic_maps_service\" }, \"url\":\"\" } } }" }, "coreMigrationVersion": "7.0.0", "id": "Apache-access-unique-IPs-map-ecs", "migrationVersion": { "visualization": "7.10.0" }, "references": [ { "id": "Apache-access-logs-ecs", "name": "search_0", "type": "search" } ], "type": "visualization", "updated_at": "2021-08-04T16:33:55.372Z", "version": "WzQzMzcsMV0=" } +{ "attributes": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{ \"filter\": [] }" }, "savedSearchRefName": "search_0", "title": "Operating systems breakdown ", "uiStateJSON": "{}", "version": 1, "visState": "{ \"aggs\": [ { \"enabled\": true, \"id\": \"1\", \"params\": { \"field\": \"communication.source.address.keyword\" }, \"schema\": \"metric\", \"type\": \"cardinality\" }, { \"enabled\": true, \"id\": \"2\", \"params\": { \"field\": \"http.user_agent.os.name\", \"order\": \"desc\", \"orderBy\": \"1\", \"size\": 5 }, \"schema\": \"segment\", \"type\": \"terms\" }, { \"enabled\": true, \"id\": \"3\", \"params\": { \"field\": \"http.user_agent.os.version\", \"order\": \"desc\", \"orderBy\": \"1\", \"size\": 5 }, \"schema\": \"segment\", \"type\": \"terms\" } ], \"listeners\": {}, \"params\": { \"addLegend\": true, \"addTooltip\": true, \"distinctColors\": true, \"isDonut\": true, \"legendPosition\": \"bottom\", \"palette\": { \"name\": \"kibana_palette\", \"type\": \"palette\" }, \"shareYAxis\": true }, \"title\": \"Apache operating systems ECS\", \"type\": \"pie\" }" }, "coreMigrationVersion": "7.0.0", "id": "Apache-operating-systems-ecs", "migrationVersion": { "visualization": "7.10.0" }, "references": [ { "id": "Apache-access-logs-ecs", "name": "search_0", "type": "search" } ], "type": "visualization", "updated_at": "2021-08-04T16:33:55.372Z", "version": "WzQzNDAsMV0=" } +{ "attributes": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{ \"filter\": [] }" }, "savedSearchRefName": "search_0", "title": "Error logs over time ", "uiStateJSON": "{}", "version": 1, "visState": "{ \"aggs\": [ { \"enabled\": true, \"id\": \"1\", \"params\": {}, \"schema\": \"metric\", \"type\": \"count\" }, { \"enabled\": true, \"id\": \"2\", \"params\": { \"extended_bounds\": {}, \"field\": \"@timestamp\", \"interval\": \"auto\", \"min_doc_count\": 1 }, \"schema\": \"segment\", \"type\": \"date_histogram\" }, { \"enabled\": true, \"id\": \"3\", \"params\": { \"field\": \"severity.text.keyword\", \"order\": \"desc\", \"orderBy\": \"1\", \"size\": 5 }, \"schema\": \"group\", \"type\": \"terms\" } ], \"listeners\": {}, \"params\": { \"addLegend\": true, \"addTimeMarker\": false, \"addTooltip\": true, \"defaultYExtents\": false, \"legendPosition\": \"right\", \"mode\": \"stacked\", \"scale\": \"linear\", \"setYExtents\": false, \"shareYAxis\": true, \"times\": [], \"yAxis\": {} }, \"title\": \"Apache error logs over time ECS\", \"type\": \"histogram\" }" }, "coreMigrationVersion": "7.10.0", "id": "Apache-error-logs-over-time-ecs", "migrationVersion": { "visualization": "7.10.0" }, "references": [ { "id": "Apache-errors-log-ecs", "name": "search_0", "type": "search" } ], "type": "visualization", "updated_at": "2021-08-04T16:33:55.372Z", "version": "WzQzNDEsMV0=" } +{ "attributes": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{ \"filter\": [] }" }, "savedSearchRefName": "search_0", "title": "Top URLs by response code ", "uiStateJSON": "{ \"vis\": { \"colors\": { \"200\": \"#7EB26D\", \"404\": \"#EF843C\" } } }", "version": 1, "visState": "{ \"aggs\": [ { \"enabled\": true, \"id\": \"1\", \"params\": { }, \"schema\": \"metric\", \"type\": \"count\" }, { \"enabled\": true, \"id\": \"3\", \"params\": { \"customLabel\": \"URL\", \"field\": \"http.url\", \"order\": \"desc\", \"orderBy\": \"1\", \"size\": 5 }, \"schema\": \"split\", \"type\": \"terms\" }, { \"enabled\": true, \"id\": \"2\", \"params\": { \"field\": \"http.response.status_code\", \"order\": \"desc\", \"orderBy\": \"1\", \"size\": 5 }, \"schema\": \"segment\", \"type\": \"terms\" } ], \"listeners\": { }, \"params\": { \"addLegend\": true, \"addTooltip\": true, \"distinctColors\": true, \"isDonut\": false, \"legendPosition\": \"right\", \"palette\": { \"name\": \"kibana_palette\", \"type\": \"palette\" }, \"row\": false, \"shareYAxis\": true }, \"title\": \"Apache response codes of top URLs ECS\", \"type\": \"pie\" }" }, "coreMigrationVersion": "8.0.0", "id": "Apache-response-codes-of-top-URLs-ecs", "migrationVersion": { "visualization": "7.10.0" }, "references": [ { "id": "Apache-access-logs-ecs", "name": "search_0", "type": "search" } ], "type": "visualization", "updated_at": "2021-08-04T16:33:55.372Z", "version": "WzQzMzgsMV0=" } +{ "attributes": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": "{ \"filter\": [] }" }, "savedSearchRefName": "search_0", "title": "Response codes over time ", "uiStateJSON": "{ \"vis\": { \"colors\": { \"200\": \"#629E51\", \"404\": \"#EF843C\" } } }", "version": 1, "visState": "{ \"aggs\": [ { \"enabled\": true, \"id\": \"1\", \"params\": {}, \"schema\": \"metric\", \"type\": \"count\" }, { \"enabled\": true, \"id\": \"2\", \"params\": { \"extended_bounds\": {}, \"field\": \"@timestamp\", \"interval\": \"auto\", \"min_doc_count\": 1 }, \"schema\": \"segment\", \"type\": \"date_histogram\" }, { \"enabled\": true, \"id\": \"3\", \"params\": { \"field\": \"http.response.status_code\", \"order\": \"desc\", \"orderBy\": \"1\", \"size\": 5 }, \"schema\": \"group\", \"type\": \"terms\" } ], \"listeners\": { }, \"params\": { \"addLegend\": true, \"addTimeMarker\": false, \"addTooltip\": true, \"defaultYExtents\": false, \"legendPosition\": \"right\", \"mode\": \"stacked\", \"scale\": \"linear\", \"setYExtents\": false, \"shareYAxis\": true, \"times\": [ ], \"yAxis\": { } }, \"title\": \"Apache response codes over time ECS\", \"type\": \"histogram\" }" }, "coreMigrationVersion": "8.0.0", "id": "Apache-response-codes-over-time-ecs", "migrationVersion": { "visualization": "7.10.0" }, "references": [ { "id": "Apache-access-logs-ecs", "name": "search_0", "type": "search" } ], "type": "visualization", "updated_at": "2021-08-04T16:33:55.372Z", "version": "WzQzNDIsMV0=" } +{ "attributes": { "description": "Filebeat Apache module dashboard", "hits": 0, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" }, "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", "refreshInterval": { "pause": true, "value": 0 }, "timeFrom": "now-15m", "timeRestore": true, "panelsJSON": "[ { \"embeddableConfig\": { \"enhancements\": {}, \"mapBounds\": { \"bottom_right\": { \"lat\": -3.864254615721396, \"lon\": 205.3125 }, \"top_left\": { \"lat\": 67.7427590666639, \"lon\": -205.6640625 } }, \"mapCenter\": [ 40.713955826286046, -0.17578125 ], \"mapCollar\": { \"bottom_right\": { \"lat\": -39.667755, \"lon\": 180 }, \"top_left\": { \"lat\": 90, \"lon\": -180 }, \"zoom\": 2 }, \"mapZoom\": 2 }, \"gridData\": { \"h\": 12, \"i\": \"1\", \"w\": 48, \"x\": 0, \"y\": 0 }, \"panelIndex\": \"1\", \"panelRefName\": \"panel_1\", \"type\": \"visualization\", \"version\": \"7.3.0\" }, { \"embeddableConfig\": { \"enhancements\": {} }, \"gridData\": { \"h\": 12, \"i\": \"2\", \"w\": 32, \"x\": 0, \"y\": 20 }, \"panelIndex\": \"2\", \"panelRefName\": \"panel_2\", \"type\": \"visualization\", \"version\": \"7.3.0\" }, { \"embeddableConfig\": { \"enhancements\": {} }, \"gridData\": { \"h\": 12, \"i\": \"3\", \"w\": 16, \"x\": 32, \"y\": 20 }, \"panelIndex\": \"3\", \"panelRefName\": \"panel_3\", \"type\": \"visualization\", \"version\": \"7.3.0\" }, { \"embeddableConfig\": { \"enhancements\": { } }, \"gridData\": { \"h\": 8, \"i\": \"4\", \"w\": 8, \"x\": 40, \"y\": 12 }, \"panelIndex\": \"4\", \"panelRefName\": \"panel_4\", \"type\": \"visualization\", \"version\": \"7.3.0\" }, { \"embeddableConfig\": { \"enhancements\": { } }, \"gridData\": { \"h\": 8, \"i\": \"5\", \"w\": 48, \"x\": 0, \"y\": 32 }, \"panelIndex\": \"5\", \"panelRefName\": \"panel_5\", \"type\": \"visualization\", \"version\": \"7.3.0\" }, { \"embeddableConfig\": { \"enhancements\": { } }, \"gridData\": { \"h\": 8, \"i\": \"6\", \"w\": 40, \"x\": 0, \"y\": 12 }, \"panelIndex\": \"6\", \"panelRefName\": \"panel_6\", \"type\": \"visualization\", \"version\": \"7.3.0\" }, { \"embeddableConfig\": { \"columns\": [ \"communication.source.address\", \"severity.number\", \"severity.text\", \"body\" ], \"enhancements\": { }, \"sort\": [ \"@timestamp\", \"desc\" ] }, \"gridData\": { \"h\": 12, \"i\": \"7\", \"w\": 48, \"x\": 0, \"y\": 50 }, \"panelIndex\": \"7\", \"panelRefName\": \"panel_7\", \"type\": \"search\", \"version\": \"7.3.0\" } ]", "title": "Access and error logs ECS", "version": 1 }, "coreMigrationVersion": "7.9.3", "id": "Filebeat-Apache-Dashboard-ecs", "migrationVersion": { "dashboard": "7.9.3" }, "references": [ { "id": "Apache-access-unique-IPs-map-ecs", "name": "panel_1", "type": "visualization" }, { "id": "Apache-response-codes-of-top-URLs-ecs", "name": "panel_2", "type": "visualization" }, { "id": "Apache-browsers-ecs", "name": "panel_3", "type": "visualization" }, { "id": "Apache-operating-systems-ecs", "name": "panel_4", "type": "visualization" }, { "id": "Apache-error-logs-over-time-ecs", "name": "panel_5", "type": "visualization" }, { "id": "Apache-response-codes-over-time-ecs", "name": "panel_6", "type": "visualization" }, { "id": "Apache-errors-log-ecs", "name": "panel_7", "type": "search" } ], "type": "dashboard", "updated_at": "2021-08-04T16:33:55.372Z", "version": "WzQzNDUsMV0=" } +{"exportedCount":10,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/server/adaptors/integrations/__data__/repository/apache/data/sample.json b/server/adaptors/integrations/__data__/repository/apache/data/sample.json new file mode 100644 index 000000000..5d41e04c9 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/apache/data/sample.json @@ -0,0 +1,206 @@ +[ + { + "observedTimestamp": "2023-07-21T16:52:08.000Z", + "http": { + "response": { + "status_code": 406, + "bytes": 6141 + }, + "url": "/strategize", + "flavor": "1.1", + "request": { + "method": "GET" + }, + "user_agent": { + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36", + "name": "Chrome", + "version": "114.0.0", + "os": { + "name": "Mac OS X", + "full": "Mac OS X 10.15.7", + "version": "10.15.7", + "device": { + "name": "Mac" + } + } + } + }, + "attributes": { + "data_stream": { + "dataset": "apache.access", + "namespace": "production", + "type": "logs" + } + }, + "event": { + "result": "success", + "category": "web", + "name": "access", + "type": "access", + "domain": "apache.access", + "kind": "event" + }, + "communication": { + "source": { + "address": "127.0.0.1", + "ip": "42.204.151.42", + "geo": { + "country": "China", + "country_iso_code": "CN" + } + } + }, + "body": "15.248.1.132 - - [21/Jun/2023:21:35:24 +0000] \"GET / HTTP/1.1\" 403 45 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36\"", + "traceId": "d09d293a27c9a754f2bf0196b5a1c9bc", + "spanId": "18ba0e515e42dad0", + "@timestamp": "2023-07-21T16:52:08.000Z" + }, + { + "observedTimestamp": "2023-07-21T16:52:08.000Z", + "http": { + "response": { + "status_code": 406, + "bytes": 6141 + }, + "url": "/strategize", + "flavor": "1.1", + "request": { + "method": "GET" + }, + "user_agent": { + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36", + "name": "Chrome", + "version": "114.0.0", + "os": { + "name": "Mac OS X", + "full": "Mac OS X 10.15.7", + "version": "10.15.7", + "device": { + "name": "Mac" + } + } + } + }, + "attributes": { + "data_stream": { + "dataset": "apache.access", + "namespace": "production", + "type": "logs" + } + }, + "event": { + "result": "success", + "category": "web", + "name": "access", + "type": "access", + "domain": "apache.access", + "kind": "event" + }, + "communication": { + "source": { + "address": "127.0.0.1", + "ip": "42.204.151.42", + "geo": { + "country": "China", + "country_iso_code": "CN" + } + } + }, + + "body": "15.248.1.132 - - [21/Jun/2023:21:35:24 +0000] \"GET / HTTP/1.1\" 403 45 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36\"", + "traceId": "d09d293a27c9a754f2bf0196b5a1c9bc", + "spanId": "18ba0e515e42dad0", + "@timestamp": "2023-07-21T16:52:08.000Z" + }, + { + "observedTimestamp": "2023-07-25:52:08.000Z", + "http": { + "response": { + "status_code": 400, + "bytes": 6141 + }, + "url": "/strategize", + "flavor": "1.1", + "request": { + "method": "GET" + }, + "user_agent": { + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36", + "name": "Chrome", + "version": "114.0.0", + "os": { + "name": "Mac OS X", + "full": "Mac OS X 10.15.7", + "version": "10.15.7", + "device": { + "name": "Mac" + } + } + } + }, + "attributes": { + "data_stream": { + "dataset": "apache.access", + "namespace": "production", + "type": "logs" + } + }, + "event": { + "result": "success", + "category": "web", + "name": "access", + "type": "access", + "domain": "apache.access", + "kind": "event" + }, + "communication": { + "source": { + "address": "127.0.0.1", + "ip": "42.204.151.42", + "geo": { + "country": "United States", + "country_iso_code": "US" + } + } + }, + "body": "15.248.1.132 - - [21/Jun/2023:21:35:24 +0000] \"GET / HTTP/1.1\" 403 45 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36\"", + "traceId": "d09d293a27c9a754f2bf0196b5a1c9bc", + "spanId": "18ba0e515e42dad0", + "@timestamp": "2023-07-21T16:52:08.000Z" + }, + { + "attributes": { + "data_stream": { + "dataset": "apache.error", + "namespace": "production", + "type": "logs" + } + }, + "observedTimestamp": "2023-07-21T16:52:08.000Z", + "@timestamp": "2023-07-21T16:52:08.000Z", + "severity": { + "text": "cgid:error" + }, + "communication": { + "source": { + "address": "127.0.0.1", + "ip": "42.204.151.42", + "geo": { + "country": "France", + "country_iso_code": "FR" + } + } + }, + "event": { + "result": "error", + "category": "web", + "name": "error", + "type": "error", + "domain": "apache.error", + "kind": "error" + }, + "traceId": "d09d293a27c9a754f2bf0196b5a1c9bc", + "spanId": "18ba0e515e42dad0", + "body": "[Sat Aug 12 04:05:51 2006] [notice] Apache/1.3.11 (Unix) mod_perl/1.21 configured -- resuming normal operations" + } +] \ No newline at end of file diff --git a/server/adaptors/integrations/__data__/repository/apache/info/INGESTION.md b/server/adaptors/integrations/__data__/repository/apache/info/INGESTION.md new file mode 100644 index 000000000..f9f64bcdd --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/apache/info/INGESTION.md @@ -0,0 +1,149 @@ +# Ingestion Pipeline +To set up an ingestion pipeline, I used Docker to run Fluent-bit and an Apache fake log generator, along with an instance of OpenSearch. + +## FluentBit and OpenSearch Setup +First, create a `docker-compose.yml` instance like [this](https://github.com/opensearch-project/data-prepper/blob/93d06db5cad280e2e4c53e12dfb47c7cbaa7b364/examples/log-ingestion/docker-compose.yaml). This will pull FluentBit and OpenSearch Docker images and run them in `opensearch-net` Docker network. + +Next, use an Apache log generator to generate sample logs. I used `mingrammer/flog` in my `docker-compose.yml` file: +``` + apache: + image: mingrammer/flog + command: "--loop -d 5s -f apache_combined" + container_name: apache + networks: + - opensearch-net # All of the containers will join the same Docker bridge network + links: + - fluentbit + logging: + driver: "fluentd" + options: + fluentd-address: 127.0.0.1:24224 + tag: apache.access + fluentd-async: "true" +``` + +Then, create a `fluent-bit.conf` as follows: +``` +[SERVICE] + Parsers_File parsers.conf + +[INPUT] + Name forward + Port 24224 + +[FILTER] + Name parser + Match apache.access + Key_Name log + Parser apache + +[FILTER] + Name lua + Match apache.access + Script otel-converter.lua + call convert_to_otel + +[OUTPUT] + Name opensearch + Match apache.access + Host opensearch + Port 9200 + Index ss4o_logs-apache-prod-sample + Suppress_Type_Name On +``` +This creates a Fluent-Bit pipeline that forwards your generated apache logs through a parser to OpenSearch. + +Create a `parsers.conf` file as follows: +``` +[PARSER] + Name apache + Format regex + Regex ^(?[^ ]*) [^ ]* (?[^ ]*) \[(?