default.yml

---
description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent
  plugins.
processors:
  - pipeline:
      if: ctx.message.startsWith('{')
      name: '{{ IngestPipeline "third-party" }}'
  - set:
      field: event.ingested
      value: '{{_ingest.timestamp}}'
  - set:
      field: ecs.version
      value: '8.11.0'
  - rename:
      field: message
      target_field: event.original
      ignore_missing: true
      if: 'ctx.event?.original == null'
  - grok:
      field: event.original
      patterns:
      - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address})
        - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}"
        %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long}
        "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})"
      pattern_definitions:
        NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})?
        NGINX_NOTSEPARATOR: "[^\t ,:]+"
        NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))*
      ignore_missing: true
  - grok:
      field: nginx.access.info
      patterns:
      - '%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}'
      - ""
      ignore_missing: true
  - uri_parts:
      field: _tmp.url_orig
      ignore_failure: true
  - set:
      field: url.domain
      value: "{{destination.domain}}"
      if: ctx.url?.domain == null && ctx.destination?.domain != null
  - remove:
      field:
        - nginx.access.info
        - _tmp.url_orig
      ignore_missing: true
  - split:
      field: nginx.access.remote_ip_list
      separator: '"?,?\s+'
      ignore_missing: true
  - split:
      field: nginx.access.origin
      separator: '"?,?\s+'
      ignore_missing: true
  - set:
      field: source.address
      if: ctx.source?.address == null
      value: ""
  - script:
      if: ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.access.remote_ip_list.length > 0
      lang: painless
      source: >-
        boolean isPrivate(def dot, def ip) {
          try {
            StringTokenizer tok = new StringTokenizer(ip, dot);
            int firstByte = Integer.parseInt(tok.nextToken());
            int secondByte = Integer.parseInt(tok.nextToken());
            if (firstByte == 10) {
              return true;
            }
            if (firstByte == 192 && secondByte == 168) {
              return true;
            }
            if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) {
              return true;
            }
            if (firstByte == 127) {
              return true;
            }
            return false;
          }
          catch (Exception e) {
            return false;
          }
        }
        try {
          ctx.source.address = null;
          if (ctx.nginx.access.remote_ip_list == null) {
            return;
          }
          def found = false;
          for (def item : ctx.nginx.access.remote_ip_list) {
            if (!isPrivate(params.dot, item)) {
              ctx.source.address = item;
              found = true;
              break;
            }
          }
          if (!found) {
            ctx.source.address = ctx.nginx.access.remote_ip_list[0];
          }
        }
        catch (Exception e) {
          ctx.source.address = null;
        }
      params:
        dot: .
  - remove:
      field: source.address
      if: ctx.source.address == null
  - grok:
      field: source.address
      patterns:
      - ^%{IP:source.ip}$
      ignore_failure: true
  - remove:
      field: event.created
      ignore_missing: true
      ignore_failure: true
  - rename:
      field: '@timestamp'
      target_field: event.created
  - convert:
      field: destination.port
      type: long
      if: ctx.destination?.port != null
  - date:
      field: nginx.access.time
      target_field: '@timestamp'
      formats:
      - dd/MMM/yyyy:H:m:s Z
      on_failure:
      - append:
          field: error.message
          value: '{{ _ingest.on_failure_message }}'
  - remove:
      field: nginx.access.time
  - user_agent:
      field: user_agent.original
      ignore_missing: true
  - geoip:
      field: source.ip
      target_field: source.geo
      ignore_missing: true
  - geoip:
      database_file: GeoLite2-ASN.mmdb
      field: source.ip
      target_field: source.as
      properties:
      - asn
      - organization_name
      ignore_missing: true
  - rename:
      field: source.as.asn
      target_field: source.as.number
      ignore_missing: true
  - rename:
      field: source.as.organization_name
      target_field: source.as.organization.name
      ignore_missing: true
  - set:
      field: event.kind
      value: event
  - append:
      field: event.category
      value: web
  - append:
      field: event.type
      value: access
  - set:
      field: event.outcome
      value: success
      if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
  - set:
      field: event.outcome
      value: failure
      if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400"      
  - append:
      field: related.ip
      value: "{{source.ip}}"
      if: "ctx?.source?.ip != null"
  - append:
      field: related.ip
      value: "{{destination.ip}}"
      if: "ctx?.destination?.ip != null"
  - append:
      field: related.user
      value: "{{user.name}}"
      if: "ctx?.user?.name != null"
  - script:
      lang: painless
      description: This script processor iterates over the whole document to remove fields with null values.
      source: |
        void handleMap(Map map) {
          for (def x : map.values()) {
            if (x instanceof Map) {
                handleMap(x);
            } else if (x instanceof List) {
                handleList(x);
            }
          }
          map.values().removeIf(v -> v == null);
        }
        void handleList(List list) {
          for (def x : list) {
              if (x instanceof Map) {
                  handleMap(x);
              } else if (x instanceof List) {
                  handleList(x);
              }
          }
        }
        handleMap(ctx);
  - remove:
      field: event.original
      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
      ignore_failure: true
      ignore_missing: true
on_failure:
  - set:
      field: error.message
      value: '{{ _ingest.on_failure_message }}'