1801-preprocess-azure.conf

# SOF-ELK® Configuration File
# (C)2023 Lewes Technology Consulting, LLC
#
# This file parses CSV-formatted Azure linux syslog logs
# At this time, the only known CSV logs from Azure that will be handled are of this type
# This preprocessor normalizes the content to match other syslog pipelines
# The resulting event will be handled as a normal syslog event

filter {
  if [type] == "azure" {

    if "csv" in [tags] {
      ### Azure Linux Logs, in CSV format
      csv {
        separator => ","
        skip_empty_rows => "true"
        columns => [ "partition_key", "row_key", "datetime", "deployment_id", "deployment_id_type","event_time", "event_time_type", "facility", "facility_type","fluentd_ingest_timestamp", "fluentd_ingest_timestamp_type", "host", "host_type", "ignore", "ignore_type", "msg", "msg_type", "n", "n_type", "precise_timestamp", "precise_timestamp_type", "row_index", "row_index_type", "sending_host", "sending_host_type", "severity", "severity_type", "timestamp", "timestamp_type", "hostname", "hostname_type", "ident", "ident_type", "pid", "pid_type" ]
        remove_field => "message"
        target => "raw"
        add_tag => [ "azure_linux_syslog" ]
      }

      mutate {
        rename => {
          "[raw][event_time]" => "event_time"
          "[raw][facility]" => "facility"
          "[raw][severity]" => "severity"
          "[raw][host]" => "syslog_hostname"
          "[raw][ident]" => "syslog_program"
          "[raw][pid]" => "syslog_pid"
          "[raw][msg]" => "message"
        }
      }

      if [event_time] == "EventTime" {
        drop {}  # drop the first line that contains the column names.
      }

      date {
        match => [ "event_time", "ISO8601" ]
      }

      # convert facility and severity back to integers
      translate {
        source => "facility"
        target => "facility_int"
        dictionary_path => "/usr/local/sof-elk/lib/dictionaries/syslog_facility2int.yaml"
      }
      translate {
        source => "severity"
        target => "severity_int"
        dictionary_path => "/usr/local/sof-elk/lib/dictionaries/syslog_severity2int.yaml"
      }

      mutate {
        remove_field => [ "raw", "event_time", "facility", "severity" ]
        replace => { "type" => "syslog" }
      }
      mutate {
        rename => {
          "facility_int" => "facility"
          "severity_int" => "severity"
        }
      }
    }
  }
}