Audit

// create mappings

$> curl -X PUT http://localhost:9200/_template/logstash-template -d '...'

$> curl -X PUT http://localhost:9200/_template/audit-template -d '...'

// on app machine


$> NODE_ENV=production node . &>log/production.log

$> filebeat -e -c configs/filebeat.yml -d "publish"

$> logstash -f configs/logstash-agent.conf

// filebeat.yml

filebeat:
  prospectors:
    -
      paths:
        - "/Users/horiaradu/dev/scotch-box/airlift-hive/log/production.log"
      fields:
        type: syslog
output:
  logstash:
    hosts: ["127.0.0.1:5044"]

// on app machine


$> NODE_ENV=production node . &>log/production.log

$> filebeat -e -c configs/filebeat.yml -d "publish"

$> logstash -f configs/logstash-agent.conf

# logstash-agent.conf
input {
  beats {
    port => 5044
  }
}
output {
  redis {
    host => "127.0.0.1"
    data_type => "list"
    key => "logstash"
  }
  stdout {
    codec  => rubydebug {
      metadata => true
    }
  }
}

// on collector machine


$> logstash -f configs/logstash-server.conf

# The # character at the beginning of a line indicates a comment. Use
# comments to describe your configuration.
input {
  redis {
    host => "127.0.0.1"
    data_type => "list"
    key => "logstash"
  }
}
# The filter part of this file is commented out to indicate that it is
# optional.
filter {
  # try to parse access logs
  grok {
    match => { "message" => "%{COMBINEDAPACHELOG}" }
    add_field => {
      'logstash_event_type' => 'access-log'
    }
  }
  if [logstash_event_type] != 'access-log' {
    # try to parse audit logs
    ruby {
      code => "
        require 'json'
        message = JSON.parse(event['message'])

        if message['audit']
          audit_info = message.delete('audit_info')

          payload = audit_info.delete('payload')
          where = audit_info.delete('where')
          migration = audit_info.delete('migration')
          event['payload'] = payload.to_json.to_s
          event['where'] = where.to_json.to_s
          event['migration'] = migration.to_json.to_s

          audit_info.each do |k, v|
            event[k] = v
          end

          event['logstash_event_type'] = 'audit-log'
        else
          event['logstash_event_type'] = 'plain-log'
        end

        event['msg'] = message
      "
    }
  }
  if "_rubyexception" in [tags] {
    mutate {
      add_field => {
        'logstash_event_type' => 'plain-log'
      }
    }
  }
}
output {
  if [logstash_event_type] == 'audit-log' {
    elasticsearch {
      hosts => "localhost:9200"
      manage_template => false
      index => "v1-audit-%{+YYYY.MM.dd}"
      document_type => "audit"
      template => 'index-templates/audit_template.json'
    }
  } else {
    elasticsearch {
      hosts => "localhost:9200"
      manage_template => false
      index => "v1-logstash-%{+YYYY.MM.dd}"
      document_type => "logstash"
      template => 'index-templates/logstash_template.json'
    }
  }
  stdout {
    codec  => rubydebug {
      metadata => true
    }
  }
}

// on collector machine


$> redis-server

$> elasticsearch

$> kibana

// event structure


{
    "uuid": "AircraftProfile-12",
    "entity_type": "AircraftProfile",
    "entity_id": "12",
    "event_timestamp": "2016-03-09T20:18:00.000Z",
    "payload": {
        "field": "value"
    },
    ...
}

// event structure


{
    "uuid": "AircraftProfile--1",
    "entity_type": "AircraftProfile",
    "event_timestamp": "2016-03-09T20:18:00.000Z",
    "payload": {
        "anotherField": "someValue"
    },
    "where": {
        "field": "value"
    }
    ...
}