Update logstash parsing contribution

Author: mohierf

contrib/logstash/README.rst

@@ -2,7 +2,7 @@
 Alignak Logstash
 ================
 
-Alignak monitoring log is easily parsable thanks to logstash to store all the monitoring events into an Elasticsearch database. Logstash is a powerful and easy to use log parser... and Kibana alllows to easily build dashboards from the data collected ;)
+Alignak monitoring events log is easily parsable thanks to logstash to store all the monitoring events into an Elasticsearch database or any other...). Logstash is a powerful and easy to use log parser... and Kibana allows to easily build dashboards from the data collected ;)
 
 
 Installation
@@ -15,12 +15,21 @@ A `logstash.conf` example file is available in the same directory as this doc fi
 Configuration
 -------------
 
-Copy the `logstash.conf` in the logstash configuration directory (eg. */usr/local/etc/logstash*) and copy the *patterns* directory of this repository in the same place.
+Copy the `logstash.conf` in the logstash configuration directory (eg. */etc/logstash*) and copy the *patterns* directory of this repository in the same place.
 
 Update the `logstash.conf` according to your configuration. Some important updates:
 - the date inserted in each log is formatted according to the logger configuration. Often it is an ISO date yyyy-mm-dd hh:mm:ss but you may have set this date as a unix timestamp. Update the patterns and the `logstash.conf` accordingly
 - the elasticsearch URL must be updated to connect your own ES cluster
 
+Using an output plugin for MongoDB allows to get Alignak events log in a MongoDB collection::
+
+   # Install the output plugin for MongoDB
+   $ sudo /usr/share/logstash/bin/logstash-plugin install logstash-output-mongodb
+   Validating logstash-output-mongodb
+   Installing logstash-output-mongodb
+   Installation successful
+
+
 Collected information
 ---------------------
 
@@ -37,3 +46,4 @@ The logstash parser is able to analyse the Alignak daemons log files. Extracted
 Monitoring log
 ~~~~~~~~~~~~~~
 
+All the monitoring events are extracted from the monitoring events log and pushed to the output plugins defined in the logstash.conf file: elasticsearch and / or mongodb. Default is to push to elasticsearch; you can uncomment to push the parsed log to a Mongo database.

contrib/logstash/logstash.conf

@@ -1,15 +1,16 @@
 input {
    # Monitoring events log
    file {
-      type => "alignak_monitoring_log"
+      type => "alignak_events_log"
       path => [
-        "/usr/local/var/log/alignak/monitoring-log/*"
+#        "/usr/local/var/log/alignak/monitoring-log/*"
+        "/tmp/var/log/alignak/alignak-events.log"
       ]
-      tags => [ "alignak-monitoring" ]
+      tags => [ "alignak-events" ]
       start_position => "beginning"
-      sincedb_path => "/var/db/logstash/since_alignak_monitoring"
+      sincedb_path => "/var/run/logstash/since_alignak_events"
       codec => multiline {
-         patterns_dir => ["/usr/local/etc/logstash/patterns"]
+         patterns_dir => ["/etc/logstash/patterns"]
          pattern => "^%{ALIGNAK_TIME}"
          negate => true
          what => "previous"
@@ -24,9 +25,9 @@ input {
       ]
       tags => [ "alignak-daemon" ]
       start_position => "beginning"
-      sincedb_path => "/var/db/logstash/since_alignak_daemon"
+      sincedb_path => "/var/run/logstash/since_alignak_daemon"
       codec => multiline {
-         patterns_dir => ["/usr/local/etc/logstash/patterns"]
+         patterns_dir => ["/etc/logstash/patterns"]
          pattern => "^%{ALIGNAK_DAEMON_TIME}"
          negate => true
          what => "previous"
@@ -36,11 +37,19 @@ input {
 
 filter {
    # Monitoring events log
-   if [type] == "alignak_monitoring_log" {
+   if [type] == "alignak_events_log" {
       grok {
-         patterns_dir => ["/usr/local/etc/logstash/patterns"]
+         patterns_dir => ["/etc/logstash/patterns"]
          match => { "message" => "%{ALIGNAK_LOG}" }
       }
+      date {
+        match => [ "[alignak][timestamp]", "yyyy-MM-dd HH:mm:ss" ]
+        target => "@timestamp"
+      }
+      date {
+        match => [ "[alignak][timestamp]", "yyyy-MM-dd HH:mm:ss" ]
+        target => "[alignak][timestamp]"
+      }
       csv {
          source => "[alignak][check_result]"
          separator => ";"
@@ -61,10 +70,6 @@ filter {
           columns => [ "dummy", "[alignak][host_name]", "[alignak][service]", "[alignak][state_id]", "[alignak][check_result]" ]
         }
       }
-      date {
-        match => [ "[alignak][timestamp]", "yyyy-MM-dd HH:mm:ss" ]
-        target => "[alignak][timestamp]"
-      }
       if [alignak][ext_cmd_timestamp] {
         date {
           match => [ "[alignak][ext_cmd_timestamp]", "UNIX" ]
@@ -77,22 +82,17 @@ filter {
           ruby {
             code => "event.set('@timestamp', event.get('[alignak][ext_cmd_timestamp]'));"
           }
-          #mutate {
-          #  update => { "@timestamp" => "[alignak][ext_cmd_timestamp]" }
-          #}
-        }
-      } else {
-        date {
-          match => [ "[alignak][timestamp]", "yyyy-MM-dd HH:mm:ss" ]
-          target => "@timestamp"
+          mutate {
+            update => { "@timestamp" => "[alignak][ext_cmd_timestamp]" }
+          }
         }
       }
    }
 
    # Daemons log
    if [type] == "alignak_daemon" {
       grok {
-         patterns_dir => ["/usr/local/etc/logstash/patterns"]
+         patterns_dir => ["/etc/logstash/patterns"]
          match => { "message" => "%{ALIGNAK_DAEMON_LOG}" }
       }
       date {
@@ -107,21 +107,35 @@ filter {
 
 output {
    # Emit events to stdout for easy debugging of what is going through logstash.
-#   stdout { debug => "true" }
-#   stdout { codec => rubydebug }
+   stdout { codec => rubydebug }
+
+   # Alignak daemons log
+   # if [type] == "alignak-daemon" {
+      # This will use elasticsearch to store your logs.
+      # elasticsearch {
+      #    hosts => [ "es1:9200" ]
+      #    index => "logstash-alignak-daemon-%{+YYYY.MM.dd}"
+      # }
+   # }
 
-  if [type] == "alignak-daemon" {
-    # This will use elasticsearch to store your logs.
-    elasticsearch {
-      hosts => [ "es1:9200" ]
-      index => "logstash-alignak-daemon-%{+YYYY.MM.dd}"
-    }
-  }
+   # Alignak events log
+   if [type] == "alignak_events_log" {
+      elasticsearch {
+         hosts => ["es1:9200"]
+         index => "logstash-alignak-events-%{+YYYY.MM.dd}"
+      }
 
-  if [type] == "alignak-monitoring" {
-    elasticsearch {
-      hosts => ["es1:9200"]
-      index => "logstash-alignak-monitoring-%{+YYYY.MM.dd}"
-    }
-  }
+#      mongodb {
+#         id => "alignak_mongodb_plugin_id"
+#         collection => "alignak_events"
+#         database => "alignak"
+#         uri => "mongodb://localhost:27017"
+#         # bulk => "true"
+#         # bulk_interval => 10
+#         # bulk_size => 50
+#         # generateId => "true"
+#         # isodate => "true"
+#         # codec => "json"
+#      }
+   }
 }