From 1237eafdf29f327af9e7c122976386feeaa4c6a9 Mon Sep 17 00:00:00 2001 From: Lennart Koopmann Date: Tue, 19 Apr 2011 22:55:04 +0200 Subject: [PATCH 01/19] blacklist regex matcher now having same behavior as streams message matcher. #SERVER-20 (cherry picked from commit b25214ee7db256f93b46e0c30970f0c0d4b4b7f1) --- .../messagehandlers/gelf/GELFMessage.java | 3 +- .../messagehandlers/gelf/GELFMessageTest.java | 33 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/graylog2/messagehandlers/gelf/GELFMessage.java b/src/main/java/org/graylog2/messagehandlers/gelf/GELFMessage.java index 11c31e3057c4..72563a963be9 100644 --- a/src/main/java/org/graylog2/messagehandlers/gelf/GELFMessage.java +++ b/src/main/java/org/graylog2/messagehandlers/gelf/GELFMessage.java @@ -31,6 +31,7 @@ import org.graylog2.streams.matchers.StreamRuleMatcherIF; import java.util.*; +import java.util.regex.Pattern; import java.util.zip.Deflater; import org.json.simple.JSONObject; import org.json.simple.JSONValue; @@ -313,7 +314,7 @@ public boolean blacklisted(List blacklists) { for (Blacklist blacklist : blacklists) { for (BlacklistRule rule : blacklist.getRules()) { - if (this.getShortMessage().matches(rule.getTerm())) { + if (Pattern.compile(rule.getTerm(), Pattern.DOTALL).matcher(this.getShortMessage()).matches()) { LOG.info("Message <" + this.toString() + "> is blacklisted. First match on " + rule.getTerm()); return true; } diff --git a/src/test/java/org/graylog2/messagehandlers/gelf/GELFMessageTest.java b/src/test/java/org/graylog2/messagehandlers/gelf/GELFMessageTest.java index fa3b854b70d4..37f73fdfa906 100644 --- a/src/test/java/org/graylog2/messagehandlers/gelf/GELFMessageTest.java +++ b/src/test/java/org/graylog2/messagehandlers/gelf/GELFMessageTest.java @@ -97,4 +97,37 @@ public void testBlacklistedWithPositiveResult() { assertTrue(msg.blacklisted(blacklists)); } + @Test + public void testBlacklistedWithPositiveResultAndNewline() { + BasicDBObject mongoList = new BasicDBObject(); + mongoList.put("_id", new ObjectId()); + mongoList.put("title", "foo"); + + BasicDBObject mongoRule1 = new BasicDBObject(); + mongoRule1.put("_id", new ObjectId()); + mongoRule1.put("blacklist_id", mongoList.get("_id")); + mongoRule1.put("term", "^ohai.+"); + + BasicDBObject mongoRule2 = new BasicDBObject(); + mongoRule2.put("_id", new ObjectId()); + mongoRule1.put("blacklist_id", mongoList.get("_id")); + mongoRule2.put("term", ".+aarrghhhllll"); + + BasicDBList rules = new BasicDBList(); + rules.add(mongoRule1); + rules.add(mongoRule2); + + mongoList.put("blacklisted_terms", rules); + + Blacklist blacklist = new Blacklist(mongoList); + + GELFMessage msg = new GELFMessage(); + msg.setShortMessage("ohai thar\nfoo"); + + List blacklists = new ArrayList(); + blacklists.add(blacklist); + + assertTrue(msg.blacklisted(blacklists)); + } + } \ No newline at end of file From cbfe7bde86f5fe8fcf2497e824c8d522b2230f00 Mon Sep 17 00:00:00 2001 From: Dennis Oelkers Date: Tue, 9 Aug 2011 12:01:17 +0200 Subject: [PATCH 02/19] - added new matcher class which matches the pattern to the full message of an event --- .../java/org/graylog2/streams/StreamRule.java | 1 + .../streams/StreamRuleMatcherFactory.java | 3 ++ .../streams/matchers/FullMessageMatcher.java | 43 +++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 src/main/java/org/graylog2/streams/matchers/FullMessageMatcher.java diff --git a/src/main/java/org/graylog2/streams/StreamRule.java b/src/main/java/org/graylog2/streams/StreamRule.java index 188b926dcb49..56aae9f3404b 100644 --- a/src/main/java/org/graylog2/streams/StreamRule.java +++ b/src/main/java/org/graylog2/streams/StreamRule.java @@ -36,6 +36,7 @@ public class StreamRule { public final static int TYPE_HOST = 2; public final static int TYPE_SEVERITY = 3; public final static int TYPE_FACILITY = 4; + public final static int TYPE_FULL_MESSAGE = 5; public final static int TYPE_ADDITIONAL = 6; private ObjectId objectId = null; diff --git a/src/main/java/org/graylog2/streams/StreamRuleMatcherFactory.java b/src/main/java/org/graylog2/streams/StreamRuleMatcherFactory.java index 253b7b5a513d..7f0f287ac4b5 100644 --- a/src/main/java/org/graylog2/streams/StreamRuleMatcherFactory.java +++ b/src/main/java/org/graylog2/streams/StreamRuleMatcherFactory.java @@ -39,6 +39,9 @@ public static StreamRuleMatcherIF build(int ruleType) throws InvalidStreamRuleTy case StreamRule.TYPE_MESSAGE: matcher = new MessageMatcher(); break; + case StreamRule.TYPE_FULL_MESSAGE: + matcher = new FullMessageMatcher(); + break; case StreamRule.TYPE_HOST: matcher = new HostMatcher(); break; diff --git a/src/main/java/org/graylog2/streams/matchers/FullMessageMatcher.java b/src/main/java/org/graylog2/streams/matchers/FullMessageMatcher.java new file mode 100644 index 000000000000..54c13aefa45e --- /dev/null +++ b/src/main/java/org/graylog2/streams/matchers/FullMessageMatcher.java @@ -0,0 +1,43 @@ +/** + * Copyright 2011 Lennart Koopmann + * + * This file is part of Graylog2. + * + * Graylog2 is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog2. If not, see . + * + */ + +package org.graylog2.streams.matchers; + +import java.util.regex.Pattern; +import org.graylog2.messagehandlers.gelf.GELFMessage; +import org.graylog2.streams.StreamRule; + +/** + * FullMessageMatcher.java: Mar 27, 2011 4:50:34 PM + * + * [description] + * + * @author: Dennis Oelkers + */ +public class FullMessageMatcher implements StreamRuleMatcherIF { + + public boolean match(GELFMessage msg, StreamRule rule) { + Pattern messagePattern = Pattern.compile(rule.getValue(), Pattern.DOTALL); + String fullMessage = msg.getFullMessage(); + + return (fullMessage != null && messagePattern.matcher(msg.getFullMessage()).matches()); + } + +} \ No newline at end of file From 56894088b0850c44e6c7379ba932048a9a63a370 Mon Sep 17 00:00:00 2001 From: Dennis Oelkers Date: Tue, 9 Aug 2011 12:09:09 +0200 Subject: [PATCH 03/19] Revert "- added new matcher class which matches the pattern to the full message of an event" -> was supposed to go to develop branch, d'oh! This reverts commit cbfe7bde86f5fe8fcf2497e824c8d522b2230f00. --- .../java/org/graylog2/streams/StreamRule.java | 1 - .../streams/StreamRuleMatcherFactory.java | 3 -- .../streams/matchers/FullMessageMatcher.java | 43 ------------------- 3 files changed, 47 deletions(-) delete mode 100644 src/main/java/org/graylog2/streams/matchers/FullMessageMatcher.java diff --git a/src/main/java/org/graylog2/streams/StreamRule.java b/src/main/java/org/graylog2/streams/StreamRule.java index 56aae9f3404b..188b926dcb49 100644 --- a/src/main/java/org/graylog2/streams/StreamRule.java +++ b/src/main/java/org/graylog2/streams/StreamRule.java @@ -36,7 +36,6 @@ public class StreamRule { public final static int TYPE_HOST = 2; public final static int TYPE_SEVERITY = 3; public final static int TYPE_FACILITY = 4; - public final static int TYPE_FULL_MESSAGE = 5; public final static int TYPE_ADDITIONAL = 6; private ObjectId objectId = null; diff --git a/src/main/java/org/graylog2/streams/StreamRuleMatcherFactory.java b/src/main/java/org/graylog2/streams/StreamRuleMatcherFactory.java index 7f0f287ac4b5..253b7b5a513d 100644 --- a/src/main/java/org/graylog2/streams/StreamRuleMatcherFactory.java +++ b/src/main/java/org/graylog2/streams/StreamRuleMatcherFactory.java @@ -39,9 +39,6 @@ public static StreamRuleMatcherIF build(int ruleType) throws InvalidStreamRuleTy case StreamRule.TYPE_MESSAGE: matcher = new MessageMatcher(); break; - case StreamRule.TYPE_FULL_MESSAGE: - matcher = new FullMessageMatcher(); - break; case StreamRule.TYPE_HOST: matcher = new HostMatcher(); break; diff --git a/src/main/java/org/graylog2/streams/matchers/FullMessageMatcher.java b/src/main/java/org/graylog2/streams/matchers/FullMessageMatcher.java deleted file mode 100644 index 54c13aefa45e..000000000000 --- a/src/main/java/org/graylog2/streams/matchers/FullMessageMatcher.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2011 Lennart Koopmann - * - * This file is part of Graylog2. - * - * Graylog2 is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Graylog2 is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Graylog2. If not, see . - * - */ - -package org.graylog2.streams.matchers; - -import java.util.regex.Pattern; -import org.graylog2.messagehandlers.gelf.GELFMessage; -import org.graylog2.streams.StreamRule; - -/** - * FullMessageMatcher.java: Mar 27, 2011 4:50:34 PM - * - * [description] - * - * @author: Dennis Oelkers - */ -public class FullMessageMatcher implements StreamRuleMatcherIF { - - public boolean match(GELFMessage msg, StreamRule rule) { - Pattern messagePattern = Pattern.compile(rule.getValue(), Pattern.DOTALL); - String fullMessage = msg.getFullMessage(); - - return (fullMessage != null && messagePattern.matcher(msg.getFullMessage()).matches()); - } - -} \ No newline at end of file From 43899858e63204360719f57580cab68f83b6fa03 Mon Sep 17 00:00:00 2001 From: Lennart Koopmann Date: Sat, 24 Dec 2011 16:51:16 +0100 Subject: [PATCH 04/19] allowe to override date of syslog messages to NOW this takes place if the date could not be parsed. configurable via graylog2.conf. fixes #SERVER-90 (cherry picked from commit 00e964ecf199e767d09aa024d69f3207e7ceb729) Conflicts: src/main/java/org/graylog2/messagehandlers/syslog/SyslogEventHandler.java --- misc/graylog2.conf | 2 ++ src/main/java/org/graylog2/Configuration.java | 7 +++++++ .../messagehandlers/syslog/SyslogEventHandler.java | 14 +++++++++++--- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/misc/graylog2.conf b/misc/graylog2.conf index 3f3b896a0e34..f4aee9302c97 100644 --- a/misc/graylog2.conf +++ b/misc/graylog2.conf @@ -8,6 +8,8 @@ elasticsearch_index_name = graylog2 # Always try a reverse DNS lookup instead of parsing hostname from syslog message? force_syslog_rdns = false +# Set time to NOW if parsing date/time from syslog message failed instead of rejecting it? +allow_override_syslog_date = true # MongoDB Configuration mongodb_useauth = true diff --git a/src/main/java/org/graylog2/Configuration.java b/src/main/java/org/graylog2/Configuration.java index 8c80400ddeac..eb3d71b7b078 100644 --- a/src/main/java/org/graylog2/Configuration.java +++ b/src/main/java/org/graylog2/Configuration.java @@ -59,6 +59,9 @@ public class Configuration { @Parameter(value = "mongodb_useauth", required = true) private boolean mongoUseAuth = false; + @Parameter(value = "allow_override_syslog_date", required = true) + private boolean allowOverrideSyslogDate = true; + @Parameter(value = "elasticsearch_url", required = true) private String elasticsearchUrl = "http://localhost:9200/"; @@ -149,6 +152,10 @@ public boolean getForceSyslogRdns() { return forceSyslogRdns; } + public boolean getAllowOverrideSyslogDate() { + return allowOverrideSyslogDate; + } + public String getElasticSearchUrl() { String ret = elasticsearchUrl; diff --git a/src/main/java/org/graylog2/messagehandlers/syslog/SyslogEventHandler.java b/src/main/java/org/graylog2/messagehandlers/syslog/SyslogEventHandler.java index 65ea683d5061..fc1e13c60240 100644 --- a/src/main/java/org/graylog2/messagehandlers/syslog/SyslogEventHandler.java +++ b/src/main/java/org/graylog2/messagehandlers/syslog/SyslogEventHandler.java @@ -30,6 +30,7 @@ import java.net.SocketAddress; import java.net.UnknownHostException; +import java.util.Date; import org.graylog2.Main; /** @@ -70,10 +71,17 @@ public void event(SyslogServerIF syslogServer, SocketAddress socketAddress, Sysl LOG.debug("Raw: " + new String(event.getRaw())); LOG.debug("Host stripped from message? " + event.isHostStrippedFromMessage()); - // Manually check for provided date because it's necessary to parse the GELF message. Second check for completness later. + // Check if date could be parsed. if (event.getDate() == null) { - LOG.info("Syslog message is missing date or could not be parsed. Not further handling. Message was: " + event.getRaw()); - return; + if (Main.configuration.getAllowOverrideSyslogDate()) { + // empty Date constructor allocates a Date object and initializes it so that it represents the time at which it was allocated. + event.setDate(new Date()); + LOG.info("Date could not be parsed. Was set to NOW because allow_override_syslog_date is true."); + } else { + LOG.info("Syslog message is missing date or date could not be parsed. (Possibly set allow_override_syslog_date to true) " + + "Not further handling. Message was: " + new String(event.getRaw())); + return; + } } // Possibly overwrite host with RNDS if configured. From 23e92d9b2136cd7417d41ce3a478e55e2df81c59 Mon Sep 17 00:00:00 2001 From: Lennart Koopmann Date: Sat, 31 Dec 2011 15:15:41 +0100 Subject: [PATCH 05/19] Several retries for ES index check on startup gives ElasticSearch some time to come up. Useful when used in init scripts etc. fixes #SERVER-96 --- src/main/java/org/graylog2/Main.java | 58 +++++++++++++++++++++------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/graylog2/Main.java b/src/main/java/org/graylog2/Main.java index 03dafc205bfb..b51f617a2879 100644 --- a/src/main/java/org/graylog2/Main.java +++ b/src/main/java/org/graylog2/Main.java @@ -65,6 +65,9 @@ public final class Main { private static final Logger LOG = Logger.getLogger(Main.class); private static final String GRAYLOG2_VERSION = "0.9.6"; + private static final int INDEX_CHECK_RETRIES = 15; + private static final int INDEX_CHECK_INTERVAL_SEC = 5; + public static RulesEngine drools = null; private static final int SCHEDULED_THREADS_POOL_SIZE = 7; @@ -125,23 +128,33 @@ public static void main(String[] args) { System.exit(0); } - // XXX ELASTIC: put in own method - // Check if the index exists. Create it if not. - try { - if (Indexer.indexExists()) { - LOG.info("Index exists. Not creating it."); - } else { - LOG.info("Index does not exist! Trying to create it ..."); - if (Indexer.createIndex()) { - LOG.info("Successfully created index."); - } else { - LOG.fatal("Could not create Index. Terminating."); - System.exit(1); + /* + * Check if the index exists. Create it if not. + * (Try this a few times to allow ElasticSearch to come up.) + */ + boolean indexNotChecked = true; + int indexCheckIntervalCount = 1; + while(indexNotChecked) { + try { + if (checkAndCreateIndex()) { + break; } + } catch (IOException e) { + LOG.warn("Could not check for or create ElasticSearch index. [" + indexCheckIntervalCount + "] " + + "Retrying in " + INDEX_CHECK_INTERVAL_SEC + " seconds.", e); } - } catch (IOException e) { - LOG.fatal("IOException while trying to check Index. Make sure that your ElasticSearch server is running.", e); - System.exit(1); + + // Abort if last retry failed. + if (indexCheckIntervalCount == INDEX_CHECK_RETRIES) { + LOG.fatal("Could not check for ElasticSearch index after " + INDEX_CHECK_RETRIES + " retries." + + "Make sure that your ElasticSearch server is running."); + System.exit(1); + } + + indexCheckIntervalCount++; + try { + Thread.sleep(INDEX_CHECK_INTERVAL_SEC * 1000); + } catch (InterruptedException e) { } } savePidFile(commandLineArguments.getPidFile()); @@ -349,4 +362,19 @@ public static void writeInitialServerValues(Configuration configuration) { ServerValue.writeMessageQueueBatchSize(configuration.getMessageQueueBatchSize()); ServerValue.writeMessageQueuePollFrequency(configuration.getMessageQueuePollFrequency()); } + + public static boolean checkAndCreateIndex() throws IOException { + if (Indexer.indexExists()) { + LOG.info("Index exists. Not creating it."); + return true; + } else { + LOG.info("Index does not exist! Trying to create it ..."); + if (Indexer.createIndex()) { + LOG.info("Successfully created index."); + return true; + } else { + return false; + } + } + } } From 80030874e758a306688bce5bff98a1ffdac72fc0 Mon Sep 17 00:00:00 2001 From: Lennart Koopmann Date: Sat, 31 Dec 2011 15:54:27 +0100 Subject: [PATCH 06/19] bumped version to 0.9.6p1 --- pom.xml | 2 +- src/main/java/org/graylog2/Main.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 64ae4a20f164..b40d5d2c9d6d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.graylog2 graylog2-server - 0.9.6 + 0.9.6p1 jar diff --git a/src/main/java/org/graylog2/Main.java b/src/main/java/org/graylog2/Main.java index b51f617a2879..4096ca7bb12f 100644 --- a/src/main/java/org/graylog2/Main.java +++ b/src/main/java/org/graylog2/Main.java @@ -63,7 +63,7 @@ public final class Main { private static final Logger LOG = Logger.getLogger(Main.class); - private static final String GRAYLOG2_VERSION = "0.9.6"; + private static final String GRAYLOG2_VERSION = "0.9.6p1"; private static final int INDEX_CHECK_RETRIES = 15; private static final int INDEX_CHECK_INTERVAL_SEC = 5; From 9bace34b804cb936aefd11efe4e28ef69041d125 Mon Sep 17 00:00:00 2001 From: Lennart Koopmann Date: Thu, 1 Mar 2012 17:45:57 +0100 Subject: [PATCH 07/19] made getMessageCountsColl() synchronized #SERVER-102 --- src/main/java/org/graylog2/database/MongoConnection.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/graylog2/database/MongoConnection.java b/src/main/java/org/graylog2/database/MongoConnection.java index 964af0e655e4..dc74fca99832 100644 --- a/src/main/java/org/graylog2/database/MongoConnection.java +++ b/src/main/java/org/graylog2/database/MongoConnection.java @@ -162,7 +162,7 @@ public DBCollection getMessagesColl() { * * @return The messages collection */ - public DBCollection getMessageCountsColl() { + public synchronized DBCollection getMessageCountsColl() { if (this.messageCountsCollection != null) { return this.messageCountsCollection; } From ab05200cbd3f1cf7a4d314aa7a133fd19816ba91 Mon Sep 17 00:00:00 2001 From: Lennart Koopmann Date: Wed, 11 Apr 2012 20:27:33 +0200 Subject: [PATCH 08/19] make message counter hashmaps concurrent (cherry picked from commit 7e7725dd8131263024858eae523852d703f4c310) --- .../org/graylog2/messagehandlers/common/MessageCounter.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/graylog2/messagehandlers/common/MessageCounter.java b/src/main/java/org/graylog2/messagehandlers/common/MessageCounter.java index 38da7a47662d..75f540a335e3 100644 --- a/src/main/java/org/graylog2/messagehandlers/common/MessageCounter.java +++ b/src/main/java/org/graylog2/messagehandlers/common/MessageCounter.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.graylog2.Tools; /** @@ -87,11 +88,11 @@ public void resetAllCounts() { } public void resetHostCounts() { - this.hosts = new HashMap(); + this.hosts = new ConcurrentHashMap(); } public void resetStreamCounts() { - this.streams = new HashMap(); + this.streams = new ConcurrentHashMap(); } public void resetTotal() { From a196f4d860140b2d61d944f886cfb18411ac82f7 Mon Sep 17 00:00:00 2001 From: Lennart Koopmann Date: Mon, 16 Apr 2012 13:39:14 +0200 Subject: [PATCH 09/19] travis configuration file (cherry picked from commit e67571d90c1a7811aa97e084d4bed790fcd3d564) --- .travis.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000000..dff5f3a5d02a --- /dev/null +++ b/.travis.yml @@ -0,0 +1 @@ +language: java From 788807b19ffdcb7e4663985ccc6650a79cc88c8c Mon Sep 17 00:00:00 2001 From: Lennart Koopmann Date: Mon, 16 Apr 2012 13:49:42 +0200 Subject: [PATCH 10/19] let's try this: before_install script for travis to install syslog4j jar locally (cherry picked from commit ef2675615047945cbb9ec6e473b76f4b8e9fcd80) --- .travis.yml | 1 + install-syslog4j-jar.sh | 2 ++ 2 files changed, 3 insertions(+) create mode 100755 install-syslog4j-jar.sh diff --git a/.travis.yml b/.travis.yml index dff5f3a5d02a..596805d99bcf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1 +1,2 @@ language: java +before_install: install-syslog4j-jar.sh # installs syslog4j jar to local maven repo diff --git a/install-syslog4j-jar.sh b/install-syslog4j-jar.sh new file mode 100755 index 000000000000..af788ced896c --- /dev/null +++ b/install-syslog4j-jar.sh @@ -0,0 +1,2 @@ +echo "Installing provided syslog4j jar to local mvn repository for great justice!" +mvn install:install-file -DgroupId=org.syslog4j -DartifactId=syslog4j -Dversion=0.9.46 -Dpackaging=jar -Dfile=lib/syslog4j-0.9.46-bin.jar From 7a46a6d4a6926a179462a8dc6316d81eface11fe Mon Sep 17 00:00:00 2001 From: Lennart Koopmann Date: Mon, 16 Apr 2012 13:52:13 +0200 Subject: [PATCH 11/19] travis-ci before_install script path has to be local (cherry picked from commit 5bb2792d334e8b4e7770a7de7fc0babadf5841d9) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 596805d99bcf..3f9e1ea75b8b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,2 +1,2 @@ language: java -before_install: install-syslog4j-jar.sh # installs syslog4j jar to local maven repo +before_install: ./install-syslog4j-jar.sh # installs syslog4j jar to local maven repo From f7586d42c82971b281ef13b074c3b0348e0cb1e5 Mon Sep 17 00:00:00 2001 From: Lennart Koopmann Date: Tue, 8 May 2012 14:47:06 +0200 Subject: [PATCH 12/19] bumped version to 0.9.6p1-RC2 --- Makefile | 4 ++-- pom.xml | 2 +- src/main/java/org/graylog2/Main.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index e756dd4f3dff..6126e859309b 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,8 @@ NAME=graylog2-server PREFIX=/usr DESTDIR= -SERVER_W_DEP=target/graylog2-server-0.9.6-jar-with-dependencies.jar -SERVER=target/graylog2-server-0.9.6.jar +SERVER_W_DEP=target/graylog2-server-0.9.6p1-RC2-jar-with-dependencies.jar +SERVER=target/graylog2-server-0.9.6p1-RC2.jar SYSLOG4J=lib/syslog4j-0.9.46-bin.jar INITD=contrib/distro/generic/graylog2-server.init.d CONF=misc/graylog2.conf diff --git a/pom.xml b/pom.xml index b40d5d2c9d6d..b96bc81006e7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.graylog2 graylog2-server - 0.9.6p1 + 0.9.6p1-RC2 jar diff --git a/src/main/java/org/graylog2/Main.java b/src/main/java/org/graylog2/Main.java index 4096ca7bb12f..7165222b0f78 100644 --- a/src/main/java/org/graylog2/Main.java +++ b/src/main/java/org/graylog2/Main.java @@ -63,7 +63,7 @@ public final class Main { private static final Logger LOG = Logger.getLogger(Main.class); - private static final String GRAYLOG2_VERSION = "0.9.6p1"; + private static final String GRAYLOG2_VERSION = "0.9.6p1-RC2"; private static final int INDEX_CHECK_RETRIES = 15; private static final int INDEX_CHECK_INTERVAL_SEC = 5; From a22f9df3140343444ccb43dd133f01a3b6e9c569 Mon Sep 17 00:00:00 2001 From: Michael Hart Date: Mon, 18 Jun 2012 10:16:11 +1000 Subject: [PATCH 13/19] Updated syslog install script to include mvn path --- install-syslog4j-jar.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/install-syslog4j-jar.sh b/install-syslog4j-jar.sh index af788ced896c..ebea1578ff48 100755 --- a/install-syslog4j-jar.sh +++ b/install-syslog4j-jar.sh @@ -1,2 +1,5 @@ +MVN_REPO="/tmp/graylog2-server-build-${USER}" +MVN_OPTS=-Dmaven.repo.local=${MVN_REPO} + echo "Installing provided syslog4j jar to local mvn repository for great justice!" -mvn install:install-file -DgroupId=org.syslog4j -DartifactId=syslog4j -Dversion=0.9.46 -Dpackaging=jar -Dfile=lib/syslog4j-0.9.46-bin.jar +mvn install:install-file ${MVN_OPTS} -DgroupId=org.syslog4j -DartifactId=syslog4j -Dversion=0.9.46 -Dpackaging=jar -Dfile=lib/syslog4j-0.9.46-bin.jar From c796649ac397b913518209841b0f1486da6e6cb1 Mon Sep 17 00:00:00 2001 From: Michael Hart Date: Mon, 18 Jun 2012 11:29:47 +1000 Subject: [PATCH 14/19] Eliminate nasty race condition when handling chunked GELF messages --- .../gelf/ChunkedGELFClientManager.java | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/graylog2/messagehandlers/gelf/ChunkedGELFClientManager.java b/src/main/java/org/graylog2/messagehandlers/gelf/ChunkedGELFClientManager.java index ae6a216029d0..717b2f3ac7fd 100644 --- a/src/main/java/org/graylog2/messagehandlers/gelf/ChunkedGELFClientManager.java +++ b/src/main/java/org/graylog2/messagehandlers/gelf/ChunkedGELFClientManager.java @@ -51,7 +51,7 @@ public static synchronized ChunkedGELFClientManager getInstance() { } /** - * Add a chunk to it's message + * Add a chunk to its message * * @param chunk * @return NULL if message is not yet complete, the complete ChunkedGELFMessage if this was the last missing chunk @@ -60,23 +60,26 @@ public static synchronized ChunkedGELFClientManager getInstance() { */ public ChunkedGELFMessage insertChunk(GELFClientChunk chunk) throws ForeignGELFChunkException, InvalidGELFChunkException { ChunkedGELFMessage fullMessage = messageMap.get(chunk.getHash()); - + if (fullMessage == null) { // This is the first chunk of this message. Create a new message. - fullMessage = new ChunkedGELFMessage(); + final ChunkedGELFMessage newMessage = new ChunkedGELFMessage(); + fullMessage = messageMap.putIfAbsent(chunk.getHash(), newMessage); + if (fullMessage == null) { + fullMessage = newMessage; + } } - - // Add this chunk to the message. - fullMessage.insertChunk(chunk); - // Save the message with the new chunk. - messageMap.put(chunk.getHash(), fullMessage); + synchronized (fullMessage) { + // Add this chunk to the message. + fullMessage.insertChunk(chunk); - if (fullMessage.isComplete()) { - return fullMessage; - } + if (fullMessage.isComplete()) { + return fullMessage; + } - return null; + return null; + } } /** From a573c5a0f0982ff11173178ea8a7a314a6844f0e Mon Sep 17 00:00:00 2001 From: Lennart Koopmann Date: Mon, 25 Jun 2012 16:54:52 +0200 Subject: [PATCH 15/19] bumped version to 0.9.6p1 --- src/main/java/org/graylog2/Main.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/graylog2/Main.java b/src/main/java/org/graylog2/Main.java index 7165222b0f78..4096ca7bb12f 100644 --- a/src/main/java/org/graylog2/Main.java +++ b/src/main/java/org/graylog2/Main.java @@ -63,7 +63,7 @@ public final class Main { private static final Logger LOG = Logger.getLogger(Main.class); - private static final String GRAYLOG2_VERSION = "0.9.6p1-RC2"; + private static final String GRAYLOG2_VERSION = "0.9.6p1"; private static final int INDEX_CHECK_RETRIES = 15; private static final int INDEX_CHECK_INTERVAL_SEC = 5; From 5bce534851591acfa461217b4f1ab173d090ba07 Mon Sep 17 00:00:00 2001 From: Peter Donald Date: Thu, 5 Jul 2012 22:45:46 +1000 Subject: [PATCH 16/19] Update the before_install script for travis to install the syslog jar into the correct location --- install-syslog4j-jar.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/install-syslog4j-jar.sh b/install-syslog4j-jar.sh index ebea1578ff48..af788ced896c 100755 --- a/install-syslog4j-jar.sh +++ b/install-syslog4j-jar.sh @@ -1,5 +1,2 @@ -MVN_REPO="/tmp/graylog2-server-build-${USER}" -MVN_OPTS=-Dmaven.repo.local=${MVN_REPO} - echo "Installing provided syslog4j jar to local mvn repository for great justice!" -mvn install:install-file ${MVN_OPTS} -DgroupId=org.syslog4j -DartifactId=syslog4j -Dversion=0.9.46 -Dpackaging=jar -Dfile=lib/syslog4j-0.9.46-bin.jar +mvn install:install-file -DgroupId=org.syslog4j -DartifactId=syslog4j -Dversion=0.9.46 -Dpackaging=jar -Dfile=lib/syslog4j-0.9.46-bin.jar From d50db0d1bff4d12e736fd62739c0f28b9b661230 Mon Sep 17 00:00:00 2001 From: kbrockhoff Date: Tue, 15 Jan 2013 09:07:16 -0600 Subject: [PATCH 17/19] added implementation of commons-daemon Daemon interface and modified Core so it will work with or without the daemon --- pom.xml | 5 + src/main/java/org/graylog2/Core.java | 28 ++- src/main/java/org/graylog2/ServerDaemon.java | 215 +++++++++++++++++++ 3 files changed, 243 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/graylog2/ServerDaemon.java diff --git a/pom.xml b/pom.xml index 30f0000bca48..45d3b02e13f3 100644 --- a/pom.xml +++ b/pom.xml @@ -163,6 +163,11 @@ graylog2-plugin 0.10.9 + + commons-daemon + commons-daemon + 1.0.10 + diff --git a/src/main/java/org/graylog2/Core.java b/src/main/java/org/graylog2/Core.java index 7566d102a23b..88115d6f7457 100644 --- a/src/main/java/org/graylog2/Core.java +++ b/src/main/java/org/graylog2/Core.java @@ -39,6 +39,8 @@ import org.graylog2.streams.StreamCache; import com.google.common.collect.Lists; + +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import com.google.common.collect.Maps; import java.util.Map; @@ -112,6 +114,8 @@ public class Core implements GraylogServer { private OutputBuffer outputBuffer; private AtomicInteger outputBufferWatermark = new AtomicInteger(); private AtomicInteger processBufferWatermark = new AtomicInteger(); + private AtomicBoolean componentsInitialized = new AtomicBoolean(false); + private AtomicBoolean processing = new AtomicBoolean(false); private Deflector deflector; @@ -198,10 +202,28 @@ public void registerTransport(Transport transport) { public void registerAlarmCallback(AlarmCallback alarmCallback) { this.alarmCallbacks.add(alarmCallback); } + + public boolean isProcessing() { + return processing.get(); + } + + public void setProcessing(boolean processing) { + this.processing.set(processing); + } @Override public void run() { + if (!componentsInitialized.get()) { + initializeComponents(); + } + + while (isProcessing()) { + try { Thread.sleep(1000); } catch (InterruptedException e) { /* lol, i don't care */ } + } + + } + void initializeComponents() { gelfChunkManager.start(); BlacklistCache.initialize(this); StreamCache.initialize(this); @@ -323,13 +345,9 @@ public void run() { } } + componentsInitialized.set(true); activityWriter.write(new Activity("Started up.", GraylogServer.class)); LOG.info("Graylog2 up and running."); - - while (true) { - try { Thread.sleep(1000); } catch (InterruptedException e) { /* lol, i don't care */ } - } - } private void loadPlugins(Class type, String subDirectory) { diff --git a/src/main/java/org/graylog2/ServerDaemon.java b/src/main/java/org/graylog2/ServerDaemon.java new file mode 100644 index 000000000000..43663a1bea8f --- /dev/null +++ b/src/main/java/org/graylog2/ServerDaemon.java @@ -0,0 +1,215 @@ +/** + * Copyright 2010, 2011, 2012 Lennart Koopmann + * + * This file is part of Graylog2. + * + * Graylog2 is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog2. If not, see . + * + */ + +package org.graylog2; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.commons.daemon.Daemon; +import org.apache.commons.daemon.DaemonContext; +import org.apache.commons.daemon.DaemonInitException; +import org.apache.log4j.Level; +import org.graylog2.activities.Activity; +import org.graylog2.alarms.transports.EmailTransport; +import org.graylog2.alarms.transports.JabberTransport; +import org.graylog2.filters.BlacklistFilter; +import org.graylog2.filters.CounterUpdateFilter; +import org.graylog2.filters.RewriteFilter; +import org.graylog2.filters.StreamMatcherFilter; +import org.graylog2.filters.TokenizerFilter; +import org.graylog2.initializers.AMQPSyncInitializer; +import org.graylog2.initializers.AlarmScannerInitializer; +import org.graylog2.initializers.AnonymousInformationCollectorInitializer; +import org.graylog2.initializers.BufferWatermarkInitializer; +import org.graylog2.initializers.DeflectorThreadsInitializer; +import org.graylog2.initializers.DroolsInitializer; +import org.graylog2.initializers.GraphiteInitializer; +import org.graylog2.initializers.HostCounterCacheWriterInitializer; +import org.graylog2.initializers.IndexRetentionInitializer; +import org.graylog2.initializers.LibratoMetricsInitializer; +import org.graylog2.initializers.MessageCounterInitializer; +import org.graylog2.initializers.ServerValueWriterInitializer; +import org.graylog2.initializers.StatisticsPrinterInitializer; +import org.graylog2.inputs.amqp.AMQPInput; +import org.graylog2.inputs.gelf.GELFTCPInput; +import org.graylog2.inputs.gelf.GELFUDPInput; +import org.graylog2.inputs.http.GELFHttpInput; +import org.graylog2.inputs.syslog.SyslogTCPInput; +import org.graylog2.inputs.syslog.SyslogUDPInput; +import org.graylog2.plugin.Tools; +import org.graylog2.outputs.ElasticSearchOutput; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.beust.jcommander.JCommander; +import com.github.joschi.jadconfig.JadConfig; +import com.github.joschi.jadconfig.RepositoryException; +import com.github.joschi.jadconfig.ValidationException; +import com.github.joschi.jadconfig.repositories.PropertiesRepository; + +/** + * Wrapper for use with Apache Commons Daemon. + * + * @author $Author: kbrockhoff $ + * @version $Revision: 201129 $, $Date: 2013-01-03 08:26:44 -0600 (Thu, 03 Jan 2013) $ + */ +public class ServerDaemon implements Daemon { + + private static final String[] EMPTY_ARGS = {}; + private static final Logger LOG = LoggerFactory.getLogger(ServerDaemon.class); + + private String[] args = EMPTY_ARGS; + private Core server; + private ExecutorService executorService = Executors.newSingleThreadExecutor(); + + public ServerDaemon() { + super(); + } + + @Override + public void init(final DaemonContext context) throws DaemonInitException { + args = context.getArguments(); + } + + @Override + public void start() throws RepositoryException, ValidationException { + + final CommandLineArguments commandLineArguments = new CommandLineArguments(); + final JCommander jCommander = new JCommander(commandLineArguments, args); + jCommander.setProgramName("graylog2"); + + // Are we in debug mode? + if (commandLineArguments.isDebug()) { + LOG.info("Running in Debug mode"); + org.apache.log4j.Logger.getRootLogger().setLevel(Level.ALL); + org.apache.log4j.Logger.getLogger(Main.class.getPackage().getName()).setLevel(Level.ALL); + } + + LOG.info("Graylog2 {} starting up. (JRE: {})", Core.GRAYLOG2_VERSION, Tools.getSystemInformation()); + + String configFile = commandLineArguments.getConfigFile(); + LOG.info("Using config file: {}", configFile); + + final Configuration configuration = new Configuration(); + JadConfig jadConfig = new JadConfig(new PropertiesRepository(configFile), configuration); + + LOG.info("Loading configuration"); + jadConfig.process(); + + // Le server object. This is where all the magic happens. + server = new Core(); + server.initialize(configuration); + + // Could it be that there is another master instance already? + if (configuration.isMaster() && server.cluster().masterCountExcept(server.getServerId()) != 0) { + // All devils here. + String what = "Detected other master node in the cluster! Starting as non-master! " + + "This is a mis-configuration you should fix."; + LOG.warn(what); + server.getActivityWriter().write(new Activity(what, Main.class)); + + configuration.setIsMaster(false); + } + + // Enable local mode? + if (commandLineArguments.isLocal() || commandLineArguments.isDebug()) { + // In local mode, systemstats are sent to localhost for example. + LOG.info("Running in local mode"); + server.setLocalMode(true); + } + + // Are we in stats mode? + if (commandLineArguments.isStats()) { + LOG.info("Printing system utilization information."); + server.setStatsMode(true); + } + + // Register transports. + if (configuration.isTransportEmailEnabled()) { server.registerTransport(new EmailTransport()); } + if (configuration.isTransportJabberEnabled()) { server.registerTransport(new JabberTransport()); } + + // Register initializers. + server.registerInitializer(new ServerValueWriterInitializer()); + server.registerInitializer(new DroolsInitializer()); + server.registerInitializer(new HostCounterCacheWriterInitializer()); + server.registerInitializer(new MessageCounterInitializer()); + server.registerInitializer(new AlarmScannerInitializer()); + if (configuration.isEnableGraphiteOutput()) { server.registerInitializer(new GraphiteInitializer()); } + if (configuration.isEnableLibratoMetricsOutput()) { server.registerInitializer(new LibratoMetricsInitializer()); } + server.registerInitializer(new DeflectorThreadsInitializer()); + server.registerInitializer(new AnonymousInformationCollectorInitializer()); + if (configuration.performRetention() && commandLineArguments.performRetention()) { + server.registerInitializer(new IndexRetentionInitializer()); + } + if (configuration.isAmqpEnabled()) { + server.registerInitializer(new AMQPSyncInitializer()); + } + server.registerInitializer(new BufferWatermarkInitializer()); + if (commandLineArguments.isStats()) { server.registerInitializer(new StatisticsPrinterInitializer()); } + + // Register inputs. + if (configuration.isUseGELF()) { + server.registerInput(new GELFUDPInput()); + server.registerInput(new GELFTCPInput()); + } + + if (configuration.isSyslogUdpEnabled()) { server.registerInput(new SyslogUDPInput()); } + if (configuration.isSyslogTcpEnabled()) { server.registerInput(new SyslogTCPInput()); } + + if (configuration.isAmqpEnabled()) { server.registerInput(new AMQPInput()); } + + if (configuration.isHttpEnabled()) { server.registerInput(new GELFHttpInput()); } + + // Register message filters. + server.registerFilter(new RewriteFilter()); + server.registerFilter(new BlacklistFilter()); + if (configuration.isEnableTokenizerFilter()) { server.registerFilter(new TokenizerFilter()); } + server.registerFilter(new StreamMatcherFilter()); + server.registerFilter(new CounterUpdateFilter()); + + // Register outputs. + server.registerOutput(new ElasticSearchOutput()); + + // initialize the components + server.initializeComponents(); + + // Set running + executorService.execute(server); + } + + @Override + public void stop() { + if (server != null) { + LOG.info("Graylog2 {} exiting.", Core.GRAYLOG2_VERSION); + server.setProcessing(false); + executorService.shutdown(); + } + } + + @Override + public void destroy() { + if (!executorService.isShutdown()) { + stop(); + } + server = null; + } + +} From 19700e81247c54b5c6bf0c184675c9c00f75ada4 Mon Sep 17 00:00:00 2001 From: kbrockhoff Date: Tue, 15 Jan 2013 10:04:28 -0600 Subject: [PATCH 18/19] added init.d script and rpm spec that uses commons-daemon --- .../SOURCES/graylog2-elasticsearch.yml | 65 +++++++ .../SOURCES/graylog2-log4j.xml | 24 +++ .../SOURCES/graylog2.conf | 167 ++++++++++++++++++ .../SOURCES/graylog2.drl | 26 +++ .../SOURCES/graylog2.init | 150 ++++++++++++++++ .../SPECS/graylog2-server.spec | 128 ++++++++++++++ 6 files changed, 560 insertions(+) create mode 100644 contrib/distro/commons-daemon-redhat/SOURCES/graylog2-elasticsearch.yml create mode 100644 contrib/distro/commons-daemon-redhat/SOURCES/graylog2-log4j.xml create mode 100644 contrib/distro/commons-daemon-redhat/SOURCES/graylog2.conf create mode 100644 contrib/distro/commons-daemon-redhat/SOURCES/graylog2.drl create mode 100755 contrib/distro/commons-daemon-redhat/SOURCES/graylog2.init create mode 100644 contrib/distro/commons-daemon-redhat/SPECS/graylog2-server.spec diff --git a/contrib/distro/commons-daemon-redhat/SOURCES/graylog2-elasticsearch.yml b/contrib/distro/commons-daemon-redhat/SOURCES/graylog2-elasticsearch.yml new file mode 100644 index 000000000000..40215ac8f245 --- /dev/null +++ b/contrib/distro/commons-daemon-redhat/SOURCES/graylog2-elasticsearch.yml @@ -0,0 +1,65 @@ +# this must be the same as for your elasticsearch cluster +cluster.name: graylog2 + +# you could also leave this out, but makes it easier to identify the graylog2 client instance +node.name: "graylog2-server" + +# we don't want the graylog2 client to store any data, or be master node +node.master: false +node.data: false + +# you might need to bind to a certain IP address, do that here +#network.host: 172.24.0.14 +# use a different port if you run multiple elasticsearch nodes on one machine +#transport.tcp.port: 9350 + +# we don't need to run the embedded HTTP server here +http.enabled: false + +# adapt these for discovery to work in your network! multicast can be tricky +#discovery.zen.ping.multicast.address: 172.24.0.14 +#discovery.zen.ping.multicast.group: 224.0.0.1 + + +################################## Discovery ################################## + +# Discovery infrastructure ensures nodes can be found within a cluster +# and master node is elected. Multicast discovery is the default. + +# Set to ensure a node sees N other master eligible nodes to be considered +# operational within the cluster. Set this option to a higher value (2-4) +# for large clusters (>3 nodes): +# +# discovery.zen.minimum_master_nodes: 1 + +# Set the time to wait for ping responses from other nodes when discovering. +# Set this option to a higher value on a slow or congested network +# to minimize discovery failures: +# +# discovery.zen.ping.timeout: 3s + +# See +# for more information. + +# Unicast discovery allows to explicitly control which nodes will be used +# to discover the cluster. It can be used when multicast is not present, +# or to restrict the cluster communication-wise. +# +# 1. Disable multicast discovery (enabled by default): +# +# discovery.zen.ping.multicast.enabled: false +# +# 2. Configure an initial list of master nodes in the cluster +# to perform discovery when new nodes (master or data) are started: +# +# discovery.zen.ping.unicast.hosts: ["host1", "host2:port", "host3[portX-portY]"] + +# EC2 discovery allows to use AWS EC2 API in order to perform discovery. +# +# You have to install the cloud-aws plugin for enabling the EC2 discovery. +# +# See +# for more information. +# +# See +# for a step-by-step tutorial. diff --git a/contrib/distro/commons-daemon-redhat/SOURCES/graylog2-log4j.xml b/contrib/distro/commons-daemon-redhat/SOURCES/graylog2-log4j.xml new file mode 100644 index 000000000000..89ef57af1e49 --- /dev/null +++ b/contrib/distro/commons-daemon-redhat/SOURCES/graylog2-log4j.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/contrib/distro/commons-daemon-redhat/SOURCES/graylog2.conf b/contrib/distro/commons-daemon-redhat/SOURCES/graylog2.conf new file mode 100644 index 000000000000..563a201a7ffc --- /dev/null +++ b/contrib/distro/commons-daemon-redhat/SOURCES/graylog2.conf @@ -0,0 +1,167 @@ +# If you are running more than one instances of graylog2-server you have to select one of these +# instances as master. The master will perform some periodical tasks that non-masters won't perform. +is_master = true + +# Set plugin directory here (relative or absolute) +plugin_dir = plugin + +# On which port (UDP) should we listen for Syslog messages? (Standard: 514) +syslog_listen_port = 514 +syslog_listen_address = 0.0.0.0 +syslog_enable_udp = true +syslog_enable_tcp = false +# Standard delimiter is LF. You can force using a NUL byte delimiter using this option. +syslog_use_nul_delimiter = false +# The raw syslog message is stored as full_message of the message if not disabled here. +syslog_store_full_message = true + +# Socket receive buffer size (bytes) for UDP syslog and UDP GELF. +udp_recvbuffer_sizes = 1048576 + +# Embedded elasticsearch configuration file +# pay attention to the working directory of the server, maybe use an absolute path here +elasticsearch_config_file = /etc/graylog2-server/elasticsearch.yml +elasticsearch_max_docs_per_index = 20000000 + +elasticsearch_index_prefix = graylog2 + +# How many indices do you want to keep? If the number of indices exceeds this number, older indices will be dropped. +# elasticsearch_max_number_of_indices*elasticsearch_max_docs_per_index=total number of messages in your setup +elasticsearch_max_number_of_indices = 20 + +# How many ElasticSearch shards and replicas should be used per index? Note that this only applies to newly created indices. +elasticsearch_shards = 4 +elasticsearch_replicas = 0 + +# Analyzer (tokenizer) to use for message and full_message field. The "standard" filter usually is a good idea. +# All supported analyzers are: standard, simple, whitespace, stop, keyword, pattern, language, snowball, custom +# ElasticSearch documentation: http://www.elasticsearch.org/guide/reference/index-modules/analysis/ +# Note that this setting only takes effect on newly created indices. +elasticsearch_analyzer = standard + +# How many minutes of messages do you want to keep in the recent index? This index lives in memory only and is used to build the overview and stream pages. Raise this value if you want to see more messages in the overview pages. This is not affecting for example searches which are always targeting *all* indices. +recent_index_ttl_minutes = 60 + +# Storage type of recent index. Allowed values: niofs, simplefs, mmapfs, memory +# Standard: niofs - Set to memory for best speed but keep in mind that the whole recent index has to fit into the memory of your ElasticSearch machines. Set recent_index_ttl_minutes to a reasonable amount that will let the messages fit into memory. +recent_index_store_type = niofs + +# Always try a reverse DNS lookup instead of parsing hostname from syslog message? +force_syslog_rdns = false +# Set time to NOW if parsing date/time from syslog message failed instead of rejecting it? +allow_override_syslog_date = true + +# Batch size for all outputs. This is the maximum (!) number of messages an output module will get at once. +# For example, if this is set to 5000 (default), the ElasticSearch Output will not index more than 5000 messages +# at once. After that index operation is performed, the next batch will be indexed. If there is only 1 message +# waiting, it will only index that single message. It is important to raise this parameter if you send in so +# many messages that it is not enough to index 5000 messages at once. (Only at *really* high message rates) +output_batch_size = 5000 + +# The number of parallel running processors. +# Raise this number if your buffers are filling up. +processbuffer_processors = 5 +outputbuffer_processors = 5 + +# Wait strategy describing how buffer processors wait on a cursor sequence. (default: sleeping) +# Possible types: +# - yielding +# Compromise between performance and CPU usage. +# - sleeping +# Compromise between performance and CPU usage. Latency spikes can occur after quiet periods. +# - blocking +# High throughput, low latency, higher CPU usage. +# - busy_spinning +# Avoids syscalls which could introduce latency jitter. Best when threads can be bound to specific CPU cores. +processor_wait_strategy = sleeping + +# Size of internal ring buffers. Raise this if raising outputbuffer_processors does not help anymore. +# For optimum performance your LogMessage objects in the ring buffer should fit in your CPU L3 cache. +# Start server with --statistics flag to see buffer utilization. +# Must be a power of 2. (512, 1024, 2048, ...) +ring_size = 1024 + +# MongoDB Configuration +mongodb_useauth = true +mongodb_user = grayloguser +mongodb_password = 123 +mongodb_host = 127.0.0.1 +#mongodb_replica_set = localhost:27017,localhost:27018,localhost:27019 +mongodb_database = graylog2 +mongodb_port = 27017 + +# Raise this according to the maximum connections your MongoDB server can handle if you encounter MongoDB connection problems. +mongodb_max_connections = 100 + +# Number of threads allowed to be blocked by MongoDB connections multiplier. Default: 5 +# If mongodb_max_connections is 100, and mongodb_threads_allowed_to_block_multiplier is 5, then 500 threads can block. More than that and an exception will be thrown. +# http://api.mongodb.org/java/current/com/mongodb/MongoOptions.html#threadsAllowedToBlockForConnectionMultiplier +mongodb_threads_allowed_to_block_multiplier = 5 + +# Graylog Extended Log Format (GELF) +use_gelf = true +gelf_listen_address = 0.0.0.0 +gelf_listen_port = 12201 + +# Drools Rule File (Use to rewrite incoming log messages) +# rules_file = /etc/graylog2.d/rules/graylog2.drl + +# AMQP +amqp_enabled = false +amqp_host = localhost +amqp_port = 5672 +amqp_username = guest +amqp_password = guest +amqp_virtualhost = / + +# HTTP input +# the server will accept PUT requests to /gelf or /gelf/raw +# /gelf can process all standard GELF messages containing the two header bytes +# /gelf/raw can only process uncompressed GELF messages without any header bytes. +# the HTTP server allows keep-alive connections and supports compression. +http_enabled = false +http_listen_address = 0.0.0.0 +http_listen_port = 12202 + +# Email transport +transport_email_enabled = false +transport_email_hostname = mail.example.com +transport_email_port = 587 +transport_email_use_auth = true +transport_email_use_tls = true +transport_email_auth_username = you@example.com +transport_email_auth_password = secret +transport_email_subject_prefix = [graylog2] +transport_email_from_email = graylog2@example.com +transport_email_from_name = Graylog2 + +# Jabber/XMPP transport +transport_jabber_enabled = false +transport_jabber_hostname = jabber.example.com +transport_jabber_port = 5222 +transport_jabber_use_sasl_auth = true +transport_jabber_allow_selfsigned_certs = false +transport_jabber_auth_username = your_user +transport_jabber_auth_password = secret +transport_jabber_message_prefix = [graylog2] + +# Filters +# Enable the filter that tries to extract additional fields from k=v values in the log message? +enable_tokenizer_filter = true + +# Additional modules +# Graphite +#enable_graphite_output = false +#graphite_carbon_host = 127.0.0.1 +#graphite_carbon_tcp_port = 2003 +#graphite_prefix = logs + +# Librato Metrics (http://support.torch.sh/help/kb/graylog2-server/using-librato-metrics-with-graylog2) +#enable_libratometrics_output = false +#enable_libratometrics_system_metrics = false +#libratometrics_api_user = you@example.com +#libratometrics_api_token = abcdefg12345 +#libratometrics_prefix = gl2- +#libratometrics_interval = 60 +#libratometrics_stream_filter = +#libratometrics_host_filter = diff --git a/contrib/distro/commons-daemon-redhat/SOURCES/graylog2.drl b/contrib/distro/commons-daemon-redhat/SOURCES/graylog2.drl new file mode 100644 index 000000000000..3794d3b039a5 --- /dev/null +++ b/contrib/distro/commons-daemon-redhat/SOURCES/graylog2.drl @@ -0,0 +1,26 @@ +# Example Drools file. +#import org.graylog2.messagehandlers.gelf.GELFMessage +# +#rule "Overwrite localhost host" +# when +# m : GELFMessage( host == "localhost" && version == "1.0" ) +# then +# m.setHost( "localhost.example.com" ); +# System.out.println( "[Overwrite localhost rule fired] : " + m.toString() ); +#end +# +#rule "Drop all messages from www1 facility" +# when +# m : GELFMessage( host == "www1" && facility == "graylog2-test" ) +# then +# m.setFilterOut(true); +# System.out.println( "[Drop all messages from www1 facility rule fired] : " + m.toString() ); +#end +# +#rule "Drop UDP and ICMP Traffic from firewall" +# when +# m : GELFMessage( fullMessage matches "(?i).*(ICMP|UDP) Packet(.|\n|\r)*" ) +# then +# m.setFilterOut(true); +# System.out.println("[Drop all syslog ICMP and UDP traffic] : " + m.toString() ); +#end diff --git a/contrib/distro/commons-daemon-redhat/SOURCES/graylog2.init b/contrib/distro/commons-daemon-redhat/SOURCES/graylog2.init new file mode 100755 index 000000000000..03a88439a32b --- /dev/null +++ b/contrib/distro/commons-daemon-redhat/SOURCES/graylog2.init @@ -0,0 +1,150 @@ +#!/bin/bash +# +# graylog2-server - This script starts and stops the graylog2-server daemon +# +# chkconfig: - 85 15 +# description: graylog2 is a syslog implementation that stores logs in Elasticsearch + +### BEGIN INIT INFO +# Provides: graylog2-server +# Required-Start: $network +# Required-Stop: $network +# Default-Stop: 0 1 2 6 +# Short-Description: Start/stop the graylog2-server daemon +# Description: A syslog implementation that stores logs in Elasticsearch +### END INIT INFO + +# Source function library. +. /etc/rc.d/init.d/functions + +# Check networking +. /etc/sysconfig/network +[[ $NETWORKING == no ]] && exit 0 + +NAME=graylog2-server +PID_FILE=${PIDFILE:-/var/run/${NAME}/${NAME}.pid} +LOCK_FILE=${LOCKFILE:-/var/lock/subsys/${NAME}} + +GRAYLOG2_HOME=/opt/graylog2-server +GRAYLOG2_USER=graylog2 +GRAYLOG2_CONFIG=/etc/${NAME}/graylog2.conf +GRAYLOG2_OUT=/var/log/${NAME}/${NAME}.log +LOG4J_CONFIG=file:/etc/${NAME}/log4j.xml + +GRAYLOG2_OPTS="-Xms256m -Xmx256m" +GRAYLOG2_OPTS="$GRAYLOG2_OPTS -Dcom.sun.management.jmxremote.ssl=false" +GRAYLOG2_OPTS="$GRAYLOG2_OPTS -Dcom.sun.management.jmxremote.authenticate=false" +GRAYLOG2_OPTS="$GRAYLOG2_OPTS -Dcom.sun.management.jmxremote.port=12203" + +CLASSPATH="$GRAYLOG2_HOME/$NAME.jar" + +JSVC_EXEC=`which jsvc` +JAVA_HOME=${JAVA_HOME:-/usr/java/default} + +case `echo "testing\c"`,`echo -n testing` in + *c*,-n*) echo_n= echo_c= ;; + *c*,*) echo_n=-n echo_c= ;; + *) echo_n= echo_c='\c' ;; +esac + +start() { + echo -n $"Starting ${NAME}: " + # check if graylog2 is already booted + __pids_var_run "${JAVA_HOME}/bin/java" "$PID_FILE" + RC=$? + if [ -z "$PID_FILE" -a -z "$pid" ]; then + pid="$(__pids_pidof "$1")" + fi + if [ -n "$pid" ]; then + echo_success + echo + return 0 + fi + rm -f $PID_FILE + + $JSVC_EXEC \ + -user $GRAYLOG2_USER \ + -home $JAVA_HOME \ + -server \ + $GRAYLOG2_OPTS \ + -wait 30 \ + -pidfile $PID_FILE \ + -outfile $GRAYLOG2_OUT \ + -errfile '&1' \ + -cp $CLASSPATH \ + -Dlog4j.configuration=$LOG4J_CONFIG \ + org.graylog2.ServerDaemon --configfile ${GRAYLOG2_CONFIG} + exitValue=$? + + if [ $exitValue -eq 0 ]; then + echo_success + else + echo_failure + fi + echo +} + +stop() { + echo -n $"Shutting down ${NAME}: " + + $JSVC_EXEC \ + -stop -pidfile $PID_FILE \ + -user $GRAYLOG2_USER \ + -home $JAVA_HOME \ + org.graylog2.ServerDaemon --configfile ${GRAYLOG2_CONFIG} + if [ $? -eq 0 ]; then + echo_success + else + echo_failure + fi + echo +} + +status() { + __pids_var_run "${JAVA_HOME}/bin/java" "$PID_FILE" + RC=$? + if [ -z "$PID_FILE" -a -z "$pid" ]; then + pid="$(__pids_pidof "$1")" + fi + if [ -n "$pid" ]; then + echo $"${NAME} (pid $pid) is running..." + return 0 + fi + + case "$RC" in + 0) + echo $"${NAME} (pid $pid) is running..." + return 0 + ;; + 1) + echo $"${NAME} dead but pid file exists" + return 1 + ;; + esac + echo $"${NAME} is stopped" + return 2 +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + status + RETVAL=$? + ;; + restart|force-reload) + stop + start + ;; + *) + N=/etc/init.d/${NAME} + echo "Usage: $N {start|stop|restart|force-reload}" >&2 + RETVAL=2 + ;; +esac + +exit 0 diff --git a/contrib/distro/commons-daemon-redhat/SPECS/graylog2-server.spec b/contrib/distro/commons-daemon-redhat/SPECS/graylog2-server.spec new file mode 100644 index 000000000000..16164beb418c --- /dev/null +++ b/contrib/distro/commons-daemon-redhat/SPECS/graylog2-server.spec @@ -0,0 +1,128 @@ +%define _prefix /opt +%define _docdir /usr/share/doc + +Name: graylog2-server +Version: 0.10.0.rc.1 +Release: 2%{?dist} +Summary: Graylog2 Server + +Group: System Environment/Daemons +License: GPLv3 +URL: http://graylog2.org/ +Source0: https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-0.10.0-rc.1.tar.gz +Source1: graylog2.drl +Source2: graylog2.conf +Source3: graylog2.init +Source4: graylog2-elasticsearch.yml +Source5: graylog2-log4j.xml +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-server-%{version}-%{release}-XXXXXX) + +Requires: java-1.6.0-openjdk +Requires: apache-commons-daemon-jsvc + +%description +Graylog2 is an open source syslog implementation that stores your logs in ElasticSearch. It consists of a server written in Java that accepts your syslog messages via TCP or UDP and stores it in the database. The second part is a Ruby on Rails web interface that allows you to view the log messages. + +%prep +%setup -q -n graylog2-server-0.10.0-rc.1 + + +%build +true + +%install +rm -rf $RPM_BUILD_ROOT + +# Base +%{__mkdir} -p %{buildroot}%{_prefix}/%{name} +%{__install} -p -D -m 0644 build_date %{buildroot}%{_prefix}/%{name} +%{__install} -p -D -m 0644 COPYING %{buildroot}%{_prefix}/%{name} +%{__install} -p -D -m 0644 graylog2.conf.example %{buildroot}%{_prefix}/%{name} +%{__install} -p -D -m 0644 graylog2-server.jar %{buildroot}%{_prefix}/%{name} +%{__install} -p -D -m 0644 README.markdown %{buildroot}%{_prefix}/%{name} +%{__mkdir} -p %{buildroot}%{_prefix}/%{name}/bin +%{__install} -p -D -m 754 bin/* %{buildroot}%{_prefix}/%{name}/bin +%{__mkdir} -p %{buildroot}%{_prefix}/%{name}/plugin +%{__mkdir} -p %{buildroot}%{_prefix}/%{name}/plugin/alarm_callbacks +%{__mkdir} -p %{buildroot}%{_prefix}/%{name}/plugin/filters +%{__mkdir} -p %{buildroot}%{_prefix}/%{name}/plugin/initializers +%{__mkdir} -p %{buildroot}%{_prefix}/%{name}/plugin/inputs +%{__mkdir} -p %{buildroot}%{_prefix}/%{name}/plugin/outputs +%{__mkdir} -p %{buildroot}%{_prefix}/%{name}/plugin/transports + +# config +%{__mkdir} -p %{buildroot}%{_sysconfdir}/%{name}/rules +%{__install} -p -D -m 0644 %{SOURCE2} %{buildroot}%{_sysconfdir}/%{name}/graylog2.conf +%{__install} -p -D -m 0644 %{SOURCE4} %{buildroot}%{_sysconfdir}/%{name}/elasticsearch.yml +%{__install} -p -D -m 0644 %{SOURCE5} %{buildroot}%{_sysconfdir}/%{name}/log4j.xml +%{__install} -p -D -m 0644 %{SOURCE1} %{buildroot}%{_sysconfdir}/%{name}/rules/graylog2.drl + +# logs +%{__mkdir} -p %{buildroot}%{_localstatedir}/log/%{name} + +# sysconfig and init +%{__mkdir} -p %{buildroot}%{_sysconfdir}/rc.d/init.d +%{__install} -m 755 %{SOURCE3} %{buildroot}%{_sysconfdir}/rc.d/init.d/%{name} + +%{__mkdir} -p %{buildroot}%{_localstatedir}/run/%{name} +%{__mkdir} -p %{buildroot}%{_localstatedir}/lock/subsys/%{name} + +%pre +# create graylog2 group +if ! getent group graylog2 >/dev/null; then + /usr/sbin/groupadd -r graylog2 +fi + +# create graylog2 user +if ! getent passwd graylog2 >/dev/null; then + /usr/sbin/useradd -r -g graylog2 -d %{_prefix}/%{name} \ + -s /sbin/nologin -c "log aggregator" graylog2 +fi + +%post +/sbin/chkconfig --add %{name} + + +%preun +if [[ $1 -ge 1 ]] +then + /sbin/service %{name} stop > /dev/null 2>&1 + /sbin/chkconfig --del %{name} +fi + + +%clean +rm -rf $RPM_BUILD_ROOT + + +%files +%defattr(-,root,root,-) +%{_sysconfdir}/rc.d/init.d/%{name} +%dir %{_sysconfdir}/%{name} +%dir %{_sysconfdir}/%{name}/rules +%config(noreplace) %{_sysconfdir}/%{name}/*.conf +%config(noreplace) %{_sysconfdir}/%{name}/*.yml +%config(noreplace) %{_sysconfdir}/%{name}/*.xml +%config(noreplace) %{_sysconfdir}/%{name}/rules/*.drl + +%defattr(-,graylog2,graylog2,-) +%{_prefix}/%{name} +%{_prefix}/%{name}/bin +%{_prefix}/%{name}/plugin +%{_prefix}/%{name}/plugin/alarm_callbacks +%{_prefix}/%{name}/plugin/filters +%{_prefix}/%{name}/plugin/initializers +%{_prefix}/%{name}/plugin/inputs +%{_prefix}/%{name}/plugin/outputs +%{_prefix}/%{name}/plugin/transports +%dir %{_localstatedir}/run/%{name} +%dir %{_localstatedir}/log/%{name} + + +%changelog +* Wed Dec 26 2012 kbrockhoff@codekaizen.org 0.10.0-rc.1 +- Update to latest graylog2 release + +* Wed Oct 24 2012 kbrockhoff@codekaizen.org 0.9.6p1-2 +- Initial definition + From 9e3060dd3580d2e027d8c91e263efb29401b72bb Mon Sep 17 00:00:00 2001 From: kbrockhoff Date: Thu, 21 Feb 2013 12:41:22 -0600 Subject: [PATCH 19/19] Added handling for AlreadyColsedException and fixed executor cleanup --- .../graylog2/inputs/amqp/AMQPConsumer.java | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/graylog2/inputs/amqp/AMQPConsumer.java b/src/main/java/org/graylog2/inputs/amqp/AMQPConsumer.java index 9155e3c151f3..52ed17382915 100644 --- a/src/main/java/org/graylog2/inputs/amqp/AMQPConsumer.java +++ b/src/main/java/org/graylog2/inputs/amqp/AMQPConsumer.java @@ -20,6 +20,7 @@ package org.graylog2.inputs.amqp; import com.rabbitmq.client.AMQP; +import com.rabbitmq.client.AlreadyClosedException; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; @@ -31,6 +32,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -58,6 +60,7 @@ public class AMQPConsumer implements Runnable { Connection connection; Channel channel; + ExecutorService executor; private final Meter handledMessages = Metrics.newMeter(AMQPConsumer.class, "HandledAMQPMessages", "messages", TimeUnit.SECONDS); private final Meter handledSyslogMessages = Metrics.newMeter(AMQPConsumer.class, "HandledAMQPSyslogMessages", "messages", TimeUnit.SECONDS); @@ -139,9 +142,24 @@ public void disconnect() { AMQPInput.getConsumers().remove(queueConfig.getId()); channel.close(); - connection.close(); + } catch (AlreadyClosedException ignore) { + // do nothing } catch(IOException e) { LOG.error("Could not disconnect from AMQP broker!", e); + } finally { + if (executor != null) { + executor.shutdownNow(); + executor = null; + } + try { + if (connection != null && connection.isOpen()) { + connection.close(); + } + } catch (AlreadyClosedException ignore) { + // do nothing + } catch (IOException e) { + LOG.error("Could not disconnect from AMQP broker!", e); + } } } @@ -159,11 +177,11 @@ private Channel connect() throws IOException { factory.setHost(server.getConfiguration().getAmqpHost()); factory.setPort(server.getConfiguration().getAmqpPort()); - connection = factory.newConnection(Executors.newCachedThreadPool( - new ThreadFactoryBuilder() - .setNameFormat("amqp-consumer-" + queueConfig.getId() + "-%d") - .build() - )); + executor = Executors.newCachedThreadPool( + new ThreadFactoryBuilder() + .setNameFormat("amqp-consumer-" + queueConfig.getId() + "-%d") + .build()); + connection = factory.newConnection(executor); return connection.createChannel(); }