Improve test stability and pass rate for distributed tests.

   - configure travis to run separate jobs for multi-concurrency ReaperCassandraIT executions.
   - reduce ccm count and memory use
   - increase ccm request timeouts (tests can run very slowly)
   - after a failure display ccm errors
   - upgrade to dropwizard 1.0.8
   - upgrade jerysey-client to 2.25.1 due to a bug where requests can be sent twice (still a problem but happens less)
   - use `allow_failures` on the longer "smoke" tests

  ref: #124

.travis.yml

@@ -1,15 +1,33 @@
+sudo: required
 language: java
 jdk:
 - oraclejdk8
-install: true
 env:
   global:
     - LOCAL_JMX=no
+    - TEST=""
   matrix:
-    - CASSANDRA_VERSION=2.1.16
-    - CASSANDRA_VERSION=2.2.8
-    - CASSANDRA_VERSION=3.0.9
-    - CASSANDRA_VERSION=3.10
+      # 2.1.18 runs all tests together because it's also for sonar reporting, see below
+    - CASSANDRA_VERSION=2.1.18
+    - CASSANDRA_VERSION=2.1.18 GRIM_MIN=2 GRIM_MAX=2
+    - CASSANDRA_VERSION=2.1.18 GRIM_MIN=2 GRIM_MAX=4
+    - CASSANDRA_VERSION=2.2.10 GRIM_MIN=1 GRIM_MAX=1
+    - CASSANDRA_VERSION=2.2.10 GRIM_MIN=2 GRIM_MAX=2 
+    - CASSANDRA_VERSION=2.2.10 GRIM_MIN=2 GRIM_MAX=4
+    - CASSANDRA_VERSION=3.0.10 GRIM_MIN=1 GRIM_MAX=1
+    - CASSANDRA_VERSION=3.0.14 GRIM_MIN=2 GRIM_MAX=2
+    - CASSANDRA_VERSION=3.0.14 GRIM_MIN=2 GRIM_MAX=4
+    - CASSANDRA_VERSION=3.11.0 GRIM_MIN=1 GRIM_MAX=1
+    - CASSANDRA_VERSION=3.11.0 GRIM_MIN=2 GRIM_MAX=2
+    - CASSANDRA_VERSION=3.11.0 GRIM_MIN=2 GRIM_MAX=4
+matrix:
+  fast_finish: true
+  allow_failures:
+  - env: CASSANDRA_VERSION=2.2.10 GRIM_MIN=2 GRIM_MAX=2
+  - env: CASSANDRA_VERSION=2.2.10 GRIM_MIN=2 GRIM_MAX=4
+  - env: CASSANDRA_VERSION=3.0.14 GRIM_MIN=2 GRIM_MAX=2
+  - env: CASSANDRA_VERSION=3.0.14 GRIM_MIN=2 GRIM_MAX=4
+  - env: CASSANDRA_VERSION=3.11.0 GRIM_MIN=2 GRIM_MAX=4
 services:
   - postgresql
 before_script:
@@ -23,19 +41,37 @@ before_install:
 
 install:
   - ccm create test -v $CASSANDRA_VERSION > /dev/null
-  - ccm populate -n 3 > /dev/null
+  - ccm populate --vnodes -n 2 > /dev/null
   - sed -i 's/jmxremote.authenticate=true/jmxremote.authenticate=false/' /home/travis/.ccm/test/node1/conf/cassandra-env.sh
   - sed -i 's/jmxremote.authenticate=true/jmxremote.authenticate=false/' /home/travis/.ccm/test/node2/conf/cassandra-env.sh
-  - sed -i 's/jmxremote.authenticate=true/jmxremote.authenticate=false/' /home/travis/.ccm/test/node3/conf/cassandra-env.sh
-  - ccm start > /dev/null
+  - sed -i 's/#MAX_HEAP_SIZE="4G"/MAX_HEAP_SIZE="256m"/' /home/travis/.ccm/test/node1/conf/cassandra-env.sh
+  - sed -i 's/#MAX_HEAP_SIZE="4G"/MAX_HEAP_SIZE="256m"/' /home/travis/.ccm/test/node2/conf/cassandra-env.sh
+  - "sudo sed -i 's/_timeout_in_ms:.*/_timeout_in_ms: 60000/' /home/travis/.ccm/test/node1/conf/cassandra.yaml"
+  - "sudo sed -i 's/_timeout_in_ms:.*/_timeout_in_ms: 60000/' /home/travis/.ccm/test/node2/conf/cassandra.yaml"
+  - "sudo sed -i 's/start_rpc: true/start_rpc: false/' /home/travis/.ccm/test/node1/conf/cassandra.yaml"
+  - "sudo sed -i 's/start_rpc: true/start_rpc: false/' /home/travis/.ccm/test/node2/conf/cassandra.yaml"
+  - "sudo sed -i 's/cross_node_timeout: false/cross_node_timeout: true/' /home/travis/.ccm/test/node1/conf/cassandra.yaml"
+  - "sudo sed -i 's/cross_node_timeout: false/cross_node_timeout: true/' /home/travis/.ccm/test/node2/conf/cassandra.yaml"
+  - "sudo sed -i 's/concurrent_reads: 32/concurrent_reads: 2/' /home/travis/.ccm/test/node1/conf/cassandra.yaml"
+  - "sudo sed -i 's/concurrent_reads: 32/concurrent_reads: 2/' /home/travis/.ccm/test/node2/conf/cassandra.yaml"
+  - "sudo sed -i 's/concurrent_writes: 32/concurrent_writes: 2/' /home/travis/.ccm/test/node1/conf/cassandra.yaml"
+  - "sudo sed -i 's/concurrent_writes: 32/concurrent_writes: 2/' /home/travis/.ccm/test/node2/conf/cassandra.yaml"
+  - "sudo sed -i 's/concurrent_counter_writes: 32/concurrent_counter_writes: 2/' /home/travis/.ccm/test/node1/conf/cassandra.yaml"
+  - "sudo sed -i 's/concurrent_counter_writes: 32/concurrent_counter_writes: 2/' /home/travis/.ccm/test/node2/conf/cassandra.yaml"
+  - "sudo sed -i 's/num_tokens: 256/num_tokens: 32/' /home/travis/.ccm/test/node1/conf/cassandra.yaml"
+  - "sudo sed -i 's/num_tokens: 256/num_tokens: 32/' /home/travis/.ccm/test/node2/conf/cassandra.yaml"
+  - ccm start -v
 
 #before_script:
 #  - npm install -g bower
 script:
-  - mvn clean package
-  - mvn surefire:test -Pintegration-tests
+  - ccm status
+  - ccm node1 nodetool status
+  - 'if [ "x$GRIM_MIN" = "x" ]; then mvn clean package && mvn surefire:test -Dtest=ReaperIT && mvn surefire:test -Dtest=ReaperH2IT && mvn surefire:test -Dtest=ReaperPostgresIT && mvn surefire:test -Dtest=ReaperCassandraIT  ; else  mvn clean package && mvn surefire:test -Dtest=ReaperCassandraIT -Dgrim.reaper.min=$GRIM_MIN -Dgrim.reaper.max=$GRIM_MAX ; fi'
 after_success:
-  - 'if [ "$TRAVIS_PULL_REQUEST" = "false" -a "$TRAVIS_BRANCH" = "master" -a "$CASSANDRA_VERSION" = "2.1.16" ]; then mvn sonar:sonar -Dsonar.host.url=https://sonarqube.com -Dsonar.login=$SONAR_TOKEN -Dsonar.projectKey=tlp-cassandra-reaper -Dsonar.github.oauth=$GITHUB_TOKEN -Dsonar.github.repository=thelastpickle/cassandra-reaper; fi'
+  - 'if [ "$TRAVIS_PULL_REQUEST" = "false" -a "$TRAVIS_BRANCH" = "master" -a "$CASSANDRA_VERSION" = "2.1.18" ]; then mvn sonar:sonar -Dsonar.host.url=https://sonarqube.com -Dsonar.login=$SONAR_TOKEN -Dsonar.projectKey=tlp-cassandra-reaper -Dsonar.github.oauth=$GITHUB_TOKEN -Dsonar.github.repository=thelastpickle/cassandra-reaper; fi'
+after_failure:
+  - ccm checklogerror
 before_deploy: 
   - "mkdir cassandra-reaper-${TRAVIS_TAG}"
   - "mkdir cassandra-reaper-${TRAVIS_TAG}/target"
@@ -47,8 +83,7 @@ deploy:
   provider: releases
   api_key: $GITHUB_TOKEN
   file_glob: true
-  file: 
-    - "cassandra-reaper-${TRAVIS_TAG}-release.tar.gz"
+  file: "cassandra-reaper-${TRAVIS_TAG}-release.tar.gz"
   skip_cleanup: true
   on:
     tags: true 

pom.xml

@@ -11,7 +11,7 @@
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <dropwizard.version>1.0.7</dropwizard.version>
+        <dropwizard.version>1.0.8</dropwizard.version>
         <dropwizard.cassandra.version>4.1.0</dropwizard.cassandra.version>
         <cassandra.version>2.2.7</cassandra.version>
         <cucumber.version>1.2.5</cucumber.version>
@@ -57,7 +57,7 @@
         <dependency>
             <groupId>io.dropwizard</groupId>
             <artifactId>dropwizard-assets</artifactId>
-            <version>1.0.0</version>
+            <version>${dropwizard.version}</version>
         </dependency>
         <dependency>
             <groupId>systems.composable</groupId>
@@ -164,6 +164,14 @@
             <version>2.0M10</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <!-- fixes bug around duplicate requests
+              - https://stackoverflow.com/questions/37956741/jersey-resource-receiving-duplicate-requests-from-jersey-client#39377403
+             -->
+            <groupId>org.glassfish.jersey.core</groupId>
+            <artifactId>jersey-client</artifactId>
+            <version>2.25.1</version>
+        </dependency>
     </dependencies>
 
     <profiles>

src/test/java/com/spotify/reaper/acceptance/BasicSteps.java

@@ -946,7 +946,7 @@ private static void createKeyspace(String keyspaceName) {
       try (Cluster cluster = buildCluster(); Session tmpSession = cluster.connect()) {
           tmpSession.execute(
                   "CREATE KEYSPACE IF NOT EXISTS " + keyspaceName
-                          + " WITH replication={'class':'SimpleStrategy', 'replication_factor':3}");
+                          + " WITH replication={'class':'SimpleStrategy', 'replication_factor':2}");
       }
   }
 

src/test/java/com/spotify/reaper/acceptance/ReaperCassandraIT.java

@@ -106,7 +106,7 @@ public static void initSchema() throws IOException {
               }
           });
           tmpSession.execute(
-                  "CREATE KEYSPACE reaper_db WITH replication = {'class':'SimpleStrategy', 'replication_factor':3}");
+                  "CREATE KEYSPACE reaper_db WITH replication = {'class':'SimpleStrategy', 'replication_factor':2}");
       }
   }
 

src/test/resources/cassandra-reaper-cassandra-at.yaml

@@ -11,11 +11,7 @@ incrementalRepair: false
 jmxConnectionTimeoutInSeconds: 300
 
 logging:
-  level: DEBUG
-  loggers:
-    io.dropwizard: INFO
-    org.eclipse.jetty: INFO
-    com.datastax.driver.core.QueryLogger.NORMAL: INFO
+  level: WARN
   appenders:
     - type: console
 
@@ -47,3 +43,6 @@ cassandra:
   clusterName: "test"
   contactPoints: ["127.0.0.1"]
   keyspace: reaper_db
+  socketOptions:
+      connectTimeoutMillis: 20000
+      readTimeoutMillis: 40000