Generalize the trace rewrite utility

Author: ben-manes

config/spotbugs/exclude.xml

@@ -151,7 +151,7 @@
     <Bug pattern="RV_RETURN_VALUE_IGNORED"/>
   </Match>
   <Match>
-    <Class name="com.github.benmanes.caffeine.cache.simulator.parser.rewrite.Rewriter"/>
+    <Class name="com.github.benmanes.caffeine.cache.simulator.parser.Rewriter"/>
     <Method name="run"/>
     <Bug code="RCN"/>
   </Match>

gradle/codeQuality.gradle

@@ -180,6 +180,9 @@ afterEvaluate {
   tasks.findAll { it.name.startsWith('spotbugs') }.each {
     it.enabled = System.properties.containsKey('spotbugs')
     it.group = 'SpotBugs'
+    it.reports {
+      html { enabled = true }
+    }
   }
 }
 

gradle/dependencies.gradle

@@ -48,12 +48,12 @@ ext {
     javaObjectLayout: '0.10',
     javapoet: '1.12.1',
     jcache: '1.1.1',
-    jcommander: '1.78',
     jmh: '1.23',
     jsr330: '1',
     koloboke: '1.0.0',
     nullaway: '0.7.9',
     ohc: '0.6.1',
+    picocli: '4.3.2',
     slf4j: '1.7.30',
     tcache: '2.0.1',
     stream: '2.9.8',
@@ -123,14 +123,14 @@ ext {
     javaObjectLayout: "org.openjdk.jol:jol-cli:${versions.javaObjectLayout}",
     javapoet: "com.squareup:javapoet:${versions.javapoet}",
     jcache: "javax.cache:cache-api:${versions.jcache}",
-    jcommander: "com.beust:jcommander:${versions.jcommander}",
     jsr330: "javax.inject:javax.inject:${versions.jsr330}",
     koloboke: [
       "com.koloboke:koloboke-api-jdk8:${versions.koloboke}",
       "com.koloboke:koloboke-impl-jdk8:${versions.koloboke}",
     ],
     nullaway: "com.uber.nullaway:nullaway:${versions.nullaway}",
     ohc: "org.caffinitas.ohc:ohc-core-j8:${versions.ohc}",
+    picocli: "info.picocli:picocli:${versions.picocli}",
     slf4jNop: "org.slf4j:slf4j-nop:${versions.slf4j}",
     tcache: "com.trivago:triava:${versions.tcache}",
     stream: "com.clearspring.analytics:stream:${versions.stream}",

simulator/build.gradle

@@ -16,13 +16,13 @@ dependencies {
   implementation libraries.stream
   implementation libraries.tcache
   implementation libraries.cache2k
+  implementation libraries.picocli
   implementation libraries.ehcache3
   implementation libraries.fastutil
   implementation libraries.slf4jNop
   implementation libraries.collision
   implementation libraries.fastfilter
   implementation libraries.flipTables
-  implementation libraries.jcommander
   implementation libraries.expiringMap
   implementation libraries.commonsLang3
   implementation libraries.commonsMath3
@@ -80,10 +80,10 @@ run {
 }
 
 task rewrite(type: JavaExec) {
-  main = 'com.github.benmanes.caffeine.cache.simulator.parser.rewrite.Rewriter'
+  main = 'com.github.benmanes.caffeine.cache.simulator.parser.Rewriter'
   classpath = sourceSets.main.runtimeClasspath
 
-  def arguments = ['inputFiles', 'outputFile', 'outputFormat']
+  def arguments = ['inputFormat', 'inputFiles', 'outputFile', 'outputFormat']
   for (def argument : arguments) {
     if (project.hasProperty(argument)) {
       args "--${argument}", project.property(argument)

simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/OutputFormat.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2019 Ben Manes. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.github.benmanes.caffeine.cache.simulator.parser;
+
+import java.io.BufferedOutputStream;
+import java.io.OutputStream;
+import java.util.function.Function;
+
+import com.github.benmanes.caffeine.cache.simulator.parser.adapt_size.AdaptSizeTraceWriter;
+import com.github.benmanes.caffeine.cache.simulator.parser.climb.ClimbTraceWriter;
+
+/**
+ * The trace output format.
+ *
+ * @author ben.manes@gmail.com (Ben Manes)
+ */
+@SuppressWarnings("ImmutableEnumChecker")
+public enum OutputFormat {
+  ADAPT_SIZE(AdaptSizeTraceWriter::new),
+  CLIMB(ClimbTraceWriter::new);
+
+  private final Function<OutputStream, TraceWriter> factory;
+
+  OutputFormat(Function<OutputStream, TraceWriter> factory) {
+    this.factory = factory;
+  }
+
+  public TraceWriter writer(OutputStream output) {
+    return factory.apply(new BufferedOutputStream(output));
+  }
+}

simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/rewrite/Rewriter.java simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/Rewriter.java

@@ -13,75 +13,79 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.github.benmanes.caffeine.cache.simulator.parser.rewrite;
+package com.github.benmanes.caffeine.cache.simulator.parser;
 
-import java.io.BufferedWriter;
+import java.io.BufferedOutputStream;
 import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.stream.Stream;
 
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.github.benmanes.caffeine.cache.simulator.parser.TraceFormat;
 import com.github.benmanes.caffeine.cache.simulator.policy.AccessEvent;
 import com.google.common.base.Stopwatch;
 
+import picocli.CommandLine;
+import picocli.CommandLine.Help;
+import picocli.CommandLine.Option;
+
 /**
  * A simple utility to rewrite traces for into the format used by other simulators. This lets us
  * run multiple simulators in parallel for a quick-and-dirty analysis, rather than port their code
  * into Java.
  * <p>
  * <pre>{@code
- *   ./gradlew :simulator:rewrite -PinputFiles=? -PoutputFile=? -PoutputFormat=?
+ *   ./gradlew :simulator:rewrite \
+ *      -PinputFormat=? \
+ *      -PinputFiles=? \
+ *      -PoutputFile=? \
+ *      -PoutputFormat=?
  * }</pre>
  *
  * @author ben.manes@gmail.com (Ben Manes)
  */
 @SuppressWarnings("PMD.ImmutableField")
-public final class Rewriter {
-  @Parameter(names = "--inputFiles", required = true, description = "The trace input files. To use "
+public final class Rewriter implements Runnable {
+  @Option(names = "--inputFiles", required = true, description = "The trace input files. To use "
       + "a mix of formats, specify the entry as format:path, e.g. lirs:loop.trace.gz")
   private List<String> inputFiles = new ArrayList<>();
-  @Parameter(names = "--inputFormat", description = "The default trace input format")
-  private TraceFormat inputFormat = TraceFormat.LIRS;
+  @Option(names = "--inputFormat", required = true, description = "The default trace input format")
+  private TraceFormat inputFormat;
 
-  @Parameter(names = "--outputFile", description = "The trace output file", required = true)
+  @Option(names = "--outputFile", required = true, description = "The trace output file")
   private Path outputFile;
-  @Parameter(names = "--outputFormat", description = "The trace output format", required = true)
+  @Option(names = "--outputFormat", required = true, description = "The trace output format")
   private OutputFormat outputFormat;
 
-  @Parameter(names = "--help", help = true, hidden = true)
-  private boolean help;
-
+  @Override
   @SuppressWarnings("PMD.ForLoopCanBeForeach")
-  public void run() throws IOException {
-    long count = 0;
+  public void run() {
+    int tick = 0;
     Stopwatch stopwatch = Stopwatch.createStarted();
-    try (Stream<AccessEvent> events = inputFormat.readFiles(inputFiles).events();
-         BufferedWriter writer = Files.newBufferedWriter(outputFile)) {
+    try (OutputStream output = new BufferedOutputStream(Files.newOutputStream(outputFile));
+         Stream<AccessEvent> events = inputFormat.readFiles(inputFiles).events();
+         TraceWriter writer = outputFormat.writer(output)) {
+      writer.writeHeader();
       for (Iterator<AccessEvent> i = events.iterator(); i.hasNext();) {
-        outputFormat.write(writer, i.next(), count);
-        count++;
+        writer.writeEvent(tick, i.next());
+        tick++;
       }
+      writer.writeFooter();
+      System.out.printf("Rewrote %,d events in %s%n", tick, stopwatch);
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
     }
-    System.out.printf("Rewrote %,d events in %s%n", count, stopwatch);
   }
 
   public static void main(String[] args) throws IOException {
-    Rewriter rewriter = new Rewriter();
-    JCommander commander =  JCommander.newBuilder()
-        .programName(Rewriter.class.getSimpleName())
-        .addObject(rewriter)
-        .build();
-    commander.parse(args);
-    if (rewriter.help) {
-      commander.usage();
-    } else {
-      rewriter.run();
-    }
+    new CommandLine(Rewriter.class)
+        .setColorScheme(Help.defaultColorScheme(Help.Ansi.ON))
+        .setCommandName(Rewriter.class.getSimpleName())
+        .setCaseInsensitiveEnumValuesAllowed(true)
+        .execute(args);
   }
 }

simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/TextTraceWriter.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2020 Ben Manes. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.github.benmanes.caffeine.cache.simulator.parser;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+/**
+ * A skeletal implementation that writes to the trace file as textual data.
+ *
+ * @author ben.manes@gmail.com (Ben Manes)
+ */
+public abstract class TextTraceWriter implements TraceWriter {
+  private final BufferedWriter writer;
+
+  public TextTraceWriter(OutputStream output) {
+    this.writer = new BufferedWriter(new OutputStreamWriter(output, UTF_8));
+  }
+
+  protected BufferedWriter writer() {
+    return writer;
+  }
+
+  @Override
+  public void close() throws IOException {
+    writer.close();
+  }
+}

simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/TraceFormat.java

@@ -24,7 +24,7 @@
 import java.util.function.Function;
 import java.util.stream.Stream;
 
-import com.github.benmanes.caffeine.cache.simulator.parser.adapt_size.AdaptSizeReader;
+import com.github.benmanes.caffeine.cache.simulator.parser.adapt_size.AdaptSizeTraceReader;
 import com.github.benmanes.caffeine.cache.simulator.parser.address.AddressTraceReader;
 import com.github.benmanes.caffeine.cache.simulator.parser.address.penalties.AddressPenaltiesTraceReader;
 import com.github.benmanes.caffeine.cache.simulator.parser.arc.ArcTraceReader;
@@ -54,7 +54,7 @@
 public enum TraceFormat {
   ADDRESS(AddressTraceReader::new),
   ADDRESS_PENALTIES(AddressPenaltiesTraceReader::new),
-  ADAPT_SIZE(AdaptSizeReader::new),
+  ADAPT_SIZE(AdaptSizeTraceReader::new),
   ARC(ArcTraceReader::new),
   CACHE2K(Cache2kTraceReader::new),
   CAMELAB(CamelabTraceReader::new),

simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/TraceWriter.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2020 Ben Manes. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.github.benmanes.caffeine.cache.simulator.parser;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import com.github.benmanes.caffeine.cache.simulator.policy.AccessEvent;
+
+/**
+ * A writer to output to an access trace format.
+ *
+ * @author ben.manes@gmail.com (Ben Manes)
+ */
+public interface TraceWriter extends Closeable {
+
+  /** Writes the header for the trace format. */
+  default void writeHeader() throws IOException {};
+
+  /** Writes the event in the trace format. */
+  void writeEvent(int tick, AccessEvent event) throws IOException;
+
+  /** Writes the footer for the trace format. */
+  default void writeFooter() throws IOException {};
+}

simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/adapt_size/AdaptSizeReader.java simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/adapt_size/AdaptSizeTraceReader.java

@@ -32,9 +32,9 @@
  *
  * @author ben.manes@gmail.com (Ben Manes)
  */
-public final class AdaptSizeReader extends TextTraceReader {
+public final class AdaptSizeTraceReader extends TextTraceReader {
 
-  public AdaptSizeReader(String filePath) {
+  public AdaptSizeTraceReader(String filePath) {
     super(filePath);
   }
 

simulator/src/main/java/com/github/benmanes/caffeine/cache/simulator/parser/adapt_size/AdaptSizeTraceWriter.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020 Ben Manes. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.github.benmanes.caffeine.cache.simulator.parser.adapt_size;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import com.github.benmanes.caffeine.cache.simulator.parser.TextTraceWriter;
+import com.github.benmanes.caffeine.cache.simulator.policy.AccessEvent;
+
+/**
+ * A writer for the trace format used by the authors of the AdaptSize simulator.
+ *
+ * @author ben.manes@gmail.com (Ben Manes)
+ */
+public final class AdaptSizeTraceWriter extends TextTraceWriter {
+
+  public AdaptSizeTraceWriter(OutputStream output) {
+    super(output);
+  }
+
+  @Override
+  public void writeEvent(int tick, AccessEvent event) throws IOException {
+    writer().write(Long.toString(tick));
+    writer().write(" ");
+    writer().write(Long.toString(event.key()));
+    writer().write(" ");
+    writer().write(Integer.toString(event.weight()));
+    writer().newLine();
+  }
+}