0.18.0 - Update jfile-io@0.8.3 (fix loading empty stream/file), new '-threads #' and '-debug' command line arguments, additional debugging/performance information.

Author: TeamworkGuy2

CHANGELOG.md

@@ -4,7 +4,25 @@ This project does its best to adhere to [Semantic Versioning](http://semver.org/
 
 
 --------
-### [0.17.0](N/A) - 2019-03-30
+### [0.18.0](N/A) - 2019-03-30
+#### Changed
+* Added `-debug` and `-threads #` command line arguments
+* More detailed debug and log file information
+* Renamed `TokenizeStepLogger` -> `ParserActionLogger`
+* `ParseTimes.log()` renamed `setActionTime()`
+* Simplified and synchronized `PerformanceTrackers` so instance can be shared across threads
+* Renamed node.js plugin file `plugins/node-js/jparser-tools-cli` -> `plugins/node-js/jparse-code-cli` and renamed associated test file
+
+#### Removed
+* Unused `ScopeType` enum
+
+#### Fixed
+* Fixed multi-threaded parsing! Handling of FileReadUtil, performance logs, and result lists are now synchronized in `ParserMisc.parseFileSet()`
+* Update dependency jfile-io@0.8.3 (fix for decoding empty streams/files)
+
+
+--------
+### [0.17.0](https://github.com/TeamworkGuy2/JParseCode/commit/7745de03fb4ce135ddb8e6a7f158f6d1f27c5329) - 2019-03-30
 Performance refactor, several libraries updated: JArrays, JCollectionUtil, JFileIo, JTextParser, and JTextTokenizer
 #### Changed
 * Changed to new `FileReadUtil.readChars(InputStream)` (`jfile-io@0.8.2`)

README.md

@@ -179,20 +179,22 @@ JSON Result (printed to System.out):
 
 --------
 
-### Command Line Interface (CLI)
+## Command Line Interface (CLI)
 
 A command line call looks like:
 ```
 path/to/java -jar path/to/jparse-code.jar 
  -sources './src/java/Server/Services=1,[cs];./src/java/Server/Models=3,[cs]'
  -destinations './output/Services.json=[App.Services];./output/Models.json=[App.Entities]'
  -log './output/parser.log'
+ -threads 1
+ -debug
 ```
-Where ./src/java/Server/** is where source files are kept
-And the files in ./src/java/Server/Services belong to the C# namespace `App.Services` and ./src/java/Server/Models/ belong to the C# namespace `App.Entities`
+Where `./src/java/Server/**` is where source files are kept
+And the files in `./src/java/Server/Services` belong to the C# namespace `App.Services` and `./src/java/Server/Models` belong to the C# namespace `App.Entities`
 
 
-#### Sources
+### Sources
 A semicolon separated list of paths set equal to a directory depth followed by a comma and a comma separated, brackets wrapped, list of file extensions. 
 The path, child directory depth, and file extensions are used to create a file system filter and all matching files are parsed.
 The following formats are valid:
@@ -204,21 +206,29 @@ Example: ```/project/myApp/Models=3,[java,json]```
 Note: the brackets around the '[java,json]' file extension list are literal.
 
 
-#### Destinations
+### Destinations
 A semicolon separated list of output file names associated with lists of namespaces.  Each parsed file who's namespace falls into one of these lists is written to that file. 
 The following format is valid:
 'path=[namespace,namespace,...]'
 
 Example: ```/project/output/models.json=[MyApp.Models]```
 
 
-#### Log
+### Log
 An optional log file name to write parser information to, in the format:
 'path'
 
 Example: ```/project/output/parser-log.log```
 
 
+### Threads
+An optional number of threads to run parsing in parallel, 0 uses the logical number of processors on the current machine, default is 1
+
+
+### Debug
+An optional flag which causes extra debug and performance information to be logged
+
+
 --------
-Plugins:
+### Plugins:
 Currently there is one plugin, a dual purpose TypeScript/Javascript plugin for create the CLI argument strings used by jparse-code.jar

bin/jparse_code-with-tests.jar

@@ -1,5 +1,5 @@
 {
-	"version" : "0.17.0",
+	"version" : "0.18.0",
 	"name" : "jparse-code",
 	"description" : "An in-progress suite of parsing/transpilation tools for C#, Java, and TypeScript code.  Generates simple JSON ASTs.",
 	"homepage" : "https://github.com/TeamworkGuy2/JParseCode",

bin/jparse_code.jar

@@ -1,13 +1,13 @@
 "use strict";
-var JParserToolsCli = require("./jparser-tools-cli");
+var JParseCodeCli = require("./jparse-code-cli");
 /** Simple test for jparser-tools-cli
  * @author TeamworkGuy2
  * @since 2016-06-21
  */
-var JParserToolsCliTest;
-(function (JParserToolsCliTest) {
+var JParseCodeCliTest;
+(function (JParseCodeCliTest) {
     function testJsCli() {
-        var opts = JParserToolsCli.createOptions({
+        var opts = JParseCodeCli.createOptions({
             sources: [
                 { path: "src/services", depth: 1, fileExtensions: ["java"] },
                 { path: "src/models", depth: 2, fileExtensions: ["java"] },
@@ -16,23 +16,27 @@ var JParserToolsCliTest;
                 { path: "../models/service-defs.json", namespaces: ["App.Services"] },
                 { path: "../models/model-defs.json", namespaces: ["App.Models", "App.Core"] },
             ],
-            log: "../output/jparser-tools.log"
+            log: "../output/jparser-tools.log",
+            threads: 5,
+            debug: true
         });
-        var optsStr = JParserToolsCli.stringifyOptions(opts, '"');
+        var optsStr = JParseCodeCli.stringifyOptions(opts, '"');
         var expStr = ' -sources "' + "src/services=1,[java]" + ";" + "src/models=2,[java]" + '"' +
             ' -destinations "' + "../models/service-defs.json=[App.Services]" + ";" + "../models/model-defs.json=[App.Models,App.Core]" + '"' +
-            ' -log "' + "../output/jparser-tools.log" + '"';
+            ' -log "' + "../output/jparser-tools.log" + '"' +
+            ' -threads ' + 5 +
+            ' -debug';
         if (expStr !== optsStr) {
             throw new Error("options string mismatch, expected: \n'" + expStr + "'\nactual: \n'" + optsStr + "'");
         }
         else {
             console.log("success 'testJsCli()'");
         }
     }
-    JParserToolsCliTest.testJsCli = testJsCli;
-})(JParserToolsCliTest || (JParserToolsCliTest = {}));
+    JParseCodeCliTest.testJsCli = testJsCli;
+})(JParseCodeCliTest || (JParseCodeCliTest = {}));
 function main() {
-    JParserToolsCliTest.testJsCli();
+    JParseCodeCliTest.testJsCli();
 }
 main();
-module.exports = JParserToolsCliTest;
+module.exports = JParseCodeCliTest;

package-lib.json

@@ -1,13 +1,13 @@
-import JParserToolsCli = require("./jparser-tools-cli");
+import JParseCodeCli = require("./jparse-code-cli");
 
 /** Simple test for jparser-tools-cli
  * @author TeamworkGuy2
  * @since 2016-06-21
  */
-module JParserToolsCliTest {
+module JParseCodeCliTest {
 
     export function testJsCli() {
-        var opts = JParserToolsCli.createOptions({
+        var opts = JParseCodeCli.createOptions({
             sources: [
                 { path: "src/services", depth: 1, fileExtensions: ["java"] },
                 { path: "src/models", depth: 2, fileExtensions: ["java"] },
@@ -17,13 +17,17 @@ module JParserToolsCliTest {
                 { path: "../models/model-defs.json", namespaces: ["App.Models", "App.Core"] },
             ],
             log: "../output/jparser-tools.log",
+            threads: 5,
+            debug: true
         });
 
-        var optsStr = JParserToolsCli.stringifyOptions(opts, '"');
+        var optsStr = JParseCodeCli.stringifyOptions(opts, '"');
 
         var expStr = ' -sources "' + "src/services=1,[java]" + ";" + "src/models=2,[java]" + '"' +
             ' -destinations "' + "../models/service-defs.json=[App.Services]" + ";" + "../models/model-defs.json=[App.Models,App.Core]" + '"' +
-            ' -log "' + "../output/jparser-tools.log" + '"';
+            ' -log "' + "../output/jparser-tools.log" + '"' +
+            ' -threads ' + 5 +
+            ' -debug';
 
         if (expStr !== optsStr) {
             throw new Error("options string mismatch, expected: \n'" + expStr + "'\nactual: \n'" + optsStr + "'");
@@ -37,10 +41,10 @@ module JParserToolsCliTest {
 
 
 function main() {
-    JParserToolsCliTest.testJsCli();
+    JParseCodeCliTest.testJsCli();
 }
 
 main();
 
 
-export = JParserToolsCliTest;
+export = JParseCodeCliTest;

plugins/node-js/jparser-tools-cli-tests.js plugins/node-js/jparse-code-cli-tests.js

@@ -1,22 +1,24 @@
 "use strict";
-/** Generate CLI strings for jparser-tools.jar from javascript objects
+/** Generate CLI strings for jparse-code.jar from javascript objects
  * @author TeamworkGuy2
  * @since 2016-06-21
  */
-var JParserToolsCli;
-(function (JParserToolsCli) {
+var JParseCodeCli;
+(function (JParseCodeCli) {
     function createOptions(opts) {
         return opts;
     }
-    JParserToolsCli.createOptions = createOptions;
+    JParseCodeCli.createOptions = createOptions;
     function stringifyOptions(opts, quoteChar) {
         if (quoteChar === void 0) { quoteChar = '"'; }
         var srcs = opts.sources;
         var dsts = opts.destinations;
         return " -sources " + quoteChar + srcs.map(function (s) { return s.path + "=" + s.depth + "," + "[" + s.fileExtensions.join(",") + "]"; }).join(";") + quoteChar +
             " -destinations " + quoteChar + dsts.map(function (d) { return d.path + "=" + "[" + d.namespaces.join(",") + "]"; }).join(";") + quoteChar +
-            (opts.log ? " -log " + quoteChar + opts.log + quoteChar : "");
+            (opts.log ? " -log " + quoteChar + opts.log + quoteChar : "") +
+            (!isNaN(opts.threads) ? " -threads " + opts.threads : "") +
+            (opts.debug === true ? " -debug" : "");
     }
-    JParserToolsCli.stringifyOptions = stringifyOptions;
-})(JParserToolsCli || (JParserToolsCli = {}));
-module.exports = JParserToolsCli;
+    JParseCodeCli.stringifyOptions = stringifyOptions;
+})(JParseCodeCli || (JParseCodeCli = {}));
+module.exports = JParseCodeCli;

plugins/node-js/jparser-tools-cli-tests.ts plugins/node-js/jparse-code-cli-tests.ts

@@ -1,31 +1,35 @@
 
-/** Generate CLI strings for jparser-tools.jar from javascript objects
+/** Generate CLI strings for jparse-code.jar from javascript objects
  * @author TeamworkGuy2
  * @since 2016-06-21
  */
-module JParserToolsCli {
+module JParseCodeCli {
 
-    export interface Options {
+    export interface ParseOptions {
         sources: { path: string; depth: number; fileExtensions: string[]; }[];
         destinations: { path: string; namespaces: string[]; }[];
-        log: string;
+        log?: string | null;
+        threads?: number | null;
+        debug?: boolean | null;
     }
 
 
-    export function createOptions(opts: Options) {
+    export function createOptions(opts: ParseOptions) {
         return opts;
     }
 
 
-    export function stringifyOptions(opts: Options, quoteChar = '"') {
+    export function stringifyOptions(opts: ParseOptions, quoteChar = '"') {
         var srcs = opts.sources;
         var dsts = opts.destinations;
 
         return " -sources " + quoteChar + srcs.map(s => s.path + "=" + s.depth + "," + "[" + s.fileExtensions.join(",") + "]").join(";") + quoteChar +
             " -destinations " + quoteChar + dsts.map(d => d.path + "=" + "[" + d.namespaces.join(",") + "]").join(";") + quoteChar +
-            (opts.log ? " -log " + quoteChar + opts.log + quoteChar : "");
+            (opts.log ? " -log " + quoteChar + opts.log + quoteChar : "") +
+            (!isNaN(opts.threads) ? " -threads " + opts.threads : "") +
+            (opts.debug === true ? " -debug" : "");
     }
 
 }
 
-export = JParserToolsCli;
+export = JParseCodeCli;

plugins/node-js/jparser-tools-cli.js plugins/node-js/jparse-code-cli.js

@@ -3,7 +3,6 @@
 import java.io.IOException;
 import java.util.Map;
 
-import lombok.Getter;
 import twg2.dataUtil.dataUtils.EnumError;
 import twg2.io.files.FileUtil;
 import twg2.io.json.stringify.JsonStringify;
@@ -27,18 +26,38 @@ public static enum TrackerAction {
 
 
 
-	@Getter long setupNs;
-	@Getter long loadNs;
-	@Getter long tokenizeNs;
-	@Getter long parseNs;
+	long setupNs;
+	long loadNs;
+	long tokenizeNs;
+	long parseNs;
+
+
+	public long getSetupNs() {
+		return setupNs;
+	}
+
+
+	public long getLoadNs() {
+		return loadNs;
+	}
+
+
+	public long getTokenizeNs() {
+		return tokenizeNs;
+	}
+
+
+	public long getParseNs() {
+		return parseNs;
+	}
 
 
 	public long getTotalNs() {
 		return setupNs + loadNs + tokenizeNs + parseNs;
 	}
 
 
-	public void log(TrackerAction action, long timeNanos) {
+	public void setActionTime(TrackerAction action, long timeNanos) {
 		switch(action) {
 		case SETUP:
 			this.setupNs = timeNanos;