0.19.2 - Add varargs '...' parser support and update 'jtext-parser' and 'jtext-tokenizer' dependencies.

Author: TeamworkGuy2

CHANGELOG.md

@@ -4,7 +4,23 @@ This project does its best to adhere to [Semantic Versioning](http://semver.org/
 
 
 --------
-### [0.19.1](N/A) - 2020-04-20
+### [0.19.2](N/A) - 2020-05-23
+__Parameter varargs parsing support__ (i.e. 'int...' in Java).
+#### Changed
+* Update dependency `jtext-parser@0.16.0` and `jtext-tokenizer@0.4.0`
+  * Classes combined, class names simplified, and unused classes and methods removed from libraries
+  * Code identifier parser now provided by `jtext-tokenizer`
+  * Several bug fixes around compound optional parser conditions
+* Added `char[] src, int srcOff, int srcLen` parameters to `CodeTokenizer.tokenizeDocument()`
+* Renamed `IdentifierTokenizer` `newIdentifierTokenizer()` to `createIdentifierTokenizer()`
+* Improved unit tests
+
+#### Removed
+* `IdentifierTokenizer.createIdentifierTokenizer()`
+
+
+--------
+### [0.19.1](https://github.com/TeamworkGuy2/JParseCode/commit/691c019ee2b8a889bd44a8048957fdf86a02bcd4) - 2020-04-20
 #### Changed
 * Finish `CommentAndWhitespaceExtractor` and tests for it
 * `TextToken` interface now includes `hashCode()` and `equals(Object)`

README.md

@@ -177,7 +177,7 @@ JSON Result (printed to System.out):
 
 ## Command Line Interface (CLI)
 
-A command line call looks like:
+A command line call looks like this:
 ```
 path/to/java -jar path/to/jparse-code.jar 
  -sources './src/java/Server/Services=1,[cs];./src/java/Server/Models=3,[cs]'
@@ -190,7 +190,7 @@ Where `./src/java/Server/**` is where source files are kept
 And the files in `./src/java/Server/Services` belong to the C# namespace `App.Services` and `./src/java/Server/Models` belong to the C# namespace `App.Entities`
 
 
-### Sources
+### -sources
 A semicolon separated list of paths set equal to a directory depth followed by a comma and a comma separated, brackets wrapped, list of file extensions. 
 The path, child directory depth, and file extensions are used to create a file system filter and all matching files are parsed.
 The following formats are valid:
@@ -202,26 +202,26 @@ Example: ```/project/myApp/Models=3,[java,json]```
 Note: the brackets around the '[java,json]' file extension list are literal.
 
 
-### Destinations
+### -destinations
 A semicolon separated list of output file names associated with lists of namespaces.  Each parsed file who's namespace falls into one of these lists is written to that file. 
 The following format is valid:
 'path=[namespace,namespace,...]'
 
 Example: ```/project/output/models.json=[MyApp.Models]```
 
 
-### Log
+### -log
 An optional log file name to write parser information to, in the format:
 'path'
 
 Example: ```/project/output/parser-log.log```
 
 
-### Threads
+### -threads
 An optional number of threads to run parsing in parallel, 0 uses the logical number of processors on the current machine, default is 1
 
 
-### Debug
+### -debug
 An optional flag which causes extra debug and performance information to be logged
 
 

bin/jparse_code-with-tests.jar

@@ -1,5 +1,5 @@
 {
-	"version" : "0.19.1",
+	"version" : "0.19.2",
 	"name" : "jparse-code",
 	"description" : "An in-progress suite of parsing/transpilation tools for C#, Java, and TypeScript code.  Generates simple JSON ASTs.",
 	"homepage" : "https://github.com/TeamworkGuy2/JParseCode",

bin/jparse_code.jar

@@ -16,7 +16,7 @@ public class AlbumInfo {
         public string AlbumName { get; set; }
 
         /// <value>The track duration in milliseconds</value>
-        public IList<TrackInfo> Tracks { get; set }
+        public IList<TrackInfo> Tracks { get; set; }
 
     }
 

package-lib.json

@@ -8,7 +8,7 @@
 /// <threadsafety>
 /// This class is mutable. And it is not thread-safe.
 /// </threadsafety>
-[DataContract]
+@DataContract
 public class AlbumInfo {
 
 	/// <value>The track name.</value>

rsc/csharp/ParserExamples/Models/AlbumInfo.cs

@@ -10,7 +10,7 @@
 /// <threadsafety>
 /// This class is mutable. And it is not thread-safe.
 /// </threadsafety>
-[DataContract]
+@DataContract
 public class TrackInfo implements Serializable, Comparable<TrackInfo> {
 
 	/// <value>The track name.</value>

rsc/java/ParserExamples/Models/AlbumInfo.java

@@ -1,7 +1,6 @@
 package twg2.parser.codeParser.csharp;
 
 import twg2.collections.dataStructures.PairList;
-import twg2.parser.Inclusion;
 import twg2.parser.codeParser.CommentStyle;
 import twg2.parser.fragment.CodeTokenType;
 import twg2.parser.language.CodeLanguageOptions;
@@ -13,7 +12,7 @@
 import twg2.parser.tokenizers.IdentifierTokenizer;
 import twg2.parser.tokenizers.NumberTokenizer;
 import twg2.text.tokenizer.CharParserFactory;
-import twg2.text.tokenizer.StringBoundedParserBuilder;
+import twg2.text.tokenizer.Inclusion;
 import twg2.text.tokenizer.StringParserBuilder;
 
 import static twg2.parser.tokenizers.CodeTokenizer.ofType;
@@ -57,7 +56,7 @@ public static PairList<CharParserFactory, TextTransformer<CodeTokenType>> create
 
 
 	public static CharParserFactory createAnnotationTokenizer() {
-		CharParserFactory annotationParser = new StringBoundedParserBuilder("C# annotation")
+		CharParserFactory annotationParser = new StringParserBuilder("C# annotation")
 			.addStartEndNotPrecededByMarkers("block [ ]", '[', '[', ']', Inclusion.INCLUDE)
 			.isCompound(true)
 			.build();

rsc/java/ParserExamples/Models/TrackInfo.java

@@ -1,8 +1,8 @@
 package twg2.parser.tokenizers;
 
-import twg2.parser.Inclusion;
 import twg2.text.tokenizer.CharParserFactory;
-import twg2.text.tokenizer.StringBoundedParserBuilder;
+import twg2.text.tokenizer.Inclusion;
+import twg2.text.tokenizer.StringParserBuilder;
 
 /**
  * @author TeamworkGuy2
@@ -14,7 +14,7 @@ public class CodeBlockTokenizer {
 
 
 	public static CharParserFactory createBlockTokenizer(char startChar, char endChar) {
-		CharParserFactory commentParser = new StringBoundedParserBuilder("block")
+		CharParserFactory commentParser = new StringParserBuilder("block")
 			.addStartEndMarkers("block " + startChar + " " + endChar, startChar, endChar, Inclusion.INCLUDE)
 			.isCompound(true)
 			.build();

src/twg2/parser/codeParser/csharp/CsFileTokenizer.java

@@ -1,8 +1,8 @@
 package twg2.parser.tokenizers;
 
-import twg2.parser.Inclusion;
 import twg2.text.tokenizer.CharParserFactory;
-import twg2.text.tokenizer.StringBoundedParserBuilder;
+import twg2.text.tokenizer.Inclusion;
+import twg2.text.tokenizer.StringParserBuilder;
 
 /**
  * @author TeamworkGuy2
@@ -14,7 +14,7 @@ public final class CodeStringTokenizer {
 
 
 	public static final CharParserFactory createStringTokenizerForJava() {
-		CharParserFactory stringParser = new StringBoundedParserBuilder("Java string")
+		CharParserFactory stringParser = new StringParserBuilder("Java string")
 			.addStartEndNotPrecededByMarkers("string literal", '"', '\\', '"', Inclusion.INCLUDE)
 			.addStartEndNotPrecededByMarkers("char literal", '\'', '\\', '\'', Inclusion.INCLUDE)
 			.build();
@@ -24,7 +24,7 @@ public static final CharParserFactory createStringTokenizerForJava() {
 
 	// TODO make parser work with all types of C# string literals
 	public static final CharParserFactory createStringTokenizerForCSharp() {
-		CharParserFactory stringParser = new StringBoundedParserBuilder("C# string")
+		CharParserFactory stringParser = new StringParserBuilder("C# string")
 			.addStartEndNotPrecededByMarkers("string literal", '"', '\\', '"', Inclusion.INCLUDE)
 			.addStartEndNotPrecededByMarkers("char literal", '\'', '\\', '\'', Inclusion.INCLUDE)
 			.build();
@@ -33,7 +33,7 @@ public static final CharParserFactory createStringTokenizerForCSharp() {
 
 
 	public static final CharParserFactory createStringTokenizerForJavascript() {
-		CharParserFactory stringParser = new StringBoundedParserBuilder("JS string")
+		CharParserFactory stringParser = new StringParserBuilder("JS string")
 			.addStartEndNotPrecededByMarkers("string literal", '"', '\\', '"', Inclusion.INCLUDE)
 			.addStartEndNotPrecededByMarkers("char literal", '\'', '\\', '\'', Inclusion.INCLUDE)
 			.build();

src/twg2/parser/tokenizers/CodeBlockTokenizer.java

@@ -14,7 +14,8 @@
 import twg2.parser.fragment.CodeTokenType;
 import twg2.parser.fragment.TextToken;
 import twg2.parser.language.CodeLanguage;
-import twg2.parser.textFragment.TextConsumer;
+import twg2.parser.textFragment.TextFragmentConsumer;
+import twg2.parser.textFragment.TextFragmentRef;
 import twg2.parser.textFragment.TextFragmentRefImpl;
 import twg2.parser.textFragment.TextFragmentRefImplMut;
 import twg2.parser.textFragment.TextTransformer;
@@ -100,10 +101,10 @@ public static <_T_LANG extends CodeLanguage> CodeFileSrc tokenizeCodeFile(PairLi
 		var input = TextCharsParser.of(src, srcOff, srcLen);
 
 		var docTextFragment = new TextFragmentRefImplMut(srcOff, srcOff + srcLen, 0, 0, -1, -1);
-		var docRoot = new CodeToken(CodeTokenType.DOCUMENT, docTextFragment, docTextFragment.getText(src, srcOff, srcLen).toString());
+		var docRoot = new CodeToken(CodeTokenType.DOCUMENT, docTextFragment, docTextFragment.getText(0, src, srcOff, srcLen).toString());
 
-		SimpleTree<CodeToken> docTree = tokenizeDocument(srcName, input, stepsDetails, tokenizers, docRoot,
-				(type, frag) -> new CodeToken(type, frag, frag.getText(src, srcOff, srcLen).toString()),
+		SimpleTree<CodeToken> docTree = tokenizeDocument(srcName, input, src, srcOff, srcLen, stepsDetails, tokenizers, docRoot,
+				(type, frag) -> new CodeToken(type, frag, frag.getText(0, src, srcOff, srcLen).toString()),
 				(docFrag) -> docFrag.getTokenType().isCompound(),
 				(parent, child) -> parent != child && parent.getToken().contains(child.getToken()));
 
@@ -126,6 +127,7 @@ public static <_T_LANG extends CodeLanguage> CodeFileSrc tokenizeCodeFile(PairLi
 	public static <D extends TextToken<S, T>, T, S> SimpleTree<D> tokenizeDocument(
 		String srcName,
 		TextParser input,
+		char[] src, int srcOff, int srcLen,
 		ParserActionLogger stepsDetails,
 		PairList<? extends CharParserFactory, ? extends TextTransformer<T>> tokenizers,
 		D root,
@@ -135,12 +137,13 @@ public static <D extends TextToken<S, T>, T, S> SimpleTree<D> tokenizeDocument(
 	) {
 		SimpleTreeImpl<D> tree = new SimpleTreeImpl<>(root);
 
-		List<Entry<CharParserFactory, TextConsumer>> conditions = new ArrayList<>();
+		List<Entry<CharParserFactory, TextFragmentConsumer>> conditions = new ArrayList<>();
 
 		for(int i = 0, size = tokenizers.size(); i < size; i++) {
 			TextTransformer<T> transformer = tokenizers.getValue(i);
 
-			conditions.add(Tuples.of(tokenizers.getKey(i), (text, off, len, lineStart, columnStart, lineEnd, columnEnd) -> {
+			conditions.add(Tuples.of(tokenizers.getKey(i), (off, len, lineStart, columnStart, lineEnd, columnEnd) -> {
+				var text = TextFragmentRef.getText(srcOff, src, srcOff, srcLen, off, off + len);
 				T elemType = transformer.apply(text, off, len);
 				var textFragment = new TextFragmentRefImpl(off, off + len, lineStart, columnStart, lineEnd, columnEnd);
 

src/twg2/parser/tokenizers/CodeStringTokenizer.java

@@ -2,10 +2,10 @@
 
 import java.util.EnumSet;
 
-import twg2.parser.Inclusion;
 import twg2.parser.codeParser.CommentStyle;
 import twg2.text.tokenizer.CharParserFactory;
-import twg2.text.tokenizer.StringBoundedParserBuilder;
+import twg2.text.tokenizer.Inclusion;
+import twg2.text.tokenizer.StringParserBuilder;
 
 /**
  * @author TeamworkGuy2
@@ -17,7 +17,7 @@ public final class CommentTokenizer {
 
 
 	public static final CharParserFactory createCommentTokenizerForJava() {
-		CharParserFactory commentParser = new StringBoundedParserBuilder("comment")
+		CharParserFactory commentParser = new StringParserBuilder("comment")
 			.addStartEndMarkers("multi-line comment", "/*", "*/", Inclusion.INCLUDE)
 			.addStartEndMarkers("single-line comment", "//", '\n', Inclusion.EXCLUDE)
 			.build();
@@ -26,7 +26,7 @@ public static final CharParserFactory createCommentTokenizerForJava() {
 
 
 	public static final CharParserFactory createCommentTokenizer(EnumSet<CommentStyle> style) {
-		StringBoundedParserBuilder commentParser = new StringBoundedParserBuilder("comment " + style);
+		StringParserBuilder commentParser = new StringParserBuilder("comment " + style);
 		int markerCount = 0;
 
 		if(style.contains(CommentStyle.MULTILINE_C_STYLE)) {

src/twg2/parser/tokenizers/CodeTokenizer.java

@@ -4,10 +4,10 @@
 import java.util.function.Supplier;
 
 import twg2.collections.primitiveCollections.CharArrayList;
-import twg2.parser.Inclusion;
 import twg2.parser.condition.text.CharParserMatchable;
 import twg2.text.tokenizer.CharConditionPipe;
 import twg2.text.tokenizer.CharConditions;
+import twg2.text.tokenizer.Inclusion;
 import twg2.text.tokenizer.StringConditions;
 
 /** Static methods for creating generic type tokenizers that support nesting (i.e. for tokenizing '{@code HashMap<Entry<String, Integer>, List<String>>}').
@@ -35,20 +35,17 @@ private static CharParserMatchable _createGenericTypeTokenizer(int recursionDept
 
 		var typeIdentifierParser = Arrays.asList(singleIdentifierParserConstructor.get());
 		// TODO only matches generic types in the format '<a, b>', allow whitespace between '<'/'>' and after ','
-		var genericParamsParser = Arrays.asList(CharConditionPipe.createPipeOptionalSuffixesAny("generic type and array dimensions", Arrays.asList(
-			CharConditionPipe.createPipeAllRequired("generic type signature", Arrays.asList(
-				new CharConditions.Literal("<", CharArrayList.of('<'), Inclusion.INCLUDE),
-				CharConditionPipe.createPipeRepeatableSeparator("generic type params",
-					Arrays.asList(nestedGenericTypeIdentifierCond),
-					Arrays.asList(new StringConditions.Literal("separator", new String[] { ", " }, Inclusion.INCLUDE))
-				),
-				new CharConditions.Literal(">", CharArrayList.of('>'), Inclusion.INCLUDE)
-			))), Arrays.asList(
-				CharConditionPipe.createPipeRepeatableSeparator("array dimensions '[]'...", Arrays.asList(new StringConditions.Literal("array dimension '[]'", new String[] { "[]" }, Inclusion.INCLUDE)), null)
-			)
-		));
-
-		return CharConditionPipe.createPipeOptionalSuffix("type parser", typeIdentifierParser, genericParamsParser);
+		var genericParamsParser = CharConditionPipe.createPipeAllRequired("generic type signature",
+			new CharConditions.Literal("<", CharArrayList.of('<'), Inclusion.INCLUDE),
+			CharConditionPipe.createPipeRepeatableSeparator("generic type params",
+				Arrays.asList(nestedGenericTypeIdentifierCond),
+				Arrays.asList(new StringConditions.Literal("separator", new String[] { ", " }, Inclusion.INCLUDE))
+			),
+			new CharConditions.Literal(">", CharArrayList.of('>'), Inclusion.INCLUDE)
+		);
+		var arrayDimensionsParser = CharConditionPipe.createPipeRepeatableSeparator("array dimensions '[]'...", Arrays.asList(new StringConditions.Literal("array dimension '[]'", new String[] { "[]" }, Inclusion.INCLUDE)), null);
+
+		return CharConditionPipe.createPipeOptionalSuffix("type parser", typeIdentifierParser, Arrays.asList(genericParamsParser, arrayDimensionsParser));
 	}
 
 }

src/twg2/parser/tokenizers/CommentTokenizer.java

@@ -3,17 +3,13 @@
 import java.util.Arrays;
 
 import twg2.collections.primitiveCollections.CharArrayList;
-import twg2.functions.predicates.CharPredicate;
-import twg2.parser.Inclusion;
 import twg2.parser.condition.text.CharParserMatchable;
-import twg2.parser.condition.text.CharParserPredicate;
-import twg2.parser.textParser.TextParser;
-import twg2.ranges.CharSearchSet;
 import twg2.text.tokenizer.CharConditionPipe;
 import twg2.text.tokenizer.CharConditions;
 import twg2.text.tokenizer.CharParserFactory;
 import twg2.text.tokenizer.CharParserMatchableFactory;
-import twg2.text.tokenizer.StringParserBuilder;
+import twg2.text.tokenizer.Inclusion;
+import twg2.text.tokenizer.StringConditions;
 import twg2.tuple.Tuples;
 
 /** Static methods for creating C language like identifier parsers (i.e. parsing strings '_myVar', '$num', 'camelCaseStr', etc.)
@@ -31,44 +27,24 @@ public static CharParserFactory createIdentifierWithGenericTypeTokenizer(int max
 	}
 
 
-	public static CharParserFactory createIdentifierTokenizer() {
-		CharParserFactory identifierParser = new StringParserBuilder("identifier")
-			.addConditionMatcher(newIdentifierTokenizer())
-			.build();
-		return identifierParser;
-	}
-
-
 	/**
-	 * @return a basic parser for a string of contiguous characters matching those allowed in identifiers (i.e. 'mySpecialLoopCount', '$thing', or '_stspr')
+	 * @return a basic parser for a string of contiguous characters matching those allowed in identifiers (i.e. 'FancyObject.LoopCount', '$thing', or '_stspr')
 	 */
-	public static CharConditions.BaseCharParserMatchable newIdentifierTokenizer() {
-		CharSearchSet firstCharSet = new CharSearchSet();
-		firstCharSet.addChar('$');
-		firstCharSet.addChar('_');
-		firstCharSet.addRange('a', 'z');
-		firstCharSet.addRange('A', 'Z');
-		CharParserPredicate firstCharCheck = (char ch, TextParser parser) -> (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || ch == '_' || ch == '$';
-
-		CharSearchSet charSet = firstCharSet.copy();
-		charSet.addRange('0', '9');
-		CharPredicate charCheck = (char ch) -> (ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || ch == '_' || ch == '$';
-
-		var cond = new CharConditions.ContainsFirstSpecial("identifier", charCheck, firstCharCheck, firstCharSet.toCharList().toArray(), Inclusion.INCLUDE);
-		return cond;
+	public static CharConditions.BaseCharParserMatchable createIdentifierTokenizer() {
+		return CharConditions.Identifier.newInstance("identifier");
 	}
 
 
 	/**
 	 * @return a compound identifier parser (i.e. can parse 'Aa.Bb.Cc' as one identifier token')
 	 */
 	public static CharParserMatchable createCompoundIdentifierTokenizer() {
-		var identifierParser = Arrays.asList(newIdentifierTokenizer());
-		var separatorParser = Arrays.asList(new CharConditions.Literal("identifier namespace separator", CharArrayList.of('.'), Inclusion.INCLUDE));
+		var identifierParser = createIdentifierTokenizer();
 
 		return CharConditionPipe.createPipeOptionalSuffix("compound identifier (nullable)",
-			Arrays.asList(CharConditionPipe.createPipeRepeatableSeparator("compound identifier", identifierParser, separatorParser)),
-			Arrays.asList(new CharConditions.Literal("nullable '?' type", CharArrayList.of('?'), Inclusion.INCLUDE))
+			Arrays.asList(identifierParser),
+			Arrays.asList(new CharConditions.Literal("nullable '?' suffix", CharArrayList.of('?'), Inclusion.INCLUDE)
+					, new StringConditions.Literal("params '...' suffix", new String[] { "..." }, Inclusion.INCLUDE))
 		);
 	}