0.14.1 - Updated dependencies, new parsing strategy from jtext-tokenizer, fixed type parser to better handle C# nullable '?' types

Author: TeamworkGuy2

CHANGELOG.md

@@ -4,9 +4,17 @@ This project does its best to adhere to [Semantic Versioning](http://semver.org/
 
 
 --------
-###[0.14.0](N/A) - 2016-10-30
+###[0.14.1](N/A) - 2016-12-03
 #### Changed
+* Updated dependencies to latest versions: jtext-parser@0.11.0, jtext-tokenizer@0.2.0, jparser-primitive@0.2.0
+  * This includes a new parsing strategy which tries to parse non-compound tokens from start to finish using one parser at a time without passing the characters to compound parser, this improves performance and simplifies some of the compound parsers, but makes some compound parsers more difficult, such as ending conditions that try to keep track of characters between the start and end of the compound parser segment 
+* GenericTypeTokenizer and IdentifierTokenizer changes to properly parse nullable generic parameters
+
+
+--------
+###[0.14.0](https://github.com/TeamworkGuy2/JParseCode/commit/07bd715ecd29a42d781b663cd3a20c2436e69bff) - 2016-10-30
 __Reduced library complexity/scope by moving twg2.parser.text conditions/tokenizers to separate [jtext-tokenizer](https://github.com/TeamworkGuy2/JTextTokenizer) library__
+#### Changed
 * Moved twg2.parser.text package to jtext-tokenizer library
 * Moved twg2.parser.Inclusion to jtext-parser library
 * Renamed classes *Fragment -> *Token:
@@ -26,12 +34,12 @@ __Reduced library complexity/scope by moving twg2.parser.text conditions/tokeniz
 
 --------
 ###[0.13.0](https://github.com/TeamworkGuy2/JParseCode/commit/76734b17d16c67a89df7245a2cea2a1133c3b6b0) - 2016-10-26
+__Parameter modifier parsing support__ (i.e. 'final' in Java or 'params' in C#):
 #### Added
 * Added ParameterSig and ParameterSigResolved 'parameterModifiers' field
 * Added KeywordUtil parameterModifiers() and isParameterModifier()
 
 #### Changed
-__Parameter modifier parsing support__ (i.e. 'final' in Java or 'params' in C#):
 * MethodParametersParser.extractParamsFromSignature() to support parameter modifiers
 * Renamed CodeTokenizerBuilder addConstParser() -> addParser()
 * Added FieldDef.initializerToJson() 'preClosingComma' parameter

README.md

@@ -1,6 +1,6 @@
 JParseCode
 ==============
-version: 0.14.0
+version: 0.14.1
 
 In progress C#/Java/TypeScript parser tools built atop [JTextParser] (https://github.com/TeamworkGuy2/JTextParser), [Jackson] (https://github.com/FasterXML/jackson-core/) (core, databind, annotations) and half a dozen other utility libraries. 
 

bin/jparse_code-with-tests.jar

@@ -1,5 +1,5 @@
 {
-	"version" : "0.14.0",
+	"version" : "0.14.1",
 	"name" : "jparse-code",
 	"description" : "An in-progress suite of parsing/transpilation tools for C#, Java, and TypeScript code.  Generates simple JSON ASTs.",
 	"homepage" : "https://github.com/TeamworkGuy2/JParseCode",

bin/jparse_code.jar

@@ -118,7 +118,7 @@ public void toJson(String srcName, boolean includeSurroundingBrackets, Appendabl
 		if(includeSurroundingBrackets) { dst.append("{\n"); }
 		if(srcName != null) {
 			dst.append("\t\"file\": \"");
-			StringEscapeJson.toJsonString(srcName, 0, srcName.length(), dst);
+			StringEscapeJson.toJsonString(srcName, dst);
 			dst.append("\"");
 		}
 

package-lib.json

@@ -44,7 +44,7 @@ public static CodeTokenizerBuilder<CodeLanguageOptions.CSharp> createFileParser(
 
 	public static CharParserFactory createAnnotationTokenizer() {
 		CharParserFactory annotationParser = new StringBoundedParserBuilder("C# annotation")
-			.addStartEndNotPrecededByMarkers("block [ ]", '[', '[', ']', 1, Inclusion.INCLUDE)
+			.addStartEndNotPrecededByMarkers("block [ ]", '[', '[', ']', Inclusion.INCLUDE)
 			.isCompound(true)
 			.build();
 		return annotationParser;

src/twg2/parser/codeParser/analytics/TokenizeStepLogger.java

@@ -180,7 +180,8 @@ public static <D extends TextToken<S, T>, T, S> SimpleTree<D> tokenizeDocument(S
 
 		while(input.hasNext()) {
 			char ch = input.nextChar();
-			parser.acceptNext(ch, input);
+			//System.out.println(input.getPositionDisplayText());
+			parser.parse(ch, input);
 		}
 
 		return tree;

src/twg2/parser/codeParser/csharp/CsFileTokenizer.java

@@ -30,9 +30,9 @@ public static CharParserMatchable _createGenericTypeTokenizer(int recursionDepth
 		// the condition that parses identifiers nested inside the generic type definition
 		val nestedGenericTypeIdentifierCond = recursionDepth > 1 ? _createGenericTypeTokenizer(recursionDepth - 1, singleIdentifierParserConstructor) : singleIdentifierParserConstructor.get();
 
-		val requiredParser = Arrays.asList(singleIdentifierParserConstructor.get());
+		val typeIdentifierParser = Arrays.asList(singleIdentifierParserConstructor.get());
 		// TODO only matches generic types in the format '<a, b>', allow whitespace between '<'/'>' and after ','
-		val optionalParser = Arrays.asList(CharConditionPipe.createPipeOptionalSuffixesAny("generic type and array dimensions", Arrays.asList(
+		val genericParamsParser = Arrays.asList(CharConditionPipe.createPipeOptionalSuffixesAny("generic type and array dimensions", Arrays.asList(
 			CharConditionPipe.createPipeAllRequired("generic type signature", Arrays.asList(
 				new CharConditions.Literal("<", CharArrayList.of('<'), Inclusion.INCLUDE),
 				CharConditionPipe.createPipeRepeatableSeparator("generic type params",
@@ -45,7 +45,7 @@ public static CharParserMatchable _createGenericTypeTokenizer(int recursionDepth
 			)
 		));
 
-		return CharConditionPipe.createPipeOptionalSuffix("type parser", requiredParser, optionalParser);
+		return CharConditionPipe.createPipeOptionalSuffix("type parser", typeIdentifierParser, genericParamsParser);
 	}
 
 }

src/twg2/parser/tokenizers/CodeTokenizerBuilder.java

@@ -23,8 +23,7 @@ public class IdentifierTokenizer {
 
 	public static CharParserFactory createIdentifierWithGenericTypeTokenizer() {
 		val typeStatementCond = GenericTypeTokenizer.createGenericTypeTokenizer(genericTypeDepth, IdentifierTokenizer::createCompoundIdentifierTokenizer);
-		CharParserFactory identifierWithGenericTypeParser = new CharParserMatchableFactory<>("compound identifier with optional generic type", false, Tuples.of(typeStatementCond.getFirstCharMatcher(), typeStatementCond));
-		return identifierWithGenericTypeParser;
+		return new CharParserMatchableFactory<>("compound identifier with optional generic type", false, Tuples.of(typeStatementCond.getFirstCharMatcher(), typeStatementCond));
 	}
 
 
@@ -60,7 +59,10 @@ public static CharConditions.BaseCharParserMatchable newIdentifierTokenizer() {
 	public static CharParserMatchable createCompoundIdentifierTokenizer() {
 		val identifierParser = Arrays.asList(newIdentifierTokenizer());
 		val separatorParser = Arrays.asList(new CharConditions.Literal("identifier namespace separator", CharArrayList.of('.'), Inclusion.INCLUDE));
-		return CharConditionPipe.createPipeRepeatableSeparator("compound identifier", identifierParser, separatorParser);
+		return CharConditionPipe.createPipeOptionalSuffix("compound identifier (nullable)",
+			Arrays.asList(CharConditionPipe.createPipeRepeatableSeparator("compound identifier", identifierParser, separatorParser)),
+			Arrays.asList(new CharConditions.Literal("nullable '?' type", CharArrayList.of('?'), Inclusion.INCLUDE))
+		);
 	}
 
 }

src/twg2/parser/tokenizers/GenericTypeTokenizer.java

@@ -3,6 +3,7 @@
 import java.io.IOException;
 import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.List;
 
 import lombok.val;
 
@@ -31,7 +32,7 @@
  * @since 2016-1-1
  */
 public class CsClassParseTest {
-	private static CodeFileAndAst<CsBlock> simpleCs = CodeFileAndAst.<CsBlock>parse(CodeLanguageOptions.C_SHARP, "SimpleCs.cs", "ParserExamples.Samples.SimpleCs", true, Arrays.asList(
+	private static List<String> srcLines = Arrays.asList(
 		"namespace ParserExamples.Samples {",
 		"",
 		"  /// <summary>",
@@ -94,8 +95,10 @@ public class CsClassParseTest {
 		"  }",
 		"",
 		"}"
-	));
+	);
 
+	@Parameter
+	private CodeFileAndAst<CsBlock> simpleCs = CodeFileAndAst.<CsBlock>parse(CodeLanguageOptions.C_SHARP, "SimpleCs.cs", "ParserExamples.Samples.SimpleCs", true, srcLines);
 
 	@Parameter
 	private CodeFileSrc<CodeLanguage> file = ParseCodeFile.parseFiles(Arrays.asList(Paths.get("rsc/csharp/ParserExamples/Models/TrackInfo.cs")), FileReadUtil.threadLocalInst(), null).get(0);

src/twg2/parser/tokenizers/IdentifierTokenizer.java

@@ -3,6 +3,7 @@
 import java.io.IOException;
 import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.List;
 
 import lombok.val;
 
@@ -31,7 +32,7 @@
  * @since 2016-1-15
  */
 public class JavaClassParseTest {
-	private static CodeFileAndAst<JavaBlock> simpleJava = CodeFileAndAst.<JavaBlock>parse(CodeLanguageOptions.JAVA, "SimpleJava.java", "ParserExamples.Samples.SimpleJava", true, Arrays.asList(
+	private static List<String> srcLines = Arrays.asList(
 		"package ParserExamples.Samples;",
 		"",
 		"/** A simple class to test parsing.",
@@ -80,8 +81,10 @@ public class JavaClassParseTest {
 		"    }",
 		"",
 		"}"
-	));
+	);
 
+	@Parameter
+	private CodeFileAndAst<JavaBlock> simpleJava = CodeFileAndAst.<JavaBlock>parse(CodeLanguageOptions.JAVA, "SimpleJava.java", "ParserExamples.Samples.SimpleJava", true, srcLines);
 
 	@Parameter
 	private CodeFileSrc<CodeLanguage> file = ParseCodeFile.parseFiles(Arrays.asList(Paths.get("rsc/java/ParserExamples/Models/TrackInfo.java")), FileReadUtil.threadLocalInst(), null).get(0);