Skip to content

Commit 7745de0

Browse files
TeamworkGuy2TeamworkGuy2
committed
0.17.0 - performance refactor, several libraries updated, removed most lombok 'val' usages in favor of Java 9 'var'.
1 parent 4d9ae10 commit 7745de0

39 files changed

+274
-247
lines changed

‎.classpath

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
</classpathentry>
1212
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/5"/>
1313
<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/ANTLR"/>
14-
<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/Lombok"/>
1514
<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/TemplateUtil"/>
1615
<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/TestChecks"/>
1716
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jackson/jar/jackson-annotations-2.5.0.jar" sourcepath="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jackson/src/jackson-annotations-2.5.0-sources.jar">
@@ -54,5 +53,6 @@
5453
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jtree-walker/bin/jtree_walker.jar" sourcepath="/JTreeWalker"/>
5554
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jtuples/bin/jtuples.jar" sourcepath="/JTuples"/>
5655
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jtwg2-logging/bin/jtwg2_logging.jar" sourcepath="/JTwg2Logging"/>
56+
<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/Lombok"/>
5757
<classpathentry kind="output" path="bin"/>
5858
</classpath>

‎CHANGELOG.md

+11-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,17 @@ This project does its best to adhere to [Semantic Versioning](http://semver.org/
44

55

66
--------
7-
### [0.16.1](N/A) - 2019-03-17
7+
### [0.17.0](N/A) - 2019-03-30
8+
Performance refactor, several libraries updated: JArrays, JCollectionUtil, JFileIo, JTextParser, and JTextTokenizer
9+
#### Changed
10+
* Changed to new `FileReadUtil.readChars(InputStream)` (`jfile-io@0.8.2`)
11+
* Switched lombok `val` usage to Java 9 `var`
12+
* Added `HashMap<String, *Keyword> keywordSet` field to `CsKeyword` and `JavaKeyword` for performance
13+
* Added some duplicate code in `IdentifierTokenizer` to work with optimized `CharConditions.ContainsFirstSpecial` constructor
14+
15+
16+
--------
17+
### [0.16.1](https://github.com/TeamworkGuy2/JParseCode/commit/4d9ae1065f328e0354e979ab90c21eee5ade2338) - 2019-03-17
818
#### Fixed
919
* Accidentally deleted compiled *.jar files in 0.16.0 release
1020

‎README.md

-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
JParseCode
22
==============
3-
version: 0.16.1
43

54
In progress C#/Java/TypeScript parser tools built atop [JTextParser](https://github.com/TeamworkGuy2/JTextParser), [Jackson](https://github.com/FasterXML/jackson-core/) (core, databind, annotations) and half a dozen other utility libraries.
65

‎bin/jparse_code-with-tests.jar

1.02 KB
Binary file not shown.

‎bin/jparse_code.jar

851 Bytes
Binary file not shown.

‎package-lib.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version" : "0.16.1",
2+
"version" : "0.17.0",
33
"name" : "jparse-code",
44
"description" : "An in-progress suite of parsing/transpilation tools for C#, Java, and TypeScript code. Generates simple JSON ASTs.",
55
"homepage" : "https://github.com/TeamworkGuy2/JParseCode",

‎src/twg2/parser/codeParser/codeStats/ParseDirectoryCodeFiles.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package twg2.parser.codeParser.codeStats;
22

33
import java.io.File;
4-
import java.io.FileReader;
4+
import java.io.FileInputStream;
55
import java.io.IOException;
66
import java.nio.file.Files;
77
import java.nio.file.Path;
@@ -137,7 +137,7 @@ public static ParseDirectoryCodeFiles parseFileStats(Path relativePath, List<Pat
137137
for(Path path : files) {
138138
File file = path.toFile();
139139
String fullFileName = file.getName();
140-
char[] src = fileReader.readChars(new FileReader(file));
140+
char[] src = fileReader.readChars(new FileInputStream(file));
141141
int srcOff = 0;
142142
int srcLen = src.length;
143143
Entry<String, String> fileNameExt = StringSplit.lastMatchParts(fullFileName, ".");

‎src/twg2/parser/codeParser/csharp/CsAnnotationExtractor.java

+4-6
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import java.util.ArrayList;
44
import java.util.List;
55

6-
import lombok.val;
76
import twg2.ast.interm.annotation.AnnotationSig;
87
import twg2.parser.codeParser.extractors.AnnotationExtractor;
98
import twg2.parser.fragment.AstFragType;
@@ -31,13 +30,13 @@ public CsAnnotationExtractor() {
3130

3231
@Override
3332
public boolean acceptNext(SimpleTree<CodeToken> tokenNode) {
34-
val lang = CodeLanguageOptions.C_SHARP;
33+
var lang = CodeLanguageOptions.C_SHARP;
3534

3635
if(state != State.FAILED) {
37-
val childs = tokenNode.getChildren();
36+
var childs = tokenNode.getChildren();
3837
CodeToken annotTypeFrag = null;
3938
if(AstFragType.isBlock(tokenNode.getData(), "[") && childs != null && childs.size() > 0 && AstFragType.isIdentifier(annotTypeFrag = childs.get(0).getData())) {
40-
val annot = AnnotationExtractor.parseAnnotationBlock(lang, annotTypeFrag.getTokenType(), annotTypeFrag.getText(), (childs.size() > 1 ? childs.get(1) : null));
39+
var annot = AnnotationExtractor.parseAnnotationBlock(lang, annotTypeFrag.getTokenType(), annotTypeFrag.getText(), (childs.size() > 1 ? childs.get(1) : null));
4140
annotations.add(annot);
4241
state = State.COMPLETE;
4342
return true;
@@ -64,8 +63,7 @@ public CsAnnotationExtractor recycle() {
6463

6564
@Override
6665
public CsAnnotationExtractor copy() {
67-
val copy = new CsAnnotationExtractor();
68-
return copy;
66+
return new CsAnnotationExtractor();
6967
}
7068

7169

‎src/twg2/parser/codeParser/csharp/CsAstUtil.java

+8-9
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package twg2.parser.codeParser.csharp;
22

3-
import lombok.val;
43
import twg2.dataUtil.dataUtils.EnumError;
54
import twg2.parser.codeParser.AccessModifierEnum;
65
import twg2.parser.codeParser.AccessModifierParser;
@@ -132,23 +131,23 @@ public boolean isKeyword(CodeToken node, CsKeyword keyword1, CsKeyword keyword2,
132131
@Override
133132
public boolean isFieldBlock(SimpleTree<CodeToken> block) {
134133
if(block == null) { return true; }
135-
val childs = block.getChildren();
134+
var childs = block.getChildren();
136135
// properties must have at-least one indexer (i.e. 'get' or 'set')
137136
if(childs.size() == 0) { return false; }
138137

139-
val keywords = this.getLanguage().getKeywordUtil();
138+
var keywords = this.getLanguage().getKeywordUtil();
140139

141140
boolean prevWasGetOrSet = false;
142141
for(int i = 0, size = childs.size(); i < size; i++) {
143-
val child = childs.get(i);
144-
val nextChild = i < size - 1 ? childs.get(i + 1) : null;
145-
val frag = child.getData();
146-
val fragType = frag.getTokenType();
142+
var child = childs.get(i);
143+
var nextChild = i < size - 1 ? childs.get(i + 1) : null;
144+
var frag = child.getData();
145+
var fragType = frag.getTokenType();
147146
if(fragType == CodeTokenType.COMMENT) {
148147
continue;
149148
}
150-
val isGetOrSet = isGetOrSet(frag);
151-
val isAccessMod = keywords.fieldModifiers().is(frag);
149+
boolean isGetOrSet = isGetOrSet(frag);
150+
boolean isAccessMod = keywords.fieldModifiers().is(frag);
152151
if(isGetOrSet ||
153152
(prevWasGetOrSet && (fragType == CodeTokenType.BLOCK || fragType == CodeTokenType.SEPARATOR)) ||
154153
(isAccessMod && nextChild != null && isGetOrSet(nextChild.getData()))) {

‎src/twg2/parser/codeParser/csharp/CsEnumMemberExtractor.java

+5-7
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import java.util.Collections;
66
import java.util.List;
77

8-
import lombok.val;
98
import twg2.arrays.ArrayUtil;
109
import twg2.ast.interm.annotation.AnnotationSig;
1110
import twg2.ast.interm.block.BlockAst;
@@ -58,7 +57,7 @@ public CsEnumMemberExtractor(KeywordUtil<? extends Keyword> keywordUtil, BlockAs
5857
this.state = State.INIT;
5958

6059
// determine the enum's base type
61-
val enumExtends = parentBlock.declaration.getExtendImplementSimpleNames();
60+
var enumExtends = parentBlock.declaration.getExtendImplementSimpleNames();
6261
if(enumExtends == null || enumExtends.isEmpty()) {
6362
enumType = new TypeSig.TypeSigSimpleBase(CsKeyword.INT.toSrc(), 0, false, true);
6463
}
@@ -112,7 +111,7 @@ else if(state == State.FOUND_NAME) {
112111
// if a '=' symbol is found, the enum has a custom value (i.e. 'MY_ENUM = 2, ...;')
113112
if(AstFragType.isOperator(tokenData, CsOperator.ASSIGNMENT)) {
114113
// remove the minimum viable enum that was added when the previous identifier node was found, this is going to be a full enum with a value
115-
val minimumEnum = enumMembers.remove(enumMembers.size() - 1);
114+
var minimumEnum = enumMembers.remove(enumMembers.size() - 1);
116115
nextMemberComments = minimumEnum.getComments();
117116
state = State.FOUND_ASSIGNMENT_SYMBOL;
118117
return true;
@@ -160,8 +159,7 @@ public CsEnumMemberExtractor recycle() {
160159

161160
@Override
162161
public AstParser<List<FieldDef>> copy() {
163-
val copy = new CsEnumMemberExtractor(keywordUtil, parentBlock, commentParser.copy());
164-
return copy;
162+
return new CsEnumMemberExtractor(keywordUtil, parentBlock, commentParser.copy());
165163
}
166164

167165

@@ -172,8 +170,8 @@ void reset() {
172170

173171

174172
private void addEnumMember(String memberName, SimpleTree<CodeToken> tokenNode) {
175-
val comments = (nextMemberComments != null ? nextMemberComments : new ArrayList<>(commentParser.getParserResult()));
176-
val field = new FieldDef(memberName, NameUtil.newFqName(parentBlock.declaration.getFullName(), memberName), enumType,
173+
var comments = (nextMemberComments != null ? nextMemberComments : new ArrayList<>(commentParser.getParserResult()));
174+
var field = new FieldDef(memberName, NameUtil.newFqName(parentBlock.declaration.getFullName(), memberName), enumType,
177175
Arrays.asList(CsKeyword.PUBLIC), Collections.<AnnotationSig>emptyList(), comments, tokenNode);
178176
nextMemberComments = null;
179177
commentParser.recycle();

‎src/twg2/parser/codeParser/csharp/CsFileTokenizer.java

+6-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package twg2.parser.codeParser.csharp;
22

3-
import lombok.val;
43
import twg2.parser.Inclusion;
54
import twg2.parser.codeParser.CommentStyle;
65
import twg2.parser.fragment.CodeTokenType;
@@ -20,18 +19,21 @@
2019
* @since 2015-2-9
2120
*/
2221
public class CsFileTokenizer {
22+
public static int cnt = 0;
2323

2424
public static CodeTokenizerBuilder<CodeLanguageOptions.CSharp> createFileParser() {
25-
val identifierParser = IdentifierTokenizer.createIdentifierWithGenericTypeTokenizer();
26-
val numericLiteralParser = NumberTokenizer.createNumericLiteralTokenizer();
25+
var identifierParser = IdentifierTokenizer.createIdentifierWithGenericTypeTokenizer();
26+
var numericLiteralParser = NumberTokenizer.createNumericLiteralTokenizer();
2727

28-
val parser = new CodeTokenizerBuilder<>(CodeLanguageOptions.C_SHARP)
28+
var parser = new CodeTokenizerBuilder<>(CodeLanguageOptions.C_SHARP)
2929
.addParser(CommentTokenizer.createCommentTokenizer(CommentStyle.multiAndSingleLine()), CodeTokenType.COMMENT)
3030
.addParser(CodeStringTokenizer.createStringTokenizerForCSharp(), CodeTokenType.STRING)
3131
.addParser(CodeBlockTokenizer.createBlockTokenizer('{', '}'), CodeTokenType.BLOCK)
3232
.addParser(CodeBlockTokenizer.createBlockTokenizer('(', ')'), CodeTokenType.BLOCK)
3333
.addParser(createAnnotationTokenizer(), CodeTokenType.BLOCK)
3434
.addParser(identifierParser, (text, off, len) -> {
35+
cnt++;
36+
// TODO performance
3537
return CsKeyword.check.isKeyword(text.toString()) ? CodeTokenType.KEYWORD : CodeTokenType.IDENTIFIER; // possible bad performance
3638
})
3739
.addParser(createOperatorTokenizer(), CodeTokenType.OPERATOR)

‎src/twg2/parser/codeParser/csharp/CsKeyword.java

+15-6
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
package twg2.parser.codeParser.csharp;
22

33
import java.util.Arrays;
4+
import java.util.HashMap;
45

56
import lombok.Getter;
6-
import lombok.val;
77
import lombok.experimental.Accessors;
88
import twg2.arrays.ArrayUtil;
99
import twg2.parser.codeParser.Keyword;
@@ -148,6 +148,7 @@ public String toSrc() {
148148
@Accessors(fluent = true)
149149
public static class CsKeywordUtil implements KeywordUtil<CsKeyword> {
150150
public final String[] keywords;
151+
public final HashMap<String, CsKeyword> keywordSet;
151152
private final CsKeyword[] values;
152153
private final String[] primitives;
153154
@Getter private final CodeTokenEnumSubSet<CsKeyword> types;
@@ -162,7 +163,7 @@ public static class CsKeywordUtil implements KeywordUtil<CsKeyword> {
162163

163164
{
164165
this.values = CsKeyword.values();
165-
val enumData = EnumSplitter.split(this.values, (e) -> e.srcName,
166+
var enumData = EnumSplitter.split(this.values, (e) -> e.srcName,
166167
(e) -> e.isType,
167168
(e) -> e.isClassModifier,
168169
(e) -> e.isFieldModifier,
@@ -173,9 +174,15 @@ public static class CsKeywordUtil implements KeywordUtil<CsKeyword> {
173174
(e) -> e.isTypeLiteral
174175
);
175176
this.keywords = enumData.getKey();
177+
this.keywordSet = new HashMap<>(this.keywords.length);
178+
int k = 0;
179+
for(String keyword : this.keywords) {
180+
this.keywordSet.put(keyword, this.values[k]);
181+
k++;
182+
}
176183

177184
int i = 0;
178-
val enumSets = ArrayUtil.map(enumData.getValue(), CodeTokenEnumSubSet.class, (es) -> new CodeTokenEnumSubSet<>(CodeTokenType.KEYWORD, es));
185+
var enumSets = ArrayUtil.map(enumData.getValue(), CodeTokenEnumSubSet.class, (es) -> new CodeTokenEnumSubSet<>(CodeTokenType.KEYWORD, es));
179186
types = enumSets[i++];
180187
classModifiers = enumSets[i++];
181188
fieldModifiers = enumSets[i++];
@@ -206,8 +213,9 @@ public CsKeyword toKeyword(String str) {
206213

207214
@Override
208215
public CsKeyword tryToKeyword(String str) {
209-
int idx = Arrays.binarySearch(keywords, str);
210-
return idx > -1 ? values[idx] : null;
216+
//int idx = Arrays.binarySearch(keywords, str);
217+
//return idx > -1 ? values[idx] : null;
218+
return this.keywordSet.get(str);
211219
}
212220

213221

@@ -219,7 +227,8 @@ public boolean isInheritanceKeyword(String str) {
219227

220228
@Override
221229
public boolean isKeyword(String str) {
222-
return Arrays.binarySearch(keywords, str) > -1;
230+
//return Arrays.binarySearch(keywords, str) > -1;
231+
return this.keywordSet.containsKey(str);
223232
}
224233

225234

‎src/twg2/parser/codeParser/csharp/CsOperator.java

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package twg2.parser.codeParser.csharp;
22

33
import lombok.Getter;
4-
import lombok.val;
54
import lombok.experimental.Accessors;
65
import twg2.arrays.ArrayUtil;
76
import twg2.collections.primitiveCollections.IntArrayList;
@@ -140,7 +139,7 @@ public static class Inst implements OperatorUtil<CsOperator> {
140139

141140
{
142141
this.values = CsOperator.values();
143-
val enumData = EnumSplitter.split(this.values, (e) -> e.srcSymbol,
142+
var enumData = EnumSplitter.split(this.values, (e) -> e.srcSymbol,
144143
(e) -> e.isArithmetic,
145144
(e) -> e.isAssignment,
146145
(e) -> e.isBitwise,
@@ -153,7 +152,7 @@ public static class Inst implements OperatorUtil<CsOperator> {
153152
this.keywords = enumData.getKey();
154153

155154
int i = 0;
156-
val enumSets = ArrayUtil.map(enumData.getValue(), CodeTokenEnumSubSet.class, (es) -> new CodeTokenEnumSubSet<>(CodeTokenType.OPERATOR, es));
155+
var enumSets = ArrayUtil.map(enumData.getValue(), CodeTokenEnumSubSet.class, (es) -> new CodeTokenEnumSubSet<>(CodeTokenType.OPERATOR, es));
157156
arithmeticOperators = enumSets[i++];
158157
assignmentOperators = enumSets[i++];
159158
bitwiseOperators = enumSets[i++];

‎src/twg2/parser/codeParser/csharp/CsUsingStatementExtractor.java

+2-4
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import java.util.ArrayList;
44
import java.util.List;
55

6-
import lombok.val;
76
import twg2.parser.codeParser.tools.NameUtil;
87
import twg2.parser.fragment.AstFragType;
98
import twg2.parser.fragment.CodeToken;
@@ -43,7 +42,7 @@ public boolean acceptNext(SimpleTree<CodeToken> tokenNode) {
4342
}
4443
}
4544
else if(state == State.FOUND_USING) {
46-
val data = tokenNode.getData();
45+
var data = tokenNode.getData();
4746
if(AstFragType.isIdentifier(data)) {
4847
usingStatements.add(NameUtil.splitFqName(data.getText()));
4948
state = State.COMPLETE;
@@ -72,8 +71,7 @@ public CsUsingStatementExtractor recycle() {
7271

7372
@Override
7473
public CsUsingStatementExtractor copy() {
75-
val copy = new CsUsingStatementExtractor();
76-
return copy;
74+
return new CsUsingStatementExtractor();
7775
}
7876

7977

‎src/twg2/parser/codeParser/extractors/AccessModifierExtractor.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import java.util.ArrayList;
44
import java.util.List;
55

6-
import lombok.val;
76
import twg2.parser.codeParser.Keyword;
87
import twg2.parser.codeParser.KeywordUtil;
98
import twg2.parser.fragment.CodeToken;
@@ -28,7 +27,7 @@ public static Keyword parseAccessModifier(KeywordUtil<? extends Keyword> keyword
2827
*/
2928
public static List<String> readAccessModifiers(KeywordUtil<? extends Keyword> keywordUtil, EnhancedListBuilderIterator<SimpleTree<CodeToken>> iter) {
3029
int prevCount = 0;
31-
val accessModifiers = new ArrayList<String>();
30+
var accessModifiers = new ArrayList<String>();
3231
SimpleTree<CodeToken> child = iter.hasPrevious() ? iter.previous() : null;
3332

3433
while(child != null && keywordUtil.classModifiers().is(child.getData())) {

‎src/twg2/parser/codeParser/extractors/AnnotationExtractor.java

+5-6
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import java.util.HashMap;
44

5-
import lombok.val;
65
import twg2.ast.interm.annotation.AnnotationSig;
76
import twg2.parser.codeParser.tools.NameUtil;
87
import twg2.parser.fragment.CodeToken;
@@ -25,17 +24,17 @@ public class AnnotationExtractor {
2524
* @return a parsed annotation
2625
*/
2726
public static AnnotationSig parseAnnotationBlock(CodeLanguage lang, CodeTokenType annotNameType, String annotName, SimpleTree<CodeToken> annotParamsNode) {
28-
val paramChilds = annotParamsNode != null ? annotParamsNode.getChildren() : null;
29-
val size = paramChilds != null ? paramChilds.size() : 0;
27+
var paramChilds = annotParamsNode != null ? annotParamsNode.getChildren() : null;
28+
int size = paramChilds != null ? paramChilds.size() : 0;
3029

3130
if(annotNameType != CodeTokenType.IDENTIFIER) { throw new IllegalArgumentException("annotation node expected to contain identifier, found '" + annotName + "'"); }
3231

33-
val params = new HashMap<String, String>();
32+
var params = new HashMap<String, String>();
3433
boolean firstParamUnnamed = false;
3534

3635
// parse an annotation '(arguments, ...)'
3736
if(size > 0) {
38-
val annotParamsBlock = annotParamsNode.getData();
37+
var annotParamsBlock = annotParamsNode.getData();
3938
if(annotParamsBlock.getTokenType() != CodeTokenType.BLOCK) { throw new IllegalArgumentException("annotation node expected to contain identifier, found '" + annotParamsBlock.getText() + "'"); }
4039

4140
// += 2, for the value and the separator
@@ -62,7 +61,7 @@ public static AnnotationSig parseAnnotationBlock(CodeLanguage lang, CodeTokenTyp
6261
// number: 'Annotation(1)' or 'Annotation(-15)'
6362
int num;
6463
if((num = DataTypeExtractor.isNumber(param, (i + 1 < size ? paramChilds.get(i + 1).getData() : null))) > 0) {
65-
val paramValue = param.getText() + (i + 1 < size && num > 1 ? paramChilds.get(i + 1).getData().getText() : "");
64+
String paramValue = param.getText() + (i + 1 < size && num > 1 ? paramChilds.get(i + 1).getData().getText() : "");
6665
params.put(paramName, paramValue);
6766
i += (num - 1);
6867
}

0 commit comments

Comments
 (0)