Skip to content

Commit 32ee2a5

Browse files
TeamworkGuy2TeamworkGuy2
committed
0.8.0 - Moved ParserWorkFlow.SourceInfo and ParserWorkFlow.LoadResult (new name: SourceFiles) classes and ParserMain.getFilesByExtension() method to JFileIo library. Renamed ParserMain -> ParserMisc. Use latest version of JFileIo and JTextParser and switch from old CharParser.WithMarks interface to new CharParserMatchable.
1 parent 218036c commit 32ee2a5

39 files changed

+254
-436
lines changed

‎.classpath

+1
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,6 @@
2727
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jtext-util/jar/jtext_util.jar" sourcepath="/JTextUtil"/>
2828
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jtree-walker/jar/jtree_walker.jar" sourcepath="/JTreeWalker"/>
2929
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jfile-io/jar/jfile_io.jar" sourcepath="/JFileIo"/>
30+
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jtwg2-logging/jar/jtwg2_logging.jar" sourcepath="/JTwg2Logging"/>
3031
<classpathentry kind="output" path="bin"/>
3132
</classpath>

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
ParserToolsTmp
22
==============
3-
version: 0.7.0
3+
version: 0.8.0
44

55
In progress C#/Java/TypeScript parser tools built atop [JTextParser] (https://github.com/TeamworkGuy2/JTextParser), [Jackson] (https://github.com/FasterXML/jackson-core/) (core, databind, annotations) and half a dozen other utility libraries.
66

‎package-lib.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version" : "0.7.0",
2+
"version" : "0.8.0",
33
"name" : "jparser-tools",
44
"description" : "An in-progress suite of parsing tools for C#, Java, and TypeScript source code",
55
"homepage" : "https://github.com/TeamworkGuy2/JFileIo",

‎src/twg2/parser/baseAst/csharp/CsAstUtil.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package twg2.parser.baseAst.csharp;
22

33
import lombok.val;
4+
import twg2.dataUtil.dataUtils.EnumUtil;
45
import twg2.parser.baseAst.AccessModifierEnum;
56
import twg2.parser.baseAst.AccessModifierParser;
67
import twg2.parser.baseAst.AstTypeChecker;
@@ -12,7 +13,6 @@
1213
import twg2.parser.codeParser.csharp.CsKeyword;
1314
import twg2.parser.documentParser.DocumentFragmentText;
1415
import twg2.treeLike.simpleTree.SimpleTree;
15-
import dataUtils.EnumUtil;
1616

1717
/**
1818
* @author TeamworkGuy2

‎src/twg2/parser/baseAst/java/JavaAstUtil.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package twg2.parser.baseAst.java;
22

3+
import twg2.dataUtil.dataUtils.EnumUtil;
34
import twg2.parser.baseAst.AccessModifierEnum;
45
import twg2.parser.baseAst.AccessModifierParser;
56
import twg2.parser.baseAst.AstTypeChecker;
@@ -11,7 +12,6 @@
1112
import twg2.parser.codeParser.java.JavaKeyword;
1213
import twg2.parser.documentParser.DocumentFragmentText;
1314
import twg2.treeLike.simpleTree.SimpleTree;
14-
import dataUtils.EnumUtil;
1515

1616
/**
1717
* @author TeamworkGuy2

‎src/twg2/parser/codeParser/CodeFileSrc.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
* @since 2015-11-22
1313
* @param <T_LANG> the code language of the source file
1414
*/
15-
public class CodeFileSrc<T_LANG extends CodeLanguage> {
15+
public class CodeFileSrc<T_LANG> {
1616
@Getter SimpleTree<DocumentFragmentText<CodeFragmentType>> doc;
1717
@Getter String srcName;
1818
@Getter String src;

‎src/twg2/parser/codeParser/NumberParser.java

+7-16
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
11
package twg2.parser.codeParser;
22

3-
import java.util.AbstractMap;
4-
53
import lombok.val;
6-
import twg2.arrays.ArrayUtil;
7-
import twg2.collections.primitiveCollections.CharArrayList;
4+
import twg2.collections.tuple.Tuples;
85
import twg2.functions.BiPredicates;
96
import twg2.parser.condition.text.CharParser;
107
import twg2.parser.primitive.NumericParser;
@@ -28,23 +25,17 @@ public static CharParserPlainFactoryImpl<CharParser> createNumericLiteralParser(
2825

2926
val numParser = new NumericParser("numeric literal");
3027

31-
val charList = new CharArrayList();
32-
numParser.getMatchFirstChars(charList);
33-
char[] allowChars = charList.toArray();
34-
3528
BiPredicates.CharObject<TextParser> charCheck = (ch, buf) -> {
36-
// TODO this should be getPosition(), but buffer doesn't allow unread() into previous lines
37-
int off = buf.getColumnNumber();
38-
if(off < 2) { return ArrayUtil.indexOf(allowChars, ch) > -1; }
29+
boolean isFirst = numParser.getFirstCharMatcher().test(ch, buf);
30+
boolean hasPrev = buf.hasPrevChar();
31+
if(!hasPrev) { return isFirst; }
3932
// TODO somewhat messy hack to look back at the previous character and ensure that it's not one of certain chars that never precede numbers
4033
// (e.g. if an A-Z character preceds a digit, it's not a number, it's part of an identifer)
41-
buf.unread(2);
42-
char prevCh = buf.nextChar();
43-
buf.nextChar();
44-
return ArrayUtil.indexOf(allowChars, ch) > -1 && !notPreceedingSet.contains(prevCh);
34+
char prevCh = buf.prevChar();
35+
return isFirst && !notPreceedingSet.contains(prevCh);
4536
};
4637

47-
val numericLiteralParser = new CharParserPlainFactoryImpl<>("numeric literal", false, new AbstractMap.SimpleImmutableEntry<>(charCheck, numParser));
38+
val numericLiteralParser = new CharParserPlainFactoryImpl<>("numeric literal", false, Tuples.of(charCheck, numParser));
4839
return (CharParserPlainFactoryImpl)numericLiteralParser;
4940
}
5041

‎src/twg2/parser/codeParser/ParseCommentsAndWhitespace.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ public static CodeFileSrc<CodeLanguage> buildCommentsAndWhitespaceTree(EnumSet<C
3939
ParserBuilder parser = new ParserBuilder()
4040
.addConstParser(commentParser, CodeFragmentType.COMMENT)
4141
.addConstParser(stringParser, CodeFragmentType.STRING);
42-
return parser.buildAndParse(src, null, srcName);
42+
return parser.buildAndParse(src, null, srcName, true);
4343
}
4444

4545

‎src/twg2/parser/codeParser/ParserBuilder.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ public class ParserBuilder {
2323
private DocumentParser<CodeFragmentType> fileParser;
2424

2525

26-
{
26+
public ParserBuilder() {
2727
this.fileParser = new DocumentParser<>();
2828
}
2929

@@ -46,14 +46,14 @@ public ParserBuilder addConstParser(CharParserFactory parser, CodeFragmentType t
4646
* @param srcName optional
4747
* @return a parsed {@link CodeFileSrc} containing {@link DocumentFragmentText} nodes represented the tokens parsed from {@code src}
4848
*/
49-
public <L extends CodeLanguage> CodeFileSrc<L> buildAndParse(String src, L language, String srcName) {
49+
public <L> CodeFileSrc<L> buildAndParse(String src, L language, String srcName, boolean includeLines) {
5050
List<char[]> lines = new ArrayList<>();
5151

5252
// intercept each line request and add the line to our list of lines
5353
CharsLineSupplier srcLineReader = new CharsLineSupplier(src, 0, src.length(), true, true, true, true);
5454
EnhancedIterator<char[]> lineReader = new EnhancedIterator<>(() -> {
5555
char[] chs = srcLineReader.get();
56-
if(chs != null) {
56+
if(includeLines && chs != null) {
5757
lines.add(chs);
5858
}
5959
return chs;

‎src/twg2/parser/codeParser/csharp/CsBlock.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package twg2.parser.codeParser.csharp;
22

3+
import twg2.dataUtil.dataUtils.EnumUtil;
34
import twg2.parser.baseAst.CompoundBlock;
4-
import dataUtils.EnumUtil;
55

66
/**
77
* @author TeamworkGuy2

‎src/twg2/parser/codeParser/csharp/CsClassParser.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public static CodeFileSrc<CodeLanguageOptions.CSharp> parse(ParseInput params) {
4040
.addConstParser(createOperatorParser(), CodeFragmentType.OPERATOR)
4141
.addConstParser(createSeparatorParser(), CodeFragmentType.SEPARATOR)
4242
.addConstParser(numericLiteralParser, CodeFragmentType.NUMBER);
43-
return parser.buildAndParse(params.getSrc(), CodeLanguageOptions.C_SHARP, params.getFileName());
43+
return parser.buildAndParse(params.getSrc(), CodeLanguageOptions.C_SHARP, params.getFileName(), true);
4444
} catch(Exception e) {
4545
if(params.getErrorHandler() != null) {
4646
params.getErrorHandler().accept(e);

‎src/twg2/parser/codeParser/java/JavaBlock.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package twg2.parser.codeParser.java;
22

3+
import twg2.dataUtil.dataUtils.EnumUtil;
34
import twg2.parser.baseAst.CompoundBlock;
4-
import dataUtils.EnumUtil;
55

66
/**
77
* @author TeamworkGuy2

‎src/twg2/parser/codeParser/java/JavaClassParser.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ public static CodeFileSrc<CodeLanguageOptions.Java> parse(ParseInput params) {
3939
.addConstParser(createOperatorParser(), CodeFragmentType.OPERATOR)
4040
.addConstParser(createSeparatorParser(), CodeFragmentType.SEPARATOR)
4141
.addConstParser(numericLiteralParser, CodeFragmentType.NUMBER);
42-
return parser.buildAndParse(params.getSrc(), CodeLanguageOptions.JAVA, params.getFileName());
42+
return parser.buildAndParse(params.getSrc(), CodeLanguageOptions.JAVA, params.getFileName(), true);
4343
} catch(Exception e) {
4444
if(params.getErrorHandler() != null) {
4545
params.getErrorHandler().accept(e);

‎src/twg2/parser/codeParser/parsers/GenericTypeParser.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import twg2.collections.primitiveCollections.CharArrayList;
88
import twg2.parser.Inclusion;
99
import twg2.parser.condition.text.CharParser;
10+
import twg2.parser.condition.text.CharParserMatchable;
1011
import twg2.parser.text.CharConditionPipe;
1112
import twg2.parser.text.CharConditions;
1213
import twg2.parser.text.StringConditions;
@@ -21,12 +22,12 @@ public class GenericTypeParser {
2122
* @param genericTypeDepth the nesting depth of generic type statements that the returned parser can support.
2223
* i.e. {@code A<B>} would require depth=1, {@code A<B<C>>} would require depth=2
2324
*/
24-
public static CharParser createGenericTypeParser(int genericTypeDepth, Supplier<CharParser> singleIdentifierParserConstructor) {
25+
public static CharParserMatchable createGenericTypeParser(int genericTypeDepth, Supplier<CharParserMatchable> singleIdentifierParserConstructor) {
2526
return _createGenericTypeParser(genericTypeDepth, singleIdentifierParserConstructor);
2627
}
2728

2829

29-
public static CharParser _createGenericTypeParser(int recursionDepth, Supplier<CharParser> singleIdentifierParserConstructor) {
30+
public static CharParserMatchable _createGenericTypeParser(int recursionDepth, Supplier<CharParserMatchable> singleIdentifierParserConstructor) {
3031
// the condition that parses identifiers nested inside the generic type definition
3132
val nestedGenericTypeIdentifierCond = recursionDepth > 1 ? _createGenericTypeParser(recursionDepth - 1, singleIdentifierParserConstructor) : singleIdentifierParserConstructor.get();
3233

‎src/twg2/parser/codeParser/parsers/IdentifierParser.java

+7-6
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,13 @@
44

55
import lombok.val;
66
import twg2.collections.primitiveCollections.CharArrayList;
7+
import twg2.collections.tuple.Tuples;
78
import twg2.parser.Inclusion;
8-
import twg2.parser.condition.text.CharParser;
9+
import twg2.parser.condition.text.CharParserMatchable;
910
import twg2.parser.text.CharConditionPipe;
1011
import twg2.parser.text.CharConditions;
1112
import twg2.parser.text.CharParserFactory;
12-
import twg2.parser.text.CharParserFactoryImpl;
13+
import twg2.parser.text.CharParserPlainFactoryImpl;
1314
import twg2.parser.text.StringParserBuilder;
1415
import twg2.ranges.CharSearchSet;
1516

@@ -21,8 +22,8 @@ public class IdentifierParser {
2122
static int genericTypeDepth = 3;
2223

2324
public static CharParserFactory createIdentifierWithGenericTypeParser() {
24-
val typeStatementCondition = GenericTypeParser.createGenericTypeParser(genericTypeDepth, IdentifierParser::createCompoundIdentifierParser);
25-
CharParserFactory identifierWithGenericTypeParser = new CharParserFactoryImpl<>("compound identifier with optional generic type", false, typeStatementCondition);
25+
val typeStatementCond = GenericTypeParser.createGenericTypeParser(genericTypeDepth, IdentifierParser::createCompoundIdentifierParser);
26+
CharParserFactory identifierWithGenericTypeParser = new CharParserPlainFactoryImpl<>("compound identifier with optional generic type", false, Tuples.of(typeStatementCond.getFirstCharMatcher(), typeStatementCond));
2627
return identifierWithGenericTypeParser;
2728
}
2829

@@ -38,7 +39,7 @@ public static CharParserFactory createIdentifierParser() {
3839
/**
3940
* @return a basic parser for a string of contiguous characters matching those allowed in identifiers (i.e. 'mySpecialLoopCount', '$thing', or '_stspr')
4041
*/
41-
public static CharConditions.BaseCharParserWithMarks newIdentifierParser() {
42+
public static CharConditions.BaseCharParserMatchable newIdentifierParser() {
4243
CharSearchSet firstCharSet = new CharSearchSet();
4344
firstCharSet.addChar('$');
4445
firstCharSet.addChar('_');
@@ -56,7 +57,7 @@ public static CharConditions.BaseCharParserWithMarks newIdentifierParser() {
5657
/**
5758
* @return a compound identifier parser (i.e. can parse 'Aa.Bb.Cc' as one identifier token')
5859
*/
59-
public static CharParser createCompoundIdentifierParser() {
60+
public static CharParserMatchable createCompoundIdentifierParser() {
6061
val identifierParser = Arrays.asList(newIdentifierParser());
6162
val separatorParser = Arrays.asList(new CharConditions.Literal("identifier namespace separator", CharArrayList.of('.'), Inclusion.INCLUDE));
6263
return CharConditionPipe.createPipeRepeatableSeparator("compound identifier", identifierParser, separatorParser);

‎src/twg2/parser/intermAst/classes/IntermClass.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import lombok.Getter;
77
import lombok.val;
88
import twg2.annotations.Immutable;
9+
import twg2.io.write.JsonWrite;
910
import twg2.parser.baseAst.CompoundBlock;
1011
import twg2.parser.baseAst.tools.NameUtil;
1112
import twg2.parser.intermAst.field.IntermFieldSig;
@@ -14,7 +15,6 @@
1415
import twg2.parser.intermAst.method.IntermParameterSig;
1516
import twg2.parser.intermAst.method.ResolvedParameterSig;
1617
import twg2.parser.output.JsonWritableSig;
17-
import twg2.parser.output.JsonWrite;
1818
import twg2.parser.output.WriteSettings;
1919
import twg2.text.stringUtils.StringJoin;
2020

‎src/twg2/parser/intermAst/classes/IntermClassSig.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import lombok.Getter;
1212
import lombok.val;
1313
import twg2.annotations.Immutable;
14+
import twg2.io.write.JsonWrite;
1415
import twg2.parser.baseAst.AccessModifier;
1516
import twg2.parser.baseAst.CompoundBlock;
1617
import twg2.parser.baseAst.tools.NameUtil;
@@ -19,7 +20,6 @@
1920
import twg2.parser.intermAst.project.ProjectClassSet;
2021
import twg2.parser.intermAst.type.TypeSig;
2122
import twg2.parser.output.JsonWritableSig;
22-
import twg2.parser.output.JsonWrite;
2323
import twg2.parser.output.WriteSettings;
2424

2525
/**

‎src/twg2/parser/intermAst/field/IntermFieldSig.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import lombok.AllArgsConstructor;
88
import lombok.Getter;
99
import twg2.annotations.Immutable;
10+
import twg2.io.write.JsonWrite;
1011
import twg2.parser.baseAst.AccessModifier;
1112
import twg2.parser.baseAst.CompoundBlock;
1213
import twg2.parser.baseAst.tools.NameUtil;
@@ -16,7 +17,6 @@
1617
import twg2.parser.intermAst.project.ProjectClassSet;
1718
import twg2.parser.intermAst.type.TypeSig;
1819
import twg2.parser.output.JsonWritableSig;
19-
import twg2.parser.output.JsonWrite;
2020
import twg2.parser.output.WriteSettings;
2121

2222
/**

‎src/twg2/parser/intermAst/field/ResolvedFieldSig.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@
66
import lombok.AllArgsConstructor;
77
import lombok.Getter;
88
import twg2.annotations.Immutable;
9+
import twg2.io.write.JsonWrite;
910
import twg2.parser.baseAst.AccessModifier;
1011
import twg2.parser.baseAst.tools.NameUtil;
1112
import twg2.parser.intermAst.annotation.AnnotationSig;
1213
import twg2.parser.intermAst.type.TypeSig;
1314
import twg2.parser.output.JsonWritableSig;
14-
import twg2.parser.output.JsonWrite;
1515
import twg2.parser.output.WriteSettings;
1616

1717

‎src/twg2/parser/intermAst/method/IntermMethodSig.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import lombok.Getter;
99
import lombok.val;
1010
import twg2.annotations.Immutable;
11+
import twg2.io.write.JsonWrite;
1112
import twg2.parser.baseAst.AccessModifier;
1213
import twg2.parser.baseAst.CompoundBlock;
1314
import twg2.parser.baseAst.tools.NameUtil;
@@ -19,7 +20,6 @@
1920
import twg2.parser.intermAst.project.ProjectClassSet;
2021
import twg2.parser.intermAst.type.TypeSig;
2122
import twg2.parser.output.JsonWritableSig;
22-
import twg2.parser.output.JsonWrite;
2323
import twg2.parser.output.WriteSettings;
2424

2525
/**

‎src/twg2/parser/intermAst/method/IntermParameterSig.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import twg2.annotations.Immutable;
88
import twg2.parser.output.JsonWritableSig;
99
import twg2.parser.output.WriteSettings;
10-
import twg2.text.stringUtils.StringEscape;
10+
import twg2.text.stringEscape.StringEscapeJson;
1111

1212
/** Represents a method parameter
1313
* @author TeamworkGuy2
@@ -36,7 +36,7 @@ public void toJson(Appendable dst, WriteSettings st) throws IOException {
3636
if(defaultValue != null) {
3737
dst.append(", ");
3838
dst.append("\"defaultValue\": \"");
39-
StringEscape.toJsonString(defaultValue, 0, defaultValue.length(), dst);
39+
StringEscapeJson.toJsonString(defaultValue, 0, defaultValue.length(), dst);
4040
dst.append("\"");
4141
}
4242

‎src/twg2/parser/intermAst/method/ResolvedParameterSig.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import twg2.parser.intermAst.type.TypeSig;
99
import twg2.parser.output.JsonWritableSig;
1010
import twg2.parser.output.WriteSettings;
11-
import twg2.text.stringUtils.StringEscape;
11+
import twg2.text.stringEscape.StringEscapeJson;
1212

1313

1414
/**
@@ -42,7 +42,7 @@ public void toJson(Appendable dst, WriteSettings st) throws IOException {
4242
if(defaultValue != null) {
4343
dst.append(", ");
4444
dst.append("\"defaultValue\": \"");
45-
StringEscape.toJsonString(defaultValue, 0, defaultValue.length(), dst);
45+
StringEscapeJson.toJsonString(defaultValue, 0, defaultValue.length(), dst);
4646
dst.append("\"");
4747
}
4848

‎src/twg2/parser/intermAst/type/TypeSig.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,13 @@
1010
import lombok.val;
1111
import twg2.annotations.Immutable;
1212
import twg2.collections.builder.ListBuilder;
13+
import twg2.io.write.JsonWrite;
1314
import twg2.parser.baseAst.CompoundBlock;
1415
import twg2.parser.baseAst.tools.NameUtil;
1516
import twg2.parser.intermAst.classes.IntermClass;
1617
import twg2.parser.intermAst.classes.IntermClassSig;
1718
import twg2.parser.intermAst.project.ProjectClassSet;
1819
import twg2.parser.output.JsonWritableSig;
19-
import twg2.parser.output.JsonWrite;
2020
import twg2.parser.output.WriteSettings;
2121
import twg2.text.stringUtils.StringJoin;
2222

0 commit comments

Comments
 (0)