Skip to content

Commit ab23d86

Browse files
TeamworkGuy2TeamworkGuy2
committed
0.4.0 - Fixed C# method signature parsing regression, refactored some minor things, removed unused twg2.parser.documentParser.block package.
1 parent 96bddfc commit ab23d86

25 files changed

+110
-1004
lines changed

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
ParserToolsTmp
22
==============
3-
version: 0.3.0
3+
version: 0.4.0
44

55
In progress C#/Java/TypeScript parser tools built atop [JTextParser] (https://github.com/TeamworkGuy2/JTextParser), [Jackson] (https://github.com/FasterXML/jackson-core/) (core, databind, annotations) and half a dozen other utility libraries.
66

‎src/twg2/parser/condition/AstParser.java ‎src/twg2/parser/baseAst/AstParser.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
package twg2.parser.condition;
1+
package twg2.parser.baseAst;
22

33
import twg2.parser.codeParser.CodeFragmentType;
4+
import twg2.parser.condition.TokenParser;
45
import twg2.parser.documentParser.DocumentFragmentText;
56
import twg2.treeLike.simpleTree.SimpleTree;
67

‎src/twg2/parser/codeParser/AstExtractor.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
import java.util.List;
44
import java.util.Map.Entry;
55

6+
import twg2.parser.baseAst.AstParser;
67
import twg2.parser.baseAst.CompoundBlock;
7-
import twg2.parser.condition.AstParser;
88
import twg2.parser.documentParser.DocumentFragmentText;
99
import twg2.parser.intermAst.annotation.AnnotationSig;
1010
import twg2.parser.intermAst.block.IntermBlock;
@@ -27,9 +27,9 @@ public interface AstExtractor<T_BLOCK extends CompoundBlock> {
2727

2828
public AstParser<List<AnnotationSig>> createAnnotationParser(IntermBlock<T_BLOCK> block);
2929

30-
public AstParser<List<IntermFieldSig>> createFieldParser(IntermBlock<T_BLOCK> block, AstParser<TypeSig.Simple> typeParser, AstParser<List<AnnotationSig>> annotationParser);
30+
public AstParser<List<IntermFieldSig>> createFieldParser(IntermBlock<T_BLOCK> block, AstParser<List<AnnotationSig>> annotationParser);
3131

32-
public AstParser<List<IntermMethodSig.SimpleImpl>> createMethodParser(IntermBlock<T_BLOCK> block, AstParser<TypeSig.Simple> typeParser, AstParser<List<AnnotationSig>> annotationParser);
32+
public AstParser<List<IntermMethodSig.SimpleImpl>> createMethodParser(IntermBlock<T_BLOCK> block, AstParser<List<AnnotationSig>> annotationParser);
3333

3434

3535
public List<IntermBlock<T_BLOCK>> extractBlocks(List<String> nameScope, SimpleTree<DocumentFragmentText<CodeFragmentType>> astTree, IntermBlock<T_BLOCK> parentScope);

‎src/twg2/parser/codeParser/BaseBlockParser.java

+12-16
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,11 @@
33
import java.util.ArrayList;
44
import java.util.List;
55
import java.util.Map.Entry;
6-
import java.util.function.Predicate;
76

87
import lombok.val;
98
import twg2.collections.tuple.Tuples;
9+
import twg2.parser.baseAst.AstParser;
1010
import twg2.parser.baseAst.CompoundBlock;
11-
import twg2.parser.condition.AstParser;
1211
import twg2.parser.documentParser.DocumentFragmentText;
1312
import twg2.parser.intermAst.annotation.AnnotationSig;
1413
import twg2.parser.intermAst.block.IntermBlock;
@@ -26,12 +25,11 @@ public class BaseBlockParser {
2625
/** Parses a simple AST tree using an {@link AstExtractor}
2726
* @param extractor provides parsers and extract method to consume the astTree
2827
* @param astTree the tree of basic {@link DocumentFragmentText}{@code <}{@link CodeFragmentType}{@code >} tokens
29-
* @param isValidBlock a function that checks if an {@link IntermBlock} appears to be valid before attempting to parse it
3028
* @return a list of entries with simple AST tree blocks as keys and {@link IntermClass} as values
3129
*/
3230
// TODO this only parses some fields and interface methods
3331
public static <_T_BLOCK extends CompoundBlock> List<Entry<SimpleTree<DocumentFragmentText<CodeFragmentType>>, IntermClass.SimpleImpl<_T_BLOCK>>> extractBlockFieldsAndInterfaceMethods(
34-
AstExtractor<_T_BLOCK> extractor, SimpleTree<DocumentFragmentText<CodeFragmentType>> astTree, Predicate<IntermBlock<_T_BLOCK>> isValidBlock) {
32+
AstExtractor<_T_BLOCK> extractor, SimpleTree<DocumentFragmentText<CodeFragmentType>> astTree) {
3533

3634
val nameScope = new ArrayList<String>();
3735

@@ -46,23 +44,19 @@ public static <_T_BLOCK extends CompoundBlock> List<Entry<SimpleTree<DocumentFra
4644
List<List<String>> usingStatements = new ArrayList<>(usingStatementExtractor.getParserResult());
4745

4846
for(val block : blockDeclarations) {
49-
if(!isValidBlock.test(block)) {
50-
continue;
51-
}
52-
53-
List<IntermFieldSig> fields = null;
54-
List<IntermMethodSig.SimpleImpl> intfMethods = null;
55-
56-
AstParser<List<AnnotationSig>> annotationExtractor = extractor.createAnnotationParser(block);
57-
AstParser<List<IntermFieldSig>> fieldExtractor = extractor.createFieldParser(block, extractor.createTypeParser(), annotationExtractor);
58-
AstParser<List<IntermMethodSig.SimpleImpl>> methodExtractor = extractor.createMethodParser(block, extractor.createTypeParser(), annotationExtractor);
59-
6047
usingStatementExtractor.recycle();
6148
runParsers(block.getBlockTree(), usingStatementExtractor);
6249

6350
List<List<String>> tmpUsingStatements = usingStatementExtractor.getParserResult();
6451
usingStatements.addAll(tmpUsingStatements);
6552

53+
List<IntermFieldSig> fields = null;
54+
List<IntermMethodSig.SimpleImpl> intfMethods = null;
55+
56+
AstParser<List<AnnotationSig>> annotationExtractor = extractor.createAnnotationParser(block);
57+
AstParser<List<IntermFieldSig>> fieldExtractor = extractor.createFieldParser(block, annotationExtractor);
58+
AstParser<List<IntermMethodSig.SimpleImpl>> methodExtractor = extractor.createMethodParser(block, annotationExtractor);
59+
6660
runParsers(block.getBlockTree(), annotationExtractor, fieldExtractor, methodExtractor);
6761

6862
if(block.getBlockType().canContainFields()) {
@@ -72,7 +66,9 @@ public static <_T_BLOCK extends CompoundBlock> List<Entry<SimpleTree<DocumentFra
7266
intfMethods = methodExtractor.getParserResult();
7367
}
7468

75-
resBlocks.add(Tuples.of(block.getBlockTree(), new IntermClass.SimpleImpl<>(block.getDeclaration(), usingStatements, fields, intfMethods, block.getBlockType())));
69+
if(block.getBlockType().canContainFields() && block.getBlockType().canContainMethods()) {
70+
resBlocks.add(Tuples.of(block.getBlockTree(), new IntermClass.SimpleImpl<>(block.getDeclaration(), usingStatements, fields, intfMethods, block.getBlockType())));
71+
}
7672
}
7773

7874
return resBlocks;

‎src/twg2/parser/codeParser/BaseDataTypeExtractor.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66

77
import lombok.val;
88
import twg2.collections.util.ListBuilder;
9+
import twg2.parser.baseAst.AstParser;
910
import twg2.parser.baseAst.tools.AstFragType;
10-
import twg2.parser.condition.AstParser;
1111
import twg2.parser.documentParser.DocumentFragmentText;
1212
import twg2.parser.intermAst.type.TypeSig;
1313
import twg2.text.stringUtils.StringCheck;

‎src/twg2/parser/codeParser/BaseFieldExtractor.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,11 @@
44
import java.util.List;
55

66
import lombok.val;
7+
import twg2.parser.baseAst.AstParser;
78
import twg2.parser.baseAst.CompoundBlock;
89
import twg2.parser.baseAst.tools.AstFragType;
910
import twg2.parser.baseAst.tools.AstUtil;
1011
import twg2.parser.baseAst.tools.NameUtil;
11-
import twg2.parser.condition.AstParser;
1212
import twg2.parser.documentParser.DocumentFragmentText;
1313
import twg2.parser.intermAst.annotation.AnnotationSig;
1414
import twg2.parser.intermAst.block.IntermBlock;

‎src/twg2/parser/codeParser/BaseMethodExtractor.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@
44
import java.util.List;
55

66
import lombok.val;
7+
import twg2.parser.baseAst.AstParser;
78
import twg2.parser.baseAst.CompoundBlock;
89
import twg2.parser.baseAst.tools.AstFragType;
910
import twg2.parser.baseAst.tools.NameUtil;
10-
import twg2.parser.condition.AstParser;
1111
import twg2.parser.documentParser.DocumentFragmentText;
1212
import twg2.parser.intermAst.annotation.AnnotationSig;
1313
import twg2.parser.intermAst.block.IntermBlock;

‎src/twg2/parser/codeParser/CodeLanguageOptions.java

+44-32
Original file line numberDiff line numberDiff line change
@@ -25,17 +25,54 @@
2525
* @author TeamworkGuy2
2626
* @since 2015-9-19
2727
*/
28-
public class CodeLanguageOptions<T_KEYWORD, T_LANG extends CodeLanguage, T_AST_UTIL extends AstUtil<? extends CompoundBlock, T_KEYWORD>, T_AST_EXTRACTOR extends AstExtractor<? extends CompoundBlock>> implements CodeLanguage {
28+
public enum CodeLanguageOptions {
29+
;
2930

30-
public static class CSharp extends CodeLanguageOptions<CsKeyword, CSharp, CsAstUtil, AstExtractor<CsBlock>> {
31+
/**
32+
* @author TeamworkGuy2
33+
* @since 2016-1-16
34+
* @param <T_KEYWORD> the keyword enum containing this language's keywords
35+
* @param <T_LANG> this language's {@link CodeLanguage} type
36+
* @param <T_AST_UTIL> the {@link AstUtil} type for this language
37+
* @param <T_AST_EXTRACTOR> {@link AstExtractor} type for this language
38+
*/
39+
public static class CodeLanguageImpl<T_KEYWORD, T_LANG extends CodeLanguage, T_AST_UTIL extends AstUtil<? extends CompoundBlock, T_KEYWORD>, T_AST_EXTRACTOR extends AstExtractor<? extends CompoundBlock>> implements CodeLanguage {
40+
final String displayName;
41+
@Getter final Function<ParseInput, CodeFileSrc<T_LANG>> parser;
42+
@Getter final List<String> fileExtensions;
43+
@Getter final T_AST_UTIL astUtil;
44+
@Getter final Keyword keyword;
45+
@Getter final T_AST_EXTRACTOR extractor;
46+
47+
48+
// package-private
49+
CodeLanguageImpl(String displayName, T_AST_UTIL astUtil, Keyword keyword, Function<ParseInput, CodeFileSrc<T_LANG>> parser, T_AST_EXTRACTOR extractor, String... fileExtensions) {
50+
this.displayName = displayName;
51+
this.parser = parser;
52+
this.fileExtensions = new ArrayList<>(Arrays.asList(fileExtensions));
53+
this.astUtil = astUtil;
54+
this.keyword = keyword;
55+
this.extractor = extractor;
56+
}
57+
58+
59+
@Override
60+
public String displayName() {
61+
return displayName;
62+
}
63+
64+
}
65+
66+
67+
public static class CSharp extends CodeLanguageImpl<CsKeyword, CSharp, CsAstUtil, AstExtractor<CsBlock>> {
3168
CSharp(String displayName, CsAstUtil astUtil, Keyword keyword, Function<ParseInput, CodeFileSrc<CSharp>> parser, AstExtractor<CsBlock> extractor, String... fileExtensions) {
3269
super(displayName, astUtil, keyword, parser, extractor, fileExtensions);
3370
}
3471

3572
}
3673

3774

38-
public static class Java extends CodeLanguageOptions<JavaKeyword, Java, JavaAstUtil, AstExtractor<JavaBlock>> {
75+
public static class Java extends CodeLanguageImpl<JavaKeyword, Java, JavaAstUtil, AstExtractor<JavaBlock>> {
3976
Java(String displayName, JavaAstUtil astUtil, Keyword keyword, Function<ParseInput, CodeFileSrc<Java>> parser, AstExtractor<JavaBlock> extractor, String... fileExtensions) {
4077
super(displayName, astUtil, keyword, parser, extractor, fileExtensions);
4178
}
@@ -44,10 +81,10 @@ public static class Java extends CodeLanguageOptions<JavaKeyword, Java, JavaAstU
4481

4582

4683
public static final Java JAVA = new Java("Java", new JavaAstUtil(), JavaKeyword.check, JavaClassParser::parse, new JavaBlockParser(), "java");
47-
public static final CodeLanguageOptions<Object, CodeLanguage, AstUtil<CompoundBlock, Object>, AstExtractor<CompoundBlock>> JAVASCRIPT = new CodeLanguageOptions<>("Javascript", null, null, null, null, "js", "ts");
84+
public static final CodeLanguageImpl<Object, CodeLanguage, AstUtil<CompoundBlock, Object>, AstExtractor<CompoundBlock>> JAVASCRIPT = new CodeLanguageImpl<>("Javascript", null, null, null, null, "js", "ts");
4885
public static final CSharp C_SHARP = new CSharp("C#", new CsAstUtil(), CsKeyword.check, CsClassParser::parse, new CsBlockParser(), "cs");
49-
public static final CodeLanguageOptions<Object, CodeLanguage, AstUtil<CompoundBlock, Object>, AstExtractor<CompoundBlock>> CSS = new CodeLanguageOptions<>("CSS", null, null, null, null, "css");
50-
public static final CodeLanguageOptions<Object, CodeLanguage, AstUtil<CompoundBlock, Object>, AstExtractor<CompoundBlock>> XML = new CodeLanguageOptions<>("XML", null, null, null, null, "html", "xml");
86+
public static final CodeLanguageImpl<Object, CodeLanguage, AstUtil<CompoundBlock, Object>, AstExtractor<CompoundBlock>> CSS = new CodeLanguageImpl<>("CSS", null, null, null, null, "css");
87+
public static final CodeLanguageImpl<Object, CodeLanguage, AstUtil<CompoundBlock, Object>, AstExtractor<CompoundBlock>> XML = new CodeLanguageImpl<>("XML", null, null, null, null, "html", "xml");
5188

5289
private static CopyOnWriteArrayList<CodeLanguage> values;
5390

@@ -62,31 +99,6 @@ public static class Java extends CodeLanguageOptions<JavaKeyword, Java, JavaAstU
6299
}
63100

64101

65-
final String displayName;
66-
@Getter final Function<ParseInput, CodeFileSrc<T_LANG>> parser;
67-
@Getter final List<String> fileExtensions;
68-
@Getter final T_AST_UTIL astUtil;
69-
@Getter final Keyword keyword;
70-
@Getter final T_AST_EXTRACTOR extractor;
71-
72-
73-
// package-private
74-
CodeLanguageOptions(String displayName, T_AST_UTIL astUtil, Keyword keyword, Function<ParseInput, CodeFileSrc<T_LANG>> parser, T_AST_EXTRACTOR extractor, String... fileExtensions) {
75-
this.displayName = displayName;
76-
this.parser = parser;
77-
this.fileExtensions = new ArrayList<>(Arrays.asList(fileExtensions));
78-
this.astUtil = astUtil;
79-
this.keyword = keyword;
80-
this.extractor = extractor;
81-
}
82-
83-
84-
@Override
85-
public String displayName() {
86-
return displayName;
87-
}
88-
89-
90102
/**
91103
* @return a list of all registered languages
92104
*/
@@ -106,7 +118,7 @@ public static List<CodeLanguage> getLanguages() {
106118
*/
107119
public static <_T_BLOCK extends CompoundBlock, _T_KEYWORD, _T_LANG extends CodeLanguage, _T_AST_UTIL extends AstUtil<_T_BLOCK, _T_KEYWORD>, T_AST_EXTRACTOR extends AstExtractor<_T_BLOCK>> CodeLanguage registerCodeLanguage(
108120
String displayName, _T_AST_UTIL astUtil, Keyword keyword, Function<ParseInput, CodeFileSrc<_T_LANG>> parser, T_AST_EXTRACTOR extractor, String... fileExtensions) {
109-
val inst = new CodeLanguageOptions<>(displayName, astUtil, keyword, parser, extractor, fileExtensions);
121+
val inst = new CodeLanguageImpl<>(displayName, astUtil, keyword, parser, extractor, fileExtensions);
110122
_registerNewLanguage(inst);
111123
return inst;
112124
}

‎src/twg2/parser/codeParser/csharp/CsAnnotationExtractor.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@
66
import java.util.Map;
77

88
import lombok.val;
9+
import twg2.parser.baseAst.AstParser;
910
import twg2.parser.baseAst.tools.AstFragType;
1011
import twg2.parser.baseAst.tools.NameUtil;
1112
import twg2.parser.codeParser.CodeFragmentType;
12-
import twg2.parser.condition.AstParser;
1313
import twg2.parser.documentParser.DocumentFragmentText;
1414
import twg2.parser.intermAst.annotation.AnnotationSig;
1515
import twg2.text.stringUtils.StringTrim;

‎src/twg2/parser/codeParser/csharp/CsBlockParser.java

+7-5
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import lombok.val;
99
import twg2.collections.tuple.Tuples;
1010
import twg2.parser.baseAst.AccessModifierEnum;
11+
import twg2.parser.baseAst.AstParser;
1112
import twg2.parser.baseAst.tools.AstFragType;
1213
import twg2.parser.baseAst.tools.NameUtil;
1314
import twg2.parser.codeParser.AstExtractor;
@@ -18,7 +19,6 @@
1819
import twg2.parser.codeParser.CodeFragmentType;
1920
import twg2.parser.codeParser.CodeLanguageOptions;
2021
import twg2.parser.codeParser.tools.TokenListIterable;
21-
import twg2.parser.condition.AstParser;
2222
import twg2.parser.documentParser.DocumentFragmentText;
2323
import twg2.parser.intermAst.annotation.AnnotationSig;
2424
import twg2.parser.intermAst.block.IntermBlock;
@@ -48,7 +48,7 @@ public AstParser<List<List<String>>> createImportStatementParser() {
4848

4949
@Override
5050
public AstParser<Simple> createTypeParser() {
51-
return new BaseDataTypeExtractor(CodeLanguageOptions.C_SHARP, false);
51+
return new BaseDataTypeExtractor(CodeLanguageOptions.C_SHARP, true);
5252
}
5353

5454

@@ -59,20 +59,22 @@ public AstParser<List<AnnotationSig>> createAnnotationParser(IntermBlock<CsBlock
5959

6060

6161
@Override
62-
public AstParser<List<IntermFieldSig>> createFieldParser(IntermBlock<CsBlock> block, AstParser<TypeSig.Simple> typeParser, AstParser<List<AnnotationSig>> annotationParser) {
62+
public AstParser<List<IntermFieldSig>> createFieldParser(IntermBlock<CsBlock> block, AstParser<List<AnnotationSig>> annotationParser) {
63+
val typeParser = new BaseDataTypeExtractor(CodeLanguageOptions.C_SHARP, false);
6364
return new BaseFieldExtractor("C#", CsKeyword.check, block, typeParser, annotationParser);
6465
}
6566

6667

6768
@Override
68-
public AstParser<List<IntermMethodSig.SimpleImpl>> createMethodParser(IntermBlock<CsBlock> block, AstParser<TypeSig.Simple> typeParser, AstParser<List<AnnotationSig>> annotationParser) {
69+
public AstParser<List<IntermMethodSig.SimpleImpl>> createMethodParser(IntermBlock<CsBlock> block, AstParser<List<AnnotationSig>> annotationParser) {
70+
val typeParser = new BaseDataTypeExtractor(CodeLanguageOptions.C_SHARP, true);
6971
return new BaseMethodExtractor("C#", CsKeyword.check, block, typeParser, annotationParser);
7072
}
7173

7274

7375
@Override
7476
public List<Entry<SimpleTree<DocumentFragmentText<CodeFragmentType>>, IntermClass.SimpleImpl<CsBlock>>> extractClassFieldsAndMethodSignatures(SimpleTree<DocumentFragmentText<CodeFragmentType>> astTree) {
75-
val blocks = BaseBlockParser.extractBlockFieldsAndInterfaceMethods(this, astTree, (block) -> block.getBlockType() != CsBlock.NAMESPACE);
77+
val blocks = BaseBlockParser.extractBlockFieldsAndInterfaceMethods(this, astTree);
7678
return blocks;
7779
}
7880

‎src/twg2/parser/codeParser/csharp/CsUsingStatementExtractor.java

+2-4
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,11 @@
44
import java.util.List;
55

66
import lombok.val;
7-
import twg2.parser.baseAst.csharp.CsAstUtil;
7+
import twg2.parser.baseAst.AstParser;
88
import twg2.parser.baseAst.tools.AstFragType;
99
import twg2.parser.baseAst.tools.NameUtil;
10-
import twg2.parser.codeParser.AstExtractor;
1110
import twg2.parser.codeParser.CodeFragmentType;
1211
import twg2.parser.codeParser.CodeLanguageOptions;
13-
import twg2.parser.condition.AstParser;
1412
import twg2.parser.documentParser.DocumentFragmentText;
1513
import twg2.treeLike.simpleTree.SimpleTree;
1614

@@ -28,7 +26,7 @@ static enum State {
2826
}
2927

3028

31-
private static final CodeLanguageOptions<CsKeyword, CodeLanguageOptions.CSharp, CsAstUtil, AstExtractor<CsBlock>> lang = CodeLanguageOptions.C_SHARP;
29+
private static final CodeLanguageOptions.CSharp lang = CodeLanguageOptions.C_SHARP;
3230

3331
List<List<String>> usingStatements = new ArrayList<>();
3432
State state = State.INIT;

‎src/twg2/parser/codeParser/java/JavaAnnotationExtractor.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@
66
import java.util.Map;
77

88
import lombok.val;
9+
import twg2.parser.baseAst.AstParser;
910
import twg2.parser.baseAst.tools.AstFragType;
1011
import twg2.parser.baseAst.tools.NameUtil;
1112
import twg2.parser.codeParser.CodeFragmentType;
12-
import twg2.parser.condition.AstParser;
1313
import twg2.parser.documentParser.DocumentFragmentText;
1414
import twg2.parser.intermAst.annotation.AnnotationSig;
1515
import twg2.text.stringUtils.StringTrim;

0 commit comments

Comments
 (0)