Skip to content

Commit 3e8a324

Browse files
TeamworkGuy2TeamworkGuy2
committed
0.10.0 - Better annotation parsing. New CodeFragment class replaces verbose 'DocumentFragmentText<CodeFragmentType>' type. Added Operator and OperatorUtil (similar to KeywordUtil and AccessModifier). Changed how KeywordUtil exposes its enum sub-categories and designed OperatorUtil to follow the same pattern. Moved extractor classes (i.e. twg2.parser.codeParser.AccessModifierExtractor) to new 'twg2.parser.codeParser.extractors' package.
1 parent 679778b commit 3e8a324

File tree

67 files changed

+1420
-847
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+1420
-847
lines changed

‎README.md

+15-8
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
ParserToolsTmp
22
==============
3-
version: 0.9.0
3+
version: 0.10.0
44

55
In progress C#/Java/TypeScript parser tools built atop [JTextParser] (https://github.com/TeamworkGuy2/JTextParser), [Jackson] (https://github.com/FasterXML/jackson-core/) (core, databind, annotations) and half a dozen other utility libraries.
66

@@ -92,14 +92,16 @@ JSON Result (printed to System.out):
9292
"primitive": true
9393
},
9494
"accessModifiers": ["private"],
95-
"annotations": []
95+
"annotations": [],
96+
"comments": [" <value>The modification count.</value>\n"]
9697
}, {
9798
"name": "ParserExamples.Samples.SimpleCs._name",
9899
"type": {
99100
"typeName": "string"
100101
},
101102
"accessModifiers": ["private"],
102-
"annotations": []
103+
"annotations": [],
104+
"comments": [" <value>The name.</value>\n"]
103105
}, {
104106
"name": "ParserExamples.Samples.SimpleCs.Names",
105107
"type": {
@@ -109,30 +111,34 @@ JSON Result (printed to System.out):
109111
}]
110112
},
111113
"accessModifiers": ["public"],
112-
"annotations": []
114+
"annotations": [],
115+
"comments": [" <value>The names.</value>\n"]
113116
}, {
114117
"name": "ParserExamples.Samples.SimpleCs.Count",
115118
"type": {
116119
"typeName": "int",
117120
"primitive": true
118121
},
119122
"accessModifiers": ["public"],
120-
"annotations": []
123+
"annotations": [],
124+
"comments": [" <value>The number of names.</value>\n"]
121125
}, {
122126
"name": "ParserExamples.Samples.SimpleCs.accesses",
123127
"type": {
124128
"typeName": "DateTime",
125129
"arrayDimensions": 1
126130
},
127131
"accessModifiers": ["public"],
128-
"annotations": []
132+
"annotations": [],
133+
"comments": [" <value>The access timestamps.</value>\n"]
129134
}, {
130135
"name": "ParserExamples.Samples.SimpleCs.name",
131136
"type": {
132137
"typeName": "string"
133138
},
134139
"accessModifiers": ["public"],
135-
"annotations": []
140+
"annotations": [],
141+
"comments": [" <value>The access timestamps.</value>\n"]
136142
}],
137143
"methods": [{
138144
"name": "ParserExamples.Samples.SimpleCs.AddName",
@@ -165,7 +171,8 @@ JSON Result (printed to System.out):
165171
"typeName": "String"
166172
}]
167173
}]
168-
}
174+
},
175+
"comments": [" <summary>Add name</summary>\n", " <param name=\"name\">the name</param>\n", " <returns>the names</returns>\n"]
169176
}]
170177
}
171178
```

‎package-lib.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version" : "0.9.0",
2+
"version" : "0.10.0",
33
"name" : "jparser-tools",
44
"description" : "An in-progress suite of parsing tools for C#, Java, and TypeScript source code",
55
"homepage" : "https://github.com/TeamworkGuy2/JFileIo",

‎src/twg2/ast/interm/block/BlockAst.java

+3-4
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@
44
import twg2.annotations.Immutable;
55
import twg2.ast.interm.classes.ClassSig;
66
import twg2.parser.baseAst.CompoundBlock;
7-
import twg2.parser.codeParser.CodeFragmentType;
8-
import twg2.parser.documentParser.DocumentFragmentText;
7+
import twg2.parser.documentParser.CodeFragment;
98
import twg2.treeLike.simpleTree.SimpleTree;
109

1110
/**
@@ -15,11 +14,11 @@
1514
@Immutable
1615
public class BlockAst<T_BLOCK extends CompoundBlock> {
1716
private final @Getter ClassSig.SimpleImpl declaration;
18-
private final @Getter SimpleTree<DocumentFragmentText<CodeFragmentType>> blockTree;
17+
private final @Getter SimpleTree<CodeFragment> blockTree;
1918
private final @Getter T_BLOCK blockType;
2019

2120

22-
public BlockAst(ClassSig.SimpleImpl declaration, SimpleTree<DocumentFragmentText<CodeFragmentType>> blockTree, T_BLOCK blockType) {
21+
public BlockAst(ClassSig.SimpleImpl declaration, SimpleTree<CodeFragment> blockTree, T_BLOCK blockType) {
2322
this.declaration = declaration;
2423
this.blockTree = blockTree;
2524
this.blockType = blockType;

‎src/twg2/parser/baseAst/AstParser.java

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,15 @@
11
package twg2.parser.baseAst;
22

3-
import twg2.parser.codeParser.CodeFragmentType;
43
import twg2.parser.condition.TokenParser;
5-
import twg2.parser.documentParser.DocumentFragmentText;
4+
import twg2.parser.documentParser.CodeFragment;
65
import twg2.treeLike.simpleTree.SimpleTree;
76

87
/**
98
* @author TeamworkGuy2
109
* @since 2015-12-12
1110
* @param <T_RESULT> the type of result object that parsed data is store in
1211
*/
13-
public interface AstParser<T_RESULT> extends TokenParser<SimpleTree<DocumentFragmentText<CodeFragmentType>>, T_RESULT> {
12+
public interface AstParser<T_RESULT> extends TokenParser<SimpleTree<CodeFragment>, T_RESULT> {
1413

1514
@Override
1615
public AstParser<T_RESULT> copy();

‎src/twg2/parser/baseAst/AstTypeChecker.java

+5-6
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package twg2.parser.baseAst;
22

3-
import twg2.parser.codeParser.CodeFragmentType;
4-
import twg2.parser.documentParser.DocumentFragmentText;
3+
import twg2.parser.documentParser.CodeFragment;
54
import twg2.treeLike.simpleTree.SimpleTree;
65

76
/**
@@ -16,12 +15,12 @@ public interface AstTypeChecker<T_KEYWORD> {
1615
* @param tokenNode the document code fragment
1716
* @return true if the token represents a field block, false if not
1817
*/
19-
public boolean isFieldBlock(SimpleTree<DocumentFragmentText<CodeFragmentType>> tokenNode);
18+
public boolean isFieldBlock(SimpleTree<CodeFragment> tokenNode);
2019

21-
public boolean isKeyword(DocumentFragmentText<CodeFragmentType> node, T_KEYWORD keyword1);
20+
public boolean isKeyword(CodeFragment node, T_KEYWORD keyword1);
2221

23-
public boolean isKeyword(DocumentFragmentText<CodeFragmentType> node, T_KEYWORD keyword1, T_KEYWORD keyword2);
22+
public boolean isKeyword(CodeFragment node, T_KEYWORD keyword1, T_KEYWORD keyword2);
2423

25-
public boolean isKeyword(DocumentFragmentText<CodeFragmentType> node, T_KEYWORD keyword1, T_KEYWORD keyword2, T_KEYWORD keyword3);
24+
public boolean isKeyword(CodeFragment node, T_KEYWORD keyword1, T_KEYWORD keyword2, T_KEYWORD keyword3);
2625

2726
}

‎src/twg2/parser/baseAst/Operator.java

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package twg2.parser.baseAst;
2+
3+
import twg2.collections.primitiveCollections.IntListReadOnly;
4+
5+
/**
6+
* @author TeamworkGuy2
7+
* @since 2016-4-13
8+
*/
9+
public interface Operator {
10+
11+
public IntListReadOnly operandCount();
12+
13+
}

‎src/twg2/parser/baseAst/csharp/CsAstUtil.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import twg2.parser.codeParser.CodeFragmentType;
1010
import twg2.parser.codeParser.csharp.CsBlock;
1111
import twg2.parser.codeParser.csharp.CsKeyword;
12-
import twg2.parser.documentParser.DocumentFragmentText;
12+
import twg2.parser.documentParser.CodeFragment;
1313
import twg2.parser.language.CodeLanguage;
1414
import twg2.parser.language.CodeLanguageOptions;
1515
import twg2.treeLike.simpleTree.SimpleTree;
@@ -108,25 +108,25 @@ public final AccessModifierEnum tryFromLanguageSrc(String src) {
108108

109109

110110
@Override
111-
public boolean isKeyword(DocumentFragmentText<CodeFragmentType> node, CsKeyword keyword1) {
111+
public boolean isKeyword(CodeFragment node, CsKeyword keyword1) {
112112
return node != null && (node.getFragmentType() == CodeFragmentType.KEYWORD && keyword1.toSrc().equals(node.getText()));
113113
}
114114

115115

116116
@Override
117-
public boolean isKeyword(DocumentFragmentText<CodeFragmentType> node, CsKeyword keyword1, CsKeyword keyword2) {
117+
public boolean isKeyword(CodeFragment node, CsKeyword keyword1, CsKeyword keyword2) {
118118
return node != null && (node.getFragmentType() == CodeFragmentType.KEYWORD && (keyword1.toSrc().equals(node.getText()) || keyword2.toSrc().equals(node.getText())));
119119
}
120120

121121

122122
@Override
123-
public boolean isKeyword(DocumentFragmentText<CodeFragmentType> node, CsKeyword keyword1, CsKeyword keyword2, CsKeyword keyword3) {
123+
public boolean isKeyword(CodeFragment node, CsKeyword keyword1, CsKeyword keyword2, CsKeyword keyword3) {
124124
return node != null && (node.getFragmentType() == CodeFragmentType.KEYWORD && (keyword1.toSrc().equals(node.getText()) || keyword2.toSrc().equals(node.getText()) || keyword3.toSrc().equals(node.getText())));
125125
}
126126

127127

128128
@Override
129-
public boolean isFieldBlock(SimpleTree<DocumentFragmentText<CodeFragmentType>> block) {
129+
public boolean isFieldBlock(SimpleTree<CodeFragment> block) {
130130
if(block == null) { return true; }
131131
val childs = block.getChildren();
132132
// properties must have at-least one indexer

‎src/twg2/parser/baseAst/java/JavaAstUtil.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import twg2.parser.codeParser.CodeFragmentType;
99
import twg2.parser.codeParser.java.JavaBlock;
1010
import twg2.parser.codeParser.java.JavaKeyword;
11-
import twg2.parser.documentParser.DocumentFragmentText;
11+
import twg2.parser.documentParser.CodeFragment;
1212
import twg2.parser.language.CodeLanguage;
1313
import twg2.parser.language.CodeLanguageOptions;
1414
import twg2.treeLike.simpleTree.SimpleTree;
@@ -110,27 +110,27 @@ public final AccessModifierEnum tryFromLanguageSrc(String src) {
110110

111111

112112
@Override
113-
public boolean isKeyword(DocumentFragmentText<CodeFragmentType> node, JavaKeyword keyword1) {
113+
public boolean isKeyword(CodeFragment node, JavaKeyword keyword1) {
114114
return node != null && (node.getFragmentType() == CodeFragmentType.KEYWORD && keyword1.toSrc().equals(node.getText()));
115115
}
116116

117117

118118
@Override
119-
public boolean isKeyword(DocumentFragmentText<CodeFragmentType> node, JavaKeyword keyword1, JavaKeyword keyword2) {
119+
public boolean isKeyword(CodeFragment node, JavaKeyword keyword1, JavaKeyword keyword2) {
120120
return node != null && (node.getFragmentType() == CodeFragmentType.KEYWORD && (keyword1.toSrc().equals(node.getText()) || keyword2.toSrc().equals(node.getText())));
121121
}
122122

123123

124124
@Override
125-
public boolean isKeyword(DocumentFragmentText<CodeFragmentType> node, JavaKeyword keyword1, JavaKeyword keyword2, JavaKeyword keyword3) {
125+
public boolean isKeyword(CodeFragment node, JavaKeyword keyword1, JavaKeyword keyword2, JavaKeyword keyword3) {
126126
return node != null && (node.getFragmentType() == CodeFragmentType.KEYWORD && (keyword1.toSrc().equals(node.getText()) || keyword2.toSrc().equals(node.getText()) || keyword3.toSrc().equals(node.getText())));
127127
}
128128

129129

130130
/** Java does not have a field block
131131
*/
132132
@Override
133-
public boolean isFieldBlock(SimpleTree<DocumentFragmentText<CodeFragmentType>> tokenNode) {
133+
public boolean isFieldBlock(SimpleTree<CodeFragment> tokenNode) {
134134
return false;
135135
}
136136

‎src/twg2/parser/baseAst/tools/AstFragType.java

+7-7
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@
55
import lombok.val;
66
import twg2.arrays.ArrayUtil;
77
import twg2.parser.codeParser.CodeFragmentType;
8+
import twg2.parser.documentParser.CodeFragment;
89
import twg2.parser.documentParser.DocumentFragment;
9-
import twg2.parser.documentParser.DocumentFragmentText;
1010
import twg2.treeLike.simpleTree.SimpleTree;
1111

1212
/**
@@ -53,33 +53,33 @@ public static final boolean isType(DocumentFragment<?, CodeFragmentType> node, C
5353
}
5454

5555

56-
public static final boolean isOptionalTypeMarker(DocumentFragmentText<CodeFragmentType> node) {
56+
public static final boolean isOptionalTypeMarker(CodeFragment node) {
5757
return node != null && (node.getFragmentType() == CodeFragmentType.OPERATOR && "?".equals(node.getText()));
5858
}
5959

6060

61-
public static final boolean isIdentifier(DocumentFragmentText<CodeFragmentType> node) {
61+
public static final boolean isIdentifier(CodeFragment node) {
6262
return node != null && (node.getFragmentType() == CodeFragmentType.IDENTIFIER);
6363
}
6464

6565

66-
public static final boolean isIdentifierOrKeyword(DocumentFragmentText<CodeFragmentType> node) {
66+
public static final boolean isIdentifierOrKeyword(CodeFragment node) {
6767
return node != null && (node.getFragmentType() == CodeFragmentType.KEYWORD || node.getFragmentType() == CodeFragmentType.IDENTIFIER);
6868
}
6969

7070

71-
public static final boolean isKeyword(DocumentFragmentText<CodeFragmentType> node) {
71+
public static final boolean isKeyword(CodeFragment node) {
7272
return node != null && (node.getFragmentType() == CodeFragmentType.KEYWORD);
7373
}
7474

7575

76-
public static final boolean isBlock(DocumentFragmentText<CodeFragmentType> node, String blockSymbol) {
76+
public static final boolean isBlock(CodeFragment node, String blockSymbol) {
7777
return node != null && node.getFragmentType().isCompound() && node.getText().startsWith(blockSymbol);
7878
}
7979

8080

8181
// TODO unused
82-
public static final boolean blockContainsOnly(SimpleTree<DocumentFragmentText<CodeFragmentType>> block, BiPredicate<DocumentFragmentText<CodeFragmentType>, CodeFragmentType> cond, boolean emptyTreeValid, CodeFragmentType... optionalAllows) {
82+
public static final boolean blockContainsOnly(SimpleTree<CodeFragment> block, BiPredicate<CodeFragment, CodeFragmentType> cond, boolean emptyTreeValid, CodeFragmentType... optionalAllows) {
8383
if(block == null) {
8484
return emptyTreeValid;
8585
}

‎src/twg2/parser/codeParser/AstExtractor.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
import twg2.ast.interm.type.TypeSig;
1212
import twg2.parser.baseAst.AstParser;
1313
import twg2.parser.baseAst.CompoundBlock;
14-
import twg2.parser.documentParser.DocumentFragmentText;
14+
import twg2.parser.documentParser.CodeFragment;
1515
import twg2.treeLike.simpleTree.SimpleTree;
1616

1717
/**
@@ -34,9 +34,9 @@ public interface AstExtractor<T_BLOCK extends CompoundBlock> {
3434
public AstParser<List<MethodSig.SimpleImpl>> createMethodParser(BlockAst<T_BLOCK> block, AstParser<List<AnnotationSig>> annotationParser, AstParser<List<String>> commentParser);
3535

3636

37-
public List<BlockAst<T_BLOCK>> extractBlocks(List<String> nameScope, SimpleTree<DocumentFragmentText<CodeFragmentType>> astTree, BlockAst<T_BLOCK> parentScope);
37+
public List<BlockAst<T_BLOCK>> extractBlocks(List<String> nameScope, SimpleTree<CodeFragment> astTree, BlockAst<T_BLOCK> parentScope);
3838

3939

40-
public List<Entry<SimpleTree<DocumentFragmentText<CodeFragmentType>>, ClassAst.SimpleImpl<T_BLOCK>>> extractClassFieldsAndMethodSignatures(SimpleTree<DocumentFragmentText<CodeFragmentType>> astTree);
40+
public List<Entry<SimpleTree<CodeFragment>, ClassAst.SimpleImpl<T_BLOCK>>> extractClassFieldsAndMethodSignatures(SimpleTree<CodeFragment> astTree);
4141

4242
}

‎src/twg2/parser/codeParser/CodeFileParsed.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import twg2.ast.interm.classes.ClassAst;
66
import twg2.ast.interm.classes.ClassSig;
77
import twg2.parser.baseAst.CompoundBlock;
8-
import twg2.parser.documentParser.DocumentFragmentText;
8+
import twg2.parser.documentParser.CodeFragment;
99
import twg2.parser.output.JsonWritableSig;
1010
import twg2.treeLike.simpleTree.SimpleTree;
1111

@@ -17,14 +17,14 @@
1717
public class CodeFileParsed<T_ID, T_CLASS extends ClassAst<? extends ClassSig, ? extends JsonWritableSig, ? extends CompoundBlock>> {
1818
@Getter T_ID id;
1919
@Getter T_CLASS parsedClass;
20-
@Getter SimpleTree<DocumentFragmentText<CodeFragmentType>> astTree;
20+
@Getter SimpleTree<CodeFragment> astTree;
2121

2222

2323

2424

2525
public static class Simple<T_ID, T_BLOCK extends CompoundBlock> extends CodeFileParsed<T_ID, ClassAst.SimpleImpl<T_BLOCK>> {
2626

27-
public Simple(T_ID id, ClassAst.SimpleImpl<T_BLOCK> parsedClass, SimpleTree<DocumentFragmentText<CodeFragmentType>> astTree) {
27+
public Simple(T_ID id, ClassAst.SimpleImpl<T_BLOCK> parsedClass, SimpleTree<CodeFragment> astTree) {
2828
super(id, parsedClass, astTree);
2929
}
3030

@@ -35,7 +35,7 @@ public Simple(T_ID id, ClassAst.SimpleImpl<T_BLOCK> parsedClass, SimpleTree<Docu
3535

3636
public static class Resolved<T_ID, T_BLOCK extends CompoundBlock> extends CodeFileParsed<T_ID, ClassAst.ResolvedImpl<T_BLOCK>> {
3737

38-
public Resolved(T_ID id, ClassAst.ResolvedImpl<T_BLOCK> parsedClass, SimpleTree<DocumentFragmentText<CodeFragmentType>> astTree) {
38+
public Resolved(T_ID id, ClassAst.ResolvedImpl<T_BLOCK> parsedClass, SimpleTree<CodeFragment> astTree) {
3939
super(id, parsedClass, astTree);
4040
}
4141

‎src/twg2/parser/codeParser/CodeFileSrc.java

+6-5
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,17 @@
33
import java.util.List;
44

55
import lombok.Getter;
6-
import twg2.parser.documentParser.DocumentFragmentText;
6+
import twg2.parser.documentParser.CodeFragment;
77
import twg2.treeLike.simpleTree.SimpleTree;
88

99
/** All the information resulting from building a source string into an AST.
10-
* Contains an {@link DocumentFragmentText} AST tree, source name, source string, source string as list of lines, and a language.
10+
* Contains an {@link CodeFragment} AST tree, source name, source string, source string as list of lines, and a language.
1111
* @author TeamworkGuy2
1212
* @since 2015-11-22
1313
* @param <T_LANG> the code language of the source file
1414
*/
1515
public class CodeFileSrc<T_LANG> {
16-
@Getter SimpleTree<DocumentFragmentText<CodeFragmentType>> doc;
16+
@Getter SimpleTree<CodeFragment> doc;
1717
@Getter String srcName;
1818
@Getter String src;
1919
@Getter List<char[]> lines;
@@ -26,8 +26,9 @@ public class CodeFileSrc<T_LANG> {
2626
* @param lines
2727
* @param language optional
2828
*/
29-
public CodeFileSrc(SimpleTree<DocumentFragmentText<CodeFragmentType>> doc, String srcName, String src, List<char[]> lines, T_LANG language) {
30-
this.doc = doc;
29+
@SuppressWarnings("unchecked")
30+
public CodeFileSrc(SimpleTree<? extends CodeFragment> doc, String srcName, String src, List<char[]> lines, T_LANG language) {
31+
this.doc = (SimpleTree<CodeFragment>) doc;
3132
this.srcName = srcName;
3233
this.src = src;
3334
this.lines = lines;

0 commit comments

Comments
 (0)