Skip to content

Commit eea353c

Browse files
TeamworkGuy2TeamworkGuy2
committed
0.5.0 - Added array type parsing (i.e. able to parse IList<int[]>), some minor refactoring and additional tests
1 parent ab23d86 commit eea353c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+733
-600
lines changed

‎.classpath

+16-14
Original file line numberDiff line numberDiff line change
@@ -4,25 +4,27 @@
44
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"/>
55
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
66
<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/ANTLR"/>
7-
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jtext-util/jar/jtext_util.jar" sourcepath="/JTextUtility"/>
8-
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jdata-util/jar/jdata_util.jar" sourcepath="/JDataUtility"/>
97
<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/TestUtil"/>
10-
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jcollection-util/jar/jcollection_util.jar" sourcepath="/JCollectionUtil"/>
11-
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jprimitive-collections/jar/jprimitive_collections.jar" sourcepath="/JPrimitiveCollections"/>
128
<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/Lombok"/>
13-
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jsimple-types/jar/jsimple_types.jar" sourcepath="/JSimpleTypes/src"/>
14-
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jtext-parser/jar/jtext_parser.jar" sourcepath="/JTextParser/src"/>
15-
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jstreamish/jar/jstreamish.jar" sourcepath="/JStreamish"/>
16-
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jrange/jar/jrange.jar" sourcepath="/JRange"/>
17-
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jfunc/jar/jfunc.jar" sourcepath="/JFunc"/>
18-
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/json-io/jar/json_io.jar" sourcepath="/JsonIo"/>
9+
<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/TemplateUtil"/>
10+
<classpathentry kind="output" path="bin"/>
11+
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/data-transfer/jar/data_transfer.jar"/>
1912
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jackson/jar/jackson-annotations-2.5.0.jar"/>
2013
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jackson/jar/jackson-core-2.5.0.jar"/>
2114
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jackson/jar/jackson-databind-2.5.4.jar"/>
15+
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jarray-util/jar/jarray_util.jar" sourcepath="/JArrayUtil"/>
16+
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jcollection-util/jar/jcollection_util.jar" sourcepath="/JCollectionUtil"/>
17+
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jdata-util/jar/jdata_util.jar" sourcepath="/JDataUtil"/>
2218
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jfile-io/jar/jfile_io.jar" sourcepath="/JFileIo"/>
23-
<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/TemplateUtil"/>
24-
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jtree-walker/jar/jtree_walker.jar" sourcepath="/JTreeWalker"/>
25-
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/data-transfer/jar/data_transfer.jar"/>
19+
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jfunc/jar/jfunc.jar" sourcepath="/JFunc"/>
20+
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jinterfaces/jar/jinterfaces.jar" sourcepath="/JInterfaces"/>
2621
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jparser-data-type-like/jar/jparser_data_type_like.jar" sourcepath="/JParserDataTypeLike"/>
27-
<classpathentry kind="output" path="bin"/>
22+
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jprimitive-collections/jar/jprimitive_collections.jar" sourcepath="/JPrimitiveCollections"/>
23+
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jrange/jar/jrange.jar" sourcepath="/JRange"/>
24+
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jsimple-types/jar/jsimple_types.jar" sourcepath="/JSimpleTypes/src"/>
25+
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/json-io/jar/json_io.jar" sourcepath="/JsonIo"/>
26+
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jstreamish/jar/jstreamish.jar" sourcepath="/JStreamish"/>
27+
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jtext-parser/jar/jtext_parser.jar" sourcepath="/JTextParser/src"/>
28+
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jtext-util/jar/jtext_util.jar" sourcepath="/JTextUtil"/>
29+
<classpathentry kind="lib" path="C:/Users/TeamworkGuy2/Documents/Java/Libraries/jtree-walker/jar/jtree_walker.jar" sourcepath="/JTreeWalker"/>
2830
</classpath>

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
ParserToolsTmp
22
==============
3-
version: 0.4.0
3+
version: 0.5.0
44

55
In progress C#/Java/TypeScript parser tools built atop [JTextParser] (https://github.com/TeamworkGuy2/JTextParser), [Jackson] (https://github.com/FasterXML/jackson-core/) (core, databind, annotations) and half a dozen other utility libraries.
66

‎package-lib.json

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"version" : "0.5.0",
3+
"name" : "jparser-tools",
4+
"description" : "An in-progress suite of parsing tools for C#, Java, and TypeScript source code",
5+
"homepage" : "https://github.com/TeamworkGuy2/JFileIo",
6+
"license" : "MIT",
7+
"main" : "./jar/jparser_tools.jar",
8+
"dependencies" : {
9+
"data-transfer": "*",
10+
"jackson": "*",
11+
"jackson": "*",
12+
"jackson": "*",
13+
"jarray-util": "*",
14+
"jcollection-util": "*",
15+
"jdata-util": "*",
16+
"jfile-io": "*",
17+
"jfunc": "*",
18+
"jinterfaces": "*",
19+
"jparser-data-type-like": "*",
20+
"jprimitive-collections": "*",
21+
"jrange": "*",
22+
"jsimple-types": "*",
23+
"json-io": "*",
24+
"jstreamish": "*",
25+
"jtext-parser": "*",
26+
"jtext-util": "*",
27+
"jtree-walker": "*"
28+
}
29+
}

‎src/twg2/parser/codeParser/BaseDataTypeExtractor.java

+16-10
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,9 @@
22

33
import java.util.ArrayList;
44
import java.util.List;
5-
import java.util.Map;
65

76
import lombok.val;
8-
import twg2.collections.util.ListBuilder;
7+
import twg2.collections.builder.ListBuilder;
98
import twg2.parser.baseAst.AstParser;
109
import twg2.parser.baseAst.tools.AstFragType;
1110
import twg2.parser.documentParser.DocumentFragmentText;
@@ -151,7 +150,7 @@ public static TypeSig.Simple extractGenericTypes(String typeSig) {
151150
throw new IllegalArgumentException("cannot parse a type signature containing '" + genericMark + "' (because this is a simple parser implementation)");
152151
}
153152

154-
StringBuilder sb = new StringBuilder(typeSig);
153+
val sb = new StringBuilder(typeSig);
155154
val genericParamSets = new ArrayList<String>();
156155
int i = 0;
157156
while(true) {
@@ -168,14 +167,17 @@ public static TypeSig.Simple extractGenericTypes(String typeSig) {
168167
}
169168

170169
// convert the generic parameters to TypeSig nested
171-
Map.Entry<String, String> rootNameAndMarker = StringSplit.firstMatchParts(sb.toString(), "#");
172-
TypeSig.SimpleBaseImpl root = new TypeSig.SimpleBaseImpl(StringTrim.trimTrailing(rootNameAndMarker.getKey(), '?'), rootNameAndMarker.getKey().endsWith("?"));
170+
val rootNameAndMarker = StringSplit.firstMatchParts(sb.toString(), "#");
171+
val paramName = StringTrim.trimTrailing(rootNameAndMarker.getKey(), '?');
172+
val nameAndArrayDimensions = StringTrim.countAndTrimTrailing(paramName, "[]", true);
173+
val isOptional = rootNameAndMarker.getKey().endsWith("?");
174+
TypeSig.SimpleBaseImpl root = new TypeSig.SimpleBaseImpl(nameAndArrayDimensions.getValue(), nameAndArrayDimensions.getKey(), isOptional);
173175
TypeSig.Simple sig;
174176

175177
int rootMarker = !StringCheck.isNullOrEmpty(rootNameAndMarker.getValue()) ? Integer.parseInt(rootNameAndMarker.getValue()) : -1;
176178
if(rootMarker > -1) {
177179
val sigChilds = expandGenericParamSet(root, rootMarker, genericParamSets);
178-
sig = new TypeSig.SimpleGenericImpl(root.getTypeName(), sigChilds, root.isNullable());
180+
sig = new TypeSig.SimpleGenericImpl(root.getTypeName(), sigChilds, root.getArrayDimensions(), root.isNullable());
179181
}
180182
else {
181183
sig = root;
@@ -188,21 +190,25 @@ public static TypeSig.Simple extractGenericTypes(String typeSig) {
188190
public static List<TypeSig.Simple> expandGenericParamSet(TypeSig.SimpleBaseImpl parent, int parentParamMarker, List<String> remainingParamSets) {
189191
String paramSetStr = remainingParamSets.get(parentParamMarker);
190192
remainingParamSets.remove(parentParamMarker);
191-
List<String> params = ListBuilder.newMutable(paramSetStr.split(", "));
192193
List<TypeSig.Simple> paramSigs = new ArrayList<>();
194+
List<String> params = ListBuilder.mutable(paramSetStr.split(", "));
195+
193196
for(String param : params) {
194197
// Split the generic parameter name and possible marker indicating further nested generic type
195-
Map.Entry<String, String> paramNameAndMarker = StringSplit.firstMatchParts(param, "#");
198+
val paramNameAndMarker = StringSplit.firstMatchParts(param, "#");
196199

197200
// Create basic generic parameter using the name
198-
TypeSig.SimpleBaseImpl paramSigInit = new TypeSig.SimpleBaseImpl(StringTrim.trimTrailing(paramNameAndMarker.getKey(), '?'), paramNameAndMarker.getKey().endsWith("?"));
201+
val paramName = StringTrim.trimTrailing(paramNameAndMarker.getKey(), '?');
202+
val nameAndArrayDimensions = StringTrim.countAndTrimTrailing(paramName, "[]", true);
203+
val isOptional = paramNameAndMarker.getKey().endsWith("?");
204+
TypeSig.SimpleBaseImpl paramSigInit = new TypeSig.SimpleBaseImpl(nameAndArrayDimensions.getValue(), nameAndArrayDimensions.getKey(), isOptional);
199205
TypeSig.Simple paramSig;
200206

201207
// if this generic parameter has a marker, parse it's sub parameters and add them to a new compound generic type signature
202208
int paramMarker = !StringCheck.isNullOrEmpty(paramNameAndMarker.getValue()) ? Integer.parseInt(paramNameAndMarker.getValue()) : -1;
203209
if(paramMarker > -1) {
204210
val childParams = expandGenericParamSet(paramSigInit, paramMarker, remainingParamSets);
205-
paramSig = new TypeSig.SimpleGenericImpl(paramSigInit.getTypeName(), childParams, paramSigInit.isNullable());
211+
paramSig = new TypeSig.SimpleGenericImpl(paramSigInit.getTypeName(), childParams, paramSigInit.getArrayDimensions(), paramSigInit.isNullable());
206212
}
207213
// else just use the generic parameter's basic signature (no nested generic types)
208214
else {

‎src/twg2/parser/codeParser/CodeLanguage.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
import java.util.List;
44
import java.util.function.Function;
55

6-
import twg2.parser.baseAst.CompoundBlock;
76
import twg2.parser.baseAst.AstUtil;
7+
import twg2.parser.baseAst.CompoundBlock;
88

99
/**
1010
* @author TeamworkGuy2

‎src/twg2/parser/codeParser/CodeLanguageOptions.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88

99
import lombok.Getter;
1010
import lombok.val;
11-
import twg2.parser.baseAst.CompoundBlock;
1211
import twg2.parser.baseAst.AstUtil;
12+
import twg2.parser.baseAst.CompoundBlock;
1313
import twg2.parser.baseAst.csharp.CsAstUtil;
1414
import twg2.parser.baseAst.java.JavaAstUtil;
1515
import twg2.parser.codeParser.csharp.CsBlock;

‎src/twg2/parser/codeParser/codeStats/ParseDirectoryCodeFiles.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@
1616
import lombok.Getter;
1717
import lombok.NoArgsConstructor;
1818
import lombok.val;
19+
import twg2.collections.builder.ListUtil;
1920
import twg2.collections.tuple.Tuples;
20-
import twg2.collections.util.ListUtil;
2121
import twg2.io.files.FileReadUtil;
2222
import twg2.io.files.FileVisitorUtil;
2323
import twg2.io.json.Json;
@@ -133,13 +133,13 @@ public static List<Path> loadFiles(Path projectDir, String... allowedFileExtensi
133133
}
134134

135135

136-
public static ParseDirectoryCodeFiles parseFileStats(Path relativePath, List<Path> files) throws IOException {
136+
public static ParseDirectoryCodeFiles parseFileStats(Path relativePath, List<Path> files, FileReadUtil fileReader) throws IOException {
137137
List<ParsedFileStats> filesStats = new ArrayList<>();
138138

139139
for(Path path : files) {
140140
File file = path.toFile();
141141
String fullFileName = file.getName();
142-
String srcStr = StringReplace.replace(FileReadUtil.defaultInst.readString(new FileReader(file)), "\r\n", "\n");
142+
String srcStr = StringReplace.replace(fileReader.readString(new FileReader(file)), "\r\n", "\n");
143143
Entry<String, String> fileNameExt = StringSplit.lastMatchParts(fullFileName, ".");
144144
if("json".equals(fileNameExt.getValue())) {
145145
int lineCount = StringSplit.countMatches(srcStr, "\n");

‎src/twg2/parser/codeParser/csharp/CsBlockParser.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -125,11 +125,11 @@ public static void _extractBlocksFromTree(List<String> nameScope, SimpleTree<Doc
125125

126126
nameScope.add(nameCompoundRes.getKey());
127127

128-
val blockGenericTypesSig = BaseDataTypeExtractor.extractGenericTypes(NameUtil.joinFqName(nameScope));
129-
val blockGenericTypes = blockGenericTypesSig.isGeneric() ? blockGenericTypesSig.getGenericParams() : Collections.<TypeSig.Simple>emptyList();
130-
val blockFqName = NameUtil.splitFqName(blockGenericTypesSig.getTypeName());
128+
val blockSig = BaseDataTypeExtractor.extractGenericTypes(NameUtil.joinFqName(nameScope));
129+
val blockTypes = blockSig.isGeneric() ? blockSig.getParams() : Collections.<TypeSig.Simple>emptyList();
130+
val blockFqName = NameUtil.splitFqName(blockSig.getTypeName());
131131

132-
blocks.add(new IntermBlock<>(new IntermClassSig.SimpleImpl(access, blockFqName, blockGenericTypes, blockTypeStr, nameCompoundRes.getValue()), child, blockType));
132+
blocks.add(new IntermBlock<>(new IntermClassSig.SimpleImpl(access, blockFqName, blockTypes, blockTypeStr, nameCompoundRes.getValue()), child, blockType));
133133
}
134134

135135
childIter.reset(mark);

‎src/twg2/parser/codeParser/csharp/CsClassParser.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -47,17 +47,17 @@ public static CodeFileSrc<CodeLanguageOptions.CSharp> parse(ParseInput params) {
4747
}
4848

4949

50-
static CharParserFactory createAnnotationParser() {
50+
public static CharParserFactory createAnnotationParser() {
5151
CharParserFactory annotationParser = new StringBoundedParserBuilder("C# annotation")
52-
.addStartEndNotPrecededByMarkers("block [ ]", '[', '[', ']', Inclusion.INCLUDE)
52+
.addStartEndNotPrecededByMarkers("block [ ]", '[', '[', ']', 1, Inclusion.INCLUDE)
5353
.isCompound(true)
5454
.build();
5555
return annotationParser;
5656
}
5757

5858

5959
// TODO only partially implemented
60-
static CharParserFactory createOperatorParser() {
60+
public static CharParserFactory createOperatorParser() {
6161
CharParserFactory operatorParser = new StringParserBuilder("C# operator")
6262
.addCharLiteralMarker("+", '+')
6363
.addCharLiteralMarker("-", '-')
@@ -70,7 +70,7 @@ static CharParserFactory createOperatorParser() {
7070

7171

7272
// TODO couldn't get this working with identifier parser which needs to parse ', ' in strings like 'Map<String, String>'
73-
static CharParserFactory createSeparatorParser() {
73+
public static CharParserFactory createSeparatorParser() {
7474
CharParserFactory annotationParser = new StringBoundedParserBuilder("C# separator")
7575
//.addCharLiteralMarker(',')
7676
.addCharLiteralMarker(";", ';')

‎src/twg2/parser/codeParser/eclipseProject/EclipseClasspathFile.java

-145
This file was deleted.

0 commit comments

Comments
 (0)