Skip to content

Commit 218036c

Browse files
TeamworkGuy2TeamworkGuy2
committed
0.7.0 - Fixed toJson() issue and default behavior now prints empty annotation arguments map. Updated to use latest version of JTextParser and JStreamish, including switch from StringLineSupplier to new CharLineSuppler. Moved twg2.parser.test package to separate test directory.
1 parent 5ae0793 commit 218036c

24 files changed

+195
-86
lines changed

‎.classpath

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
<?xml version="1.0" encoding="UTF-8"?>
22
<classpath>
33
<classpathentry kind="src" path="src"/>
4+
<classpathentry kind="src" path="test"/>
45
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"/>
56
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
67
<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/ANTLR"/>

‎README.md

+112-64
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
ParserToolsTmp
22
==============
3-
version: 0.6.0
3+
version: 0.7.0
44

55
In progress C#/Java/TypeScript parser tools built atop [JTextParser] (https://github.com/TeamworkGuy2/JTextParser), [Jackson] (https://github.com/FasterXML/jackson-core/) (core, databind, annotations) and half a dozen other utility libraries.
66

7-
####The Goal:
7+
####Goals:
88
* A competent source code parser that can turn C#, Java, or JavaScript/TypeScript code into a simple AST like structure ('competent' meaning this project aims to support common use cases, not every syntatic feature of the supported languages).
99
* A 'code first' parser aimed at manipulating the resulting AST and writing it back as source code or JSON. With the goal of allowing simple language constructs like interfaces and data models to be transpiled to different languages.
1010

@@ -19,39 +19,51 @@ Example:
1919
Source Code (SimpleCs.cs):
2020
```C#
2121
namespace ParserExamples.Samples {
22-
22+
2323
/// <summary>
2424
/// A simple class to test parsing.
2525
/// </summary>
2626
public class SimpleCs {
27-
27+
28+
/// <value>The modification count.</value>
29+
private int mod;
30+
31+
/// <value>The name.</value>
32+
private string _name;
33+
2834
/// <value>The names.</value>
29-
public IList<string> Names { get; set; }
30-
35+
public IList<string> Names { get; }
36+
3137
/// <value>The number of names.</value>
32-
public int Count { get; set; }
33-
38+
public int Count { set; }
39+
40+
/// <value>The access timestamps.</value>
41+
public DateTime[] accesses { set { this.mod++; this.accesses = value; } }
42+
43+
/// <value>The access timestamps.</value>
44+
public string name { get { this.mod++; return this._name; } set { this.mod++; this._name = value; } }
45+
3446
/// <summary>Add name</summary>
3547
/// <param name="name">the name</param>
3648
/// <returns>the names</returns>
3749
[OperationContract]
3850
[WebInvoke(Method = "POST", UriTemplate = "/AddName?name={name}",
39-
ResponseFormat = WebMessageFormat.Json)]
51+
ResponseFormat = WebMessageFormat.Json)]
4052
[TransactionFlow(TransactionFlowOption.Allowed)]
4153
Result<IList<String>> AddName(string name) {
4254
content of block;
4355
}
44-
56+
4557
}
46-
58+
4759
}
4860
```
4961

5062

5163
Java code to parser SimpleCs.cs (simple_cs_source_string is a string containing the contents of SimpleCs.cs):
5264
```Java
5365
CodeFileSrc<CodeLanguage> simpleCsAst = ParseCodeFile.parseCode("SimpleCs.cs", CodeLanguageOptions.C_SHARP, simple_cs_source_string);
54-
WriteSettings ws = new WriteSettings(true, true, true);
66+
WriteSettings ws = new WriteSettings(true, true, true, true);
5567

5668
for(Map.Entry<SimpleTree<DocumentFragmentText<CodeFragmentType>>, IntermClass.SimpleImpl<CsBlock>> block : CodeLanguageOptions.C_SHARP.getExtractor().extractClassFieldsAndMethodSignatures(simpleCsAst.getDoc())) {
5769
CodeFileParsed.Simple<String, CsBlock> fileParsed = new CodeFileParsed.Simple<>("SimpleCs.cs", block.getValue(), block.getKey());
@@ -65,59 +77,95 @@ Java code to parser SimpleCs.cs (simple_cs_source_string is a string containing
6577

6678
JSON Result (printed to System.out):
6779
```JSON
68-
{
69-
"classSignature" : {
70-
"access" : "PUBLIC",
71-
"name" : "ParserExamples.Samples.SimpleCs",
72-
"declarationType" : "class"
80+
{
81+
"classSignature": {
82+
"access": "PUBLIC",
83+
"name": "ParserExamples.Samples.SimpleCs",
84+
"declarationType": "class"
85+
},
86+
"blockType": "CLASS",
87+
"using": [],
88+
"fields": [{
89+
"name": "ParserExamples.Samples.SimpleCs.mod",
90+
"type": {
91+
"typeName": "int",
92+
"primitive": true
93+
},
94+
"accessModifiers": ["private"],
95+
"annotations": []
96+
}, {
97+
"name": "ParserExamples.Samples.SimpleCs._name",
98+
"type": {
99+
"typeName": "string"
100+
},
101+
"accessModifiers": ["private"],
102+
"annotations": []
103+
}, {
104+
"name": "ParserExamples.Samples.SimpleCs.Names",
105+
"type": {
106+
"typeName": "IList",
107+
"genericParameters": [{
108+
"typeName": "string"
109+
}]
73110
},
74-
"blockType" : "CLASS",
75-
"using" : [],
76-
"fields" : [{
77-
"name" : "ParserExamples.Samples.SimpleCs.Names",
78-
"type" : "IList[string]"
79-
}, {
80-
"name" : "ParserExamples.Samples.SimpleCs.Count",
81-
"type" : "int"
111+
"accessModifiers": ["public"],
112+
"annotations": []
113+
}, {
114+
"name": "ParserExamples.Samples.SimpleCs.Count",
115+
"type": {
116+
"typeName": "int",
117+
"primitive": true
118+
},
119+
"accessModifiers": ["public"],
120+
"annotations": []
121+
}, {
122+
"name": "ParserExamples.Samples.SimpleCs.accesses",
123+
"type": {
124+
"typeName": "DateTime",
125+
"arrayDimensions": 1
126+
},
127+
"accessModifiers": ["public"],
128+
"annotations": []
129+
}, {
130+
"name": "ParserExamples.Samples.SimpleCs.name",
131+
"type": {
132+
"typeName": "string"
133+
},
134+
"accessModifiers": ["public"],
135+
"annotations": []
136+
}],
137+
"methods": [{
138+
"name": "ParserExamples.Samples.SimpleCs.AddName",
139+
"parameters": [{
140+
"type": "string",
141+
"name": "name"
142+
}],
143+
"accessModifiers": [],
144+
"annotations": [{
145+
"name": "OperationContract",
146+
"arguments": {}
147+
}, {
148+
"name": "WebInvoke",
149+
"arguments": {
150+
"ResponseFormat": "WebMessageFormat.Json",
151+
"Method": "POST",
152+
"UriTemplate": "/AddName?name={name}"
82153
}
83-
],
84-
"methods" : [{
85-
"name" : "ParserExamples.Samples.SimpleCs.AddName",
86-
"parameters" : [{
87-
"type" : "string",
88-
"name" : "name"
89-
}
90-
],
91-
"annotations" : [{
92-
"name" : "OperationContract",
93-
"arguments" : {}
94-
95-
}, {
96-
"name" : "WebInvoke",
97-
"arguments" : {
98-
"ResponseFormat" : "WebMessageFormat.Json",
99-
"Method" : "POST",
100-
"UriTemplate" : "/AddName?name={name}"
101-
}
102-
}, {
103-
"name" : "TransactionFlow",
104-
"arguments" : {
105-
"value" : "TransactionFlowOption.Allowed"
106-
}
107-
}
108-
],
109-
"returnType" : {
110-
"typeName" : "Result",
111-
"genericParameters" : [{
112-
"typeName" : "IList",
113-
"genericParameters" : [{
114-
"typeName" : "String"
115-
}
116-
]
117-
}
118-
]
119-
}
154+
}, {
155+
"name": "TransactionFlow",
156+
"arguments": {
157+
"value": "TransactionFlowOption.Allowed"
120158
}
121-
]
122-
}
159+
}],
160+
"returnType": {
161+
"typeName": "Result",
162+
"genericParameters": [{
163+
"typeName": "IList",
164+
"genericParameters": [{
165+
"typeName": "String"
166+
}]
167+
}]
168+
}
169+
}]
170+
}
123171
```

‎package-lib.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version" : "0.6.0",
2+
"version" : "0.7.0",
33
"name" : "jparser-tools",
44
"description" : "An in-progress suite of parsing tools for C#, Java, and TypeScript source code",
55
"homepage" : "https://github.com/TeamworkGuy2/JFileIo",

‎rsc/csharp/ParserExamples/Services/ITrackSearchService.cs

+41-1
Original file line numberDiff line numberDiff line change
@@ -1 +1,41 @@
1-
using System.ServiceModel;using System.ServiceModel.Web;using ParserExamples.Models;using ParserExamples.Searching;namespace ParserExamples.Services{ /// <summary> /// This interface provides the contract for track searching. /// </summary> /// <remarks> /// Implementations are expected to be effectively thread-safe. /// </remarks> [ServiceContract] public interface ITrackSearchService { /// <summary> /// Searches tracks. /// </summary> /// <param name="criteria">The search criteria</param> /// <returns>The search result</returns> [OperationContract] [WebInvoke(Method = "POST", UriTemplate = "/TrackSearch", RequestFormat = WebMessageFormat.Json, ResponseFormat = WebMessageFormat.Json)] [TransactionFlow(TransactionFlowOption.Allowed)] SearchResult<TrackInfo> Search(TrackSearchCriteria criteria) { ; } /// <summary> /// Searches tracks that have past due date. /// </summary> /// <param name="albumName">The album name</param> /// <returns>The search result</returns> [OperationContract] [WebInvoke(Method = "POST", UriTemplate = "/GetAlbumTracks?albumName={albumName}", ResponseFormat = WebMessageFormat.Json)] [TransactionFlow(TransactionFlowOption.Allowed)] SearchResult<IDictionary<AlbumInfo, IList<TrackInfo>>> GetAlbumTracks(string albumName) { content of block; } }}
1+
using System.ServiceModel;
2+
using System.ServiceModel.Web;
3+
using ParserExamples.Models;
4+
using ParserExamples.Searching;
5+
6+
namespace ParserExamples.Services
7+
{
8+
/// <summary>
9+
/// This interface provides the contract for track searching.
10+
/// </summary>
11+
/// <remarks>
12+
/// Implementations are expected to be effectively thread-safe.
13+
/// </remarks>
14+
[ServiceContract]
15+
public interface ITrackSearchService
16+
{
17+
/// <summary>
18+
/// Searches tracks.
19+
/// </summary>
20+
/// <param name="criteria">The search criteria</param>
21+
/// <returns>The search result</returns>
22+
[OperationContract]
23+
[WebInvoke(Method = "POST", UriTemplate = "/TrackSearch",
24+
RequestFormat = WebMessageFormat.Json, ResponseFormat = WebMessageFormat.Json)]
25+
[TransactionFlow(TransactionFlowOption.Allowed)]
26+
SearchResult<TrackInfo> Search(TrackSearchCriteria criteria) { ; }
27+
28+
/// <summary>
29+
/// Searches tracks that have past due date.
30+
/// </summary>
31+
/// <param name="albumName">The album name</param>
32+
/// <returns>The search result</returns>
33+
[OperationContract]
34+
[WebInvoke(Method = "POST", UriTemplate = "/GetAlbumTracks?albumName={albumName}",
35+
ResponseFormat = WebMessageFormat.Json)]
36+
[TransactionFlow(TransactionFlowOption.Allowed)]
37+
SearchResult<IDictionary<AlbumInfo, IList<TrackInfo>>> GetAlbumTracks(string albumName) {
38+
content of block;
39+
}
40+
}
41+
}

‎src/twg2/parser/codeParser/CodeFileSrc.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ public class CodeFileSrc<T_LANG extends CodeLanguage> {
1616
@Getter SimpleTree<DocumentFragmentText<CodeFragmentType>> doc;
1717
@Getter String srcName;
1818
@Getter String src;
19-
@Getter List<String> lines;
19+
@Getter List<char[]> lines;
2020
@Getter T_LANG language;
2121

2222

@@ -26,7 +26,7 @@ public class CodeFileSrc<T_LANG extends CodeLanguage> {
2626
* @param lines
2727
* @param language optional
2828
*/
29-
public CodeFileSrc(SimpleTree<DocumentFragmentText<CodeFragmentType>> doc, String srcName, String src, List<String> lines, T_LANG language) {
29+
public CodeFileSrc(SimpleTree<DocumentFragmentText<CodeFragmentType>> doc, String srcName, String src, List<char[]> lines, T_LANG language) {
3030
this.doc = doc;
3131
this.srcName = srcName;
3232
this.src = src;

‎src/twg2/parser/codeParser/ParseCommentsAndWhitespace.java

+9-6
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import twg2.parser.documentParser.DocumentFragmentText;
1414
import twg2.parser.text.CharParserFactory;
1515
import twg2.parser.textFragment.TextFragmentRef;
16+
import twg2.text.stringUtils.StringCheck;
1617
import twg2.treeLike.TreeTraversalOrder;
1718
import twg2.treeLike.TreeTraverse;
1819
import twg2.treeLike.parameters.SimpleTreeTraverseParameters;
@@ -42,14 +43,16 @@ public static CodeFileSrc<CodeLanguage> buildCommentsAndWhitespaceTree(EnumSet<C
4243
}
4344

4445

45-
public static ParsedFileStats calcCommentsAndWhitespaceLinesTreeStats(String srcId, int srcCharCount, List<String> lines, SimpleTree<DocumentFragmentText<CodeFragmentType>> tree) {
46+
public static ParsedFileStats calcCommentsAndWhitespaceLinesTreeStats(String srcId, int srcCharCount, List<char[]> lines, SimpleTree<DocumentFragmentText<CodeFragmentType>> tree) {
4647
// flatten the document tree into a nested list of tokens per source line of text
4748
val tokensPerLine = documentTreeToTokensPerLine(tree);
4849

4950
// find lines containing only comments (with optional whitespace)
5051
IntArrayList commentLines = new IntArrayList();
5152
IntArrayList whitespaceLines = new IntArrayList();
5253
for(int i = 0, size = lines.size(); i < size; i++) {
54+
char[] line = lines.get(i);
55+
5356
if(tokensPerLine.size() <= i) {
5457
tokensPerLine.add(new ArrayList<>());
5558
}
@@ -62,20 +65,20 @@ public static ParsedFileStats calcCommentsAndWhitespaceLinesTreeStats(String src
6265
}
6366
else {
6467
TextFragmentRef comment = lineTokens.get(0).getTextFragment();
65-
String line = lines.get(i);
66-
String prefix = comment.getLineStart() < i ? "" : line.substring(0, comment.getColumnStart()); // if the token started on a previous line, there is no prefix text before it starts on this line
68+
String prefix = comment.getLineStart() < i ? "" : new String(line, 0, comment.getColumnStart()); // if the token started on a previous line, there is no prefix text before it starts on this line
6769
String suffix = "";
6870
// in case the token ends at the end of the line
69-
if(comment.getColumnEnd() + 1 < line.length()) {
70-
line.substring(comment.getColumnEnd() + 1, line.length());
71+
if(comment.getColumnEnd() + 1 < line.length) {
72+
// TODO not sure why this was originally here
73+
//new String(line, comment.getColumnEnd() + 1, line.length - (comment.getColumnEnd() + 1));
7174
}
7275

7376
if(prefix.trim().length() == 0 && suffix.trim().length() == 0) {
7477
commentLines.add(i + 1);
7578
}
7679
}
7780
}
78-
else if(lines.get(i).trim().length() == 0) {
81+
else if(StringCheck.isWhitespace(line, 0, line.length)) {
7982
whitespaceLines.add(i + 1);
8083
}
8184
System.out.println("line " + i + " tokens: " + lineTokens);

‎src/twg2/parser/codeParser/ParserBuilder.java

+9-9
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@
1111
import twg2.parser.textFragment.TextTransformer;
1212
import twg2.parser.textParser.TextParser;
1313
import twg2.parser.textParser.TextParserImpl;
14+
import twg2.parser.textStream.CharsLineSupplier;
1415
import twg2.streams.EnhancedIterator;
15-
import twg2.streams.StringLineSupplier;
1616
import twg2.treeLike.simpleTree.SimpleTree;
1717

1818
/**
@@ -47,19 +47,19 @@ public ParserBuilder addConstParser(CharParserFactory parser, CodeFragmentType t
4747
* @return a parsed {@link CodeFileSrc} containing {@link DocumentFragmentText} nodes represented the tokens parsed from {@code src}
4848
*/
4949
public <L extends CodeLanguage> CodeFileSrc<L> buildAndParse(String src, L language, String srcName) {
50-
List<String> lines = new ArrayList<>();
50+
List<char[]> lines = new ArrayList<>();
5151

5252
// intercept each line request and add the line to our list of lines
53-
StringLineSupplier srcLineReader = new StringLineSupplier(src, 0, src.length(), true, true, true, true);
54-
EnhancedIterator<String> lineReader = new EnhancedIterator<>(() -> {
55-
String str = srcLineReader.get();
56-
if(str != null) {
57-
lines.add(str);
53+
CharsLineSupplier srcLineReader = new CharsLineSupplier(src, 0, src.length(), true, true, true, true);
54+
EnhancedIterator<char[]> lineReader = new EnhancedIterator<>(() -> {
55+
char[] chs = srcLineReader.get();
56+
if(chs != null) {
57+
lines.add(chs);
5858
}
59-
return str;
59+
return chs;
6060
});
6161

62-
TextParser input = new TextParserImpl(lineReader);
62+
TextParser input = TextParserImpl.fromCharArrays(lineReader);
6363

6464
val docTextFragment = new TextFragmentRef.ImplMut(0, src.length(), 0, 0, -1, -1);
6565
val docRoot = new DocumentFragmentText<>(CodeFragmentType.DOCUMENT, docTextFragment, docTextFragment.getText(src).toString());

0 commit comments

Comments
 (0)