Skip to content

Commit a33f37a

Browse files
TeamworkGuy2TeamworkGuy2
committed
Added simple C# parse example with resulting JSON. Added and refactored some test cases.
1 parent 5b692bc commit a33f37a

File tree

5 files changed

+372
-151
lines changed

5 files changed

+372
-151
lines changed

‎README.md

+112-5
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,120 @@
11
ParserToolsTmp
22
==============
3-
version: 0.2.0
3+
version: 0.2.1
44

5-
In progress parser tools built atop [JTextParser] (https://github.com/TeamworkGuy2/JTextParser), [Jackson] (https://github.com/FasterXML/jackson-core/) (core, databind, annotations) and half a dozen other utility libraries.
5+
In progress C#/Java/TypeScript parser tools built atop [JTextParser] (https://github.com/TeamworkGuy2/JTextParser), [Jackson] (https://github.com/FasterXML/jackson-core/) (core, databind, annotations) and half a dozen other utility libraries.
66

7-
####The goal:
7+
####The Goal:
88
* A competent source code parser that can turn C#, Java, or JavaScript/TypeScript code into a simple AST like structure ('competent' meaning this project aims to support common use cases, not every syntatic feature of the supported languages).
9-
* A 'code first' parser aimed at manipulating the resulting AST and writing it back as source code. With the goal of allowing simple language constructs like interfaces and data models to be transpiled to different languages.
9+
* A 'code first' parser aimed at manipulating the resulting AST and writing it back as source code or JSON. With the goal of allowing simple language constructs like interfaces and data models to be transpiled to different languages.
1010

11-
####Not-goals:
11+
####Not Goals:
1212
* NOT to create another compiler for C#, Java, or JS/TS. This project's parser expects valid code as input, the few error messages that are present are NOT design to highlight syntax errors in the input.
1313
* NOT to create a valid AST for each supported language. Rather an AST like structure that supports the lowest common denominator between the targeted languages. This means many AST compromises are inevitable due to differences in language specs.
14+
15+
16+
Example:
17+
--------
18+
19+
Source Code:
20+
21+
namespace ParserExamples.Samples {
22+
23+
/// <summary>
24+
/// A simple class to test parsing.
25+
/// </summary>
26+
public class SimpleCs {
27+
28+
/// <value>The names.</value>
29+
public IList<string> Names { get; set; }
30+
31+
/// <value>The number of names.</value>
32+
public int Count { get; set }
33+
34+
/// <summary>
35+
/// Add name
36+
/// </summary>
37+
/// <param name="name">the name</param>
38+
/// <returns>the names</returns>
39+
[OperationContract]
40+
[WebInvoke(Method = "POST", UriTemplate = "/AddName?name={name}",
41+
ResponseFormat = WebMessageFormat.Json)]
42+
[TransactionFlow(TransactionFlowOption.Allowed)]
43+
Result<IList<String>> AddName(string name) {
44+
content of block;
45+
}
46+
47+
}
48+
49+
50+
Parser Code:
51+
52+
CodeFileSrc<CodeLanguage> simpleCsAst = ParseCodeFile.parseCode(simpleCsName, CodeLanguageOptions.C_SHARP, simpleCsCode);
53+
WriteSettings ws = new WriteSettings(true, true, true);
54+
55+
for(val block : CodeLanguageOptions.C_SHARP.getExtractor().extractClassFieldsAndMethodSignatures(simpleCsAst.getDoc())) {
56+
CodeFileParsed.Simple<String, CsBlock> fileParsed = new CodeFileParsed.Simple<>(simpleCsName, block.getValue(), block.getKey());
57+
58+
StringBuilder sb = new StringBuilder();
59+
fileParsed.getParsedClass().toJson(sb, ws);
60+
System.out.println(sb.toString());
61+
}
62+
63+
64+
JSON Result:
65+
66+
{
67+
"classSignature" : {
68+
"access" : "PUBLIC",
69+
"name" : "SimpleCs",
70+
"declarationType" : "class"
71+
},
72+
"blockType" : "CLASS",
73+
"using" : [],
74+
"fields" : [{
75+
"name" : "SimpleCs.Names",
76+
"type" : "IList[string]"
77+
}, {
78+
"name" : "SimpleCs.Count",
79+
"type" : "int"
80+
}
81+
],
82+
"methods" : [{
83+
"name" : "SimpleCs.AddName",
84+
"parameters" : [{
85+
"type" : "string",
86+
"name" : "name"
87+
}
88+
],
89+
"annotations" : [{
90+
"name" : "OperationContract",
91+
"arguments" : {}
92+
93+
}, {
94+
"name" : "WebInvoke",
95+
"arguments" : {
96+
"ResponseFormat" : "WebMessageFormat.Json",
97+
"Method" : "POST",
98+
"UriTemplate" : "/AddName?name={name}"
99+
}
100+
}, {
101+
"name" : "TransactionFlow",
102+
"arguments" : {
103+
"value" : "TransactionFlowOption.Allowed"
104+
}
105+
}
106+
],
107+
"returnType" : {
108+
"typeName" : "Result",
109+
"genericParameters" : [{
110+
"typeName" : "IList",
111+
"genericParameters" : [{
112+
"typeName" : "String"
113+
}
114+
]
115+
}
116+
]
117+
}
118+
}
119+
]
120+
}

‎src/twg2/parser/main/ParserMain.java

-1
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,6 @@ public static <T_BLOCK extends CompoundBlock> void parseFileSet(List<Path> files
9494
val blockDeclarations = ((AstExtractor<CompoundBlock>)parsedFile.getLanguage().getExtractor()).extractClassFieldsAndMethodSignatures(parsedFile.getDoc());
9595

9696
for(val block : blockDeclarations) {
97-
//CodeFileParsed.Simple<CodeFileSrc<DocumentFragmentText<CodeFragmentType>, CodeLanguage>, CompoundBlock> fileParsed = new CodeFileParsed.Simple<>(parsedFile, block.getValue(), block.getKey());
9897
val fileParsed = new CodeFileParsed.Simple<>(parsedFile, block.getValue(), block.getKey());
9998
dstFileSetCast.addCompilationUnit(block.getValue().getSignature().getFullyQualifyingName(), fileParsed);
10099
}

‎src/twg2/parser/test/MiscStringTests.java

+127
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
package twg2.parser.test;
2+
3+
import java.io.IOException;
4+
import java.util.Arrays;
5+
import java.util.List;
6+
import java.util.function.Function;
7+
8+
import lombok.val;
9+
10+
import org.junit.Assert;
11+
import org.junit.Test;
12+
13+
import twg2.arrays.ArrayUtil;
14+
import twg2.parser.Inclusion;
15+
import twg2.parser.text.CharParserCondition;
16+
import twg2.parser.text.CharPrecondition;
17+
import twg2.parser.text.StringBoundedParserBuilder;
18+
import twg2.parser.textParser.TextParser;
19+
import twg2.parser.textParser.TextParserImpl;
20+
import twg2.parser.textParserUtils.EscapeSequences;
21+
import twg2.text.stringUtils.StringCompare;
22+
import twg2.text.stringUtils.StringIndex;
23+
import checks.CheckTask;
24+
import checks.TestData;
25+
import checks.TestDataObj;
26+
27+
/**
28+
* @author TeamworkGuy2
29+
* @since 2016-0-0
30+
*/
31+
public class MiscStringTests {
32+
33+
@Test
34+
public void startsWithTest() {
35+
List<TestData<String, String>> startsWithStrs = Arrays.asList(
36+
TestDataObj.matchFalse("this a lz3", "thisa"),
37+
TestDataObj.matchTrue("this a lz3", "this"),
38+
TestDataObj.matchFalse("a", "ab"),
39+
TestDataObj.matchTrue("a", "a"),
40+
TestDataObj.matchFalse("", "a"),
41+
TestDataObj.matchTrue("", "")
42+
);
43+
44+
for(TestData<String, String> test : startsWithStrs) {
45+
Assert.assertTrue(test.isShouldInputEqualExpected() ==
46+
StringCompare.startsWith(test.getInput().toCharArray(), 0, test.getExpected().toCharArray(), 0));
47+
}
48+
}
49+
50+
51+
@Test
52+
public void indexOfTest() {
53+
String searchString = "this 32a this 32.1f is_a string";
54+
String[] strs = new String[] {"32a", "32z", " ", " ", "this", "string"};
55+
Integer[] expect = { 5, -1, 4, -1, 0, 25 };
56+
char[] searchChars = searchString.toCharArray();
57+
58+
CheckTask.assertTests(strs, expect, (str) -> StringIndex.indexOf(searchChars, 0, str, 0));
59+
}
60+
61+
62+
@Test
63+
public void lineBufferTest() {
64+
String line = "Abbb abab [very awesome] 132\n" +
65+
"few 142345 52132";
66+
67+
TextParser tool = TextParserImpl.of(line);
68+
StringBuilder dst = new StringBuilder();
69+
70+
tool.nextIf('A', dst);
71+
tool.nextIf((c) -> (c == 'b'), 2, dst);
72+
check(tool.nextIf('b', dst), 1, "could not read 'b'");
73+
check(tool.nextIf(' ', dst), 1, "could not read ' '");
74+
check(tool.nextIf('a', 'b', 0, dst), 4, "could not read 'a' or 'b'");
75+
tool.nextIf((c) -> (true), 0, dst);
76+
77+
tool.nextIf((c) -> (true), 3, dst);
78+
tool.nextIf((c) -> (ArrayUtil.indexOf(new char[] {'1', '2', '3', '4', '5', ' '}, c) > -1) , 0, dst);
79+
tool.nextIf((c) -> (ArrayUtil.indexOf(new char[] {'1', '2', '3', '4', '5', ' '}, c) > -1) , 0, dst);
80+
81+
Assert.assertEquals("parsed: '" + dst.toString() + "', does not match: '" + line + "'", line, dst.toString());
82+
}
83+
84+
85+
@Test
86+
public void stringBoundedSegmentParserTest() throws IOException {
87+
88+
// single-character start and end markers and single-character escape markers
89+
String[] strs = new String[] { "\"a \\\" b \\\"", "\"\" !", "alpha", "\"a \n\\\"\\\" z\" echo" };
90+
String[] expect = new String[] { "\"a \" b \"", "\"\"", "", "\"a \n\"\" z\"" };
91+
92+
CharPrecondition parser1 = new StringBoundedParserBuilder("stringBoundedSegmentParserTest").addStartEndNotPrecededByMarkers("string literal", '"', '\\', '"', Inclusion.INCLUDE).build();
93+
94+
Function<String, String> escSeqDecoder = EscapeSequences.unicodeEscapeDecoder();
95+
CheckTask.assertTests(strs, expect, (s, i) -> {
96+
val dst = new StringBuilder();
97+
//Assert.assertTrue("i=" + i + " first char '" + s.charAt(0) + "' of '" + s + "'", parser1.isMatch(s.charAt(0)));
98+
CharParserCondition cond = parser1.createParser();
99+
cond.readConditional(TextParserImpl.of(s), dst);
100+
return escSeqDecoder.apply(dst.toString());
101+
});
102+
103+
// TODO reimplement string markers
104+
// multi-character start and end markers and multi-character escape markers
105+
//strs = new String[] { "to /**string @@/** and @@**/", "/**/", "alpha", "@@/**start \n@@/**@@**/ end**/ echo" };
106+
//expect = new String[] { "/**string @@/** and **/", "/**/", "", "/**start \n@@/****/ end**/" };
107+
/*
108+
settings.setEscapeString("@@")
109+
.setHandleEscapes(true)
110+
.setReadMultiline(true);
111+
input = new ParserHelper();
112+
*/
113+
//StringBoundedParser parser2 = new StringBoundedParser(settings, input, new String[] {"/**"}, new String[] {"**/"});
114+
/*
115+
Check.assertTests(strs, expect, "", "mismatch", (s) -> {
116+
dst.setLength(0);
117+
return parser2.readElement(LineBufferImpl.of(s), dst).toString();
118+
});
119+
*/
120+
}
121+
122+
123+
private static void check(int input, int expected, String message) {
124+
Assert.assertEquals(message + " (input: " + input + ", expected: " + expected + ")", expected, input);
125+
}
126+
127+
}

0 commit comments

Comments
 (0)