Skip to content

Commit 6136531

Browse files
TeamworkGuy2TeamworkGuy2
committed
0.15.1 - Update jtext-parser@0.13.0, added some test cases.
1 parent db38f7e commit 6136531

8 files changed

+234
-4
lines changed

‎CHANGELOG.md

+8-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,14 @@ This project does its best to adhere to [Semantic Versioning](http://semver.org/
44

55

66
--------
7-
### [0.15.0](N/A) - 2017-10-15
7+
### [0.15.1](N/A) - 2017-11-11
8+
#### Changed
9+
* Update dependency jtext-parser@0.13.0
10+
* Add some test cases
11+
12+
13+
--------
14+
### [0.15.0](https://github.com/TeamworkGuy2/JParseCode/commit/db38f7ec5d369b60ef8e79480d981d3bcdf3006c) - 2017-10-15
815
#### Changed
916
Simplified class names and generic type signatures:
1017
* Changed `AnnotationSig`, `BlockAst`, `MethodSig`, `CodeFileSrc`, and `CodeFileParsed` to have `public final` properties and remove getters

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
JParseCode
22
==============
3-
version: 0.15.0
3+
version: 0.15.1
44

55
In progress C#/Java/TypeScript parser tools built atop [JTextParser] (https://github.com/TeamworkGuy2/JTextParser), [Jackson] (https://github.com/FasterXML/jackson-core/) (core, databind, annotations) and half a dozen other utility libraries.
66

‎bin/jparse_code-with-tests.jar

5.38 KB
Binary file not shown.

‎bin/jparse_code.jar

4.79 KB
Binary file not shown.

‎package-lib.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version" : "0.15.0",
2+
"version" : "0.15.1",
33
"name" : "jparse-code",
44
"description" : "An in-progress suite of parsing/transpilation tools for C#, Java, and TypeScript code. Generates simple JSON ASTs.",
55
"homepage" : "https://github.com/TeamworkGuy2/JParseCode",

‎src/twg2/parser/tokenizers/CodeTokenizerBuilder.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ public static <_T_LANG> Function<ParseInput, CodeFileSrc> createTokenizerWithTim
117117
public static <_T_LANG extends CodeLanguage> CodeFileSrc tokenizeCodeFile(PairList<CharParserFactory, TextTransformer<CodeTokenType>> tokenizers,
118118
char[] src, int srcOff, int srcLen, _T_LANG lang, String srcName, TokenizeStepLogger stepsDetails) {
119119

120-
val input = TextCharsParser.of(src, srcOff, srcLen, true, true, true);
120+
val input = TextCharsParser.of(src, srcOff, srcLen);
121121

122122
val docTextFragment = new TextFragmentRefImplMut(srcOff, srcOff + srcLen, 0, 0, -1, -1);
123123
val docRoot = new CodeToken(CodeTokenType.DOCUMENT, docTextFragment, docTextFragment.getText(src, srcOff, srcLen).toString());

‎test/twg2/parser/test/HtmlTemplateTest.java

+153
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
package twg2.parser.test;
2+
3+
import java.util.Arrays;
4+
import java.util.List;
5+
6+
import lombok.val;
7+
8+
import org.junit.Assert;
9+
import org.junit.Test;
10+
11+
import twg2.collections.primitiveCollections.CharArrayList;
12+
import twg2.parser.Inclusion;
13+
import twg2.parser.condition.text.CharParserMatchable;
14+
import twg2.parser.fragment.CodeTokenType;
15+
import twg2.parser.language.CodeLanguage;
16+
import twg2.parser.tokenizers.CodeTokenizer;
17+
import twg2.parser.tokenizers.CodeTokenizerBuilder;
18+
import twg2.parser.workflow.CodeFileSrc;
19+
import twg2.text.tokenizer.CharConditionPipe;
20+
import twg2.text.tokenizer.CharConditions;
21+
import twg2.text.tokenizer.CharParserFactory;
22+
import twg2.text.tokenizer.CharParserMatchableFactory;
23+
import twg2.text.tokenizer.StringConditions;
24+
25+
/**
26+
* @author TeamworkGuy2
27+
* @since 2017-10-21
28+
*/
29+
public class HtmlTemplateTest {
30+
31+
@Test
32+
public void compileInputSinglelineTest() {
33+
String ln = "\n";
34+
String srcName = "compile-input-test-1";
35+
36+
String src =
37+
"<head>" + ln +
38+
" <body>" + ln +
39+
" <stuff>things 1</stuff>" + ln +
40+
" $importHtml(name=\"arc 12\")$" + ln +
41+
" </body>" + ln +
42+
"</head>";
43+
44+
String expect =
45+
"<head>" + ln +
46+
" <body>" + ln +
47+
" <stuff>things 1</stuff>" + ln +
48+
" custom(arc 12)" + ln +
49+
" </body>" + ln +
50+
"</head>";
51+
52+
StringBuilder dst = new StringBuilder(src);
53+
compileTemplate(createHtmlParser().build(), src, dst, srcName, true, createHtmlVarsSingleline());
54+
55+
Assert.assertEquals(expect, dst.toString());
56+
}
57+
58+
59+
@Test
60+
public void compileInputMultilineTest() {
61+
String ln = "\n";
62+
String srcName = "compile-input-test-2";
63+
64+
String src =
65+
"<head>" + ln +
66+
" <body>" + ln +
67+
" <stuff>things 1</stuff>" + ln +
68+
" $importHtml(name=\"arc 12\")$" + ln +
69+
" </body>" + ln +
70+
"</head>";
71+
72+
String expect =
73+
"<head>" + ln +
74+
" <body>" + ln +
75+
" <stuff>things 1</stuff>" + ln +
76+
" custom: {" + ln +
77+
" arc 12" + ln +
78+
" }" + ln +
79+
" </body>" + ln +
80+
"</head>";
81+
82+
StringBuilder dst = new StringBuilder(src);
83+
compileTemplate(createHtmlParser().build(), src, dst, srcName, true, createHtmlVarsMultiline());
84+
85+
Assert.assertEquals(expect, dst.toString());
86+
}
87+
88+
89+
private static final CodeTokenizerBuilder<CodeLanguage> createHtmlParser() {
90+
CharParserFactory parser = createHtmlNamedVarParser("$", "importHtml", "$");
91+
CodeTokenizerBuilder<CodeLanguage> docParser = new CodeTokenizerBuilder<CodeLanguage>((CodeLanguage)null).addParser(parser, CodeTokenType.IDENTIFIER);
92+
return docParser;
93+
}
94+
95+
96+
private static final List<TemplateVar> createHtmlVarsSingleline() {
97+
return Arrays.asList(
98+
new TemplateVar("$", "$", "importHtml", (s) -> {
99+
return Arrays.asList("custom(" + s + ")");
100+
})
101+
);
102+
103+
}
104+
105+
106+
private static final List<TemplateVar> createHtmlVarsMultiline() {
107+
return Arrays.asList(
108+
new TemplateVar("$", "$", "importHtml", (s) -> {
109+
return Arrays.asList("custom: {", "\t" + s, "}");
110+
})
111+
);
112+
113+
}
114+
115+
116+
public static final CharParserFactory createHtmlNamedVarParser(String startMark, String templateName, String endMark) {
117+
CharParserFactory htmlParser = new CharParserMatchableFactory<CharParserMatchable>("HTML Named Var Template", false, Arrays.asList(CharConditionPipe.createPipeAllRequired("HTML Named Var Template", Arrays.asList(
118+
new StringConditions.Literal("start tag", new String[] { startMark + templateName + "(name=" }, Inclusion.INCLUDE),
119+
new CharConditions.Start("attribute-string-start", CharArrayList.of('"'), Inclusion.INCLUDE),
120+
new CharConditions.EndNotPrecededBy("attribute-string-end", CharArrayList.of('"'), Inclusion.INCLUDE, CharArrayList.of('\\')),
121+
new StringConditions.Literal("end tag", new String[] { ")" + endMark }, Inclusion.INCLUDE)
122+
))));
123+
return htmlParser;
124+
}
125+
126+
127+
public static void compileTemplate(CodeTokenizer parser, String src, StringBuilder srcDst, String srcName, boolean preserveIndentation, List<TemplateVar> vars) {
128+
char[] srcChars = src.toCharArray();
129+
CodeFileSrc docParser = parser.tokenizeDocument(srcChars, 0, srcChars.length, srcName, null);
130+
131+
val childs = docParser.astTree.getChildren();
132+
for(int i = childs.size() - 1; i > -1; i--) {
133+
val child = childs.get(i);
134+
val frag = child.getData().getToken();
135+
val text = child.getData().getText();
136+
val startI = frag.getOffsetStart();
137+
val endI = frag.getOffsetEnd();
138+
boolean foundOne = false;
139+
140+
for(val var : vars) {
141+
if(var.isMatch(text)) {
142+
if(foundOne) {
143+
throw new IllegalStateException("found two vars both matching token " + i + ": " + frag + ", text: '" + text + "'");
144+
}
145+
146+
var.insert(child.getData(), true, srcDst, startI, endI - startI);
147+
foundOne = true;
148+
}
149+
}
150+
}
151+
}
152+
153+
}

‎test/twg2/parser/test/TemplateVar.java

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package twg2.parser.test;
2+
3+
import java.util.List;
4+
import java.util.function.Function;
5+
6+
import lombok.val;
7+
import twg2.parser.fragment.CodeTokenType;
8+
import twg2.parser.fragment.TextFragmentRefToken;
9+
import twg2.text.stringSearch.StringIndex;
10+
import twg2.text.stringUtils.StringJoin;
11+
12+
13+
/**
14+
* @author TeamworkGuy2
15+
* @since 2016-2-28
16+
*/
17+
public class TemplateVar {
18+
public final String name;
19+
public final Function<String, List<String>> linesGetter;
20+
public final String startMark;
21+
public final String endMark;
22+
public final String startName;
23+
24+
25+
/**
26+
* @param name
27+
* @param linesGetter
28+
*/
29+
public TemplateVar(String startMark, String endMark, String name, Function<String, List<String>> linesGetter) {
30+
this.name = name;
31+
this.linesGetter = linesGetter;
32+
this.startMark = startMark;
33+
this.endMark = endMark;
34+
this.startName = startMark + name;
35+
}
36+
37+
38+
public boolean isMatch(String text) {
39+
return text.startsWith(startName);
40+
}
41+
42+
43+
public int insert(TextFragmentRefToken<? extends CodeTokenType> frag, boolean preserveIndentation, StringBuilder srcDst, int off, int len) {
44+
val name = inbetweenString(frag.getText(), startName + "(name=\"", "\")" + endMark);
45+
String replacementStr = null;
46+
val lines = linesGetter.apply(name);
47+
48+
if(preserveIndentation) {
49+
int ldx = StringIndex.lastIndexOf(srcDst, 0, off, '\n');
50+
String indentation = ldx > -1 ? srcDst.substring(ldx + 1, off) : "";
51+
indentation = "\n" + indentation;
52+
replacementStr = StringJoin.join(lines, indentation);
53+
}
54+
else {
55+
replacementStr = StringJoin.join(lines, "\n");
56+
}
57+
58+
srcDst.replace(off, off + len, replacementStr);
59+
return off + replacementStr.length();
60+
}
61+
62+
63+
public static String inbetweenString(String src, String start, String end) {
64+
int off = 0;
65+
int idx1 = src.indexOf(start, off);
66+
int idx2 = src.indexOf(end, idx1 + start.length());
67+
return idx1 > -1 && idx2 > -1 ? src.substring(idx1 + start.length(), idx2) : null;
68+
}
69+
70+
}

0 commit comments

Comments
 (0)