Skip to content

Commit 44139d5

Browse files
author
Andy Chu
committed
[oil-lang] Copied the expression type in Python's ASDL schema.
I lightly modified it, and it will need further modifications. Addresses issue #387.
1 parent 6a120df commit 44139d5

File tree

6 files changed

+117
-27
lines changed

6 files changed

+117
-27
lines changed

bin/osh_parse.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ def main(argv):
2525
arena = alloc.Arena()
2626
arena.PushSource(source.Stdin(''))
2727

28-
line_reader = reader.FileLineReader(sys.stdin, arena)
2928
# Dummy value; not respecting aliases!
3029
aliases = {} # type: Dict[str, Any]
3130
# parse `` and a[x+1]=bar differently
@@ -35,6 +34,8 @@ def main(argv):
3534

3635
parse_ctx = parse_lib.ParseContext(arena, aliases, oil_grammar,
3736
one_pass_parse=True)
37+
38+
line_reader = reader.FileLineReader(sys.stdin, arena)
3839
c_parser = parse_ctx.MakeOshParser(line_reader)
3940

4041
try:

core/test_lib.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -232,10 +232,10 @@ def InitCommandParser(code_str, arena=None):
232232
return c_parser
233233

234234

235-
def InitOilParser(code_str, arena=None):
235+
def InitOilCommandParser(code_str, arena=None):
236236
# NOTE: aliases don't exist in the Oil parser?
237-
arena = arena or MakeArena('<cmd_exec_test.py>')
237+
arena = arena or MakeArena('')
238238
parse_ctx = parse_lib.ParseContext(arena, {}, None)
239239
line_reader, _ = InitLexer(code_str, arena)
240-
c_parser = parse_ctx.MakeOilParser(line_reader)
240+
c_parser = parse_ctx.MakeOilCommandParser(line_reader)
241241
return arena, c_parser

frontend/parse_lib.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ def MakeOshParser(self, line_reader, emit_comp_dummy=False,
239239
aliases_in_flight=aliases_in_flight)
240240
return c_parser
241241

242-
def MakeOilParser(self, line_reader):
242+
def MakeOilCommandParser(self, line_reader):
243243
# type: (_Reader) -> None
244244
# Same lexer as Oil? It just doesn't start in the OUTER state?
245245
lx = self._MakeLexer(line_reader)
@@ -279,6 +279,7 @@ def MakeWordParserForPlugin(self, code_str):
279279

280280
def ParseOilAssign(self, lexer, start_symbol, print_parse_tree=False):
281281
# type: (Lexer, int, bool) -> Tuple[command_t, token]
282+
"""e.g. var mylist = [1, 2, 3]"""
282283
pnode, last_token = self.e_parser.Parse(lexer, start_symbol)
283284

284285
if print_parse_tree:
@@ -289,6 +290,7 @@ def ParseOilAssign(self, lexer, start_symbol, print_parse_tree=False):
289290

290291
def ParseOilExpr(self, lexer, start_symbol, print_parse_tree=False):
291292
# type: (Lexer, int, bool) -> Tuple[expr_t, token]
293+
"""For Oil expressions that aren't assignments. Currently unused."""
292294
pnode, last_token = self.e_parser.Parse(lexer, start_symbol)
293295

294296
if print_parse_tree:

frontend/syntax.asdl

+43
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,49 @@ module syntax
305305
-- should 1:2 be an expression then?
306306
| Subscript(expr collection, expr* indices)
307307

308+
-- COPIED from Python-3.7/Parser/Python.asdl and lightly modified.
309+
-- Will need more changes.
310+
311+
| IfExp(expr test, expr body, expr orelse)
312+
| Dict(expr* keys, expr* values)
313+
| Set(expr* elts)
314+
| ListComp(expr elt, comprehension* generators)
315+
| SetComp(expr elt, comprehension* generators)
316+
| DictComp(expr key, expr value, comprehension* generators)
317+
| GeneratorExp(expr elt, comprehension* generators)
318+
-- need sequences for compare to distinguish between
319+
-- x < 4 < 3 and (x < 4) < 3 (NOTE: unused; Oil currently uses Binary)
320+
| Compare(expr left, cmpop* ops, expr* comparators)
321+
| Call(expr func, expr* args, keyword* keywords)
322+
| Num(int n) -- do we need this for Oil?
323+
| Str(string s) -- need to specify raw, unicode, etc?
324+
| FormattedValue(expr value, int? conversion, expr? format_spec)
325+
| JoinedStr(expr* values)
326+
| Ellipsis -- do we need this?
327+
328+
-- the following expressions can appear in assignment context
329+
| Attribute(expr value, token attr, expr_context ctx)
330+
-- TODO: we used Subscript() above, might want to migrate?
331+
| Subscript_PYTHON(expr value, slice slice, expr_context ctx)
332+
| Starred(expr value, expr_context ctx)
333+
| Name(token identifier, expr_context ctx)
334+
| List(expr* elts, expr_context ctx)
335+
| Tuple(expr* elts, expr_context ctx)
336+
337+
expr_context = Load | Store | Del | AugLoad | AugStore | Param
338+
339+
slice = Slice(expr? lower, expr? upper, expr? step)
340+
| ExtSlice(slice* dims)
341+
| Index(expr value)
342+
343+
comprehension = (expr target, expr iter, expr* ifs, int is_async)
344+
345+
cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn
346+
347+
-- keyword arguments supplied to call (NULL identifier for **kwargs)
348+
keyword = (token? arg, expr value)
349+
350+
308351
-- NOTE: Remove these for the first cut? We could mostly reuse OSH nodes.
309352
-- And then we can share the evaluator to have identical semantics?
310353

oil_lang/cmd_parse_test.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ class OilParseTest(unittest.TestCase):
1616
#
1717
# That could be part of 'wild'? Or oil-wild?
1818
def testSimple(self):
19-
_, c_parser = test_lib.InitOilParser('echo hi')
19+
_, c_parser = test_lib.InitOilCommandParser('echo hi')
2020
node = c_parser.ParseLogicalLine()
2121
print(node)
2222

oil_lang/expr_parse_test.py

+65-21
Original file line numberDiff line numberDiff line change
@@ -6,41 +6,85 @@
66

77
import unittest
88

9+
#from _devbuild.gen import grammar_nt # names for integer nonterminal IDs
910
from _devbuild.gen.id_kind_asdl import Kind
11+
from _devbuild.gen.syntax_asdl import source
12+
1013
from core.meta import ID_SPEC
11-
#from frontend import lex
12-
#from pgen2.pgen2_main import OilTokenDef # module under test
14+
from core import alloc
15+
from core import meta
16+
from core import pyutil
17+
from frontend import parse_lib
18+
from frontend import reader
19+
1320

21+
class ExprParseTest(unittest.TestCase):
1422

15-
class FooTest(unittest.TestCase):
1623
def setUp(self):
17-
pass
24+
"""Done on every test."""
25+
self.arena = alloc.Arena()
26+
self.arena.PushSource(source.Unused(''))
27+
28+
loader = pyutil.GetResourceLoader()
29+
oil_grammar = meta.LoadOilGrammar(loader)
30+
31+
self.parse_ctx = parse_lib.ParseContext(self.arena, {}, oil_grammar,
32+
one_pass_parse=True)
33+
34+
def _ParseOsh(self, code_str):
35+
"""Parse a line of OSH, which can include Oil assignments."""
36+
line_reader = reader.StringLineReader(code_str, self.arena)
37+
# the OSH parser hooks into the Oil parser
38+
c_parser = self.parse_ctx.MakeOshParser(line_reader)
39+
node = c_parser.ParseLogicalLine()
40+
node.PrettyPrint()
41+
return node
42+
43+
def _ParseOilExpression(self, code_str):
44+
"""Convenient shortcut."""
45+
node = self._ParseOsh('var x = %s\n' % code_str)
46+
47+
def testPythonLike(self):
48+
# This works.
49+
node = self._ParseOsh('var x = y + 2 * 3;')
1850

19-
def tearDown(self):
20-
pass
51+
# The lexer isn't handling single quotes yet.
52+
#node = self._ParseOsh(r"var x = 'one\ntwo\n';")
2153

22-
def testOilTokenDef(self):
23-
# Used for
24-
#tok_def = OilTokenDef()
54+
# NOTE: C-escapes aren't parsed properly.
55+
node = self._ParseOsh(r'var x = "one\ntwo\n";')
2556

26-
# NOTE: These overlap with Kind.Op.
57+
# These raise NotImplementedError
2758

28-
# We need ID_SPEC.ExprOperators(), which has Op, Arith, and Expr kinds.
59+
#node = self._ParseOsh('var x = [1,2,3];')
60+
#node = self._ParseOilExpression('[]')
61+
#node = self._ParseOilExpression('{foo: bar}')
2962

30-
# We don't have:
31-
# LexerPairs(Kind.Op)
32-
# LexerPairs(Kind.Expr)
63+
def testOtherExpr(self):
64+
"""Some examples copied from pgen2/pgen2-test.sh mode-test."""
3365

34-
# Because we really need a lookup for a MODE.
35-
# Problem: _UNQUOTED is used in both DBracket and ShCommand mode.
66+
node = self._ParseOsh('@[1 2 3];')
3667

68+
CASES = [
69+
'@[1 2 3]',
70+
'$/ x /',
71+
'$/ "." [a-z A-Z] y /',
72+
'$[echo hi]',
73+
'$(1 + 2)',
74+
'${x}',
75+
'"quoted ${x}"',
76+
]
3777

38-
arith = ID_SPEC.LexerPairs(Kind.Arith)
39-
print(arith)
78+
# array literal
79+
for c in CASES:
80+
node = self._ParseOilExpression(c)
4081

41-
# Doesn't have one.
42-
#left = ID_SPEC.LexerPairs(Kind.Left)
43-
#print(left)
82+
def testLexer(self):
83+
# NOTE: Kind.Expr for Oil doesn't have LexerPairs
84+
pairs = ID_SPEC.LexerPairs(Kind.Arith)
85+
for p in pairs:
86+
#print(p)
87+
pass
4488

4589

4690
if __name__ == '__main__':

0 commit comments

Comments
 (0)