Skip to content

Commit 49b0649

Browse files
author
Andy Chu
committed
[oil-language] Transform list literals.
- Stubbed out list comprehensions. - Extracted a method for the 'atom' production. - Added comments on how it works.
1 parent 9bf65d0 commit 49b0649

File tree

4 files changed

+73
-13
lines changed

4 files changed

+73
-13
lines changed

core/process.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
from _devbuild.gen.id_kind_asdl import Id
1919
from _devbuild.gen.runtime_asdl import redirect_e, job_state_e
20-
from asdl import const, pretty
20+
from asdl import pretty
2121
from core import ui
2222
from core.util import log
2323
from frontend import match

frontend/parse_lib.py

+1
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,7 @@ def ParseOilAssign(self, lexer, start_symbol, print_parse_tree=False):
282282
"""e.g. var mylist = [1, 2, 3]"""
283283
pnode, last_token = self.e_parser.Parse(lexer, start_symbol)
284284

285+
#print_parse_tree = True
285286
if print_parse_tree:
286287
self.p_printer.Print(pnode)
287288

oil_lang/expr_parse_test.py

+9-2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
from core import alloc
1515
from core import meta
1616
from core import pyutil
17+
from core.util import log
1718
from frontend import parse_lib
1819
from frontend import reader
1920

@@ -37,7 +38,10 @@ def _ParseOsh(self, code_str):
3738
# the OSH parser hooks into the Oil parser
3839
c_parser = self.parse_ctx.MakeOshParser(line_reader)
3940
node = c_parser.ParseLogicalLine()
41+
print('')
42+
log('\t%s', code_str)
4043
node.PrettyPrint()
44+
print('')
4145
return node
4246

4347
def _ParseOilExpression(self, code_str):
@@ -56,8 +60,11 @@ def testPythonLike(self):
5660

5761
# These raise NotImplementedError
5862

59-
#node = self._ParseOsh('var x = [1,2,3];')
60-
#node = self._ParseOilExpression('[]')
63+
node = self._ParseOsh('var x = [1,2,3];')
64+
node = self._ParseOilExpression('[4+5, 6+7*8]')
65+
node = self._ParseOilExpression('[]')
66+
67+
node = self._ParseOilExpression('[x for x in y]')
6168
#node = self._ParseOilExpression('{foo: bar}')
6269

6370
def testOtherExpr(self):

oil_lang/expr_to_ast.py

+62-10
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,15 @@
77
from _devbuild.gen import syntax_asdl
88
from _devbuild.gen.syntax_asdl import (
99
command, command__OilAssign,
10-
expr, expr_t, regex, regex_t, word, word_t, word_part,
10+
expr, expr_t, expr_context_e, regex, regex_t, word, word_t, word_part,
1111
oil_word_part, oil_word_part_t,
1212
)
1313
from _devbuild.gen import grammar_nt
14+
from pgen2.parse import PNode
1415
#from core.util import log
1516

1617
from typing import TYPE_CHECKING, List
1718
if TYPE_CHECKING:
18-
from pgen2.parse import PNode
1919
from pgen2.grammar import Grammar
2020

2121

@@ -28,7 +28,25 @@ def ISNONTERMINAL(x):
2828

2929

3030
class Transformer(object):
31-
31+
"""Homogeneous parse tree -> heterogeneous AST ("lossless syntax tree")
32+
33+
pgen2 (Python's LL parser generator) doesn't have semantic actions like yacc,
34+
so this "transformer" is the equivalent.
35+
36+
Files to refer to when modifying this function:
37+
38+
oil_lang/grammar.pgen2 (generates _devbuild/gen/grammar_nt.py)
39+
frontend/syntax.asdl (generates _devbuild/gen/syntax_asdl.py)
40+
41+
Related examples:
42+
43+
opy/compiler2/transformer.py (Python's parse tree -> AST, ~1500 lines)
44+
Python-2.7.13/Python/ast.c (the "real" CPython version, ~3600 lines)
45+
46+
Other:
47+
frontend/parse_lib.py (turn on print_parse_tree)
48+
49+
"""
3250
def __init__(self, gr):
3351
# type: (Grammar) -> None
3452
self.number2symbol = gr.number2symbol
@@ -40,8 +58,8 @@ def _AssocBinary(self, children):
4058
We don't care if it's (1+2)+3 or 1+(2+3).
4159
"""
4260
assert len(children) >= 3, children
43-
# NOTE: opy/compiler2/transformer.py has an interative version of this in
44-
# com_binary.
61+
# Note: Compare the iteractive com_binary() method in
62+
# opy/compiler2/transformer.py.
4563

4664
left, op = children[0], children[1]
4765
if len(children) == 3:
@@ -89,6 +107,7 @@ def _Trailer(self, base, p_trailer):
89107

90108
def OilAssign(self, pnode):
91109
# type: (PNode) -> command__OilAssign
110+
"""Transform an Oil assignment statement."""
92111
typ = pnode.typ
93112
children = pnode.children
94113

@@ -127,9 +146,45 @@ def OilAssign(self, pnode):
127146
raise AssertionError(
128147
"PNode type %d (%s) wasn't handled" % (typ, nt_name))
129148

149+
def atom(self, children):
150+
# type: (List[PNode]) -> expr_t
151+
"""Handles alternatives of 'atom' where there is more than one child."""
152+
153+
id_ = children[0].tok.id
154+
155+
if id_ == Id.Op_LParen:
156+
# atom: '(' [yield_expr|testlist_comp] ')' | ...
157+
return self.Expr(children[1])
158+
159+
if id_ == Id.Op_LBracket:
160+
# atom: ... | '[' [testlist_comp] ']' | ...
161+
162+
if len(children) == 2: # []
163+
return expr.List([], expr_context_e.Store) # unused expr_context_e
164+
165+
p_list = children[1].children # what's between [ and ]
166+
167+
# [x for x in y]
168+
if len(p_list) == 2 and p_list[1].typ == grammar_nt.sync_comp_for:
169+
elt = self.Expr(p_list[0])
170+
171+
# TODO: transform 'for', 'if', etc.
172+
return expr.ListComp(elt, [])
173+
174+
# [1, 2, 3]
175+
n = len(p_list)
176+
elts = []
177+
for i in xrange(0, n, 2): # skip commas
178+
p_node = p_list[i]
179+
elts.append(self.Expr(p_node))
180+
181+
return expr.List(elts, expr_context_e.Store) # unused expr_context_e
182+
183+
raise NotImplementedError
184+
130185
def Expr(self, pnode):
131186
# type: (PNode) -> expr_t
132-
"""Walk the homogeneous parse tree and create a typed AST."""
187+
"""Transform expressions (as opposed to statements)."""
133188
typ = pnode.typ
134189
tok = pnode.tok
135190
children = pnode.children
@@ -143,10 +198,7 @@ def Expr(self, pnode):
143198
return self._AssocBinary(children)
144199

145200
if typ == grammar_nt.atom:
146-
if children[0].tok.id == Id.Op_LParen:
147-
return self.Expr(children[1])
148-
else:
149-
raise NotImplementedError
201+
return self.atom(children)
150202

151203
if typ == grammar_nt.eval_input:
152204
# testlist_input: testlist NEWLINE* ENDMARKER

0 commit comments

Comments
 (0)