7
7
from _devbuild .gen import syntax_asdl
8
8
from _devbuild .gen .syntax_asdl import (
9
9
command , command__OilAssign ,
10
- expr , expr_t , regex , regex_t , word , word_t , word_part ,
10
+ expr , expr_t , expr_context_e , regex , regex_t , word , word_t , word_part ,
11
11
oil_word_part , oil_word_part_t ,
12
12
)
13
13
from _devbuild .gen import grammar_nt
14
+ from pgen2 .parse import PNode
14
15
#from core.util import log
15
16
16
17
from typing import TYPE_CHECKING , List
17
18
if TYPE_CHECKING :
18
- from pgen2 .parse import PNode
19
19
from pgen2 .grammar import Grammar
20
20
21
21
@@ -28,7 +28,25 @@ def ISNONTERMINAL(x):
28
28
29
29
30
30
class Transformer (object ):
31
-
31
+ """Homogeneous parse tree -> heterogeneous AST ("lossless syntax tree")
32
+
33
+ pgen2 (Python's LL parser generator) doesn't have semantic actions like yacc,
34
+ so this "transformer" is the equivalent.
35
+
36
+ Files to refer to when modifying this function:
37
+
38
+ oil_lang/grammar.pgen2 (generates _devbuild/gen/grammar_nt.py)
39
+ frontend/syntax.asdl (generates _devbuild/gen/syntax_asdl.py)
40
+
41
+ Related examples:
42
+
43
+ opy/compiler2/transformer.py (Python's parse tree -> AST, ~1500 lines)
44
+ Python-2.7.13/Python/ast.c (the "real" CPython version, ~3600 lines)
45
+
46
+ Other:
47
+ frontend/parse_lib.py (turn on print_parse_tree)
48
+
49
+ """
32
50
def __init__ (self , gr ):
33
51
# type: (Grammar) -> None
34
52
self .number2symbol = gr .number2symbol
@@ -40,8 +58,8 @@ def _AssocBinary(self, children):
40
58
We don't care if it's (1+2)+3 or 1+(2+3).
41
59
"""
42
60
assert len (children ) >= 3 , children
43
- # NOTE: opy/compiler2/transformer.py has an interative version of this in
44
- # com_binary .
61
+ # Note: Compare the iteractive com_binary() method in
62
+ # opy/compiler2/transformer.py .
45
63
46
64
left , op = children [0 ], children [1 ]
47
65
if len (children ) == 3 :
@@ -89,6 +107,7 @@ def _Trailer(self, base, p_trailer):
89
107
90
108
def OilAssign (self , pnode ):
91
109
# type: (PNode) -> command__OilAssign
110
+ """Transform an Oil assignment statement."""
92
111
typ = pnode .typ
93
112
children = pnode .children
94
113
@@ -127,9 +146,45 @@ def OilAssign(self, pnode):
127
146
raise AssertionError (
128
147
"PNode type %d (%s) wasn't handled" % (typ , nt_name ))
129
148
149
+ def atom (self , children ):
150
+ # type: (List[PNode]) -> expr_t
151
+ """Handles alternatives of 'atom' where there is more than one child."""
152
+
153
+ id_ = children [0 ].tok .id
154
+
155
+ if id_ == Id .Op_LParen :
156
+ # atom: '(' [yield_expr|testlist_comp] ')' | ...
157
+ return self .Expr (children [1 ])
158
+
159
+ if id_ == Id .Op_LBracket :
160
+ # atom: ... | '[' [testlist_comp] ']' | ...
161
+
162
+ if len (children ) == 2 : # []
163
+ return expr .List ([], expr_context_e .Store ) # unused expr_context_e
164
+
165
+ p_list = children [1 ].children # what's between [ and ]
166
+
167
+ # [x for x in y]
168
+ if len (p_list ) == 2 and p_list [1 ].typ == grammar_nt .sync_comp_for :
169
+ elt = self .Expr (p_list [0 ])
170
+
171
+ # TODO: transform 'for', 'if', etc.
172
+ return expr .ListComp (elt , [])
173
+
174
+ # [1, 2, 3]
175
+ n = len (p_list )
176
+ elts = []
177
+ for i in xrange (0 , n , 2 ): # skip commas
178
+ p_node = p_list [i ]
179
+ elts .append (self .Expr (p_node ))
180
+
181
+ return expr .List (elts , expr_context_e .Store ) # unused expr_context_e
182
+
183
+ raise NotImplementedError
184
+
130
185
def Expr (self , pnode ):
131
186
# type: (PNode) -> expr_t
132
- """Walk the homogeneous parse tree and create a typed AST ."""
187
+ """Transform expressions (as opposed to statements) ."""
133
188
typ = pnode .typ
134
189
tok = pnode .tok
135
190
children = pnode .children
@@ -143,10 +198,7 @@ def Expr(self, pnode):
143
198
return self ._AssocBinary (children )
144
199
145
200
if typ == grammar_nt .atom :
146
- if children [0 ].tok .id == Id .Op_LParen :
147
- return self .Expr (children [1 ])
148
- else :
149
- raise NotImplementedError
201
+ return self .atom (children )
150
202
151
203
if typ == grammar_nt .eval_input :
152
204
# testlist_input: testlist NEWLINE* ENDMARKER
0 commit comments