[oil-lang] Copied the expression type in Python's ASDL schema.

I lightly modified it, and it will need further modifications.

Addresses issue #387.

Author: Andy Chu

bin/osh_parse.py

@@ -25,7 +25,6 @@ def main(argv):
   arena = alloc.Arena()
   arena.PushSource(source.Stdin(''))
 
-  line_reader = reader.FileLineReader(sys.stdin, arena)
   # Dummy value; not respecting aliases!
   aliases = {}  # type: Dict[str, Any]
   # parse `` and a[x+1]=bar differently
@@ -35,6 +34,8 @@ def main(argv):
 
   parse_ctx = parse_lib.ParseContext(arena, aliases, oil_grammar,
                                      one_pass_parse=True)
+
+  line_reader = reader.FileLineReader(sys.stdin, arena)
   c_parser = parse_ctx.MakeOshParser(line_reader)
 
   try:

core/test_lib.py

@@ -232,10 +232,10 @@ def InitCommandParser(code_str, arena=None):
   return c_parser
 
 
-def InitOilParser(code_str, arena=None):
+def InitOilCommandParser(code_str, arena=None):
   # NOTE: aliases don't exist in the Oil parser?
-  arena = arena or MakeArena('<cmd_exec_test.py>')
+  arena = arena or MakeArena('')
   parse_ctx = parse_lib.ParseContext(arena, {}, None)
   line_reader, _ = InitLexer(code_str, arena)
-  c_parser = parse_ctx.MakeOilParser(line_reader)
+  c_parser = parse_ctx.MakeOilCommandParser(line_reader)
   return arena, c_parser

frontend/parse_lib.py

@@ -239,7 +239,7 @@ def MakeOshParser(self, line_reader, emit_comp_dummy=False,
                                        aliases_in_flight=aliases_in_flight)
     return c_parser
 
-  def MakeOilParser(self, line_reader):
+  def MakeOilCommandParser(self, line_reader):
     # type: (_Reader) -> None
     # Same lexer as Oil?  It just doesn't start in the OUTER state?
     lx = self._MakeLexer(line_reader)
@@ -279,6 +279,7 @@ def MakeWordParserForPlugin(self, code_str):
 
   def ParseOilAssign(self, lexer, start_symbol, print_parse_tree=False):
     # type: (Lexer, int, bool) -> Tuple[command_t, token]
+    """e.g. var mylist = [1, 2, 3]"""
     pnode, last_token = self.e_parser.Parse(lexer, start_symbol)
 
     if print_parse_tree:
@@ -289,6 +290,7 @@ def ParseOilAssign(self, lexer, start_symbol, print_parse_tree=False):
 
   def ParseOilExpr(self, lexer, start_symbol, print_parse_tree=False):
     # type: (Lexer, int, bool) -> Tuple[expr_t, token]
+    """For Oil expressions that aren't assignments.  Currently unused."""
     pnode, last_token = self.e_parser.Parse(lexer, start_symbol)
 
     if print_parse_tree:

frontend/syntax.asdl

@@ -305,6 +305,49 @@ module syntax
     -- should 1:2 be an expression then?
   | Subscript(expr collection, expr* indices)
 
+  -- COPIED from Python-3.7/Parser/Python.asdl and lightly modified.
+  -- Will need more changes.
+
+  | IfExp(expr test, expr body, expr orelse)
+  | Dict(expr* keys, expr* values)
+  | Set(expr* elts)
+  | ListComp(expr elt, comprehension* generators)
+  | SetComp(expr elt, comprehension* generators)
+  | DictComp(expr key, expr value, comprehension* generators)
+  | GeneratorExp(expr elt, comprehension* generators)
+  -- need sequences for compare to distinguish between
+  -- x < 4 < 3 and (x < 4) < 3 (NOTE: unused; Oil currently uses Binary)
+  | Compare(expr left, cmpop* ops, expr* comparators)
+  | Call(expr func, expr* args, keyword* keywords)
+  | Num(int n) -- do we need this for Oil?
+  | Str(string s) -- need to specify raw, unicode, etc?
+  | FormattedValue(expr value, int? conversion, expr? format_spec)
+  | JoinedStr(expr* values)
+  | Ellipsis -- do we need this?
+
+  -- the following expressions can appear in assignment context
+  | Attribute(expr value, token attr, expr_context ctx)
+    -- TODO: we used Subscript() above, might want to migrate?
+  | Subscript_PYTHON(expr value, slice slice, expr_context ctx)
+  | Starred(expr value, expr_context ctx)
+  | Name(token identifier, expr_context ctx)
+  | List(expr* elts, expr_context ctx)
+  | Tuple(expr* elts, expr_context ctx)
+
+  expr_context = Load | Store | Del | AugLoad | AugStore | Param
+
+  slice = Slice(expr? lower, expr? upper, expr? step)
+        | ExtSlice(slice* dims)
+        | Index(expr value)
+
+  comprehension = (expr target, expr iter, expr* ifs, int is_async)
+
+  cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn
+
+  -- keyword arguments supplied to call (NULL identifier for **kwargs)
+  keyword = (token? arg, expr value)
+
+
   -- NOTE: Remove these for the first cut?  We could mostly reuse OSH nodes.
   -- And then we can share the evaluator to have identical semantics?
 

oil_lang/cmd_parse_test.py

@@ -16,7 +16,7 @@ class OilParseTest(unittest.TestCase):
   #
   # That could be part of 'wild'?  Or oil-wild?
   def testSimple(self):
-    _, c_parser = test_lib.InitOilParser('echo hi')
+    _, c_parser = test_lib.InitOilCommandParser('echo hi')
     node = c_parser.ParseLogicalLine()
     print(node)
 

oil_lang/expr_parse_test.py

@@ -6,41 +6,85 @@
 
 import unittest
 
+#from _devbuild.gen import grammar_nt  # names for integer nonterminal IDs
 from _devbuild.gen.id_kind_asdl import Kind
+from _devbuild.gen.syntax_asdl import source
+
 from core.meta import ID_SPEC
-#from frontend import lex
-#from pgen2.pgen2_main import OilTokenDef # module under test
+from core import alloc
+from core import meta
+from core import pyutil
+from frontend import parse_lib
+from frontend import reader
+
 
+class ExprParseTest(unittest.TestCase):
 
-class FooTest(unittest.TestCase):
   def setUp(self):
-    pass
+    """Done on every test."""
+    self.arena = alloc.Arena()
+    self.arena.PushSource(source.Unused(''))
+
+    loader = pyutil.GetResourceLoader()
+    oil_grammar = meta.LoadOilGrammar(loader)
+
+    self.parse_ctx = parse_lib.ParseContext(self.arena, {}, oil_grammar,
+                                            one_pass_parse=True)
+
+  def _ParseOsh(self, code_str):
+    """Parse a line of OSH, which can include Oil assignments."""
+    line_reader = reader.StringLineReader(code_str, self.arena)
+    # the OSH parser hooks into the Oil parser
+    c_parser = self.parse_ctx.MakeOshParser(line_reader)
+    node = c_parser.ParseLogicalLine()
+    node.PrettyPrint()
+    return node
+
+  def _ParseOilExpression(self, code_str):
+    """Convenient shortcut."""
+    node = self._ParseOsh('var x = %s\n' % code_str)
+
+  def testPythonLike(self):
+    # This works.
+    node = self._ParseOsh('var x = y + 2 * 3;')
 
-  def tearDown(self):
-    pass
+    # The lexer isn't handling single quotes yet.
+    #node = self._ParseOsh(r"var x = 'one\ntwo\n';")
 
-  def testOilTokenDef(self):
-    # Used for
-    #tok_def = OilTokenDef()
+    # NOTE: C-escapes aren't parsed properly.
+    node = self._ParseOsh(r'var x = "one\ntwo\n";')
 
-    # NOTE: These overlap with Kind.Op.
+    # These raise NotImplementedError
 
-    # We need ID_SPEC.ExprOperators(), which has Op, Arith, and Expr kinds.
+    #node = self._ParseOsh('var x = [1,2,3];')
+    #node = self._ParseOilExpression('[]')
+    #node = self._ParseOilExpression('{foo: bar}')
 
-    # We don't have:
-    # LexerPairs(Kind.Op)
-    # LexerPairs(Kind.Expr)
+  def testOtherExpr(self):
+    """Some examples copied from pgen2/pgen2-test.sh mode-test."""
 
-    # Because we really need a lookup for a MODE.
-    # Problem: _UNQUOTED is used in both DBracket and ShCommand mode.
+    node = self._ParseOsh('@[1 2 3];')
 
+    CASES = [
+      '@[1 2 3]',
+      '$/ x /',
+      '$/ "." [a-z A-Z] y /',
+      '$[echo hi]',
+      '$(1 + 2)',
+      '${x}',
+      '"quoted ${x}"',
+    ]
 
-    arith = ID_SPEC.LexerPairs(Kind.Arith)
-    print(arith)
+    # array literal
+    for c in CASES:
+      node = self._ParseOilExpression(c)
 
-    # Doesn't have one.
-    #left = ID_SPEC.LexerPairs(Kind.Left)
-    #print(left)
+  def testLexer(self):
+    # NOTE: Kind.Expr for Oil doesn't have LexerPairs
+    pairs = ID_SPEC.LexerPairs(Kind.Arith)
+    for p in pairs:
+      #print(p)
+      pass
 
 
 if __name__ == '__main__':