[osh] Implement InitializerList, e.g. a=(v [k]=v [k]+=v) (#2279)

* [osh/word_parse] Introduce InitializerLiteral for `(v [k]=v [k]+=v ...)`

* [osh/word_parse] Allow mixed forms of initializer in BashAssocLiteral

  We allow mixed forms of initializer such as (1 2 [3]=4).  We extend
  the element type of "BashAssocLiteral.pairs" to "InitializerWord",
  which is either "ArrayWord" or "AssocPair". "ArrayWord" is the non
  array-assignment form of a word.  In addition, "AssocPair" now
  records also whether the word in initializer has the form "[]=" or
  "[]+=".

* [refactor frontend/syntax] Rename "{BashAssoc => Initializer}Literal"

* [osh/cmd_eval] Process InitializerList for tempenv

* [core/bash_impl] Implement InitializerList_ToStrForShellPrint for xtrace

* [refactor frontend/id_kind_def] Rename "Id.Right_{ShArrayLiteral => Initializer}"

* [osh] Add temporary adjustment for "assoc=(1 2)"

The current OSH behavior of overwriting "assoc" with a new indexed
array for "assoc=(1 2)" conflicts with the Bash 5.1 feature
"assoc=(key value)".  The current master behavior will soon be changed
when the Bash 5.1 feature is implemented, yet we keep the current
master behavior for now to avoid changing the behavior too many times.
This commit adjusts the behavior to be backward compatible with the
current master version.

* [spec] Update spec tests for "arr=([index]=value)"

* [spec/assoc] Update spec tests for "assoc=([key]=value)"

  We now support the sparse-array initialization of the form
  "arr=([index]=value)", so an attempt of assigning an initializer
  list to an indexed array no longer overwrites the original array
  with BashAssoc.  With this change, the expected behaviors of three
  spec tests in spec/assoc.test.sh are changed.  We adjust the
  expected results of the three tests and copy them into
  spec/ble-sparse.test.sh.  We also leave BashAssoc versions of spec
  tests in spec/assoc.test.sh.

* [spec/ysh-printing] Update spec tests for "assoc=([key]=value)"

* [spec/assign-extended] Confirm that "declare -p sp" is correctly parsed

The initialier-list implementation resolves the inconsistency in the
sparse array representation of the form "declare -a sp=([2]=v)".  This
commit removes the "oils_failures_allowed" introduced in Ref. [1] (as
a part of PR 2257, which turned on the sparse array representation of
indexed arrays).

[1] bfa7497#diff-097d35f191fa3ada9a05d61d3020a2b3acb43e800141b67021094fb05d8e769e

* [spec/assoc] Reduce "oils_failures_allowed"

OSH now implements the feature required by spec/assoc#37 "Implicit
increment of keys" so passes the test.

* [doc] Update reference

* [doc/{ref/{chap-type-method,toc-osh},known-differences,quirks}] Update

* [refactor] Rename "{Sh => Ysh}ArrayLiteral"

Author: akinomyoga

builtin/assign_osh.py

@@ -14,7 +14,7 @@
 
 from core import bash_impl
 from core import error
-from core.error import e_usage
+from core.error import e_usage, e_die
 from core import state
 from core import vm
 from display import ui
@@ -193,8 +193,9 @@ def _PrintVariables(mem, cmd_val, attrs, print_flags, builtin=_OTHER):
         return 1
 
 
-def _AssignVarForBuiltin(mem, rval, pair, which_scopes, flags):
-    # type: (Mem, value_t, AssignArg, scope_t, int) -> None
+def _AssignVarForBuiltin(mem, rval, pair, which_scopes, flags, arith_ev,
+                         flag_a, flag_A):
+    # type: (Mem, value_t, AssignArg, scope_t, int, sh_expr_eval.ArithEvaluator, bool, bool) -> None
     """For 'export', 'readonly', and NewVar to respect += and flags.
 
     Like 'setvar' (scope_e.LocalOnly), unless dynamic scope is on.  That is, it
@@ -203,28 +204,85 @@ def _AssignVarForBuiltin(mem, rval, pair, which_scopes, flags):
     Used for assignment builtins, (( a = b )), {fd}>out, ${x=}, etc.
     """
     lval = LeftName(pair.var_name, pair.blame_word)
-    if pair.plus_eq:
+
+    initializer = None  # type: value.InitializerList
+    if rval is None:
+        # When 'export e+=', then rval is value.Str('')
+        # When 'export foo', the pair.plus_eq flag is false.
+        # Thus, when rval is None, plus_eq cannot be True.
+        assert not pair.plus_eq, pair.plus_eq
+        # NOTE: when rval is None, only flags are changed
+        val = None  # type: value_t
+    elif rval.tag() == value_e.InitializerList:
+        old_val = sh_expr_eval.OldValue(
+            lval,
+            mem,
+            None,  # ignore set -u
+            pair.blame_word)
+        initializer = cast(value.InitializerList, rval)
+
+        # OSH compatibility: "declare -A assoc=(1 2)" is a usage
+        # error.  This code will soon be removed when the
+        # initialization of the form "assoc=(key value)" is supported.
+        if (flag_A and len(initializer.assigns) > 0 and
+                initializer.assigns[0].key is None):
+            e_usage("Can't initialize assoc array with indexed array",
+                    pair.blame_word)
+
+        val = old_val
+        if flag_a:
+            if old_val.tag() == value_e.BashAssoc:
+                # Note: OSH allows overwriting an existing BashArray with an
+                #   empty BashArray.  Bash does not allow this.
+                val = bash_impl.BashArray_New()
+            elif old_val.tag() in (value_e.Undef, value_e.Str,
+                                   value_e.BashArray):
+                # We do not need adjustemnts for -a.  These types are
+                # consistently handled within ListInitialize
+                pass
+            else:
+                e_die(
+                    "Can't convert type %s into BashArray" %
+                    ui.ValType(old_val), pair.blame_word)
+        elif flag_A:
+            if old_val.tag() in (value_e.Undef, value_e.Str,
+                                 value_e.BashArray):
+                # Note: We explicitly initialize BashAssoc for Undef.
+                # Note: OSH allows overwriting an existing BashArray with an
+                #   empty BashAssoc.  Bash does not allow this.
+                val = bash_impl.BashAssoc_New()
+            elif old_val.tag() == value_e.BashAssoc:
+                # We do not need adjustments for -A.
+                pass
+            else:
+                e_die(
+                    "Can't convert type %s into BashAssoc" %
+                    ui.ValType(old_val), pair.blame_word)
+
+        val = cmd_eval.ListInitializeTarget(val, initializer, pair.plus_eq,
+                                            pair.blame_word)
+    elif pair.plus_eq:
         old_val = sh_expr_eval.OldValue(
             lval,
             mem,
             None,  # ignore set -u
             pair.blame_word)
-        # When 'export e+=', then rval is value.Str('')
-        # When 'export foo', the pair.plus_eq flag is false.
-        assert rval is not None
         val = cmd_eval.PlusEquals(old_val, rval)
     else:
-        # NOTE: when rval is None, only flags are changed
         val = rval
 
     mem.SetNamed(lval, val, which_scopes, flags=flags)
+    if initializer is not None:
+        cmd_eval.ListInitialize(val, initializer, pair.plus_eq,
+                                pair.blame_word, arith_ev)
 
 
 class Export(vm._AssignBuiltin):
 
-    def __init__(self, mem, errfmt):
-        # type: (Mem, ui.ErrorFormatter) -> None
+    def __init__(self, mem, arith_ev, errfmt):
+        # type: (Mem, sh_expr_eval.ArithEvaluator, ui.ErrorFormatter) -> None
         self.mem = mem
+        self.arith_ev = arith_ev
         self.errfmt = errfmt
 
     def Run(self, cmd_val):
@@ -266,7 +324,8 @@ def Run(self, cmd_val):
             which_scopes = self.mem.ScopesForWriting()
             for pair in cmd_val.pairs:
                 _AssignVarForBuiltin(self.mem, pair.rval, pair, which_scopes,
-                                     state.SetExport)
+                                     state.SetExport, self.arith_ev, False,
+                                     False)
 
         return 0
 
@@ -298,32 +357,23 @@ def _ReconcileTypes(rval, flag_a, flag_A, pair, mem):
                 rval = bash_impl.BashAssoc_New()
     else:
         if flag_a:
-            if rval.tag() not in (value_e.InternalStringArray,
-                                  value_e.BashArray):
-                e_usage("Got -a but RHS isn't an array",
+            if rval.tag() != value_e.InitializerList:
+                e_usage("Got -a but RHS isn't an initializer list",
                         loc.Word(pair.blame_word))
-
         elif flag_A:
-            # Special case: declare -A A=() is OK.  The () is changed to mean
-            # an empty associative array.
-            if rval.tag() == value_e.BashArray:
-                sparse_val = cast(value.BashArray, rval)
-                if bash_impl.BashArray_IsEmpty(sparse_val):
-                    return bash_impl.BashAssoc_New()
-                    #return bash_impl.BashArray_New()
-
-            if rval.tag() != value_e.BashAssoc:
-                e_usage("Got -A but RHS isn't an associative array",
+            if rval.tag() != value_e.InitializerList:
+                e_usage("Got -A but RHS isn't an initializer list",
                         loc.Word(pair.blame_word))
 
     return rval
 
 
 class Readonly(vm._AssignBuiltin):
 
-    def __init__(self, mem, errfmt):
-        # type: (Mem, ui.ErrorFormatter) -> None
+    def __init__(self, mem, arith_ev, errfmt):
+        # type: (Mem, sh_expr_eval.ArithEvaluator, ui.ErrorFormatter) -> None
         self.mem = mem
+        self.arith_ev = arith_ev
         self.errfmt = errfmt
 
     def Run(self, cmd_val):
@@ -348,19 +398,21 @@ def Run(self, cmd_val):
             # - when rval is None, only flags are changed
             # - dynamic scope because flags on locals can be changed, etc.
             _AssignVarForBuiltin(self.mem, rval, pair, which_scopes,
-                                 state.SetReadOnly)
+                                 state.SetReadOnly, self.arith_ev, arg.a,
+                                 arg.A)
 
         return 0
 
 
 class NewVar(vm._AssignBuiltin):
     """declare/typeset/local."""
 
-    def __init__(self, mem, procs, exec_opts, errfmt):
-        # type: (Mem, state.Procs, optview.Exec, ui.ErrorFormatter) -> None
+    def __init__(self, mem, procs, exec_opts, arith_ev, errfmt):
+        # type: (Mem, state.Procs, optview.Exec, sh_expr_eval.ArithEvaluator, ui.ErrorFormatter) -> None
         self.mem = mem
         self.procs = procs
         self.exec_opts = exec_opts
+        self.arith_ev = arith_ev
         self.errfmt = errfmt
 
     def _PrintFuncs(self, names):
@@ -463,7 +515,8 @@ def Run(self, cmd_val):
         for pair in cmd_val.pairs:
             rval = _ReconcileTypes(pair.rval, arg.a, arg.A, pair, self.mem)
 
-            _AssignVarForBuiltin(self.mem, rval, pair, which_scopes, flags)
+            _AssignVarForBuiltin(self.mem, rval, pair, which_scopes, flags,
+                                 self.arith_ev, arg.a, arg.A)
 
         return status
 

core/bash_impl.py

@@ -2,7 +2,9 @@
 
 from _devbuild.gen.runtime_asdl import error_code_e, error_code_t
 from _devbuild.gen.value_asdl import value
+from _devbuild.gen.syntax_asdl import loc_t
 
+from core.error import e_die
 from data_lang import j8_lite
 from mycpp import mops
 from mycpp import mylib
@@ -30,6 +32,21 @@ def BigInt_LessEq(a, b):
     return not mops.Greater(a, b)
 
 
+class ArrayIndexEvaluator(object):
+    """Interface class for implementing the evaluation of array indices in
+    initializer-list items of the form [i]=1."""
+
+    def __init__(self):
+        # type: () -> None
+        """Empty constructor for mycpp."""
+        pass
+
+    def StringToBigInt(self, s, blame_loc):
+        # type: (str, loc_t) -> mops.BigInt
+        """Returns an array index obtained by evaluating the specified string."""
+        raise NotImplementedError()
+
+
 #------------------------------------------------------------------------------
 # All InternalStringArray operations depending on the internal representation
 # of InternalStringArray come here.
@@ -267,6 +284,35 @@ def BashAssoc_New():
     return value.BashAssoc(d)
 
 
+def BashAssoc_Copy(val):
+    # type: (value.BashAssoc) -> value.BashAssoc
+    # mycpp limitation: NewDict() needs to be typed
+    d = mylib.NewDict()  # type: Dict[str, str]
+    for key in val.d:
+        d[key] = val.d[key]
+    return value.BashAssoc(d)
+
+
+def BashAssoc_ListInitialize(val, initializer, has_plus, blame_loc):
+    # type: (value.BashAssoc, value.InitializerList, bool, loc_t) -> None
+
+    if not has_plus:
+        val.d.clear()
+
+    for triplet in initializer.assigns:
+        if triplet.key is None:
+            e_die(
+                "Key is missing. BashAssoc requires a key for %r" %
+                triplet.rval, blame_loc)
+
+        s = triplet.rval
+        if triplet.plus_eq:
+            old_s = val.d.get(triplet.key)
+            if old_s is not None:
+                s = old_s + s
+        val.d[triplet.key] = s
+
+
 def BashAssoc_IsEmpty(assoc_val):
     # type: (value.BashAssoc) -> bool
     return len(assoc_val.d) == 0
@@ -361,6 +407,14 @@ def BashArray_New():
     return value.BashArray(d, max_index)
 
 
+def BashArray_Copy(val):
+    # type: (value.BashArray) -> value.BashArray
+    d = {}  # type: Dict[mops.BigInt, str]
+    for index in val.d:
+        d[index] = val.d[index]
+    return value.BashArray(d, val.max_index)
+
+
 def BashArray_FromList(strs):
     # type: (List[str]) -> value.BashArray
     d = {}  # type: Dict[mops.BigInt, str]
@@ -373,6 +427,30 @@ def BashArray_FromList(strs):
     return value.BashArray(d, max_index)
 
 
+def BashArray_ListInitialize(val, initializer, has_plus, blame_loc, arith_ev):
+    # type: (value.BashArray, value.InitializerList, bool, loc_t, ArrayIndexEvaluator) -> None
+    if not has_plus:
+        val.d.clear()
+        val.max_index = mops.MINUS_ONE
+
+    array_index = val.max_index
+    for triplet in initializer.assigns:
+        if triplet.key is not None:
+            array_index = arith_ev.StringToBigInt(triplet.key, blame_loc)
+        else:
+            array_index = mops.Add(array_index, mops.ONE)
+
+        s = triplet.rval
+        if triplet.plus_eq:
+            old_s = val.d.get(array_index)
+            if old_s is not None:
+                s = old_s + s
+
+        val.d[array_index] = s
+        if BigInt_Greater(array_index, val.max_index):
+            val.max_index = array_index
+
+
 def BashArray_IsEmpty(sparse_val):
     # type: (value.BashArray) -> bool
     return len(sparse_val.d) == 0
@@ -505,3 +583,26 @@ def BashArray_ToStrForShellPrint(sparse_val):
 
         body.append(j8_lite.MaybeShellEncode(sparse_val.d[index]))
     return "(%s)" % ''.join(body)
+
+
+#------------------------------------------------------------------------------
+# InitializerList operations depending on its internal representation come
+# here.
+
+
+def InitializerList_ToStrForShellPrint(val):
+    # type: (value.InitializerList) -> str
+    body = []  # type: List[str]
+
+    for init in val.assigns:
+        if len(body) > 0:
+            body.append(" ")
+        if init.key is not None:
+            key = j8_lite.MaybeShellEncode(init.key)
+            if init.plus_eq:
+                body.extend(["[", key, "]+="])
+            else:
+                body.extend(["[", key, "]="])
+        body.append(j8_lite.MaybeShellEncode(init.rval))
+
+    return "(%s)" % ''.join(body)

core/dev.py

@@ -225,6 +225,10 @@ def _PrintShValue(val, buf):
             val = cast(value.BashArray, UP_val)
             result = bash_impl.BashArray_ToStrForShellPrint(val)
 
+        elif case(value_e.InitializerList):
+            val = cast(value.InitializerList, UP_val)
+            result = bash_impl.InitializerList_ToStrForShellPrint(val)
+
     buf.write(result)
 
 

core/shell.py

@@ -211,19 +211,20 @@ def InitAssignmentBuiltins(
         mem,  # type: state.Mem
         procs,  # type: state.Procs
         exec_opts,  # type: optview.Exec
+        arith_ev,  # type: sh_expr_eval.ArithEvaluator
         errfmt,  # type: ui.ErrorFormatter
 ):
     # type: (...) -> Dict[int, vm._AssignBuiltin]
 
     assign_b = {}  # type: Dict[int, vm._AssignBuiltin]
 
-    new_var = assign_osh.NewVar(mem, procs, exec_opts, errfmt)
+    new_var = assign_osh.NewVar(mem, procs, exec_opts, arith_ev, errfmt)
     assign_b[builtin_i.declare] = new_var
     assign_b[builtin_i.typeset] = new_var
     assign_b[builtin_i.local] = new_var
 
-    assign_b[builtin_i.export_] = assign_osh.Export(mem, errfmt)
-    assign_b[builtin_i.readonly] = assign_osh.Readonly(mem, errfmt)
+    assign_b[builtin_i.export_] = assign_osh.Export(mem, arith_ev, errfmt)
+    assign_b[builtin_i.readonly] = assign_osh.Readonly(mem, arith_ev, errfmt)
 
     return assign_b
 
@@ -506,7 +507,7 @@ def Main(
     word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, mutable_opts,
                                             tilde_ev, splitter, errfmt)
 
-    assign_b = InitAssignmentBuiltins(mem, procs, exec_opts, errfmt)
+    assign_b = InitAssignmentBuiltins(mem, procs, exec_opts, arith_ev, errfmt)
     cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs, assign_b,
                                        arena, cmd_deps, trap_state,
                                        signal_safe)