tests: ovsdb-idl: Fix flaky insert and delete tests.

The weak reference test sends two transactions and then waits for an
update and sends a third transaction.  First transaction adds a row
and the second one deletes it.  The test relies on an IDL run between
the second and the third transactions to receive and process updates
from both previous transactions, and that is not guaranteed.  This
is causing frequent test failures in CI.

If the server is slow to send an update for the second transaction,
the IDL run will return without receiving it and the test application
has no way to know that it needs to wait for one more update.  The
change sequence number doesn't have a meaning other than it changes
on updates, so we can't rely on it to wait for two updates.

The strong reference test has somewhat similar issue, it sends two
transactions and then expects both updates to be received together.
And that is, again, not guaranteed.  However, the difference here is
that the issue (crash) that it tries to reproduce requires both
updates to be received within the same IDL run, which was not a hard
requirement for the weak reference test.  The weak reference test
tolerates receiving updates in two separate IDL runs, so we could have
fixed it by waiting for an update after the first transaction without
clearing the tracking and then waiting again after the second.  But
such approach would not work for the strong reference test.  And there
is actually no way for us to ensure that both updates will be received
within the same IDL run.  Best we can do is to sleep for some time
hoping that it's enough for both updates to be queued up.  So, let's
do that.  Tests are very simple and fast, so sleeping for 1 second
should be enough.

'refs to link[12]' tests are similar to a strong reference test in
terms that they require changes to be processed within the same
IDL run.

Fixes: 02f31a1 ("ovsdb-idl: Preserve references for rows deleted in same IDL run as their insertion.")
Fixes: 91e1ff5 ("ovsdb-idl: Don't reparse orphaned rows.")
Fixes: 4102674 ("ovsdb-idl: Preserve change_seqno when deleting rows.")
Acked-by: Mike Pattrick <mkp@redhat.com>
Signed-off-by: Ilya Maximets <i.maximets@ovn.org>

Author: igsilya

tests/ovsdb-idl.at

@@ -1562,6 +1562,7 @@ OVSDB_CHECK_IDL_TRACK([track, simple idl, initially populated, weak references,
       {"op": "delete",
        "table": "simple",
        "where": [["s", "==", "row1_s"]]}]' \
+    '+sleep' \
     '["idltest",
       {"op": "insert",
        "table": "simple",
@@ -1574,16 +1575,17 @@ OVSDB_CHECK_IDL_TRACK([track, simple idl, initially populated, weak references,
 001: table simple: updated columns: s
 002: {"error":null,"result":[{"uuid":["uuid","<3>"]},{"count":1}]}
 003: {"error":null,"result":[{"count":1}]}
-004: table simple6: name=row0_s6 weak_ref=[<0>] uuid=<1>
-004: table simple6: updated columns: weak_ref
-004: table simple: inserted/deleted row: i=0 r=0 b=false s=row1_s u=<2> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<3>
-004: table simple: updated columns: s
-005: {"error":null,"result":[{"uuid":["uuid","<4>"]}]}
-006: table simple6: name=row0_s6 weak_ref=[<0>] uuid=<1>
-006: table simple: i=0 r=0 b=false s=row0_s u=<2> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<0>
-006: table simple: inserted row: i=0 r=0 b=false s=row2_s u=<2> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<4>
-006: table simple: updated columns: s
-007: done
+004: sleep
+005: table simple6: name=row0_s6 weak_ref=[<0>] uuid=<1>
+005: table simple6: updated columns: weak_ref
+005: table simple: inserted/deleted row: i=0 r=0 b=false s=row1_s u=<2> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<3>
+005: table simple: updated columns: s
+006: {"error":null,"result":[{"uuid":["uuid","<4>"]}]}
+007: table simple6: name=row0_s6 weak_ref=[<0>] uuid=<1>
+007: table simple: i=0 r=0 b=false s=row0_s u=<2> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<0>
+007: table simple: inserted row: i=0 r=0 b=false s=row2_s u=<2> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<4>
+007: table simple: updated columns: s
+008: done
 ]])
 
 dnl This test checks that deleting both the destination and source of the
@@ -1757,14 +1759,16 @@ OVSDB_CHECK_IDL_TRACK([track, simple idl, initially empty, strong references, in
     '+["idltest",
       {"op": "delete",
        "table": "link2",
-       "where": [["i", "==", 2]]}]'
+       "where": [["i", "==", 2]]}]' \
+    '+sleep'
   ]],
   [[000: empty
 001: {"error":null,"result":[{"uuid":["uuid","<0>"]},{"uuid":["uuid","<1>"]}]}
 002: {"error":null,"result":[{"count":1}]}
-003: table link1: inserted row: i=1 k=1 ka=[] l2= uuid=<0>
-003: table link1: updated columns: i k
-004: done
+003: sleep
+004: table link1: inserted row: i=1 k=1 ka=[] l2= uuid=<0>
+004: table link1: updated columns: i k
+005: done
 ]])
 
 OVSDB_CHECK_IDL_TRACK([track, simple idl, initially empty, various ops],
@@ -2741,6 +2745,7 @@ OVSDB_CHECK_IDL_TRACK([track, insert and delete, refs to link1],
        "table": "link2",
        "where": [["i", "==", 2]]}
        ]' \
+    '+sleep' \
     '["idltest",
       {"op": "delete",
        "table": "link2",
@@ -2750,15 +2755,16 @@ OVSDB_CHECK_IDL_TRACK([track, insert and delete, refs to link1],
   [[000: empty
 001: {"error":null,"result":[{"uuid":["uuid","<0>"]},{"uuid":["uuid","<1>"]},{"uuid":["uuid","<2>"]}]}
 002: {"error":null,"result":[{"count":1}]}
-003: table link1: inserted row: i=1 k=1 ka=[] l2= uuid=<1>
-003: table link1: updated columns: i k
-003: table link2: inserted row: i=1 l1=1 uuid=<0>
-003: table link2: inserted/deleted row: i=2 l1=1 uuid=<2>
-003: table link2: updated columns: i l1
-003: table link2: updated columns: i l1
-004: {"error":null,"result":[{"count":1}]}
-005: table link1: i=1 k=1 ka=[] l2= uuid=<1>
-006: done
+003: sleep
+004: table link1: inserted row: i=1 k=1 ka=[] l2= uuid=<1>
+004: table link1: updated columns: i k
+004: table link2: inserted row: i=1 l1=1 uuid=<0>
+004: table link2: inserted/deleted row: i=2 l1=1 uuid=<2>
+004: table link2: updated columns: i l1
+004: table link2: updated columns: i l1
+005: {"error":null,"result":[{"count":1}]}
+006: table link1: i=1 k=1 ka=[] l2= uuid=<1>
+007: done
 ]])
 OVSDB_CHECK_IDL_TRACK([track, insert and delete, refs to link2],
   [],
@@ -2784,6 +2790,7 @@ OVSDB_CHECK_IDL_TRACK([track, insert and delete, refs to link2],
        "table": "link1",
        "where": [["i", "==", 2]]}
        ]' \
+    '+sleep' \
     '["idltest",
       {"op": "delete",
        "table": "link1",
@@ -2793,15 +2800,16 @@ OVSDB_CHECK_IDL_TRACK([track, insert and delete, refs to link2],
   [[000: empty
 001: {"error":null,"result":[{"uuid":["uuid","<0>"]},{"uuid":["uuid","<1>"]},{"uuid":["uuid","<2>"]}]}
 002: {"error":null,"result":[{"count":1}]}
-003: table link1: inserted row: i=1 k=1 ka=[] l2=1 uuid=<0>
-003: table link1: inserted/deleted row: i=2 k=1 ka=[] l2=1 uuid=<2>
-003: table link1: updated columns: i k l2
-003: table link1: updated columns: i k l2
-003: table link2: inserted row: i=1 l1= uuid=<1>
-003: table link2: updated columns: i
-004: {"error":null,"result":[{"count":1}]}
-005: table link2: i=1 l1= uuid=<1>
-006: done
+003: sleep
+004: table link1: inserted row: i=1 k=1 ka=[] l2=1 uuid=<0>
+004: table link1: inserted/deleted row: i=2 k=1 ka=[] l2=1 uuid=<2>
+004: table link1: updated columns: i k l2
+004: table link1: updated columns: i k l2
+004: table link2: inserted row: i=1 l1= uuid=<1>
+004: table link2: updated columns: i
+005: {"error":null,"result":[{"count":1}]}
+006: table link2: i=1 l1= uuid=<1>
+007: done
 ]])
 
 m4_define([OVSDB_CHECK_IDL_PERS_UUID_INSERT_C],

tests/test-ovsdb.c

@@ -2884,6 +2884,9 @@ do_idl(struct ovs_cmdl_context *ctx)
         if (!strcmp(arg, "reconnect")) {
             print_and_log("%03d: reconnect", step++);
             ovsdb_idl_force_reconnect(idl);
+        } else if (!strcmp(arg, "sleep")) {
+            print_and_log("%03d: sleep", step++);
+            xsleep(1);
         }  else if (!strncmp(arg, remote_s, strlen(remote_s))) {
             ovsdb_idl_set_remote(idl, arg + strlen(remote_s), true);
             print_and_log("%03d: new remotes: %s, is connected: %s", step++,