more automated checking, improve output

Signed-off-by: Hugo Lefeuvre <hugo.lefeuvre@manchester.ac.uk>

Author: hlef

explore.py

@@ -640,8 +640,8 @@ def explore_perf(errno, syscalls):
     if u:
         info("Final check analysis for stubbing succeeded; the entire set can be stubbed simultaneously.")
     else:
-        info("Final check analysis for stubbing failed; the entire set cannot be stubbed simultaneously.")
-        info("We recommend a manual pass with seccomp_run to find the culprit.")
+        warning("Final check analysis for stubbing failed; the entire set cannot be stubbed simultaneously.")
+        warning("We recommend a manual pass with seccomp_run to find the culprit.")
 
 info("Finding system calls that work when we fake (errno = 0)...")
 
@@ -654,8 +654,8 @@ def explore_perf(errno, syscalls):
     if u:
         info("Final check analysis for faking succeeded; the entire set can be faked simultaneously.")
     else:
-        info("Final check analysis for faking failed; the entire set cannot be faked simultaneously.")
-        info("We recommend a manual pass with seccomp_run to find the culprit.")
+        warning("Final check analysis for faking failed; the entire set cannot be faked simultaneously.")
+        warning("We recommend a manual pass with seccomp_run to find the culprit.")
 
 require_impl = list(((set(used)) - set(probably_works_stubbed)) - set(probably_works_lying))
 require_impl.sort()

loupe

@@ -623,9 +623,9 @@ def run_tests(path_db, application, workload, path_dockerfile, only_build_docker
     results = [None] * NUMBER_GENERATE_REPLICAS
     static_results_array = [None]
     for j in range(NUMBER_GENERATE_REPLICAS // NUMBER_PARALLEL_REPLICAS):
-        debug("Starting replica batch (%d/%d)" % ((
+        debug("Starting replica batch (%d/%d), batch size %d" % ((
             j * NUMBER_PARALLEL_REPLICAS) + NUMBER_PARALLEL_REPLICAS,
-            NUMBER_GENERATE_REPLICAS))
+            NUMBER_GENERATE_REPLICAS, NUMBER_PARALLEL_REPLICAS))
 
         for i in range(NUMBER_PARALLEL_REPLICAS):
             t = threading.Thread(target=_run_test, args=(containername,
@@ -784,7 +784,7 @@ def run_tests(path_db, application, workload, path_dockerfile, only_build_docker
         with open(os.path.join(runpath, "data", "static_binary.csv"), "a+") as outf:
             outf.write(static_results)
 
-    debug("Outputing additional reproducibility information...")
+    debug("Outputing additional reproducibility and debugging information...")
 
     with open(os.path.join(runpath, "cmd.txt"), "a+") as outf:
         outf.write(" ".join(sys.argv))
@@ -801,6 +801,25 @@ def run_tests(path_db, application, workload, path_dockerfile, only_build_docker
                         outf.write(logf.read())
                     repnum += 1
 
+    debug("Checking explorer logs...")
+    with open(os.path.join(runpath, "explore.logs")) as logf:
+        content = logf.read()
+        err = False
+
+        if "Final check analysis for faking failed; the entire set cannot" in content:
+            warning("At least one replica reported issues when consolidating stubbing")
+            warning("and faking measurements. Please check %s." %
+                    os.path.join(runpath, "explore.logs"))
+            err = True
+
+        if "[E] " in content:
+            warning("At least one replica reported an error. Please check %s."
+                    % os.path.join(runpath, "explore.logs"))
+            err = True
+
+        if not err:
+            debug("explorer logs all OK.")
+
     end = datetime.now()
 
     info("Done! Full analysis last %s." % str(end - start))
@@ -1329,6 +1348,7 @@ if (args.cmd == "generate"):
     # check Dockerfile for obvious errors
     # we just provide that as a quick sanity check for confused users
     # hopefully this does reduce the amount of bug reports we get :)
+    debug("Checking Dockerfile...")
     with open(args.dockerfile) as dockerf:
         content = dockerf.read()
         if "loupe-base:latest" not in content:
@@ -1345,6 +1365,10 @@ if (args.cmd == "generate"):
                   "option, which is mandatory when using the loupe wrapper." % args.dockerfile)
             exit(1)
 
+        if "--final-check" not in content:
+            warning("Dockerfile (%s) calls explore.py without the --final-check "
+                  "option, which is recommended." % args.dockerfile)
+
     # remove potential logs from previous runs
     try:
        os.remove(QUIET_LOG)