Improve test runner.

tests/ci_proxy.py

@@ -16,7 +16,7 @@ def recv_file(conn, target_root, mylogger):
     file_dict = json.loads(header.decode())
     file_name = file_dict['file_name']
     file_size = file_dict['file_size']
-    mylogger.debug('recv: file = {0}, size = {1}'.format(file_name, file_size))
+    mylogger.debug('recv begin: file = {0}, size = {1}'.format(file_name, file_size))
     conn.sendall(f"pls send {file_name}".encode())
 
     full_file = os.path.join(target_root, file_name)
@@ -28,6 +28,7 @@ def recv_file(conn, target_root, mylogger):
             recv_size += len(slice)
 
     os.chmod(full_file, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
+    mylogger.debug('recv end')
     return file_name
 
 
@@ -39,15 +40,13 @@ def Consumer(target, q, nfs_root, ip, port):
 
     # logging
     mylogger = logging.getLogger()
-    mylogger.setLevel(logging.DEBUG)
+    mylogger.setLevel(logging.INFO)
     rf_handler = logging.handlers.RotatingFileHandler(
         f'ci_proxy_{target}.log', mode='a', maxBytes=32 * 1024 * 1024, backupCount=10)
-    # rf_handler.setLevel(logging.INFO)
-    mylogger.setLevel(logging.DEBUG)
+    rf_handler.setLevel(logging.INFO)
     rf_handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(message)s'))
     mylogger.addHandler(rf_handler)
 
-    # telnet_client = TelnetClient(mylogger)
     while True:
         cmd = './'
         conn = q.get()
@@ -67,8 +66,6 @@ def Consumer(target, q, nfs_root, ip, port):
             else:
                 cmd = cmd + ' ' + file
 
-        # print('cmd = {0}'.format(cmd))
-
         # connect nuc_proxy server
         client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         client_socket.connect((ip, int(port)))
@@ -87,7 +84,6 @@ def Consumer(target, q, nfs_root, ip, port):
 
         infer_result = client_socket.recv(1024).decode()
         client_socket.close()
-        # print('infer_result = {0}'.format(infer_result))
         if infer_result.find('succeed') == -1:
             conn.sendall(f'infer failed on {target} board: {infer_result}'.encode())
         else:
@@ -101,10 +97,11 @@ def Consumer(target, q, nfs_root, ip, port):
                 conn.sendall(str(file_size).encode())
                 dummy = conn.recv(1024)
 
+                mylogger.debug('send begin: file = {0}, size = {1}'.format(file, file_size))
                 with open(file, 'rb') as f:
                     conn.sendall(f.read())
+                mylogger.debug('send end')
                 dummy = conn.recv(1024)
-                mylogger.debug('send: file = {0}, size = {1}'.format(file, file_size))
 
         conn.close()
 

tests/config.toml

@@ -1,6 +1,6 @@
 name = 'default_config'
 root = 'tests_output'
-dump_hist = true
+dump_hist = false
 
 [compile_opt]
 preprocess = false

tests/nuc_proxy.py

@@ -7,148 +7,73 @@
 import queue
 import logging
 import logging.handlers
-import telnetlib
 import time
 import serial
 import toml
 
 
-class TelnetClient():
-    def __init__(self, mylogger):
-        self.tn = telnetlib.Telnet()
-        self.logger = mylogger
-        self.ip = '10.99.105.216'
-        self.timeout = 60
-
-    def login(self, ip, username, password):
-        try:
-            self.tn.open(ip, port=23)
-        except:
-            self.logger.error('telnet {0} failed'.format(ip))
-            return False
-
-        self.ip = ip
-        self.tn.read_until(b'login: ', timeout=self.timeout)
-        self.tn.write(username.encode() + b'\r\n')
-
-        cmd_result = self.tn.read_very_eager().decode()
-        if 'Login incorrect' not in cmd_result:
-            self.logger.info('{0} login succeed'.format(ip))
-            return True
-        else:
-            self.logger.error('{0} login failed'.format(ip))
-            return False
-
-    def logout(self):
-        self.tn.close()
-        self.logger.info('{0} logout succeed'.format(self.ip))
-
-    def execute(self, cmd, flag):
-        self.logger.debug('execute: cmd = {0}, flag = {1}'.format(cmd, flag))
-        self.tn.write(cmd.encode() + b'\r\n')
-        cmd_result = self.tn.read_until(flag.encode(), timeout=self.timeout).decode()
-        if flag not in cmd_result:
-            # time out
-            self.tn.write(telnetlib.IP)
-            cmd_result = f'timeout for {self.timeout} seconds'
-            self.logger.error('execute {0} failed: {1}'.format(cmd, cmd_result))
-            return cmd_result, False
-        else:
-            self.tn.write('echo $?'.encode() + b'\r\n')
-            cmd_status = self.tn.read_until(flag.encode(), self.timeout).decode()
-            if cmd_status.find('\r\n0\r\n') == -1:
-                self.logger.error('execute {0} failed: {1}'.format(cmd, cmd_result))
-                return cmd_result, False
-            else:
-                return cmd_result, True
-
-
-def recv_file(conn, target_root, mylogger):
-    conn.sendall(f"pls send file info".encode())
-    header = conn.recv(1024)
-    file_dict = json.loads(header.decode())
-    file_name = file_dict['file_name']
-    file_size = file_dict['file_size']
-    mylogger.debug('recv: file = {0}, size = {1}'.format(file_name, file_size))
-    conn.sendall(f"pls send {file_name}".encode())
-
-    full_file = os.path.join(target_root, file_name)
-    with open(full_file, 'wb') as f:
-        recv_size = 0
-        while recv_size < file_size:
-            slice = conn.recv(4096)
-            f.write(slice)
-            recv_size += len(slice)
-
-    # conn.sendall(f"recv {file_name} succeed".encode())
-    os.chmod(full_file, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
-    return file_name
-
-
 class MySerial:
-    def __init__(self, target, port, baudrate):
+    def __init__(self, port, baudrate, logger):
         self.s = None
-        self.target = target
         self.port = port
         self.baudrate = baudrate
-        self.timeout = 60
-        # self.log_file = '{0}_uart.log'.format(target)
+        self.logger = logger
+        self.timeout = 20
 
     def open(self):
+        self.logger.debug('open serial begin')
         self.s = serial.Serial(port=self.port, baudrate=self.baudrate, timeout=self.timeout)
+        self.logger.debug('open serial end')
         # if (self.s.isOpen()):
         #     print('open {0} succeed'.format(self.port))
         # else:
         #     print('open {0} failed'.format(self.port))
 
     def close(self):
+        self.logger.debug('close serial begin')
         self.s.close()
+        self.logger.debug('close serial end')
 
     def write(self, cmd):
+        self.logger.debug('write begin')
         self.s.write(cmd.encode())
-
-    # def readall(self):
-    #     f = open(self.log_file, 'a')
-    #     while True:
-    #         # line = self.s.readline().decode()
-    #         #print('readline from serial {0}: {1}'.format(self.port, line))
-    #         print('{0}'.format(line), end='')
-    #         f.write(line)
-    #         if '}\n' in line:
-    #             break
-    #     f.close()
+        self.logger.debug('write end')
 
     def read_until(self, expected):
+        self.logger.debug('read begin')
         data = self.s.read_until(expected).decode()
-        # print('read: {0}'.format(data), end='')
+        self.logger.debug('read end: data = {0}'.format(data))
         return data
 
+    def run_cmd(self, cmd):
+        self.open()
+        self.write(cmd + '\r')
+        self.close()
 
-def run_cmds(s, cmds):
-    s.open()
-
-    for cmd in cmds.split('&&'):
-        s.write(cmd + '\r')
+    def run_cmds(self, cmds):
+        self.open()
 
-    data = s.read_until(b'}')
-    s.close()
+        for cmd in cmds.split('&&'):
+            self.write(cmd + '\r')
 
-    return data
+        data = self.read_until(b'}')
+        self.close()
+        return data
 
 
-def Consumer(target, q, ip, username, password, working_dir, uart, baudrate):
+def Consumer(target, q, working_dir, uart0, baudrate0, uart1, baudrate1):
     # logging
     mylogger = logging.getLogger()
-    mylogger.setLevel(logging.DEBUG)
+    mylogger.setLevel(logging.INFO)
     rf_handler = logging.handlers.RotatingFileHandler(
         f'nuc_proxy_{target}.log', mode='a', maxBytes=32 * 1024 * 1024, backupCount=10)
     rf_handler.setLevel(logging.INFO)
     rf_handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(message)s'))
     mylogger.addHandler(rf_handler)
-    telnet_client = TelnetClient(mylogger)
 
     # serial
-    s = MySerial(target, uart, baudrate)
+    s0 = MySerial(uart0, baudrate0, mylogger)
+    s1 = MySerial(uart1, baudrate1, mylogger)
 
     while True:
         conn = q.get()
@@ -158,43 +83,43 @@ def Consumer(target, q, ip, username, password, working_dir, uart, baudrate):
         header = conn.recv(1024)
         dict = json.loads(header.decode())
         cmd = dict['cmd']
-        mylogger.info("cmd = {0}".format(dict['cmd']))
+        mylogger.debug("cmd = {0}".format(dict['cmd']))
 
         # run cmds on device
         cmds = 'cd {0}/{1} && {2}'.format(working_dir, target, cmd)
-        ret_str = run_cmds(s, cmds)
+        ret_str = s1.run_cmds(cmds)
 
         # infer result
         if ret_str.find('terminate') != -1 or ret_str.find('Exception') != -1:
-            conn.sendall(f'infer exception: {ret_str}'.encode())
+            err = f'infer exception: {ret_str}'
+            mylogger.error('infer exception')
+            conn.sendall(err[0:1024].encode())
         elif ret_str.find('}') == -1:
             # reboot target when timeout
             conn.sendall(f'infer timeout'.encode())
-            mylogger.error('reboot {0}({1}) for timeout'.format(target, ip))
-            telnet_client.login(ip, username, password)
-            flag = f'[{username}@canaan ~ ]'
-            telnet_client.execute('reboot', flag)
-            telnet_client.logout()
-            time.sleep(30)
+            mylogger.error('reboot {0} for timeout'.format(target))
+            s0.run_cmd('reboot')
+            time.sleep(20)
         else:
             conn.sendall(f'infer succeed'.encode())
+            mylogger.debug('infer succeed')
         conn.close()
 
 
 def main():
     # default config
     config = '''
     [k230]
-    ip = '192.168.1.230'
-    username = 'root'
-    password = ''
     working_dir = '/sharefs'
-    uart = '/dev/ttyUSB1'
-    baudrate = 2500000
+    uart0 = '/dev/ttyUSB0'
+    baudrate0 = 115200
+    uart1 = '/dev/ttyUSB1'
+    baudrate1 = 115200
     '''
 
     # args
     parser = argparse.ArgumentParser(prog="nuc_proxy")
+    parser.add_argument("--ip", help='ip to connect', type=str, default='10.99.105.216')
     parser.add_argument("--port", help='listening port', type=int, default=10001)
     parser.add_argument("--config", help='config str or file', type=str, default=config)
     args = parser.parse_args()
@@ -211,13 +136,13 @@ def main():
     for k in cfg:
         q = queue.Queue(maxsize=size)
         t_consumer = threading.Thread(target=Consumer, args=(
-            k, q, cfg[k]["ip"], cfg[k]['username'], cfg[k]['password'], cfg[k]['working_dir'], cfg[k]['uart'], cfg[k]['baudrate']))
+            k, q, cfg[k]['working_dir'], cfg[k]['uart0'], cfg[k]['baudrate0'], cfg[k]['uart1'], cfg[k]['baudrate1']))
         t_consumer.start()
         cfg[k]['queue'] = q
 
     # server socket
     server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    server_socket.bind(('localhost', args.port))
+    server_socket.bind((args.ip, args.port))
     server_socket.listen(size)
     while True:
         conn, addr = server_socket.accept()

tests/test_runner.py

@@ -250,7 +250,7 @@ def run(self, model_file: Union[List[str], str]):
                             mode_dir = os.path.join(target_dir, k_mode)
                             shutil.move(tmp_dir, mode_dir)
                             judge, result = self.compare_results(
-                                expected, actual, stage, k_target, v_target['similarity_name'], k_mode, v_mode['threshold'], self.cfg['dump_hist'], mode_dir)
+                                expected, actual, stage, k_target, v_target['similarity_name'], k_mode, v_mode['threshold'], dump_hist, mode_dir)
 
                             if not judge:
                                 if test_utils.in_ci():