Logging supervisor's child process logs

compose.yaml

@@ -17,6 +17,7 @@ services:
       - SUPERVISOR_URL=127.0.0.1:9001
       - AUTOTESTER_CONFIG=/app/.dockerfiles/docker-config.yml
       - STACK_ROOT=/home/docker/.autotesting/.stack
+      - SUPERVISOR_CHILD_LOG=/home/docker/.autotesting/supervisor_child_log
     depends_on:
       - postgres
       - redis

server/autotest_server/settings.yml

@@ -1,6 +1,7 @@
 workspace: !ENV ${WORKSPACE}
 redis_url: !ENV ${REDIS_URL}
 supervisor_url: !ENV ${SUPERVISOR_URL}
+supervisor_child_log: !ENV ${SUPERVISOR_CHILD_LOG}
 workers:
   - user: !ENV ${USER}
     queues:

server/install.py

@@ -62,6 +62,11 @@ def create_workspace():
     os.makedirs(config["workspace"], exist_ok=True)
 
 
+def create_supervisor_child_log():
+    _print(f'creating supervisor log folder for its children at {config["supervisor_child_log"]}')
+    os.makedirs(config["supervisor_child_log"], exist_ok=True)
+
+
 def install_all_testers():
     settings = install_testers()
     skeleton_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "autotest_server", "schema_skeleton.json")
@@ -76,6 +81,7 @@ def install():
     check_dependencies()
     check_users_exist()
     create_workspace()
+    create_supervisor_child_log()
     install_all_testers()
 
 

server/start_stop.py

@@ -40,6 +40,12 @@
 autorestart=true
 stopasgroup=true
 killasgroup=true
+stdout_logfile={stdout_logfile}
+stdout_logfile_maxbytes=1MB
+stdout_logfile_backups=10
+stderr_logfile={stderr_logfile}
+stderr_logfile_maxbytes=1MB
+stderr_logfile_backups=10
 
 """
 
@@ -57,6 +63,12 @@ def create_enqueuer_wrapper(rq):
                 queues=" ".join(worker_data["queues"]),
                 numprocs=1,
                 directory=os.path.dirname(os.path.realpath(__file__)),
+                stdout_logfile=os.path.join(
+                    _THIS_DIR, f'{config["supervisor_child_log"]}/{worker_data["user"]}_stdout.log'
+                ),
+                stderr_logfile=os.path.join(
+                    _THIS_DIR, f'{config["supervisor_child_log"]}/{worker_data["user"]}_stderr.log'
+                ),
             )
             f.write(c)