From a53430c0e9ed0c428a58655346d520ca3a75d5df Mon Sep 17 00:00:00 2001 From: Jiri Popelka Date: Wed, 9 Oct 2019 15:48:22 +0200 Subject: [PATCH 1/3] Update Dockerfile & files/recipe.yaml --- Dockerfile.app | 19 ++++++------------- Makefile | 3 +-- files/recipe.yaml | 20 ++++++++------------ 3 files changed, 15 insertions(+), 27 deletions(-) diff --git a/Dockerfile.app b/Dockerfile.app index b1d7161..738d352 100644 --- a/Dockerfile.app +++ b/Dockerfile.app @@ -1,15 +1,13 @@ FROM registry.fedoraproject.org/fedora:30 -ENV LANG=en_US.UTF-8 -# nicer output from the playbook run -ENV ANSIBLE_STDOUT_CALLBACK=debug +ENV LANG=en_US.UTF-8 \ + ANSIBLE_STDOUT_CALLBACK=debug \ + USER=release-bot \ + HOME=/home/release-bot + # Ansible doesn't like /tmp COPY files/ /src/files/ -USER 0 - -RUN mkdir /home/release-bot && chmod 0776 /home/release-bot - # Install packages first and reuse the cache as much as possible RUN dnf install -y ansible \ && cd /src/ \ @@ -24,9 +22,4 @@ COPY release_bot/ /src/release_bot/ RUN cd /src/ \ && ansible-playbook -vv -c local -i localhost, files/recipe.yaml - -USER 1001 -ENV USER=release-bot -ENV HOME=/home/release-bot - -CMD ["release-bot -c /home/release-bot/.config/conf.yaml"] \ No newline at end of file +CMD ["/usr/bin/run.sh"] diff --git a/Makefile b/Makefile index afb02c9..5ccb31d 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ .PHONY: test -IMAGE_NAME := usercont/release-bot -IMAGE_NAME_DEV := usercont/release-bot:dev +IMAGE_NAME := usercont/release-bot:dev TEST_IMAGE_NAME := release-bot-tests image: files/install-rpm-packages.yaml files/recipe.yaml diff --git a/files/recipe.yaml b/files/recipe.yaml index c062f08..6f39ba5 100644 --- a/files/recipe.yaml +++ b/files/recipe.yaml @@ -3,18 +3,12 @@ hosts: all vars: home_path: "{{ lookup('env', 'HOME') }}" - ansible_bender: - base_image: FROM registry.fedoraproject.org/fedora:30 - #base_image: fedora:29 - target_image: - name: docker.io/usercont/release-bot - environment: - FLASK_APP: releasebot.service.web_hook - cmd: flask-3 run -h 0.0.0.0 - working_container: - volumes: - - '{{ playbook_dir }}:/src:Z' tasks: + - name: Create home + file: + state: directory + path: "{{ home_path }}" + mode: 0776 - name: Create some home dirs file: state: directory @@ -52,4 +46,6 @@ name: /src executable: pip3 - name: Clean all the cache files (especially pip) - shell: rm -rf ~/.cache/* \ No newline at end of file + file: + state: absent + path: ~/.cache/ From 65360bc95d1949d519366277e3c4125020ecbc82 Mon Sep 17 00:00:00 2001 From: Jiri Popelka Date: Wed, 9 Oct 2019 15:50:15 +0200 Subject: [PATCH 2/3] Dockerfile.app -> Dockerfile --- Dockerfile.app => Dockerfile | 0 Makefile | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename Dockerfile.app => Dockerfile (100%) diff --git a/Dockerfile.app b/Dockerfile similarity index 100% rename from Dockerfile.app rename to Dockerfile diff --git a/Makefile b/Makefile index 5ccb31d..5b3f4ee 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ IMAGE_NAME := usercont/release-bot:dev TEST_IMAGE_NAME := release-bot-tests image: files/install-rpm-packages.yaml files/recipe.yaml - docker build --rm -f Dockerfile.app --tag=$(IMAGE_NAME) . + docker build --rm -f Dockerfile --tag=$(IMAGE_NAME) . image-test: docker build --tag=$(TEST_IMAGE_NAME) -f Dockerfile.test . From ccd9b322834a39d382b56bd60e14cf4d262fcf9d Mon Sep 17 00:00:00 2001 From: Jiri Popelka Date: Wed, 9 Oct 2019 15:50:50 +0200 Subject: [PATCH 3/3] More debug logs & set celery worker log level to debug --- files/run.sh | 4 +--- release_bot/celery_task.py | 20 ++++++++++++++------ release_bot/configuration.py | 1 + release_bot/webhooks.py | 4 ++-- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/files/run.sh b/files/run.sh index bdb3d62..820e6cb 100644 --- a/files/run.sh +++ b/files/run.sh @@ -1,6 +1,4 @@ #!/usr/bin/bash -export RELEASE_BOT_HOME=/home/release-bot - exec release-bot -c /home/release-bot/.config/conf.yaml & -exec celery -A release_bot.celery_task worker -l info +exec celery -A release_bot.celery_task worker -l debug diff --git a/release_bot/celery_task.py b/release_bot/celery_task.py index a502252..93c88c4 100644 --- a/release_bot/celery_task.py +++ b/release_bot/celery_task.py @@ -23,6 +23,7 @@ from release_bot.releasebot import ReleaseBot DEFAULT_CONF_FILE = "/home/release-bot/.config/conf.yaml" +logger = configuration.logger @celery_app.task(name="task.celery_task.parse_web_hook_payload") @@ -44,7 +45,6 @@ def parse_web_hook_payload(webhook_payload): installation_id = webhook_payload['installation']['id'] repositories_added = webhook_payload['repositories_added'] save_new_installations(installation_id, repositories_added, db) - if webhook_payload['action'] == 'removed': # detect when repo uninstall app repositories_removed = webhook_payload['repositories_removed'] delete_installations(repositories_removed, db) @@ -88,7 +88,7 @@ def set_configuration(webhook_payload, db, issue=True): f'{configuration.github_token}@github.com/' \ f'{configuration.repository_owner}/{configuration.repository_name}.git' - return ReleaseBot(configuration), configuration.logger + return ReleaseBot(configuration) def handle_issue(webhook_payload, db): @@ -98,7 +98,7 @@ def handle_issue(webhook_payload, db): :param db: Redis instance :return: """ - release_bot, logger = set_configuration(webhook_payload, db=db, issue=True) + release_bot = set_configuration(webhook_payload, db=db, issue=True) logger.info("Resolving opened issue") release_bot.git.pull() @@ -122,7 +122,7 @@ def handle_pr(webhook_payload, db): :param db: Redis instance :return: """ - release_bot, logger = set_configuration(webhook_payload, db=db, issue=False) + release_bot = set_configuration(webhook_payload, db=db, issue=False) logger.info("Resolving opened PR") release_bot.git.pull() @@ -150,11 +150,15 @@ def save_new_installations(installation_id, repositories_added, db): :param db: Redis instance :return: True if data was saved successfully into Redis """ + logger.debug(f"Saving: {installation_id} added to {repositories_added}") with db.pipeline() as pipe: for repo in repositories_added: pipe.set(repo["full_name"], installation_id) pipe.execute() - return db.save() + success = db.save() + if not success: + logger.error(f"Failed to save {installation_id}:{repositories_added}") + return success def delete_installations(repositories_removed, db): @@ -164,8 +168,12 @@ def delete_installations(repositories_removed, db): :param db: Redis instance :return: True if data was deleted successfully into Redis """ + logger.debug(f"Removing: {repositories_removed}") with db.pipeline() as pipe: for repo in repositories_removed: pipe.delete(repo["full_name"]) pipe.execute() - return db.save() + success = db.save() + if not success: + logger.error(f"Failed to remove {repositories_removed}") + return success diff --git a/release_bot/configuration.py b/release_bot/configuration.py index e7eba2f..47b12f5 100644 --- a/release_bot/configuration.py +++ b/release_bot/configuration.py @@ -152,6 +152,7 @@ def load_release_conf(self, conf): self.logger.warning(msg) parsed_conf['trigger_on_issue'] = False + self.logger.debug(f"Loaded release-conf.yaml: {parsed_conf}") return parsed_conf def set_pypi_project(self, parsed_conf, setup_cfg=None): diff --git a/release_bot/webhooks.py b/release_bot/webhooks.py index 9559ce3..d0a31f6 100644 --- a/release_bot/webhooks.py +++ b/release_bot/webhooks.py @@ -31,10 +31,10 @@ def __init__(self, conf): self.logger = conf.logger def dispatch_request(self): - self.logger.info(f'New github webhook call from detected') + self.logger.info('New github webhook call detected') if request.is_json: celery_app.send_task(name="task.celery_task.parse_web_hook_payload", - kwargs={"webhook_payload": request.get_json()}) + kwargs={"webhook_payload": request.json}) else: self.logger.error("This webhook doesn't contain JSON") return jsonify(result={"status": 200})