From 00fed97f0f212638e1ba0b6a2994f835689b905e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20L=C3=B3pez?= Date: Thu, 8 Dec 2016 04:03:08 +0000 Subject: [PATCH 1/4] [FIX] runbot_travis2docker: Fixing overlapped items - Helps to fix https://github.com/OCA/runbot-addons/issues/111 - Use a unique name of image - Use a unique path of t2d scripts - Remove use of global sys.args (Using directly the t2d class) - Add _log db and logger in order to self-debug - Use a copy of ids list instead of original pointer --- runbot_travis2docker/models/runbot_build.py | 57 +++++++++++++++------ 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/runbot_travis2docker/models/runbot_build.py b/runbot_travis2docker/models/runbot_build.py index efc1330c..20740813 100644 --- a/runbot_travis2docker/models/runbot_build.py +++ b/runbot_travis2docker/models/runbot_build.py @@ -7,13 +7,9 @@ import os import requests import subprocess -import sys import time import traceback -from travis2docker.git_run import GitRun -from travis2docker.cli import main as t2d - import openerp from openerp import fields, models, api from openerp.addons.runbot.runbot import (_re_error, _re_warning, grep, rfind, @@ -23,6 +19,13 @@ _logger = logging.getLogger(__name__) +try: + from travis2docker.git_run import GitRun + from travis2docker.cli import get_git_data + from travis2docker.travis2docker import Travis2Docker +except ImportError as err: + _logger.debug(err) + def custom_build(func): # TODO: Make this method more generic for re-use in all custom modules @@ -83,7 +86,8 @@ def get_docker_image(self, branch_closest=None): if build.repo_id.docker_registry_server else "" image_name = registry_host + \ git_obj.owner + '-' + git_obj.repo + ':' + branch + \ - '_' + os.path.basename(build.dockerfile_path) + '_' + os.path.basename(build.dockerfile_path) + \ + '_' + str(build.id) if branch_closest: image_name += '_cached' return image_name.lower() @@ -277,20 +281,36 @@ def use_build_cache(self): @custom_build def checkout(self, cr, uid, ids, context=None): """Save travis2docker output""" - to_be_skipped_ids = ids + to_be_skipped_ids = ids[:] for build in self.browse(cr, uid, ids, context=context): branch_short_name = build.branch_id.name.replace( 'refs/heads/', '', 1).replace('refs/pull/', 'pull/', 1) t2d_path = os.path.join(build.repo_id.root(), 'travis2docker') - sys.argv = [ - 'travisfile2dockerfile', build.repo_id.name, - branch_short_name, '--root-path=' + t2d_path, - ] + repo_name = build.repo_id.name + if not (repo_name.startswith('https://') or + repo_name.startswith('git@')): + repo_name = 'https://' + repo_name + sha = build.name + git_data = get_git_data( + repo_name, os.path.join(t2d_path, 'repo'), sha) + git_data['revision'] = branch_short_name + yml_content = git_data['content'] + t2d_e = None try: - path_scripts = t2d() - except BaseException: # TODO: Add custom exception to t2d - _logger.error(traceback.format_exc()) + t2d_obj = Travis2Docker( + yml_buffer=yml_content, + work_path=os.path.join(t2d_path, 'script', + str(build.id) + "_" + sha[:7]), + os_kwargs=git_data, + copy_paths=[("~/.ssh", "$HOME/.ssh")], + ) + path_scripts = t2d_obj.compute_dockerfile( + skip_after_success=True) + except BaseException as t2d_e: path_scripts = [] + build._log('t2d error', t2d_e.message) + _logger.error('t2d build#%d: "%s"', build.id, t2d_e.message) + _logger.error(traceback.format_exc()) for path_script in path_scripts: df_content = open(os.path.join( path_script, 'Dockerfile')).read() @@ -312,10 +332,13 @@ def checkout(self, cr, uid, ids, context=None): if build.id in to_be_skipped_ids: to_be_skipped_ids.remove(build.id) break - if to_be_skipped_ids: - _logger.info('Dockerfile without TESTS=1 env. ' - 'Skipping builds %s', to_be_skipped_ids) - self.skip(cr, uid, to_be_skipped_ids, context=context) + for build in self.browse(cr, uid, to_be_skipped_ids, context=context): + build._log('Dockerfile without TESTS=1 env.', 'Skipping') + _logger.warning('Dockerfile without TESTS=1 env. ' + 'Skipping build %d: %s %s', + build.id, build.repo_id.name, build.branch_id.name, + ) + build.skip() def docker_rm_container(self): for build in self: From 8813070048e4bfac450b7f6e68fd09043bda93c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20L=C3=B3pez?= Date: Tue, 27 Sep 2016 13:54:52 -0500 Subject: [PATCH 2/4] [REF] runbot_travis2docker: Long term test --- runbot_travis2docker/tests/test_runbot_build.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/runbot_travis2docker/tests/test_runbot_build.py b/runbot_travis2docker/tests/test_runbot_build.py index 68a8fdab..4047c03b 100644 --- a/runbot_travis2docker/tests/test_runbot_build.py +++ b/runbot_travis2docker/tests/test_runbot_build.py @@ -89,13 +89,15 @@ def run_jobs(self, branch): ('name', '=', branch)], limit=1) self.assertEqual(len(branch), 1, "Branch not found") self.build_obj.search([('branch_id', '=', branch.id)]).unlink() - - self.repo.update() - build = self.build_obj.search([ - ('branch_id', '=', branch.id)], limit=1, order='id desc') - self.assertEqual(len(build) == 0, False, "Build not found") - - if build.state == 'done' and build.result == 'skipped': + self.build_obj.create({'branch_id': branch.id, 'name': 'HEAD'}) + # runbot module has a inherit in create method + # but a "return id" is missed. Then we need to search it. + # https://github.com/odoo/odoo-extra/blob/038fd3e/runbot/runbot.py#L599 + self.build = self.build_obj.search([('branch_id', '=', branch.id)], + limit=1) + self.assertEqual(len(self.build) == 0, False, "Build not found") + + if self.build.state == 'done' and self.build.result == 'skipped': # When the last commit of the repo is too old, # runbot will skip this build then we are forcing it build.force() From 768e4d564b7551a8dbfae5e95cd33acd8ce3d36c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20L=C3=B3pez?= Date: Thu, 8 Dec 2016 15:50:26 -0600 Subject: [PATCH 3/4] [REF] runbot_travis2docker: Assign to object variable self.build --- .../tests/test_runbot_build.py | 61 ++++++++++--------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/runbot_travis2docker/tests/test_runbot_build.py b/runbot_travis2docker/tests/test_runbot_build.py index 4047c03b..05847edc 100644 --- a/runbot_travis2docker/tests/test_runbot_build.py +++ b/runbot_travis2docker/tests/test_runbot_build.py @@ -26,12 +26,13 @@ def setUp(self): self.repo = self.repo_obj.search([ ('is_travis2docker_build', '=', True)], limit=1) self.repo_domain = [('repo_id', '=', self.repo.id)] + self.build = None def delete_build_path(self, build): - subprocess.check_output(['rm', '-rf', build.path()]) + subprocess.check_output(['rm', '-rf', self.build.path()]) def delete_image_cache(self, build): - cmd = ['docker', 'rmi', '-f', build.docker_image_cache] + cmd = ['docker', 'rmi', '-f', self.build.docker_image_cache] res = -1 try: res = subprocess.check_output(cmd) @@ -40,7 +41,7 @@ def delete_image_cache(self, build): return res def delete_container(self, build): - cmd = ['docker', 'rm', '-f', build.get_docker_container()] + cmd = ['docker', 'rm', '-f', self.build.get_docker_container()] res = -1 try: res = subprocess.check_output(cmd) @@ -54,8 +55,8 @@ def wait_change_job(self, current_job, build, _logger.info("Waiting change of current job: %s", current_job) for count in range(loops): self.repo.cron() - if build.job != current_job: - return build.job + if self.build.job != current_job: + return self.build.job time.sleep(timeout) if divmod(count + 1, 5)[1] == 0: _logger.info("...") @@ -100,77 +101,77 @@ def run_jobs(self, branch): if self.build.state == 'done' and self.build.result == 'skipped': # When the last commit of the repo is too old, # runbot will skip this build then we are forcing it - build.force() + self.build.force() - build.checkout() - self.delete_build_path(build) + self.build.checkout() + self.delete_build_path(self.build) self.assertEqual( - build.state, u'pending', "State should be pending") + self.build.state, u'pending', "State should be pending") self.repo.cron() self.assertEqual( - build.state, u'testing', "State should be testing") + self.build.state, u'testing', "State should be testing") images_result = subprocess.check_output(['docker', 'images']) _logger.info(images_result) containers_result = subprocess.check_output(['docker', 'ps']) _logger.info(containers_result) - if not build.is_pull_request: + if not self.build.is_pull_request: self.assertEqual( - build.job, u'job_10_test_base', + self.build.job, u'job_10_test_base', "Job should be job_10_test_base") - new_current_job = self.wait_change_job(build.job, build) + new_current_job = self.wait_change_job(self.build.job, self.build) _logger.info( - open(os.path.join(build.path(), "logs", + open(os.path.join(self.build.path(), "logs", "job_10_test_base.txt")).read()) else: self.assertTrue( - self.docker_registry_test(build), + self.docker_registry_test(self.build), "Docker image don't found in registry to re-use in PR.", ) new_current_job = u'job_20_test_all' self.assertEqual( new_current_job, u'job_20_test_all') - new_current_job = self.wait_change_job(new_current_job, build) + new_current_job = self.wait_change_job(new_current_job, self.build) self.assertEqual( new_current_job, u'job_30_run', "Job should be job_30_run, found %s" % new_current_job) _logger.info(open( - os.path.join(build.path(), "logs", + os.path.join(self.build.path(), "logs", "job_20_test_all.txt")).read()) self.assertEqual( - build.state, u'running', + self.build.state, u'running', "Job state should be running") - user_ids = self.connection_test(build, 36, 10) + user_ids = self.connection_test(self.build, 36, 10) _logger.info(open( - os.path.join(build.path(), "logs", + os.path.join(self.build.path(), "logs", "job_30_run.txt")).read()) self.assertEqual( - build.state, u'running', + self.build.state, u'running', "Job state should be running still") self.assertEqual( len(user_ids) >= 1, True, "Failed connection test") self.assertEqual( - build.result, u'ok', "Job result should be ok") + self.build.result, u'ok', "Job result should be ok") self.assertTrue( - self.exists_container(build.docker_container), + self.exists_container(self.build.docker_container), "Container dont't exists") - build.kill() + self.build.kill() self.assertEqual( - build.state, u'done', "Job state should be done") + self.build.state, u'done', "Job state should be done") self.assertFalse( - self.exists_container(build.docker_container), + self.exists_container(self.build.docker_container), "Container don't deleted") - if not build.is_pull_request: + if not self.build.is_pull_request: self.assertTrue( - self.docker_registry_test(build), + self.docker_registry_test(self.build), "Docker image don't found in registry.", ) - self.delete_image_cache(build) + self.delete_image_cache(self.build) # Runbot original module use cr.commit :( # This explicit commit help us to avoid believe # that we will have a rollback of the data @@ -192,7 +193,7 @@ def docker_registry_test(self, build): "vauxoo-dev-runbot_branch_remote_name_grp_feature2/tags/list", ] tag_list_output = subprocess.check_output(cmd) - tag_build = build.docker_image_cache.split(':')[-1] + tag_build = self.build.docker_image_cache.split(':')[-1] return tag_build in tag_list_output def connection_test(self, build, attempts=1, delay=0): From f67f408115e0897c9d568a83433c6e1e00ab5f4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20L=C3=B3pez?= Date: Thu, 8 Dec 2016 16:17:04 -0600 Subject: [PATCH 4/4] =?UTF-8?q?[FIX]=C2=A0runbot=5Ftravis2docker:=20Fixing?= =?UTF-8?q?=20cache=20image=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- runbot_travis2docker/models/runbot_build.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/runbot_travis2docker/models/runbot_build.py b/runbot_travis2docker/models/runbot_build.py index 20740813..f1ca4dbe 100644 --- a/runbot_travis2docker/models/runbot_build.py +++ b/runbot_travis2docker/models/runbot_build.py @@ -86,10 +86,11 @@ def get_docker_image(self, branch_closest=None): if build.repo_id.docker_registry_server else "" image_name = registry_host + \ git_obj.owner + '-' + git_obj.repo + ':' + branch + \ - '_' + os.path.basename(build.dockerfile_path) + \ - '_' + str(build.id) + '_' + os.path.basename(build.dockerfile_path) + '_' if branch_closest: - image_name += '_cached' + image_name += 'cached' + else: + image_name += str(build.id) return image_name.lower() def get_docker_container(self):