Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FIX] runbot_travis2docker: Fixing overlapped items #86

Merged
merged 4 commits into from
Dec 8, 2016
Merged

[FIX] runbot_travis2docker: Fixing overlapped items #86

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
00fed97
[FIX] runbot_travis2docker: Fixing overlapped items
moylop260 Dec 8, 2016
8813070
[REF] runbot_travis2docker: Long term test
moylop260 Sep 27, 2016
768e4d5
[REF] runbot_travis2docker: Assign to object variable self.build
moylop260 Dec 8, 2016
f67f408
[FIX] runbot_travis2docker: Fixing cache image name
moylop260 Dec 8, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 42 additions & 18 deletions runbot_travis2docker/models/runbot_build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,9 @@
import os
import requests
import subprocess
import sys
import time
import traceback

from travis2docker.git_run import GitRun
from travis2docker.cli import main as t2d

import openerp
from openerp import fields, models, api
from openerp.addons.runbot.runbot import (_re_error, _re_warning, grep, rfind,
Expand All @@ -23,6 +19,13 @@

_logger = logging.getLogger(__name__)

try:
from travis2docker.git_run import GitRun
from travis2docker.cli import get_git_data
from travis2docker.travis2docker import Travis2Docker
except ImportError as err:
_logger.debug(err)


def custom_build(func):
# TODO: Make this method more generic for re-use in all custom modules
Expand Down Expand Up @@ -83,9 +86,11 @@ def get_docker_image(self, branch_closest=None):
if build.repo_id.docker_registry_server else ""
image_name = registry_host + \
git_obj.owner + '-' + git_obj.repo + ':' + branch + \
'_' + os.path.basename(build.dockerfile_path)
'_' + os.path.basename(build.dockerfile_path) + '_'
if branch_closest:
image_name += '_cached'
image_name += 'cached'
else:
image_name += str(build.id)
return image_name.lower()

def get_docker_container(self):
Expand Down Expand Up @@ -277,20 +282,36 @@ def use_build_cache(self):
@custom_build
def checkout(self, cr, uid, ids, context=None):
"""Save travis2docker output"""
to_be_skipped_ids = ids
to_be_skipped_ids = ids[:]
for build in self.browse(cr, uid, ids, context=context):
branch_short_name = build.branch_id.name.replace(
'refs/heads/', '', 1).replace('refs/pull/', 'pull/', 1)
t2d_path = os.path.join(build.repo_id.root(), 'travis2docker')
sys.argv = [
'travisfile2dockerfile', build.repo_id.name,
branch_short_name, '--root-path=' + t2d_path,
]
repo_name = build.repo_id.name
if not (repo_name.startswith('https://') or
repo_name.startswith('git@')):
repo_name = 'https://' + repo_name
sha = build.name
git_data = get_git_data(
repo_name, os.path.join(t2d_path, 'repo'), sha)
git_data['revision'] = branch_short_name
yml_content = git_data['content']
t2d_e = None
try:
path_scripts = t2d()
except BaseException: # TODO: Add custom exception to t2d
_logger.error(traceback.format_exc())
t2d_obj = Travis2Docker(
yml_buffer=yml_content,
work_path=os.path.join(t2d_path, 'script',
str(build.id) + "_" + sha[:7]),
os_kwargs=git_data,
copy_paths=[("~/.ssh", "$HOME/.ssh")],
)
path_scripts = t2d_obj.compute_dockerfile(
skip_after_success=True)
except BaseException as t2d_e:
path_scripts = []
build._log('t2d error', t2d_e.message)
_logger.error('t2d build#%d: "%s"', build.id, t2d_e.message)
_logger.error(traceback.format_exc())
for path_script in path_scripts:
df_content = open(os.path.join(
path_script, 'Dockerfile')).read()
Expand All @@ -312,10 +333,13 @@ def checkout(self, cr, uid, ids, context=None):
if build.id in to_be_skipped_ids:
to_be_skipped_ids.remove(build.id)
break
if to_be_skipped_ids:
_logger.info('Dockerfile without TESTS=1 env. '
'Skipping builds %s', to_be_skipped_ids)
self.skip(cr, uid, to_be_skipped_ids, context=context)
for build in self.browse(cr, uid, to_be_skipped_ids, context=context):
build._log('Dockerfile without TESTS=1 env.', 'Skipping')
_logger.warning('Dockerfile without TESTS=1 env. '
'Skipping build %d: %s %s',
build.id, build.repo_id.name, build.branch_id.name,
)
build.skip()

def docker_rm_container(self):
for build in self:
Expand Down
77 changes: 40 additions & 37 deletions runbot_travis2docker/tests/test_runbot_build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,13 @@ def setUp(self):
self.repo = self.repo_obj.search([
('is_travis2docker_build', '=', True)], limit=1)
self.repo_domain = [('repo_id', '=', self.repo.id)]
self.build = None

def delete_build_path(self, build):
subprocess.check_output(['rm', '-rf', build.path()])
subprocess.check_output(['rm', '-rf', self.build.path()])

def delete_image_cache(self, build):
cmd = ['docker', 'rmi', '-f', build.docker_image_cache]
cmd = ['docker', 'rmi', '-f', self.build.docker_image_cache]
res = -1
try:
res = subprocess.check_output(cmd)
Expand All @@ -40,7 +41,7 @@ def delete_image_cache(self, build):
return res

def delete_container(self, build):
cmd = ['docker', 'rm', '-f', build.get_docker_container()]
cmd = ['docker', 'rm', '-f', self.build.get_docker_container()]
res = -1
try:
res = subprocess.check_output(cmd)
Expand All @@ -54,8 +55,8 @@ def wait_change_job(self, current_job, build,
_logger.info("Waiting change of current job: %s", current_job)
for count in range(loops):
self.repo.cron()
if build.job != current_job:
return build.job
if self.build.job != current_job:
return self.build.job
time.sleep(timeout)
if divmod(count + 1, 5)[1] == 0:
_logger.info("...")
Expand Down Expand Up @@ -89,86 +90,88 @@ def run_jobs(self, branch):
('name', '=', branch)], limit=1)
self.assertEqual(len(branch), 1, "Branch not found")
self.build_obj.search([('branch_id', '=', branch.id)]).unlink()

self.repo.update()
build = self.build_obj.search([
('branch_id', '=', branch.id)], limit=1, order='id desc')
self.assertEqual(len(build) == 0, False, "Build not found")

if build.state == 'done' and build.result == 'skipped':
self.build_obj.create({'branch_id': branch.id, 'name': 'HEAD'})
# runbot module has a inherit in create method
# but a "return id" is missed. Then we need to search it.
# https://github.com/odoo/odoo-extra/blob/038fd3e/runbot/runbot.py#L599
self.build = self.build_obj.search([('branch_id', '=', branch.id)],
limit=1)
self.assertEqual(len(self.build) == 0, False, "Build not found")

if self.build.state == 'done' and self.build.result == 'skipped':
# When the last commit of the repo is too old,
# runbot will skip this build then we are forcing it
build.force()
self.build.force()

build.checkout()
self.delete_build_path(build)
self.build.checkout()
self.delete_build_path(self.build)
self.assertEqual(
build.state, u'pending', "State should be pending")
self.build.state, u'pending', "State should be pending")

self.repo.cron()
self.assertEqual(
build.state, u'testing', "State should be testing")
self.build.state, u'testing', "State should be testing")
images_result = subprocess.check_output(['docker', 'images'])
_logger.info(images_result)
containers_result = subprocess.check_output(['docker', 'ps'])
_logger.info(containers_result)
if not build.is_pull_request:
if not self.build.is_pull_request:
self.assertEqual(
build.job, u'job_10_test_base',
self.build.job, u'job_10_test_base',
"Job should be job_10_test_base")
new_current_job = self.wait_change_job(build.job, build)
new_current_job = self.wait_change_job(self.build.job, self.build)
_logger.info(
open(os.path.join(build.path(), "logs",
open(os.path.join(self.build.path(), "logs",
"job_10_test_base.txt")).read())
else:
self.assertTrue(
self.docker_registry_test(build),
self.docker_registry_test(self.build),
"Docker image don't found in registry to re-use in PR.",
)
new_current_job = u'job_20_test_all'

self.assertEqual(
new_current_job, u'job_20_test_all')
new_current_job = self.wait_change_job(new_current_job, build)
new_current_job = self.wait_change_job(new_current_job, self.build)
self.assertEqual(
new_current_job, u'job_30_run',
"Job should be job_30_run, found %s" % new_current_job)
_logger.info(open(
os.path.join(build.path(), "logs",
os.path.join(self.build.path(), "logs",
"job_20_test_all.txt")).read())

self.assertEqual(
build.state, u'running',
self.build.state, u'running',
"Job state should be running")

user_ids = self.connection_test(build, 36, 10)
user_ids = self.connection_test(self.build, 36, 10)
_logger.info(open(
os.path.join(build.path(), "logs",
os.path.join(self.build.path(), "logs",
"job_30_run.txt")).read())

self.assertEqual(
build.state, u'running',
self.build.state, u'running',
"Job state should be running still")
self.assertEqual(
len(user_ids) >= 1, True, "Failed connection test")

self.assertEqual(
build.result, u'ok', "Job result should be ok")
self.build.result, u'ok', "Job result should be ok")
self.assertTrue(
self.exists_container(build.docker_container),
self.exists_container(self.build.docker_container),
"Container dont't exists")
build.kill()
self.build.kill()
self.assertEqual(
build.state, u'done', "Job state should be done")
self.build.state, u'done', "Job state should be done")
self.assertFalse(
self.exists_container(build.docker_container),
self.exists_container(self.build.docker_container),
"Container don't deleted")
if not build.is_pull_request:
if not self.build.is_pull_request:
self.assertTrue(
self.docker_registry_test(build),
self.docker_registry_test(self.build),
"Docker image don't found in registry.",
)
self.delete_image_cache(build)
self.delete_image_cache(self.build)
# Runbot original module use cr.commit :(
# This explicit commit help us to avoid believe
# that we will have a rollback of the data
Expand All @@ -190,7 +193,7 @@ def docker_registry_test(self, build):
"vauxoo-dev-runbot_branch_remote_name_grp_feature2/tags/list",
]
tag_list_output = subprocess.check_output(cmd)
tag_build = build.docker_image_cache.split(':')[-1]
tag_build = self.build.docker_image_cache.split(':')[-1]
return tag_build in tag_list_output

def connection_test(self, build, attempts=1, delay=0):
Expand Down