diff --git a/Makefile b/Makefile index 9f784cb875..666b42cd07 100644 --- a/Makefile +++ b/Makefile @@ -329,6 +329,13 @@ build-debs-notest: ## Build SecureDrop Debian packages without running tests. @$(SDROOT)/devops/scripts/build-debs.sh notest @echo +.PHONY: build-debs-focal +build-debs-focal: ## Build and test SecureDrop Debian packages. + @echo "Building SecureDrop Debian packages..." + @$(SDROOT)/devops/scripts/build-debs.sh notest focal + @echo + + ######################## # diff --git a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/defaults/main.yml b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/defaults/main.yml index 80643216ff..23fb2cb720 100644 --- a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/defaults/main.yml +++ b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/defaults/main.yml @@ -45,6 +45,4 @@ securedrop_app_rsync_opts: securedrop_local_build: "../../build" -# Always build packages targeting Xenial. -securedrop_build_xenial_support: True securedrop_package_dist: xenial diff --git a/molecule/builder-focal/Dockerfile b/molecule/builder-focal/Dockerfile new file mode 100644 index 0000000000..a75cde3029 --- /dev/null +++ b/molecule/builder-focal/Dockerfile @@ -0,0 +1,50 @@ +# ubuntu:focal-20200720 +FROM ubuntu@sha256:60f560e52264ed1cb7829a0d59b1ee7740d7580e0eb293aca2d722136edb1e24 + + +# additional meta-data makes it easier to clean up, find +LABEL org="Freedom of the Press" +LABEL image_name="focal-sd-builder-app" +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get -y update && apt-get upgrade -y && apt-get install -y \ + apache2-dev \ + aptitude \ + coreutils \ + debhelper \ + devscripts \ + dh-python \ + dh-systemd \ + gdb \ + git \ + gnupg2 \ + haveged \ + inotify-tools \ + libffi-dev \ + libssl-dev \ + make \ + ntp \ + paxctl \ + python3-all \ + python3-pip \ + python3-setuptools \ + rsync \ + ruby \ + sqlite \ + sudo \ + tzdata \ + libevent-dev \ + unzip + + +# TEMPORARY: install dh-virtualenv from debian unstable, pending focal package: +# https://github.com/spotify/dh-virtualenv/issues/298 +RUN echo "deb https://deb.debian.org/debian unstable main contrib" > /etc/apt/sources.list.d/debian-unstable.list +COPY aptpreferences.conf /etc/apt/preferences.d/debian-unstable +RUN apt-get install -y debian-archive-keyring +RUN ln -s /usr/share/keyrings/debian-archive-keyring.gpg /etc/apt/trusted.gpg.d/ + +RUN apt-get update && apt-get install -y dh-virtualenv + +RUN paxctl -cm /usr/bin/python3.8 && mkdir -p /tmp/build +RUN apt-get clean \ + && rm -rf /var/lib/apt/lists/* diff --git a/molecule/builder-focal/Makefile b/molecule/builder-focal/Makefile new file mode 100644 index 0000000000..23d07d213a --- /dev/null +++ b/molecule/builder-focal/Makefile @@ -0,0 +1,20 @@ +DATE_STR := $(shell date +"%Y_%m_%d") +BUILDER_IMAGE ?= "quay.io/freedomofpress/sd-docker-builder-focal:$(DATE_STR)" + +.PHONY: build-container +build-container: ## Build Docker image for Debian package creation + @echo "███Building Docker image $(BUILDER_IMAGE) for Debian package creation..." + @docker build --no-cache -t $(BUILDER_IMAGE) . + +.PHONY: push-container +push-container: ## Push the Docker image for Debian package creation to quay.io + @echo "███Pushing Docker image for Debian package creation to quay.io..." + @./push.sh + +.PHONY: help +help: ## Print this message and exit. + @printf "Molecule scenario for building a Docker container for Debian package creation.\n" + @printf "Subcommands:\n\n" + @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \ + | sort \ + | column -s ':' -t diff --git a/molecule/builder-focal/ansible-override-vars.yml b/molecule/builder-focal/ansible-override-vars.yml new file mode 100644 index 0000000000..38ee4f7669 --- /dev/null +++ b/molecule/builder-focal/ansible-override-vars.yml @@ -0,0 +1,3 @@ +--- +securedrop_build_focal_support: True +securedrop_staging_install_target_distro: "focal" diff --git a/molecule/builder-focal/aptpreferences.conf b/molecule/builder-focal/aptpreferences.conf new file mode 100644 index 0000000000..58a36d3f3b --- /dev/null +++ b/molecule/builder-focal/aptpreferences.conf @@ -0,0 +1,7 @@ +Package: * +Pin: release a=focal +Pin-Priority: 700 + +Package: * +Pin: release a=unstable +Pin-Priority: 1 diff --git a/molecule/builder-focal/create.yml b/molecule/builder-focal/create.yml new file mode 100644 index 0000000000..b26a45c58f --- /dev/null +++ b/molecule/builder-focal/create.yml @@ -0,0 +1,28 @@ +--- +- name: Create + hosts: localhost + connection: local + gather_facts: False + vars: + molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}" + molecule_ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + molecule_scenario_directory: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}" + molecule_yml: "{{ lookup('file', molecule_file) | from_yaml }}" + image_hash: "{{ lookup('pipe', 'egrep -v ^# image_hash') }}" + default_image: "quay.io/freedomofpress/sd-docker-builder-focal@sha256:{{image_hash}}" + image: "{{ lookup('env', 'BUILDER_IMAGE') | default(default_image, true) }}" + tasks: + - debug: + msg: "Building with Docker image {{ image }}" + + - name: Create builders + docker_container: + name: "{{ item.name }}" + hostname: "{{ item.name }}" + image: "{{ image }}" + state: started + command: "tail -f /dev/null" + privileged: "{{ item.privileged | default(omit) }}" + volumes: "{{ item.volumes | default(omit) }}" + capabilities: "{{ item.capabilities | default(omit) }}" + with_items: "{{ molecule_yml.platforms }}" diff --git a/molecule/builder-focal/destroy.yml b/molecule/builder-focal/destroy.yml new file mode 100644 index 0000000000..f804be7578 --- /dev/null +++ b/molecule/builder-focal/destroy.yml @@ -0,0 +1,15 @@ +--- +- name: Destroy + hosts: localhost + connection: local + gather_facts: False + vars: + molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}" + molecule_yml: "{{ lookup('file', molecule_file) | from_yaml }}" + tasks: + - name: Destroy molecule instance(s) + docker_container: + name: "{{ item.name }}" + state: absent + force_kill: "{{ item.force_kill | default(True) }}" + with_items: "{{ molecule_yml.platforms }}" diff --git a/molecule/builder-focal/image_hash b/molecule/builder-focal/image_hash new file mode 100644 index 0000000000..bd105f0d50 --- /dev/null +++ b/molecule/builder-focal/image_hash @@ -0,0 +1,2 @@ +# sha256 digest quay.io/freedomofpress/sd-docker-builder-focal:2020_08_13 +47f7e1af8cc54bf9cb0afdef599b76d063c614943e54443ea660d6b53615814c diff --git a/molecule/builder-focal/molecule.yml b/molecule/builder-focal/molecule.yml new file mode 100644 index 0000000000..fb3b221a17 --- /dev/null +++ b/molecule/builder-focal/molecule.yml @@ -0,0 +1,79 @@ +--- +driver: + name: docker +lint: + name: yamllint +platforms: + - name: focal-sd-app + groups: + - builders + - name: focal-sd-generic-ossec-agent + groups: + - builders + - name: focal-sd-generic-ossec-server + groups: + - builders + - name: focal-sd-generic-ossec-agent2 + groups: + - builders + - name: focal-sd-generic-ossec-server2 + groups: + - builders + - name: focal-sd-grsec + groups: + - builders + - name: focal-sd-config + groups: + - builders + - name: focal-sd-keyring + groups: + - builders + - name: focal-sd-sec-update + groups: + - builders + - name: focal-sd-dpkg-verification + groups: + - testers +provisioner: + name: ansible + inventory: + links: + group_vars: ../../install_files/ansible-base/group_vars + options: + e: "@ansible-override-vars.yml" + config_options: + defaults: + interpreter_python: auto + env: + ANSIBLE_ROLES_PATH: ../../install_files/ansible-base/roles + ANSIBLE_ACTION_PLUGINS: ../../install_files/ansible-base/action_plugins + ANSIBLE_CALLBACK_WHITELIST: skippy + ANSIBLE_STDOUT_CALLBACK: skippy + ANSIBLE_GATHER_TIMEOUT: "120" + lint: + name: ansible-lint + playbooks: + converge: playbook.yml +scenario: + name: builder-focal + converge_sequence: + - destroy + - create + - converge + - destroy + test_sequence: + - destroy + - create + - converge + - verify + - destroy +verifier: + name: testinfra + options: + # provided by pytest-xdist + n: auto + env: + SECUREDROP_TARGET_PLATFORM: focal + directory: tests/ + lint: + name: flake8 diff --git a/molecule/builder-focal/playbook.yml b/molecule/builder-focal/playbook.yml new file mode 100644 index 0000000000..3564675bfd --- /dev/null +++ b/molecule/builder-focal/playbook.yml @@ -0,0 +1,80 @@ +--- +- name: Build SecureDrop application Debian package from local repository. + hosts: builders + # Build as fast as possible with each host going individually + strategy: free + become: yes + tasks: + - name: Update apt-cache for our security checker + apt: + update_cache: yes + when: ansible_host.endswith("-sd-sec-update") + roles: + - role: build-securedrop-app-code-deb-pkg + tags: app-deb + when: ansible_host.endswith("-sd-app") + + - role: build-ossec-deb-pkg + tags: ossec-server + purpose: server + when: ansible_host.endswith("-sd-generic-ossec-server") + + - role: build-ossec-deb-pkg + tags: ossec-agent + purpose: agent + when: ansible_host.endswith("-sd-generic-ossec-agent") + + - role: build-generic-pkg + tags: securedrop-ossec-server + package_name: securedrop-ossec-server + when: ansible_host.endswith("-sd-generic-ossec-server2") or ansible_host == "localhost" + + - role: build-generic-pkg + tags: securedrop-ossec-agent + package_name: securedrop-ossec-agent + when: ansible_host.endswith("-sd-generic-ossec-agent2") or ansible_host == "localhost" + + - role: build-generic-pkg + tags: securedrop-keyring + package_name: securedrop-keyring + when: ansible_host.endswith("-sd-keyring") or ansible_host == "localhost" + + - role: build-generic-pkg + tags: securedrop-grsec + package_name: securedrop-grsec + when: ansible_host.endswith("-sd-grsec") or ansible_host == "localhost" + + - role: build-generic-pkg + tags: securedrop-config + package_name: securedrop-config + when: ansible_host.endswith("-sd-config") or ansible_host == "localhost" + tags: rebuild + +# Typically we'd perform volume mounting here but to work around docker +# remote calls (in CircleCI) we have to copy the files instead +- name: Give dpkg verify container access to debs + hosts: testers + gather_facts: false + vars: + sd_build_root: "{{ playbook_dir + '/../../build' }}" + sd_target_distro: focal + sd_build_dest: "{{ sd_build_root + '/' + sd_target_distro }}" + + tasks: + - name: Discover local debian build files + find: + paths: "{{ sd_build_dest }}" + patterns: '*.deb' + delegate_to: localhost + register: debian_files + + - name: Create build dir + file: + state: directory + path: /tmp/build + + - name: Drop debian files into container + copy: + src: "{{ item.path }}" + dest: /tmp/build/ + with_items: "{{ debian_files.files }}" diff --git a/molecule/builder-focal/push.sh b/molecule/builder-focal/push.sh new file mode 100755 index 0000000000..e5a7ed42d1 --- /dev/null +++ b/molecule/builder-focal/push.sh @@ -0,0 +1,12 @@ +#!/bin/bash +DATE_STR=$(date +"%Y_%m_%d") +QUAY_REPO=quay.io/freedomofpress/sd-docker-builder-focal + +set -e +set -x + +docker push "${QUAY_REPO}:${DATE_STR}" + +echo "# sha256 digest ${QUAY_REPO}:${DATE_STR}" > image_hash +docker inspect --format='{{index .RepoDigests 0}}' "${QUAY_REPO}:${DATE_STR}" \ + | sed 's/.*://g' >> image_hash diff --git a/molecule/builder-focal/tests/conftest.py b/molecule/builder-focal/tests/conftest.py new file mode 100644 index 0000000000..da01aa9600 --- /dev/null +++ b/molecule/builder-focal/tests/conftest.py @@ -0,0 +1,22 @@ +""" +Import variables from vars.yml and inject into pytest namespace +""" + +import os +import io +import yaml + + +def pytest_namespace(): + """ Return dict of vars imported as 'securedrop_test_vars' into pytest + global namespace + """ + filepath = os.path.join(os.path.dirname(__file__), "vars.yml") + with io.open(filepath, 'r') as f: + securedrop_test_vars = yaml.safe_load(f) + + # Tack on target OS for use in tests + securedrop_target_platform = os.environ.get("SECUREDROP_TARGET_PLATFORM") + securedrop_test_vars["securedrop_target_platform"] = securedrop_target_platform + # Wrapping the return value to accommodate for pytest namespacing + return dict(securedrop_test_vars=securedrop_test_vars) diff --git a/molecule/builder-focal/tests/test_build_dependencies.py b/molecule/builder-focal/tests/test_build_dependencies.py new file mode 100644 index 0000000000..6cfce29ba3 --- /dev/null +++ b/molecule/builder-focal/tests/test_build_dependencies.py @@ -0,0 +1,40 @@ +import pytest +import os + + +SECUREDROP_TARGET_PLATFORM = os.environ.get("SECUREDROP_TARGET_PLATFORM") +testinfra_hosts = [ + "docker://{}-sd-app".format(SECUREDROP_TARGET_PLATFORM) +] + + +def test_sass_gem_installed(host): + """ + Ensure the `sass` Ruby gem is installed, for compiling SASS to CSS. + """ + c = host.run("gem list") + assert "sass (3.4.23)" in c.stdout + assert c.rc == 0 + + +def test_pip_dependencies_installed(host): + """ + Ensure the development pip dependencies are installed + """ + c = host.run("pip3 list installed") + assert "Flask-Babel" in c.stdout + assert c.rc == 0 + + +@pytest.mark.xfail(reason="This check conflicts with the concept of pegging" + "dependencies") +def test_build_all_packages_updated(host): + """ + Ensure a dist-upgrade has already been run, by checking that no + packages are eligible for upgrade currently. This will ensure that + all upgrades, security and otherwise, have been applied to the VM + used to build packages. + """ + c = host.run('aptitude --simulate -y dist-upgrade') + assert c.rc == 0 + assert "No packages will be installed, upgraded, or removed." in c.stdout diff --git a/molecule/builder-focal/tests/test_legacy_paths.py b/molecule/builder-focal/tests/test_legacy_paths.py new file mode 100644 index 0000000000..8b02df9171 --- /dev/null +++ b/molecule/builder-focal/tests/test_legacy_paths.py @@ -0,0 +1,20 @@ +import pytest + + +@pytest.mark.parametrize('build_path', [ + '/tmp/build-', + '/tmp/rsync-filter', + '/tmp/src_install_files', + '/tmp/build-securedrop-keyring', + '/tmp/build-securedrop-ossec-agent', + '/tmp/build-securedrop-ossec-server', +]) +def test_build_ossec_apt_dependencies(host, build_path): + """ + Ensure that unwanted build paths are absent. Most of these were created + as unwanted side-effects during CI-related changes to the build scripts. + + All paths are rightly considered "legacy" and should never be present on + the build host. This test is strictly for guarding against regressions. + """ + assert not host.file(build_path).exists diff --git a/molecule/builder-focal/tests/test_securedrop_deb_package.py b/molecule/builder-focal/tests/test_securedrop_deb_package.py new file mode 100644 index 0000000000..0f88c24b78 --- /dev/null +++ b/molecule/builder-focal/tests/test_securedrop_deb_package.py @@ -0,0 +1,484 @@ +import pytest +import os +import re +import tempfile + + +SECUREDROP_TARGET_PLATFORM = os.environ.get("SECUREDROP_TARGET_PLATFORM") +testinfra_hosts = [ + "docker://{}-sd-dpkg-verification".format(SECUREDROP_TARGET_PLATFORM) +] +securedrop_test_vars = pytest.securedrop_test_vars + + +def extract_package_name_from_filepath(filepath): + """ + Helper function to infer intended package name from + the absolute filepath, using a rather garish regex. + E.g., given: + securedrop-ossec-agent-2.8.2+0.3.10-amd64.deb + + retuns: + + securedrop-ossec-agent + + which can then be used for comparisons in dpkg output. + """ + deb_basename = os.path.basename(filepath) + package_name = re.search(r'^([a-z\-]+(?!\d))', deb_basename).groups()[0] + assert deb_basename.startswith(package_name) + return package_name + + +def get_deb_packages(): + """ + Helper function to retrieve module-namespace test vars and format + the strings to interpolate version info. Keeps the test vars DRY + in terms of version info, and required since we can't rely on + Jinja-based evaluation of the YAML files (so we can't trivially + reuse vars in other var values, as is the case with Ansible). + """ + substitutions = dict( + securedrop_version=securedrop_test_vars.securedrop_version, + ossec_version=securedrop_test_vars.ossec_version, + keyring_version=securedrop_test_vars.keyring_version, + config_version=securedrop_test_vars.config_version, + grsec_version=securedrop_test_vars.grsec_version, + securedrop_target_platform=securedrop_test_vars.securedrop_target_platform, + ) + + deb_packages = [d.format(**substitutions) for d + in securedrop_test_vars.build_deb_packages] + return deb_packages + + +deb_packages = get_deb_packages() + + +def get_deb_tags(): + """ + Helper function to build array of package and tag tuples + for lintian. + """ + deb_tags = [] + + for deb in get_deb_packages(): + for tag in securedrop_test_vars.lintian_tags: + deb_tags.append((deb, tag)) + + return deb_tags + + +deb_tags = get_deb_tags() + + +@pytest.mark.parametrize("deb", deb_packages) +def test_build_deb_packages(host, deb): + """ + Sanity check the built Debian packages for Control field + values and general package structure. + """ + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + assert deb_package.is_file + + +@pytest.mark.parametrize("deb", deb_packages) +def test_deb_packages_appear_installable(host, deb): + """ + Confirms that a dry-run of installation reports no errors. + Simple check for valid Debian package structure, but not thorough. + When run on a malformed package, `dpkg` will report: + + dpkg-deb: error: `foo.deb' is not a debian format archive + + Testing application behavior is left to the functional tests. + """ + + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + + deb_basename = os.path.basename(deb_package.path) + package_name = extract_package_name_from_filepath(deb_package.path) + assert deb_basename.startswith(package_name) + + # sudo is required to call `dpkg --install`, even as dry-run. + with host.sudo(): + c = host.run("dpkg --install --dry-run {}".format(deb_package.path)) + assert "Selecting previously unselected package {}".format( + package_name) in c.stdout + regex = "Preparing to unpack [./]+{} ...".format( + re.escape(deb_basename)) + assert re.search(regex, c.stdout, re.M) + assert c.rc == 0 + + +@pytest.mark.parametrize("deb", deb_packages) +def test_deb_package_control_fields(host, deb): + """ + Ensure Debian Control fields are populated as expected in the package. + These checks are rather superficial, and don't actually confirm that the + .deb files are not broken. At a later date, consider integration tests + that actually use these built files during an Ansible provisioning run. + """ + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + package_name = extract_package_name_from_filepath(deb_package.path) + # The `--field` option will display all fields if none are specified. + c = host.run("dpkg-deb --field {}".format(deb_package.path)) + + assert "Maintainer: SecureDrop Team " in c.stdout + # The securedrop-config package is architecture indepedent + if package_name == "securedrop-config": + assert "Architecture: all" in c.stdout + else: + assert "Architecture: amd64" in c.stdout + + assert "Package: {}".format(package_name) in c.stdout + assert c.rc == 0 + + +@pytest.mark.parametrize("deb", deb_packages) +def test_deb_package_control_fields_homepage(host, deb): + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + # The `--field` option will display all fields if none are specified. + c = host.run("dpkg-deb --field {}".format(deb_package.path)) + # The OSSEC source packages will have a different homepage; + # all other packages should set securedrop.org as homepage. + if os.path.basename(deb_package.path).startswith('ossec-'): + assert "Homepage: http://ossec.net" in c.stdout + else: + assert "Homepage: https://securedrop.org" in c.stdout + + +@pytest.mark.parametrize("deb", deb_packages) +def test_deb_package_contains_no_config_file(host, deb): + """ + Ensures the `securedrop-app-code` package does not ship a `config.py` + file. Doing so would clobber the site-specific changes made via Ansible. + + Somewhat lazily checking all deb packages, rather than just the app-code + package, but it accomplishes the same in a DRY manner. + """ + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + c = host.run("dpkg-deb --contents {}".format(deb_package.path)) + assert not re.search(r"^ ./var/www/securedrop/config.py$", c.stdout, re.M) + + +@pytest.mark.parametrize("deb", deb_packages) +def test_deb_package_contains_pot_file(host, deb): + """ + Ensures the `securedrop-app-code` package has the + messages.pot file + """ + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + c = host.run("dpkg-deb --contents {}".format(deb_package.path)) + # Only relevant for the securedrop-app-code package: + if "securedrop-app-code" in deb_package.path: + assert re.search("^.*messages.pot$", c.stdout, re.M) + + +@pytest.mark.parametrize("deb", deb_packages) +def test_deb_package_contains_mo_file(host, deb): + """ + Ensures the `securedrop-app-code` package has at least one + compiled mo file. + """ + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + c = host.run("dpkg-deb --contents {}".format(deb_package.path)) + # Only relevant for the securedrop-app-code package: + if "securedrop-app-code" in deb_package.path: + assert re.search(r"^.*messages\.mo$", c.stdout, re.M) + + +@pytest.mark.parametrize("deb", deb_packages) +def test_deb_package_contains_no_generated_assets(host, deb): + """ + Ensures the `securedrop-app-code` package does not ship minified + static assets, which are built automatically via Flask-Assets, and + may be present in the source directory used to build from. + """ + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + + # Only relevant for the securedrop-app-code package: + if "securedrop-app-code" in deb_package.path: + c = host.run("dpkg-deb --contents {}".format(deb_package.path)) + # static/gen/ directory should exist + assert re.search(r"^.*\./var/www/securedrop" + "/static/gen/$", c.stdout, re.M) + # static/gen/ directory should be empty + assert not re.search(r"^.*\./var/www/securedrop" + "/static/gen/.+$", c.stdout, re.M) + + # static/.webassets-cache/ directory should exist + assert re.search(r"^.*\./var/www/securedrop" + "/static/.webassets-cache/$", c.stdout, re.M) + # static/.webassets-cache/ directory should be empty + assert not re.search(r"^.*\./var/www/securedrop" + "/static/.webassets-cache/.+$", c.stdout, re.M) + + # no SASS files should exist; only the generated CSS files. + assert not re.search("^.*sass$", c.stdout, re.M) + + # no .map files should exist; only the generated CSS files. + assert not re.search("^.*css.map$", c.stdout, re.M) + + +@pytest.mark.parametrize("deb", deb_packages) +def test_deb_package_contains_expected_conffiles(host, deb): + """ + Ensures the `securedrop-app-code` package declares only whitelisted + `conffiles`. Several files in `/etc/` would automatically be marked + conffiles, which would break unattended updates to critical package + functionality such as AppArmor profiles. This test validates overrides + in the build logic to unset those conffiles. + """ + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + + # For the securedrop-app-code package: + if "securedrop-app-code" in deb_package.path: + tmpdir = tempfile.mkdtemp() + # The `--raw-extract` flag includes `DEBIAN/` dir with control files + host.run("dpkg-deb --raw-extract {} {}".format(deb, tmpdir)) + conffiles_path = os.path.join(tmpdir, "DEBIAN", "conffiles") + f = host.file(conffiles_path) + + assert f.is_file + # Ensure that the entirety of the file lists only the logo as conffile; + # effectively ensures e.g. AppArmor profiles are not conffiles. + conffiles = f.content_string.rstrip() + assert conffiles == "/var/www/securedrop/static/i/logo.png" + + # For the securedrop-config package, we want to ensure there are no + # conffiles so securedrop_additions.sh is squashed every time + if "securedrop-config" in deb_package.path: + c = host.run("dpkg-deb -I {}".format(deb)) + assert "conffiles" not in c.stdout + + +@pytest.mark.parametrize("deb", deb_packages) +def test_deb_package_contains_css(host, deb): + """ + Ensures the `securedrop-app-code` package contains files that + are generated during the `sass` build process. + """ + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + + # Only relevant for the securedrop-app-code package: + if "securedrop-app-code" in deb_package.path: + c = host.run("dpkg-deb --contents {}".format(deb_package.path)) + + for css_type in ['journalist', 'source']: + assert re.search(r"^.*\./var/www/securedrop/static/" + "css/{}.css$".format(css_type), c.stdout, re.M) + + +@pytest.mark.parametrize("deb, tag", deb_tags) +def test_deb_package_lintian(host, deb, tag): + """ + Ensures lintian likes our Debian packages. + """ + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + c = host.run("lintian --tags {} --no-tag-display-limit {}".format( + tag, deb_package.path)) + assert len(c.stdout) == 0 + + +@pytest.mark.parametrize("deb", deb_packages) +def test_deb_app_package_contains_https_validate_dir(host, deb): + """ + Ensures the `securedrop-app-code` package ships with a validation + '.well-known/pki-validation' directory + """ + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + + # Only relevant for the securedrop-app-code package: + if "securedrop-app-code" in deb_package.path: + c = host.run("dpkg-deb --contents {}".format(deb_package.path)) + # well-known/pki-validation directory should exist + assert re.search(r"^.*\./var/www/securedrop/" + ".well-known/pki-validation/$", c.stdout, re.M) + + +@pytest.mark.parametrize("deb", deb_packages) +def test_grsec_metapackage(host, deb): + """ + Sanity checks on the securedrop-grsec metapackage. Mostly checks + for presence of PaX flags hook and sysctl settings. + Does not validate file contents, just presence. + """ + + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + + if "securedrop-grsec" in deb_package.path: + c = host.run("dpkg-deb --contents {}".format(deb_package.path)) + # Custom sysctl options should be present + assert re.search(r"^.*\./etc/sysctl.d/30-securedrop.conf$", + c.stdout, re.M) + c = host.run("dpkg-deb --contents {}".format(deb_package.path)) + # Post-install kernel hook for managing PaX flags must exist. + assert re.search(r"^.*\./etc/kernel/postinst.d/paxctl-grub$", + c.stdout, re.M) + + +@pytest.mark.parametrize("deb", deb_packages) +def test_control_helper_files_are_present(host, deb): + """ + Inspect the package info to get a list of helper scripts + that should be shipped with the package, e.g. postinst, prerm, etc. + Necessary due to package build logic retooling. + + Example output from package info, for reference: + + $ dpkg-deb --info securedrop-app-code_0.12.0~rc1_amd64.deb + new debian package, version 2.0. + size 13583186 bytes: control archive=11713 bytes. + 62 bytes, 2 lines conffiles + 657 bytes, 10 lines control + 26076 bytes, 298 lines md5sums + 5503 bytes, 159 lines * postinst #!/bin/bash + + Note that the actual output will have trailing whitespace, removed + from this text description to satisfy linters. + """ + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + # Only relevant for the securedrop-app-code package: + if "securedrop-app-code" in deb_package.path: + wanted_files = [ + "conffiles", + "config", + "control", + "postinst", + "postrm", + "preinst", + "prerm", + "templates", + ] + c = host.run("dpkg-deb --info {}".format(deb_package.path)) + for wanted_file in wanted_files: + assert re.search(r"^\s+?\d+ bytes,\s+\d+ lines[\s*]+"+wanted_file+r"\s+.*$", + c.stdout, re.M) + + +@pytest.mark.parametrize("deb", deb_packages) +def test_jinja_files_not_present(host, deb): + """ + Make sure that jinja (.j2) files were not copied over + as-is into the debian packages. + """ + + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + + c = host.run("dpkg-deb --contents {}".format(deb_package.path)) + # There shouldn't be any files with a .j2 ending + assert not re.search(r"^.*\.j2$", c.stdout, re.M) + + +@pytest.mark.parametrize("deb", deb_packages) +def test_ossec_binaries_are_present_agent(host, deb): + """ + Inspect the package contents to ensure all ossec agent binaries are properly + included in the package. + """ + deb_package = host.file(deb.format( + securedrop_test_vars.ossec_version)) + # Only relevant for the ossec-agent package and not securedrop-ossec-agent: + if "ossec-agent" in deb_package.path and "securedrop" not in deb_package.path: + wanted_files = [ + "/var/ossec/bin/agent-auth", + "/var/ossec/bin/ossec-syscheckd", + "/var/ossec/bin/ossec-agentd", + "/var/ossec/bin/manage_agents", + "/var/ossec/bin/ossec-control", + "/var/ossec/bin/ossec-logcollector", + "/var/ossec/bin/util.sh", + "/var/ossec/bin/ossec-execd", + ] + c = host.run("dpkg-deb -c {}".format(deb_package.path)) + for wanted_file in wanted_files: + assert wanted_file in c.stdout + + +@pytest.mark.parametrize("deb", deb_packages) +def test_ossec_binaries_are_present_server(host, deb): + """ + Inspect the package contents to ensure all ossec server binaries are properly + included in the package. + """ + deb_package = host.file(deb.format( + securedrop_test_vars.ossec_version)) + # Only relevant for the ossec-agent package and not securedrop-ossec-agent: + if "ossec-server" in deb_package.path and "securedrop" not in deb_package.path: + wanted_files = [ + "/var/ossec/bin/ossec-maild", + "/var/ossec/bin/ossec-remoted", + "/var/ossec/bin/ossec-syscheckd", + "/var/ossec/bin/ossec-makelists", + "/var/ossec/bin/ossec-logtest", + "/var/ossec/bin/syscheck_update", + "/var/ossec/bin/ossec-reportd", + "/var/ossec/bin/ossec-agentlessd", + "/var/ossec/bin/manage_agents", + "/var/ossec/bin/rootcheck_control", + "/var/ossec/bin/ossec-control", + "/var/ossec/bin/ossec-dbd", + "/var/ossec/bin/ossec-csyslogd", + "/var/ossec/bin/ossec-regex", + "/var/ossec/bin/agent_control", + "/var/ossec/bin/ossec-monitord", + "/var/ossec/bin/clear_stats", + "/var/ossec/bin/ossec-logcollector", + "/var/ossec/bin/list_agents", + "/var/ossec/bin/verify-agent-conf", + "/var/ossec/bin/syscheck_control", + "/var/ossec/bin/util.sh", + "/var/ossec/bin/ossec-analysisd", + "/var/ossec/bin/ossec-execd", + "/var/ossec/bin/ossec-authd", + ] + c = host.run("dpkg-deb --contents {}".format(deb_package.path)) + for wanted_file in wanted_files: + assert wanted_file in c.stdout + + +@pytest.mark.parametrize("deb", deb_packages) +def test_config_package_contains_expected_files(host, deb): + """ + Inspect the package contents to ensure all config files are included in + the package. + """ + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + if "securedrop-config" in deb_package.path: + wanted_files = [ + "/etc/cron-apt/action.d/9-remove", + "/etc/profile.d/securedrop_additions.sh", + ] + c = host.run("dpkg-deb --contents {}".format(deb_package.path)) + for wanted_file in wanted_files: + assert wanted_file in c.stdout + + +@pytest.mark.parametrize("deb", deb_packages) +def test_app_package_does_not_contain_custom_logo(host, deb): + """ + Inspect the package contents to ensure custom_logo.png is not present. This + is because custom_logo.png superceeds logo.png. + """ + deb_package = host.file(deb.format( + securedrop_test_vars.securedrop_version)) + if "securedrop-app-code" in deb_package.path: + c = host.run("dpkg-deb --contents {}".format(deb_package.path)) + assert "/var/www/static/i/custom_logo.png" not in c.stdout diff --git a/molecule/builder-focal/tests/test_security_updates.py b/molecule/builder-focal/tests/test_security_updates.py new file mode 100644 index 0000000000..654ac6b717 --- /dev/null +++ b/molecule/builder-focal/tests/test_security_updates.py @@ -0,0 +1,43 @@ +import os +from subprocess import check_output +import re +import pytest + +SECUREDROP_TARGET_PLATFORM = os.environ.get("SECUREDROP_TARGET_PLATFORM") +testinfra_hosts = [ + "docker://{}-sd-sec-update".format(SECUREDROP_TARGET_PLATFORM) +] + + +def test_should_run(): + command = ["git", "describe", "--all"] + version = check_output(command).decode("utf8")[0:-1] + candidates = (r"(^tags/[\d]+\.[\d]+\.[\d]+-rc[\d]+)|" + r"(^tags/[\d]+\.[\d]+\.[\d]+)|" + r"(^heads/release/[\d]+\.[\d]+\.[\d]+)|" + r"(^heads/update-builder.*)") + result = re.match(candidates, version) + if result: + return True + else: + return False + + +@pytest.mark.skipif(not test_should_run(), reason="Only tested for RCs and builder updates") +def test_ensure_no_updates_avail(host): + """ + Test to make sure that there are no security-updates in the + base builder container. + """ + # Filter out all the security repos to their own file + # without this change all the package updates appeared as if they were + # coming from normal ubuntu update channel (since they get posted to both) + host.run('egrep "^deb.*security" /etc/apt/sources.list > /tmp/sec.list') + + dist_upgrade_simulate = host.run('apt-get -s dist-upgrade ' + '-oDir::Etc::Sourcelist=/tmp/sec.list ' + '|grep "^Inst" |grep -i security') + + # If the grep was successful that means security package updates found + # otherwise we get a non-zero exit code so no updates needed. + assert dist_upgrade_simulate.rc != 0 diff --git a/molecule/builder-focal/tests/vars.yml b/molecule/builder-focal/tests/vars.yml new file mode 100644 index 0000000000..35f8932cba --- /dev/null +++ b/molecule/builder-focal/tests/vars.yml @@ -0,0 +1,36 @@ +--- +securedrop_version: "1.5.0~rc1" +ossec_version: "3.6.0" +keyring_version: "0.1.4" +config_version: "0.1.3" +grsec_version: "4.14.175" + +# These values will be interpolated with values populated above +# via helper functions in the tests. +build_directories: + # The build scripts for securedrop-app-code run separate from the others, + # i.e. lacking the `/tmp/build` pardir. + - /tmp/securedrop-app-code-{securedrop_version}_amd64/ + - /tmp/build/securedrop-keyring-{keyring_version}+{securedrop_version}-amd64/ + - /tmp/build/securedrop-config-{config_version}+{securedrop_version}-amd64/ + - /tmp/build/securedrop-ossec-agent-{ossec_version}+{securedrop_version}-amd64/ + - /tmp/build/securedrop-ossec-server-{ossec_version}+{securedrop_version}-amd64/ + - /tmp/build/ossec-agent-{ossec_version}-amd64/ + - /tmp/build/ossec-server-{ossec_version}-amd64/ + - /tmp/build + +build_deb_packages: + - /tmp/build/securedrop-app-code_{securedrop_version}+{securedrop_target_platform}_amd64.deb + - /tmp/build/securedrop-ossec-agent-{ossec_version}+{securedrop_version}-amd64.deb + - /tmp/build/securedrop-ossec-server-{ossec_version}+{securedrop_version}-amd64.deb + - /tmp/build/ossec-server-{ossec_version}-amd64.deb + - /tmp/build/ossec-agent-{ossec_version}-amd64.deb + - /tmp/build/securedrop-keyring-{keyring_version}+{securedrop_version}-amd64.deb + - /tmp/build/securedrop-config-{config_version}+{securedrop_version}-amd64.deb + - /tmp/build/securedrop-grsec-{grsec_version}-amd64.deb + +lintian_tags: + # - non-standard-file-perm + - package-contains-vcs-control-file + - package-installs-python-bytecode + # - wrong-file-owner-uid-or-gid diff --git a/molecule/builder-xenial/ansible-override-vars.yml b/molecule/builder-xenial/ansible-override-vars.yml deleted file mode 100644 index 8b317960eb..0000000000 --- a/molecule/builder-xenial/ansible-override-vars.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -securedrop_build_xenial_support: True diff --git a/molecule/builder-xenial/molecule.yml b/molecule/builder-xenial/molecule.yml index d32f66a349..4ed6a362b5 100644 --- a/molecule/builder-xenial/molecule.yml +++ b/molecule/builder-xenial/molecule.yml @@ -42,8 +42,6 @@ provisioner: config_options: defaults: interpreter_python: auto - options: - e: "@ansible-override-vars.yml" env: ANSIBLE_ROLES_PATH: ../../install_files/ansible-base/roles ANSIBLE_ACTION_PLUGINS: ../../install_files/ansible-base/action_plugins