From 4c023d5215fb8451a8ada82d11ebe565a7ac4c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 25 Feb 2018 19:20:24 +0100 Subject: [PATCH 001/136] Enable CI backed by GitLab CI and CircleCI --- .ci/README.md | 9 ++ .ci/build-docker.sh | 46 ++++++++ .ci/protect-secrets.sh | 40 +++++++ .ci/pull-gitlab.sh | 33 ++++++ .ci/push-dockerhub.sh | 30 +++++ .ci/push-gitlab.sh | 27 +++++ .ci/setup-make-parallelity.sh | 26 ++++ .ci/test-cli.sh | 21 ++++ .ci/test-dev.sh | 35 ++++++ .ci/test-jupyter.sh | 23 ++++ .circleci/config.yml | 41 +++++++ .dockerignore | 1 + .gitignore | 6 + .gitlab-ci.yml | 99 ++++++++++++++++ Makefile | 26 ++++ configure.ac | 6 +- docker/.gitignore | 2 + docker/Dockerfile | 176 ++++++++++++++++++++++++++++ docker/README.md | 51 ++++++++ docker/entrypoint-dev.sh | 4 + docker/entrypoint.sh | 2 + docker/hooks/build | 1 + docker/hooks/push | 1 + src/sage_setup/docbuild/__init__.py | 7 +- 24 files changed, 709 insertions(+), 4 deletions(-) create mode 100644 .ci/README.md create mode 100755 .ci/build-docker.sh create mode 100755 .ci/protect-secrets.sh create mode 100755 .ci/pull-gitlab.sh create mode 100755 .ci/push-dockerhub.sh create mode 100755 .ci/push-gitlab.sh create mode 100644 .ci/setup-make-parallelity.sh create mode 100755 .ci/test-cli.sh create mode 100755 .ci/test-dev.sh create mode 100755 .ci/test-jupyter.sh create mode 100644 .circleci/config.yml create mode 120000 .dockerignore create mode 100644 .gitlab-ci.yml create mode 100644 docker/.gitignore create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100755 docker/entrypoint-dev.sh create mode 100755 docker/entrypoint.sh create mode 100644 docker/hooks/build create mode 100644 docker/hooks/push diff --git a/.ci/README.md b/.ci/README.md new file mode 100644 index 00000000000..e2b165cc518 --- /dev/null +++ b/.ci/README.md @@ -0,0 +1,9 @@ +# Continuous Integration (CI) + +We support several implementations of CI. All these implementations rely on +[docker](https://docker.com) in some way. This directory contains bits which +are shared between these CI implementations. The relevant docker files can be +found in `/docker/`. + +* [CircleCI](https://circleci.com) is configured in `/.circleci/`. +* [GitLab CI](https://gitlab.com) is configured in `/.gitlab-ci.yml`. diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh new file mode 100755 index 00000000000..11651e5dd38 --- /dev/null +++ b/.ci/build-docker.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# This script gets called from CI to build several flavours of docker images +# which contain Sage. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** +set -ex + +[[ -z "$DOCKER_TAG" ]] && DOCKER_TAG=none +[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest + +. .ci/setup-make-parallelity.sh + +# We speed up the build process by copying built artifacts from ARTIFACT_BASE +# during docker build. See /docker/Dockerfile for more details. +ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:latest} + +# Seed our cache with $ARTIFACT_BASE if it exists +docker pull $ARTIFACT_BASE || true + +function docker_build { + time docker build -f docker/Dockerfile --build-arg "MAKE=${MAKE}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ +} + +# We use a multi-stage build /docker/Dockerfile. For the caching to be +# effective, we populate the cache by building the make-all target. (Just +# building the last target is not enough as intermediate targets would be +# discarded from the cache and therefore the caching would fail for our actual +# builds below.) +docker_build --pull --tag make-all --target make-all . + +# Build the release image without build artifacts. +DOCKER_IMAGE_CLI=${DOCKER_USER:-sagemath}/sagemath:$DOCKER_TAG +docker_build --target sagemath --tag "$DOCKER_IMAGE_CLI" . +# Build the developer image with the build artifacts intact. +# Note: It's important to build the dev image last because it might be tagged as ARTIFACT_BASE. +DOCKER_IMAGE_DEV=${DOCKER_USER:-sagemath}/sagemath-dev:$DOCKER_TAG +docker_build --target sagemath-dev --tag "$DOCKER_IMAGE_DEV" . diff --git a/.ci/protect-secrets.sh b/.ci/protect-secrets.sh new file mode 100755 index 00000000000..a3ae9239988 --- /dev/null +++ b/.ci/protect-secrets.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# This script protects all environment variables that start with "SECRET_". +# It puts them in a temporary file. The name of the variable contains the path +# of that file. This filename can then safely be used in `cat` even if `set +# -x` has been turned on. Also you can run "export" to understand the +# environment without danger. +# Be careful, however, not to use this like the following: +# docker login $DOCKER_USER $(cat $SECRET_DOCKER_PASS) +# as this would expose the password if `set -x` has been turned on. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -eo pipefail +set +x + +function encrypt { + RET=`mktemp` + eval " echo \$$1" > "$RET" + echo $RET +} + +for name in `awk 'END { for (name in ENVIRON) { print name; } }' < /dev/null`; do +case "$name" in + SECRET_*) + export $name="$(encrypt $name)" + echo "Protected $name" + ;; +esac +done + +unset encrypt diff --git a/.ci/pull-gitlab.sh b/.ci/pull-gitlab.sh new file mode 100755 index 00000000000..01bb10cdd08 --- /dev/null +++ b/.ci/pull-gitlab.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# This script gets called from CI to pull the Sage docker images that were +# built during the "build" phase to pull all the connected docker daemon +# (likely a docker-in-docker.) +# This script expects a single parameter, the base name of the docker image +# such as sagemath or sagemath-dev. +# The variable $DOCKER_IMAGE is set to the full name of the pulled image; +# source this script to use it. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +[[ -z "$DOCKER_TAG" ]] && (echo "Can not pull untagged build."; exit 0) +[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest + +# Pull the built images from the gitlab registry and give them the original +# names they had after built. +# Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it +# automatically from the log output. +docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY +docker pull $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG +DOCKER_IMAGE="${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG" +docker tag $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG $DOCKER_IMAGE diff --git a/.ci/push-dockerhub.sh b/.ci/push-dockerhub.sh new file mode 100755 index 00000000000..ca2c4906eec --- /dev/null +++ b/.ci/push-dockerhub.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# This script gets called from CI to push our docker images to +# $DOCKER_USER/sagemath* on the Docker Hub. +# This script expects a single parameter, the base name of the docker image +# such as sagemath or sagemath-dev. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +[[ -z "$DOCKER_TAG" ]] && (echo "Can not push untagged build."; exit 0) +[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest + +# Push the built images to the docker hub (and fail silently if +# DOCKER_USER/SECRET_DOCKER_PASS have not been configured.) +if [[ -z "$DOCKER_USER" || -z "$SECRET_DOCKER_PASS" ]]; then + echo "DOCKER_USER/SECRET_DOCKER_PASS variables have not been configured in your Continuous Integration setup. Not pushing built images to Docker Hub." +else + cat "$SECRET_DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin + docker push ${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG +fi diff --git a/.ci/push-gitlab.sh b/.ci/push-gitlab.sh new file mode 100755 index 00000000000..9d98dca7e49 --- /dev/null +++ b/.ci/push-gitlab.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -ex + +# This script gets called from CI to push our docker images to registry +# configured in GitLab. (Mostly, so we can pull them again to push them to the +# Docker Hub.) +# This script expects a single parameter, the base name of the docker image +# such as sagemath or sagemath-dev. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +[[ -z "$DOCKER_TAG" ]] && (echo "Can not push untagged build."; exit 0) +[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest + +# Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it +# automatically from the log output. +docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY +docker tag ${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG +docker push $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG diff --git a/.ci/setup-make-parallelity.sh b/.ci/setup-make-parallelity.sh new file mode 100644 index 00000000000..12f3a252179 --- /dev/null +++ b/.ci/setup-make-parallelity.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** +set -ex + +# Determine the number of threads that can run simultaneously on this system +# (we might not have nproc available.) +# Note that this value is incorrect for some CI providers (notably CircleCI: +# https://circleci.com/docs/2.0/configuration-reference/#resource_class) which +# provision fewer vCPUs than shown in /proc/cpuinfo. Also, setting this value +# too high can lead to RAM being insufficient, so it's best to set this +# variable manually in your CI configuration. +[[ -z "$NTHREADS" ]] && NTHREADS=`grep -E '^processor' /proc/cpuinfo | wc -l` || true +# Set -j and -l for make (though -l is probably stripped by Sage) +[[ -z "$MAKEOPTS" ]] && MAKEOPTS="-j $NTHREADS -l $((NTHREADS-1)).8" || true +# Not all parts of Sage seem to honor MAKEOPTS, so the current way of telling +# the system which concurrency to use, seems to be setting $MAKE. +[[ -z "$MAKE" ]] && MAKE="make $MAKEOPTS" || true diff --git a/.ci/test-cli.sh b/.ci/test-cli.sh new file mode 100755 index 00000000000..a9e8c059ffe --- /dev/null +++ b/.ci/test-cli.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# This script gets called from CI to run minimal tests on the sagemath image. + +# Usage: ./test-cli.sh sage-cli-image + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** +set -exo pipefail + +echo "Checking that Sage starts and can calculate 1+1…" +# Calculate 1+1 (remove startup messages and leading & trailing whitespace) +TWO=`docker run "$1" sage -c "'print(1+1)'" | tail -1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'` +[[ "x$TWO" = "x2" ]] diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh new file mode 100755 index 00000000000..60b59d6a5ca --- /dev/null +++ b/.ci/test-dev.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# This script gets called from CI to run minimal tests on the sagemath-dev image. +# This script expects a single argument, the full name of the docker image to +# test. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** +set -exo pipefail + +IMAGE="$1" + +. .ci/setup-make-parallelity.sh + +# Usage: timed_run limit args +# Runs $IMAGE with args and check that it terminates with a zero exit code in at most limit seconds. +function timed_run { + START=`date +%s` + docker run -e MAKE="$MAKE" "$IMAGE" "$2" + END=`date +%s` + TOTAL=$((END-START)) + echo "Checking that \"$2\" was fast…" + [[ $TOTAL -lt $1 ]] +} + +timed_run 60 true # runs make build +# TODO: Can't we get this faster than that? +timed_run 300 make # runs make build and then make diff --git a/.ci/test-jupyter.sh b/.ci/test-jupyter.sh new file mode 100755 index 00000000000..982f06fbeac --- /dev/null +++ b/.ci/test-jupyter.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# This script gets called from CI to run minimal tests on the sagemath-jupyter +# image. + +# Usage: ./test-jupyter.sh sage-jupyter-image [host] + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** +set -ex + +docker run --name sage-jupyter -p 8888:8888 -d "$1" "sage -n jupyter --no-browser --ip='*' --port=8888" +echo "Checking that the Jupyter notebook is running…" +sleep 10 # giving the server some time to start +docker logs sage-jupyter +wget --retry-connrefused --tries=10 --wait=3 "http://${2:-localhost}:8888" diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..13f8b567f5f --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,41 @@ +# This file configures automatic builds of Sage on [CircleCI](https://circleci.com). +# To make the build time not too excessive, we seed the build cache with +# sagemath/sagemath-dev:latest. When basic SPKGs change, this does not help much +# and the full build might exceed CircleCI's limits for open source projcets (five +# hours on 2 vCPUs as of early 2018.) +# You might want to try to build locally or with GitLab CI, see +# `.gitlab-ci.yml` for more options. +# Note that we do not use workflows because it was not clear whether the docker +# images sizes would not exceed the size limits of CircleCI workspaces. Also, +# workflows would not make things faster. We do not get more machines for free +# and the frequent loading of docker images probably exceeds the cost of the +# actual tests. + +version: 2 +jobs: + # As https://circleci.com/docs/2.0/docker-layer-caching/ is a paid feature, + # we do build & test & release in one step. + build: + machine: true + steps: + - checkout + - run: + # The docker commands sometimes take a while to produce output + no_output_timeout: 30m + command: | + # CircleCI has no mechanism to hide secret variables. + # Therefore we roll our own to protect $SECRET_* variables. + . .ci/protect-secrets.sh + + export DOCKER_TAG=${CIRCLE_TAG:-$CIRCLE_BRANCH} + # Build docker images + # TODO: Change this line to sagemath/sagemath-dev:latest + export ARTIFACT_BASE=saraedum/sagemath-dev:gitlabci + . .ci/build-docker.sh + # Test that the images work + . .ci/test-dev.sh $DOCKER_IMAGE_DEV + . .ci/test-cli.sh $DOCKER_IMAGE_CLI + . .ci/test-jupyter.sh $DOCKER_IMAGE_CLI localhost + # Push docker images to dockerhub if a dockerhub user has been configured + . .ci/push-dockerhub.sh sagemath-dev + . .ci/push-dockerhub.sh sagemath diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 00000000000..3e4e48b0b5f --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index 101d349690c..498b92c73b9 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,9 @@ $RECYCLE.BIN/ ########### .ipynb_checkpoints Untitled*.ipynb + +############################# +# GitLab CI generated files # +############################# +gitlab-build-docker.log + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000000..e256dd1effe --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,99 @@ +# This file configures automatic builds of Sage on [GitLab](https://gitlab.com). +# To make the build time not too excessive, we seed the build cache with +# sagemath/sagemath-dev:latest. When basic SPKGs changed, this does not help +# much and the full build might the set time limit in GitLab. You can increase +# that limit in Settings → CI/CD. +# You can also provision your own private more powerful runner in the same +# place; or set up your favourite cloud service to provide an on-demand +# autoscale runner. + +image: docker:latest + +stages: + - build + - test + - release + +variables: + DOCKER_TAG: $CI_COMMIT_REF_SLUG + # Builds are very I/O intensive; make sure we have a fast file system. + DOCKER_DRIVER: overlay2 + # TODO: Change this line to sagemath/sagemath-dev:latest + ARTIFACT_BASE: saraedum/sagemath-dev:gitlabci + +before_script: + # GitLab has no mechanism yet to hide secret variables: https://gitlab.com/gitlab-org/gitlab-ce/issues/13784 + # So we roll our own which protects all variables that start with SECRET_ + - . .ci/protect-secrets.sh + +# We use docker-in-docker to build our docker images. It can be faster to +# expose your outer docker daemon by mounting /var/run/docker.sock to +# /var/run/docker.sock and setting DOCKER_HOST in Settings -> CI/CD -> Secret +# variable to unix:///var/run/docker.sock +services: +- docker:dind + +# Build Sage and its documentation. +# The build starts from the build artifacts of ARTIFACT_BASE which is usually +# much faster than building from a clean checkout of Sage. +build-from-latest: &build + stage: build + artifacts: + when: always + paths: + - gitlab-build-docker.log + expire_in: 1 month + script: + # The output of the build can get larger than gitlab.com's limit; only print the first 3MB. + - . .ci/build-docker.sh | tee gitlab-build-docker.log | head -c 3m + - . .ci/push-gitlab.sh sagemath-dev + - . .ci/push-gitlab.sh sagemath + except: + - master + +# Build Sage and its documentation from a clean checkout of Sage. +# Note that this takes a very long time. You probably want to run this on your +# own gitlab-runner and increase the standard GitLab time limit for CI runs. +build-from-clean: + << : *build + variables: + ARTIFACT_BASE: "source-clean" + only: + - master + +test-dev: + stage: test + script: + - . .ci/pull-gitlab.sh sagemath-dev + - . .ci/test-dev.sh "$DOCKER_IMAGE" + +test-cli: + stage: test + script: + - . .ci/pull-gitlab.sh sagemath + - . .ci/test-cli.sh "$DOCKER_IMAGE" + +test-jupyter: + stage: test + script: + # Force usage of docker-in-docker (and don't start docker on a bind-mounted + # /var/run/docker.sock set through a proivate GitLab CI variable) so that + # the -p flag to docker run works as expected. + - export DOCKER_HOST='tcp://docker:2375' + - apk update + - apk add wget + - . .ci/pull-gitlab.sh sagemath + - . .ci/test-jupyter.sh "$DOCKER_IMAGE" docker + +# Pushes the built images to Docker Hub if the Settings -> CI/CD -> Secret +# variables DOCKER_USER and SECRET_DOCKER_PASS have been set up. +dockerhub: + stage: release + only: + - branches + - tags + script: + - . .ci/pull-gitlab.sh sagemath-dev + - . .ci/push-dockerhub.sh sagemath-dev + - . .ci/pull-gitlab.sh sagemath + - . .ci/push-dockerhub.sh sagemath diff --git a/Makefile b/Makefile index 92457458fe5..d2dfc71256d 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,7 @@ misc-clean: rm -f build/make/Makefile build/make/Makefile-auto rm -f .BUILDSTART +# TODO: What is a "bdist"? A binary distribution? bdist-clean: clean $(MAKE) misc-clean @@ -89,6 +90,31 @@ maintainer-clean: distclean bootstrap-clean micro_release: bdist-clean sagelib-clean @echo "Stripping binaries ..." LC_ALL=C find local/lib local/bin -type f -exec strip '{}' ';' 2>&1 | grep -v "File format not recognized" | grep -v "File truncated" || true + @echo "Removing .py files that have a corresponding .pyc or .pyo file as they are not needed when running code with CPython..." + find local/lib/python* -name '*.py' | while IFS= read -r fname; do [ -e "$${fname}c" -o -e "$${fname}o" ] && rm "$$fname"; done || true + @echo "Removing sphinx artifacts..." + rm -rf local/share/doc/sage/doctrees local/share/doc/sage/inventory + @echo "Removing unnecessary files & directories - make will not be functional afterwards anymore" + @# We need src/sage/ for introspection with "??" + @# We need src/sage/bin/ for the scripts that invoke Sage + @# We need sage, the script to start Sage + @# We need local/, the dependencies and the built Sage library itself. + @# We keep VERSION.txt. + @# We keep COPYING.txt so we ship a license with this distribution. + find . -name . -o -prune ! -name src ! -name sage ! -name local ! -name VERSION.txt ! -name COPYING.txt ! -name build -exec rm -rf \{\} \; + cd src && find . -name . -o -prune ! -name sage ! -name bin -exec rm -rf \{\} \; + +# Leaves everything that is needed to make the next "make" fast but removes +# all the cheap build artifacts that can be quickly regenerated. +fast-rebuild-clean: misc-clean bdist-clean + rm -rf upstream/ + rm -rf src/build/temp.* + # Without site-packages/sage sage does not start but copying/compiling + # them from src/build is very fast. + rm -rf local/lib/python*/site-packages/sage + # The .py files in src/build are restored from src/sage without their + # mtimes changed. + find src/build -name '*.py' -exec rm \{\} \; TESTALL = ./sage -t --all PTESTALL = ./sage -t -p --all diff --git a/configure.ac b/configure.ac index 5c61a89704e..2d86803c6c9 100644 --- a/configure.ac +++ b/configure.ac @@ -285,9 +285,9 @@ fi AC_CHECK_PROG(found_latex, latex, yes, no) if test x$found_latex != xyes then - AC_MSG_WARN([You do not have 'latex', which is recommended, but not]) - AC_MSG_WARN([required. Latex is only really used for building pdf]) - AC_MSG_WARN([documents and for %latex mode in the AC_PACKAGE_NAME notebook.]) + AC_MSG_NOTICE([You do not have 'latex', which is recommended, but not]) + AC_MSG_NOTICE([required. Latex is only really used for building pdf]) + AC_MSG_NOTICE([documents and for %latex mode in the AC_PACKAGE_NAME notebook.]) fi # Check that perl is available, with version 5.8.0 or later. diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 00000000000..a38152cb5ee --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1,2 @@ +# Stores the commit that was used to create a sagemath-dev image +commit diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000000..503f9eb579f --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,176 @@ +################################################################################ +# SageMath images for Docker # +################################################################################ +# This is a description of the layout of this Dockerfile; for details on the # +# created docker images, see the README.md please. # +# # +# This Dockerfile builds sagemath (for end-users) and sagemath-dev (for # +# developers) it consists of lots of intermediate targets, mostly to shrink # +# the resulting images but also to make this hopefully easier to maintain. # +# The aims of this Dockerfile are: # +# (1) Make it build in reasonable time. # +# (2) It should be self-contained and work on its own, i.e., just by invoking # +# docker build without any external orchestration script. # +# # +# The idea to achieve (1) is to reuse the build artifacts from the latest # +# master build. This is slightly against the philosophy of a Dockerfile (which # +# should produce perfectly reproducible outputs) but building Sage from scratch# +# just takes too long at the moment to do this all the time. ARTIFACT_BASE # +# controls which build artifacts are used. You probably want to set this to # +# sagemath/sagemath-dev:latest which takes the latest build from the official # +# master branch. The default is source-clean which builds Sage from scratch. # +# If you want to understand how this works, have a look at source-from-context # +# which merges ARTIFACT_BASE with the context, i.e., the contents of the sage # +# source directory. # +################################################################################ + + +ARG ARTIFACT_BASE=source-clean + +################################################################################ +# Image containing the run-time dependencies for Sage # +################################################################################ +FROM ubuntu:xenial as run-time-dependencies +LABEL maintainer="Erik M. Bray , Julian Rüth " +# Set sane defaults for common environment variables. +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 +ENV SHELL /bin/bash +# Create symlinks for sage and sagemath - we copy a built sage to the target of these symlinks later. +ARG SAGE_ROOT=/home/sage/sage +RUN ln -s "$SAGE_ROOT/sage" /usr/bin/sage +RUN ln -s /usr/bin/sage /usr/bin/sagemath +# Sage needs the fortran libraries at run-time because we do not build gfortran +# with Sage but use the system's. +# We also install sudo for the sage user, see below. +RUN apt-get update -qq \ + && apt-get install -y --no-install-recommends gfortran sudo \ + && apt-get clean \ + && rm -r /var/lib/apt/lists/* +# Sage refuses to build as root, so we add a "sage" user that can sudo without a password. +# We also want this user at runtime as some commands in sage know about the user that was used during build. +ARG HOME=/home/sage +RUN adduser --quiet --shell /bin/bash --gecos "Sage user,101,," --disabled-password --home "$HOME" sage \ + && echo "sage ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/01-sage \ + && chmod 0440 /etc/sudoers.d/01-sage +# Run everything from now on as the sage user in sage's home +USER sage +ENV HOME $HOME +WORKDIR $HOME + +################################################################################ +# Image containing everything so that a make in a clone of the Sage repository # +# completes without errors # +################################################################################ +FROM run-time-dependencies as build-time-dependencies +# Install the build time dependencies & git +RUN sudo apt-get update -qq \ + && sudo apt-get install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git \ + && sudo apt-get clean \ + && sudo rm -r /var/lib/apt/lists/* + +################################################################################ +# Image with an empty git repository in $SAGE_ROOT. # +################################################################################ +FROM build-time-dependencies as source-clean +ARG SAGE_ROOT=/home/sage/sage +RUN mkdir -p "$SAGE_ROOT" +WORKDIR $SAGE_ROOT +RUN git init +RUN git remote add trac git@trac.sagemath.org:sage.git + +################################################################################ +# Image with the build context added, i.e., the directory from which `docker # +# build` has been called in a separate directory so we can copy files from # +# there. # +# This blows up the size of this docker image significantly, but we only use # +# this image to create artifacts for our final image. # +# Warning: If you set ARTIFACT_BASE to something else than source-clean, the # +# build is not going to use the build-time-dependencies target but rely on # +# whatever tools are installed in ARTIFACT_BASE. # +################################################################################ +FROM $ARTIFACT_BASE as source-from-context +WORKDIR $HOME +COPY --chown=sage:sage . sage-context +# Checkout the commit that is in $HOME/sage-context +ARG SAGE_ROOT=/home/sage/sage +WORKDIR $SAGE_ROOT +RUN git fetch "$HOME/sage-context" HEAD \ + && git reset FETCH_HEAD \ + && git checkout -f FETCH_HEAD \ + && git log -1 --format=%H > docker/commit \ + && rm -rf .git +# Copy over all the untracked/staged/unstaged changes from sage-context +WORKDIR $HOME/sage-context +RUN if git status --porcelain | read CHANGES; then \ + git -c user.name=docker-build -c user.email=docker-build@sage.invalid stash -u \ + && git stash show -p > "$HOME"/sage-context.patch; \ + else \ + touch "$HOME"/sage-context.patch; \ + fi +WORKDIR $SAGE_ROOT +RUN patch -p1 < "$HOME"/sage-context.patch + +################################################################################ +# Image with a built sage but without sage's documentation. # +################################################################################ +FROM source-from-context as make-build +# Make sure that the results runs on most CPUs. +ENV SAGE_FAT_BINARY yes +# Just to be sure Sage doesn't try to build its own GCC (even though +# it shouldn't with a recent GCC package from the system and with gfortran) +ENV SAGE_INSTALL_GCC no +# Make everything in the build use multiple cores (this causes trouble for some packages outside Sage but it still seems to be they way Sage is doing this.) +ARG MAKE="make -j2" +ENV MAKE $MAKE +RUN make build + +################################################################################ +# Image with a full build of sage and its documentation. # +################################################################################ +FROM make-build as make-all +RUN make + +################################################################################ +# Image with a full build of sage and its documentation but everything # +# stripped that can be quickly rebuild by make. # +################################################################################ +FROM make-all as make-fast-rebuild-clean +RUN make fast-rebuild-clean + +################################################################################ +# A releasable (relatively small, but still huge) image of this build with all # +# the build artifacts intact so developers can make changes and rebuild # +# quickly # +################################################################################ +FROM source-clean as sagemath-dev +ARG SAGE_ROOT=/home/sage/sage +COPY --chown=sage:sage --from=make-fast-rebuild-clean $SAGE_ROOT $SAGE_ROOT +COPY ./docker/entrypoint-dev.sh /usr/local/bin/sage-entrypoint +ENTRYPOINT ["/usr/local/bin/sage-entrypoint"] +CMD ["bash"] + +################################################################################ +# Image with a full build of sage, ready to release, i.e., with stripped # +# binaries and some extras to run the jupyter notebook. # +################################################################################ +FROM make-all as make-release +RUN sage -pip install terminado "notebook>=5" "ipykernel>=4.6" +RUN sage -i gap_jupyter singular_jupyter pari_jupyter +RUN make micro_release + +################################################################################ +# A releasable (relatively small) image which contains a copy of sage without # +# temporary build artifacts which is set up to start the command line # +# interface if no parameters are passed in. # +################################################################################ +FROM run-time-dependencies as sagemath +ARG SAGE_ROOT=/home/sage/sage +COPY --chown=sage:sage --from=make-release $SAGE_ROOT/ $SAGE_ROOT/ +# Put scripts to start gap, gp, maxima, ... in /usr/bin +WORKDIR $SAGE_ROOT +RUN sudo sage --nodotsage -c "install_scripts('/usr/bin')" +COPY ./docker/entrypoint.sh /usr/local/bin/sage-entrypoint +ENTRYPOINT ["/usr/local/bin/sage-entrypoint"] +EXPOSE 8888 +CMD ["sage"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000000..9b653689031 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,51 @@ +[![License: GPL v3](https://img.shields.io/badge/License-GPL%20v3-blue.svg)](https://github.com/sagemath/sage/COPYING.txt) [![Maintained](https://img.shields.io/maintenance/yes/2018.svg)](https://github.com/sagemath/sage/commits/master) [![CircleCI](https://circleci.com/gh/saraedum/sage.svg?style=svg)](https://circleci.com/gh/saraedum/sage) [![GitLab CI](https://gitlab.com/saraedum/sage/badges/gitlabci/pipeline.svg)](https://gitlab.com/saraedum/sage/commits/gitlabci) + +# Supported tags + +* `latest` — the stable `master` branch +* `x.x.x` — all stable releases of Sage are tagged with their version number. +* `develop` — the current development version of Sage which gets merged into the `master` branch when a new version of Sage is released + +# What is SageMath + +SageMath is a free open-source mathematics software system licensed under the GPL. It builds on top of many existing open-source packages: NumPy, SciPy, matplotlib, Sympy, Maxima, GAP, FLINT, R and many more. Access their combined power through a common, Python-based language or directly via interfaces or wrappers. + +**Mission**: *Creating a viable free open source alternative to Magma, Maple, Mathematica and Matlab.* + +# What's in this image + +There are several flavours of this image. + +* [`sagemath/sagemath`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath.svg)](https://hub.docker.com/sagemath/sagemath) contains everything necessary to run Sage on the command line. Run it with: + ``` + docker run -it sagemath/sagemath:latest + ``` + You can start a graphical [Jupyter Notebook](https://jupyter.org) at http://localhost:8888 instead. To use the notebook, follow the instructions printed when you run: + ``` + docker run -p8888:8888 sagemath/sagemath:latest "sage -n jupyter --no-browser --ip='*' --port=8888" + ``` +* [`sagemath/sagemath-dev`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath-dev.svg)](https://hub.docker.com/sagemath/sagemath-dev) contains all the build artifacts to rebuild Sage quickly. This version is probably only relevant for Sage developers. Run this image with: + ``` + docker run -it sagemath/sagemath-dev:latest + ``` + This triggers a rebuild and drops you in a shell afterwards. Note that the git repository has been emptied to save space. If you want to use git, fetch from your git repository with `git fetch trac` and go to the commit that was used to create this image with + ``` + git reset $(cat docker/commit) + ``` + +# How to build your own SageMath images + +Run `docker build -f docker/Dockerfile --target TARGET .` in the Sage repository with `TARGET` one of `sage`, `sage-jupyter`, or `dev`. + +# How these images get updated + +Every push to our [github repository](https://github.com/sagemath/sage) triggers a build in [CircleCI](https://circleci.com) which builds and pushes the docker images. +A push to master also triggers a "build" on our [Docker Hub](https://hub.docker.com) repositories. The latter build is mostly disabled by the `hooks/` and only updates the `README.md`. + +Every push to our [GitLab repository](https://gitlab.com/sagemath/sage) triggers a pipeline in GitLab CI. This build also pushes images to Docker Hub. + +Have a look at `.circleci/` and `.gitlab-ci.yml` if you want to setup either continuous integration service for your own fork of the SageMath repository. + +# License + +The whole Sage software distribution is licensed under the General Public License, version 3. More details can be found in our [COPYING.txt](https://github.com/sagemath/sage/blob/master/COPYING.txt) diff --git a/docker/entrypoint-dev.sh b/docker/entrypoint-dev.sh new file mode 100755 index 00000000000..67299a5a92d --- /dev/null +++ b/docker/entrypoint-dev.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e +make build +exec "$@" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000000..692b5c83390 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec sage -sh -c "$*" diff --git a/docker/hooks/build b/docker/hooks/build new file mode 100644 index 00000000000..b23e55619b2 --- /dev/null +++ b/docker/hooks/build @@ -0,0 +1 @@ +#!/bin/true diff --git a/docker/hooks/push b/docker/hooks/push new file mode 100644 index 00000000000..b23e55619b2 --- /dev/null +++ b/docker/hooks/push @@ -0,0 +1 @@ +#!/bin/true diff --git a/src/sage_setup/docbuild/__init__.py b/src/sage_setup/docbuild/__init__.py index b8f79c2269a..bf48bf57cdf 100644 --- a/src/sage_setup/docbuild/__init__.py +++ b/src/sage_setup/docbuild/__init__.py @@ -911,7 +911,12 @@ def get_new_and_updated_modules(self): except ImportError as err: logger.error("Warning: Could not import %s %s", module_name, err) raise - newtime = os.path.getmtime(sys.modules[module_name].__file__) + + module_filename = sys.modules[module_name].__file__ + if (module_filename.endswith('.pyc') or module_filename.endswith('.pyo')): + source_filename = module_filename[:-1] + if (os.path.exists(source_filename)): module_filename = source_filename + newtime = os.path.getmtime(module_filename) if newtime > mtime: updated_modules.append(module_name) From 6ccbaa3130cf397128b99dad75cbbe99c8abe745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 25 Feb 2018 20:44:29 +0100 Subject: [PATCH 002/136] Be more agressive about requiring the latest develop otherwise the builds will just take forever and it is good practice to merge develop frequently to prevent future conflicts. --- .ci/build-docker.sh | 2 +- .circleci/config.yml | 17 +++++++++++++---- .gitlab-ci.yml | 9 ++++++--- docker/Dockerfile | 33 ++++++++++++++++++++++++++------- docker/README.md | 2 +- 5 files changed, 47 insertions(+), 16 deletions(-) diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh index 11651e5dd38..9ec55ab149a 100755 --- a/.ci/build-docker.sh +++ b/.ci/build-docker.sh @@ -21,7 +21,7 @@ set -ex # We speed up the build process by copying built artifacts from ARTIFACT_BASE # during docker build. See /docker/Dockerfile for more details. -ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:latest} +ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop} # Seed our cache with $ARTIFACT_BASE if it exists docker pull $ARTIFACT_BASE || true diff --git a/.circleci/config.yml b/.circleci/config.yml index 13f8b567f5f..8a2cc7ca71d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ # This file configures automatic builds of Sage on [CircleCI](https://circleci.com). # To make the build time not too excessive, we seed the build cache with -# sagemath/sagemath-dev:latest. When basic SPKGs change, this does not help much +# sagemath/sagemath-dev:develop. When basic SPKGs change, this does not help much # and the full build might exceed CircleCI's limits for open source projcets (five # hours on 2 vCPUs as of early 2018.) # You might want to try to build locally or with GitLab CI, see @@ -27,15 +27,24 @@ jobs: # Therefore we roll our own to protect $SECRET_* variables. . .ci/protect-secrets.sh - export DOCKER_TAG=${CIRCLE_TAG:-$CIRCLE_BRANCH} # Build docker images - # TODO: Change this line to sagemath/sagemath-dev:latest - export ARTIFACT_BASE=saraedum/sagemath-dev:gitlabci + export DOCKER_TAG=${CIRCLE_TAG:-$CIRCLE_BRANCH} + case $DOCKER_TAG in + "develop" | "latest") + export ARTIFACT_BASE=source-clean + ;; + *) + # TODO: Change this line to sagemath/sagemath-dev:develop + export ARTIFACT_BASE=saraedum/sagemath-dev:develop + ;; + esac . .ci/build-docker.sh + # Test that the images work . .ci/test-dev.sh $DOCKER_IMAGE_DEV . .ci/test-cli.sh $DOCKER_IMAGE_CLI . .ci/test-jupyter.sh $DOCKER_IMAGE_CLI localhost + # Push docker images to dockerhub if a dockerhub user has been configured . .ci/push-dockerhub.sh sagemath-dev . .ci/push-dockerhub.sh sagemath diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e256dd1effe..d56fb0e28a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ # This file configures automatic builds of Sage on [GitLab](https://gitlab.com). # To make the build time not too excessive, we seed the build cache with -# sagemath/sagemath-dev:latest. When basic SPKGs changed, this does not help +# sagemath/sagemath-dev:develop. When basic SPKGs changed, this does not help # much and the full build might the set time limit in GitLab. You can increase # that limit in Settings → CI/CD. # You can also provision your own private more powerful runner in the same @@ -18,8 +18,8 @@ variables: DOCKER_TAG: $CI_COMMIT_REF_SLUG # Builds are very I/O intensive; make sure we have a fast file system. DOCKER_DRIVER: overlay2 - # TODO: Change this line to sagemath/sagemath-dev:latest - ARTIFACT_BASE: saraedum/sagemath-dev:gitlabci + # TODO: Change this line to sagemath/sagemath-dev:develop + ARTIFACT_BASE: saraedum/sagemath-dev:develop before_script: # GitLab has no mechanism yet to hide secret variables: https://gitlab.com/gitlab-org/gitlab-ce/issues/13784 @@ -50,6 +50,7 @@ build-from-latest: &build - . .ci/push-gitlab.sh sagemath except: - master + - develop # Build Sage and its documentation from a clean checkout of Sage. # Note that this takes a very long time. You probably want to run this on your @@ -60,6 +61,8 @@ build-from-clean: ARTIFACT_BASE: "source-clean" only: - master + - develop + except: [] test-dev: stage: test diff --git a/docker/Dockerfile b/docker/Dockerfile index 503f9eb579f..d62f4c60cd3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,12 +13,12 @@ # docker build without any external orchestration script. # # # # The idea to achieve (1) is to reuse the build artifacts from the latest # -# master build. This is slightly against the philosophy of a Dockerfile (which # +# develop build. This is slightly against the philosophy of a Dockerfile (which# # should produce perfectly reproducible outputs) but building Sage from scratch# # just takes too long at the moment to do this all the time. ARTIFACT_BASE # # controls which build artifacts are used. You probably want to set this to # -# sagemath/sagemath-dev:latest which takes the latest build from the official # -# master branch. The default is source-clean which builds Sage from scratch. # +# sagemath/sagemath-dev:develop which takes the latest build from the official # +# develop branch. The default is source-clean which builds Sage from scratch. # # If you want to understand how this works, have a look at source-from-context # # which merges ARTIFACT_BASE with the context, i.e., the contents of the sage # # source directory. # @@ -92,15 +92,34 @@ RUN git remote add trac git@trac.sagemath.org:sage.git FROM $ARTIFACT_BASE as source-from-context WORKDIR $HOME COPY --chown=sage:sage . sage-context -# Checkout the commit that is in $HOME/sage-context +# Checkout the commit that checked out in $HOME/sage-context +# This is a bit complicated because our local .git/ is empty and we want to +# make sure that we only change the mtimes of a minimal number of files. +# 1) Restore the git checkout ARTIFACT_BASE was built from, recorded in +# docker/commit. (Or go directly to FETCH_HEAD if there is no history to +# restore.) +# 2) Merge in FETCH_HEAD but only if it is a fast-forward, i.e., if it is an +# ancestor of the commit restored in 1. If we would not do that we would get +# a new commit hash in docker/commit that is not known outside of this build +# run. Since docker/commit was in the history of FETCH_HEAD this should +# automatically be a fast-forward. +# 3) Trash .git again to save some space. ARG SAGE_ROOT=/home/sage/sage WORKDIR $SAGE_ROOT RUN git fetch "$HOME/sage-context" HEAD \ - && git reset FETCH_HEAD \ - && git checkout -f FETCH_HEAD \ + && if [ -e docker/commit ]; then \ + git reset `cat docker/commit` \ + || ( echo "Could not find commit `cat docker/commit` in your local Git history. Please merge in the latest develop branch to fix this: git fetch trac && git merge trac/develop." && exit 1 ) \ + else \ + echo "You are building from $ARTIFACT_BASE which has no docker/commit file. That's a bug unless you are building from source-clean or something similar." \ + && git reset FETCH_HEAD \ + && git checkout -f FETCH_HEAD; \ + fi \ + && git merge --ff-only FETCH_HEAD \ && git log -1 --format=%H > docker/commit \ && rm -rf .git -# Copy over all the untracked/staged/unstaged changes from sage-context +# Copy over all the untracked/staged/unstaged changes from sage-context. This +# is relevant for non-CI invocations of this Dockerfile. WORKDIR $HOME/sage-context RUN if git status --porcelain | read CHANGES; then \ git -c user.name=docker-build -c user.email=docker-build@sage.invalid stash -u \ diff --git a/docker/README.md b/docker/README.md index 9b653689031..0e388d95e5a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -26,7 +26,7 @@ There are several flavours of this image. ``` * [`sagemath/sagemath-dev`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath-dev.svg)](https://hub.docker.com/sagemath/sagemath-dev) contains all the build artifacts to rebuild Sage quickly. This version is probably only relevant for Sage developers. Run this image with: ``` - docker run -it sagemath/sagemath-dev:latest + docker run -it sagemath/sagemath-dev:develop ``` This triggers a rebuild and drops you in a shell afterwards. Note that the git repository has been emptied to save space. If you want to use git, fetch from your git repository with `git fetch trac` and go to the commit that was used to create this image with ``` From 7f90d86a5716391952675f7ef45104ff4f85e925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 26 Feb 2018 00:45:40 +0100 Subject: [PATCH 003/136] be more precise about failed latest merge --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d62f4c60cd3..ca3e40a6af8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -109,7 +109,7 @@ WORKDIR $SAGE_ROOT RUN git fetch "$HOME/sage-context" HEAD \ && if [ -e docker/commit ]; then \ git reset `cat docker/commit` \ - || ( echo "Could not find commit `cat docker/commit` in your local Git history. Please merge in the latest develop branch to fix this: git fetch trac && git merge trac/develop." && exit 1 ) \ + || ( echo "Could not find commit `cat docker/commit` in your local Git history. Please merge in the latest built develop branch to fix this: git fetch trac && git merge `cat docker/commit`." && exit 1 ) \ else \ echo "You are building from $ARTIFACT_BASE which has no docker/commit file. That's a bug unless you are building from source-clean or something similar." \ && git reset FETCH_HEAD \ From 8efe3a9d501d97b89575952166109bb770b9026f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 26 Feb 2018 00:59:17 +0100 Subject: [PATCH 004/136] more and more specific badges --- docker/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docker/README.md b/docker/README.md index 0e388d95e5a..e64d90937a3 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,10 +1,11 @@ -[![License: GPL v3](https://img.shields.io/badge/License-GPL%20v3-blue.svg)](https://github.com/sagemath/sage/COPYING.txt) [![Maintained](https://img.shields.io/maintenance/yes/2018.svg)](https://github.com/sagemath/sage/commits/master) [![CircleCI](https://circleci.com/gh/saraedum/sage.svg?style=svg)](https://circleci.com/gh/saraedum/sage) [![GitLab CI](https://gitlab.com/saraedum/sage/badges/gitlabci/pipeline.svg)](https://gitlab.com/saraedum/sage/commits/gitlabci) +[![License: GPL v3](https://img.shields.io/badge/License-GPL%20v3-blue.svg)](https://github.com/sagemath/sage/COPYING.txt) [![Maintained](https://img.shields.io/maintenance/yes/2018.svg)](https://github.com/sagemath/sage/commits/master) # Supported tags -* `latest` — the stable `master` branch +* `latest` — the stable `master` branch [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/saraedum/sage/master.svg)](https://github.com/saraedum/sage/commits/master) [![CircleCI branch](https://img.shields.io/circleci/project/github/saraedum/sage/master.svg)](https://circleci.com/gh/saraedum/sage/tree/master) [![GitLab CI](https://gitlab.com/saraedum/sage/badges/master/pipeline.svg)](https://gitlab.com/saraedum/sage/commits/master) * `x.x.x` — all stable releases of Sage are tagged with their version number. -* `develop` — the current development version of Sage which gets merged into the `master` branch when a new version of Sage is released +* `develop` — the current development version of Sage which gets merged into the `master` branch when a new version of Sage is released [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/saraedum/sage/develop.svg)](https://github.com/saraedum/sage/commits/develop) [![CircleCI branch](https://img.shields.io/circleci/project/github/saraedum/sage/master.svg)](https://circleci.com/gh/saraedum/sage/tree/master) [![GitLab CI](https://gitlab.com/saraedum/sage/badges/develop/pipeline.svg)](https://gitlab.com/saraedum/sage/commits/develop) + # What is SageMath @@ -16,7 +17,7 @@ SageMath is a free open-source mathematics software system licensed under the GP There are several flavours of this image. -* [`sagemath/sagemath`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath.svg)](https://hub.docker.com/sagemath/sagemath) contains everything necessary to run Sage on the command line. Run it with: +* [`sagemath/sagemath`![image size](https://img.shields.io/microbadger/image-size/saraedum/sagemath:latest.svg)](https://hub.docker.com/saraedum/sagemath) contains everything necessary to run Sage on the command line. Run it with: ``` docker run -it sagemath/sagemath:latest ``` @@ -24,7 +25,7 @@ There are several flavours of this image. ``` docker run -p8888:8888 sagemath/sagemath:latest "sage -n jupyter --no-browser --ip='*' --port=8888" ``` -* [`sagemath/sagemath-dev`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath-dev.svg)](https://hub.docker.com/sagemath/sagemath-dev) contains all the build artifacts to rebuild Sage quickly. This version is probably only relevant for Sage developers. Run this image with: +* [`sagemath/sagemath-dev`![image size](https://img.shields.io/microbadger/image-size/saraedum/sagemath-dev:develop.svg)](https://hub.docker.com/saraedum/sagemath-dev) contains all the build artifacts to rebuild Sage quickly. This version is probably only relevant for Sage developers. Run this image with: ``` docker run -it sagemath/sagemath-dev:develop ``` From 15f3ae761650c4d3c441e3ab2b68f26e290db9db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 26 Feb 2018 01:16:42 +0100 Subject: [PATCH 005/136] fix "build your own" instructions --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index e64d90937a3..aa938e7122a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -36,7 +36,7 @@ There are several flavours of this image. # How to build your own SageMath images -Run `docker build -f docker/Dockerfile --target TARGET .` in the Sage repository with `TARGET` one of `sage`, `sage-jupyter`, or `dev`. +Run `docker build -f docker/Dockerfile --build-arg ARTIFACT_BASE=saraedum/sagemath-dev:develop --target TARGET .` in the Sage repository with `TARGET` one of `sagemath` or `sagemath-dev`. # How these images get updated From 595be9c6431f9e406b5cf14277b8ec0399966226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 26 Feb 2018 09:07:44 +0100 Subject: [PATCH 006/136] POSIX scripts so they run on Alpine --- .ci/build-docker.sh | 3 ++- .ci/protect-secrets.sh | 2 +- .ci/pull-gitlab.sh | 2 +- .ci/push-dockerhub.sh | 2 +- .ci/push-gitlab.sh | 5 +++-- .ci/setup-make-parallelity.sh | 3 ++- .ci/test-cli.sh | 3 ++- .ci/test-dev.sh | 3 ++- .ci/test-jupyter.sh | 3 ++- 9 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh index 9ec55ab149a..d6d082db7fb 100755 --- a/.ci/build-docker.sh +++ b/.ci/build-docker.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # This script gets called from CI to build several flavours of docker images # which contain Sage. @@ -12,6 +12,7 @@ # (at your option) any later version. # http://www.gnu.org/licenses/ # **************************************************************************** + set -ex [[ -z "$DOCKER_TAG" ]] && DOCKER_TAG=none diff --git a/.ci/protect-secrets.sh b/.ci/protect-secrets.sh index a3ae9239988..ee781dddca8 100755 --- a/.ci/protect-secrets.sh +++ b/.ci/protect-secrets.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # This script protects all environment variables that start with "SECRET_". # It puts them in a temporary file. The name of the variable contains the path diff --git a/.ci/pull-gitlab.sh b/.ci/pull-gitlab.sh index 01bb10cdd08..19be8c9ff7c 100755 --- a/.ci/pull-gitlab.sh +++ b/.ci/pull-gitlab.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # This script gets called from CI to pull the Sage docker images that were # built during the "build" phase to pull all the connected docker daemon diff --git a/.ci/push-dockerhub.sh b/.ci/push-dockerhub.sh index ca2c4906eec..5055fc056ac 100755 --- a/.ci/push-dockerhub.sh +++ b/.ci/push-dockerhub.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # This script gets called from CI to push our docker images to # $DOCKER_USER/sagemath* on the Docker Hub. diff --git a/.ci/push-gitlab.sh b/.ci/push-gitlab.sh index 9d98dca7e49..2e491e0a5b6 100755 --- a/.ci/push-gitlab.sh +++ b/.ci/push-gitlab.sh @@ -1,5 +1,4 @@ -#!/bin/bash -set -ex +#!/bin/sh # This script gets called from CI to push our docker images to registry # configured in GitLab. (Mostly, so we can pull them again to push them to the @@ -17,6 +16,8 @@ set -ex # http://www.gnu.org/licenses/ # **************************************************************************** +set -ex + [[ -z "$DOCKER_TAG" ]] && (echo "Can not push untagged build."; exit 0) [[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest diff --git a/.ci/setup-make-parallelity.sh b/.ci/setup-make-parallelity.sh index 12f3a252179..a69bdbc37f6 100644 --- a/.ci/setup-make-parallelity.sh +++ b/.ci/setup-make-parallelity.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # **************************************************************************** # Copyright (C) 2018 Julian Rüth @@ -9,6 +9,7 @@ # (at your option) any later version. # http://www.gnu.org/licenses/ # **************************************************************************** + set -ex # Determine the number of threads that can run simultaneously on this system diff --git a/.ci/test-cli.sh b/.ci/test-cli.sh index a9e8c059ffe..ab58474f74f 100755 --- a/.ci/test-cli.sh +++ b/.ci/test-cli.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # This script gets called from CI to run minimal tests on the sagemath image. @@ -13,6 +13,7 @@ # (at your option) any later version. # http://www.gnu.org/licenses/ # **************************************************************************** + set -exo pipefail echo "Checking that Sage starts and can calculate 1+1…" diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh index 60b59d6a5ca..c8fd0af1be8 100755 --- a/.ci/test-dev.sh +++ b/.ci/test-dev.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # This script gets called from CI to run minimal tests on the sagemath-dev image. # This script expects a single argument, the full name of the docker image to @@ -13,6 +13,7 @@ # (at your option) any later version. # http://www.gnu.org/licenses/ # **************************************************************************** + set -exo pipefail IMAGE="$1" diff --git a/.ci/test-jupyter.sh b/.ci/test-jupyter.sh index 982f06fbeac..b2360b72722 100755 --- a/.ci/test-jupyter.sh +++ b/.ci/test-jupyter.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # This script gets called from CI to run minimal tests on the sagemath-jupyter # image. @@ -14,6 +14,7 @@ # (at your option) any later version. # http://www.gnu.org/licenses/ # **************************************************************************** + set -ex docker run --name sage-jupyter -p 8888:8888 -d "$1" "sage -n jupyter --no-browser --ip='*' --port=8888" From e4010d8f91082ac1b4d9c2cd2955edda30c638f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 26 Feb 2018 09:09:36 +0100 Subject: [PATCH 007/136] Fix branding of CI files and refer to sagemath instead of saraedum (that I had used for testing.) --- .circleci/config.yml | 3 +-- .gitlab-ci.yml | 3 +-- docker/README.md | 10 +++++----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8a2cc7ca71d..504680deb98 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,8 +34,7 @@ jobs: export ARTIFACT_BASE=source-clean ;; *) - # TODO: Change this line to sagemath/sagemath-dev:develop - export ARTIFACT_BASE=saraedum/sagemath-dev:develop + export ARTIFACT_BASE=sagemath/sagemath-dev:develop ;; esac . .ci/build-docker.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d56fb0e28a9..1616e411980 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,8 +18,7 @@ variables: DOCKER_TAG: $CI_COMMIT_REF_SLUG # Builds are very I/O intensive; make sure we have a fast file system. DOCKER_DRIVER: overlay2 - # TODO: Change this line to sagemath/sagemath-dev:develop - ARTIFACT_BASE: saraedum/sagemath-dev:develop + ARTIFACT_BASE: sagemath/sagemath-dev:develop before_script: # GitLab has no mechanism yet to hide secret variables: https://gitlab.com/gitlab-org/gitlab-ce/issues/13784 diff --git a/docker/README.md b/docker/README.md index aa938e7122a..0a48435d9f9 100644 --- a/docker/README.md +++ b/docker/README.md @@ -2,9 +2,9 @@ # Supported tags -* `latest` — the stable `master` branch [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/saraedum/sage/master.svg)](https://github.com/saraedum/sage/commits/master) [![CircleCI branch](https://img.shields.io/circleci/project/github/saraedum/sage/master.svg)](https://circleci.com/gh/saraedum/sage/tree/master) [![GitLab CI](https://gitlab.com/saraedum/sage/badges/master/pipeline.svg)](https://gitlab.com/saraedum/sage/commits/master) +* `latest` — the stable `master` branch [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/sagemath/sage/master.svg)](https://github.com/sagemath/sage/commits/master) [![CircleCI branch](https://img.shields.io/circleci/project/github/sagemath/sage/master.svg)](https://circleci.com/gh/sagemath/sage/tree/master) [![GitLab CI](https://gitlab.com/sagemath/sage/badges/master/pipeline.svg)](https://gitlab.com/sagemath/sage/commits/master) * `x.x.x` — all stable releases of Sage are tagged with their version number. -* `develop` — the current development version of Sage which gets merged into the `master` branch when a new version of Sage is released [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/saraedum/sage/develop.svg)](https://github.com/saraedum/sage/commits/develop) [![CircleCI branch](https://img.shields.io/circleci/project/github/saraedum/sage/master.svg)](https://circleci.com/gh/saraedum/sage/tree/master) [![GitLab CI](https://gitlab.com/saraedum/sage/badges/develop/pipeline.svg)](https://gitlab.com/saraedum/sage/commits/develop) +* `develop` — the current development version of Sage which gets merged into the `master` branch when a new version of Sage is released [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/sagemath/sage/develop.svg)](https://github.com/sagemath/sage/commits/develop) [![CircleCI branch](https://img.shields.io/circleci/project/github/sagemath/sage/master.svg)](https://circleci.com/gh/sagemath/sage/tree/master) [![GitLab CI](https://gitlab.com/sagemath/sage/badges/develop/pipeline.svg)](https://gitlab.com/sagemath/sage/commits/develop) # What is SageMath @@ -17,7 +17,7 @@ SageMath is a free open-source mathematics software system licensed under the GP There are several flavours of this image. -* [`sagemath/sagemath`![image size](https://img.shields.io/microbadger/image-size/saraedum/sagemath:latest.svg)](https://hub.docker.com/saraedum/sagemath) contains everything necessary to run Sage on the command line. Run it with: +* [`sagemath/sagemath`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath:latest.svg)](https://hub.docker.com/sagemath/sagemath) contains everything necessary to run Sage on the command line. Run it with: ``` docker run -it sagemath/sagemath:latest ``` @@ -25,7 +25,7 @@ There are several flavours of this image. ``` docker run -p8888:8888 sagemath/sagemath:latest "sage -n jupyter --no-browser --ip='*' --port=8888" ``` -* [`sagemath/sagemath-dev`![image size](https://img.shields.io/microbadger/image-size/saraedum/sagemath-dev:develop.svg)](https://hub.docker.com/saraedum/sagemath-dev) contains all the build artifacts to rebuild Sage quickly. This version is probably only relevant for Sage developers. Run this image with: +* [`sagemath/sagemath-dev`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath-dev:develop.svg)](https://hub.docker.com/sagemath/sagemath-dev) contains all the build artifacts to rebuild Sage quickly. This version is probably only relevant for Sage developers. Run this image with: ``` docker run -it sagemath/sagemath-dev:develop ``` @@ -36,7 +36,7 @@ There are several flavours of this image. # How to build your own SageMath images -Run `docker build -f docker/Dockerfile --build-arg ARTIFACT_BASE=saraedum/sagemath-dev:develop --target TARGET .` in the Sage repository with `TARGET` one of `sagemath` or `sagemath-dev`. +Run `docker build -f docker/Dockerfile --build-arg ARTIFACT_BASE=sagemath/sagemath-dev:develop --target TARGET .` in the Sage repository with `TARGET` one of `sagemath` or `sagemath-dev`. # How these images get updated From 10d2093e21b637f31f2f6fef0a994dd14f3cd48f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 26 Feb 2018 13:24:35 +0100 Subject: [PATCH 008/136] Comment on timings and possible speedups --- .circleci/config.yml | 12 +++++++++--- .gitlab-ci.yml | 8 ++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 504680deb98..2c6354d9a86 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,8 +1,8 @@ # This file configures automatic builds of Sage on [CircleCI](https://circleci.com). # To make the build time not too excessive, we seed the build cache with -# sagemath/sagemath-dev:develop. When basic SPKGs change, this does not help much -# and the full build might exceed CircleCI's limits for open source projcets (five -# hours on 2 vCPUs as of early 2018.) +# sagemath/sagemath-dev:develop. When basic SPKGs change, this does not help much, +# still the full build does usually not exceed CircleCI's limits for open +# source projcets (five hours on 2 vCPUs as of early 2018.) # You might want to try to build locally or with GitLab CI, see # `.gitlab-ci.yml` for more options. # Note that we do not use workflows because it was not clear whether the docker @@ -11,6 +11,12 @@ # and the frequent loading of docker images probably exceeds the cost of the # actual tests. +# As of early 2018, a run on CircleCI takes usually about 35 minutes. Most of +# the time is spent pulling/pushing from/to Docker Hub and copying files +# locally during the docker build. We could probably save 5 minutes by not +# pushing the sagemath-dev image to Docker Hub. We could save another five +# minutes by not building and testing the sagemath-dev image in these cases. + version: 2 jobs: # As https://circleci.com/docs/2.0/docker-layer-caching/ is a paid feature, diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1616e411980..38241e6ecd0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,14 @@ # place; or set up your favourite cloud service to provide an on-demand # autoscale runner. +# As of early 2018 a run on GitLab CI takes about 50 minutes (with +# build-from-latest.) We could probably save 5 minutes by not pushing the +# sagemath-dev image for branches other than develop and master. We could +# save another 10 minutes by not building/pushing/testing that image in these +# cases. +# Note that most of the time during CI is spent with pulling and pushing of +# docker images and copying files locally as part of the docker build. + image: docker:latest stages: From f1d33ecf2a9283aeef48379490e23b50f8cf5da6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 26 Feb 2018 15:06:40 +0100 Subject: [PATCH 009/136] Explain how docker can be used for local development --- docker/Dockerfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index ca3e40a6af8..2468c13b79f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,6 +24,22 @@ # source directory. # ################################################################################ +################################################################################ +# HOWTO use this file for local builds # +################################################################################ +# If you don't mind downloading a 2GB docker image from time to time, you # +# could use this file for local Sage development. As of early 2018 each build # +# takes about five minutes but you don't have to through the sadly frequent # +# rebuilds of Sage the whole Sage distribution... # +# To build Sage, run this command from your sage/ directory: # +# $ docker build --build-arg MAKE="make -j4" --build-arg ARTIFACT_BASE="sagemath/sagemath-dev:develop" -f docker/Dockerfile --target=make-build --tag sage . +# To run Sage: # +# $ docker run -it sage # +# To run doctests: # +# $ docker run -e "MAKE=make -j4" -it sage sage -tp src/sage # +# Make sure that you always have the latest develop branch merged into your # +# local branch for this to work. # +################################################################################ ARG ARTIFACT_BASE=source-clean From d0b3d7fb1d288885fe42bc7988ed23bc513582a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 26 Feb 2018 18:09:41 +0100 Subject: [PATCH 010/136] To make this truly a micro-release drop all documentation that is not accessible from IPython/Jupyter --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index d2dfc71256d..9f91b71e921 100644 --- a/Makefile +++ b/Makefile @@ -94,6 +94,8 @@ micro_release: bdist-clean sagelib-clean find local/lib/python* -name '*.py' | while IFS= read -r fname; do [ -e "$${fname}c" -o -e "$${fname}o" ] && rm "$$fname"; done || true @echo "Removing sphinx artifacts..." rm -rf local/share/doc/sage/doctrees local/share/doc/sage/inventory + @echo "Removing documentation. Inspection in IPython still works." + rm -rf local/share/doc local/share/*/doc local/share/*/examples local/share/singular/html @echo "Removing unnecessary files & directories - make will not be functional afterwards anymore" @# We need src/sage/ for introspection with "??" @# We need src/sage/bin/ for the scripts that invoke Sage From 069e722679c5f5e612a7214e51d98cf89b07d137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 26 Feb 2018 13:53:34 +0100 Subject: [PATCH 011/136] Allow user to override ARTIFACT_BASE on CircleCI --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2c6354d9a86..ae4fc6dc985 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,7 +40,10 @@ jobs: export ARTIFACT_BASE=source-clean ;; *) - export ARTIFACT_BASE=sagemath/sagemath-dev:develop + # Select sagemath/sagemath-dev:develop as the ARTIFACT_BASE + # unless the user has explicitly selected a differnt one as a + # CircleCI environment variable. + export ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop} ;; esac . .ci/build-docker.sh From cd951b82acb8e9b2245a43cc319c8fe7038a749b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 26 Feb 2018 18:50:20 +0100 Subject: [PATCH 012/136] Allow overriding of ARTIFACT_BASE but still use source-clean on develop build --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 38241e6ecd0..f5848e84567 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,7 +26,7 @@ variables: DOCKER_TAG: $CI_COMMIT_REF_SLUG # Builds are very I/O intensive; make sure we have a fast file system. DOCKER_DRIVER: overlay2 - ARTIFACT_BASE: sagemath/sagemath-dev:develop + DEFAULT_ARTIFACT_BASE: sagemath/sagemath-dev:develop before_script: # GitLab has no mechanism yet to hide secret variables: https://gitlab.com/gitlab-org/gitlab-ce/issues/13784 @@ -45,6 +45,8 @@ services: # much faster than building from a clean checkout of Sage. build-from-latest: &build stage: build + variables: + ARTIFACT_BASE: $DEFAULT_ARTIFACT_BASE artifacts: when: always paths: From ffe50a74936f9f3df94b9f89282a2972276a4afd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 27 Feb 2018 14:23:27 +0100 Subject: [PATCH 013/136] Include gcc in the docker images --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2468c13b79f..262d156be50 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -58,9 +58,10 @@ RUN ln -s "$SAGE_ROOT/sage" /usr/bin/sage RUN ln -s /usr/bin/sage /usr/bin/sagemath # Sage needs the fortran libraries at run-time because we do not build gfortran # with Sage but use the system's. +# We need gcc to allow compilation of cython at run-time from the notebook. # We also install sudo for the sage user, see below. RUN apt-get update -qq \ - && apt-get install -y --no-install-recommends gfortran sudo \ + && apt-get install -y --no-install-recommends gfortran gcc sudo \ && apt-get clean \ && rm -r /var/lib/apt/lists/* # Sage refuses to build as root, so we add a "sage" user that can sudo without a password. From 09b8527363e8a33a34b31e755765f2d172dce0a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 27 Feb 2018 14:25:10 +0100 Subject: [PATCH 014/136] build-time-dependencies are not a requirement for ARTIFACT_BASE usually. --- .ci/build-docker.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh index d6d082db7fb..2a1d3f8f4a8 100755 --- a/.ci/build-docker.sh +++ b/.ci/build-docker.sh @@ -32,10 +32,11 @@ function docker_build { } # We use a multi-stage build /docker/Dockerfile. For the caching to be -# effective, we populate the cache by building the make-all target. (Just -# building the last target is not enough as intermediate targets would be -# discarded from the cache and therefore the caching would fail for our actual -# builds below.) +# effective, we populate the cache by building the build-time-dependencies and +# the make-all target. (Just building the last target is not enough as +# intermediate targets would be discarded from the cache and therefore the +# caching would fail for our actual builds below.) +docker_build --pull --target build-time-dependencies . docker_build --pull --tag make-all --target make-all . # Build the release image without build artifacts. From 0e2aebfdfd12ce97984b87c165309d6137e3d550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 27 Feb 2018 15:09:37 +0100 Subject: [PATCH 015/136] Are 2GB instances big enough for our build? --- .gitlab-ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f5848e84567..3ee2fbbaacd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,8 @@ # that limit in Settings → CI/CD. # You can also provision your own private more powerful runner in the same # place; or set up your favourite cloud service to provide an on-demand -# autoscale runner. +# autoscale runner. (Make sure to tag them as "2gb", otherwise the build step +# is not going to run on them.) # As of early 2018 a run on GitLab CI takes about 50 minutes (with # build-from-latest.) We could probably save 5 minutes by not pushing the @@ -57,6 +58,11 @@ build-from-latest: &build - . .ci/build-docker.sh | tee gitlab-build-docker.log | head -c 3m - . .ci/push-gitlab.sh sagemath-dev - . .ci/push-gitlab.sh sagemath + tags: + # We need enough disk space for the build to work. + # The 2gb instances on gitlab.com have 50GB of disk: + # https://www.digitalocean.com/pricing/ + - 2gb except: - master - develop From 7b1f937c0018d22c95a56a64782af220904ca1e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 27 Feb 2018 15:18:30 +0100 Subject: [PATCH 016/136] Trying the do instances instead the 2gb instances do not seem to have --privileged enabled. *** WARNING: Service runner-9a6801bd-project-5193394-concurrent-0-docker-0 probably didn't start properly. Error response from daemon: Cannot link to a non running container: /runner-9a6801bd-project-5193394-concurrent-0-docker-0 AS /runner-9a6801bd-project-5193394-concurrent-0-docker-0-wait-for-service/runner-9a6801bd-project-5193394-concurrent-0-docker-0 2018-02-27T14:11:49.193457177Z mount: permission denied (are you root?) 2018-02-27T14:11:49.193483054Z Could not mount /sys/kernel/security. 2018-02-27T14:11:49.193487156Z AppArmor detection and --privileged mode might break. 2018-02-27T14:11:49.197419963Z mount: permission denied (are you root?) ********* --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3ee2fbbaacd..ca4ee580f43 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,9 +60,9 @@ build-from-latest: &build - . .ci/push-gitlab.sh sagemath tags: # We need enough disk space for the build to work. - # The 2gb instances on gitlab.com have 50GB of disk: - # https://www.digitalocean.com/pricing/ - - 2gb + # The do(=digitalocean) instances on gitlab.com should be the 4GB instances + # with 80GB of disk: https://www.digitalocean.com/pricing/ + - do except: - master - develop From 30176eee0bbc2264464482501362d86faeb37914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 27 Feb 2018 16:20:26 +0100 Subject: [PATCH 017/136] make doc-build depends heavily on the number of threads --- .ci/test-dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh index c8fd0af1be8..75d1f0a812b 100755 --- a/.ci/test-dev.sh +++ b/.ci/test-dev.sh @@ -33,4 +33,4 @@ function timed_run { timed_run 60 true # runs make build # TODO: Can't we get this faster than that? -timed_run 300 make # runs make build and then make +timed_run $(( 600/$NTHREADS )) make # runs make build and then make From 8ac71a5b2d33104ecc3621985dc0f8f72537b8e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 27 Feb 2018 18:03:44 +0100 Subject: [PATCH 018/136] doc-build sometimes (?) needs pkg-config ``` [dochtml] Error building the documentation. [dochtml] Traceback (most recent call last): [dochtml] File "/home/sage/sage/local/lib/python2.7/runpy.py", line 174, in _run_module_as_main [dochtml] "__main__", fname, loader, pkg_name) [dochtml] File "/home/sage/sage/local/lib/python2.7/runpy.py", line 72, in _run_code [dochtml] exec code in run_globals [dochtml] File "/home/sage/sage/local/lib/python2.7/site-packages/sage_setup/docbuild/__main__.py", line 2, in [dochtml] main() [dochtml] File "/home/sage/sage/local/lib/python2.7/site-packages/sage_setup/docbuild/__init__.py", line 1680, in main [dochtml] builder() [dochtml] File "/home/sage/sage/local/lib/python2.7/site-packages/sage_setup/docbuild/__init__.py", line 314, in _wrapper [dochtml] getattr(get_builder(document), name)(*args, **kwds) [dochtml] File "/home/sage/sage/local/lib/python2.7/site-packages/sage_setup/docbuild/__init__.py", line 505, in _wrapper [dochtml] build_many(build_ref_doc, L) [dochtml] File "/home/sage/sage/local/lib/python2.7/site-packages/sage_setup/docbuild/__init__.py", line 246, in build_many [dochtml] ret = x.get(99999) [dochtml] File "/home/sage/sage/local/lib/python2.7/multiprocessing/pool.py", line 572, in get [dochtml] raise self._value [dochtml] EnvironmentError: pkg-config is not installed ``` --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 262d156be50..5bd9185c0df 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -82,7 +82,7 @@ WORKDIR $HOME FROM run-time-dependencies as build-time-dependencies # Install the build time dependencies & git RUN sudo apt-get update -qq \ - && sudo apt-get install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git \ + && sudo apt-get install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git pkg-config \ && sudo apt-get clean \ && sudo rm -r /var/lib/apt/lists/* From 711409f416fad2d9cad8d9a1ff3ae661976f1391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Wed, 28 Feb 2018 00:21:32 +0100 Subject: [PATCH 019/136] slow CPUs need more time to make --- .ci/test-dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh index 75d1f0a812b..dcbf988abcb 100755 --- a/.ci/test-dev.sh +++ b/.ci/test-dev.sh @@ -33,4 +33,4 @@ function timed_run { timed_run 60 true # runs make build # TODO: Can't we get this faster than that? -timed_run $(( 600/$NTHREADS )) make # runs make build and then make +timed_run $(( 1200/$NTHREADS )) make # runs make build and then make From 29d7d9ab8a6f47f5d9b7883be93c3fa4b0b5aac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Wed, 28 Feb 2018 11:04:57 +0100 Subject: [PATCH 020/136] Update timings I started running these on "do" machines now, not on my own GCE machines that were faster. --- .gitlab-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ca4ee580f43..3a436999536 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,13 +8,16 @@ # autoscale runner. (Make sure to tag them as "2gb", otherwise the build step # is not going to run on them.) -# As of early 2018 a run on GitLab CI takes about 50 minutes (with +# As of early 2018 a run on GitLab CI takes about an hour (with # build-from-latest.) We could probably save 5 minutes by not pushing the # sagemath-dev image for branches other than develop and master. We could # save another 10 minutes by not building/pushing/testing that image in these # cases. # Note that most of the time during CI is spent with pulling and pushing of # docker images and copying files locally as part of the docker build. +# At the moment there is no way of passing the docker images to the following +# stages without explicit pushing/pulling or similar: +# https://gitlab.com/gitlab-org/gitlab-runner/issues/1107 image: docker:latest From cf303a14641fbbeb83fbe5e9375d00010a6dec1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Wed, 28 Feb 2018 15:48:50 +0100 Subject: [PATCH 021/136] Instead of deleting .py files in local hardlink them this also removes some 50MB in other places --- Makefile | 8 ++++++-- docker/Dockerfile | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9f91b71e921..c3b92d101bf 100644 --- a/Makefile +++ b/Makefile @@ -90,8 +90,6 @@ maintainer-clean: distclean bootstrap-clean micro_release: bdist-clean sagelib-clean @echo "Stripping binaries ..." LC_ALL=C find local/lib local/bin -type f -exec strip '{}' ';' 2>&1 | grep -v "File format not recognized" | grep -v "File truncated" || true - @echo "Removing .py files that have a corresponding .pyc or .pyo file as they are not needed when running code with CPython..." - find local/lib/python* -name '*.py' | while IFS= read -r fname; do [ -e "$${fname}c" -o -e "$${fname}o" ] && rm "$$fname"; done || true @echo "Removing sphinx artifacts..." rm -rf local/share/doc/sage/doctrees local/share/doc/sage/inventory @echo "Removing documentation. Inspection in IPython still works." @@ -105,6 +103,12 @@ micro_release: bdist-clean sagelib-clean @# We keep COPYING.txt so we ship a license with this distribution. find . -name . -o -prune ! -name src ! -name sage ! -name local ! -name VERSION.txt ! -name COPYING.txt ! -name build -exec rm -rf \{\} \; cd src && find . -name . -o -prune ! -name sage ! -name bin -exec rm -rf \{\} \; + if command -v rdfind > /dev/null; then \ + @echo "Hardlinking identical files."; \ + rdfind -makeresultsfile false -makehardlinks true .; \ + else \ + @echo "rdfind not installed. Not hardlinking identical files."; \ + fi # Leaves everything that is needed to make the next "make" fast but removes # all the cheap build artifacts that can be quickly regenerated. diff --git a/docker/Dockerfile b/docker/Dockerfile index 5bd9185c0df..7b7d4345324 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -193,6 +193,7 @@ CMD ["bash"] FROM make-all as make-release RUN sage -pip install terminado "notebook>=5" "ipykernel>=4.6" RUN sage -i gap_jupyter singular_jupyter pari_jupyter +RUN apt-get update && apt-get install rdfind RUN make micro_release ################################################################################ From 34522cdb0109202286909a563265613254e32ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Wed, 28 Feb 2018 16:46:01 +0100 Subject: [PATCH 022/136] We do not need pkg-config it's part of Sage-the-distribution --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7b7d4345324..e8f144f82ef 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -82,7 +82,7 @@ WORKDIR $HOME FROM run-time-dependencies as build-time-dependencies # Install the build time dependencies & git RUN sudo apt-get update -qq \ - && sudo apt-get install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git pkg-config \ + && sudo apt-get install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git && sudo apt-get clean \ && sudo rm -r /var/lib/apt/lists/* From 13938095674f6c927ac012ff1c8f974db6a537e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Wed, 28 Feb 2018 17:53:57 +0100 Subject: [PATCH 023/136] fix line terminator --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e8f144f82ef..c92d3f10024 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -82,7 +82,7 @@ WORKDIR $HOME FROM run-time-dependencies as build-time-dependencies # Install the build time dependencies & git RUN sudo apt-get update -qq \ - && sudo apt-get install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git + && sudo apt-get install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git \ && sudo apt-get clean \ && sudo rm -r /var/lib/apt/lists/* From 2fc2da6d443e66771f1da6bef9b122a0328a1aaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Wed, 28 Feb 2018 18:36:00 +0100 Subject: [PATCH 024/136] Add rdfind properly to the dependencies --- docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c92d3f10024..fbab7266d24 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -82,7 +82,7 @@ WORKDIR $HOME FROM run-time-dependencies as build-time-dependencies # Install the build time dependencies & git RUN sudo apt-get update -qq \ - && sudo apt-get install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git \ + && sudo apt-get install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git rdfind \ && sudo apt-get clean \ && sudo rm -r /var/lib/apt/lists/* @@ -193,7 +193,6 @@ CMD ["bash"] FROM make-all as make-release RUN sage -pip install terminado "notebook>=5" "ipykernel>=4.6" RUN sage -i gap_jupyter singular_jupyter pari_jupyter -RUN apt-get update && apt-get install rdfind RUN make micro_release ################################################################################ From 8ce8f41ca3ec0a752628556150af1e670a7b390a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 4 Mar 2018 01:06:27 +0100 Subject: [PATCH 025/136] add more debug info about the host we are running on --- .ci/describe-system.sh | 18 ++++++++++++++++++ .gitlab-ci.yml | 2 ++ 2 files changed, 20 insertions(+) create mode 100644 .ci/describe-system.sh diff --git a/.ci/describe-system.sh b/.ci/describe-system.sh new file mode 100644 index 00000000000..c8cf7a3048c --- /dev/null +++ b/.ci/describe-system.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set +e -x + +uname -a +cat /proc/cpuinfo +cat /proc/meminfo +docker info diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3a436999536..4af063fd93a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,6 +36,8 @@ before_script: # GitLab has no mechanism yet to hide secret variables: https://gitlab.com/gitlab-org/gitlab-ce/issues/13784 # So we roll our own which protects all variables that start with SECRET_ - . .ci/protect-secrets.sh + # Collect debug infos about the system we are running on + - . .ci/describe-system.sh # We use docker-in-docker to build our docker images. It can be faster to # expose your outer docker daemon by mounting /var/run/docker.sock to From e4d5e4e574387bca71e7290ccb5d7c9757425d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 4 Mar 2018 01:35:39 +0100 Subject: [PATCH 026/136] fix doctest errors --- Makefile | 5 +++-- docker/Dockerfile | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c3b92d101bf..72d5f03956b 100644 --- a/Makefile +++ b/Makefile @@ -95,14 +95,15 @@ micro_release: bdist-clean sagelib-clean @echo "Removing documentation. Inspection in IPython still works." rm -rf local/share/doc local/share/*/doc local/share/*/examples local/share/singular/html @echo "Removing unnecessary files & directories - make will not be functional afterwards anymore" - @# We need src/sage/ for introspection with "??" + @# We need src/sage/ and src/doc/common for introspection with "??" @# We need src/sage/bin/ for the scripts that invoke Sage @# We need sage, the script to start Sage @# We need local/, the dependencies and the built Sage library itself. @# We keep VERSION.txt. @# We keep COPYING.txt so we ship a license with this distribution. find . -name . -o -prune ! -name src ! -name sage ! -name local ! -name VERSION.txt ! -name COPYING.txt ! -name build -exec rm -rf \{\} \; - cd src && find . -name . -o -prune ! -name sage ! -name bin -exec rm -rf \{\} \; + cd src && find . -name . -o -prune ! -name sage ! -name bin ! -name doc -exec rm -rf \{\} \; + cd doc && find . -name . -o -prune ! -name common -exec rm -rf \{\} \; if command -v rdfind > /dev/null; then \ @echo "Hardlinking identical files."; \ rdfind -makeresultsfile false -makehardlinks true .; \ diff --git a/docker/Dockerfile b/docker/Dockerfile index fbab7266d24..5da501c23a1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -58,10 +58,10 @@ RUN ln -s "$SAGE_ROOT/sage" /usr/bin/sage RUN ln -s /usr/bin/sage /usr/bin/sagemath # Sage needs the fortran libraries at run-time because we do not build gfortran # with Sage but use the system's. -# We need gcc to allow compilation of cython at run-time from the notebook. +# We need gcc and libstdc++-5-dev to allow compilation of cython at run-time from the notebook. # We also install sudo for the sage user, see below. RUN apt-get update -qq \ - && apt-get install -y --no-install-recommends gfortran gcc sudo \ + && apt-get install -y --no-install-recommends gfortran gcc libstdc++-5-dev sudo \ && apt-get clean \ && rm -r /var/lib/apt/lists/* # Sage refuses to build as root, so we add a "sage" user that can sudo without a password. From bbe279e974ab3cd98cde207ed7cb757d360b2a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 4 Mar 2018 01:46:41 +0100 Subject: [PATCH 027/136] same log info on CircleCI --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ae4fc6dc985..2d38cef4204 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,6 +32,8 @@ jobs: # CircleCI has no mechanism to hide secret variables. # Therefore we roll our own to protect $SECRET_* variables. . .ci/protect-secrets.sh + # Collect debug infos about the system we are running on + . .ci/describe-system.sh # Build docker images export DOCKER_TAG=${CIRCLE_TAG:-$CIRCLE_BRANCH} From 9aaca79bc7e99bc613e5a83016175fa418dcdb4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 4 Mar 2018 02:10:04 +0100 Subject: [PATCH 028/136] sh commands in Makefile do not modify the cwd --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 72d5f03956b..fffaac2f563 100644 --- a/Makefile +++ b/Makefile @@ -103,7 +103,7 @@ micro_release: bdist-clean sagelib-clean @# We keep COPYING.txt so we ship a license with this distribution. find . -name . -o -prune ! -name src ! -name sage ! -name local ! -name VERSION.txt ! -name COPYING.txt ! -name build -exec rm -rf \{\} \; cd src && find . -name . -o -prune ! -name sage ! -name bin ! -name doc -exec rm -rf \{\} \; - cd doc && find . -name . -o -prune ! -name common -exec rm -rf \{\} \; + cd src/doc && find . -name . -o -prune ! -name common -exec rm -rf \{\} \; if command -v rdfind > /dev/null; then \ @echo "Hardlinking identical files."; \ rdfind -makeresultsfile false -makehardlinks true .; \ From fa932819caf9cf56d2bba8dd41bf14ac23308908 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 4 Mar 2018 13:53:37 +0100 Subject: [PATCH 029/136] Fix introspection it is easier to keep the full src/doc around. It's not worth the cherry-picking for just a few MB. --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index fffaac2f563..468302be943 100644 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ micro_release: bdist-clean sagelib-clean @echo "Removing documentation. Inspection in IPython still works." rm -rf local/share/doc local/share/*/doc local/share/*/examples local/share/singular/html @echo "Removing unnecessary files & directories - make will not be functional afterwards anymore" - @# We need src/sage/ and src/doc/common for introspection with "??" + @# We need src/sage/, src/doc/common, src/doc/en/introspect for introspection with "??" @# We need src/sage/bin/ for the scripts that invoke Sage @# We need sage, the script to start Sage @# We need local/, the dependencies and the built Sage library itself. @@ -103,7 +103,6 @@ micro_release: bdist-clean sagelib-clean @# We keep COPYING.txt so we ship a license with this distribution. find . -name . -o -prune ! -name src ! -name sage ! -name local ! -name VERSION.txt ! -name COPYING.txt ! -name build -exec rm -rf \{\} \; cd src && find . -name . -o -prune ! -name sage ! -name bin ! -name doc -exec rm -rf \{\} \; - cd src/doc && find . -name . -o -prune ! -name common -exec rm -rf \{\} \; if command -v rdfind > /dev/null; then \ @echo "Hardlinking identical files."; \ rdfind -makeresultsfile false -makehardlinks true .; \ From 5e387366d4078454ceb6dd144c4974c6b8c66706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 4 Mar 2018 13:58:19 +0100 Subject: [PATCH 030/136] Make tests requiring documentation optional as proposed by roed who seems to be the original author of some of this code. --- src/doc/common/conf.py | 2 +- src/sage/misc/sagedoc.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/doc/common/conf.py b/src/doc/common/conf.py index 97491ad8e1d..bb959ea04de 100644 --- a/src/doc/common/conf.py +++ b/src/doc/common/conf.py @@ -624,7 +624,7 @@ def call_intersphinx(app, env, node, contnode): sage: from sage.env import SAGE_DOC sage: thematic_index = os.path.join(SAGE_DOC, "html", "en", "thematic_tutorials", "index.html") - sage: for line in open(thematic_index).readlines(): + sage: for line in open(thematic_index).readlines(): # requires a built documentation, optional: doc ....: if "padics" in line: ....: sys.stdout.write(line)
  • Introduction to the -adics
    • diff --git a/src/sage/misc/sagedoc.py b/src/sage/misc/sagedoc.py index ebe1015364f..4649c116a43 100644 --- a/src/sage/misc/sagedoc.py +++ b/src/sage/misc/sagedoc.py @@ -20,7 +20,7 @@ sage: from sage.env import SAGE_DOC sage: docfilename = os.path.join(SAGE_DOC, 'html', 'en', 'reference', 'calculus', 'sage', 'symbolic', 'expression.html') - sage: for line in open(docfilename): + sage: for line in open(docfilename): # requires a built documentation, optional: doc ....: if "#sage.symbolic.expression.Expression.numerical_approx" in line: ....: print(line) numerical_approx(prec=None, digits=None, algorithm=None)... @@ -1331,7 +1331,7 @@ class _sage_doc: EXAMPLES:: - sage: browse_sage_doc._open("reference", testing=True)[0] # indirect doctest + sage: browse_sage_doc._open("reference", testing=True)[0] # indirect doctest, requires a built documentation, optional: doc 'http://localhost:8000/doc/live/reference/index.html' sage: browse_sage_doc(identity_matrix, 'rst')[-107:-47] 'Full MatrixSpace of 3 by 3 sparse matrices over Integer Ring' @@ -1493,9 +1493,9 @@ def _open(self, name, testing=False): EXAMPLES:: - sage: browse_sage_doc._open("reference", testing=True)[0] + sage: browse_sage_doc._open("reference", testing=True)[0] # requires a built documentation, optional: doc 'http://localhost:8000/doc/live/reference/index.html' - sage: browse_sage_doc._open("tutorial", testing=True)[1] + sage: browse_sage_doc._open("tutorial", testing=True)[1] # requires a built documentation, optional: doc '.../html/en/tutorial/index.html' """ url = self._base_url + os.path.join(name, "index.html") From fb22aa2984d5d2da6501dd422d0abf27b7ebc2db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 4 Mar 2018 14:02:54 +0100 Subject: [PATCH 031/136] Fix cython compilation errors during doctesting --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5da501c23a1..fc2664a13e9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -58,10 +58,10 @@ RUN ln -s "$SAGE_ROOT/sage" /usr/bin/sage RUN ln -s /usr/bin/sage /usr/bin/sagemath # Sage needs the fortran libraries at run-time because we do not build gfortran # with Sage but use the system's. -# We need gcc and libstdc++-5-dev to allow compilation of cython at run-time from the notebook. +# We need gcc/g++ and libstdc++-5-dev to allow compilation of cython at run-time from the notebook. # We also install sudo for the sage user, see below. RUN apt-get update -qq \ - && apt-get install -y --no-install-recommends gfortran gcc libstdc++-5-dev sudo \ + && apt-get install -y --no-install-recommends gfortran gcc g++ libstdc++-5-dev sudo \ && apt-get clean \ && rm -r /var/lib/apt/lists/* # Sage refuses to build as root, so we add a "sage" user that can sudo without a password. From 6aeebe66ce81c4122ba1bbee24a82d891dc70dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 4 Mar 2018 23:55:37 +0100 Subject: [PATCH 032/136] simplify environment variables --- .ci/build-docker.sh | 7 ------- .ci/pull-gitlab.sh | 3 --- .ci/push-gitlab.sh | 3 --- .ci/update-env.sh | 22 ++++++++++++++++++++++ .gitlab-ci.yml | 19 ++++++++++++++++--- 5 files changed, 38 insertions(+), 16 deletions(-) create mode 100644 .ci/update-env.sh diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh index 2a1d3f8f4a8..6ad77571efa 100755 --- a/.ci/build-docker.sh +++ b/.ci/build-docker.sh @@ -15,11 +15,6 @@ set -ex -[[ -z "$DOCKER_TAG" ]] && DOCKER_TAG=none -[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest - -. .ci/setup-make-parallelity.sh - # We speed up the build process by copying built artifacts from ARTIFACT_BASE # during docker build. See /docker/Dockerfile for more details. ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop} @@ -40,9 +35,7 @@ docker_build --pull --target build-time-dependencies . docker_build --pull --tag make-all --target make-all . # Build the release image without build artifacts. -DOCKER_IMAGE_CLI=${DOCKER_USER:-sagemath}/sagemath:$DOCKER_TAG docker_build --target sagemath --tag "$DOCKER_IMAGE_CLI" . # Build the developer image with the build artifacts intact. # Note: It's important to build the dev image last because it might be tagged as ARTIFACT_BASE. -DOCKER_IMAGE_DEV=${DOCKER_USER:-sagemath}/sagemath-dev:$DOCKER_TAG docker_build --target sagemath-dev --tag "$DOCKER_IMAGE_DEV" . diff --git a/.ci/pull-gitlab.sh b/.ci/pull-gitlab.sh index 19be8c9ff7c..1aa316a625c 100755 --- a/.ci/pull-gitlab.sh +++ b/.ci/pull-gitlab.sh @@ -20,9 +20,6 @@ set -ex -[[ -z "$DOCKER_TAG" ]] && (echo "Can not pull untagged build."; exit 0) -[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest - # Pull the built images from the gitlab registry and give them the original # names they had after built. # Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it diff --git a/.ci/push-gitlab.sh b/.ci/push-gitlab.sh index 2e491e0a5b6..e4ceb30abcd 100755 --- a/.ci/push-gitlab.sh +++ b/.ci/push-gitlab.sh @@ -18,9 +18,6 @@ set -ex -[[ -z "$DOCKER_TAG" ]] && (echo "Can not push untagged build."; exit 0) -[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest - # Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it # automatically from the log output. docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY diff --git a/.ci/update-env.sh b/.ci/update-env.sh new file mode 100644 index 00000000000..9b4223fae3f --- /dev/null +++ b/.ci/update-env.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +# This script gets called from CI to establish the name of the current docker +# tag to build from the name of the branch/tag provided by CI. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +[[ -z "$DOCKER_TAG" ]] && DOCKER_TAG=none +[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest + +DOCKER_IMAGE_CLI=${DOCKER_USER:-sagemath}/sagemath:$DOCKER_TAG +DOCKER_IMAGE_DEV=${DOCKER_USER:-sagemath}/sagemath-dev:$DOCKER_TAG diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4af063fd93a..7df98cc5710 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,6 +38,10 @@ before_script: - . .ci/protect-secrets.sh # Collect debug infos about the system we are running on - . .ci/describe-system.sh + # Set DOCKER_TAG according to the current branch/tag + - . .ci/update-env.sh + # Set MAKE and NTHREADS according to the machine we are running on + - . .ci/setup-make-parallelity.sh # We use docker-in-docker to build our docker images. It can be faster to # expose your outer docker daemon by mounting /var/run/docker.sock to @@ -110,13 +114,22 @@ test-jupyter: # Pushes the built images to Docker Hub if the Settings -> CI/CD -> Secret # variables DOCKER_USER and SECRET_DOCKER_PASS have been set up. -dockerhub: +push-dockerhub: stage: release only: - branches - tags script: - - . .ci/pull-gitlab.sh sagemath-dev - - . .ci/push-dockerhub.sh sagemath-dev - . .ci/pull-gitlab.sh sagemath - . .ci/push-dockerhub.sh sagemath + +# Pushes the built dev images to Docker Hub if the Settings -> CI/CD -> Secret +# variables DOCKER_USER and SECRET_DOCKER_PASS have been set up. +push-dockerhub-dev: + stage: release + only: + - master + - develop + script: + - . .ci/pull-gitlab.sh sagemath-dev + - . .ci/push-dockerhub.sh sagemath-dev From 4818f9bc54c2d71c12097f5b0ff10335718429e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 5 Mar 2018 03:28:07 +0100 Subject: [PATCH 033/136] An attempt at incremental builds --- .gitignore | 5 +++++ docker/Dockerfile | 34 ++++++++++++++++++++++++++++++++-- src/.gitignore | 4 ---- 3 files changed, 37 insertions(+), 6 deletions(-) delete mode 100644 src/.gitignore diff --git a/.gitignore b/.gitignore index 498b92c73b9..c98ce30b813 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,8 @@ Untitled*.ipynb ############################# gitlab-build-docker.log +/src/.cython_version +/src/build +/src/Makefile +/src/bin/sage-env-config + diff --git a/docker/Dockerfile b/docker/Dockerfile index fc2664a13e9..3273509bf7c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -123,6 +123,7 @@ COPY --chown=sage:sage . sage-context # 3) Trash .git again to save some space. ARG SAGE_ROOT=/home/sage/sage WORKDIR $SAGE_ROOT +RUN find . -type f > $HOME/artifact-base.manifest RUN git fetch "$HOME/sage-context" HEAD \ && if [ -e docker/commit ]; then \ git reset `cat docker/commit` \ @@ -174,14 +175,43 @@ RUN make FROM make-all as make-fast-rebuild-clean RUN make fast-rebuild-clean +################################################################################ +# Depending on whether we built from source-clean or not, this image is either # +# identical to make-fast-rebuild-clean or contains a patch which can be used # +# to upgrade ARTIFACT_BASE to make-fast-rebuild-clean. # +################################################################################ +FROM make-fast-rebuild-clean as sagemath-dev-patch +ARG ARTIFACT_BASE=source-clean +ARG SAGE_ROOT=/home/sage/sage +# Build a patch containing of a tar file which contains all the modified files +# and a list of all modified files (added/updated/removed). +RUN if [ x"$ARTIFACT_BASE" != x"source-clean" ]; then \ + mkdir -p $HOME/patch \ + && find . -type f > $HOME/make-fast-rebuild-clean.manifest \ + && cat $HOME/make-fast-rebuild-clean.manifest $HOME/artifact-base.manifest | sort | uniq -u > $HOME/obsolete \ + && find . -type f -cnewer $HOME/artifact-base.manifest > $HOME/modified \ + && tar -cJf $HOME/patch/modified.tar.xz -T $HOME/modified \ + && cat $HOME/obsolete $HOME/modified | xz > $HOME/patch/modified.xz \ + && rm -rf $SAGE_ROOT \ + && mkdir -p $SAGE_ROOT \ + && mv $HOME/patch $SAGE_ROOT/; \ + fi + ################################################################################ # A releasable (relatively small, but still huge) image of this build with all # # the build artifacts intact so developers can make changes and rebuild # # quickly # ################################################################################ -FROM source-clean as sagemath-dev +FROM $ARTIFACT_BASE as sagemath-dev ARG SAGE_ROOT=/home/sage/sage -COPY --chown=sage:sage --from=make-fast-rebuild-clean $SAGE_ROOT $SAGE_ROOT +COPY --chown=sage:sage --from=sagemath-dev-patch $SAGE_ROOT $SAGE_ROOT +ARG ARTIFACT_BASE=source-clean +# Apply the patch from sagemath-dev-patch if we created one. +RUN if [ x"$ARTIFACT_BASE" != x"source-clean" ]; then \ + xzcat patch/modified.xz | xargs rm -rf \ + && tar -Jxf patch/modified.tar.xz \ + && rm -rf patch; \ + fi COPY ./docker/entrypoint-dev.sh /usr/local/bin/sage-entrypoint ENTRYPOINT ["/usr/local/bin/sage-entrypoint"] CMD ["bash"] diff --git a/src/.gitignore b/src/.gitignore deleted file mode 100644 index e85aa7f89e4..00000000000 --- a/src/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/.cython_version -/build -/Makefile -/bin/sage-env-config From c821ba052ef6b3c5bb625a1178663292e32f188b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 5 Mar 2018 13:18:28 +0100 Subject: [PATCH 034/136] Fix build on CircleCI --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2d38cef4204..8b13cbcd6ab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,6 +34,10 @@ jobs: . .ci/protect-secrets.sh # Collect debug infos about the system we are running on . .ci/describe-system.sh + # Set DOCKER_TAG according to the current branch/tag + . .ci/update-env.sh + # Set MAKE and NTHREADS according to the machine we are running on + . .ci/setup-make-parallelity.sh # Build docker images export DOCKER_TAG=${CIRCLE_TAG:-$CIRCLE_BRANCH} From 07602e5e137cdbe499733868f24628c2ee8449d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 5 Mar 2018 14:07:03 +0100 Subject: [PATCH 035/136] fix CirceCI tag selection --- .ci/push-dockerhub.sh | 1 - .circleci/config.yml | 9 ++++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.ci/push-dockerhub.sh b/.ci/push-dockerhub.sh index 5055fc056ac..1252b5f93ab 100755 --- a/.ci/push-dockerhub.sh +++ b/.ci/push-dockerhub.sh @@ -18,7 +18,6 @@ set -ex [[ -z "$DOCKER_TAG" ]] && (echo "Can not push untagged build."; exit 0) -[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest # Push the built images to the docker hub (and fail silently if # DOCKER_USER/SECRET_DOCKER_PASS have not been configured.) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8b13cbcd6ab..dec0d9763cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,13 +34,14 @@ jobs: . .ci/protect-secrets.sh # Collect debug infos about the system we are running on . .ci/describe-system.sh - # Set DOCKER_TAG according to the current branch/tag - . .ci/update-env.sh # Set MAKE and NTHREADS according to the machine we are running on . .ci/setup-make-parallelity.sh - # Build docker images + # Set DOCKER_TAG according to the current branch/tag export DOCKER_TAG=${CIRCLE_TAG:-$CIRCLE_BRANCH} + . .ci/update-env.sh + + # Select ARTIFACT_BASE depending on the current branch/tag case $DOCKER_TAG in "develop" | "latest") export ARTIFACT_BASE=source-clean @@ -52,6 +53,8 @@ jobs: export ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop} ;; esac + + # Build docker images . .ci/build-docker.sh # Test that the images work From 3f088bfc5dce7211fd401566b4cc0907f8618369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 5 Mar 2018 15:31:00 +0100 Subject: [PATCH 036/136] enforce valid docker tags and make them the same on GitLab CI and CircleCI --- .ci/update-env.sh | 6 ++++++ .gitlab-ci.yml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.ci/update-env.sh b/.ci/update-env.sh index 9b4223fae3f..ab7db44f5ce 100644 --- a/.ci/update-env.sh +++ b/.ci/update-env.sh @@ -15,6 +15,12 @@ set -ex +# From the docker documentation: "A tag name must be valid ASCII and may +# contain lowercase and uppercase letters, digits, underscores, periods and +# dashes. A tag name may not start with a period or a dash and may contain a +# maximum of 128 characters." +DOCKER_TAG=`echo $DOCKER_TAG | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128` + [[ -z "$DOCKER_TAG" ]] && DOCKER_TAG=none [[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7df98cc5710..9b2ef782023 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,7 +27,7 @@ stages: - release variables: - DOCKER_TAG: $CI_COMMIT_REF_SLUG + DOCKER_TAG: $CI_COMMIT_REF_NAME # Builds are very I/O intensive; make sure we have a fast file system. DOCKER_DRIVER: overlay2 DEFAULT_ARTIFACT_BASE: sagemath/sagemath-dev:develop From 96a20eaa0cef2ce4a26867b86d16f1b813ec2f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 5 Mar 2018 19:07:56 +0100 Subject: [PATCH 037/136] explain why "make" is not as fast as it could be --- .ci/test-dev.sh | 6 +++++- src/sage_setup/docbuild/ext/sage_autodoc.py | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh index dcbf988abcb..4a3bac3ffbd 100755 --- a/.ci/test-dev.sh +++ b/.ci/test-dev.sh @@ -32,5 +32,9 @@ function timed_run { } timed_run 60 true # runs make build -# TODO: Can't we get this faster than that? +# Building the documentation is quite slow at the moment: +# Currently, this detects some dependencies as changed that have not changed. +# The parser in Sphinx fails to parse some .py files and adds the (more +# recently modified) .pyc files as dependencies instead. (Have a look the +# changeset that introduced this comment for more details.) timed_run $(( 1200/$NTHREADS )) make # runs make build and then make diff --git a/src/sage_setup/docbuild/ext/sage_autodoc.py b/src/sage_setup/docbuild/ext/sage_autodoc.py index 7590e229186..d5c45ba54a4 100644 --- a/src/sage_setup/docbuild/ext/sage_autodoc.py +++ b/src/sage_setup/docbuild/ext/sage_autodoc.py @@ -938,7 +938,12 @@ def generate(self, more_content=None, real_modname=None, self.analyzer.find_attr_docs() except PycodeError as err: self.env.app.debug('[autodoc] module analyzer failed: %s', err) - # no source file -- e.g. for builtin and C modules + # A few things could have happened here: + # * there is no source file -- e.g. for builtin and C modules + # * the source file contains syntax that Sphinx can not parse, + # e.g., "print(1, end=' ')"; see + # https://github.com/sphinx-doc/sphinx/issues/1641, + # fixed in Sphinx 1.7. self.analyzer = None # at least add the module.__file__ as a dependency if hasattr(self.module, '__file__') and self.module.__file__: From 3f9a0193369fe11dbe194a302e444b825a1de4a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 5 Mar 2018 19:47:40 +0100 Subject: [PATCH 038/136] minor documentation improvements --- docker/Dockerfile | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3273509bf7c..d59b924b91d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,7 +5,7 @@ # created docker images, see the README.md please. # # # # This Dockerfile builds sagemath (for end-users) and sagemath-dev (for # -# developers) it consists of lots of intermediate targets, mostly to shrink # +# developers.) It consists of lots of intermediate targets, mostly to shrink # # the resulting images but also to make this hopefully easier to maintain. # # The aims of this Dockerfile are: # # (1) Make it build in reasonable time. # @@ -29,8 +29,8 @@ ################################################################################ # If you don't mind downloading a 2GB docker image from time to time, you # # could use this file for local Sage development. As of early 2018 each build # -# takes about five minutes but you don't have to through the sadly frequent # -# rebuilds of Sage the whole Sage distribution... # +# takes about five minutes but you don't have to go through the sadly frequent # +# rebuilds the whole Sage distribution... # # To build Sage, run this command from your sage/ directory: # # $ docker build --build-arg MAKE="make -j4" --build-arg ARTIFACT_BASE="sagemath/sagemath-dev:develop" -f docker/Dockerfile --target=make-build --tag sage . # To run Sage: # @@ -123,6 +123,8 @@ COPY --chown=sage:sage . sage-context # 3) Trash .git again to save some space. ARG SAGE_ROOT=/home/sage/sage WORKDIR $SAGE_ROOT +# We create a list of all files present in the artifact-base (with a timestamp +# of now) so we can find out later which files were added/changed/removed. RUN find . -type f > $HOME/artifact-base.manifest RUN git fetch "$HOME/sage-context" HEAD \ && if [ -e docker/commit ]; then \ @@ -152,12 +154,14 @@ RUN patch -p1 < "$HOME"/sage-context.patch # Image with a built sage but without sage's documentation. # ################################################################################ FROM source-from-context as make-build -# Make sure that the results runs on most CPUs. +# Make sure that the result runs on most CPUs. ENV SAGE_FAT_BINARY yes # Just to be sure Sage doesn't try to build its own GCC (even though # it shouldn't with a recent GCC package from the system and with gfortran) ENV SAGE_INSTALL_GCC no -# Make everything in the build use multiple cores (this causes trouble for some packages outside Sage but it still seems to be they way Sage is doing this.) +# Make everything in the build use multiple cores (setting this variable causes +# trouble for some packages outside Sage but it still seems to be they way Sage +# is doing this.) ARG MAKE="make -j2" ENV MAKE $MAKE RUN make build @@ -177,7 +181,7 @@ RUN make fast-rebuild-clean ################################################################################ # Depending on whether we built from source-clean or not, this image is either # -# identical to make-fast-rebuild-clean or contains a patch which can be used # +# identical to make-fast-rebuild-clean or contains a "patch" which can be used # # to upgrade ARTIFACT_BASE to make-fast-rebuild-clean. # ################################################################################ FROM make-fast-rebuild-clean as sagemath-dev-patch From 2a57dd007a662286ec6de6c88770c67f72ed102b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 5 Mar 2018 21:04:45 +0100 Subject: [PATCH 039/136] update timings --- .circleci/config.yml | 7 +++---- .gitlab-ci.yml | 12 +++++------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dec0d9763cd..ba52319eb1f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,11 +11,10 @@ # and the frequent loading of docker images probably exceeds the cost of the # actual tests. -# As of early 2018, a run on CircleCI takes usually about 35 minutes. Most of +# As of early 2018, a run on CircleCI takes usually about 25 minutes. Most of # the time is spent pulling/pushing from/to Docker Hub and copying files -# locally during the docker build. We could probably save 5 minutes by not -# pushing the sagemath-dev image to Docker Hub. We could save another five -# minutes by not building and testing the sagemath-dev image in these cases. +# locally during the docker build. We could probably save five minutes by not +# building and testing the sagemath-dev image for most branches. version: 2 jobs: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9b2ef782023..39902bb09af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,15 +8,13 @@ # autoscale runner. (Make sure to tag them as "2gb", otherwise the build step # is not going to run on them.) -# As of early 2018 a run on GitLab CI takes about an hour (with -# build-from-latest.) We could probably save 5 minutes by not pushing the -# sagemath-dev image for branches other than develop and master. We could -# save another 10 minutes by not building/pushing/testing that image in these -# cases. +# As of early 2018 a run on GitLab CI takes about 35 minutes (with +# build-from-latest.) We could probably save 10 minutes by not +# building/pushing/testing that image for branches other than master/develop. # Note that most of the time during CI is spent with pulling and pushing of # docker images and copying files locally as part of the docker build. -# At the moment there is no way of passing the docker images to the following -# stages without explicit pushing/pulling or similar: +# At the moment there is no reliable way of passing the docker images to the +# following stages without explicit pushing/pulling or similar: # https://gitlab.com/gitlab-org/gitlab-runner/issues/1107 image: docker:latest From 28445f7107762401af831bf3e018fbbfdf6e6fd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 8 Mar 2018 09:00:02 +0100 Subject: [PATCH 040/136] Fix comments in Makefile I still don't really understand what bdist is good for, so rather keep it uncommented. --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 468302be943..f0e01715f8c 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,6 @@ misc-clean: rm -f build/make/Makefile build/make/Makefile-auto rm -f .BUILDSTART -# TODO: What is a "bdist"? A binary distribution? bdist-clean: clean $(MAKE) misc-clean @@ -87,6 +86,8 @@ bootstrap-clean: maintainer-clean: distclean bootstrap-clean rm -rf upstream +# Remove everything that is not necessary to run Sage and pass all its +# doctests. micro_release: bdist-clean sagelib-clean @echo "Stripping binaries ..." LC_ALL=C find local/lib local/bin -type f -exec strip '{}' ';' 2>&1 | grep -v "File format not recognized" | grep -v "File truncated" || true @@ -95,7 +96,9 @@ micro_release: bdist-clean sagelib-clean @echo "Removing documentation. Inspection in IPython still works." rm -rf local/share/doc local/share/*/doc local/share/*/examples local/share/singular/html @echo "Removing unnecessary files & directories - make will not be functional afterwards anymore" - @# We need src/sage/, src/doc/common, src/doc/en/introspect for introspection with "??" + @# We need src/doc/common, src/doc/en/introspect for introspection with "??" + @# We keep src/sage for some doctests that it expect it to be there and + @# also because it does not add any weight with rdfind below. @# We need src/sage/bin/ for the scripts that invoke Sage @# We need sage, the script to start Sage @# We need local/, the dependencies and the built Sage library itself. From 2767adef1af044a105332a84e4f1370d94b50d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 24 Mar 2018 00:39:42 +0200 Subject: [PATCH 041/136] Source only the scripts that set environment variables and run everything else with sh --- .circleci/config.yml | 12 ++++++------ .gitlab-ci.yml | 18 +++++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba52319eb1f..3cdf69f7b27 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,7 +32,7 @@ jobs: # Therefore we roll our own to protect $SECRET_* variables. . .ci/protect-secrets.sh # Collect debug infos about the system we are running on - . .ci/describe-system.sh + sh .ci/describe-system.sh # Set MAKE and NTHREADS according to the machine we are running on . .ci/setup-make-parallelity.sh @@ -57,10 +57,10 @@ jobs: . .ci/build-docker.sh # Test that the images work - . .ci/test-dev.sh $DOCKER_IMAGE_DEV - . .ci/test-cli.sh $DOCKER_IMAGE_CLI - . .ci/test-jupyter.sh $DOCKER_IMAGE_CLI localhost + sh .ci/test-dev.sh $DOCKER_IMAGE_DEV + sh .ci/test-cli.sh $DOCKER_IMAGE_CLI + sh .ci/test-jupyter.sh $DOCKER_IMAGE_CLI localhost # Push docker images to dockerhub if a dockerhub user has been configured - . .ci/push-dockerhub.sh sagemath-dev - . .ci/push-dockerhub.sh sagemath + sh .ci/push-dockerhub.sh sagemath-dev + sh .ci/push-dockerhub.sh sagemath diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 39902bb09af..241fed309a2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,7 @@ before_script: # So we roll our own which protects all variables that start with SECRET_ - . .ci/protect-secrets.sh # Collect debug infos about the system we are running on - - . .ci/describe-system.sh + - sh .ci/describe-system.sh # Set DOCKER_TAG according to the current branch/tag - . .ci/update-env.sh # Set MAKE and NTHREADS according to the machine we are running on @@ -62,9 +62,9 @@ build-from-latest: &build expire_in: 1 month script: # The output of the build can get larger than gitlab.com's limit; only print the first 3MB. - - . .ci/build-docker.sh | tee gitlab-build-docker.log | head -c 3m - - . .ci/push-gitlab.sh sagemath-dev - - . .ci/push-gitlab.sh sagemath + - sh .ci/build-docker.sh | tee gitlab-build-docker.log | head -c 3m + - sh .ci/push-gitlab.sh sagemath-dev + - sh .ci/push-gitlab.sh sagemath tags: # We need enough disk space for the build to work. # The do(=digitalocean) instances on gitlab.com should be the 4GB instances @@ -90,13 +90,13 @@ test-dev: stage: test script: - . .ci/pull-gitlab.sh sagemath-dev - - . .ci/test-dev.sh "$DOCKER_IMAGE" + - sh .ci/test-dev.sh "$DOCKER_IMAGE" test-cli: stage: test script: - . .ci/pull-gitlab.sh sagemath - - . .ci/test-cli.sh "$DOCKER_IMAGE" + - sh .ci/test-cli.sh "$DOCKER_IMAGE" test-jupyter: stage: test @@ -108,7 +108,7 @@ test-jupyter: - apk update - apk add wget - . .ci/pull-gitlab.sh sagemath - - . .ci/test-jupyter.sh "$DOCKER_IMAGE" docker + - sh .ci/test-jupyter.sh "$DOCKER_IMAGE" docker # Pushes the built images to Docker Hub if the Settings -> CI/CD -> Secret # variables DOCKER_USER and SECRET_DOCKER_PASS have been set up. @@ -119,7 +119,7 @@ push-dockerhub: - tags script: - . .ci/pull-gitlab.sh sagemath - - . .ci/push-dockerhub.sh sagemath + - sh .ci/push-dockerhub.sh sagemath # Pushes the built dev images to Docker Hub if the Settings -> CI/CD -> Secret # variables DOCKER_USER and SECRET_DOCKER_PASS have been set up. @@ -130,4 +130,4 @@ push-dockerhub-dev: - develop script: - . .ci/pull-gitlab.sh sagemath-dev - - . .ci/push-dockerhub.sh sagemath-dev + - sh .ci/push-dockerhub.sh sagemath-dev From 834f66bd79845e4844c3f6a2b7d0b130d9ef2c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 24 Mar 2018 00:47:21 +0200 Subject: [PATCH 042/136] We need long lines in the README because dockerhub is not very smart --- docker/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/README.md b/docker/README.md index 0a48435d9f9..1d2ce27bb7f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -50,3 +50,5 @@ Have a look at `.circleci/` and `.gitlab-ci.yml` if you want to setup either con # License The whole Sage software distribution is licensed under the General Public License, version 3. More details can be found in our [COPYING.txt](https://github.com/sagemath/sage/blob/master/COPYING.txt) + +[//]: # (Please don't break long lines in this files as dockerhub then gets the formatting of this file wrong.) From 4ee63131bad067e53df4369dacb30461a58c97fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 24 Mar 2018 00:51:58 +0200 Subject: [PATCH 043/136] make usage easier to understand --- .ci/test-cli.sh | 2 +- .ci/test-dev.sh | 2 ++ .ci/test-jupyter.sh | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/test-cli.sh b/.ci/test-cli.sh index ab58474f74f..9048a9877cf 100755 --- a/.ci/test-cli.sh +++ b/.ci/test-cli.sh @@ -2,7 +2,7 @@ # This script gets called from CI to run minimal tests on the sagemath image. -# Usage: ./test-cli.sh sage-cli-image +# Usage: ./test-cli.sh IMAGE-NAME # **************************************************************************** # Copyright (C) 2018 Julian Rüth diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh index 4a3bac3ffbd..4ac3a4e7655 100755 --- a/.ci/test-dev.sh +++ b/.ci/test-dev.sh @@ -4,6 +4,8 @@ # This script expects a single argument, the full name of the docker image to # test. +# Usage: ./test-dev.sh IMAGE-NAME + # **************************************************************************** # Copyright (C) 2018 Julian Rüth # diff --git a/.ci/test-jupyter.sh b/.ci/test-jupyter.sh index b2360b72722..edcc0bcd44a 100755 --- a/.ci/test-jupyter.sh +++ b/.ci/test-jupyter.sh @@ -3,7 +3,7 @@ # This script gets called from CI to run minimal tests on the sagemath-jupyter # image. -# Usage: ./test-jupyter.sh sage-jupyter-image [host] +# Usage: ./test-jupyter.sh IMAGE-NAME [HOST] # **************************************************************************** # Copyright (C) 2018 Julian Rüth From 0892d35ba0c1a6dee2df8faa74cd1155e9964c4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Wed, 28 Mar 2018 19:17:22 +0300 Subject: [PATCH 044/136] Explain the pros and cons of docker:dind --- .gitlab-ci.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 241fed309a2..616e62dd383 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,10 +41,21 @@ before_script: # Set MAKE and NTHREADS according to the machine we are running on - . .ci/setup-make-parallelity.sh -# We use docker-in-docker to build our docker images. It can be faster to -# expose your outer docker daemon by mounting /var/run/docker.sock to -# /var/run/docker.sock and setting DOCKER_HOST in Settings -> CI/CD -> Secret -# variable to unix:///var/run/docker.sock +# We use docker-in-docker to build our docker images, i.e., we run a +# docker:dind "service" container and link to it from the container running the +# actual scripts below. +# Our scripts automatically connect to this service (unless you override it by +# setting DOCKER_HOST.) For example, each RUN statement in the Dockerfile +# spawns a docker container inside the docker:dind container to perform the RUN +# command there. +# It can be faster to expose your outer docker daemon by mounting +# /var/run/docker.sock to /var/run/docker.sock and setting DOCKER_HOST in +# Settings -> CI/CD -> Secret variable to unix:///var/run/docker.sock. (The +# speedup is mostly due to sharing layers of intermediate images.) However, +# this is only possible if you provision your own runners. Shared gitlab +# runners, do not bind mount /var/run/docker.sock. Also, docker:dind provides +# better isolation. If you expect many builds to run simultaneously on a host, +# conflicting tags can cause issues with a mounted DOCKER_HOST. services: - docker:dind From 290bec4ccc73a977fdff9d7539092fa2dc211d83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Wed, 28 Mar 2018 19:43:00 +0300 Subject: [PATCH 045/136] clarify comment --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d59b924b91d..cc157c7f409 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -114,7 +114,8 @@ COPY --chown=sage:sage . sage-context # make sure that we only change the mtimes of a minimal number of files. # 1) Restore the git checkout ARTIFACT_BASE was built from, recorded in # docker/commit. (Or go directly to FETCH_HEAD if there is no history to -# restore.) +# restore, i.e., set ARTIFACT_BASE=source-clean if you want to build from +# scratch.) # 2) Merge in FETCH_HEAD but only if it is a fast-forward, i.e., if it is an # ancestor of the commit restored in 1. If we would not do that we would get # a new commit hash in docker/commit that is not known outside of this build From c3eeb65d4adb6f77ecdb76f5c016c70973c935ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Wed, 28 Mar 2018 19:47:38 +0300 Subject: [PATCH 046/136] do not source scripts without side effects --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3cdf69f7b27..f2c3cc16f3a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,7 +54,7 @@ jobs: esac # Build docker images - . .ci/build-docker.sh + sh .ci/build-docker.sh # Test that the images work sh .ci/test-dev.sh $DOCKER_IMAGE_DEV From bf56bcae1c28545666ed7ed489c8bcba3e3422ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Wed, 28 Mar 2018 20:13:05 +0300 Subject: [PATCH 047/136] howto provision your own runners --- .gitlab-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 616e62dd383..048dae29e99 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,9 +4,11 @@ # much and the full build might the set time limit in GitLab. You can increase # that limit in Settings → CI/CD. # You can also provision your own private more powerful runner in the same -# place; or set up your favourite cloud service to provide an on-demand -# autoscale runner. (Make sure to tag them as "2gb", otherwise the build step -# is not going to run on them.) +# place +# https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-in-docker-executor; +# or set up your favourite cloud service to provide an on-demand autoscale +# runner. (Make sure to tag them as "do", otherwise the build step is not going +# to run on them.) # As of early 2018 a run on GitLab CI takes about 35 minutes (with # build-from-latest.) We could probably save 10 minutes by not From fa4599a6c6849e5a2ac2a1f738eed437843454fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 3 Apr 2018 00:42:06 +0300 Subject: [PATCH 048/136] Rename commit to .commit so it does no show up in ls as we usually don't care about it. --- docker/.gitignore | 2 +- docker/Dockerfile | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docker/.gitignore b/docker/.gitignore index a38152cb5ee..579da245b87 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -1,2 +1,2 @@ # Stores the commit that was used to create a sagemath-dev image -commit +.commit diff --git a/docker/Dockerfile b/docker/Dockerfile index cc157c7f409..1c28a0572c9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -113,13 +113,13 @@ COPY --chown=sage:sage . sage-context # This is a bit complicated because our local .git/ is empty and we want to # make sure that we only change the mtimes of a minimal number of files. # 1) Restore the git checkout ARTIFACT_BASE was built from, recorded in -# docker/commit. (Or go directly to FETCH_HEAD if there is no history to +# docker/.commit. (Or go directly to FETCH_HEAD if there is no history to # restore, i.e., set ARTIFACT_BASE=source-clean if you want to build from # scratch.) # 2) Merge in FETCH_HEAD but only if it is a fast-forward, i.e., if it is an # ancestor of the commit restored in 1. If we would not do that we would get -# a new commit hash in docker/commit that is not known outside of this build -# run. Since docker/commit was in the history of FETCH_HEAD this should +# a new commit hash in docker/.commit that is not known outside of this build +# run. Since docker/.commit was in the history of FETCH_HEAD this should # automatically be a fast-forward. # 3) Trash .git again to save some space. ARG SAGE_ROOT=/home/sage/sage @@ -128,16 +128,16 @@ WORKDIR $SAGE_ROOT # of now) so we can find out later which files were added/changed/removed. RUN find . -type f > $HOME/artifact-base.manifest RUN git fetch "$HOME/sage-context" HEAD \ - && if [ -e docker/commit ]; then \ - git reset `cat docker/commit` \ - || ( echo "Could not find commit `cat docker/commit` in your local Git history. Please merge in the latest built develop branch to fix this: git fetch trac && git merge `cat docker/commit`." && exit 1 ) \ + && if [ -e docker/.commit ]; then \ + git reset `cat docker/.commit` \ + || ( echo "Could not find commit `cat docker/.commit` in your local Git history. Please merge in the latest built develop branch to fix this: git fetch trac && git merge `cat docker/.commit`." && exit 1 ) \ else \ - echo "You are building from $ARTIFACT_BASE which has no docker/commit file. That's a bug unless you are building from source-clean or something similar." \ + echo "You are building from $ARTIFACT_BASE which has no docker/.commit file. That's a bug unless you are building from source-clean or something similar." \ && git reset FETCH_HEAD \ && git checkout -f FETCH_HEAD; \ fi \ && git merge --ff-only FETCH_HEAD \ - && git log -1 --format=%H > docker/commit \ + && git log -1 --format=%H > docker/.commit \ && rm -rf .git # Copy over all the untracked/staged/unstaged changes from sage-context. This # is relevant for non-CI invocations of this Dockerfile. From 71d9ce7391c6550ba0653bd5421a3745e28417d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 3 Apr 2018 01:09:00 +0300 Subject: [PATCH 049/136] export variables in sourced scripts otherwise they do not get passed on the the subshells (before we sourced everything, so exported variables were essentially the same to non-exported ones.) --- .ci/pull-gitlab.sh | 2 +- .ci/setup-make-parallelity.sh | 3 +++ .ci/update-env.sh | 10 +++++----- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.ci/pull-gitlab.sh b/.ci/pull-gitlab.sh index 1aa316a625c..c8235e64068 100755 --- a/.ci/pull-gitlab.sh +++ b/.ci/pull-gitlab.sh @@ -26,5 +26,5 @@ set -ex # automatically from the log output. docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY docker pull $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG -DOCKER_IMAGE="${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG" +export DOCKER_IMAGE="${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG" docker tag $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG $DOCKER_IMAGE diff --git a/.ci/setup-make-parallelity.sh b/.ci/setup-make-parallelity.sh index a69bdbc37f6..761578a5b07 100644 --- a/.ci/setup-make-parallelity.sh +++ b/.ci/setup-make-parallelity.sh @@ -20,8 +20,11 @@ set -ex # too high can lead to RAM being insufficient, so it's best to set this # variable manually in your CI configuration. [[ -z "$NTHREADS" ]] && NTHREADS=`grep -E '^processor' /proc/cpuinfo | wc -l` || true +export NTHREADS="$NTHREADS" # Set -j and -l for make (though -l is probably stripped by Sage) [[ -z "$MAKEOPTS" ]] && MAKEOPTS="-j $NTHREADS -l $((NTHREADS-1)).8" || true +export MAKEOPTS="$MAKEOPTS" # Not all parts of Sage seem to honor MAKEOPTS, so the current way of telling # the system which concurrency to use, seems to be setting $MAKE. [[ -z "$MAKE" ]] && MAKE="make $MAKEOPTS" || true +export MAKE="$MAKE" diff --git a/.ci/update-env.sh b/.ci/update-env.sh index ab7db44f5ce..ee6a2110a56 100644 --- a/.ci/update-env.sh +++ b/.ci/update-env.sh @@ -19,10 +19,10 @@ set -ex # contain lowercase and uppercase letters, digits, underscores, periods and # dashes. A tag name may not start with a period or a dash and may contain a # maximum of 128 characters." -DOCKER_TAG=`echo $DOCKER_TAG | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128` +export DOCKER_TAG=`echo $DOCKER_TAG | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128` -[[ -z "$DOCKER_TAG" ]] && DOCKER_TAG=none -[[ "$DOCKER_TAG" = "master" ]] && DOCKER_TAG=latest +[[ -z "$DOCKER_TAG" ]] && export DOCKER_TAG=none +[[ "$DOCKER_TAG" = "master" ]] && export DOCKER_TAG=latest -DOCKER_IMAGE_CLI=${DOCKER_USER:-sagemath}/sagemath:$DOCKER_TAG -DOCKER_IMAGE_DEV=${DOCKER_USER:-sagemath}/sagemath-dev:$DOCKER_TAG +export DOCKER_IMAGE_CLI=${DOCKER_USER:-sagemath}/sagemath:$DOCKER_TAG +export DOCKER_IMAGE_DEV=${DOCKER_USER:-sagemath}/sagemath-dev:$DOCKER_TAG From 2493bb3e942ccfb4fa3fa0b11439062125d54445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 3 Apr 2018 01:15:39 +0300 Subject: [PATCH 050/136] Fixup for fa4599a6c6849e5a2ac2a1f738eed437843454fa forgot to rename commit to .commit in the docker README --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 1d2ce27bb7f..27fce3d8e18 100644 --- a/docker/README.md +++ b/docker/README.md @@ -31,7 +31,7 @@ There are several flavours of this image. ``` This triggers a rebuild and drops you in a shell afterwards. Note that the git repository has been emptied to save space. If you want to use git, fetch from your git repository with `git fetch trac` and go to the commit that was used to create this image with ``` - git reset $(cat docker/commit) + git reset $(cat docker/.commit) ``` # How to build your own SageMath images From b3c6d14e3c61efa0868e02bd481340aa408905c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 3 Apr 2018 01:19:44 +0300 Subject: [PATCH 051/136] Make docker-build.sh POSIX compliant --- .ci/build-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh index 6ad77571efa..ee42df6830f 100755 --- a/.ci/build-docker.sh +++ b/.ci/build-docker.sh @@ -22,7 +22,7 @@ ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop} # Seed our cache with $ARTIFACT_BASE if it exists docker pull $ARTIFACT_BASE || true -function docker_build { +docker_build() { time docker build -f docker/Dockerfile --build-arg "MAKE=${MAKE}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ } From 183ddd40b2ebd9f53d242a112325bda119b8298c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 3 Apr 2018 01:42:11 +0300 Subject: [PATCH 052/136] Make jupyter invocation easier to remember --- .ci/test-jupyter.sh | 2 +- docker/README.md | 2 +- docker/entrypoint.sh | 9 ++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.ci/test-jupyter.sh b/.ci/test-jupyter.sh index edcc0bcd44a..a569013c99a 100755 --- a/.ci/test-jupyter.sh +++ b/.ci/test-jupyter.sh @@ -17,7 +17,7 @@ set -ex -docker run --name sage-jupyter -p 8888:8888 -d "$1" "sage -n jupyter --no-browser --ip='*' --port=8888" +docker run --name sage-jupyter -p 8888:8888 -d "$1" jupyter echo "Checking that the Jupyter notebook is running…" sleep 10 # giving the server some time to start docker logs sage-jupyter diff --git a/docker/README.md b/docker/README.md index 27fce3d8e18..314cdc91d8a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -23,7 +23,7 @@ There are several flavours of this image. ``` You can start a graphical [Jupyter Notebook](https://jupyter.org) at http://localhost:8888 instead. To use the notebook, follow the instructions printed when you run: ``` - docker run -p8888:8888 sagemath/sagemath:latest "sage -n jupyter --no-browser --ip='*' --port=8888" + docker run -p8888:8888 sagemath/sagemath:latest jupyter ``` * [`sagemath/sagemath-dev`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath-dev:develop.svg)](https://hub.docker.com/sagemath/sagemath-dev) contains all the build artifacts to rebuild Sage quickly. This version is probably only relevant for Sage developers. Run this image with: ``` diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 692b5c83390..52a67592834 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,2 +1,9 @@ #!/bin/bash -exec sage -sh -c "$*" +if [ x"$1" = jupyter ]; then + # If "jupyter" is given as a first argument, we start a jupyter notebook + # with reasonable default parameters for running it inside a container. + shift + exec sage -n jupyter --no-browser --ip='*' --port=8888 "$@" +else + exec sage -sh -c "$*" +fi From 59952f343884585bccaba2e2cb27e8393d3df985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 3 Apr 2018 19:50:21 +0300 Subject: [PATCH 053/136] Drop pipefail as it is not supported by CircleCI's "sh" and we actually do not need it here. --- .ci/test-cli.sh | 2 +- .ci/test-dev.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/test-cli.sh b/.ci/test-cli.sh index 9048a9877cf..fb0a4cb101a 100755 --- a/.ci/test-cli.sh +++ b/.ci/test-cli.sh @@ -14,7 +14,7 @@ # http://www.gnu.org/licenses/ # **************************************************************************** -set -exo pipefail +set -ex echo "Checking that Sage starts and can calculate 1+1…" # Calculate 1+1 (remove startup messages and leading & trailing whitespace) diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh index 4ac3a4e7655..daf2c49794a 100755 --- a/.ci/test-dev.sh +++ b/.ci/test-dev.sh @@ -16,7 +16,7 @@ # http://www.gnu.org/licenses/ # **************************************************************************** -set -exo pipefail +set -ex IMAGE="$1" From a727ec548c321df24a6ac9bd789f700a8d31e2b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 3 Apr 2018 19:54:30 +0300 Subject: [PATCH 054/136] Compilation sometimes takes a bit longer and the error message was a bit confusing --- .ci/test-dev.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh index daf2c49794a..a5d852e7350 100755 --- a/.ci/test-dev.sh +++ b/.ci/test-dev.sh @@ -29,11 +29,14 @@ function timed_run { docker run -e MAKE="$MAKE" "$IMAGE" "$2" END=`date +%s` TOTAL=$((END-START)) - echo "Checking that \"$2\" was fast…" + echo "Checking whether running \"$2\" was fast…" [[ $TOTAL -lt $1 ]] } -timed_run 60 true # runs make build +# Most setup should be done in well under 120 seconds but some CI machines tend +# to be really slow so we better give them some space here. Better miss a +# regression at first than create a lot of noise. +timed_run 120 true # runs make build # Building the documentation is quite slow at the moment: # Currently, this detects some dependencies as changed that have not changed. # The parser in Sphinx fails to parse some .py files and adds the (more From 2e74da9f1e408d8f569151198b4c739b2208749f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 5 Apr 2018 00:01:06 +0300 Subject: [PATCH 055/136] POSIX complaint tests so we can run test-dev.sh with a POSIX shell --- .ci/setup-make-parallelity.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/setup-make-parallelity.sh b/.ci/setup-make-parallelity.sh index 761578a5b07..655bd86b762 100644 --- a/.ci/setup-make-parallelity.sh +++ b/.ci/setup-make-parallelity.sh @@ -19,12 +19,12 @@ set -ex # provision fewer vCPUs than shown in /proc/cpuinfo. Also, setting this value # too high can lead to RAM being insufficient, so it's best to set this # variable manually in your CI configuration. -[[ -z "$NTHREADS" ]] && NTHREADS=`grep -E '^processor' /proc/cpuinfo | wc -l` || true +[ -z "$NTHREADS" ] && NTHREADS=`grep -E '^processor' /proc/cpuinfo | wc -l` || true export NTHREADS="$NTHREADS" # Set -j and -l for make (though -l is probably stripped by Sage) -[[ -z "$MAKEOPTS" ]] && MAKEOPTS="-j $NTHREADS -l $((NTHREADS-1)).8" || true +[ -z "$MAKEOPTS" ] && MAKEOPTS="-j $NTHREADS -l $((NTHREADS-1)).8" || true export MAKEOPTS="$MAKEOPTS" # Not all parts of Sage seem to honor MAKEOPTS, so the current way of telling # the system which concurrency to use, seems to be setting $MAKE. -[[ -z "$MAKE" ]] && MAKE="make $MAKEOPTS" || true +[ -z "$MAKE" ] && MAKE="make $MAKEOPTS" || true export MAKE="$MAKE" From 410c5465a79f7204d1fd0e98d10a5f8879e627e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 5 Apr 2018 00:07:25 +0300 Subject: [PATCH 056/136] use sage-jupyter instead of jupyter jupyter is actually a command in the PATH so we should not hide it --- .ci/test-jupyter.sh | 2 +- docker/entrypoint.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/test-jupyter.sh b/.ci/test-jupyter.sh index a569013c99a..9df157b763b 100755 --- a/.ci/test-jupyter.sh +++ b/.ci/test-jupyter.sh @@ -17,7 +17,7 @@ set -ex -docker run --name sage-jupyter -p 8888:8888 -d "$1" jupyter +docker run --name sage-jupyter -p 8888:8888 -d "$1" sage-jupyter echo "Checking that the Jupyter notebook is running…" sleep 10 # giving the server some time to start docker logs sage-jupyter diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 52a67592834..bec95735ec8 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [ x"$1" = jupyter ]; then - # If "jupyter" is given as a first argument, we start a jupyter notebook +if [ x"$1" = sage-jupyter ]; then + # If "sage-jupyter" is given as a first argument, we start a jupyter notebook # with reasonable default parameters for running it inside a container. shift exec sage -n jupyter --no-browser --ip='*' --port=8888 "$@" From 93c98e0f1c4724a451b04c8a7230661b3b464911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 7 Apr 2018 00:43:12 +0300 Subject: [PATCH 057/136] POSIX compliant function call in sh --- .ci/test-dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh index a5d852e7350..1c9519d778d 100755 --- a/.ci/test-dev.sh +++ b/.ci/test-dev.sh @@ -24,7 +24,7 @@ IMAGE="$1" # Usage: timed_run limit args # Runs $IMAGE with args and check that it terminates with a zero exit code in at most limit seconds. -function timed_run { +timed_run() { START=`date +%s` docker run -e MAKE="$MAKE" "$IMAGE" "$2" END=`date +%s` From 474b181411500f59c21d1d58223a4edf510b4fc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 7 Apr 2018 01:13:34 +0300 Subject: [PATCH 058/136] fix typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 048dae29e99..9540316dceb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -115,7 +115,7 @@ test-jupyter: stage: test script: # Force usage of docker-in-docker (and don't start docker on a bind-mounted - # /var/run/docker.sock set through a proivate GitLab CI variable) so that + # /var/run/docker.sock set through a private GitLab CI variable) so that # the -p flag to docker run works as expected. - export DOCKER_HOST='tcp://docker:2375' - apk update From 4e714a2cbc26e623979912126fd1c6d2b897ff0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 7 Apr 2018 01:18:09 +0300 Subject: [PATCH 059/136] Add checks for binder binder needs jupyter to be on the path. --- .ci/test-cli.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.ci/test-cli.sh b/.ci/test-cli.sh index fb0a4cb101a..dfcedfc5ec0 100755 --- a/.ci/test-cli.sh +++ b/.ci/test-cli.sh @@ -20,3 +20,11 @@ echo "Checking that Sage starts and can calculate 1+1…" # Calculate 1+1 (remove startup messages and leading & trailing whitespace) TWO=`docker run "$1" sage -c "'print(1+1)'" | tail -1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'` [[ "x$TWO" = "x2" ]] + +echo "Checking that some binaries that should be distributed with Sage are on the PATH…" +# We could also run minimal tests on these but we don't yet. +# Check that Singular and GAP are present +docker run "$1" which Singular +docker run "$1" which gap +# Check that jupyter is present (for binder) +docker run "$1" which jupyter From 84d541c289760839b99c5a471607058f3e1cb518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 7 Apr 2018 01:24:39 +0300 Subject: [PATCH 060/136] fix README the command is now called sage-jupyter not just jupyter as that one already exists --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 314cdc91d8a..0028e8b9798 100644 --- a/docker/README.md +++ b/docker/README.md @@ -23,7 +23,7 @@ There are several flavours of this image. ``` You can start a graphical [Jupyter Notebook](https://jupyter.org) at http://localhost:8888 instead. To use the notebook, follow the instructions printed when you run: ``` - docker run -p8888:8888 sagemath/sagemath:latest jupyter + docker run -p8888:8888 sagemath/sagemath:latest sage-jupyter ``` * [`sagemath/sagemath-dev`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath-dev:develop.svg)](https://hub.docker.com/sagemath/sagemath-dev) contains all the build artifacts to rebuild Sage quickly. This version is probably only relevant for Sage developers. Run this image with: ``` From 73b963b3f1013ed90aff9ca9776d515a19748b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 12 Apr 2018 19:49:18 +0200 Subject: [PATCH 061/136] POSIX compliant test, [ instead of [[. --- .ci/test-cli.sh | 2 +- .ci/test-dev.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/test-cli.sh b/.ci/test-cli.sh index dfcedfc5ec0..0ad6a8f3d0a 100755 --- a/.ci/test-cli.sh +++ b/.ci/test-cli.sh @@ -19,7 +19,7 @@ set -ex echo "Checking that Sage starts and can calculate 1+1…" # Calculate 1+1 (remove startup messages and leading & trailing whitespace) TWO=`docker run "$1" sage -c "'print(1+1)'" | tail -1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'` -[[ "x$TWO" = "x2" ]] +[ "x$TWO" = "x2" ] echo "Checking that some binaries that should be distributed with Sage are on the PATH…" # We could also run minimal tests on these but we don't yet. diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh index 1c9519d778d..3dde76bbbb0 100755 --- a/.ci/test-dev.sh +++ b/.ci/test-dev.sh @@ -30,7 +30,7 @@ timed_run() { END=`date +%s` TOTAL=$((END-START)) echo "Checking whether running \"$2\" was fast…" - [[ $TOTAL -lt $1 ]] + [ "$TOTAL" -lt "$1" ] } # Most setup should be done in well under 120 seconds but some CI machines tend From 0d5dd3d9f854ad6053ec527caacd737fe0e78023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 12 Apr 2018 19:57:25 +0200 Subject: [PATCH 062/136] Fix sage-jupyter command in entrypoint --- docker/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index bec95735ec8..bc841382eaf 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,5 +1,5 @@ #!/bin/bash -if [ x"$1" = sage-jupyter ]; then +if [ x"$1" = x"sage-jupyter" ]; then # If "sage-jupyter" is given as a first argument, we start a jupyter notebook # with reasonable default parameters for running it inside a container. shift From 7cb69d7b5a8bfa13e78a3c81ec7f864385505c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 12 Apr 2018 20:48:36 +0200 Subject: [PATCH 063/136] Mention that timings may vary --- .gitlab-ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9540316dceb..46d209060cc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,8 @@ # This file configures automatic builds of Sage on [GitLab](https://gitlab.com). # To make the build time not too excessive, we seed the build cache with # sagemath/sagemath-dev:develop. When basic SPKGs changed, this does not help -# much and the full build might the set time limit in GitLab. You can increase -# that limit in Settings → CI/CD. +# much and the full build might exceed the set time limit in GitLab. You can +# increase that limit in Settings → CI/CD. # You can also provision your own private more powerful runner in the same # place # https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-in-docker-executor; @@ -18,6 +18,10 @@ # At the moment there is no reliable way of passing the docker images to the # following stages without explicit pushing/pulling or similar: # https://gitlab.com/gitlab-org/gitlab-runner/issues/1107 +# The timings mentioned above are typical values. The shared runners provided +# on gitlab.com are sometimes by a factor of two slower. It's unclear what is +# the source of this slowdown. There is nothing different in the logs, so it's +# probably just an overcommittment of virtual machines on hardware. image: docker:latest From 6aade395c5bb9cf92118c317942a7398f22ad08b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 13 Apr 2018 21:12:28 +0200 Subject: [PATCH 064/136] Silence docker pull errors are still printed (to stderr.) Note that as of early 2018 there is no -q switch for docker pull yet. --- .ci/build-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh index ee42df6830f..afd898e6fb2 100755 --- a/.ci/build-docker.sh +++ b/.ci/build-docker.sh @@ -20,7 +20,7 @@ set -ex ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop} # Seed our cache with $ARTIFACT_BASE if it exists -docker pull $ARTIFACT_BASE || true +docker pull "$ARTIFACT_BASE" > /dev/null || true docker_build() { time docker build -f docker/Dockerfile --build-arg "MAKE=${MAKE}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ From 93a7ba6f9f07e6076bed9002a58b62a1f3c3beec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 13 Apr 2018 21:13:28 +0200 Subject: [PATCH 065/136] Silence apt-get we only care about errors here --- docker/Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1c28a0572c9..2e3383a1ebe 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -60,9 +60,9 @@ RUN ln -s /usr/bin/sage /usr/bin/sagemath # with Sage but use the system's. # We need gcc/g++ and libstdc++-5-dev to allow compilation of cython at run-time from the notebook. # We also install sudo for the sage user, see below. -RUN apt-get update -qq \ - && apt-get install -y --no-install-recommends gfortran gcc g++ libstdc++-5-dev sudo \ - && apt-get clean \ +RUN apt-get -qq update \ + && apt-get -qq install -y --no-install-recommends gfortran gcc g++ libstdc++-5-dev sudo \ + && apt-get -qq clean \ && rm -r /var/lib/apt/lists/* # Sage refuses to build as root, so we add a "sage" user that can sudo without a password. # We also want this user at runtime as some commands in sage know about the user that was used during build. @@ -81,9 +81,9 @@ WORKDIR $HOME ################################################################################ FROM run-time-dependencies as build-time-dependencies # Install the build time dependencies & git -RUN sudo apt-get update -qq \ - && sudo apt-get install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git rdfind \ - && sudo apt-get clean \ +RUN sudo apt-get -qq update \ + && sudo apt-get -qq install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git rdfind \ + && sudo apt-get -qq clean \ && sudo rm -r /var/lib/apt/lists/* ################################################################################ From 158af15bf74e2604c6d3606867f0b6143bfe58ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 13 Apr 2018 21:13:46 +0200 Subject: [PATCH 066/136] Silence setup.py The "cythonizing" messages, compiler/cython warnings are still shown. This mostly removes the messages that about byte-compilations of python files and copying of python files to site-packages. --- src/Makefile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile.in b/src/Makefile.in index 81a28b3758e..0233601cbbc 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -37,7 +37,7 @@ sage: SAGE_DOC_SRC=/doesnotexist \ SAGE_BUILD_DIR=/doesnotexist \ SAGE_PKGS=$(abs_top_srcdir)/build/pkgs \ - && sage-python23 -u setup.py --no-user-cfg build install + && sage-python23 -u setup.py --quiet --no-user-cfg build install if [ "$$UNAME" = "CYGWIN" ]; then \ sage-rebase.sh "$$SAGE_LOCAL" 2>/dev/null; \ fi From 3c78351fdcbfe93c2c353d0aef23bc2e976d7e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 13 Apr 2018 21:14:45 +0200 Subject: [PATCH 067/136] Ignore much more useless chatter in sphinx build this currently makes CI builds exceed their output limit. I don't think we care about these "progress bars". Errors and warnings are still printed (and definitely detected.) --- src/sage_setup/docbuild/sphinxbuild.py | 52 ++++++++++++++++++-------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/src/sage_setup/docbuild/sphinxbuild.py b/src/sage_setup/docbuild/sphinxbuild.py index 40aac600b36..a5b65baca0c 100644 --- a/src/sage_setup/docbuild/sphinxbuild.py +++ b/src/sage_setup/docbuild/sphinxbuild.py @@ -1,7 +1,7 @@ r""" This is Sage's version of the sphinx-build script -We redirect stdout to our own logger, and remove some unwanted chatter. +We redirect stdout and stderr to our own logger, and remove some unwanted chatter. """ # **************************************************************************** # Copyright (C) 2013-2014 Volker Braun @@ -31,8 +31,8 @@ def term_width_line(text): class SageSphinxLogger(object): r""" - This implements the file object interface to serve as sys.stdout - replacement. + This implements the file object interface to serve as + ``sys.stdout``/``sys.stderr`` replacement. """ ansi_color = re.compile(r'\x1b\[[0-9;]*m') ansi_reset = re.compile(r'\x1b\[39;49;00m') @@ -59,25 +59,41 @@ def __init__(self, stream, prefix): self._error = None def _init_chatter(self): - # useless_chatter: regular expressions to be filtered from - # Sphinx output. - self.useless_chatter = ( + # We drop any messages from the output that match these regular + # expressions. These just bloat the output and do not contain any + # information that we care about. + self._useless_chatter = ( re.compile('^$'), re.compile('^Running Sphinx v'), re.compile('^loading intersphinx inventory from '), + re.compile('^loading pickled environment... done'), + re.compile('^loading cross citations... done \([0-9]* citations\).'), re.compile('^Compiling a sub-document'), re.compile('^updating environment: 0 added, 0 changed, 0 removed'), re.compile('^looking for now-outdated files... none found'), re.compile('^building \[.*\]: targets for 0 source files that are out of date'), - re.compile('^loading pickled environment... done'), - re.compile('^loading cross citations... done \([0-9]* citations\).'), + re.compile('^building \[.*\]: targets for 0 po files that are out of date'), + re.compile('^building \[.*\]: targets for 0 mo files that are out of date'), + re.copmile('^pickling environment... done'), + re.copmile('^dumping object inventory... done'), + # We still have "Build finished." + re.compile('^build succeeded.'), + re.copmile('^checking consistency... done'), + re.copmile('^preparing documents... done'), + re.compile('^writing output... \[.*\] '), + re.compile('^reading sources... \[.*\] '), + re.compile('language "hu" not supported'), + ) + + # We fail whenever a line starts with "WARNING:", however, we ignore + # these warnings, as they are not relevant. + self._ignored_warnings = ( re.compile('WARNING: favicon file \'favicon.ico\' does not exist'), re.compile('WARNING: html_static_path entry .* does not exist'), re.compile('WARNING: while setting up extension'), re.compile('WARNING: Any IDs not assiend for figure node'), re.compile('WARNING: .* is not referenced'), re.compile('WARNING: Build finished'), - re.compile('language "hu" not supported'), ) # replacements: pairs of regular expressions and their replacements, @@ -88,7 +104,7 @@ def _init_chatter(self): if 'inventory' in sys.argv: # When building the inventory, ignore warnings about missing # citations and the search index. - self.useless_chatter += ( + self._useless_chatter += ( re.compile('WARNING: citation not found:'), re.compile('WARNING: search index couldn\'t be loaded, but not all documents will be built: the index will be incomplete.') ) @@ -119,7 +135,7 @@ def _filter_out(self, line): # swallow non-errors after an error occurred return True line = re.sub(self.ansi_color, '', line) - for regex in self.useless_chatter: + for regex in self._useless_chatter: if regex.search(line) is not None: return True return False @@ -143,10 +159,14 @@ def _check_errors(self, line): """ if self._error: return # we already have found an error - for regex in self._error_patterns: - if regex.search(line) is not None: - self._error = line - return + for error in self._error_patterns: + if error.search(line) is not None: + for ignored in self._ignored_warnings: + if ignored.search(line) is not None: + break + else: + self._error = line + return def _log_line(self, line): r""" @@ -182,6 +202,7 @@ def _log_line(self, line): [#25160 ] Exception: artificial exception """ + self._check_errors(line) if self._filter_out(line): return for (old, new) in self.replacements: @@ -191,7 +212,6 @@ def _log_line(self, line): line = self.ansi_color.sub('', line) self._stream.write(line) self._stream.flush() - self._check_errors(line) def raise_errors(self): r""" From 7d5f3898f3bff423efe3c3e97997de6af5ccd573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 13 Apr 2018 21:44:01 +0200 Subject: [PATCH 068/136] Smarter truncation of output this feels a bit overengineered but just using head makes the output stop without an additional message to the user. --- .ci/head-tail.sh | 45 +++++++++++++++++++++++++++++++++++++++++++++ .gitlab-ci.yml | 2 +- 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100755 .ci/head-tail.sh diff --git a/.ci/head-tail.sh b/.ci/head-tail.sh new file mode 100755 index 00000000000..8ecf9f879f0 --- /dev/null +++ b/.ci/head-tail.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +OFFSET=80 +# This script reads from stdin and prints to stdout as long as a the output +# does not exceed a certain number of bytes. When reading an EOF it prints the +# last $OFFSET lines if they have not been printed normally already. +# This script expects one argument, the number of bytes. + +# Heavily inspired by a simlar strategy in make, https://stackoverflow.com/a/44849696/812379. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +stdbuf -i0 -o0 -e0 awk -v limit=$1 -v firstMissingNR=-1 -v offset=$OFFSET -v bytes=0 \ +'{ + if (bytes < limit) { + # this probably gets multi-byte characters wrong, but that probably does + # not matter for our purposes. (We add 1 for a UNIX newline.) + bytes += length($0) + 1; + print; + } else { + if (firstMissingNR == -1){ + print "[…output truncated…]"; + firstMissingNR = NR; + } + a[NR] = $0; + delete a[NR-offset]; + printf "." > "/dev/stderr" + } +} +END { + if (firstMissingNR != -1) { + print "" > "/dev/stderr"; + for(i = NR-offset+1 > firstMissingNR ? NR-offset-1 : firstMissingNR; i<=NR ; i++){ print a[i]; } + } +} +' + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 46d209060cc..386a0e2ba81 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,7 +79,7 @@ build-from-latest: &build expire_in: 1 month script: # The output of the build can get larger than gitlab.com's limit; only print the first 3MB. - - sh .ci/build-docker.sh | tee gitlab-build-docker.log | head -c 3m + - sh .ci/build-docker.sh | tee gitlab-build-docker.log | .ci/head-tail.sh 3145728 - sh .ci/push-gitlab.sh sagemath-dev - sh .ci/push-gitlab.sh sagemath tags: From 321658ed55349977819da30e83cbb1850aa05536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 13 Apr 2018 21:51:47 +0200 Subject: [PATCH 069/136] head-tail.sh needs stdbuf from coreutils --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 386a0e2ba81..414810105c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -78,7 +78,8 @@ build-from-latest: &build - gitlab-build-docker.log expire_in: 1 month script: - # The output of the build can get larger than gitlab.com's limit; only print the first 3MB. + - apk update && apk add coreutils + # The output of the build can get larger than gitlab.com's limit; only print the first 3MB (and the last 80 lines.) - sh .ci/build-docker.sh | tee gitlab-build-docker.log | .ci/head-tail.sh 3145728 - sh .ci/push-gitlab.sh sagemath-dev - sh .ci/push-gitlab.sh sagemath From 7735a2c0b60f71731c550544acb295300aee52bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 13 Apr 2018 22:05:32 +0200 Subject: [PATCH 070/136] POSIX compliant push-dockerhub.sh script --- .ci/push-dockerhub.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/push-dockerhub.sh b/.ci/push-dockerhub.sh index 1252b5f93ab..c2c46062ffa 100755 --- a/.ci/push-dockerhub.sh +++ b/.ci/push-dockerhub.sh @@ -17,11 +17,11 @@ set -ex -[[ -z "$DOCKER_TAG" ]] && (echo "Can not push untagged build."; exit 0) +[ -z "$DOCKER_TAG" ] && (echo "Can not push untagged build."; exit 0) # Push the built images to the docker hub (and fail silently if # DOCKER_USER/SECRET_DOCKER_PASS have not been configured.) -if [[ -z "$DOCKER_USER" || -z "$SECRET_DOCKER_PASS" ]]; then +if [ -z "$DOCKER_USER" -o -z "$SECRET_DOCKER_PASS" ]; then echo "DOCKER_USER/SECRET_DOCKER_PASS variables have not been configured in your Continuous Integration setup. Not pushing built images to Docker Hub." else cat "$SECRET_DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin From 3112236a66637c4ad27f69294c35277aca904968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 13 Apr 2018 22:07:15 +0200 Subject: [PATCH 071/136] Try to get rid of the explicit sh invocation --- .ci/describe-system.sh | 0 .ci/setup-make-parallelity.sh | 0 .ci/update-env.sh | 0 .circleci/config.yml | 14 +++++++------- .gitlab-ci.yml | 8 ++++---- 5 files changed, 11 insertions(+), 11 deletions(-) mode change 100644 => 100755 .ci/describe-system.sh mode change 100644 => 100755 .ci/setup-make-parallelity.sh mode change 100644 => 100755 .ci/update-env.sh diff --git a/.ci/describe-system.sh b/.ci/describe-system.sh old mode 100644 new mode 100755 diff --git a/.ci/setup-make-parallelity.sh b/.ci/setup-make-parallelity.sh old mode 100644 new mode 100755 diff --git a/.ci/update-env.sh b/.ci/update-env.sh old mode 100644 new mode 100755 diff --git a/.circleci/config.yml b/.circleci/config.yml index f2c3cc16f3a..0404cc6b8bb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,7 +32,7 @@ jobs: # Therefore we roll our own to protect $SECRET_* variables. . .ci/protect-secrets.sh # Collect debug infos about the system we are running on - sh .ci/describe-system.sh + .ci/describe-system.sh # Set MAKE and NTHREADS according to the machine we are running on . .ci/setup-make-parallelity.sh @@ -54,13 +54,13 @@ jobs: esac # Build docker images - sh .ci/build-docker.sh + .ci/build-docker.sh # Test that the images work - sh .ci/test-dev.sh $DOCKER_IMAGE_DEV - sh .ci/test-cli.sh $DOCKER_IMAGE_CLI - sh .ci/test-jupyter.sh $DOCKER_IMAGE_CLI localhost + .ci/test-dev.sh $DOCKER_IMAGE_DEV + .ci/test-cli.sh $DOCKER_IMAGE_CLI + .ci/test-jupyter.sh $DOCKER_IMAGE_CLI localhost # Push docker images to dockerhub if a dockerhub user has been configured - sh .ci/push-dockerhub.sh sagemath-dev - sh .ci/push-dockerhub.sh sagemath + .ci/push-dockerhub.sh sagemath-dev + .ci/push-dockerhub.sh sagemath diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 414810105c3..8b9760b08c8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,7 @@ before_script: # So we roll our own which protects all variables that start with SECRET_ - . .ci/protect-secrets.sh # Collect debug infos about the system we are running on - - sh .ci/describe-system.sh + - .ci/describe-system.sh # Set DOCKER_TAG according to the current branch/tag - . .ci/update-env.sh # Set MAKE and NTHREADS according to the machine we are running on @@ -80,9 +80,9 @@ build-from-latest: &build script: - apk update && apk add coreutils # The output of the build can get larger than gitlab.com's limit; only print the first 3MB (and the last 80 lines.) - - sh .ci/build-docker.sh | tee gitlab-build-docker.log | .ci/head-tail.sh 3145728 - - sh .ci/push-gitlab.sh sagemath-dev - - sh .ci/push-gitlab.sh sagemath + - .ci/build-docker.sh | tee gitlab-build-docker.log | .ci/head-tail.sh 3145728 + - .ci/push-gitlab.sh sagemath-dev + - .ci/push-gitlab.sh sagemath tags: # We need enough disk space for the build to work. # The do(=digitalocean) instances on gitlab.com should be the 4GB instances From 05328a37ecc0705658327db56ba56c36a485f9f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 13 Apr 2018 22:08:31 +0200 Subject: [PATCH 072/136] Fix typo in 3c78351fdcbfe93c2c353d0aef23bc2e976d7e17 --- src/sage_setup/docbuild/sphinxbuild.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sage_setup/docbuild/sphinxbuild.py b/src/sage_setup/docbuild/sphinxbuild.py index a5b65baca0c..6bd5913d8ad 100644 --- a/src/sage_setup/docbuild/sphinxbuild.py +++ b/src/sage_setup/docbuild/sphinxbuild.py @@ -74,12 +74,12 @@ def _init_chatter(self): re.compile('^building \[.*\]: targets for 0 source files that are out of date'), re.compile('^building \[.*\]: targets for 0 po files that are out of date'), re.compile('^building \[.*\]: targets for 0 mo files that are out of date'), - re.copmile('^pickling environment... done'), - re.copmile('^dumping object inventory... done'), + re.compile('^pickling environment... done'), + re.compile('^dumping object inventory... done'), # We still have "Build finished." re.compile('^build succeeded.'), - re.copmile('^checking consistency... done'), - re.copmile('^preparing documents... done'), + re.compile('^checking consistency... done'), + re.compile('^preparing documents... done'), re.compile('^writing output... \[.*\] '), re.compile('^reading sources... \[.*\] '), re.compile('language "hu" not supported'), From 77eeea94b2b31328f3543a167e3440a22f19ab7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 13 Apr 2018 22:37:08 +0200 Subject: [PATCH 073/136] Do not print any of these ignored warnings we might want to add more ignored warnings that we actually print later --- src/sage_setup/docbuild/sphinxbuild.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sage_setup/docbuild/sphinxbuild.py b/src/sage_setup/docbuild/sphinxbuild.py index 6bd5913d8ad..ae0b6531dca 100644 --- a/src/sage_setup/docbuild/sphinxbuild.py +++ b/src/sage_setup/docbuild/sphinxbuild.py @@ -95,6 +95,7 @@ def _init_chatter(self): re.compile('WARNING: .* is not referenced'), re.compile('WARNING: Build finished'), ) + self._useless_chatter += self._ignored_warnings # replacements: pairs of regular expressions and their replacements, # to be applied to Sphinx output. @@ -104,10 +105,12 @@ def _init_chatter(self): if 'inventory' in sys.argv: # When building the inventory, ignore warnings about missing # citations and the search index. - self._useless_chatter += ( + ignored = ( re.compile('WARNING: citation not found:'), re.compile('WARNING: search index couldn\'t be loaded, but not all documents will be built: the index will be incomplete.') ) + self._ignored_warnings += ignored + self._useless_chatter += ignored # Regular expressions indicating a problem with docbuilding. Raise an # exception if any of these occur. From aa13c840c2be80d905c243ff630a95d4516c3bcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 14 Apr 2018 00:31:08 +0200 Subject: [PATCH 074/136] Ignore more chatter in docbuild --- src/sage_setup/docbuild/sphinxbuild.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/sage_setup/docbuild/sphinxbuild.py b/src/sage_setup/docbuild/sphinxbuild.py index 45fbaae315b..efca70476d2 100644 --- a/src/sage_setup/docbuild/sphinxbuild.py +++ b/src/sage_setup/docbuild/sphinxbuild.py @@ -80,9 +80,20 @@ def _init_chatter(self): re.compile('^build succeeded.'), re.compile('^checking consistency... done'), re.compile('^preparing documents... done'), + re.compile('^copying extra files... done'), + re.compile('^writing additional pages... search'), + re.compile('^Writing js search indexes...writing additional pages... .*'), + re.compile('^generating indices... .*'), + re.compile('^dumping search index in .* ... done'), + re.compile('^linking _static directory'), + re.compile('^copying static files... done'), + re.compile('^copying extra files... done'), + re.compile('^loading transations \[.*\]... done'), re.compile('^writing output... \[.*\] '), + re.compile('^copying images... \[.*\] '), re.compile('^reading sources... \[.*\] '), re.compile('language "hu" not supported'), + re.compile('^WARNING:$'), ) # We fail whenever a line starts with "WARNING:", however, we ignore From 8884d9b2b809e9ddcf0c899508dbb84051e1c6b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 14 Apr 2018 01:19:30 +0200 Subject: [PATCH 075/136] Print available disk space we need quite a bit of space for the CI to work --- .ci/describe-system.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/describe-system.sh b/.ci/describe-system.sh index c8cf7a3048c..b33a84e2428 100755 --- a/.ci/describe-system.sh +++ b/.ci/describe-system.sh @@ -13,6 +13,7 @@ set +e -x uname -a +df -h cat /proc/cpuinfo cat /proc/meminfo docker info From 090c25af45187150e501188715fb5943bcc91ff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 14 Apr 2018 01:20:36 +0200 Subject: [PATCH 076/136] Trying to find out which of the shared runners actually provide enough power --- .gitlab-ci.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8b9760b08c8..19b1d42939a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,8 +7,7 @@ # place # https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-in-docker-executor; # or set up your favourite cloud service to provide an on-demand autoscale -# runner. (Make sure to tag them as "do", otherwise the build step is not going -# to run on them.) +# runner. # As of early 2018 a run on GitLab CI takes about 35 minutes (with # build-from-latest.) We could probably save 10 minutes by not @@ -83,11 +82,6 @@ build-from-latest: &build - .ci/build-docker.sh | tee gitlab-build-docker.log | .ci/head-tail.sh 3145728 - .ci/push-gitlab.sh sagemath-dev - .ci/push-gitlab.sh sagemath - tags: - # We need enough disk space for the build to work. - # The do(=digitalocean) instances on gitlab.com should be the 4GB instances - # with 80GB of disk: https://www.digitalocean.com/pricing/ - - do except: - master - develop From a788043e2c0182c0cba45554ecd8b7ee5cb8c8dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 14 Apr 2018 01:41:16 +0200 Subject: [PATCH 077/136] Make NTHREADS respect the available RAM sadly we need 2GB per thread for the docbuild to go through --- .ci/setup-make-parallelity.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/setup-make-parallelity.sh b/.ci/setup-make-parallelity.sh index 655bd86b762..18cfcf5675f 100755 --- a/.ci/setup-make-parallelity.sh +++ b/.ci/setup-make-parallelity.sh @@ -17,9 +17,11 @@ set -ex # Note that this value is incorrect for some CI providers (notably CircleCI: # https://circleci.com/docs/2.0/configuration-reference/#resource_class) which # provision fewer vCPUs than shown in /proc/cpuinfo. Also, setting this value -# too high can lead to RAM being insufficient, so it's best to set this +# too high can lead to RAM being insufficient, so it's best to set the NTHREADS # variable manually in your CI configuration. -[ -z "$NTHREADS" ] && NTHREADS=`grep -E '^processor' /proc/cpuinfo | wc -l` || true +RAMTHREADS=$(( `grep MemTotal /proc/meminfo | awk '{ print $2 }'` / 1024 / 1024 / 2 )) +CPUTHREADS=`grep -E '^processor' /proc/cpuinfo | wc -l` +[ -z "$NTHREADS" ] && NTHREADS=$([ $RAMTHREADS -le $CPUTHREADS ] && echo "$RAMTHREADS" || echo "$CPUTHREADS") || true export NTHREADS="$NTHREADS" # Set -j and -l for make (though -l is probably stripped by Sage) [ -z "$MAKEOPTS" ] && MAKEOPTS="-j $NTHREADS -l $((NTHREADS-1)).8" || true From 9e317e6318275e971896f2dac14c3df483123132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 14 Apr 2018 01:49:24 +0200 Subject: [PATCH 078/136] Allow at least one thread --- .ci/setup-make-parallelity.sh | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.ci/setup-make-parallelity.sh b/.ci/setup-make-parallelity.sh index 18cfcf5675f..cafe538cdc8 100755 --- a/.ci/setup-make-parallelity.sh +++ b/.ci/setup-make-parallelity.sh @@ -19,14 +19,25 @@ set -ex # provision fewer vCPUs than shown in /proc/cpuinfo. Also, setting this value # too high can lead to RAM being insufficient, so it's best to set the NTHREADS # variable manually in your CI configuration. -RAMTHREADS=$(( `grep MemTotal /proc/meminfo | awk '{ print $2 }'` / 1024 / 1024 / 2 )) -CPUTHREADS=`grep -E '^processor' /proc/cpuinfo | wc -l` -[ -z "$NTHREADS" ] && NTHREADS=$([ $RAMTHREADS -le $CPUTHREADS ] && echo "$RAMTHREADS" || echo "$CPUTHREADS") || true +if [ -z "$NTHREADS" ]; then + CPUTHREADS=`grep -E '^processor' /proc/cpuinfo | wc -l` + RAMTHREADS=$(( `grep MemTotal /proc/meminfo | awk '{ print $2 }'` / 1024 / 1024 / 2 )) + if [ $RAMTHREADS = 0 ];then + RAMTHREADS=1; + fi + NTHREADS=$([ $RAMTHREADS -le $CPUTHREADS ] && echo "$RAMTHREADS" || echo "$CPUTHREADS") +fi export NTHREADS="$NTHREADS" + # Set -j and -l for make (though -l is probably stripped by Sage) -[ -z "$MAKEOPTS" ] && MAKEOPTS="-j $NTHREADS -l $((NTHREADS-1)).8" || true +if [ -z "$MAKEOPTS" ]; then + MAKEOPTS="-j $NTHREADS -l $((NTHREADS-1)).8" +fi export MAKEOPTS="$MAKEOPTS" + # Not all parts of Sage seem to honor MAKEOPTS, so the current way of telling # the system which concurrency to use, seems to be setting $MAKE. -[ -z "$MAKE" ] && MAKE="make $MAKEOPTS" || true +if [ -z "$MAKE" ];then + MAKE="make $MAKEOPTS" +fi export MAKE="$MAKE" From d7e743ad626fd7a95046985bdca4b9ee0199af1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 14 Apr 2018 02:02:32 +0200 Subject: [PATCH 079/136] Fixed comments about the do tag Apparently we don't need "do" for build-from-latest. --- .gitlab-ci.yml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 19b1d42939a..39abb3a30f6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,18 +9,18 @@ # or set up your favourite cloud service to provide an on-demand autoscale # runner. -# As of early 2018 a run on GitLab CI takes about 35 minutes (with -# build-from-latest.) We could probably save 10 minutes by not -# building/pushing/testing that image for branches other than master/develop. +# As of early 2018 a run on GitLab CI takes about 35 minutes. We could probably +# save 10 minutes by not building/pushing/testing that image for branches other +# than master/develop. # Note that most of the time during CI is spent with pulling and pushing of -# docker images and copying files locally as part of the docker build. -# At the moment there is no reliable way of passing the docker images to the -# following stages without explicit pushing/pulling or similar: +# docker images and copying files locally as part of the docker build. At the +# moment there is no reliable way of passing the docker images to the following +# stages without explicit pushing/pulling or similar: # https://gitlab.com/gitlab-org/gitlab-runner/issues/1107 # The timings mentioned above are typical values. The shared runners provided -# on gitlab.com are sometimes by a factor of two slower. It's unclear what is -# the source of this slowdown. There is nothing different in the logs, so it's -# probably just an overcommittment of virtual machines on hardware. +# on gitlab.com are sometimes much slower depending on the runner you are +# scheduled on. Sometimes it's slower for no apparent reason, probably just an +# overcommittment of virtual machines on hardware. image: docker:latest @@ -87,8 +87,12 @@ build-from-latest: &build - develop # Build Sage and its documentation from a clean checkout of Sage. -# Note that this takes a very long time. You probably want to run this on your -# own gitlab-runner and increase the standard GitLab time limit for CI runs. +# Note that this takes several hours. You probably want to run this on your own +# gitlab-runner and increase the standard GitLab time limit for CI runs. +# Some of the shared runners provided by GitLab for free do not have enough +# disk space for this to work. If a build fails with "no space left on device", +# you could just retry it and hope to be scheduled on a machine with more disk +# space, or provision your own runner. build-from-clean: << : *build variables: From 49b21e16cbc288121fed109bc063629c1d285529 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 14 Apr 2018 02:43:24 +0200 Subject: [PATCH 080/136] Properly ignore undefined labels on first pass --- src/sage_setup/docbuild/sphinxbuild.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sage_setup/docbuild/sphinxbuild.py b/src/sage_setup/docbuild/sphinxbuild.py index efca70476d2..ef539d6df56 100644 --- a/src/sage_setup/docbuild/sphinxbuild.py +++ b/src/sage_setup/docbuild/sphinxbuild.py @@ -138,11 +138,11 @@ def _init_chatter(self): # - undefined labels upon the first pass of the compilation: some # cross links may legitimately not yet be resolvable at this point. if 'latex' not in sys.argv: + self._error_patterns += (re.compile('WARNING:'),) if 'multidoc_first_pass=1' in sys.argv: - # Catch all warnings except 'WARNING: undefined label' - self._error_patterns += (re.compile('WARNING: (?!undefined label)'),) - else: - self._error_patterns += (re.compile('WARNING:'),) + ignore = (re.compile('WARNING: undefined label'),) + self._ignored_warnings += ignore + self._useless_chatter += ignore def _filter_out(self, line): if self._error and self._is_stdout: From bb1a40743fbfde18ecd949c0f504781729097534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 14 Apr 2018 03:33:27 +0200 Subject: [PATCH 081/136] Fix incorrect @echo command --- Makefile | 4 ++-- docker/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d7e1671d4de..df8a7cbc101 100644 --- a/Makefile +++ b/Makefile @@ -114,10 +114,10 @@ micro_release: bdist-clean sagelib-clean find . -name . -o -prune ! -name src ! -name sage ! -name local ! -name VERSION.txt ! -name COPYING.txt ! -name build -exec rm -rf \{\} \; cd src && find . -name . -o -prune ! -name sage ! -name bin ! -name doc -exec rm -rf \{\} \; if command -v rdfind > /dev/null; then \ - @echo "Hardlinking identical files."; \ + echo "Hardlinking identical files."; \ rdfind -makeresultsfile false -makehardlinks true .; \ else \ - @echo "rdfind not installed. Not hardlinking identical files."; \ + echo "rdfind not installed. Not hardlinking identical files."; \ fi # Leaves everything that is needed to make the next "make" fast but removes diff --git a/docker/Dockerfile b/docker/Dockerfile index 2e3383a1ebe..7c061d944a8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -80,7 +80,7 @@ WORKDIR $HOME # completes without errors # ################################################################################ FROM run-time-dependencies as build-time-dependencies -# Install the build time dependencies & git +# Install the build time dependencies & git & rdfind RUN sudo apt-get -qq update \ && sudo apt-get -qq install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git rdfind \ && sudo apt-get -qq clean \ From 52c2136eb4f7de55a5a1107bc29c58bc4652e912 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 14 Apr 2018 11:44:38 +0200 Subject: [PATCH 082/136] ignoring more chatter during docbuild --- src/sage_setup/docbuild/sphinxbuild.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/sage_setup/docbuild/sphinxbuild.py b/src/sage_setup/docbuild/sphinxbuild.py index ef539d6df56..f55e25e46ac 100644 --- a/src/sage_setup/docbuild/sphinxbuild.py +++ b/src/sage_setup/docbuild/sphinxbuild.py @@ -88,11 +88,14 @@ def _init_chatter(self): re.compile('^linking _static directory'), re.compile('^copying static files... done'), re.compile('^copying extra files... done'), - re.compile('^loading transations \[.*\]... done'), + re.compile('^loading translations \[.*\]... done'), + re.compile('^Compiling the master document'), + re.compile('^Saved pickle file: citations.pickle'), re.compile('^writing output... \[.*\] '), re.compile('^copying images... \[.*\] '), re.compile('^reading sources... \[.*\] '), re.compile('language "hu" not supported'), + re.compile('^$'), re.compile('^WARNING:$'), ) @@ -149,6 +152,7 @@ def _filter_out(self, line): # swallow non-errors after an error occurred return True line = re.sub(self.ansi_color, '', line) + line = line.strip() for regex in self._useless_chatter: if regex.search(line) is not None: return True From f3bfa5fe9971fa2a04190b92da501cbf9724905b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 14 Apr 2018 12:18:34 +0200 Subject: [PATCH 083/136] Upgraded .gitlab-ci.yml to provide proper tags that actually work It's really unfortunate that GitLab CI machines are so underpowered. --- .gitlab-ci.yml | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 39abb3a30f6..fd776acf157 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,21 +7,35 @@ # place # https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-in-docker-executor; # or set up your favourite cloud service to provide an on-demand autoscale -# runner. +# runner. More details below. -# As of early 2018 a run on GitLab CI takes about 35 minutes. We could probably -# save 10 minutes by not building/pushing/testing that image for branches other +# As of early 2018 a run on GitLab CI takes about 45 minutes. We could probably +# save 10 minutes by not building/pushing/testing dev images for branches other # than master/develop. + # Note that most of the time during CI is spent with pulling and pushing of # docker images and copying files locally as part of the docker build. At the # moment there is no reliable way of passing the docker images to the following # stages without explicit pushing/pulling or similar: # https://gitlab.com/gitlab-org/gitlab-runner/issues/1107 + # The timings mentioned above are typical values. The shared runners provided # on gitlab.com are sometimes much slower depending on the runner you are # scheduled on. Sometimes it's slower for no apparent reason, probably just an # overcommittment of virtual machines on hardware. +# GitLab provides several flavours of shared runners (as of early 2018): +# * runners tagged as "do" (digitalocean.com) provide about 60GB of HDD, two +# cores, but only 2GB of RAM. The RAM is often not sufficient to build the +# documentation. +# * runners tagged as "gce" (Google Compute Engine) provide about 22GB of HDD, +# a single core, 4GB of RAM. Since we are relying on OverlayFS, the disk +# space is not sufficient to build sage from scratch. + +# If you want to provide your own runners, make sure to tag them as follows: +# * "60hdd" (60GB of disk space are available) to make build-from-clean pass. +# * "gce" (4GB of RAM are available) to make build-from-latest pass. + image: docker:latest stages: @@ -85,6 +99,9 @@ build-from-latest: &build except: - master - develop + tags: + # 4GB of RAM are available + - gce # Build Sage and its documentation from a clean checkout of Sage. # Note that this takes several hours. You probably want to run this on your own @@ -101,6 +118,11 @@ build-from-clean: - master - develop except: [] + tags: + # 4 GB of RAM are available + - gce + # 60 GB of HDD are available + - 60hdd test-dev: stage: test From 5719a728d42ab8876b3e06c6bf7582f85e804591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 15 Apr 2018 23:51:55 +0200 Subject: [PATCH 084/136] Improve caching during build I do not fully understand why but somehow --cache-from works more reliably than just pulling an image and hoping for the docker daemon to realize that it can reuse some layers. It might be related to the docker version running or whether we are running docker:dind. --- .ci/build-docker.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh index afd898e6fb2..75bad64e037 100755 --- a/.ci/build-docker.sh +++ b/.ci/build-docker.sh @@ -19,11 +19,12 @@ set -ex # during docker build. See /docker/Dockerfile for more details. ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop} -# Seed our cache with $ARTIFACT_BASE if it exists -docker pull "$ARTIFACT_BASE" > /dev/null || true +# Seed our cache with $ARTIFACT_BASE if it exists (otherwise take alpine which is tiny) +(docker pull "$ARTIFACT_BASE" > /dev/null && docker tag "$ARTIFACT_BASE" cache) || \ + (docker pull alpine > /dev/null && docker tag alpine cache) docker_build() { - time docker build -f docker/Dockerfile --build-arg "MAKE=${MAKE}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ + time docker build -f docker/Dockerfile --cache-from cache --build-arg "MAKE=${MAKE}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ } # We use a multi-stage build /docker/Dockerfile. For the caching to be From e4887e92aa0011d6debd6e59bca3e715fdf6fba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 16 Apr 2018 01:22:26 +0200 Subject: [PATCH 085/136] Fix caching during build It does not matter to docker anymore whether --cache-from exists. So we can just --cache-from everything that we could possibly find. --- .ci/build-docker.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh index 75bad64e037..7f4ac7b7d3e 100755 --- a/.ci/build-docker.sh +++ b/.ci/build-docker.sh @@ -19,12 +19,16 @@ set -ex # during docker build. See /docker/Dockerfile for more details. ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop} -# Seed our cache with $ARTIFACT_BASE if it exists (otherwise take alpine which is tiny) -(docker pull "$ARTIFACT_BASE" > /dev/null && docker tag "$ARTIFACT_BASE" cache) || \ - (docker pull alpine > /dev/null && docker tag alpine cache) +# Seed our cache with $ARTIFACT_BASE if it exists +docker pull "$ARTIFACT_BASE" > /dev/null || true docker_build() { - time docker build -f docker/Dockerfile --cache-from cache --build-arg "MAKE=${MAKE}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ + # Docker's --cache-from does not really work with multi-stage builds: https://github.com/moby/moby/issues/34715 + # We work around that by specifying all possible tags (docker does not + # fail anymore if a cache-from tag can not be found.) + time docker build -f docker/Dockerfile \ +--cache-from "$ARTIFACT_BASE" --cache-from build-time-dependencies --cache-from make-all --cache-from "$DOCKER_IMAGE_CLI" --cache-from "$DOCKER_IMAGE_DEV" \ +--build-arg "MAKE=${MAKE}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ } # We use a multi-stage build /docker/Dockerfile. For the caching to be @@ -32,8 +36,8 @@ docker_build() { # the make-all target. (Just building the last target is not enough as # intermediate targets would be discarded from the cache and therefore the # caching would fail for our actual builds below.) -docker_build --pull --target build-time-dependencies . -docker_build --pull --tag make-all --target make-all . +docker_build --pull --target build-time-dependencies --tag build-time-dependencies . +docker_build --pull --target make-all --tag make-all . # Build the release image without build artifacts. docker_build --target sagemath --tag "$DOCKER_IMAGE_CLI" . From be3a0f733950345bee7f4237db20558e1a06cefc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 19 Apr 2018 22:20:28 +0200 Subject: [PATCH 086/136] The docbuild crashes happen during an os.fork to spawn tachyon This is a cheap operation but it might exceed our overcommittment, let's check if that's the case. --- .ci/describe-system.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/describe-system.sh b/.ci/describe-system.sh index b33a84e2428..327243a4070 100755 --- a/.ci/describe-system.sh +++ b/.ci/describe-system.sh @@ -16,4 +16,6 @@ uname -a df -h cat /proc/cpuinfo cat /proc/meminfo +cat /proc/sys/vm/overcommit_memory +cat /proc/sys/vm/overcommit_ratio docker info From 7d85dc796c58c3de57401bc22d3587b94e205091 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 19 Apr 2018 23:01:30 +0200 Subject: [PATCH 087/136] Something related to the sphinxbuild seems to be leaking memory and this might make us exceed the overcommit limit on an os.fork() call then. Using the Pool() even if we only want one thread, means that we get a constant penalty at the beginning but then every build runs in a clean process. --- src/sage_setup/docbuild/__init__.py | 44 ++++++++++------------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/src/sage_setup/docbuild/__init__.py b/src/sage_setup/docbuild/__init__.py index 6940ebe5997..806018edcf4 100644 --- a/src/sage_setup/docbuild/__init__.py +++ b/src/sage_setup/docbuild/__init__.py @@ -264,35 +264,21 @@ def clean(self, *args): # import the customized builder for object.inv files inventory = builder_helper('inventory') -if NUM_THREADS > 1: - def build_many(target, args): - from multiprocessing import Pool - pool = Pool(NUM_THREADS, maxtasksperchild=1) - # map_async handles KeyboardInterrupt correctly. Plain map and - # apply_async does not, so don't use it. - x = pool.map_async(target, args, 1) - try: - ret = x.get(99999) - pool.close() - pool.join() - except Exception: - pool.terminate() - if ABORT_ON_ERROR: - raise - return ret -else: - def build_many(target, args): - results = [] - - for arg in args: - try: - results.append(target(arg)) - except Exception: - if ABORT_ON_ERROR: - raise - - return results - +def build_many(target, args): + from multiprocessing import Pool + pool = Pool(NUM_THREADS, maxtasksperchild=1) + # map_async handles KeyboardInterrupt correctly. Plain map and + # apply_async does not, so don't use it. + x = pool.map_async(target, args, 1) + try: + ret = x.get(99999) + pool.close() + pool.join() + except Exception: + pool.terminate() + if ABORT_ON_ERROR: + raise + return ret ########################################## # Parallel Building Ref Manual # From 50898feaef6499180ab2ec5b21e4b6b18032a77d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 19 Apr 2018 23:15:31 +0200 Subject: [PATCH 088/136] Try to build-from-scratch on GitLab's underpowered CI machines now that we are probably saving some (virtual) RAM in the docbuild. --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fd776acf157..7d4672f5500 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -119,10 +119,8 @@ build-from-clean: - develop except: [] tags: - # 4 GB of RAM are available - - gce # 60 GB of HDD are available - - 60hdd + - do test-dev: stage: test From 95c6275adac26be537c11a084114d8a8dc576392 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 21 Apr 2018 12:04:42 +0200 Subject: [PATCH 089/136] Comment on fork logic in docbuilding --- src/sage_setup/docbuild/__init__.py | 8 ++++++++ src/sage_setup/docbuild/sphinxbuild.py | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/src/sage_setup/docbuild/__init__.py b/src/sage_setup/docbuild/__init__.py index 806018edcf4..3425a2686c7 100644 --- a/src/sage_setup/docbuild/__init__.py +++ b/src/sage_setup/docbuild/__init__.py @@ -265,6 +265,14 @@ def clean(self, *args): inventory = builder_helper('inventory') def build_many(target, args): + # Pool() uses an actual fork() to run each new instance. This is important + # for performance reasons, i.e., don't use a forkserver when it becomes + # available with Python 3: Here, sage is already initialized which is quite + # costly, with a forkserver we would have to reinitialize it for every + # document we build. At the same time, don't serialize this by taking the + # pool (and thus the call to fork()) out completely: The call to Sphinx + # leaks memory, so we need to build each document in its own process to + # control the RAM usage. from multiprocessing import Pool pool = Pool(NUM_THREADS, maxtasksperchild=1) # map_async handles KeyboardInterrupt correctly. Plain map and diff --git a/src/sage_setup/docbuild/sphinxbuild.py b/src/sage_setup/docbuild/sphinxbuild.py index f55e25e46ac..fbc66d4d28f 100644 --- a/src/sage_setup/docbuild/sphinxbuild.py +++ b/src/sage_setup/docbuild/sphinxbuild.py @@ -309,6 +309,11 @@ def runsphinx(): try: sys.stdout = SageSphinxLogger(sys.stdout, os.path.basename(output_dir)) sys.stderr = SageSphinxLogger(sys.stderr, os.path.basename(output_dir)) + # Note that this call as of eraly 2018 leaks memory. So make sure that + # you don't call runsphinx() several times in a row. (i.e., you want to + # fork() somewhere before this call.) + # We don't use subprocess here, as we don't want to re-initialize Sage + # for every docbuild as this takes a while. sphinx.cmdline.main(sys.argv) sys.stderr.raise_errors() sys.stdout.raise_errors() From 329fdab86a1a3edf03c88780104a1471105fbcd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 23 Apr 2018 10:07:47 +0200 Subject: [PATCH 090/136] fix tags for build-from-latest --- .gitlab-ci.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7d4672f5500..d06104fae3d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,15 +26,14 @@ # GitLab provides several flavours of shared runners (as of early 2018): # * runners tagged as "do" (digitalocean.com) provide about 60GB of HDD, two -# cores, but only 2GB of RAM. The RAM is often not sufficient to build the -# documentation. +# cores, but only 2GB of RAM. The RAM is sometimes not sufficient to build +# the documentation. # * runners tagged as "gce" (Google Compute Engine) provide about 22GB of HDD, # a single core, 4GB of RAM. Since we are relying on OverlayFS, the disk # space is not sufficient to build sage from scratch. # If you want to provide your own runners, make sure to tag them as follows: -# * "60hdd" (60GB of disk space are available) to make build-from-clean pass. -# * "gce" (4GB of RAM are available) to make build-from-latest pass. +# * "do" (60GB of disk space are available) to make build-from-clean pass. image: docker:latest @@ -99,9 +98,6 @@ build-from-latest: &build except: - master - develop - tags: - # 4GB of RAM are available - - gce # Build Sage and its documentation from a clean checkout of Sage. # Note that this takes several hours. You probably want to run this on your own From 048e2d8f0a25bada68693f9b22a8e0426eab2fb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 27 Apr 2018 15:57:21 +0200 Subject: [PATCH 091/136] silence unicode warnings from the patchbot --- src/sage_setup/docbuild/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sage_setup/docbuild/__init__.py b/src/sage_setup/docbuild/__init__.py index 3425a2686c7..7dc8ff783c5 100644 --- a/src/sage_setup/docbuild/__init__.py +++ b/src/sage_setup/docbuild/__init__.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ The documentation builder From f9c772e5003fda874e82edd5c785eab2657cee1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 27 Apr 2018 16:21:33 +0200 Subject: [PATCH 092/136] Fix filtering logic in sphinx logger we still want to print the error lines, so we need to check for filter_out first. --- src/sage_setup/docbuild/sphinxbuild.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sage_setup/docbuild/sphinxbuild.py b/src/sage_setup/docbuild/sphinxbuild.py index fbc66d4d28f..bb3f49ae927 100644 --- a/src/sage_setup/docbuild/sphinxbuild.py +++ b/src/sage_setup/docbuild/sphinxbuild.py @@ -220,16 +220,16 @@ def _log_line(self, line): [#25160 ] Exception: artificial exception """ + skip_this_line = self._filter_out(line): self._check_errors(line) - if self._filter_out(line): - return for (old, new) in self.replacements: line = old.sub(new, line) line = self._prefix + ' ' + line.rstrip() + '\n' if not self._color: line = self.ansi_color.sub('', line) - self._stream.write(line) - self._stream.flush() + if not skip_this_line: + self._stream.write(line) + self._stream.flush() def raise_errors(self): r""" From f0431637a02819116781d87825523a068bf88ee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 27 Apr 2018 16:30:31 +0200 Subject: [PATCH 093/136] Fixed some minor doctest issues --- src/sage_setup/docbuild/sphinxbuild.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/sage_setup/docbuild/sphinxbuild.py b/src/sage_setup/docbuild/sphinxbuild.py index bb3f49ae927..a7a6afa81ec 100644 --- a/src/sage_setup/docbuild/sphinxbuild.py +++ b/src/sage_setup/docbuild/sphinxbuild.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- r""" This is Sage's version of the sphinx-build script @@ -172,7 +173,7 @@ def _check_errors(self, line): sage: logger.raise_errors() Traceback (most recent call last): ... - OSError: [doctestin] Segmentation fault! + OSError: Segmentation fault! """ if self._error: @@ -220,7 +221,7 @@ def _log_line(self, line): [#25160 ] Exception: artificial exception """ - skip_this_line = self._filter_out(line): + skip_this_line = self._filter_out(line) self._check_errors(line) for (old, new) in self.replacements: line = old.sub(new, line) @@ -246,7 +247,7 @@ def raise_errors(self): sage: logger.raise_errors() Traceback (most recent call last): ... - OSError: [doctestin] This is a SEVERE error + OSError: This is a SEVERE error """ if self._error: From 8f3480e71cafcef96b1c21aaf3219e8c81e5e9e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Wed, 2 May 2018 20:04:08 +0200 Subject: [PATCH 094/136] It seems that MAKEOPTS/SAGE_NUM_THREADS parsing has been fixed now let's not mangle MAKE anymore --- .ci/build-docker.sh | 2 +- .ci/setup-make-parallelity.sh | 8 +------- .ci/test-dev.sh | 2 +- docker/Dockerfile | 16 +++++++++------- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh index 7f4ac7b7d3e..5bff3091814 100755 --- a/.ci/build-docker.sh +++ b/.ci/build-docker.sh @@ -28,7 +28,7 @@ docker_build() { # fail anymore if a cache-from tag can not be found.) time docker build -f docker/Dockerfile \ --cache-from "$ARTIFACT_BASE" --cache-from build-time-dependencies --cache-from make-all --cache-from "$DOCKER_IMAGE_CLI" --cache-from "$DOCKER_IMAGE_DEV" \ ---build-arg "MAKE=${MAKE}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ +--build-arg "MAKEOPTS=${MAKEOPTS}" --build-arg "SAGE_NUM_THREADS=${SAGE_NUM_THREADS}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ } # We use a multi-stage build /docker/Dockerfile. For the caching to be diff --git a/.ci/setup-make-parallelity.sh b/.ci/setup-make-parallelity.sh index cafe538cdc8..a188d0d4269 100755 --- a/.ci/setup-make-parallelity.sh +++ b/.ci/setup-make-parallelity.sh @@ -28,16 +28,10 @@ if [ -z "$NTHREADS" ]; then NTHREADS=$([ $RAMTHREADS -le $CPUTHREADS ] && echo "$RAMTHREADS" || echo "$CPUTHREADS") fi export NTHREADS="$NTHREADS" +export SAGE_NUM_THREADS="$NTHREADS" # Set -j and -l for make (though -l is probably stripped by Sage) if [ -z "$MAKEOPTS" ]; then MAKEOPTS="-j $NTHREADS -l $((NTHREADS-1)).8" fi export MAKEOPTS="$MAKEOPTS" - -# Not all parts of Sage seem to honor MAKEOPTS, so the current way of telling -# the system which concurrency to use, seems to be setting $MAKE. -if [ -z "$MAKE" ];then - MAKE="make $MAKEOPTS" -fi -export MAKE="$MAKE" diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh index 3dde76bbbb0..b1216377049 100755 --- a/.ci/test-dev.sh +++ b/.ci/test-dev.sh @@ -26,7 +26,7 @@ IMAGE="$1" # Runs $IMAGE with args and check that it terminates with a zero exit code in at most limit seconds. timed_run() { START=`date +%s` - docker run -e MAKE="$MAKE" "$IMAGE" "$2" + docker run -e MAKEOPTS="$MAKEOPTS" -e SAGE_NUM_THREADS="$SAGE_NUM_THREADS" "$IMAGE" "$2" END=`date +%s` TOTAL=$((END-START)) echo "Checking whether running \"$2\" was fast…" diff --git a/docker/Dockerfile b/docker/Dockerfile index 7c061d944a8..90266204d97 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -32,11 +32,11 @@ # takes about five minutes but you don't have to go through the sadly frequent # # rebuilds the whole Sage distribution... # # To build Sage, run this command from your sage/ directory: # -# $ docker build --build-arg MAKE="make -j4" --build-arg ARTIFACT_BASE="sagemath/sagemath-dev:develop" -f docker/Dockerfile --target=make-build --tag sage . +# $ docker build --build-arg MAKEOPTS="-j4" --build-arg SAGE_NUM_THREADS="4" --build-arg ARTIFACT_BASE="sagemath/sagemath-dev:develop" -f docker/Dockerfile --target=make-build --tag sage . # To run Sage: # # $ docker run -it sage # # To run doctests: # -# $ docker run -e "MAKE=make -j4" -it sage sage -tp src/sage # +# $ docker run -e "MAKEOPTS=-j4" -e "SAGE_NUM_THREADS=4" -it sage sage -tp src/sage # Make sure that you always have the latest develop branch merged into your # # local branch for this to work. # ################################################################################ @@ -160,11 +160,13 @@ ENV SAGE_FAT_BINARY yes # Just to be sure Sage doesn't try to build its own GCC (even though # it shouldn't with a recent GCC package from the system and with gfortran) ENV SAGE_INSTALL_GCC no -# Make everything in the build use multiple cores (setting this variable causes -# trouble for some packages outside Sage but it still seems to be they way Sage -# is doing this.) -ARG MAKE="make -j2" -ENV MAKE $MAKE +# Set MAKEOPTS and SAGE_NUM_THREADS to build things in parallel during the +# docker build. Note that these do not leak into the sagemath and sagemath-dev +# images. +ARG MAKEOPTS="-j2" +ENV MAKEOPTS $MAKEOPTS +ARG SAGE_NUM_THREADS="2" +ENV SAGE_NUM_THREADS $SAGE_NUM_THREADS RUN make build ################################################################################ From 99355ad8eb1402dd09ec4dd7bd0f898271147b8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 3 May 2018 00:43:49 +0200 Subject: [PATCH 095/136] Build images in the same order in the Dockerfile and in build-docker.sh Otherwise we build sagemath-dev twice (probably depending on how aggresively exactly the docker daemon caches.) --- docker/Dockerfile | 50 +++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 90266204d97..e24f9e84bf2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -175,6 +175,31 @@ RUN make build FROM make-build as make-all RUN make +################################################################################ +# Image with a full build of sage, ready to release, i.e., with stripped # +# binaries and some extras to run the jupyter notebook. # +################################################################################ +FROM make-all as make-release +RUN sage -pip install terminado "notebook>=5" "ipykernel>=4.6" +RUN sage -i gap_jupyter singular_jupyter pari_jupyter +RUN make micro_release + +################################################################################ +# A releasable (relatively small) image which contains a copy of sage without # +# temporary build artifacts which is set up to start the command line # +# interface if no parameters are passed in. # +################################################################################ +FROM run-time-dependencies as sagemath +ARG SAGE_ROOT=/home/sage/sage +COPY --chown=sage:sage --from=make-release $SAGE_ROOT/ $SAGE_ROOT/ +# Put scripts to start gap, gp, maxima, ... in /usr/bin +WORKDIR $SAGE_ROOT +RUN sudo sage --nodotsage -c "install_scripts('/usr/bin')" +COPY ./docker/entrypoint.sh /usr/local/bin/sage-entrypoint +ENTRYPOINT ["/usr/local/bin/sage-entrypoint"] +EXPOSE 8888 +CMD ["sage"] + ################################################################################ # Image with a full build of sage and its documentation but everything # # stripped that can be quickly rebuild by make. # @@ -222,28 +247,3 @@ RUN if [ x"$ARTIFACT_BASE" != x"source-clean" ]; then \ COPY ./docker/entrypoint-dev.sh /usr/local/bin/sage-entrypoint ENTRYPOINT ["/usr/local/bin/sage-entrypoint"] CMD ["bash"] - -################################################################################ -# Image with a full build of sage, ready to release, i.e., with stripped # -# binaries and some extras to run the jupyter notebook. # -################################################################################ -FROM make-all as make-release -RUN sage -pip install terminado "notebook>=5" "ipykernel>=4.6" -RUN sage -i gap_jupyter singular_jupyter pari_jupyter -RUN make micro_release - -################################################################################ -# A releasable (relatively small) image which contains a copy of sage without # -# temporary build artifacts which is set up to start the command line # -# interface if no parameters are passed in. # -################################################################################ -FROM run-time-dependencies as sagemath -ARG SAGE_ROOT=/home/sage/sage -COPY --chown=sage:sage --from=make-release $SAGE_ROOT/ $SAGE_ROOT/ -# Put scripts to start gap, gp, maxima, ... in /usr/bin -WORKDIR $SAGE_ROOT -RUN sudo sage --nodotsage -c "install_scripts('/usr/bin')" -COPY ./docker/entrypoint.sh /usr/local/bin/sage-entrypoint -ENTRYPOINT ["/usr/local/bin/sage-entrypoint"] -EXPOSE 8888 -CMD ["sage"] From 81babb8af3c494be4ce89c586c6168ae80682560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 3 May 2018 00:58:15 +0200 Subject: [PATCH 096/136] Simplify caching during docker build all the fancy --cache-from setup does not work with the latest docker:dind for some reason that I do not understand. If we also don't --pull caching seems to work fine on both GitLabCI (dind) and CircleCI. --- .ci/build-docker.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh index 5bff3091814..cff26132d6f 100755 --- a/.ci/build-docker.sh +++ b/.ci/build-docker.sh @@ -19,25 +19,25 @@ set -ex # during docker build. See /docker/Dockerfile for more details. ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop} -# Seed our cache with $ARTIFACT_BASE if it exists +# Seed our cache with $ARTIFACT_BASE if it exists. docker pull "$ARTIFACT_BASE" > /dev/null || true docker_build() { # Docker's --cache-from does not really work with multi-stage builds: https://github.com/moby/moby/issues/34715 - # We work around that by specifying all possible tags (docker does not - # fail anymore if a cache-from tag can not be found.) + # So we just have to rely on the local cache. time docker build -f docker/Dockerfile \ ---cache-from "$ARTIFACT_BASE" --cache-from build-time-dependencies --cache-from make-all --cache-from "$DOCKER_IMAGE_CLI" --cache-from "$DOCKER_IMAGE_DEV" \ --build-arg "MAKEOPTS=${MAKEOPTS}" --build-arg "SAGE_NUM_THREADS=${SAGE_NUM_THREADS}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ } # We use a multi-stage build /docker/Dockerfile. For the caching to be -# effective, we populate the cache by building the build-time-dependencies and -# the make-all target. (Just building the last target is not enough as -# intermediate targets would be discarded from the cache and therefore the -# caching would fail for our actual builds below.) -docker_build --pull --target build-time-dependencies --tag build-time-dependencies . -docker_build --pull --target make-all --tag make-all . +# effective, we populate the cache by building the run/build-time-dependencies +# and the make-all target. (Just building the last target is not enough as +# intermediate targets could be discarded from the cache [depending on the +# docker version] and therefore the caching would fail for our actual builds +# below.) +docker_build --target run-time-dependencies --tag run-time-dependencies:$DOCKER_TAG . +docker_build --target build-time-dependencies --tag build-time-dependencies:$DOCKER_TAG . +docker_build --target make-all --tag make-all:$DOCKER_TAG . # Build the release image without build artifacts. docker_build --target sagemath --tag "$DOCKER_IMAGE_CLI" . From ad39cc9de9dd8a3f6092c7153278982393da57b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 3 May 2018 18:48:54 +0200 Subject: [PATCH 097/136] Complicate parallelization quite a bit to make the build fast and the docbuild not crash --- .ci/build-docker.sh | 2 +- .ci/setup-make-parallelity.sh | 55 ++++++++++++++++++++++++----------- .ci/test-dev.sh | 4 +-- .circleci/config.yml | 2 +- .gitlab-ci.yml | 2 +- docker/Dockerfile | 8 +++++ 6 files changed, 51 insertions(+), 22 deletions(-) diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh index cff26132d6f..404362cc1bb 100755 --- a/.ci/build-docker.sh +++ b/.ci/build-docker.sh @@ -26,7 +26,7 @@ docker_build() { # Docker's --cache-from does not really work with multi-stage builds: https://github.com/moby/moby/issues/34715 # So we just have to rely on the local cache. time docker build -f docker/Dockerfile \ ---build-arg "MAKEOPTS=${MAKEOPTS}" --build-arg "SAGE_NUM_THREADS=${SAGE_NUM_THREADS}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ +--build-arg "MAKEOPTS=${MAKEOPTS}" --build-arg "SAGE_NUM_THREADS=${SAGE_NUM_THREADS}" --build-arg "MAKEOPTS_DOCBUILD=${MAKEOPTS}" --build-arg "SAGE_NUM_THREADS_DOCBUILD=${SAGE_NUM_THREADS_DOCBUILD}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ } # We use a multi-stage build /docker/Dockerfile. For the caching to be diff --git a/.ci/setup-make-parallelity.sh b/.ci/setup-make-parallelity.sh index a188d0d4269..1998b8bc423 100755 --- a/.ci/setup-make-parallelity.sh +++ b/.ci/setup-make-parallelity.sh @@ -1,5 +1,15 @@ #!/bin/sh +# Source this to set CPUTHREADS (the number of apparent cores) and +# RAMTHREADS (free RAM divided by the maximum amount needed per thread +# typically) +# From this this script exports reasonable defaults for SAGE_NUM_THREADS and +# MAKEOPTS. + +# We do exactly the same for CPUTHREADS_DOCBUILD, RAMTHREADS_DOCBUILD, +# SAGE_NUM_THREADS_DOCBUILD, MAKEOPTS_DOCBUILD. As the docbuild needs +# substantially more RAM as of May 2018. + # **************************************************************************** # Copyright (C) 2018 Julian Rüth # @@ -12,26 +22,37 @@ set -ex -# Determine the number of threads that can run simultaneously on this system -# (we might not have nproc available.) -# Note that this value is incorrect for some CI providers (notably CircleCI: -# https://circleci.com/docs/2.0/configuration-reference/#resource_class) which -# provision fewer vCPUs than shown in /proc/cpuinfo. Also, setting this value -# too high can lead to RAM being insufficient, so it's best to set the NTHREADS -# variable manually in your CI configuration. -if [ -z "$NTHREADS" ]; then +if [ -z "$CPUTHREADS" ]; then + # Determine the number of threads that can run simultaneously on this system + # (we might not have nproc available.) + # Note that this value is incorrect for some CI providers (notably CircleCI: + # https://circleci.com/docs/2.0/configuration-reference/#resource_class) which + # provision fewer vCPUs than shown in /proc/cpuinfo. So it is probably better + # to set CPUTHREADS manuall in your CI configuration. CPUTHREADS=`grep -E '^processor' /proc/cpuinfo | wc -l` - RAMTHREADS=$(( `grep MemTotal /proc/meminfo | awk '{ print $2 }'` / 1024 / 1024 / 2 )) +fi +if [ -z "$CPUTHREADS_DOCBUILD" ]; then + CPUTHREADS_DOCBUILD=$CPUTHREADS +fi + +if [ -z "$RAMTHREADS" ]; then + RAMTHREADS=$(( `grep MemTotal /proc/meminfo | awk '{ print $2 }'` / 1048576 )) if [ $RAMTHREADS = 0 ];then RAMTHREADS=1; fi - NTHREADS=$([ $RAMTHREADS -le $CPUTHREADS ] && echo "$RAMTHREADS" || echo "$CPUTHREADS") fi -export NTHREADS="$NTHREADS" -export SAGE_NUM_THREADS="$NTHREADS" - -# Set -j and -l for make (though -l is probably stripped by Sage) -if [ -z "$MAKEOPTS" ]; then - MAKEOPTS="-j $NTHREADS -l $((NTHREADS-1)).8" +if [ -z "$RAMTHREADS_DOCBUILD" ]; then + RAMTHREADS_DOCBUILD=$(( `grep MemTotal /proc/meminfo | awk '{ print $2 }'` / 2097152 )) + if [ $RAMTHREADS_DOCBUILD = 0 ];then + RAMTHREADS_DOCBUILD=1; + fi fi -export MAKEOPTS="$MAKEOPTS" + +# On CI machines with their virtual CPUs, it seems to be quite beneficial to +# overcommit on CPU usage. We only need to make sure that we do not exceed RAM +# (as there is no swap.) +export SAGE_NUM_THREADS=$RAMTHREADS +export SAGE_NUM_THREADS_DOCBUILD=$RAMTHREADS_DOCBUILD +# Set -j and -l for make (though -l is probably ignored by Sage) +export MAKEOPTS="-j $RAMTHREADS -l $((CPUTHREADS-1)).8" +export MAKEOPTS_DOCBUILD="-j $RAMTHREADS_DOCBUILD -l $((CPUTHREADS_DOCBUILD-1)).8" diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh index b1216377049..5ba2cce16ca 100755 --- a/.ci/test-dev.sh +++ b/.ci/test-dev.sh @@ -26,7 +26,7 @@ IMAGE="$1" # Runs $IMAGE with args and check that it terminates with a zero exit code in at most limit seconds. timed_run() { START=`date +%s` - docker run -e MAKEOPTS="$MAKEOPTS" -e SAGE_NUM_THREADS="$SAGE_NUM_THREADS" "$IMAGE" "$2" + docker run -e MAKEOPTS="$MAKEOPTS_DOCBUILD" -e SAGE_NUM_THREADS="$SAGE_NUM_THREADS_DOCBUILD" "$IMAGE" "$2" END=`date +%s` TOTAL=$((END-START)) echo "Checking whether running \"$2\" was fast…" @@ -42,4 +42,4 @@ timed_run 120 true # runs make build # The parser in Sphinx fails to parse some .py files and adds the (more # recently modified) .pyc files as dependencies instead. (Have a look the # changeset that introduced this comment for more details.) -timed_run $(( 1200/$NTHREADS )) make # runs make build and then make +timed_run $(( 1200/$SAGE_NUM_THREADS_DOCBUILD )) make # runs make build and then make diff --git a/.circleci/config.yml b/.circleci/config.yml index 0404cc6b8bb..4869e5003dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,7 +33,7 @@ jobs: . .ci/protect-secrets.sh # Collect debug infos about the system we are running on .ci/describe-system.sh - # Set MAKE and NTHREADS according to the machine we are running on + # Set MAKEOPTS and SAGE_NUM_THREADS . .ci/setup-make-parallelity.sh # Set DOCKER_TAG according to the current branch/tag diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d06104fae3d..b4f9dfb97a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,7 +56,7 @@ before_script: - .ci/describe-system.sh # Set DOCKER_TAG according to the current branch/tag - . .ci/update-env.sh - # Set MAKE and NTHREADS according to the machine we are running on + # Set MAKEOPTS and SAGE_NUM_THREADS according to the machine we are running on - . .ci/setup-make-parallelity.sh # We use docker-in-docker to build our docker images, i.e., we run a diff --git a/docker/Dockerfile b/docker/Dockerfile index e24f9e84bf2..325e5d7f922 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -173,6 +173,14 @@ RUN make build # Image with a full build of sage and its documentation. # ################################################################################ FROM make-build as make-all +# The docbuild needs quite some RAM (as of May 2018). It sometimes calls +# os.fork() to spawn an external program which then exceeds easily the +# overcommit limit of the system (no RAM is actually used, but this limit is +# very low because there is not even swap on most CI systems.) +ARG MAKEOPTS_DOCBUILD=$MAKEOPTS +ENV MAKEOPTS_DOCBUILD $MAKEOPTS_DOCBUILD +ARG SAGE_NUM_THREADS_DOCBUILD=$SAGE_NUM_THREADS +ENV SAGE_NUM_THREADS_DOCBUILD $SAGE_NUM_THREADS_DOCBUILD RUN make ################################################################################ From ae119f23c62921e7e01e5169b52124a2243a8ab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 3 May 2018 22:56:22 +0200 Subject: [PATCH 098/136] Do not overcommit that much on CircleCI we sometimes get single core but 8GB of RAM --- .ci/setup-make-parallelity.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.ci/setup-make-parallelity.sh b/.ci/setup-make-parallelity.sh index 1998b8bc423..77c1bbb9877 100755 --- a/.ci/setup-make-parallelity.sh +++ b/.ci/setup-make-parallelity.sh @@ -51,8 +51,16 @@ fi # On CI machines with their virtual CPUs, it seems to be quite beneficial to # overcommit on CPU usage. We only need to make sure that we do not exceed RAM # (as there is no swap.) -export SAGE_NUM_THREADS=$RAMTHREADS -export SAGE_NUM_THREADS_DOCBUILD=$RAMTHREADS_DOCBUILD +if [ $CPUTHREADS -lt $RAMTHREADS ]; then + export SAGE_NUM_THREADS=$((CPUTHREADS + 1)) +else + export SAGE_NUM_THREADS=$RAMTHREADS +fi +if [ $CPUTHREADS_DOCBUILD -lt $RAMTHREADS_DOCBUILD ]; then + export SAGE_NUM_THREADS_DOCBUILD=$((CPUTHREADS_DOCBUILD + 1)) +else + export SAGE_NUM_THREADS_DOCBUILD=$RAMTHREADS_DOCBUILD +fi # Set -j and -l for make (though -l is probably ignored by Sage) -export MAKEOPTS="-j $RAMTHREADS -l $((CPUTHREADS-1)).8" -export MAKEOPTS_DOCBUILD="-j $RAMTHREADS_DOCBUILD -l $((CPUTHREADS_DOCBUILD-1)).8" +export MAKEOPTS="-j $SAGE_NUM_THREADS -l $((CPUTHREADS - 1)).8" +export MAKEOPTS_DOCBUILD="-j $SAGE_NUM_THREADS_DOCBUILD -l $((CPUTHREADS_DOCBUILD - 1)).8" From a5914a4917e6dbbbc99c8424250f010ee54de809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 13 May 2018 20:13:49 +0200 Subject: [PATCH 099/136] Build tags from clean --- .circleci/config.yml | 9 +++------ .gitlab-ci.yml | 3 +++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4869e5003dc..5fb65914ecb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,17 +41,14 @@ jobs: . .ci/update-env.sh # Select ARTIFACT_BASE depending on the current branch/tag - case $DOCKER_TAG in - "develop" | "latest") + if [ -n $CIRCLE_TAG ] || [ $DOCKER_TAG = master ] || [ $DOCKER_TAG = develop ];then export ARTIFACT_BASE=source-clean - ;; - *) + else # Select sagemath/sagemath-dev:develop as the ARTIFACT_BASE # unless the user has explicitly selected a differnt one as a # CircleCI environment variable. export ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop} - ;; - esac + fi # Build docker images .ci/build-docker.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4f9dfb97a1..dcf46066619 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -98,6 +98,7 @@ build-from-latest: &build except: - master - develop + - tags # Build Sage and its documentation from a clean checkout of Sage. # Note that this takes several hours. You probably want to run this on your own @@ -113,6 +114,7 @@ build-from-clean: only: - master - develop + - tags except: [] tags: # 60 GB of HDD are available @@ -160,6 +162,7 @@ push-dockerhub-dev: only: - master - develop + - tags script: - . .ci/pull-gitlab.sh sagemath-dev - sh .ci/push-dockerhub.sh sagemath-dev From 49bb769a5adec5dae2e407c0e72d50d278f1a8c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 14 May 2018 15:09:28 +0200 Subject: [PATCH 100/136] Work around CircleCI escaping -n does not work correctly in CircleCI's shell. Quoting might help. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5fb65914ecb..f8ad75a05ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,7 +41,7 @@ jobs: . .ci/update-env.sh # Select ARTIFACT_BASE depending on the current branch/tag - if [ -n $CIRCLE_TAG ] || [ $DOCKER_TAG = master ] || [ $DOCKER_TAG = develop ];then + if [ -n "$CIRCLE_TAG" ] || [ $DOCKER_TAG = "master" ] || [ $DOCKER_TAG = "develop" ];then export ARTIFACT_BASE=source-clean else # Select sagemath/sagemath-dev:develop as the ARTIFACT_BASE From f9c68a64834343db79cd5953c493728c30e71735 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 14 May 2018 15:19:03 +0200 Subject: [PATCH 101/136] Tell CircleCI to build for all tags the branches bit is not necessary but for symmetry I think it looks better like that. --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index f8ad75a05ed..27b27b1829c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,6 +22,12 @@ jobs: # we do build & test & release in one step. build: machine: true + # Build for all tags + tags: + only: /.*/ + # Build for all branches + branches: + only: /.*/ steps: - checkout - run: From b1508cf144c5b31fb7f4f1ff7a1f22155130900d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 15 May 2018 10:37:16 +0200 Subject: [PATCH 102/136] adapt to microbadger URL change --- docker/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index 0028e8b9798..a33ea0b81cc 100644 --- a/docker/README.md +++ b/docker/README.md @@ -17,7 +17,7 @@ SageMath is a free open-source mathematics software system licensed under the GP There are several flavours of this image. -* [`sagemath/sagemath`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath:latest.svg)](https://hub.docker.com/sagemath/sagemath) contains everything necessary to run Sage on the command line. Run it with: +* [`sagemath/sagemath`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath/latest.svg)](https://hub.docker.com/r/sagemath/sagemath) contains everything necessary to run Sage on the command line. Run it with: ``` docker run -it sagemath/sagemath:latest ``` @@ -25,7 +25,7 @@ There are several flavours of this image. ``` docker run -p8888:8888 sagemath/sagemath:latest sage-jupyter ``` -* [`sagemath/sagemath-dev`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath-dev:develop.svg)](https://hub.docker.com/sagemath/sagemath-dev) contains all the build artifacts to rebuild Sage quickly. This version is probably only relevant for Sage developers. Run this image with: +* [`sagemath/sagemath-dev`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath-dev.svg)](https://hub.docker.com/r/sagemath/sagemath-dev) contains all the build artifacts to rebuild Sage quickly. This version is probably only relevant for Sage developers. Run this image with: ``` docker run -it sagemath/sagemath-dev:develop ``` From a4487bf1fa369edb3084bcb9ba37755156eea770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 15 May 2018 10:47:56 +0200 Subject: [PATCH 103/136] Fix format of stable releases and mention betas --- docker/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index a33ea0b81cc..4b627c11d46 100644 --- a/docker/README.md +++ b/docker/README.md @@ -3,7 +3,8 @@ # Supported tags * `latest` — the stable `master` branch [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/sagemath/sage/master.svg)](https://github.com/sagemath/sage/commits/master) [![CircleCI branch](https://img.shields.io/circleci/project/github/sagemath/sage/master.svg)](https://circleci.com/gh/sagemath/sage/tree/master) [![GitLab CI](https://gitlab.com/sagemath/sage/badges/master/pipeline.svg)](https://gitlab.com/sagemath/sage/commits/master) -* `x.x.x` — all stable releases of Sage are tagged with their version number. +* `x.x` — all stable releases of Sage are tagged with their version number. +* `x.x.{beta,rc}x` - betas and release candidadets of Sage as [tagged in our git repository](https://github.com/sagemath/sage/tags). * `develop` — the current development version of Sage which gets merged into the `master` branch when a new version of Sage is released [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/sagemath/sage/develop.svg)](https://github.com/sagemath/sage/commits/develop) [![CircleCI branch](https://img.shields.io/circleci/project/github/sagemath/sage/master.svg)](https://circleci.com/gh/sagemath/sage/tree/master) [![GitLab CI](https://gitlab.com/sagemath/sage/badges/develop/pipeline.svg)](https://gitlab.com/sagemath/sage/commits/develop) From 2d33a626429f5ae07dbe3dd7791b238341e37ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 15 May 2018 10:49:48 +0200 Subject: [PATCH 104/136] fix typo --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 4b627c11d46..b9f9240f9e4 100644 --- a/docker/README.md +++ b/docker/README.md @@ -4,7 +4,7 @@ * `latest` — the stable `master` branch [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/sagemath/sage/master.svg)](https://github.com/sagemath/sage/commits/master) [![CircleCI branch](https://img.shields.io/circleci/project/github/sagemath/sage/master.svg)](https://circleci.com/gh/sagemath/sage/tree/master) [![GitLab CI](https://gitlab.com/sagemath/sage/badges/master/pipeline.svg)](https://gitlab.com/sagemath/sage/commits/master) * `x.x` — all stable releases of Sage are tagged with their version number. -* `x.x.{beta,rc}x` - betas and release candidadets of Sage as [tagged in our git repository](https://github.com/sagemath/sage/tags). +* `x.x.{beta,rc}x` - betas and release candidates of Sage as [tagged in our git repository](https://github.com/sagemath/sage/tags). * `develop` — the current development version of Sage which gets merged into the `master` branch when a new version of Sage is released [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/sagemath/sage/develop.svg)](https://github.com/sagemath/sage/commits/develop) [![CircleCI branch](https://img.shields.io/circleci/project/github/sagemath/sage/master.svg)](https://circleci.com/gh/sagemath/sage/tree/master) [![GitLab CI](https://gitlab.com/sagemath/sage/badges/develop/pipeline.svg)](https://gitlab.com/sagemath/sage/commits/develop) From aa49196c4f0ed33da7a45b80c7a4a01bd46b7d61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 15 May 2018 10:58:36 +0200 Subject: [PATCH 105/136] Backport https://trac.sagemath.org/ticket/24655 for automated docker builds --- .ci/README.md | 9 + .ci/build-docker.sh | 46 ++++ .ci/describe-system.sh | 21 ++ .ci/head-tail.sh | 45 ++++ .ci/protect-secrets.sh | 40 +++ .ci/pull-gitlab.sh | 30 +++ .ci/push-dockerhub.sh | 29 +++ .ci/push-gitlab.sh | 25 ++ .ci/setup-make-parallelity.sh | 66 +++++ .ci/test-cli.sh | 30 +++ .ci/test-dev.sh | 45 ++++ .ci/test-jupyter.sh | 24 ++ .ci/update-env.sh | 28 +++ .circleci/config.yml | 69 ++++++ .dockerignore | 1 + .gitignore | 11 + .gitlab-ci.yml | 168 +++++++++++++ Makefile | 35 +++ configure.ac | 6 +- docker/.gitignore | 2 + docker/Dockerfile | 257 ++++++++++++++++++++ docker/README.md | 55 +++++ docker/entrypoint-dev.sh | 4 + docker/entrypoint.sh | 9 + docker/hooks/build | 1 + docker/hooks/push | 1 + src/.gitignore | 4 - src/Makefile.in | 2 +- src/doc/common/conf.py | 2 +- src/sage/misc/sagedoc.py | 8 +- src/sage_setup/docbuild/__init__.py | 59 +++-- src/sage_setup/docbuild/ext/sage_autodoc.py | 7 +- src/sage_setup/docbuild/sphinxbuild.py | 231 +++++++++++++----- 33 files changed, 1264 insertions(+), 106 deletions(-) create mode 100644 .ci/README.md create mode 100755 .ci/build-docker.sh create mode 100755 .ci/describe-system.sh create mode 100755 .ci/head-tail.sh create mode 100755 .ci/protect-secrets.sh create mode 100755 .ci/pull-gitlab.sh create mode 100755 .ci/push-dockerhub.sh create mode 100755 .ci/push-gitlab.sh create mode 100755 .ci/setup-make-parallelity.sh create mode 100755 .ci/test-cli.sh create mode 100755 .ci/test-dev.sh create mode 100755 .ci/test-jupyter.sh create mode 100755 .ci/update-env.sh create mode 100644 .circleci/config.yml create mode 120000 .dockerignore create mode 100644 .gitlab-ci.yml create mode 100644 docker/.gitignore create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100755 docker/entrypoint-dev.sh create mode 100755 docker/entrypoint.sh create mode 100644 docker/hooks/build create mode 100644 docker/hooks/push delete mode 100644 src/.gitignore diff --git a/.ci/README.md b/.ci/README.md new file mode 100644 index 00000000000..e2b165cc518 --- /dev/null +++ b/.ci/README.md @@ -0,0 +1,9 @@ +# Continuous Integration (CI) + +We support several implementations of CI. All these implementations rely on +[docker](https://docker.com) in some way. This directory contains bits which +are shared between these CI implementations. The relevant docker files can be +found in `/docker/`. + +* [CircleCI](https://circleci.com) is configured in `/.circleci/`. +* [GitLab CI](https://gitlab.com) is configured in `/.gitlab-ci.yml`. diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh new file mode 100755 index 00000000000..404362cc1bb --- /dev/null +++ b/.ci/build-docker.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# This script gets called from CI to build several flavours of docker images +# which contain Sage. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +# We speed up the build process by copying built artifacts from ARTIFACT_BASE +# during docker build. See /docker/Dockerfile for more details. +ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop} + +# Seed our cache with $ARTIFACT_BASE if it exists. +docker pull "$ARTIFACT_BASE" > /dev/null || true + +docker_build() { + # Docker's --cache-from does not really work with multi-stage builds: https://github.com/moby/moby/issues/34715 + # So we just have to rely on the local cache. + time docker build -f docker/Dockerfile \ +--build-arg "MAKEOPTS=${MAKEOPTS}" --build-arg "SAGE_NUM_THREADS=${SAGE_NUM_THREADS}" --build-arg "MAKEOPTS_DOCBUILD=${MAKEOPTS}" --build-arg "SAGE_NUM_THREADS_DOCBUILD=${SAGE_NUM_THREADS_DOCBUILD}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ +} + +# We use a multi-stage build /docker/Dockerfile. For the caching to be +# effective, we populate the cache by building the run/build-time-dependencies +# and the make-all target. (Just building the last target is not enough as +# intermediate targets could be discarded from the cache [depending on the +# docker version] and therefore the caching would fail for our actual builds +# below.) +docker_build --target run-time-dependencies --tag run-time-dependencies:$DOCKER_TAG . +docker_build --target build-time-dependencies --tag build-time-dependencies:$DOCKER_TAG . +docker_build --target make-all --tag make-all:$DOCKER_TAG . + +# Build the release image without build artifacts. +docker_build --target sagemath --tag "$DOCKER_IMAGE_CLI" . +# Build the developer image with the build artifacts intact. +# Note: It's important to build the dev image last because it might be tagged as ARTIFACT_BASE. +docker_build --target sagemath-dev --tag "$DOCKER_IMAGE_DEV" . diff --git a/.ci/describe-system.sh b/.ci/describe-system.sh new file mode 100755 index 00000000000..327243a4070 --- /dev/null +++ b/.ci/describe-system.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set +e -x + +uname -a +df -h +cat /proc/cpuinfo +cat /proc/meminfo +cat /proc/sys/vm/overcommit_memory +cat /proc/sys/vm/overcommit_ratio +docker info diff --git a/.ci/head-tail.sh b/.ci/head-tail.sh new file mode 100755 index 00000000000..8ecf9f879f0 --- /dev/null +++ b/.ci/head-tail.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +OFFSET=80 +# This script reads from stdin and prints to stdout as long as a the output +# does not exceed a certain number of bytes. When reading an EOF it prints the +# last $OFFSET lines if they have not been printed normally already. +# This script expects one argument, the number of bytes. + +# Heavily inspired by a simlar strategy in make, https://stackoverflow.com/a/44849696/812379. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +stdbuf -i0 -o0 -e0 awk -v limit=$1 -v firstMissingNR=-1 -v offset=$OFFSET -v bytes=0 \ +'{ + if (bytes < limit) { + # this probably gets multi-byte characters wrong, but that probably does + # not matter for our purposes. (We add 1 for a UNIX newline.) + bytes += length($0) + 1; + print; + } else { + if (firstMissingNR == -1){ + print "[…output truncated…]"; + firstMissingNR = NR; + } + a[NR] = $0; + delete a[NR-offset]; + printf "." > "/dev/stderr" + } +} +END { + if (firstMissingNR != -1) { + print "" > "/dev/stderr"; + for(i = NR-offset+1 > firstMissingNR ? NR-offset-1 : firstMissingNR; i<=NR ; i++){ print a[i]; } + } +} +' + diff --git a/.ci/protect-secrets.sh b/.ci/protect-secrets.sh new file mode 100755 index 00000000000..ee781dddca8 --- /dev/null +++ b/.ci/protect-secrets.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +# This script protects all environment variables that start with "SECRET_". +# It puts them in a temporary file. The name of the variable contains the path +# of that file. This filename can then safely be used in `cat` even if `set +# -x` has been turned on. Also you can run "export" to understand the +# environment without danger. +# Be careful, however, not to use this like the following: +# docker login $DOCKER_USER $(cat $SECRET_DOCKER_PASS) +# as this would expose the password if `set -x` has been turned on. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -eo pipefail +set +x + +function encrypt { + RET=`mktemp` + eval " echo \$$1" > "$RET" + echo $RET +} + +for name in `awk 'END { for (name in ENVIRON) { print name; } }' < /dev/null`; do +case "$name" in + SECRET_*) + export $name="$(encrypt $name)" + echo "Protected $name" + ;; +esac +done + +unset encrypt diff --git a/.ci/pull-gitlab.sh b/.ci/pull-gitlab.sh new file mode 100755 index 00000000000..c8235e64068 --- /dev/null +++ b/.ci/pull-gitlab.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# This script gets called from CI to pull the Sage docker images that were +# built during the "build" phase to pull all the connected docker daemon +# (likely a docker-in-docker.) +# This script expects a single parameter, the base name of the docker image +# such as sagemath or sagemath-dev. +# The variable $DOCKER_IMAGE is set to the full name of the pulled image; +# source this script to use it. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +# Pull the built images from the gitlab registry and give them the original +# names they had after built. +# Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it +# automatically from the log output. +docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY +docker pull $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG +export DOCKER_IMAGE="${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG" +docker tag $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG $DOCKER_IMAGE diff --git a/.ci/push-dockerhub.sh b/.ci/push-dockerhub.sh new file mode 100755 index 00000000000..c2c46062ffa --- /dev/null +++ b/.ci/push-dockerhub.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# This script gets called from CI to push our docker images to +# $DOCKER_USER/sagemath* on the Docker Hub. +# This script expects a single parameter, the base name of the docker image +# such as sagemath or sagemath-dev. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +[ -z "$DOCKER_TAG" ] && (echo "Can not push untagged build."; exit 0) + +# Push the built images to the docker hub (and fail silently if +# DOCKER_USER/SECRET_DOCKER_PASS have not been configured.) +if [ -z "$DOCKER_USER" -o -z "$SECRET_DOCKER_PASS" ]; then + echo "DOCKER_USER/SECRET_DOCKER_PASS variables have not been configured in your Continuous Integration setup. Not pushing built images to Docker Hub." +else + cat "$SECRET_DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin + docker push ${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG +fi diff --git a/.ci/push-gitlab.sh b/.ci/push-gitlab.sh new file mode 100755 index 00000000000..e4ceb30abcd --- /dev/null +++ b/.ci/push-gitlab.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +# This script gets called from CI to push our docker images to registry +# configured in GitLab. (Mostly, so we can pull them again to push them to the +# Docker Hub.) +# This script expects a single parameter, the base name of the docker image +# such as sagemath or sagemath-dev. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +# Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it +# automatically from the log output. +docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY +docker tag ${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG +docker push $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG diff --git a/.ci/setup-make-parallelity.sh b/.ci/setup-make-parallelity.sh new file mode 100755 index 00000000000..77c1bbb9877 --- /dev/null +++ b/.ci/setup-make-parallelity.sh @@ -0,0 +1,66 @@ +#!/bin/sh + +# Source this to set CPUTHREADS (the number of apparent cores) and +# RAMTHREADS (free RAM divided by the maximum amount needed per thread +# typically) +# From this this script exports reasonable defaults for SAGE_NUM_THREADS and +# MAKEOPTS. + +# We do exactly the same for CPUTHREADS_DOCBUILD, RAMTHREADS_DOCBUILD, +# SAGE_NUM_THREADS_DOCBUILD, MAKEOPTS_DOCBUILD. As the docbuild needs +# substantially more RAM as of May 2018. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +if [ -z "$CPUTHREADS" ]; then + # Determine the number of threads that can run simultaneously on this system + # (we might not have nproc available.) + # Note that this value is incorrect for some CI providers (notably CircleCI: + # https://circleci.com/docs/2.0/configuration-reference/#resource_class) which + # provision fewer vCPUs than shown in /proc/cpuinfo. So it is probably better + # to set CPUTHREADS manuall in your CI configuration. + CPUTHREADS=`grep -E '^processor' /proc/cpuinfo | wc -l` +fi +if [ -z "$CPUTHREADS_DOCBUILD" ]; then + CPUTHREADS_DOCBUILD=$CPUTHREADS +fi + +if [ -z "$RAMTHREADS" ]; then + RAMTHREADS=$(( `grep MemTotal /proc/meminfo | awk '{ print $2 }'` / 1048576 )) + if [ $RAMTHREADS = 0 ];then + RAMTHREADS=1; + fi +fi +if [ -z "$RAMTHREADS_DOCBUILD" ]; then + RAMTHREADS_DOCBUILD=$(( `grep MemTotal /proc/meminfo | awk '{ print $2 }'` / 2097152 )) + if [ $RAMTHREADS_DOCBUILD = 0 ];then + RAMTHREADS_DOCBUILD=1; + fi +fi + +# On CI machines with their virtual CPUs, it seems to be quite beneficial to +# overcommit on CPU usage. We only need to make sure that we do not exceed RAM +# (as there is no swap.) +if [ $CPUTHREADS -lt $RAMTHREADS ]; then + export SAGE_NUM_THREADS=$((CPUTHREADS + 1)) +else + export SAGE_NUM_THREADS=$RAMTHREADS +fi +if [ $CPUTHREADS_DOCBUILD -lt $RAMTHREADS_DOCBUILD ]; then + export SAGE_NUM_THREADS_DOCBUILD=$((CPUTHREADS_DOCBUILD + 1)) +else + export SAGE_NUM_THREADS_DOCBUILD=$RAMTHREADS_DOCBUILD +fi +# Set -j and -l for make (though -l is probably ignored by Sage) +export MAKEOPTS="-j $SAGE_NUM_THREADS -l $((CPUTHREADS - 1)).8" +export MAKEOPTS_DOCBUILD="-j $SAGE_NUM_THREADS_DOCBUILD -l $((CPUTHREADS_DOCBUILD - 1)).8" diff --git a/.ci/test-cli.sh b/.ci/test-cli.sh new file mode 100755 index 00000000000..0ad6a8f3d0a --- /dev/null +++ b/.ci/test-cli.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# This script gets called from CI to run minimal tests on the sagemath image. + +# Usage: ./test-cli.sh IMAGE-NAME + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +echo "Checking that Sage starts and can calculate 1+1…" +# Calculate 1+1 (remove startup messages and leading & trailing whitespace) +TWO=`docker run "$1" sage -c "'print(1+1)'" | tail -1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'` +[ "x$TWO" = "x2" ] + +echo "Checking that some binaries that should be distributed with Sage are on the PATH…" +# We could also run minimal tests on these but we don't yet. +# Check that Singular and GAP are present +docker run "$1" which Singular +docker run "$1" which gap +# Check that jupyter is present (for binder) +docker run "$1" which jupyter diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh new file mode 100755 index 00000000000..5ba2cce16ca --- /dev/null +++ b/.ci/test-dev.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +# This script gets called from CI to run minimal tests on the sagemath-dev image. +# This script expects a single argument, the full name of the docker image to +# test. + +# Usage: ./test-dev.sh IMAGE-NAME + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +IMAGE="$1" + +. .ci/setup-make-parallelity.sh + +# Usage: timed_run limit args +# Runs $IMAGE with args and check that it terminates with a zero exit code in at most limit seconds. +timed_run() { + START=`date +%s` + docker run -e MAKEOPTS="$MAKEOPTS_DOCBUILD" -e SAGE_NUM_THREADS="$SAGE_NUM_THREADS_DOCBUILD" "$IMAGE" "$2" + END=`date +%s` + TOTAL=$((END-START)) + echo "Checking whether running \"$2\" was fast…" + [ "$TOTAL" -lt "$1" ] +} + +# Most setup should be done in well under 120 seconds but some CI machines tend +# to be really slow so we better give them some space here. Better miss a +# regression at first than create a lot of noise. +timed_run 120 true # runs make build +# Building the documentation is quite slow at the moment: +# Currently, this detects some dependencies as changed that have not changed. +# The parser in Sphinx fails to parse some .py files and adds the (more +# recently modified) .pyc files as dependencies instead. (Have a look the +# changeset that introduced this comment for more details.) +timed_run $(( 1200/$SAGE_NUM_THREADS_DOCBUILD )) make # runs make build and then make diff --git a/.ci/test-jupyter.sh b/.ci/test-jupyter.sh new file mode 100755 index 00000000000..9df157b763b --- /dev/null +++ b/.ci/test-jupyter.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +# This script gets called from CI to run minimal tests on the sagemath-jupyter +# image. + +# Usage: ./test-jupyter.sh IMAGE-NAME [HOST] + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +docker run --name sage-jupyter -p 8888:8888 -d "$1" sage-jupyter +echo "Checking that the Jupyter notebook is running…" +sleep 10 # giving the server some time to start +docker logs sage-jupyter +wget --retry-connrefused --tries=10 --wait=3 "http://${2:-localhost}:8888" diff --git a/.ci/update-env.sh b/.ci/update-env.sh new file mode 100755 index 00000000000..ee6a2110a56 --- /dev/null +++ b/.ci/update-env.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +# This script gets called from CI to establish the name of the current docker +# tag to build from the name of the branch/tag provided by CI. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +# From the docker documentation: "A tag name must be valid ASCII and may +# contain lowercase and uppercase letters, digits, underscores, periods and +# dashes. A tag name may not start with a period or a dash and may contain a +# maximum of 128 characters." +export DOCKER_TAG=`echo $DOCKER_TAG | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128` + +[[ -z "$DOCKER_TAG" ]] && export DOCKER_TAG=none +[[ "$DOCKER_TAG" = "master" ]] && export DOCKER_TAG=latest + +export DOCKER_IMAGE_CLI=${DOCKER_USER:-sagemath}/sagemath:$DOCKER_TAG +export DOCKER_IMAGE_DEV=${DOCKER_USER:-sagemath}/sagemath-dev:$DOCKER_TAG diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..27b27b1829c --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,69 @@ +# This file configures automatic builds of Sage on [CircleCI](https://circleci.com). +# To make the build time not too excessive, we seed the build cache with +# sagemath/sagemath-dev:develop. When basic SPKGs change, this does not help much, +# still the full build does usually not exceed CircleCI's limits for open +# source projcets (five hours on 2 vCPUs as of early 2018.) +# You might want to try to build locally or with GitLab CI, see +# `.gitlab-ci.yml` for more options. +# Note that we do not use workflows because it was not clear whether the docker +# images sizes would not exceed the size limits of CircleCI workspaces. Also, +# workflows would not make things faster. We do not get more machines for free +# and the frequent loading of docker images probably exceeds the cost of the +# actual tests. + +# As of early 2018, a run on CircleCI takes usually about 25 minutes. Most of +# the time is spent pulling/pushing from/to Docker Hub and copying files +# locally during the docker build. We could probably save five minutes by not +# building and testing the sagemath-dev image for most branches. + +version: 2 +jobs: + # As https://circleci.com/docs/2.0/docker-layer-caching/ is a paid feature, + # we do build & test & release in one step. + build: + machine: true + # Build for all tags + tags: + only: /.*/ + # Build for all branches + branches: + only: /.*/ + steps: + - checkout + - run: + # The docker commands sometimes take a while to produce output + no_output_timeout: 30m + command: | + # CircleCI has no mechanism to hide secret variables. + # Therefore we roll our own to protect $SECRET_* variables. + . .ci/protect-secrets.sh + # Collect debug infos about the system we are running on + .ci/describe-system.sh + # Set MAKEOPTS and SAGE_NUM_THREADS + . .ci/setup-make-parallelity.sh + + # Set DOCKER_TAG according to the current branch/tag + export DOCKER_TAG=${CIRCLE_TAG:-$CIRCLE_BRANCH} + . .ci/update-env.sh + + # Select ARTIFACT_BASE depending on the current branch/tag + if [ -n "$CIRCLE_TAG" ] || [ $DOCKER_TAG = "master" ] || [ $DOCKER_TAG = "develop" ];then + export ARTIFACT_BASE=source-clean + else + # Select sagemath/sagemath-dev:develop as the ARTIFACT_BASE + # unless the user has explicitly selected a differnt one as a + # CircleCI environment variable. + export ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop} + fi + + # Build docker images + .ci/build-docker.sh + + # Test that the images work + .ci/test-dev.sh $DOCKER_IMAGE_DEV + .ci/test-cli.sh $DOCKER_IMAGE_CLI + .ci/test-jupyter.sh $DOCKER_IMAGE_CLI localhost + + # Push docker images to dockerhub if a dockerhub user has been configured + .ci/push-dockerhub.sh sagemath-dev + .ci/push-dockerhub.sh sagemath diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 00000000000..3e4e48b0b5f --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index 101d349690c..c98ce30b813 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,14 @@ $RECYCLE.BIN/ ########### .ipynb_checkpoints Untitled*.ipynb + +############################# +# GitLab CI generated files # +############################# +gitlab-build-docker.log + +/src/.cython_version +/src/build +/src/Makefile +/src/bin/sage-env-config + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000000..dcf46066619 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,168 @@ +# This file configures automatic builds of Sage on [GitLab](https://gitlab.com). +# To make the build time not too excessive, we seed the build cache with +# sagemath/sagemath-dev:develop. When basic SPKGs changed, this does not help +# much and the full build might exceed the set time limit in GitLab. You can +# increase that limit in Settings → CI/CD. +# You can also provision your own private more powerful runner in the same +# place +# https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-in-docker-executor; +# or set up your favourite cloud service to provide an on-demand autoscale +# runner. More details below. + +# As of early 2018 a run on GitLab CI takes about 45 minutes. We could probably +# save 10 minutes by not building/pushing/testing dev images for branches other +# than master/develop. + +# Note that most of the time during CI is spent with pulling and pushing of +# docker images and copying files locally as part of the docker build. At the +# moment there is no reliable way of passing the docker images to the following +# stages without explicit pushing/pulling or similar: +# https://gitlab.com/gitlab-org/gitlab-runner/issues/1107 + +# The timings mentioned above are typical values. The shared runners provided +# on gitlab.com are sometimes much slower depending on the runner you are +# scheduled on. Sometimes it's slower for no apparent reason, probably just an +# overcommittment of virtual machines on hardware. + +# GitLab provides several flavours of shared runners (as of early 2018): +# * runners tagged as "do" (digitalocean.com) provide about 60GB of HDD, two +# cores, but only 2GB of RAM. The RAM is sometimes not sufficient to build +# the documentation. +# * runners tagged as "gce" (Google Compute Engine) provide about 22GB of HDD, +# a single core, 4GB of RAM. Since we are relying on OverlayFS, the disk +# space is not sufficient to build sage from scratch. + +# If you want to provide your own runners, make sure to tag them as follows: +# * "do" (60GB of disk space are available) to make build-from-clean pass. + +image: docker:latest + +stages: + - build + - test + - release + +variables: + DOCKER_TAG: $CI_COMMIT_REF_NAME + # Builds are very I/O intensive; make sure we have a fast file system. + DOCKER_DRIVER: overlay2 + DEFAULT_ARTIFACT_BASE: sagemath/sagemath-dev:develop + +before_script: + # GitLab has no mechanism yet to hide secret variables: https://gitlab.com/gitlab-org/gitlab-ce/issues/13784 + # So we roll our own which protects all variables that start with SECRET_ + - . .ci/protect-secrets.sh + # Collect debug infos about the system we are running on + - .ci/describe-system.sh + # Set DOCKER_TAG according to the current branch/tag + - . .ci/update-env.sh + # Set MAKEOPTS and SAGE_NUM_THREADS according to the machine we are running on + - . .ci/setup-make-parallelity.sh + +# We use docker-in-docker to build our docker images, i.e., we run a +# docker:dind "service" container and link to it from the container running the +# actual scripts below. +# Our scripts automatically connect to this service (unless you override it by +# setting DOCKER_HOST.) For example, each RUN statement in the Dockerfile +# spawns a docker container inside the docker:dind container to perform the RUN +# command there. +# It can be faster to expose your outer docker daemon by mounting +# /var/run/docker.sock to /var/run/docker.sock and setting DOCKER_HOST in +# Settings -> CI/CD -> Secret variable to unix:///var/run/docker.sock. (The +# speedup is mostly due to sharing layers of intermediate images.) However, +# this is only possible if you provision your own runners. Shared gitlab +# runners, do not bind mount /var/run/docker.sock. Also, docker:dind provides +# better isolation. If you expect many builds to run simultaneously on a host, +# conflicting tags can cause issues with a mounted DOCKER_HOST. +services: +- docker:dind + +# Build Sage and its documentation. +# The build starts from the build artifacts of ARTIFACT_BASE which is usually +# much faster than building from a clean checkout of Sage. +build-from-latest: &build + stage: build + variables: + ARTIFACT_BASE: $DEFAULT_ARTIFACT_BASE + artifacts: + when: always + paths: + - gitlab-build-docker.log + expire_in: 1 month + script: + - apk update && apk add coreutils + # The output of the build can get larger than gitlab.com's limit; only print the first 3MB (and the last 80 lines.) + - .ci/build-docker.sh | tee gitlab-build-docker.log | .ci/head-tail.sh 3145728 + - .ci/push-gitlab.sh sagemath-dev + - .ci/push-gitlab.sh sagemath + except: + - master + - develop + - tags + +# Build Sage and its documentation from a clean checkout of Sage. +# Note that this takes several hours. You probably want to run this on your own +# gitlab-runner and increase the standard GitLab time limit for CI runs. +# Some of the shared runners provided by GitLab for free do not have enough +# disk space for this to work. If a build fails with "no space left on device", +# you could just retry it and hope to be scheduled on a machine with more disk +# space, or provision your own runner. +build-from-clean: + << : *build + variables: + ARTIFACT_BASE: "source-clean" + only: + - master + - develop + - tags + except: [] + tags: + # 60 GB of HDD are available + - do + +test-dev: + stage: test + script: + - . .ci/pull-gitlab.sh sagemath-dev + - sh .ci/test-dev.sh "$DOCKER_IMAGE" + +test-cli: + stage: test + script: + - . .ci/pull-gitlab.sh sagemath + - sh .ci/test-cli.sh "$DOCKER_IMAGE" + +test-jupyter: + stage: test + script: + # Force usage of docker-in-docker (and don't start docker on a bind-mounted + # /var/run/docker.sock set through a private GitLab CI variable) so that + # the -p flag to docker run works as expected. + - export DOCKER_HOST='tcp://docker:2375' + - apk update + - apk add wget + - . .ci/pull-gitlab.sh sagemath + - sh .ci/test-jupyter.sh "$DOCKER_IMAGE" docker + +# Pushes the built images to Docker Hub if the Settings -> CI/CD -> Secret +# variables DOCKER_USER and SECRET_DOCKER_PASS have been set up. +push-dockerhub: + stage: release + only: + - branches + - tags + script: + - . .ci/pull-gitlab.sh sagemath + - sh .ci/push-dockerhub.sh sagemath + +# Pushes the built dev images to Docker Hub if the Settings -> CI/CD -> Secret +# variables DOCKER_USER and SECRET_DOCKER_PASS have been set up. +push-dockerhub-dev: + stage: release + only: + - master + - develop + - tags + script: + - . .ci/pull-gitlab.sh sagemath-dev + - sh .ci/push-dockerhub.sh sagemath-dev diff --git a/Makefile b/Makefile index d81d450ef4e..89ac3e4ecf1 100644 --- a/Makefile +++ b/Makefile @@ -104,9 +104,44 @@ bootstrap-clean: maintainer-clean: distclean bootstrap-clean rm -rf upstream +# Remove everything that is not necessary to run Sage and pass all its +# doctests. micro_release: bdist-clean sagelib-clean @echo "Stripping binaries ..." LC_ALL=C find local/lib local/bin -type f -exec strip '{}' ';' 2>&1 | grep -v "File format not recognized" | grep -v "File truncated" || true + @echo "Removing sphinx artifacts..." + rm -rf local/share/doc/sage/doctrees local/share/doc/sage/inventory + @echo "Removing documentation. Inspection in IPython still works." + rm -rf local/share/doc local/share/*/doc local/share/*/examples local/share/singular/html + @echo "Removing unnecessary files & directories - make will not be functional afterwards anymore" + @# We need src/doc/common, src/doc/en/introspect for introspection with "??" + @# We keep src/sage for some doctests that it expect it to be there and + @# also because it does not add any weight with rdfind below. + @# We need src/sage/bin/ for the scripts that invoke Sage + @# We need sage, the script to start Sage + @# We need local/, the dependencies and the built Sage library itself. + @# We keep VERSION.txt. + @# We keep COPYING.txt so we ship a license with this distribution. + find . -name . -o -prune ! -name src ! -name sage ! -name local ! -name VERSION.txt ! -name COPYING.txt ! -name build -exec rm -rf \{\} \; + cd src && find . -name . -o -prune ! -name sage ! -name bin ! -name doc -exec rm -rf \{\} \; + if command -v rdfind > /dev/null; then \ + echo "Hardlinking identical files."; \ + rdfind -makeresultsfile false -makehardlinks true .; \ + else \ + echo "rdfind not installed. Not hardlinking identical files."; \ + fi + +# Leaves everything that is needed to make the next "make" fast but removes +# all the cheap build artifacts that can be quickly regenerated. +fast-rebuild-clean: misc-clean bdist-clean + rm -rf upstream/ + rm -rf src/build/temp.* + # Without site-packages/sage sage does not start but copying/compiling + # them from src/build is very fast. + rm -rf local/lib/python*/site-packages/sage + # The .py files in src/build are restored from src/sage without their + # mtimes changed. + find src/build -name '*.py' -exec rm \{\} \; TESTALL = ./sage -t --all PTESTALL = ./sage -t -p --all diff --git a/configure.ac b/configure.ac index 569caf911cf..6a948395098 100644 --- a/configure.ac +++ b/configure.ac @@ -285,9 +285,9 @@ fi AC_CHECK_PROG(found_latex, latex, yes, no) if test x$found_latex != xyes then - AC_MSG_WARN([You do not have 'latex', which is recommended, but not]) - AC_MSG_WARN([required. Latex is only really used for building pdf]) - AC_MSG_WARN([documents and for %latex mode in the AC_PACKAGE_NAME notebook.]) + AC_MSG_NOTICE([You do not have 'latex', which is recommended, but not]) + AC_MSG_NOTICE([required. Latex is only really used for building pdf]) + AC_MSG_NOTICE([documents and for %latex mode in the AC_PACKAGE_NAME notebook.]) fi # Check that perl is available, with version 5.8.0 or later. diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 00000000000..579da245b87 --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1,2 @@ +# Stores the commit that was used to create a sagemath-dev image +.commit diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000000..325e5d7f922 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,257 @@ +################################################################################ +# SageMath images for Docker # +################################################################################ +# This is a description of the layout of this Dockerfile; for details on the # +# created docker images, see the README.md please. # +# # +# This Dockerfile builds sagemath (for end-users) and sagemath-dev (for # +# developers.) It consists of lots of intermediate targets, mostly to shrink # +# the resulting images but also to make this hopefully easier to maintain. # +# The aims of this Dockerfile are: # +# (1) Make it build in reasonable time. # +# (2) It should be self-contained and work on its own, i.e., just by invoking # +# docker build without any external orchestration script. # +# # +# The idea to achieve (1) is to reuse the build artifacts from the latest # +# develop build. This is slightly against the philosophy of a Dockerfile (which# +# should produce perfectly reproducible outputs) but building Sage from scratch# +# just takes too long at the moment to do this all the time. ARTIFACT_BASE # +# controls which build artifacts are used. You probably want to set this to # +# sagemath/sagemath-dev:develop which takes the latest build from the official # +# develop branch. The default is source-clean which builds Sage from scratch. # +# If you want to understand how this works, have a look at source-from-context # +# which merges ARTIFACT_BASE with the context, i.e., the contents of the sage # +# source directory. # +################################################################################ + +################################################################################ +# HOWTO use this file for local builds # +################################################################################ +# If you don't mind downloading a 2GB docker image from time to time, you # +# could use this file for local Sage development. As of early 2018 each build # +# takes about five minutes but you don't have to go through the sadly frequent # +# rebuilds the whole Sage distribution... # +# To build Sage, run this command from your sage/ directory: # +# $ docker build --build-arg MAKEOPTS="-j4" --build-arg SAGE_NUM_THREADS="4" --build-arg ARTIFACT_BASE="sagemath/sagemath-dev:develop" -f docker/Dockerfile --target=make-build --tag sage . +# To run Sage: # +# $ docker run -it sage # +# To run doctests: # +# $ docker run -e "MAKEOPTS=-j4" -e "SAGE_NUM_THREADS=4" -it sage sage -tp src/sage +# Make sure that you always have the latest develop branch merged into your # +# local branch for this to work. # +################################################################################ + +ARG ARTIFACT_BASE=source-clean + +################################################################################ +# Image containing the run-time dependencies for Sage # +################################################################################ +FROM ubuntu:xenial as run-time-dependencies +LABEL maintainer="Erik M. Bray , Julian Rüth " +# Set sane defaults for common environment variables. +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 +ENV SHELL /bin/bash +# Create symlinks for sage and sagemath - we copy a built sage to the target of these symlinks later. +ARG SAGE_ROOT=/home/sage/sage +RUN ln -s "$SAGE_ROOT/sage" /usr/bin/sage +RUN ln -s /usr/bin/sage /usr/bin/sagemath +# Sage needs the fortran libraries at run-time because we do not build gfortran +# with Sage but use the system's. +# We need gcc/g++ and libstdc++-5-dev to allow compilation of cython at run-time from the notebook. +# We also install sudo for the sage user, see below. +RUN apt-get -qq update \ + && apt-get -qq install -y --no-install-recommends gfortran gcc g++ libstdc++-5-dev sudo \ + && apt-get -qq clean \ + && rm -r /var/lib/apt/lists/* +# Sage refuses to build as root, so we add a "sage" user that can sudo without a password. +# We also want this user at runtime as some commands in sage know about the user that was used during build. +ARG HOME=/home/sage +RUN adduser --quiet --shell /bin/bash --gecos "Sage user,101,," --disabled-password --home "$HOME" sage \ + && echo "sage ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/01-sage \ + && chmod 0440 /etc/sudoers.d/01-sage +# Run everything from now on as the sage user in sage's home +USER sage +ENV HOME $HOME +WORKDIR $HOME + +################################################################################ +# Image containing everything so that a make in a clone of the Sage repository # +# completes without errors # +################################################################################ +FROM run-time-dependencies as build-time-dependencies +# Install the build time dependencies & git & rdfind +RUN sudo apt-get -qq update \ + && sudo apt-get -qq install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git rdfind \ + && sudo apt-get -qq clean \ + && sudo rm -r /var/lib/apt/lists/* + +################################################################################ +# Image with an empty git repository in $SAGE_ROOT. # +################################################################################ +FROM build-time-dependencies as source-clean +ARG SAGE_ROOT=/home/sage/sage +RUN mkdir -p "$SAGE_ROOT" +WORKDIR $SAGE_ROOT +RUN git init +RUN git remote add trac git@trac.sagemath.org:sage.git + +################################################################################ +# Image with the build context added, i.e., the directory from which `docker # +# build` has been called in a separate directory so we can copy files from # +# there. # +# This blows up the size of this docker image significantly, but we only use # +# this image to create artifacts for our final image. # +# Warning: If you set ARTIFACT_BASE to something else than source-clean, the # +# build is not going to use the build-time-dependencies target but rely on # +# whatever tools are installed in ARTIFACT_BASE. # +################################################################################ +FROM $ARTIFACT_BASE as source-from-context +WORKDIR $HOME +COPY --chown=sage:sage . sage-context +# Checkout the commit that checked out in $HOME/sage-context +# This is a bit complicated because our local .git/ is empty and we want to +# make sure that we only change the mtimes of a minimal number of files. +# 1) Restore the git checkout ARTIFACT_BASE was built from, recorded in +# docker/.commit. (Or go directly to FETCH_HEAD if there is no history to +# restore, i.e., set ARTIFACT_BASE=source-clean if you want to build from +# scratch.) +# 2) Merge in FETCH_HEAD but only if it is a fast-forward, i.e., if it is an +# ancestor of the commit restored in 1. If we would not do that we would get +# a new commit hash in docker/.commit that is not known outside of this build +# run. Since docker/.commit was in the history of FETCH_HEAD this should +# automatically be a fast-forward. +# 3) Trash .git again to save some space. +ARG SAGE_ROOT=/home/sage/sage +WORKDIR $SAGE_ROOT +# We create a list of all files present in the artifact-base (with a timestamp +# of now) so we can find out later which files were added/changed/removed. +RUN find . -type f > $HOME/artifact-base.manifest +RUN git fetch "$HOME/sage-context" HEAD \ + && if [ -e docker/.commit ]; then \ + git reset `cat docker/.commit` \ + || ( echo "Could not find commit `cat docker/.commit` in your local Git history. Please merge in the latest built develop branch to fix this: git fetch trac && git merge `cat docker/.commit`." && exit 1 ) \ + else \ + echo "You are building from $ARTIFACT_BASE which has no docker/.commit file. That's a bug unless you are building from source-clean or something similar." \ + && git reset FETCH_HEAD \ + && git checkout -f FETCH_HEAD; \ + fi \ + && git merge --ff-only FETCH_HEAD \ + && git log -1 --format=%H > docker/.commit \ + && rm -rf .git +# Copy over all the untracked/staged/unstaged changes from sage-context. This +# is relevant for non-CI invocations of this Dockerfile. +WORKDIR $HOME/sage-context +RUN if git status --porcelain | read CHANGES; then \ + git -c user.name=docker-build -c user.email=docker-build@sage.invalid stash -u \ + && git stash show -p > "$HOME"/sage-context.patch; \ + else \ + touch "$HOME"/sage-context.patch; \ + fi +WORKDIR $SAGE_ROOT +RUN patch -p1 < "$HOME"/sage-context.patch + +################################################################################ +# Image with a built sage but without sage's documentation. # +################################################################################ +FROM source-from-context as make-build +# Make sure that the result runs on most CPUs. +ENV SAGE_FAT_BINARY yes +# Just to be sure Sage doesn't try to build its own GCC (even though +# it shouldn't with a recent GCC package from the system and with gfortran) +ENV SAGE_INSTALL_GCC no +# Set MAKEOPTS and SAGE_NUM_THREADS to build things in parallel during the +# docker build. Note that these do not leak into the sagemath and sagemath-dev +# images. +ARG MAKEOPTS="-j2" +ENV MAKEOPTS $MAKEOPTS +ARG SAGE_NUM_THREADS="2" +ENV SAGE_NUM_THREADS $SAGE_NUM_THREADS +RUN make build + +################################################################################ +# Image with a full build of sage and its documentation. # +################################################################################ +FROM make-build as make-all +# The docbuild needs quite some RAM (as of May 2018). It sometimes calls +# os.fork() to spawn an external program which then exceeds easily the +# overcommit limit of the system (no RAM is actually used, but this limit is +# very low because there is not even swap on most CI systems.) +ARG MAKEOPTS_DOCBUILD=$MAKEOPTS +ENV MAKEOPTS_DOCBUILD $MAKEOPTS_DOCBUILD +ARG SAGE_NUM_THREADS_DOCBUILD=$SAGE_NUM_THREADS +ENV SAGE_NUM_THREADS_DOCBUILD $SAGE_NUM_THREADS_DOCBUILD +RUN make + +################################################################################ +# Image with a full build of sage, ready to release, i.e., with stripped # +# binaries and some extras to run the jupyter notebook. # +################################################################################ +FROM make-all as make-release +RUN sage -pip install terminado "notebook>=5" "ipykernel>=4.6" +RUN sage -i gap_jupyter singular_jupyter pari_jupyter +RUN make micro_release + +################################################################################ +# A releasable (relatively small) image which contains a copy of sage without # +# temporary build artifacts which is set up to start the command line # +# interface if no parameters are passed in. # +################################################################################ +FROM run-time-dependencies as sagemath +ARG SAGE_ROOT=/home/sage/sage +COPY --chown=sage:sage --from=make-release $SAGE_ROOT/ $SAGE_ROOT/ +# Put scripts to start gap, gp, maxima, ... in /usr/bin +WORKDIR $SAGE_ROOT +RUN sudo sage --nodotsage -c "install_scripts('/usr/bin')" +COPY ./docker/entrypoint.sh /usr/local/bin/sage-entrypoint +ENTRYPOINT ["/usr/local/bin/sage-entrypoint"] +EXPOSE 8888 +CMD ["sage"] + +################################################################################ +# Image with a full build of sage and its documentation but everything # +# stripped that can be quickly rebuild by make. # +################################################################################ +FROM make-all as make-fast-rebuild-clean +RUN make fast-rebuild-clean + +################################################################################ +# Depending on whether we built from source-clean or not, this image is either # +# identical to make-fast-rebuild-clean or contains a "patch" which can be used # +# to upgrade ARTIFACT_BASE to make-fast-rebuild-clean. # +################################################################################ +FROM make-fast-rebuild-clean as sagemath-dev-patch +ARG ARTIFACT_BASE=source-clean +ARG SAGE_ROOT=/home/sage/sage +# Build a patch containing of a tar file which contains all the modified files +# and a list of all modified files (added/updated/removed). +RUN if [ x"$ARTIFACT_BASE" != x"source-clean" ]; then \ + mkdir -p $HOME/patch \ + && find . -type f > $HOME/make-fast-rebuild-clean.manifest \ + && cat $HOME/make-fast-rebuild-clean.manifest $HOME/artifact-base.manifest | sort | uniq -u > $HOME/obsolete \ + && find . -type f -cnewer $HOME/artifact-base.manifest > $HOME/modified \ + && tar -cJf $HOME/patch/modified.tar.xz -T $HOME/modified \ + && cat $HOME/obsolete $HOME/modified | xz > $HOME/patch/modified.xz \ + && rm -rf $SAGE_ROOT \ + && mkdir -p $SAGE_ROOT \ + && mv $HOME/patch $SAGE_ROOT/; \ + fi + +################################################################################ +# A releasable (relatively small, but still huge) image of this build with all # +# the build artifacts intact so developers can make changes and rebuild # +# quickly # +################################################################################ +FROM $ARTIFACT_BASE as sagemath-dev +ARG SAGE_ROOT=/home/sage/sage +COPY --chown=sage:sage --from=sagemath-dev-patch $SAGE_ROOT $SAGE_ROOT +ARG ARTIFACT_BASE=source-clean +# Apply the patch from sagemath-dev-patch if we created one. +RUN if [ x"$ARTIFACT_BASE" != x"source-clean" ]; then \ + xzcat patch/modified.xz | xargs rm -rf \ + && tar -Jxf patch/modified.tar.xz \ + && rm -rf patch; \ + fi +COPY ./docker/entrypoint-dev.sh /usr/local/bin/sage-entrypoint +ENTRYPOINT ["/usr/local/bin/sage-entrypoint"] +CMD ["bash"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000000..b9f9240f9e4 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,55 @@ +[![License: GPL v3](https://img.shields.io/badge/License-GPL%20v3-blue.svg)](https://github.com/sagemath/sage/COPYING.txt) [![Maintained](https://img.shields.io/maintenance/yes/2018.svg)](https://github.com/sagemath/sage/commits/master) + +# Supported tags + +* `latest` — the stable `master` branch [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/sagemath/sage/master.svg)](https://github.com/sagemath/sage/commits/master) [![CircleCI branch](https://img.shields.io/circleci/project/github/sagemath/sage/master.svg)](https://circleci.com/gh/sagemath/sage/tree/master) [![GitLab CI](https://gitlab.com/sagemath/sage/badges/master/pipeline.svg)](https://gitlab.com/sagemath/sage/commits/master) +* `x.x` — all stable releases of Sage are tagged with their version number. +* `x.x.{beta,rc}x` - betas and release candidates of Sage as [tagged in our git repository](https://github.com/sagemath/sage/tags). +* `develop` — the current development version of Sage which gets merged into the `master` branch when a new version of Sage is released [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/sagemath/sage/develop.svg)](https://github.com/sagemath/sage/commits/develop) [![CircleCI branch](https://img.shields.io/circleci/project/github/sagemath/sage/master.svg)](https://circleci.com/gh/sagemath/sage/tree/master) [![GitLab CI](https://gitlab.com/sagemath/sage/badges/develop/pipeline.svg)](https://gitlab.com/sagemath/sage/commits/develop) + + +# What is SageMath + +SageMath is a free open-source mathematics software system licensed under the GPL. It builds on top of many existing open-source packages: NumPy, SciPy, matplotlib, Sympy, Maxima, GAP, FLINT, R and many more. Access their combined power through a common, Python-based language or directly via interfaces or wrappers. + +**Mission**: *Creating a viable free open source alternative to Magma, Maple, Mathematica and Matlab.* + +# What's in this image + +There are several flavours of this image. + +* [`sagemath/sagemath`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath/latest.svg)](https://hub.docker.com/r/sagemath/sagemath) contains everything necessary to run Sage on the command line. Run it with: + ``` + docker run -it sagemath/sagemath:latest + ``` + You can start a graphical [Jupyter Notebook](https://jupyter.org) at http://localhost:8888 instead. To use the notebook, follow the instructions printed when you run: + ``` + docker run -p8888:8888 sagemath/sagemath:latest sage-jupyter + ``` +* [`sagemath/sagemath-dev`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath-dev.svg)](https://hub.docker.com/r/sagemath/sagemath-dev) contains all the build artifacts to rebuild Sage quickly. This version is probably only relevant for Sage developers. Run this image with: + ``` + docker run -it sagemath/sagemath-dev:develop + ``` + This triggers a rebuild and drops you in a shell afterwards. Note that the git repository has been emptied to save space. If you want to use git, fetch from your git repository with `git fetch trac` and go to the commit that was used to create this image with + ``` + git reset $(cat docker/.commit) + ``` + +# How to build your own SageMath images + +Run `docker build -f docker/Dockerfile --build-arg ARTIFACT_BASE=sagemath/sagemath-dev:develop --target TARGET .` in the Sage repository with `TARGET` one of `sagemath` or `sagemath-dev`. + +# How these images get updated + +Every push to our [github repository](https://github.com/sagemath/sage) triggers a build in [CircleCI](https://circleci.com) which builds and pushes the docker images. +A push to master also triggers a "build" on our [Docker Hub](https://hub.docker.com) repositories. The latter build is mostly disabled by the `hooks/` and only updates the `README.md`. + +Every push to our [GitLab repository](https://gitlab.com/sagemath/sage) triggers a pipeline in GitLab CI. This build also pushes images to Docker Hub. + +Have a look at `.circleci/` and `.gitlab-ci.yml` if you want to setup either continuous integration service for your own fork of the SageMath repository. + +# License + +The whole Sage software distribution is licensed under the General Public License, version 3. More details can be found in our [COPYING.txt](https://github.com/sagemath/sage/blob/master/COPYING.txt) + +[//]: # (Please don't break long lines in this files as dockerhub then gets the formatting of this file wrong.) diff --git a/docker/entrypoint-dev.sh b/docker/entrypoint-dev.sh new file mode 100755 index 00000000000..67299a5a92d --- /dev/null +++ b/docker/entrypoint-dev.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e +make build +exec "$@" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000000..bc841382eaf --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash +if [ x"$1" = x"sage-jupyter" ]; then + # If "sage-jupyter" is given as a first argument, we start a jupyter notebook + # with reasonable default parameters for running it inside a container. + shift + exec sage -n jupyter --no-browser --ip='*' --port=8888 "$@" +else + exec sage -sh -c "$*" +fi diff --git a/docker/hooks/build b/docker/hooks/build new file mode 100644 index 00000000000..b23e55619b2 --- /dev/null +++ b/docker/hooks/build @@ -0,0 +1 @@ +#!/bin/true diff --git a/docker/hooks/push b/docker/hooks/push new file mode 100644 index 00000000000..b23e55619b2 --- /dev/null +++ b/docker/hooks/push @@ -0,0 +1 @@ +#!/bin/true diff --git a/src/.gitignore b/src/.gitignore deleted file mode 100644 index e85aa7f89e4..00000000000 --- a/src/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/.cython_version -/build -/Makefile -/bin/sage-env-config diff --git a/src/Makefile.in b/src/Makefile.in index 81a28b3758e..0233601cbbc 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -37,7 +37,7 @@ sage: SAGE_DOC_SRC=/doesnotexist \ SAGE_BUILD_DIR=/doesnotexist \ SAGE_PKGS=$(abs_top_srcdir)/build/pkgs \ - && sage-python23 -u setup.py --no-user-cfg build install + && sage-python23 -u setup.py --quiet --no-user-cfg build install if [ "$$UNAME" = "CYGWIN" ]; then \ sage-rebase.sh "$$SAGE_LOCAL" 2>/dev/null; \ fi diff --git a/src/doc/common/conf.py b/src/doc/common/conf.py index 9e57f8c761a..a9c4eb0fb6c 100644 --- a/src/doc/common/conf.py +++ b/src/doc/common/conf.py @@ -624,7 +624,7 @@ def call_intersphinx(app, env, node, contnode): sage: from sage.env import SAGE_DOC sage: thematic_index = os.path.join(SAGE_DOC, "html", "en", "thematic_tutorials", "index.html") - sage: for line in open(thematic_index).readlines(): + sage: for line in open(thematic_index).readlines(): # requires a built documentation, optional: doc ....: if "padics" in line: ....: sys.stdout.write(line)
  • Introduction to the -adics
    • diff --git a/src/sage/misc/sagedoc.py b/src/sage/misc/sagedoc.py index 9255aa848ff..ec2c435cd8a 100644 --- a/src/sage/misc/sagedoc.py +++ b/src/sage/misc/sagedoc.py @@ -20,7 +20,7 @@ sage: from sage.env import SAGE_DOC sage: docfilename = os.path.join(SAGE_DOC, 'html', 'en', 'reference', 'calculus', 'sage', 'symbolic', 'expression.html') - sage: with open(docfilename) as fobj: + sage: with open(docfilename) as fobj: # requires a built documentation, optional: doc ....: for line in fobj: ....: if "#sage.symbolic.expression.Expression.numerical_approx" in line: ....: print(line) @@ -1332,7 +1332,7 @@ class _sage_doc: EXAMPLES:: - sage: browse_sage_doc._open("reference", testing=True)[0] # indirect doctest + sage: browse_sage_doc._open("reference", testing=True)[0] # indirect doctest, requires a built documentation, optional: doc 'http://localhost:8000/doc/live/reference/index.html' sage: browse_sage_doc(identity_matrix, 'rst')[-107:-47] 'Full MatrixSpace of 3 by 3 sparse matrices over Integer Ring' @@ -1494,9 +1494,9 @@ def _open(self, name, testing=False): EXAMPLES:: - sage: browse_sage_doc._open("reference", testing=True)[0] + sage: browse_sage_doc._open("reference", testing=True)[0] # requires a built documentation, optional: doc 'http://localhost:8000/doc/live/reference/index.html' - sage: browse_sage_doc._open("tutorial", testing=True)[1] + sage: browse_sage_doc._open("tutorial", testing=True)[1] # requires a built documentation, optional: doc '.../html/en/tutorial/index.html' """ url = self._base_url + os.path.join(name, "index.html") diff --git a/src/sage_setup/docbuild/__init__.py b/src/sage_setup/docbuild/__init__.py index 5877b5b2a76..7dc8ff783c5 100644 --- a/src/sage_setup/docbuild/__init__.py +++ b/src/sage_setup/docbuild/__init__.py @@ -265,35 +265,29 @@ def clean(self, *args): # import the customized builder for object.inv files inventory = builder_helper('inventory') -if NUM_THREADS > 1: - def build_many(target, args): - from multiprocessing import Pool - pool = Pool(NUM_THREADS, maxtasksperchild=1) - # map_async handles KeyboardInterrupt correctly. Plain map and - # apply_async does not, so don't use it. - x = pool.map_async(target, args, 1) - try: - ret = x.get(99999) - pool.close() - pool.join() - except Exception: - pool.terminate() - if ABORT_ON_ERROR: - raise - return ret -else: - def build_many(target, args): - results = [] - - for arg in args: - try: - results.append(target(arg)) - except Exception: - if ABORT_ON_ERROR: - raise - - return results - +def build_many(target, args): + # Pool() uses an actual fork() to run each new instance. This is important + # for performance reasons, i.e., don't use a forkserver when it becomes + # available with Python 3: Here, sage is already initialized which is quite + # costly, with a forkserver we would have to reinitialize it for every + # document we build. At the same time, don't serialize this by taking the + # pool (and thus the call to fork()) out completely: The call to Sphinx + # leaks memory, so we need to build each document in its own process to + # control the RAM usage. + from multiprocessing import Pool + pool = Pool(NUM_THREADS, maxtasksperchild=1) + # map_async handles KeyboardInterrupt correctly. Plain map and + # apply_async does not, so don't use it. + x = pool.map_async(target, args, 1) + try: + ret = x.get(99999) + pool.close() + pool.join() + except Exception: + pool.terminate() + if ABORT_ON_ERROR: + raise + return ret ########################################## # Parallel Building Ref Manual # @@ -940,7 +934,12 @@ def get_new_and_updated_modules(self): except ImportError as err: logger.error("Warning: Could not import %s %s", module_name, err) raise - newtime = os.path.getmtime(sys.modules[module_name].__file__) + + module_filename = sys.modules[module_name].__file__ + if (module_filename.endswith('.pyc') or module_filename.endswith('.pyo')): + source_filename = module_filename[:-1] + if (os.path.exists(source_filename)): module_filename = source_filename + newtime = os.path.getmtime(module_filename) if newtime > mtime: updated_modules.append(module_name) diff --git a/src/sage_setup/docbuild/ext/sage_autodoc.py b/src/sage_setup/docbuild/ext/sage_autodoc.py index f241e09f8a5..2c399bad51b 100644 --- a/src/sage_setup/docbuild/ext/sage_autodoc.py +++ b/src/sage_setup/docbuild/ext/sage_autodoc.py @@ -988,7 +988,12 @@ def generate(self, more_content=None, real_modname=None, self.analyzer.find_attr_docs() except PycodeError as err: self.env.app.debug('[autodoc] module analyzer failed: %s', err) - # no source file -- e.g. for builtin and C modules + # A few things could have happened here: + # * there is no source file -- e.g. for builtin and C modules + # * the source file contains syntax that Sphinx can not parse, + # e.g., "print(1, end=' ')"; see + # https://github.com/sphinx-doc/sphinx/issues/1641, + # fixed in Sphinx 1.7. self.analyzer = None # at least add the module.__file__ as a dependency if hasattr(self.module, '__file__') and self.module.__file__: diff --git a/src/sage_setup/docbuild/sphinxbuild.py b/src/sage_setup/docbuild/sphinxbuild.py index fda76a41748..a7a6afa81ec 100644 --- a/src/sage_setup/docbuild/sphinxbuild.py +++ b/src/sage_setup/docbuild/sphinxbuild.py @@ -1,8 +1,25 @@ -""" +# -*- coding: utf-8 -*- +r""" This is Sage's version of the sphinx-build script -We redirect stdout to our own logger, and remove some unwanted chatter. +We redirect stdout and stderr to our own logger, and remove some unwanted chatter. """ +# **************************************************************************** +# Copyright (C) 2013-2014 Volker Braun +# 2013-2017 J. H. Palmieri < +# 2013-2017 Jeroen Demeyer +# 2014 Christopher Schwan +# 2014 Nicolas M. Thiéry +# 2015 Marc Mezzarobba +# 2015 André Apitzsch +# 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** import os, sys, re, sphinx @@ -14,9 +31,9 @@ def term_width_line(text): class SageSphinxLogger(object): - """ - This implements the file object interface to serve as sys.stdout - replacement. + r""" + This implements the file object interface to serve as + ``sys.stdout``/``sys.stderr`` replacement. """ ansi_color = re.compile(r'\x1b\[[0-9;]*m') ansi_reset = re.compile(r'\x1b\[39;49;00m') @@ -37,28 +54,63 @@ def __init__(self, stream, prefix): else: color = 'lightgray' self._prefix = sphinx.util.console.colorize(color, prefix) + # When we see an error in the log, we store it here and raise it at the + # end of the file (sometimes the lines following the error still + # contain valuable information.) + self._error = None def _init_chatter(self): - # useless_chatter: regular expressions to be filtered from - # Sphinx output. - self.useless_chatter = ( + # We drop any messages from the output that match these regular + # expressions. These just bloat the output and do not contain any + # information that we care about. + self._useless_chatter = ( re.compile('^$'), re.compile('^Running Sphinx v'), re.compile('^loading intersphinx inventory from '), + re.compile('^loading pickled environment... done'), + re.compile('^loading cross citations... done \([0-9]* citations\).'), re.compile('^Compiling a sub-document'), re.compile('^updating environment: 0 added, 0 changed, 0 removed'), re.compile('^looking for now-outdated files... none found'), re.compile('^building \[.*\]: targets for 0 source files that are out of date'), - re.compile('^loading pickled environment... done'), - re.compile('^loading cross citations... done \([0-9]* citations\).'), + re.compile('^building \[.*\]: targets for 0 po files that are out of date'), + re.compile('^building \[.*\]: targets for 0 mo files that are out of date'), + re.compile('^pickling environment... done'), + re.compile('^dumping object inventory... done'), + # We still have "Build finished." + re.compile('^build succeeded.'), + re.compile('^checking consistency... done'), + re.compile('^preparing documents... done'), + re.compile('^copying extra files... done'), + re.compile('^writing additional pages... search'), + re.compile('^Writing js search indexes...writing additional pages... .*'), + re.compile('^generating indices... .*'), + re.compile('^dumping search index in .* ... done'), + re.compile('^linking _static directory'), + re.compile('^copying static files... done'), + re.compile('^copying extra files... done'), + re.compile('^loading translations \[.*\]... done'), + re.compile('^Compiling the master document'), + re.compile('^Saved pickle file: citations.pickle'), + re.compile('^writing output... \[.*\] '), + re.compile('^copying images... \[.*\] '), + re.compile('^reading sources... \[.*\] '), + re.compile('language "hu" not supported'), + re.compile('^$'), + re.compile('^WARNING:$'), + ) + + # We fail whenever a line starts with "WARNING:", however, we ignore + # these warnings, as they are not relevant. + self._ignored_warnings = ( re.compile('WARNING: favicon file \'favicon.ico\' does not exist'), re.compile('WARNING: html_static_path entry .* does not exist'), re.compile('WARNING: while setting up extension'), re.compile('WARNING: Any IDs not assiend for figure node'), re.compile('WARNING: .* is not referenced'), re.compile('WARNING: Build finished'), - re.compile('language "hu" not supported'), ) + self._useless_chatter += self._ignored_warnings # replacements: pairs of regular expressions and their replacements, # to be applied to Sphinx output. @@ -68,14 +120,16 @@ def _init_chatter(self): if 'inventory' in sys.argv: # When building the inventory, ignore warnings about missing # citations and the search index. - self.useless_chatter += ( + ignored = ( re.compile('WARNING: citation not found:'), re.compile('WARNING: search index couldn\'t be loaded, but not all documents will be built: the index will be incomplete.') ) + self._ignored_warnings += ignored + self._useless_chatter += ignored - # warnings: regular expressions (or strings) indicating a problem with - # docbuilding. Raise an exception if any of these occur. - self.warnings = (re.compile('Segmentation fault'), + # Regular expressions indicating a problem with docbuilding. Raise an + # exception if any of these occur. + self._error_patterns = (re.compile('Segmentation fault'), re.compile('SEVERE'), re.compile('ERROR'), re.compile('^make.*Error'), @@ -88,66 +142,121 @@ def _init_chatter(self): # - undefined labels upon the first pass of the compilation: some # cross links may legitimately not yet be resolvable at this point. if 'latex' not in sys.argv: + self._error_patterns += (re.compile('WARNING:'),) if 'multidoc_first_pass=1' in sys.argv: - # Catch all warnings except 'WARNING: undefined label' - self.warnings += (re.compile('WARNING: (?!undefined label)'),) - else: - self.warnings += (re.compile('WARNING:'),) + ignore = (re.compile('WARNING: undefined label'),) + self._ignored_warnings += ignore + self._useless_chatter += ignore def _filter_out(self, line): - if exception is not None and self._is_stdout: + if self._error and self._is_stdout: # swallow non-errors after an error occurred return True line = re.sub(self.ansi_color, '', line) - for regex in self.useless_chatter: + line = line.strip() + for regex in self._useless_chatter: if regex.search(line) is not None: return True return False - def _check_warnings(self, line): - global exception - if exception is not None: + def _check_errors(self, line): + r""" + Search for errors in line. + + EXAMPLES:: + + sage: from sys import stdout + sage: from sage_setup.docbuild.sphinxbuild import SageSphinxLogger + sage: logger = SageSphinxLogger(stdout, "doctesting") + sage: logger._log_line("Segmentation fault!\n") # indirect doctest + [doctestin] Segmentation fault! + sage: logger.raise_errors() + Traceback (most recent call last): + ... + OSError: Segmentation fault! + + """ + if self._error: return # we already have found an error - for regex in self.warnings: - if regex.search(line) is not None: - exception = OSError(line) - return + for error in self._error_patterns: + if error.search(line) is not None: + for ignored in self._ignored_warnings: + if ignored.search(line) is not None: + break + else: + self._error = line + return def _log_line(self, line): - if self._filter_out(line): - return + r""" + Write ``line`` to the output stream with some mangling. + + EXAMPLES:: + + sage: from sys import stdout + sage: from sage_setup.docbuild.sphinxbuild import SageSphinxLogger + sage: logger = SageSphinxLogger(stdout, "doctesting") + sage: logger._log_line("building documentation…\n") + [doctestin] building documentation… + + TESTS: + + Verify that :trac:`25160` has been resolved:: + + sage: logger = SageSphinxLogger(stdout, "#25160") + sage: raise Exception("artificial exception") + Traceback (most recent call last): + ... + Exception: artificial exception + sage: import traceback + sage: for line in traceback.format_exc().split('\n'): + ....: logger._log_line(line) + [#25160 ] Traceback (most recent call last): + [#25160 ] File ... + [#25160 ] self.compile_and_execute(example, compiler, test.globs) + [#25160 ] File ... + [#25160 ] exec(compiled, globs) + [#25160 ] File ... + [#25160 ] raise Exception("artificial exception") + [#25160 ] Exception: artificial exception + + """ + skip_this_line = self._filter_out(line) + self._check_errors(line) for (old, new) in self.replacements: line = old.sub(new, line) - line = self._prefix + ' ' + line.strip() + '\n' + line = self._prefix + ' ' + line.rstrip() + '\n' if not self._color: line = self.ansi_color.sub('', line) - self._stream.write(line) - self._stream.flush() - self._check_warnings(line) + if not skip_this_line: + self._stream.write(line) + self._stream.flush() + + def raise_errors(self): + r""" + Raise an exceptions if any errors have been found while parsing the + Sphinx output. + + EXAMPLES:: + + sage: from sys import stdout + sage: from sage_setup.docbuild.sphinxbuild import SageSphinxLogger + sage: logger = SageSphinxLogger(stdout, "doctesting") + sage: logger._log_line("This is a SEVERE error\n") + [doctestin] This is a SEVERE error + sage: logger.raise_errors() + Traceback (most recent call last): + ... + OSError: This is a SEVERE error - _line_buffer = '' - - def _break_long_lines(self): - """ - Break text that has been formated with string.ljust() back - into individual lines. Return partial output. Do nothing if - the filter rule matches, otherwise subsequent lines would be - not filtered out. """ - if self._filter_out(self._line_buffer): - return - cols = sphinx.util.console._tw - lines = [] - buf = self._line_buffer - while len(buf) > cols: - lines.append(buf[0:cols]) - buf = buf[cols:] - lines.append(buf) - self._line_buffer = '\n'.join(lines) + if self._error: + raise OSError(self._error) + + _line_buffer = '' def _write(self, string): self._line_buffer += string - #self._break_long_lines() lines = self._line_buffer.splitlines() for i, line in enumerate(lines): last = (i == len(lines)-1) @@ -193,12 +302,6 @@ def writelines(self, sequence): def runsphinx(): - # Do not error out at the first warning, sometimes there is more - # information. So we run until the end of the file and only then - # raise the error. - global exception - exception = None - output_dir = sys.argv[-1] saved_stdout = sys.stdout @@ -207,12 +310,16 @@ def runsphinx(): try: sys.stdout = SageSphinxLogger(sys.stdout, os.path.basename(output_dir)) sys.stderr = SageSphinxLogger(sys.stderr, os.path.basename(output_dir)) + # Note that this call as of eraly 2018 leaks memory. So make sure that + # you don't call runsphinx() several times in a row. (i.e., you want to + # fork() somewhere before this call.) + # We don't use subprocess here, as we don't want to re-initialize Sage + # for every docbuild as this takes a while. sphinx.cmdline.main(sys.argv) + sys.stderr.raise_errors() + sys.stdout.raise_errors() finally: sys.stdout = saved_stdout sys.stderr = saved_stderr sys.stdout.flush() sys.stderr.flush() - - if exception is not None: - raise exception From 90ab8f67bf99b3fc988e562b8c82b051dbbc151a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sat, 19 May 2018 14:46:17 +0200 Subject: [PATCH 106/136] Put URL for anynomous access in the docker image so git fetch trac works without an SSH key --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 325e5d7f922..852dbb7af9d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -94,7 +94,7 @@ ARG SAGE_ROOT=/home/sage/sage RUN mkdir -p "$SAGE_ROOT" WORKDIR $SAGE_ROOT RUN git init -RUN git remote add trac git@trac.sagemath.org:sage.git +RUN git remote add trac git://trac.sagemath.org/sage.git ################################################################################ # Image with the build context added, i.e., the directory from which `docker # From 686fe86c7fb1f13f3d9d6ece7828107ff988e7c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 20 May 2018 22:56:23 +0200 Subject: [PATCH 107/136] An attempt to use workflows on CircleCI workflows do not really provide any advantage but they are necessary to build for a tagged branch, i.e., otherwise we can not tell CircleCI to build a for a git tag (such as 8.2) if we tell it to build for the git branch "master" at the same time. --- .ci/test-jupyter.sh | 5 +- .ci/update-env.sh | 6 ++- .circleci/before-script.sh | 26 ++++++++++ .circleci/config.yml | 103 ++++++++++++++++++++++--------------- .gitlab-ci.yml | 12 +---- 5 files changed, 96 insertions(+), 56 deletions(-) create mode 100644 .circleci/before-script.sh diff --git a/.ci/test-jupyter.sh b/.ci/test-jupyter.sh index 9df157b763b..b3ade38556a 100755 --- a/.ci/test-jupyter.sh +++ b/.ci/test-jupyter.sh @@ -16,9 +16,8 @@ # **************************************************************************** set -ex - -docker run --name sage-jupyter -p 8888:8888 -d "$1" sage-jupyter +docker run --name sage-jupyter -p -d "$1" sage-jupyter echo "Checking that the Jupyter notebook is running…" sleep 10 # giving the server some time to start docker logs sage-jupyter -wget --retry-connrefused --tries=10 --wait=3 "http://${2:-localhost}:8888" +docker run --link sage-jupyter alpine sh -c "apk --update add wget; wget --retry-connrefused --tries=10 --wait=3 http://sage-jupyter:8888" diff --git a/.ci/update-env.sh b/.ci/update-env.sh index ee6a2110a56..32ed5df6e8f 100755 --- a/.ci/update-env.sh +++ b/.ci/update-env.sh @@ -1,7 +1,7 @@ #!/bin/sh # This script gets called from CI to establish the name of the current docker -# tag to build from the name of the branch/tag provided by CI. +# tag to build and also the image which is used to seed the cache. # **************************************************************************** # Copyright (C) 2018 Julian Rüth @@ -26,3 +26,7 @@ export DOCKER_TAG=`echo $DOCKER_TAG | tr -d '[:space:]' | tr -c '[:alnum:]_.-' ' export DOCKER_IMAGE_CLI=${DOCKER_USER:-sagemath}/sagemath:$DOCKER_TAG export DOCKER_IMAGE_DEV=${DOCKER_USER:-sagemath}/sagemath-dev:$DOCKER_TAG + +# Seed the build cache with this image (set to source-clean to build from +# scratch.) +export ARTIFACT_BASE=${ARTIFACT_BASE:-$DEFAULT_ARTIFACT_BASE} diff --git a/.circleci/before-script.sh b/.circleci/before-script.sh new file mode 100644 index 00000000000..51127362f15 --- /dev/null +++ b/.circleci/before-script.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# Source this script during a CI run to set environment variables and print +# some informational messages about the system we are running on. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +# CircleCI has no mechanism to hide secret variables. +# Therefore we roll our own to protect $SECRET_* variables. +. .ci/protect-secrets.sh +# Collect debug infos about the system we are running on +.ci/describe-system.sh +# Set MAKEOPTS and SAGE_NUM_THREADS +. .ci/setup-make-parallelity.sh + +# Set DOCKER_TAG according to the current branch/tag +export DOCKER_TAG=${CIRCLE_TAG:-$CIRCLE_BRANCH} +. .ci/update-env.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 27b27b1829c..65c00dcced9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,11 +5,6 @@ # source projcets (five hours on 2 vCPUs as of early 2018.) # You might want to try to build locally or with GitLab CI, see # `.gitlab-ci.yml` for more options. -# Note that we do not use workflows because it was not clear whether the docker -# images sizes would not exceed the size limits of CircleCI workspaces. Also, -# workflows would not make things faster. We do not get more machines for free -# and the frequent loading of docker images probably exceeds the cost of the -# actual tests. # As of early 2018, a run on CircleCI takes usually about 25 minutes. Most of # the time is spent pulling/pushing from/to Docker Hub and copying files @@ -18,52 +13,76 @@ version: 2 jobs: - # As https://circleci.com/docs/2.0/docker-layer-caching/ is a paid feature, - # we do build & test & release in one step. - build: - machine: true - # Build for all tags - tags: - only: /.*/ - # Build for all branches - branches: - only: /.*/ + build-test-release: &build-test-release + docker: + - image: docker:latest + environment: + DEFAULT_ARTIFACT_BASE: sagemath/sagemath-dev:latest steps: + - run: apk --update add git openssh - checkout - - run: + - setup_remote_docker + - run: &build # The docker commands sometimes take a while to produce output no_output_timeout: 30m + name: build command: | - # CircleCI has no mechanism to hide secret variables. - # Therefore we roll our own to protect $SECRET_* variables. - . .ci/protect-secrets.sh - # Collect debug infos about the system we are running on - .ci/describe-system.sh - # Set MAKEOPTS and SAGE_NUM_THREADS - . .ci/setup-make-parallelity.sh - - # Set DOCKER_TAG according to the current branch/tag - export DOCKER_TAG=${CIRCLE_TAG:-$CIRCLE_BRANCH} - . .ci/update-env.sh - - # Select ARTIFACT_BASE depending on the current branch/tag - if [ -n "$CIRCLE_TAG" ] || [ $DOCKER_TAG = "master" ] || [ $DOCKER_TAG = "develop" ];then - export ARTIFACT_BASE=source-clean - else - # Select sagemath/sagemath-dev:develop as the ARTIFACT_BASE - # unless the user has explicitly selected a differnt one as a - # CircleCI environment variable. - export ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop} - fi - - # Build docker images + . .circleci/before-script.sh .ci/build-docker.sh - - # Test that the images work + - run: &test-dev + name: test-dev + command: | + . .circleci/before-script.sh .ci/test-dev.sh $DOCKER_IMAGE_DEV + - run: &test-cli + name: test-cli + command: | + . .circleci/before-script.sh .ci/test-cli.sh $DOCKER_IMAGE_CLI + - run: &test-jupyter + name: test-jupyter + command: | + . .circleci/before-script.sh .ci/test-jupyter.sh $DOCKER_IMAGE_CLI localhost - + - run: &release + # The docker commands sometimes take a while to produce output + no_output_timeout: 30m + name: release + command: | + . .circleci/before-script.sh # Push docker images to dockerhub if a dockerhub user has been configured .ci/push-dockerhub.sh sagemath-dev .ci/push-dockerhub.sh sagemath + build-from-latest-test-release: + <<: *build-test-release + build-from-clean-test-release: + <<: *build-test-release + environment: + ARTIFACT_BASE: source-clean + +workflows: + version: 2 + build-branch-from-clean: + jobs: + - build-from-clean-test-release: + filters: + branches: + only: + - master + - develop + build-tag-from-clean: + jobs: + - build-from-clean-test-release: + filters: + branches: + ignore: /.*/ + tags: + only: /.*/ + build-branch-from-latest: + jobs: + - build-from-latest-test-release: + filters: + branches: + ignore: + - master + - develop diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dcf46066619..b00a14498bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -78,12 +78,10 @@ services: - docker:dind # Build Sage and its documentation. -# The build starts from the build artifacts of ARTIFACT_BASE which is usually -# much faster than building from a clean checkout of Sage. +# The build starts from the build artifacts of DEFAULT_ARTIFACT_BASE which is +# usually much faster than building from a clean checkout of Sage. build-from-latest: &build stage: build - variables: - ARTIFACT_BASE: $DEFAULT_ARTIFACT_BASE artifacts: when: always paths: @@ -135,12 +133,6 @@ test-cli: test-jupyter: stage: test script: - # Force usage of docker-in-docker (and don't start docker on a bind-mounted - # /var/run/docker.sock set through a private GitLab CI variable) so that - # the -p flag to docker run works as expected. - - export DOCKER_HOST='tcp://docker:2375' - - apk update - - apk add wget - . .ci/pull-gitlab.sh sagemath - sh .ci/test-jupyter.sh "$DOCKER_IMAGE" docker From 79003ab53cb5dcc4ade75573021dec91c763360c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 20 May 2018 23:27:46 +0200 Subject: [PATCH 108/136] Do not deduce CPUs/RAM by looking at the local machine but by looking at the limits of the machine running the docker containers --- .ci/setup-make-parallelity.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.ci/setup-make-parallelity.sh b/.ci/setup-make-parallelity.sh index 77c1bbb9877..4e33e11b006 100755 --- a/.ci/setup-make-parallelity.sh +++ b/.ci/setup-make-parallelity.sh @@ -1,9 +1,8 @@ #!/bin/sh -# Source this to set CPUTHREADS (the number of apparent cores) and -# RAMTHREADS (free RAM divided by the maximum amount needed per thread -# typically) -# From this this script exports reasonable defaults for SAGE_NUM_THREADS and +# Source this to set CPUTHREADS (the number of apparent cores) and RAMTHREADS +# (free RAM divided by the maximum amount needed per thread typically) +# From this this script infers reasonable defaults for SAGE_NUM_THREADS and # MAKEOPTS. # We do exactly the same for CPUTHREADS_DOCBUILD, RAMTHREADS_DOCBUILD, @@ -29,20 +28,20 @@ if [ -z "$CPUTHREADS" ]; then # https://circleci.com/docs/2.0/configuration-reference/#resource_class) which # provision fewer vCPUs than shown in /proc/cpuinfo. So it is probably better # to set CPUTHREADS manuall in your CI configuration. - CPUTHREADS=`grep -E '^processor' /proc/cpuinfo | wc -l` + CPUTHREADS=`docker run docker cat /proc/cpuinfo | grep -E '^processor' | wc -l` fi if [ -z "$CPUTHREADS_DOCBUILD" ]; then CPUTHREADS_DOCBUILD=$CPUTHREADS fi if [ -z "$RAMTHREADS" ]; then - RAMTHREADS=$(( `grep MemTotal /proc/meminfo | awk '{ print $2 }'` / 1048576 )) + RAMTHREADS=$(( `docker run docker cat /proc/meminfo | grep MemTotal | awk '{ print $2 }'` / 1048576 )) if [ $RAMTHREADS = 0 ];then RAMTHREADS=1; fi fi if [ -z "$RAMTHREADS_DOCBUILD" ]; then - RAMTHREADS_DOCBUILD=$(( `grep MemTotal /proc/meminfo | awk '{ print $2 }'` / 2097152 )) + RAMTHREADS_DOCBUILD=$(( `docker run docker cat /proc/meminfo | grep MemTotal | awk '{ print $2 }'` / 2097152 )) if [ $RAMTHREADS_DOCBUILD = 0 ];then RAMTHREADS_DOCBUILD=1; fi From 906ebeab69c1307582d6f299c8062837f9af30ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 20 May 2018 23:37:51 +0200 Subject: [PATCH 109/136] Collect system information on the docker host not on the local machine. For docker-in-docker (gitlab) this is the same but for CircleCI this is vastly different. --- .ci/describe-system.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.ci/describe-system.sh b/.ci/describe-system.sh index 327243a4070..6bd3b0efd4a 100755 --- a/.ci/describe-system.sh +++ b/.ci/describe-system.sh @@ -12,10 +12,12 @@ set +e -x -uname -a -df -h -cat /proc/cpuinfo -cat /proc/meminfo -cat /proc/sys/vm/overcommit_memory -cat /proc/sys/vm/overcommit_ratio docker info +docker run docker sh -c " + set -x + uname -a + df -h + cat /proc/cpuinfo + cat /proc/meminfo + cat /proc/sys/vm/overcommit_memory + cat /proc/sys/vm/overcommit_ratio" From 60ddb4b0486ee73fcb608d3efe60532fa5300fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 20 May 2018 23:44:43 +0200 Subject: [PATCH 110/136] simplify apk add command for GitLab CI --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b00a14498bf..1caebb802db 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -88,7 +88,7 @@ build-from-latest: &build - gitlab-build-docker.log expire_in: 1 month script: - - apk update && apk add coreutils + - apk --update add coreutils # The output of the build can get larger than gitlab.com's limit; only print the first 3MB (and the last 80 lines.) - .ci/build-docker.sh | tee gitlab-build-docker.log | .ci/head-tail.sh 3145728 - .ci/push-gitlab.sh sagemath-dev From 25f934e5ec1bd8344b9e372f46387b569d379a8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 20 May 2018 23:54:27 +0200 Subject: [PATCH 111/136] Build from the latest develop image not from the latest (=master) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 65c00dcced9..bb28ca1b160 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,7 +17,7 @@ jobs: docker: - image: docker:latest environment: - DEFAULT_ARTIFACT_BASE: sagemath/sagemath-dev:latest + DEFAULT_ARTIFACT_BASE: sagemath/sagemath-dev:develop steps: - run: apk --update add git openssh - checkout From 11f8a6cc24e9a1853a85f45d48210bdfb35254ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 21 May 2018 00:34:07 +0200 Subject: [PATCH 112/136] no port exposure needed for jupyter anymore --- .ci/test-jupyter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test-jupyter.sh b/.ci/test-jupyter.sh index b3ade38556a..4578fb5cbd8 100755 --- a/.ci/test-jupyter.sh +++ b/.ci/test-jupyter.sh @@ -16,7 +16,7 @@ # **************************************************************************** set -ex -docker run --name sage-jupyter -p -d "$1" sage-jupyter +docker run --name sage-jupyter -d "$1" sage-jupyter echo "Checking that the Jupyter notebook is running…" sleep 10 # giving the server some time to start docker logs sage-jupyter From 1960ba1e0674435f222af242df6252d2b7ea50c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 21 May 2018 02:59:17 +0200 Subject: [PATCH 113/136] add openssl/libssl-dev to the docker images so pip works --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 852dbb7af9d..2ac43bda981 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -61,7 +61,7 @@ RUN ln -s /usr/bin/sage /usr/bin/sagemath # We need gcc/g++ and libstdc++-5-dev to allow compilation of cython at run-time from the notebook. # We also install sudo for the sage user, see below. RUN apt-get -qq update \ - && apt-get -qq install -y --no-install-recommends gfortran gcc g++ libstdc++-5-dev sudo \ + && apt-get -qq install -y --no-install-recommends gfortran gcc g++ libstdc++-5-dev sudo openssl \ && apt-get -qq clean \ && rm -r /var/lib/apt/lists/* # Sage refuses to build as root, so we add a "sage" user that can sudo without a password. @@ -82,7 +82,7 @@ WORKDIR $HOME FROM run-time-dependencies as build-time-dependencies # Install the build time dependencies & git & rdfind RUN sudo apt-get -qq update \ - && sudo apt-get -qq install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git rdfind \ + && sudo apt-get -qq install -y wget build-essential automake m4 dpkg-dev python libssl-dev git rdfind libssl-dev \ && sudo apt-get -qq clean \ && sudo rm -r /var/lib/apt/lists/* From 14926cfe70ca7f26f97cbf69a4f6e7298f6fc3d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 21 May 2018 14:59:09 +0200 Subject: [PATCH 114/136] Revert "add openssl/libssl-dev to the docker images" This reverts commit 1960ba1e0674435f222af242df6252d2b7ea50c8. sage -pip does not actually need this, so let's not install it. --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2ac43bda981..852dbb7af9d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -61,7 +61,7 @@ RUN ln -s /usr/bin/sage /usr/bin/sagemath # We need gcc/g++ and libstdc++-5-dev to allow compilation of cython at run-time from the notebook. # We also install sudo for the sage user, see below. RUN apt-get -qq update \ - && apt-get -qq install -y --no-install-recommends gfortran gcc g++ libstdc++-5-dev sudo openssl \ + && apt-get -qq install -y --no-install-recommends gfortran gcc g++ libstdc++-5-dev sudo \ && apt-get -qq clean \ && rm -r /var/lib/apt/lists/* # Sage refuses to build as root, so we add a "sage" user that can sudo without a password. @@ -82,7 +82,7 @@ WORKDIR $HOME FROM run-time-dependencies as build-time-dependencies # Install the build time dependencies & git & rdfind RUN sudo apt-get -qq update \ - && sudo apt-get -qq install -y wget build-essential automake m4 dpkg-dev python libssl-dev git rdfind libssl-dev \ + && sudo apt-get -qq install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git rdfind \ && sudo apt-get -qq clean \ && sudo rm -r /var/lib/apt/lists/* From f2c641a9d3a99fd0a1dc4f95c801c600e226f1f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 21 May 2018 15:01:27 +0200 Subject: [PATCH 115/136] Add debug information on patch size somehow I am again at 4GB for the sagemath-dev image. Let's see why this happened. --- .ci/build-docker.sh | 4 ++++ docker/Dockerfile | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh index 404362cc1bb..82fb2c87c5a 100755 --- a/.ci/build-docker.sh +++ b/.ci/build-docker.sh @@ -41,6 +41,10 @@ docker_build --target make-all --tag make-all:$DOCKER_TAG . # Build the release image without build artifacts. docker_build --target sagemath --tag "$DOCKER_IMAGE_CLI" . +# Display the layers of this image +docker history "$DOCKER_IMAGE_CLI" # Build the developer image with the build artifacts intact. # Note: It's important to build the dev image last because it might be tagged as ARTIFACT_BASE. docker_build --target sagemath-dev --tag "$DOCKER_IMAGE_DEV" . +# Display the layers of this image +docker history "$DOCKER_IMAGE_DEV" diff --git a/docker/Dockerfile b/docker/Dockerfile index 852dbb7af9d..0e772b987bd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -248,7 +248,8 @@ COPY --chown=sage:sage --from=sagemath-dev-patch $SAGE_ROOT $SAGE_ROOT ARG ARTIFACT_BASE=source-clean # Apply the patch from sagemath-dev-patch if we created one. RUN if [ x"$ARTIFACT_BASE" != x"source-clean" ]; then \ - xzcat patch/modified.xz | xargs rm -rf \ + echo "Applying `du -hs patch/modified.tar.xz` patch" \ + xzcat patch/modified.xz | xargs rm -rvf \ && tar -Jxf patch/modified.tar.xz \ && rm -rf patch; \ fi From 83f71e7948c8eb8a72e63d4d1451f9f20779bb8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 22 May 2018 22:32:20 +0200 Subject: [PATCH 116/136] Don't pipe "echo" into "rm" --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0e772b987bd..cb9bf4230d9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -249,7 +249,7 @@ ARG ARTIFACT_BASE=source-clean # Apply the patch from sagemath-dev-patch if we created one. RUN if [ x"$ARTIFACT_BASE" != x"source-clean" ]; then \ echo "Applying `du -hs patch/modified.tar.xz` patch" \ - xzcat patch/modified.xz | xargs rm -rvf \ + && xzcat patch/modified.xz | xargs rm -rvf \ && tar -Jxf patch/modified.tar.xz \ && rm -rf patch; \ fi From d28f601f53ddae2cecadc95854fbd91a525ac8f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 24 May 2018 23:55:25 +0200 Subject: [PATCH 117/136] Make sage -pip work I think I had tested this with the wrong sagemath docker image. pip does not work without these changes. (Probably it does not work with these changes either.) Revert "Revert "add openssl/libssl-dev to the docker images"" This reverts commit 14926cfe70ca7f26f97cbf69a4f6e7298f6fc3d3. --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index cb9bf4230d9..36f9d4ef76a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -61,7 +61,7 @@ RUN ln -s /usr/bin/sage /usr/bin/sagemath # We need gcc/g++ and libstdc++-5-dev to allow compilation of cython at run-time from the notebook. # We also install sudo for the sage user, see below. RUN apt-get -qq update \ - && apt-get -qq install -y --no-install-recommends gfortran gcc g++ libstdc++-5-dev sudo \ + && apt-get -qq install -y --no-install-recommends gfortran gcc g++ libstdc++-5-dev sudo openssl \ && apt-get -qq clean \ && rm -r /var/lib/apt/lists/* # Sage refuses to build as root, so we add a "sage" user that can sudo without a password. @@ -82,7 +82,7 @@ WORKDIR $HOME FROM run-time-dependencies as build-time-dependencies # Install the build time dependencies & git & rdfind RUN sudo apt-get -qq update \ - && sudo apt-get -qq install -y wget build-essential automake m4 dpkg-dev sudo python libssl-dev git rdfind \ + && sudo apt-get -qq install -y wget build-essential automake m4 dpkg-dev python libssl-dev git rdfind libssl-dev \ && sudo apt-get -qq clean \ && sudo rm -r /var/lib/apt/lists/* From b8e6077366e318a542299adfa1920424944e696c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Sun, 27 May 2018 01:33:43 +0200 Subject: [PATCH 118/136] Do not install libssl-dev twice --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 36f9d4ef76a..ad330cb7c13 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -82,7 +82,7 @@ WORKDIR $HOME FROM run-time-dependencies as build-time-dependencies # Install the build time dependencies & git & rdfind RUN sudo apt-get -qq update \ - && sudo apt-get -qq install -y wget build-essential automake m4 dpkg-dev python libssl-dev git rdfind libssl-dev \ + && sudo apt-get -qq install -y wget build-essential automake m4 dpkg-dev python libssl-dev git rdfind \ && sudo apt-get -qq clean \ && sudo rm -r /var/lib/apt/lists/* From 4ddd34683e4e85c5121c720c0d790c4acfca1347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Mon, 28 May 2018 14:10:53 +0200 Subject: [PATCH 119/136] Backport #24655 to build docker images --- .ci/README.md | 9 + .ci/build-docker.sh | 50 ++++ .ci/describe-system.sh | 23 ++ .ci/head-tail.sh | 45 ++++ .ci/protect-secrets.sh | 40 +++ .ci/pull-gitlab.sh | 30 +++ .ci/push-dockerhub.sh | 29 +++ .ci/push-gitlab.sh | 25 ++ .ci/setup-make-parallelity.sh | 65 +++++ .ci/test-cli.sh | 30 +++ .ci/test-dev.sh | 45 ++++ .ci/test-jupyter.sh | 23 ++ .ci/update-env.sh | 32 +++ .circleci/before-script.sh | 26 ++ .circleci/config.yml | 88 +++++++ .dockerignore | 1 + .gitignore | 11 + .gitlab-ci.yml | 160 ++++++++++++ Makefile | 35 +++ configure.ac | 6 +- docker/.gitignore | 2 + docker/Dockerfile | 258 ++++++++++++++++++++ docker/README.md | 55 +++++ docker/entrypoint-dev.sh | 4 + docker/entrypoint.sh | 9 + docker/hooks/build | 1 + docker/hooks/push | 1 + src/.gitignore | 4 - src/Makefile.in | 2 +- src/doc/common/conf.py | 2 +- src/sage/misc/sagedoc.py | 8 +- src/sage_setup/docbuild/__init__.py | 59 +++-- src/sage_setup/docbuild/ext/sage_autodoc.py | 7 +- src/sage_setup/docbuild/sphinxbuild.py | 95 +++++-- 34 files changed, 1210 insertions(+), 70 deletions(-) create mode 100644 .ci/README.md create mode 100755 .ci/build-docker.sh create mode 100755 .ci/describe-system.sh create mode 100755 .ci/head-tail.sh create mode 100755 .ci/protect-secrets.sh create mode 100755 .ci/pull-gitlab.sh create mode 100755 .ci/push-dockerhub.sh create mode 100755 .ci/push-gitlab.sh create mode 100755 .ci/setup-make-parallelity.sh create mode 100755 .ci/test-cli.sh create mode 100755 .ci/test-dev.sh create mode 100755 .ci/test-jupyter.sh create mode 100755 .ci/update-env.sh create mode 100644 .circleci/before-script.sh create mode 100644 .circleci/config.yml create mode 120000 .dockerignore create mode 100644 .gitlab-ci.yml create mode 100644 docker/.gitignore create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100755 docker/entrypoint-dev.sh create mode 100755 docker/entrypoint.sh create mode 100644 docker/hooks/build create mode 100644 docker/hooks/push delete mode 100644 src/.gitignore diff --git a/.ci/README.md b/.ci/README.md new file mode 100644 index 00000000000..e2b165cc518 --- /dev/null +++ b/.ci/README.md @@ -0,0 +1,9 @@ +# Continuous Integration (CI) + +We support several implementations of CI. All these implementations rely on +[docker](https://docker.com) in some way. This directory contains bits which +are shared between these CI implementations. The relevant docker files can be +found in `/docker/`. + +* [CircleCI](https://circleci.com) is configured in `/.circleci/`. +* [GitLab CI](https://gitlab.com) is configured in `/.gitlab-ci.yml`. diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh new file mode 100755 index 00000000000..82fb2c87c5a --- /dev/null +++ b/.ci/build-docker.sh @@ -0,0 +1,50 @@ +#!/bin/sh + +# This script gets called from CI to build several flavours of docker images +# which contain Sage. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +# We speed up the build process by copying built artifacts from ARTIFACT_BASE +# during docker build. See /docker/Dockerfile for more details. +ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop} + +# Seed our cache with $ARTIFACT_BASE if it exists. +docker pull "$ARTIFACT_BASE" > /dev/null || true + +docker_build() { + # Docker's --cache-from does not really work with multi-stage builds: https://github.com/moby/moby/issues/34715 + # So we just have to rely on the local cache. + time docker build -f docker/Dockerfile \ +--build-arg "MAKEOPTS=${MAKEOPTS}" --build-arg "SAGE_NUM_THREADS=${SAGE_NUM_THREADS}" --build-arg "MAKEOPTS_DOCBUILD=${MAKEOPTS}" --build-arg "SAGE_NUM_THREADS_DOCBUILD=${SAGE_NUM_THREADS_DOCBUILD}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ +} + +# We use a multi-stage build /docker/Dockerfile. For the caching to be +# effective, we populate the cache by building the run/build-time-dependencies +# and the make-all target. (Just building the last target is not enough as +# intermediate targets could be discarded from the cache [depending on the +# docker version] and therefore the caching would fail for our actual builds +# below.) +docker_build --target run-time-dependencies --tag run-time-dependencies:$DOCKER_TAG . +docker_build --target build-time-dependencies --tag build-time-dependencies:$DOCKER_TAG . +docker_build --target make-all --tag make-all:$DOCKER_TAG . + +# Build the release image without build artifacts. +docker_build --target sagemath --tag "$DOCKER_IMAGE_CLI" . +# Display the layers of this image +docker history "$DOCKER_IMAGE_CLI" +# Build the developer image with the build artifacts intact. +# Note: It's important to build the dev image last because it might be tagged as ARTIFACT_BASE. +docker_build --target sagemath-dev --tag "$DOCKER_IMAGE_DEV" . +# Display the layers of this image +docker history "$DOCKER_IMAGE_DEV" diff --git a/.ci/describe-system.sh b/.ci/describe-system.sh new file mode 100755 index 00000000000..6bd3b0efd4a --- /dev/null +++ b/.ci/describe-system.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set +e -x + +docker info +docker run docker sh -c " + set -x + uname -a + df -h + cat /proc/cpuinfo + cat /proc/meminfo + cat /proc/sys/vm/overcommit_memory + cat /proc/sys/vm/overcommit_ratio" diff --git a/.ci/head-tail.sh b/.ci/head-tail.sh new file mode 100755 index 00000000000..8ecf9f879f0 --- /dev/null +++ b/.ci/head-tail.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +OFFSET=80 +# This script reads from stdin and prints to stdout as long as a the output +# does not exceed a certain number of bytes. When reading an EOF it prints the +# last $OFFSET lines if they have not been printed normally already. +# This script expects one argument, the number of bytes. + +# Heavily inspired by a simlar strategy in make, https://stackoverflow.com/a/44849696/812379. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +stdbuf -i0 -o0 -e0 awk -v limit=$1 -v firstMissingNR=-1 -v offset=$OFFSET -v bytes=0 \ +'{ + if (bytes < limit) { + # this probably gets multi-byte characters wrong, but that probably does + # not matter for our purposes. (We add 1 for a UNIX newline.) + bytes += length($0) + 1; + print; + } else { + if (firstMissingNR == -1){ + print "[…output truncated…]"; + firstMissingNR = NR; + } + a[NR] = $0; + delete a[NR-offset]; + printf "." > "/dev/stderr" + } +} +END { + if (firstMissingNR != -1) { + print "" > "/dev/stderr"; + for(i = NR-offset+1 > firstMissingNR ? NR-offset-1 : firstMissingNR; i<=NR ; i++){ print a[i]; } + } +} +' + diff --git a/.ci/protect-secrets.sh b/.ci/protect-secrets.sh new file mode 100755 index 00000000000..ee781dddca8 --- /dev/null +++ b/.ci/protect-secrets.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +# This script protects all environment variables that start with "SECRET_". +# It puts them in a temporary file. The name of the variable contains the path +# of that file. This filename can then safely be used in `cat` even if `set +# -x` has been turned on. Also you can run "export" to understand the +# environment without danger. +# Be careful, however, not to use this like the following: +# docker login $DOCKER_USER $(cat $SECRET_DOCKER_PASS) +# as this would expose the password if `set -x` has been turned on. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -eo pipefail +set +x + +function encrypt { + RET=`mktemp` + eval " echo \$$1" > "$RET" + echo $RET +} + +for name in `awk 'END { for (name in ENVIRON) { print name; } }' < /dev/null`; do +case "$name" in + SECRET_*) + export $name="$(encrypt $name)" + echo "Protected $name" + ;; +esac +done + +unset encrypt diff --git a/.ci/pull-gitlab.sh b/.ci/pull-gitlab.sh new file mode 100755 index 00000000000..c8235e64068 --- /dev/null +++ b/.ci/pull-gitlab.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# This script gets called from CI to pull the Sage docker images that were +# built during the "build" phase to pull all the connected docker daemon +# (likely a docker-in-docker.) +# This script expects a single parameter, the base name of the docker image +# such as sagemath or sagemath-dev. +# The variable $DOCKER_IMAGE is set to the full name of the pulled image; +# source this script to use it. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +# Pull the built images from the gitlab registry and give them the original +# names they had after built. +# Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it +# automatically from the log output. +docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY +docker pull $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG +export DOCKER_IMAGE="${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG" +docker tag $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG $DOCKER_IMAGE diff --git a/.ci/push-dockerhub.sh b/.ci/push-dockerhub.sh new file mode 100755 index 00000000000..c2c46062ffa --- /dev/null +++ b/.ci/push-dockerhub.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# This script gets called from CI to push our docker images to +# $DOCKER_USER/sagemath* on the Docker Hub. +# This script expects a single parameter, the base name of the docker image +# such as sagemath or sagemath-dev. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +[ -z "$DOCKER_TAG" ] && (echo "Can not push untagged build."; exit 0) + +# Push the built images to the docker hub (and fail silently if +# DOCKER_USER/SECRET_DOCKER_PASS have not been configured.) +if [ -z "$DOCKER_USER" -o -z "$SECRET_DOCKER_PASS" ]; then + echo "DOCKER_USER/SECRET_DOCKER_PASS variables have not been configured in your Continuous Integration setup. Not pushing built images to Docker Hub." +else + cat "$SECRET_DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin + docker push ${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG +fi diff --git a/.ci/push-gitlab.sh b/.ci/push-gitlab.sh new file mode 100755 index 00000000000..e4ceb30abcd --- /dev/null +++ b/.ci/push-gitlab.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +# This script gets called from CI to push our docker images to registry +# configured in GitLab. (Mostly, so we can pull them again to push them to the +# Docker Hub.) +# This script expects a single parameter, the base name of the docker image +# such as sagemath or sagemath-dev. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +# Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it +# automatically from the log output. +docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY +docker tag ${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG +docker push $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG diff --git a/.ci/setup-make-parallelity.sh b/.ci/setup-make-parallelity.sh new file mode 100755 index 00000000000..4e33e11b006 --- /dev/null +++ b/.ci/setup-make-parallelity.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +# Source this to set CPUTHREADS (the number of apparent cores) and RAMTHREADS +# (free RAM divided by the maximum amount needed per thread typically) +# From this this script infers reasonable defaults for SAGE_NUM_THREADS and +# MAKEOPTS. + +# We do exactly the same for CPUTHREADS_DOCBUILD, RAMTHREADS_DOCBUILD, +# SAGE_NUM_THREADS_DOCBUILD, MAKEOPTS_DOCBUILD. As the docbuild needs +# substantially more RAM as of May 2018. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +if [ -z "$CPUTHREADS" ]; then + # Determine the number of threads that can run simultaneously on this system + # (we might not have nproc available.) + # Note that this value is incorrect for some CI providers (notably CircleCI: + # https://circleci.com/docs/2.0/configuration-reference/#resource_class) which + # provision fewer vCPUs than shown in /proc/cpuinfo. So it is probably better + # to set CPUTHREADS manuall in your CI configuration. + CPUTHREADS=`docker run docker cat /proc/cpuinfo | grep -E '^processor' | wc -l` +fi +if [ -z "$CPUTHREADS_DOCBUILD" ]; then + CPUTHREADS_DOCBUILD=$CPUTHREADS +fi + +if [ -z "$RAMTHREADS" ]; then + RAMTHREADS=$(( `docker run docker cat /proc/meminfo | grep MemTotal | awk '{ print $2 }'` / 1048576 )) + if [ $RAMTHREADS = 0 ];then + RAMTHREADS=1; + fi +fi +if [ -z "$RAMTHREADS_DOCBUILD" ]; then + RAMTHREADS_DOCBUILD=$(( `docker run docker cat /proc/meminfo | grep MemTotal | awk '{ print $2 }'` / 2097152 )) + if [ $RAMTHREADS_DOCBUILD = 0 ];then + RAMTHREADS_DOCBUILD=1; + fi +fi + +# On CI machines with their virtual CPUs, it seems to be quite beneficial to +# overcommit on CPU usage. We only need to make sure that we do not exceed RAM +# (as there is no swap.) +if [ $CPUTHREADS -lt $RAMTHREADS ]; then + export SAGE_NUM_THREADS=$((CPUTHREADS + 1)) +else + export SAGE_NUM_THREADS=$RAMTHREADS +fi +if [ $CPUTHREADS_DOCBUILD -lt $RAMTHREADS_DOCBUILD ]; then + export SAGE_NUM_THREADS_DOCBUILD=$((CPUTHREADS_DOCBUILD + 1)) +else + export SAGE_NUM_THREADS_DOCBUILD=$RAMTHREADS_DOCBUILD +fi +# Set -j and -l for make (though -l is probably ignored by Sage) +export MAKEOPTS="-j $SAGE_NUM_THREADS -l $((CPUTHREADS - 1)).8" +export MAKEOPTS_DOCBUILD="-j $SAGE_NUM_THREADS_DOCBUILD -l $((CPUTHREADS_DOCBUILD - 1)).8" diff --git a/.ci/test-cli.sh b/.ci/test-cli.sh new file mode 100755 index 00000000000..0ad6a8f3d0a --- /dev/null +++ b/.ci/test-cli.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# This script gets called from CI to run minimal tests on the sagemath image. + +# Usage: ./test-cli.sh IMAGE-NAME + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +echo "Checking that Sage starts and can calculate 1+1…" +# Calculate 1+1 (remove startup messages and leading & trailing whitespace) +TWO=`docker run "$1" sage -c "'print(1+1)'" | tail -1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'` +[ "x$TWO" = "x2" ] + +echo "Checking that some binaries that should be distributed with Sage are on the PATH…" +# We could also run minimal tests on these but we don't yet. +# Check that Singular and GAP are present +docker run "$1" which Singular +docker run "$1" which gap +# Check that jupyter is present (for binder) +docker run "$1" which jupyter diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh new file mode 100755 index 00000000000..5ba2cce16ca --- /dev/null +++ b/.ci/test-dev.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +# This script gets called from CI to run minimal tests on the sagemath-dev image. +# This script expects a single argument, the full name of the docker image to +# test. + +# Usage: ./test-dev.sh IMAGE-NAME + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +IMAGE="$1" + +. .ci/setup-make-parallelity.sh + +# Usage: timed_run limit args +# Runs $IMAGE with args and check that it terminates with a zero exit code in at most limit seconds. +timed_run() { + START=`date +%s` + docker run -e MAKEOPTS="$MAKEOPTS_DOCBUILD" -e SAGE_NUM_THREADS="$SAGE_NUM_THREADS_DOCBUILD" "$IMAGE" "$2" + END=`date +%s` + TOTAL=$((END-START)) + echo "Checking whether running \"$2\" was fast…" + [ "$TOTAL" -lt "$1" ] +} + +# Most setup should be done in well under 120 seconds but some CI machines tend +# to be really slow so we better give them some space here. Better miss a +# regression at first than create a lot of noise. +timed_run 120 true # runs make build +# Building the documentation is quite slow at the moment: +# Currently, this detects some dependencies as changed that have not changed. +# The parser in Sphinx fails to parse some .py files and adds the (more +# recently modified) .pyc files as dependencies instead. (Have a look the +# changeset that introduced this comment for more details.) +timed_run $(( 1200/$SAGE_NUM_THREADS_DOCBUILD )) make # runs make build and then make diff --git a/.ci/test-jupyter.sh b/.ci/test-jupyter.sh new file mode 100755 index 00000000000..4578fb5cbd8 --- /dev/null +++ b/.ci/test-jupyter.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# This script gets called from CI to run minimal tests on the sagemath-jupyter +# image. + +# Usage: ./test-jupyter.sh IMAGE-NAME [HOST] + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex +docker run --name sage-jupyter -d "$1" sage-jupyter +echo "Checking that the Jupyter notebook is running…" +sleep 10 # giving the server some time to start +docker logs sage-jupyter +docker run --link sage-jupyter alpine sh -c "apk --update add wget; wget --retry-connrefused --tries=10 --wait=3 http://sage-jupyter:8888" diff --git a/.ci/update-env.sh b/.ci/update-env.sh new file mode 100755 index 00000000000..32ed5df6e8f --- /dev/null +++ b/.ci/update-env.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +# This script gets called from CI to establish the name of the current docker +# tag to build and also the image which is used to seed the cache. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +# From the docker documentation: "A tag name must be valid ASCII and may +# contain lowercase and uppercase letters, digits, underscores, periods and +# dashes. A tag name may not start with a period or a dash and may contain a +# maximum of 128 characters." +export DOCKER_TAG=`echo $DOCKER_TAG | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128` + +[[ -z "$DOCKER_TAG" ]] && export DOCKER_TAG=none +[[ "$DOCKER_TAG" = "master" ]] && export DOCKER_TAG=latest + +export DOCKER_IMAGE_CLI=${DOCKER_USER:-sagemath}/sagemath:$DOCKER_TAG +export DOCKER_IMAGE_DEV=${DOCKER_USER:-sagemath}/sagemath-dev:$DOCKER_TAG + +# Seed the build cache with this image (set to source-clean to build from +# scratch.) +export ARTIFACT_BASE=${ARTIFACT_BASE:-$DEFAULT_ARTIFACT_BASE} diff --git a/.circleci/before-script.sh b/.circleci/before-script.sh new file mode 100644 index 00000000000..51127362f15 --- /dev/null +++ b/.circleci/before-script.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# Source this script during a CI run to set environment variables and print +# some informational messages about the system we are running on. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +# CircleCI has no mechanism to hide secret variables. +# Therefore we roll our own to protect $SECRET_* variables. +. .ci/protect-secrets.sh +# Collect debug infos about the system we are running on +.ci/describe-system.sh +# Set MAKEOPTS and SAGE_NUM_THREADS +. .ci/setup-make-parallelity.sh + +# Set DOCKER_TAG according to the current branch/tag +export DOCKER_TAG=${CIRCLE_TAG:-$CIRCLE_BRANCH} +. .ci/update-env.sh diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..bb28ca1b160 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,88 @@ +# This file configures automatic builds of Sage on [CircleCI](https://circleci.com). +# To make the build time not too excessive, we seed the build cache with +# sagemath/sagemath-dev:develop. When basic SPKGs change, this does not help much, +# still the full build does usually not exceed CircleCI's limits for open +# source projcets (five hours on 2 vCPUs as of early 2018.) +# You might want to try to build locally or with GitLab CI, see +# `.gitlab-ci.yml` for more options. + +# As of early 2018, a run on CircleCI takes usually about 25 minutes. Most of +# the time is spent pulling/pushing from/to Docker Hub and copying files +# locally during the docker build. We could probably save five minutes by not +# building and testing the sagemath-dev image for most branches. + +version: 2 +jobs: + build-test-release: &build-test-release + docker: + - image: docker:latest + environment: + DEFAULT_ARTIFACT_BASE: sagemath/sagemath-dev:develop + steps: + - run: apk --update add git openssh + - checkout + - setup_remote_docker + - run: &build + # The docker commands sometimes take a while to produce output + no_output_timeout: 30m + name: build + command: | + . .circleci/before-script.sh + .ci/build-docker.sh + - run: &test-dev + name: test-dev + command: | + . .circleci/before-script.sh + .ci/test-dev.sh $DOCKER_IMAGE_DEV + - run: &test-cli + name: test-cli + command: | + . .circleci/before-script.sh + .ci/test-cli.sh $DOCKER_IMAGE_CLI + - run: &test-jupyter + name: test-jupyter + command: | + . .circleci/before-script.sh + .ci/test-jupyter.sh $DOCKER_IMAGE_CLI localhost + - run: &release + # The docker commands sometimes take a while to produce output + no_output_timeout: 30m + name: release + command: | + . .circleci/before-script.sh + # Push docker images to dockerhub if a dockerhub user has been configured + .ci/push-dockerhub.sh sagemath-dev + .ci/push-dockerhub.sh sagemath + build-from-latest-test-release: + <<: *build-test-release + build-from-clean-test-release: + <<: *build-test-release + environment: + ARTIFACT_BASE: source-clean + +workflows: + version: 2 + build-branch-from-clean: + jobs: + - build-from-clean-test-release: + filters: + branches: + only: + - master + - develop + build-tag-from-clean: + jobs: + - build-from-clean-test-release: + filters: + branches: + ignore: /.*/ + tags: + only: /.*/ + build-branch-from-latest: + jobs: + - build-from-latest-test-release: + filters: + branches: + ignore: + - master + - develop diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 00000000000..3e4e48b0b5f --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index 101d349690c..c98ce30b813 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,14 @@ $RECYCLE.BIN/ ########### .ipynb_checkpoints Untitled*.ipynb + +############################# +# GitLab CI generated files # +############################# +gitlab-build-docker.log + +/src/.cython_version +/src/build +/src/Makefile +/src/bin/sage-env-config + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000000..1caebb802db --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,160 @@ +# This file configures automatic builds of Sage on [GitLab](https://gitlab.com). +# To make the build time not too excessive, we seed the build cache with +# sagemath/sagemath-dev:develop. When basic SPKGs changed, this does not help +# much and the full build might exceed the set time limit in GitLab. You can +# increase that limit in Settings → CI/CD. +# You can also provision your own private more powerful runner in the same +# place +# https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-in-docker-executor; +# or set up your favourite cloud service to provide an on-demand autoscale +# runner. More details below. + +# As of early 2018 a run on GitLab CI takes about 45 minutes. We could probably +# save 10 minutes by not building/pushing/testing dev images for branches other +# than master/develop. + +# Note that most of the time during CI is spent with pulling and pushing of +# docker images and copying files locally as part of the docker build. At the +# moment there is no reliable way of passing the docker images to the following +# stages without explicit pushing/pulling or similar: +# https://gitlab.com/gitlab-org/gitlab-runner/issues/1107 + +# The timings mentioned above are typical values. The shared runners provided +# on gitlab.com are sometimes much slower depending on the runner you are +# scheduled on. Sometimes it's slower for no apparent reason, probably just an +# overcommittment of virtual machines on hardware. + +# GitLab provides several flavours of shared runners (as of early 2018): +# * runners tagged as "do" (digitalocean.com) provide about 60GB of HDD, two +# cores, but only 2GB of RAM. The RAM is sometimes not sufficient to build +# the documentation. +# * runners tagged as "gce" (Google Compute Engine) provide about 22GB of HDD, +# a single core, 4GB of RAM. Since we are relying on OverlayFS, the disk +# space is not sufficient to build sage from scratch. + +# If you want to provide your own runners, make sure to tag them as follows: +# * "do" (60GB of disk space are available) to make build-from-clean pass. + +image: docker:latest + +stages: + - build + - test + - release + +variables: + DOCKER_TAG: $CI_COMMIT_REF_NAME + # Builds are very I/O intensive; make sure we have a fast file system. + DOCKER_DRIVER: overlay2 + DEFAULT_ARTIFACT_BASE: sagemath/sagemath-dev:develop + +before_script: + # GitLab has no mechanism yet to hide secret variables: https://gitlab.com/gitlab-org/gitlab-ce/issues/13784 + # So we roll our own which protects all variables that start with SECRET_ + - . .ci/protect-secrets.sh + # Collect debug infos about the system we are running on + - .ci/describe-system.sh + # Set DOCKER_TAG according to the current branch/tag + - . .ci/update-env.sh + # Set MAKEOPTS and SAGE_NUM_THREADS according to the machine we are running on + - . .ci/setup-make-parallelity.sh + +# We use docker-in-docker to build our docker images, i.e., we run a +# docker:dind "service" container and link to it from the container running the +# actual scripts below. +# Our scripts automatically connect to this service (unless you override it by +# setting DOCKER_HOST.) For example, each RUN statement in the Dockerfile +# spawns a docker container inside the docker:dind container to perform the RUN +# command there. +# It can be faster to expose your outer docker daemon by mounting +# /var/run/docker.sock to /var/run/docker.sock and setting DOCKER_HOST in +# Settings -> CI/CD -> Secret variable to unix:///var/run/docker.sock. (The +# speedup is mostly due to sharing layers of intermediate images.) However, +# this is only possible if you provision your own runners. Shared gitlab +# runners, do not bind mount /var/run/docker.sock. Also, docker:dind provides +# better isolation. If you expect many builds to run simultaneously on a host, +# conflicting tags can cause issues with a mounted DOCKER_HOST. +services: +- docker:dind + +# Build Sage and its documentation. +# The build starts from the build artifacts of DEFAULT_ARTIFACT_BASE which is +# usually much faster than building from a clean checkout of Sage. +build-from-latest: &build + stage: build + artifacts: + when: always + paths: + - gitlab-build-docker.log + expire_in: 1 month + script: + - apk --update add coreutils + # The output of the build can get larger than gitlab.com's limit; only print the first 3MB (and the last 80 lines.) + - .ci/build-docker.sh | tee gitlab-build-docker.log | .ci/head-tail.sh 3145728 + - .ci/push-gitlab.sh sagemath-dev + - .ci/push-gitlab.sh sagemath + except: + - master + - develop + - tags + +# Build Sage and its documentation from a clean checkout of Sage. +# Note that this takes several hours. You probably want to run this on your own +# gitlab-runner and increase the standard GitLab time limit for CI runs. +# Some of the shared runners provided by GitLab for free do not have enough +# disk space for this to work. If a build fails with "no space left on device", +# you could just retry it and hope to be scheduled on a machine with more disk +# space, or provision your own runner. +build-from-clean: + << : *build + variables: + ARTIFACT_BASE: "source-clean" + only: + - master + - develop + - tags + except: [] + tags: + # 60 GB of HDD are available + - do + +test-dev: + stage: test + script: + - . .ci/pull-gitlab.sh sagemath-dev + - sh .ci/test-dev.sh "$DOCKER_IMAGE" + +test-cli: + stage: test + script: + - . .ci/pull-gitlab.sh sagemath + - sh .ci/test-cli.sh "$DOCKER_IMAGE" + +test-jupyter: + stage: test + script: + - . .ci/pull-gitlab.sh sagemath + - sh .ci/test-jupyter.sh "$DOCKER_IMAGE" docker + +# Pushes the built images to Docker Hub if the Settings -> CI/CD -> Secret +# variables DOCKER_USER and SECRET_DOCKER_PASS have been set up. +push-dockerhub: + stage: release + only: + - branches + - tags + script: + - . .ci/pull-gitlab.sh sagemath + - sh .ci/push-dockerhub.sh sagemath + +# Pushes the built dev images to Docker Hub if the Settings -> CI/CD -> Secret +# variables DOCKER_USER and SECRET_DOCKER_PASS have been set up. +push-dockerhub-dev: + stage: release + only: + - master + - develop + - tags + script: + - . .ci/pull-gitlab.sh sagemath-dev + - sh .ci/push-dockerhub.sh sagemath-dev diff --git a/Makefile b/Makefile index d81d450ef4e..89ac3e4ecf1 100644 --- a/Makefile +++ b/Makefile @@ -104,9 +104,44 @@ bootstrap-clean: maintainer-clean: distclean bootstrap-clean rm -rf upstream +# Remove everything that is not necessary to run Sage and pass all its +# doctests. micro_release: bdist-clean sagelib-clean @echo "Stripping binaries ..." LC_ALL=C find local/lib local/bin -type f -exec strip '{}' ';' 2>&1 | grep -v "File format not recognized" | grep -v "File truncated" || true + @echo "Removing sphinx artifacts..." + rm -rf local/share/doc/sage/doctrees local/share/doc/sage/inventory + @echo "Removing documentation. Inspection in IPython still works." + rm -rf local/share/doc local/share/*/doc local/share/*/examples local/share/singular/html + @echo "Removing unnecessary files & directories - make will not be functional afterwards anymore" + @# We need src/doc/common, src/doc/en/introspect for introspection with "??" + @# We keep src/sage for some doctests that it expect it to be there and + @# also because it does not add any weight with rdfind below. + @# We need src/sage/bin/ for the scripts that invoke Sage + @# We need sage, the script to start Sage + @# We need local/, the dependencies and the built Sage library itself. + @# We keep VERSION.txt. + @# We keep COPYING.txt so we ship a license with this distribution. + find . -name . -o -prune ! -name src ! -name sage ! -name local ! -name VERSION.txt ! -name COPYING.txt ! -name build -exec rm -rf \{\} \; + cd src && find . -name . -o -prune ! -name sage ! -name bin ! -name doc -exec rm -rf \{\} \; + if command -v rdfind > /dev/null; then \ + echo "Hardlinking identical files."; \ + rdfind -makeresultsfile false -makehardlinks true .; \ + else \ + echo "rdfind not installed. Not hardlinking identical files."; \ + fi + +# Leaves everything that is needed to make the next "make" fast but removes +# all the cheap build artifacts that can be quickly regenerated. +fast-rebuild-clean: misc-clean bdist-clean + rm -rf upstream/ + rm -rf src/build/temp.* + # Without site-packages/sage sage does not start but copying/compiling + # them from src/build is very fast. + rm -rf local/lib/python*/site-packages/sage + # The .py files in src/build are restored from src/sage without their + # mtimes changed. + find src/build -name '*.py' -exec rm \{\} \; TESTALL = ./sage -t --all PTESTALL = ./sage -t -p --all diff --git a/configure.ac b/configure.ac index 569caf911cf..6a948395098 100644 --- a/configure.ac +++ b/configure.ac @@ -285,9 +285,9 @@ fi AC_CHECK_PROG(found_latex, latex, yes, no) if test x$found_latex != xyes then - AC_MSG_WARN([You do not have 'latex', which is recommended, but not]) - AC_MSG_WARN([required. Latex is only really used for building pdf]) - AC_MSG_WARN([documents and for %latex mode in the AC_PACKAGE_NAME notebook.]) + AC_MSG_NOTICE([You do not have 'latex', which is recommended, but not]) + AC_MSG_NOTICE([required. Latex is only really used for building pdf]) + AC_MSG_NOTICE([documents and for %latex mode in the AC_PACKAGE_NAME notebook.]) fi # Check that perl is available, with version 5.8.0 or later. diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 00000000000..579da245b87 --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1,2 @@ +# Stores the commit that was used to create a sagemath-dev image +.commit diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000000..ad330cb7c13 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,258 @@ +################################################################################ +# SageMath images for Docker # +################################################################################ +# This is a description of the layout of this Dockerfile; for details on the # +# created docker images, see the README.md please. # +# # +# This Dockerfile builds sagemath (for end-users) and sagemath-dev (for # +# developers.) It consists of lots of intermediate targets, mostly to shrink # +# the resulting images but also to make this hopefully easier to maintain. # +# The aims of this Dockerfile are: # +# (1) Make it build in reasonable time. # +# (2) It should be self-contained and work on its own, i.e., just by invoking # +# docker build without any external orchestration script. # +# # +# The idea to achieve (1) is to reuse the build artifacts from the latest # +# develop build. This is slightly against the philosophy of a Dockerfile (which# +# should produce perfectly reproducible outputs) but building Sage from scratch# +# just takes too long at the moment to do this all the time. ARTIFACT_BASE # +# controls which build artifacts are used. You probably want to set this to # +# sagemath/sagemath-dev:develop which takes the latest build from the official # +# develop branch. The default is source-clean which builds Sage from scratch. # +# If you want to understand how this works, have a look at source-from-context # +# which merges ARTIFACT_BASE with the context, i.e., the contents of the sage # +# source directory. # +################################################################################ + +################################################################################ +# HOWTO use this file for local builds # +################################################################################ +# If you don't mind downloading a 2GB docker image from time to time, you # +# could use this file for local Sage development. As of early 2018 each build # +# takes about five minutes but you don't have to go through the sadly frequent # +# rebuilds the whole Sage distribution... # +# To build Sage, run this command from your sage/ directory: # +# $ docker build --build-arg MAKEOPTS="-j4" --build-arg SAGE_NUM_THREADS="4" --build-arg ARTIFACT_BASE="sagemath/sagemath-dev:develop" -f docker/Dockerfile --target=make-build --tag sage . +# To run Sage: # +# $ docker run -it sage # +# To run doctests: # +# $ docker run -e "MAKEOPTS=-j4" -e "SAGE_NUM_THREADS=4" -it sage sage -tp src/sage +# Make sure that you always have the latest develop branch merged into your # +# local branch for this to work. # +################################################################################ + +ARG ARTIFACT_BASE=source-clean + +################################################################################ +# Image containing the run-time dependencies for Sage # +################################################################################ +FROM ubuntu:xenial as run-time-dependencies +LABEL maintainer="Erik M. Bray , Julian Rüth " +# Set sane defaults for common environment variables. +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 +ENV SHELL /bin/bash +# Create symlinks for sage and sagemath - we copy a built sage to the target of these symlinks later. +ARG SAGE_ROOT=/home/sage/sage +RUN ln -s "$SAGE_ROOT/sage" /usr/bin/sage +RUN ln -s /usr/bin/sage /usr/bin/sagemath +# Sage needs the fortran libraries at run-time because we do not build gfortran +# with Sage but use the system's. +# We need gcc/g++ and libstdc++-5-dev to allow compilation of cython at run-time from the notebook. +# We also install sudo for the sage user, see below. +RUN apt-get -qq update \ + && apt-get -qq install -y --no-install-recommends gfortran gcc g++ libstdc++-5-dev sudo openssl \ + && apt-get -qq clean \ + && rm -r /var/lib/apt/lists/* +# Sage refuses to build as root, so we add a "sage" user that can sudo without a password. +# We also want this user at runtime as some commands in sage know about the user that was used during build. +ARG HOME=/home/sage +RUN adduser --quiet --shell /bin/bash --gecos "Sage user,101,," --disabled-password --home "$HOME" sage \ + && echo "sage ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/01-sage \ + && chmod 0440 /etc/sudoers.d/01-sage +# Run everything from now on as the sage user in sage's home +USER sage +ENV HOME $HOME +WORKDIR $HOME + +################################################################################ +# Image containing everything so that a make in a clone of the Sage repository # +# completes without errors # +################################################################################ +FROM run-time-dependencies as build-time-dependencies +# Install the build time dependencies & git & rdfind +RUN sudo apt-get -qq update \ + && sudo apt-get -qq install -y wget build-essential automake m4 dpkg-dev python libssl-dev git rdfind \ + && sudo apt-get -qq clean \ + && sudo rm -r /var/lib/apt/lists/* + +################################################################################ +# Image with an empty git repository in $SAGE_ROOT. # +################################################################################ +FROM build-time-dependencies as source-clean +ARG SAGE_ROOT=/home/sage/sage +RUN mkdir -p "$SAGE_ROOT" +WORKDIR $SAGE_ROOT +RUN git init +RUN git remote add trac git://trac.sagemath.org/sage.git + +################################################################################ +# Image with the build context added, i.e., the directory from which `docker # +# build` has been called in a separate directory so we can copy files from # +# there. # +# This blows up the size of this docker image significantly, but we only use # +# this image to create artifacts for our final image. # +# Warning: If you set ARTIFACT_BASE to something else than source-clean, the # +# build is not going to use the build-time-dependencies target but rely on # +# whatever tools are installed in ARTIFACT_BASE. # +################################################################################ +FROM $ARTIFACT_BASE as source-from-context +WORKDIR $HOME +COPY --chown=sage:sage . sage-context +# Checkout the commit that checked out in $HOME/sage-context +# This is a bit complicated because our local .git/ is empty and we want to +# make sure that we only change the mtimes of a minimal number of files. +# 1) Restore the git checkout ARTIFACT_BASE was built from, recorded in +# docker/.commit. (Or go directly to FETCH_HEAD if there is no history to +# restore, i.e., set ARTIFACT_BASE=source-clean if you want to build from +# scratch.) +# 2) Merge in FETCH_HEAD but only if it is a fast-forward, i.e., if it is an +# ancestor of the commit restored in 1. If we would not do that we would get +# a new commit hash in docker/.commit that is not known outside of this build +# run. Since docker/.commit was in the history of FETCH_HEAD this should +# automatically be a fast-forward. +# 3) Trash .git again to save some space. +ARG SAGE_ROOT=/home/sage/sage +WORKDIR $SAGE_ROOT +# We create a list of all files present in the artifact-base (with a timestamp +# of now) so we can find out later which files were added/changed/removed. +RUN find . -type f > $HOME/artifact-base.manifest +RUN git fetch "$HOME/sage-context" HEAD \ + && if [ -e docker/.commit ]; then \ + git reset `cat docker/.commit` \ + || ( echo "Could not find commit `cat docker/.commit` in your local Git history. Please merge in the latest built develop branch to fix this: git fetch trac && git merge `cat docker/.commit`." && exit 1 ) \ + else \ + echo "You are building from $ARTIFACT_BASE which has no docker/.commit file. That's a bug unless you are building from source-clean or something similar." \ + && git reset FETCH_HEAD \ + && git checkout -f FETCH_HEAD; \ + fi \ + && git merge --ff-only FETCH_HEAD \ + && git log -1 --format=%H > docker/.commit \ + && rm -rf .git +# Copy over all the untracked/staged/unstaged changes from sage-context. This +# is relevant for non-CI invocations of this Dockerfile. +WORKDIR $HOME/sage-context +RUN if git status --porcelain | read CHANGES; then \ + git -c user.name=docker-build -c user.email=docker-build@sage.invalid stash -u \ + && git stash show -p > "$HOME"/sage-context.patch; \ + else \ + touch "$HOME"/sage-context.patch; \ + fi +WORKDIR $SAGE_ROOT +RUN patch -p1 < "$HOME"/sage-context.patch + +################################################################################ +# Image with a built sage but without sage's documentation. # +################################################################################ +FROM source-from-context as make-build +# Make sure that the result runs on most CPUs. +ENV SAGE_FAT_BINARY yes +# Just to be sure Sage doesn't try to build its own GCC (even though +# it shouldn't with a recent GCC package from the system and with gfortran) +ENV SAGE_INSTALL_GCC no +# Set MAKEOPTS and SAGE_NUM_THREADS to build things in parallel during the +# docker build. Note that these do not leak into the sagemath and sagemath-dev +# images. +ARG MAKEOPTS="-j2" +ENV MAKEOPTS $MAKEOPTS +ARG SAGE_NUM_THREADS="2" +ENV SAGE_NUM_THREADS $SAGE_NUM_THREADS +RUN make build + +################################################################################ +# Image with a full build of sage and its documentation. # +################################################################################ +FROM make-build as make-all +# The docbuild needs quite some RAM (as of May 2018). It sometimes calls +# os.fork() to spawn an external program which then exceeds easily the +# overcommit limit of the system (no RAM is actually used, but this limit is +# very low because there is not even swap on most CI systems.) +ARG MAKEOPTS_DOCBUILD=$MAKEOPTS +ENV MAKEOPTS_DOCBUILD $MAKEOPTS_DOCBUILD +ARG SAGE_NUM_THREADS_DOCBUILD=$SAGE_NUM_THREADS +ENV SAGE_NUM_THREADS_DOCBUILD $SAGE_NUM_THREADS_DOCBUILD +RUN make + +################################################################################ +# Image with a full build of sage, ready to release, i.e., with stripped # +# binaries and some extras to run the jupyter notebook. # +################################################################################ +FROM make-all as make-release +RUN sage -pip install terminado "notebook>=5" "ipykernel>=4.6" +RUN sage -i gap_jupyter singular_jupyter pari_jupyter +RUN make micro_release + +################################################################################ +# A releasable (relatively small) image which contains a copy of sage without # +# temporary build artifacts which is set up to start the command line # +# interface if no parameters are passed in. # +################################################################################ +FROM run-time-dependencies as sagemath +ARG SAGE_ROOT=/home/sage/sage +COPY --chown=sage:sage --from=make-release $SAGE_ROOT/ $SAGE_ROOT/ +# Put scripts to start gap, gp, maxima, ... in /usr/bin +WORKDIR $SAGE_ROOT +RUN sudo sage --nodotsage -c "install_scripts('/usr/bin')" +COPY ./docker/entrypoint.sh /usr/local/bin/sage-entrypoint +ENTRYPOINT ["/usr/local/bin/sage-entrypoint"] +EXPOSE 8888 +CMD ["sage"] + +################################################################################ +# Image with a full build of sage and its documentation but everything # +# stripped that can be quickly rebuild by make. # +################################################################################ +FROM make-all as make-fast-rebuild-clean +RUN make fast-rebuild-clean + +################################################################################ +# Depending on whether we built from source-clean or not, this image is either # +# identical to make-fast-rebuild-clean or contains a "patch" which can be used # +# to upgrade ARTIFACT_BASE to make-fast-rebuild-clean. # +################################################################################ +FROM make-fast-rebuild-clean as sagemath-dev-patch +ARG ARTIFACT_BASE=source-clean +ARG SAGE_ROOT=/home/sage/sage +# Build a patch containing of a tar file which contains all the modified files +# and a list of all modified files (added/updated/removed). +RUN if [ x"$ARTIFACT_BASE" != x"source-clean" ]; then \ + mkdir -p $HOME/patch \ + && find . -type f > $HOME/make-fast-rebuild-clean.manifest \ + && cat $HOME/make-fast-rebuild-clean.manifest $HOME/artifact-base.manifest | sort | uniq -u > $HOME/obsolete \ + && find . -type f -cnewer $HOME/artifact-base.manifest > $HOME/modified \ + && tar -cJf $HOME/patch/modified.tar.xz -T $HOME/modified \ + && cat $HOME/obsolete $HOME/modified | xz > $HOME/patch/modified.xz \ + && rm -rf $SAGE_ROOT \ + && mkdir -p $SAGE_ROOT \ + && mv $HOME/patch $SAGE_ROOT/; \ + fi + +################################################################################ +# A releasable (relatively small, but still huge) image of this build with all # +# the build artifacts intact so developers can make changes and rebuild # +# quickly # +################################################################################ +FROM $ARTIFACT_BASE as sagemath-dev +ARG SAGE_ROOT=/home/sage/sage +COPY --chown=sage:sage --from=sagemath-dev-patch $SAGE_ROOT $SAGE_ROOT +ARG ARTIFACT_BASE=source-clean +# Apply the patch from sagemath-dev-patch if we created one. +RUN if [ x"$ARTIFACT_BASE" != x"source-clean" ]; then \ + echo "Applying `du -hs patch/modified.tar.xz` patch" \ + && xzcat patch/modified.xz | xargs rm -rvf \ + && tar -Jxf patch/modified.tar.xz \ + && rm -rf patch; \ + fi +COPY ./docker/entrypoint-dev.sh /usr/local/bin/sage-entrypoint +ENTRYPOINT ["/usr/local/bin/sage-entrypoint"] +CMD ["bash"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000000..b9f9240f9e4 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,55 @@ +[![License: GPL v3](https://img.shields.io/badge/License-GPL%20v3-blue.svg)](https://github.com/sagemath/sage/COPYING.txt) [![Maintained](https://img.shields.io/maintenance/yes/2018.svg)](https://github.com/sagemath/sage/commits/master) + +# Supported tags + +* `latest` — the stable `master` branch [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/sagemath/sage/master.svg)](https://github.com/sagemath/sage/commits/master) [![CircleCI branch](https://img.shields.io/circleci/project/github/sagemath/sage/master.svg)](https://circleci.com/gh/sagemath/sage/tree/master) [![GitLab CI](https://gitlab.com/sagemath/sage/badges/master/pipeline.svg)](https://gitlab.com/sagemath/sage/commits/master) +* `x.x` — all stable releases of Sage are tagged with their version number. +* `x.x.{beta,rc}x` - betas and release candidates of Sage as [tagged in our git repository](https://github.com/sagemath/sage/tags). +* `develop` — the current development version of Sage which gets merged into the `master` branch when a new version of Sage is released [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/sagemath/sage/develop.svg)](https://github.com/sagemath/sage/commits/develop) [![CircleCI branch](https://img.shields.io/circleci/project/github/sagemath/sage/master.svg)](https://circleci.com/gh/sagemath/sage/tree/master) [![GitLab CI](https://gitlab.com/sagemath/sage/badges/develop/pipeline.svg)](https://gitlab.com/sagemath/sage/commits/develop) + + +# What is SageMath + +SageMath is a free open-source mathematics software system licensed under the GPL. It builds on top of many existing open-source packages: NumPy, SciPy, matplotlib, Sympy, Maxima, GAP, FLINT, R and many more. Access their combined power through a common, Python-based language or directly via interfaces or wrappers. + +**Mission**: *Creating a viable free open source alternative to Magma, Maple, Mathematica and Matlab.* + +# What's in this image + +There are several flavours of this image. + +* [`sagemath/sagemath`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath/latest.svg)](https://hub.docker.com/r/sagemath/sagemath) contains everything necessary to run Sage on the command line. Run it with: + ``` + docker run -it sagemath/sagemath:latest + ``` + You can start a graphical [Jupyter Notebook](https://jupyter.org) at http://localhost:8888 instead. To use the notebook, follow the instructions printed when you run: + ``` + docker run -p8888:8888 sagemath/sagemath:latest sage-jupyter + ``` +* [`sagemath/sagemath-dev`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath-dev.svg)](https://hub.docker.com/r/sagemath/sagemath-dev) contains all the build artifacts to rebuild Sage quickly. This version is probably only relevant for Sage developers. Run this image with: + ``` + docker run -it sagemath/sagemath-dev:develop + ``` + This triggers a rebuild and drops you in a shell afterwards. Note that the git repository has been emptied to save space. If you want to use git, fetch from your git repository with `git fetch trac` and go to the commit that was used to create this image with + ``` + git reset $(cat docker/.commit) + ``` + +# How to build your own SageMath images + +Run `docker build -f docker/Dockerfile --build-arg ARTIFACT_BASE=sagemath/sagemath-dev:develop --target TARGET .` in the Sage repository with `TARGET` one of `sagemath` or `sagemath-dev`. + +# How these images get updated + +Every push to our [github repository](https://github.com/sagemath/sage) triggers a build in [CircleCI](https://circleci.com) which builds and pushes the docker images. +A push to master also triggers a "build" on our [Docker Hub](https://hub.docker.com) repositories. The latter build is mostly disabled by the `hooks/` and only updates the `README.md`. + +Every push to our [GitLab repository](https://gitlab.com/sagemath/sage) triggers a pipeline in GitLab CI. This build also pushes images to Docker Hub. + +Have a look at `.circleci/` and `.gitlab-ci.yml` if you want to setup either continuous integration service for your own fork of the SageMath repository. + +# License + +The whole Sage software distribution is licensed under the General Public License, version 3. More details can be found in our [COPYING.txt](https://github.com/sagemath/sage/blob/master/COPYING.txt) + +[//]: # (Please don't break long lines in this files as dockerhub then gets the formatting of this file wrong.) diff --git a/docker/entrypoint-dev.sh b/docker/entrypoint-dev.sh new file mode 100755 index 00000000000..67299a5a92d --- /dev/null +++ b/docker/entrypoint-dev.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e +make build +exec "$@" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000000..bc841382eaf --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash +if [ x"$1" = x"sage-jupyter" ]; then + # If "sage-jupyter" is given as a first argument, we start a jupyter notebook + # with reasonable default parameters for running it inside a container. + shift + exec sage -n jupyter --no-browser --ip='*' --port=8888 "$@" +else + exec sage -sh -c "$*" +fi diff --git a/docker/hooks/build b/docker/hooks/build new file mode 100644 index 00000000000..b23e55619b2 --- /dev/null +++ b/docker/hooks/build @@ -0,0 +1 @@ +#!/bin/true diff --git a/docker/hooks/push b/docker/hooks/push new file mode 100644 index 00000000000..b23e55619b2 --- /dev/null +++ b/docker/hooks/push @@ -0,0 +1 @@ +#!/bin/true diff --git a/src/.gitignore b/src/.gitignore deleted file mode 100644 index e85aa7f89e4..00000000000 --- a/src/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/.cython_version -/build -/Makefile -/bin/sage-env-config diff --git a/src/Makefile.in b/src/Makefile.in index 81a28b3758e..0233601cbbc 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -37,7 +37,7 @@ sage: SAGE_DOC_SRC=/doesnotexist \ SAGE_BUILD_DIR=/doesnotexist \ SAGE_PKGS=$(abs_top_srcdir)/build/pkgs \ - && sage-python23 -u setup.py --no-user-cfg build install + && sage-python23 -u setup.py --quiet --no-user-cfg build install if [ "$$UNAME" = "CYGWIN" ]; then \ sage-rebase.sh "$$SAGE_LOCAL" 2>/dev/null; \ fi diff --git a/src/doc/common/conf.py b/src/doc/common/conf.py index 9e57f8c761a..a9c4eb0fb6c 100644 --- a/src/doc/common/conf.py +++ b/src/doc/common/conf.py @@ -624,7 +624,7 @@ def call_intersphinx(app, env, node, contnode): sage: from sage.env import SAGE_DOC sage: thematic_index = os.path.join(SAGE_DOC, "html", "en", "thematic_tutorials", "index.html") - sage: for line in open(thematic_index).readlines(): + sage: for line in open(thematic_index).readlines(): # requires a built documentation, optional: doc ....: if "padics" in line: ....: sys.stdout.write(line)
  • Introduction to the -adics
    • diff --git a/src/sage/misc/sagedoc.py b/src/sage/misc/sagedoc.py index 9255aa848ff..ec2c435cd8a 100644 --- a/src/sage/misc/sagedoc.py +++ b/src/sage/misc/sagedoc.py @@ -20,7 +20,7 @@ sage: from sage.env import SAGE_DOC sage: docfilename = os.path.join(SAGE_DOC, 'html', 'en', 'reference', 'calculus', 'sage', 'symbolic', 'expression.html') - sage: with open(docfilename) as fobj: + sage: with open(docfilename) as fobj: # requires a built documentation, optional: doc ....: for line in fobj: ....: if "#sage.symbolic.expression.Expression.numerical_approx" in line: ....: print(line) @@ -1332,7 +1332,7 @@ class _sage_doc: EXAMPLES:: - sage: browse_sage_doc._open("reference", testing=True)[0] # indirect doctest + sage: browse_sage_doc._open("reference", testing=True)[0] # indirect doctest, requires a built documentation, optional: doc 'http://localhost:8000/doc/live/reference/index.html' sage: browse_sage_doc(identity_matrix, 'rst')[-107:-47] 'Full MatrixSpace of 3 by 3 sparse matrices over Integer Ring' @@ -1494,9 +1494,9 @@ def _open(self, name, testing=False): EXAMPLES:: - sage: browse_sage_doc._open("reference", testing=True)[0] + sage: browse_sage_doc._open("reference", testing=True)[0] # requires a built documentation, optional: doc 'http://localhost:8000/doc/live/reference/index.html' - sage: browse_sage_doc._open("tutorial", testing=True)[1] + sage: browse_sage_doc._open("tutorial", testing=True)[1] # requires a built documentation, optional: doc '.../html/en/tutorial/index.html' """ url = self._base_url + os.path.join(name, "index.html") diff --git a/src/sage_setup/docbuild/__init__.py b/src/sage_setup/docbuild/__init__.py index 5877b5b2a76..7dc8ff783c5 100644 --- a/src/sage_setup/docbuild/__init__.py +++ b/src/sage_setup/docbuild/__init__.py @@ -265,35 +265,29 @@ def clean(self, *args): # import the customized builder for object.inv files inventory = builder_helper('inventory') -if NUM_THREADS > 1: - def build_many(target, args): - from multiprocessing import Pool - pool = Pool(NUM_THREADS, maxtasksperchild=1) - # map_async handles KeyboardInterrupt correctly. Plain map and - # apply_async does not, so don't use it. - x = pool.map_async(target, args, 1) - try: - ret = x.get(99999) - pool.close() - pool.join() - except Exception: - pool.terminate() - if ABORT_ON_ERROR: - raise - return ret -else: - def build_many(target, args): - results = [] - - for arg in args: - try: - results.append(target(arg)) - except Exception: - if ABORT_ON_ERROR: - raise - - return results - +def build_many(target, args): + # Pool() uses an actual fork() to run each new instance. This is important + # for performance reasons, i.e., don't use a forkserver when it becomes + # available with Python 3: Here, sage is already initialized which is quite + # costly, with a forkserver we would have to reinitialize it for every + # document we build. At the same time, don't serialize this by taking the + # pool (and thus the call to fork()) out completely: The call to Sphinx + # leaks memory, so we need to build each document in its own process to + # control the RAM usage. + from multiprocessing import Pool + pool = Pool(NUM_THREADS, maxtasksperchild=1) + # map_async handles KeyboardInterrupt correctly. Plain map and + # apply_async does not, so don't use it. + x = pool.map_async(target, args, 1) + try: + ret = x.get(99999) + pool.close() + pool.join() + except Exception: + pool.terminate() + if ABORT_ON_ERROR: + raise + return ret ########################################## # Parallel Building Ref Manual # @@ -940,7 +934,12 @@ def get_new_and_updated_modules(self): except ImportError as err: logger.error("Warning: Could not import %s %s", module_name, err) raise - newtime = os.path.getmtime(sys.modules[module_name].__file__) + + module_filename = sys.modules[module_name].__file__ + if (module_filename.endswith('.pyc') or module_filename.endswith('.pyo')): + source_filename = module_filename[:-1] + if (os.path.exists(source_filename)): module_filename = source_filename + newtime = os.path.getmtime(module_filename) if newtime > mtime: updated_modules.append(module_name) diff --git a/src/sage_setup/docbuild/ext/sage_autodoc.py b/src/sage_setup/docbuild/ext/sage_autodoc.py index f241e09f8a5..2c399bad51b 100644 --- a/src/sage_setup/docbuild/ext/sage_autodoc.py +++ b/src/sage_setup/docbuild/ext/sage_autodoc.py @@ -988,7 +988,12 @@ def generate(self, more_content=None, real_modname=None, self.analyzer.find_attr_docs() except PycodeError as err: self.env.app.debug('[autodoc] module analyzer failed: %s', err) - # no source file -- e.g. for builtin and C modules + # A few things could have happened here: + # * there is no source file -- e.g. for builtin and C modules + # * the source file contains syntax that Sphinx can not parse, + # e.g., "print(1, end=' ')"; see + # https://github.com/sphinx-doc/sphinx/issues/1641, + # fixed in Sphinx 1.7. self.analyzer = None # at least add the module.__file__ as a dependency if hasattr(self.module, '__file__') and self.module.__file__: diff --git a/src/sage_setup/docbuild/sphinxbuild.py b/src/sage_setup/docbuild/sphinxbuild.py index fadc7ab0184..25257adbdd6 100644 --- a/src/sage_setup/docbuild/sphinxbuild.py +++ b/src/sage_setup/docbuild/sphinxbuild.py @@ -2,7 +2,7 @@ r""" This is Sage's version of the sphinx-build script -We redirect stdout to our own logger, and remove some unwanted chatter. +We redirect stdout and stderr to our own logger, and remove some unwanted chatter. """ # **************************************************************************** # Copyright (C) 2013-2014 Volker Braun @@ -32,8 +32,8 @@ def term_width_line(text): class SageSphinxLogger(object): r""" - This implements the file object interface to serve as sys.stdout - replacement. + This implements the file object interface to serve as + ``sys.stdout``/``sys.stderr`` replacement. """ ansi_color = re.compile(r'\x1b\[[0-9;]*m') ansi_reset = re.compile(r'\x1b\[39;49;00m') @@ -60,26 +60,57 @@ def __init__(self, stream, prefix): self._error = None def _init_chatter(self): - # useless_chatter: regular expressions to be filtered from - # Sphinx output. - self.useless_chatter = ( + # We drop any messages from the output that match these regular + # expressions. These just bloat the output and do not contain any + # information that we care about. + self._useless_chatter = ( re.compile('^$'), re.compile('^Running Sphinx v'), re.compile('^loading intersphinx inventory from '), + re.compile('^loading pickled environment... done'), + re.compile('^loading cross citations... done \([0-9]* citations\).'), re.compile('^Compiling a sub-document'), re.compile('^updating environment: 0 added, 0 changed, 0 removed'), re.compile('^looking for now-outdated files... none found'), re.compile('^building \[.*\]: targets for 0 source files that are out of date'), - re.compile('^loading pickled environment... done'), - re.compile('^loading cross citations... done \([0-9]* citations\).'), + re.compile('^building \[.*\]: targets for 0 po files that are out of date'), + re.compile('^building \[.*\]: targets for 0 mo files that are out of date'), + re.compile('^pickling environment... done'), + re.compile('^dumping object inventory... done'), + # We still have "Build finished." + re.compile('^build succeeded.'), + re.compile('^checking consistency... done'), + re.compile('^preparing documents... done'), + re.compile('^copying extra files... done'), + re.compile('^writing additional pages... search'), + re.compile('^Writing js search indexes...writing additional pages... .*'), + re.compile('^generating indices... .*'), + re.compile('^dumping search index in .* ... done'), + re.compile('^linking _static directory'), + re.compile('^copying static files... done'), + re.compile('^copying extra files... done'), + re.compile('^loading translations \[.*\]... done'), + re.compile('^Compiling the master document'), + re.compile('^Saved pickle file: citations.pickle'), + re.compile('^writing output... \[.*\] '), + re.compile('^copying images... \[.*\] '), + re.compile('^reading sources... \[.*\] '), + re.compile('language "hu" not supported'), + re.compile('^$'), + re.compile('^WARNING:$'), + ) + + # We fail whenever a line starts with "WARNING:", however, we ignore + # these warnings, as they are not relevant. + self._ignored_warnings = ( re.compile('WARNING: favicon file \'favicon.ico\' does not exist'), re.compile('WARNING: html_static_path entry .* does not exist'), re.compile('WARNING: while setting up extension'), re.compile('WARNING: Any IDs not assiend for figure node'), re.compile('WARNING: .* is not referenced'), re.compile('WARNING: Build finished'), - re.compile('language "hu" not supported'), ) + self._useless_chatter += self._ignored_warnings # replacements: pairs of regular expressions and their replacements, # to be applied to Sphinx output. @@ -89,10 +120,12 @@ def _init_chatter(self): if 'inventory' in sys.argv: # When building the inventory, ignore warnings about missing # citations and the search index. - self.useless_chatter += ( + ignored = ( re.compile('WARNING: citation not found:'), re.compile('WARNING: search index couldn\'t be loaded, but not all documents will be built: the index will be incomplete.') ) + self._ignored_warnings += ignored + self._useless_chatter += ignored # Regular expressions indicating a problem with docbuilding. Raise an # exception if any of these occur. @@ -109,18 +142,19 @@ def _init_chatter(self): # - undefined labels upon the first pass of the compilation: some # cross links may legitimately not yet be resolvable at this point. if 'latex' not in sys.argv: + self._error_patterns += (re.compile('WARNING:'),) if 'multidoc_first_pass=1' in sys.argv: - # Catch all warnings except 'WARNING: undefined label' - self._error_patterns += (re.compile('WARNING: (?!undefined label)'),) - else: - self._error_patterns += (re.compile('WARNING:'),) + ignore = (re.compile('WARNING: undefined label'),) + self._ignored_warnings += ignore + self._useless_chatter += ignore def _filter_out(self, line): if self._error is not None and self._is_stdout: # swallow non-errors after an error occurred return True line = re.sub(self.ansi_color, '', line) - for regex in self.useless_chatter: + line = line.strip() + for regex in self._useless_chatter: if regex.search(line) is not None: return True return False @@ -139,15 +173,19 @@ def _check_errors(self, line): sage: logger.raise_errors() Traceback (most recent call last): ... - OSError: [doctestin] Segmentation fault! + OSError: Segmentation fault! """ if self._error is not None: return # we already have found an error - for regex in self._error_patterns: - if regex.search(line) is not None: - self._error = line - return + for error in self._error_patterns: + if error.search(line) is not None: + for ignored in self._ignored_warnings: + if ignored.search(line) is not None: + break + else: + self._error = line + return def _log_line(self, line): r""" @@ -183,16 +221,16 @@ def _log_line(self, line): [#25160 ] Exception: artificial exception """ - if self._filter_out(line): - return + skip_this_line = self._filter_out(line) + self._check_errors(line) for (old, new) in self.replacements: line = old.sub(new, line) line = self._prefix + ' ' + line.rstrip() + '\n' if not self._color: line = self.ansi_color.sub('', line) - self._stream.write(line) - self._stream.flush() - self._check_errors(line) + if not skip_this_line: + self._stream.write(line) + self._stream.flush() def raise_errors(self): r""" @@ -209,7 +247,7 @@ def raise_errors(self): sage: logger.raise_errors() Traceback (most recent call last): ... - OSError: [doctestin] This is a SEVERE error + OSError: This is a SEVERE error """ if self._error is not None: @@ -272,6 +310,11 @@ def runsphinx(): try: sys.stdout = SageSphinxLogger(sys.stdout, os.path.basename(output_dir)) sys.stderr = SageSphinxLogger(sys.stderr, os.path.basename(output_dir)) + # Note that this call as of eraly 2018 leaks memory. So make sure that + # you don't call runsphinx() several times in a row. (i.e., you want to + # fork() somewhere before this call.) + # We don't use subprocess here, as we don't want to re-initialize Sage + # for every docbuild as this takes a while. sphinx.cmdline.main(sys.argv) sys.stderr.raise_errors() sys.stdout.raise_errors() From 511bf3ebfc3c5fef8320c5116592aca875971b85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 1 Jun 2018 16:54:43 +0200 Subject: [PATCH 120/136] Introduce monkey-patching in CI --- .ci/update-env.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.ci/update-env.sh b/.ci/update-env.sh index 32ed5df6e8f..22c7a3a7ce3 100755 --- a/.ci/update-env.sh +++ b/.ci/update-env.sh @@ -15,6 +15,18 @@ set -ex +# The maintainer of the CI environment, e.g., the people administrating the +# SageMath account on gitlab.com, can decide to inject an arbitrary +# base64-encoded script early into the CI execution. The script has access to +# all the CI variables, e.g., to find out which job is being executed, and +# it could do things such as "exit 1" to fail early, or "git merge" a fix for a +# known bug in CI. The CI_MONKEY_PATCH could of course also curl a more +# complicated script and execute that. +if [ -n "$CI_MONKEY_PATCH" ]; then + SCRIPT=$(base64 -d <<< "$CI_MONKEY_PATCH") + $SCRIPT +fi + # From the docker documentation: "A tag name must be valid ASCII and may # contain lowercase and uppercase letters, digits, underscores, periods and # dashes. A tag name may not start with a period or a dash and may contain a From 162fe52d0efcc665e71520d6aedb61faa1785a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 5 Jun 2018 12:19:26 +0200 Subject: [PATCH 121/136] Reset WORKDIR to HOME nthiery reported that this used to be the case with our previous 8.1 images. So let's not break compatibility. --- docker/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ad330cb7c13..18df023aa23 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -198,12 +198,13 @@ RUN make micro_release # interface if no parameters are passed in. # ################################################################################ FROM run-time-dependencies as sagemath +ARG HOME=/home/sage ARG SAGE_ROOT=/home/sage/sage COPY --chown=sage:sage --from=make-release $SAGE_ROOT/ $SAGE_ROOT/ # Put scripts to start gap, gp, maxima, ... in /usr/bin -WORKDIR $SAGE_ROOT -RUN sudo sage --nodotsage -c "install_scripts('/usr/bin')" +RUN sudo $SAGE_ROOT/sage --nodotsage -c "install_scripts('/usr/bin')" COPY ./docker/entrypoint.sh /usr/local/bin/sage-entrypoint +WORKDIR $HOME ENTRYPOINT ["/usr/local/bin/sage-entrypoint"] EXPOSE 8888 CMD ["sage"] From 4ef4e1830f78d45e9faf0ac48fa44f9d57da33ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 5 Jun 2018 12:26:55 +0200 Subject: [PATCH 122/136] Where to report bugs for the docker images --- docker/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/README.md b/docker/README.md index b9f9240f9e4..216fcc77364 100644 --- a/docker/README.md +++ b/docker/README.md @@ -48,6 +48,10 @@ Every push to our [GitLab repository](https://gitlab.com/sagemath/sage) triggers Have a look at `.circleci/` and `.gitlab-ci.yml` if you want to setup either continuous integration service for your own fork of the SageMath repository. +# Report bugs and issues + +Please tell us of any bugs or omissions at our [Issue Tracker](https://trac.sagemath.org) or contact us through the [sage-support](https://groups.google.com/forum/#!forum/sage-support) or the [sage-devel](https://groups.google.com/forum/#!forum/sage-devel) mailing lists. + # License The whole Sage software distribution is licensed under the General Public License, version 3. More details can be found in our [COPYING.txt](https://github.com/sagemath/sage/blob/master/COPYING.txt) From 6a3d5d0c70310ddcf28ac684c56bf279d189179c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 5 Jun 2018 12:31:11 +0200 Subject: [PATCH 123/136] drop trailing space --- .ci/protect-secrets.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/protect-secrets.sh b/.ci/protect-secrets.sh index ee781dddca8..527604106ca 100755 --- a/.ci/protect-secrets.sh +++ b/.ci/protect-secrets.sh @@ -29,7 +29,7 @@ function encrypt { } for name in `awk 'END { for (name in ENVIRON) { print name; } }' < /dev/null`; do -case "$name" in +case "$name" in SECRET_*) export $name="$(encrypt $name)" echo "Protected $name" From f7ea0f3c7a10baf475e96bd0bacd09f06039b14b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 5 Jun 2018 12:47:01 +0200 Subject: [PATCH 124/136] Make update-env posix compliant again --- .ci/update-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/update-env.sh b/.ci/update-env.sh index 22c7a3a7ce3..574595e48ac 100755 --- a/.ci/update-env.sh +++ b/.ci/update-env.sh @@ -23,7 +23,7 @@ set -ex # known bug in CI. The CI_MONKEY_PATCH could of course also curl a more # complicated script and execute that. if [ -n "$CI_MONKEY_PATCH" ]; then - SCRIPT=$(base64 -d <<< "$CI_MONKEY_PATCH") + SCRIPT=$(echo "$CI_MONKEY_PATCH" | base64 -d) $SCRIPT fi From 5d9b1a5c9ec4588567a8f34bdb895082a6c1c26d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Wed, 4 Jul 2018 17:32:08 +0200 Subject: [PATCH 125/136] Allow to push to another's user's namespace on the Docker Hub --- .ci/push-dockerhub.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/push-dockerhub.sh b/.ci/push-dockerhub.sh index c2c46062ffa..a685ae8dd6e 100755 --- a/.ci/push-dockerhub.sh +++ b/.ci/push-dockerhub.sh @@ -1,7 +1,7 @@ #!/bin/sh # This script gets called from CI to push our docker images to -# $DOCKER_USER/sagemath* on the Docker Hub. +# $DOCKER_NAMESPACE/sagemath* on the Docker Hub. # This script expects a single parameter, the base name of the docker image # such as sagemath or sagemath-dev. @@ -25,5 +25,5 @@ if [ -z "$DOCKER_USER" -o -z "$SECRET_DOCKER_PASS" ]; then echo "DOCKER_USER/SECRET_DOCKER_PASS variables have not been configured in your Continuous Integration setup. Not pushing built images to Docker Hub." else cat "$SECRET_DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin - docker push ${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG + docker push ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG fi From 4bdd32a4c7a9a434bc992be8d4e84ef7f2c89748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 5 Jul 2018 17:01:52 +0200 Subject: [PATCH 126/136] Retry build-from-clean on github --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1caebb802db..50445f8b16d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -117,6 +117,11 @@ build-from-clean: tags: # 60 GB of HDD are available - do + # This build takes several CPU hours. It is very unlikely that there are any + # actual build errors for a tagged release but the (discounted) cloud + # machines this is running on might be preempted during the long build time. + # So let's try three times before we give up. + retry: 2 test-dev: stage: test From 754c52bcf2ded4813b440e8f15e4ee2f295c290a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 5 Jul 2018 17:23:51 +0200 Subject: [PATCH 127/136] Skip publication steps if SECRET_DOCKER_PASS has not been set Note that there is no way to check whether both SECRET_DOCKER_PASS && DOCKER_USER have been set. Also, this is an alpha feature of GitLab, so the syntax might change. As an alternative we could check this at the beginning of the script block and just "exit 0"; but currently a lot of time and in particular bandwidth is wasted by pulling the docker images and then not pushing them because of missing credentials. --- .gitlab-ci.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 50445f8b16d..ae5bd00de5e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -146,8 +146,11 @@ test-jupyter: push-dockerhub: stage: release only: - - branches - - tags + refs: + - branches + - tags + variables: + - $SECRET_DOCKER_PASS script: - . .ci/pull-gitlab.sh sagemath - sh .ci/push-dockerhub.sh sagemath @@ -157,9 +160,12 @@ push-dockerhub: push-dockerhub-dev: stage: release only: - - master - - develop - - tags + refs: + - master + - develop + - tags + variables: + - $SECRET_DOCKER_PASS script: - . .ci/pull-gitlab.sh sagemath-dev - sh .ci/push-dockerhub.sh sagemath-dev From 894c5b8b5c55324015e1d3c0664409bc60c8b992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 5 Jul 2018 17:28:22 +0200 Subject: [PATCH 128/136] Fixup for 5d9b1a5c9ec4588567a8f34bdb895082a6c1c26d use DOCKER_NAMESPACE consistently --- .ci/pull-gitlab.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/pull-gitlab.sh b/.ci/pull-gitlab.sh index c8235e64068..58b17016b90 100755 --- a/.ci/pull-gitlab.sh +++ b/.ci/pull-gitlab.sh @@ -26,5 +26,5 @@ set -ex # automatically from the log output. docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY docker pull $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG -export DOCKER_IMAGE="${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG" +export DOCKER_IMAGE="${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG" docker tag $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG $DOCKER_IMAGE From 2cec8ac704cddc2c65cbc439a7b07c7bcc953f1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Wed, 11 Jul 2018 06:56:12 +0200 Subject: [PATCH 129/136] Replace do tag with big Currently, the do runners are insufficient to build Sage from scratch so there is no point in using that confusing tag anymore. --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ae5bd00de5e..6bf63f4240c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,9 +31,11 @@ # * runners tagged as "gce" (Google Compute Engine) provide about 22GB of HDD, # a single core, 4GB of RAM. Since we are relying on OverlayFS, the disk # space is not sufficient to build sage from scratch. +# The shared runners are terminated after three hours. Currently, this is often +# insufficient to build sage from scratch. # If you want to provide your own runners, make sure to tag them as follows: -# * "do" (60GB of disk space are available) to make build-from-clean pass. +# * "big" (60GB of disk space are available) to make build-from-clean pass. image: docker:latest @@ -116,7 +118,7 @@ build-from-clean: except: [] tags: # 60 GB of HDD are available - - do + - big # This build takes several CPU hours. It is very unlikely that there are any # actual build errors for a tagged release but the (discounted) cloud # machines this is running on might be preempted during the long build time. From 624101115b9af149f8bf0d7cfe2f63d9ccf5201d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Wed, 11 Jul 2018 10:56:51 +0200 Subject: [PATCH 130/136] Fix namespacing for CircleCI --- .ci/update-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/update-env.sh b/.ci/update-env.sh index 574595e48ac..470529d742a 100755 --- a/.ci/update-env.sh +++ b/.ci/update-env.sh @@ -36,8 +36,8 @@ export DOCKER_TAG=`echo $DOCKER_TAG | tr -d '[:space:]' | tr -c '[:alnum:]_.-' ' [[ -z "$DOCKER_TAG" ]] && export DOCKER_TAG=none [[ "$DOCKER_TAG" = "master" ]] && export DOCKER_TAG=latest -export DOCKER_IMAGE_CLI=${DOCKER_USER:-sagemath}/sagemath:$DOCKER_TAG -export DOCKER_IMAGE_DEV=${DOCKER_USER:-sagemath}/sagemath-dev:$DOCKER_TAG +export DOCKER_IMAGE_CLI=${DOCKER_NAMESPACE:-sagemath}/sagemath:$DOCKER_TAG +export DOCKER_IMAGE_DEV=${DOCKER_NAMESPACE:-sagemath}/sagemath-dev:$DOCKER_TAG # Seed the build cache with this image (set to source-clean to build from # scratch.) From 6e267b63dd1ce92e7cbd6ef0cdf99ed7be66ec53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Thu, 12 Jul 2018 10:59:35 +0200 Subject: [PATCH 131/136] Fix namespacing for GitLab CI --- .ci/push-gitlab.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/push-gitlab.sh b/.ci/push-gitlab.sh index e4ceb30abcd..a1609995eed 100755 --- a/.ci/push-gitlab.sh +++ b/.ci/push-gitlab.sh @@ -21,5 +21,5 @@ set -ex # Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it # automatically from the log output. docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY -docker tag ${DOCKER_USER:-sagemath}/$1:$DOCKER_TAG $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG +docker tag ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG docker push $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG From b12cd9ffba2c3e89c06f1051122a3bd3bd69167f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 17 Jul 2018 12:16:13 +0200 Subject: [PATCH 132/136] Do not use broken docker version on CircleCI CircleCI currently uses an old docker version. Let's use the one that we used on GitLab CI as we have done more extensive testing with that one. Strangely, this increases the size of the resulting docker image from 611MB to 635MB on CircleCI (same as on GitLab.) I have not checked what's the difference between the two but I guess it's safer to go with the later version. --- .circleci/config.yml | 3 ++- .gitlab-ci.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bb28ca1b160..635ea14f16e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,8 @@ jobs: steps: - run: apk --update add git openssh - checkout - - setup_remote_docker + - setup_remote_docker: + version: 18.05.0-ce - run: &build # The docker commands sometimes take a while to produce output no_output_timeout: 30m diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6bf63f4240c..0319ba1c410 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,7 +77,7 @@ before_script: # better isolation. If you expect many builds to run simultaneously on a host, # conflicting tags can cause issues with a mounted DOCKER_HOST. services: -- docker:dind +- docker:18.05.0-ce-dind # Build Sage and its documentation. # The build starts from the build artifacts of DEFAULT_ARTIFACT_BASE which is From 220d948c93048bab2a5a909b377f012374c01b06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 17 Jul 2018 13:44:04 +0200 Subject: [PATCH 133/136] Warn about aufs as it creates 4GB sagemath-dev images --- docker/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 18df023aa23..8b1b5110170 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -245,6 +245,11 @@ RUN if [ x"$ARTIFACT_BASE" != x"source-clean" ]; then \ ################################################################################ FROM $ARTIFACT_BASE as sagemath-dev ARG SAGE_ROOT=/home/sage/sage +# If docker is backed by aufs, then the following command adds the size of +# ARTIFACT_BASE to the image size. As of mid 2018 this is notably the case with +# the docker instances provided by setup_remote_docker on CircleCI. As a +# result, the sagemath-dev images that are "build-from-latest" are twice as big +# as the ones that are build on GitLab. COPY --chown=sage:sage --from=sagemath-dev-patch $SAGE_ROOT $SAGE_ROOT ARG ARTIFACT_BASE=source-clean # Apply the patch from sagemath-dev-patch if we created one. From c40bb81d324fc08f1ea6179626e0a5ddcddd5fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 17 Jul 2018 13:52:05 +0200 Subject: [PATCH 134/136] Add reference to known chown bug in docker/aufs --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8b1b5110170..4c8bc553959 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -249,7 +249,8 @@ ARG SAGE_ROOT=/home/sage/sage # ARTIFACT_BASE to the image size. As of mid 2018 this is notably the case with # the docker instances provided by setup_remote_docker on CircleCI. As a # result, the sagemath-dev images that are "build-from-latest" are twice as big -# as the ones that are build on GitLab. +# as the ones that are build on GitLab: +# https://github.com/moby/moby/issues/6119#issuecomment-268870519 COPY --chown=sage:sage --from=sagemath-dev-patch $SAGE_ROOT $SAGE_ROOT ARG ARTIFACT_BASE=source-clean # Apply the patch from sagemath-dev-patch if we created one. From 6fa8bc6a16cfb7ce61598f38052729775cf898c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Tue, 7 Aug 2018 20:59:48 +0200 Subject: [PATCH 135/136] Builds fail with 8.4.beta0 but the output is truncated and it is not clear what's the issue. In general it makes sense to keep a bitter chunk of the output as the part towards the end is usually more interesting. --- .ci/head-tail.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/head-tail.sh b/.ci/head-tail.sh index 8ecf9f879f0..036161dbc37 100755 --- a/.ci/head-tail.sh +++ b/.ci/head-tail.sh @@ -1,6 +1,6 @@ #!/bin/sh -OFFSET=80 +OFFSET=1024 # This script reads from stdin and prints to stdout as long as a the output # does not exceed a certain number of bytes. When reading an EOF it prints the # last $OFFSET lines if they have not been printed normally already. From ac6201a942aa19b6181c463cac8e9588e92b3d8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20R=C3=BCth?= Date: Fri, 17 Aug 2018 15:39:32 +0200 Subject: [PATCH 136/136] Timeouts DO happen on CircleCI I am not sure when that changed. The builds used to take 4:30 hours and now the take almost exactly 5 hours. Sometimes more, sometimes less. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 635ea14f16e..2b425783ce6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,8 +1,8 @@ # This file configures automatic builds of Sage on [CircleCI](https://circleci.com). # To make the build time not too excessive, we seed the build cache with # sagemath/sagemath-dev:develop. When basic SPKGs change, this does not help much, -# still the full build does usually not exceed CircleCI's limits for open -# source projcets (five hours on 2 vCPUs as of early 2018.) +# as the full build will often exceed CircleCI's limits for open source +# projcets (five hours on 2 vCPUs as of early 2018.) # You might want to try to build locally or with GitLab CI, see # `.gitlab-ci.yml` for more options.