diff --git a/.ci/README.md b/.ci/README.md new file mode 100644 index 00000000000..e2b165cc518 --- /dev/null +++ b/.ci/README.md @@ -0,0 +1,9 @@ +# Continuous Integration (CI) + +We support several implementations of CI. All these implementations rely on +[docker](https://docker.com) in some way. This directory contains bits which +are shared between these CI implementations. The relevant docker files can be +found in `/docker/`. + +* [CircleCI](https://circleci.com) is configured in `/.circleci/`. +* [GitLab CI](https://gitlab.com) is configured in `/.gitlab-ci.yml`. diff --git a/.ci/build-docker.sh b/.ci/build-docker.sh new file mode 100755 index 00000000000..82fb2c87c5a --- /dev/null +++ b/.ci/build-docker.sh @@ -0,0 +1,50 @@ +#!/bin/sh + +# This script gets called from CI to build several flavours of docker images +# which contain Sage. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +# We speed up the build process by copying built artifacts from ARTIFACT_BASE +# during docker build. See /docker/Dockerfile for more details. +ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop} + +# Seed our cache with $ARTIFACT_BASE if it exists. +docker pull "$ARTIFACT_BASE" > /dev/null || true + +docker_build() { + # Docker's --cache-from does not really work with multi-stage builds: https://github.com/moby/moby/issues/34715 + # So we just have to rely on the local cache. + time docker build -f docker/Dockerfile \ +--build-arg "MAKEOPTS=${MAKEOPTS}" --build-arg "SAGE_NUM_THREADS=${SAGE_NUM_THREADS}" --build-arg "MAKEOPTS_DOCBUILD=${MAKEOPTS}" --build-arg "SAGE_NUM_THREADS_DOCBUILD=${SAGE_NUM_THREADS_DOCBUILD}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@ +} + +# We use a multi-stage build /docker/Dockerfile. For the caching to be +# effective, we populate the cache by building the run/build-time-dependencies +# and the make-all target. (Just building the last target is not enough as +# intermediate targets could be discarded from the cache [depending on the +# docker version] and therefore the caching would fail for our actual builds +# below.) +docker_build --target run-time-dependencies --tag run-time-dependencies:$DOCKER_TAG . +docker_build --target build-time-dependencies --tag build-time-dependencies:$DOCKER_TAG . +docker_build --target make-all --tag make-all:$DOCKER_TAG . + +# Build the release image without build artifacts. +docker_build --target sagemath --tag "$DOCKER_IMAGE_CLI" . +# Display the layers of this image +docker history "$DOCKER_IMAGE_CLI" +# Build the developer image with the build artifacts intact. +# Note: It's important to build the dev image last because it might be tagged as ARTIFACT_BASE. +docker_build --target sagemath-dev --tag "$DOCKER_IMAGE_DEV" . +# Display the layers of this image +docker history "$DOCKER_IMAGE_DEV" diff --git a/.ci/describe-system.sh b/.ci/describe-system.sh new file mode 100755 index 00000000000..6bd3b0efd4a --- /dev/null +++ b/.ci/describe-system.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set +e -x + +docker info +docker run docker sh -c " + set -x + uname -a + df -h + cat /proc/cpuinfo + cat /proc/meminfo + cat /proc/sys/vm/overcommit_memory + cat /proc/sys/vm/overcommit_ratio" diff --git a/.ci/head-tail.sh b/.ci/head-tail.sh new file mode 100755 index 00000000000..036161dbc37 --- /dev/null +++ b/.ci/head-tail.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +OFFSET=1024 +# This script reads from stdin and prints to stdout as long as a the output +# does not exceed a certain number of bytes. When reading an EOF it prints the +# last $OFFSET lines if they have not been printed normally already. +# This script expects one argument, the number of bytes. + +# Heavily inspired by a simlar strategy in make, https://stackoverflow.com/a/44849696/812379. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +stdbuf -i0 -o0 -e0 awk -v limit=$1 -v firstMissingNR=-1 -v offset=$OFFSET -v bytes=0 \ +'{ + if (bytes < limit) { + # this probably gets multi-byte characters wrong, but that probably does + # not matter for our purposes. (We add 1 for a UNIX newline.) + bytes += length($0) + 1; + print; + } else { + if (firstMissingNR == -1){ + print "[…output truncated…]"; + firstMissingNR = NR; + } + a[NR] = $0; + delete a[NR-offset]; + printf "." > "/dev/stderr" + } +} +END { + if (firstMissingNR != -1) { + print "" > "/dev/stderr"; + for(i = NR-offset+1 > firstMissingNR ? NR-offset-1 : firstMissingNR; i<=NR ; i++){ print a[i]; } + } +} +' + diff --git a/.ci/protect-secrets.sh b/.ci/protect-secrets.sh new file mode 100755 index 00000000000..527604106ca --- /dev/null +++ b/.ci/protect-secrets.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +# This script protects all environment variables that start with "SECRET_". +# It puts them in a temporary file. The name of the variable contains the path +# of that file. This filename can then safely be used in `cat` even if `set +# -x` has been turned on. Also you can run "export" to understand the +# environment without danger. +# Be careful, however, not to use this like the following: +# docker login $DOCKER_USER $(cat $SECRET_DOCKER_PASS) +# as this would expose the password if `set -x` has been turned on. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -eo pipefail +set +x + +function encrypt { + RET=`mktemp` + eval " echo \$$1" > "$RET" + echo $RET +} + +for name in `awk 'END { for (name in ENVIRON) { print name; } }' < /dev/null`; do +case "$name" in + SECRET_*) + export $name="$(encrypt $name)" + echo "Protected $name" + ;; +esac +done + +unset encrypt diff --git a/.ci/pull-gitlab.sh b/.ci/pull-gitlab.sh new file mode 100755 index 00000000000..58b17016b90 --- /dev/null +++ b/.ci/pull-gitlab.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# This script gets called from CI to pull the Sage docker images that were +# built during the "build" phase to pull all the connected docker daemon +# (likely a docker-in-docker.) +# This script expects a single parameter, the base name of the docker image +# such as sagemath or sagemath-dev. +# The variable $DOCKER_IMAGE is set to the full name of the pulled image; +# source this script to use it. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +# Pull the built images from the gitlab registry and give them the original +# names they had after built. +# Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it +# automatically from the log output. +docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY +docker pull $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG +export DOCKER_IMAGE="${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG" +docker tag $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG $DOCKER_IMAGE diff --git a/.ci/push-dockerhub.sh b/.ci/push-dockerhub.sh new file mode 100755 index 00000000000..a685ae8dd6e --- /dev/null +++ b/.ci/push-dockerhub.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# This script gets called from CI to push our docker images to +# $DOCKER_NAMESPACE/sagemath* on the Docker Hub. +# This script expects a single parameter, the base name of the docker image +# such as sagemath or sagemath-dev. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +[ -z "$DOCKER_TAG" ] && (echo "Can not push untagged build."; exit 0) + +# Push the built images to the docker hub (and fail silently if +# DOCKER_USER/SECRET_DOCKER_PASS have not been configured.) +if [ -z "$DOCKER_USER" -o -z "$SECRET_DOCKER_PASS" ]; then + echo "DOCKER_USER/SECRET_DOCKER_PASS variables have not been configured in your Continuous Integration setup. Not pushing built images to Docker Hub." +else + cat "$SECRET_DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin + docker push ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG +fi diff --git a/.ci/push-gitlab.sh b/.ci/push-gitlab.sh new file mode 100755 index 00000000000..a1609995eed --- /dev/null +++ b/.ci/push-gitlab.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +# This script gets called from CI to push our docker images to registry +# configured in GitLab. (Mostly, so we can pull them again to push them to the +# Docker Hub.) +# This script expects a single parameter, the base name of the docker image +# such as sagemath or sagemath-dev. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +# Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it +# automatically from the log output. +docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY +docker tag ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG +docker push $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG diff --git a/.ci/setup-make-parallelity.sh b/.ci/setup-make-parallelity.sh new file mode 100755 index 00000000000..4e33e11b006 --- /dev/null +++ b/.ci/setup-make-parallelity.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +# Source this to set CPUTHREADS (the number of apparent cores) and RAMTHREADS +# (free RAM divided by the maximum amount needed per thread typically) +# From this this script infers reasonable defaults for SAGE_NUM_THREADS and +# MAKEOPTS. + +# We do exactly the same for CPUTHREADS_DOCBUILD, RAMTHREADS_DOCBUILD, +# SAGE_NUM_THREADS_DOCBUILD, MAKEOPTS_DOCBUILD. As the docbuild needs +# substantially more RAM as of May 2018. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +if [ -z "$CPUTHREADS" ]; then + # Determine the number of threads that can run simultaneously on this system + # (we might not have nproc available.) + # Note that this value is incorrect for some CI providers (notably CircleCI: + # https://circleci.com/docs/2.0/configuration-reference/#resource_class) which + # provision fewer vCPUs than shown in /proc/cpuinfo. So it is probably better + # to set CPUTHREADS manuall in your CI configuration. + CPUTHREADS=`docker run docker cat /proc/cpuinfo | grep -E '^processor' | wc -l` +fi +if [ -z "$CPUTHREADS_DOCBUILD" ]; then + CPUTHREADS_DOCBUILD=$CPUTHREADS +fi + +if [ -z "$RAMTHREADS" ]; then + RAMTHREADS=$(( `docker run docker cat /proc/meminfo | grep MemTotal | awk '{ print $2 }'` / 1048576 )) + if [ $RAMTHREADS = 0 ];then + RAMTHREADS=1; + fi +fi +if [ -z "$RAMTHREADS_DOCBUILD" ]; then + RAMTHREADS_DOCBUILD=$(( `docker run docker cat /proc/meminfo | grep MemTotal | awk '{ print $2 }'` / 2097152 )) + if [ $RAMTHREADS_DOCBUILD = 0 ];then + RAMTHREADS_DOCBUILD=1; + fi +fi + +# On CI machines with their virtual CPUs, it seems to be quite beneficial to +# overcommit on CPU usage. We only need to make sure that we do not exceed RAM +# (as there is no swap.) +if [ $CPUTHREADS -lt $RAMTHREADS ]; then + export SAGE_NUM_THREADS=$((CPUTHREADS + 1)) +else + export SAGE_NUM_THREADS=$RAMTHREADS +fi +if [ $CPUTHREADS_DOCBUILD -lt $RAMTHREADS_DOCBUILD ]; then + export SAGE_NUM_THREADS_DOCBUILD=$((CPUTHREADS_DOCBUILD + 1)) +else + export SAGE_NUM_THREADS_DOCBUILD=$RAMTHREADS_DOCBUILD +fi +# Set -j and -l for make (though -l is probably ignored by Sage) +export MAKEOPTS="-j $SAGE_NUM_THREADS -l $((CPUTHREADS - 1)).8" +export MAKEOPTS_DOCBUILD="-j $SAGE_NUM_THREADS_DOCBUILD -l $((CPUTHREADS_DOCBUILD - 1)).8" diff --git a/.ci/test-cli.sh b/.ci/test-cli.sh new file mode 100755 index 00000000000..0ad6a8f3d0a --- /dev/null +++ b/.ci/test-cli.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# This script gets called from CI to run minimal tests on the sagemath image. + +# Usage: ./test-cli.sh IMAGE-NAME + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +echo "Checking that Sage starts and can calculate 1+1…" +# Calculate 1+1 (remove startup messages and leading & trailing whitespace) +TWO=`docker run "$1" sage -c "'print(1+1)'" | tail -1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'` +[ "x$TWO" = "x2" ] + +echo "Checking that some binaries that should be distributed with Sage are on the PATH…" +# We could also run minimal tests on these but we don't yet. +# Check that Singular and GAP are present +docker run "$1" which Singular +docker run "$1" which gap +# Check that jupyter is present (for binder) +docker run "$1" which jupyter diff --git a/.ci/test-dev.sh b/.ci/test-dev.sh new file mode 100755 index 00000000000..5ba2cce16ca --- /dev/null +++ b/.ci/test-dev.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +# This script gets called from CI to run minimal tests on the sagemath-dev image. +# This script expects a single argument, the full name of the docker image to +# test. + +# Usage: ./test-dev.sh IMAGE-NAME + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +IMAGE="$1" + +. .ci/setup-make-parallelity.sh + +# Usage: timed_run limit args +# Runs $IMAGE with args and check that it terminates with a zero exit code in at most limit seconds. +timed_run() { + START=`date +%s` + docker run -e MAKEOPTS="$MAKEOPTS_DOCBUILD" -e SAGE_NUM_THREADS="$SAGE_NUM_THREADS_DOCBUILD" "$IMAGE" "$2" + END=`date +%s` + TOTAL=$((END-START)) + echo "Checking whether running \"$2\" was fast…" + [ "$TOTAL" -lt "$1" ] +} + +# Most setup should be done in well under 120 seconds but some CI machines tend +# to be really slow so we better give them some space here. Better miss a +# regression at first than create a lot of noise. +timed_run 120 true # runs make build +# Building the documentation is quite slow at the moment: +# Currently, this detects some dependencies as changed that have not changed. +# The parser in Sphinx fails to parse some .py files and adds the (more +# recently modified) .pyc files as dependencies instead. (Have a look the +# changeset that introduced this comment for more details.) +timed_run $(( 1200/$SAGE_NUM_THREADS_DOCBUILD )) make # runs make build and then make diff --git a/.ci/test-jupyter.sh b/.ci/test-jupyter.sh new file mode 100755 index 00000000000..4578fb5cbd8 --- /dev/null +++ b/.ci/test-jupyter.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# This script gets called from CI to run minimal tests on the sagemath-jupyter +# image. + +# Usage: ./test-jupyter.sh IMAGE-NAME [HOST] + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex +docker run --name sage-jupyter -d "$1" sage-jupyter +echo "Checking that the Jupyter notebook is running…" +sleep 10 # giving the server some time to start +docker logs sage-jupyter +docker run --link sage-jupyter alpine sh -c "apk --update add wget; wget --retry-connrefused --tries=10 --wait=3 http://sage-jupyter:8888" diff --git a/.ci/update-env.sh b/.ci/update-env.sh new file mode 100755 index 00000000000..470529d742a --- /dev/null +++ b/.ci/update-env.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +# This script gets called from CI to establish the name of the current docker +# tag to build and also the image which is used to seed the cache. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +set -ex + +# The maintainer of the CI environment, e.g., the people administrating the +# SageMath account on gitlab.com, can decide to inject an arbitrary +# base64-encoded script early into the CI execution. The script has access to +# all the CI variables, e.g., to find out which job is being executed, and +# it could do things such as "exit 1" to fail early, or "git merge" a fix for a +# known bug in CI. The CI_MONKEY_PATCH could of course also curl a more +# complicated script and execute that. +if [ -n "$CI_MONKEY_PATCH" ]; then + SCRIPT=$(echo "$CI_MONKEY_PATCH" | base64 -d) + $SCRIPT +fi + +# From the docker documentation: "A tag name must be valid ASCII and may +# contain lowercase and uppercase letters, digits, underscores, periods and +# dashes. A tag name may not start with a period or a dash and may contain a +# maximum of 128 characters." +export DOCKER_TAG=`echo $DOCKER_TAG | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128` + +[[ -z "$DOCKER_TAG" ]] && export DOCKER_TAG=none +[[ "$DOCKER_TAG" = "master" ]] && export DOCKER_TAG=latest + +export DOCKER_IMAGE_CLI=${DOCKER_NAMESPACE:-sagemath}/sagemath:$DOCKER_TAG +export DOCKER_IMAGE_DEV=${DOCKER_NAMESPACE:-sagemath}/sagemath-dev:$DOCKER_TAG + +# Seed the build cache with this image (set to source-clean to build from +# scratch.) +export ARTIFACT_BASE=${ARTIFACT_BASE:-$DEFAULT_ARTIFACT_BASE} diff --git a/.circleci/before-script.sh b/.circleci/before-script.sh new file mode 100644 index 00000000000..51127362f15 --- /dev/null +++ b/.circleci/before-script.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# Source this script during a CI run to set environment variables and print +# some informational messages about the system we are running on. + +# **************************************************************************** +# Copyright (C) 2018 Julian Rüth +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +# **************************************************************************** + +# CircleCI has no mechanism to hide secret variables. +# Therefore we roll our own to protect $SECRET_* variables. +. .ci/protect-secrets.sh +# Collect debug infos about the system we are running on +.ci/describe-system.sh +# Set MAKEOPTS and SAGE_NUM_THREADS +. .ci/setup-make-parallelity.sh + +# Set DOCKER_TAG according to the current branch/tag +export DOCKER_TAG=${CIRCLE_TAG:-$CIRCLE_BRANCH} +. .ci/update-env.sh diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..2b425783ce6 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,89 @@ +# This file configures automatic builds of Sage on [CircleCI](https://circleci.com). +# To make the build time not too excessive, we seed the build cache with +# sagemath/sagemath-dev:develop. When basic SPKGs change, this does not help much, +# as the full build will often exceed CircleCI's limits for open source +# projcets (five hours on 2 vCPUs as of early 2018.) +# You might want to try to build locally or with GitLab CI, see +# `.gitlab-ci.yml` for more options. + +# As of early 2018, a run on CircleCI takes usually about 25 minutes. Most of +# the time is spent pulling/pushing from/to Docker Hub and copying files +# locally during the docker build. We could probably save five minutes by not +# building and testing the sagemath-dev image for most branches. + +version: 2 +jobs: + build-test-release: &build-test-release + docker: + - image: docker:latest + environment: + DEFAULT_ARTIFACT_BASE: sagemath/sagemath-dev:develop + steps: + - run: apk --update add git openssh + - checkout + - setup_remote_docker: + version: 18.05.0-ce + - run: &build + # The docker commands sometimes take a while to produce output + no_output_timeout: 30m + name: build + command: | + . .circleci/before-script.sh + .ci/build-docker.sh + - run: &test-dev + name: test-dev + command: | + . .circleci/before-script.sh + .ci/test-dev.sh $DOCKER_IMAGE_DEV + - run: &test-cli + name: test-cli + command: | + . .circleci/before-script.sh + .ci/test-cli.sh $DOCKER_IMAGE_CLI + - run: &test-jupyter + name: test-jupyter + command: | + . .circleci/before-script.sh + .ci/test-jupyter.sh $DOCKER_IMAGE_CLI localhost + - run: &release + # The docker commands sometimes take a while to produce output + no_output_timeout: 30m + name: release + command: | + . .circleci/before-script.sh + # Push docker images to dockerhub if a dockerhub user has been configured + .ci/push-dockerhub.sh sagemath-dev + .ci/push-dockerhub.sh sagemath + build-from-latest-test-release: + <<: *build-test-release + build-from-clean-test-release: + <<: *build-test-release + environment: + ARTIFACT_BASE: source-clean + +workflows: + version: 2 + build-branch-from-clean: + jobs: + - build-from-clean-test-release: + filters: + branches: + only: + - master + - develop + build-tag-from-clean: + jobs: + - build-from-clean-test-release: + filters: + branches: + ignore: /.*/ + tags: + only: /.*/ + build-branch-from-latest: + jobs: + - build-from-latest-test-release: + filters: + branches: + ignore: + - master + - develop diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 00000000000..3e4e48b0b5f --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index 101d349690c..c98ce30b813 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,14 @@ $RECYCLE.BIN/ ########### .ipynb_checkpoints Untitled*.ipynb + +############################# +# GitLab CI generated files # +############################# +gitlab-build-docker.log + +/src/.cython_version +/src/build +/src/Makefile +/src/bin/sage-env-config + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000000..0319ba1c410 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,173 @@ +# This file configures automatic builds of Sage on [GitLab](https://gitlab.com). +# To make the build time not too excessive, we seed the build cache with +# sagemath/sagemath-dev:develop. When basic SPKGs changed, this does not help +# much and the full build might exceed the set time limit in GitLab. You can +# increase that limit in Settings → CI/CD. +# You can also provision your own private more powerful runner in the same +# place +# https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-in-docker-executor; +# or set up your favourite cloud service to provide an on-demand autoscale +# runner. More details below. + +# As of early 2018 a run on GitLab CI takes about 45 minutes. We could probably +# save 10 minutes by not building/pushing/testing dev images for branches other +# than master/develop. + +# Note that most of the time during CI is spent with pulling and pushing of +# docker images and copying files locally as part of the docker build. At the +# moment there is no reliable way of passing the docker images to the following +# stages without explicit pushing/pulling or similar: +# https://gitlab.com/gitlab-org/gitlab-runner/issues/1107 + +# The timings mentioned above are typical values. The shared runners provided +# on gitlab.com are sometimes much slower depending on the runner you are +# scheduled on. Sometimes it's slower for no apparent reason, probably just an +# overcommittment of virtual machines on hardware. + +# GitLab provides several flavours of shared runners (as of early 2018): +# * runners tagged as "do" (digitalocean.com) provide about 60GB of HDD, two +# cores, but only 2GB of RAM. The RAM is sometimes not sufficient to build +# the documentation. +# * runners tagged as "gce" (Google Compute Engine) provide about 22GB of HDD, +# a single core, 4GB of RAM. Since we are relying on OverlayFS, the disk +# space is not sufficient to build sage from scratch. +# The shared runners are terminated after three hours. Currently, this is often +# insufficient to build sage from scratch. + +# If you want to provide your own runners, make sure to tag them as follows: +# * "big" (60GB of disk space are available) to make build-from-clean pass. + +image: docker:latest + +stages: + - build + - test + - release + +variables: + DOCKER_TAG: $CI_COMMIT_REF_NAME + # Builds are very I/O intensive; make sure we have a fast file system. + DOCKER_DRIVER: overlay2 + DEFAULT_ARTIFACT_BASE: sagemath/sagemath-dev:develop + +before_script: + # GitLab has no mechanism yet to hide secret variables: https://gitlab.com/gitlab-org/gitlab-ce/issues/13784 + # So we roll our own which protects all variables that start with SECRET_ + - . .ci/protect-secrets.sh + # Collect debug infos about the system we are running on + - .ci/describe-system.sh + # Set DOCKER_TAG according to the current branch/tag + - . .ci/update-env.sh + # Set MAKEOPTS and SAGE_NUM_THREADS according to the machine we are running on + - . .ci/setup-make-parallelity.sh + +# We use docker-in-docker to build our docker images, i.e., we run a +# docker:dind "service" container and link to it from the container running the +# actual scripts below. +# Our scripts automatically connect to this service (unless you override it by +# setting DOCKER_HOST.) For example, each RUN statement in the Dockerfile +# spawns a docker container inside the docker:dind container to perform the RUN +# command there. +# It can be faster to expose your outer docker daemon by mounting +# /var/run/docker.sock to /var/run/docker.sock and setting DOCKER_HOST in +# Settings -> CI/CD -> Secret variable to unix:///var/run/docker.sock. (The +# speedup is mostly due to sharing layers of intermediate images.) However, +# this is only possible if you provision your own runners. Shared gitlab +# runners, do not bind mount /var/run/docker.sock. Also, docker:dind provides +# better isolation. If you expect many builds to run simultaneously on a host, +# conflicting tags can cause issues with a mounted DOCKER_HOST. +services: +- docker:18.05.0-ce-dind + +# Build Sage and its documentation. +# The build starts from the build artifacts of DEFAULT_ARTIFACT_BASE which is +# usually much faster than building from a clean checkout of Sage. +build-from-latest: &build + stage: build + artifacts: + when: always + paths: + - gitlab-build-docker.log + expire_in: 1 month + script: + - apk --update add coreutils + # The output of the build can get larger than gitlab.com's limit; only print the first 3MB (and the last 80 lines.) + - .ci/build-docker.sh | tee gitlab-build-docker.log | .ci/head-tail.sh 3145728 + - .ci/push-gitlab.sh sagemath-dev + - .ci/push-gitlab.sh sagemath + except: + - master + - develop + - tags + +# Build Sage and its documentation from a clean checkout of Sage. +# Note that this takes several hours. You probably want to run this on your own +# gitlab-runner and increase the standard GitLab time limit for CI runs. +# Some of the shared runners provided by GitLab for free do not have enough +# disk space for this to work. If a build fails with "no space left on device", +# you could just retry it and hope to be scheduled on a machine with more disk +# space, or provision your own runner. +build-from-clean: + << : *build + variables: + ARTIFACT_BASE: "source-clean" + only: + - master + - develop + - tags + except: [] + tags: + # 60 GB of HDD are available + - big + # This build takes several CPU hours. It is very unlikely that there are any + # actual build errors for a tagged release but the (discounted) cloud + # machines this is running on might be preempted during the long build time. + # So let's try three times before we give up. + retry: 2 + +test-dev: + stage: test + script: + - . .ci/pull-gitlab.sh sagemath-dev + - sh .ci/test-dev.sh "$DOCKER_IMAGE" + +test-cli: + stage: test + script: + - . .ci/pull-gitlab.sh sagemath + - sh .ci/test-cli.sh "$DOCKER_IMAGE" + +test-jupyter: + stage: test + script: + - . .ci/pull-gitlab.sh sagemath + - sh .ci/test-jupyter.sh "$DOCKER_IMAGE" docker + +# Pushes the built images to Docker Hub if the Settings -> CI/CD -> Secret +# variables DOCKER_USER and SECRET_DOCKER_PASS have been set up. +push-dockerhub: + stage: release + only: + refs: + - branches + - tags + variables: + - $SECRET_DOCKER_PASS + script: + - . .ci/pull-gitlab.sh sagemath + - sh .ci/push-dockerhub.sh sagemath + +# Pushes the built dev images to Docker Hub if the Settings -> CI/CD -> Secret +# variables DOCKER_USER and SECRET_DOCKER_PASS have been set up. +push-dockerhub-dev: + stage: release + only: + refs: + - master + - develop + - tags + variables: + - $SECRET_DOCKER_PASS + script: + - . .ci/pull-gitlab.sh sagemath-dev + - sh .ci/push-dockerhub.sh sagemath-dev diff --git a/Makefile b/Makefile index d81d450ef4e..89ac3e4ecf1 100644 --- a/Makefile +++ b/Makefile @@ -104,9 +104,44 @@ bootstrap-clean: maintainer-clean: distclean bootstrap-clean rm -rf upstream +# Remove everything that is not necessary to run Sage and pass all its +# doctests. micro_release: bdist-clean sagelib-clean @echo "Stripping binaries ..." LC_ALL=C find local/lib local/bin -type f -exec strip '{}' ';' 2>&1 | grep -v "File format not recognized" | grep -v "File truncated" || true + @echo "Removing sphinx artifacts..." + rm -rf local/share/doc/sage/doctrees local/share/doc/sage/inventory + @echo "Removing documentation. Inspection in IPython still works." + rm -rf local/share/doc local/share/*/doc local/share/*/examples local/share/singular/html + @echo "Removing unnecessary files & directories - make will not be functional afterwards anymore" + @# We need src/doc/common, src/doc/en/introspect for introspection with "??" + @# We keep src/sage for some doctests that it expect it to be there and + @# also because it does not add any weight with rdfind below. + @# We need src/sage/bin/ for the scripts that invoke Sage + @# We need sage, the script to start Sage + @# We need local/, the dependencies and the built Sage library itself. + @# We keep VERSION.txt. + @# We keep COPYING.txt so we ship a license with this distribution. + find . -name . -o -prune ! -name src ! -name sage ! -name local ! -name VERSION.txt ! -name COPYING.txt ! -name build -exec rm -rf \{\} \; + cd src && find . -name . -o -prune ! -name sage ! -name bin ! -name doc -exec rm -rf \{\} \; + if command -v rdfind > /dev/null; then \ + echo "Hardlinking identical files."; \ + rdfind -makeresultsfile false -makehardlinks true .; \ + else \ + echo "rdfind not installed. Not hardlinking identical files."; \ + fi + +# Leaves everything that is needed to make the next "make" fast but removes +# all the cheap build artifacts that can be quickly regenerated. +fast-rebuild-clean: misc-clean bdist-clean + rm -rf upstream/ + rm -rf src/build/temp.* + # Without site-packages/sage sage does not start but copying/compiling + # them from src/build is very fast. + rm -rf local/lib/python*/site-packages/sage + # The .py files in src/build are restored from src/sage without their + # mtimes changed. + find src/build -name '*.py' -exec rm \{\} \; TESTALL = ./sage -t --all PTESTALL = ./sage -t -p --all diff --git a/configure.ac b/configure.ac index 5a830e50f25..a5540ef97be 100644 --- a/configure.ac +++ b/configure.ac @@ -285,9 +285,9 @@ fi AC_CHECK_PROG(found_latex, latex, yes, no) if test x$found_latex != xyes then - AC_MSG_WARN([You do not have 'latex', which is recommended, but not]) - AC_MSG_WARN([required. Latex is only really used for building pdf]) - AC_MSG_WARN([documents and for %latex mode in the AC_PACKAGE_NAME notebook.]) + AC_MSG_NOTICE([You do not have 'latex', which is recommended, but not]) + AC_MSG_NOTICE([required. Latex is only really used for building pdf]) + AC_MSG_NOTICE([documents and for %latex mode in the AC_PACKAGE_NAME notebook.]) fi # Check that perl is available, with version 5.8.0 or later. diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 00000000000..579da245b87 --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1,2 @@ +# Stores the commit that was used to create a sagemath-dev image +.commit diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000000..4c8bc553959 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,265 @@ +################################################################################ +# SageMath images for Docker # +################################################################################ +# This is a description of the layout of this Dockerfile; for details on the # +# created docker images, see the README.md please. # +# # +# This Dockerfile builds sagemath (for end-users) and sagemath-dev (for # +# developers.) It consists of lots of intermediate targets, mostly to shrink # +# the resulting images but also to make this hopefully easier to maintain. # +# The aims of this Dockerfile are: # +# (1) Make it build in reasonable time. # +# (2) It should be self-contained and work on its own, i.e., just by invoking # +# docker build without any external orchestration script. # +# # +# The idea to achieve (1) is to reuse the build artifacts from the latest # +# develop build. This is slightly against the philosophy of a Dockerfile (which# +# should produce perfectly reproducible outputs) but building Sage from scratch# +# just takes too long at the moment to do this all the time. ARTIFACT_BASE # +# controls which build artifacts are used. You probably want to set this to # +# sagemath/sagemath-dev:develop which takes the latest build from the official # +# develop branch. The default is source-clean which builds Sage from scratch. # +# If you want to understand how this works, have a look at source-from-context # +# which merges ARTIFACT_BASE with the context, i.e., the contents of the sage # +# source directory. # +################################################################################ + +################################################################################ +# HOWTO use this file for local builds # +################################################################################ +# If you don't mind downloading a 2GB docker image from time to time, you # +# could use this file for local Sage development. As of early 2018 each build # +# takes about five minutes but you don't have to go through the sadly frequent # +# rebuilds the whole Sage distribution... # +# To build Sage, run this command from your sage/ directory: # +# $ docker build --build-arg MAKEOPTS="-j4" --build-arg SAGE_NUM_THREADS="4" --build-arg ARTIFACT_BASE="sagemath/sagemath-dev:develop" -f docker/Dockerfile --target=make-build --tag sage . +# To run Sage: # +# $ docker run -it sage # +# To run doctests: # +# $ docker run -e "MAKEOPTS=-j4" -e "SAGE_NUM_THREADS=4" -it sage sage -tp src/sage +# Make sure that you always have the latest develop branch merged into your # +# local branch for this to work. # +################################################################################ + +ARG ARTIFACT_BASE=source-clean + +################################################################################ +# Image containing the run-time dependencies for Sage # +################################################################################ +FROM ubuntu:xenial as run-time-dependencies +LABEL maintainer="Erik M. Bray , Julian Rüth " +# Set sane defaults for common environment variables. +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 +ENV SHELL /bin/bash +# Create symlinks for sage and sagemath - we copy a built sage to the target of these symlinks later. +ARG SAGE_ROOT=/home/sage/sage +RUN ln -s "$SAGE_ROOT/sage" /usr/bin/sage +RUN ln -s /usr/bin/sage /usr/bin/sagemath +# Sage needs the fortran libraries at run-time because we do not build gfortran +# with Sage but use the system's. +# We need gcc/g++ and libstdc++-5-dev to allow compilation of cython at run-time from the notebook. +# We also install sudo for the sage user, see below. +RUN apt-get -qq update \ + && apt-get -qq install -y --no-install-recommends gfortran gcc g++ libstdc++-5-dev sudo openssl \ + && apt-get -qq clean \ + && rm -r /var/lib/apt/lists/* +# Sage refuses to build as root, so we add a "sage" user that can sudo without a password. +# We also want this user at runtime as some commands in sage know about the user that was used during build. +ARG HOME=/home/sage +RUN adduser --quiet --shell /bin/bash --gecos "Sage user,101,," --disabled-password --home "$HOME" sage \ + && echo "sage ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/01-sage \ + && chmod 0440 /etc/sudoers.d/01-sage +# Run everything from now on as the sage user in sage's home +USER sage +ENV HOME $HOME +WORKDIR $HOME + +################################################################################ +# Image containing everything so that a make in a clone of the Sage repository # +# completes without errors # +################################################################################ +FROM run-time-dependencies as build-time-dependencies +# Install the build time dependencies & git & rdfind +RUN sudo apt-get -qq update \ + && sudo apt-get -qq install -y wget build-essential automake m4 dpkg-dev python libssl-dev git rdfind \ + && sudo apt-get -qq clean \ + && sudo rm -r /var/lib/apt/lists/* + +################################################################################ +# Image with an empty git repository in $SAGE_ROOT. # +################################################################################ +FROM build-time-dependencies as source-clean +ARG SAGE_ROOT=/home/sage/sage +RUN mkdir -p "$SAGE_ROOT" +WORKDIR $SAGE_ROOT +RUN git init +RUN git remote add trac git://trac.sagemath.org/sage.git + +################################################################################ +# Image with the build context added, i.e., the directory from which `docker # +# build` has been called in a separate directory so we can copy files from # +# there. # +# This blows up the size of this docker image significantly, but we only use # +# this image to create artifacts for our final image. # +# Warning: If you set ARTIFACT_BASE to something else than source-clean, the # +# build is not going to use the build-time-dependencies target but rely on # +# whatever tools are installed in ARTIFACT_BASE. # +################################################################################ +FROM $ARTIFACT_BASE as source-from-context +WORKDIR $HOME +COPY --chown=sage:sage . sage-context +# Checkout the commit that checked out in $HOME/sage-context +# This is a bit complicated because our local .git/ is empty and we want to +# make sure that we only change the mtimes of a minimal number of files. +# 1) Restore the git checkout ARTIFACT_BASE was built from, recorded in +# docker/.commit. (Or go directly to FETCH_HEAD if there is no history to +# restore, i.e., set ARTIFACT_BASE=source-clean if you want to build from +# scratch.) +# 2) Merge in FETCH_HEAD but only if it is a fast-forward, i.e., if it is an +# ancestor of the commit restored in 1. If we would not do that we would get +# a new commit hash in docker/.commit that is not known outside of this build +# run. Since docker/.commit was in the history of FETCH_HEAD this should +# automatically be a fast-forward. +# 3) Trash .git again to save some space. +ARG SAGE_ROOT=/home/sage/sage +WORKDIR $SAGE_ROOT +# We create a list of all files present in the artifact-base (with a timestamp +# of now) so we can find out later which files were added/changed/removed. +RUN find . -type f > $HOME/artifact-base.manifest +RUN git fetch "$HOME/sage-context" HEAD \ + && if [ -e docker/.commit ]; then \ + git reset `cat docker/.commit` \ + || ( echo "Could not find commit `cat docker/.commit` in your local Git history. Please merge in the latest built develop branch to fix this: git fetch trac && git merge `cat docker/.commit`." && exit 1 ) \ + else \ + echo "You are building from $ARTIFACT_BASE which has no docker/.commit file. That's a bug unless you are building from source-clean or something similar." \ + && git reset FETCH_HEAD \ + && git checkout -f FETCH_HEAD; \ + fi \ + && git merge --ff-only FETCH_HEAD \ + && git log -1 --format=%H > docker/.commit \ + && rm -rf .git +# Copy over all the untracked/staged/unstaged changes from sage-context. This +# is relevant for non-CI invocations of this Dockerfile. +WORKDIR $HOME/sage-context +RUN if git status --porcelain | read CHANGES; then \ + git -c user.name=docker-build -c user.email=docker-build@sage.invalid stash -u \ + && git stash show -p > "$HOME"/sage-context.patch; \ + else \ + touch "$HOME"/sage-context.patch; \ + fi +WORKDIR $SAGE_ROOT +RUN patch -p1 < "$HOME"/sage-context.patch + +################################################################################ +# Image with a built sage but without sage's documentation. # +################################################################################ +FROM source-from-context as make-build +# Make sure that the result runs on most CPUs. +ENV SAGE_FAT_BINARY yes +# Just to be sure Sage doesn't try to build its own GCC (even though +# it shouldn't with a recent GCC package from the system and with gfortran) +ENV SAGE_INSTALL_GCC no +# Set MAKEOPTS and SAGE_NUM_THREADS to build things in parallel during the +# docker build. Note that these do not leak into the sagemath and sagemath-dev +# images. +ARG MAKEOPTS="-j2" +ENV MAKEOPTS $MAKEOPTS +ARG SAGE_NUM_THREADS="2" +ENV SAGE_NUM_THREADS $SAGE_NUM_THREADS +RUN make build + +################################################################################ +# Image with a full build of sage and its documentation. # +################################################################################ +FROM make-build as make-all +# The docbuild needs quite some RAM (as of May 2018). It sometimes calls +# os.fork() to spawn an external program which then exceeds easily the +# overcommit limit of the system (no RAM is actually used, but this limit is +# very low because there is not even swap on most CI systems.) +ARG MAKEOPTS_DOCBUILD=$MAKEOPTS +ENV MAKEOPTS_DOCBUILD $MAKEOPTS_DOCBUILD +ARG SAGE_NUM_THREADS_DOCBUILD=$SAGE_NUM_THREADS +ENV SAGE_NUM_THREADS_DOCBUILD $SAGE_NUM_THREADS_DOCBUILD +RUN make + +################################################################################ +# Image with a full build of sage, ready to release, i.e., with stripped # +# binaries and some extras to run the jupyter notebook. # +################################################################################ +FROM make-all as make-release +RUN sage -pip install terminado "notebook>=5" "ipykernel>=4.6" +RUN sage -i gap_jupyter singular_jupyter pari_jupyter +RUN make micro_release + +################################################################################ +# A releasable (relatively small) image which contains a copy of sage without # +# temporary build artifacts which is set up to start the command line # +# interface if no parameters are passed in. # +################################################################################ +FROM run-time-dependencies as sagemath +ARG HOME=/home/sage +ARG SAGE_ROOT=/home/sage/sage +COPY --chown=sage:sage --from=make-release $SAGE_ROOT/ $SAGE_ROOT/ +# Put scripts to start gap, gp, maxima, ... in /usr/bin +RUN sudo $SAGE_ROOT/sage --nodotsage -c "install_scripts('/usr/bin')" +COPY ./docker/entrypoint.sh /usr/local/bin/sage-entrypoint +WORKDIR $HOME +ENTRYPOINT ["/usr/local/bin/sage-entrypoint"] +EXPOSE 8888 +CMD ["sage"] + +################################################################################ +# Image with a full build of sage and its documentation but everything # +# stripped that can be quickly rebuild by make. # +################################################################################ +FROM make-all as make-fast-rebuild-clean +RUN make fast-rebuild-clean + +################################################################################ +# Depending on whether we built from source-clean or not, this image is either # +# identical to make-fast-rebuild-clean or contains a "patch" which can be used # +# to upgrade ARTIFACT_BASE to make-fast-rebuild-clean. # +################################################################################ +FROM make-fast-rebuild-clean as sagemath-dev-patch +ARG ARTIFACT_BASE=source-clean +ARG SAGE_ROOT=/home/sage/sage +# Build a patch containing of a tar file which contains all the modified files +# and a list of all modified files (added/updated/removed). +RUN if [ x"$ARTIFACT_BASE" != x"source-clean" ]; then \ + mkdir -p $HOME/patch \ + && find . -type f > $HOME/make-fast-rebuild-clean.manifest \ + && cat $HOME/make-fast-rebuild-clean.manifest $HOME/artifact-base.manifest | sort | uniq -u > $HOME/obsolete \ + && find . -type f -cnewer $HOME/artifact-base.manifest > $HOME/modified \ + && tar -cJf $HOME/patch/modified.tar.xz -T $HOME/modified \ + && cat $HOME/obsolete $HOME/modified | xz > $HOME/patch/modified.xz \ + && rm -rf $SAGE_ROOT \ + && mkdir -p $SAGE_ROOT \ + && mv $HOME/patch $SAGE_ROOT/; \ + fi + +################################################################################ +# A releasable (relatively small, but still huge) image of this build with all # +# the build artifacts intact so developers can make changes and rebuild # +# quickly # +################################################################################ +FROM $ARTIFACT_BASE as sagemath-dev +ARG SAGE_ROOT=/home/sage/sage +# If docker is backed by aufs, then the following command adds the size of +# ARTIFACT_BASE to the image size. As of mid 2018 this is notably the case with +# the docker instances provided by setup_remote_docker on CircleCI. As a +# result, the sagemath-dev images that are "build-from-latest" are twice as big +# as the ones that are build on GitLab: +# https://github.com/moby/moby/issues/6119#issuecomment-268870519 +COPY --chown=sage:sage --from=sagemath-dev-patch $SAGE_ROOT $SAGE_ROOT +ARG ARTIFACT_BASE=source-clean +# Apply the patch from sagemath-dev-patch if we created one. +RUN if [ x"$ARTIFACT_BASE" != x"source-clean" ]; then \ + echo "Applying `du -hs patch/modified.tar.xz` patch" \ + && xzcat patch/modified.xz | xargs rm -rvf \ + && tar -Jxf patch/modified.tar.xz \ + && rm -rf patch; \ + fi +COPY ./docker/entrypoint-dev.sh /usr/local/bin/sage-entrypoint +ENTRYPOINT ["/usr/local/bin/sage-entrypoint"] +CMD ["bash"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000000..216fcc77364 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,59 @@ +[![License: GPL v3](https://img.shields.io/badge/License-GPL%20v3-blue.svg)](https://github.com/sagemath/sage/COPYING.txt) [![Maintained](https://img.shields.io/maintenance/yes/2018.svg)](https://github.com/sagemath/sage/commits/master) + +# Supported tags + +* `latest` — the stable `master` branch [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/sagemath/sage/master.svg)](https://github.com/sagemath/sage/commits/master) [![CircleCI branch](https://img.shields.io/circleci/project/github/sagemath/sage/master.svg)](https://circleci.com/gh/sagemath/sage/tree/master) [![GitLab CI](https://gitlab.com/sagemath/sage/badges/master/pipeline.svg)](https://gitlab.com/sagemath/sage/commits/master) +* `x.x` — all stable releases of Sage are tagged with their version number. +* `x.x.{beta,rc}x` - betas and release candidates of Sage as [tagged in our git repository](https://github.com/sagemath/sage/tags). +* `develop` — the current development version of Sage which gets merged into the `master` branch when a new version of Sage is released [![GitHub last commit (branch)](https://img.shields.io/github/last-commit/sagemath/sage/develop.svg)](https://github.com/sagemath/sage/commits/develop) [![CircleCI branch](https://img.shields.io/circleci/project/github/sagemath/sage/master.svg)](https://circleci.com/gh/sagemath/sage/tree/master) [![GitLab CI](https://gitlab.com/sagemath/sage/badges/develop/pipeline.svg)](https://gitlab.com/sagemath/sage/commits/develop) + + +# What is SageMath + +SageMath is a free open-source mathematics software system licensed under the GPL. It builds on top of many existing open-source packages: NumPy, SciPy, matplotlib, Sympy, Maxima, GAP, FLINT, R and many more. Access their combined power through a common, Python-based language or directly via interfaces or wrappers. + +**Mission**: *Creating a viable free open source alternative to Magma, Maple, Mathematica and Matlab.* + +# What's in this image + +There are several flavours of this image. + +* [`sagemath/sagemath`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath/latest.svg)](https://hub.docker.com/r/sagemath/sagemath) contains everything necessary to run Sage on the command line. Run it with: + ``` + docker run -it sagemath/sagemath:latest + ``` + You can start a graphical [Jupyter Notebook](https://jupyter.org) at http://localhost:8888 instead. To use the notebook, follow the instructions printed when you run: + ``` + docker run -p8888:8888 sagemath/sagemath:latest sage-jupyter + ``` +* [`sagemath/sagemath-dev`![image size](https://img.shields.io/microbadger/image-size/sagemath/sagemath-dev.svg)](https://hub.docker.com/r/sagemath/sagemath-dev) contains all the build artifacts to rebuild Sage quickly. This version is probably only relevant for Sage developers. Run this image with: + ``` + docker run -it sagemath/sagemath-dev:develop + ``` + This triggers a rebuild and drops you in a shell afterwards. Note that the git repository has been emptied to save space. If you want to use git, fetch from your git repository with `git fetch trac` and go to the commit that was used to create this image with + ``` + git reset $(cat docker/.commit) + ``` + +# How to build your own SageMath images + +Run `docker build -f docker/Dockerfile --build-arg ARTIFACT_BASE=sagemath/sagemath-dev:develop --target TARGET .` in the Sage repository with `TARGET` one of `sagemath` or `sagemath-dev`. + +# How these images get updated + +Every push to our [github repository](https://github.com/sagemath/sage) triggers a build in [CircleCI](https://circleci.com) which builds and pushes the docker images. +A push to master also triggers a "build" on our [Docker Hub](https://hub.docker.com) repositories. The latter build is mostly disabled by the `hooks/` and only updates the `README.md`. + +Every push to our [GitLab repository](https://gitlab.com/sagemath/sage) triggers a pipeline in GitLab CI. This build also pushes images to Docker Hub. + +Have a look at `.circleci/` and `.gitlab-ci.yml` if you want to setup either continuous integration service for your own fork of the SageMath repository. + +# Report bugs and issues + +Please tell us of any bugs or omissions at our [Issue Tracker](https://trac.sagemath.org) or contact us through the [sage-support](https://groups.google.com/forum/#!forum/sage-support) or the [sage-devel](https://groups.google.com/forum/#!forum/sage-devel) mailing lists. + +# License + +The whole Sage software distribution is licensed under the General Public License, version 3. More details can be found in our [COPYING.txt](https://github.com/sagemath/sage/blob/master/COPYING.txt) + +[//]: # (Please don't break long lines in this files as dockerhub then gets the formatting of this file wrong.) diff --git a/docker/entrypoint-dev.sh b/docker/entrypoint-dev.sh new file mode 100755 index 00000000000..67299a5a92d --- /dev/null +++ b/docker/entrypoint-dev.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e +make build +exec "$@" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000000..bc841382eaf --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash +if [ x"$1" = x"sage-jupyter" ]; then + # If "sage-jupyter" is given as a first argument, we start a jupyter notebook + # with reasonable default parameters for running it inside a container. + shift + exec sage -n jupyter --no-browser --ip='*' --port=8888 "$@" +else + exec sage -sh -c "$*" +fi diff --git a/docker/hooks/build b/docker/hooks/build new file mode 100644 index 00000000000..b23e55619b2 --- /dev/null +++ b/docker/hooks/build @@ -0,0 +1 @@ +#!/bin/true diff --git a/docker/hooks/push b/docker/hooks/push new file mode 100644 index 00000000000..b23e55619b2 --- /dev/null +++ b/docker/hooks/push @@ -0,0 +1 @@ +#!/bin/true diff --git a/src/.gitignore b/src/.gitignore deleted file mode 100644 index e85aa7f89e4..00000000000 --- a/src/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/.cython_version -/build -/Makefile -/bin/sage-env-config diff --git a/src/Makefile.in b/src/Makefile.in index 81a28b3758e..0233601cbbc 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -37,7 +37,7 @@ sage: SAGE_DOC_SRC=/doesnotexist \ SAGE_BUILD_DIR=/doesnotexist \ SAGE_PKGS=$(abs_top_srcdir)/build/pkgs \ - && sage-python23 -u setup.py --no-user-cfg build install + && sage-python23 -u setup.py --quiet --no-user-cfg build install if [ "$$UNAME" = "CYGWIN" ]; then \ sage-rebase.sh "$$SAGE_LOCAL" 2>/dev/null; \ fi diff --git a/src/sage_setup/docbuild/__init__.py b/src/sage_setup/docbuild/__init__.py index 5877b5b2a76..7dc8ff783c5 100644 --- a/src/sage_setup/docbuild/__init__.py +++ b/src/sage_setup/docbuild/__init__.py @@ -265,35 +265,29 @@ def clean(self, *args): # import the customized builder for object.inv files inventory = builder_helper('inventory') -if NUM_THREADS > 1: - def build_many(target, args): - from multiprocessing import Pool - pool = Pool(NUM_THREADS, maxtasksperchild=1) - # map_async handles KeyboardInterrupt correctly. Plain map and - # apply_async does not, so don't use it. - x = pool.map_async(target, args, 1) - try: - ret = x.get(99999) - pool.close() - pool.join() - except Exception: - pool.terminate() - if ABORT_ON_ERROR: - raise - return ret -else: - def build_many(target, args): - results = [] - - for arg in args: - try: - results.append(target(arg)) - except Exception: - if ABORT_ON_ERROR: - raise - - return results - +def build_many(target, args): + # Pool() uses an actual fork() to run each new instance. This is important + # for performance reasons, i.e., don't use a forkserver when it becomes + # available with Python 3: Here, sage is already initialized which is quite + # costly, with a forkserver we would have to reinitialize it for every + # document we build. At the same time, don't serialize this by taking the + # pool (and thus the call to fork()) out completely: The call to Sphinx + # leaks memory, so we need to build each document in its own process to + # control the RAM usage. + from multiprocessing import Pool + pool = Pool(NUM_THREADS, maxtasksperchild=1) + # map_async handles KeyboardInterrupt correctly. Plain map and + # apply_async does not, so don't use it. + x = pool.map_async(target, args, 1) + try: + ret = x.get(99999) + pool.close() + pool.join() + except Exception: + pool.terminate() + if ABORT_ON_ERROR: + raise + return ret ########################################## # Parallel Building Ref Manual # @@ -940,7 +934,12 @@ def get_new_and_updated_modules(self): except ImportError as err: logger.error("Warning: Could not import %s %s", module_name, err) raise - newtime = os.path.getmtime(sys.modules[module_name].__file__) + + module_filename = sys.modules[module_name].__file__ + if (module_filename.endswith('.pyc') or module_filename.endswith('.pyo')): + source_filename = module_filename[:-1] + if (os.path.exists(source_filename)): module_filename = source_filename + newtime = os.path.getmtime(module_filename) if newtime > mtime: updated_modules.append(module_name) diff --git a/src/sage_setup/docbuild/ext/sage_autodoc.py b/src/sage_setup/docbuild/ext/sage_autodoc.py index f241e09f8a5..2c399bad51b 100644 --- a/src/sage_setup/docbuild/ext/sage_autodoc.py +++ b/src/sage_setup/docbuild/ext/sage_autodoc.py @@ -988,7 +988,12 @@ def generate(self, more_content=None, real_modname=None, self.analyzer.find_attr_docs() except PycodeError as err: self.env.app.debug('[autodoc] module analyzer failed: %s', err) - # no source file -- e.g. for builtin and C modules + # A few things could have happened here: + # * there is no source file -- e.g. for builtin and C modules + # * the source file contains syntax that Sphinx can not parse, + # e.g., "print(1, end=' ')"; see + # https://github.com/sphinx-doc/sphinx/issues/1641, + # fixed in Sphinx 1.7. self.analyzer = None # at least add the module.__file__ as a dependency if hasattr(self.module, '__file__') and self.module.__file__: diff --git a/src/sage_setup/docbuild/sphinxbuild.py b/src/sage_setup/docbuild/sphinxbuild.py index 6b7ac9e8947..da29430dd75 100644 --- a/src/sage_setup/docbuild/sphinxbuild.py +++ b/src/sage_setup/docbuild/sphinxbuild.py @@ -2,7 +2,7 @@ r""" This is Sage's version of the sphinx-build script -We redirect stdout to our own logger, and remove some unwanted chatter. +We redirect stdout and stderr to our own logger, and remove some unwanted chatter. """ # **************************************************************************** # Copyright (C) 2013-2014 Volker Braun @@ -32,8 +32,8 @@ def term_width_line(text): class SageSphinxLogger(object): r""" - This implements the file object interface to serve as sys.stdout - replacement. + This implements the file object interface to serve as + ``sys.stdout``/``sys.stderr`` replacement. """ ansi_color = re.compile(r'\x1b\[[0-9;]*m') ansi_reset = re.compile(r'\x1b\[39;49;00m') @@ -60,26 +60,57 @@ def __init__(self, stream, prefix): self._error = None def _init_chatter(self): - # useless_chatter: regular expressions to be filtered from - # Sphinx output. - self.useless_chatter = ( + # We drop any messages from the output that match these regular + # expressions. These just bloat the output and do not contain any + # information that we care about. + self._useless_chatter = ( re.compile('^$'), re.compile('^Running Sphinx v'), re.compile('^loading intersphinx inventory from '), + re.compile('^loading pickled environment... done'), + re.compile('^loading cross citations... done \([0-9]* citations\).'), re.compile('^Compiling a sub-document'), re.compile('^updating environment: 0 added, 0 changed, 0 removed'), re.compile('^looking for now-outdated files... none found'), re.compile('^building \[.*\]: targets for 0 source files that are out of date'), - re.compile('^loading pickled environment... done'), - re.compile('^loading cross citations... done \([0-9]* citations\).'), + re.compile('^building \[.*\]: targets for 0 po files that are out of date'), + re.compile('^building \[.*\]: targets for 0 mo files that are out of date'), + re.compile('^pickling environment... done'), + re.compile('^dumping object inventory... done'), + # We still have "Build finished." + re.compile('^build succeeded.'), + re.compile('^checking consistency... done'), + re.compile('^preparing documents... done'), + re.compile('^copying extra files... done'), + re.compile('^writing additional pages... search'), + re.compile('^Writing js search indexes...writing additional pages... .*'), + re.compile('^generating indices... .*'), + re.compile('^dumping search index in .* ... done'), + re.compile('^linking _static directory'), + re.compile('^copying static files... done'), + re.compile('^copying extra files... done'), + re.compile('^loading translations \[.*\]... done'), + re.compile('^Compiling the master document'), + re.compile('^Saved pickle file: citations.pickle'), + re.compile('^writing output... \[.*\] '), + re.compile('^copying images... \[.*\] '), + re.compile('^reading sources... \[.*\] '), + re.compile('language "hu" not supported'), + re.compile('^$'), + re.compile('^WARNING:$'), + ) + + # We fail whenever a line starts with "WARNING:", however, we ignore + # these warnings, as they are not relevant. + self._ignored_warnings = ( re.compile('WARNING: favicon file \'favicon.ico\' does not exist'), re.compile('WARNING: html_static_path entry .* does not exist'), re.compile('WARNING: while setting up extension'), re.compile('WARNING: Any IDs not assiend for figure node'), re.compile('WARNING: .* is not referenced'), re.compile('WARNING: Build finished'), - re.compile('language "hu" not supported'), ) + self._useless_chatter += self._ignored_warnings # replacements: pairs of regular expressions and their replacements, # to be applied to Sphinx output. @@ -89,10 +120,12 @@ def _init_chatter(self): if 'inventory' in sys.argv: # When building the inventory, ignore warnings about missing # citations and the search index. - self.useless_chatter += ( + ignored = ( re.compile('WARNING: citation not found:'), re.compile('WARNING: search index couldn\'t be loaded, but not all documents will be built: the index will be incomplete.') ) + self._ignored_warnings += ignored + self._useless_chatter += ignored # Regular expressions indicating a problem with docbuilding. Raise an # exception if any of these occur. @@ -109,18 +142,19 @@ def _init_chatter(self): # - undefined labels upon the first pass of the compilation: some # cross links may legitimately not yet be resolvable at this point. if 'latex' not in sys.argv: + self._error_patterns += (re.compile('WARNING:'),) if 'multidoc_first_pass=1' in sys.argv: - # Catch all warnings except 'WARNING: undefined label' - self._error_patterns += (re.compile('WARNING: (?!undefined label)'),) - else: - self._error_patterns += (re.compile('WARNING:'),) + ignore = (re.compile('WARNING: undefined label'),) + self._ignored_warnings += ignore + self._useless_chatter += ignore def _filter_out(self, line): if self._error is not None and self._is_stdout: # swallow non-errors after an error occurred return True line = re.sub(self.ansi_color, '', line) - for regex in self.useless_chatter: + line = line.strip() + for regex in self._useless_chatter: if regex.search(line) is not None: return True return False @@ -139,15 +173,19 @@ def _check_errors(self, line): sage: logger.raise_errors() Traceback (most recent call last): ... - OSError: [doctestin] Segmentation fault! + OSError: Segmentation fault! """ if self._error is not None: return # we already have found an error - for regex in self._error_patterns: - if regex.search(line) is not None: - self._error = line - return + for error in self._error_patterns: + if error.search(line) is not None: + for ignored in self._ignored_warnings: + if ignored.search(line) is not None: + break + else: + self._error = line + return def _log_line(self, line): r""" @@ -183,16 +221,16 @@ def _log_line(self, line): [#25160 ] Exception: artificial exception """ - if self._filter_out(line): - return + skip_this_line = self._filter_out(line) + self._check_errors(line) for (old, new) in self.replacements: line = old.sub(new, line) line = self._prefix + ' ' + line.rstrip() + '\n' if not self._color: line = self.ansi_color.sub('', line) - self._stream.write(line) - self._stream.flush() - self._check_errors(line) + if not skip_this_line: + self._stream.write(line) + self._stream.flush() def raise_errors(self): r""" @@ -209,7 +247,7 @@ def raise_errors(self): sage: logger.raise_errors() Traceback (most recent call last): ... - OSError: [doctestin] This is a SEVERE error + OSError: This is a SEVERE error """ if self._error is not None: @@ -272,6 +310,11 @@ def runsphinx(): try: sys.stdout = SageSphinxLogger(sys.stdout, os.path.basename(output_dir)) sys.stderr = SageSphinxLogger(sys.stderr, os.path.basename(output_dir)) + # Note that this call as of eraly 2018 leaks memory. So make sure that + # you don't call runsphinx() several times in a row. (i.e., you want to + # fork() somewhere before this call.) + # We don't use subprocess here, as we don't want to re-initialize Sage + # for every docbuild as this takes a while. sphinx.cmdline.main(sys.argv[1:]) sys.stderr.raise_errors() sys.stdout.raise_errors()