1.16: CI fixes backport (#13665)

Backport following commits to 1.16: 748b2ab (mac ci: try ignoring update failure (#13658), 2020-10-20) f95f539 (ci: various improvements (#13660), 2020-10-20) 6481a27 (ci: stop building alpine-debug images in favor of ubuntu-based debug image (#13598), 2020-10-15) 73d78f8 (ci: use multiple stage (#13557), 2020-10-15) b7a4756 (ci: use azp for api and go-control-plane sync (#13550), 2020-10-14) 876a6bb (ci use azp to sync filter example (#13501), 2020-10-12) a0f31ee (ci: use azp to generate docs (#13481), 2020-10-12) 7e5d854 (ci: bring BAZEL_BUILD_OPTIONS back for format and docs (#13480), 2020-10-11) 7af2b2b (ci: use same flaky test process script on macOS (#13485), 2020-10-10) Signed-off-by: Lizan Zhou <lizan@tetrate.io> Co-authored-by: asraa <asraa@google.com>
envoyproxy · Oct 21, 2020 · 8620920 · 8620920
1 parent b8a804a
commit 8620920
Show file tree

Hide file tree

Showing 14 changed files with 419 additions and 351 deletions.
diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml
diff --git a/.circleci/config.yml b/.circleci/config.yml
diff --git a/ci/api_mirror.sh b/ci/api_mirror.sh
@@ -3,16 +3,15 @@
 set -e
 
 CHECKOUT_DIR=../data-plane-api
+MAIN_BRANCH="refs/heads/master"
+API_MAIN_BRANCH="master"
 
-if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ]
-then
+if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then
   echo "Cloning..."
-  git clone git@github.com:envoyproxy/data-plane-api "$CHECKOUT_DIR"
+  git clone git@github.com:envoyproxy/data-plane-api "$CHECKOUT_DIR" -b "${API_MAIN_BRANCH}"
 
-  git -C "$CHECKOUT_DIR" config user.name "data-plane-api(CircleCI)"
+  git -C "$CHECKOUT_DIR" config user.name "data-plane-api(Azure Pipelines)"
   git -C "$CHECKOUT_DIR" config user.email data-plane-api@users.noreply.github.com
-  git -C "$CHECKOUT_DIR" fetch
-  git -C "$CHECKOUT_DIR" checkout -B master origin/master
 
   # Determine last envoyproxy/envoy SHA in envoyproxy/data-plane-api
   MIRROR_MSG="Mirrored from https://github.com/envoyproxy/envoy"
@@ -40,6 +39,6 @@ then
   done
 
   echo "Pushing..."
-  git -C "$CHECKOUT_DIR" push origin master
+  git -C "$CHECKOUT_DIR" push origin "${API_MAIN_BRANCH}"
   echo "Done"
 fi
diff --git a/ci/do_ci.sh b/ci/do_ci.sh
@@ -311,19 +311,21 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then
   collect_build_profile build
   exit 0
 elif [[ "$CI_TARGET" == "bazel.api" ]]; then
+  # Use libstdc++ because the API booster links to prebuilt libclang*/libLLVM* installed in /opt/llvm/lib,
+  # which is built with libstdc++. Using libstdc++ for whole of the API CI job to avoid unnecessary rebuild.
+  ENVOY_STDLIB="libstdc++"
   setup_clang_toolchain
+  export LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config
   echo "Validating API structure..."
   ./tools/api/validate_structure.py
+  echo "Testing API and API Boosting..."
+  bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api_canonical//test/... @envoy_api_canonical//tools/... \
+    @envoy_api_canonical//tools:tap2pcap_test @envoy_dev//clang_tools/api_booster/...
   echo "Building API..."
   bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api_canonical//envoy/...
-  echo "Testing API..."
-  bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_api_canonical//test/... @envoy_api_canonical//tools/... \
-    @envoy_api_canonical//tools:tap2pcap_test
-  echo "Testing API boosting (unit tests)..."
-  bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild @envoy_dev//clang_tools/api_booster/...
   echo "Testing API boosting (golden C++ tests)..."
   # We use custom BAZEL_BUILD_OPTIONS here; the API booster isn't capable of working with libc++ yet.
-  LLVM_CONFIG="${LLVM_ROOT}"/bin/llvm-config BAZEL_BUILD_OPTIONS="--config=clang" python3.8 ./tools/api_boost/api_boost_test.py
+  BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS[*]}" python3.8 ./tools/api_boost/api_boost_test.py
   exit 0
 elif [[ "$CI_TARGET" == "bazel.coverage" || "$CI_TARGET" == "bazel.fuzz_coverage" ]]; then
   setup_clang_toolchain
@@ -368,21 +370,23 @@ elif [[ "$CI_TARGET" == "bazel.fuzz" ]]; then
 elif [[ "$CI_TARGET" == "fix_format" ]]; then
   # proto_format.sh needs to build protobuf.
   setup_clang_toolchain
+
   echo "fix_format..."
   ./tools/code_format/check_format.py fix
   ./tools/code_format/format_python_tools.sh fix
-  ./tools/proto_format/proto_format.sh fix --test
+  BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS[*]}" ./tools/proto_format/proto_format.sh fix --test
   exit 0
 elif [[ "$CI_TARGET" == "check_format" ]]; then
   # proto_format.sh needs to build protobuf.
   setup_clang_toolchain
+
   echo "check_format_test..."
   ./tools/code_format/check_format_test_helper.sh --log=WARN
   echo "check_format..."
   ./tools/code_format/check_shellcheck_format.sh
   ./tools/code_format/check_format.py check
   ./tools/code_format/format_python_tools.sh check
-  ./tools/proto_format/proto_format.sh check --test
+  BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS[*]}" ./tools/proto_format/proto_format.sh check --test
   exit 0
 elif [[ "$CI_TARGET" == "check_repositories" ]]; then
   echo "check_repositories..."
@@ -410,7 +414,7 @@ elif [[ "$CI_TARGET" == "docs" ]]; then
   tools/dependency/validate_test.py
   tools/dependency/validate.py
   # Build docs.
-  docs/build.sh
+  BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS[*]}" docs/build.sh
   exit 0
 elif [[ "$CI_TARGET" == "verify_examples" ]]; then
   echo "verify examples..."

diff --git a/ci/filter_example_mirror.sh b/ci/filter_example_mirror.sh
@@ -4,16 +4,15 @@ set -e
 
 ENVOY_SRCDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../" && pwd)
 CHECKOUT_DIR=../envoy-filter-example
+MAIN_BRANCH="refs/heads/master"
+FILTER_EXAMPLE_MAIN_BRANCH="master"
 
-if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ]
-then
+if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then
   echo "Cloning..."
-  git clone git@github.com:envoyproxy/envoy-filter-example "$CHECKOUT_DIR"
+  git clone git@github.com:envoyproxy/envoy-filter-example "$CHECKOUT_DIR" -b "${FILTER_EXAMPLE_MAIN_BRANCH}"
 
-  git -C "$CHECKOUT_DIR" config user.name "envoy-filter-example(CircleCI)"
+  git -C "$CHECKOUT_DIR" config user.name "envoy-filter-example(Azure Pipelines)"
   git -C "$CHECKOUT_DIR" config user.email envoy-filter-example@users.noreply.github.com
-  git -C "$CHECKOUT_DIR" fetch
-  git -C "$CHECKOUT_DIR" checkout -B master origin/master
 
   echo "Updating Submodule..."
   # Update submodule to latest Envoy SHA
@@ -26,6 +25,6 @@ then
 
   echo "Committing, and Pushing..."
   git -C "$CHECKOUT_DIR" commit -a -m "Update Envoy submodule to $ENVOY_SHA"
-  git -C "$CHECKOUT_DIR" push origin master
+  git -C "$CHECKOUT_DIR" push origin "${FILTER_EXAMPLE_MAIN_BRANCH}"
   echo "Done"
 fi
diff --git a/ci/flaky_test/run_process_xml.sh b/ci/flaky_test/run_process_xml.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+export ENVOY_SRCDIR=${ENVOY_SRCDIR:-.}
+
 # shellcheck source=tools/shell_utils.sh
 . "${ENVOY_SRCDIR}"/tools/shell_utils.sh
 

diff --git a/ci/flaky_test/run_process_xml_mac.sh b/ci/flaky_test/run_process_xml_mac.sh
diff --git a/ci/go_mirror.sh b/ci/go_mirror.sh
@@ -2,7 +2,11 @@
 
 set -e
 
-if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ]
-then
-  tools/api/generate_go_protobuf.py
+MAIN_BRANCH="refs/heads/master"
+
+# shellcheck source=ci/setup_cache.sh
+. "$(dirname "$0")"/setup_cache.sh
+
+if [[ "${AZP_BRANCH}" == "${MAIN_BRANCH}" ]]; then
+  BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_EXTRA_OPTIONS}" tools/api/generate_go_protobuf.py
 fi
diff --git a/ci/mac_ci_setup.sh b/ci/mac_ci_setup.sh
@@ -38,8 +38,8 @@ function retry () {
 }
 
 if ! retry brew update; then
-    echo "Failed to update homebrew"
-    exit 1
+  # Do not exit early if update fails.
+  echo "Failed to update homebrew"
 fi
 
 DEPS="automake cmake coreutils go libtool wget ninja"
@@ -65,4 +65,4 @@ fi
 
 bazel version
 
-pip3 install slackclient
+pip3 install virtualenv
diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh
@@ -56,6 +56,7 @@ mkdir -p "${ENVOY_DOCKER_BUILD_DIR}"
 
 [[ -t 1 ]] && ENVOY_DOCKER_OPTIONS+=("-it")
 [[ -f .git ]] && [[ ! -d .git ]] && ENVOY_DOCKER_OPTIONS+=(-v "$(git rev-parse --git-common-dir):$(git rev-parse --git-common-dir)")
+[[ -n "${SSH_AUTH_SOCK}" ]] && ENVOY_DOCKER_OPTIONS+=(-v "${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK}" -e SSH_AUTH_SOCK)
 
 export ENVOY_BUILD_IMAGE="${IMAGE_NAME}:${IMAGE_ID}"
 
@@ -64,6 +65,7 @@ docker run --rm \
        "${ENVOY_DOCKER_OPTIONS[@]}" \
        -v "${ENVOY_DOCKER_BUILD_DIR}":"${BUILD_DIR_MOUNT_DEST}" \
        -v "${SOURCE_DIR}":"${SOURCE_DIR_MOUNT_DEST}" \
+       -e AZP_BRANCH \
        -e HTTP_PROXY \
        -e HTTPS_PROXY \
        -e NO_PROXY \

diff --git a/docs/build.sh b/docs/build.sh
@@ -7,25 +7,30 @@
 
 set -e
 
+RELEASE_TAG_REGEX="^refs/tags/v.*"
+
+if [[ "${AZP_BRANCH}" =~ ${RELEASE_TAG_REGEX} ]]; then
+  DOCS_TAG="${AZP_BRANCH/refs\/tags\//}"
+fi
+
 # We need to set ENVOY_DOCS_VERSION_STRING and ENVOY_DOCS_RELEASE_LEVEL for Sphinx.
 # We also validate that the tag and version match at this point if needed.
-if [ -n "$CIRCLE_TAG" ]
-then
+if [[ -n "${DOCS_TAG}" ]]; then
   # Check the git tag matches the version number in the VERSION file.
   VERSION_NUMBER=$(cat VERSION)
-  if [ "v${VERSION_NUMBER}" != "${CIRCLE_TAG}" ]; then
+  if [[ "v${VERSION_NUMBER}" != "${DOCS_TAG}" ]]; then
     echo "Given git tag does not match the VERSION file content:"
-    echo "${CIRCLE_TAG} vs $(cat VERSION)"
+    echo "${DOCS_TAG} vs $(cat VERSION)"
     exit 1
   fi
   # Check the version_history.rst contains current release version.
   grep --fixed-strings "$VERSION_NUMBER" docs/root/version_history/current.rst \
     || (echo "Git tag not found in version_history/current.rst" && exit 1)
 
   # Now that we know there is a match, we can use the tag.
-  export ENVOY_DOCS_VERSION_STRING="tag-$CIRCLE_TAG"
+  export ENVOY_DOCS_VERSION_STRING="tag-${DOCS_TAG}"
   export ENVOY_DOCS_RELEASE_LEVEL=tagged
-  export ENVOY_BLOB_SHA="$CIRCLE_TAG"
+  export ENVOY_BLOB_SHA="${DOCS_TAG}"
 else
   BUILD_SHA=$(git rev-parse HEAD)
   VERSION_NUM=$(cat VERSION)

diff --git a/docs/publish.sh b/docs/publish.sh
@@ -10,35 +10,36 @@
 set -e
 
 DOCS_DIR=generated/docs
-CHECKOUT_DIR=../envoy-docs
+CHECKOUT_DIR=envoy-docs
 BUILD_SHA=$(git rev-parse HEAD)
 
-if [ -n "$CIRCLE_TAG" ]
-then
-  PUBLISH_DIR="$CHECKOUT_DIR"/docs/envoy/"$CIRCLE_TAG"
-elif [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ]
-then
-  PUBLISH_DIR="$CHECKOUT_DIR"/docs/envoy/latest
+MAIN_BRANCH="refs/heads/master"
+RELEASE_TAG_REGEX="^refs/tags/v.*"
+
+if [[ "${AZP_BRANCH}" =~ ${RELEASE_TAG_REGEX} ]]; then
+  PUBLISH_DIR="${CHECKOUT_DIR}"/docs/envoy/"${AZP_BRANCH/refs\/tags\//}"
+elif [[ "$AZP_BRANCH" == "${MAIN_BRANCH}" ]]; then
+  PUBLISH_DIR="${CHECKOUT_DIR}"/docs/envoy/latest
 else
   echo "Ignoring docs push"
   exit 0
 fi
 
+DOCS_MAIN_BRANCH="master"
+
 echo 'cloning'
-git clone git@github.com:envoyproxy/envoyproxy.github.io "$CHECKOUT_DIR"
+git clone git@github.com:envoyproxy/envoyproxy.github.io "${CHECKOUT_DIR}" -b "${DOCS_MAIN_BRANCH}" --depth 1
 
-git -C "$CHECKOUT_DIR" fetch
-git -C "$CHECKOUT_DIR" checkout -B master origin/master
 rm -fr "$PUBLISH_DIR"
 mkdir -p "$PUBLISH_DIR"
 cp -r "$DOCS_DIR"/* "$PUBLISH_DIR"
-cd "$CHECKOUT_DIR"
+cd "${CHECKOUT_DIR}"
 
-git config user.name "envoy-docs(travis)"
+git config user.name "envoy-docs(Azure Pipelines)"
 git config user.email envoy-docs@users.noreply.github.com
-echo 'add'
+
+set -x
+
 git add .
-echo 'commit'
 git commit -m "docs envoy@$BUILD_SHA"
-echo 'push'
-git push origin master
+git push origin "${DOCS_MAIN_BRANCH}"
diff --git a/tools/api/generate_go_protobuf.py b/tools/api/generate_go_protobuf.py
@@ -4,17 +4,21 @@
 from subprocess import check_call
 import glob
 import os
+import shlex
 import shutil
 import sys
 import re
 
+# Needed for CI to pass down bazel options.
+BAZEL_BUILD_OPTIONS = shlex.split(os.environ.get('BAZEL_BUILD_OPTIONS', ''))
+
 TARGETS = '@envoy_api//...'
 IMPORT_BASE = 'github.com/envoyproxy/go-control-plane'
 OUTPUT_BASE = 'build_go'
 REPO_BASE = 'go-control-plane'
 BRANCH = 'master'
 MIRROR_MSG = 'Mirrored from envoyproxy/envoy @ '
-USER_NAME = 'go-control-plane(CircleCI)'
+USER_NAME = 'go-control-plane(Azure Pipelines)'
 USER_EMAIL = 'go-control-plane@users.noreply.github.com'
 
 
@@ -32,7 +36,7 @@ def generateProtobufs(output):
   check_call([
       'bazel', 'build', '-c', 'fastbuild',
       '--experimental_proto_descriptor_sets_include_source_info'
-  ] + go_protos)
+  ] + BAZEL_BUILD_OPTIONS + go_protos)
 
   for rule in go_protos:
     # Example rule:
@@ -67,9 +71,7 @@ def git(repo, *args):
 
 def cloneGoProtobufs(repo):
   # Create a local clone of go-control-plane
-  git(None, 'clone', 'git@github.com:envoyproxy/go-control-plane', repo)
-  git(repo, 'fetch')
-  git(repo, 'checkout', '-B', BRANCH, 'origin/master')
+  git(None, 'clone', 'git@github.com:envoyproxy/go-control-plane', repo, '-b', BRANCH)
 
 
 def findLastSyncSHA(repo):

diff --git a/tools/api_boost/api_boost.py b/tools/api_boost/api_boost.py
@@ -132,7 +132,6 @@ def ApiBoostTree(target_paths,
     sp.run([
         'bazel',
         'build',
-        '--config=libc++',
         '--strip=always',
     ] + BAZEL_BUILD_OPTIONS + dep_lib_build_targets,
            check=True)