Merge pull request #5 from OSGeo/master

Refresh

.github/workflows/android.yml

@@ -0,0 +1,24 @@
+name: Android build
+
+on: [push, pull_request]
+
+jobs:
+
+  android_build:
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]')"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Cache
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+                ${{ github.workspace }}/ccache.tar.gz
+          key: ${{ runner.os }}-cache-android-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-cache-android-
+
+      - name: Build
+        run: docker run --privileged=true -e CI -e WORK_DIR="$PWD" -v $PWD:$PWD -v /var/run/docker.sock:/var/run/docker.sock ubuntu:20.04 $PWD/.github/workflows/android/start.sh

.github/workflows/android/start.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+
+set -e
+
+apt-get update -y
+
+# python needed for make-standalone-toolchain.sh
+# pkg-config sqlite3 for proj compilation
+# libncurses5 since android clang links against it
+DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    wget unzip ccache curl ca-certificates \
+    python \
+    pkg-config sqlite3 \
+    libncurses5
+
+cd "$WORK_DIR"
+
+if test -f "$WORK_DIR/ccache.tar.gz"; then
+    echo "Restoring ccache..."
+    (cd $HOME && tar xzf "$WORK_DIR/ccache.tar.gz")
+fi
+
+wget -q https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip
+unzip -q android-ndk-r16b-linux-x86_64.zip
+android-ndk-r16b/build/tools/make-standalone-toolchain.sh --platform=android-24 --install-dir=$HOME/android-toolchain --stl=libc++ --verbose
+
+export PATH=$HOME/android-toolchain/bin:$PATH
+export CC="ccache arm-linux-androideabi-clang"
+export CXX="ccache arm-linux-androideabi-clang++"
+export CFLAGS="-mthumb -Qunused-arguments"
+export CXXFLAGS="-mthumb -Qunused-arguments"
+
+ccache -M 1G
+ccache -s
+
+SCRIPT_DIR=$(dirname "$0")
+case $SCRIPT_DIR in
+    "/"*)
+        ;;
+    ".")
+        SCRIPT_DIR=$(pwd)
+        ;;
+    *)
+        SCRIPT_DIR=$(pwd)/$(dirname "$0")
+        ;;
+esac
+"$SCRIPT_DIR"/../common_install.sh
+
+# build sqlite3
+wget -q https://sqlite.org/2018/sqlite-autoconf-3250100.tar.gz
+tar xzf sqlite-autoconf-3250100.tar.gz
+(cd sqlite-autoconf-3250100 && ./configure --host=arm-linux-androideabi --prefix=/tmp/install && make -j3 && make install)
+
+# Build proj
+(cd proj;  ./autogen.sh && PKG_CONFIG_PATH=/tmp/install/lib/pkgconfig ./configure --host=arm-linux-androideabi --prefix=/tmp/install --disable-static && make -j3 && make install)
+
+cd gdal
+./configure --host=arm-linux-androideabi --with-proj=/tmp/install --with-sqlite3=/tmp/install
+make USER_DEFS="-Wextra -Werror" -j3
+
+ccache -s
+
+echo "Saving ccache..."
+rm -f "$WORK_DIR/ccache.tar.gz"
+(cd $HOME && tar czf "$WORK_DIR/ccache.tar.gz" .ccache)

.github/workflows/asan/start.sh

@@ -0,0 +1,165 @@
+#!/bin/sh
+
+set -e
+
+apt-get update -y
+DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    sudo wget tzdata
+
+# Emulate 'sanitize' Travis-CI target for the purpose of test skipping
+TRAVIS=yes
+export TRAVIS
+TRAVIS_BRANCH=sanitize
+export TRAVIS_BRANCH
+
+LANG=en_US.UTF-8
+export LANG
+apt-get install -y locales && \
+    echo "$LANG UTF-8" > /etc/locale.gen && \
+    dpkg-reconfigure --frontend=noninteractive locales && \
+    update-locale LANG=$LANG
+
+USER=root
+export USER
+
+cd "$WORK_DIR"
+
+if test -f "$WORK_DIR/ccache.tar.gz"; then
+    echo "Restoring ccache..."
+    (cd $HOME && tar xzf "$WORK_DIR/ccache.tar.gz")
+fi
+
+
+sudo apt-get install -y software-properties-common python-software-properties
+sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
+sudo add-apt-repository -y ppa:deadsnakes/ppa
+sudo apt-get update
+sudo apt-get install -y python3.6 python3.6-dev
+sudo apt-get install -y --allow-unauthenticated libpng12-dev libjpeg-dev libgif-dev liblzma-dev libgeos-dev libcurl4-gnutls-dev libproj-dev libxml2-dev libexpat-dev libxerces-c-dev libnetcdf-dev netcdf-bin libpoppler-dev libpoppler-private-dev libsqlite3-dev gpsbabel swig libhdf4-alt-dev libhdf5-serial-dev libpodofo-dev poppler-utils libfreexl-dev unixodbc-dev libwebp-dev libepsilon-dev liblcms2-2 libpcre3-dev libcrypto++-dev libdap-dev libfyba-dev libmysqlclient-dev libogdi3.2-dev libcfitsio-dev openjdk-8-jdk couchdb libzstd1-dev ccache curl autoconf automake sqlite3 libspatialite-dev make g++ libssl-dev libsfcgal-dev libgeotiff-dev libcharls-dev libopenjp2-7-dev libcairo2-dev
+
+wget https://github.com/Esri/file-geodatabase-api/raw/master/FileGDB_API_1.5/FileGDB_API_1_5_64gcc51.tar.gz
+tar xzf FileGDB_API_1_5_64gcc51.tar.gz
+sudo cp FileGDB_API-64gcc51/lib/* /usr/lib
+sudo ldconfig
+
+FILE=clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
+URL_ROOT=https://github.com/rouault/gdal_ci_tools/raw/master/${FILE}
+curl -Ls ${URL_ROOT}aa ${URL_ROOT}ab ${URL_ROOT}ac ${URL_ROOT}ad ${URL_ROOT}ae | tar xJf -
+
+
+SCRIPT_DIR=$(dirname "$0")
+case $SCRIPT_DIR in
+    "/"*)
+        ;;
+    ".")
+        SCRIPT_DIR=$(pwd)
+        ;;
+    *)
+        SCRIPT_DIR=$(pwd)/$(dirname "$0")
+        ;;
+esac
+$SCRIPT_DIR/../common_install.sh
+
+export ASAN_OPTIONS=allocator_may_return_null=1
+
+export CCACHE_CPP2=yes
+export CC="ccache $PWD/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang"
+export CXX="ccache $PWD/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang++"
+
+ccache -M 1G
+ccache -s
+
+# Build proj
+(cd proj;  ./autogen.sh && CFLAGS='-DPROJ_RENAME_SYMBOLS' CXXFLAGS='-DPROJ_RENAME_SYMBOLS' ./configure --disable-static --prefix=/usr/local && make -j3)
+(cd proj; sudo make -j3 install && sudo mv /usr/local/lib/libproj.so.15.0.0 /usr/local/lib/libinternalproj.so.15.0.0 && sudo rm /usr/local/lib/libproj.so*  && sudo rm /usr/local/lib/libproj.la && sudo ln -f -s libinternalproj.so.15.0.0  /usr/local/lib/libinternalproj.so.15 && sudo ln -f -s libinternalproj.so.15.0.0  /usr/local/lib/libinternalproj.so)
+
+CURRENT_DIR=$PWD
+cd gdal
+
+SANITIZE_FLAGS="-DMAKE_SANITIZE_HAPPY -fsanitize=undefined -fsanitize=address -fsanitize=unsigned-integer-overflow"
+CFLAGS=$SANITIZE_FLAGS CXXFLAGS=$SANITIZE_FLAGS LDFLAGS="-fsanitize=undefined -fsanitize=address -lstdc++" ./configure --prefix=/usr --without-libtool --enable-debug --with-jpeg12 --with-poppler --without-podofo --with-spatialite --with-mysql --with-liblzma --with-webp --with-epsilon --with-libtiff=internal --with-rename-internal-libtiff-symbols --with-hide-internal-symbols --with-gnm --with-proj=/usr/local --with-fgdb=$PWD/../FileGDB_API-64gcc51
+sed -i "s/-fsanitize=address/-fsanitize=address -shared-libasan/g" GDALmake.opt
+sed -i "s/-fsanitize=unsigned-integer-overflow/-fsanitize=unsigned-integer-overflow -fno-sanitize-recover=unsigned-integer-overflow/g" GDALmake.opt
+make USER_DEFS="-Werror" -j3
+(cd apps && make USER_DEFS="-Werror" test_ogrsf)
+(cd swig/python && \
+    echo "#!/bin/sh" > mycc.sh && \
+    echo "$CC -fsanitize=undefined -fsanitize=address -shared-libasan \$*" >> mycc.sh && \
+    cat mycc.sh && \
+    chmod +x mycc.sh && \
+    PATH=$PWD:$PATH CC=mycc.sh python3.6 setup.py build
+)
+
+sudo rm -f /usr/lib/libgdal.so*
+sudo make install
+curl -sSL 'https://bootstrap.pypa.io/get-pip.py' | sudo python3.6
+sudo python3.6 -m pip install numpy
+(cd swig/python && sudo python3.6 setup.py install)
+
+sudo ldconfig
+
+ccache -s
+
+cd "$CURRENT_DIR"
+
+echo "Saving ccache..."
+rm -f "$WORK_DIR/ccache.tar.gz"
+(cd $HOME && tar czf "$WORK_DIR/ccache.tar.gz" .ccache)
+
+
+export PRELOAD=$PWD/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-16.04/lib/clang/9.0.0/lib/linux/libclang_rt.asan-x86_64.so
+
+cd autotest
+
+# Don't run these
+rm -f ogr/ogr_fgdb.py ogr/ogr_pgeo.py
+
+# Too old spatialite version
+rm -f ogr/ogr_sqlite.py gdrivers/rasterlite.py
+
+# install test dependencies
+sudo python3.6 -m pip install -U -r ./requirements.txt
+
+# Run each module in its own pytest process.
+# This makes sure the output from the address sanitizer is relevant
+# and it doesn't blow out RAM too much.
+# Unfortunately it's also a reasonably large slowdown since we have to wait
+# for a python interpreter and all modules to load between each module.
+# (and add a grep to get rid of the extra pytest header headers/etc)
+#
+# NOTE: `find ... -exec` always exits with 0 even when the tests failed.
+# That turns out to be what we want here though, since we want
+# to not fail when the address sanitizer finds errors.
+# So we tee the output to a file and grep it to discover if the tests failed.
+export SKIP_MEM_INTENSIVE_TEST=YES SKIP_VIRTUALMEM=YES LD_PRELOAD=$PRELOAD \
+    LSAN_OPTIONS=detect_leaks=1,print_suppressions=0,suppressions=$PWD/asan_suppressions.txt
+
+gdalinfo gcore/data/byte.tif
+python3.6 -c "from osgeo import gdal; print('yes')"
+
+echo "#!/bin/sh" > pytest_wrapper.sh
+echo 'ARGS="$*"' >> pytest_wrapper.sh
+echo "python3.6 -m pytest --capture=no -ra -vv -p no:sugar --color=no -o console_output_style=classic \${ARGS} 2>&1" >> pytest_wrapper.sh
+cat pytest_wrapper.sh
+chmod +x pytest_wrapper.sh
+
+# Error on ogdi_5 test
+rm ogr/ogr_ogdi.py
+
+find \
+    ogr gcore gdrivers osr alg gnm utilities pyscripts \
+    -name '*.py' ! -name netcdf_cfchecks.py ! -name "__init__.py" ! -path 'ogr/data/*' \
+    -print \
+    -exec ./pytest_wrapper.sh {} \; \
+    | tee ./test-output.txt
+
+# Check if the tests failed and error out.
+if grep -P '===.*\d+ failed' ./test-output.txt > /dev/null ; then
+    echo 'Tests failed'
+    exit 1
+elif grep '==ABORTING' ./test-output.txt; then
+    echo 'Tests crashed'
+    exit 1
+else
+    echo 'Tests passed'
+fi

.github/workflows/asan_build.yml

@@ -0,0 +1,24 @@
+name: ASAN build
+
+on: [push, pull_request]
+
+jobs:
+
+  asan_build:
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]')"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Cache
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+                ${{ github.workspace }}/ccache.tar.gz
+          key: ${{ runner.os }}-cache-asan-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-cache-asan-
+
+      - name: Build
+        run: docker run --privileged=true -e CI -e WORK_DIR="$PWD" -v $PWD:$PWD -v /var/run/docker.sock:/var/run/docker.sock ubuntu:16.04 $PWD/.github/workflows/asan/start.sh

.github/workflows/code_checks.yml

@@ -31,7 +31,7 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CFLAGS=-O0 CXXFLAGS=-O0 ./configure --without-static --prefix=/tmp/projinstall
             make -j$(nproc)
             make install -j$(nproc)
- 
+
       - name: Run configure
         run: (cd gdal && ./configure --with-proj=/tmp/projinstall)
 
@@ -65,7 +65,7 @@ jobs:
             CC="ccache gcc" CXX="ccache g++" CFLAGS=-O0 CXXFLAGS=-O0 ./configure --without-static --prefix=/tmp/projinstall
             make -j$(nproc)
             make install -j$(nproc)
- 
+
       - name: Run configure
         run: (cd gdal && ./configure --with-proj=/tmp/projinstall)
 
@@ -91,7 +91,7 @@ jobs:
         run: ./gdal/scripts/detect_suspicious_char_digit_zero.sh
 
       - name: Shellcheck
-        run: shellcheck -e SC2086,SC2046 $(find gdal -name '*.sh' -a -not -name ltmain.sh)
+        run: shellcheck -e SC2086,SC2046,SC2164,SC2054 $(find gdal -name '*.sh' -a -not -name ltmain.sh)
 
   flake8:
     runs-on: ubuntu-latest
@@ -111,6 +111,8 @@ jobs:
             $FLAKE8 autotest
             $FLAKE8 gdal/swig/python/scripts
             $FLAKE8 gdal/swig/python/samples
+            $FLAKE8 gdal/swig/python/osgeo/utils
+            $FLAKE8 gdal/swig/python/osgeo/auxiliary
 
   doxygen:
     runs-on: ubuntu-latest
@@ -120,6 +122,7 @@ jobs:
 
       - name: Install Requirements
         run: |
+            sudo apt update
             sudo apt install doxygen texlive-latex-base
 
       - name: Run doxygen

.github/workflows/common_install.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -e
+
+# Clone proj
+#git clone --depth=1 https://github.com/OSGeo/proj.4 proj
+curl http://download.osgeo.org/proj/proj-6.0.0.tar.gz > proj-6.0.0.tar.gz
+tar xzf proj-6.0.0.tar.gz
+mv proj-6.0.0 proj
+echo "#!/bin/sh" > proj/autogen.sh
+chmod +x proj/autogen.sh
+(cd proj/data && curl http://download.osgeo.org/proj/proj-datumgrid-1.8.tar.gz > proj-datumgrid-1.8.tar.gz && tar xvzf proj-datumgrid-1.8.tar.gz)
+

.github/workflows/conda.yml

@@ -0,0 +1,57 @@
+name: Conda
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: Conda ${{ matrix.platform }}
+    if: "!contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]')"
+
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      fail-fast: true
+      matrix:
+        platform: ['ubuntu-latest','windows-latest','macos-latest']
+
+    env:
+      PLATFORM: ${{ matrix.platform }}
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        channels: conda-forge
+        auto-update-conda: true
+
+    - name: Setup
+      shell: bash -l {0}
+      run: |
+          source ./gdal/ci/travis/conda/setup.sh
+
+    - name: Support longpaths
+      run: git config --system core.longpaths true
+      if: matrix.platform == 'windows-latest'
+
+    - name: Build
+      shell: bash -l {0}
+      run: |
+          source ../gdal/ci/travis/conda/compile.sh
+      working-directory: ./gdal-feedstock
+
+    - uses: ilammy/msvc-dev-cmd@v1
+      if: matrix.platform == 'windows-latest'
+
+    - uses: actions/upload-artifact@v2
+      with:
+        name: ${{ matrix.platform }}-conda-package
+        path: ./gdal-feedstock/packages/
+
+    - name: Deploy to gdal-master Conda channel
+      if: github.ref == 'refs/heads/master'
+      shell: bash -l {0}
+      env:
+        ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
+      run: |
+          source ../gdal/ci/travis/conda/upload.sh || true
+      working-directory: ./gdal-feedstock