fix: memory leaks (DEV-237) (#365)

Co-authored-by: Lukas Rosenthaler <lukas.rosenthaler@unibas.ch>
dasch-swiss · Jan 15, 2022 · c3b9b35 · c3b9b35
1 parent 2e257c6
commit c3b9b35
Show file tree

Hide file tree

Showing 36 changed files with 741 additions and 516 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,4 +1,9 @@
-**/*.pyc
+.git
+.gitignore
+build
+build-linux
+Dockerfile
+README.md
 **/__pycache__/
 wheels/
 pip-log.txt

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -6,8 +6,8 @@ on:
     types: [published]
 
 jobs:
-  test:
-    name: Build and Test
+  unit:
+    name: Run Unit Tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout source
@@ -22,54 +22,50 @@ jobs:
           path: ci
       - name: Copy ci-assets
         run: |
-          cp $GITHUB_WORKSPACE/ci/kakadu/v8_0_5-01727L.zip $GITHUB_WORKSPACE/vendor/v8_0_5-01727L.zip
+          cp $GITHUB_WORKSPACE/ci/kakadu/v8_2_1-01727L.zip $GITHUB_WORKSPACE/vendor/v8_2_1-01727L.zip
       - name: Set up Python 3.9
         uses: actions/setup-python@v2
         with:
           python-version: 3.9
       - name: install requirements
         run: make install-requirements
-      - name: run build
+      - name: compile
         run: make compile-ci
-      - name: run test
+      - name: test
         run: make test-ci
-      - name: run integration tests
-        run: make test-integration
 
-  # build documentation
-  docs-build-test:
-    name: Docs Build Test
+  integration:
+    name: Run Integration Tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout source
+        uses: actions/checkout@v1
+        with:
+          fetch-depth: 50
+      - name: Checkout private ci-assets
         uses: actions/checkout@v2
         with:
-          fetch-depth: 1
-      - name: Set up Python 3.8
+          repository: dasch-swiss/dsp-ci-assets
+          token: ${{ secrets.DASCHBOT_PAT }} # `GitHub_PAT` is a secret that contains your PAT
+          path: ci
+      - name: Copy ci-assets
+        run: |
+          cp $GITHUB_WORKSPACE/ci/kakadu/v8_2_1-01727L.zip $GITHUB_WORKSPACE/vendor/v8_2_1-01727L.zip
+      - name: Set up Python 3.9
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-      - name: Disk Free
-        run: |
-          df -h
-          docker system df
-          docker system prune --all --force --volumes
-          df -h
-      - name: run docs build
-        run: make docs-build
-      - name: Disk Free After
-        run: |
-          df -h
-          docker system df
+          python-version: 3.9
+      - name: install requirements
+        run: make install-requirements
+      - name: build docker image
+        run: make docker-build
+      - name: integration tests
+        run: make test-integration
 
   # publish only on release
-  publish:
-    name: Publish to Dockerhub
-    needs: test
+  publish-images:
+    name: Build and Publish Images to Dockerhub
+    needs: [unit, integration]
     runs-on: ubuntu-latest
     if: github.event_name == 'release' && startsWith(github.ref, 'refs/tags')
     steps:
@@ -85,20 +81,44 @@ jobs:
           path: ci
       - name: copy ci-assets
         run: |
-          cp $GITHUB_WORKSPACE/ci/kakadu/v8_0_5-01727L.zip $GITHUB_WORKSPACE/vendor/v8_0_5-01727L.zip
-      - name: build and publish image
+          cp $GITHUB_WORKSPACE/ci/kakadu/v8_2_1-01727L.zip $GITHUB_WORKSPACE/vendor/v8_2_1-01727L.zip
+      - name: build and publish production image
         run: |
           echo ${{ secrets.DOCKER_HUB_TOKEN }} | docker login -u ${{ secrets.DOCKER_USER }} --password-stdin
           make docker-publish
+      - name: build and publish debug image
+        run: |
+          echo ${{ secrets.DOCKER_HUB_TOKEN }} | docker login -u ${{ secrets.DOCKER_USER }} --password-stdin
+          make docker-publish-debug
       - name: Update release notes
         uses: lakto/gren-action@v1.1.0
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
 
-  # deploy documentation when merged into main branch
-  deploy-docs:
-    name: Deploy docs
-    needs: [docs-build-test]
+  # build documentation
+  docs:
+    name: Build Docs Testrun
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      - name: run docs build
+        run: make docs-build
+
+  # publish documentation when merged into main branch
+  publish-docs:
+    name: Build and Publish Docs
+    needs: [docs]
     runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/main'
     steps:
@@ -109,7 +129,3 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
           REQUIREMENTS: requirements.txt
-      - name: Disk Free After
-        run: |
-          df -h
-          docker system df
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,24 @@
-# STAGE 1: Build
-FROM daschswiss/sipi-base:2.4.2 as builder
+# Expose (global) variables (ARGs before FROM can only be used on FROM lines and not afterwards)
+ARG BUILD_TYPE=production
+ARG SIPI_BASE=daschswiss/sipi-base:2.5.0
+ARG UBUNTU_BASE=ubuntu:20.04
+
+# STAGE 1: Build debug
+FROM $SIPI_BASE as builder-debug
+
+WORKDIR /sipi
+
+# Add everything to image.
+COPY . .
+
+# Build SIPI.
+RUN mkdir -p /sipi/build-linux && \
+    cd /sipi/build-linux && \
+    cmake -DMAKE_DEBUG:BOOL=ON .. && \
+    make
+
+# STAGE 1: Build production
+FROM $SIPI_BASE as builder-production
 
 WORKDIR /sipi
 
@@ -12,8 +31,63 @@ RUN mkdir -p /sipi/build-linux && \
     cmake -DMAKE_DEBUG:BOOL=OFF .. && \
     make
 
-# STAGE 2: Setup
-FROM ubuntu:20.04
+# STAGE 2: Setup debug
+FROM $UBUNTU_BASE as debug
+
+MAINTAINER Ivan Subotic <400790+subotic@users.noreply.github.com>
+
+# Silence debconf messages
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN sed -i 's/# \(.*multiverse$\)/\1/g' /etc/apt/sources.list && \
+    apt-get clean && apt-get -qq update && apt-get -y install \
+    ca-certificates \
+    gnupg2
+
+# Install build dependencies.
+RUN sed -i 's/# \(.*multiverse$\)/\1/g' /etc/apt/sources.list && \
+    echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-11 main' | tee -a /etc/apt/sources.list && \
+    echo 'deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-11 main' | tee -a /etc/apt/sources.list && \
+    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 15CF4D18AF4F7421 && \
+    apt-get clean && apt-get -qq update && apt-get -y install \
+    ca-certificates \
+    byobu curl git htop man vim wget unzip \
+    libllvm11 llvm-11-runtime \
+    openssl \
+    libidn11-dev \
+    locales \
+    uuid \
+    ffmpeg \
+    at \
+    valgrind
+
+# add locales
+RUN locale-gen en_US.UTF-8 && \
+    locale-gen sr_RS.UTF-8
+
+ENV LC_ALL=en_US.UTF-8
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US.UTF-8
+
+WORKDIR /sipi
+
+EXPOSE 1024
+
+RUN mkdir -p /sipi/images/knora && \
+    mkdir -p /sipi/cache
+
+# Copy Sipi binary and other files from the build stage
+COPY --from=builder-debug /sipi/build-linux/sipi /sipi/sipi
+COPY --from=builder-debug /sipi/config/sipi.config.lua /sipi/config/sipi.config.lua
+COPY --from=builder-debug /sipi/config/sipi.init.lua /sipi/config/sipi.init.lua
+COPY --from=builder-debug /sipi/server/test.html /sipi/server/test.html
+
+ENTRYPOINT [ "/sipi/sipi" ]
+
+CMD ["--config=/sipi/config/sipi.config.lua"]
+
+# STAGE 2: Setup production
+FROM $UBUNTU_BASE as production
 
 MAINTAINER Ivan Subotic <400790+subotic@users.noreply.github.com>
 
@@ -57,11 +131,15 @@ RUN mkdir -p /sipi/images/knora && \
     mkdir -p /sipi/cache
 
 # Copy Sipi binary and other files from the build stage
-COPY --from=builder /sipi/build-linux/sipi /sipi/sipi
-COPY --from=builder /sipi/config/sipi.config.lua /sipi/config/sipi.config.lua
-COPY --from=builder /sipi/config/sipi.init.lua /sipi/config/sipi.init.lua
-COPY --from=builder /sipi/server/test.html /sipi/server/test.html
+COPY --from=builder-production /sipi/build-linux/sipi /sipi/sipi
+COPY --from=builder-production /sipi/config/sipi.config.lua /sipi/config/sipi.config.lua
+COPY --from=builder-production /sipi/config/sipi.init.lua /sipi/config/sipi.init.lua
+COPY --from=builder-production /sipi/server/test.html /sipi/server/test.html
 
 ENTRYPOINT [ "/sipi/sipi" ]
 
 CMD ["--config=/sipi/config/sipi.config.lua"]
+
+#
+# Stage 3: The final build type specific image
+FROM $BUILD_TYPE