Add initial GitHub workflows (#676)

* Add GitHub workflows

* Workflows with CMake compilation on Windows

* Ubuntu workflow with Boost

* Ignore warnings from Boost

* Compile unit tests on Windows

* Disable cpuinfo tools if compiled with ninja

* Use a separate CMakeSettings.json for CI

* Disable CMake debugs

* Fix unit tests compilation with Ninja Release

* Use FBGEMM in Windows workflow; add comments

* Fix C4706 warning

* Update CHANGELOG

* Run Windows build on pull requests

* Compile SentencePiece statically in Windows workflow

* Add GitHub workflow on MacOS

* Address review comments

* Disable C4702 globally, not only in Debug

* Update CHANGELOG and workflows names

* Update VERSION

.github/workflows/build-macos-10.15-cpu.yml

@@ -0,0 +1,51 @@
+name: macos-10.5-cpu
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: macos-10.15
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Install dependencies
+      run: brew install openblas protobuf
+
+    # Openblas location is exported explicitly because openblas is keg-only,
+    # which means it was not symlinked into /usr/local/.
+    # CMake cannot find BLAS on GitHub runners if Marian is being compiled
+    # statically, hence USE_STATIC_LIBS=off
+    - name: Configure CMake
+      run: |
+        export LDFLAGS="-L/usr/local/opt/openblas/lib"
+        export CPPFLAGS="-I/usr/local/opt/openblas/include"
+        mkdir -p build
+        cd build
+        cmake .. -DCOMPILE_CPU=on -DCOMPILE_CUDA=off -DCOMPILE_EXAMPLES=on -DCOMPILE_SERVER=on -DCOMPILE_TESTS=on \
+          -DUSE_FBGEMM=on -DUSE_SENTENCEPIECE=on -DUSE_STATIC_LIBS=off
+
+    - name: Compile
+      working-directory: build
+      run: make -j2
+
+    - name: Run unit tests
+      working-directory: build
+      run: make test
+
+    - name: Print versions
+      working-directory: build
+      run: |
+        ./marian --version
+        ./marian-decoder --version
+        ./marian-scorer --version
+        ./spm_encode --version
+

.github/workflows/build-ubuntu-18.04-cpu.yml

@@ -0,0 +1,64 @@
+name: ubuntu-18.04-cpu
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-18.04
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    # The following packages are already installed on GitHub-hosted runners: build-essential openssl libssl-dev
+    - name: Install dependencies
+      run: sudo apt-get install --no-install-recommends libgoogle-perftools-dev libprotobuf10 libprotobuf-dev protobuf-compiler
+
+    # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
+    - name: Install MKL
+      run: |
+        wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add -
+        sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list"
+        sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list"
+        sudo apt-get install --no-install-recommends intel-mkl-64bit-2020.0-088
+
+    - name: Print Boost paths
+      run: |
+        ls $BOOST_ROOT_1_69_0
+        ls $BOOST_ROOT_1_69_0/include
+        ls $BOOST_ROOT_1_69_0/lib
+
+    # Boost is already installed on GitHub-hosted runners in a non-standard location
+    # https://github.com/actions/virtual-environments/issues/687#issuecomment-610471671
+    - name: Configure CMake
+      run: |
+        mkdir -p build
+        cd build
+        cmake .. -DCOMPILE_CPU=on -DCOMPILE_CUDA=off -DCOMPILE_EXAMPLES=on -DCOMPILE_SERVER=on -DCOMPILE_TESTS=on \
+          -DUSE_FBGEMM=on -DUSE_SENTENCEPIECE=on \
+          -DBOOST_ROOT=$BOOST_ROOT_1_69_0 -DBOOST_INCLUDEDIR=$BOOST_ROOT_1_69_0/include -DBOOST_LIBRARYDIR=$BOOST_ROOT_1_69_0/lib \
+          -DBoost_ARCHITECTURE=-x64
+
+    - name: Compile
+      working-directory: build
+      run: make -j2
+
+    - name: Run unit tests
+      working-directory: build
+      run: make test
+
+    - name: Print versions
+      working-directory: build
+      run: |
+        ./marian --version
+        ./marian-decoder --version
+        ./marian-scorer --version
+        ./spm_encode --version
+

.github/workflows/build-windows-2019-cpu.yml

@@ -0,0 +1,49 @@
+name: windows-2019-cpu
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: windows-2019
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Prepare vcpkg
+      uses: lukka/run-vcpkg@v3
+      with:
+        vcpkgArguments: protobuf
+        vcpkgGitCommitId: 6185aa76504a5025f36754324abf307cc776f3da
+        vcpkgDirectory: ${{ github.workspace }}/vcpkg/
+        vcpkgTriplet: x64-windows-static
+
+    # Note that we build with a simplified CMake settings JSON file
+    - name: Run CMake
+      uses: lukka/run-cmake@v2
+      with:
+        buildDirectory: ${{ github.workspace }}/build/
+        cmakeAppendedArgs: -G Ninja
+        cmakeListsOrSettingsJson: CMakeSettingsJson
+        cmakeSettingsJsonPath: ${{ github.workspace }}/CMakeSettingsCI.json
+        useVcpkgToolchainFile: true
+
+    - name: Run unit tests
+      working-directory: build/Debug/
+      run: ctest
+
+    - name: Print versions
+      working-directory: build/Debug/
+      run: |
+        .\marian.exe --version
+        .\marian-decoder.exe --version
+        .\marian-scorer.exe --version
+        .\spm_encode.exe --version
+

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [Unreleased]
 
 ### Added
+- GitHub workflows on Ubuntu, Windows, and MacOS
 - LSH indexing to replace short list
 - ONNX support for transformer models
 - Add topk operator like PyTorch's topk
@@ -19,6 +20,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
   and translation with options --tsv and --tsv-fields n.
 
 ### Fixed
+- CMake-based compilation on Windows
 - Fix minor issues with compilation on MacOS
 - Fix warnings in Windows MSVC builds using CMake
 - Fix building server with Boost 1.72

CMakeLists.txt

@@ -51,6 +51,8 @@ message(STATUS "Project version: ${PROJECT_VERSION_STRING_FULL}")
 execute_process(COMMAND git submodule update --init --recursive --no-fetch
                 WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 
+# Note that with CMake MSVC build, the option CMAKE_BUILD_TYPE is automatically derived from the key
+# 'configurationType' in CMakeSettings.json configurations
 if(NOT CMAKE_BUILD_TYPE)
   message(WARNING "CMAKE_BUILD_TYPE not set; setting to Release")
   set(CMAKE_BUILD_TYPE "Release")
@@ -65,7 +67,8 @@ if(MSVC)
   # Disabled bogus warnings for CPU intrinsics:
   # C4310: cast truncates constant value
   # C4324: 'marian::cpu::int16::`anonymous-namespace'::ScatterPut': structure was padded due to alignment specifier
-  set(DISABLE_GLOBALLY "/wd\"4310\" /wd\"4324\"")
+  # C4702: unreachable code; note it is also disabled globally in the VS project file
+  set(DISABLE_GLOBALLY "/wd\"4310\" /wd\"4324\" /wd\"4702\"")
 
   # set(INTRINSICS "/arch:AVX")
   add_definitions(-DUSE_SSE2=1)

CMakeSettingsCI.json

@@ -0,0 +1,52 @@
+{
+  "configurations": [
+    {
+      "name": "Release",
+      "generator": "Ninja",
+      "configurationType": "Release",
+      "inheritEnvironments": [ "msvc_x64" ],
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "-v",
+      "ctestCommandArgs": "",
+      "variables": [
+        { "name": "OPENSSL_USE_STATIC_LIBS:BOOL", "value": "TRUE" },
+        { "name": "OPENSSL_MSVC_STATIC_RT:BOOL", "value": "TRUE" },
+
+        { "name": "COMPILE_CUDA:BOOL", "value": "FALSE" },
+        { "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
+        { "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
+        { "name": "COMPILE_SERVER:BOOL", "value": "FALSE" },
+        { "name": "COMPILE_TESTS:BOOL", "value": "TRUE" },
+
+        { "name": "USE_FBGEMM:BOOL", "value": "TRUE" },
+        { "name": "USE_MPI:BOOL", "value": "FALSE" },
+        { "name": "USE_SENTENCEPIECE:BOOL", "value": "TRUE" },
+        { "name": "USE_STATIC_LIBS:BOOL", "value": "TRUE" }
+      ]
+    },
+    {
+      "name": "Debug",
+      "generator": "Ninja",
+      "configurationType": "Debug",
+      "inheritEnvironments": [ "msvc_x64" ],
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "-v",
+      "ctestCommandArgs": "",
+      "variables": [
+        { "name": "OPENSSL_MSVC_STATIC_RT:BOOL", "value": "TRUE" },
+        { "name": "OPENSSL_USE_STATIC_LIBS:BOOL", "value": "TRUE" },
+
+        { "name": "COMPILE_CUDA:BOOL", "value": "FALSE" },
+        { "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
+        { "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
+        { "name": "COMPILE_SERVER:BOOL", "value": "FALSE" },
+        { "name": "COMPILE_TESTS:BOOL", "value": "TRUE" },
+
+        { "name": "USE_FBGEMM:BOOL", "value": "TRUE" },
+        { "name": "USE_MPI:BOOL", "value": "FALSE" },
+        { "name": "USE_SENTENCEPIECE:BOOL", "value": "TRUE" },
+        { "name": "USE_STATIC_LIBS:BOOL", "value": "TRUE" }
+      ]
+    }
+  ]
+}

VERSION

@@ -1,2 +1,2 @@
-v1.9.28
+v1.9.29
 

cmake/GetCacheVariables.cmake

@@ -34,17 +34,18 @@ foreach(_variableName ${_variableNames})
         (NOT "${_variableType}" STREQUAL "INTERNAL") AND
         (NOT "${_variableValue}" STREQUAL "") )
 
-
-        set(PROJECT_CMAKE_CACHE_ADVANCED "${PROJECT_CMAKE_CACHE_ADVANCED}    \"${_variableName}=${_variableValue}\\n\"\n")
+        string(REPLACE "\"" " " _variableValueEscapedQuotes ${_variableValue})
+        string(REPLACE "\\" "/" _variableValueEscaped ${_variableValueEscapedQuotes})
+        set(PROJECT_CMAKE_CACHE_ADVANCED "${PROJECT_CMAKE_CACHE_ADVANCED}    \"${_variableName}=${_variableValueEscaped}\\n\"\n")
 
         # Get the variable's advanced flag
         get_property(_isAdvanced CACHE ${_variableName} PROPERTY ADVANCED SET)
         if(NOT _isAdvanced)
-          set(PROJECT_CMAKE_CACHE "${PROJECT_CMAKE_CACHE}    \"${_variableName}=${_variableValue}\\n\"\n")
+          set(PROJECT_CMAKE_CACHE "${PROJECT_CMAKE_CACHE}    \"${_variableName}=${_variableValueEscaped}\\n\"\n")
         endif()
 
         # Print variables for debugging
-        #message(STATUS "${_variableName}=${${_variableName}}")
+        #message(STATUS "${_variableName}=${_variableValueEscaped}")
         #message(STATUS "  Type=${_variableType}")
         #message(STATUS "  Advanced=${_isAdvanced}")
     endif()

src/3rd_party/CMakeLists.txt

@@ -19,6 +19,9 @@ if(USE_FBGEMM)
     # only locally disabled for the 3rd_party folder
     # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-value -Wno-unused-parameter -Wno-unused-variable -Wno-unused-function")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused")
+  else()
+    # Do not compile cpuinfo executables due to a linker error, and they are not needed
+    set(CPUINFO_BUILD_TOOLS OFF CACHE BOOL "Build command-line tools")
   endif()
 
   set(FBGEMM_BUILD_TESTS OFF CACHE BOOL "Disable fbgemm tests")

src/3rd_party/phf/phf.cc

@@ -679,7 +679,8 @@ PHF_PUBLIC void PHF::compact(struct phf *phf) {
 	}
 
 	/* simply keep old array if realloc fails */
-	if ((tmp = realloc(phf->g, phf->r * size)))
+	tmp = realloc(phf->g, phf->r * size);
+	if (tmp != 0)
 		phf->g = static_cast<uint32_t *>(tmp);
 } /* PHF::compact() */
 

src/3rd_party/zlib/CMakeLists.txt

@@ -2,11 +2,11 @@
 file(GLOB ZLIB_SRC *.c)
 file(GLOB ZLIB_INC *.h)
 
-# add sources of the wrapper as a "SQLiteCpp" static library
+# add sources of the wrapper as a "zlib" static library
 add_library(zlib OBJECT ${ZLIB_SRC} ${ZLIB_INC})
 
 if(MSVC)
-  target_compile_options(zlib PUBLIC /wd"4996" /wd"4267")
+  target_compile_options(zlib PUBLIC /wd4996 /wd4267)
 else()
   target_compile_options(zlib PUBLIC -Wno-implicit-function-declaration)
 endif()

src/CMakeLists.txt

@@ -221,7 +221,13 @@ endif(USE_STATIC_LIBS)
 if(COMPILE_SERVER)
   add_executable(marian_server command/marian_server.cpp)
   set_target_properties(marian_server PROPERTIES OUTPUT_NAME marian-server)
-  target_compile_options(marian_server PUBLIC ${ALL_WARNINGS})
+  if(MSVC)
+    # Disable warnings from the SimpleWebSocketServer library needed for compilation of marian-server
+    target_compile_options(marian_server PUBLIC ${ALL_WARNINGS})
+  else(MSVC)
+    # -Wno-suggest-override disables warnings from Boost 1.69+
+    target_compile_options(marian_server PUBLIC ${ALL_WARNINGS} -Wno-suggest-override)
+  endif(MSVC)
   set(EXECUTABLES ${EXECUTABLES} marian_server)
 endif(COMPILE_SERVER)
 

src/common/timer.h

@@ -3,6 +3,7 @@
 #include <iostream>
 #include <sstream>
 #include <chrono>
+#include <ctime>
 
 namespace marian {
 namespace timer {

src/tests/CMakeLists.txt

@@ -1,25 +1,26 @@
 # Unit tests
 add_subdirectory(units)
 
+if(NOT MSVC)
+  # Testing apps
+  set(APP_TESTS
+      logger
+      dropout
+      sqlite
+      prod
+      cli
+      pooling
+  )
 
-# Testing apps
-set(APP_TESTS
-    logger
-    dropout
-    sqlite
-    prod
-    cli
-    pooling
-)
+  foreach(test ${APP_TESTS})
+    add_executable("test_${test}" "${test}.cpp")
 
-foreach(test ${APP_TESTS})
-  add_executable("test_${test}" "${test}.cpp")
+    if(CUDA_FOUND)
+      target_link_libraries("test_${test}" ${EXT_LIBS} marian ${EXT_LIBS} marian_cuda ${EXT_LIBS})
+    else(CUDA_FOUND)
+      target_link_libraries("test_${test}" marian ${EXT_LIBS})
+    endif(CUDA_FOUND)
 
-  if(CUDA_FOUND)
-    target_link_libraries("test_${test}" ${EXT_LIBS} marian ${EXT_LIBS} marian_cuda ${EXT_LIBS})
-  else(CUDA_FOUND)
-    target_link_libraries("test_${test}" marian ${EXT_LIBS})
-  endif(CUDA_FOUND)
-
-  set_target_properties("test_${test}" PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
-endforeach(test)
+    set_target_properties("test_${test}" PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
+  endforeach(test)
+endif(NOT MSVC)

src/tests/units/CMakeLists.txt

@@ -17,5 +17,10 @@ foreach(test ${UNIT_TESTS})
     target_link_libraries("run_${test}" marian ${EXT_LIBS} Catch)
   endif(CUDA_FOUND)
 
+  if(MSVC)
+    # Disable C4305: truncation from 'double' to '_Ty'
+    target_compile_options("run_${test}" PUBLIC /wd4305)
+  endif(MSVC)
+
   add_test(NAME ${test} COMMAND "run_${test}")
 endforeach(test)