Skip to content
This repository was archived by the owner on Dec 16, 2020. It is now read-only.

Use Emscripten with STANDALONE_WASM=1. #260

Merged
merged 3 commits into from
Oct 19, 2019
Merged

Use Emscripten with STANDALONE_WASM=1. #260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c9b00bc
Use Emscripten with STANDALONE_WASM=1.
PiotrSikora Oct 17, 2019
cc62b9a
review: avoid pulling WAVM's Emscripten implementation.
PiotrSikora Oct 19, 2019
6a820a9
review: re-add SIGSEGV test and check for "unreachable".
PiotrSikora Oct 19, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api/wasm/cpp/Dockerfile-sdk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@ FROM ubuntu:bionic

COPY ./sdk_container.sh /
COPY ./build_wasm.sh /
COPY *.cc *.h *.js *.proto Makefile* *.bc /sdk/
COPY *.cc *.h *.js *.proto Makefile* *.a /sdk/

RUN ./sdk_container.sh
4 changes: 2 additions & 2 deletions api/wasm/cpp/Makefile.base
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ else
endif

%.wasm %.wat: %.cc ${CPP_API}/proxy_wasm_intrinsics.h ${CPP_API}/proxy_wasm_enums.h ${CPP_API}/proxy_wasm_externs.h ${CPP_API}/proxy_wasm_api.h ${CPP_API}/proxy_wasm_intrinsics.js ${CPP_CONTEXT_LIB}
em++ -s WASM=1 -s BINARYEN_TRAP_MODE='clamp' -s LEGALIZE_JS_FFI=0 -s EMIT_EMSCRIPTEN_METADATA=1 --std=c++17 -O3 -g3 -I${CPP_API} -I/usr/local/include --js-library ${CPP_API}/proxy_wasm_intrinsics.js $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.bc -o $*.js
rm -f $*.js $*.wast
em++ -s STANDALONE_WASM=1 -s EMIT_EMSCRIPTEN_METADATA=1 --std=c++17 -O3 -g3 -I${CPP_API} -I/usr/local/include --js-library ${CPP_API}/proxy_wasm_intrinsics.js $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.a -o $*.wasm
rm -f $*.wast

clean:
rm *.wasm
4 changes: 2 additions & 2 deletions api/wasm/cpp/Makefile.base_lite
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ else
endif

%.wasm %.wat: %.cc ${CPP_API}/proxy_wasm_intrinsics.h ${CPP_API}/proxy_wasm_enums.h ${CPP_API}/proxy_wasm_externs.h ${CPP_API}/proxy_wasm_api.h ${CPP_API}/proxy_wasm_intrinsics.js ${CPP_CONTEXT_LIB}
em++ -s WASM=1 -s BINARYEN_TRAP_MODE='clamp' -s LEGALIZE_JS_FFI=0 -s EMIT_EMSCRIPTEN_METADATA=1 --std=c++17 -O3 -g3 -DEMSCRIPTEN_PROTOBUF_LITE=1 -I${CPP_API} -I/usr/local/include --js-library ${CPP_API}/proxy_wasm_intrinsics.js $*.cc ${CPP_API}/proxy_wasm_intrinsics_lite.pb.cc ${CPP_API}/struct_lite.pb.cc ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf-lite.bc -o $*.js
rm -f $*.js $*.wast
em++ -s STANDALONE_WASM=1 -s EMIT_EMSCRIPTEN_METADATA=1 --std=c++17 -O3 -g3 -DEMSCRIPTEN_PROTOBUF_LITE=1 -I${CPP_API} -I/usr/local/include --js-library ${CPP_API}/proxy_wasm_intrinsics.js $*.cc ${CPP_API}/proxy_wasm_intrinsics_lite.pb.cc ${CPP_API}/struct_lite.pb.cc ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf-lite.a -o $*.wasm
rm -f $*.wast

clean:
rm *.wasm
36 changes: 18 additions & 18 deletions api/wasm/cpp/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ A Dockerfile for the C++ SDK is provided in Dockerfile-sdk.

It can built in this directory by:

docker build -t wasmsdk:v1 -f Dockerfile-sdk .
docker build -t wasmsdk:v2 -f Dockerfile-sdk .

The docker image can be used for compiling wasm files.

Expand Down Expand Up @@ -57,7 +57,7 @@ void ExampleContext::onDone() { logInfo("onDone " + std::to_string(id())); }
Run docker:

```bash
docker run -v $PWD:/work -w /work wasmsdk:v1 /build_wasm.sh
docker run -v $PWD:/work -w /work wasmsdk:v2 /build_wasm.sh
```

### Caching the standard libraries
Expand All @@ -66,7 +66,7 @@ The first time that emscripten runs it will generate the standard libraries. To
after the first successful compilation (e.g myproject.cc above), commit the image with the standard libraries:

```bash
docker commit `docker ps -l | grep wasmsdk:v1 | awk '{print $1}'` wasmsdk:v1
docker commit `docker ps -l | grep wasmsdk:v2 | awk '{print $1}'` wasmsdk:v2
```

This will save time on subsequent compiles.
Expand All @@ -88,7 +88,7 @@ include ${DOCKER_SDK}/Makefile.base_lite
Run docker pointing to Envoy sources in a directory parallel (at the same level) as your project directory:

```bash
docker run -v $PWD:/work -v $PWD/../envoy/api/wasm/cpp:/work/sdk -w /work wasmsdk:v1 bash /build_wasm.sh
docker run -v $PWD:/work -v $PWD/../envoy/api/wasm/cpp:/work/sdk -w /work wasmsdk:v2 bash /build_wasm.sh
```

### Using abseil form the image
Expand All @@ -106,7 +106,7 @@ all: plugin.wasm

%.wasm %.wat: %.cc ${CPP_API}/proxy_wasm_intrinsics.h ${CPP_API}/proxy_wasm_enums.h ${CPP_API}/proxy_wasm_externs.h ${CPP_API}/proxy_wasm_api.h ${CPP_API}/proxy_wasm_intrinsics.js ${CPP_CONTEXT_LIB}
ls /root
em++ -s WASM=1 -s BINARYEN_TRAP_MODE='clamp' -s LEGALIZE_JS_FFI=0 -s EMIT_EMSCRIPTEN_METADATA=1 --std=c++17 -O3 -g3 -I${CPP_API} -I${CPP_API}/google/protobuf -I/usr/local/include -I${ABSL} --js-library ${CPP_API}/proxy_wasm_intrinsics.js ${ABSL_CPP} $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.bc -o $*.js
em++ -s STANDALONE_WASM=1 -s EMIT_EMSCRIPTEN_METADATA=1 --std=c++17 -O3 -g3 -I${CPP_API} -I${CPP_API}/google/protobuf -I/usr/local/include -I${ABSL} --js-library ${CPP_API}/proxy_wasm_intrinsics.js ${ABSL_CPP} $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.a -o $*.wasm
```

Precompiled abseil libraries are also available, so the above can also be done as:
Expand All @@ -122,7 +122,7 @@ all: plugin.wasm

%.wasm %.wat: %.cc ${CPP_API}/proxy_wasm_intrinsics.h ${CPP_API}/proxy_wasm_enums.h ${CPP_API}/proxy_wasm_externs.h ${CPP_API}/proxy_wasm_api.h ${CPP_API}/proxy_wasm_intrinsics.js ${CPP_CONTEXT_LIB}
ls /root
em++ -s WASM=1 -s BINARYEN_TRAP_MODE='clamp' -s LEGALIZE_JS_FFI=0 -s EMIT_EMSCRIPTEN_METADATA=1 --std=c++17 -O3 -g3 -I${CPP_API} -I${CPP_API}/google/protobuf -I/usr/local/include -I${ABSL} --js-library ${CPP_API}/proxy_wasm_intrinsics.js $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.bc ${ABSL_LIBS} -o $*.js
em++ -s STANDALONE_WASM=1 -s EMIT_EMSCRIPTEN_METADATA=1 --std=c++17 -O3 -g3 -I${CPP_API} -I${CPP_API}/google/protobuf -I/usr/local/include -I${ABSL} --js-library ${CPP_API}/proxy_wasm_intrinsics.js $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.a ${ABSL_LIBS} -o $*.wasm
```

### Ownership of the resulting .wasm files
Expand All @@ -142,7 +142,7 @@ Invocation file (e.g. build.sh):

```bash
#!/bin/bash
docker run -e uid="$(id -u)" -e gid="$(id -g)" -v $PWD:/work -w /work wasmsdk:v1 /build_wasm.sh
docker run -e uid="$(id -u)" -e gid="$(id -g)" -v $PWD:/work -w /work wasmsdk:v2 /build_wasm.sh
```

## Dependencies for building WASM modules:
Expand All @@ -151,7 +151,7 @@ If you do not wish to use the Docker file, the dependencies can be installed by

### protobuf v3.9.1

You must install the version of protobuf on your build system that matches the libprotobuf.bc files (without any patches) so that the generated code matches the .bc library. Currently this is based on tag v3.9.1 of https://github.com/protocolbuffers/protobuf.
You must install the version of protobuf on your build system that matches the libprotobuf.a files (without any patches) so that the generated code matches the .a library. Currently this is based on tag v3.9.1 of https://github.com/protocolbuffers/protobuf.

```bash
git clone https://github.com/protocolbuffers/protobuf
Expand All @@ -171,8 +171,8 @@ sudo make install
git clone https://github.com/emscripten-core/emsdk.git
cd emsdk
./emsdk update-tags
./emsdk install 1.38.46
./emsdk activate 1.38.46
./emsdk install 1.38.48-upstream
./emsdk activate 1.38.48-upstream

source ./emsdk\_env.sh
```
Expand All @@ -181,15 +181,15 @@ It is possible later versions will work, e.g.

```bash
./emsdk update-tags
./emsdk install latest
./emsdk activate latest
./emsdk install latest-upstream
./emsdk activate latest-upstream
```

However 1.38.46 is known to work.
However 1.38.48-upstream is known to work.

### Rebuilding the protobuf.bc files
### Rebuilding the libprotobuf.a files

If want to rebuild the .bc files or use a different version see the instructions at https://github.com/kwonoj/protobuf-wasm. Commit 4bba8b2f38b5004f87489642b6ca4525ae72fe7f works for protobuf v3.9.x.
If want to rebuild the libprotobuf.a files or use a different version see the instructions at https://github.com/kwonoj/protobuf-wasm. Commit 4bba8b2f38b5004f87489642b6ca4525ae72fe7f works for protobuf v3.9.x.

```bash
git clone https://github.com/protocolbuffers/protobuf protobuf-wasm
Expand All @@ -199,10 +199,10 @@ git clone https://github.com/kwonoj/protobuf-wasm wasm-patches
cd wasm-patches && git checkout 4bba8b2f38b5004f87489642b6ca4525ae72fe7f && cd ..
git apply wasm-patches/*.patch
./autogen.sh
emconfigure ./configure CXXFLAGS="-O3"
emconfigure ./configure --disable-shared CXXFLAGS="-O3"
emmake make
cp src/.libs/libprotobuf-lite.so ${CPP_API}/libprotobuf-lite.bc
cp src/.libs/libprotobuf.so ${CPP_API}/libprotobuf.bc
cp src/.libs/libprotobuf-lite.a ${CPP_API}/libprotobuf-lite.a
cp src/.libs/libprotobuf.a ${CPP_API}/libprotobuf.a
```

### WAVM binaries
Expand Down
Binary file added api/wasm/cpp/libprotobuf-lite.a
View file
Open in desktop
Binary file not shown.
Binary file removed api/wasm/cpp/libprotobuf-lite.bc
View file
Open in desktop
Binary file not shown.
Binary file added api/wasm/cpp/libprotobuf.a
View file
Open in desktop
Binary file not shown.
Binary file removed api/wasm/cpp/libprotobuf.bc
View file
Open in desktop
Binary file not shown.
10 changes: 6 additions & 4 deletions api/wasm/cpp/sdk_container.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
#!/bin/bash

set -e

# basics
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get upgrade -y
apt-get install -y --no-install-recommends apt-utils ca-certificates
apt-get autoremove -y
apt-get clean
apt-get install -y --no-install-recommends software-properties-common apt-transport-https git wget curl libglib2.0-dev autoconf autotools-dev automake libtool cmake python
apt-get install -y --no-install-recommends software-properties-common apt-transport-https git wget curl pkg-config autoconf autotools-dev automake libtool cmake python zlib1g-dev
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just curious what caused these changes?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I removed libglib2.0-dev which wasn't necessary (it used to be needed by V8, but it's not anymore, and I'm not sure why it was here in the first place), so I had to add pkg-config and zlib1g-dev directly, since those are needed and were previously installed as transitive dependencies of libglib2.0-dev.


# gcc-7
apt-get install -y --no-install-recommends gcc-7 g++-7 cpp-7
Expand All @@ -31,8 +33,8 @@ cd
git clone https://github.com/emscripten-core/emsdk.git
cd emsdk
./emsdk update-tags
./emsdk install 1.38.46
./emsdk activate 1.38.46
./emsdk install 1.38.48-upstream
./emsdk activate 1.38.48-upstream
source ./emsdk_env.sh
cd

Expand All @@ -48,7 +50,7 @@ cd
apt-get install -y --no-install-recommends llvm-6.0-dev
git clone https://github.com/WAVM/WAVM
cd WAVM
git checkout 5e69711c074d16e717e476e3e7365505ebb0c42b -b Aug292019 # Aug 29 2019
git checkout 1ec06cd202a922015c9041c5ed84f875453c4dc7 -b Oct152019 # Oct 15 2019
cmake "."
make
make install
Expand Down
5 changes: 2 additions & 3 deletions api/wasm/rust/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ fn __post_instantiate() {

/// Allow host to allocate memory.
#[no_mangle]
fn _malloc(size: usize) -> *mut u8 {
fn malloc(size: usize) -> *mut u8 {
let mut vec: Vec<u8> = Vec::with_capacity(size);
unsafe {
vec.set_len(size);
Expand All @@ -54,7 +54,7 @@ fn _malloc(size: usize) -> *mut u8 {

/// Allow host to free memory.
#[no_mangle]
fn _free(ptr: *mut u8) {
fn free(ptr: *mut u8) {
if !ptr.is_null() {
unsafe {
Box::from_raw(ptr);
Expand All @@ -65,7 +65,6 @@ fn _free(ptr: *mut u8) {
/// Low-level Proxy-WASM APIs for the host functions.
pub mod host {
extern "C" {
#[link_name = "_proxy_log"]
pub fn proxy_log(level: u32, message_data: *const u8, message_size: usize) -> u32;
}
}
1 change: 0 additions & 1 deletion bazel/foreign_cc/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,6 @@ envoy_cmake_external(
"//conditions:default": [
"libWAVM.a",
"libWAVMUnwind.a",
"libWAVMgdtoa.a",
],
}),
deps = [":llvm"],
Expand Down
6 changes: 3 additions & 3 deletions bazel/repository_locations.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,9 +258,9 @@ REPOSITORY_LOCATIONS = dict(
urls = ["https://github.com/llvm/llvm-project/releases/download/llvmorg-8.0.1/llvm-8.0.1.src.tar.xz"],
),
com_github_wavm_wavm = dict(
sha256 = "7afe97f3833dfd58dffe0b525d669e66a30d2a8f4edbeb70924ffb43d01c80e6",
strip_prefix = "WAVM-6146b4e95d4b5d80d1fde9bb920791836a9078b4",
urls = ["https://github.com/WAVM/WAVM/archive/6146b4e95d4b5d80d1fde9bb920791836a9078b4.tar.gz"],
sha256 = "48939906ff44a0e26e2c4c63132eebd75cfd768fac989a874395121e139933ec",
strip_prefix = "WAVM-1ec06cd202a922015c9041c5ed84f875453c4dc7",
urls = ["https://github.com/WAVM/WAVM/archive/1ec06cd202a922015c9041c5ed84f875453c4dc7.tar.gz"],
),
com_googlesource_chromium_v8 = dict(
# This archive was created using https://storage.googleapis.com/envoyproxy-wee8/wee8-archive.sh
Expand Down
2 changes: 1 addition & 1 deletion examples/wasm/docker_build_cpp_wasm.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
#!/bin/bash
docker run -e uid="$(id -u)" -e gid="$(id -g)" -v $PWD:/work -v $(realpath ../../api/wasm/cpp):/external_sdk -w /work wasmsdk:v1 bash ./docker_cpp_builder.sh
docker run -e uid="$(id -u)" -e gid="$(id -g)" -v $PWD:/work -v $(realpath ../../api/wasm/cpp):/external_sdk -w /work wasmsdk:v2 bash ./docker_cpp_builder.sh
Binary file modified examples/wasm/envoy_filter_http_wasm_example.wasm
View file
Open in desktop
Binary file not shown.
Loading