Skip to content

Commit 3df1879

Browse files
csukuangfjcsukuangfj
authored
Add address sanitizer and undefined behavior sanitizer (k2-fsa#951)
1 parent a6a938a commit 3df1879

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+354
-132
lines changed

‎.github/workflows/android.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ jobs:
137137
git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main
138138
139139
- name: Release android libs
140-
if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/')
140+
if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/')
141141
uses: svenstaro/upload-release-action@v2
142142
with:
143143
file_glob: true

‎.github/workflows/flutter-linux.yaml

+8
Original file line numberDiff line numberDiff line change
@@ -217,3 +217,11 @@ jobs:
217217
with:
218218
name: flutter-sherpa-onnx-linux-x64
219219
path: ./*.tar.bz2
220+
221+
# - name: Release android libs
222+
# if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/')
223+
# uses: svenstaro/upload-release-action@v2
224+
# with:
225+
# file_glob: true
226+
# overwrite: true
227+
# file: flutter*.tar.bz2

‎.github/workflows/flutter-macos.yaml

+8
Original file line numberDiff line numberDiff line change
@@ -180,3 +180,11 @@ jobs:
180180
with:
181181
name: flutter-sherpa-onnx-app-macos-${{ matrix.arch }}
182182
path: ./*.tar.bz2
183+
184+
- name: Release android libs
185+
if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/')
186+
uses: svenstaro/upload-release-action@v2
187+
with:
188+
file_glob: true
189+
overwrite: true
190+
file: flutter*.tar.bz2

‎.github/workflows/flutter-windows-x64.yaml

+8
Original file line numberDiff line numberDiff line change
@@ -157,3 +157,11 @@ jobs:
157157
with:
158158
name: flutter-sherpa-onnx-windows-x64
159159
path: ./*.tar.bz2
160+
161+
- name: Release android libs
162+
if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/')
163+
uses: svenstaro/upload-release-action@v2
164+
with:
165+
file_glob: true
166+
overwrite: true
167+
file: flutter*.tar.bz2

‎.github/workflows/sanitizer.yaml

+186
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
name: sanitizer
2+
3+
on:
4+
workflow_dispatch:
5+
6+
schedule:
7+
# minute (0-59)
8+
# hour (0-23)
9+
# day of the month (1-31)
10+
# month (1-12)
11+
# day of the week (0-6)
12+
# nightly build at 22:50 UTC time every day
13+
- cron: "50 22 * * *"
14+
15+
concurrency:
16+
group: sanitizer-${{ github.ref }}
17+
cancel-in-progress: true
18+
19+
jobs:
20+
sanitizer:
21+
runs-on: ${{ matrix.os }}
22+
name: sanitizer
23+
strategy:
24+
fail-fast: false
25+
matrix:
26+
os: [macos-latest]
27+
28+
steps:
29+
- uses: actions/checkout@v4
30+
with:
31+
fetch-depth: 0
32+
33+
- name: ccache
34+
uses: hendrikmuhs/ccache-action@v1.2
35+
with:
36+
key: ${{ matrix.os }}-sanitizer
37+
38+
- name: Configure CMake
39+
shell: bash
40+
run: |
41+
export CMAKE_CXX_COMPILER_LAUNCHER=ccache
42+
export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
43+
cmake --version
44+
45+
mkdir build
46+
cd build
47+
48+
cmake \
49+
-DSHERPA_ONNX_ENABLE_PYTHON=ON \
50+
-DSHERPA_ONNX_ENABLE_TESTS=ON \
51+
-DSHERPA_ONNX_ENABLE_JNI=ON \
52+
-DSHERPA_ONNX_ENABLE_SANITIZER=ON \
53+
-D BUILD_SHARED_LIBS=ON \
54+
-D CMAKE_BUILD_TYPE=Release \
55+
-DCMAKE_INSTALL_PREFIX=./install \
56+
..
57+
58+
- name: Build sherpa-onnx
59+
shell: bash
60+
run: |
61+
export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
62+
63+
cd build
64+
make -j2
65+
make install
66+
67+
ls -lh lib
68+
ls -lh bin
69+
70+
file ./bin/sherpa-onnx
71+
72+
- name: Display dependencies of sherpa-onnx for macos
73+
shell: bash
74+
run: |
75+
file bin/sherpa-onnx
76+
otool -L build/bin/sherpa-onnx
77+
otool -l build/bin/sherpa-onnx
78+
79+
- name: Test offline transducer
80+
shell: bash
81+
run: |
82+
export PATH=$PWD/build/bin:$PATH
83+
export EXE=sherpa-onnx-offline
84+
85+
.github/scripts/test-offline-transducer.sh
86+
87+
- name: Test online CTC
88+
shell: bash
89+
run: |
90+
export PATH=$PWD/build/bin:$PATH
91+
export EXE=sherpa-onnx
92+
93+
.github/scripts/test-online-ctc.sh
94+
95+
- name: Test offline punctuation
96+
shell: bash
97+
run: |
98+
export PATH=$PWD/build/bin:$PATH
99+
export EXE=sherpa-onnx-offline-punctuation
100+
101+
.github/scripts/test-offline-punctuation.sh
102+
103+
- name: Test C API
104+
shell: bash
105+
run: |
106+
export PATH=$PWD/build/bin:$PATH
107+
export SLID_EXE=spoken-language-identification-c-api
108+
export SID_EXE=speaker-identification-c-api
109+
export AT_EXE=audio-tagging-c-api
110+
export PUNCT_EXE=add-punctuation-c-api
111+
112+
.github/scripts/test-c-api.sh
113+
114+
- name: Test Audio tagging
115+
shell: bash
116+
run: |
117+
export PATH=$PWD/build/bin:$PATH
118+
export EXE=sherpa-onnx-offline-audio-tagging
119+
120+
.github/scripts/test-audio-tagging.sh
121+
122+
- name: Test spoken language identification (C++ API)
123+
shell: bash
124+
run: |
125+
export PATH=$PWD/build/bin:$PATH
126+
export EXE=sherpa-onnx-offline-language-identification
127+
128+
.github/scripts/test-spoken-language-identification.sh
129+
130+
- name: Test transducer kws
131+
shell: bash
132+
run: |
133+
export PATH=$PWD/build/bin:$PATH
134+
export EXE=sherpa-onnx-keyword-spotter
135+
136+
.github/scripts/test-kws.sh
137+
138+
- name: Test offline TTS
139+
if: matrix.with_tts == 'ON'
140+
shell: bash
141+
run: |
142+
export PATH=$PWD/build/bin:$PATH
143+
export EXE=sherpa-onnx-offline-tts
144+
145+
.github/scripts/test-offline-tts.sh
146+
147+
- name: Test online paraformer
148+
shell: bash
149+
run: |
150+
export PATH=$PWD/build/bin:$PATH
151+
export EXE=sherpa-onnx
152+
153+
.github/scripts/test-online-paraformer.sh
154+
155+
- name: Test offline Whisper
156+
if: matrix.build_type != 'Debug'
157+
shell: bash
158+
run: |
159+
export PATH=$PWD/build/bin:$PATH
160+
export EXE=sherpa-onnx-offline
161+
162+
.github/scripts/test-offline-whisper.sh
163+
164+
- name: Test offline CTC
165+
shell: bash
166+
run: |
167+
export PATH=$PWD/build/bin:$PATH
168+
export EXE=sherpa-onnx-offline
169+
170+
.github/scripts/test-offline-ctc.sh
171+
172+
- name: Test online transducer
173+
shell: bash
174+
run: |
175+
export PATH=$PWD/build/bin:$PATH
176+
export EXE=sherpa-onnx
177+
178+
.github/scripts/test-online-transducer.sh
179+
180+
- name: Test online transducer (C API)
181+
shell: bash
182+
run: |
183+
export PATH=$PWD/build/bin:$PATH
184+
export EXE=decode-file-c-api
185+
186+
.github/scripts/test-online-transducer.sh

‎CMakeLists.txt

+31
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
22

33
set(CMAKE_OSX_DEPLOYMENT_TARGET "10.14" CACHE STRING "Minimum OS X deployment version. Used only for macOS")
44

5+
6+
57
project(sherpa-onnx)
68

79
set(SHERPA_ONNX_VERSION "1.9.26")
@@ -32,6 +34,7 @@ option(SHERPA_ONNX_ENABLE_BINARY "Whether to build binaries" ON)
3234
option(SHERPA_ONNX_ENABLE_TTS "Whether to build TTS related code" ON)
3335
option(SHERPA_ONNX_LINK_LIBSTDCPP_STATICALLY "True to link libstdc++ statically. Used only when BUILD_SHARED_LIBS is OFF on Linux" ON)
3436
option(SHERPA_ONNX_USE_PRE_INSTALLED_ONNXRUNTIME_IF_AVAILABLE "True to use pre-installed onnxruntime if available" ON)
37+
option(SHERPA_ONNX_ENABLE_SANITIZER "Whether to enable ubsan and asan" OFF)
3538

3639
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
3740
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
@@ -121,6 +124,7 @@ message(STATUS "SHERPA_ONNX_ENABLE_BINARY ${SHERPA_ONNX_ENABLE_BINARY}")
121124
message(STATUS "SHERPA_ONNX_ENABLE_TTS ${SHERPA_ONNX_ENABLE_TTS}")
122125
message(STATUS "SHERPA_ONNX_LINK_LIBSTDCPP_STATICALLY ${SHERPA_ONNX_LINK_LIBSTDCPP_STATICALLY}")
123126
message(STATUS "SHERPA_ONNX_USE_PRE_INSTALLED_ONNXRUNTIME_IF_AVAILABLE ${SHERPA_ONNX_USE_PRE_INSTALLED_ONNXRUNTIME_IF_AVAILABLE}")
127+
message(STATUS "SHERPA_ONNX_ENABLE_SANITIZER: ${SHERPA_ONNX_ENABLE_SANITIZER}")
124128

125129
if(SHERPA_ONNX_ENABLE_TTS)
126130
message(STATUS "TTS is enabled")
@@ -267,6 +271,33 @@ if(SHERPA_ONNX_ENABLE_TTS)
267271
include(cppjieba) # For Chinese TTS. It is a header-only C++ library
268272
endif()
269273

274+
# if(NOT MSVC AND CMAKE_BUILD_TYPE STREQUAL Debug AND (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang"))
275+
if(SHERPA_ONNX_ENABLE_SANITIZER)
276+
message(WARNING "enable ubsan and asan")
277+
set(CMAKE_REQUIRED_LIBRARIES -lubsan -lasan)
278+
include(CheckCCompilerFlag)
279+
280+
set(flags -fsanitize=undefined )
281+
string(APPEND flags " -fno-sanitize-recover=undefined ")
282+
string(APPEND flags " -fsanitize=integer ")
283+
string(APPEND flags " -fsanitize=nullability ")
284+
string(APPEND flags " -fsanitize=implicit-conversion ")
285+
string(APPEND flags " -fsanitize=bounds ")
286+
string(APPEND flags " -fsanitize=address ")
287+
288+
if(OFF)
289+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${flags} -Wall -Wextra")
290+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${flags} -Wall -Wextra")
291+
else()
292+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${flags}")
293+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${flags}")
294+
endif()
295+
296+
set(CMAKE_EXECUTBLE_LINKER_FLAGS "${CMAKE_EXECUTBLE_LINKER_FLAGS} ${flags}")
297+
298+
add_compile_options(-fno-omit-frame-pointer)
299+
endif()
300+
270301
add_subdirectory(sherpa-onnx)
271302

272303
if(SHERPA_ONNX_ENABLE_C_API AND SHERPA_ONNX_ENABLE_BINARY)

‎c-api-examples/add-punctuation-c-api.c

+2-2
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ int32_t main() {
4343
const char *texts[] = {
4444
"这是一个测试你好吗How are you我很好thank you are you ok谢谢你",
4545
"我们都是木头人不会说话不会动",
46-
"The African blogosphere is rapidly expanding bringing more voices "
47-
"online in the form of commentaries opinions analyses rants and poetry",
46+
("The African blogosphere is rapidly expanding bringing more voices "
47+
"online in the form of commentaries opinions analyses rants and poetry"),
4848
};
4949

5050
int32_t n = sizeof(texts) / sizeof(const char *);

‎c-api-examples/decode-file-c-api.c

-2
Original file line numberDiff line numberDiff line change
@@ -179,8 +179,6 @@ int32_t main(int32_t argc, char *argv[]) {
179179

180180
#define N 3200 // 0.2 s. Sample rate is fixed to 16 kHz
181181

182-
int16_t buffer[N];
183-
float samples[N];
184182
fprintf(stderr, "sample rate: %d, num samples: %d, duration: %.2f s\n",
185183
wave->sample_rate, wave->num_samples,
186184
(float)wave->num_samples / wave->sample_rate);

‎c-api-examples/streaming-hlg-decode-file-c-api.c

-2
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,6 @@ int32_t main() {
6565
// simulate streaming. You can choose an arbitrary N
6666
#define N 3200
6767

68-
int16_t buffer[N];
69-
float samples[N];
7068
fprintf(stderr, "sample rate: %d, num samples: %d, duration: %.2f s\n",
7169
wave->sample_rate, wave->num_samples,
7270
(float)wave->num_samples / wave->sample_rate);

‎cmake/espeak-ng-for-piper.cmake

+1-4
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,7 @@ function(download_espeak_ng_for_piper)
1414
set(USE_SPEECHPLAYER OFF CACHE BOOL "" FORCE)
1515
set(EXTRA_cmn ON CACHE BOOL "" FORCE)
1616
set(EXTRA_ru ON CACHE BOOL "" FORCE)
17-
18-
if(SHERPA_ONNX_ENABLE_WASM)
19-
set(BUILD_ESPEAK_NG_EXE OFF CACHE BOOL "" FORCE)
20-
endif()
17+
set(BUILD_ESPEAK_NG_EXE OFF CACHE BOOL "" FORCE)
2118

2219
# If you don't have access to the Internet,
2320
# please pre-download kaldi-decoder

‎sherpa-onnx/c-api/c-api.cc

+3-3
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ const SherpaOnnxOnlineRecognizerResult *GetOnlineStreamResult(
187187
r->text = pText;
188188

189189
// copy json
190-
const auto &json = result.AsJsonString();
190+
std::string json = result.AsJsonString();
191191
char *pJson = new char[json.size() + 1];
192192
std::copy(json.begin(), json.end(), pJson);
193193
pJson[json.size()] = 0;
@@ -445,7 +445,7 @@ const SherpaOnnxOfflineRecognizerResult *GetOfflineStreamResult(
445445
r->text = pText;
446446

447447
// copy json
448-
const auto &json = result.AsJsonString();
448+
std::string json = result.AsJsonString();
449449
char *pJson = new char[json.size() + 1];
450450
std::copy(json.begin(), json.end(), pJson);
451451
pJson[json.size()] = 0;
@@ -643,7 +643,7 @@ const SherpaOnnxKeywordResult *GetKeywordResult(
643643
r->keyword = pKeyword;
644644

645645
// copy json
646-
const auto &json = result.AsJsonString();
646+
std::string json = result.AsJsonString();
647647
char *pJson = new char[json.size() + 1];
648648
std::copy(json.begin(), json.end(), pJson);
649649
pJson[json.size()] = 0;

‎sherpa-onnx/csrc/audio-tagging-ced-impl.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ class AudioTaggingCEDImpl : public AudioTaggingImpl {
7373
std::vector<float> f = s->GetFrames();
7474

7575
int32_t num_frames = f.size() / feat_dim;
76-
assert(feat_dim * num_frames == f.size());
76+
assert(feat_dim * num_frames == static_cast<int32_t>(f.size()));
7777

7878
std::array<int64_t, 3> shape = {1, num_frames, feat_dim};
7979

‎sherpa-onnx/csrc/audio-tagging-label-file.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ void AudioTaggingLabels::Init(std::istream &is) {
6565
exit(-1);
6666
}
6767

68-
if (i != names_.size()) {
68+
if (i != static_cast<int32_t>(names_.size())) {
6969
SHERPA_ONNX_LOGE(
7070
"Index should be sorted and contiguous. Expected index: %d, given: "
7171
"%d.",

‎sherpa-onnx/csrc/audio-tagging-zipformer-impl.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ class AudioTaggingZipformerImpl : public AudioTaggingImpl {
7474

7575
int32_t num_frames = f.size() / feat_dim;
7676

77-
assert(feat_dim * num_frames == f.size());
77+
assert(feat_dim * num_frames == static_cast<int32_t>(f.size()));
7878

7979
std::array<int64_t, 3> shape = {1, num_frames, feat_dim};
8080

‎sherpa-onnx/csrc/cat.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ Ort::Value Cat(OrtAllocator *allocator,
9292
}
9393
}
9494

95-
return std::move(ans);
95+
return ans;
9696
}
9797

9898
template Ort::Value Cat<float>(OrtAllocator *allocator,

0 commit comments

Comments
 (0)