Add a byte-level BPE Chinese+English non-streaming zipformer model (#1645)

Author: csukuangfj

.github/scripts/test-python.sh

@@ -8,6 +8,27 @@ log() {
   echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
 }
 
+log "test offline zipformer (byte-level bpe, Chinese+English)"
+curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-zh-en-2023-11-22.tar.bz2
+tar xvf sherpa-onnx-zipformer-zh-en-2023-11-22.tar.bz2
+rm sherpa-onnx-zipformer-zh-en-2023-11-22.tar.bz2
+
+repo=sherpa-onnx-zipformer-zh-en-2023-11-22
+
+./python-api-examples/offline-decode-files.py  \
+  --tokens=$repo/tokens.txt \
+  --encoder=$repo/encoder-epoch-34-avg-19.int8.onnx \
+  --decoder=$repo/decoder-epoch-34-avg-19.onnx \
+  --joiner=$repo/joiner-epoch-34-avg-19.int8.onnx \
+  --num-threads=2 \
+  --decoding-method=greedy_search \
+  --debug=true \
+  $repo/test_wavs/0.wav \
+  $repo/test_wavs/1.wav \
+  $repo/test_wavs/2.wav
+
+rm -rf sherpa-onnx-zipformer-zh-en-2023-11-22
+
 log "test offline Moonshine"
 
 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2

.github/workflows/add-new-asr-models.yaml

@@ -0,0 +1,61 @@
+name: add-new-asr-models
+
+on:
+  # push:
+  #   branches:
+  #     - new-asr-models
+  workflow_dispatch:
+
+concurrency:
+  group: add-new-asr-models-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  add-new-asr-models:
+    runs-on: ${{ matrix.os }}
+    name: New asr models
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Download icefall-asr-zipformer-multi-zh-en-2023-11-22
+        shell: bash
+        run: |
+          d=sherpa-onnx-zipformer-zh-en-2023-11-22
+          mkdir $d
+          pushd $d
+
+          wget -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/data/lang_bbpe_2000/tokens.txt
+          wget -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/data/lang_bbpe_2000/bbpe.model
+          wget -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/exp/decoder-epoch-34-avg-19.onnx
+          wget -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/exp/encoder-epoch-34-avg-19.int8.onnx
+          wget -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/exp/encoder-epoch-34-avg-19.onnx
+          wget -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/exp/joiner-epoch-34-avg-19.int8.onnx
+          wget -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/exp/joiner-epoch-34-avg-19.onnx
+
+          mkdir test_wavs
+          cd test_wavs
+          wget -O 0.wav -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/test_wavs/_1634_210_2577_1_1525157964032_3712259_29.wav
+          wget -O 1.wav -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/test_wavs/_1634_210_2577_1_1525157964032_3712259_55.wav
+
+          wget -O 2.wav -q https://huggingface.co/zrjin/icefall-asr-zipformer-multi-zh-en-2023-11-22/resolve/main/test_wavs/_1634_210_2577_1_1525157964032_3712259_75.wav
+          popd
+          tar cvjf $d.tar.bz2 $d
+          ls -lh $d
+          rm -rf $d
+
+      - name: Release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          file_glob: true
+          file: ./*.tar.bz2
+          overwrite: true
+          repo_name: k2-fsa/sherpa-onnx
+          repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }}
+          tag: asr-models

harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets

@@ -229,6 +229,18 @@ export function getOfflineModelConfig(type: number): OfflineModelConfig {
 
       break;
     }
+
+    case 23: {
+      const modelDir = "sherpa-onnx-zipformer-zh-en-2023-11-22";
+      c.transducer.encoder = `${modelDir}/encoder-epoch-34-avg-19.int8.onnx`;
+      c.transducer.decoder = `${modelDir}/decoder-epoch-34-avg-19.onnx`;
+      c.transducer.joiner = `${modelDir}/joiner-epoch-34-avg-19.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+      c.modelType = "transducer";
+
+      break;
+    }
+
     default: {
       console.log(`Please specify a supported type. Given type ${type}`);
     }

scripts/apk/generate-vad-asr-apk-script.py

@@ -420,6 +420,26 @@ def get_models():
 
             ls -lh
 
+            popd
+            """,
+        ),
+        Model(
+            model_name="sherpa-onnx-zipformer-zh-en-2023-11-22",
+            idx=23,
+            lang="zh_en",
+            lang2="Chinese,English",
+            short_name="zipformer",
+            cmd="""
+            pushd $model_name
+
+            rm -rfv test_wavs
+
+            rm -fv encoder-epoch-34-avg-19.onnx
+            rm -fv joiner-epoch-34-avg-19.onnx
+            rm -fv bbpe.model
+
+            ls -lh
+
             popd
             """,
         ),

sherpa-onnx/kotlin-api/OfflineRecognizer.kt

@@ -451,6 +451,19 @@ fun getOfflineModelConfig(type: Int): OfflineModelConfig? {
                 tokens = "$modelDir/tokens.txt",
             )
         }
+
+        23 -> {
+            val modelDir = "sherpa-onnx-zipformer-zh-en-2023-11-22"
+            return OfflineModelConfig(
+                transducer = OfflineTransducerModelConfig(
+                    encoder = "$modelDir/encoder-epoch-34-avg-19.int8.onnx",
+                    decoder = "$modelDir/decoder-epoch-34-avg-19.onnx",
+                    joiner = "$modelDir/joiner-epoch-34-avg-19.int8.onnx",
+                ),
+                tokens = "$modelDir/tokens.txt",
+                modelType = "transducer",
+            )
+        }
     }
     return null
 }