Fix tokenization of Korean chars fix: #1877

hankcs · hankcs · commit 785bed5f2738 · 2024-02-23T20:53:43.000-08:00
diff --git a/hanlp/components/tokenizers/transformer.py b/hanlp/components/tokenizers/transformer.py
@@ -147,7 +147,7 @@ def tag_to_span(self, batch_tags, batch: dict):
                             tags[i - 1] = 'B'
                         elif prev_tag == 'E':
                             tags[i - 1] = 'M'
-                        tags[i] = 'M'
+                        tags[i] = tag = 'M'
                     offset = e
                     prev_tag = tag
         for tags in batch_tags:
diff --git a/hanlp/version.py b/hanlp/version.py
@@ -2,7 +2,7 @@
 # Author: hankcs
 # Date: 2019-12-28 19:26
 
-__version__ = '2.1.0-beta.55'
+__version__ = '2.1.0-beta.56'
 """HanLP version"""
 
 
