同步个bug修复

同步个修改过来 infinilabs#998

Author: lizongbo

src/main/java/org/wltea/analyzer/core/AnalyzeContext.java

@@ -50,7 +50,8 @@ class AnalyzeContext {
     private int cursor;
     //最近一次读入的,可处理的字串长度
     private int available;
-
+    //末尾非CJK字符数目
+    private int lastUselessCharNum;
 
     //子分词器锁
     //该集合非空，说明有子分词器在占用segmentBuff
@@ -107,6 +108,7 @@ int fillBuffer(Reader reader) throws IOException {
         if (this.buffOffset == 0) {
             //首次读取reader
             readCount = reader.read(segmentBuff);
+            this.lastUselessCharNum = 0;
         } else {
             int offset = this.available - this.cursor;
             if (offset > 0) {
@@ -242,8 +244,11 @@ void outputToResult() {
             //跳过非CJK字符
             if (CharacterUtil.CHAR_USELESS == this.charTypes[index]) {
                 index++;
+                this.lastUselessCharNum++;
                 continue;
             }
+            // 清空数值
+            this.lastUselessCharNum = 0;
             //从pathMap找出对应index位置的LexemePath
             LexemePath path = this.pathMap.get(index);
             if (path != null) {
@@ -354,6 +359,15 @@ Lexeme getNextLexeme() {
         return result;
     }
 
+
+    /**
+     * 返回末尾非CJK字符字符数目
+     */
+    public int getLastUselessCharNum() {
+        return this.lastUselessCharNum;
+    }
+
+
     /**
      * 重置分词上下文状态
      */

src/main/java/org/wltea/analyzer/core/IKSegmenter.java

@@ -139,4 +139,11 @@ public synchronized void reset(Reader input) {
             segmenter.reset();
         }
     }
+
+    /**
+     * 返回末尾非CJK字符字符数目
+     */
+    public int getLastUselessCharNum() {
+        return this.context.getLastUselessCharNum();
+    }
 }

src/main/java/org/wltea/analyzer/lucene/IKTokenizer.java

@@ -109,7 +109,7 @@ public void reset() throws IOException {
     public void end() throws IOException {
         super.end();
         // set final offset
-        int finalOffset = correctOffset(this.endPosition);
+        int finalOffset = correctOffset(this.endPosition + _IKImplement.getLastUselessCharNum());
         offsetAtt.setOffset(finalOffset, finalOffset);
         posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);
     }