A slightly better CharStream implementation

apache · Sep 3, 2022 · 52409ea · 52409ea
1 parent 95f381f
commit 52409ea
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 7 deletions.
diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java
@@ -28,8 +28,8 @@
  */
 public class LexerInputCharStream implements CharStream {
     private final LexerInput input;
-    private final StringBuilder readBuffer = new StringBuilder();
 
+    private int tokenMark = Integer.MAX_VALUE;
     private int index = 0;
 
     public LexerInputCharStream(LexerInput input) {
@@ -38,8 +38,12 @@ public LexerInputCharStream(LexerInput input) {
 
     @Override
     public String getText(Interval intrvl) {
-        int end = Math.min(intrvl.b + 1, readBuffer.length());
-        return readBuffer.substring(intrvl.a, end);
+        if (intrvl.a < tokenMark) {
+            throw new UnsupportedOperationException("Read before the current token start is not supported: " + intrvl.a + " < " + tokenMark);
+        }
+        int start = intrvl.a - tokenMark;
+        int end = intrvl.b - tokenMark + 1;
+        return String.valueOf(input.readText(start, end));
     }
 
     @Override
@@ -97,9 +101,6 @@ public void seek(int i) {
 
     private int read() {
         int ret = input.read();
-        if ((readBuffer.length() == index) && (ret != EOF)) {
-            readBuffer.append((char)ret);
-        }
         index += 1;
         return ret;
     }
@@ -109,6 +110,10 @@ private void backup(int count) {
         input.backup(count);
     }
 
+    public final void markToken() {
+        tokenMark = index;
+    }
+
     @Override
     public int size() {
         throw new UnsupportedOperationException("Stream size is unknown.");

diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java
@@ -36,10 +36,12 @@ public final class TomlLexer implements Lexer<TomlTokenId> {
 
     private final TokenFactory<TomlTokenId> tokenFactory;
     private final org.tomlj.internal.TomlLexer lexer;
+    private final LexerInputCharStream input;
 
     public TomlLexer(LexerRestartInfo<TomlTokenId> info) {
         this.tokenFactory = info.tokenFactory();
-        this.lexer = new org.tomlj.internal.TomlLexer(new LexerInputCharStream(info.input()));
+        this.input = new LexerInputCharStream(info.input());
+        this.lexer = new org.tomlj.internal.TomlLexer(input);
         if (info.state() != null) {
             ((LexerState) info.state()).restore(lexer);
         }
@@ -130,6 +132,7 @@ public void release() {
     }
 
     private Token<TomlTokenId> token(TomlTokenId id) {
+        input.markToken();
         return tokenFactory.createToken(id);
     }