diff --git a/ide/languages.toml/nbproject/project.xml b/ide/languages.toml/nbproject/project.xml index 1b7f3de63089..1afa2c5bb40c 100644 --- a/ide/languages.toml/nbproject/project.xml +++ b/ide/languages.toml/nbproject/project.xml @@ -59,6 +59,15 @@ 2.74 + + org.netbeans.modules.csl.types + + + + 1 + 1.17 + + org.netbeans.modules.editor.completion @@ -104,35 +113,28 @@ - org.openide.awt - - - - 7.85 - - - - org.openide.filesystems + org.netbeans.modules.parsing.api - 9.29 + 1 + 9.24 - org.openide.loaders + org.openide.awt - 7.87 + 7.85 - org.openide.nodes + org.openide.filesystems - 7.62 + 9.29 diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java index 307a13e5e29a..5f359d8485d7 100644 --- a/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java +++ b/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java @@ -18,7 +18,6 @@ */ package org.netbeans.modules.languages.toml; -import java.util.Stack; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.misc.Interval; import org.netbeans.spi.lexer.*; @@ -29,21 +28,27 @@ */ public class LexerInputCharStream implements CharStream { private final LexerInput input; - private final Stack markers = new Stack<>(); + + private int tokenMark = Integer.MAX_VALUE; + private int index = 0; public LexerInputCharStream(LexerInput input) { this.input = input; - } @Override public String getText(Interval intrvl) { - return input.readText(intrvl.a, intrvl.b).toString(); + if (intrvl.a < tokenMark) { + throw new UnsupportedOperationException("Read before the current token start is not supported: " + intrvl.a + " < " + tokenMark); + } + int start = intrvl.a - tokenMark; + int end = intrvl.b - tokenMark + 1; + return String.valueOf(input.readText(start, end)); } @Override public void consume() { - input.read(); + read(); } @Override @@ -55,51 +60,63 @@ public int LA(int count) { int c = 0; if (count > 0) { for (int i = 0; i < count; i++) { - c = input.read(); + c = read(); } - input.backup(count); + backup(count); } else { - input.backup(count); - c = input.read(); + backup(count); + c = read(); } return c; } + //Marks are for buffering in ANTLR4, we do not really need them @Override public int mark() { - markers.push(index()); - return markers.size() - 1; + return -1; } @Override public void release(int marker) { - while (marker < markers.size()) { - markers.pop(); - } } @Override public int index() { - return input.readLengthEOF(); + return index; } @Override public void seek(int i) { - if (i < input.readLengthEOF()) { - input.backup(index() - i); + if (i < index()) { + backup(index() - i); } else { - while (input.readLengthEOF() < i) { - if (input.read() == LexerInput.EOF) { + while (index() < i) { + if (read() == LexerInput.EOF) { break; } } } } + + private int read() { + int ret = input.read(); + index += 1; + return ret; + } + + private void backup(int count) { + index -= count; + input.backup(count); + } + + public final void markToken() { + tokenMark = index; + } + @Override public int size() { - return -1; - //throw new UnsupportedOperationException("Stream size is unknown."); + throw new UnsupportedOperationException("Stream size is unknown."); } @Override diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlDataObject.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlDataObject.java deleted file mode 100644 index 644c4b7903b5..000000000000 --- a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlDataObject.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.netbeans.modules.languages.toml; - -import java.io.IOException; -import org.netbeans.core.spi.multiview.MultiViewElement; -import org.netbeans.core.spi.multiview.text.MultiViewEditorElement; -import org.openide.awt.ActionID; -import org.openide.awt.ActionReference; -import org.openide.awt.ActionReferences; -import org.openide.filesystems.FileObject; -import org.openide.filesystems.MIMEResolver; -import org.openide.loaders.DataNode; -import org.openide.loaders.DataObject; -import org.openide.loaders.DataObjectExistsException; -import org.openide.loaders.MultiDataObject; -import org.openide.loaders.MultiFileLoader; -import org.openide.nodes.Children; -import org.openide.nodes.Node; -import org.openide.util.*; -import org.openide.windows.TopComponent; - -/** - * - * @author lkishalmi - */ -@NbBundle.Messages( - "TOMLResolver=Toml File" -) -@MIMEResolver.ExtensionRegistration(displayName = "#TOMLResolver", - extension = "toml", - mimeType = TomlTokenId.TOML_MIME_TYPE, - position = 285 -) - -@ActionReferences({ - @ActionReference( - path = "Loaders/text/x-toml/Actions", - id = @ActionID(category = "System", id = "org.openide.actions.OpenAction"), - position = 100, - separatorAfter = 300 - ), - @ActionReference( - path = "Loaders/text/x-toml/Actions", - id = @ActionID(category = "Edit", id = "org.openide.actions.CutAction"), - position = 400 - ), - @ActionReference( - path = "Loaders/text/x-toml/Actions", - id = @ActionID(category = "Edit", id = "org.openide.actions.CopyAction"), - position = 500, - separatorAfter = 600 - ), - @ActionReference( - path = "Loaders/text/x-toml/Actions", - id = @ActionID(category = "Edit", id = "org.openide.actions.DeleteAction"), - position = 700 - ), - @ActionReference( - path = "Loaders/text/x-toml/Actions", - id = @ActionID(category = "System", id = "org.openide.actions.RenameAction"), - position = 800, - separatorAfter = 900 - ), - @ActionReference( - path = "Loaders/text/x-toml/Actions", - id = @ActionID(category = "System", id = "org.openide.actions.SaveAsTemplateAction"), - position = 1000, - separatorAfter = 1100 - ), - @ActionReference( - path = "Loaders/text/x-toml/Actions", - id = @ActionID(category = "System", id = "org.openide.actions.FileSystemAction"), - position = 1200, - separatorAfter = 1300 - ), - @ActionReference( - path = "Loaders/text/x-toml/Actions", - id = @ActionID(category = "System", id = "org.openide.actions.ToolsAction"), - position = 1400 - ), - @ActionReference( - path = "Loaders/text/x-toml/Actions", - id = @ActionID(category = "System", id = "org.openide.actions.PropertiesAction"), - position = 1500 - ) -}) - -@DataObject.Registration( - mimeType = TomlTokenId.TOML_MIME_TYPE, - iconBase = "org/netbeans/modules/languages/toml/toml-icon.png", - displayName = "#TOMLResolver", - position = 303 -) -public final class TomlDataObject extends MultiDataObject { - - public TomlDataObject(FileObject pf, MultiFileLoader loader) throws DataObjectExistsException, IOException { - super(pf, loader); - registerEditor(TomlTokenId.TOML_MIME_TYPE, true); - } - - @Override - protected Node createNodeDelegate() { - return new DataNode(this, Children.LEAF, getLookup()); - } - - @Override - protected int associateLookup() { - return 1; - } - - @NbBundle.Messages("Source=&Source") - @MultiViewElement.Registration( - displayName = "#Source", - iconBase = "org/netbeans/modules/languages/toml/toml-icon.png", - persistenceType = TopComponent.PERSISTENCE_ONLY_OPENED, - mimeType = TomlTokenId.TOML_MIME_TYPE, - preferredID = "neon.source", - position = 1 - ) - public static MultiViewEditorElement createMultiViewEditorElement(Lookup context) { - return new MultiViewEditorElement(context); - } - -} diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLanguage.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLanguage.java index 55e62bf1483f..24576dcc0183 100644 --- a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLanguage.java +++ b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLanguage.java @@ -19,14 +19,90 @@ package org.netbeans.modules.languages.toml; import org.netbeans.api.lexer.Language; +import org.netbeans.core.spi.multiview.MultiViewElement; +import org.netbeans.core.spi.multiview.text.MultiViewEditorElement; import org.netbeans.modules.csl.spi.DefaultLanguageConfig; import org.netbeans.modules.csl.spi.LanguageRegistration; +import org.netbeans.modules.parsing.spi.Parser; +import org.openide.awt.ActionID; +import org.openide.awt.ActionReference; +import org.openide.awt.ActionReferences; +import org.openide.filesystems.MIMEResolver; +import org.openide.util.Lookup; +import org.openide.util.NbBundle; +import org.openide.windows.TopComponent; /** * * @author lkishalmi */ -@LanguageRegistration(mimeType = TomlTokenId.TOML_MIME_TYPE) //NOI18N +@NbBundle.Messages( + "TOMLResolver=Toml File" +) +@MIMEResolver.ExtensionRegistration(displayName = "#TOMLResolver", + extension = "toml", + mimeType = TomlTokenId.TOML_MIME_TYPE, + position = 285 +) + +@ActionReferences({ + @ActionReference( + path = "Loaders/text/x-toml/Actions", + id = @ActionID(category = "System", id = "org.openide.actions.OpenAction"), + position = 100, + separatorAfter = 200 + ), + @ActionReference( + path = "Loaders/text/x-toml/Actions", + id = @ActionID(category = "Edit", id = "org.openide.actions.CutAction"), + position = 300 + ), + @ActionReference( + path = "Loaders/text/x-toml/Actions", + id = @ActionID(category = "Edit", id = "org.openide.actions.CopyAction"), + position = 400 + ), + @ActionReference( + path = "Loaders/text/x-toml/Actions", + id = @ActionID(category = "Edit", id = "org.openide.actions.PasteAction"), + position = 500, + separatorAfter = 600 + ), + @ActionReference( + path = "Loaders/text/x-toml/Actions", + id = @ActionID(category = "Edit", id = "org.openide.actions.DeleteAction"), + position = 700 + ), + @ActionReference( + path = "Loaders/text/x-toml/Actions", + id = @ActionID(category = "System", id = "org.openide.actions.RenameAction"), + position = 800, + separatorAfter = 900 + ), + @ActionReference( + path = "Loaders/text/x-toml/Actions", + id = @ActionID(category = "System", id = "org.openide.actions.SaveAsTemplateAction"), + position = 1000, + separatorAfter = 1100 + ), + @ActionReference( + path = "Loaders/text/x-toml/Actions", + id = @ActionID(category = "System", id = "org.openide.actions.FileSystemAction"), + position = 1200, + separatorAfter = 1300 + ), + @ActionReference( + path = "Loaders/text/x-toml/Actions", + id = @ActionID(category = "System", id = "org.openide.actions.ToolsAction"), + position = 1400 + ), + @ActionReference( + path = "Loaders/text/x-toml/Actions", + id = @ActionID(category = "System", id = "org.openide.actions.PropertiesAction"), + position = 1500 + ) +}) +@LanguageRegistration(mimeType = TomlTokenId.TOML_MIME_TYPE, useMultiview = true) public class TomlLanguage extends DefaultLanguageConfig { @Override @@ -43,4 +119,22 @@ public String getDisplayName() { public String getLineCommentPrefix() { return "#"; // NOI18N } + + @Override + public Parser getParser() { + return new TomlParser(); + } + + @NbBundle.Messages("Source=&Source") + @MultiViewElement.Registration( + displayName = "#Source", + iconBase = "org/netbeans/modules/languages/toml/toml-icon.png", + persistenceType = TopComponent.PERSISTENCE_ONLY_OPENED, + mimeType = TomlTokenId.TOML_MIME_TYPE, + preferredID = "toml.source", + position = 100 + ) + public static MultiViewEditorElement createMultiViewEditorElement(Lookup context) { + return new MultiViewEditorElement(context); + } } diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java index d7bc97eb3eac..dcbced435f15 100644 --- a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java +++ b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java @@ -18,7 +18,8 @@ */ package org.netbeans.modules.languages.toml; -import org.antlr.v4.runtime.misc.IntegerList; +import java.lang.reflect.Field; +import org.antlr.v4.runtime.misc.IntegerStack; import org.netbeans.api.lexer.Token; import org.netbeans.spi.lexer.Lexer; import org.netbeans.spi.lexer.LexerRestartInfo; @@ -35,10 +36,12 @@ public final class TomlLexer implements Lexer { private final TokenFactory tokenFactory; private final org.tomlj.internal.TomlLexer lexer; + private final LexerInputCharStream input; public TomlLexer(LexerRestartInfo info) { this.tokenFactory = info.tokenFactory(); - this.lexer = new org.tomlj.internal.TomlLexer(new LexerInputCharStream(info.input())); + this.input = new LexerInputCharStream(info.input()); + this.lexer = new org.tomlj.internal.TomlLexer(input); if (info.state() != null) { ((LexerState) info.state()).restore(lexer); } @@ -46,80 +49,76 @@ public TomlLexer(LexerRestartInfo info) { @Override public Token nextToken() { - try { - org.antlr.v4.runtime.Token nextToken = lexer.nextToken(); - if (nextToken.getType() == EOF) { - return null; - } - switch (nextToken.getType()) { - case TripleQuotationMark: - case TripleApostrophe: - return token(ML_STRING_START); - - case StringChar: - case QuotationMark: - case Apostrophe: - return token(STRING); - - case MLBasicStringEnd: - case MLLiteralStringEnd: - return token(ML_STRING_END); - - case Comma: - case ArrayStart: - case ArrayEnd: - case InlineTableStart: - case InlineTableEnd: - - case Dot: - return token(DOT); - - case Equals: - return token(EQUALS); - - case TableKeyStart: - case TableKeyEnd: - case ArrayTableKeyStart: - case ArrayTableKeyEnd: - return token(TABLE_MARK); - case UnquotedKey: - return token(KEY); - case Comment: - return token(COMMENT); - case WS: - case NewLine: - return token(TomlTokenId.WHITESPACE); - case Error: - return token(ERROR); - case DecimalInteger: - case HexInteger: - case OctalInteger: - case BinaryInteger: - case FloatingPoint: - case FloatingPointInf: - case FloatingPointNaN: - return token(NUMBER); - - case TrueBoolean: - case FalseBoolean: - return token(BOOLEAN); - - case EscapeSequence: - return token(ESCAPE_SEQUENCE); - - case Dash: - case Plus: - case Colon: - case Z: - case TimeDelimiter: - case DateDigits: - case DateComma: - return token(DATE); - default: - return token(ERROR); - } - } catch (IndexOutOfBoundsException ex) { - return token(ERROR); + org.antlr.v4.runtime.Token nextToken = lexer.nextToken(); + if (nextToken.getType() == EOF) { + return null; + } + switch (nextToken.getType()) { + case TripleQuotationMark: + case TripleApostrophe: + return token(ML_STRING_START); + + case StringChar: + case QuotationMark: + case Apostrophe: + return token(STRING); + + case MLBasicStringEnd: + case MLLiteralStringEnd: + return token(ML_STRING_END); + + case Comma: + case ArrayStart: + case ArrayEnd: + case InlineTableStart: + case InlineTableEnd: + + case Dot: + return token(DOT); + + case Equals: + return token(EQUALS); + + case TableKeyStart: + case TableKeyEnd: + case ArrayTableKeyStart: + case ArrayTableKeyEnd: + return token(TABLE_MARK); + case UnquotedKey: + return token(KEY); + case Comment: + return token(COMMENT); + case WS: + case NewLine: + return token(TomlTokenId.WHITESPACE); + case Error: + return token(ERROR); + case DecimalInteger: + case HexInteger: + case OctalInteger: + case BinaryInteger: + case FloatingPoint: + case FloatingPointInf: + case FloatingPointNaN: + return token(NUMBER); + + case TrueBoolean: + case FalseBoolean: + return token(BOOLEAN); + + case EscapeSequence: + return token(ESCAPE_SEQUENCE); + + case Dash: + case Plus: + case Colon: + case Z: + case TimeDelimiter: + case DateDigits: + case DateComma: + return token(DATE); + default: + return token(ERROR); } } @@ -133,25 +132,58 @@ public void release() { } private Token token(TomlTokenId id) { + input.markToken(); return tokenFactory.createToken(id); } private static class LexerState { + private static Field ARRAY_DEPTH; + private static Field ARRAY_DEPTH_STACK; + final int state; final int mode; - final IntegerList modes; + final IntegerStack modes; + + final int arrayDepth; + final IntegerStack arrayDepthStack; + + static { + try { + // Hack accessing private state parts of TomlLexer + // See: https://github.com/tomlj/tomlj/pull/42 + ARRAY_DEPTH = org.tomlj.internal.TomlLexer.class.getDeclaredField("arrayDepth"); + ARRAY_DEPTH.setAccessible(true); + ARRAY_DEPTH_STACK = org.tomlj.internal.TomlLexer.class.getDeclaredField("arrayDepthStack"); + ARRAY_DEPTH_STACK.setAccessible(true); + } catch (ReflectiveOperationException ex) { + } + } LexerState(org.tomlj.internal.TomlLexer lexer) { this.state= lexer.getState(); this.mode = lexer._mode; - this.modes = new IntegerList(lexer._modeStack); + this.modes = new IntegerStack(lexer._modeStack); + + try { + this.arrayDepth = ARRAY_DEPTH.getInt(lexer); + this.arrayDepthStack = new IntegerStack((IntegerStack)ARRAY_DEPTH_STACK.get(lexer)); + } catch (IllegalAccessException ex) { + throw new RuntimeException(ex); + } } public void restore(org.tomlj.internal.TomlLexer lexer) { lexer.setState(state); lexer._modeStack.addAll(modes); lexer._mode = mode; + + try { + ARRAY_DEPTH.setInt(lexer, arrayDepth); + ((IntegerStack) ARRAY_DEPTH_STACK.get(lexer)).addAll(arrayDepthStack); + } catch (IllegalAccessException ex) { + throw new RuntimeException(ex); + } } @Override diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlParser.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlParser.java new file mode 100644 index 000000000000..12bf61a819f0 --- /dev/null +++ b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlParser.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.netbeans.modules.languages.toml; + +import java.util.ArrayList; +import java.util.List; +import javax.swing.event.ChangeListener; +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.Token; +import org.netbeans.modules.csl.api.Error; +import org.netbeans.modules.csl.api.Severity; +import org.netbeans.modules.csl.spi.DefaultError; +import org.netbeans.modules.csl.spi.ParserResult; +import org.netbeans.modules.parsing.api.Snapshot; +import org.netbeans.modules.parsing.api.Task; +import org.netbeans.modules.parsing.spi.ParseException; +import org.netbeans.modules.parsing.spi.Parser; +import org.netbeans.modules.parsing.spi.SourceModificationEvent; + +/** + * + * @author Laszlo Kishalmi + */ +public class TomlParser extends Parser{ + + private Result lastResult; + + @Override + public void parse(Snapshot snapshot, Task task, SourceModificationEvent event) throws ParseException { + org.tomlj.internal.TomlLexer lexer = new org.tomlj.internal.TomlLexer(CharStreams.fromString(String.valueOf(snapshot.getText()))); + org.tomlj.internal.TomlParser parser = new org.tomlj.internal.TomlParser(new CommonTokenStream(lexer)); + final List errors = new ArrayList<>(); + parser.addErrorListener(new BaseErrorListener() { + @Override + public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { + int errorBegin = 0; + int errorEnd = 0; + if (offendingSymbol instanceof Token) { + Token token = (Token) offendingSymbol; + errorBegin = token.getStartIndex(); + errorEnd = token.getStopIndex() + 1; + } + errors.add(new DefaultError(null, msg, null, snapshot.getSource().getFileObject(), errorBegin, errorEnd, Severity.ERROR)); + } + + }); + parser.toml(); + lastResult = new TomlParser.Result(snapshot, errors); + } + + @Override + public Result getResult(Task task) throws ParseException { + return lastResult; + } + + @Override + public void addChangeListener(ChangeListener changeListener) { + } + + @Override + public void removeChangeListener(ChangeListener changeListener) { + } + + public static class Result extends ParserResult { + + private final List errors; + + public Result(Snapshot snapshot, List errors) { + super(snapshot); + this.errors = errors; + } + + @Override + public List getDiagnostics() { + return errors; + } + + @Override + protected void invalidate() { + } + + } +} diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/layer.xml b/ide/languages.toml/src/org/netbeans/modules/languages/toml/layer.xml index 59d0f2b1c6eb..8aa5b7f258c1 100644 --- a/ide/languages.toml/src/org/netbeans/modules/languages/toml/layer.xml +++ b/ide/languages.toml/src/org/netbeans/modules/languages/toml/layer.xml @@ -42,6 +42,13 @@ + + + + + + +