From d5b2f6484e536f88a0f189cd5f43a7b46bde7250 Mon Sep 17 00:00:00 2001 From: Michael Dowling Date: Sun, 14 Jun 2020 23:55:46 -0700 Subject: [PATCH] Add support message templates in EmitEachSelector This commit adds support for templated messages in the EmitEachSelector that are able to reference variables captured while shapes are matched. This allows for very rich and descriptive error messages that generally are as good as error messages in code-driven Smithy validators. In order to pull this, it required that attribute value selector context values could be parsed from a message template. Since context values are already parsed as part of parsing selectors, this motivated me to reuse the existing selector's parsing functionality. To do this, I created a SimpleParser abstraction that extracts out lots of the duplicate parsing code in Smithy into a single class. This allowed parsing of attributes to be shared, but it was also an opportunity to trim down duplicate code and reuse parsing logic across the IDL and selectors. --- .../source/1.0/spec/core/model-validation.rst | 216 +++++++++- docs/themes/smithy/static/default.css_t | 7 +- .../smithy/model/loader/IdlModelParser.java | 254 ++++-------- .../smithy/model/loader/IdlNodeParser.java | 24 +- .../smithy/model/loader/IdlNumberParser.java | 85 ---- .../smithy/model/loader/IdlShapeIdParser.java | 88 ----- .../smithy/model/loader/IdlTextParser.java | 6 +- .../smithy/model/loader/IdlTraitParser.java | 26 +- .../smithy/model/loader/ParserUtils.java | 195 +++++++++ .../smithy/model/pattern/SmithyPattern.java | 9 +- .../smithy/model/selector/AttributeValue.java | 14 + .../model/selector/AttributeValueImpl.java | 15 - .../smithy/model/selector/Selector.java | 2 +- .../{Parser.java => SelectorParser.java} | 369 ++++++------------ .../selector/SelectorSyntaxException.java | 10 +- .../amazon/smithy/model/shapes/ShapeId.java | 17 +- .../linters/EmitEachSelectorValidator.java | 204 +++++++++- .../model/loader/IdlModelLoaderTest.java | 2 +- .../model/selector/AttributeValueTest.java | 10 +- .../smithy/model/selector/SelectorTest.java | 4 +- .../EmitEachSelectorValidatorTest.java | 157 ++++++++ .../emit-each-selector-validator.errors | 60 +-- .../invalid/apply-requires-newline.smithy | 2 +- .../loader/invalid/newline-after-shape.smithy | 2 +- .../amazon/smithy/utils/CodeFormatter.java | 3 + .../amazon/smithy/utils/SimpleParser.java | 365 +++++++++++++++++ .../smithy/utils/OptionalUtilsTest.java | 1 - .../amazon/smithy/utils/SimpleParserTest.java | 216 ++++++++++ 28 files changed, 1656 insertions(+), 707 deletions(-) delete mode 100644 smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNumberParser.java delete mode 100644 smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlShapeIdParser.java create mode 100644 smithy-model/src/main/java/software/amazon/smithy/model/loader/ParserUtils.java rename smithy-model/src/main/java/software/amazon/smithy/model/selector/{Parser.java => SelectorParser.java} (62%) create mode 100644 smithy-model/src/test/java/software/amazon/smithy/model/validation/linters/EmitEachSelectorValidatorTest.java create mode 100644 smithy-utils/src/main/java/software/amazon/smithy/utils/SimpleParser.java create mode 100644 smithy-utils/src/test/java/software/amazon/smithy/utils/SimpleParserTest.java diff --git a/docs/source/1.0/spec/core/model-validation.rst b/docs/source/1.0/spec/core/model-validation.rst index 8878ee31cba..76cf81c6945 100644 --- a/docs/source/1.0/spec/core/model-validation.rst +++ b/docs/source/1.0/spec/core/model-validation.rst @@ -309,9 +309,18 @@ Configuration - Description * - selector - ``string`` - - **Required**. A valid :ref:`selector `. Each shape in - the model that is returned from the selector with emit a validation - event. + - **Required**. A valid :ref:`selector `. A validation + event is emitted for each shape in the model that matches the + ``selector``. + * - :ref:`bindToTrait ` + - ``string`` + - An optional string that MUST be a valid :ref:`shape ID ` + that targets a :ref:`trait definition `. + A validation event is only emitted for shapes that have this trait. + * - :ref:`messageTemplate ` + - ``string`` + - A custom template that is expanded for each matching shape and + assigned as the message for the emitted validation event. The following example detects if a shape is missing documentation with the following constraints: @@ -393,6 +402,207 @@ as lifecycle 'read' or 'delete' that has a shape name that does not start with ] +.. _emit-each-bind-to-trait: + +Binding events to traits +------------------------ + +The ``bindToTrait`` property contains a :ref:`shape ID ` that MUST +reference a :ref:`trait definition ` shape. When set, this +property causes the ``EmitEachSelector`` validator to only emit validation +events for shapes that have the referenced trait. The contextual location of +where the violation occurred in the model SHOULD point to the location where +the trait is applied to the matched shape. + +Consider the following model: + +.. code-block:: smithy + + metadata validators = [ + { + name: "EmitEachSelector", + id: "DocumentedString", + configuration: { + // matches all shapes + selector: "*", + // Only emitted for shapes with the documentation + // trait, and each event points to where the + // trait is defined. + bindToTrait: documentation + } + } + ] + + namespace smithy.example + + @documentation("Hello") + string A // <-- Emits an event + + string B // <-- Does not emit an event + +The ``DocumentedString`` validator will only emit an event for +``smithy.example#A`` because ``smithy.example#B`` does not have the +:ref:`documentation-trait`. + + +.. _emit-each-message-template: + +Message templates +----------------- + +A ``messageTemplate`` is used to create more granular error messages. The +template consists of literal spans and :token:`selector context value ` +templates (for example, ``@{id}``). A selector context value MAY be escaped +by placing a ``@`` before a ``@`` character (for example, ``@@`` expands to +``@``). ``@`` characters in the message template that are not escaped MUST +form a valid ``selector_context_value`` production. + +For each shaped matched by the ``selector`` of an ``EmitEachSelector``, a +:ref:`selector attribute ` is created from the shape +along with all of the :ref:`selector variables ` that were +assigned when the shape was matched. Each ``selector_context_value`` in the +template is then expanded by retrieving nested properties from the shape +using a pipe-delimited path (for example, ``@{id|name}`` expands to the +name of the matching shape's :ref:`shape ID `). + +Consider the following model: + +.. code-block:: smithy + + metadata validators = [ + { + name: "EmitEachSelector", + configuration: { + selector: "[trait|documentation]", + messageTemplate: """ + This shape has a name of @{id|name} and a @@documentation \ + trait of "@{trait|documentation}".""" + } + } + ] + + namespace smithy.example + + @documentation("Hello") + string A + + @documentation("Goodbye") + string B + +The above selector will emit two validation events: + +.. list-table:: + :header-rows: 1 + :widths: 20 80 + + * - Shape ID + - Expanded message + * - ``smithy.example#A`` + - This shape has a name of A and a @documentation trait of "Hello". + * - ``smithy.example#B`` + - This shape has a name of B and a @documentation trait of "Goodbye". + +:ref:`Selector variables ` can be used in the selector +to make message templates more descriptive. Consider the following example: + +.. code-block:: smithy + + metadata validators = [ + { + name: "EmitEachSelector", + id: "UnstableTrait", + configuration: { + selector: """ + $matches(-[trait]-> [trait|unstable]) + ${matches}""", + messageTemplate: "This shape applies traits(s) that are unstable: @{var|matches|id}" + } + } + ] + + namespace smithy.example + + @trait + @unstable + structure doNotUseMe {} + + @doNotUseMe + string A + +The above selector will emit the following validation event: + +.. list-table:: + :header-rows: 1 + :widths: 20 80 + + * - Shape ID + - Expanded message + * - ``smithy.example#A`` + - This shape applies traits(s) that are unstable: [smithy.example#doNotUseMe] + + +Variable message formatting +--------------------------- + +Different types of variables expand to different kinds of strings in message +templates. + +.. list-table:: + :header-rows: 1 + :widths: 20 80 + + * - Attribute + - Expansion + * - empty values + - An empty value expands to nothingness [#comparison]_. Empty values are + created when a selector context value attempts to access a variable + or nested property that does not exist. + + Consider the following message template: ``Hello, @{foo}.``. Because + ``foo`` is not a valid selector attribute, the message expands to: + + .. code-block:: none + + Hello, . + * - :ref:`id ` + - Expands to the absolute :ref:`shape ID ` of a shape + [#comparison]_. + * - literal values + - Literal values are created when descending into nested properties of + an ``id``, ``service``, or projection attribute. A literal string is + expanded to the the contents of the string with no wrapping quotes. + A literal integer is expanded to the string representation of the + number. [#comparison]_ + * - :ref:`node ` + - A JSON formatted string representation of a trait or nested property + of a trait. The JSON is *not* pretty-printed, meaning there is no + indentation or newlines inserted into the JSON output for formatting. + For example, a template of ``@{trait|tags}`` applied to a shape with + a :ref:`tags-trait` that contains "a" and "b" would expand to: + + .. code-block:: none + + ["a","b"] + * - :ref:`projection ` + - Expands to a list that starts with ``[`` and ends with ``]``. Each + shape in the projection is inserted into the list using variable + message formatting. Subsequent shapes are separated from the previous + shape by a comma followed by a space. If a variable projection + (for example, ``@{var|foo}``) contains two shape IDs, + ``smithy.example#A`` and ``smithy.example#B``, the attribute expands + to: + + .. code-block:: none + + [smithy.example#A, smithy.example#B] + * - :ref:`service ` + - Expands to the absolute shape ID of a service shape [#comparison]_. + * - :ref:`trait ` + - Expands to nothingness [#comparison]_. + +.. [#comparison] This is the same behavior that is used when the attribute is used in a :ref:`string comparison `. + + .. _EmitNoneSelector: EmitNoneSelector diff --git a/docs/themes/smithy/static/default.css_t b/docs/themes/smithy/static/default.css_t index 247f2118b70..984cd180363 100644 --- a/docs/themes/smithy/static/default.css_t +++ b/docs/themes/smithy/static/default.css_t @@ -133,6 +133,11 @@ h7 { color: #24292e } +/* Make the code block on the homepage have the right size */ +#splash pre { + font-size: 1em; +} + #splash .splash-column:first-child { padding-right: 10%; text-align: center; @@ -529,7 +534,7 @@ pre { line-height: 1.45; background-color: #222; border-radius: 3px; - font-size: 1em; + font-size: 14px; color: #A9B7C6; overflow-x: auto white-space: pre; diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelParser.java index 3c9f3e68cca..97ff5f8cdd5 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelParser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelParser.java @@ -28,11 +28,11 @@ import java.util.Set; import java.util.StringJoiner; import java.util.function.Consumer; -import java.util.function.Predicate; import java.util.stream.Collectors; import software.amazon.smithy.model.FromSourceLocation; import software.amazon.smithy.model.SourceLocation; import software.amazon.smithy.model.node.Node; +import software.amazon.smithy.model.node.NumberNode; import software.amazon.smithy.model.node.ObjectNode; import software.amazon.smithy.model.node.StringNode; import software.amazon.smithy.model.shapes.AbstractShapeBuilder; @@ -68,9 +68,10 @@ import software.amazon.smithy.model.validation.Validator; import software.amazon.smithy.utils.ListUtils; import software.amazon.smithy.utils.SetUtils; +import software.amazon.smithy.utils.SimpleParser; import software.amazon.smithy.utils.StringUtils; -final class IdlModelParser { +final class IdlModelParser extends SimpleParser { /** Only allow nesting up to 250 arrays/objects in node values. */ private static final int MAX_NESTING_LEVEL = 250; @@ -105,16 +106,10 @@ final class IdlModelParser { } private final String filename; - private final String model; private final LoaderVisitor visitor; - private final int length; - private int position = 0; - private int line = 1; - private int column = 1; private String namespace; private String definedVersion; private TraitEntry pendingDocumentationComment; - private int nestingLevel; /** Map of shape aliases to their targets. */ private final Map useShapes = new HashMap<>(); @@ -133,10 +128,9 @@ static final class TraitEntry { } IdlModelParser(String filename, String model, LoaderVisitor visitor) { + super(model, MAX_NESTING_LEVEL); this.filename = filename; this.visitor = visitor; - this.model = model; - this.length = model.length(); } void parse() { @@ -146,8 +140,37 @@ void parse() { parseShapeSection(); } + /** + * Overrides whitespace parsing to handle comments. + */ + @Override + public void ws() { + while (!eof()) { + char c = peek(); + if (c == '/') { + if (peekDocComment()) { + parseDocComment(); + } else { + parseComment(); + } + } else if (!(c == ' ' || c == '\t' || c == '\r' || c == '\n')) { + break; + } else { + skip(); + } + } + } + + @Override + public ModelSyntaxException syntax(String message) { + String formatted = format( + "Parse error at line %d, column %d near `%s`: %s", + line(), column(), peekDebugMessage(), message); + return new ModelSyntaxException(formatted, filename, line(), column()); + } + private void parseControlSection() { - while (charPeek() == '$') { + while (peek() == '$') { expect('$'); ws(); String key = IdlNodeParser.parseNodeObjectKey(this); @@ -193,7 +216,7 @@ private void onVersion(Node value) { } private void parseMetadataSection() { - while (charPeek() == 'm') { + while (peek() == 'm') { expect('m'); expect('e'); expect('t'); @@ -214,7 +237,7 @@ private void parseMetadataSection() { } private void parseShapeSection() { - if (charPeek() == 'n') { + if (peek() == 'n') { expect('n'); expect('a'); expect('m'); @@ -225,7 +248,12 @@ private void parseShapeSection() { expect('c'); expect('e'); ws(); - namespace = IdlShapeIdParser.parseNamespace(this); + + // Parse the namespace. + int start = position(); + ParserUtils.consumeNamespace(this); + namespace = sliceFrom(start); + br(); // Clear out any erroneous documentation comments. clearPendingDocs(); @@ -233,7 +261,7 @@ private void parseShapeSection() { parseUseSection(); parseShapeStatements(); } else if (!eof()) { - if (!IdlShapeIdParser.isIdentifierStart(charPeek())) { + if (!ParserUtils.isIdentifierStart(peek())) { throw syntax("Expected a namespace definition, but found unexpected syntax"); } else { throw syntax("A namespace must be defined before a use statement or shapes"); @@ -242,16 +270,16 @@ private void parseShapeSection() { } private void parseUseSection() { - while (charPeek() == 'u' && charPeek(1) == 's') { + while (peek() == 'u' && peek(1) == 's') { expect('u'); expect('s'); expect('e'); ws(); - int start = position; - IdlShapeIdParser.consumeNamespace(this); + int start = position(); + ParserUtils.consumeNamespace(this); expect('#'); - IdlShapeIdParser.consumeIdentifier(this); + ParserUtils.consumeIdentifier(this); String lexeme = sliceFrom(start); br(); // Clear out any erroneous documentation comments. @@ -270,7 +298,7 @@ private void parseUseSection() { private void parseShapeStatements() { while (!eof()) { ws(); - if (charPeek() == 'a') { + if (peek() == 'a') { parseApplyStatement(); } else { boolean docsOnly = pendingDocumentationComment != null; @@ -314,7 +342,7 @@ private void parseShape(List traits) { SourceLocation location = currentLocation(); // Do a check here to give better parsing error messages. - String shapeType = IdlShapeIdParser.parseIdentifier(this); + String shapeType = ParserUtils.parseIdentifier(this); if (!SHAPE_TYPES.contains(shapeType)) { switch (shapeType) { case "use": @@ -406,7 +434,7 @@ private void parseShape(List traits) { } private ShapeId parseShapeName() { - String name = IdlShapeIdParser.parseIdentifier(this); + String name = ParserUtils.parseIdentifier(this); if (useShapes.containsKey(name)) { throw syntax(String.format( @@ -440,19 +468,19 @@ private void parseMembers(ShapeId id, Set requiredMembers) { clearPendingDocs(); ws(); - if (charPeek() != '}') { + if (peek() != '}') { // Remove the parsed member from the remaining set to detect // when duplicates are found, or when members are missing. remaining.remove(parseMember(id, remaining)); while (!eof()) { ws(); - if (charPeek() == ',') { + if (peek() == ',') { expect(','); // A comma clears out any previously captured documentation // comments that may have been found when parsing the member. clearPendingDocs(); ws(); - if (charPeek() == '}') { + if (peek() == '}') { // Trailing comma: "," "}" break; } @@ -484,7 +512,7 @@ private String parseMember(ShapeId parent, Set requiredMembers) { // Parse optional member traits. List memberTraits = parseDocsAndTraits(); SourceLocation memberLocation = currentLocation(); - String memberName = IdlShapeIdParser.parseIdentifier(this); + String memberName = ParserUtils.parseIdentifier(this); // Only enforce "allowedMembers" if it isn't empty. if (!requiredMembers.isEmpty() && !requiredMembers.contains(memberName)) { @@ -497,7 +525,7 @@ private String parseMember(ShapeId parent, Set requiredMembers) { ShapeId memberId = parent.withMember(memberName); MemberShape.Builder memberBuilder = MemberShape.builder().id(memberId).source(memberLocation); SourceLocation targetLocation = currentLocation(); - String target = IdlShapeIdParser.parseShapeId(this); + String target = ParserUtils.parseShapeId(this); visitor.onShape(memberBuilder); onShapeTarget(target, targetLocation, memberBuilder::target); addTraits(memberId, memberTraits); @@ -601,7 +629,7 @@ private void parseResourceStatement(ShapeId id, SourceLocation location) { // "//" *(not_newline) private void parseComment() { expect('/'); - notNewline(); + consumeRemainingCharactersOnLine(); } private void parseDocComment() { @@ -615,7 +643,7 @@ private void parseDocComment() { } private boolean peekDocComment() { - return charPeek() == '/' && charPeek(1) == '/' && charPeek(2) == '/'; + return peek() == '/' && peek(1) == '/' && peek(2) == '/'; } // documentation_comment = "///" *(not_newline) @@ -624,11 +652,11 @@ private String parseDocCommentLine() { expect('/'); expect('/'); // Skip a leading space, if present. - if (charPeek() == ' ') { + if (peek() == ' ') { skip(); } int start = position(); - notNewline(); + consumeRemainingCharactersOnLine(); br(); return StringUtils.stripEnd(sliceFrom(start), " \t\r\n"); } @@ -642,7 +670,7 @@ private void parseApplyStatement() { ws(); SourceLocation location = currentLocation(); - String name = IdlShapeIdParser.parseShapeId(this); + String name = ParserUtils.parseShapeId(this); ws(); TraitEntry traitEntry = IdlTraitParser.parseTraitValue(this); @@ -743,81 +771,33 @@ private void onDeferredTrait(ShapeId target, String traitName, Node traitValue, }); } - boolean eof() { - return position >= length; - } - - void ws() { - while (!eof()) { - char c = charPeek(); - if (c == '/') { - if (peekDocComment()) { - parseDocComment(); - } else { - parseComment(); - } - } else if (!(c == ' ' || c == '\t' || c == '\r' || c == '\n')) { - break; - } else { - skip(); - } - } - } - - private void sp() { - while (!eof()) { - char c = charPeek(); - if (!(c == ' ' || c == '\t')) { - break; - } - skip(); - } + SourceLocation currentLocation() { + return new SourceLocation(filename, line(), column()); } - private void br() { - sp(); - - // EOF can also be considered a line break to end a file. - if (eof()) { - return; - } - - char c = charPeek(); - if (c == '\n') { - skip(); - } else if (c == '\r') { - skip(); - if (charPeek() == '\n') { - skip(); - } + NumberNode parseNumberNode() { + SourceLocation location = currentLocation(); + String lexeme = ParserUtils.parseNumber(this); + if (lexeme.contains("e") || lexeme.contains(".")) { + return new NumberNode(Double.valueOf(lexeme), location); } else { - throw syntax("Expected a line break"); - } - } - - private void notNewline() { - while (!eof()) { - char c = charPeek(); - if (c == '\r' || c == '\n') { - break; - } - skip(); + return new NumberNode(Long.parseLong(lexeme), location); } } private String peekDebugMessage() { - StringBuilder result = new StringBuilder(length); + StringBuilder result = new StringBuilder(expression().length()); - char c = charPeek(); + char c = peek(); // Try to read an entire identifier for context (16 char max) if that's what's being peeked. - if (c == ' ' || IdlShapeIdParser.isIdentifierStart(c) || IdlShapeIdParser.isDigit(c)) { + if (c == ' ' || ParserUtils.isIdentifierStart(c) || ParserUtils.isDigit(c)) { if (c == ' ') { result.append(' '); } for (int i = c == ' ' ? 1 : 0; i < 16; i++) { - c = charPeek(i); - if (IdlShapeIdParser.isIdentifierStart(c) || IdlShapeIdParser.isDigit(c)) { + c = peek(i); + if (ParserUtils.isIdentifierStart(c) || ParserUtils.isDigit(c)) { result.append(c); } else { break; @@ -828,7 +808,7 @@ private String peekDebugMessage() { // Take two characters for context. for (int i = 0; i < 2; i++) { - char peek = charPeek(i); + char peek = peek(i); if (peek == Character.MIN_VALUE) { result.append("[EOF]"); break; @@ -838,90 +818,4 @@ private String peekDebugMessage() { return result.toString(); } - - char charPeek() { - return charPeek(0); - } - - char charPeek(int offset) { - return position + offset >= length - ? Character.MIN_VALUE - : model.charAt(position + offset); - } - - char expect(char token) { - if (charPeek() == token) { - skip(); - return token; - } - - throw syntax(String.format("Expected: '%s', but found '%s'", token, peekSingleCharForMessage())); - } - - String peekSingleCharForMessage() { - char peek = charPeek(); - return peek == Character.MIN_VALUE ? "[EOF]" : String.valueOf(peek); - } - - SourceLocation currentLocation() { - return new SourceLocation(filename, line, column); - } - - ModelSyntaxException syntax(String message) { - String formatted = format( - "Parse error at line %d, column %d near `%s`: %s", - line, column, peekDebugMessage(), message); - return new ModelSyntaxException(formatted, filename, line, column); - } - - String sliceFrom(int start) { - return model.substring(start, position); - } - - int consumeUntilNoLongerMatches(Predicate predicate) { - int startPosition = position; - while (!eof()) { - char peekedChar = charPeek(); - if (!predicate.test(peekedChar)) { - break; - } - skip(); - } - - return position - startPosition; - } - - void skip() { - switch (model.charAt(position)) { - case '\r': - if (charPeek(1) == '\n') { - position++; - } - line++; - column = 1; - break; - case '\n': - line++; - column = 1; - break; - default: - column++; - } - - position++; - } - - int position() { - return position; - } - - void increaseNestingLevel() { - if (++nestingLevel >= MAX_NESTING_LEVEL) { - throw syntax("Node value nesting too deep"); - } - } - - void decreaseNestingLevel() { - nestingLevel--; - } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java index 5fea4a2a2ff..baf6cfb3fa8 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNodeParser.java @@ -37,7 +37,7 @@ final class IdlNodeParser { private IdlNodeParser() {} static Node parseNode(IdlModelParser parser) { - char c = parser.charPeek(); + char c = parser.peek(); switch (c) { case '{': return parseObjectNode(parser); @@ -62,10 +62,10 @@ static Node parseNode(IdlModelParser parser) { case '8': case '9': case '-': - return IdlNumberParser.parse(parser); + return parser.parseNumberNode(); default: { SourceLocation location = parser.currentLocation(); - return parseNodeTextWithKeywords(parser, location, IdlShapeIdParser.parseShapeId(parser)); + return parseNodeTextWithKeywords(parser, location, ParserUtils.parseShapeId(parser)); } } } @@ -90,9 +90,9 @@ static Node parseNodeTextWithKeywords(IdlModelParser parser, SourceLocation loca } static boolean peekTextBlock(IdlModelParser parser) { - return parser.charPeek() == '"' - && parser.charPeek(1) == '"' - && parser.charPeek(2) == '"'; + return parser.peek() == '"' + && parser.peek(1) == '"' + && parser.peek(2) == '"'; } static Node parseTextBlock(IdlModelParser parser) { @@ -111,7 +111,7 @@ static ObjectNode parseObjectNode(IdlModelParser parser) { parser.ws(); while (!parser.eof()) { - char c = parser.charPeek(); + char c = parser.peek(); if (c == '}') { break; } else { @@ -123,7 +123,7 @@ static ObjectNode parseObjectNode(IdlModelParser parser) { Node value = parseNode(parser); entries.put(new StringNode(key, keyLocation), value); parser.ws(); - if (parser.charPeek() == ',') { + if (parser.peek() == ',') { parser.skip(); parser.ws(); } else { @@ -138,10 +138,10 @@ static ObjectNode parseObjectNode(IdlModelParser parser) { } static String parseNodeObjectKey(IdlModelParser parser) { - if (parser.charPeek() == '"') { + if (parser.peek() == '"') { return IdlTextParser.parseQuotedString(parser); } else { - return IdlShapeIdParser.parseIdentifier(parser); + return ParserUtils.parseIdentifier(parser); } } @@ -153,13 +153,13 @@ private static ArrayNode parseArrayNode(IdlModelParser parser) { parser.ws(); while (!parser.eof()) { - char c = parser.charPeek(); + char c = parser.peek(); if (c == ']') { break; } else { items.add(parseNode(parser)); parser.ws(); - if (parser.charPeek() == ',') { + if (parser.peek() == ',') { parser.skip(); parser.ws(); } else { diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNumberParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNumberParser.java deleted file mode 100644 index 1d8c11ba02a..00000000000 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlNumberParser.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.smithy.model.loader; - -import software.amazon.smithy.model.SourceLocation; -import software.amazon.smithy.model.node.NumberNode; - -/** - * Parses IDL numbers. - */ -final class IdlNumberParser { - - private IdlNumberParser() {} - - static NumberNode parse(IdlModelParser parser) { - SourceLocation location = parser.currentLocation(); - String lexeme = parseNumber(parser); - if (lexeme.contains("e") || lexeme.contains(".")) { - return new NumberNode(Double.valueOf(lexeme), location); - } else { - return new NumberNode(Long.parseLong(lexeme), location); - } - } - - // -?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)? - private static String parseNumber(IdlModelParser parser) { - int startPosition = parser.position(); - - char current = parser.charPeek(); - - if (current == '-') { - parser.skip(); - if (!IdlShapeIdParser.isDigit(parser.charPeek())) { - throw parser.syntax(createInvalidString( - parser, startPosition, "'-' must be followed by a digit")); - } - } - - parser.consumeUntilNoLongerMatches(IdlShapeIdParser::isDigit); - - // Consume decimals. - char peek = parser.charPeek(); - if (peek == '.') { - parser.skip(); - if (parser.consumeUntilNoLongerMatches(IdlShapeIdParser::isDigit) == 0) { - throw parser.syntax(createInvalidString( - parser, startPosition, "'.' must be followed by a digit")); - } - } - - // Consume scientific notation. - peek = parser.charPeek(); - if (peek == 'e' || peek == 'E') { - parser.skip(); - peek = parser.charPeek(); - if (peek == '+' || peek == '-') { - parser.skip(); - } - if (parser.consumeUntilNoLongerMatches(IdlShapeIdParser::isDigit) == 0) { - throw parser.syntax(createInvalidString( - parser, startPosition, "'e', '+', and '-' must be followed by a digit")); - } - } - - return parser.sliceFrom(startPosition); - } - - private static String createInvalidString(IdlModelParser parser, int startPosition, String message) { - String lexeme = parser.sliceFrom(startPosition); - return String.format("Invalid number '%s': %s", lexeme, message); - } -} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlShapeIdParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlShapeIdParser.java deleted file mode 100644 index a2f47024609..00000000000 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlShapeIdParser.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.smithy.model.loader; - -/** - * Parses IDL shape IDs. - */ -final class IdlShapeIdParser { - - private IdlShapeIdParser() {} - - static String parseShapeId(IdlModelParser parser) { - int start = parser.position(); - consumeShapeId(parser); - return parser.sliceFrom(start); - } - - static String parseIdentifier(IdlModelParser parser) { - int start = parser.position(); - consumeIdentifier(parser); - return parser.sliceFrom(start); - } - - // identifier = (ALPHA / "_") *(ALPHA / DIGIT / "_") - static void consumeIdentifier(IdlModelParser parser) { - // (ALPHA / "_") - if (!isIdentifierStart(parser.charPeek())) { - throw parser.syntax("Expected a valid identifier character, but found '" - + parser.peekSingleCharForMessage() + '\''); - } - - // *(ALPHA / DIGIT / "_") - parser.consumeUntilNoLongerMatches(IdlShapeIdParser::isValidIdentifierCharacter); - } - - private static boolean isValidIdentifierCharacter(char c) { - return isIdentifierStart(c) || isDigit(c); - } - - static String parseNamespace(IdlModelParser parser) { - int start = parser.position(); - consumeNamespace(parser); - return parser.sliceFrom(start); - } - - static boolean isIdentifierStart(char c) { - return c == '_' || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); - } - - static boolean isDigit(char c) { - return c >= '0' && c <= '9'; - } - - static void consumeNamespace(IdlModelParser parser) { - consumeIdentifier(parser); - while (parser.charPeek() == '.') { - parser.skip(); - consumeIdentifier(parser); - } - } - - private static void consumeShapeId(IdlModelParser parser) { - consumeNamespace(parser); - - if (parser.charPeek() == '#') { - parser.skip(); - consumeIdentifier(parser); - } - - if (parser.charPeek() == '$') { - parser.skip(); - consumeIdentifier(parser); - } - } -} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTextParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTextParser.java index 1e05d910599..e60440ccaa2 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTextParser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTextParser.java @@ -25,7 +25,7 @@ private IdlTextParser() {} // quoted_text = DQUOTE *quoted_char DQUOTE static String parseQuotedString(IdlModelParser parser) { parser.expect('"'); - if (parser.charPeek() == '"') { // open and closed string. + if (parser.peek() == '"') { // open and closed string. parser.skip(); return ""; // "" } else { // " @@ -38,8 +38,8 @@ static String parseQuotedTextAndTextBlock(IdlModelParser parser, boolean triple) int start = parser.position(); while (!parser.eof()) { - char next = parser.charPeek(); - if (next == '"' && (!triple || (parser.charPeek(1) == '"' && parser.charPeek(2) == '"'))) { + char next = parser.peek(); + if (next == '"' && (!triple || (parser.peek(1) == '"' && parser.peek(2) == '"'))) { break; } parser.skip(); diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java index e2653401c06..0c60b830f9d 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java @@ -32,7 +32,7 @@ private IdlTraitParser() {} // trait_statements = *(ws trait) static List parseTraits(IdlModelParser parser) { List entries = new ArrayList<>(); - while (parser.charPeek() == '@') { + while (parser.peek() == '@') { entries.add(parseTraitValue(parser)); parser.ws(); } @@ -43,10 +43,10 @@ static List parseTraits(IdlModelParser parser) { static IdlModelParser.TraitEntry parseTraitValue(IdlModelParser parser) { // "@" shape_id parser.expect('@'); - String id = IdlShapeIdParser.parseShapeId(parser); + String id = ParserUtils.parseShapeId(parser); // No (): it's an annotation trait. - if (parser.charPeek() != '(') { + if (parser.peek() != '(') { return new IdlModelParser.TraitEntry(id, new NullNode(parser.currentLocation()), true); } @@ -54,7 +54,7 @@ static IdlModelParser.TraitEntry parseTraitValue(IdlModelParser parser) { parser.ws(); // (): it's also an annotation trait. - if (parser.charPeek() == ')') { + if (parser.peek() == ')') { parser.expect(')'); return new IdlModelParser.TraitEntry(id, new NullNode(parser.currentLocation()), true); } @@ -69,7 +69,7 @@ static IdlModelParser.TraitEntry parseTraitValue(IdlModelParser parser) { private static Node parseTraitValueBody(IdlModelParser parser) { SourceLocation keyLocation = parser.currentLocation(); - char c = parser.charPeek(); + char c = parser.peek(); switch (c) { case '{': case '[': @@ -85,11 +85,11 @@ private static Node parseTraitValueBody(IdlModelParser parser) { return parseTraitValueBodyIdentifierOrQuotedString(parser, keyLocation, key, false); } default: { // Parser numbers. - if (c == '-' || IdlShapeIdParser.isDigit(c)) { - return IdlNumberParser.parse(parser); + if (c == '-' || ParserUtils.isDigit(c)) { + return parser.parseNumberNode(); } else { // Parse unquoted strings or possibly a structured trait. - String key = IdlShapeIdParser.parseIdentifier(parser); + String key = ParserUtils.parseIdentifier(parser); return parseTraitValueBodyIdentifierOrQuotedString(parser, keyLocation, key, true); } } @@ -105,7 +105,7 @@ private static Node parseTraitValueBodyIdentifierOrQuotedString( parser.ws(); // If the next character is ':', this it's a KVP. - if (parser.charPeek() == ':') { + if (parser.peek() == ':') { parser.expect(':'); parser.ws(); return parseStructuredTrait(parser, new StringNode(key, location)); @@ -124,14 +124,14 @@ private static ObjectNode parseStructuredTrait(IdlModelParser parser, StringNode entries.put(startingKey, firstValue); parser.ws(); - while (!parser.eof() && parser.charPeek() != ')') { + while (!parser.eof() && parser.peek() != ')') { parser.expect(','); parser.ws(); - if (parser.charPeek() == ')') { + if (parser.peek() == ')') { break; } - char c = parser.charPeek(); - if (IdlShapeIdParser.isIdentifierStart(c) || c == '"') { + char c = parser.peek(); + if (ParserUtils.isIdentifierStart(c) || c == '"') { parseTraitStructureKvp(parser, entries); } else { throw parser.syntax("Unexpected object key character: '" + c + '\''); diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/loader/ParserUtils.java b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ParserUtils.java new file mode 100644 index 00000000000..016d5d1d16d --- /dev/null +++ b/smithy-model/src/main/java/software/amazon/smithy/model/loader/ParserUtils.java @@ -0,0 +1,195 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.model.loader; + +import software.amazon.smithy.utils.SimpleParser; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * Utility methods that act on a {@link SimpleParser} and parse + * Smithy grammar productions. + */ +@SmithyUnstableApi +public final class ParserUtils { + + private ParserUtils() {} + + /** + * Parses a Smithy number production into a string. + * + * 
-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?
+ * + * @param parser Parser to consume tokens from. + * @return Returns the parsed number lexeme. + */ + public static String parseNumber(SimpleParser parser) { + int startPosition = parser.position(); + char current = parser.peek(); + + if (current == '-') { + parser.skip(); + if (!isDigit(parser.peek())) { + throw parser.syntax(createInvalidString(parser, startPosition, "'-' must be followed by a digit")); + } + } + + parser.consumeUntilNoLongerMatches(ParserUtils::isDigit); + + // Consume decimals. + char peek = parser.peek(); + if (peek == '.') { + parser.skip(); + if (parser.consumeUntilNoLongerMatches(ParserUtils::isDigit) == 0) { + throw parser.syntax(createInvalidString(parser, startPosition, "'.' must be followed by a digit")); + } + } + + // Consume scientific notation. + peek = parser.peek(); + if (peek == 'e' || peek == 'E') { + parser.skip(); + peek = parser.peek(); + if (peek == '+' || peek == '-') { + parser.skip(); + } + if (parser.consumeUntilNoLongerMatches(ParserUtils::isDigit) == 0) { + throw parser.syntax( + createInvalidString(parser, startPosition, "'e', '+', and '-' must be followed by a digit")); + } + } + + return parser.sliceFrom(startPosition); + } + + private static String createInvalidString(SimpleParser parser, int startPosition, String message) { + String lexeme = parser.sliceFrom(startPosition); + return String.format("Invalid number '%s': %s", lexeme, message); + } + + /** + * Expects and returns a parsed Smithy identifier production. + * + * @param parser Parser to consume tokens from. + * @return Returns the parsed identifier. + */ + public static String parseIdentifier(SimpleParser parser) { + int start = parser.position(); + consumeIdentifier(parser); + return parser.sliceFrom(start); + } + + /** + * Expects and returns a parsed absolute Smithy Shape ID that + * does not include a member. + * + * @param parser Parser to consume tokens from. + * @return Returns the parsed Shape ID as a string. + */ + public static String parseRootShapeId(SimpleParser parser) { + int start = parser.position(); + consumeShapeId(parser, false); + return parser.sliceFrom(start); + } + + /** + * Expects and returns a parsed absolute Smithy Shape ID. + * + * @param parser Parser to consume tokens from. + * @return Returns the parsed Shape ID as a string. + */ + public static String parseShapeId(SimpleParser parser) { + int start = parser.position(); + consumeShapeId(parser, true); + return parser.sliceFrom(start); + } + + private static void consumeShapeId(SimpleParser parser, boolean parseMember) { + consumeNamespace(parser); + + if (parser.peek() == '#') { + parser.skip(); + consumeIdentifier(parser); + } + + if (parseMember && parser.peek() == '$') { + parser.skip(); + consumeIdentifier(parser); + } + } + + /** + * Expects and consumes a valid Smithy shape ID namespace. + * + * @param parser Parser to consume tokens from. + */ + public static void consumeNamespace(SimpleParser parser) { + consumeIdentifier(parser); + while (parser.peek() == '.') { + parser.skip(); + consumeIdentifier(parser); + } + } + + /** + * Expects and skips over a Smithy identifier production. + * + * 
+     *     identifier = (ALPHA / "_") *(ALPHA / DIGIT / "_")
+     *
+ * + * @param parser Parser to consume tokens from. + */ + public static void consumeIdentifier(SimpleParser parser) { + // (ALPHA / "_") + if (!isIdentifierStart(parser.peek())) { + throw parser.syntax("Expected a valid identifier character, but found '" + + parser.peekSingleCharForMessage() + '\''); + } + + // *(ALPHA / DIGIT / "_") + parser.consumeUntilNoLongerMatches(ParserUtils::isValidIdentifierCharacter); + } + + /** + * Returns true if the given character is allowed in an identifier. + * + * @param c Character to check. + * @return Returns true if the character is allowed in an identifier. + */ + public static boolean isValidIdentifierCharacter(char c) { + return isIdentifierStart(c) || isDigit(c); + } + + /** + * Returns true if the given character is allowed to start an identifier. + * + * @param c Character to check. + * @return Returns true if the character can start an identifier. + */ + public static boolean isIdentifierStart(char c) { + return c == '_' || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); + } + + /** + * Returns true if the given value is a digit 0-9. + * + * @param c Character to check. + * @return Returns true if the character is a digit. + */ + public static boolean isDigit(char c) { + return c >= '0' && c <= '9'; + } +} diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/pattern/SmithyPattern.java b/smithy-model/src/main/java/software/amazon/smithy/model/pattern/SmithyPattern.java index 1984e1a3d1c..03205d9ccc9 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/pattern/SmithyPattern.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/pattern/SmithyPattern.java @@ -25,6 +25,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import software.amazon.smithy.model.shapes.ShapeId; /** * Represents a contained pattern. @@ -188,9 +189,6 @@ public static final class Segment { public enum Type { LITERAL, LABEL, GREEDY_LABEL } - private static final java.util.regex.Pattern LABEL_PATTERN = java.util.regex.Pattern.compile( - "^[a-zA-Z0-9_]+$"); - private final String asString; private final String content; private final Type segmentType; @@ -220,10 +218,9 @@ private void checkForInvalidContents() { } } else if (content.isEmpty()) { throw new InvalidPatternException("Empty label declaration in pattern."); - } else if (!LABEL_PATTERN.matcher(content).matches()) { + } else if (!ShapeId.isValidIdentifier(content)) { throw new InvalidPatternException( - "Invalid label name in pattern: '" + content + "'. Labels must satisfy the " - + "following regular expression: " + LABEL_PATTERN.pattern()); + "Invalid label name in pattern: '" + content + "'. Labels must contain value identifiers."); } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/selector/AttributeValue.java b/smithy-model/src/main/java/software/amazon/smithy/model/selector/AttributeValue.java index 1d74d7ea6a9..8067178bf5b 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/selector/AttributeValue.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/selector/AttributeValue.java @@ -24,6 +24,8 @@ import software.amazon.smithy.model.shapes.ServiceShape; import software.amazon.smithy.model.shapes.Shape; import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.validation.linters.EmitEachSelectorValidator; +import software.amazon.smithy.utils.SimpleParser; /** * Selector attribute values are the data model of selectors. @@ -178,4 +180,16 @@ static AttributeValue node(Node node) { static AttributeValue projection(Collection values) { return new AttributeValueImpl.Projection(values); } + + /** + * Uses the given parser to parse a scoped attribute production. + * + * @param parser Parser to consume. + * @return Returns the list of validated scoped attribute path segments. + * @throws RuntimeException if the parser does not contain a valid scoped attribute production. + * @see EmitEachSelectorValidator for an example of how this is used. + */ + static List parseScopedAttribute(SimpleParser parser) { + return SelectorParser.parseScopedValuePath(parser); + } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/selector/AttributeValueImpl.java b/smithy-model/src/main/java/software/amazon/smithy/model/selector/AttributeValueImpl.java index e9b903b34b1..0b51d6f3e86 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/selector/AttributeValueImpl.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/selector/AttributeValueImpl.java @@ -377,7 +377,6 @@ public AttributeValue getProperty(String property) { */ static final class Traits implements AttributeValue { private final Shape shape; - private String messageString; Traits(Shape shape) { this.shape = Objects.requireNonNull(shape); @@ -388,20 +387,6 @@ public String toString() { return ""; } - @Override - public String toMessageString() { - String str = messageString; - if (str == null) { - // Returns a sorted, comma separated list of absolute trait shape IDs. - str = shape.getAllTraits().keySet().stream() - .map(ShapeId::toString) - .sorted() - .collect(Collectors.joining(", ")); - messageString = str; - } - return str; - } - @Override public AttributeValue getProperty(String property) { switch (property) { diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/selector/Selector.java b/smithy-model/src/main/java/software/amazon/smithy/model/selector/Selector.java index cf61049cc51..85e55cc66b4 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/selector/Selector.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/selector/Selector.java @@ -42,7 +42,7 @@ public interface Selector { * @return Returns the parsed {@link Selector}. */ static Selector parse(String expression) { - return Parser.parse(expression); + return SelectorParser.parse(expression); } /** diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/selector/Parser.java b/smithy-model/src/main/java/software/amazon/smithy/model/selector/SelectorParser.java similarity index 62% rename from smithy-model/src/main/java/software/amazon/smithy/model/selector/Parser.java rename to smithy-model/src/main/java/software/amazon/smithy/model/selector/SelectorParser.java index 0f251406230..5bf377c4bae 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/selector/Parser.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/selector/SelectorParser.java @@ -23,6 +23,7 @@ import java.util.Set; import java.util.function.BiFunction; import java.util.logging.Logger; +import software.amazon.smithy.model.loader.ParserUtils; import software.amazon.smithy.model.neighbor.RelationshipType; import software.amazon.smithy.model.shapes.CollectionShape; import software.amazon.smithy.model.shapes.NumberShape; @@ -31,13 +32,14 @@ import software.amazon.smithy.model.shapes.SimpleShape; import software.amazon.smithy.utils.ListUtils; import software.amazon.smithy.utils.SetUtils; +import software.amazon.smithy.utils.SimpleParser; /** * Parses a selector expression. */ -final class Parser { +final class SelectorParser extends SimpleParser { - private static final Logger LOGGER = Logger.getLogger(Parser.class.getName()); + private static final Logger LOGGER = Logger.getLogger(SelectorParser.class.getName()); private static final Set BREAK_TOKENS = SetUtils.of(',', ']', ')'); private static final Set REL_TYPES = new HashSet<>(); @@ -48,20 +50,15 @@ final class Parser { } } - private final String expression; - private final int length; - private int position = 0; - - private Parser(String selector) { - expression = selector; - length = expression.length(); + private SelectorParser(String selector) { + super(selector); } static Selector parse(String selector) { - return new WrappedSelector(selector, new Parser(selector).expression()); + return new WrappedSelector(selector, new SelectorParser(selector).parse()); } - private List expression() { + List parse() { return recursiveParse(); } @@ -75,7 +72,7 @@ private List recursiveParse() { ws(); // Parse until a break token: ",", "]", and ")". - while (position != length && !BREAK_TOKENS.contains(expression.charAt(position))) { + while (!eof() && !BREAK_TOKENS.contains(peek())) { selectors.add(createSelector()); // Always skip ws after calling createSelector. ws(); @@ -88,47 +85,47 @@ private InternalSelector createSelector() { ws(); // Require at least one selector. - switch (charPeek()) { + switch (peek()) { case ':': // function - position++; + skip(); return parseSelectorFunction(); case '[': // attribute - position++; - if (charPeek() == '@') { - position++; + skip(); + if (peek() == '@') { + skip(); return parseScopedAttribute(); } else { return parseAttribute(); } case '>': // forward undirected neighbor - position++; + skip(); return new ForwardNeighborSelector(ListUtils.of()); case '<': // reverse [un]directed neighbor - position++; - if (charPeek() == '-') { // reverse directed neighbor (<-[X, Y, Z]-) - position++; + skip(); + if (peek() == '-') { // reverse directed neighbor (<-[X, Y, Z]-) + skip(); expect('['); return parseSelectorDirectedReverseNeighbor(); } else { // reverse undirected neighbor (<) return new ReverseNeighborSelector(ListUtils.of()); } case '~': // ~> - position++; + skip(); expect('>'); return new RecursiveNeighborSelector(); case '-': // forward directed neighbor - position++; + skip(); expect('['); return parseSelectorForwardDirectedNeighbor(); case '*': // Any shape - position++; + skip(); return InternalSelector.IDENTITY; case '$': // variable - position++; + skip(); return parseVariable(); default: - if (validIdentifierStart(charPeek())) { - String identifier = parseIdentifier(); + if (ParserUtils.isIdentifierStart(peek())) { + String identifier = ParserUtils.parseIdentifier(this); switch (identifier) { case "number": return new ShapeTypeCategorySelector(NumberShape.class); @@ -141,72 +138,32 @@ private InternalSelector createSelector() { .orElseThrow(() -> syntax("Unknown shape type: " + identifier)); return new ShapeTypeSelector(shape); } + } else if (peek() == Character.MIN_VALUE) { + throw syntax("Unexpected selector EOF"); } else { - char c = charPeek(); - if (c == Character.MIN_VALUE) { - throw syntax("Unexpected selector EOF"); - } else { - throw syntax("Unexpected selector character: " + charPeek()); - } + throw syntax("Unexpected selector character: " + peek()); } } } - private void ws() { - for (; position < length; position++) { - char c = expression.charAt(position); - if (c != ' ' && c != '\t' && c != '\r' && c != '\n') { - break; - } - } - } - - private char charPeek() { - return position == length ? Character.MIN_VALUE : expression.charAt(position); - } - - private char expect(char token) { - if (charPeek() == token) { - position++; - return token; - } - - throw syntax("Expected: '" + token + "'"); - } - - private char expect(char... tokens) { - for (char token : tokens) { - if (charPeek() == token) { - position++; - return token; - } - } - - StringBuilder message = new StringBuilder("Expected one of the following tokens:"); - for (char c : tokens) { - message.append(' ').append('\'').append(c).append('\''); - } - - throw syntax(message.toString()); - } - - private SelectorSyntaxException syntax(String message) { - return new SelectorSyntaxException(message, expression, position); + @Override + public SelectorSyntaxException syntax(String message) { + return new SelectorSyntaxException(message, expression(), position(), line(), column()); } private InternalSelector parseVariable() { ws(); - if (charPeek() == '{') { - position++; + if (peek() == '{') { + skip(); ws(); - String variableName = parseIdentifier(); + String variableName = ParserUtils.parseIdentifier(this); ws(); expect('}'); return new VariableGetSelector(variableName); } - String name = parseIdentifier(); + String name = ParserUtils.parseIdentifier(this); ws(); expect('('); ws(); @@ -240,14 +197,14 @@ private List parseSelectorDirectedRelationships() { do { // Requires at least one relationship type. ws(); - next = parseIdentifier(); + next = ParserUtils.parseIdentifier(this); relationships.add(next); // Tolerate unknown relationships, but log a warning. if (!REL_TYPES.contains(next)) { LOGGER.warning(String.format( "Unknown relationship type '%s' found near %s. Expected one of: %s", - next, position - next.length(), REL_TYPES)); + next, position() - next.length(), REL_TYPES)); } ws(); @@ -258,14 +215,15 @@ private List parseSelectorDirectedRelationships() { } private InternalSelector parseSelectorFunction() { - int functionPosition = position; - String name = parseIdentifier(); + int functionPosition = position(); + String name = ParserUtils.parseIdentifier(this); List selectors = parseSelectorFunctionArgs(); switch (name) { case "not": if (selectors.size() != 1) { throw new SelectorSyntaxException( - "The :not function requires a single selector argument", expression, functionPosition); + "The :not function requires a single selector argument", + expression(), functionPosition, line(), column()); } return new NotSelector(selectors.get(0)); case "test": @@ -273,10 +231,11 @@ private InternalSelector parseSelectorFunction() { case "is": return IsSelector.of(selectors); case "each": - LOGGER.warning("The `:each` selector function has been renamed to `:is`: " + expression); + LOGGER.warning("The `:each` selector function has been renamed to `:is`: " + expression()); return IsSelector.of(selectors); default: - LOGGER.warning(String.format("Unknown function name `%s` found in selector: %s", name, expression)); + LOGGER.warning(String.format("Unknown function name `%s` found in selector: %s", + name, expression())); return (context, shape, next) -> true; } } @@ -315,9 +274,9 @@ private InternalSelector parseAttribute() { private boolean parseCaseInsensitiveToken() { ws(); - boolean insensitive = charPeek() == 'i'; + boolean insensitive = peek() == 'i'; if (insensitive) { - position++; + skip(); ws(); } return insensitive; @@ -350,16 +309,16 @@ private AttributeComparator parseComparator(char next) { comparator = AttributeComparator.EXISTS; break; case '>': - if (charPeek() == '=') { // >= - position++; + if (peek() == '=') { // >= + skip(); comparator = AttributeComparator.GTE; } else { // > comparator = AttributeComparator.GT; } break; case '<': - if (charPeek() == '=') { // <= - position++; + if (peek() == '=') { // <= + skip(); comparator = AttributeComparator.LTE; } else { // < comparator = AttributeComparator.LT; @@ -368,7 +327,7 @@ private AttributeComparator parseComparator(char next) { case '{': // projection comparators char nextSet = expect('<', '=', '!'); if (nextSet == '<') { - if (charPeek() == '<') { + if (peek() == '<') { expect('<'); // {<<} comparator = AttributeComparator.PROPER_SUBSET; } else { // {<} @@ -407,7 +366,7 @@ private List parseScopedAssertions() { assertions.add(parseScopedAssertion()); ws(); - while (charPeek() == '&') { + while (peek() == '&') { expect('&'); expect('&'); ws(); @@ -420,15 +379,15 @@ private List parseScopedAssertions() { private ScopedAttributeSelector.Assertion parseScopedAssertion() { ScopedAttributeSelector.ScopedFactory lhs = parseScopedValue(); - char next = charPeek(); - position++; + char next = peek(); + skip(); AttributeComparator comparator = parseComparator(next); List rhs = new ArrayList<>(); rhs.add(parseScopedValue()); - while (charPeek() == ',') { - position++; + while (peek() == ',') { + skip(); rhs.add(parseScopedValue()); } @@ -438,19 +397,12 @@ private ScopedAttributeSelector.Assertion parseScopedAssertion() { private ScopedAttributeSelector.ScopedFactory parseScopedValue() { ws(); - if (charPeek() == '@') { - position++; - expect('{'); - // parse at least one path segment, followed by any number of - // comma separated segments. - List path = new ArrayList<>(); - path.add(parseSelectorPathSegment()); - path.addAll(parseSelectorPath()); - expect('}'); + if (peek() == '@') { + List path = parseScopedValuePath(this); ws(); return value -> value.getPath(path); } else { - String parsedValue = parseAttributeValue(); + String parsedValue = parseAttributeValue(this); ws(); return value -> AttributeValue.literal(parsedValue); } @@ -460,75 +412,78 @@ private BiFunction>, AttributeValue> parseAttribut ws(); // '[@:' binds the current shape as the context. - if (charPeek() == ':') { + if (peek() == ':') { return AttributeValue::shape; } List path = new ArrayList<>(); // Parse the top-level namespace key. - path.add(parseIdentifier()); + path.add(ParserUtils.parseIdentifier(this)); // It is optionally followed by "|" delimited path keys. - path.addAll(parseSelectorPath()); + path.addAll(parseSelectorPath(this)); return (shape, variables) -> AttributeValue.shape(shape, variables).getPath(path); } - // Can be a shape_id, quoted string, number, or function key. - private List parseSelectorPath() { + private List parseAttributeValues() { + List result = new ArrayList<>(); + result.add(parseAttributeValue(this)); ws(); - if (charPeek() != '|') { - return Collections.emptyList(); + while (peek() == ',') { + skip(); + result.add(parseAttributeValue(this)); + ws(); } - List result = new ArrayList<>(); - do { - position++; // skip '|' - result.add(parseSelectorPathSegment()); - } while (charPeek() == '|'); - return result; } - private String parseSelectorPathSegment() { - ws(); + /* + * The following methods are static methods that aren't coupled to the + * SelectorParser, but rather a SimpleParser. This allows the AttributeValue#parseScopedAttribute + * method to accept a SimpleParser and then use this method to perform the actual + * parsing of a scoped attribute value. + * + * This is used to parse scoped attribute values from EmitEachSelector message + * templates. + */ + + static List parseScopedValuePath(SimpleParser parser) { + parser.expect('@'); + parser.expect('{'); + // parse at least one path segment, followed by any number of + // comma separated segments. + List path = new ArrayList<>(); + path.add(parseSelectorPathSegment(parser)); + path.addAll(parseSelectorPath(parser)); + parser.expect('}'); + return path; + } + + private static String parseSelectorPathSegment(SimpleParser parser) { + parser.ws(); // Handle function properties enclosed in "(" identifier ")". - if (charPeek() == '(') { - position++; - String propertyName = parseIdentifier(); - expect(')'); + if (parser.peek() == '(') { + parser.skip(); + String propertyName = ParserUtils.parseIdentifier(parser); + parser.expect(')'); return "(" + propertyName + ")"; } else { - return parseAttributeValue(); - } - } - - private List parseAttributeValues() { - List result = new ArrayList<>(); - result.add(parseAttributeValue()); - ws(); - - while (charPeek() == ',') { - position++; - result.add(parseAttributeValue()); - ws(); + return parseAttributeValue(parser); } - - return result; } - private String parseAttributeValue() { - ws(); + private static String parseAttributeValue(SimpleParser parser) { + parser.ws(); - switch (charPeek()) { + switch (parser.peek()) { case '\'': - return consumeInside('\''); + return consumeInside(parser, '\''); case '"': - return consumeInside('"'); + return consumeInside(parser, '"'); case '-': - position++; - return parseNumber(true); case '0': case '1': case '2': @@ -539,115 +494,43 @@ private String parseAttributeValue() { case '7': case '8': case '9': - return parseNumber(false); + return ParserUtils.parseNumber(parser); default: - return parseShapeId(); + return ParserUtils.parseRootShapeId(parser); } } - private String consumeInside(char c) { - int i = ++position; - while (i < length) { - if (expression.charAt(i) == c) { - String result = expression.substring(position, i); - position = i + 1; - ws(); + private static String consumeInside(SimpleParser parser, char c) { + parser.skip(); // skip the opening character. + int start = parser.position(); + + while (!parser.eof()) { + if (parser.peek() == c) { + String result = parser.sliceFrom(start); + parser.skip(); + parser.ws(); return result; } - i++; + parser.skip(); } - throw syntax("Expected " + c + " to close " + expression.substring(position)); - } - - private String parseIdentifier() { - StringBuilder builder = new StringBuilder(); - char current = charPeek(); - - // needs at least one character - if (!validIdentifierStart(current)) { - throw syntax("Invalid attribute start character `" + current + "`"); - } - - builder.append(current); - position++; - - current = charPeek(); - while (validIdentifierInner(current)) { - builder.append(current); - position++; - current = charPeek(); - } - - return builder.toString(); - } - - private boolean validIdentifierStart(char c) { - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; - } - private boolean validIdentifierInner(char c) { - return validIdentifierStart(c) || (c >= '0' && c <= '9'); + throw parser.syntax("Expected " + c + " to close " + parser.sliceFrom(start)); } - private String parseShapeId() { - StringBuilder builder = new StringBuilder(); - builder.append(parseIdentifier()); - - if (charPeek() == '.') { - do { - position++; - builder.append('.').append(parseIdentifier()); - } while (charPeek() == '.'); - // "." is only allowed in the namespace part, so it must be followed by a "#". - expect('#'); - builder.append('#').append(parseIdentifier()); - } else if (charPeek() == '#') { // a shape id with no namespace dots, but with a namespace. - position++; - builder.append('#').append(parseIdentifier()); - } - - // Note that members are not supported in this production! - return builder.toString(); - } - - private String parseNumber(boolean negative) { - StringBuilder result = new StringBuilder(); - - if (negative) { - result.append('-'); - } - - addSimpleNumberToBuilder(result); - - // Consume the fraction part. - if (charPeek() == '.') { - result.append('.'); - position++; - addSimpleNumberToBuilder(result); - } + // Can be a shape_id, quoted string, number, or function key. + private static List parseSelectorPath(SimpleParser parser) { + parser.ws(); - // Consume the exponent, if present. - if (charPeek() == 'e') { - result.append('e'); - position++; - if (charPeek() == '-' || charPeek() == '+') { - result.append(charPeek()); - position++; - } - addSimpleNumberToBuilder(result); + if (parser.peek() != '|') { + return Collections.emptyList(); } - return result.toString(); - } - - private void addSimpleNumberToBuilder(StringBuilder result) { - // Require at least one numeric value. - result.append(expect('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')); + List result = new ArrayList<>(); + do { + parser.skip(); // skip '|' + result.add(parseSelectorPathSegment(parser)); + } while (parser.peek() == '|'); - // Consume all numbers after the first number. - while (Character.isDigit(charPeek())) { - result.append(charPeek()); - position++; - } + return result; } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/selector/SelectorSyntaxException.java b/smithy-model/src/main/java/software/amazon/smithy/model/selector/SelectorSyntaxException.java index bdc0a1d4599..6599b15182f 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/selector/SelectorSyntaxException.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/selector/SelectorSyntaxException.java @@ -19,15 +19,17 @@ * Exception thrown when a selector expression is invalid. */ public final class SelectorSyntaxException extends RuntimeException { - SelectorSyntaxException(String message, String expression, int pos) { - super(createMessage(message, expression, pos)); + SelectorSyntaxException(String message, String expression, int pos, int line, int column) { + super(createMessage(message, expression, pos, line, column)); } - private static String createMessage(String message, String expression, int pos) { - String result = "Syntax error at character " + pos + " of " + expression.length(); + private static String createMessage(String message, String expression, int pos, int line, int column) { + String result = "Syntax error at line " + line + " column " + column; + if (pos <= expression.length()) { result += ", near `" + expression.substring(pos) + "`"; } + return result + ": " + message + "; expression: " + expression; } } diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/shapes/ShapeId.java b/smithy-model/src/main/java/software/amazon/smithy/model/shapes/ShapeId.java index a5ebcb6ff8a..745a0f0d93b 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/shapes/ShapeId.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/shapes/ShapeId.java @@ -21,6 +21,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; +import software.amazon.smithy.model.loader.ParserUtils; import software.amazon.smithy.model.loader.Prelude; /** @@ -96,12 +97,12 @@ public static boolean isValidNamespace(CharSequence namespace) { char c = namespace.charAt(position); if (start) { start = false; - if (!isValidIdentifierStart(c)) { + if (!ParserUtils.isIdentifierStart(c)) { return false; } } else if (c == '.') { start = true; - } else if (!isValidIdentifierAfterStart(c)) { + } else if (!ParserUtils.isValidIdentifierCharacter(c)) { return false; } } @@ -120,12 +121,12 @@ public static boolean isValidIdentifier(CharSequence identifier) { return false; } - if (!isValidIdentifierStart(identifier.charAt(0))) { + if (!ParserUtils.isIdentifierStart(identifier.charAt(0))) { return false; } for (int i = 1; i < identifier.length(); i++) { - if (!isValidIdentifierAfterStart(identifier.charAt(i))) { + if (!ParserUtils.isValidIdentifierCharacter(identifier.charAt(i))) { return false; } } @@ -133,14 +134,6 @@ public static boolean isValidIdentifier(CharSequence identifier) { return true; } - private static boolean isValidIdentifierStart(char c) { - return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'; - } - - private static boolean isValidIdentifierAfterStart(char c) { - return isValidIdentifierStart(c) || (c >= '0' && c <= '9'); - } - /** * Creates an absolute shape ID from parts of a shape ID. * diff --git a/smithy-model/src/main/java/software/amazon/smithy/model/validation/linters/EmitEachSelectorValidator.java b/smithy-model/src/main/java/software/amazon/smithy/model/validation/linters/EmitEachSelectorValidator.java index 31507e4a768..87eb6318bbb 100644 --- a/smithy-model/src/main/java/software/amazon/smithy/model/validation/linters/EmitEachSelectorValidator.java +++ b/smithy-model/src/main/java/software/amazon/smithy/model/validation/linters/EmitEachSelectorValidator.java @@ -15,15 +15,28 @@ package software.amazon.smithy.model.validation.linters; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; +import software.amazon.smithy.model.FromSourceLocation; import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.loader.ModelSyntaxException; import software.amazon.smithy.model.node.NodeMapper; +import software.amazon.smithy.model.selector.AttributeValue; import software.amazon.smithy.model.selector.Selector; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.ShapeId; import software.amazon.smithy.model.validation.AbstractValidator; import software.amazon.smithy.model.validation.ValidationEvent; import software.amazon.smithy.model.validation.ValidatorService; +import software.amazon.smithy.utils.OptionalUtils; +import software.amazon.smithy.utils.SimpleParser; /** * Emits a validation event for each shape that matches a selector. @@ -34,11 +47,16 @@ public final class EmitEachSelectorValidator extends AbstractValidator { * EmitEachSelector configuration settings. */ public static final class Config { + private Selector selector; + private ShapeId bindToTrait; + private MessageTemplate messageTemplate; /** - * Each shape that matches the given selector will emit a validation - * event. + * Gets the required selector that matches shapes. + * + *

Each shape that matches the given selector will emit a + * validation event. * * @return Selector to match on. */ @@ -49,6 +67,40 @@ public Selector getSelector() { public void setSelector(Selector selector) { this.selector = selector; } + + /** + * Gets the optional trait that each emitted event is bound to. + * + *

An event is only emitted for shapes that have this trait. + * + * @return Returns the trait to bind each event to. + */ + public ShapeId getBindToTrait() { + return bindToTrait; + } + + public void setBindToTrait(ShapeId bindToTrait) { + this.bindToTrait = bindToTrait; + } + + /** + * Gets the optional message template that can reference selector variables. + * + * @return Returns the message template. + */ + public String getMessageTemplate() { + return messageTemplate == null ? null : messageTemplate.toString(); + } + + /** + * Sets the optional message template for each emitted event. + * + * @param messageTemplate Message template to set. + * @throws ModelSyntaxException if the message template is invalid. + */ + public void setMessageTemplate(String messageTemplate) { + this.messageTemplate = new MessageTemplateParser(messageTemplate).parse(); + } } public static final class Provider extends ValidatorService.Provider { @@ -63,15 +115,159 @@ public Provider() { private final Config config; - private EmitEachSelectorValidator(Config config) { + /** + * @param config Validator configuration. + */ + public EmitEachSelectorValidator(Config config) { this.config = config; Objects.requireNonNull(config.selector, "selector is required"); } @Override public List validate(Model model) { + // Short-circuit the validation if the binding trait is never used. + if (config.bindToTrait != null && !model.getAppliedTraits().contains(config.getBindToTrait())) { + return Collections.emptyList(); + } else if (config.messageTemplate == null) { + return validateWithSimpleMessages(model); + } else { + return validateWithTemplate(model); + } + } + + private List validateWithSimpleMessages(Model model) { return config.getSelector().select(model).stream() - .map(shape -> danger(shape, "Selector capture matched selector: " + config.getSelector())) + .flatMap(shape -> OptionalUtils.stream(createSimpleEvent(shape))) .collect(Collectors.toList()); } + + private Optional createSimpleEvent(Shape shape) { + FromSourceLocation location = determineEventLocation(shape); + // Only create a validation event if the bound trait (if any) is present on the shape. + if (location == null) { + return Optional.empty(); + } + + return Optional.of(danger(shape, location, "Selector capture matched selector: " + config.getSelector())); + } + + // Determine where to bind the event. Only emit an event when `bindToTrait` is + // set if the shape actually has the trait. + private FromSourceLocation determineEventLocation(Shape shape) { + return config.bindToTrait == null + ? shape.getSourceLocation() + : shape.findTrait(config.bindToTrait).orElse(null); + } + + // Created events with a message template requires emitting matches + // into a BiConsumer and building up a mutated List of events. + private List validateWithTemplate(Model model) { + List events = new ArrayList<>(); + config.getSelector() + .runner() + .model(model) + .selectMatches((shape, vars) -> createTemplatedEvent(shape, vars).ifPresent(events::add)); + return events; + } + + private Optional createTemplatedEvent(Shape shape, Map> vars) { + FromSourceLocation location = determineEventLocation(shape); + // Only create a validation event if the bound trait (if any) is present on the shape. + if (location == null) { + return Optional.empty(); + } + + // Create an AttributeValue from the matched shape and context vars. + // This is then used to expand message template scoped attributes. + AttributeValue value = AttributeValue.shape(shape, vars); + return Optional.of(danger(shape, location, config.messageTemplate.expand(value))); + } + + /** + * A message template is made up of "parts", where each part is a function that accepts + * an {@link AttributeValue} and returns a String. + */ + private static final class MessageTemplate { + private final String template; + private final List> parts; + + private MessageTemplate(String template, List> parts) { + this.template = template; + this.parts = parts; + } + + /** + * Expands the MessageTemplate using the provided AttributeValue. + * + *

Each selector result shape along with the variables that were captured + * when the shape was matched are used to create an AttributeValue which + * is then passed to this message to create a validation event message. + * + * @param value The attribute value to pass to each part. + * @return Returns the expanded message template. + */ + private String expand(AttributeValue value) { + StringBuilder builder = new StringBuilder(); + for (Function part : parts) { + builder.append(part.apply(value)); + } + return builder.toString(); + } + + @Override + public String toString() { + return template; + } + } + + /** + * Parses message templates by slicing out literals and scoped attribute selectors. + * + *

Two "@" characters in a row (@@) are considered a single "@" because the + * first "@" acts as an escape character for the second. + */ + private static final class MessageTemplateParser extends SimpleParser { + private int mark = 0; + private final List> parts = new ArrayList<>(); + + private MessageTemplateParser(String expression) { + super(expression); + } + + MessageTemplate parse() { + while (!eof()) { + consumeUntilNoLongerMatches(c -> c != '@'); + // '@' followed by '@' is an escaped '@", so keep parsing + // the marked literal if that's the case. + if (peek(1) == '@') { + skip(); // consume the first @. + addLiteralPartIfNecessary(); + skip(); // skip the escaped @. + mark++; + } else if (!eof()) { + addLiteralPartIfNecessary(); + List path = AttributeValue.parseScopedAttribute(this); + parts.add(attributeValue -> attributeValue.getPath(path).toMessageString()); + mark = position(); + } + } + + addLiteralPartIfNecessary(); + return new MessageTemplate(expression(), parts); + } + + @Override + public RuntimeException syntax(String message) { + return new RuntimeException("Syntax error at line " + line() + " column " + column() + + " of EmitEachSelector message template: " + message); + } + + private void addLiteralPartIfNecessary() { + String slice = sliceFrom(mark); + if (!slice.isEmpty()) { + parts.add(ignoredAttribute -> slice); + } + mark = position(); + } + } } diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlModelLoaderTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlModelLoaderTest.java index 32649d0957d..9d187bbfd61 100644 --- a/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlModelLoaderTest.java +++ b/smithy-model/src/test/java/software/amazon/smithy/model/loader/IdlModelLoaderTest.java @@ -114,6 +114,6 @@ public void limitsRecursion() { Model.assembler().addUnparsedModel("/foo.smithy", nodeBuilder.toString()).assemble().unwrap(); }); - assertThat(e.getMessage(), containsString("Node value nesting too deep")); + assertThat(e.getMessage(), containsString("Parser exceeded the maximum allowed depth of")); } } diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/selector/AttributeValueTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/selector/AttributeValueTest.java index 5577cb0a90d..62f61633f2a 100644 --- a/smithy-model/src/test/java/software/amazon/smithy/model/selector/AttributeValueTest.java +++ b/smithy-model/src/test/java/software/amazon/smithy/model/selector/AttributeValueTest.java @@ -18,6 +18,7 @@ import software.amazon.smithy.utils.ListUtils; import software.amazon.smithy.utils.MapUtils; import software.amazon.smithy.utils.SetUtils; +import software.amazon.smithy.utils.SimpleParser; public class AttributeValueTest { @Test @@ -260,7 +261,7 @@ public void createsTraitValue() { AttributeValue service = AttributeValue.shape(serviceShape, MapUtils.of()).getProperty("trait"); assertThat(service.toString(), equalTo("")); - assertThat(service.toMessageString(), equalTo("smithy.api#documentation, smithy.api#tags")); + assertThat(service.toMessageString(), equalTo("")); assertThat(service.getPath(ListUtils.of("tags")).toMessageString(), equalTo("[\"hi\"]")); assertThat(service.getPath(ListUtils.of("smithy.api#tags")).toMessageString(), equalTo("[\"hi\"]")); assertThat(service.getPath(ListUtils.of("documentation")).toString(), equalTo("hi")); @@ -301,4 +302,11 @@ public void shapeProvidesVarAccess() { assertThat(attr.getPath(ListUtils.of("var", "a")).getProperty("id").toMessageString(), equalTo("[foo.baz#Foo2, foo.baz#Foo3]")); } + + @Test + public void canParseScopedSelectorFromParser() { + SimpleParser parser = new SimpleParser("@{var|bar|baz}"); + + assertThat(AttributeValue.parseScopedAttribute(parser), contains("var", "bar", "baz")); + } } diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/selector/SelectorTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/selector/SelectorTest.java index fabae800fbf..2e52928c254 100644 --- a/smithy-model/src/test/java/software/amazon/smithy/model/selector/SelectorTest.java +++ b/smithy-model/src/test/java/software/amazon/smithy/model/selector/SelectorTest.java @@ -238,7 +238,7 @@ public void selectsCustomTraits() { public void requiresValidAttribute() { Throwable thrown = Assertions.assertThrows(SelectorSyntaxException.class, () -> Selector.parse("[id=-]")); - assertThat(thrown.getMessage(), containsString("Syntax error at character 5 of 6, near `]`")); + assertThat(thrown.getMessage(), containsString("Syntax error at line 1 column 6, near `]`")); } @Test @@ -321,7 +321,7 @@ public void throwsOnInvalidComparator() { SelectorSyntaxException.class, () -> Selector.parse("[id%=100]")); - assertThat(e.getMessage(), containsString("Expected one of the following tokens")); + assertThat(e.getMessage(), containsString("Found '%', but expected one of the following tokens")); } @Test diff --git a/smithy-model/src/test/java/software/amazon/smithy/model/validation/linters/EmitEachSelectorValidatorTest.java b/smithy-model/src/test/java/software/amazon/smithy/model/validation/linters/EmitEachSelectorValidatorTest.java new file mode 100644 index 00000000000..63f6c2e2f89 --- /dev/null +++ b/smithy-model/src/test/java/software/amazon/smithy/model/validation/linters/EmitEachSelectorValidatorTest.java @@ -0,0 +1,157 @@ +package software.amazon.smithy.model.validation.linters; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.nullValue; + +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.selector.Selector; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.traits.DocumentationTrait; +import software.amazon.smithy.model.validation.ValidationEvent; + +public class EmitEachSelectorValidatorTest { + + @Test + public void messageTemplateCanBeCastToString() { + EmitEachSelectorValidator.Config config = new EmitEachSelectorValidator.Config(); + + assertThat(config.getMessageTemplate(), nullValue()); + + config.setMessageTemplate("Hi"); + + assertThat(config.getMessageTemplate(), equalTo("Hi")); + } + + @Test + public void expandsMessageTemplates() { + EmitEachSelectorValidator.Config config = new EmitEachSelectorValidator.Config(); + Model model = Model.builder() + .addShape(StringShape.builder() + .id(ShapeId.from("foo.bar#Baz")) + .addTrait(new DocumentationTrait("hello")) + .build()) + .build(); + config.setSelector(Selector.parse("$foo(*)")); + config.setMessageTemplate("before `@{trait|documentation}` after. ID: @{id}. Var: @{var|foo|id}."); + EmitEachSelectorValidator validator = new EmitEachSelectorValidator(config); + List events = validator.validate(model); + + assertThat(events.get(0).getMessage(), + equalTo("before `\"hello\"` after. ID: foo.bar#Baz. Var: [foo.bar#Baz].")); + } + + @Test + public void onlyEmitsEventsWhenShapeHasBoundTraitAndNoTemplate() { + EmitEachSelectorValidator.Config config = new EmitEachSelectorValidator.Config(); + Model model = Model.builder() + .addShape(StringShape.builder() + .id(ShapeId.from("foo.bar#A")) + .addTrait(new DocumentationTrait("hello")) + .build()) + .addShape(StringShape.builder().id(ShapeId.from("foo.bar#B")).build()) + .build(); + config.setSelector(Selector.parse("*")); + config.setBindToTrait(DocumentationTrait.ID); + EmitEachSelectorValidator validator = new EmitEachSelectorValidator(config); + List events = validator.validate(model); + + assertThat(events, hasSize(1)); + assertThat(events.get(0).getShapeId(), equalTo(Optional.of(ShapeId.from("foo.bar#A")))); + } + + @Test + public void onlyEmitsEventsWhenShapeHasBoundTraitAndHasTemplate() { + EmitEachSelectorValidator.Config config = new EmitEachSelectorValidator.Config(); + Model model = Model.builder() + .addShape(StringShape.builder() + .id(ShapeId.from("foo.bar#A")) + .addTrait(new DocumentationTrait("hello")) + .build()) + .addShape(StringShape.builder().id(ShapeId.from("foo.bar#B")).build()) + .build(); + config.setSelector(Selector.parse("*")); + config.setMessageTemplate("This is only set to test the necessary code path of using templates..."); + config.setBindToTrait(DocumentationTrait.ID); + EmitEachSelectorValidator validator = new EmitEachSelectorValidator(config); + List events = validator.validate(model); + + assertThat(events, hasSize(1)); + assertThat(events.get(0).getShapeId(), equalTo(Optional.of(ShapeId.from("foo.bar#A")))); + } + + @Test + public void skipsUsingTheActualValidatorIfNoTraitsUseTheBoundTrait() { + EmitEachSelectorValidator.Config config = new EmitEachSelectorValidator.Config(); + Model model = Model.builder() + .addShape(StringShape.builder().id(ShapeId.from("foo.bar#A")).build()) + .build(); + config.setSelector(Selector.parse("*")); + config.setBindToTrait(DocumentationTrait.ID); + EmitEachSelectorValidator validator = new EmitEachSelectorValidator(config); + + assertThat(validator.validate(model), empty()); + } + + @Test + public void handlesEscapesAtSymbols() { + EmitEachSelectorValidator.Config config = new EmitEachSelectorValidator.Config(); + config.setSelector(Selector.parse("string")); + config.setMessageTemplate("A@@@@B"); + EmitEachSelectorValidator validator = new EmitEachSelectorValidator(config); + Model model = Model.builder() + .addShape(StringShape.builder().id(ShapeId.from("foo.bar#Baz")).build()) + .build(); + List events = validator.validate(model); + + assertThat(events, hasSize(1)); + assertThat(events.get(0).getMessage(), equalTo("A@@B")); + } + + @Test + public void validatesMessageTemplateIsNotUnclosed() { + RuntimeException e = Assertions.assertThrows(RuntimeException.class, () -> { + EmitEachSelectorValidator.Config config = new EmitEachSelectorValidator.Config(); + config.setSelector(Selector.parse("string")); + config.setMessageTemplate("...hello @{"); + new EmitEachSelectorValidator(config); + }); + + assertThat(e.getMessage(), + containsString("Syntax error at line 1 column 12 of EmitEachSelector message template")); + } + + @Test + public void validatesMessageTemplateIsNotEmpty() { + RuntimeException e = Assertions.assertThrows(RuntimeException.class, () -> { + EmitEachSelectorValidator.Config config = new EmitEachSelectorValidator.Config(); + config.setSelector(Selector.parse("string")); + config.setMessageTemplate("@{}"); + new EmitEachSelectorValidator(config); + }); + + assertThat(e.getMessage(), + containsString("Syntax error at line 1 column 3 of EmitEachSelector message template")); + } + + @Test + public void validatesMessageTemplateWithTrailingPipe() { + RuntimeException e = Assertions.assertThrows(RuntimeException.class, () -> { + EmitEachSelectorValidator.Config config = new EmitEachSelectorValidator.Config(); + config.setSelector(Selector.parse("string")); + config.setMessageTemplate("@{var|}"); + new EmitEachSelectorValidator(config); + }); + + assertThat(e.getMessage(), + containsString("Syntax error at line 1 column 7 of EmitEachSelector message template")); + } +} diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/validators/linters/emit-each-selector-validator.errors b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/validators/linters/emit-each-selector-validator.errors index 62c1972bfa8..6ed101d8733 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/validators/linters/emit-each-selector-validator.errors +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/validators/linters/emit-each-selector-validator.errors @@ -96,33 +96,33 @@ [DANGER] other.ns#String: Selector capture matched selector: [id|name='String'] | shapeName [DANGER] other.ns#String: Selector capture matched selector: simpleType | simpleType [NOTE] other.ns#String: The string shape is not connected to from any service shape. | UnreferencedShape -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "": Unable to deserialize Node using fromNode method: Syntax error at character 0 of 0, near ``: Unexpected selector EOF; expression: | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "": Unable to deserialize Node using fromNode method: Syntax error at character 0 of 0, near ``: Unexpected selector EOF; expression: | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "!": Unable to deserialize Node using fromNode method: Syntax error at character 0 of 1, near `!`: Unexpected selector character: !; expression: ! | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "'foo'": Unable to deserialize Node using fromNode method: Syntax error at character 0 of 5, near `'foo'`: Unexpected selector character: '; expression: 'foo' | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "\"foo\"": Unable to deserialize Node using fromNode method: Syntax error at character 0 of 5, near `"foo"`: Unexpected selector character: "; expression: "foo" | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "invalid": Unable to deserialize Node using fromNode method: Syntax error at character 7 of 7, near ``: Unknown shape type: invalid; expression: invalid | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[]": Unable to deserialize Node using fromNode method: Syntax error at character 1 of 2, near `]`: Invalid attribute start character `]`; expression: [] | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[foo|]": Unable to deserialize Node using fromNode method: Syntax error at character 5 of 6, near `]`: Invalid attribute start character `]`; expression: [foo|] | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[|]": Unable to deserialize Node using fromNode method: Syntax error at character 1 of 3, near `|]`: Invalid attribute start character `|`; expression: [|] | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[a=]": Unable to deserialize Node using fromNode method: Syntax error at character 3 of 4, near `]`: Invalid attribute start character `]`; expression: [a=] | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[a=b": Unable to deserialize Node using fromNode method: Syntax error at character 4 of 4, near ``: Expected: ']'; expression: [a=b | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "string=b": Unable to deserialize Node using fromNode method: Syntax error at character 6 of 8, near `=b`: Unexpected selector character: =; expression: string=b | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[foo=']": Unable to deserialize Node using fromNode method: Syntax error at character 6 of 7, near `]`: Expected ' to close ]; expression: [foo='] | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[foo=\"]": Unable to deserialize Node using fromNode method: Syntax error at character 6 of 7, near `]`: Expected " to close ]; expression: [foo="] | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[foo==value]": Unable to deserialize Node using fromNode method: Syntax error at character 5 of 12, near `=value]`: Invalid attribute start character `=`; expression: [foo==value] | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[foo^foo]": Unable to deserialize Node using fromNode method: Syntax error at character 5 of 9, near `foo]`: Expected: '='; expression: [foo^foo] | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":is(:not(string) > list": Unable to deserialize Node using fromNode method: Syntax error at character 23 of 23, near ``: Expected one of the following tokens: ')' ','; expression: :is(:not(string) > list | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "foo -[]->": Unable to deserialize Node using fromNode method: Syntax error at character 3 of 9, near ` -[]->`: Unknown shape type: foo; expression: foo -[]-> | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "foo -[input]->": Unable to deserialize Node using fromNode method: Syntax error at character 3 of 14, near ` -[input]->`: Unknown shape type: foo; expression: foo -[input]-> | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":not": Unable to deserialize Node using fromNode method: Syntax error at character 4 of 4, near ``: Expected: '('; expression: :not | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":not(": Unable to deserialize Node using fromNode method: Syntax error at character 5 of 5, near ``: Unexpected selector EOF; expression: :not( | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":not()": Unable to deserialize Node using fromNode method: Syntax error at character 5 of 6, near `)`: Unexpected selector character: ); expression: :not() | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":not(string": Unable to deserialize Node using fromNode method: Syntax error at character 11 of 11, near ``: Expected one of the following tokens: ')' ','; expression: :not(string | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":is": Unable to deserialize Node using fromNode method: Syntax error at character 3 of 3, near ``: Expected: '('; expression: :is | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":is(": Unable to deserialize Node using fromNode method: Syntax error at character 4 of 4, near ``: Unexpected selector EOF; expression: :is( | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":nay()": Unable to deserialize Node using fromNode method: Syntax error at character 5 of 6, near `)`: Unexpected selector character: ); expression: :nay() | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":is(string": Unable to deserialize Node using fromNode method: Syntax error at character 10 of 10, near ``: Expected one of the following tokens: ')' ','; expression: :is(string | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":is(string, ": Unable to deserialize Node using fromNode method: Syntax error at character 12 of 12, near ``: Unexpected selector EOF; expression: :is(string, | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":is(string, )": Unable to deserialize Node using fromNode method: Syntax error at character 12 of 13, near `)`: Unexpected selector character: ); expression: :is(string, ) | Model -[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":is(string, :not())": Unable to deserialize Node using fromNode method: Syntax error at character 17 of 19, near `))`: Unexpected selector character: ); expression: :is(string, :not()) | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 1, near ``: Unexpected selector EOF; expression: | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 1, near ``: Unexpected selector EOF; expression: | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "!": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 1, near `!`: Unexpected selector character: !; expression: ! | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "'foo'": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 1, near `'foo'`: Unexpected selector character: '; expression: 'foo' | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "\"foo\"": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 1, near `"foo"`: Unexpected selector character: "; expression: "foo" | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "invalid": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 8, near ``: Unknown shape type: invalid; expression: invalid | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[]": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 2, near `]`: Expected a valid identifier character, but found ']'; expression: [] | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[foo|]": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 6, near `]`: Expected a valid identifier character, but found ']'; expression: [foo|] | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[|]": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 2, near `|]`: Expected a valid identifier character, but found '|'; expression: [|] | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[a=]": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 4, near `]`: Expected a valid identifier character, but found ']'; expression: [a=] | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[a=b": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 5, near ``: Expected: ']', but found '[EOF]'; expression: [a=b | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "string=b": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 7, near `=b`: Unexpected selector character: =; expression: string=b | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[foo=']": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 8, near ``: Expected ' to close ]; expression: [foo='] | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[foo=\"]": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 8, near ``: Expected " to close ]; expression: [foo="] | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[foo==value]": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 6, near `=value]`: Expected a valid identifier character, but found '='; expression: [foo==value] | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "[foo^foo]": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 6, near `foo]`: Expected: '=', but found 'f'; expression: [foo^foo] | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":is(:not(string) > list": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 24, near ``: Found '[EOF]', but expected one of the following tokens: ')' ','; expression: :is(:not(string) > list | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "foo -[]->": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 4, near ` -[]->`: Unknown shape type: foo; expression: foo -[]-> | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from "foo -[input]->": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 4, near ` -[input]->`: Unknown shape type: foo; expression: foo -[input]-> | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":not": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 5, near ``: Expected: '(', but found '[EOF]'; expression: :not | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":not(": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 6, near ``: Unexpected selector EOF; expression: :not( | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":not()": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 6, near `)`: Unexpected selector character: ); expression: :not() | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":not(string": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 12, near ``: Found '[EOF]', but expected one of the following tokens: ')' ','; expression: :not(string | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":is": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 4, near ``: Expected: '(', but found '[EOF]'; expression: :is | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":is(": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 5, near ``: Unexpected selector EOF; expression: :is( | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":nay()": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 6, near `)`: Unexpected selector character: ); expression: :nay() | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":is(string": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 11, near ``: Found '[EOF]', but expected one of the following tokens: ')' ','; expression: :is(string | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":is(string, ": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 13, near ``: Unexpected selector EOF; expression: :is(string, | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":is(string, )": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 13, near `)`: Unexpected selector character: ); expression: :is(string, ) | Model +[ERROR] -: Error creating `EmitEachSelector` validator: Deserialization error at (/selector): unable to create software.amazon.smithy.model.selector.Selector from ":is(string, :not())": Unable to deserialize Node using fromNode method: Syntax error at line 1 column 18, near `))`: Unexpected selector character: ); expression: :is(string, :not()) | Model diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply-requires-newline.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply-requires-newline.smithy index 915ee2df32c..0e936945728 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply-requires-newline.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/apply-requires-newline.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 29 near `string`: Expected a line break | Model +// Parse error at line 4, column 29 near `string`: Expected a line break, but found 's' | Model namespace com.foo string SomeShape apply SomeShape @deprecated string AnotherString diff --git a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/newline-after-shape.smithy b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/newline-after-shape.smithy index 1ebffa56ed0..bc72d43ba4d 100644 --- a/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/newline-after-shape.smithy +++ b/smithy-model/src/test/resources/software/amazon/smithy/model/loader/invalid/newline-after-shape.smithy @@ -1,4 +1,4 @@ -// Parse error at line 4, column 17 near `string`: Expected a line break | Model +// Parse error at line 4, column 17 near `string`: Expected a line break, but found 's' | Model namespace com.test string MyString string OtherString diff --git a/smithy-utils/src/main/java/software/amazon/smithy/utils/CodeFormatter.java b/smithy-utils/src/main/java/software/amazon/smithy/utils/CodeFormatter.java index 611bd82ac7b..82156ddeb2e 100644 --- a/smithy-utils/src/main/java/software/amazon/smithy/utils/CodeFormatter.java +++ b/smithy-utils/src/main/java/software/amazon/smithy/utils/CodeFormatter.java @@ -21,6 +21,9 @@ import java.util.function.BiFunction; import java.util.regex.Pattern; +/** + * TODO: Rewrite the formatter parser to use a custom {@link SimpleParser}. + */ final class CodeFormatter { private static final Pattern NAME_PATTERN = Pattern.compile("^[a-z]+[a-zA-Z0-9_.#$]*$"); private static final Set VALID_FORMATTER_CHARS = SetUtils.of( diff --git a/smithy-utils/src/main/java/software/amazon/smithy/utils/SimpleParser.java b/smithy-utils/src/main/java/software/amazon/smithy/utils/SimpleParser.java new file mode 100644 index 00000000000..a7049346d38 --- /dev/null +++ b/smithy-utils/src/main/java/software/amazon/smithy/utils/SimpleParser.java @@ -0,0 +1,365 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.utils; + +import java.util.Objects; +import java.util.function.Predicate; + +/** + * A simple expression parser that can be extended to implement parsers + * for small domain specific languages. + * + *

This parser consumes characters of an in-memory string while tracking + * the current 0-based position, 1-based line, and 1-based column. + * Expectations can be made on the parser to require specific characters, + * and when those expectations are not met, a syntax exception is thrown. + */ +public class SimpleParser { + + private final String expression; + private final int length; + private final int maxNestingLevel; + private int position = 0; + private int line = 1; + private int column = 1; + private int nestingLevel = 0; + + /** + * Creates a new SimpleParser and sets the expression to parse. + * + * @param expression Expression to parser. + */ + public SimpleParser(String expression) { + this(expression, 0); + } + + /** + * Creates a new SimpleParser and sets the expression to parse. + * + *

By default, no maximum parsing level is enforced. Setting the + * {@code maxParsingLevel} to 0 disables the enforcement of a + * maximum parsing level. + * + * @param expression Expression to parse that must not be null. + * @param maxNestingLevel The maximum allowed nesting level of the parser. + */ + public SimpleParser(String expression, int maxNestingLevel) { + this.expression = Objects.requireNonNull(expression, "expression must not be null"); + this.length = expression.length(); + + if (maxNestingLevel < 0) { + throw new IllegalArgumentException("maxNestingLevel must be >= 0"); + } + + this.maxNestingLevel = maxNestingLevel; + } + + /** + * Gets the expression being parsed. + * + * @return Returns the expression being parsed. + */ + public final String expression() { + return expression; + } + + /** + * Gets the current 0-based position of the parser. + * + * @return Returns the parser character position. + */ + public final int position() { + return position; + } + + /** + * Gets the current 1-based line number of the parser. + * + * @return Returns the current line number. + */ + public final int line() { + return line; + } + + /** + * Gets the current 1-based column number of the parser. + * + * @return Returns the current column number. + */ + public final int column() { + return column; + } + + /** + * Checks if the parser has reached the end of the expression. + * + * @return Returns true if the parser has reached the end. + */ + public final boolean eof() { + return position >= length; + } + + /** + * Returns the current character of the expression, but does not consume it. + * + * @return Returns the peeked character. + */ + public final char peek() { + return peek(0); + } + + /** + * Returns the current character of the expression + {@code offset} + * characters, but does not consume it. + * + *

If the end of the expression is reached or if the peeked offset is + * calculated to be less than 0, {@link Character#MIN_VALUE} is returned + * (that is, '\0'). + * + * @param offset The number of characters to peek ahead (positive or negative). + * @return Returns the peeked character. + */ + public final char peek(int offset) { + int target = position + offset; + if (target >= length || target < 0) { + return Character.MIN_VALUE; + } + + return expression.charAt(target); + } + + /** + * Expects that the next character is the given character and consumes it. + * + * @param token The character to expect. + * @return Returns the expected character. + */ + public final char expect(char token) { + if (peek() == token) { + skip(); + return token; + } + + throw syntax(String.format("Expected: '%s', but found '%s'", token, peekSingleCharForMessage())); + } + + /** + * Peeks the next character and returns [EOF] if the next character is past + * the end of the expression. + * + * @return Returns the peeked next character. + */ + public final String peekSingleCharForMessage() { + char peek = peek(); + return peek == Character.MIN_VALUE ? "[EOF]" : String.valueOf(peek); + } + + /** + * Expects that the next character is one of a fixed set of possible characters. + * + * @param tokens Characters to expect. + * @return Returns the consumed character. + + */ + public final char expect(char... tokens) { + for (char token : tokens) { + if (peek() == token) { + skip(); + return token; + } + } + + StringBuilder message = new StringBuilder("Found '") + .append(peekSingleCharForMessage()) + .append("', but expected one of the following tokens:"); + for (char c : tokens) { + message.append(' ').append('\'').append(c).append('\''); + } + + throw syntax(message.toString()); + } + + /** + * Creates a syntax error that adds some context to the given message. + * + * @param message Message for why the error occurred. + * @return Returns the created syntax error. + */ + public RuntimeException syntax(String message) { + return new RuntimeException("Syntax error at line " + line() + " column " + column() + ": " + message); + } + + /** + * Skip 0 or more whitespace characters (that is, ' ', '\t', '\r', and '\n'). + */ + public void ws() { + while (!eof()) { + char c = peek(); + if (!(c == ' ' || c == '\t' || c == '\r' || c == '\n')) { + break; + } else { + skip(); + } + } + } + + /** + * Skip 0 or more spaces (that is, ' ' and '\t'). + */ + public void sp() { + while (!eof()) { + char c = peek(); + if (!(c == ' ' || c == '\t')) { + break; + } + skip(); + } + } + + /** + * Skips spaces then expects that the next character is either the end of + * the expression or a line break (\n, \r\n, or \r). + * + * @throws RuntimeException if the next non-space character is not EOF or a line break. + */ + public void br() { + sp(); + + // EOF can also be considered a line break to end a file. + if (eof()) { + return; + } + + char c = peek(); + if (c == '\n' || c == '\r') { + skip(); + } else { + throw syntax("Expected a line break, but found '" + c + "'"); + } + } + + /** + * Skips a single character while tracking lines and columns. + */ + public void skip() { + if (eof()) { + return; + } + + switch (expression.charAt(position)) { + case '\r': + if (peek(1) == '\n') { + position++; + } + line++; + column = 1; + break; + case '\n': + line++; + column = 1; + break; + default: + column++; + } + + position++; + } + + /** + * Skips over the remaining characters on a line but does not consume + * the newline character (\n or \r\n). + * + *

This method will also terminate when the end of the expression + * is encountered. + * + *

This method is useful, for example, for skipping the text of a + * commented out line in an expression. If the contents of the skipped + * line are required, then store the current position before invoking + * this method using {@link #position()}, then call this method, then get + * the contents of the skipped characters using {@link #sliceFrom(int)}. + */ + public void consumeRemainingCharactersOnLine() { + consumeUntilNoLongerMatches(c -> c != '\n' && c != '\r'); + } + + /** + * Gets a slice of the expression starting from the given 0-based + * character position, read all the way through to the current + * position of the parser. + * + * @param start Position to slice from, ending at the current position. + * @return Returns the slice of the expression from {@code start} to {@link #position}. + */ + public final String sliceFrom(int start) { + return expression().substring(start, position); + } + + /** + * Reads a lexeme from the expression while the given {@code predicate} + * matches each peeked character. + * + * @param predicate Predicate that filters characters. + * @return Returns the consumed lexeme (or an empty string on no matches). + */ + public final int consumeUntilNoLongerMatches(Predicate predicate) { + int startPosition = position; + while (!eof()) { + char peekedChar = peek(); + if (!predicate.test(peekedChar)) { + break; + } + skip(); + } + + return position - startPosition; + } + + /** + * Increases the current nesting level of the parser. + * + *

This method can be manually invoked when parsing in order to + * prevent parsing too deeply using recursive descent parsers. + * + * @throws RuntimeException if the nesting level is deeper than the max allowed nesting. + */ + public final void increaseNestingLevel() { + nestingLevel++; + + if (maxNestingLevel > 0 && nestingLevel > maxNestingLevel) { + throw syntax("Parser exceeded the maximum allowed depth of " + maxNestingLevel); + } + } + + /** + * Decreases the current nesting level of the parser. + */ + public final void decreaseNestingLevel() { + nestingLevel--; + + if (nestingLevel < 0) { + throw syntax("Invalid parser state. Nesting level set to -1"); + } + } + + /** + * Gets the current 0-based nesting level of the parser. + * + * @return Returns the current nesting level. + */ + public int nestingLevel() { + return nestingLevel; + } +} diff --git a/smithy-utils/src/test/java/software/amazon/smithy/utils/OptionalUtilsTest.java b/smithy-utils/src/test/java/software/amazon/smithy/utils/OptionalUtilsTest.java index 331e9d7abe3..4983e1140d3 100644 --- a/smithy-utils/src/test/java/software/amazon/smithy/utils/OptionalUtilsTest.java +++ b/smithy-utils/src/test/java/software/amazon/smithy/utils/OptionalUtilsTest.java @@ -40,7 +40,6 @@ public void streamUsesValue() { @Test public void streamIsEmpty() { assertThat(OptionalUtils.stream(Optional.empty()).count(), equalTo(0L)); - } @Test diff --git a/smithy-utils/src/test/java/software/amazon/smithy/utils/SimpleParserTest.java b/smithy-utils/src/test/java/software/amazon/smithy/utils/SimpleParserTest.java new file mode 100644 index 00000000000..c474b0d30c6 --- /dev/null +++ b/smithy-utils/src/test/java/software/amazon/smithy/utils/SimpleParserTest.java @@ -0,0 +1,216 @@ +package software.amazon.smithy.utils; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class SimpleParserTest { + + @Test + public void simpleParserBasics() { + SimpleParser p = new SimpleParser("foo", 10); + + assertThat(p.expression(), equalTo("foo")); + assertThat(p.line(), equalTo(1)); + assertThat(p.column(), equalTo(1)); + assertThat(p.position(), equalTo(0)); + } + + @Test + public void parserValidatesMaxNestingLevelGreaterThanZero() { + Assertions.assertThrows(IllegalArgumentException.class, () -> new SimpleParser("foo", -100)); + } + + @Test + public void expectThrowsWhenNotMatching() { + RuntimeException e = Assertions.assertThrows(RuntimeException.class, () -> { + new SimpleParser("foo").expect('!'); + }); + + assertThat(e.getMessage(), equalTo("Syntax error at line 1 column 1: Expected: '!', but found 'f'")); + } + + @Test + public void expectThrowsWhenNotMatchingOneOfMany() { + RuntimeException e = Assertions.assertThrows(RuntimeException.class, () -> { + new SimpleParser("foo").expect('!', '?'); + }); + + assertThat(e.getMessage(), equalTo( + "Syntax error at line 1 column 1: Found 'f', but expected one of the following tokens: '!' '?'")); + } + + @Test + public void expectThrowsWhenNotMatchingOneOfManyAndDueToEof() { + RuntimeException e = Assertions.assertThrows(RuntimeException.class, () -> { + new SimpleParser("").expect('!', '?'); + }); + + assertThat(e.getMessage(), equalTo( + "Syntax error at line 1 column 1: Found '[EOF]', but expected one of the following tokens: '!' '?'")); + } + + @Test + public void tracksLinesAndColumnsAndSlices() { + SimpleParser p = new SimpleParser("foo\nbaz\r\nBam!\nHi"); + + p.consumeRemainingCharactersOnLine(); + assertThat(p.line(), equalTo(1)); + assertThat(p.column(), equalTo(4)); + assertThat(p.sliceFrom(0), equalTo("foo")); + p.skip(); + + int position = p.position(); + + p.consumeRemainingCharactersOnLine(); + assertThat(p.line(), equalTo(2)); + assertThat(p.column(), equalTo(4)); + assertThat(p.sliceFrom(position), equalTo("baz")); + p.skip(); + + position = p.position(); + + p.consumeRemainingCharactersOnLine(); + assertThat(p.line(), equalTo(3)); + assertThat(p.column(), equalTo(5)); + assertThat(p.sliceFrom(position), equalTo("Bam!")); + p.skip(); + + position = p.position(); + + p.consumeRemainingCharactersOnLine(); + assertThat(p.line(), equalTo(4)); + assertThat(p.column(), equalTo(3)); + assertThat(p.sliceFrom(position), equalTo("Hi")); + p.skip(); + + assertThat(p.eof(), equalTo(true)); + } + + @Test + public void expectConsumesAndDoesNotThrowWhenValid() { + SimpleParser p = new SimpleParser("hi"); + + assertThat(p.expect('h'), equalTo('h')); + assertThat(p.expect('i', '!'), equalTo('i')); + } + + @Test + public void canPeekForwardAndBehind() { + SimpleParser p = new SimpleParser("foo"); + + assertThat(p.peek(), equalTo('f')); + assertThat(p.peek(0), equalTo('f')); + assertThat(p.peek(1), equalTo('o')); + assertThat(p.peek(2), equalTo('o')); + assertThat(p.peek(3), equalTo(Character.MIN_VALUE)); + assertThat(p.peek(-1), equalTo(Character.MIN_VALUE)); + + p.expect('f'); + assertThat(p.peek(-1), equalTo('f')); + } + + @Test + public void assertsNestingNotTooDeep() { + SimpleParser p = new SimpleParser("foo", 2); + p.increaseNestingLevel(); + p.increaseNestingLevel(); + p.decreaseNestingLevel(); + p.increaseNestingLevel(); + + // Hits 3 at this point, so it throws. + Assertions.assertThrows(RuntimeException.class, p::increaseNestingLevel); + } + + @Test + public void doesNotValidateNestingLevelWhenItIs0() { + SimpleParser p = new SimpleParser("foo"); + + // These are all fine. + p.increaseNestingLevel(); + p.increaseNestingLevel(); + p.increaseNestingLevel(); + assertThat(p.nestingLevel(), equalTo(3)); + } + + @Test + public void validatesNestingLevelDoesNotGoBelow0() { + SimpleParser p = new SimpleParser("foo", 2); + + Assertions.assertThrows(RuntimeException.class, p::decreaseNestingLevel); + } + + @Test + public void skipsWhitespace() { + SimpleParser p = new SimpleParser(" \n\t\r hi"); + p.ws(); + + assertThat(p.position(), equalTo(6)); + assertThat(p.line(), equalTo(3)); + assertThat(p.column(), equalTo(3)); + p.expect('h'); + p.expect('i'); + + // Skipping ws at EOF is fine. + assertThat(p.eof(), is(true)); + p.ws(); + assertThat(p.eof(), is(true)); + } + + @Test + public void skipsSpaces() { + SimpleParser p = new SimpleParser(" \n\t\r hi"); + p.sp(); + + assertThat(p.position(), equalTo(1)); + assertThat(p.line(), equalTo(1)); + assertThat(p.column(), equalTo(2)); + p.expect('\n'); + p.sp(); // skips the tab! + assertThat(p.position(), equalTo(3)); + p.sp(); // no-op + p.expect('\r'); + p.sp(); + p.expect('h'); + p.sp(); + p.expect('i'); + + // Skipping sp at EOF is fine. + p.sp(); + } + + @Test + public void expectsNewlineAndSkipsSpaces() { + SimpleParser p = new SimpleParser(" \n.\r.\r\n."); + p.br(); + + assertThat(p.position(), equalTo(4)); + assertThat(p.line(), equalTo(2)); + assertThat(p.column(), equalTo(1)); + p.expect('.'); + p.br(); + p.expect('.'); + p.br(); + p.expect('.'); + } + + @Test + public void eofIsAcceptableNewline() { + SimpleParser p = new SimpleParser(""); + p.br(); + } + + @Test + public void throwsWhenNotNewline() { + RuntimeException e = Assertions.assertThrows(RuntimeException.class, () -> { + SimpleParser p = new SimpleParser("H"); + p.br(); + }); + + assertThat(e.getMessage(), containsString("Expected a line break, but found 'H'")); + } +}