Port babel-parser changes from 2023-03-20 to 2023-06-15

src/parser/plugins/typescript.ts

@@ -9,6 +9,7 @@ import {
   nextTemplateToken,
   popTypeContext,
   pushTypeContext,
+  rescan_gt,
 } from "../tokenizer/index";
 import {ContextualKeyword} from "../tokenizer/keywords";
 import {TokenType, TokenType as tt} from "../tokenizer/types";
@@ -1153,11 +1154,26 @@ function tsParseTypeArgumentsWithPossibleBitshift(): void {
 function tsParseTypeArguments(): void {
   const oldIsType = pushTypeContext(0);
   expect(tt.lessThan);
-  while (!eat(tt.greaterThan) && !state.error) {
+  while (!match(tt.greaterThan) && !state.error) {
     tsParseType();
     eat(tt.comma);
   }
-  popTypeContext(oldIsType);
+  if (!oldIsType) {
+    // If the type arguments are present in an expression context, e.g.
+    // f<number>(), then the > sign should be tokenized as a non-type token.
+    // In particular, f(a < b, c >= d) should parse the >= as a single token,
+    // resulting in a syntax error and fallback to the non-type-args
+    // interpretation. In the success case, even though the > is tokenized as a
+    // non-type token, it still must be marked as a type token so that it is
+    // erased.
+    popTypeContext(oldIsType);
+    rescan_gt();
+    expect(tt.greaterThan);
+    state.tokens[state.tokens.length - 1].isType = true;
+  } else {
+    expect(tt.greaterThan);
+    popTypeContext(oldIsType);
+  }
 }
 
 export function tsIsDeclarationStart(): boolean {

src/parser/tokenizer/index.ts

@@ -571,15 +571,18 @@ function readToken_gt(): void {
 }
 
 /**
- * Called after `as` expressions in TS; we're switching from a type to a
- * non-type context, so a > token may actually be >= . This is needed because >=
- * must be tokenized as a > in a type context because of code like
- * `const x: Array<T>=[];`, but `a as T >= 1` is a code example where it must be
- * treated as >=.
+ * Reinterpret a possible > token when transitioning from a type to a non-type
+ * context.
  *
- * Notably, this only applies to >, not <. In a code snippet like `a as T <= 1`,
- * we must NOT tokenize as <, or else the type parser will start parsing a type
- * argument and fail.
+ * This comes up in two situations where >= needs to be treated as one token:
+ * - After an `as` expression, like in the code `a as T >= 1`.
+ * - In a type argument in an expression context, e.g. `f(a < b, c >= d)`, we
+ *   need to see the token as >= so that we get an error and backtrack to
+ *   normal expression parsing.
+ *
+ * Other situations require >= to be seen as two tokens, e.g.
+ * `const x: Array<T>=[];`, so it's important to treat > as its own token in
+ * typical type parsing situations.
  */
 export function rescan_gt(): void {
   if (state.type === tt.greaterThan) {

src/parser/traverser/statement.ts

@@ -223,6 +223,10 @@ function parseStatementContent(declaration: boolean): void {
       ) {
         parseVarStatement(true);
         return;
+      } else if (startsAwaitUsing()) {
+        expectContextual(ContextualKeyword._await);
+        parseVarStatement(true);
+        return;
       }
     default:
       // Do nothing.
@@ -255,6 +259,48 @@ function parseStatementContent(declaration: boolean): void {
   }
 }
 
+/**
+ * Determine if we're positioned at an `await using` declaration.
+ *
+ * Note that this can happen either in place of a regular variable declaration
+ * or in a loop body, and in both places, there are similar-looking cases where
+ * we need to return false.
+ *
+ * Examples returning true:
+ * await using foo = bar();
+ * for (await using a of b) {}
+ *
+ * Examples returning false:
+ * await using
+ * await using + 1
+ * await using instanceof T
+ * for (await using;;) {}
+ *
+ * For now, we early return if we don't see `await`, then do a simple
+ * backtracking-based lookahead for the `using` and identifier tokens. In the
+ * future, this could be optimized with a character-based approach.
+ */
+function startsAwaitUsing(): boolean {
+  if (!isContextual(ContextualKeyword._await)) {
+    return false;
+  }
+  const snapshot = state.snapshot();
+  // await
+  next();
+  if (!isContextual(ContextualKeyword._using) || hasPrecedingLineBreak()) {
+    state.restoreFromSnapshot(snapshot);
+    return false;
+  }
+  // using
+  next();
+  if (!match(tt.name) || hasPrecedingLineBreak()) {
+    state.restoreFromSnapshot(snapshot);
+    return false;
+  }
+  state.restoreFromSnapshot(snapshot);
+  return true;
+}
+
 export function parseDecorators(): void {
   while (match(tt.at)) {
     parseDecorator();
@@ -361,7 +407,11 @@ function parseAmbiguousForStatement(): void {
     return;
   }
 
-  if (match(tt._var) || match(tt._let) || match(tt._const) || isUsingInLoop()) {
+  const isAwaitUsing = startsAwaitUsing();
+  if (isAwaitUsing || match(tt._var) || match(tt._let) || match(tt._const) || isUsingInLoop()) {
+    if (isAwaitUsing) {
+      expectContextual(ContextualKeyword._await);
+    }
     next();
     parseVar(true, state.type !== tt._var);
     if (match(tt._in) || isContextual(ContextualKeyword._of)) {
@@ -1024,7 +1074,7 @@ function parseExportSpecifiersMaybe(): void {
 export function parseExportFrom(): void {
   if (eatContextual(ContextualKeyword._from)) {
     parseExprAtom();
-    maybeParseImportAssertions();
+    maybeParseImportAttributes();
   }
   semicolon();
 }
@@ -1192,7 +1242,7 @@ export function parseImport(): void {
     expectContextual(ContextualKeyword._from);
     parseExprAtom();
   }
-  maybeParseImportAssertions();
+  maybeParseImportAttributes();
   semicolon();
 }
 
@@ -1268,13 +1318,14 @@ function parseImportSpecifier(): void {
 }
 
 /**
- * Parse import assertions like `assert {type: "json"}`.
+ * Parse import attributes like `with {type: "json"}`, or the legacy form
+ * `assert {type: "json"}`.
  *
- * Import assertions technically have their own syntax, but are always parseable
+ * Import attributes technically have their own syntax, but are always parseable
  * as a plain JS object, so just do that for simplicity.
  */
-function maybeParseImportAssertions(): void {
-  if (isContextual(ContextualKeyword._assert) && !hasPrecedingLineBreak()) {
+function maybeParseImportAttributes(): void {
+  if (match(tt._with) || (isContextual(ContextualKeyword._assert) && !hasPrecedingLineBreak())) {
     next();
     parseObj(false, false);
   }

src/transformers/CJSImportTransformer.ts

@@ -11,7 +11,7 @@ import getDeclarationInfo, {
   EMPTY_DECLARATION_INFO,
 } from "../util/getDeclarationInfo";
 import getImportExportSpecifierInfo from "../util/getImportExportSpecifierInfo";
-import {removeMaybeImportAssertion} from "../util/removeMaybeImportAssertion";
+import {removeMaybeImportAttributes} from "../util/removeMaybeImportAttributes";
 import shouldElideDefaultExport from "../util/shouldElideDefaultExport";
 import type ReactHotLoaderTransformer from "./ReactHotLoaderTransformer";
 import type RootTransformer from "./RootTransformer";
@@ -150,7 +150,7 @@ export default class CJSImportTransformer extends Transformer {
       this.tokens.replaceTokenTrimmingLeftWhitespace(this.importProcessor.claimImportCode(path));
       this.tokens.appendCode(this.importProcessor.claimImportCode(path));
     }
-    removeMaybeImportAssertion(this.tokens);
+    removeMaybeImportAttributes(this.tokens);
     if (this.tokens.matches1(tt.semi)) {
       this.tokens.removeToken();
     }
@@ -354,7 +354,7 @@ export default class CJSImportTransformer extends Transformer {
       ) {
         this.tokens.removeToken();
         this.tokens.removeToken();
-        removeMaybeImportAssertion(this.tokens);
+        removeMaybeImportAttributes(this.tokens);
       }
       return true;
     } else {
@@ -829,7 +829,7 @@ export default class CJSImportTransformer extends Transformer {
       this.tokens.removeToken();
       const path = this.tokens.stringValue();
       this.tokens.replaceTokenTrimmingLeftWhitespace(this.importProcessor.claimImportCode(path));
-      removeMaybeImportAssertion(this.tokens);
+      removeMaybeImportAttributes(this.tokens);
     } else {
       // This is a normal named export, so use that.
       this.tokens.appendCode(exportStatements.join(" "));
@@ -847,7 +847,7 @@ export default class CJSImportTransformer extends Transformer {
     }
     const path = this.tokens.stringValue();
     this.tokens.replaceTokenTrimmingLeftWhitespace(this.importProcessor.claimImportCode(path));
-    removeMaybeImportAssertion(this.tokens);
+    removeMaybeImportAttributes(this.tokens);
     if (this.tokens.matches1(tt.semi)) {
       this.tokens.removeToken();
     }

src/transformers/ESMImportTransformer.ts

@@ -11,7 +11,7 @@ import getDeclarationInfo, {
 } from "../util/getDeclarationInfo";
 import getImportExportSpecifierInfo from "../util/getImportExportSpecifierInfo";
 import {getNonTypeIdentifiers} from "../util/getNonTypeIdentifiers";
-import {removeMaybeImportAssertion} from "../util/removeMaybeImportAssertion";
+import {removeMaybeImportAttributes} from "../util/removeMaybeImportAttributes";
 import shouldElideDefaultExport from "../util/shouldElideDefaultExport";
 import type ReactHotLoaderTransformer from "./ReactHotLoaderTransformer";
 import Transformer from "./Transformer";
@@ -118,7 +118,7 @@ export default class ESMImportTransformer extends Transformer {
       ) {
         this.tokens.removeToken();
         this.tokens.removeToken();
-        removeMaybeImportAssertion(this.tokens);
+        removeMaybeImportAttributes(this.tokens);
       }
       return true;
     }
@@ -162,7 +162,7 @@ export default class ESMImportTransformer extends Transformer {
         this.tokens.removeToken();
       }
       this.tokens.removeToken();
-      removeMaybeImportAssertion(this.tokens);
+      removeMaybeImportAttributes(this.tokens);
       if (this.tokens.matches1(tt.semi)) {
         this.tokens.removeToken();
       }

src/util/removeMaybeImportAttributes.ts

@@ -0,0 +1,22 @@
+import {ContextualKeyword} from "../parser/tokenizer/keywords";
+import {TokenType as tt} from "../parser/tokenizer/types";
+import type TokenProcessor from "../TokenProcessor";
+
+/**
+ * Starting at a potential `with` or (legacy) `assert` token, remove the import
+ * attributes if they exist.
+ */
+export function removeMaybeImportAttributes(tokens: TokenProcessor): void {
+  if (
+    tokens.matches2(tt._with, tt.braceL) ||
+    (tokens.matches2(tt.name, tt.braceL) && tokens.matchesContextual(ContextualKeyword._assert))
+  ) {
+    // assert
+    tokens.removeToken();
+    // {
+    tokens.removeToken();
+    tokens.removeBalancedCode();
+    // }
+    tokens.removeToken();
+  }
+}

test/imports-test.ts

@@ -327,7 +327,7 @@ return obj && obj.__esModule ? obj : { default: obj }; }
     );
   });
 
-  it("removes import assertions", () => {
+  it("removes legacy import assertions", () => {
     assertResult(
       `
       import DefaultName from 'module1' assert {type: "json"};
@@ -350,6 +350,29 @@ return obj && obj.__esModule ? obj : { default: obj }; }
     );
   });
 
+  it("removes import attributes", () => {
+    assertResult(
+      `
+      import DefaultName from 'module1' with {type: "json"};
+      import {namedName} from 'module2' with {type: "json"};
+      import "module3" with {type: "json"};
+      export * from "module4" with {type: "json"};
+      // Arbitrary expressions like these aren't actually allowed right now, but
+      // exercise the ability to detect matching braces.
+      import test from "module5" with {type: {foo: "test"}};
+    `,
+      `"use strict";${ESMODULE_PREFIX}${IMPORT_DEFAULT_PREFIX}${CREATE_STAR_EXPORT_PREFIX}
+      var _module1 = require('module1'); var _module12 = _interopRequireDefault(_module1);
+      var _module2 = require('module2');
+      require('module3');
+      var _module4 = require('module4'); _createStarExport(_module4);
+      // Arbitrary expressions like these aren't actually allowed right now, but
+      // exercise the ability to detect matching braces.
+      var _module5 = require('module5'); var _module52 = _interopRequireDefault(_module5);
+    `,
+    );
+  });
+
   it("allows an import statement with no import bindings", () => {
     assertResult(
       `

test/sucrase-test.ts

@@ -1780,4 +1780,54 @@ describe("sucrase", () => {
       {transforms: []},
     );
   });
+
+  it("correctly handles `await using`", () => {
+    assertResult(
+      `
+      async function foo() {
+        await using x = blah();
+      }
+    `,
+      `
+      async function foo() {
+        await using x = blah();
+      }
+    `,
+      {transforms: []},
+    );
+  });
+
+  it("correctly handles `await using` in a loop", () => {
+    assertResult(
+      `
+      async function foo() {
+        for (await using a of b) {}
+      }
+    `,
+      `
+      async function foo() {
+        for (await using a of b) {}
+      }
+    `,
+      {transforms: []},
+    );
+  });
+
+  it("is not confused by `await using` in other contexts", () => {
+    assertResult(
+      `
+      await using
+      await using instanceof Foo
+      await using + 1
+      for (await using;;) {}
+    `,
+      `
+      await using
+      await using instanceof Foo
+      await using + 1
+      for (await using;;) {}
+    `,
+      {transforms: []},
+    );
+  });
 });