fix(napi/parser): fix unicode comment panic (#9084)

- closes #9079

I'm still leaving an existing TODO for `module_record` and `errors` span not being converted as they won't cause an immediate issue, but I'll work on that shortly.

Author: hi-ogawa

crates/oxc_ast/src/utf8_to_utf16.rs

@@ -29,7 +29,7 @@ impl Utf8ToUtf16 {
     }
 
     /// Convert all spans in the AST to UTF-16.
-    pub fn convert(mut self, program: &mut Program<'_>) {
+    pub fn convert(&mut self, program: &mut Program<'_>) {
         self.build_table(program.source_text);
         // Skip if source is entirely ASCII
         if self.translations.len() == 1 {
@@ -81,7 +81,8 @@ impl Utf8ToUtf16 {
         span.end = self.convert_offset(span.end);
     }
 
-    fn convert_offset(&self, utf8_offset: u32) -> u32 {
+    /// Convert UTF-8 offset to UTF-16.
+    pub fn convert_offset(&self, utf8_offset: u32) -> u32 {
         // Find the first entry in table *after* the UTF-8 offset.
         // The difference we need to subtract is recorded in the entry prior to it.
         let index =

napi/parser/src/lib.rs

@@ -69,15 +69,9 @@ fn parse_with_return(filename: &str, source_text: String, options: &ParserOption
     let allocator = Allocator::default();
     let source_type = get_source_type(filename, options);
     let mut ret = parse(&allocator, source_type, &source_text, options);
-    if options.convert_span_utf16.unwrap_or(false) {
-        // TODO: fix spans in `module_record` and `errors`
-        Utf8ToUtf16::new().convert(&mut ret.program);
-    }
-    let program = serde_json::to_string(&ret.program).unwrap();
-
     let errors = ret.errors.into_iter().map(OxcError::from).collect::<Vec<_>>();
 
-    let comments = ret
+    let mut comments = ret
         .program
         .comments
         .iter()
@@ -93,6 +87,18 @@ fn parse_with_return(filename: &str, source_text: String, options: &ParserOption
         .collect::<Vec<Comment>>();
 
     let module = EcmaScriptModule::from(&ret.module_record);
+
+    if options.convert_span_utf16.unwrap_or(false) {
+        // TODO: fix spans in `module_record` and `errors`
+        let mut converter = Utf8ToUtf16::new();
+        converter.convert(&mut ret.program);
+        for comment in &mut comments {
+            comment.start = converter.convert_offset(comment.start);
+            comment.end = converter.convert_offset(comment.end);
+        }
+    }
+    let program = serde_json::to_string(&ret.program).unwrap();
+
     ParseResult { source_text, program, module, comments, errors }
 }
 

napi/parser/test/parse.test.ts

@@ -105,6 +105,22 @@ it('utf16 span', async () => {
     });
     expect(ret.program.end).toMatchInlineSnapshot(`4`);
   }
+  {
+    const code = `// ∞`;
+    const ret = await parseAsync('test.js', code, {
+      convertSpanUtf16: true,
+    });
+    expect(ret.comments).toMatchInlineSnapshot(`
+      [
+        {
+          "end": 4,
+          "start": 0,
+          "type": "Line",
+          "value": " ∞",
+        },
+      ]
+    `);
+  }
 });
 
 describe('error', () => {