2992: Use unicode character classes to keep unicode letters and numbers

Author: steffenkleinle

shared/utils/__tests__/normalizeString.spec.ts

@@ -15,4 +15,15 @@ describe('normalizeString', () => {
     expect(normalizeString('Bergstraße')).toBe('bergstrasse')
     expect(normalizeString('äßoßsß')).toBe('assosssss')
   })
+
+  it('should keep characters in other alphabets', () => {
+    expect(normalizeString('اللغة')).toBe('اللغة')
+    expect(normalizeString('በጀርመን')).toBe('በጀርመን')
+    expect(normalizeString('त')).toBe('त')
+    expect(normalizeString('本页内容来自')).toBe('本页内容来自')
+  })
+
+  it('should keep numbers', () => {
+    expect(normalizeString('غ12te57ة')).toBe('غ12te57ة')
+  })
 })

shared/utils/normalizeString.ts

@@ -1,5 +1,4 @@
-// eslint-disable-next-line no-control-regex
-const nonAsciiRegex = /[^\x00-\x7F\xDF]/g
+const nonAsciiRegex = /[^\p{Letter}|\p{Number}]/gu
 
 const normalizeToAscii = (str: string): string => str.normalize('NFKD').replace(nonAsciiRegex, '').replace(/ß/g, 'ss')