improve-url-extraction

Author: uriva

src/index.test.ts

@@ -251,9 +251,13 @@ hahev11@walla.com
 
 רוצה לדעת לפני כולם מה קורה?
 להצטרפות לקבוצות של מה קורה היום? >> makorehayom.info/48YR0Sw`,
-    ["http://shiri-livny.com/adamaretreat", "http://makorehayom.info/48YR0Sw"],
+    ["Shiri-livny.com/adamaretreat", "makorehayom.info/48YR0Sw"],
   ],
   ["", []],
+  [
+    "https://gigtickets.co.il/il//eid/101",
+    ["https://gigtickets.co.il/il//eid/101"],
+  ],
 ]);
 
 testUnaryFn("telegram handlers", telegramHandlesInText)([

src/index.ts

@@ -393,9 +393,24 @@ const removeEmails = replace(
   "",
 );
 
+const extractUrls = (text: string) => {
+  const urlRegex =
+    /(?:(?:https?|ftp):\/\/)?[\w.-]+(?:\.[\w.-]+)+[\w\-._~:/?#[\]@!$&'()*+,;=]*/g;
+  const urls = text.match(urlRegex) || [];
+  // Filter out things that look like version numbers or other unwanted patterns
+  const filteredUrls = urls.filter((url) => {
+    // Basic filter to exclude things that look like version numbers or paths beginning with numbers.
+    if (/^\d+\.\d+/.test(url) || /^\d+\/\//.test(url)) {
+      return false;
+    }
+    return true;
+  });
+  return filteredUrls;
+};
+
 export const urlsInText = (
   x: string,
-) => [...getUrls(removeEmails(x).replace(/\b[^\s]+.(jpg|png|jpeg)\b/g, ""))];
+) => extractUrls(removeEmails(x).replace(/\b[^\s]+.(jpg|png|jpeg)\b/g, ""));
 
 type Keywords = { keywords: string[]; antiKeywords?: string[] };