fix(crawler): check for more strings

mendableai · Mar 5, 2025 · c22c87a · c22c87a
1 parent 4902d0a
commit c22c87a
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -179,7 +179,7 @@ export class WebCrawler {
 
         const isAllowed = this.ignoreRobotsTxt
           ? true
-          : (this.robots.isAllowed(link, "FireCrawlAgent") ?? true);
+          : ((this.robots.isAllowed(link, "FireCrawlAgent") || this.robots.isAllowed(link, "FirecrawlAgent")) ?? true);
         // Check if the link is disallowed by robots.txt
         if (!isAllowed) {
           this.logger.debug(`Link disallowed by robots.txt: ${link}`, {
@@ -453,7 +453,7 @@ export class WebCrawler {
     return ignoreRobotsTxt
       ? true
       : this.robots
-        ? (this.robots.isAllowed(url, "FireCrawlAgent") ?? true)
+        ? ((this.robots.isAllowed(url, "FireCrawlAgent") || this.robots.isAllowed(url, "FirecrawlAgent")) ?? true)
         : true;
   }