feat(api): allow index with enhanced mode (firecrawl#3158)

Author: tomsideguide

apps/api/src/lib/native-logging.ts

@@ -3,7 +3,7 @@ import type { Logger } from "winston";
 const NATIVE_LOGS_SEPARATOR = "\n__native_logs__:";
 
 /** Matches the NativeLogEntry struct from Rust (@mendable/firecrawl-rs). */
-export interface NativeLogEntry {
+interface NativeLogEntry {
   level: string;
   target: string;
   message: string;

apps/api/src/scraper/scrapeURL/engines/index.ts

@@ -213,7 +213,7 @@ const engineOptions: {
       location: true,
       skipTlsVerification: true,
       useFastMode: true,
-      stealthProxy: false,
+      stealthProxy: true,
       branding: false,
       disableAdblock: true,
     },
@@ -270,7 +270,7 @@ const engineOptions: {
       mobile: true,
       skipTlsVerification: true,
       useFastMode: true,
-      stealthProxy: false,
+      stealthProxy: true,
       branding: false,
       disableAdblock: false,
     },
@@ -467,8 +467,7 @@ export function shouldUseIndex(meta: Meta) {
     meta.options.maxAge !== 0 &&
     (meta.options.headers === undefined ||
       Object.keys(meta.options.headers).length === 0) &&
-    (meta.options.actions === undefined || meta.options.actions.length === 0) &&
-    meta.options.proxy !== "stealth"
+    (meta.options.actions === undefined || meta.options.actions.length === 0)
   );
 }
 
@@ -561,7 +560,11 @@ export async function buildFallbackList(meta: Meta): Promise<
     }
   }
 
-  if (selectedEngines.some(x => engineOptions[x.engine].quality > 0)) {
+  if (
+    selectedEngines.some(
+      x => engineOptions[x.engine].quality > 0 && !x.engine.startsWith("index"),
+    )
+  ) {
     selectedEngines = selectedEngines.filter(
       x => engineOptions[x.engine].quality > 0,
     );
@@ -628,7 +631,10 @@ export async function scrapeURLWithEngine(
   });
 
   const featureFlags = new Set(meta.featureFlags);
-  if (engineOptions[engine].features.stealthProxy) {
+  if (
+    engineOptions[engine].features.stealthProxy &&
+    !engine.startsWith("index") // don't force stealth proxy for index
+  ) {
     featureFlags.add("stealthProxy");
   }
 

apps/api/src/scraper/scrapeURL/engines/index/index.ts

@@ -151,6 +151,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
           location_languages: meta.options.location?.languages ?? null,
           status: document.metadata.statusCode,
           is_precrawl: meta.internalOptions.isPreCrawl === true,
+          is_stealth: meta.featureFlags.has("stealthProxy"),
           wait_time_ms: meta.options.waitFor > 0 ? meta.options.waitFor : null,
           ...urlSplitsHash.slice(0, 10).reduce(
             (a, x, i) => ({
@@ -259,7 +260,7 @@ export async function scrapeURLWithIndex(
   const checkpoint1 = Date.now();
 
   const { data, error } = await index_supabase_service.rpc(
-    "index_get_recent_3",
+    "index_get_recent_4",
     {
       p_url_hash: urlHash,
       p_max_age_ms: maxAge,
@@ -275,6 +276,7 @@ export async function scrapeURLWithIndex(
           ? meta.options.location?.languages
           : null,
       p_wait_time_ms: meta.options.waitFor,
+      p_is_stealth: meta.featureFlags.has("stealthProxy"),
       p_min_age_ms: meta.options.minAge ?? null,
     },
   );