ParallelScraperManager.cs

using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Web.WebView2.Core;
using Microsoft.Web.WebView2.WinForms;
using System.Windows.Forms;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

namespace WebToolsDataMonitor
{
    public class ParallelScraperManager
    {
        private readonly Form1 _parentForm;
        private readonly bool _runPagesInParallel;
        public event EventHandler<ScrapedDataEventArgs> DataScraped;

        public ParallelScraperManager(Form1 parentForm, bool runPagesInParallel)
        {
            _parentForm = parentForm;
            _runPagesInParallel = runPagesInParallel;
        }

        // --- LAYER A: TOOLS ROUTING CONFIGURATIONS ---

        public async Task RunAllToolsInParallelAsync(List<ToolConfig> tools, CancellationToken token)
        {
            var tasks = new List<Task>();
            Logger.Instance.Info("SCRAPER_MANAGER", $"Executing parallel execution routines across ({tools.Count}) tracking profiles.");

            foreach (var tool in tools)
            {
                tasks.Add(Task.Run(() => ProcessSingleToolEngineAsync(tool, token), token));
            }

            await Task.WhenAll(tasks);
            Logger.Instance.Info("SCRAPER_MANAGER", "All parallel asynchronous engine tasks complete.");
        }

        public async Task RunAllToolsInSequenceAsync(List<ToolConfig> tools, CancellationToken token)
        {
            Logger.Instance.Info("SCRAPER_MANAGER", $"Executing sequential execution routines across ({tools.Count}) tracking profiles.");

            foreach (var tool in tools)
            {
                token.ThrowIfCancellationRequested();
                await ProcessSingleToolEngineAsync(tool, token);
            }

            Logger.Instance.Info("SCRAPER_MANAGER", "All sequential engine tasks complete.");
        }

        // --- LAYER B: TARGET PROFILE ORCHESTRATION ---

        private async Task ProcessSingleToolEngineAsync(ToolConfig tool, CancellationToken token)
        {
            // Per-tool forcePageRunMode (tools/*.yaml) wins over the global "Execute Tool Pages in Parallel"
            // checkbox when set to a recognized value ("parallel" | "sequential"); otherwise fall back to it.
            bool runPagesInParallel = tool.ResolvedForcePagesInParallel ?? _runPagesInParallel;

            if (!string.IsNullOrWhiteSpace(tool.ForcePageRunMode) && tool.ResolvedForcePagesInParallel == null)
            {
                Logger.Instance.Warn(tool.ToolName, $"Unrecognized forcePageRunMode value '{tool.ForcePageRunMode}' (expected 'parallel' or 'sequential') - falling back to the global toggle.");
            }

            string runModeSource = tool.ResolvedForcePagesInParallel.HasValue ? $"forced via tool config ('{tool.ForcePageRunMode}')" : "global toggle";
            Logger.Instance.Info(tool.ToolName, $"Background runtime worker spin-up confirmed. Pages Parallel mode: {runPagesInParallel} [{runModeSource}]");

            // Create an isolated User Data Folder profile caching path unique to this tracking tool configuration profile
            string toolCachePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "CacheData", tool.ToolName);

            try
            {
                // 🔥 FIXED: Create ONE CoreWebView2Environment for the whole tool run and reuse the SAME
                // object for the master login worker and every parallel page worker. Pointing separate
                // environment objects at the same userDataFolder path is not equivalent to sharing a
                // browser session (each CreateAsync call can spin up its own browser host process and
                // race for the profile's lock/cookie store). Passing the identical environment instance
                // into every EnsureCoreWebView2Async call is what actually makes the workers behave like
                // tabs in one shared browser instance.
                CoreWebView2Environment environment = await CreateEnvironmentOnUiAsync(toolCachePath, tool.IgnoreSslCertError);

                if (runPagesInParallel)
                {
                    // 🔥 STRATEGY B: RUN PAGES IN PARALLEL
                    WebView2 masterView = null;

                    if (tool.BypassLogin)
                    {
                        Logger.Instance.Info(tool.ToolName, "bypassLogin=true - skipping master login handshake, going straight to page workers.");
                    }
                    else
                    {
                        // Phase 1: Initialize a single session-builder browser instance context to populate the authentication cache disk structures
                        Logger.Instance.Info(tool.ToolName, "Spawning central master initialization browser sequence context to anchor session state...");
                        masterView = await ExecuteMasterLoginSequenceAsync(tool, environment, token);

                        if (masterView == null)
                        {
                            const string msg = "Central master initialization authentication handshake routine faulted. Abandoning parallel extraction routines tracking.";
                            Logger.Instance.Error(tool.ToolName, msg);
                            OnLogTraced(tool.ToolName, null, $"[ERROR] {msg}");
                            return;
                        }
                    }

                    try
                    {
                        // Phase 2: Fire concurrent extraction task paths across all assigned pages simultaneously using the shared browser environment/session
                        Logger.Instance.Info(tool.ToolName, $"Launching parallel execution routines across ({tool.ScrapPages.Count}) target page arrays.");
                        var pageTasks = new List<Task>();

                        int trackerIndex = 0;
                        foreach (var page in tool.ScrapPages)
                        {
                            trackerIndex++;
                            int instanceId = trackerIndex;
                            pageTasks.Add(Task.Run(() => ProcessSinglePageScrapContextAsync(tool, page, environment, instanceId, token), token));
                        }

                        await Task.WhenAll(pageTasks);
                    }
                    finally
                    {
                        // Candidate fix under test: the master login tab is now kept open for the entire
                        // parallel phase and only torn down here, after every page worker has finished,
                        // instead of being disposed right after login. Not yet confirmed to fix the
                        // extraction failures - being tried empirically per user direction.
                        // masterView is null when bypassLogin=true (no master worker was ever created) -
                        // TeardownBrowserContextFromUiAsync already no-ops on a null view.
                        await TeardownBrowserContextFromUiAsync(masterView);
                    }
                }
                else
                {
                    // 🔒 STRATEGY A: RUN PAGES IN SEQUENCE (Original Single Browser Context Footprint)
                    await ProcessPagesSequentiallyAsync(tool, environment, token);
                }
            }
            catch (Exception ex)
            {
                Logger.Instance.Critical(tool.ToolName, $"Fatal engine collapse exception captured: {ex.Message}");
                OnLogTraced(tool.ToolName, null, $"[ERROR] Fatal engine failure: {ex.Message}");
            }
            finally
            {
                // Wipe this tool's own WebView2 profile folder once every worker for this tool has finished
                // (success or failure) - forces a fresh login next run instead of silently reusing a
                // persisted session cookie. A tool whose site issues persistent (non-session) cookies would
                // otherwise stay logged in indefinitely across continuous-run cycles, and since the login
                // form then genuinely isn't present on an already-authenticated page, that was being
                // misread as a login failure and aborting the whole tool every cycle after the first.
                //
                // Per-tool disableCache (tools/*.yaml) can opt a tool OUT of this: some devices are slow/weak
                // enough that a fully cold profile (no cached JS/HTML, not just no cookies) can't finish
                // bootstrapping its SPA within the login retry window, which needs the cache preserved to
                // stay fast across runs instead.
                bool shouldClearCache = tool.DisableCache ?? AppConfig.Instance.Data.DefaultDisableCache;

                if (shouldClearCache)
                {
                    await ClearToolCacheAsync(tool, toolCachePath);
                }
                else
                {
                    Logger.Instance.Debug(tool.ToolName, $"disableCache=false for this tool - preserving cache folder: {toolCachePath}");
                }
            }
        }

        // Deletes ONLY this tool's own CacheData/<ToolName> folder - never touches sibling tools' folders,
        // since toolCachePath is already scoped per-tool by the caller.
        private async Task ClearToolCacheAsync(ToolConfig tool, string toolCachePath)
        {
            if (!Directory.Exists(toolCachePath)) return;

            const int maxAttempts = 3;

            for (int attempt = 1; attempt <= maxAttempts; attempt++)
            {
                try
                {
                    Directory.Delete(toolCachePath, recursive: true);
                    Logger.Instance.Debug(tool.ToolName, $"Cleared tool cache folder after run: {toolCachePath}");
                    return;
                }
                catch (Exception ex) when (attempt < maxAttempts && (ex is IOException || ex is UnauthorizedAccessException))
                {
                    // The underlying WebView2 browser process may not have fully released its file locks
                    // yet right after the last controller was disposed - give it a brief moment and retry.
                    await Task.Delay(500);
                }
                catch (Exception ex)
                {
                    Logger.Instance.Warn(tool.ToolName, $"Failed to clear tool cache folder '{toolCachePath}': {ex.Message}");
                    return;
                }
            }
        }

        // --- LAYER C: STRATEGY A CODE ROUTINE (SEQUENTIAL EXTRACTION LOOP) ---

        private async Task ProcessPagesSequentiallyAsync(ToolConfig tool, CoreWebView2Environment environment, CancellationToken token)
        {
            WebView2 workerView = null;
            try
            {
                workerView = await InstantiateBrowserContextOnUiAsync(environment, tool.ToolName, tool.IgnoreSslCertError);

                if (tool.BypassLogin)
                {
                    Logger.Instance.Info(tool.ToolName, "bypassLogin=true - skipping login handshake, going straight to page scripts.");
                }
                else
                {
                    bool loginSuccess = await AttemptLoginWithRetryAsync(tool, workerView, token);
                    if (!loginSuccess)
                    {
                        const string msg = "Sequential login handshake failed. Abandoning page extraction for this tool.";
                        Logger.Instance.Error(tool.ToolName, msg);
                        OnLogTraced(tool.ToolName, null, $"[ERROR] {msg}");
                        return;
                    }

                    Logger.Instance.Debug(tool.ToolName, "Holding for session settlement...");
                    await Task.Delay(4000, token);
                }

                int pageCounter = 0;
                foreach (var page in tool.ScrapPages)
                {
                    pageCounter++;
                    token.ThrowIfCancellationRequested();

                    _parentForm.Invoke((MethodInvoker)(() => workerView.CoreWebView2.Navigate(page.DestinationUrl)));
                    await WaitForNavigationCompleteAsync(workerView, token);

                    await RunPageScriptsAsync(tool, page, workerView, "[SEQ]", token);
                }
            }
            finally
            {
                await TeardownBrowserContextFromUiAsync(workerView);
            }
        }

        // --- LAYER D: STRATEGY B CODE ROUTINES (PARALLEL CONTAINER LOGIC) ---

        // Returns the live master WebView2 on success (caller now owns disposing it - kept open for the
        // whole parallel phase instead of being torn down immediately after login), or null on failure
        // (torn down internally in that case, since the caller never gets a reference to it).
        private async Task<WebView2> ExecuteMasterLoginSequenceAsync(ToolConfig tool, CoreWebView2Environment environment, CancellationToken token)
        {
            WebView2 masterView = null;
            try
            {
                masterView = await InstantiateBrowserContextOnUiAsync(environment, tool.ToolName, tool.IgnoreSslCertError);

                bool loginSuccess = await AttemptLoginWithRetryAsync(tool, masterView, token);
                if (!loginSuccess)
                {
                    await TeardownBrowserContextFromUiAsync(masterView);
                    return null;
                }

                // Let the login POST/redirect settle so the server-issued session cookie is fully applied
                await Task.Delay(5000, token);
                return masterView;
            }
            catch (Exception ex)
            {
                Logger.Instance.Error(tool.ToolName, $"Master initialization login routine failed: {ex.Message}");
                await TeardownBrowserContextFromUiAsync(masterView);
                return null;
            }
        }

        private async Task ProcessSinglePageScrapContextAsync(ToolConfig tool, ScrapPageConfig page, CoreWebView2Environment environment, int subId, CancellationToken token)
        {
            Logger.Instance.Info(tool.ToolName, $"[PARALLEL WORKER #{subId}] Initializing isolated view context channel for URL target mapping target...");
            WebView2 subPageView = null;

            try
            {
                token.ThrowIfCancellationRequested();

                // Mounts a new WebView2 controller from the SAME shared CoreWebView2Environment used for the
                // master login worker — this is what actually makes it "another tab" of the same browser/session
                // rather than a separate process that merely points at the same folder on disk. One login only
                // (the master, kept alive for the whole parallel phase - see ExecuteMasterLoginSequenceAsync);
                // workers rely purely on the shared environment/cookies, exactly like opening more tabs in the
                // same browser window after already being logged in. No per-worker login - that was tried and
                // reverted: it's both unnecessary (this tab is already authenticated via the shared session,
                // so the login form genuinely isn't there to find) and actively harmful (navigating this tab to
                // the bare root first, then to the same origin+path with only the hash differing, turns the
                // second navigation into a same-document/fragment navigation that never reliably fires
                // NavigationCompleted, hanging WaitForNavigationCompleteAsync forever).
                subPageView = await InstantiateBrowserContextOnUiAsync(environment, tool.ToolName, tool.IgnoreSslCertError);

                Logger.Instance.Info(tool.ToolName, $"[PARALLEL WORKER #{subId}] Navigating straight toward destination target point: {page.DestinationUrl}");
                _parentForm.Invoke((MethodInvoker)(() => subPageView.CoreWebView2.Navigate(page.DestinationUrl)));
                await WaitForNavigationCompleteAsync(subPageView, token);

                await RunPageScriptsAsync(tool, page, subPageView, $"[PARALLEL WORKER #{subId}]", token);
            }
            catch (Exception ex)
            {
                Logger.Instance.Error(tool.ToolName, $"[PARALLEL WORKER #{subId}] Fault encountered processing page array operations logic: {ex.Message}");
                OnLogTraced(tool.ToolName, null, $"[ERROR] [PARALLEL WORKER #{subId}] Page processing failed: {ex.Message}");
            }
            finally
            {
                await TeardownBrowserContextFromUiAsync(subPageView);
            }
        }

        // --- LAYER E: SUB-ROUTINES INFRASTRUCTURE ASSIGNMENTS ---

        // Debug-only diagnostic aid: dumps the page's live DOM (outerHTML, post-render) to disk right
        // before a script runs, so a failed extraction can be inspected after the fact instead of guessing
        // at delay/timing values blind. Gated on Logger.Instance.IsDebugEnabled so it never touches disk or
        // the WebView2 tab in normal INFO-level operation.
        private async Task CaptureDebugSnapshotAsync(string toolName, string scriptName, WebView2 view)
        {
            if (!Logger.Instance.IsDebugEnabled) return;

            try
            {
                string rawHtmlResult = await ControlExtensions.InvokeAsync<string>(_parentForm, () =>
                    view.CoreWebView2.ExecuteScriptAsync("document.documentElement.outerHTML")
                );

                string html = UnwrapJsonEncodedString(rawHtmlResult);

                string snapshotsDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "snapshots");
                Directory.CreateDirectory(snapshotsDir);

                string fileName = $"{SanitizeForFileName(toolName)}_{SanitizeForFileName(scriptName)}_{DateTime.Now:yyyyMMdd_HHmmss_fff}.html";
                string filePath = Path.Combine(snapshotsDir, fileName);

                File.WriteAllText(filePath, html);

                Logger.Instance.Debug(toolName, $"Snapshot for script '{scriptName}' created: {fileName}");
            }
            catch (Exception ex)
            {
                Logger.Instance.Warn(toolName, $"Failed to capture DOM snapshot for script '{scriptName}': {ex.Message}");
            }
        }

        private static string SanitizeForFileName(string value)
        {
            foreach (char c in Path.GetInvalidFileNameChars())
                value = value.Replace(c, '_');
            return value;
        }

        // ExecuteScriptAsync/ExecuteScriptWithArgsAsync return their result JSON-encoded (a quoted,
        // escaped JSON string) - this unwraps it back to the real string value.
        private static string UnwrapJsonEncodedString(string rawResult)
        {
            if (rawResult == null) return null;

            try
            {
                return JsonConvert.DeserializeObject<string>(rawResult) ?? rawResult;
            }
            catch (JsonException)
            {
                return rawResult;
            }
        }

        // How a script's execution failure affects the rest of its page's run - resolved from
        // options.FailureLevel / optionsOverride.FailureLevel, same base/override precedence as
        // DelaySeconds and RunIndex.
        private enum ScriptFailureLevel { Common, Important, Critical }

        // "important" scripts get this many total attempts (1 initial + retries) before giving up, waiting
        // the script's own resolved DelaySeconds between attempts. Kept as a fixed constant rather than a
        // new YAML knob - the ask was "retry a few times", not a fully tunable retry count.
        private const int ImportantFailureMaxAttempts = 3;

        // Runs every script on a page against the given (already-navigated) view. Scripts are grouped by
        // their resolved RunIndex and groups execute in ascending index order; scripts sharing the same
        // RunIndex run concurrently against each other (default RunIndex is 0, so scripts with no explicit
        // index configured all run in parallel by default - opt into strict ordering by giving scripts
        // increasing indices). Running scripts "in parallel" here means issuing their ExecuteScriptAsync
        // calls without waiting for one to finish before starting the next - they still execute against
        // the same single-threaded page/JS context, so this only saves the accumulated per-script
        // DelaySeconds/round-trip wait time, it doesn't create real thread-level concurrency inside the page.
        // If any script in a group is marked FailureLevel: critical and ultimately fails, no further
        // RunIndex groups run for this page.
        private async Task RunPageScriptsAsync(ToolConfig tool, ScrapPageConfig page, WebView2 view, string logPrefix, CancellationToken token)
        {
            var groups = new SortedDictionary<int, List<TargetScriptExecution>>();

            foreach (var scriptExec in page.Scripts)
            {
                var globalScriptData = ScriptsRepository.Instance.GetScript(scriptExec.Name);
                if (globalScriptData == null) continue;

                int runIndex = ResolveRunIndex(globalScriptData, scriptExec);
                if (!groups.TryGetValue(runIndex, out var group))
                {
                    group = new List<TargetScriptExecution>();
                    groups[runIndex] = group;
                }
                group.Add(scriptExec);
            }

            foreach (var group in groups)
            {
                token.ThrowIfCancellationRequested();

                Logger.Instance.Debug(tool.ToolName, $"{logPrefix} Running runIndex={group.Key} group ({group.Value.Count} script(s)) in parallel...");

                var groupTasks = new List<Task<bool>>();
                foreach (var scriptExec in group.Value)
                {
                    groupTasks.Add(RunSingleScriptAsync(tool, page.DestinationUrl, scriptExec, view, logPrefix, token));
                }

                bool[] abortSignals = await Task.WhenAll(groupTasks);

                if (abortSignals.Any(shouldAbort => shouldAbort))
                {
                    string msg = $"{logPrefix} A critical script failed in runIndex={group.Key} - skipping remaining RunIndex groups for this page.";
                    Logger.Instance.Error(tool.ToolName, msg);
                    OnLogTraced(tool.ToolName, null, $"[CRITICAL] {msg}");
                    break;
                }
            }
        }

        // 🔥 FIXED: this had no try/catch of its own - an exception here (WebView2 fault, script timeout,
        // navigation drop, etc.) used to propagate up and abort the ENTIRE page/tool run, and was only ever
        // reported to the file logger, never to Form1's UI. Now a single script's failure is isolated and
        // reported clearly in Form1's status window via OnLogTraced, not just scraper_execution.log.
        // Returns true if the caller should abort remaining RunIndex groups for this page (only ever true
        // for a "critical" script that failed after its attempt(s)); false on success or a non-critical failure.
        private async Task<bool> RunSingleScriptAsync(ToolConfig tool, string pageUrl, TargetScriptExecution scriptExec, WebView2 view, string logPrefix, CancellationToken token)
        {
            var globalScriptData = ScriptsRepository.Instance.GetScript(scriptExec.Name);
            if (globalScriptData == null) return false;

            ScriptFailureLevel failureLevel = ResolveFailureLevel(tool, globalScriptData, scriptExec);
            int maxAttempts = failureLevel == ScriptFailureLevel.Important ? ImportantFailureMaxAttempts : 1;

            for (int attempt = 1; attempt <= maxAttempts; attempt++)
            {
                try
                {
                    int waitDelaySeconds = ResolveDelaySeconds(globalScriptData, scriptExec);

                    Logger.Instance.Debug(tool.ToolName, $"{logPrefix} Stabilizing window frame delay: {waitDelaySeconds}s...");
                    await Task.Delay(waitDelaySeconds * 1000, token);

                    Logger.Instance.Info(tool.ToolName, $"{logPrefix} Interrogating element properties metrics using: '{scriptExec.Name}'...");
                    Logger.Instance.Debug(tool.ToolName, $"{logPrefix} Script '{scriptExec.Name}' full text being executed:{Environment.NewLine}{globalScriptData.ScriptText}");

                    bool useExperimentalCdpAsync = ResolveUseExperimentalCdpAsync(globalScriptData, scriptExec);
                    if (useExperimentalCdpAsync)
                    {
                        Logger.Instance.Debug(tool.ToolName, $"{logPrefix} Script '{scriptExec.Name}' running via experimental CDP async path (useExperimentalCdpAsync=true).");
                    }

                    string rawJsonResult = await ControlExtensions.InvokeAsync<string>(_parentForm, () =>
                        useExperimentalCdpAsync
                            ? ExperimentalCdpScriptExecutor.ExecuteScriptAsync(view.CoreWebView2, globalScriptData.ScriptText)
                            : view.CoreWebView2.ExecuteScriptAsync(globalScriptData.ScriptText)
                    );

                    Logger.Instance.Debug(tool.ToolName, $"{logPrefix} Script '{scriptExec.Name}' raw result: {rawJsonResult}");

                    DataScraped?.Invoke(this, new ScrapedDataEventArgs(tool.ToolName, pageUrl, scriptExec.Name, rawJsonResult, null, globalScriptData.ReturnType));

                    OnLogTraced(tool.ToolName, scriptExec.Name, $"{logPrefix} Extraction operations successfully reported data back for '{scriptExec.Name}'.");
                    return false;
                }
                catch (OperationCanceledException)
                {
                    throw; // user-requested cancellation, not a script failure - let it propagate normally
                }
                catch (Exception ex)
                {
                    bool willRetry = failureLevel == ScriptFailureLevel.Important && attempt < maxAttempts;

                    Logger.Instance.Error(tool.ToolName, $"{logPrefix} Script '{scriptExec.Name}' failed (attempt {attempt}/{maxAttempts}, level={failureLevel}): {ex.Message}");

                    // Snapshot moved here from right before execution - only capture the DOM when a script
                    // actually fails, not on every successful run. One snapshot per failed attempt, same
                    // pattern as the login handshake's per-attempt snapshots.
                    await CaptureDebugSnapshotAsync(tool.ToolName, $"{scriptExec.Name}_attempt{attempt}", view);

                    if (willRetry) continue;

                    string severityTag = failureLevel == ScriptFailureLevel.Critical ? "[CRITICAL]" : "[ERROR]";
                    OnLogTraced(tool.ToolName, scriptExec.Name, $"{severityTag} {logPrefix} Script '{scriptExec.Name}' FAILED: {ex.Message}");

                    return failureLevel == ScriptFailureLevel.Critical;
                }
            }

            return false; // unreachable
        }

        // Same base/override precedence as ResolveDelaySeconds/ResolveRunIndex. Default "common" - a script
        // failure is isolated and doesn't affect anything else. Unrecognized values fall back to "common"
        // with a warning, same pattern as ToolConfig.ForcePageRunMode.
        private static ScriptFailureLevel ResolveFailureLevel(ToolConfig tool, ScrapScript globalScriptData, TargetScriptExecution scriptExec)
        {
            string raw = null;

            if (globalScriptData.Options != null && globalScriptData.Options.TryGetValue("FailureLevel", out var baseVal))
                raw = baseVal?.ToString();

            if (scriptExec.OptionsOverride != null && scriptExec.OptionsOverride.TryGetValue("FailureLevel", out var overrideVal))
                raw = overrideVal?.ToString();

            if (string.IsNullOrWhiteSpace(raw)) return ScriptFailureLevel.Common;
            if (string.Equals(raw, "critical", StringComparison.OrdinalIgnoreCase)) return ScriptFailureLevel.Critical;
            if (string.Equals(raw, "important", StringComparison.OrdinalIgnoreCase)) return ScriptFailureLevel.Important;
            if (string.Equals(raw, "common", StringComparison.OrdinalIgnoreCase)) return ScriptFailureLevel.Common;

            Logger.Instance.Warn(tool.ToolName, $"Unrecognized FailureLevel value '{raw}' on script '{scriptExec.Name}' (expected 'common', 'important', or 'critical') - falling back to 'common'.");
            return ScriptFailureLevel.Common;
        }

        // 🔥 FIXED: waitDelaySeconds used to only ever check scriptExec.OptionsOverride (the page mapping's
        // per-usage override), hardcoding 5 as the fallback and never reading the script's own base
        // options.DelaySeconds (tools/scripts/*.yaml) at all - so a base delay set on the script itself was
        // silently ignored. Base Options is the default; page-level OptionsOverride wins when present.
        private static int ResolveDelaySeconds(ScrapScript globalScriptData, TargetScriptExecution scriptExec)
        {
            int delaySeconds = 5;

            if (globalScriptData.Options != null && globalScriptData.Options.TryGetValue("DelaySeconds", out var baseDelay))
                delaySeconds = Convert.ToInt32(baseDelay);

            if (scriptExec.OptionsOverride != null && scriptExec.OptionsOverride.TryGetValue("DelaySeconds", out var localOverride))
                delaySeconds = Convert.ToInt32(localOverride);

            return delaySeconds;
        }

        // Same base/override precedence as ResolveDelaySeconds: script's own options.RunIndex is the
        // default, a page mapping's optionsOverride.RunIndex wins for that specific invocation. Default 0.
        private static int ResolveRunIndex(ScrapScript globalScriptData, TargetScriptExecution scriptExec)
        {
            int runIndex = 0;

            if (globalScriptData.Options != null && globalScriptData.Options.TryGetValue("RunIndex", out var baseRunIndex))
                runIndex = Convert.ToInt32(baseRunIndex);

            if (scriptExec.OptionsOverride != null && scriptExec.OptionsOverride.TryGetValue("RunIndex", out var localOverride))
                runIndex = Convert.ToInt32(localOverride);

            return runIndex;
        }

        // Same base/override precedence as ResolveDelaySeconds/ResolveRunIndex. Default false - a script
        // only takes the experimental CDP Runtime.evaluate path (ExperimentalCdpScriptExecutor) when it
        // explicitly opts in, since real async/await/fetch is otherwise silently broken through the
        // standard CoreWebView2.ExecuteScriptAsync (see README's "Experimental CDP Async Execution"
        // section for the full investigation). Deliberately per-script, not per-tool or global, so opting
        // one script into the experimental path never affects any other script's execution.
        private static bool ResolveUseExperimentalCdpAsync(ScrapScript globalScriptData, TargetScriptExecution scriptExec)
        {
            bool useExperimentalCdpAsync = false;

            if (globalScriptData.Options != null && globalScriptData.Options.TryGetValue("UseExperimentalCdpAsync", out var baseVal))
                useExperimentalCdpAsync = Convert.ToBoolean(baseVal);

            if (scriptExec.OptionsOverride != null && scriptExec.OptionsOverride.TryGetValue("UseExperimentalCdpAsync", out var overrideVal))
                useExperimentalCdpAsync = Convert.ToBoolean(overrideVal);

            return useExperimentalCdpAsync;
        }

        // Login protection for SPAs: the login form fields may not exist in the DOM yet when the first
        // attempt runs (page still bootstrapping). StandardLoginHandshake already reports this case by
        // returning an "ERROR: ..." string instead of throwing, so it doubles as a DOM-readiness probe.
        // On that signal, wait LoginRetryDelaySeconds and re-run the script (no re-navigation - the page
        // is still loading, it just needs more time) up to LoginRetryAttempts times total, per tool config.
        private async Task<bool> AttemptLoginWithRetryAsync(ToolConfig tool, WebView2 view, CancellationToken token)
        {
            var loginScriptData = ScriptsRepository.Instance.GetScript("StandardLoginHandshake");
            if (loginScriptData == null)
            {
                Logger.Instance.Error(tool.ToolName, "StandardLoginHandshake script not found in Scripts Repository.");
                return false;
            }

            _parentForm.Invoke((MethodInvoker)(() => view.CoreWebView2.Navigate(tool.BaseLoginUrl)));
            await WaitForNavigationCompleteAsync(view, token);
            await Task.Delay(2500, token);

            int totalAttempts = Math.Max(1, tool.LoginRetryAttempts);

            for (int attempt = 1; attempt <= totalAttempts; attempt++)
            {
                token.ThrowIfCancellationRequested();

                string rawLoginStatusResult = await ControlExtensions.InvokeAsync<string>(_parentForm, () =>
                    view.CoreWebView2.ExecuteScriptWithArgsAsync(
                        loginScriptData.ScriptText,
                        new object[] { tool.PredefinedUser, tool.PredefinedPass, tool.ElementName, tool.ElementPass, tool.ElementSubmit }
                    )
                );

                // 🔥 FIXED: ExecuteScriptWithArgsAsync/ExecuteScriptAsync return the result JSON-encoded
                // (a quoted string), so the raw value was "\"ERROR: ...\"" - StartsWith("ERROR") was always
                // false because the actual first character was a quote, silently treating every genuine
                // DOM-not-ready error as a login success. Unwrap it back to the real string first.
                string loginStatusResult = UnwrapJsonEncodedString(rawLoginStatusResult);

                bool domNotReady = loginStatusResult != null && loginStatusResult.Trim().StartsWith("ERROR", StringComparison.OrdinalIgnoreCase);

                if (!domNotReady)
                {
                    Logger.Instance.Debug(tool.ToolName, $"Login attempt {attempt}/{totalAttempts} response: {loginStatusResult?.Trim()}");
                    return true;
                }

                Logger.Instance.Warn(tool.ToolName, $"Login attempt {attempt}/{totalAttempts} - form inputs not yet in DOM ({loginStatusResult.Trim()}).");

                await CaptureDebugSnapshotAsync(tool.ToolName, $"StandardLoginHandshake_attempt{attempt}", view);

                if (attempt < totalAttempts)
                {
                    await Task.Delay(tool.LoginRetryDelaySeconds * 1000, token);
                }
            }

            Logger.Instance.Error(tool.ToolName, $"Login form inputs never appeared in DOM after {totalAttempts} attempt(s). Giving up.");
            return false;
        }

        // 🔥 FIXED: Builds the CoreWebView2Environment exactly ONCE per tool run. Every WebView2 controller
        // (master login worker + all parallel page workers) must be handed this SAME object via
        // EnsureCoreWebView2Async so they share one underlying browser process/session — like tabs in one
        // browser window — instead of each independently deriving its own environment from the folder path,
        // which races for the profile lock and can silently leave a worker on an unauthenticated session.
        private async Task<CoreWebView2Environment> CreateEnvironmentOnUiAsync(string cachePath, bool ignoreSslCertError)
        {
            // 🔥 ADDED: ignoreSslCertError=true launches the underlying Chromium host with
            // --ignore-certificate-errors, so navigations to a device with an untrusted/self-signed/expired
            // certificate don't get blocked by ERR_CERT_AUTHORITY_INVALID/ERR_CERT_DATE_INVALID/etc. in the
            // first place - this covers the error at the browser-process level, before any page-level event
            // would even fire. See also the ServerCertificateErrorDetected handler in
            // InstantiateBrowserContextOnUiAsync, which is a second layer for whatever this launch argument
            // doesn't catch.
            CoreWebView2EnvironmentOptions options = ignoreSslCertError
                ? new CoreWebView2EnvironmentOptions { AdditionalBrowserArguments = "--ignore-certificate-errors" }
                : null;

            return await ControlExtensions.InvokeAsync<CoreWebView2Environment>(_parentForm, async () =>
                await CoreWebView2Environment.CreateAsync(userDataFolder: cachePath, options: options));
        }

        private async Task<WebView2> InstantiateBrowserContextOnUiAsync(CoreWebView2Environment environment, string toolName, bool ignoreSslCertError = false)
        {
            WebView2 view = null;
            await ControlExtensions.InvokeAsync(_parentForm, async () =>
            {
                // 🔥 FIXED (under test): previously left at WinForms' default control size - not a real
                // browser viewport. Some SPAs do viewport-dependent layout/initialization and can silently
                // fail to render correctly in a degenerate-sized surface, independent of timing or cache.
                // Give it a real browser-window-sized viewport. Location is pushed far off-screen rather
                // than setting Visible = false, since Visible = false would make Chromium report
                // document.visibilityState as "hidden" to the page - some SPAs use that to defer rendering,
                // which would reintroduce the same class of problem this is meant to fix.
                view = new WebView2
                {
                    Size = new Size(1280, 800),
                    Location = new Point(-10000, -10000),
                    Visible = true
                };
                _parentForm.Controls.Add(view);
                await view.EnsureCoreWebView2Async(environment);

                if (ignoreSslCertError)
                {
                    // Second layer alongside the --ignore-certificate-errors launch argument above: handles
                    // any certificate error WebView2 still surfaces as an event rather than silently passing
                    // through the launch flag (behavior here isn't officially itemized per error code, so
                    // covering both is cheap insurance for the same "just let it through" outcome).
                    view.CoreWebView2.ServerCertificateErrorDetected += (s, e) =>
                    {
                        Logger.Instance.Warn(toolName, $"Ignoring TLS certificate error ({e.ErrorStatus}) for {e.RequestUri} per ignoreSslCertError=true.");
                        e.Action = CoreWebView2ServerCertificateErrorAction.AlwaysAllow;
                    };
                }

                if (Logger.Instance.IsDebugEnabled)
                {
                    await EnableBrowserConsoleLoggingAsync(view, toolName);
                }
            });
            return view;
        }

        // Surfaces the page's own DevTools Console (console.log/warn/error and uncaught JS exceptions) to
        // scraper_execution.log via the Chrome DevTools Protocol - WebView2 has no simpler high-level
        // "console message" event. Debug-only: enabling the Runtime CDP domain has real overhead and this
        // is a diagnostic aid, not something needed for normal extraction.
        private async Task EnableBrowserConsoleLoggingAsync(WebView2 view, string toolName)
        {
            try
            {
                CoreWebView2 core = view.CoreWebView2;
                await core.CallDevToolsProtocolMethodAsync("Runtime.enable", "{}");

                var consoleReceiver = core.GetDevToolsProtocolEventReceiver("Runtime.consoleAPICalled");
                consoleReceiver.DevToolsProtocolEventReceived += (s, e) =>
                    HandleBrowserConsoleApiCalled(toolName, e.ParameterObjectAsJson);

                var exceptionReceiver = core.GetDevToolsProtocolEventReceiver("Runtime.exceptionThrown");
                exceptionReceiver.DevToolsProtocolEventReceived += (s, e) =>
                    HandleBrowserExceptionThrown(toolName, e.ParameterObjectAsJson);
            }
            catch (Exception ex)
            {
                Logger.Instance.Warn(toolName, $"Failed to attach browser console logging: {ex.Message}");
            }
        }

        private void HandleBrowserConsoleApiCalled(string toolName, string json)
        {
            try
            {
                var obj = JObject.Parse(json);
                string type = obj["type"]?.ToString() ?? "log";
                var argsArray = obj["args"] as JArray;

                string message = argsArray != null
                    ? string.Join(" ", argsArray.Select(a => a["value"]?.ToString() ?? a["description"]?.ToString() ?? a.ToString()))
                    : json;

                if (string.Equals(type, "error", StringComparison.OrdinalIgnoreCase))
                    Logger.Instance.Error(toolName, $"[BROWSER CONSOLE:{type}] {message}");
                else if (string.Equals(type, "warning", StringComparison.OrdinalIgnoreCase))
                    Logger.Instance.Warn(toolName, $"[BROWSER CONSOLE:{type}] {message}");
                else
                    Logger.Instance.Debug(toolName, $"[BROWSER CONSOLE:{type}] {message}");
            }
            catch (Exception ex)
            {
                Logger.Instance.Debug(toolName, $"[BROWSER CONSOLE] (unparsed - {ex.Message}) {json}");
            }
        }

        private void HandleBrowserExceptionThrown(string toolName, string json)
        {
            try
            {
                var obj = JObject.Parse(json);
                var details = obj["exceptionDetails"];
                string text = details?["text"]?.ToString() ?? "Uncaught exception";
                string description = details?["exception"]?["description"]?.ToString()
                    ?? details?["exception"]?["value"]?.ToString();
                string url = details?["url"]?.ToString();
                string line = details?["lineNumber"]?.ToString();

                Logger.Instance.Error(toolName, $"[BROWSER EXCEPTION] {text}: {description} (at {url}:{line})");
            }
            catch (Exception ex)
            {
                Logger.Instance.Debug(toolName, $"[BROWSER EXCEPTION] (unparsed - {ex.Message}) {json}");
            }
        }

        private async Task TeardownBrowserContextFromUiAsync(WebView2 view)
        {
            if (view == null) return;
            await ControlExtensions.InvokeAsync(_parentForm, () =>
            {
                view.Dispose();
                _parentForm.Controls.Remove(view);
                return Task.CompletedTask;
            });
        }

        private async Task WaitForNavigationCompleteAsync(WebView2 webView, CancellationToken token)
        {
            var tcs = new TaskCompletionSource<bool>();
            EventHandler<CoreWebView2NavigationCompletedEventArgs> handler = null;
            handler = (s, e) => { webView.NavigationCompleted -= handler; tcs.TrySetResult(e.IsSuccess); };
            webView.NavigationCompleted += handler;
            using (token.Register(() => { webView.NavigationCompleted -= handler; tcs.TrySetCanceled(); })) { await tcs.Task; }
        }

        private void OnLogTraced(string toolName, string scriptName, string text)
        {
            DataScraped?.Invoke(this, new ScrapedDataEventArgs(toolName, null, scriptName, null, text, null));
        }
    }
}