/**
 * Technical Audit Runner — Crawl-Based
 * --------------------------------------
 * Real site-wide crawl that fetches and inspects each page's HTML:
 *   1. Discovers internal pages from homepage links
 *   2. Fetches each page and runs HTML-based checks:
 *      - Title tag exists
 *      - Meta description exists
 *      - H1 exists
 *      - Heading order (no skipped levels)
 *      - Images missing alt text
 *      - Canonical tag presence
 *      - HTTP status issues
 *      - Broken internal links
 *   3. Detects site-wide issues:
 *      - Duplicate titles across pages
 *      - Duplicate meta descriptions across pages
 *   4. Stores one issue per check, with affected URLs per issue
 *   5. pagesScanned = actual pages fetched and inspected
 */

import { prisma } from "@/lib/prisma";
import { log } from "@/lib/logger";

// Crawl coverage. Tuned against real production timings:
//   Modern Heart ran ~165 pages in ≤290s → comfortably under cap.
//   Life Imaging ran 172 pages then hit Vercel's 300s kill → cap
//     was too permissive for slower origins.
//
// 100 pages gives representative sample coverage (top GSC pages +
// top Site Tag entry pages + homepage BFS) while keeping total wall
// time bounded even when the target site returns 2–3s per page.
// 100 × 2s = 200s crawl, leaving 100s for broken-link analysis +
// cross-page aggregation + DB writes.
//
// CRAWL_DEPTH stays at 5 — BFS depth limits the FAN-OUT, not the
// total page count, and MAX_PAGES is the real cap.
const MAX_PAGES = 100;
const FETCH_TIMEOUT = 12000;
const CRAWL_DEPTH = 5;
const HEADLESS_THRESHOLD = 3;
const AUDIT_HARD_TIMEOUT = 290000; // 290s — must finish before Vercel kills at 300s (Pro maxDuration)
// Per-stage time budgets. Replaces the old "check the global 290s
// budget on every page" heuristic, which let the crawl spend 280s
// on pages and leave only 10s for analysis + save — the observed
// failure mode on Life Imaging. Each phase now has its own ceiling
// and bails gracefully when hit so downstream phases still get to
// run and we always persist SOMETHING.
//
// Budgets intentionally sum to less than AUDIT_HARD_TIMEOUT so the
// global throw-on-exceed fallback at `checkTimeout()` only fires
// when something went genuinely wrong (DB stall, clock skew).
const CRAWL_TIME_LIMIT = 200_000;        // 200s for crawling pages
const BROKEN_LINK_TIME_LIMIT = 45_000;   // 45s for cross-page broken-link checks
const STALE_RUN_MINUTES = 5; // Runs older than this with status "running" are considered dead
const DB_TASK_TIMEOUT = 30000; // per-query ceiling for the discovery-stage Promise.all
const BROWSER_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";

/** Race a promise against a millisecond deadline, resolving to the
 *  fallback value when the deadline fires first. Prevents a single
 *  hung Prisma call (connection-pool exhaustion, slow-running query
 *  under index miss) from blackholing the whole audit — worst case
 *  the discovery stage loses one source of URL candidates, which is
 *  a soft degradation rather than the full 300s Vercel kill we used
 *  to see. Errors inside the promise also resolve to the fallback
 *  (same shape as a deadline miss from the caller's perspective). */
function withTimeout<T>(promise: Promise<T>, ms: number, fallback: T, label: string): Promise<T> {
  let timer: NodeJS.Timeout | null = null;
  const timeoutPromise = new Promise<T>((resolve) => {
    timer = setTimeout(() => {
      log.warn("api", `[audit-runner] ${label} timed out after ${ms}ms — using fallback`);
      resolve(fallback);
    }, ms);
  });
  return Promise.race([
    promise.then((v) => { if (timer) clearTimeout(timer); return v; })
           .catch((err) => {
             if (timer) clearTimeout(timer);
             log.warn("api", `[audit-runner] ${label} rejected — using fallback`, {
               metadata: { error: err instanceof Error ? err.message : String(err) },
             });
             return fallback;
           }),
    timeoutPromise,
  ]);
}

export interface AuditResult {
  auditId: string;
  status: "completed" | "failed";
  crawlHealth: number;
  issues: number;
  message: string;
}

// ─── Issue collected per page ───────────────────────────────────────────────

interface BrokenLinkEvidence {
  sourcePageUrl: string;
  brokenHref: string;
  anchorText: string;
  selector: string;
  xpath: string;
  snippet: string;
  statusCode: number;
  auditId?: string;
}

interface RawIssue {
  category: string;
  severity: string;
  title: string;
  detail: string;
  pageUrl: string;
  evidence?: BrokenLinkEvidence[];
}

// ─── Main runner ────────────────────────────────────────────────────────────

export async function runTechnicalAudit(brandId: string, domain: string): Promise<AuditResult> {
  const startTime = Date.now();
  let stage = "initializing";

  const audit = await prisma.technicalAudit.create({
    data: { brandId, status: "running", pagesScanned: 0 },
  });

  log.info("api", `Technical audit started for ${domain}`, {
    metadata: { brandId, auditId: audit.id, domain },
  });

  // Heartbeat writer — stamps the current stage into the audit row's
  // errorMessage column every time the runner moves to a new phase.
  // Purely advisory: if the Vercel function is SIGKILL'd before the
  // catch block runs, the DB still reflects "audit stalled at stage
  // X" instead of the generic "no response after 15 minutes" the
  // GET route's auto-fail rewrites. Fire-and-forget — a write
  // failure here must not break the audit itself.
  function heartbeat(next: string) {
    stage = next;
    log.info("api", `[audit-runner] stage → ${next}`, {
      metadata: { brandId, auditId: audit.id },
    });
    prisma.technicalAudit.update({
      where: { id: audit.id },
      data: { errorMessage: `stage: ${next}` },
    }).catch(() => {});
  }

  // Hard timeout wrapper — ensures we never exceed Vercel function limit
  function checkTimeout() {
    if (Date.now() - startTime > AUDIT_HARD_TIMEOUT) {
      throw new Error(`Audit timed out at stage: ${stage}. Exceeded ${Math.round(AUDIT_HARD_TIMEOUT / 1000)}s limit.`);
    }
  }

  try {
    // Normalize domain and resolve the actual canonical URL (follows redirects)
    heartbeat("resolving domain");
    const { baseUrl, resolvedDomain } = await resolveBaseUrl(domain);
    checkTimeout();
    log.info("api", `Resolved ${domain} → ${baseUrl} (domain: ${resolvedDomain})`, {
      metadata: { auditId: audit.id },
    });

    // Step 1: Discover pages (returns URLs + cached HTML from BFS)
    heartbeat("discovering pages");
    const { urls: pageUrls, htmlCache } = await discoverPages(brandId, baseUrl, resolvedDomain);
    checkTimeout();
    log.info("api", `Discovered ${pageUrls.length} pages on ${resolvedDomain}`, {
      metadata: { auditId: audit.id, pages: pageUrls.length },
    });

    // Step 2: Crawl each page and run checks (with retry + failure tracking)
    heartbeat("crawling pages");
    const allIssues: RawIssue[] = [];
    const pageTitles: Map<string, string[]> = new Map();
    const pageDescriptions: Map<string, string[]> = new Map();
    const internalLinks: Map<string, string[]> = new Map();
    // target href → LinkDetail + sourcePageUrl (from all source pages)
    type LinkDetailWithSource = LinkDetail & { sourcePageUrl: string };
    const allLinkDetails = new Map<string, LinkDetailWithSource[]>();
    const crawlFailures: Array<{ url: string; reason: string }> = [];
    const crawledHtml: Map<string, string> = new Map(); // cache from BFS
    let pagesScanned = 0;
    // Crawl-phase timer + termination reason. Propagated into the
    // audit's coverage note so the dashboard can tell the operator
    // which ceiling fired: the page cap (representative sample) vs
    // the time ceiling (slow origin, partial sample).
    const crawlStart = Date.now();
    let crawlStopReason: "complete" | "page_cap" | "time_cap" = "complete";
    const pagesDiscovered = pageUrls.length;
    // How many pages must complete between TechnicalAudit.pagesScanned
    // DB writes. Without incremental writes the client poll at
    // /api/technical-audit saw pagesScanned=0 the entire crawl (only
    // the final write at audit completion updated the column). Ten
    // pages is a good trade-off: at ~500-1500ms per page the client
    // sees a progress bump every 5-15s (matches the 3s poll interval
    // with ~3-5 poll cycles between bumps), while still amortising
    // the DB write cost over a meaningful batch instead of hitting
    // the connection pool 200 times per audit. */
    const PROGRESS_WRITE_EVERY = 10;

    for (const pageUrl of pageUrls) {
      // Time cap fires first — a slow origin that's taken 200s+ to
      // crawl the fetched-so-far set needs the remaining budget for
      // analysis and DB save, even if we're under MAX_PAGES. Bails
      // gracefully with whatever pages were scanned so the audit
      // still reports results instead of dying on Vercel's kill.
      if (Date.now() - crawlStart > CRAWL_TIME_LIMIT) {
        crawlStopReason = "time_cap";
        log.warn("api", `Crawl time cap reached (${Math.round((Date.now() - crawlStart) / 1000)}s) — stopping at ${pagesScanned} of ${pagesDiscovered} pages, saving partial results`, {
          metadata: { auditId: audit.id },
        });
        break;
      }
      // Page cap — representative-sample cutoff. Hit naturally on
      // most brands and signals "normal completion, capped at 100".
      if (pagesScanned >= MAX_PAGES) {
        crawlStopReason = "page_cap";
        log.info("api", `Page cap reached (${MAX_PAGES}) — stopping crawl, running analysis on representative sample`, {
          metadata: { auditId: audit.id },
        });
        break;
      }
      try {
        // Delay between pages to avoid rate limiting (increases after failures)
        if (pagesScanned > 0 || crawlFailures.length > 0) {
          const delay = crawlFailures.length > 0 ? 1500 : 500;
          await new Promise(r => setTimeout(r, delay));
        }
        stage = `crawling ${pageUrl.replace(/^https?:\/\/[^/]+/, "")}`;
        const result = await crawlAndCheckWithRetry(pageUrl, baseUrl, resolvedDomain, crawledHtml);
        if (!result) continue;
        pagesScanned++;
        // Flush progress every PROGRESS_WRITE_EVERY pages (and on
        // completion via the outer update call). Non-awaited so a
        // slow DB write can't block the crawl loop; errors are
        // swallowed because missing one progress bump is harmless.
        if (pagesScanned % PROGRESS_WRITE_EVERY === 0) {
          prisma.technicalAudit.update({
            where: { id: audit.id },
            data: { pagesScanned },
          }).catch((err) => {
            log.warn("api", `Progress write failed at ${pagesScanned} pages: ${err instanceof Error ? err.message : "unknown"}`, { metadata: { auditId: audit.id } });
          });
        }

        for (const issue of result.issues) allIssues.push(issue);

        if (result.title) {
          const existing = pageTitles.get(result.title) ?? [];
          existing.push(pageUrl);
          pageTitles.set(result.title, existing);
        }
        if (result.metaDescription) {
          const existing = pageDescriptions.get(result.metaDescription) ?? [];
          existing.push(pageUrl);
          pageDescriptions.set(result.metaDescription, existing);
        }
        for (const link of result.internalLinks) {
          const existing = internalLinks.get(link) ?? [];
          existing.push(pageUrl);
          internalLinks.set(link, existing);
        }
        for (const ld of result.internalLinkDetails) {
          const existing: LinkDetailWithSource[] = allLinkDetails.get(ld.href) ?? [];
          existing.push({ ...ld, sourcePageUrl: pageUrl });
          allLinkDetails.set(ld.href, existing);
        }
      } catch (err) {
        const reason = err instanceof Error ? err.message : "unknown";
        crawlFailures.push({ url: pageUrl, reason });
        log.warn("api", `Crawl failed for ${pageUrl}: ${reason}`, {
          metadata: { auditId: audit.id, pageUrl },
        });
      }
    }

    // Log crawl failures summary
    if (crawlFailures.length > 0) {
      log.info("api", `${crawlFailures.length} page(s) failed to crawl`, {
        metadata: { auditId: audit.id, failures: crawlFailures },
      });
    }

    if (pagesScanned === 0) throw new Error(`Could not crawl any pages. ${crawlFailures.length} failed: ${crawlFailures.map(f => f.reason).join("; ")}`);

    // Step 3: Cross-page checks
    heartbeat("checking cross-page issues");
    checkTimeout();
    // Duplicate titles
    for (const [title, urls] of pageTitles) {
      if (urls.length > 1) {
        for (const url of urls) {
          allIssues.push({
            category: "On-page SEO",
            severity: "high",
            title: "Duplicate title tag",
            detail: `Title "${title.slice(0, 80)}" is shared by ${urls.length} pages`,
            pageUrl: url,
          });
        }
      }
    }

    // Duplicate descriptions
    for (const [desc, urls] of pageDescriptions) {
      if (urls.length > 1) {
        for (const url of urls) {
          allIssues.push({
            category: "On-page SEO",
            severity: "medium",
            title: "Duplicate meta description",
            detail: `Description "${desc.slice(0, 80)}…" is shared by ${urls.length} pages`,
            pageUrl: url,
          });
        }
      }
    }

    // Broken internal links — check discovered links that returned errors.
    // Time-capped: a site with many external/link-like URLs could
    // otherwise spend 10-20 minutes here on 8s HEAD requests. Bail at
    // BROKEN_LINK_TIME_LIMIT and report broken-link findings only
    // for the batches that completed. crawlStopReason isn't touched —
    // this is a separate phase — but the coverage note records it.
    const checkedLinks = new Set(pageUrls);
    const uncheckedInternalLinks = [...internalLinks.keys()].filter(l => !checkedLinks.has(l));
    const brokenLinkStart = Date.now();
    let brokenLinksBailed = false;
    for (const batch of chunk(uncheckedInternalLinks, 10)) {
      if (Date.now() - brokenLinkStart > BROKEN_LINK_TIME_LIMIT) {
        brokenLinksBailed = true;
        log.warn("api", `Broken-link analysis time cap reached (${Math.round((Date.now() - brokenLinkStart) / 1000)}s) — stopping early, keeping results so far`, {
          metadata: { auditId: audit.id },
        });
        break;
      }
      const results = await Promise.allSettled(
        batch.map(async (link) => {
          const res = await fetch(link, {
            method: "HEAD",
            signal: AbortSignal.timeout(8000),
            headers: { "User-Agent": BROWSER_UA },
            redirect: "follow",
          });
          return { url: link, status: res.status };
        })
      );
      for (const result of results) {
        if (result.status === "fulfilled" && result.value.status >= 400) {
          const brokenUrl = result.value.url;
          const statusCode = result.value.status;
          const linkDetails = allLinkDetails.get(brokenUrl) ?? [];
          const sources = internalLinks.get(brokenUrl) ?? [];

          // Build evidence from link details
          const evidence: BrokenLinkEvidence[] = linkDetails.map(ld => ({
            sourcePageUrl: ld.sourcePageUrl,
            brokenHref: brokenUrl,
            anchorText: ld.anchorText,
            selector: ld.selector,
            xpath: ld.xpath,
            snippet: ld.snippet,
            statusCode,
            auditId: audit.id,
          }));

          // Also add sources that had no detail extraction (fallback)
          for (const source of sources) {
            if (!evidence.some(e => e.sourcePageUrl === source)) {
              evidence.push({
                sourcePageUrl: source,
                brokenHref: brokenUrl,
                anchorText: "(unknown)",
                selector: "",
                xpath: "",
                snippet: "",
                statusCode,
                auditId: audit.id,
              });
            }
          }

          for (const source of sources) {
            allIssues.push({
              category: "Links & Crawling",
              severity: statusCode === 404 ? "high" : "medium",
              title: `Broken internal link (${statusCode})`,
              detail: `Link to ${brokenUrl} returns ${statusCode}`,
              pageUrl: source,
              evidence,
            });
          }
        }
      }
    }

    // Step 4: Merge issues and calculate scores
    const mergedIssues = mergeIssues(allIssues);

    // Health: count unique (page, check-category) pairs that failed vs total possible
    const checksPerPage = 8; // title, meta desc, H1, heading order, alt text, canonical, http status, landmarks
    const totalChecks = pagesScanned * checksPerPage;
    // Count unique failing check types per page (not raw issue count)
    const failedCheckSet = new Set(allIssues.map(i => `${i.pageUrl}::${i.title}`));
    const totalFailing = failedCheckSet.size;
    const crawlHealth = totalChecks > 0 ? Math.max(0, Math.min(100, Math.round(((totalChecks - totalFailing) / totalChecks) * 100))) : 0;

    const errorCount = mergedIssues.filter(i => i.severity === "critical" || i.severity === "high").length;
    const warningCount = mergedIssues.filter(i => i.severity === "medium").length;
    const noticeCount = mergedIssues.filter(i => i.severity === "low").length;
    const passedCount = Math.max(0, totalChecks - totalFailing);
    const duration = Math.round((Date.now() - startTime) / 1000);

    // Step 5: Store issues
    heartbeat("saving results");
    checkTimeout();
    if (mergedIssues.length > 0) {
      await prisma.technicalIssue.createMany({
        data: mergedIssues.map(issue => ({
          auditId: audit.id,
          category: issue.category,
          severity: issue.severity,
          title: issue.title,
          detail: issue.detail,
          pageUrl: issue.affectedUrls[0] || null,
          affectedUrls: issue.affectedUrls,
          evidence: issue.evidence.length > 0 ? JSON.parse(JSON.stringify(issue.evidence)) : [],
          affected: issue.affectedUrls.length || 1,
          status: "open",
        })),
      });
    }

    // Compose coverage + failure note. Uses `errorMessage` (the only
    // free-form column on the TechnicalAudit model today) as the
    // carrier; the UI renders it as a notice, not an error. Ordering:
    //   1. coverage note — always when the crawl stopped early for
    //      page-cap or time-cap reasons, so the user understands
    //      "you have 472 pages; we audited the 100 most important"
    //      vs the generic silence they got before.
    //   2. broken-link bail — appended when the link-check phase
    //      had to cut short (usually on sites with thousands of
    //      internal links).
    //   3. crawl-failure summary — preserved from the existing
    //      implementation for pages that errored during fetch.
    const notes: string[] = [];
    if (crawlStopReason === "page_cap") {
      notes.push(
        `Audited ${pagesScanned} of ${pagesDiscovered} discovered page${pagesDiscovered === 1 ? "" : "s"} (representative sample of the top GSC + Site Tag pages + homepage BFS). Contact support for a full-site deep crawl.`,
      );
    } else if (crawlStopReason === "time_cap") {
      notes.push(
        `Crawled ${pagesScanned} of ${pagesDiscovered} page${pagesDiscovered === 1 ? "" : "s"} before hitting the 200-second time ceiling. Results cover the scanned pages; re-run after off-peak hours for deeper coverage.`,
      );
    }
    if (brokenLinksBailed) {
      notes.push("Broken-link analysis stopped early (cross-page link verification time ceiling hit).");
    }
    if (crawlFailures.length > 0) {
      notes.push(`${crawlFailures.length} page(s) failed: ${crawlFailures.slice(0, 5).map(f => `${f.url} (${f.reason})`).join("; ")}`);
    }

    // Step 6: Update audit
    await prisma.technicalAudit.update({
      where: { id: audit.id },
      data: {
        status: "completed",
        crawlHealth: Math.max(0, Math.min(100, crawlHealth)),
        pagesScanned,
        errorCount,
        warningCount,
        noticeCount,
        passedCount,
        duration,
        completedAt: new Date(),
        errorMessage: notes.length > 0 ? notes.join(" · ") : null,
      },
    });

    log.info("api", `Technical audit completed for ${resolvedDomain}: ${pagesScanned} pages, health=${crawlHealth}, issues=${mergedIssues.length}, failures=${crawlFailures.length}`, {
      metadata: { brandId, auditId: audit.id, crawlHealth, pagesScanned, issues: mergedIssues.length, duration },
    });

    return {
      auditId: audit.id,
      status: "completed",
      crawlHealth: Math.max(0, Math.min(100, crawlHealth)),
      issues: mergedIssues.length,
      message: `Audit completed: ${pagesScanned} pages scanned, ${crawlHealth}% health, ${mergedIssues.length} issues found.`,
    };
  } catch (err) {
    const rawMsg = err instanceof Error ? err.message : String(err);
    // Classify if not already classified
    const msg = rawMsg.includes("—") ? rawMsg : classifyNetworkError(err);

    await prisma.technicalAudit.update({
      where: { id: audit.id },
      data: { status: "failed", errorMessage: msg.slice(0, 500), completedAt: new Date() },
    });

    log.error("api", `Technical audit failed for ${domain}: ${msg}`, {
      metadata: { brandId, auditId: audit.id },
    });

    return { auditId: audit.id, status: "failed", crawlHealth: 0, issues: 0, message: msg };
  }
}

// ─── Domain Resolution ──────────────────────────────────────────────────────

/**
 * Normalize the brand domain and resolve the actual canonical URL.
 * Handles: www vs non-www, http vs https, trailing slashes, stored with protocol.
 * Follows redirects to find where the site actually lives.
 */
async function resolveBaseUrl(domain: string): Promise<{ baseUrl: string; resolvedDomain: string }> {
  let cleaned = domain.trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "").toLowerCase();

  const candidates: string[] = [];
  if (cleaned.startsWith("www.")) {
    candidates.push(`https://${cleaned}`, `http://${cleaned}`);
  } else {
    candidates.push(`https://${cleaned}`, `https://www.${cleaned}`, `http://${cleaned}`, `http://www.${cleaned}`);
  }

  for (const tryUrl of candidates) {
    // Strategy 1: HEAD (fast, no body download)
    try {
      const res = await fetch(tryUrl, {
        method: "HEAD",
        signal: AbortSignal.timeout(5000),
        headers: { "User-Agent": BROWSER_UA },
        redirect: "follow",
      });

      if (res.ok || res.status === 429 || res.status === 403) {
        const parsed = new URL(res.url || tryUrl);
        return { baseUrl: parsed.origin, resolvedDomain: parsed.hostname };
      }
    } catch {
      // HEAD failed (blocked, timeout, network) — try GET below
    }

    // Strategy 2: GET fallback (some hosts block HEAD)
    try {
      const res = await fetch(tryUrl, {
        method: "GET",
        signal: AbortSignal.timeout(8000),
        headers: { "User-Agent": BROWSER_UA },
        redirect: "follow",
      });

      if (res.ok || res.status === 429 || res.status === 403) {
        const parsed = new URL(res.url || tryUrl);
        await res.text().catch(() => {});
        return { baseUrl: parsed.origin, resolvedDomain: parsed.hostname };
      }

      await res.text().catch(() => {});
    } catch {
      // Both HEAD and GET failed for this variant
    }

    await new Promise(r => setTimeout(r, 300));
  }

  // All failed — use the cleaned domain as fallback (don't throw,
  // let the crawl attempt proceed and fail with a better error)
  return { baseUrl: `https://${cleaned}`, resolvedDomain: cleaned };
}

function classifyNetworkError(err: unknown): string {
  if (!(err instanceof Error)) return "Unknown error";
  const msg = err.message.toLowerCase();
  const name = err.name;

  if (name === "TimeoutError" || name === "AbortError" || msg.includes("timeout") || msg.includes("aborted")) {
    return "Connection timed out — site may be slow or blocking automated requests";
  }
  if (msg.includes("enotfound") || msg.includes("getaddrinfo") || msg.includes("dns")) {
    return "DNS lookup failed — domain may not exist or DNS is misconfigured";
  }
  if (msg.includes("ssl") || msg.includes("tls") || msg.includes("certificate") || msg.includes("cert")) {
    return "SSL/TLS error — certificate may be expired or invalid";
  }
  if (msg.includes("redirect") || msg.includes("too many") || msg.includes("max redirect")) {
    return "Redirect loop detected — too many redirects";
  }
  if (msg.includes("econnrefused") || msg.includes("connection refused")) {
    return "Connection refused — server is not accepting connections";
  }
  if (msg.includes("econnreset") || msg.includes("socket hang up")) {
    return "Connection reset — server dropped the connection (possible bot protection)";
  }
  if (msg.includes("fetch failed") || msg.includes("network")) {
    return "Network error — site may be unreachable or blocking crawlers";
  }
  return err.message.slice(0, 150);
}

// ─── Page Discovery (inventory → sitemap → BFS → headless) ─────────────────

async function discoverPages(brandId: string, baseUrl: string, domain: string): Promise<{ urls: string[]; htmlCache: Map<string, string> }> {
  const discovered = new Set<string>();
  const htmlCache = new Map<string, string>();
  const base = baseUrl.replace(/\/$/, "");
  const normalDomain = domain.replace(/^www\./, "").toLowerCase();
  discovered.add(base);

  // Helper: add a URL if it's same-domain, non-asset, and under the cap
  function addUrl(rawUrl: string): boolean {
    if (discovered.size >= MAX_PAGES) return false;
    try {
      const parsed = new URL(rawUrl);
      if (parsed.hostname.replace(/^www\./, "").toLowerCase() !== normalDomain) return false;
      if (/\.(jpg|jpeg|png|gif|svg|css|js|pdf|zip|ico|woff|woff2|ttf|mp4|webm|webp|xml|json|txt|rss|atom)$/i.test(parsed.pathname)) return false;
      discovered.add(parsed.origin + parsed.pathname.replace(/\/$/, ""));
      return true;
    } catch { return false; }
  }

  // ── Step 1: Existing page inventory for this brand ──
  // Pull known page URLs from every first-party source we have:
  //   • Site Tag page_view events (real visitor traffic — the most
  //     complete picture for live sites; catches pages that aren't
  //     in the sitemap and haven't ranked in GSC yet).
  //   • Site Tag session entryPage (landing-page heavy pages).
  //   • GSC pages (top organic entry points).
  //   • Previous audit issues (re-test pages we've already flagged).
  //   • Performance audit pages (CWV coverage).
  //   • Dead pages (cross-check whether they've been fixed).
  // Each source is added in priority order; addUrl() respects the
  // MAX_PAGES cap so we short-circuit once the quota is hit.
  let siteTagUrlCount = 0;
  let gscUrlCount = 0;
  try {
    // Resolve the brand's TrackedSite once for both Site Tag queries.
    const site = await prisma.trackedSite.findUnique({
      where: { brandId },
      select: { id: true },
    }).catch(() => null);

    const thirtyDaysAgo = new Date(Date.now() - 30 * 86_400_000);

    // Each discovery query gets its own withTimeout wrapper so one
    // slow table (e.g. a Site Tag with 30M+ TrackedEvent rows and no
    // index coverage for the distinct path) can't push the audit
    // past the Vercel function ceiling. Fallbacks are empty arrays —
    // the audit proceeds with fewer URL candidates rather than
    // failing outright.
    const [prevIssues, gscPages, perfPages, deadPages, siteTagPageViews, siteTagSessions] = await Promise.all([
      withTimeout(
        prisma.technicalIssue.findMany({
          where: { audit: { brandId, status: "completed" } },
          select: { pageUrl: true },
          distinct: ["pageUrl"],
        }),
        DB_TASK_TIMEOUT, [] as Array<{ pageUrl: string }>, "prevIssues",
      ),
      withTimeout(
        prisma.gscPage.findMany({
          where: { brandId },
          select: { pageUrl: true },
          take: MAX_PAGES,
        }),
        DB_TASK_TIMEOUT, [] as Array<{ pageUrl: string }>, "gscPages",
      ),
      withTimeout(
        prisma.performancePageScore.findMany({
          where: { audit: { brandId, status: "completed" } },
          select: { pageUrl: true },
          distinct: ["pageUrl"],
        }),
        DB_TASK_TIMEOUT, [] as Array<{ pageUrl: string }>, "perfPages",
      ),
      withTimeout(
        prisma.deadPage.findMany({
          where: { brandId },
          select: { deadUrl: true },
          take: MAX_PAGES,
        }),
        DB_TASK_TIMEOUT, [] as Array<{ deadUrl: string }>, "deadPages",
      ),
      // Site Tag page_view events — `distinct` on pageUrl returns one
      // row per unique path. Ordered isn't meaningful here; we want
      // the full set of pages the tracker has seen.
      site?.id
        ? withTimeout(
            prisma.trackedEvent.findMany({
              where: {
                trackedSiteId: site.id,
                eventType: "page_view",
                timestamp: { gte: thirtyDaysAgo },
              },
              distinct: ["pageUrl"],
              select: { pageUrl: true },
              take: MAX_PAGES * 2, // headroom; addUrl() dedupes + caps
            }),
            DB_TASK_TIMEOUT, [] as Array<{ pageUrl: string }>, "siteTagPageViews",
          )
        : Promise.resolve([] as Array<{ pageUrl: string }>),
      // Landing-page heavy pages from TrackedSession.entryPage. These
      // are over-represented in real traffic (the pages Google /
      // social shares actually send users to) so they're the highest
      // priority to audit after the homepage.
      site?.id
        ? withTimeout(
            prisma.trackedSession.findMany({
              where: { trackedSiteId: site.id, startedAt: { gte: thirtyDaysAgo } },
              distinct: ["entryPage"],
              select: { entryPage: true },
              take: MAX_PAGES,
            }),
            DB_TASK_TIMEOUT, [] as Array<{ entryPage: string }>, "siteTagSessions",
          )
        : Promise.resolve([] as Array<{ entryPage: string }>),
    ]);

    // Order matters — earlier sources get priority on the MAX_PAGES
    // cap. Site Tag first (it captures reality on the live site),
    // then GSC (organic), then historical audit context.
    for (const row of siteTagSessions) {
      if (row.entryPage && addUrl(row.entryPage)) siteTagUrlCount++;
    }
    for (const row of siteTagPageViews) {
      if (row.pageUrl && addUrl(row.pageUrl)) siteTagUrlCount++;
    }
    for (const row of gscPages) {
      if (row.pageUrl && addUrl(row.pageUrl)) gscUrlCount++;
    }
    for (const row of prevIssues) if (row.pageUrl) addUrl(row.pageUrl);
    for (const row of perfPages) if (row.pageUrl) addUrl(row.pageUrl);
    for (const row of deadPages) if (row.deadUrl) addUrl(`${base}${row.deadUrl.startsWith("/") ? "" : "/"}${row.deadUrl}`);

    log.info("api", `Audit page inventory: ${siteTagUrlCount} from Site Tag, ${gscUrlCount} from GSC, total so far: ${discovered.size}`, {
      metadata: { brandId, siteTagUrlCount, gscUrlCount, totalBeforeSitemap: discovered.size },
    });
  } catch (err) {
    // Inventory query failed — continue with other discovery methods
    log.warn("api", `Audit page inventory lookup failed: ${err instanceof Error ? err.message : "unknown"}`, { metadata: { brandId } });
  }

  if (discovered.size >= MAX_PAGES) return { urls: [...discovered], htmlCache };

  // ── Step 2: Sitemap ──
  try {
    const sitemapUrls = await fetchSitemap(base, domain);
    for (const u of sitemapUrls) {
      addUrl(u);
      if (discovered.size >= MAX_PAGES) return { urls: [...discovered], htmlCache };
    }
  } catch { /* sitemap not available */ }

  if (discovered.size >= MAX_PAGES) return { urls: [...discovered], htmlCache };

  // ── Step 3: Sequential BFS on raw HTML (avoids rate limiting) ──
  const queue: string[] = [base];
  const visited = new Set<string>();

  for (let depth = 0; depth < CRAWL_DEPTH && queue.length > 0 && discovered.size < MAX_PAGES; depth++) {
    const batch = queue.splice(0, queue.length);

    for (const url of batch) {
      if (visited.has(url) || discovered.size >= MAX_PAGES) continue;
      visited.add(url);

      try {
        const links = await fetchPageForLinks(url, base, domain, htmlCache);
        for (const href of links) {
          if (discovered.size >= MAX_PAGES) break;
          discovered.add(href);
          if (!visited.has(href)) queue.push(href);
        }
      } catch { /* skip failed page */ }

      // Delay between fetches to avoid rate limiting
      await new Promise(r => setTimeout(r, 400));
    }
  }


  // ── Step 4: Headless Chrome for JS-rendered sites ──
  if (discovered.size < HEADLESS_THRESHOLD) {
    try {
      const headlessLinks = await discoverWithHeadless(base, domain);
      for (const href of headlessLinks) {
        addUrl(href);
        if (discovered.size >= MAX_PAGES) break;
      }
    } catch (err) {
      const msg = err instanceof Error ? `${err.name}: ${err.message}` : "unknown";
      log.warn("api", `Headless crawl failed (${msg}). Chromium may not be available in this environment.`, {});
    }
  }

  return { urls: [...discovered], htmlCache };
}

// ─── Headless Browser Discovery (for JS-rendered sites) ─────────────────────

async function discoverWithHeadless(baseUrl: string, domain: string): Promise<string[]> {
  const chromium = (await import("@sparticuz/chromium")).default;
  const puppeteer = await import("puppeteer-core");

  const browser = await puppeteer.default.launch({
    args: [...chromium.args, "--no-sandbox", "--disable-gpu"],
    defaultViewport: { width: 1280, height: 720 },
    executablePath: await chromium.executablePath(),
    headless: true,
  });

  const allLinks = new Set<string>();
  const normalDomain = domain.replace(/^www\./, "").toLowerCase();
  const base = baseUrl.replace(/\/$/, "");

  try {
    const page = await browser.newPage();
    await page.setUserAgent(BROWSER_UA);

    // Render homepage and extract links
    const homepageLinks = await renderAndExtract(page, baseUrl, base, normalDomain);
    for (const l of homepageLinks) allLinks.add(l);

    // Try clicking nav/menu elements to expose hidden links
    try {
      // Common nav toggles: hamburger menus, dropdowns
      const navSelectors = [
        'button[aria-label*="menu" i]', 'button[aria-label*="nav" i]',
        '[class*="hamburger"]', '[class*="menu-toggle"]', '[class*="nav-toggle"]',
        'button[class*="menu"]', 'header button',
      ];
      for (const sel of navSelectors) {
        const btn = await page.$(sel);
        if (btn) {
          await btn.click();
          await new Promise(r => setTimeout(r, 1000));
          const moreLinks = await extractLinksFromPage(page, base, normalDomain);
          for (const l of moreLinks) allLinks.add(l);
          break;
        }
      }
    } catch { /* nav interaction failed — continue with what we have */ }

    // Render up to 3 discovered subpages to find deeper links
    const subpages = [...allLinks].filter(u => u !== base).slice(0, 3);
    for (const subUrl of subpages) {
      if (allLinks.size >= MAX_PAGES) break;
      try {
        const subLinks = await renderAndExtract(page, subUrl, base, normalDomain);
        for (const l of subLinks) allLinks.add(l);
      } catch { continue; }
    }
  } finally {
    await browser.close();
  }

  return [...allLinks];
}

/* eslint-disable @typescript-eslint/no-explicit-any */
async function renderAndExtract(page: any, url: string, base: string, normalDomain: string): Promise<string[]> {
  await page.goto(url, { waitUntil: "networkidle2", timeout: 20000 });
  return extractLinksFromPage(page, base, normalDomain);
}

async function extractLinksFromPage(page: any, base: string, normalDomain: string): Promise<string[]> {
  const hrefs = await page.evaluate(() => {
    return Array.from(document.querySelectorAll("a[href]"))
      .map(a => a.getAttribute("href"))
      .filter((h): h is string => !!h);
  });

  const links: string[] = [];
  for (const href of hrefs) {
    let resolved = href;
    if (resolved.startsWith("/")) resolved = `${base}${resolved}`;
    else if (!resolved.startsWith("http")) resolved = `${base}/${resolved}`;

    try {
      const parsed = new URL(resolved);
      if (parsed.hostname.replace(/^www\./, "").toLowerCase() !== normalDomain) continue;
      if (/\.(jpg|jpeg|png|gif|svg|css|js|pdf|zip|ico|woff|woff2|ttf|mp4|webm|webp|xml|json|txt|rss|atom)$/i.test(parsed.pathname)) continue;
      links.push(parsed.origin + parsed.pathname.replace(/\/$/, ""));
    } catch { continue; }
  }
  return links;
}

/** Fetch a page for link extraction, with 429 retry */
async function fetchPageForLinks(
  url: string, base: string, domain: string, htmlCache: Map<string, string>
): Promise<string[]> {
  for (let attempt = 0; attempt < 2; attempt++) {
    const res = await fetch(url, {
      signal: AbortSignal.timeout(FETCH_TIMEOUT),
      headers: { "User-Agent": BROWSER_UA },
      redirect: "follow",
    });

    if (res.status === 429) {
      // Rate limited — wait and retry
      const retryAfter = parseInt(res.headers.get("retry-after") || "2", 10);
      await res.text().catch(() => {});
      await new Promise(r => setTimeout(r, Math.min(retryAfter, 5) * 1000));
      continue;
    }

    if (!res.ok) {
      await res.text().catch(() => {});
      return [];
    }

    const ct = res.headers.get("content-type") ?? "";
    if (!ct.includes("text/html")) {
      await res.text().catch(() => {});
      return [];
    }

    const html = await res.text();
    htmlCache.set(url, html);
    return extractInternalHrefs(html, base, domain);
  }
  return [];
}

async function fetchSitemap(baseUrl: string, domain: string): Promise<string[]> {
  const urls: string[] = [];
  const normalDomain = domain.replace(/^www\./, "").toLowerCase();

  // Try common sitemap locations
  const sitemapPaths = ["/sitemap.xml", "/sitemap_index.xml", "/sitemap-index.xml", "/wp-sitemap.xml"];

  let xml = "";
  for (const path of sitemapPaths) {
    try {
      let res = await fetch(`${baseUrl}${path}`, {
        signal: AbortSignal.timeout(8000),
        headers: { "User-Agent": BROWSER_UA },
        redirect: "follow",
      });
      // Retry once on 429
      if (res.status === 429) {
        await res.text().catch(() => {});
        await new Promise(r => setTimeout(r, 2000));
        res = await fetch(`${baseUrl}${path}`, {
          signal: AbortSignal.timeout(8000),
          headers: { "User-Agent": BROWSER_UA },
          redirect: "follow",
        });
      }
      if (res.ok) {
        xml = await res.text();
        break;
      }
      await res.text().catch(() => {});
    } catch { continue; }
  }

  if (!xml) return [];

  // Check if it's a sitemap index — fetch child sitemaps (up to 3)
  if (xml.includes("<sitemap>") || xml.includes("<sitemapindex")) {
    const indexLocs: string[] = [];
    const idxRegex = /<loc>([^<]+)<\/loc>/gi;
    let idxMatch;
    while ((idxMatch = idxRegex.exec(xml)) !== null) {
      indexLocs.push(idxMatch[1].trim());
    }

    let combinedXml = "";
    for (const childUrl of indexLocs.slice(0, 3)) {
      try {
        const childRes = await fetch(childUrl, {
          signal: AbortSignal.timeout(8000),
          headers: { "User-Agent": BROWSER_UA },
          redirect: "follow",
        });
        if (childRes.ok) combinedXml += await childRes.text();
      } catch { continue; }
    }
    if (combinedXml) xml = combinedXml;
  }

  // Extract <loc> entries for pages
  const locRegex = /<loc>([^<]+)<\/loc>/gi;
  let match;
  while ((match = locRegex.exec(xml)) !== null) {
    const loc = match[1].trim();
    try {
      const parsed = new URL(loc);
      if (parsed.hostname.replace(/^www\./, "").toLowerCase() !== normalDomain) continue;
      if (/\.(jpg|jpeg|png|gif|svg|css|js|pdf|zip|ico|woff|woff2|ttf|mp4|webm|webp|xml)$/i.test(parsed.pathname)) continue;
      urls.push(parsed.origin + parsed.pathname.replace(/\/$/, ""));
      if (urls.length >= MAX_PAGES) break;
    } catch { /* skip */ }
  }

  return urls;
}

function extractInternalHrefs(html: string, baseUrl: string, domain: string): string[] {
  const links: string[] = [];
  const seen = new Set<string>();
  const normalDomain = domain.replace(/^www\./, "").toLowerCase();
  const base = baseUrl.replace(/\/$/, "");
  const hrefRegex = /href=["']([^"'#]+)["']/gi;
  let match;

  while ((match = hrefRegex.exec(html)) !== null) {
    let href = match[1].trim();
    if (!href || href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:") || href.startsWith("data:")) continue;

    // Resolve relative URLs
    if (href.startsWith("/")) {
      href = `${base}${href}`;
    } else if (!href.startsWith("http")) {
      // Relative without leading slash (e.g., "about", "services/seo")
      href = `${base}/${href}`;
    }

    try {
      const parsed = new URL(href);
      if (parsed.hostname.replace(/^www\./, "").toLowerCase() !== normalDomain) continue;
      if (/\.(jpg|jpeg|png|gif|svg|css|js|pdf|zip|ico|woff|woff2|ttf|mp4|webm|webp|xml|json|txt|rss|atom)$/i.test(parsed.pathname)) continue;

      const normalised = parsed.origin + parsed.pathname.replace(/\/$/, "");
      if (!seen.has(normalised)) {
        seen.add(normalised);
        links.push(normalised);
      }
    } catch {
      // skip
    }
  }

  return links;
}

// ─── Per-page crawl + HTML checks ───────────────────────────────────────────

interface LinkDetail {
  href: string;
  anchorText: string;
  selector: string;
  xpath: string;
  snippet: string;
}

interface CrawlResult {
  issues: RawIssue[];
  title: string | null;
  metaDescription: string | null;
  internalLinks: string[];
  internalLinkDetails: LinkDetail[];
}

/** Retry wrapper: tries crawlAndCheck up to 2 times on transient network failure */
async function crawlAndCheckWithRetry(
  pageUrl: string, baseUrl: string, domain: string, htmlCache: Map<string, string>
): Promise<CrawlResult | null> {
  let lastErr: Error | null = null;
  for (let attempt = 0; attempt < 2; attempt++) {
    try {
      return await crawlAndCheck(pageUrl, baseUrl, domain, htmlCache);
    } catch (err) {
      lastErr = err instanceof Error ? err : new Error(String(err));
      if (attempt === 0 && isRetryableError(lastErr)) continue;
      // Classify and rethrow with diagnostic message
      throw new Error(classifyNetworkError(lastErr));
    }
  }
  throw new Error(lastErr ? classifyNetworkError(lastErr) : "Crawl failed after retry");
}

function isRetryableError(err: Error): boolean {
  const msg = err.message.toLowerCase();
  const name = err.name;
  return name === "TimeoutError" || name === "AbortError" ||
    msg.includes("timeout") || msg.includes("aborted") ||
    msg.includes("econnreset") || msg.includes("socket hang up") ||
    msg.includes("fetch failed") || msg.includes("network");
}

async function crawlAndCheck(pageUrl: string, baseUrl: string, domain: string, htmlCache: Map<string, string>): Promise<CrawlResult | null> {
  // Use cached HTML from BFS if available (avoids double-fetch)
  const cachedHtml = htmlCache.get(pageUrl);
  if (cachedHtml) {
    return checkHtml(cachedHtml, pageUrl, baseUrl, domain);
  }

  // Fetch with 429 retry
  let res: Response | null = null;
  for (let attempt = 0; attempt < 2; attempt++) {
    res = await fetch(pageUrl, {
      signal: AbortSignal.timeout(FETCH_TIMEOUT),
      headers: { "User-Agent": BROWSER_UA },
      redirect: "follow",
    });
    if (res.status === 429) {
      const wait = parseInt(res.headers.get("retry-after") || "2", 10);
      await res.text().catch(() => {});
      await new Promise(r => setTimeout(r, Math.min(wait, 5) * 1000));
      continue;
    }
    break;
  }
  if (!res) return null;

  const issues: RawIssue[] = [];

  // HTTP status check
  if (!res.ok) {
    issues.push({
      category: "Links & Crawling",
      severity: res.status >= 500 ? "critical" : res.status === 403 ? "medium" : "high",
      title: `HTTP ${res.status} error`,
      detail: res.status === 429 ? "Page is rate-limiting requests — try again later"
        : res.status === 403 ? "Page is blocking automated access"
        : `Page returned HTTP ${res.status} status`,
      pageUrl,
    });
    return { issues, title: null, metaDescription: null, internalLinks: [], internalLinkDetails: [] };
  }

  const contentType = res.headers.get("content-type") ?? "";
  if (!contentType.includes("text/html")) return null;

  const html = await res.text();
  htmlCache.set(pageUrl, html); // Cache for potential future use
  return checkHtml(html, pageUrl, baseUrl, domain);
}

function checkHtml(html: string, pageUrl: string, baseUrl: string, domain: string): CrawlResult {
  const issues: RawIssue[] = [];

  // ── Title check ──
  const titleMatch = html.match(/<title[^>]*>([^<]*)<\/title>/i);
  const title = titleMatch ? titleMatch[1].trim() : null;
  if (!title) {
    issues.push({ category: "On-page SEO", severity: "critical", title: "Missing title tag", detail: "Page has no <title> element", pageUrl });
  } else if (title.length < 10) {
    issues.push({ category: "On-page SEO", severity: "medium", title: "Title tag too short", detail: `Title is only ${title.length} characters: "${title}"`, pageUrl });
  } else if (title.length > 70) {
    issues.push({ category: "On-page SEO", severity: "low", title: "Title tag too long", detail: `Title is ${title.length} characters (recommended: under 70)`, pageUrl });
  }

  // ── Meta description check ──
  const descMatch = html.match(/<meta[^>]+name=["']description["'][^>]+content=["']([^"']*)["']/i)
    || html.match(/<meta[^>]+content=["']([^"']*)["'][^>]+name=["']description["']/i);
  const metaDescription = descMatch ? descMatch[1].trim() : null;
  if (!metaDescription) {
    issues.push({ category: "On-page SEO", severity: "high", title: "Missing meta description", detail: "Page has no meta description tag", pageUrl });
  } else if (metaDescription.length < 50) {
    issues.push({ category: "On-page SEO", severity: "medium", title: "Meta description too short", detail: `Description is only ${metaDescription.length} characters`, pageUrl });
  }

  // ── H1 check ──
  const h1Match = html.match(/<h1[\s>]/i);
  if (!h1Match) {
    issues.push({ category: "On-page SEO", severity: "high", title: "Missing H1 tag", detail: "Page has no H1 heading element", pageUrl });
  }
  const h1Count = (html.match(/<h1[\s>]/gi) ?? []).length;
  if (h1Count > 1) {
    issues.push({ category: "On-page SEO", severity: "medium", title: "Multiple H1 tags", detail: `Page has ${h1Count} H1 elements (recommended: 1)`, pageUrl });
  }

  // ── Heading order check ──
  const headings = [...html.matchAll(/<(h[1-6])[\s>]/gi)].map(m => parseInt(m[1][1]));
  for (let i = 1; i < headings.length; i++) {
    if (headings[i] > headings[i - 1] + 1) {
      issues.push({
        category: "On-page SEO",
        severity: "low",
        title: "Heading level skipped",
        detail: `Heading jumps from H${headings[i - 1]} to H${headings[i]}`,
        pageUrl,
      });
      break; // Report once per page
    }
  }

  // ── Images missing alt text ──
  const imgsNoAlt = (html.match(/<img(?![^>]*alt=)[^>]*>/gi) ?? []).length;
  const imgsEmptyAlt = (html.match(/<img[^>]*alt=["']\s*["'][^>]*>/gi) ?? []).length;
  const missingAlt = imgsNoAlt + imgsEmptyAlt;
  if (missingAlt > 0) {
    issues.push({
      category: "Accessibility",
      severity: missingAlt >= 5 ? "high" : "medium",
      title: "Images missing alt text",
      detail: `${missingAlt} image${missingAlt !== 1 ? "s" : ""} without alt text`,
      pageUrl,
    });
  }

  // ── Canonical check ──
  const canonicalMatch = html.match(/<link[^>]+rel=["']canonical["'][^>]*>/i);
  if (!canonicalMatch) {
    issues.push({ category: "On-page SEO", severity: "medium", title: "Missing canonical tag", detail: "Page has no canonical link element", pageUrl });
  }

  // ── Accessibility landmark check ──
  const hasMain = /<main[\s>]/i.test(html) || /role=["']main["']/i.test(html);
  const hasNav = /<nav[\s>]/i.test(html) || /role=["']navigation["']/i.test(html);
  const hasHeader = /<header[\s>]/i.test(html) || /role=["']banner["']/i.test(html);
  const hasFooter = /<footer[\s>]/i.test(html) || /role=["']contentinfo["']/i.test(html);
  const missingLandmarks: string[] = [];
  if (!hasMain) missingLandmarks.push("main");
  if (!hasNav) missingLandmarks.push("nav");
  if (!hasHeader) missingLandmarks.push("header");
  if (!hasFooter) missingLandmarks.push("footer");
  if (missingLandmarks.length >= 2) {
    issues.push({
      category: "Accessibility",
      severity: missingLandmarks.includes("main") ? "high" : "medium",
      title: "Missing accessibility landmarks",
      detail: `Page is missing landmark elements: ${missingLandmarks.join(", ")}`,
      pageUrl,
    });
  }

  // ── Collect internal links with details for broken link checks ──
  const internalLinks = extractInternalHrefs(html, baseUrl, domain);
  const internalLinkDetails = extractLinkDetails(html, pageUrl, baseUrl, domain);

  return { issues, title, metaDescription, internalLinks, internalLinkDetails };
}

function extractLinkDetails(html: string, sourcePageUrl: string, baseUrl: string, domain: string): LinkDetail[] {
  const details: LinkDetail[] = [];
  const normalDomain = domain.replace(/^www\./, "").toLowerCase();
  const base = baseUrl.replace(/\/$/, "");
  // Match full <a> tags to get anchor text and surrounding context
  const aTagRegex = /<a\s[^>]*href=["']([^"'#]+)["'][^>]*>([\s\S]*?)<\/a>/gi;
  let match;
  let linkIndex = 0;

  while ((match = aTagRegex.exec(html)) !== null) {
    let href = match[1].trim();
    const rawAnchor = match[2].replace(/<[^>]+>/g, "").trim(); // strip inner HTML tags
    const fullTag = match[0];

    if (!href || href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:") || href.startsWith("data:")) continue;

    // Resolve relative
    if (href.startsWith("/")) href = `${base}${href}`;
    else if (!href.startsWith("http")) href = `${base}/${href}`;

    try {
      const parsed = new URL(href);
      if (parsed.hostname.replace(/^www\./, "").toLowerCase() !== normalDomain) continue;

      const resolved = parsed.origin + parsed.pathname.replace(/\/$/, "");
      linkIndex++;

      // Build a CSS-like selector
      const selector = `a[href="${match[1].replace(/"/g, '\\"')}"]`;
      const xpath = `//a[@href="${match[1].replace(/"/g, "&quot;")}"]`;

      // Snippet: surrounding text (up to 150 chars around the link)
      const tagPos = match.index;
      const snippetStart = Math.max(0, tagPos - 40);
      const snippetEnd = Math.min(html.length, tagPos + fullTag.length + 40);
      const snippet = html.slice(snippetStart, snippetEnd).replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim().slice(0, 150);

      details.push({
        href: resolved,
        anchorText: rawAnchor.slice(0, 200) || "(no text)",
        selector,
        xpath,
        snippet,
      });
    } catch { continue; }
  }

  return details;
}

// ─── Issue Merging ──────────────────────────────────────────────────────────

function mergeIssues(allIssues: RawIssue[]) {
  const byTitle = new Map<string, {
    category: string; severity: string; title: string; detail: string;
    affectedUrls: Set<string>;
    evidence: BrokenLinkEvidence[];
  }>();

  for (const issue of allIssues) {
    const existing = byTitle.get(issue.title);
    if (existing) {
      existing.affectedUrls.add(issue.pageUrl);
      if (severityRank(issue.severity) < severityRank(existing.severity)) {
        existing.severity = issue.severity;
      }
      // Merge evidence (dedupe by sourcePageUrl+brokenHref)
      if (issue.evidence) {
        const seen = new Set(existing.evidence.map(e => `${e.sourcePageUrl}::${e.brokenHref}`));
        for (const e of issue.evidence) {
          const key = `${e.sourcePageUrl}::${e.brokenHref}`;
          if (!seen.has(key)) {
            seen.add(key);
            existing.evidence.push(e);
          }
        }
      }
    } else {
      byTitle.set(issue.title, {
        category: issue.category,
        severity: issue.severity,
        title: issue.title,
        detail: issue.detail,
        affectedUrls: new Set([issue.pageUrl]),
        evidence: issue.evidence ? [...issue.evidence] : [],
      });
    }
  }

  return [...byTitle.values()].map(issue => ({
    category: issue.category,
    severity: issue.severity,
    title: issue.title,
    detail: issue.detail,
    affectedUrls: [...issue.affectedUrls],
    evidence: issue.evidence,
    affected: issue.affectedUrls.size,
  }));
}

function severityRank(s: string): number {
  return s === "critical" ? 0 : s === "high" ? 1 : s === "medium" ? 2 : 3;
}

function chunk<T>(arr: T[], size: number): T[][] {
  const chunks: T[][] = [];
  for (let i = 0; i < arr.length; i += size) chunks.push(arr.slice(i, i + size));
  return chunks;
}
