/**
 * Dead Page Scanner
 * ------------------
 * Discovers a brand's internal links, checks each for 404 status, and
 * records the results in DeadPage / DeadPageSource.
 *
 * Link discovery cascade (each step only runs when the previous fails):
 *
 *   1. Plain fetch with realistic browser headers. Fast path — works for
 *      most sites. Reuses the same User-Agent the technical audit
 *      crawler uses.
 *
 *   2. Headless Chrome via @sparticuz/chromium + puppeteer-core. Handles
 *      Cloudflare challenges, JS-rendered navs, and sites that outright
 *      reject non-browser fetches. Same launch config as audit-runner.ts.
 *
 *   3. Site Tag link map. If the meSEO Site Tag is installed on the
 *      customer's site, PageRecord.internalLinks already contains the
 *      real deduped internal link graph harvested from visitor traffic.
 *      Bypasses bot protection entirely because the data came from the
 *      tag running INSIDE the customer's page.
 *
 * When every step fails, the scanner records a structured FailureReason
 * so the UI can render targeted guidance instead of a generic error.
 */

import { prisma } from "@/lib/prisma";
import { log } from "@/lib/logger";

// Headers matching a real browser — sites behind Cloudflare or other
// WAFs often block requests missing Accept / Accept-Language / etc.
const BROWSER_HEADERS: Record<string, string> = {
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
  "Accept-Language": "en-US,en;q=0.9",
  "Accept-Encoding": "gzip, deflate, br",
  "Cache-Control": "no-cache",
  "Sec-Fetch-Dest": "document",
  "Sec-Fetch-Mode": "navigate",
  "Sec-Fetch-Site": "none",
  "Sec-Fetch-User": "?1",
  "Upgrade-Insecure-Requests": "1",
};

export type FailureReason = "blocked" | "timeout" | "network" | "not_found" | "unknown";

export interface ScanResult {
  scanId: string;
  status: "completed" | "failed";
  pagesChecked: number;
  deadFound: number;
  message: string;
  /** When status is "failed", a machine-readable reason the UI can use to
   *  render targeted guidance (e.g. "blocked" → show WAF allowlist hints). */
  failureReason?: FailureReason;
  /** Which discovery strategy successfully produced the link list. */
  discoverySource?: "fetch" | "headless" | "site_tag";
  /** HTTP status returned by the homepage probe, when available. */
  homepageStatus?: number;
}

export async function runDeadPageScan(brandId: string, domain: string): Promise<ScanResult> {
  const scan = await prisma.deadPageScan.create({
    data: { brandId, status: "running" },
  });

  log.info("api", `Dead page scan started for ${domain}`, { metadata: { brandId, scanId: scan.id } });

  const baseUrl = domain.startsWith("http") ? domain : `https://${domain}`;

  try {
    // ── Stage 1: discover internal links ────────────────────────────────
    const discovery = await discoverInternalLinks(brandId, baseUrl, domain);

    if (!discovery.links.length) {
      // Nothing discovered — every strategy failed or the site has no
      // internal links. Record a structured failure so the UI can surface
      // the right guidance (blocked vs no-tag-installed vs empty-site).
      await prisma.deadPageScan.update({
        where: { id: scan.id },
        data: {
          status: "failed",
          errorMessage: discovery.message.slice(0, 500),
          completedAt: new Date(),
        },
      });
      log.warn("api", `Dead page scan failed for ${domain}: ${discovery.message}`, {
        metadata: { scanId: scan.id, reason: discovery.reason, homepageStatus: discovery.homepageStatus },
      });
      return {
        scanId: scan.id,
        status: "failed",
        pagesChecked: 0,
        deadFound: 0,
        message: discovery.message,
        failureReason: discovery.reason,
        homepageStatus: discovery.homepageStatus,
      };
    }

    const uniqueLinks = Array.from(new Set(discovery.links)).slice(0, 500);
    log.info("api", `Dead page scan: checking ${uniqueLinks.length} unique links for ${domain}`, { metadata: { scanId: scan.id, source: discovery.source } });

    // ── Stage 2: check each link for dead status via HEAD request ─────
    // Runs in parallel batches of 20. Each request has a 6-second
    // timeout to keep batches fast. A runtime guard stops at 100 seconds
    // to stay within Vercel's maxDuration (120s) with room for Stage 3.
    const BATCH_SIZE = 20;
    const PER_REQUEST_TIMEOUT = 6000;
    const RUNTIME_CAP_MS = 100_000;
    const scanStartedAt = Date.now();

    const deadPages: Array<{ deadUrl: string; sourceUrl: string; status: number | null }> = [];
    let checked = 0;
    let stoppedEarly = false;

    for (let i = 0; i < uniqueLinks.length; i += BATCH_SIZE) {
      // Runtime guard — stop before we exceed the function timeout
      if (Date.now() - scanStartedAt > RUNTIME_CAP_MS) {
        stoppedEarly = true;
        log.warn("api", `Dead page scan: runtime cap reached at ${checked}/${uniqueLinks.length} links for ${domain}`, { metadata: { scanId: scan.id } });
        break;
      }

      const batch = uniqueLinks.slice(i, i + BATCH_SIZE);
      const results = await Promise.allSettled(
        batch.map(async (link) => {
          const res = await fetch(link, {
            method: "HEAD",
            signal: AbortSignal.timeout(PER_REQUEST_TIMEOUT),
            headers: BROWSER_HEADERS,
            redirect: "follow",
          });
          return { url: link, status: res.status };
        }),
      );
      for (let j = 0; j < results.length; j++) {
        checked++;
        const result = results[j];
        if (result.status === "fulfilled") {
          const s = result.value.status;
          if (s === 404 || s === 410) {
            deadPages.push({ deadUrl: result.value.url, sourceUrl: baseUrl, status: s });
          } else if (s >= 500 && s < 600) {
            deadPages.push({ deadUrl: result.value.url, sourceUrl: baseUrl, status: s });
          }
        } else {
          deadPages.push({ deadUrl: batch[j] ?? "", sourceUrl: baseUrl, status: null });
        }
      }
    }

    const remaining = stoppedEarly ? uniqueLinks.length - checked : 0;
    log.info("api", `Dead page scan: ${checked} checked, ${deadPages.length} dead for ${domain}${stoppedEarly ? ` (stopped early, ${remaining} remaining)` : ""}`, { metadata: { scanId: scan.id } });

    // ── Stage 2b: Cross-reference with Site Tag http_error events ─────
    // If the Site Tag has already captured real-user 404 encounters that
    // the crawler didn't find (e.g. pages linked from deep content), add
    // those to the results with higher priority.
    try {
      const since = new Date(Date.now() - 30 * 86400000);
      const siteTagErrors = await prisma.siteEvent.findMany({
        where: { brandId, eventType: { in: ["http_error", "soft_404"] }, timestamp: { gte: since } },
        select: { pageUrl: true, eventData: true },
        take: 500,
      });
      const crawledUrls = new Set(uniqueLinks.map((u) => extractPath(u)));
      const deadUrlPaths = new Set(deadPages.map((d) => extractPath(d.deadUrl)));
      const siteTag404s = new Map<string, number>();
      for (const e of siteTagErrors) {
        const d = (e.eventData ?? {}) as Record<string, unknown>;
        const code = typeof d.statusCode === "number" ? d.statusCode : 404;
        if (code >= 400) siteTag404s.set(e.pageUrl, (siteTag404s.get(e.pageUrl) ?? 0) + 1);
      }
      for (const [url, hitCount] of siteTag404s) {
        const path = extractPath(url);
        // Skip if already found by the crawler
        if (deadUrlPaths.has(path)) continue;
        deadPages.push({ deadUrl: url, sourceUrl: "site-tag", status: 404 });
        log.info("api", `Dead page scan: Site Tag found additional 404 '${path}' (${hitCount} user hits)`, { metadata: { scanId: scan.id } });
      }
    } catch { /* Site Tag cross-reference is best-effort */ }

    // ── Stage 3: persist new dead pages ────────────────────────────────
    for (const dp of deadPages) {
      if (!dp.deadUrl) continue;
      const path = extractPath(dp.deadUrl);
      const isServerError = dp.status !== null && dp.status >= 500;
      const isFromSiteTag = dp.sourceUrl === "site-tag";
      const priority = isServerError ? "critical"
        : isFromSiteTag ? "high"
        : dp.status === null ? "high"
        : "medium";
      const fix = isServerError
        ? `Server returned ${dp.status}. Check server logs and hosting status.`
        : isFromSiteTag
        ? "Real users hit this 404. Add a 301 redirect or restore the page."
        : "Review and redirect or restore this page.";

      const existing = await prisma.deadPage.findFirst({
        where: { brandId, deadUrl: path },
      });
      if (!existing) {
        await prisma.deadPage.create({
          data: {
            brandId,
            deadUrl: path,
            priority,
            suggestedFix: fix,
            status: "open",
            sources: { create: { sourceUrl: isFromSiteTag ? "(Site Tag)" : extractPath(dp.sourceUrl), anchorText: null } },
          },
        });
      } else if (isFromSiteTag && existing.priority !== "critical") {
        await prisma.deadPage.update({
          where: { id: existing.id },
          data: { priority: "high", suggestedFix: `Confirmed by real users. ${existing.suggestedFix ?? fix}` },
        });
      }
    }

    const partialNote = stoppedEarly ? ` (scan incomplete — ${remaining} links remaining, will be checked on next scan)` : "";
    await prisma.deadPageScan.update({
      where: { id: scan.id },
      data: {
        status: "completed",
        pagesChecked: checked,
        deadFound: deadPages.length,
        errorMessage: stoppedEarly ? `Partial scan: checked ${checked}/${uniqueLinks.length} links before runtime cap.` : null,
        completedAt: new Date(),
      },
    });

    log.info("api", `Dead page scan completed for ${domain}: ${checked} checked, ${deadPages.length} dead (via ${discovery.source})${partialNote}`, {
      metadata: { scanId: scan.id, source: discovery.source },
    });

    return {
      scanId: scan.id,
      status: "completed",
      pagesChecked: checked,
      deadFound: deadPages.length,
      message: `Scanned ${checked} pages, found ${deadPages.length} dead pages.${partialNote}`,
      discoverySource: discovery.source,
    };
  } catch (err) {
    const msg = err instanceof Error ? err.message : "Scan failed";
    await prisma.deadPageScan.update({
      where: { id: scan.id },
      data: { status: "failed", errorMessage: msg.slice(0, 500), completedAt: new Date() },
    });
    log.error("api", `Dead page scan error for ${domain}: ${msg}`, { metadata: { scanId: scan.id } });
    return {
      scanId: scan.id,
      status: "failed",
      pagesChecked: 0,
      deadFound: 0,
      message: msg,
      failureReason: "unknown",
    };
  }
}

// ═══════════════════════════════════════════════════════════════════════════
// Discovery cascade
// ═══════════════════════════════════════════════════════════════════════════

interface DiscoveryResult {
  links: string[];
  source: "fetch" | "headless" | "site_tag";
  reason: FailureReason;
  message: string;
  homepageStatus?: number;
}

async function discoverInternalLinks(brandId: string, baseUrl: string, domain: string): Promise<DiscoveryResult> {
  // ── 1. Plain fetch ──
  const fetchAttempt = await tryFetchHomepage(baseUrl, domain);
  if (fetchAttempt.links.length > 0) {
    return { ...fetchAttempt, source: "fetch" };
  }

  // ── 2. Headless Chrome ──
  // Only runs when plain fetch failed or came back empty. Reuses the
  // same @sparticuz/chromium + puppeteer-core stack as the Technical
  // Audit crawler. The dynamic import keeps Chromium out of the cold-
  // start path for sites that don't need it.
  log.info("api", `Dead page scan: fetch failed (${fetchAttempt.reason}), trying headless for ${domain}`);
  const headlessAttempt = await tryHeadlessHomepage(baseUrl, domain);
  if (headlessAttempt.links.length > 0) {
    return { ...headlessAttempt, source: "headless" };
  }

  // ── 3. Site Tag link map ──
  // Last resort for sites behind WAFs that block both fetch and headless.
  // If the meSEO Site Tag is installed, it's already collected the real
  // internal link graph from visitor traffic and stored it on
  // PageRecord.internalLinks. No remote requests needed for discovery.
  log.info("api", `Dead page scan: headless failed (${headlessAttempt.reason}), trying Site Tag for ${domain}`);
  const siteTagLinks = await collectSiteTagLinks(brandId, baseUrl);
  if (siteTagLinks.length > 0) {
    return {
      links: siteTagLinks,
      source: "site_tag",
      reason: "unknown",
      message: `Discovered ${siteTagLinks.length} links via meSEO Site Tag`,
    };
  }

  // Everything failed — bubble up the most informative failure reason.
  // Prefer the fetch attempt's reason since that's what the user will
  // most likely need to act on (e.g. "blocked" means WAF allowlist).
  return {
    links: [],
    source: "fetch",
    reason: fetchAttempt.reason,
    message: buildFailureMessage(fetchAttempt, headlessAttempt),
    homepageStatus: fetchAttempt.homepageStatus,
  };
}

interface AttemptResult {
  links: string[];
  reason: FailureReason;
  message: string;
  homepageStatus?: number;
}

async function tryFetchHomepage(baseUrl: string, domain: string): Promise<AttemptResult> {
  try {
    const res = await fetch(baseUrl, {
      signal: AbortSignal.timeout(15000),
      headers: BROWSER_HEADERS,
      redirect: "follow",
    });
    if (!res.ok) {
      return {
        links: [],
        reason: res.status === 403 || res.status === 401 || res.status === 429 ? "blocked" : "not_found",
        message: `Homepage returned HTTP ${res.status}`,
        homepageStatus: res.status,
      };
    }
    const html = await res.text();
    const links = extractInternalLinks(html, baseUrl, domain);
    return { links, reason: "unknown", message: `${links.length} links found via fetch`, homepageStatus: res.status };
  } catch (err) {
    const isTimeout = err instanceof Error && err.name === "TimeoutError";
    return {
      links: [],
      reason: isTimeout ? "timeout" : "network",
      message: err instanceof Error ? err.message : "Network error",
    };
  }
}

async function tryHeadlessHomepage(baseUrl: string, domain: string): Promise<AttemptResult> {
  let browser: unknown = null;
  try {
    // Dynamic import so Chromium stays out of the bundle for the happy path
    const chromium = (await import("@sparticuz/chromium")).default;
    const puppeteer = await import("puppeteer-core");
    browser = await puppeteer.default.launch({
      args: [...chromium.args, "--no-sandbox", "--disable-gpu"],
      defaultViewport: { width: 1280, height: 720 },
      executablePath: await chromium.executablePath(),
      headless: true,
    });
    const page = await (browser as { newPage: () => Promise<unknown> }).newPage();
    const p = page as {
      setUserAgent: (ua: string) => Promise<void>;
      setExtraHTTPHeaders: (h: Record<string, string>) => Promise<void>;
      goto: (url: string, opts: { waitUntil: string; timeout: number }) => Promise<{ status: () => number } | null>;
      content: () => Promise<string>;
      close: () => Promise<void>;
    };
    await p.setUserAgent(BROWSER_HEADERS["User-Agent"]);
    const { "User-Agent": _ua, ...rest } = BROWSER_HEADERS;
    void _ua;
    await p.setExtraHTTPHeaders(rest);
    const response = await p.goto(baseUrl, { waitUntil: "domcontentloaded", timeout: 20000 });
    const status = response?.status() ?? 0;
    if (status >= 400) {
      await p.close();
      return {
        links: [],
        reason: status === 403 || status === 401 ? "blocked" : "not_found",
        message: `Headless homepage returned HTTP ${status}`,
        homepageStatus: status,
      };
    }
    const html = await p.content();
    await p.close();
    const links = extractInternalLinks(html, baseUrl, domain);
    return { links, reason: "unknown", message: `${links.length} links found via headless`, homepageStatus: status };
  } catch (err) {
    return {
      links: [],
      reason: "network",
      message: err instanceof Error ? err.message : "Headless launch failed",
    };
  } finally {
    if (browser) {
      try { await (browser as { close: () => Promise<void> }).close(); } catch { /* swallow */ }
    }
  }
}

/**
 * Pull the internal link map that the meSEO Site Tag has been collecting
 * via POST /api/site-tag/page-links. Each PageRecord.internalLinks field
 * is a JSON array of { href, anchorText } objects where href is already
 * normalised to a path. Converts paths back to absolute URLs so the HEAD-
 * check stage downstream can hit them.
 */
async function collectSiteTagLinks(brandId: string, baseUrl: string): Promise<string[]> {
  const records = await prisma.pageRecord.findMany({
    where: { brandId, internalLinks: { not: { equals: null } } },
    select: { internalLinks: true },
    take: 500,
  });

  const origin = baseUrl.replace(/\/$/, "");
  const seen = new Set<string>();
  for (const r of records) {
    if (!Array.isArray(r.internalLinks)) continue;
    for (const link of r.internalLinks as unknown as Array<{ href?: string }>) {
      if (!link || typeof link.href !== "string") continue;
      const path = link.href.startsWith("/") ? link.href : "/" + link.href;
      seen.add(origin + path);
    }
  }
  return Array.from(seen);
}

function buildFailureMessage(fetchAttempt: AttemptResult, headlessAttempt: AttemptResult): string {
  if (fetchAttempt.reason === "blocked") {
    return `The website returned HTTP ${fetchAttempt.homepageStatus ?? "4xx"}. This usually means bot protection (Cloudflare, WAF) is blocking the scan.`;
  }
  if (fetchAttempt.reason === "timeout") {
    return "The website took too long to respond. Try again in a moment or check if the site is up.";
  }
  if (headlessAttempt.reason === "blocked") {
    return `The website is blocking automated access (HTTP ${headlessAttempt.homepageStatus ?? "4xx"}).`;
  }
  return fetchAttempt.message || "Could not discover internal links.";
}

// ─── Helpers ────────────────────────────────────────────────────────────────

function extractInternalLinks(html: string, baseUrl: string, domain: string): string[] {
  const links: string[] = [];
  const hrefRegex = /href=["']([^"']+)["']/gi;
  let match;
  const normalDomain = domain.replace(/^www\./, "").replace(/^https?:\/\//, "").toLowerCase();

  while ((match = hrefRegex.exec(html)) !== null) {
    let href = match[1];
    if (!href || href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:")) continue;

    if (href.startsWith("/")) {
      href = `${baseUrl.replace(/\/$/, "")}${href}`;
    }

    try {
      const parsed = new URL(href);
      if (parsed.hostname.replace(/^www\./, "").toLowerCase() === normalDomain) {
        links.push(parsed.origin + parsed.pathname);
      }
    } catch {
      // Skip malformed URLs
    }
  }

  return links;
}

function extractPath(fullUrl: string): string {
  try { return new URL(fullUrl).pathname; } catch { return fullUrl; }
}
