/**
 * Competitor Sitemap Crawler
 * ──────────────────────────
 * Discovers, fetches, and parses competitor sitemaps. Handles standard
 * XML sitemaps, sitemap indexes, compressed (.xml.gz) files, and
 * non-standard paths for WordPress, Shopify, Wix, Squarespace, Drupal,
 * HubSpot, Webflow, Magento, Next.js, Joomla, BigCommerce, Ghost, and
 * other platforms.
 *
 * Discovery order:
 *   1. robots.txt Sitemap: directives (canonical)
 *   2. Standard paths (/sitemap.xml, /sitemap_index.xml)
 *   3. CMS-specific paths (WordPress/Yoast/RankMath, Shopify, etc.)
 *
 * Each crawl is diffed against the stored CompetitorPage rows to detect
 * new, updated, unchanged, and removed URLs.
 */

import { prisma } from "@/lib/prisma";
import { XMLParser } from "fast-xml-parser";
import { gunzipSync } from "zlib";

// ─── Constants ──────────────────────────────────────────────────────────────

const FETCH_TIMEOUT = 10_000;
const MAX_INDEX_DEPTH = 3;     // max nesting of sitemapindex → sitemap
const MAX_CHILD_SITEMAPS = 50; // per index
const MAX_URLS = 50_000;       // safety cap per crawl
const DETAIL_FETCH_LIMIT = 50; // max pages to scrape title/h1/wordCount per crawl
const SCRAPE_DELAY_MS = 300;   // polite delay between page scrapes
const UA = "meSEO-Bot/1.0 (+https://meseo.io/bot)";

const parser = new XMLParser({ ignoreAttributes: false });

// ─── Types ──────────────────────────────────────────────────────────────────

export interface SitemapEntry {
  loc: string;
  lastmod?: string;
  changefreq?: string;
  priority?: number;
}

export interface DiscoveryResult {
  sitemapUrls: string[];
  entries: SitemapEntry[];
  detectedCms: string | null;
}

export interface CrawlDiff {
  newPages: string[];
  updatedPages: string[];
  removedPages: string[];
  unchangedCount: number;
  totalPages: number;
}

export interface CrawlResult extends CrawlDiff {
  sitemapUrls: string[];
  detectedCms: string | null;
  detailsFetched: number;
  /** Pages that need scraping but were deferred to the next crawl run. */
  deferredScrapes: number;
}

// ─── HTTP helpers ───────────────────────────────────────────────────────────

async function safeFetch(url: string): Promise<{ body: Buffer | null; contentType: string }> {
  try {
    const ctrl = new AbortController();
    const timer = setTimeout(() => ctrl.abort(), FETCH_TIMEOUT);
    const res = await fetch(url, {
      signal: ctrl.signal,
      headers: { "User-Agent": UA, "Accept-Encoding": "gzip, deflate" },
      redirect: "follow",
    });
    clearTimeout(timer);
    if (!res.ok) return { body: null, contentType: "" };
    const ct = res.headers.get("content-type") ?? "";
    const buf = Buffer.from(await res.arrayBuffer());
    return { body: buf, contentType: ct };
  } catch {
    return { body: null, contentType: "" };
  }
}

function sleep(ms: number) {
  return new Promise<void>((r) => setTimeout(r, ms));
}

/** Decompress if gzipped, otherwise return as-is. */
function maybeGunzip(buf: Buffer, url: string, contentType: string): string {
  const isGz = url.endsWith(".gz") ||
    contentType.includes("gzip") ||
    (buf[0] === 0x1f && buf[1] === 0x8b); // gzip magic bytes
  if (isGz) {
    try { return gunzipSync(buf).toString("utf-8"); } catch { /* not actually gzipped */ }
  }
  return buf.toString("utf-8");
}

// ─── XML parsing ────────────────────────────────────────────────────────────

interface ParsedSitemap {
  type: "index" | "urlset";
  /** For index: child sitemap locs. For urlset: page entries. */
  locs: string[];
  entries: SitemapEntry[];
}

function parseSitemapXml(xml: string): ParsedSitemap {
  try {
    const doc = parser.parse(xml);

    // Sitemap index
    const sitemaps = doc?.sitemapindex?.sitemap;
    if (sitemaps) {
      const items = Array.isArray(sitemaps) ? sitemaps : [sitemaps];
      const locs = items
        .map((s: Record<string, unknown>) => (s.loc ?? s["@_loc"]) as string | undefined)
        .filter(Boolean) as string[];
      return { type: "index", locs, entries: [] };
    }

    // URL set
    const urls = doc?.urlset?.url;
    if (urls) {
      const items = Array.isArray(urls) ? urls : [urls];
      const entries: SitemapEntry[] = items.map((u: Record<string, unknown>) => ({
        loc: ((u.loc ?? u["@_loc"]) as string) || "",
        lastmod: (u.lastmod as string) || undefined,
        changefreq: (u.changefreq as string) || undefined,
        priority: u.priority != null ? Number(u.priority) : undefined,
      })).filter((e) => e.loc);
      return { type: "urlset", locs: [], entries };
    }

    return { type: "urlset", locs: [], entries: [] };
  } catch {
    return { type: "urlset", locs: [], entries: [] };
  }
}

// ─── Sitemap Discovery ──────────────────────────────────────────────────────

/** Parse robots.txt for Sitemap: directives. */
function parseSitemapDirectives(robotsTxt: string): string[] {
  const urls: string[] = [];
  for (const line of robotsTxt.split("\n")) {
    const match = line.match(/^\s*Sitemap:\s*(https?:\/\/\S+)/i);
    if (match) urls.push(match[1].trim());
  }
  return urls;
}

/** CMS detection from sitemap paths. */
function detectCms(sitemapUrls: string[]): string | null {
  const joined = sitemapUrls.join(" ");
  if (/wp-sitemap/i.test(joined)) return "wordpress";
  if (/sitemap-pt-/i.test(joined) || /post-sitemap|page-sitemap|category-sitemap/i.test(joined)) return "wordpress-yoast";
  if (/local-sitemap/i.test(joined)) return "wordpress-rankmath";
  if (/sitemap_products_|sitemap_collections_|sitemap_blogs_/i.test(joined)) return "shopify";
  if (/sitemap-pages|sitemap-products|sitemap-events/i.test(joined)) return "wix-or-squarespace";
  if (/sitemap\/default\/sitemap/i.test(joined)) return "drupal";
  if (/sitemap-0\.xml|sitemap-1\.xml/i.test(joined)) return "hubspot-or-nextjs";
  if (/sitemap-collections/i.test(joined)) return "webflow";
  if (/media\/sitemap|pub\/sitemap/i.test(joined)) return "magento";
  if (/xmlsitemap\.php/i.test(joined)) return "bigcommerce";
  if (/sitemap-authors|sitemap-tags/i.test(joined)) return "ghost";
  if (/server-sitemap/i.test(joined)) return "nextjs";
  return null;
}

// Standard and CMS-specific paths to try, de-duplicated and ordered by
// likelihood. We stop as soon as we find a valid sitemap (or index).
const DISCOVERY_PATHS = [
  "/sitemap.xml",
  "/sitemap_index.xml",
  "/sitemap-index.xml",
  "/sitemaps.xml",
  // WordPress
  "/wp-sitemap.xml",
  "/post-sitemap.xml",
  "/page-sitemap.xml",
  // Shopify
  "/sitemap_products_1.xml",
  "/sitemap_pages_1.xml",
  // Wix / Squarespace
  "/sitemap-pages.xml",
  // Drupal
  "/sitemap/default/sitemap.xml",
  // HubSpot / Next.js
  "/sitemap-0.xml",
  "/server-sitemap.xml",
  "/server-sitemap-index.xml",
  // Magento
  "/media/sitemap.xml",
  "/pub/sitemap.xml",
  "/sitemap/sitemap.xml",
  // Webflow
  "/sitemap-collections.xml",
  // BigCommerce
  "/xmlsitemap.php",
  // Ghost
  "/sitemap-posts.xml",
  // GoDaddy
  "/_api/sitemap.xml",
  // Joomla
  "/component/jmap/sitemap.xml",
  // Compressed
  "/sitemap.xml.gz",
];

/**
 * Recursively fetch a sitemap URL: if it's a sitemap index, follow child
 * sitemaps up to MAX_INDEX_DEPTH. Returns all page entries found.
 */
async function fetchSitemapTree(
  url: string,
  depth: number,
  collectedUrls: string[],  // mutated: all sitemap URLs visited
): Promise<SitemapEntry[]> {
  if (depth > MAX_INDEX_DEPTH) return [];
  const { body, contentType } = await safeFetch(url);
  if (!body) return [];

  const xml = maybeGunzip(body, url, contentType);
  const parsed = parseSitemapXml(xml);
  collectedUrls.push(url);

  if (parsed.type === "index") {
    const entries: SitemapEntry[] = [];
    const children = parsed.locs.slice(0, MAX_CHILD_SITEMAPS);
    for (const child of children) {
      if (entries.length >= MAX_URLS) break;
      const childEntries = await fetchSitemapTree(child, depth + 1, collectedUrls);
      entries.push(...childEntries);
      await sleep(150); // polite delay between child fetches
    }
    return entries;
  }

  return parsed.entries;
}

/**
 * Discover and fetch all sitemaps for a domain. Returns the full set of
 * page entries found across all sitemaps, plus the sitemap URLs visited
 * and the detected CMS platform.
 */
export async function discoverAndFetch(domain: string): Promise<DiscoveryResult> {
  const clean = domain.replace(/^https?:\/\//, "").replace(/\/$/, "");
  const base = `https://${clean}`;
  const collectedSitemapUrls: string[] = [];
  let allEntries: SitemapEntry[] = [];

  // ── 1. Check robots.txt first ─────────────────────────────────────
  const { body: robotsBody } = await safeFetch(`${base}/robots.txt`);
  if (robotsBody) {
    const robotsTxt = robotsBody.toString("utf-8");
    const sitemapDirectives = parseSitemapDirectives(robotsTxt);
    if (sitemapDirectives.length > 0) {
      for (const sitemapUrl of sitemapDirectives) {
        if (allEntries.length >= MAX_URLS) break;
        const entries = await fetchSitemapTree(sitemapUrl, 0, collectedSitemapUrls);
        allEntries.push(...entries);
        await sleep(150);
      }
      if (allEntries.length > 0) {
        return {
          sitemapUrls: [...new Set(collectedSitemapUrls)],
          entries: allEntries.slice(0, MAX_URLS),
          detectedCms: detectCms(collectedSitemapUrls),
        };
      }
    }
  }

  // ── 2. Probe standard + CMS-specific paths ───────────────────────
  for (const path of DISCOVERY_PATHS) {
    const entries = await fetchSitemapTree(`${base}${path}`, 0, collectedSitemapUrls);
    if (entries.length > 0) {
      allEntries = entries;
      break; // Found a valid sitemap — stop probing
    }
    await sleep(100);
  }

  return {
    sitemapUrls: [...new Set(collectedSitemapUrls)],
    entries: allEntries.slice(0, MAX_URLS),
    detectedCms: detectCms(collectedSitemapUrls),
  };
}

// ─── Page detail scraper ────────────────────────────────────────────────────

interface PageMeta {
  title: string | null;
  metaDescription: string | null;
  h1: string | null;
  wordCount: number | null;
}

/**
 * Lightweight scrape: fetch just enough HTML to extract <title>,
 * <meta name="description">, first <h1>, and approximate word count.
 * Only reads the first 200 KB to avoid pulling huge pages.
 */
async function scrapePageMeta(url: string): Promise<PageMeta> {
  const empty: PageMeta = { title: null, metaDescription: null, h1: null, wordCount: null };
  try {
    const ctrl = new AbortController();
    const timer = setTimeout(() => ctrl.abort(), FETCH_TIMEOUT);
    const res = await fetch(url, {
      signal: ctrl.signal,
      headers: { "User-Agent": UA },
      redirect: "follow",
    });
    clearTimeout(timer);
    if (!res.ok || !res.body) return empty;

    // Read up to 200 KB
    const reader = res.body.getReader();
    const chunks: Uint8Array[] = [];
    let totalBytes = 0;
    while (totalBytes < 200_000) {
      const { done, value } = await reader.read();
      if (done || !value) break;
      chunks.push(value);
      totalBytes += value.length;
    }
    reader.cancel().catch(() => {});
    const html = Buffer.concat(chunks).toString("utf-8");

    const titleMatch = html.match(/<title[^>]*>([^<]*)<\/title>/i);
    const descMatch = html.match(/<meta\s[^>]*name=["']description["'][^>]*content=["']([^"']*)["']/i)
      ?? html.match(/<meta\s[^>]*content=["']([^"']*)["'][^>]*name=["']description["']/i);
    const h1Match = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);

    // Word count: strip all tags, collapse whitespace, count words
    const text = html
      .replace(/<script[\s\S]*?<\/script>/gi, "")
      .replace(/<style[\s\S]*?<\/style>/gi, "")
      .replace(/<[^>]+>/g, " ")
      .replace(/\s+/g, " ")
      .trim();
    const words = text.split(/\s+/).filter((w) => w.length > 0);

    return {
      title: titleMatch?.[1]?.trim().slice(0, 500) ?? null,
      metaDescription: descMatch?.[1]?.trim().slice(0, 1000) ?? null,
      h1: h1Match?.[1]?.replace(/<[^>]+>/g, "").trim().slice(0, 500) ?? null,
      wordCount: words.length > 0 ? words.length : null,
    };
  } catch {
    return empty;
  }
}

// ─── Diff + Persist ─────────────────────────────────────────────────────────

/**
 * Full crawl pipeline for one competitor:
 *   1. Discover + fetch sitemaps (or use cached sitemap URLs)
 *   2. Diff against stored CompetitorPage rows
 *   3. Upsert new/updated/removed pages
 *   4. Scrape title/h1 for new and updated pages (up to DETAIL_FETCH_LIMIT)
 *   5. Update Competitor.sitemapUrls, lastCrawledAt, detectedCms
 *   6. Create a CompetitorSnapshot for the historical record
 */
export async function crawlCompetitor(competitorId: string): Promise<CrawlResult> {
  const comp = await prisma.competitor.findUnique({
    where: { id: competitorId },
    select: { id: true, domain: true, brandId: true, sitemapUrls: true },
  });
  if (!comp) throw new Error(`Competitor ${competitorId} not found`);

  // ── Fetch ────────────────────────────────────────────────────────
  let discovery: DiscoveryResult;

  // If we have cached sitemap URLs, try them first for speed
  const cachedUrls = (comp.sitemapUrls as string[] | null) ?? [];
  if (cachedUrls.length > 0) {
    const collectedUrls: string[] = [];
    const entries: SitemapEntry[] = [];
    for (const url of cachedUrls) {
      if (entries.length >= MAX_URLS) break;
      entries.push(...await fetchSitemapTree(url, 0, collectedUrls));
      await sleep(150);
    }
    if (entries.length > 0) {
      discovery = {
        sitemapUrls: [...new Set(collectedUrls)],
        entries: entries.slice(0, MAX_URLS),
        detectedCms: detectCms(collectedUrls),
      };
    } else {
      // Cached URLs no longer work — full re-discovery
      discovery = await discoverAndFetch(comp.domain);
    }
  } else {
    discovery = await discoverAndFetch(comp.domain);
  }

  // ── Diff ─────────────────────────────────────────────────────────
  const existingPages = await prisma.competitorPage.findMany({
    where: { competitorId },
    select: { id: true, url: true, lastmod: true, status: true },
  });
  const existingByUrl = new Map(existingPages.map((p) => [p.url, p]));

  const crawledUrls = new Set(discovery.entries.map((e) => e.loc));
  const newPages: string[] = [];
  const updatedPages: string[] = [];
  let unchangedCount = 0;
  const now = new Date();

  // Process entries from the sitemap
  for (const entry of discovery.entries) {
    const existing = existingByUrl.get(entry.loc);
    const entryLastmod = entry.lastmod ? new Date(entry.lastmod) : null;

    if (!existing) {
      // New page
      newPages.push(entry.loc);
      await prisma.competitorPage.create({
        data: {
          competitorId,
          url: entry.loc,
          lastmod: entryLastmod,
          changefreq: entry.changefreq,
          priority: entry.priority,
          status: "new",
          firstSeenAt: now,
          lastCheckedAt: now,
        },
      });
    } else {
      // Existing page — check if lastmod changed
      const prevLastmod = existing.lastmod?.getTime() ?? 0;
      const currLastmod = entryLastmod?.getTime() ?? 0;
      const isUpdated = currLastmod > 0 && currLastmod !== prevLastmod;

      if (isUpdated) {
        updatedPages.push(entry.loc);
        await prisma.competitorPage.update({
          where: { id: existing.id },
          data: {
            lastmod: entryLastmod,
            changefreq: entry.changefreq,
            priority: entry.priority,
            status: "updated",
            lastCheckedAt: now,
          },
        });
      } else {
        unchangedCount++;
        if (existing.status !== "unchanged") {
          await prisma.competitorPage.update({
            where: { id: existing.id },
            data: { status: "unchanged", lastCheckedAt: now },
          });
        } else {
          await prisma.competitorPage.update({
            where: { id: existing.id },
            data: { lastCheckedAt: now },
          });
        }
      }
    }
  }

  // Pages in DB but no longer in sitemap → removed
  const removedPages: string[] = [];
  for (const [url, existing] of existingByUrl) {
    if (!crawledUrls.has(url) && existing.status !== "removed") {
      removedPages.push(url);
      await prisma.competitorPage.update({
        where: { id: existing.id },
        data: { status: "removed", lastCheckedAt: now },
      });
    }
  }

  // ── Scrape details for new + updated pages ───────────────────────
  // Priority order for the per-crawl scrape budget:
  //   1. Pages explicitly flagged `pending_scrape` from a previous run
  //      (they were deferred because the cap was reached).
  //   2. Fresh pages (new + updated in this crawl).
  //   3. Backfill pass — pages that exist in the DB but never got their
  //      wordCount populated. Each re-crawl chips away at the long
  //      tail so a 281-page competitor eventually has 100% coverage
  //      instead of stalling at the 50-page first-run cap forever.
  const pendingScrape = await prisma.competitorPage.findMany({
    where: { competitorId, status: "pending_scrape" },
    select: { url: true },
    take: DETAIL_FETCH_LIMIT,
  });
  const pendingUrls = pendingScrape.map((p) => p.url);
  const freshUrls = [...newPages, ...updatedPages];

  // Backfill pool: pages that are otherwise healthy (new/updated/
  // unchanged — not removed) but never got a wordCount. Ordered by
  // firstSeenAt descending so newer content gets filled first.
  const missingBudget = Math.max(0, DETAIL_FETCH_LIMIT - pendingUrls.length - freshUrls.length);
  const missingWordCount = missingBudget > 0
    ? await prisma.competitorPage.findMany({
        where: {
          competitorId,
          wordCount: null,
          status: { notIn: ["removed", "pending_scrape"] },
        },
        orderBy: { firstSeenAt: "desc" },
        select: { url: true },
        take: missingBudget,
      })
    : [];
  const backfillUrls = missingWordCount.map((p) => p.url);

  const toScrape = [
    ...pendingUrls,
    ...freshUrls.filter((u) => !pendingUrls.includes(u)),
    ...backfillUrls.filter((u) => !pendingUrls.includes(u) && !freshUrls.includes(u)),
  ].slice(0, DETAIL_FETCH_LIMIT);
  const deferred = freshUrls.filter((u) => !toScrape.includes(u));

  let detailsFetched = 0;
  let backfilled = 0;
  for (const url of toScrape) {
    const meta = await scrapePageMeta(url);
    // Scraped successfully — clear pending_scrape if it was set, otherwise
    // keep the page's real status (new / updated).
    await prisma.competitorPage.update({
      where: { competitorId_url: { competitorId, url } },
      data: {
        title: meta.title,
        metaDescription: meta.metaDescription,
        h1: meta.h1,
        wordCount: meta.wordCount,
        ...(pendingUrls.includes(url) ? { status: "new" } : {}),
      },
    });
    detailsFetched++;
    if (backfillUrls.includes(url)) backfilled++;
    await sleep(SCRAPE_DELAY_MS);
  }
  if (backfilled > 0) {
    console.log(`[competitor-sitemap] Backfilled wordCount for ${backfilled} older pages on ${comp.domain}`);
  }

  // Mark excess pages for scraping on the next crawl run.
  if (deferred.length > 0) {
    console.log(`[competitor-sitemap] Deferred ${deferred.length} page scrapes for ${comp.domain} to next run`);
    for (const url of deferred) {
      await prisma.competitorPage.update({
        where: { competitorId_url: { competitorId, url } },
        data: { status: "pending_scrape" },
      });
    }
  }

  // ── Update competitor record ─────────────────────────────────────
  await prisma.competitor.update({
    where: { id: competitorId },
    data: {
      sitemapUrls: discovery.sitemapUrls,
      lastCrawledAt: now,
      detectedCms: discovery.detectedCms,
    },
  });

  // ── Create snapshot for historical record ────────────────────────
  await prisma.competitorSnapshot.create({
    data: {
      competitorId,
      brandId: comp.brandId,
      totalPages: discovery.entries.length,
      newPages: newPages.slice(0, 100),       // cap stored JSON
      removedPages: removedPages.slice(0, 100),
      changedPages: updatedPages.slice(0, 100),
      sitemapUrls: discovery.sitemapUrls,
      crawledAt: now,
    },
  });

  return {
    sitemapUrls: discovery.sitemapUrls,
    detectedCms: discovery.detectedCms,
    newPages,
    updatedPages,
    removedPages,
    unchangedCount,
    totalPages: discovery.entries.length,
    detailsFetched,
    deferredScrapes: deferred.length,
  };
}
