/**
 * Content Extractor
 * ──────────────────
 * Fetches clean markdown content for key pages using a headless
 * content extraction service. Used to give the synthesis layer
 * readable prose to assess tone, depth, and messaging specificity.
 *
 * Internal module — service name must not appear in user-facing contexts.
 */

import Firecrawl from "@mendable/firecrawl-js";
import type { PageData } from "../types";

const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";

function isConfigured(): boolean {
  return !!API_KEY;
}

// URL patterns matching high-value pages from DFS crawl results
const KEY_PAGE_PATTERNS = [
  /\/about(?:-us)?/i,
  /\/services?/i,
  /\/treatments?/i,
  /\/procedures?/i,
  /\/products?/i,
  /\/solutions?/i,
  /\/contact(?:-us)?/i,
  /\/pricing/i,
  /\/locations?/i,
  /\/team/i,
  /\/why-us/i,
  /\/faq/i,
  /\/programs?/i,
  /\/academics?/i,
];

// Priority structural paths to try even when not discovered by the DFS crawl.
// Ordered by conversion value — used as last resort after sitemap fill.
const PRIORITY_PATHS = [
  "/about", "/about-us",
  "/services", "/products", "/solutions",
  "/contact", "/contact-us",
  "/pricing",
];

// Supplementary paths tried after sitemap if still under floor
const SUPPLEMENTARY_PATHS = [
  "/blog", "/resources", "/case-studies", "/work",
  "/how-it-works", "/why-us", "/team",
];

export interface ContentSample {
  url: string;
  markdown: string;
  title?: string;
  wordCount?: number;
  // Enhanced fields:
  h1?: string;
  metaDescription?: string;
  headingOutline?: string[];       // H2s extracted from markdown
  hasFAQ: boolean;                 // detected from markdown
  hasCTA: boolean;                 // detected from markdown
  hasPhoneClick: boolean;          // detected from markdown (tel: links or phone patterns)
  hasLocationSchema: boolean;      // detected from markdown (address patterns, city/state)
  ctaExamples: string[];           // up to 3 CTA text examples found
}

// ─── Detection helpers ────────────────────────────────────────────────────────

function detectFAQ(md: string): boolean {
  // FAQ heading OR 3+ question sentences OR Q&A pattern
  return /\bfaq\b|frequently asked/i.test(md) ||
    (md.match(/\?/g) || []).length >= 3;
}

function detectCTA(md: string): { hasCTA: boolean; examples: string[] } {
  const CTA_PATTERNS = [
    /\b(book|schedule|request|get started|contact us|call us|sign up|register|apply|download|get a quote|free consultation)\b/gi
  ];
  const matches: string[] = [];
  for (const pattern of CTA_PATTERNS) {
    const found = md.match(pattern) ?? [];
    matches.push(...found.slice(0, 2));
  }
  return { hasCTA: matches.length > 0, examples: [...new Set(matches)].slice(0, 3) };
}

function detectPhoneClick(md: string): boolean {
  return /tel:|(\+1[\s.-]?)?\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}/i.test(md);
}

function detectLocationSchema(md: string): boolean {
  // State abbreviations or city+state patterns in text
  return /\b[A-Z]{2}\s+\d{5}\b|\b(Houston|Dallas|Austin|New York|Los Angeles|Chicago|Phoenix)\b/i.test(md) ||
    /address|suite\s+\d|floor\s+\d|\bst\b.*\bave\b/i.test(md);
}

function extractH2s(md: string): string[] {
  const matches = md.match(/^##\s+(.+)$/gm) ?? [];
  return matches.map((m) => m.replace(/^##\s+/, "").trim()).slice(0, 10);
}

// ─── Sitemap URL discovery ────────────────────────────────────────────────────

async function discoverSitemapUrls(origin: string): Promise<string[]> {
  const candidates = [
    `${origin}/sitemap.xml`,
    `${origin}/sitemap_index.xml`,
    `${origin}/sitemap-index.xml`,
  ];

  for (const url of candidates) {
    try {
      const res = await fetch(url, {
        signal: AbortSignal.timeout(8_000),
        headers: { "User-Agent": "Mozilla/5.0 (compatible; AuditBot/1.0)" },
      });
      if (!res.ok) continue;

      const text = await res.text();
      const locs = [...text.matchAll(/<loc>\s*([^<]+)\s*<\/loc>/g)]
        .map((m) => m[1].trim())
        .filter((u) => u.startsWith("http") && !u.endsWith(".xml"));

      if (locs.length > 0) {
        console.log(`[audit-content] Sitemap found at ${url}: ${locs.length} URLs`);
        return locs.slice(0, 25);
      }
    } catch {
      // continue to next candidate
    }
  }
  return [];
}

// ─── Page selection ───────────────────────────────────────────────────────────

async function selectKeyPages(
  pages: PageData[],
  origin: string,
  maxPages = 10,
  topAiPages?: string[],
): Promise<string[]> {
  const urls = new Set<string>();

  // 0. Prepend top AI pages (1st-party audits only)
  if (topAiPages && topAiPages.length > 0) {
    for (const aiUrl of topAiPages) {
      if (urls.size >= maxPages) break;
      urls.add(aiUrl);
    }
  }

  // 1. Always include homepage
  const homepage = pages.find((p) => {
    try {
      const u = new URL(p.url);
      return u.pathname === "/" || u.pathname === "";
    } catch {
      return false;
    }
  });
  if (!urls.has(origin)) urls.add(homepage?.url ?? origin);

  // 2. Priority structural pages from DFS crawl results
  for (const pattern of KEY_PAGE_PATTERNS) {
    if (urls.size >= maxPages) break;
    const match = pages.find((p) => pattern.test(p.url) && !urls.has(p.url));
    if (match) urls.add(match.url);
  }

  // 3. Fill from sitemap (shallowest paths first, then alphabetical)
  if (urls.size < maxPages) {
    const sitemapUrls = await discoverSitemapUrls(origin);
    const sorted = [...sitemapUrls].sort((a, b) => {
      const depthA = (new URL(a).pathname.match(/\//g) ?? []).length;
      const depthB = (new URL(b).pathname.match(/\//g) ?? []).length;
      return depthA !== depthB ? depthA - depthB : a.localeCompare(b);
    });
    for (const sitemapUrl of sorted) {
      if (urls.size >= maxPages) break;
      if (!urls.has(sitemapUrl)) urls.add(sitemapUrl);
    }
  }

  // 4. Priority path fallbacks (pages not in DFS crawl due to budget)
  for (const path of PRIORITY_PATHS) {
    if (urls.size >= maxPages) break;
    const candidate = `${origin}${path}`;
    if (!urls.has(candidate) && !pages.some((p) => p.url === candidate)) {
      urls.add(candidate);
    }
  }

  // 5. Supplementary paths if still below hard floor of 5
  const floor = Math.min(5, maxPages);
  if (urls.size < floor) {
    for (const path of SUPPLEMENTARY_PATHS) {
      if (urls.size >= floor) break;
      const candidate = `${origin}${path}`;
      if (!urls.has(candidate)) urls.add(candidate);
    }
  }

  // 6. Fill remaining with DFS pages sorted by inbound link count
  if (urls.size < maxPages) {
    const byLinks = [...pages]
      .filter((p) => !urls.has(p.url))
      .sort((a, b) => b.internalLinks.length - a.internalLinks.length);
    for (const p of byLinks) {
      if (urls.size >= maxPages) break;
      urls.add(p.url);
    }
  }

  const selected = [...urls].slice(0, maxPages);
  console.log(`[audit-content] selected ${selected.length} pages for ${topAiPages ? "1st" : "3rd"}-party audit: ${selected.join(", ")}`);
  return selected;
}

// ─── Concurrent fetch ─────────────────────────────────────────────────────────

const MAX_WORDS_PER_PAGE = 500;
const CONCURRENCY = 3;

async function fetchPages(urls: string[], app: Firecrawl): Promise<ContentSample[]> {
  const results: ContentSample[] = [];

  for (let i = 0; i < urls.length; i += CONCURRENCY) {
    const batch = urls.slice(i, i + CONCURRENCY);
    const settled = await Promise.allSettled(
      batch.map(async (url): Promise<ContentSample | null> => {
        try {
          const result = await app.scrape(url, {
            formats: ["markdown"],
            onlyMainContent: true,
          });

          if (!result.markdown) return null;

          const words = result.markdown.split(/\s+/);
          const truncated = words.slice(0, MAX_WORDS_PER_PAGE).join(" ");

          // Run all signal detections on full markdown before truncation
          const fullMarkdown = result.markdown;
          const ctaResult = detectCTA(fullMarkdown);
          const headingOutline = extractH2s(fullMarkdown);

          return {
            url,
            markdown: truncated,
            title: result.metadata?.title ?? undefined,
            wordCount: words.length,
            h1: (result.metadata as Record<string, unknown> | undefined)?.["h1"] as string | undefined ?? undefined,
            metaDescription: result.metadata?.description ?? undefined,
            headingOutline,
            hasFAQ: detectFAQ(fullMarkdown),
            hasCTA: ctaResult.hasCTA,
            hasPhoneClick: detectPhoneClick(fullMarkdown),
            hasLocationSchema: detectLocationSchema(fullMarkdown),
            ctaExamples: ctaResult.examples,
          };
        } catch (err) {
          console.error(
            `[audit-content] Failed to fetch ${url}:`,
            err instanceof Error ? err.message : err,
          );
          return null;
        }
      }),
    );

    for (const outcome of settled) {
      if (outcome.status === "fulfilled" && outcome.value) {
        results.push(outcome.value);
        console.log(
          `[audit-content] Fetched ${outcome.value.url} (${outcome.value.wordCount ?? 0} words) hasCTA=${outcome.value.hasCTA} hasFAQ=${outcome.value.hasFAQ} hasPhone=${outcome.value.hasPhoneClick}`,
        );
      }
    }
  }

  return results;
}

// ─── Public exports ───────────────────────────────────────────────────────────

export interface FetchKeyPageOptions {
  topAiPages?: string[];
}

/** Fetch clean markdown content for the most strategically important pages. */
export async function fetchKeyPageContent(
  pages: PageData[],
  options?: FetchKeyPageOptions,
): Promise<ContentSample[]> {
  if (!isConfigured()) {
    console.warn("[audit-content] Content extractor not configured — skipping");
    return [];
  }

  // Derive origin from first page URL
  let origin = "";
  try {
    const first = pages[0];
    if (first) {
      const u = new URL(first.url);
      origin = u.origin;
    }
  } catch {
    // ignore
  }

  const targetUrls = await selectKeyPages(pages, origin, 10, options?.topAiPages);
  console.log(`[audit-content] Fetching ${targetUrls.length} key pages`);

  const app = new Firecrawl({ apiKey: API_KEY });

  // 90-second hard timeout across all fetches
  const timeout = new Promise<ContentSample[]>((_, reject) =>
    setTimeout(() => reject(new Error("Firecrawl total timeout (90s)")), 90_000),
  );

  try {
    return await Promise.race([fetchPages(targetUrls, app), timeout]);
  } catch (err) {
    const msg = err instanceof Error ? err.message : String(err);
    console.warn(`[audit-content] Fetch ended early: ${msg}. Returning partial results.`);
    return [];
  }
}

/** Format content samples for inclusion in the synthesis prompt. */
export function formatContentForPrompt(samples: ContentSample[]): string {
  if (samples.length === 0) return "No content samples available.";

  return samples
    .map(
      (s) =>
        `### ${s.title ?? s.url}\nURL: ${s.url}${s.wordCount ? ` (${s.wordCount} words)` : ""}\n\n${s.markdown.slice(0, 2500)}`,
    )
    .join("\n\n---\n\n");
}
