/**
 * Structural Site Scanner
 * ────────────────────────
 * Uses the On-Page API to crawl and analyze a website's structure.
 * Returns normalised PageData[] and summary metrics for the SEO analyzer.
 *
 * Internal module — third-party names must never appear in user-facing
 * contexts. Errors are wrapped as AuditDataSourceError before propagating.
 */

import type { PageData, SiteScanResult } from "../types";
import { AuditDataSourceError } from "../types";

export type CrawlErrorReason =
  | "crawl_blocked"
  | "crawl_timeout"
  | "site_unreachable"
  | "js_required"
  | "redirect_loop"
  | "firecrawl_service_error"
  | "unknown";

const BASE = "https://api.dataforseo.com/v3";
const LOGIN = process.env.DATAFORSEO_LOGIN ?? "";
const PASSWORD = process.env.DATAFORSEO_PASSWORD ?? "";

function authHeader(): string {
  return "Basic " + Buffer.from(`${LOGIN}:${PASSWORD}`).toString("base64");
}

function isConfigured(): boolean {
  return !!(LOGIN && PASSWORD);
}

// ─── Low-level request ────────────────────────────────────────────────────────

async function dfsPost<T = unknown>(endpoint: string, body: unknown[]): Promise<T> {
  if (!isConfigured()) throw new AuditDataSourceError("Structural scanner not configured", "dataforseo-onpage");

  const res = await fetch(`${BASE}${endpoint}`, {
    method: "POST",
    headers: { Authorization: authHeader(), "Content-Type": "application/json" },
    body: JSON.stringify(body),
  });

  if (!res.ok) {
    throw new AuditDataSourceError(`Structural scanner HTTP ${res.status}`, "dataforseo-onpage");
  }

  const data = (await res.json()) as { status_code?: number; status_message?: string; tasks?: unknown[] };
  if (data.status_code !== 20000) {
    throw new AuditDataSourceError(
      `Structural scanner envelope error ${data.status_code}`,
      "dataforseo-onpage",
    );
  }
  return data as T;
}

// ─── On-Page task types ───────────────────────────────────────────────────────

interface OnPageTaskResponse {
  status_code: number;
  tasks: Array<{ id: string; status_code: number; status_message: string }>;
}

interface OnPageSummaryResponse {
  tasks: Array<{
    result: Array<{
      crawl_progress?: string;
      crawl_status?: { pages_crawled?: number; pages_in_queue?: number };
      domain_info?: {
        total_pages?: number;
        checks?: {
          sitemap?: boolean;
          robots_txt?: boolean;
          mobile_friendliness?: number;
          canonical?: boolean;
        };
      };
      page_metrics?: {
        broken_links?: number;
        broken_resources?: number;
        duplicate_meta_tags?: number;
        duplicate_title?: number;
      };
    }>;
  }>;
}

interface OnPagePagesResponse {
  tasks: Array<{
    result: Array<{
      total_count?: number;
      items?: OnPageItem[];
    }>;
  }>;
}

interface OnPageItem {
  url?: string;
  status_code?: number;
  resource_type?: string;
  extended_crawl_status?: string; // e.g. "forbidden_http_header" when WAF blocks
  checks?: {
    canonical?: boolean;
    is_4xx_code?: boolean;
    is_redirect?: boolean;
    no_index_canonical?: boolean;
    has_links_from_orphan_pages?: boolean;
  };
  meta?: {
    title?: string;
    description?: string;
    canonical?: string;
    internal_links_count?: number;
    external_links_count?: number;
    images_count?: number;
    images_without_alt_attributes?: number;
    follow?: boolean;
    no_index?: boolean;
  };
  content?: {
    plain_text_word_count?: number;
  };
  page_timing?: {
    download_time?: number;
  };
  heading_info?: {
    heading?: Array<{ type?: string; content?: string }>;
  };
  relative_url?: string;
}

// ─── Polling helper ───────────────────────────────────────────────────────────

// 300s polling budget: enable_browser_rendering averages ~50s/page, so 5 pages
// needs up to 250s. 300s leaves a 50s margin before the poll window expires.
// (WAF-bypass-required: browser rendering must stay on or Cloudflare-protected
// sites return 0 pages regardless of timeout length.)
async function waitForCrawl(taskId: string, timeoutMs = 300_000): Promise<void> {
  const deadline = Date.now() + timeoutMs;
  const POLL_INTERVAL = 8_000;

  while (Date.now() < deadline) {
    await new Promise((r) => setTimeout(r, POLL_INTERVAL));

    const res = await dfsPost<OnPageSummaryResponse>("/on_page/summary", [{ id: taskId }]);
    const result = res.tasks?.[0]?.result?.[0];
    const progress = result?.crawl_progress;
    const crawled = result?.crawl_status?.pages_crawled ?? 0;
    const inQueue = result?.crawl_status?.pages_in_queue ?? 0;

    console.log(`[audit-onpage] poll taskId=${taskId} progress=${progress} crawled=${crawled} inQueue=${inQueue}`);
    console.log(`[audit-onpage-raw] poll taskId=${taskId} result=${JSON.stringify(result).slice(0, 1500)}`);

    if (progress === "finished") return;
    if (progress === "fault") {
      throw new AuditDataSourceError("Site scan reported a fault during crawl", "dataforseo-onpage");
    }
  }

  throw new AuditDataSourceError("DataForSEO crawl exceeded 5 minute budget for Phase 1", "dataforseo-onpage");
}

// ─── Fetch pages with pagination ─────────────────────────────────────────────

async function fetchAllPages(taskId: string): Promise<OnPageItem[]> {
  const items: OnPageItem[] = [];
  const limit = 100;
  let offset = 0;

  while (true) {
    const res = await dfsPost<OnPagePagesResponse>("/on_page/pages", [
      { id: taskId, limit, offset, filters: [["resource_type", "=", "html"]] },
    ]);

    const result = res.tasks?.[0]?.result?.[0];
    const batch = result?.items ?? [];

    if (offset === 0) {
      console.log(`[audit-onpage] fetchAllPages taskId=${taskId} total_count=${result?.total_count ?? "unknown"}`);
      console.log(`[audit-onpage-raw] fetchAllPages taskId=${taskId} result=${JSON.stringify(result).slice(0, 1500)}`);
    }

    const blocked = batch.filter((item) => item.extended_crawl_status === "forbidden_http_header");
    if (blocked.length > 0) {
      console.log(`[audit-onpage] WAF blocked ${blocked.length} items (forbidden_http_header): ${JSON.stringify(blocked.slice(0, 3)).slice(0, 600)}`);
    }

    items.push(...batch);

    if (batch.length < limit) break;
    offset += limit;
    if (items.length >= 300) break; // Safety cap
  }

  return items;
}

// ─── Normalise raw items to PageData ─────────────────────────────────────────

function normalizeItem(item: OnPageItem): PageData {
  const h1s = (item.heading_info?.heading ?? [])
    .filter((h) => h.type?.toLowerCase() === "h1")
    .map((h) => h.content ?? "")
    .filter(Boolean);

  return {
    url: item.url ?? "",
    title: item.meta?.title ?? undefined,
    h1: h1s[0] ?? undefined,
    metaDescription: item.meta?.description ?? undefined,
    statusCode: item.status_code ?? 200,
    wordCount: item.content?.plain_text_word_count ?? undefined,
    internalLinks: Array(item.meta?.internal_links_count ?? 0).fill(""), // count only
    externalLinks: Array(item.meta?.external_links_count ?? 0).fill(""),
    images: Array.from({ length: item.meta?.images_count ?? 0 }, (_, i) => ({
      src: `img-${i}`,
      alt: (item.meta?.images_without_alt_attributes ?? 0) > 0 && i === 0 ? undefined : "present",
    })),
    canonical: item.meta?.canonical ?? undefined,
    hasSchema: false, // populated separately if needed
    schemaTypes: [],
    isIndexable: !(item.meta?.no_index ?? false) && !(item.checks?.no_index_canonical ?? false),
    redirectTarget: item.checks?.is_redirect ? item.relative_url : undefined,
  };
}

// ─── Constants ────────────────────────────────────────────────────────────────

const CHROME_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";

// ─── Public types ─────────────────────────────────────────────────────────────

export interface OnPageSummary {
  totalPages: number;
  hasSitemap: boolean;
  hasRobotsTxt: boolean;
  mobileFriendly: boolean | null;
}

// ─── Independent robots.txt / sitemap check ───────────────────────────────────

async function checkRobotsAndSitemap(origin: string): Promise<{ hasRobotsTxt: boolean; hasSitemap: boolean }> {
  // Always probe both bare and www variants so a 301 redirect doesn't hide the file
  const wwwOrigin = origin.replace(/^(https?:\/\/)(?!www\.)/, "$1www.");
  const origins = origin === wwwOrigin ? [origin] : [origin, wwwOrigin];

  async function probe(url: string): Promise<boolean> {
    try {
      const res = await fetch(url, {
        redirect: "follow",
        signal: AbortSignal.timeout(10_000),
        headers: { "User-Agent": CHROME_UA },
      });
      console.log(`[audit-onpage] probe url="${url}" status=${res.status} ok=${res.ok}`);
      return res.ok;
    } catch (e) {
      console.log(`[audit-onpage] probe url="${url}" error=${e instanceof Error ? e.message : e}`);
      return false;
    }
  }

  const robotsUrls = origins.map((o) => `${o}/robots.txt`);
  const sitemapUrls = origins.flatMap((o) => [
    `${o}/sitemap.xml`,
    `${o}/sitemap_index.xml`,
    `${o}/sitemap-index.xml`,
  ]);

  const [robotsResults, sitemapResults] = await Promise.all([
    Promise.all(robotsUrls.map(probe)),
    Promise.all(sitemapUrls.map(probe)),
  ]);

  const hasRobotsTxt = robotsResults.some(Boolean);
  const hasSitemap = sitemapResults.some(Boolean);

  console.log(`[audit-onpage] robotsCheck hasRobotsTxt=${hasRobotsTxt} checked=${robotsUrls.join(",")}`);
  console.log(`[audit-onpage] sitemapCheck hasSitemap=${hasSitemap} checked=${sitemapUrls.join(",")}`);

  return { hasRobotsTxt, hasSitemap };
}

// ─── Single-domain crawl ──────────────────────────────────────────────────────

// TODO: max_crawl_pages is constrained by the polling budget (~50s/page with
// browser rendering). Long-term fix: use DataForSEO postback URLs for async completion
// notification, which eliminates the polling timeout and lifts this scope limit.
const CRAWL_OPTIONS = {
  max_crawl_pages: 5,
  crawl_depth: 2,
  enable_content_parsing: true,
  load_resources: true,
  enable_javascript: true,
  enable_browser_rendering: true,
  check_spell: false,
  support_cookies: false,
} as const;

async function crawlOneDomain(domain: string, maxPages?: number): Promise<{ pages: PageData[]; summary: OnPageSummary }> {
  const effectiveMaxPages = maxPages ?? CRAWL_OPTIONS.max_crawl_pages;
  console.log(`[audit-onpage] Submitting crawl target=${domain} max_pages=${effectiveMaxPages} depth=${CRAWL_OPTIONS.crawl_depth} js=${CRAWL_OPTIONS.enable_javascript}`);
  console.log(`[audit-onpage] crawl user_agent set to Chrome`);

  const taskRes = await dfsPost<OnPageTaskResponse>("/on_page/task_post", [
    { target: domain, ...CRAWL_OPTIONS, max_crawl_pages: effectiveMaxPages, user_agent: CHROME_UA },
  ]);

  console.log(`[audit-onpage-raw] task_post response=${JSON.stringify(taskRes).slice(0, 1500)}`);

  const task = taskRes.tasks?.[0];
  const taskId = task?.id;
  const taskStatusCode = task?.status_code;
  const taskStatusMsg = task?.status_message;

  // 20100 = Task Created; anything else means DFS rejected the payload
  if (taskStatusCode !== 20100 || !taskId) {
    console.log(`[audit-onpage-failure] task_post rejected status=${taskStatusCode} msg="${taskStatusMsg}" target=${domain}`);
    throw new AuditDataSourceError(
      `Site scan task rejected by data provider (status ${taskStatusCode}: ${taskStatusMsg})`,
      "dataforseo-onpage",
    );
  }
  console.log(`[audit-onpage] Task created taskId=${taskId} status=${taskStatusCode} msg="${taskStatusMsg}"`);

  await waitForCrawl(taskId);

  const summaryRes = await dfsPost<OnPageSummaryResponse>("/on_page/summary", [{ id: taskId }]);
  const summaryResult = summaryRes.tasks?.[0]?.result?.[0];
  console.log(`[audit-onpage-raw] summary response taskId=${taskId} result=${JSON.stringify(summaryResult).slice(0, 2000)}`);

  const domainInfo = summaryResult?.domain_info;
  const pageMetrics = summaryResult?.page_metrics;
  console.log(`[audit-onpage] Summary taskId=${taskId} domainInfo=${JSON.stringify(domainInfo)} pageMetrics=${JSON.stringify(pageMetrics)}`);

  const summary: OnPageSummary = {
    totalPages: domainInfo?.total_pages ?? 0,
    hasSitemap: domainInfo?.checks?.sitemap ?? false,
    hasRobotsTxt: domainInfo?.checks?.robots_txt ?? false,
    mobileFriendly:
      domainInfo?.checks?.mobile_friendliness != null
        ? domainInfo.checks.mobile_friendliness >= 60
        : null,
  };

  const rawPages = await fetchAllPages(taskId);
  const pages = rawPages.map(normalizeItem).filter((p) => p.statusCode < 400);
  console.log(`[audit-onpage] Crawl complete taskId=${taskId} target=${domain} raw=${rawPages.length} filtered=${pages.length}`);

  // Detect total WAF block (all items have forbidden_http_header or all are 4xx)
  if (rawPages.length > 0 && pages.length === 0) {
    const allForbidden = rawPages.every(
      (item) => item.extended_crawl_status === "forbidden_http_header" || (item.status_code != null && item.status_code >= 400),
    );
    if (allForbidden) {
      console.log(`[audit-onpage] WAF/bot protection blocked all ${rawPages.length} pages`);
      throw new AuditDataSourceError("Site blocked all crawl requests (WAF/bot protection)", "dataforseo-onpage");
    }
  }

  return { pages, summary };
}

// ─── Crawl error classification ───────────────────────────────────────────────

export function classifyCrawlError(err: unknown): { reason: CrawlErrorReason; detail: string } {
  const msg = err instanceof Error ? err.message : String(err);
  const detail = msg.slice(0, 500);
  if (/minute budget|timeout|exceeded/i.test(msg)) return { reason: "crawl_timeout", detail };
  if (/fault during crawl|fault/i.test(msg)) return { reason: "site_unreachable", detail };
  if (/HTTP 5\d\d|5xx|service unavailable|bad gateway/i.test(msg)) return { reason: "firecrawl_service_error", detail };
  if (/HTTP 4\d\d|forbidden|unauthorized|access denied|crawl_blocked/i.test(msg)) return { reason: "crawl_blocked", detail };
  if (/ENOTFOUND|ECONNREFUSED|ETIMEDOUT|dns/i.test(msg)) return { reason: "site_unreachable", detail };
  if (/redirect|too many redirects/i.test(msg)) return { reason: "redirect_loop", detail };
  if (/javascript|js required|script/i.test(msg)) return { reason: "js_required", detail };
  return { reason: "unknown", detail };
}

// ─── Main export ─────────────────────────────────────────────────────────────

export async function scanSite(
  website: string,
  options?: { maxPages?: number },
): Promise<{ pages: PageData[]; summary: OnPageSummary; crawlError?: { reason: CrawlErrorReason; detail: string } }> {
  const domain = website.replace(/^https?:\/\//, "").replace(/\/$/, "");
  const origin = website.startsWith("http") ? website.replace(/\/$/, "") : `https://${website}`;
  console.log(`[audit-onpage] scanSite website="${website}" domain="${domain}" origin="${origin}"${options?.maxPages ? ` maxPages=${options.maxPages}` : ""}`);

  try {
    // Crawl + independent HTTP checks run in parallel
    const [result, directChecks] = await Promise.all([
      crawlOneDomain(domain, options?.maxPages),
      checkRobotsAndSitemap(origin),
    ]);
    console.log(`[audit-onpage] Direct checks for "${origin}": ${JSON.stringify(directChecks)}`);

    let finalResult = result;

    // If 0 pages and domain isn't already www., retry crawl with www. prefix
    if (result.pages.length === 0 && !domain.startsWith("www.")) {
      const wwwDomain = `www.${domain}`;
      console.log(`[audit-onpage] 0 pages from "${domain}" — retrying with "${wwwDomain}"`);
      try {
        const wwwResult = await crawlOneDomain(wwwDomain, options?.maxPages);
        if (wwwResult.pages.length > 0) {
          console.log(`[audit-onpage] www. retry succeeded: ${wwwResult.pages.length} pages`);
          finalResult = wwwResult;
        } else {
          console.log(`[audit-onpage] www. retry also returned 0 pages`);
          console.log(`[audit-onpage-failure] ${JSON.stringify({
            originalDomain: domain,
            wwwDomain,
            directChecks,
            bareSummary: result.summary,
            wwwSummary: wwwResult.summary,
          })}`);
        }
      } catch (wwwErr) {
        const wwwMsg = wwwErr instanceof Error ? wwwErr.message : String(wwwErr);
        console.warn(`[audit-onpage] www. retry threw: ${wwwMsg}`);
        console.log(`[audit-onpage-failure] ${JSON.stringify({
          originalDomain: domain,
          wwwDomain,
          wwwError: wwwMsg,
          directChecks,
          bareSummary: result.summary,
        })}`);
      }
    }

    // Override crawl-derived robots/sitemap with independent direct HTTP checks
    const mergedSummary: OnPageSummary = {
      ...finalResult.summary,
      hasSitemap: directChecks.hasSitemap,
      hasRobotsTxt: directChecks.hasRobotsTxt,
    };

    if (finalResult.pages.length === 0) {
      return {
        pages: [],
        summary: mergedSummary,
        crawlError: { reason: "crawl_blocked", detail: `Crawl returned 0 pages for ${domain} (and www.${domain})` },
      };
    }

    return { pages: finalResult.pages, summary: mergedSummary };
  } catch (err) {
    const classified = classifyCrawlError(err);
    console.log(`[audit-onpage] scanSite failed reason=${classified.reason} detail=${classified.detail}`);
    const defaultSummary: OnPageSummary = { totalPages: 0, hasSitemap: false, hasRobotsTxt: false, mobileFriendly: null };
    return { pages: [], summary: defaultSummary, crawlError: classified };
  }
}

// ─── Backlinks summary (re-exported from data-sources for backward compat) ────

export type { BacklinksSummary } from "../data-sources/backlinks-summary";
export { fetchBacklinksSummary } from "../data-sources/backlinks-summary";

// ─── SERP positions ───────────────────────────────────────────────────────────

export interface SerpPosition {
  keyword: string;
  position: number | null;
  url: string | null;
  title: string | null;
}

export async function fetchSerpPositions(
  queries: string[],
  locationCode = 2840,
): Promise<SerpPosition[]> {
  if (!isConfigured() || queries.length === 0) return [];

  try {
    const tasks = queries.map((keyword) => ({
      keyword,
      location_code: locationCode,
      language_code: "en",
      depth: 10,
      device: "desktop",
    }));

    const res = await dfsPost<{
      status_code: number;
      tasks?: Array<{
        data?: { keyword?: string };
        result?: Array<{
          items?: Array<{
            type?: string;
            rank_absolute?: number;
            url?: string;
            title?: string;
          }>;
        }>;
      }>;
    }>("/serp/google/organic/live/advanced", tasks);

    if (!res.tasks) return [];

    return res.tasks.map((task, i) => {
      const keyword = queries[i] ?? "";
      const items = task.result?.[0]?.items ?? [];
      const organic = items.filter((it) => it.type === "organic");
      const first = organic[0];
      return {
        keyword,
        position: first?.rank_absolute ?? null,
        url: first?.url ?? null,
        title: first?.title ?? null,
      };
    });
  } catch (err) {
    console.warn(
      `[audit-serp] fetchSerpPositions failed:`,
      err instanceof Error ? err.message : err,
    );
    return [];
  }
}