/**
 * AI Crawler Accessibility Benchmark
 *
 * For each vendor domain: fetch robots.txt, then issue one HEAD request per
 * AI bot user agent and record the HTTP status. robots.txt states policy;
 * the live status reveals what the WAF/CDN actually does, which is often
 * a different answer.
 */
import { writeFileSync } from "fs";

const SEO_PLATFORMS = [
  "semrush.com", "ahrefs.com", "moz.com", "surferseo.com", "searchatlas.com",
  "seranking.com", "spyfu.com", "neilpatel.com", "majestic.com",
  "screamingfrog.co.uk", "sitebulb.com", "brightlocal.com", "whitespark.ca",
  "yext.com", "brightedge.com", "conductor.com", "agencyanalytics.com",
  "accuranker.com", "wincher.com", "clearscope.io", "marketmuse.com",
  "frase.io", "lumar.io", "botify.com", "seoclarity.net", "mangools.com",
  "serpstat.com", "cognitiveseo.com", "linkresearchtools.com", "similarweb.com",
];

const AI_VISIBILITY = [
  "profound.ai", "peec.ai", "otterly.ai", "scrunch.ai", "evertune.ai",
  "brandlight.ai", "goodie.ai", "rankscale.ai", "llmrefs.com", "xfunnel.ai",
  "knowatoa.com", "superlines.io", "omnibound.ai", "athenahq.ai",
  "dataforaisearch.com", "aisearchinsider.com",
];

const BOTS = {
  GPTBot: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.1; +https://openai.com/gptbot",
  "OAI-SearchBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot",
  "ChatGPT-User": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
  ClaudeBot: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ClaudeBot/1.0; +claudebot@anthropic.com",
  PerplexityBot: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot",
  "Google-Extended": "Mozilla/5.0 (compatible; Google-Extended/1.0)",
  "Applebot-Extended": "Mozilla/5.0 (compatible; Applebot-Extended/1.0)",
  CCBot: "CCBot/2.0 (https://commoncrawl.org/faq/)",
  "meta-externalagent": "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
  Bytespider: "Mozilla/5.0 (compatible; Bytespider; spider-feedback@bytedance.com)",
};

const BROWSER_UA =
  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";

const TIMEOUT = 20000;

async function req(url, ua, method = "GET") {
  const ctl = new AbortController();
  const t = setTimeout(() => ctl.abort(), TIMEOUT);
  try {
    const r = await fetch(url, {
      method,
      headers: { "User-Agent": ua, Accept: "*/*" },
      redirect: "follow",
      signal: ctl.signal,
    });
    return { status: r.status, ok: r.ok };
  } catch (e) {
    return { status: e.name === "AbortError" ? "timeout" : "error", ok: false };
  } finally {
    clearTimeout(t);
  }
}

/** Parse robots.txt into a policy verdict per bot. */
function parseRobots(txt, bot) {
  if (!txt) return "no-robots";
  const lines = txt.split(/\r?\n/).map((l) => l.replace(/#.*$/, "").trim());
  const groups = [];
  let cur = null;
  for (const line of lines) {
    const m = line.match(/^(user-agent|allow|disallow)\s*:\s*(.*)$/i);
    if (!m) continue;
    const [, key, val] = [m[0], m[1].toLowerCase(), m[2].trim()];
    if (key === "user-agent") {
      if (!cur || cur.rules.length) { cur = { agents: [], rules: [] }; groups.push(cur); }
      cur.agents.push(val.toLowerCase());
    } else if (cur) {
      cur.rules.push({ type: key, path: val });
    }
  }
  const named = groups.find((g) => g.agents.includes(bot.toLowerCase()));
  const wild = groups.find((g) => g.agents.includes("*"));
  const g = named || wild;
  if (!g) return "not-mentioned";
  const blocksRoot = g.rules.some((r) => r.type === "disallow" && r.path === "/");
  if (blocksRoot) return named ? "BLOCKED-named" : "BLOCKED-wildcard";
  return named ? "allowed-named" : "allowed-wildcard";
}

async function auditDomain(domain, category) {
  const base = `https://${domain}`;
  let robotsTxt = null;
  try {
    const ctl = new AbortController();
    const t = setTimeout(() => ctl.abort(), TIMEOUT);
    const r = await fetch(`${base}/robots.txt`, {
      headers: { "User-Agent": BROWSER_UA },
      redirect: "follow",
      signal: ctl.signal,
    });
    clearTimeout(t);
    if (r.ok) robotsTxt = await r.text();
  } catch { /* leave null */ }

  const baseline = await req(base, BROWSER_UA);

  const bots = {};
  for (const [name, ua] of Object.entries(BOTS)) {
    bots[name] = { ...(await req(base, ua)), robots: parseRobots(robotsTxt, name) };
    await new Promise((r) => setTimeout(r, 120)); // be polite
  }

  return {
    domain,
    category,
    hasRobots: robotsTxt !== null,
    robotsMentionsAI: robotsTxt
      ? /gptbot|claudebot|perplexitybot|google-extended|oai-searchbot|ccbot|applebot-extended/i.test(robotsTxt)
      : false,
    baseline: baseline.status,
    bots,
  };
}

async function pool(items, n, fn) {
  const out = [];
  let i = 0;
  await Promise.all(
    Array.from({ length: n }, async () => {
      while (i < items.length) {
        const idx = i++;
        out[idx] = await fn(items[idx]);
        process.stderr.write(`  ${idx + 1}/${items.length} ${items[idx][0]}\n`);
      }
    }),
  );
  return out;
}

const targets = [
  ...SEO_PLATFORMS.map((d) => [d, "SEO platform"]),
  ...AI_VISIBILITY.map((d) => [d, "AI visibility tool"]),
];

const results = await pool(targets, 6, ([d, c]) => auditDomain(d, c));
writeFileSync("crawler-study-raw.json", JSON.stringify(results, null, 2));
console.log(`\nDone. ${results.length} domains.`);