/** * DuckDuckGo Search Scraper / Wrapper (Node.js ESM) * * Output lengkap: * { *   query: string, *   engine: "duckduckgo", *   region: string, *   total: number, *   results: [ *     { *       position: number, *       title: string, *       url: string, *       description: string, *       domain: string, *       favicon?: string *     } *   ] * } */const USER_AGENT =  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";function decodeHtml(text = "") {  return String(text)    .replace(/&amp;/g, "&")    .replace(/&lt;/g, "<")    .replace(/&gt;/g, ">")    .replace(/&quot;/g, '"')    .replace(/&#x27;/g, "'")    .replace(/&#39;/g, "'")    .replace(/&nbsp;/g, " ")    .replace(/&#x2F;/g, "/")    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))    .replace(/&#x([0-9a-fA-F]+);/g, (_, h) =>      String.fromCharCode(parseInt(h, 16))    )    .replace(/\s+/g, " ")    .trim();}function extractDomain(url) {  try {    return new URL(url).hostname.replace(/^www\./, "");  } catch {    return "";  }}function unwrapDdgUrl(url) {  if (!url) return url;  const m = url.match(/[?&]uddg=([^&]+)/);  if (m) {    try {      return decodeURIComponent(m[1]);    } catch {      /* keep */    }  }  // /l/?kh=-1&uddg=...  const m2 = url.match(/\/l\/\?[^"]*uddg=([^&"]+)/);  if (m2) {    try {      return decodeURIComponent(m2[1]);    } catch {      /* keep */    }  }  return url;}/** * Parse HTML hasil DuckDuckGo (html + lite) */function parseResults(html) {  const results = [];  const seen = new Set();  // --- Pola modern: div.result ---  // Ambil setiap block result  const blocks = html.split(/<div[^>]*class="[^"]*result\s[^"]*"[^>]*>/i).slice(1);  for (const block of blocks) {    // Title + URL    const titleMatch =      block.match(        /<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i      ) ||      block.match(        /<a[^>]*href="([^"]+)"[^>]*class="[^"]*result__a[^"]*"[^>]*>([\s\S]*?)<\/a>/i      );    if (!titleMatch) continue;    let url = unwrapDdgUrl(titleMatch[1]);    const title = decodeHtml(titleMatch[2].replace(/<[^>]+>/g, ""));    // Snippet / description    const snippetMatch =      block.match(        /<(?:a|div)[^>]*class="[^"]*result__snippet[^"]*"[^>]*>([\s\S]*?)<\/(?:a|div)>/i      ) ||      block.match(        /<td[^>]*class="[^"]*result-snippet[^"]*"[^>]*>([\s\S]*?)<\/td>/i      );    const description = snippetMatch      ? decodeHtml(snippetMatch[1].replace(/<[^>]+>/g, ""))      : "";    // Favicon (kadang ada)    const faviconMatch = block.match(      /src="(\/\/external-content\.duckduckgo\.com\/ip3\/[^"]+)"/i    );    const favicon = faviconMatch      ? "https:" + faviconMatch[1]      : undefined;    if (      title &&      url &&      url.startsWith("http") &&      !url.includes("duckduckgo.com") &&      !seen.has(url)    ) {      seen.add(url);      results.push({        title,        url,        description,        domain: extractDomain(url),        ...(favicon && { favicon }),      });    }  }  // --- Fallback: pola lite (table / rel=nofollow) ---  if (results.length === 0) {    const links = [      ...html.matchAll(        /<a[^>]*rel="nofollow"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi      ),    ];    for (const m of links) {      let url = unwrapDdgUrl(m[1]);      const title = decodeHtml(m[2].replace(/<[^>]+>/g, ""));      if (        title &&        url &&        url.startsWith("http") &&        !url.includes("duckduckgo.com") &&        !seen.has(url)      ) {        seen.add(url);        results.push({          title,          url,          description: "",          domain: extractDomain(url),        });      }    }  }  // Tambah position  return results.map((r, i) => ({    position: i + 1,    ...r,  }));}/** * Pure scraper (zero dependency) * * @param {string} query * @param {object} [opts] * @param {number}  [opts.maxResults=10] * @param {string}  [opts.region="wt-wt"]   // wt-wt | id-id | us-en | uk-en | ... * @param {AbortSignal} [opts.signal] * @returns {Promise<{ *   query: string, *   engine: string, *   region: string, *   total: number, *   results: Array<{ *     position: number, *     title: string, *     url: string, *     description: string, *     domain: string, *     favicon?: string *   }> * }>} */export async function search(query, opts = {}) {  const { maxResults = 10, region = "wt-wt", signal } = opts;  if (!query || typeof query !== "string") {    throw new Error("query harus string non-kosong");  }  const params = new URLSearchParams({    q: query,    kl: region,  });  const endpoints = [    `https://html.duckduckgo.com/html/?${params}`,    `https://lite.duckduckgo.com/lite/?${params}`,  ];  let lastError;  for (const endpoint of endpoints) {    try {      const res = await fetch(endpoint, {        method: "GET",        headers: {          "User-Agent": USER_AGENT,          Accept:            "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",          "Accept-Language": "en-US,en;q=0.9,id;q=0.8",          Referer: "https://duckduckgo.com/",        },        signal,        redirect: "follow",      });      if (!res.ok) throw new Error(`HTTP ${res.status}`);      const html = await res.text();      if (        html.includes("anomaly-modal") ||        html.includes("Unfortunately, bots use DuckDuckGo") ||        html.includes("challenge-form")      ) {        throw new Error(          "DuckDuckGo menampilkan captcha (anomaly detection). Coba searchWithLib() atau tunggu beberapa menit."        );      }      const parsed = parseResults(html);      if (parsed.length > 0) {        const results = parsed.slice(0, maxResults);        return {          query,          engine: "duckduckgo",          region,          total: results.length,          results,        };      }      lastError = new Error("Tidak ada hasil yang bisa di-parse");    } catch (err) {      lastError = err;    }  }  throw lastError ?? new Error("Gagal mengambil hasil pencarian");}/** * Wrapper memakai duck-duck-scrape (lebih stabil) * Install: npm i duck-duck-scrape * * @param {string} query * @param {object} [opts] * @param {number} [opts.maxResults=10] * @param {string} [opts.safeSearch="moderate"]  // off | moderate | strict * @param {string} [opts.region="wt-wt"] */export async function searchWithLib(query, opts = {}) {  const { maxResults = 10, safeSearch = "moderate", region = "wt-wt" } = opts;  let DDG;  try {    DDG = await import("duck-duck-scrape");  } catch {    throw new Error(      'Package "duck-duck-scrape" belum terinstall.\nJalankan: npm i duck-duck-scrape'    );  }  const safeMap = {    off: DDG.SafeSearchType.OFF,    moderate: DDG.SafeSearchType.MODERATE,    strict: DDG.SafeSearchType.STRICT,  };  const data = await DDG.search(query, {    safeSearch: safeMap[safeSearch] ?? DDG.SafeSearchType.MODERATE,  });  if (data.noResults || !data.results?.length) {    return {      query,      engine: "duckduckgo",      region,      total: 0,      results: [],    };  }  const results = data.results.slice(0, maxResults).map((r, i) => {    const url = r.url || r.href || "";    return {      position: i + 1,      title: decodeHtml(r.title || ""),      url,      description: decodeHtml(r.description || r.rawDescription || ""),      domain: extractDomain(url),    };  });  return {    query,    engine: "duckduckgo",    region,    total: results.length,    results,  };}// ========== CLI ==========const isMain =  process.argv[1] &&  (import.meta.url.endsWith(process.argv[1].replace(/\\/g, "/")) ||    process.argv[1].endsWith("ddg.js"));if (isMain) {  const args = process.argv.slice(2).filter((a) => !a.startsWith("--"));  const query = args.join(" ").trim() || "Indonesia";  const useLib = process.argv.includes("--lib");  const maxResults = Number(    process.argv.find((a) => a.startsWith("--max="))?.split("=")[1] || 10  );  console.error(    `Mencari: "${query}" ${useLib ? "(via library)" : "(pure scraper)"}...\n`  );  try {    const data = useLib      ? await searchWithLib(query, { maxResults })      : await search(query, { maxResults });    console.log(JSON.stringify(data, null, 2));  } catch (err) {    console.error("Error:", err.message);    process.exit(1);  }}