scientia-website/scripts/scrape.mjs

import * as cheerio from "cheerio";
import { mkdir, writeFile } from "node:fs/promises";

const BASE = "https://scientia.ru";
const UA =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36";

const PAGES = [
  "/",
  "/about",
  "/prorock",
  "/magicore",
  "/prorockfeatures",
  "/digger",
  "/nur",
  "/emergency_care",
  "/joint_explorer",
  "/geodesy",
  "/prorock_updates",
  "/personal_data",
  "/education",
  "/scientiajoints",
  "/subscribe",
  "/diggerslope_updates",
];

const clean = (s) => s.replace(/\s+/g, " ").trim();

function extract(html, path) {
  const $ = cheerio.load(html);
  const meta = {
    title: clean($("title").first().text()),
    description: $('meta[name="description"]').attr("content")?.trim() || "",
    ogImage: $('meta[property="og:image"]').attr("content") || "",
    h1: clean($("h1").first().text()),
  };

  const blocks = [];
  const seenImages = new Set();

  $(".t-rec").each((_, rec) => {
    const $rec = $(rec);
    if ($rec.parents(".t-rec").length) return; // top-level records only
    const block = { type: "", headings: [], texts: [], images: [], links: [], form: null };

    $rec.find("h1,h2,h3,h4,h5,h6").each((__, h) => {
      const t = clean($(h).text());
      if (t) block.headings.push({ level: Number(h.tagName[1]), text: t });
    });

    $rec
      .find(
        ".t-descr, .t-text, .t-uptitle, .t-name, .t-heading, .t-subtitle, .t-section__title, .t-section__descr, .tn-atom",
      )
      .each((__, el) => {
        const $el = $(el);
        if ($el.parents("form").length) return;
        if ($el.find(".tn-atom").length) return; // nested duplicates in zero blocks
        const t = clean($el.text());
        if (t && !block.texts.includes(t)) block.texts.push(t);
      });

    $rec.find("li").each((__, li) => {
      const t = clean($(li).text());
      if (t && !block.texts.some((x) => x.includes(t))) block.texts.push("• " + t);
    });

    $rec.find("img").each((__, img) => {
      const src = ($(img).attr("data-original") || $(img).attr("src") || "").split("?")[0];
      if (src && /tildacdn|scientia\.ru/.test(src) && !seenImages.has(src)) {
        seenImages.add(src);
        block.images.push(src);
      }
    });

    $rec.find('[style*="background-image"]').each((__, el) => {
      const m = ($(el).attr("style") || "").match(/url\(['"]?([^'")]+)['"]?\)/);
      if (m && /tildacdn|scientia\.ru/.test(m[1]) && !seenImages.has(m[1])) {
        seenImages.add(m[1]);
        block.images.push(m[1].split("?")[0]);
      }
    });

    $rec.find("a[href]").each((__, a) => {
      const href = $(a).attr("href") || "";
      if (href && !href.startsWith("#") && !href.startsWith("javascript") && !block.links.includes(href)) {
        block.links.push(href);
      }
    });

    const form = $rec.find("form").first();
    if (form.length) {
      block.form = {
        action: form.attr("action") || "",
        fields: form
          .find("input,textarea,select")
          .map((__, f) => ({
            tag: f.tagName,
            name: $(f).attr("name") || "",
            type: $(f).attr("type") || "",
            placeholder: $(f).attr("placeholder") || "",
          }))
          .get(),
      };
    }

    if (block.headings.length || block.texts.length || block.images.length || block.form) {
      blocks.push(block);
    }
  });

  // Global navigation
  const nav = [];
  $(".t228__list_item a, .tmenu a, nav a, .t396__elem a[href^='/']").each((__, a) => {
    const text = clean($(a).text());
    const href = $(a).attr("href") || "";
    if (text && href && !nav.some((n) => n.href === href)) nav.push({ text, href });
  });

  return { path, meta, nav, blocks };
}

async function main() {
  await mkdir(".scratch/raw", { recursive: true });
  await mkdir(".scratch/content", { recursive: true });
  const allImages = new Map();

  for (const path of PAGES) {
    const slug = path === "/" ? "home" : path.replace(/^\//, "");
    process.stdout.write(`fetch ${path} ... `);
    try {
      const res = await fetch(BASE + path, { headers: { "User-Agent": UA } });
      if (!res.ok) {
        console.log(`HTTP ${res.status}`);
        continue;
      }
      const html = await res.text();
      await writeFile(`.scratch/raw/${slug}.html`, html);
      const data = extract(html, path);
      await writeFile(`.scratch/content/${slug}.json`, JSON.stringify(data, null, 2));
      for (const b of data.blocks) for (const img of b.images) allImages.set(img, slug);
      console.log(`${data.blocks.length} blocks, meta: "${data.meta.title}"`);
    } catch (e) {
      console.log(`ERROR ${e.message}`);
    }
  }

  await writeFile(
    ".scratch/images.json",
    JSON.stringify([...allImages.entries()].map(([url, page]) => ({ url, page })), null, 2),
  );
  console.log(`\n${allImages.size} unique images found`);
}

main();