scientia-website/scripts/fetch-telegram.mjs

// Fetches the public Telegram channel feed (t.me/s/...) — the whole first page
// chain via ?before=<id> pagination — and saves the posts for the home-page
// Telegram widget. Run manually: npm run fetch-news
import * as cheerio from "cheerio";
import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";

const CHANNEL = "scientia_ru";
const MAX_PAGES = 40; // safety cap (~20 posts per page)
const UA =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36";

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

const posts = [];
const seen = new Set();
let before = null;

for (let page = 0; page < MAX_PAGES; page++) {
  const url = `https://t.me/s/${CHANNEL}${before ? `?before=${before}` : ""}`;
  const res = await fetch(url, { headers: { "User-Agent": UA } });
  if (!res.ok) {
    console.error(`page ${page}: HTTP ${res.status}`);
    break;
  }
  const $ = cheerio.load(await res.text());
  let newCount = 0;
  let oldestId = null;

  $(".tgme_widget_message").each((_, el) => {
    const $el = $(el);
    const postId = ($el.attr("data-post") || "").split("/")[1];
    if (!postId || seen.has(postId)) return;
    seen.add(postId);
    newCount++;
    oldestId = postId;

    const textNode = $el.find(".tgme_widget_message_text").first();
    // Keep basic formatting tags, drop telegram-internal attributes
    textNode.find("a, b, i, u, s, strong, em, code, br, blockquote").each((__, keep) => {
      const attrs = $(keep).attr() || {};
      for (const name of Object.keys(attrs)) {
        if (name !== "href") $(keep).removeAttr(name);
      }
    });
    const text = textNode.html()?.trim() || "";
    const datetime = $el.find("time").first().attr("datetime") || "";
    const photoStyle = $el.find(".tgme_widget_message_photo_wrap").first().attr("style") || "";
    let photo = photoStyle.match(/url\('([^']+)'\)/)?.[1]?.split("?")[0] || "";
    const video = $el.find(".tgme_widget_message_video_wrap, .tgme_widget_message_video_player").length > 0;
    // tokenized CDN url; downloaded to public/videos/ after the scrape (see below)
    const videoSrc = video ? $el.find("video").first().attr("src") || "" : "";
    // video posts carry a full-size preview frame in .tgme_widget_message_video_thumb
    if (video && !photo) {
      const thumbStyle = $el.find(".tgme_widget_message_video_thumb").first().attr("style") || "";
      photo = thumbStyle.match(/url\('([^']+)'\)/)?.[1] || "";
    }
    const link = $el.find(".tgme_widget_message_date").first().attr("href") || `https://t.me/${CHANNEL}/${postId}`;
    if (text || photo || video) {
      posts.push({ id: Number(postId), text, datetime, photo, video, videoSrc, link });
    }
  });

  console.log(`page ${page}: +${newCount} posts (total kept ${posts.length})`);
  if (newCount === 0 || !oldestId) break;
  before = oldestId;
  await sleep(300);
}

// newest first
posts.sort((a, b) => b.id - a.id);

// telesco.pe video tokens expire within hours — persist the mp4s locally
await mkdir("public/videos", { recursive: true });
for (const post of posts) {
  if (!post.videoSrc) continue;
  const res = await fetch(post.videoSrc, { headers: { "User-Agent": UA } });
  if (!res.ok) {
    console.log(`video ${post.id}: HTTP ${res.status}, keeping poster only`);
    post.videoSrc = "";
    continue;
  }
  const buf = Buffer.from(await res.arrayBuffer());
  const file = `tg-${post.id}.mp4`;
  await writeFile(path.join("public/videos", file), buf);
  post.videoSrc = `/videos/${file}`;
  console.log(`video ${post.id}: saved ${(buf.length / 1e6).toFixed(1)} MB`);
  await sleep(300);
}

await mkdir(path.dirname("src/data/telegram-news.json"), { recursive: true });
await writeFile("src/data/telegram-news.json", JSON.stringify(posts, null, 2));
console.log(`saved ${posts.length} posts`);

// Channel avatar for the widget header
const avatar = await fetch(`https://t.me/i/userpic/320/${CHANNEL}.jpg`, { headers: { "User-Agent": UA } });
if (avatar.ok) {
  const buf = Buffer.from(await avatar.arrayBuffer());
  if (buf.length > 500) {
    await writeFile("public/images/telegram-avatar.jpg", buf);
    console.log("avatar saved");
  } else {
    console.log("avatar looks empty, skipped");
  }
} else {
  console.log(`avatar fetch failed: HTTP ${avatar.status}`);
}