|
|
@@ -1,43 +1,71 @@
|
|
|
-// Fetches the public Telegram channel feed (t.me/s/...) and saves the latest
|
|
|
-// posts for the home-page Telegram widget. Run manually: npm run fetch-news
|
|
|
+// Fetches the public Telegram channel feed (t.me/s/...) — the whole first page
|
|
|
+// chain via ?before=<id> pagination — and saves the posts for the home-page
|
|
|
+// Telegram widget. Run manually: npm run fetch-news
|
|
|
import * as cheerio from "cheerio";
|
|
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
|
import path from "node:path";
|
|
|
|
|
|
const CHANNEL = "scientia_ru";
|
|
|
+const MAX_PAGES = 40; // safety cap (~20 posts per page)
|
|
|
const UA =
|
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36";
|
|
|
|
|
|
-const res = await fetch(`https://t.me/s/${CHANNEL}`, { headers: { "User-Agent": UA } });
|
|
|
-if (!res.ok) {
|
|
|
- console.error(`Failed to fetch channel: HTTP ${res.status}`);
|
|
|
- process.exit(1);
|
|
|
-}
|
|
|
-const $ = cheerio.load(await res.text());
|
|
|
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
|
|
|
|
const posts = [];
|
|
|
-$(".tgme_widget_message").each((_, el) => {
|
|
|
- const $el = $(el);
|
|
|
- const textNode = $el.find(".tgme_widget_message_text").first();
|
|
|
- // Keep basic formatting tags, drop telegram-internal spans/attrs
|
|
|
- textNode.find("a, b, i, u, s, strong, em, code, br").each((__, keep) => {
|
|
|
- const attrs = $(keep).attr() || {};
|
|
|
- for (const name of Object.keys(attrs)) {
|
|
|
- if (name !== "href") $(keep).removeAttr(name);
|
|
|
+const seen = new Set();
|
|
|
+let before = null;
|
|
|
+
|
|
|
+for (let page = 0; page < MAX_PAGES; page++) {
|
|
|
+ const url = `https://t.me/s/${CHANNEL}${before ? `?before=${before}` : ""}`;
|
|
|
+ const res = await fetch(url, { headers: { "User-Agent": UA } });
|
|
|
+ if (!res.ok) {
|
|
|
+ console.error(`page ${page}: HTTP ${res.status}`);
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ const $ = cheerio.load(await res.text());
|
|
|
+ let newCount = 0;
|
|
|
+ let oldestId = null;
|
|
|
+
|
|
|
+ $(".tgme_widget_message").each((_, el) => {
|
|
|
+ const $el = $(el);
|
|
|
+ const postId = ($el.attr("data-post") || "").split("/")[1];
|
|
|
+ if (!postId || seen.has(postId)) return;
|
|
|
+ seen.add(postId);
|
|
|
+ newCount++;
|
|
|
+ oldestId = postId;
|
|
|
+
|
|
|
+ const textNode = $el.find(".tgme_widget_message_text").first();
|
|
|
+ // Keep basic formatting tags, drop telegram-internal attributes
|
|
|
+ textNode.find("a, b, i, u, s, strong, em, code, br, blockquote").each((__, keep) => {
|
|
|
+ const attrs = $(keep).attr() || {};
|
|
|
+ for (const name of Object.keys(attrs)) {
|
|
|
+ if (name !== "href") $(keep).removeAttr(name);
|
|
|
+ }
|
|
|
+ });
|
|
|
+ const text = textNode.html()?.trim() || "";
|
|
|
+ const datetime = $el.find("time").first().attr("datetime") || "";
|
|
|
+ const photoStyle = $el.find(".tgme_widget_message_photo_wrap").first().attr("style") || "";
|
|
|
+ const photo = photoStyle.match(/url\('([^']+)'\)/)?.[1]?.split("?")[0] || "";
|
|
|
+ const video = $el.find(".tgme_widget_message_video_wrap, .tgme_widget_message_video_player").length > 0;
|
|
|
+ const link = $el.find(".tgme_widget_message_date").first().attr("href") || `https://t.me/${CHANNEL}/${postId}`;
|
|
|
+ if (text || photo || video) {
|
|
|
+ posts.push({ id: Number(postId), text, datetime, photo, video, link });
|
|
|
}
|
|
|
});
|
|
|
- const text = textNode.html()?.trim() || "";
|
|
|
- const datetime = $el.find("time").first().attr("datetime") || "";
|
|
|
- const photoStyle = $el.find(".tgme_widget_message_photo_wrap").first().attr("style") || "";
|
|
|
- const photo = photoStyle.match(/url\('([^']+)'\)/)?.[1]?.split("?")[0] || "";
|
|
|
- const link = $el.find(".tgme_widget_message_date").first().attr("href") || `https://t.me/${CHANNEL}`;
|
|
|
- if (text || photo) posts.push({ text, datetime, photo, link });
|
|
|
-});
|
|
|
-
|
|
|
-const latest = posts.slice(-4).reverse();
|
|
|
+
|
|
|
+ console.log(`page ${page}: +${newCount} posts (total kept ${posts.length})`);
|
|
|
+ if (newCount === 0 || !oldestId) break;
|
|
|
+ before = oldestId;
|
|
|
+ await sleep(300);
|
|
|
+}
|
|
|
+
|
|
|
+// newest first
|
|
|
+posts.sort((a, b) => b.id - a.id);
|
|
|
+
|
|
|
await mkdir(path.dirname("src/data/telegram-news.json"), { recursive: true });
|
|
|
-await writeFile("src/data/telegram-news.json", JSON.stringify(latest, null, 2));
|
|
|
-console.log(`saved ${latest.length} posts`);
|
|
|
+await writeFile("src/data/telegram-news.json", JSON.stringify(posts, null, 2));
|
|
|
+console.log(`saved ${posts.length} posts`);
|
|
|
|
|
|
// Channel avatar for the widget header
|
|
|
const avatar = await fetch(`https://t.me/i/userpic/320/${CHANNEL}.jpg`, { headers: { "User-Agent": UA } });
|
xander