Widget: full channel feed (75 posts) with pagination via fetch script and 'Показать ещё' chunks

scripts/fetch-telegram.mjs

@@ -1,45 +1,71 @@
-// Fetches the public Telegram channel feed (t.me/s/...) and saves the latest
-// posts for the home-page Telegram widget. Run manually: npm run fetch-news
+// Fetches the public Telegram channel feed (t.me/s/...) — the whole first page
+// chain via ?before=<id> pagination — and saves the posts for the home-page
+// Telegram widget. Run manually: npm run fetch-news
 import * as cheerio from "cheerio";
 import { mkdir, writeFile } from "node:fs/promises";
 import path from "node:path";
 
 const CHANNEL = "scientia_ru";
+const MAX_PAGES = 40; // safety cap (~20 posts per page)
 const UA =
   "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36";
 
-const res = await fetch(`https://t.me/s/${CHANNEL}`, { headers: { "User-Agent": UA } });
-if (!res.ok) {
-  console.error(`Failed to fetch channel: HTTP ${res.status}`);
-  process.exit(1);
-}
-const $ = cheerio.load(await res.text());
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 
 const posts = [];
-$(".tgme_widget_message").each((_, el) => {
-  const $el = $(el);
-  const textNode = $el.find(".tgme_widget_message_text").first();
-  // Keep basic formatting tags, drop telegram-internal spans/attrs
-  textNode.find("a, b, i, u, s, strong, em, code, br").each((__, keep) => {
-    const attrs = $(keep).attr() || {};
-    for (const name of Object.keys(attrs)) {
-      if (name !== "href") $(keep).removeAttr(name);
+const seen = new Set();
+let before = null;
+
+for (let page = 0; page < MAX_PAGES; page++) {
+  const url = `https://t.me/s/${CHANNEL}${before ? `?before=${before}` : ""}`;
+  const res = await fetch(url, { headers: { "User-Agent": UA } });
+  if (!res.ok) {
+    console.error(`page ${page}: HTTP ${res.status}`);
+    break;
+  }
+  const $ = cheerio.load(await res.text());
+  let newCount = 0;
+  let oldestId = null;
+
+  $(".tgme_widget_message").each((_, el) => {
+    const $el = $(el);
+    const postId = ($el.attr("data-post") || "").split("/")[1];
+    if (!postId || seen.has(postId)) return;
+    seen.add(postId);
+    newCount++;
+    oldestId = postId;
+
+    const textNode = $el.find(".tgme_widget_message_text").first();
+    // Keep basic formatting tags, drop telegram-internal attributes
+    textNode.find("a, b, i, u, s, strong, em, code, br, blockquote").each((__, keep) => {
+      const attrs = $(keep).attr() || {};
+      for (const name of Object.keys(attrs)) {
+        if (name !== "href") $(keep).removeAttr(name);
+      }
+    });
+    const text = textNode.html()?.trim() || "";
+    const datetime = $el.find("time").first().attr("datetime") || "";
+    const photoStyle = $el.find(".tgme_widget_message_photo_wrap").first().attr("style") || "";
+    const photo = photoStyle.match(/url\('([^']+)'\)/)?.[1]?.split("?")[0] || "";
+    const video = $el.find(".tgme_widget_message_video_wrap, .tgme_widget_message_video_player").length > 0;
+    const link = $el.find(".tgme_widget_message_date").first().attr("href") || `https://t.me/${CHANNEL}/${postId}`;
+    if (text || photo || video) {
+      posts.push({ id: Number(postId), text, datetime, photo, video, link });
     }
   });
-  const text = textNode.html()?.trim() || "";
-  const datetime = $el.find("time").first().attr("datetime") || "";
-  const photoStyle = $el.find(".tgme_widget_message_photo_wrap").first().attr("style") || "";
-  const photo = photoStyle.match(/url\('([^']+)'\)/)?.[1]?.split("?")[0] || "";
-  const video = $el.find(".tgme_widget_message_video_wrap, .tgme_widget_message_video_player").length > 0;
-  const link = $el.find(".tgme_widget_message_date").first().attr("href") || `https://t.me/${CHANNEL}`;
-  if (text || photo || video) posts.push({ text, datetime, photo, video, link });
-});
-
-// t.me/s serves the first page oldest-first — flip so the newest post comes first
-const latest = posts.slice(0, 12).reverse();
+
+  console.log(`page ${page}: +${newCount} posts (total kept ${posts.length})`);
+  if (newCount === 0 || !oldestId) break;
+  before = oldestId;
+  await sleep(300);
+}
+
+// newest first
+posts.sort((a, b) => b.id - a.id);
+
 await mkdir(path.dirname("src/data/telegram-news.json"), { recursive: true });
-await writeFile("src/data/telegram-news.json", JSON.stringify(latest, null, 2));
-console.log(`saved ${latest.length} posts`);
+await writeFile("src/data/telegram-news.json", JSON.stringify(posts, null, 2));
+console.log(`saved ${posts.length} posts`);
 
 // Channel avatar for the widget header
 const avatar = await fetch(`https://t.me/i/userpic/320/${CHANNEL}.jpg`, { headers: { "User-Agent": UA } });

src/components/TelegramWidget.astro

@@ -26,8 +26,8 @@ const fmt = new Intl.DateTimeFormat("ru-RU", { day: "numeric", month: "long", ho
     <!-- Messages: fixed-height scroll area. Bubbles are divs because post text may contain links (nested <a> is invalid HTML) -->
     <div class="tg-wallpaper relative">
       <div class="tg-body space-y-2.5 p-3 pb-5">
-        {telegramNews.map((post) => (
-          <div class="tg-bubble relative ml-3 block rounded-xl bg-white px-3 py-2.5 shadow-[0_1px_1px_rgba(16,35,47,0.14)]">
+        {telegramNews.map((post, i) => (
+          <div class:list={["tg-bubble relative ml-3 block rounded-xl bg-white px-3 py-2.5 shadow-[0_1px_1px_rgba(16,35,47,0.14)]", i >= 8 && "hidden"]}>
             {post.photo && (
               <img src={post.photo} alt="" class="mb-2 w-full rounded-lg" loading="lazy" />
             )}
@@ -55,6 +55,13 @@ const fmt = new Intl.DateTimeFormat("ru-RU", { day: "numeric", month: "long", ho
             </a>
           </div>
         ))}
+        <button
+          type="button"
+          data-tg-more
+          class="mx-auto block rounded-full bg-white px-5 py-2 text-xs font-semibold text-[#2AABEE] shadow-sm transition-colors hover:bg-[#2AABEE] hover:text-white"
+        >
+          Показать ещё
+        </button>
       </div>
       <!-- soft fade over the scroll area -->
       <div class="pointer-events-none absolute inset-x-0 bottom-0 h-7 bg-gradient-to-t from-[#dfe6ec] to-transparent"></div>
@@ -119,3 +126,13 @@ const fmt = new Intl.DateTimeFormat("ru-RU", { day: "numeric", month: "long", ho
     }
   }
 </style>
+
+<script>
+  // "Показать ещё": reveal the next chunk of hidden bubbles
+  const more = document.querySelector("[data-tg-more]");
+  more?.addEventListener("click", () => {
+    const hidden = [...document.querySelectorAll("#tg-widget .tg-bubble.hidden")];
+    hidden.slice(0, 8).forEach((el) => el.classList.remove("hidden"));
+    if (hidden.length <= 8) more.remove();
+  });
+</script>