forust/telegram-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Activity

Merge dev into main: lint fixes and config updates

Browse Source
This commit is contained in:
mr-forust
2026-06-19 11:52:34 +02:00
parent c5361f2271 ac5beb831b
commit f5ef74bda6
19 changed files with 1567 additions and 735 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.editorconfig
+24
View File
@@ -0,0 +1,24 @@
root = true
[*]
indent_style = space
indent_size = 2
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[*.{yml,yaml}]
indent_size = 2
[*.{json,jsonc}]
indent_size = 2
[*.md]
trim_trailing_whitespace = false
[*.py]
indent_size = 4
[{Makefile,makefile}]
indent_style = tab
.hadolint.yaml
+10
View File
@@ -0,0 +1,10 @@
ignored:
- DL3008
- DL3042
- DL3018
- DL3059
trustedRegistries:
- docker.io
- ghcr.io
- quay.io
- gcr.forust.xyz
.markdownlint.json
+8
View File
@@ -0,0 +1,8 @@
{
"default": true,
"MD013": false,
"MD024": false,
"MD033": false,
"MD041": false,
"MD046": false
}
.prettierrc.yaml
+8
View File
@@ -0,0 +1,8 @@
bracketSameLine: true
htmlWhitespaceSensitivity: css
printWidth: 120
tabWidth: 2
singleQuote: true
trailingComma: all
proseWrap: preserve
endOfLine: lf
.vscode/settings.json
Vendored
+34
View File
@@ -0,0 +1,34 @@
{
"[yaml]": {
"editor.tabSize": 2,
"editor.insertSpaces": true,
"editor.formatOnSave": true,
"editor.defaultFormatter": "redhat.vscode-yaml"
},
"[python]": {
"editor.tabSize": 4,
"editor.formatOnSave": true,
"editor.defaultFormatter": "charliermarsh.ruff"
},
"[dockerfile]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "exiasr.hadolint"
},
"[markdown]": {
"editor.wordWrap": "wordWrapColumn",
"editor.wordWrapColumn": 120
},
"yaml.schemas": {
"kubernetes": ["**/k8s/*.yaml", "**/k8s/*.yml"]
},
"yaml.format.enable": true,
"yaml.validate": true,
"yaml.completion": true,
"prettier.bracketSameLine": true,
"prettier.htmlWhitespaceSensitivity": "css",
"prettier.printWidth": 120,
"editor.formatOnSave": true,
"files.autoSave": "onFocusChange",
"editor.wordWrapColumn": 120,
"editor.wordWrap": "wordWrapColumn"
}
.yamllint
+17
View File
@@ -0,0 +1,17 @@
extends: default
rules:
comments:
min-spaces-from-content: 1
comments-indentation: false
document-start: disable
line-length: disable
braces:
min-spaces-inside: 0
max-spaces-inside: 1
brackets:
min-spaces-inside: 0
max-spaces-inside: 1
indentation:
spaces: 2
indent-sequences: consistent
.zed/settings.json
+39
View File
@@ -0,0 +1,39 @@
{
"tab_size": 2,
"soft_wrap": "prefer_line",
"preferred_line_length": 120,
"format_on_save": "on",
"languages": {
"YAML": {
"tab_size": 2,
"hard_tabs": false,
"format_on_save": "on",
"formatter": {
"language_server": { "name": "yaml-language-server" }
}
},
"Python": {
"tab_size": 4,
"format_on_save": "on",
"formatter": {
"language_server": { "name": "ruff" }
}
}
},
"lsp": {
"yaml-language-server": {
"settings": {
"yaml": {
"schemas": {
"kubernetes": ["**/k8s/*.yaml", "**/k8s/*.yml"]
},
"validate": true,
"completion": true,
"format": {
"enable": true
}
}
}
}
}
}
app_state.py
+6 -2
View File
@@ -57,7 +57,9 @@ class StateStore:
self.path.parent.mkdir(parents=True, exist_ok=True) self.path.parent.mkdir(parents=True, exist_ok=True)
tmp_path = self.path.with_suffix(self.path.suffix + ".tmp") tmp_path = self.path.with_suffix(self.path.suffix + ".tmp")
with tmp_path.open("w", encoding="utf-8") as handle: with tmp_path.open("w", encoding="utf-8") as handle:
json.dump(self._merge_defaults(state), handle, ensure_ascii=False, indent=2) json.dump(
self._merge_defaults(state), handle, ensure_ascii=False, indent=2
)
handle.write("\n") handle.write("\n")
tmp_path.replace(self.path) tmp_path.replace(self.path)
self._cache = None self._cache = None
@@ -71,7 +73,9 @@ class StateStore:
def continuous_config(self) -> Dict[str, Any]: def continuous_config(self) -> Dict[str, Any]:
state = self.load() state = self.load()
return dict(state.get("continuous_scraping") or DEFAULT_STATE["continuous_scraping"]) return dict(
state.get("continuous_scraping") or DEFAULT_STATE["continuous_scraping"]
)
def save_continuous_config(self, config: Dict[str, Any]) -> Dict[str, Any]: def save_continuous_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
def mutate(state: Dict[str, Any]) -> None: def mutate(state: Dict[str, Any]) -> None:
compose.yaml
+7
View File
@@ -4,11 +4,18 @@ services:
context: . context: .
dockerfile: Dockerfile dockerfile: Dockerfile
container_name: telegram-scraper container_name: telegram-scraper
image: gcr.forust.xyz/forust/telegram-scraper:latest
user: 1000:1000 user: 1000:1000
restart: unless-stopped restart: unless-stopped
tty: true tty: true
ports: ports:
- "7887:8080" - "7887:8080"
healthcheck:
test: ["CMD", "python", "-c", "import http.client,json;c=http.client.HTTPConnection('localhost',8080);c.request('GET','/health');r=c.getresponse();exit(0)if json.loads(r.read()).get('ok')else exit(1)"]
interval: 30s
timeout: 10s
retries: 3
start_period: 15s
volumes: volumes:
- ./data:/app/data - ./data:/app/data
- ./session:/app/session - ./session:/app/session
k8s/telegram-scraper.yaml
+87
View File
@@ -0,0 +1,87 @@
apiVersion: v1
kind: Namespace
metadata:
name: telegram-scraper
---
apiVersion: v1
kind: Service
metadata:
name: telegram-scraper-service
namespace: telegram-scraper
spec:
selector:
app: telegram-scraper
ports:
- port: 8080
targetPort: 8080
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: telegram-scraper-statefulset
namespace: telegram-scraper
spec:
selector:
matchLabels:
app: telegram-scraper
serviceName: telegram-scraper-service
replicas: 1
template:
metadata:
labels:
app: telegram-scraper
spec:
containers:
- name: telegram-scraper
image: gcr.forust.xyz/forust/telegram-scraper:latest
tty: true
stdin: true
ports:
- containerPort: 8080
livenessProbe:
httpGet:
path: /health
port: 8080
initialDelaySeconds: 15
periodSeconds: 30
readinessProbe:
httpGet:
path: /health
port: 8080
initialDelaySeconds: 5
periodSeconds: 10
volumeMounts:
- name: telegram-scraper-data
mountPath: /app/data
- name: telegram-scraper-session
mountPath: /app/session
volumeClaimTemplates:
- metadata:
name: telegram-scraper-data
spec:
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 20Gi
- metadata:
name: telegram-scraper-session
spec:
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 5Mi
---
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
name: telegram-scraper-route
namespace: telegram-scraper
spec:
entryPoints:
- websecure
routes:
- match: Host(`tg.workstation.internal`)
kind: Rule
services:
- name: telegram-scraper-service
port: 8080
scraper_jobs.py
+2 -1
View File
@@ -41,7 +41,8 @@ class ScraperJobService:
) )
elif job_type == "scrape_selected": elif job_type == "scrape_selected":
await self._scrape_channels( await self._scrape_channels(
scraper, [str(channel_id) for channel_id in payload.get("channels", [])] scraper,
[str(channel_id) for channel_id in payload.get("channels", [])],
) )
elif job_type == "export_all": elif job_type == "export_all":
await scraper.export_data() await scraper.export_data()
scripts/smoke_test.py
+5 -1
View File
@@ -60,7 +60,11 @@ def main() -> int:
status, body = fetch(endpoint) status, body = fetch(endpoint)
if status != 200: if status != 200:
raise RuntimeError(f"{endpoint} returned HTTP {status}") raise RuntimeError(f"{endpoint} returned HTTP {status}")
if endpoint.endswith(".json") or endpoint.startswith("/api") or endpoint.startswith("/health"): if (
endpoint.endswith(".json")
or endpoint.startswith("/api")
or endpoint.startswith("/health")
):
json.loads(body.decode("utf-8")) json.loads(body.decode("utf-8"))
print(f"ok {endpoint}") print(f"ok {endpoint}")
return 0 return 0
telegram-scraper-OLD.py
+357 -181
View File
@@ -5,18 +5,28 @@ import csv
import asyncio import asyncio
import time import time
import sys import sys
import uuid
import warnings import warnings
from dataclasses import dataclass from dataclasses import dataclass
from typing import Dict, List, Optional, Any from typing import Dict, List, Optional, Any
from pathlib import Path from pathlib import Path
from io import StringIO from io import StringIO
from telethon import TelegramClient from telethon import TelegramClient
from telethon.tl.types import MessageMediaPhoto, MessageMediaDocument, MessageMediaWebPage, User, PeerChannel, Channel, Chat from telethon.tl.types import (
MessageMediaPhoto,
MessageMediaDocument,
MessageMediaWebPage,
User,
PeerChannel,
Channel,
Chat,
)
from telethon.errors import FloodWaitError, SessionPasswordNeededError from telethon.errors import FloodWaitError, SessionPasswordNeededError
import qrcode import qrcode
warnings.filterwarnings("ignore", message="Using async sessions support is an experimental feature") warnings.filterwarnings(
"ignore", message="Using async sessions support is an experimental feature"
)
def display_ascii_art(): def display_ascii_art():
WHITE = "\033[97m" WHITE = "\033[97m"
@@ -31,6 +41,7 @@ ___________________ _________
""" """
print(WHITE + art + RESET) print(WHITE + art + RESET)
@dataclass @dataclass
class MessageData: class MessageData:
message_id: int message_id: int
@@ -48,9 +59,10 @@ class MessageData:
forwards: Optional[int] forwards: Optional[int]
reactions: Optional[str] reactions: Optional[str]
class OptimizedTelegramScraper: class OptimizedTelegramScraper:
def __init__(self): def __init__(self):
self.STATE_FILE = 'state.json' self.STATE_FILE = "state.json"
self.state = self.load_state() self.state = self.load_state()
self.client = None self.client = None
self.continuous_scraping_active = False self.continuous_scraping_active = False
@@ -62,21 +74,21 @@ class OptimizedTelegramScraper:
def load_state(self) -> Dict[str, Any]: def load_state(self) -> Dict[str, Any]:
if os.path.exists(self.STATE_FILE): if os.path.exists(self.STATE_FILE):
try: try:
with open(self.STATE_FILE, 'r') as f: with open(self.STATE_FILE, "r") as f:
return json.load(f) return json.load(f)
except: except Exception:
pass pass
return { return {
'api_id': None, "api_id": None,
'api_hash': None, "api_hash": None,
'channels': {}, "channels": {},
'channel_names': {}, "channel_names": {},
'scrape_media': True, "scrape_media": True,
} }
def save_state(self): def save_state(self):
try: try:
with open(self.STATE_FILE, 'w') as f: with open(self.STATE_FILE, "w") as f:
json.dump(self.state, f, indent=2) json.dump(self.state, f, indent=2)
except Exception as e: except Exception as e:
print(f"Failed to save state: {e}") print(f"Failed to save state: {e}")
@@ -86,17 +98,19 @@ class OptimizedTelegramScraper:
channel_dir = Path(channel) channel_dir = Path(channel)
channel_dir.mkdir(exist_ok=True) channel_dir.mkdir(exist_ok=True)
db_file = channel_dir / f'{channel}.db' db_file = channel_dir / f"{channel}.db"
conn = sqlite3.connect(str(db_file), check_same_thread=False) conn = sqlite3.connect(str(db_file), check_same_thread=False)
conn.execute('''CREATE TABLE IF NOT EXISTS messages conn.execute("""CREATE TABLE IF NOT EXISTS messages
(id INTEGER PRIMARY KEY, message_id INTEGER UNIQUE, date TEXT, (id INTEGER PRIMARY KEY, message_id INTEGER UNIQUE, date TEXT,
sender_id INTEGER, first_name TEXT, last_name TEXT, username TEXT, sender_id INTEGER, first_name TEXT, last_name TEXT, username TEXT,
message TEXT, media_type TEXT, media_path TEXT, reply_to INTEGER, message TEXT, media_type TEXT, media_path TEXT, reply_to INTEGER,
post_author TEXT, views INTEGER, forwards INTEGER, reactions TEXT)''') post_author TEXT, views INTEGER, forwards INTEGER, reactions TEXT)""")
conn.execute('CREATE INDEX IF NOT EXISTS idx_message_id ON messages(message_id)') conn.execute(
conn.execute('CREATE INDEX IF NOT EXISTS idx_date ON messages(date)') "CREATE INDEX IF NOT EXISTS idx_message_id ON messages(message_id)"
conn.execute('PRAGMA journal_mode=WAL') )
conn.execute('PRAGMA synchronous=NORMAL') conn.execute("CREATE INDEX IF NOT EXISTS idx_date ON messages(date)")
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA synchronous=NORMAL")
conn.commit() conn.commit()
self.migrate_database(conn) self.migrate_database(conn)
@@ -111,19 +125,19 @@ class OptimizedTelegramScraper:
columns = {row[1] for row in cursor.fetchall()} columns = {row[1] for row in cursor.fetchall()}
migrations = [] migrations = []
if 'post_author' not in columns: if "post_author" not in columns:
migrations.append('ALTER TABLE messages ADD COLUMN post_author TEXT') migrations.append("ALTER TABLE messages ADD COLUMN post_author TEXT")
if 'views' not in columns: if "views" not in columns:
migrations.append('ALTER TABLE messages ADD COLUMN views INTEGER') migrations.append("ALTER TABLE messages ADD COLUMN views INTEGER")
if 'forwards' not in columns: if "forwards" not in columns:
migrations.append('ALTER TABLE messages ADD COLUMN forwards INTEGER') migrations.append("ALTER TABLE messages ADD COLUMN forwards INTEGER")
if 'reactions' not in columns: if "reactions" not in columns:
migrations.append('ALTER TABLE messages ADD COLUMN reactions TEXT') migrations.append("ALTER TABLE messages ADD COLUMN reactions TEXT")
for migration in migrations: for migration in migrations:
try: try:
conn.execute(migration) conn.execute(migration)
except: except Exception:
pass pass
if migrations: if migrations:
@@ -139,20 +153,38 @@ class OptimizedTelegramScraper:
return return
conn = self.get_db_connection(channel) conn = self.get_db_connection(channel)
data = [(msg.message_id, msg.date, msg.sender_id, msg.first_name, data = [
msg.last_name, msg.username, msg.message, msg.media_type, (
msg.media_path, msg.reply_to, msg.post_author, msg.views, msg.message_id,
msg.forwards, msg.reactions) for msg in messages] msg.date,
msg.sender_id,
msg.first_name,
msg.last_name,
msg.username,
msg.message,
msg.media_type,
msg.media_path,
msg.reply_to,
msg.post_author,
msg.views,
msg.forwards,
msg.reactions,
)
for msg in messages
]
conn.executemany('''INSERT OR IGNORE INTO messages conn.executemany(
"""INSERT OR IGNORE INTO messages
(message_id, date, sender_id, first_name, last_name, username, (message_id, date, sender_id, first_name, last_name, username,
message, media_type, media_path, reply_to, post_author, views, message, media_type, media_path, reply_to, post_author, views,
forwards, reactions) forwards, reactions)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''', data) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
data,
)
conn.commit() conn.commit()
async def download_media(self, channel: str, message) -> Optional[str]: async def download_media(self, channel: str, message) -> Optional[str]:
if not message.media or not self.state['scrape_media']: if not message.media or not self.state["scrape_media"]:
return None return None
if isinstance(message.media, MessageMediaWebPage): if isinstance(message.media, MessageMediaWebPage):
@@ -160,15 +192,15 @@ class OptimizedTelegramScraper:
try: try:
channel_dir = Path(channel) channel_dir = Path(channel)
media_folder = channel_dir / 'media' media_folder = channel_dir / "media"
media_folder.mkdir(exist_ok=True) media_folder.mkdir(exist_ok=True)
if isinstance(message.media, MessageMediaPhoto): if isinstance(message.media, MessageMediaPhoto):
original_name = getattr(message.file, 'name', None) or "photo.jpg" original_name = getattr(message.file, "name", None) or "photo.jpg"
ext = "jpg" ext = "jpg"
elif isinstance(message.media, MessageMediaDocument): elif isinstance(message.media, MessageMediaDocument):
ext = getattr(message.file, 'ext', 'bin') if message.file else 'bin' ext = getattr(message.file, "ext", "bin") if message.file else "bin"
original_name = getattr(message.file, 'name', None) or f"document.{ext}" original_name = getattr(message.file, "name", None) or f"document.{ext}"
else: else:
return None return None
@@ -205,14 +237,20 @@ class OptimizedTelegramScraper:
async def update_media_path(self, channel: str, message_id: int, media_path: str): async def update_media_path(self, channel: str, message_id: int, media_path: str):
conn = self.get_db_connection(channel) conn = self.get_db_connection(channel)
conn.execute('UPDATE messages SET media_path = ? WHERE message_id = ?', conn.execute(
(media_path, message_id)) "UPDATE messages SET media_path = ? WHERE message_id = ?",
(media_path, message_id),
)
conn.commit() conn.commit()
async def scrape_channel(self, channel: str, offset_id: int): async def scrape_channel(self, channel: str, offset_id: int):
try: try:
entity = await self.client.get_entity(PeerChannel(int(channel)) if channel.startswith('-') else channel) entity = await self.client.get_entity(
result = await self.client.get_messages(entity, offset_id=offset_id, reverse=True, limit=0) PeerChannel(int(channel)) if channel.startswith("-") else channel
)
result = await self.client.get_messages(
entity, offset_id=offset_id, reverse=True, limit=0
)
total_messages = result.total total_messages = result.total
if total_messages == 0: if total_messages == 0:
@@ -227,7 +265,9 @@ class OptimizedTelegramScraper:
last_message_id = offset_id last_message_id = offset_id
semaphore = asyncio.Semaphore(self.max_concurrent_downloads) semaphore = asyncio.Semaphore(self.max_concurrent_downloads)
async for message in self.client.iter_messages(entity, offset_id=offset_id, reverse=True): async for message in self.client.iter_messages(
entity, offset_id=offset_id, reverse=True
):
try: try:
sender = await message.get_sender() sender = await message.get_sender()
@@ -235,33 +275,45 @@ class OptimizedTelegramScraper:
if message.reactions and message.reactions.results: if message.reactions and message.reactions.results:
reactions_parts = [] reactions_parts = []
for reaction in message.reactions.results: for reaction in message.reactions.results:
emoji = getattr(reaction.reaction, 'emoticon', '') emoji = getattr(reaction.reaction, "emoticon", "")
count = reaction.count count = reaction.count
if emoji: if emoji:
reactions_parts.append(f"{emoji} {count}") reactions_parts.append(f"{emoji} {count}")
if reactions_parts: if reactions_parts:
reactions_str = ' '.join(reactions_parts) reactions_str = " ".join(reactions_parts)
msg_data = MessageData( msg_data = MessageData(
message_id=message.id, message_id=message.id,
date=message.date.strftime('%Y-%m-%d %H:%M:%S'), date=message.date.strftime("%Y-%m-%d %H:%M:%S"),
sender_id=message.sender_id, sender_id=message.sender_id,
first_name=getattr(sender, 'first_name', None) if isinstance(sender, User) else None, first_name=getattr(sender, "first_name", None)
last_name=getattr(sender, 'last_name', None) if isinstance(sender, User) else None, if isinstance(sender, User)
username=getattr(sender, 'username', None) if isinstance(sender, User) else None, else None,
message=message.message or '', last_name=getattr(sender, "last_name", None)
media_type=message.media.__class__.__name__ if message.media else None, if isinstance(sender, User)
else None,
username=getattr(sender, "username", None)
if isinstance(sender, User)
else None,
message=message.message or "",
media_type=message.media.__class__.__name__
if message.media
else None,
media_path=None, media_path=None,
reply_to=message.reply_to_msg_id if message.reply_to else None, reply_to=message.reply_to_msg_id if message.reply_to else None,
post_author=message.post_author, post_author=message.post_author,
views=message.views, views=message.views,
forwards=message.forwards, forwards=message.forwards,
reactions=reactions_str reactions=reactions_str,
) )
message_batch.append(msg_data) message_batch.append(msg_data)
if self.state['scrape_media'] and message.media and not isinstance(message.media, MessageMediaWebPage): if (
self.state["scrape_media"]
and message.media
and not isinstance(message.media, MessageMediaWebPage)
):
media_tasks.append(message) media_tasks.append(message)
last_message_id = message.id last_message_id = message.id
@@ -272,15 +324,19 @@ class OptimizedTelegramScraper:
message_batch.clear() message_batch.clear()
if processed_messages % self.state_save_interval == 0: if processed_messages % self.state_save_interval == 0:
self.state['channels'][channel] = last_message_id self.state["channels"][channel] = last_message_id
self.save_state() self.save_state()
progress = (processed_messages / total_messages) * 100 progress = (processed_messages / total_messages) * 100
bar_length = 30 bar_length = 30
filled_length = int(bar_length * processed_messages // total_messages) filled_length = int(
bar = '█' * filled_length + '░' * (bar_length - filled_length) bar_length * processed_messages // total_messages
)
bar = "█" * filled_length + "░" * (bar_length - filled_length)
sys.stdout.write(f"\r📄 Messages: [{bar}] {progress:.1f}% ({processed_messages}/{total_messages})") sys.stdout.write(
f"\r📄 Messages: [{bar}] {progress:.1f}% ({processed_messages}/{total_messages})"
)
sys.stdout.flush() sys.stdout.flush()
except Exception as e: except Exception as e:
@@ -304,13 +360,17 @@ class OptimizedTelegramScraper:
batch_size = 10 batch_size = 10
for i in range(0, len(media_tasks), batch_size): for i in range(0, len(media_tasks), batch_size):
batch = media_tasks[i : i + batch_size] batch = media_tasks[i : i + batch_size]
tasks = [asyncio.create_task(download_single_media(msg)) for msg in batch] tasks = [
asyncio.create_task(download_single_media(msg)) for msg in batch
]
for j, task in enumerate(tasks): for j, task in enumerate(tasks):
try: try:
media_path = await task media_path = await task
if media_path: if media_path:
await self.update_media_path(channel, batch[j].id, media_path) await self.update_media_path(
channel, batch[j].id, media_path
)
successful_downloads += 1 successful_downloads += 1
except Exception: except Exception:
pass pass
@@ -319,14 +379,18 @@ class OptimizedTelegramScraper:
progress = (completed_media / total_media) * 100 progress = (completed_media / total_media) * 100
bar_length = 30 bar_length = 30
filled_length = int(bar_length * completed_media // total_media) filled_length = int(bar_length * completed_media // total_media)
bar = '█' * filled_length + '░' * (bar_length - filled_length) bar = "█" * filled_length + "░" * (bar_length - filled_length)
sys.stdout.write(f"\r📥 Media: [{bar}] {progress:.1f}% ({completed_media}/{total_media})") sys.stdout.write(
f"\r📥 Media: [{bar}] {progress:.1f}% ({completed_media}/{total_media})"
)
sys.stdout.flush() sys.stdout.flush()
print(f"\n✅ Media download complete! ({successful_downloads}/{total_media} successful)") print(
f"\n✅ Media download complete! ({successful_downloads}/{total_media} successful)"
)
self.state['channels'][channel] = last_message_id self.state["channels"][channel] = last_message_id
self.save_state() self.save_state()
print(f"\nCompleted scraping channel {channel}") print(f"\nCompleted scraping channel {channel}")
@@ -336,19 +400,23 @@ class OptimizedTelegramScraper:
async def rescrape_media(self, channel: str): async def rescrape_media(self, channel: str):
conn = self.get_db_connection(channel) conn = self.get_db_connection(channel)
cursor = conn.cursor() cursor = conn.cursor()
cursor.execute('SELECT message_id FROM messages WHERE media_type IS NOT NULL AND media_type != "MessageMediaWebPage" AND media_path IS NULL') cursor.execute(
'SELECT message_id FROM messages WHERE media_type IS NOT NULL AND media_type != "MessageMediaWebPage" AND media_path IS NULL'
)
message_ids = [row[0] for row in cursor.fetchall()] message_ids = [row[0] for row in cursor.fetchall()]
channel_name = self.state.get('channel_names', {}).get(channel, 'Unknown') channel_name = self.state.get("channel_names", {}).get(channel, "Unknown")
if not message_ids: if not message_ids:
print(f"No media files to reprocess for {channel_name} (ID: {channel})") print(f"No media files to reprocess for {channel_name} (ID: {channel})")
return return
print(f"📥 Reprocessing {len(message_ids)} media files for {channel_name} (ID: {channel})") print(
f"📥 Reprocessing {len(message_ids)} media files for {channel_name} (ID: {channel})"
)
try: try:
if channel.lstrip('-').isdigit(): if channel.lstrip("-").isdigit():
entity = await self.client.get_entity(PeerChannel(int(channel))) entity = await self.client.get_entity(PeerChannel(int(channel)))
else: else:
entity = await self.client.get_entity(channel) entity = await self.client.get_entity(channel)
@@ -365,14 +433,25 @@ class OptimizedTelegramScraper:
batch_ids = message_ids[i : i + batch_size] batch_ids = message_ids[i : i + batch_size]
messages = await self.client.get_messages(entity, ids=batch_ids) messages = await self.client.get_messages(entity, ids=batch_ids)
valid_messages = [msg for msg in messages if msg and msg.media and not isinstance(msg.media, MessageMediaWebPage)] valid_messages = [
tasks = [asyncio.create_task(download_single_media(msg)) for msg in valid_messages] msg
for msg in messages
if msg
and msg.media
and not isinstance(msg.media, MessageMediaWebPage)
]
tasks = [
asyncio.create_task(download_single_media(msg))
for msg in valid_messages
]
for j, task in enumerate(tasks): for j, task in enumerate(tasks):
try: try:
media_path = await task media_path = await task
if media_path: if media_path:
await self.update_media_path(channel, valid_messages[j].id, media_path) await self.update_media_path(
channel, valid_messages[j].id, media_path
)
successful_downloads += 1 successful_downloads += 1
except Exception: except Exception:
pass pass
@@ -380,13 +459,19 @@ class OptimizedTelegramScraper:
completed_media += 1 completed_media += 1
progress = (completed_media / len(message_ids)) * 100 progress = (completed_media / len(message_ids)) * 100
bar_length = 30 bar_length = 30
filled_length = int(bar_length * completed_media // len(message_ids)) filled_length = int(
bar = '█' * filled_length + '░' * (bar_length - filled_length) bar_length * completed_media // len(message_ids)
)
bar = "█" * filled_length + "░" * (bar_length - filled_length)
sys.stdout.write(f"\r🔄 Rescrape: [{bar}] {progress:.1f}% ({completed_media}/{len(message_ids)})") sys.stdout.write(
f"\r🔄 Rescrape: [{bar}] {progress:.1f}% ({completed_media}/{len(message_ids)})"
)
sys.stdout.flush() sys.stdout.flush()
print(f"\n✅ Media reprocessing complete! ({successful_downloads}/{len(message_ids)} successful)") print(
f"\n✅ Media reprocessing complete! ({successful_downloads}/{len(message_ids)} successful)"
)
except Exception as e: except Exception as e:
print(f"Error reprocessing media: {e}") print(f"Error reprocessing media: {e}")
@@ -395,15 +480,19 @@ class OptimizedTelegramScraper:
conn = self.get_db_connection(channel) conn = self.get_db_connection(channel)
cursor = conn.cursor() cursor = conn.cursor()
cursor.execute('SELECT COUNT(*) FROM messages WHERE media_type IS NOT NULL AND media_type != "MessageMediaWebPage"') cursor.execute(
'SELECT COUNT(*) FROM messages WHERE media_type IS NOT NULL AND media_type != "MessageMediaWebPage"'
)
total_with_media = cursor.fetchone()[0] total_with_media = cursor.fetchone()[0]
cursor.execute('SELECT COUNT(*) FROM messages WHERE media_type IS NOT NULL AND media_type != "MessageMediaWebPage" AND media_path IS NOT NULL') cursor.execute(
'SELECT COUNT(*) FROM messages WHERE media_type IS NOT NULL AND media_type != "MessageMediaWebPage" AND media_path IS NOT NULL'
)
total_with_files = cursor.fetchone()[0] total_with_files = cursor.fetchone()[0]
missing_count = total_with_media - total_with_files missing_count = total_with_media - total_with_files
channel_name = self.state.get('channel_names', {}).get(channel, 'Unknown') channel_name = self.state.get("channel_names", {}).get(channel, "Unknown")
print(f"\n📊 Media Analysis for {channel_name} (ID: {channel}):") print(f"\n📊 Media Analysis for {channel_name} (ID: {channel}):")
print(f"Messages with media: {total_with_media}") print(f"Messages with media: {total_with_media}")
print(f"Media files downloaded: {total_with_files}") print(f"Media files downloaded: {total_with_files}")
@@ -413,17 +502,21 @@ class OptimizedTelegramScraper:
print("✅ All media files are already downloaded!") print("✅ All media files are already downloaded!")
return return
cursor.execute('SELECT message_id, media_type FROM messages WHERE media_type IS NOT NULL AND media_type != "MessageMediaWebPage" AND (media_path IS NULL OR media_path = "")') cursor.execute(
'SELECT message_id, media_type FROM messages WHERE media_type IS NOT NULL AND media_type != "MessageMediaWebPage" AND (media_path IS NULL OR media_path = "")'
)
missing_media = cursor.fetchall() missing_media = cursor.fetchall()
if not missing_media: if not missing_media:
print("✅ No missing media found!") print("✅ No missing media found!")
return return
print(f"\n🔧 Attempting to download {len(missing_media)} missing media files...") print(
f"\n🔧 Attempting to download {len(missing_media)} missing media files..."
)
try: try:
if channel.lstrip('-').isdigit(): if channel.lstrip("-").isdigit():
entity = await self.client.get_entity(PeerChannel(int(channel))) entity = await self.client.get_entity(PeerChannel(int(channel)))
else: else:
entity = await self.client.get_entity(channel) entity = await self.client.get_entity(channel)
@@ -441,15 +534,26 @@ class OptimizedTelegramScraper:
message_ids = [msg[0] for msg in batch] message_ids = [msg[0] for msg in batch]
messages = await self.client.get_messages(entity, ids=message_ids) messages = await self.client.get_messages(entity, ids=message_ids)
valid_messages = [msg for msg in messages if msg and msg.media and not isinstance(msg.media, MessageMediaWebPage)] valid_messages = [
msg
for msg in messages
if msg
and msg.media
and not isinstance(msg.media, MessageMediaWebPage)
]
tasks = [asyncio.create_task(download_single_media(msg)) for msg in valid_messages] tasks = [
asyncio.create_task(download_single_media(msg))
for msg in valid_messages
]
for j, task in enumerate(tasks): for j, task in enumerate(tasks):
try: try:
media_path = await task media_path = await task
if media_path: if media_path:
await self.update_media_path(channel, valid_messages[j].id, media_path) await self.update_media_path(
channel, valid_messages[j].id, media_path
)
successful_downloads += 1 successful_downloads += 1
except Exception: except Exception:
pass pass
@@ -457,13 +561,19 @@ class OptimizedTelegramScraper:
completed_media += 1 completed_media += 1
progress = (completed_media / len(missing_media)) * 100 progress = (completed_media / len(missing_media)) * 100
bar_length = 30 bar_length = 30
filled_length = int(bar_length * completed_media // len(missing_media)) filled_length = int(
bar = '█' * filled_length + '░' * (bar_length - filled_length) bar_length * completed_media // len(missing_media)
)
bar = "█" * filled_length + "░" * (bar_length - filled_length)
sys.stdout.write(f"\r🔧 Fix Media: [{bar}] {progress:.1f}% ({completed_media}/{len(missing_media)})") sys.stdout.write(
f"\r🔧 Fix Media: [{bar}] {progress:.1f}% ({completed_media}/{len(missing_media)})"
)
sys.stdout.flush() sys.stdout.flush()
print(f"\n✅ Media fix complete! ({successful_downloads}/{len(missing_media)} successful)") print(
f"\n✅ Media fix complete! ({successful_downloads}/{len(missing_media)} successful)"
)
except Exception as e: except Exception as e:
print(f"Error fixing missing media: {e}") print(f"Error fixing missing media: {e}")
@@ -475,11 +585,11 @@ class OptimizedTelegramScraper:
while self.continuous_scraping_active: while self.continuous_scraping_active:
start_time = time.time() start_time = time.time()
for channel in self.state['channels']: for channel in self.state["channels"]:
if not self.continuous_scraping_active: if not self.continuous_scraping_active:
break break
print(f"\nChecking for new messages in channel: {channel}") print(f"\nChecking for new messages in channel: {channel}")
await self.scrape_channel(channel, self.state['channels'][channel]) await self.scrape_channel(channel, self.state["channels"][channel])
elapsed = time.time() - start_time elapsed = time.time() - start_time
sleep_time = max(0, 60 - elapsed) sleep_time = max(0, 60 - elapsed)
@@ -492,19 +602,19 @@ class OptimizedTelegramScraper:
self.continuous_scraping_active = False self.continuous_scraping_active = False
def get_export_filename(self, channel: str): def get_export_filename(self, channel: str):
username = self.state.get('channel_names', {}).get(channel, 'no_username') username = self.state.get("channel_names", {}).get(channel, "no_username")
return f"{channel}_{username}" return f"{channel}_{username}"
def export_to_csv(self, channel: str): def export_to_csv(self, channel: str):
conn = self.get_db_connection(channel) conn = self.get_db_connection(channel)
filename = self.get_export_filename(channel) filename = self.get_export_filename(channel)
csv_file = Path(channel) / f'{filename}.csv' csv_file = Path(channel) / f"{filename}.csv"
cursor = conn.cursor() cursor = conn.cursor()
cursor.execute('SELECT * FROM messages ORDER BY date') cursor.execute("SELECT * FROM messages ORDER BY date")
columns = [description[0] for description in cursor.description] columns = [description[0] for description in cursor.description]
with open(csv_file, 'w', newline='', encoding='utf-8') as f: with open(csv_file, "w", newline="", encoding="utf-8") as f:
writer = csv.writer(f) writer = csv.writer(f)
writer.writerow(columns) writer.writerow(columns)
@@ -517,14 +627,14 @@ class OptimizedTelegramScraper:
def export_to_json(self, channel: str): def export_to_json(self, channel: str):
conn = self.get_db_connection(channel) conn = self.get_db_connection(channel)
filename = self.get_export_filename(channel) filename = self.get_export_filename(channel)
json_file = Path(channel) / f'{filename}.json' json_file = Path(channel) / f"{filename}.json"
cursor = conn.cursor() cursor = conn.cursor()
cursor.execute('SELECT * FROM messages ORDER BY date') cursor.execute("SELECT * FROM messages ORDER BY date")
columns = [description[0] for description in cursor.description] columns = [description[0] for description in cursor.description]
with open(json_file, 'w', encoding='utf-8') as f: with open(json_file, "w", encoding="utf-8") as f:
f.write('[\n') f.write("[\n")
first_row = True first_row = True
while True: while True:
@@ -534,21 +644,21 @@ class OptimizedTelegramScraper:
for row in rows: for row in rows:
if not first_row: if not first_row:
f.write(',\n') f.write(",\n")
else: else:
first_row = False first_row = False
data = dict(zip(columns, row)) data = dict(zip(columns, row))
json.dump(data, f, ensure_ascii=False, indent=2) json.dump(data, f, ensure_ascii=False, indent=2)
f.write('\n]') f.write("\n]")
async def export_data(self): async def export_data(self):
if not self.state['channels']: if not self.state["channels"]:
print("No channels to export") print("No channels to export")
return return
for channel in self.state['channels']: for channel in self.state["channels"]:
print(f"Exporting data for channel {channel}...") print(f"Exporting data for channel {channel}...")
try: try:
self.export_to_csv(channel) self.export_to_csv(channel)
@@ -558,22 +668,30 @@ class OptimizedTelegramScraper:
print(f"❌ Export failed for channel {channel}: {e}") print(f"❌ Export failed for channel {channel}: {e}")
async def view_channels(self): async def view_channels(self):
if not self.state['channels']: if not self.state["channels"]:
print("No channels saved") print("No channels saved")
return return
print("\nCurrent channels:") print("\nCurrent channels:")
for i, (channel, last_id) in enumerate(self.state['channels'].items(), 1): for i, (channel, last_id) in enumerate(self.state["channels"].items(), 1):
try: try:
conn = self.get_db_connection(channel) conn = self.get_db_connection(channel)
cursor = conn.cursor() cursor = conn.cursor()
cursor.execute('SELECT COUNT(*) FROM messages') cursor.execute("SELECT COUNT(*) FROM messages")
count = cursor.fetchone()[0] count = cursor.fetchone()[0]
channel_name = self.state.get('channel_names', {}).get(channel, 'Unknown') channel_name = self.state.get("channel_names", {}).get(
print(f"[{i}] {channel_name} (ID: {channel}), Last Message ID: {last_id}, Messages: {count}") channel, "Unknown"
except: )
channel_name = self.state.get('channel_names', {}).get(channel, 'Unknown') print(
print(f"[{i}] {channel_name} (ID: {channel}), Last Message ID: {last_id}") f"[{i}] {channel_name} (ID: {channel}), Last Message ID: {last_id}, Messages: {count}"
)
except Exception:
channel_name = self.state.get("channel_names", {}).get(
channel, "Unknown"
)
print(
f"[{i}] {channel_name} (ID: {channel}), Last Message ID: {last_id}"
)
async def list_channels(self): async def list_channels(self):
try: try:
@@ -582,23 +700,42 @@ class OptimizedTelegramScraper:
channels_data = [] channels_data = []
async for dialog in self.client.iter_dialogs(): async for dialog in self.client.iter_dialogs():
entity = dialog.entity entity = dialog.entity
if dialog.id != 777000 and (isinstance(entity, Channel) or isinstance(entity, Chat)): if dialog.id != 777000 and (
channel_type = "Channel" if isinstance(entity, Channel) and entity.broadcast else "Group" isinstance(entity, Channel) or isinstance(entity, Chat)
username = getattr(entity, 'username', None) or 'no_username' ):
print(f"[{count}] {dialog.title} (ID: {dialog.id}, Type: {channel_type}, Username: @{username})") channel_type = (
channels_data.append({ "Channel"
'number': count, if isinstance(entity, Channel) and entity.broadcast
'channel_name': dialog.title, else "Group"
'channel_id': str(dialog.id), )
'username': username, username = getattr(entity, "username", None) or "no_username"
'type': channel_type print(
}) f"[{count}] {dialog.title} (ID: {dialog.id}, Type: {channel_type}, Username: @{username})"
)
channels_data.append(
{
"number": count,
"channel_name": dialog.title,
"channel_id": str(dialog.id),
"username": username,
"type": channel_type,
}
)
count += 1 count += 1
if channels_data: if channels_data:
csv_file = Path('channels_list.csv') csv_file = Path("channels_list.csv")
with open(csv_file, 'w', newline='', encoding='utf-8') as f: with open(csv_file, "w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=['number', 'channel_name', 'channel_id', 'username', 'type']) writer = csv.DictWriter(
f,
fieldnames=[
"number",
"channel_name",
"channel_id",
"username",
"type",
],
)
writer.writeheader() writer.writeheader()
writer.writerows(channels_data) writer.writerows(channels_data)
print(f"\n✅ Saved channels list to {csv_file}") print(f"\n✅ Saved channels list to {csv_file}")
@@ -661,18 +798,20 @@ class OptimizedTelegramScraper:
return False return False
async def initialize_client(self): async def initialize_client(self):
if not all([self.state.get('api_id'), self.state.get('api_hash')]): if not all([self.state.get("api_id"), self.state.get("api_hash")]):
print("\n=== API Configuration Required ===") print("\n=== API Configuration Required ===")
print("You need to provide API credentials from https://my.telegram.org") print("You need to provide API credentials from https://my.telegram.org")
try: try:
self.state['api_id'] = int(input("Enter your API ID: ")) self.state["api_id"] = int(input("Enter your API ID: "))
self.state['api_hash'] = input("Enter your API Hash: ") self.state["api_hash"] = input("Enter your API Hash: ")
self.save_state() self.save_state()
except ValueError: except ValueError:
print("Invalid API ID. Must be a number.") print("Invalid API ID. Must be a number.")
return False return False
self.client = TelegramClient('session', self.state['api_id'], self.state['api_hash']) self.client = TelegramClient(
"session", self.state["api_id"], self.state["api_hash"]
)
try: try:
await self.client.connect() await self.client.connect()
@@ -687,11 +826,13 @@ class OptimizedTelegramScraper:
while True: while True:
choice = input("Enter your choice (1 or 2): ").strip() choice = input("Enter your choice (1 or 2): ").strip()
if choice in ['1', '2']: if choice in ["1", "2"]:
break break
print("Please enter 1 or 2") print("Please enter 1 or 2")
success = await self.qr_code_auth() if choice == '1' else await self.phone_auth() success = (
await self.qr_code_auth() if choice == "1" else await self.phone_auth()
)
if not success: if not success:
print("Authentication failed. Please try again.") print("Authentication failed. Please try again.")
@@ -703,16 +844,16 @@ class OptimizedTelegramScraper:
return True return True
def parse_channel_selection(self, choice): def parse_channel_selection(self, choice):
channels_list = list(self.state['channels'].keys()) channels_list = list(self.state["channels"].keys())
selected_channels = [] selected_channels = []
if choice.lower() == 'all': if choice.lower() == "all":
return channels_list return channels_list
for selection in [x.strip() for x in choice.split(',')]: for selection in [x.strip() for x in choice.split(",")]:
try: try:
if selection.startswith('-'): if selection.startswith("-"):
if selection in self.state['channels']: if selection in self.state["channels"]:
selected_channels.append(selection) selected_channels.append(selection)
else: else:
print(f"Channel ID {selection} not found in your channels") print(f"Channel ID {selection} not found in your channels")
@@ -721,14 +862,18 @@ class OptimizedTelegramScraper:
if 1 <= num <= len(channels_list): if 1 <= num <= len(channels_list):
selected_channels.append(channels_list[num - 1]) selected_channels.append(channels_list[num - 1])
else: else:
print(f"Invalid channel number: {num}. Valid range: 1-{len(channels_list)}") print(
f"Invalid channel number: {num}. Valid range: 1-{len(channels_list)}"
)
except ValueError: except ValueError:
print(f"Invalid input: {selection}. Use numbers (1,2,3) or full IDs (-100123...)") print(
f"Invalid input: {selection}. Use numbers (1,2,3) or full IDs (-100123...)"
)
return selected_channels return selected_channels
async def scrape_specific_channels(self): async def scrape_specific_channels(self):
if not self.state['channels']: if not self.state["channels"]:
print("No channels available. Use [L] to add channels first") print("No channels available. Use [L] to add channels first")
return return
@@ -745,7 +890,7 @@ class OptimizedTelegramScraper:
print(f"\n🚀 Starting scrape of {len(selected_channels)} channel(s)...") print(f"\n🚀 Starting scrape of {len(selected_channels)} channel(s)...")
for i, channel in enumerate(selected_channels, 1): for i, channel in enumerate(selected_channels, 1):
print(f"\n[{i}/{len(selected_channels)}] Scraping: {channel}") print(f"\n[{i}/{len(selected_channels)}] Scraping: {channel}")
await self.scrape_channel(channel, self.state['channels'][channel]) await self.scrape_channel(channel, self.state["channels"][channel])
print(f"\n✅ Completed scraping {len(selected_channels)} channel(s)!") print(f"\n✅ Completed scraping {len(selected_channels)} channel(s)!")
else: else:
print("❌ No valid channels selected") print("❌ No valid channels selected")
@@ -757,7 +902,9 @@ class OptimizedTelegramScraper:
print("=" * 40) print("=" * 40)
print("[S] Scrape channels") print("[S] Scrape channels")
print("[C] Continuous scraping") print("[C] Continuous scraping")
print(f"[M] Media scraping: {'ON' if self.state['scrape_media'] else 'OFF'}") print(
f"[M] Media scraping: {'ON' if self.state['scrape_media'] else 'OFF'}"
)
print("[L] List & add channels") print("[L] List & add channels")
print("[R] Remove channels") print("[R] Remove channels")
print("[E] Export data") print("[E] Export data")
@@ -769,8 +916,8 @@ class OptimizedTelegramScraper:
choice = input("Enter your choice: ").lower().strip() choice = input("Enter your choice: ").lower().strip()
try: try:
if choice == 'r': if choice == "r":
if not self.state['channels']: if not self.state["channels"]:
print("No channels to remove") print("No channels to remove")
continue continue
@@ -784,8 +931,8 @@ class OptimizedTelegramScraper:
if selected_channels: if selected_channels:
removed_count = 0 removed_count = 0
for channel in selected_channels: for channel in selected_channels:
if channel in self.state['channels']: if channel in self.state["channels"]:
del self.state['channels'][channel] del self.state["channels"][channel]
print(f"✅ Removed channel {channel}") print(f"✅ Removed channel {channel}")
removed_count += 1 removed_count += 1
else: else:
@@ -800,19 +947,21 @@ class OptimizedTelegramScraper:
else: else:
print("No valid channels selected") print("No valid channels selected")
elif choice == 's': elif choice == "s":
await self.scrape_specific_channels() await self.scrape_specific_channels()
elif choice == 'm': elif choice == "m":
self.state['scrape_media'] = not self.state['scrape_media'] self.state["scrape_media"] = not self.state["scrape_media"]
self.save_state() self.save_state()
print(f"\n✅ Media scraping {'enabled' if self.state['scrape_media'] else 'disabled'}") print(
f"\n✅ Media scraping {'enabled' if self.state['scrape_media'] else 'disabled'}"
)
elif choice == 'c': elif choice == "c":
task = asyncio.create_task(self.continuous_scraping()) task = asyncio.create_task(self.continuous_scraping())
print("Continuous scraping started. Press Ctrl+C to stop.") print("Continuous scraping started. Press Ctrl+C to stop.")
try: try:
await asyncio.sleep(float('inf')) await asyncio.sleep(float("inf"))
except KeyboardInterrupt: except KeyboardInterrupt:
self.continuous_scraping_active = False self.continuous_scraping_active = False
task.cancel() task.cancel()
@@ -822,10 +971,10 @@ class OptimizedTelegramScraper:
except asyncio.CancelledError: except asyncio.CancelledError:
pass pass
elif choice == 'e': elif choice == "e":
await self.export_data() await self.export_data()
elif choice == 'l': elif choice == "l":
channels_data = await self.list_channels() channels_data = await self.list_channels()
if not channels_data: if not channels_data:
@@ -841,24 +990,37 @@ class OptimizedTelegramScraper:
if selection: if selection:
added_count = 0 added_count = 0
if selection.lower() == 'all': if selection.lower() == "all":
for channel_info in channels_data: for channel_info in channels_data:
channel_id = channel_info['channel_id'] channel_id = channel_info["channel_id"]
if channel_id not in self.state['channels']: if channel_id not in self.state["channels"]:
self.state['channels'][channel_id] = 0 self.state["channels"][channel_id] = 0
if 'channel_names' not in self.state: if "channel_names" not in self.state:
self.state['channel_names'] = {} self.state["channel_names"] = {}
self.state['channel_names'][channel_id] = channel_info['username'] self.state["channel_names"][channel_id] = (
print(f"✅ Added channel {channel_info['channel_name']} (ID: {channel_id})") channel_info["username"]
)
print(
f"✅ Added channel {channel_info['channel_name']} (ID: {channel_id})"
)
added_count += 1 added_count += 1
else: else:
print(f"Channel {channel_info['channel_name']} already added") print(
f"Channel {channel_info['channel_name']} already added"
)
else: else:
for sel in [x.strip() for x in selection.split(',')]: for sel in [x.strip() for x in selection.split(",")]:
try: try:
if sel.startswith('-'): if sel.startswith("-"):
channel_id = sel channel_id = sel
channel_info = next((c for c in channels_data if c['channel_id'] == channel_id), None) channel_info = next(
(
c
for c in channels_data
if c["channel_id"] == channel_id
),
None,
)
if not channel_info: if not channel_info:
print(f"Channel ID {channel_id} not found") print(f"Channel ID {channel_id} not found")
continue continue
@@ -866,19 +1028,27 @@ class OptimizedTelegramScraper:
num = int(sel) num = int(sel)
if 1 <= num <= len(channels_data): if 1 <= num <= len(channels_data):
channel_info = channels_data[num - 1] channel_info = channels_data[num - 1]
channel_id = channel_info['channel_id'] channel_id = channel_info["channel_id"]
else: else:
print(f"Invalid number: {num}. Choose 1-{len(channels_data)}") print(
f"Invalid number: {num}. Choose 1-{len(channels_data)}"
)
continue continue
if channel_id in self.state['channels']: if channel_id in self.state["channels"]:
print(f"Channel {channel_info['channel_name']} already added") print(
f"Channel {channel_info['channel_name']} already added"
)
else: else:
self.state['channels'][channel_id] = 0 self.state["channels"][channel_id] = 0
if 'channel_names' not in self.state: if "channel_names" not in self.state:
self.state['channel_names'] = {} self.state["channel_names"] = {}
self.state['channel_names'][channel_id] = channel_info['username'] self.state["channel_names"][channel_id] = (
print(f"✅ Added channel {channel_info['channel_name']} (ID: {channel_id})") channel_info["username"]
)
print(
f"✅ Added channel {channel_info['channel_name']} (ID: {channel_id})"
)
added_count += 1 added_count += 1
except ValueError: except ValueError:
@@ -891,13 +1061,15 @@ class OptimizedTelegramScraper:
else: else:
print("No new channels were added") print("No new channels were added")
elif choice == 't': elif choice == "t":
if not self.state['channels']: if not self.state["channels"]:
print("No channels available. Add channels first") print("No channels available. Add channels first")
continue continue
await self.view_channels() await self.view_channels()
print("\nEnter channel NUMBER (1,2,3...) or full channel ID (-100123...)") print(
"\nEnter channel NUMBER (1,2,3...) or full channel ID (-100123...)"
)
selection = input("Enter your selection: ").strip() selection = input("Enter your selection: ").strip()
selected_channels = self.parse_channel_selection(selection) selected_channels = self.parse_channel_selection(selection)
@@ -910,13 +1082,15 @@ class OptimizedTelegramScraper:
else: else:
print("No valid channel selected") print("No valid channel selected")
elif choice == 'f': elif choice == "f":
if not self.state['channels']: if not self.state["channels"]:
print("No channels available. Add channels first") print("No channels available. Add channels first")
continue continue
await self.view_channels() await self.view_channels()
print("\nEnter channel NUMBER (1,2,3...) or full channel ID (-100123...)") print(
"\nEnter channel NUMBER (1,2,3...) or full channel ID (-100123...)"
)
selection = input("Enter your selection: ").strip() selection = input("Enter your selection: ").strip()
selected_channels = self.parse_channel_selection(selection) selected_channels = self.parse_channel_selection(selection)
@@ -928,7 +1102,7 @@ class OptimizedTelegramScraper:
else: else:
print("No valid channel selected") print("No valid channel selected")
elif choice == 'q': elif choice == "q":
print("\n👋 Goodbye!") print("\n👋 Goodbye!")
self.close_db_connections() self.close_db_connections()
if self.client: if self.client:
@@ -953,11 +1127,13 @@ class OptimizedTelegramScraper:
else: else:
print("Failed to initialize client. Exiting.") print("Failed to initialize client. Exiting.")
async def main(): async def main():
scraper = OptimizedTelegramScraper() scraper = OptimizedTelegramScraper()
await scraper.run() await scraper.run()
if __name__ == '__main__':
if __name__ == "__main__":
try: try:
asyncio.run(main()) asyncio.run(main())
except KeyboardInterrupt: except KeyboardInterrupt:
telegram_scraper_with_forwarding.py
+577 -311
View File
File diff suppressed because it is too large Load Diff
webui/index.html
+1 -1
View File
@@ -4,7 +4,7 @@
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Telegram Scraper Control Panel</title> <title>Telegram Scraper Control Panel</title>
<link rel="stylesheet" href="/static/style.css" /> <link rel="stylesheet" href="/static/style.css?v=2" />
</head> </head>
<body> <body>
<div class="app-shell"> <div class="app-shell">
webui/style.css
+115 -5
View File
@@ -47,7 +47,8 @@ button {
} }
.app-shell { .app-shell {
min-height: 100vh; height: 100vh;
overflow: hidden;
display: grid; display: grid;
grid-template-columns: 300px minmax(0, 1fr); grid-template-columns: 300px minmax(0, 1fr);
} }
@@ -59,6 +60,7 @@ button {
} }
.content { .content {
overflow-y: auto;
padding: 28px; padding: 28px;
max-width: 1320px; max-width: 1320px;
width: 100%; width: 100%;
@@ -494,6 +496,10 @@ tbody tr:hover {
overflow-y: auto; overflow-y: auto;
} }
.sidebar-toggle {
display: none;
}
.viewer-sidebar-head { .viewer-sidebar-head {
display: flex; display: flex;
align-items: flex-start; align-items: flex-start;
@@ -814,6 +820,10 @@ tbody tr:hover {
grid-template-columns: 1fr; grid-template-columns: 1fr;
} }
.app-shell {
grid-template-rows: auto 1fr;
}
.sidebar, .sidebar,
.viewer-sidebar { .viewer-sidebar {
border-right: 0; border-right: 0;
@@ -825,14 +835,114 @@ tbody tr:hover {
} }
} }
@media (max-width: 720px) { /* ---------- Mobile: tablet & phone ---------- */
.content,
.viewer-main, @media (max-width: 768px) {
.sidebar, .sidebar-toggle {
display: inline-flex;
}
.viewer-sidebar { .viewer-sidebar {
position: fixed;
left: -300px;
top: 0;
bottom: 0;
width: 280px;
z-index: 100;
transition: left 0.2s ease;
border-right: 1px dashed var(--line);
border-bottom: 0;
}
.viewer-sidebar.open {
left: 0;
}
.viewer-sidebar.open::before {
content: "";
position: fixed;
inset: 0;
background: rgba(0, 0, 0, 0.5);
z-index: -1;
}
.viewer-header {
padding: 12px 14px;
}
.messages-list {
padding: 8px 12px 16px;
}
.chat-bubble {
max-width: 92%;
}
.chat-bubble::before {
display: none;
}
.content {
padding: 18px; padding: 18px;
} }
.viewer-sidebar-head h1 {
font-size: 1.2rem;
}
}
@media (max-width: 480px) {
.content {
padding: 12px;
}
.viewer-main {
padding: 0;
}
.viewer-header {
padding: 10px 10px;
}
.viewer-header h2 {
font-size: 1.1rem;
}
.messages-list {
padding: 6px 8px 12px;
}
.chat-bubble {
max-width: 96%;
padding: 6px 9px;
font-size: 0.92rem;
}
.chat-footer {
font-size: 0.65rem;
}
.chat-sender {
font-size: 0.75rem;
}
.chat-reply-author {
font-size: 0.72rem;
}
.chat-reply-text {
font-size: 0.75rem;
}
.sidebar {
padding: 16px;
}
.viewer-sidebar {
width: 260px;
padding: 16px;
}
.panel-header, .panel-header,
.viewer-header, .viewer-header,
.dialog-header { .dialog-header {
webui/viewer.html
+2 -1
View File
@@ -4,7 +4,7 @@
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Telegram Scraper Viewer</title> <title>Telegram Scraper Viewer</title>
<link rel="stylesheet" href="/static/style.css" /> <link rel="stylesheet" href="/static/style.css?v=2" />
</head> </head>
<body> <body>
<div class="viewer-shell"> <div class="viewer-shell">
@@ -14,6 +14,7 @@
<div class="eyebrow">Export Viewer</div> <div class="eyebrow">Export Viewer</div>
<h1>Messages</h1> <h1>Messages</h1>
</div> </div>
<button id="sidebar-toggle" class="button button-small sidebar-toggle">☰</button>
<a class="button button-small" href="/">Dashboard</a> <a class="button button-small" href="/">Dashboard</a>
</div> </div>
<div id="viewer-channel-list" class="viewer-channel-list"></div> <div id="viewer-channel-list" class="viewer-channel-list"></div>
webui/viewer.js
+14
View File
@@ -81,6 +81,7 @@ function renderChannelList() {
node.classList.add("active"); node.classList.add("active");
} }
node.addEventListener("click", () => { node.addEventListener("click", () => {
document.querySelector(".viewer-sidebar")?.classList.remove("open");
viewerState.oldestMessageId = null; viewerState.oldestMessageId = null;
viewerState.newestMessageId = null; viewerState.newestMessageId = null;
loadChannel(channel.channel_id); loadChannel(channel.channel_id);
@@ -304,6 +305,19 @@ async function main() {
} }
setupInfiniteScroll(); setupInfiniteScroll();
const toggle = document.getElementById("sidebar-toggle");
const sidebar = document.querySelector(".viewer-sidebar");
if (toggle && sidebar) {
toggle.addEventListener("click", () => {
sidebar.classList.toggle("open");
});
sidebar.addEventListener("click", (e) => {
if (e.target === sidebar) {
sidebar.classList.remove("open");
}
});
}
} }
main().catch((error) => { main().catch((error) => {
webui_server.py
+35 -13
View File
@@ -306,7 +306,9 @@ class JobRunner:
if job.status == "running": if job.status == "running":
running_jobs.append(job) running_jobs.append(job)
if running_jobs: if running_jobs:
logger.warning("Waiting for %d running job(s) to finish...", len(running_jobs)) logger.warning(
"Waiting for %d running job(s) to finish...", len(running_jobs)
)
deadline = time.time() + timeout deadline = time.time() + timeout
while time.time() < deadline: while time.time() < deadline:
with self.lock: with self.lock:
@@ -561,6 +563,7 @@ class TelegramAuthManager:
async def _disconnect(): async def _disconnect():
if self.client: if self.client:
await self.client.disconnect() await self.client.disconnect()
if self.client: if self.client:
try: try:
future = asyncio.run_coroutine_threadsafe(_disconnect(), self.loop) future = asyncio.run_coroutine_threadsafe(_disconnect(), self.loop)
@@ -831,7 +834,10 @@ def openapi_payload() -> Dict[str, Any]:
"requestBody": json_body( "requestBody": json_body(
{ {
"api_id": {"type": "integer", "example": 123456}, "api_id": {"type": "integer", "example": 123456},
"api_hash": {"type": "string", "example": "0123456789abcdef"}, "api_hash": {
"type": "string",
"example": "0123456789abcdef",
},
}, },
["api_id", "api_hash"], ["api_id", "api_hash"],
), ),
@@ -929,7 +935,11 @@ def openapi_payload() -> Dict[str, Any]:
{ {
"name": "limit", "name": "limit",
"in": "query", "in": "query",
"schema": {"type": "integer", "default": 120, "maximum": 300}, "schema": {
"type": "integer",
"default": 120,
"maximum": 300,
},
}, },
{ {
"name": "before", "name": "before",
@@ -945,7 +955,10 @@ def openapi_payload() -> Dict[str, Any]:
"summary": "Track a channel", "summary": "Track a channel",
"requestBody": json_body( "requestBody": json_body(
{ {
"channel_id": {"type": "string", "example": "-1001234567890"}, "channel_id": {
"type": "string",
"example": "-1001234567890",
},
"name": {"type": "string", "example": "Research feed"}, "name": {"type": "string", "example": "Research feed"},
}, },
["channel_id"], ["channel_id"],
@@ -990,7 +1003,10 @@ def openapi_payload() -> Dict[str, Any]:
"schema": {"type": "string"}, "schema": {"type": "string"},
} }
], ],
"responses": {**json_response, "404": {"description": "Job not found"}}, "responses": {
**json_response,
"404": {"description": "Job not found"},
},
} }
}, },
"/api/jobs/scrape": { "/api/jobs/scrape": {
@@ -1416,15 +1432,18 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
try: try:
start, end = self._parse_range(range_header, file_size) start, end = self._parse_range(range_header, file_size)
except ValueError: except ValueError:
self.send_error_json(HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE, "Invalid range") self.send_error_json(
HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE, "Invalid range"
)
return return
content_length = end - start + 1 content_length = end - start + 1
self.send_response(HTTPStatus.PARTIAL_CONTENT) self.send_response(HTTPStatus.PARTIAL_CONTENT)
self.send_header("Content-Type", content_type) self.send_header("Content-Type", content_type)
self.send_header("Accept-Ranges", "bytes") self.send_header("Accept-Ranges", "bytes")
self.send_header("Last-Modified", time.strftime( self.send_header(
"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified) "Last-Modified",
)) time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)),
)
self.send_header("Content-Range", f"bytes {start}-{end}/{file_size}") self.send_header("Content-Range", f"bytes {start}-{end}/{file_size}")
self.send_header("Content-Length", str(content_length)) self.send_header("Content-Length", str(content_length))
self.end_headers() self.end_headers()
@@ -1434,9 +1453,10 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
self.send_response(HTTPStatus.OK) self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", content_type) self.send_header("Content-Type", content_type)
self.send_header("Accept-Ranges", "bytes") self.send_header("Accept-Ranges", "bytes")
self.send_header("Last-Modified", time.strftime( self.send_header(
"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified) "Last-Modified",
)) time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last_modified)),
)
self.send_header("Content-Length", str(file_size)) self.send_header("Content-Length", str(file_size))
self.end_headers() self.end_headers()
if not head_only: if not head_only:
@@ -1495,7 +1515,9 @@ class TelegramScraperWebServer(ThreadingHTTPServer):
self.job_runner = JobRunner() self.job_runner = JobRunner()
self.auth_manager = TelegramAuthManager() self.auth_manager = TelegramAuthManager()
self.continuous_manager = ContinuousScrapeManager() self.continuous_manager = ContinuousScrapeManager()
if START_CONTINUOUS and self.continuous_manager.snapshot()["config"].get("enabled", True): if START_CONTINUOUS and self.continuous_manager.snapshot()["config"].get(
"enabled", True
):
self.continuous_manager.start() self.continuous_manager.start()