forust/userbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: edu phpsessid updater bot

Browse Source
This commit is contained in:
mr-forust
2025-11-26 17:00:44 +01:00
parent 258ed04f98
commit 7cbc5bf4f3
17 changed files with 128 additions and 1081 deletions
Download Patch File Download Diff File Expand all files Collapse all files
edu_master/.env.example
+7
View File
@@ -0,0 +1,7 @@
EDU_LOGIN=login
EDU_PASSWORD=password
EDU_URL_LOGIN=https://edu.edu.vn.ua/user/login
EDU_URL_VERIFY=https://edu.edu.vn.ua/course/userlist
EDU_INTERVAL=10
EDU_USER_AGENT="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"
EDU_PHPSESSID=
edu_master/backend/Dockerfile
-15
View File
@@ -1,15 +0,0 @@
ARG VERSION
# Use the official WaterCrawl image as the base image
FROM watercrawl/watercrawl:${VERSION:-v0.10.2}
# Set working directory
WORKDIR /var/www
# Copy the extra requirements file
COPY extra_requirements.txt /var/www/extra_requirements.txt
# Install any additional packages
RUN poetry run pip install -r /var/www/extra_requirements.txt
# The rest of the configuration is inherited from the base image
# The entrypoint and command should be defined in docker-compose.yml
edu_master/backend/extra_requirements.txt
-1
View File
@@ -1 +0,0 @@
# Add your additional Python packages here, one per line
edu_master/compose.yaml
+6 -258
View File
@@ -1,262 +1,10 @@
x-app: &app
build:
context: ./backend/
dockerfile: Dockerfile
args:
- VERSION=${VERSION:-v0.10.2}
depends_on:
db:
condition: service_healthy
dns:
- 8.8.8.8
- 1.1.1.1
environment:
- SECRET_KEY=${SECRET_KEY:-django-insecure-el4wo4a4--=f0+ag#omp@^w4eq^8v4(scda&1a(td_y2@=sh6&}
- API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-8zSd6JIuC7ovfZ4AoxG_XmhubW6CPnQWW7Qe_4TD1TQ=}
- DEBUG=${DEBUG:-True}
- ALLOWED_HOSTS=${ALLOWED_HOSTS:-*}
- LANGUAGE_CODE=${LANGUAGE_CODE:-en-us}
- TIME_ZONE=${TIME_ZONE:-UTC}
- USE_I18N=${USE_I18N:-True}
- USE_TZ=${USE_TZ:-True}
- STATIC_ROOT=${STATIC_ROOT:-storage/static/}
- MEDIA_ROOT=${MEDIA_ROOT:-storage/media/}
- LOG_LEVEL=${LOG_LEVEL:-INFO}
- REDIS_URL=${REDIS_URL:-redis://redis:6379/1}
- DATABASE_URL=postgres://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-postgres}
- CELERY_BROKER_URL=${CELERY_BROKER_URL:-redis://redis:6379/0}
- CELERY_RESULT_BACKEND=${CELERY_RESULT_BACKEND:-django-db}
- REDIS_LOCKER_URL=${REDIS_LOCKER_URL:-redis://redis:6379/3}
- MINIO_ENDPOINT=minio:9000
- MINIO_EXTERNAL_ENDPOINT=nginx
- MINIO_REGION=us-east-1
- MINIO_ACCESS_KEY=minio
- MINIO_SECRET_KEY=minio123
- MINIO_USE_HTTPS=False
- MINIO_EXTERNAL_ENDPOINT_USE_HTTPS=False
- MINIO_URL_EXPIRY_HOURS=7
- MINIO_PRIVATE_BUCKET=private
- MINIO_PUBLIC_BUCKET=public
- CSRF_TRUSTED_ORIGINS=${CSRF_TRUSTED_ORIGINS:-}
- CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS:-}
- CORS_ALLOWED_ORIGIN_REGEXES=${CORS_ALLOWED_ORIGIN_REGEXES:-}
- CORS_ALLOW_ALL_ORIGINS=${CORS_ALLOW_ALL_ORIGINS:-False}
- FRONTEND_URL=${FRONTEND_URL:-http://localhost}
- IS_LOGIN_ACTIVE=${IS_LOGIN_ACTIVE:-True}
- IS_SIGNUP_ACTIVE=${IS_SIGNUP_ACTIVE:-True}
- IS_GITHUB_LOGIN_ACTIVE=${IS_GITHUB_LOGIN_ACTIVE:-True}
- IS_GOOGLE_LOGIN_ACTIVE=${IS_GOOGLE_LOGIN_ACTIVE:-True}
- GITHUB_CLIENT_ID=${GITHUB_CLIENT_ID:-}
- GITHUB_CLIENT_SECRET=${GITHUB_CLIENT_SECRET:-}
- GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID:-}
- GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET:-}
- ACCESS_TOKEN_LIFETIME_MINUTES=${ACCESS_TOKEN_LIFETIME_MINUTES:-5}
- REFRESH_TOKEN_LIFETIME_DAYS=${REFRESH_TOKEN_LIFETIME_DAYS:-30}
- EMAIL_BACKEND=${EMAIL_BACKEND:-django.core.mail.backends.smtp.EmailBackend}
- EMAIL_HOST=${EMAIL_HOST:-}
- EMAIL_PORT=${EMAIL_PORT:-587}
- EMAIL_USE_TLS=${EMAIL_USE_TLS:-True}
- EMAIL_HOST_USER=${EMAIL_HOST_USER:-}
- EMAIL_HOST_PASSWORD=${EMAIL_HOST_PASSWORD:-}
- DEFAULT_FROM_EMAIL=${DEFAULT_FROM_EMAIL:-}
- SCRAPY_USER_AGENT=${SCRAPY_USER_AGENT:-WaterCrawl/0.1 (+https://github.com/watercrawl/watercrawl)}
- SCRAPY_ROBOTSTXT_OBEY=${SCRAPY_ROBOTSTXT_OBEY:-True}
- SCRAPY_CONCURRENT_REQUESTS=${SCRAPY_CONCURRENT_REQUESTS:-16}
- SCRAPY_DOWNLOAD_DELAY=${SCRAPY_DOWNLOAD_DELAY:-0}
- SCRAPY_CONCURRENT_REQUESTS_PER_DOMAIN=${SCRAPY_CONCURRENT_REQUESTS_PER_DOMAIN:-4}
- SCRAPY_CONCURRENT_REQUESTS_PER_IP=${SCRAPY_CONCURRENT_REQUESTS_PER_IP:-4}
- SCRAPY_COOKIES_ENABLED=${SCRAPY_COOKIES_ENABLED:-False}
- SCRAPY_HTTPCACHE_ENABLED=${SCRAPY_HTTPCACHE_ENABLED:-True}
- SCRAPY_HTTPCACHE_EXPIRATION_SECS=${SCRAPY_HTTPCACHE_EXPIRATION_SECS:-3600}
- SCRAPY_HTTPCACHE_DIR=${SCRAPY_HTTPCACHE_DIR:-httpcache}
- SCRAPY_LOG_LEVEL=${SCRAPY_LOG_LEVEL:-ERROR}
- SCRAPY_GOOGLE_API_KEY=${SCRAPY_GOOGLE_API_KEY:-}
- SCRAPY_GOOGLE_CSE_ID=${SCRAPY_GOOGLE_CSE_ID:-}
- SCRAPY_MAX_NUMBER_OF_SITEMAP_URLS=${SCRAPY_MAX_NUMBER_OF_SITEMAP_URLS:-20000}
- SCRAPY_SITEMAP_CRAWL_PAGE_LIMIT=${SCRAPY_SITEMAP_CRAWL_PAGE_LIMIT:-100}
- PLAYWRIGHT_SERVER=${PLAYWRIGHT_SERVER:-http://playwright:8000}
- PLAYWRIGHT_API_KEY=${PLAYWRIGHT_API_KEY:-your-secret-api-key}
- OPENAI_API_KEY=${OPENAI_API_KEY:-}
- STRIPE_SECRET_KEY=${STRIPE_SECRET_KEY:-}
- STRIPE_WEBHOOK_SECRET=${STRIPE_WEBHOOK_SECRET:-}
- GOOGLE_ANALYTICS_ID=${GOOGLE_ANALYTICS_ID:-}
- IS_ENTERPRISE_MODE_ACTIVE=${IS_ENTERPRISE_MODE_ACTIVE:-False}
- MAX_CRAWL_DEPTH=${MAX_CRAWL_DEPTH:--1}
- CAPTURE_USAGE_HISTORY=${CAPTURE_USAGE_HISTORY:-True}
- MCP_SERVER=${MCP_SERVER:-http://localhost/sse}
networks:
- traefik-proxy
- default
- n8n
x-frontend: &frontend
image: watercrawl/frontend:${VERSION:-v0.10.2}
environment:
- VITE_API_BASE_URL=${API_BASE_URL:-http://localhost/api}
depends_on:
- app
services:
nginx:
image: nginx:alpine
volumes:
- ./nginx/nginx.conf:/etc/nginx/conf.d/default.conf.template
- ./nginx/entrypoint.sh:/entrypoint.sh
environment:
- MINIO_PRIVATE_BUCKET=${MINIO_PRIVATE_BUCKET:-private}
- MINIO_PUBLIC_BUCKET=${MINIO_PUBLIC_BUCKET:-public}
command: ["/bin/sh", "/entrypoint.sh"]
depends_on:
- app
- frontend
- minio
session-keeper:
build: ./phpsessid-bot
env_file: .env
restart: unless-stopped
networks:
- traefik-proxy
- n8n
labels:
- "traefik.enable=true"
- "traefik.docker.network=traefik-proxy"
app:
<<: *app
command: [ "gunicorn", "-b", "0.0.0.0:9000", "-w", "2", "watercrawl.wsgi:application", "--access-logfile", "-", "--error-logfile", "-", "--timeout", "60" ]
celery:
<<: *app
command: [ "celery", "-A", "watercrawl", "worker", "-l", "info", "-S", "django" ]
dns:
- 1.1.1.1
- 8.8.8.8
celery-beat:
<<: *app
command: [ "celery", "-A", "watercrawl", "beat", "-l", "info", "-S", "django" ]
frontend:
<<: *frontend
command: [ "npm", "run", "serve" ]
minio:
image: minio/minio:RELEASE.2024-11-07T00-52-20Z
restart: unless-stopped
volumes:
- ./volumes/minio-data:/data
command: server /data --console-address ":9001"
environment:
- MINIO_BROWSER_REDIRECT_URL=${MINIO_BROWSER_REDIRECT_URL:-http://localhost/minio-console/}
- MINIO_SERVER_URL=${MINIO_SERVER_URL:-http://localhost/}
- MINIO_ROOT_USER=${MINIO_ACCESS_KEY:-minio}
- MINIO_ROOT_PASSWORD=${MINIO_SECRET_KEY:-minio123}
playwright:
image: watercrawl/playwright:1.1
restart: unless-stopped
user: root
environment:
- AUTH_API_KEY=${PLAYWRIGHT_API_KEY:-your-secret-api-key}
- PORT=${PLAYWRIGHT_PORT:-8000}
- HOST=${PLAYWRIGHT_HOST:-0.0.0.0}
dns:
- 8.8.8.8
- 1.1.1.1
networks:
- traefik-proxy
- n8n
db:
image: postgres:17.2-alpine3.21
restart: unless-stopped
environment:
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
- POSTGRES_USER=${POSTGRES_USER:-postgres}
- POSTGRES_DB=${POSTGRES_DB:-postgres}
volumes:
- ./volumes/postgres-db:/var/lib/postgresql/data
healthcheck:
test: [ "CMD-SHELL", "pg_isready" ]
interval: 10s
test: ["CMD-SHELL", "test $(find /tmp/last_success -mmin -15)"]
interval: 1m
timeout: 5s
retries: 5
mcp:
image: watercrawl/mcp:v1.2.0
restart: unless-stopped
command: [ "sse", "--base-url", "http://app:9000", '--port', '3000', '--endpoint', '/sse' ]
networks:
- n8n
redis:
image: redis:latest
restart: unless-stopped
llm:
image: ollama/ollama:latest
restart: unless-stopped
volumes:
- ./volumes/ollama-models:/root/.ollama
environment:
- OLLAMA_DISABLE_TELEMETRY=true
- OLLAMA_KEEP_ALIVE=5m
- OLLAMA_HOST=0.0.0.0:11434
- OLLAMA_NUM_PARALLEL=1
- OLLAMA_MAX_LOADED_MODELS=1
dns:
- 1.1.1.1
- 8.8.8.8
networks:
- n8n
# docker exec -it edu_master-llm-1 ollama pull neural-chat:7b-q4
# docker exec -it edu_master-llm-1 ollama pull mistral:7b-q4
# lessons-bot:
# build:
# context: edu_master/lessons_bot/
# dockerfile: Dockerfile
# restart: unless-stopped
# environment:
# - LESSONS_BOT_TOKEN=${LESSONS_BOT_TOKEN}
# - N8N_WEBHOOK_URL=${N8N_WEBHOOK_URL:-http://n8n:5678/webhook-test/get-lessons}
# - N8N_SECRET=${N8N_SECRET:-your-secret-token-here}
# - WATERCRAWL_API_URL=${WATERCRAWL_API_URL:-http://app:9000/api}
# - PHPSESSID_BOT_URL=${PHPSESSID_BOT_URL:-http://phpsessid-bot:5000}
# - EDU_HOST=${EDU_HOST:-edu.edu.vn.ua}
# depends_on:
# - n8n
# - app
# - phpsessid-bot
# dns:
# - 1.1.1.1
# - 8.8.8.8
# networks:
# - default
phpsessid-bot:
build:
context: ./phpsessid_bot/
dockerfile: Dockerfile
restart: unless-stopped
environment:
- EDU_HOST=${EDU_HOST:-edu.edu.vn.ua}
- EDU_LOGIN=${EDU_LOGIN}
- EDU_PASSWORD=${EDU_PASSWORD}
- BOT_PORT=${PHPSESSID_BOT_PORT:-5000}
- BOT_HOST=${PHPSESSID_BOT_HOST:-0.0.0.0}
dns:
- 1.1.1.1
- 8.8.8.8
networks:
- default
volumes:
n8n_data:
postgres-db:
minio-data:
ollama-models:
lmstudio_data:
networks:
traefik-proxy:
external: true
retries: 3
edu_master/lessons_bot/Dockerfile
-16
View File
@@ -1,16 +0,0 @@
FROM python:3.11-slim
WORKDIR /app
# Установка зависимостей
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# Копирование кода
COPY config.py .
COPY utils.py .
COPY handlers.py .
COPY main.py .
# Запуск бота
CMD ["python", "-u", "main.py"]
edu_master/lessons_bot/config.py
-26
View File
@@ -1,26 +0,0 @@
import os
from dotenv import load_dotenv
load_dotenv()
# Telegram
BOT_TOKEN = os.getenv('LESSONS_BOT_TOKEN')
# n8n
N8N_WEBHOOK_URL = os.getenv('N8N_WEBHOOK_URL', 'http://n8n:5678/webhook/homework-check')
N8N_SECRET = os.getenv('N8N_SECRET', 'your-secret-token-here')
# WaterCrawl API
WATERCRAWL_API_URL = os.getenv('WATERCRAWL_API_URL', 'http://app:9000/api')
# PHPSESSID Bot
PHPSESSID_BOT_URL = os.getenv('PHPSESSID_BOT_URL', 'http://phpsessid-bot:5000')
# EDU site
EDU_HOST = os.getenv('EDU_HOST', 'edu.edu.vn.ua')
EDU_WEBINAR_URL = f'https://{EDU_HOST}/webinar/useractive'
# Playwright
PLAYWRIGHT_SERVER = os.getenv('PLAYWRIGHT_SERVER', 'http://playwright:8000')
PLAYWRIGHT_API_KEY = os.getenv('PLAYWRIGHT_API_KEY', 'your-secret-api-key')
WEBINAR_WAIT_TIME = int(os.getenv('WEBINAR_WAIT_TIME', '3')) # Секунды ожидания загрузки
edu_master/lessons_bot/handlers.py
-131
View File
@@ -1,131 +0,0 @@
import logging
import requests
from telegram import Update
from telegram.ext import ContextTypes
import config
from utils import fetch_webinars, format_webinar_message
logger = logging.getLogger(__name__)
async def start(update: Update, context: ContextTypes.DEFAULT_TYPE):
"""Команда /start"""
welcome_message = """
Привет! Я бот для проверки домашних заданий и вебинаров.
<b>Команды:</b>
/check - Проверить несделанные уроки
/webinar - Проверить активные онлайн уроки
/help - Помощь
"""
await update.message.reply_text(welcome_message, parse_mode='HTML')
async def help_command(update: Update, context: ContextTypes.DEFAULT_TYPE):
"""Команда /help"""
help_text = """
<b>Как пользоваться ботом:</b>
<b>/check</b> - Проверка домашних заданий
- Поиск несделанных уроков
⏱ Проверка занимает 10-30 секунд
<b>/webinar</b> - Активные онлайн уроки
- Проверка активных вебинаровв
⏱ Проверка занимает 3-5 секунд
"""
await update.message.reply_text(help_text, parse_mode='HTML')
async def check_homework(update: Update, context: ContextTypes.DEFAULT_TYPE):
"""Команда /check - запускает проверку уроков"""
chat_id = update.effective_chat.id
user_id = update.effective_user.id
username = update.effective_user.username or "unknown"
# Отправляем уведомление что начали работу
status_message = await update.message.reply_text("Запускаю проверку уроков...")
# Формируем данные для n8n
payload = {
"chat_id": chat_id,
"user_id": user_id,
"username": username,
"timestamp": update.message.date.isoformat()
}
headers = {
"Authorization": f"Bearer {config.N8N_SECRET}",
"Content-Type": "application/json"
}
try:
logger.info(f"Sending request to n8n for user {user_id}")
# Отправляем запрос в n8n
response = requests.post(
config.N8N_WEBHOOK_URL,
json=payload,
headers=headers,
timeout=5 # Короткий таймаут т.к. это асинхронный запрос
)
if response.status_code == 200:
await status_message.edit_text(
"✅ Запрос принят!\n"
"🔄 Парсинг сайта и анализ данных...\n"
"⏱ Это займет 10-30 секунд"
)
logger.info(f"Request accepted for user {user_id}")
else:
await status_message.edit_text(
f"Ошибка при отправке запроса. Функция в разработке\n"
f"Код: {response.status_code}"
)
logger.error(f"n8n returned status {response.status_code}")
except requests.Timeout:
await status_message.edit_text("⏱ Запрос обрабатывается (таймаут соединения)")
logger.warning(f"Timeout for user {user_id}")
except Exception as e:
await status_message.edit_text(f"❌ Ошибка: {str(e)}")
logger.error(f"Error for user {user_id}: {e}", exc_info=True)
async def check_webinar(update: Update, context: ContextTypes.DEFAULT_TYPE):
"""Команда /webinar - проверяет активные онлайн уроки"""
user_id = update.effective_user.id
# Отправляем уведомление что начали работу
status_message = await update.message.reply_text("Проверяю активные вебинары...")
try:
logger.info(f"Checking webinars for user {user_id}")
# Получаем список вебинаров
webinars = fetch_webinars()
if webinars is None:
await status_message.edit_text(
"❌ Не удалось получить информацию о вебинарах\n"
"Попробуйте позже или обратитесь к администратору\n"
"|@MrForust|mr.forust| Либо же прямо сюда."
)
logger.error(f"Failed to fetch webinars for user {user_id}")
return
# Форматируем и отправляем результат
message = format_webinar_message(webinars)
await status_message.edit_text(message, parse_mode='HTML', disable_web_page_preview=True)
logger.info(f"Webinar check completed for user {user_id}: found {len(webinars)} webinars")
except Exception as e:
await status_message.edit_text(f"❌ Ошибка: {str(e)}")
logger.error(f"Error checking webinars for user {user_id}: {e}", exc_info=True)
async def error_handler(update: Update, context: ContextTypes.DEFAULT_TYPE):
"""Обработчик ошибок"""
logger.error(f"Update {update} caused error {context.error}", exc_info=context.error)
edu_master/lessons_bot/main.py
-42
View File
@@ -1,42 +0,0 @@
import logging
from telegram import Update
from telegram.ext import Application, CommandHandler
import config
from handlers import start, help_command, check_homework, check_webinar, error_handler
# Настройка логирования
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.INFO
)
logger = logging.getLogger(__name__)
def main():
"""Запуск бота"""
if not config.BOT_TOKEN:
logger.error("LESSONS_BOT_TOKEN not set!")
return
# Создаем приложение
application = Application.builder().token(config.BOT_TOKEN).build()
# Регистрируем обработчики команд
application.add_handler(CommandHandler("start", start))
application.add_handler(CommandHandler("help", help_command))
application.add_handler(CommandHandler("check", check_homework))
application.add_handler(CommandHandler("webinar", check_webinar))
# Регистрируем обработчик ошибок
application.add_error_handler(error_handler)
# Запускаем бота
logger.info("Lessons Bot started!")
logger.info(f"PHPSESSID Bot URL: {config.PHPSESSID_BOT_URL}")
logger.info(f"n8n Webhook URL: {config.N8N_WEBHOOK_URL}")
application.run_polling(allowed_updates=Update.ALL_TYPES)
if __name__ == '__main__':
main()
edu_master/lessons_bot/requirements.txt
-5
View File
@@ -1,5 +0,0 @@
python-telegram-bot==20.7
requests==2.31.0
beautifulsoup4==4.12.2
python-dotenv==1.0.0
lxml==4.9.3
edu_master/lessons_bot/utils.py
-258
View File
@@ -1,258 +0,0 @@
import logging
import requests
from bs4 import BeautifulSoup
from typing import Optional, Dict, List
import config
logger = logging.getLogger(__name__)
def get_phpsessid() -> Optional[str]:
"""
Получает валидный PHPSESSID через phpsessid-bot
Returns:
str: PHPSESSID или None в случае ошибки
"""
try:
url = f"{config.PHPSESSID_BOT_URL}/get-session"
logger.info(f"Requesting PHPSESSID from {url}")
response = requests.post(url, timeout=10)
if response.status_code == 200:
data = response.json()
if data.get('success'):
phpsessid = data.get('phpsessid')
logger.info(f"Got PHPSESSID: {phpsessid[:10]}...")
return phpsessid
else:
logger.error(f"Failed to get PHPSESSID: {data.get('error')}")
return None
else:
logger.error(f"PHPSESSID bot returned status {response.status_code}")
return None
except Exception as e:
logger.error(f"Error getting PHPSESSID: {e}", exc_info=True)
return None
def parse_webinar_table(html_content: str) -> List[Dict[str, str]]:
"""
Парсит таблицу с вебинарами
Args:
html_content: HTML контент страницы
Returns:
List[Dict]: Список вебинаров или пустой список
"""
try:
soup = BeautifulSoup(html_content, 'html.parser')
# Находим таблицу с вебинарами
meetings_div = soup.find('div', {'id': 'meetings'})
if not meetings_div:
logger.warning("meetings div not found")
return []
table = meetings_div.find('table', {'class': 'table table-zebra'})
if not table:
logger.warning("table not found")
return []
tbody = table.find('tbody')
if not tbody:
logger.warning("tbody not found")
return []
rows = tbody.find_all('tr')
if not rows:
return []
# Проверяем на сообщение "Жодного онлайн уроку зараз"
first_row = rows[0]
td = first_row.find('td')
if td and 'Жодного онлайн уроку зараз' in td.get_text(strip=True):
logger.info("No webinars available")
return []
# Парсим активные вебинары
webinars = []
for row in rows:
tds = row.find_all('td')
if len(tds) >= 4:
webinar = {
'topic': tds[0].get_text(strip=True),
'course': tds[1].get_text(strip=True),
'teacher': tds[2].get_text(strip=True),
'join_link': tds[3].find('a')['href'] if tds[3].find('a') else ''
}
webinars.append(webinar)
logger.info(f"Parsed {len(webinars)} webinars")
return webinars
except Exception as e:
logger.error(f"Error parsing webinar table: {e}", exc_info=True)
return []
def fetch_webinars_with_playwright() -> Optional[List[Dict[str, str]]]:
"""
Получает список активных вебинаров используя Playwright для динамического контента
Returns:
List[Dict]: Список вебинаров или None в случае ошибки
"""
# Получаем PHPSESSID
phpsessid = get_phpsessid()
if not phpsessid:
logger.error("Failed to get PHPSESSID")
return None
try:
# Подготавливаем cookies для Playwright
cookies = [
{
'name': 'PHPSESSID',
'value': phpsessid,
'domain': config.EDU_HOST,
'path': '/'
}
]
# Запрос к Playwright серверу
playwright_request = {
'url': config.EDU_WEBINAR_URL,
'cookies': cookies,
'wait_until': 'networkidle', # Ждем пока сеть успокоится
'wait_time': config.WEBINAR_WAIT_TIME * 1000, # Дополнительное ожидание в миллисекундах
'user_agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36'
}
headers = {
'Authorization': f'Bearer {config.PLAYWRIGHT_API_KEY}',
'Content-Type': 'application/json'
}
logger.info(f"Fetching webinars via Playwright from {config.EDU_WEBINAR_URL}")
logger.info(f"Will wait {config.WEBINAR_WAIT_TIME} seconds for dynamic content")
response = requests.post(
f"{config.PLAYWRIGHT_SERVER}/render",
json=playwright_request,
headers=headers,
timeout=30
)
if response.status_code != 200:
logger.error(f"Playwright server returned status {response.status_code}")
logger.error(f"Response: {response.text}")
return None
result = response.json()
html_content = result.get('html', '')
if not html_content:
logger.error("No HTML content in Playwright response")
return None
# Парсим таблицу
webinars = parse_webinar_table(html_content)
return webinars
except Exception as e:
logger.error(f"Error fetching webinars via Playwright: {e}", exc_info=True)
return None
def fetch_webinars() -> Optional[List[Dict[str, str]]]:
"""
Получает список активных вебинаров
Сначала пробует через Playwright (для динамического контента),
при неудаче - через обычный requests
Returns:
List[Dict]: Список вебинаров или None в случае ошибки
"""
# Пробуем через Playwright
logger.info("Attempting to fetch via Playwright for dynamic content")
webinars = fetch_webinars_with_playwright()
if webinars is not None:
return webinars
# Fallback на обычный requests
logger.warning("Playwright fetch failed, falling back to simple requests")
# Получаем PHPSESSID
phpsessid = get_phpsessid()
if not phpsessid:
logger.error("Failed to get PHPSESSID")
return None
# Запрашиваем страницу с вебинарами
try:
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'ru-RU,ru;q=0.9,uk;q=0.8',
'Referer': f'https://{config.EDU_HOST}/'
}
cookies = {
'PHPSESSID': phpsessid
}
logger.info(f"Fetching webinars from {config.EDU_WEBINAR_URL}")
response = requests.get(
config.EDU_WEBINAR_URL,
headers=headers,
cookies=cookies,
timeout=15
)
if response.status_code != 200:
logger.error(f"Failed to fetch webinars page: {response.status_code}")
return None
# Парсим таблицу
webinars = parse_webinar_table(response.text)
return webinars
except Exception as e:
logger.error(f"Error fetching webinars: {e}", exc_info=True)
return None
def format_webinar_message(webinars: List[Dict[str, str]]) -> str:
"""
Форматирует список вебинаров для отправки в Telegram
Args:
webinars: Список вебинаров
Returns:
str: Отформатированное сообщение
"""
if not webinars:
return "📭 Жодного онлайн уроку зараз"
message = "🎓 <b>Активні онлайн уроки:</b>\n\n"
for i, webinar in enumerate(webinars, 1):
message += f"<b>{i}. {webinar['topic']}</b>\n"
message += f"📚 Курс: {webinar['course']}\n"
message += f"👨‍🏫 Вчитель: {webinar['teacher']}\n"
if webinar['join_link']:
full_link = webinar['join_link']
if not full_link.startswith('http'):
full_link = f"https://{config.EDU_HOST}{webinar['join_link']}"
message += f"🔗 <a href='{full_link}'>Увійти до уроку</a>\n"
message += "\n"
return message
edu_master/nginx/entrypoint.sh
-8
View File
@@ -1,8 +0,0 @@
#!/bin/sh
set -e
# Replace environment variables in the Nginx configuration template
envsubst '${MINIO_PRIVATE_BUCKET} ${MINIO_PUBLIC_BUCKET}' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf
# Start Nginx
exec nginx -g 'daemon off;'
edu_master/nginx/nginx.conf
-87
View File
@@ -1,87 +0,0 @@
server {
listen 80;
server_name localhost;
client_max_body_size 100M;
# Frontend
location / {
proxy_pass http://frontend:80;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
# API
location /api/ {
proxy_pass http://app:9000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
# MCP
location ~ ^/(sse|messages) {
proxy_pass http://mcp:3000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# Important SSE settings
proxy_http_version 1.1;
proxy_set_header Connection "";
# Disable buffering so events are sent immediately
proxy_buffering off;
proxy_cache off;
# Increase timeouts so connection stays open
proxy_read_timeout 3600s;
proxy_send_timeout 3600s;
}
# MinIO private bucket
location /${MINIO_PRIVATE_BUCKET}/ {
proxy_pass http://minio:9000/${MINIO_PRIVATE_BUCKET}/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_buffering off;
}
# MinIO public bucket
location /${MINIO_PUBLIC_BUCKET}/ {
proxy_pass http://minio:9000/${MINIO_PUBLIC_BUCKET}/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_buffering off;
}
# MinIO API - for direct S3 operations
# location /minio/api/ {
# proxy_pass http://minio:9000/;
# proxy_set_header Host $host;
# proxy_set_header X-Real-IP $remote_addr;
# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# proxy_set_header X-Forwarded-Proto $scheme;
# proxy_buffering off;
# }
# MinIO Console
location /minio-console/ {
proxy_pass http://minio:9001/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# Rewrite location headers
proxy_redirect / /minio-console/;
}
}
edu_master/phpsessid-bot/Dockerfile
+12
View File
@@ -0,0 +1,12 @@
FROM python:3.11-slim
WORKDIR /app
# Install dependencies
RUN pip install requests
# Copy application code
COPY . .
# Run the bot
CMD ["python", "bot.py"]
edu_master/phpsessid-bot/bot.py
+103
View File
@@ -0,0 +1,103 @@
import os
import time
import requests
import logging
from datetime import datetime
# Configure logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)
# Load configuration
LOGIN = os.getenv('EDU_LOGIN')
PASSWORD = os.getenv('EDU_PASSWORD')
URL_LOGIN = os.getenv('EDU_URL_LOGIN', 'https://edu.edu.vn.ua/user/login')
URL_VERIFY = os.getenv('EDU_URL_VERIFY', 'https://edu.edu.vn.ua/course/userlist')
INTERVAL = int(os.getenv('EDU_INTERVAL', 10))
USER_AGENT = os.getenv('EDU_USER_AGENT', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36')
INITIAL_PHPSESSID = os.getenv('EDU_PHPSESSID')
SUCCESS_FILE = '/tmp/last_success'
def touch_success_file():
"""Updates the timestamp of the success file for healthchecks."""
try:
with open(SUCCESS_FILE, 'w') as f:
f.write(str(datetime.now().timestamp()))
except Exception as e:
logger.error(f"Failed to touch success file: {e}")
def main():
logger.info("Starting Session Keeper Bot")
session = requests.Session()
# Set headers
headers = {
'User-Agent': USER_AGENT,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language': 'en-US,en;q=0.9',
'Cache-Control': 'max-age=0',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-User': '?1',
'Sec-Fetch-Dest': 'document',
'Sec-Ch-Ua': '"Not_A Brand";v="99", "Chromium";v="142"',
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': '"Linux"',
'Accept-Encoding': 'gzip, deflate, br',
'Priority': 'u=0, i'
}
session.headers.update(headers)
# If an initial PHPSESSID is provided, set it in the session
if INITIAL_PHPSESSID:
logger.info(f"Setting initial PHPSESSID: {INITIAL_PHPSESSID}")
session.cookies.set('PHPSESSID', INITIAL_PHPSESSID, domain='edu.edu.vn.ua')
while True:
try:
logger.info("Attempting login...")
# Login payload
payload = {
'login': LOGIN,
'password': PASSWORD
}
# Perform Login
# Note: The user request shows a POST to /user/login with form data
# We need to make sure we handle the PHPSESSID correctly.
# If we already have a PHPSESSID, requests will send it.
login_response = session.post(URL_LOGIN, data=payload, allow_redirects=True)
logger.info(f"Login Response Status: {login_response.status_code}")
logger.info(f"Cookies after login: {session.cookies.get_dict()}")
# Verify Session
logger.info("Verifying session...")
verify_response = session.get(URL_VERIFY, allow_redirects=False)
logger.info(f"Verify Response Status: {verify_response.status_code}")
if verify_response.status_code == 200:
logger.info("Session verification SUCCESS (200 OK).")
touch_success_file()
elif verify_response.status_code == 302:
logger.warning("Session verification FAILED (302 Redirect). Session might be invalid.")
else:
logger.warning(f"Session verification returned unexpected status: {verify_response.status_code}")
except Exception as e:
logger.error(f"An error occurred: {e}")
logger.info(f"Sleeping for {INTERVAL} minutes...")
time.sleep(INTERVAL * 60)
if __name__ == "__main__":
main()
edu_master/phpsessid_bot/Dockerfile
-15
View File
@@ -1,15 +0,0 @@
FROM python:3.11-slim
WORKDIR /app
# Устанавливаем зависимости
RUN pip install --no-cache-dir flask requests
# Копируем код бота
COPY main.py .
# Открываем порт
EXPOSE 5000
# Запускаем бот
CMD ["python", "-u", "main.py"]
edu_master/phpsessid_bot/main.py
-216
View File
@@ -1,216 +0,0 @@
import os
import logging
from flask import Flask, request, jsonify
import requests
from datetime import datetime
# Настройка логирования
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)
app = Flask(__name__)
# Конфигурация из переменных окружения
EDU_HOST = os.getenv('EDU_HOST', 'edu.edu.vn.ua')
EDU_LOGIN = os.getenv('EDU_LOGIN', '')
EDU_PASSWORD = os.getenv('EDU_PASSWORD', '')
BOT_PORT = int(os.getenv('BOT_PORT', '5000'))
BOT_HOST = os.getenv('BOT_HOST', '0.0.0.0')
# Кэш для хранения актуальной сессии
session_cache = {
'phpsessid': None,
'expires_at': None
}
def login_and_get_session():
"""
Выполняет логин и возвращает новый PHPSESSID
"""
url = f"https://{EDU_HOST}/user/login"
headers = {
'Cache-Control': 'max-age=0',
'Sec-Ch-Ua': '"Chromium";v="141", "Not?A_Brand";v="8"',
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': '"Linux"',
'Accept-Language': 'ru-RU,ru;q=0.9',
'Origin': f'https://{EDU_HOST}',
'Content-Type': 'application/x-www-form-urlencoded',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-User': '?1',
'Sec-Fetch-Dest': 'document',
'Referer': f'https://{EDU_HOST}/',
'Accept-Encoding': 'gzip, deflate, br',
'Priority': 'u=0, i'
}
data = {
'login': EDU_LOGIN,
'password': EDU_PASSWORD
}
try:
logger.info(f"Attempting login to {url}")
response = requests.post(
url,
data=data,
headers=headers,
allow_redirects=False,
timeout=10
)
# Получаем PHPSESSID из cookies
phpsessid = response.cookies.get('PHPSESSID')
if phpsessid:
logger.info(f"Login successful, got PHPSESSID: {phpsessid[:10]}...")
return {
'success': True,
'phpsessid': phpsessid,
'status_code': response.status_code
}
else:
logger.warning(f"Login failed: no PHPSESSID in response. Status: {response.status_code}")
return {
'success': False,
'error': 'No PHPSESSID in response',
'status_code': response.status_code
}
except requests.exceptions.RequestException as e:
logger.error(f"Login request failed: {str(e)}")
return {
'success': False,
'error': str(e)
}
def validate_phpsessid(phpsessid):
"""
Проверяет валидность существующего PHPSESSID
"""
url = f"https://{EDU_HOST}/"
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
}
cookies = {
'PHPSESSID': phpsessid
}
try:
response = requests.get(url, headers=headers, cookies=cookies, timeout=10)
# Проверяем, не редиректит ли на страницу логина
is_valid = response.status_code == 200 and '/user/login' not in response.url
return {
'valid': is_valid,
'status_code': response.status_code,
'url': response.url
}
except requests.exceptions.RequestException as e:
logger.error(f"Validation request failed: {str(e)}")
return {
'valid': False,
'error': str(e)
}
@app.route('/health', methods=['GET'])
def health():
"""Health check endpoint"""
return jsonify({'status': 'ok', 'timestamp': datetime.now().isoformat()})
@app.route('/get-session', methods=['POST', 'GET'])
def get_session():
"""
Основной endpoint для получения валидного PHPSESSID
Возвращает кэшированную сессию или создает новую
"""
result = login_and_get_session()
if result['success']:
session_cache['phpsessid'] = result['phpsessid']
session_cache['last_updated'] = datetime.now().isoformat()
return jsonify({
'success': True,
'phpsessid': result['phpsessid'],
'timestamp': datetime.now().isoformat()
})
else:
return jsonify({
'success': False,
'error': result.get('error', 'Login failed'),
'timestamp': datetime.now().isoformat()
}), 400
@app.route('/validate-session', methods=['POST'])
def validate_session():
"""
Проверяет валидность переданного PHPSESSID
"""
data = request.get_json() or {}
phpsessid = data.get('phpsessid') or request.args.get('phpsessid')
if not phpsessid:
return jsonify({
'success': False,
'error': 'PHPSESSID not provided'
}), 400
validation_result = validate_phpsessid(phpsessid)
return jsonify({
'success': True,
'valid': validation_result.get('valid', False),
'details': validation_result,
'timestamp': datetime.now().isoformat()
})
@app.route('/refresh-session', methods=['POST', 'GET'])
def refresh_session():
"""
Принудительно обновляет сессию
"""
result = login_and_get_session()
if result['success']:
return jsonify({
'success': True,
'phpsessid': result['phpsessid'],
'message': 'Session refreshed successfully',
'timestamp': datetime.now().isoformat()
})
else:
return jsonify({
'success': False,
'error': result.get('error', 'Failed to refresh session'),
'timestamp': datetime.now().isoformat()
}), 400
if __name__ == '__main__':
if not EDU_LOGIN or not EDU_PASSWORD:
logger.error("EDU_LOGIN and EDU_PASSWORD must be set!")
exit(1)
logger.info(f"Starting PHPSESSID validator bot on {BOT_HOST}:{BOT_PORT}")
logger.info(f"Target host: {EDU_HOST}")
app.run(host=BOT_HOST, port=BOT_PORT, debug=False)
edu_master/phpsessid_bot/requirements.txt
-3
View File
@@ -1,3 +0,0 @@
flask==3.0.0
requests==2.31.0
Werkzeug==3.0.1