init, .gitignore

edu_master/backend/Dockerfile

@@ -0,0 +1,15 @@
+ARG VERSION
+# Use the official WaterCrawl image as the base image
+FROM watercrawl/watercrawl:${VERSION:-v0.10.2}
+
+# Set working directory
+WORKDIR /var/www
+
+# Copy the extra requirements file
+COPY extra_requirements.txt /var/www/extra_requirements.txt
+
+# Install any additional packages
+RUN poetry run pip install -r /var/www/extra_requirements.txt
+
+# The rest of the configuration is inherited from the base image
+# The entrypoint and command should be defined in docker-compose.yml

edu_master/backend/extra_requirements.txt

@@ -0,0 +1 @@
+# Add your additional Python packages here, one per line

edu_master/docker-compose.yaml

@@ -0,0 +1,306 @@
+x-app: &app
+  build:
+    context: backend/
+    dockerfile: Dockerfile
+    args:
+      - VERSION=${VERSION:-v0.10.2}
+  depends_on:
+    db:
+      condition: service_healthy
+  dns:
+    - 8.8.8.8
+    - 1.1.1.1
+  environment:
+    - SECRET_KEY=${SECRET_KEY:-django-insecure-el4wo4a4--=f0+ag#omp@^w4eq^8v4(scda&1a(td_y2@=sh6&}
+    - API_ENCRYPTION_KEY=${API_ENCRYPTION_KEY:-8zSd6JIuC7ovfZ4AoxG_XmhubW6CPnQWW7Qe_4TD1TQ=}
+    - DEBUG=${DEBUG:-True}
+    - ALLOWED_HOSTS=${ALLOWED_HOSTS:-*}
+    - LANGUAGE_CODE=${LANGUAGE_CODE:-en-us}
+    - TIME_ZONE=${TIME_ZONE:-UTC}
+    - USE_I18N=${USE_I18N:-True}
+    - USE_TZ=${USE_TZ:-True}
+    - STATIC_ROOT=${STATIC_ROOT:-storage/static/}
+    - MEDIA_ROOT=${MEDIA_ROOT:-storage/media/}
+    - LOG_LEVEL=${LOG_LEVEL:-INFO}
+    - REDIS_URL=${REDIS_URL:-redis://redis:6379/1}
+    - DATABASE_URL=postgres://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-postgres}
+    - CELERY_BROKER_URL=${CELERY_BROKER_URL:-redis://redis:6379/0}
+    - CELERY_RESULT_BACKEND=${CELERY_RESULT_BACKEND:-django-db}
+    - REDIS_LOCKER_URL=${REDIS_LOCKER_URL:-redis://redis:6379/3}
+    - MINIO_ENDPOINT=minio:9000
+    - MINIO_EXTERNAL_ENDPOINT=nginx
+    - MINIO_REGION=us-east-1
+    - MINIO_ACCESS_KEY=minio
+    - MINIO_SECRET_KEY=minio123
+    - MINIO_USE_HTTPS=False
+    - MINIO_EXTERNAL_ENDPOINT_USE_HTTPS=False
+    - MINIO_URL_EXPIRY_HOURS=7
+    - MINIO_PRIVATE_BUCKET=private
+    - MINIO_PUBLIC_BUCKET=public
+    - CSRF_TRUSTED_ORIGINS=${CSRF_TRUSTED_ORIGINS:-}
+    - CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS:-}
+    - CORS_ALLOWED_ORIGIN_REGEXES=${CORS_ALLOWED_ORIGIN_REGEXES:-}
+    - CORS_ALLOW_ALL_ORIGINS=${CORS_ALLOW_ALL_ORIGINS:-False}
+    - FRONTEND_URL=${FRONTEND_URL:-http://localhost}
+    - IS_LOGIN_ACTIVE=${IS_LOGIN_ACTIVE:-True}
+    - IS_SIGNUP_ACTIVE=${IS_SIGNUP_ACTIVE:-True}
+    - IS_GITHUB_LOGIN_ACTIVE=${IS_GITHUB_LOGIN_ACTIVE:-True}
+    - IS_GOOGLE_LOGIN_ACTIVE=${IS_GOOGLE_LOGIN_ACTIVE:-True}
+    - GITHUB_CLIENT_ID=${GITHUB_CLIENT_ID:-}
+    - GITHUB_CLIENT_SECRET=${GITHUB_CLIENT_SECRET:-}
+    - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID:-}
+    - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET:-}
+    - ACCESS_TOKEN_LIFETIME_MINUTES=${ACCESS_TOKEN_LIFETIME_MINUTES:-5}
+    - REFRESH_TOKEN_LIFETIME_DAYS=${REFRESH_TOKEN_LIFETIME_DAYS:-30}
+    - EMAIL_BACKEND=${EMAIL_BACKEND:-django.core.mail.backends.smtp.EmailBackend}
+    - EMAIL_HOST=${EMAIL_HOST:-}
+    - EMAIL_PORT=${EMAIL_PORT:-587}
+    - EMAIL_USE_TLS=${EMAIL_USE_TLS:-True}
+    - EMAIL_HOST_USER=${EMAIL_HOST_USER:-}
+    - EMAIL_HOST_PASSWORD=${EMAIL_HOST_PASSWORD:-}
+    - DEFAULT_FROM_EMAIL=${DEFAULT_FROM_EMAIL:-}
+    - SCRAPY_USER_AGENT=${SCRAPY_USER_AGENT:-WaterCrawl/0.1 (+https://github.com/watercrawl/watercrawl)}
+    - SCRAPY_ROBOTSTXT_OBEY=${SCRAPY_ROBOTSTXT_OBEY:-True}
+    - SCRAPY_CONCURRENT_REQUESTS=${SCRAPY_CONCURRENT_REQUESTS:-16}
+    - SCRAPY_DOWNLOAD_DELAY=${SCRAPY_DOWNLOAD_DELAY:-0}
+    - SCRAPY_CONCURRENT_REQUESTS_PER_DOMAIN=${SCRAPY_CONCURRENT_REQUESTS_PER_DOMAIN:-4}
+    - SCRAPY_CONCURRENT_REQUESTS_PER_IP=${SCRAPY_CONCURRENT_REQUESTS_PER_IP:-4}
+    - SCRAPY_COOKIES_ENABLED=${SCRAPY_COOKIES_ENABLED:-False}
+    - SCRAPY_HTTPCACHE_ENABLED=${SCRAPY_HTTPCACHE_ENABLED:-True}
+    - SCRAPY_HTTPCACHE_EXPIRATION_SECS=${SCRAPY_HTTPCACHE_EXPIRATION_SECS:-3600}
+    - SCRAPY_HTTPCACHE_DIR=${SCRAPY_HTTPCACHE_DIR:-httpcache}
+    - SCRAPY_LOG_LEVEL=${SCRAPY_LOG_LEVEL:-ERROR}
+    - SCRAPY_GOOGLE_API_KEY=${SCRAPY_GOOGLE_API_KEY:-}
+    - SCRAPY_GOOGLE_CSE_ID=${SCRAPY_GOOGLE_CSE_ID:-}
+    - SCRAPY_MAX_NUMBER_OF_SITEMAP_URLS=${SCRAPY_MAX_NUMBER_OF_SITEMAP_URLS:-20000}
+    - SCRAPY_SITEMAP_CRAWL_PAGE_LIMIT=${SCRAPY_SITEMAP_CRAWL_PAGE_LIMIT:-100}
+    - PLAYWRIGHT_SERVER=${PLAYWRIGHT_SERVER:-http://playwright:8000}
+    - PLAYWRIGHT_API_KEY=${PLAYWRIGHT_API_KEY:-your-secret-api-key}
+    - OPENAI_API_KEY=${OPENAI_API_KEY:-}
+    - STRIPE_SECRET_KEY=${STRIPE_SECRET_KEY:-}
+    - STRIPE_WEBHOOK_SECRET=${STRIPE_WEBHOOK_SECRET:-}
+    - GOOGLE_ANALYTICS_ID=${GOOGLE_ANALYTICS_ID:-}
+    - IS_ENTERPRISE_MODE_ACTIVE=${IS_ENTERPRISE_MODE_ACTIVE:-False}
+    - MAX_CRAWL_DEPTH=${MAX_CRAWL_DEPTH:--1}
+    - CAPTURE_USAGE_HISTORY=${CAPTURE_USAGE_HISTORY:-True}
+    - MCP_SERVER=${MCP_SERVER:-http://localhost/sse}
+  networks:
+    - traefik-proxy 
+    - default
+    
+x-frontend: &frontend
+  image: watercrawl/frontend:${VERSION:-v0.10.2}
+  environment:
+    - VITE_API_BASE_URL=${API_BASE_URL:-http://localhost/api}
+  depends_on:
+    - app
+
+services:
+  nginx:
+    image: nginx:alpine
+    volumes:
+      - ./nginx/nginx.conf:/etc/nginx/conf.d/default.conf.template
+      - ./nginx/entrypoint.sh:/entrypoint.sh
+    environment:
+      - MINIO_PRIVATE_BUCKET=${MINIO_PRIVATE_BUCKET:-private}
+      - MINIO_PUBLIC_BUCKET=${MINIO_PUBLIC_BUCKET:-public}
+    command: ["/bin/sh", "/entrypoint.sh"]
+    depends_on:
+      - app
+      - frontend
+      - minio
+    restart: unless-stopped
+    networks:
+          - traefik-proxy 
+          - default
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.watercrawl.rule=Host(`watercrawl.workstation`)" 
+      - "traefik.http.routers.watercrawl.entrypoints=websecure"
+      - "traefik.http.services.watercrawl.loadbalancer.server.port=80"
+      - "traefik.docker.network=traefik-proxy"
+      - "traefik.http.routers.watercrawl.tls.certresolver=le"
+
+  app:
+    <<: *app
+    command: [ "gunicorn", "-b", "0.0.0.0:9000", "-w", "2", "watercrawl.wsgi:application", "--access-logfile", "-", "--error-logfile", "-", "--timeout", "60" ]
+
+  celery:
+    <<: *app
+    command: [ "celery", "-A", "watercrawl", "worker", "-l", "info", "-S", "django" ]
+    dns:
+    - 1.1.1.1
+    - 8.8.8.8
+
+  celery-beat:
+    <<: *app
+    command: [ "celery", "-A", "watercrawl", "beat", "-l", "info", "-S", "django" ]
+
+  frontend:
+    <<: *frontend
+    command: [ "npm", "run", "serve" ]
+
+  minio:
+    image: minio/minio:RELEASE.2024-11-07T00-52-20Z
+    restart: unless-stopped
+    volumes:
+      - ./volumes/minio-data:/data
+    command: server /data --console-address ":9001"
+    environment:
+      - MINIO_BROWSER_REDIRECT_URL=${MINIO_BROWSER_REDIRECT_URL:-http://localhost/minio-console/}
+      - MINIO_SERVER_URL=${MINIO_SERVER_URL:-http://localhost/}
+      - MINIO_ROOT_USER=${MINIO_ACCESS_KEY:-minio}
+      - MINIO_ROOT_PASSWORD=${MINIO_SECRET_KEY:-minio123}
+
+  playwright:
+    image: watercrawl/playwright:1.1
+    restart: unless-stopped
+    user: root
+    environment:
+      - AUTH_API_KEY=${PLAYWRIGHT_API_KEY:-your-secret-api-key}
+      - PORT=${PLAYWRIGHT_PORT:-8000}
+      - HOST=${PLAYWRIGHT_HOST:-0.0.0.0}
+    dns:
+      - 8.8.8.8
+      - 1.1.1.1
+    networks:
+      - traefik-proxy 
+      - default
+
+  db:
+    image: postgres:17.2-alpine3.21
+    restart: unless-stopped
+    environment:
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
+      - POSTGRES_USER=${POSTGRES_USER:-postgres}
+      - POSTGRES_DB=${POSTGRES_DB:-postgres}
+    volumes:
+      - ./volumes/postgres-db:/var/lib/postgresql/data
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  mcp:
+    image: watercrawl/mcp:v1.2.0
+    restart: unless-stopped
+    command: [ "sse", "--base-url", "http://app:9000", '--port', '3000', '--endpoint', '/sse' ]
+
+  redis:
+    image: redis:latest
+    restart: unless-stopped
+
+  llm:
+    image: ollama/ollama:latest
+    restart: unless-stopped
+    volumes:
+      - ./volumes/ollama-models:/root/.ollama
+    environment:
+      - OLLAMA_DISABLE_TELEMETRY=true
+      - OLLAMA_KEEP_ALIVE=5m
+      - OLLAMA_HOST=0.0.0.0:11434
+      - OLLAMA_NUM_PARALLEL=1
+      - OLLAMA_MAX_LOADED_MODELS=1
+    dns:
+      - 1.1.1.1
+      - 8.8.8.8
+
+  n8n:
+    image: docker.n8n.io/n8nio/n8n
+    restart: unless-stopped
+    environment:
+      - N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS=true
+      - N8N_PORT=5678
+      - N8N_RUNNERS_ENABLED=true
+      - NODE_ENV=production
+      - GENERIC_TIMEZONE=Europe/Bratislava
+      - TZ=Europe/Bratislava
+      - N8N_SECURE_COOKIE=false
+      - N8N_COMMUNITY_PACKAGES_ALLOW_TOOL_USAGE=true
+    volumes:
+      - n8n_data:/home/node/.n8n
+      - ./n8n/local-files:/files
+    extra_hosts:
+      - "enterprise.n8n.io:104.26.13.187"
+      - "enterprise.n8n.io:104.26.12.187"
+      - "enterprise.n8n.io:172.67.68.102"
+    dns:
+      - 1.1.1.1
+      - 8.8.8.8
+    networks:
+      - traefik-proxy 
+      - default
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.n8n.rule=Host(`n8n.workstation`)" 
+      - "traefik.http.routers.n8n.entrypoints=websecure"
+      - "traefik.http.services.n8n.loadbalancer.server.port=5678"
+      - "traefik.docker.network=traefik-proxy"
+      - "traefik.http.routers.n8n.tls.certresolver=le"
+# docker exec -it edu_master-llm-1 ollama pull neural-chat:7b-q4
+# docker exec -it edu_master-llm-1 ollama pull mistral:7b-q4
+
+  lessons-bot:
+        build:
+          context: lessons_bot/
+          dockerfile: Dockerfile
+        restart: unless-stopped
+        environment:
+          - LESSONS_BOT_TOKEN=${LESSONS_BOT_TOKEN}
+          - N8N_WEBHOOK_URL=${N8N_WEBHOOK_URL:-http://n8n:5678/webhook-test/get-lessons}
+          - N8N_SECRET=${N8N_SECRET:-your-secret-token-here}
+          - WATERCRAWL_API_URL=${WATERCRAWL_API_URL:-http://app:9000/api}
+          - PHPSESSID_BOT_URL=${PHPSESSID_BOT_URL:-http://phpsessid-bot:5000}
+          - EDU_HOST=${EDU_HOST:-edu.edu.vn.ua}
+        depends_on:
+          - n8n
+          - app
+          - phpsessid-bot
+        dns:
+          - 1.1.1.1
+          - 8.8.8.8
+        networks:
+          - default
+        healthcheck:
+          test: ["CMD-SHELL", "pgrep -f 'python.*main.py' || exit 1"]
+          interval: 30s
+          timeout: 10s
+          retries: 3
+          start_period: 10s
+  # webinar-notif-worker:
+  #   image: python:3.11-slim
+  #   restart: on-failure
+  #   volumes:
+  #     - ./webinar_notif:/app/webinar_notif:ro
+  #   working_dir: /app/webinar_notif
+  #   command: ["/bin/sh", "-c", "pip install --no-cache-dir -r requirements.txt 2>/dev/null || true; python -u main.py"]
+
+
+  phpsessid-bot:
+    build:
+      context: phpsessid_bot/
+      dockerfile: Dockerfile
+    restart: unless-stopped
+    environment:
+      - EDU_HOST=${EDU_HOST:-edu.edu.vn.ua}
+      - EDU_LOGIN=${EDU_LOGIN}
+      - EDU_PASSWORD=${EDU_PASSWORD}
+      - BOT_PORT=${PHPSESSID_BOT_PORT:-5000}
+      - BOT_HOST=${PHPSESSID_BOT_HOST:-0.0.0.0}
+    dns:
+      - 1.1.1.1
+      - 8.8.8.8
+    networks:
+      - default
+
+
+volumes:
+  n8n_data:
+  postgres-db:
+  minio-data:
+  ollama-models:
+  lmstudio_data:
+networks:
+    traefik-proxy:
+        external: true

edu_master/lessons_bot/Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Установка зависимостей
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Копирование кода
+COPY config.py .
+COPY utils.py .
+COPY handlers.py .
+COPY main.py .
+
+# Запуск бота
+CMD ["python", "-u", "main.py"]

edu_master/lessons_bot/config.py

@@ -0,0 +1,26 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# Telegram
+BOT_TOKEN = os.getenv('LESSONS_BOT_TOKEN')
+
+# n8n
+N8N_WEBHOOK_URL = os.getenv('N8N_WEBHOOK_URL', 'http://n8n:5678/webhook/homework-check')
+N8N_SECRET = os.getenv('N8N_SECRET', 'your-secret-token-here')
+
+# WaterCrawl API
+WATERCRAWL_API_URL = os.getenv('WATERCRAWL_API_URL', 'http://app:9000/api')
+
+# PHPSESSID Bot
+PHPSESSID_BOT_URL = os.getenv('PHPSESSID_BOT_URL', 'http://phpsessid-bot:5000')
+
+# EDU site
+EDU_HOST = os.getenv('EDU_HOST', 'edu.edu.vn.ua')
+EDU_WEBINAR_URL = f'https://{EDU_HOST}/webinar/useractive'
+
+# Playwright
+PLAYWRIGHT_SERVER = os.getenv('PLAYWRIGHT_SERVER', 'http://playwright:8000')
+PLAYWRIGHT_API_KEY = os.getenv('PLAYWRIGHT_API_KEY', 'your-secret-api-key')
+WEBINAR_WAIT_TIME = int(os.getenv('WEBINAR_WAIT_TIME', '3'))  # Секунды ожидания загрузки

edu_master/lessons_bot/handlers.py

@@ -0,0 +1,131 @@
+import logging
+import requests
+from telegram import Update
+from telegram.ext import ContextTypes
+import config
+from utils import fetch_webinars, format_webinar_message
+
+logger = logging.getLogger(__name__)
+
+
+async def start(update: Update, context: ContextTypes.DEFAULT_TYPE):
+    """Команда /start"""
+    welcome_message = """
+Привет! Я бот для проверки домашних заданий и вебинаров.
+
+<b>Команды:</b>
+/check - Проверить несделанные уроки 
+/webinar - Проверить активные онлайн уроки
+/help - Помощь
+    """
+    await update.message.reply_text(welcome_message, parse_mode='HTML')
+
+
+async def help_command(update: Update, context: ContextTypes.DEFAULT_TYPE):
+    """Команда /help"""
+    help_text = """
+<b>Как пользоваться ботом:</b>
+
+<b>/check</b> - Проверка домашних заданий
+- Поиск несделанных уроков
+
+⏱ Проверка занимает 10-30 секунд
+
+<b>/webinar</b> - Активные онлайн уроки
+- Проверка активных вебинаровв
+⏱ Проверка занимает 3-5 секунд
+    """
+    await update.message.reply_text(help_text, parse_mode='HTML')
+
+
+async def check_homework(update: Update, context: ContextTypes.DEFAULT_TYPE):
+    """Команда /check - запускает проверку уроков"""
+    chat_id = update.effective_chat.id
+    user_id = update.effective_user.id
+    username = update.effective_user.username or "unknown"
+    
+    # Отправляем уведомление что начали работу
+    status_message = await update.message.reply_text("Запускаю проверку уроков...")
+    
+    # Формируем данные для n8n
+    payload = {
+        "chat_id": chat_id,
+        "user_id": user_id,
+        "username": username,
+        "timestamp": update.message.date.isoformat()
+    }
+    
+    headers = {
+        "Authorization": f"Bearer {config.N8N_SECRET}",
+        "Content-Type": "application/json"
+    }
+    
+    try:
+        logger.info(f"Sending request to n8n for user {user_id}")
+        
+        # Отправляем запрос в n8n
+        response = requests.post(
+            config.N8N_WEBHOOK_URL,
+            json=payload,
+            headers=headers,
+            timeout=5  # Короткий таймаут т.к. это асинхронный запрос
+        )
+        
+        if response.status_code == 200:
+            await status_message.edit_text(
+                "✅ Запрос принят!\n"
+                "🔄 Парсинг сайта и анализ данных...\n"
+                "⏱ Это займет 10-30 секунд"
+            )
+            logger.info(f"Request accepted for user {user_id}")
+        else:
+            await status_message.edit_text(
+                f"Ошибка при отправке запроса. Функция в разработке\n"
+                f"Код: {response.status_code}"
+            )
+            logger.error(f"n8n returned status {response.status_code}")
+            
+    except requests.Timeout:
+        await status_message.edit_text("⏱ Запрос обрабатывается (таймаут соединения)")
+        logger.warning(f"Timeout for user {user_id}")
+    except Exception as e:
+        await status_message.edit_text(f"❌ Ошибка: {str(e)}")
+        logger.error(f"Error for user {user_id}: {e}", exc_info=True)
+
+
+async def check_webinar(update: Update, context: ContextTypes.DEFAULT_TYPE):
+    """Команда /webinar - проверяет активные онлайн уроки"""
+    user_id = update.effective_user.id
+    
+    # Отправляем уведомление что начали работу
+    status_message = await update.message.reply_text("Проверяю активные вебинары...")
+    
+    try:
+        logger.info(f"Checking webinars for user {user_id}")
+        
+        # Получаем список вебинаров
+        webinars = fetch_webinars()
+        
+        if webinars is None:
+            await status_message.edit_text(
+                "❌ Не удалось получить информацию о вебинарах\n"
+                "Попробуйте позже или обратитесь к администратору\n"
+                "|@MrForust|mr.forust| Либо же прямо сюда."
+            )
+            logger.error(f"Failed to fetch webinars for user {user_id}")
+            return
+        
+        # Форматируем и отправляем результат
+        message = format_webinar_message(webinars)
+        await status_message.edit_text(message, parse_mode='HTML', disable_web_page_preview=True)
+        
+        logger.info(f"Webinar check completed for user {user_id}: found {len(webinars)} webinars")
+        
+    except Exception as e:
+        await status_message.edit_text(f"❌ Ошибка: {str(e)}")
+        logger.error(f"Error checking webinars for user {user_id}: {e}", exc_info=True)
+
+
+async def error_handler(update: Update, context: ContextTypes.DEFAULT_TYPE):
+    """Обработчик ошибок"""
+    logger.error(f"Update {update} caused error {context.error}", exc_info=context.error)

edu_master/lessons_bot/main.py

@@ -0,0 +1,42 @@
+import logging
+from telegram import Update
+from telegram.ext import Application, CommandHandler
+import config
+from handlers import start, help_command, check_homework, check_webinar, error_handler
+
+# Настройка логирования
+logging.basicConfig(
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    level=logging.INFO
+)
+logger = logging.getLogger(__name__)
+
+
+def main():
+    """Запуск бота"""
+    if not config.BOT_TOKEN:
+        logger.error("LESSONS_BOT_TOKEN not set!")
+        return
+    
+    # Создаем приложение
+    application = Application.builder().token(config.BOT_TOKEN).build()
+    
+    # Регистрируем обработчики команд
+    application.add_handler(CommandHandler("start", start))
+    application.add_handler(CommandHandler("help", help_command))
+    application.add_handler(CommandHandler("check", check_homework))
+    application.add_handler(CommandHandler("webinar", check_webinar))
+    
+    # Регистрируем обработчик ошибок
+    application.add_error_handler(error_handler)
+    
+    # Запускаем бота
+    logger.info("Lessons Bot started!")
+    logger.info(f"PHPSESSID Bot URL: {config.PHPSESSID_BOT_URL}")
+    logger.info(f"n8n Webhook URL: {config.N8N_WEBHOOK_URL}")
+    
+    application.run_polling(allowed_updates=Update.ALL_TYPES)
+
+
+if __name__ == '__main__':
+    main()

edu_master/lessons_bot/requirements.txt

@@ -0,0 +1,5 @@
+python-telegram-bot==20.7
+requests==2.31.0
+beautifulsoup4==4.12.2
+python-dotenv==1.0.0
+lxml==4.9.3

edu_master/lessons_bot/utils.py

@@ -0,0 +1,258 @@
+import logging
+import requests
+from bs4 import BeautifulSoup
+from typing import Optional, Dict, List
+import config
+
+logger = logging.getLogger(__name__)
+
+
+def get_phpsessid() -> Optional[str]:
+    """
+    Получает валидный PHPSESSID через phpsessid-bot
+    
+    Returns:
+        str: PHPSESSID или None в случае ошибки
+    """
+    try:
+        url = f"{config.PHPSESSID_BOT_URL}/get-session"
+        logger.info(f"Requesting PHPSESSID from {url}")
+        
+        response = requests.post(url, timeout=10)
+        
+        if response.status_code == 200:
+            data = response.json()
+            if data.get('success'):
+                phpsessid = data.get('phpsessid')
+                logger.info(f"Got PHPSESSID: {phpsessid[:10]}...")
+                return phpsessid
+            else:
+                logger.error(f"Failed to get PHPSESSID: {data.get('error')}")
+                return None
+        else:
+            logger.error(f"PHPSESSID bot returned status {response.status_code}")
+            return None
+            
+    except Exception as e:
+        logger.error(f"Error getting PHPSESSID: {e}", exc_info=True)
+        return None
+
+
+def parse_webinar_table(html_content: str) -> List[Dict[str, str]]:
+    """
+    Парсит таблицу с вебинарами
+    
+    Args:
+        html_content: HTML контент страницы
+        
+    Returns:
+        List[Dict]: Список вебинаров или пустой список
+    """
+    try:
+        soup = BeautifulSoup(html_content, 'html.parser')
+        
+        # Находим таблицу с вебинарами
+        meetings_div = soup.find('div', {'id': 'meetings'})
+        if not meetings_div:
+            logger.warning("meetings div not found")
+            return []
+        
+        table = meetings_div.find('table', {'class': 'table table-zebra'})
+        if not table:
+            logger.warning("table not found")
+            return []
+        
+        tbody = table.find('tbody')
+        if not tbody:
+            logger.warning("tbody not found")
+            return []
+        
+        rows = tbody.find_all('tr')
+        if not rows:
+            return []
+        
+        # Проверяем на сообщение "Жодного онлайн уроку зараз"
+        first_row = rows[0]
+        td = first_row.find('td')
+        if td and 'Жодного онлайн уроку зараз' in td.get_text(strip=True):
+            logger.info("No webinars available")
+            return []
+        
+        # Парсим активные вебинары
+        webinars = []
+        for row in rows:
+            tds = row.find_all('td')
+            if len(tds) >= 4:
+                webinar = {
+                    'topic': tds[0].get_text(strip=True),
+                    'course': tds[1].get_text(strip=True),
+                    'teacher': tds[2].get_text(strip=True),
+                    'join_link': tds[3].find('a')['href'] if tds[3].find('a') else ''
+                }
+                webinars.append(webinar)
+                
+        logger.info(f"Parsed {len(webinars)} webinars")
+        return webinars
+        
+    except Exception as e:
+        logger.error(f"Error parsing webinar table: {e}", exc_info=True)
+        return []
+
+
+def fetch_webinars_with_playwright() -> Optional[List[Dict[str, str]]]:
+    """
+    Получает список активных вебинаров используя Playwright для динамического контента
+    
+    Returns:
+        List[Dict]: Список вебинаров или None в случае ошибки
+    """
+    # Получаем PHPSESSID
+    phpsessid = get_phpsessid()
+    if not phpsessid:
+        logger.error("Failed to get PHPSESSID")
+        return None
+    
+    try:
+        # Подготавливаем cookies для Playwright
+        cookies = [
+            {
+                'name': 'PHPSESSID',
+                'value': phpsessid,
+                'domain': config.EDU_HOST,
+                'path': '/'
+            }
+        ]
+        
+        # Запрос к Playwright серверу
+        playwright_request = {
+            'url': config.EDU_WEBINAR_URL,
+            'cookies': cookies,
+            'wait_until': 'networkidle',  # Ждем пока сеть успокоится
+            'wait_time': config.WEBINAR_WAIT_TIME * 1000,  # Дополнительное ожидание в миллисекундах
+            'user_agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36'
+        }
+        
+        headers = {
+            'Authorization': f'Bearer {config.PLAYWRIGHT_API_KEY}',
+            'Content-Type': 'application/json'
+        }
+        
+        logger.info(f"Fetching webinars via Playwright from {config.EDU_WEBINAR_URL}")
+        logger.info(f"Will wait {config.WEBINAR_WAIT_TIME} seconds for dynamic content")
+        
+        response = requests.post(
+            f"{config.PLAYWRIGHT_SERVER}/render",
+            json=playwright_request,
+            headers=headers,
+            timeout=30
+        )
+        
+        if response.status_code != 200:
+            logger.error(f"Playwright server returned status {response.status_code}")
+            logger.error(f"Response: {response.text}")
+            return None
+        
+        result = response.json()
+        html_content = result.get('html', '')
+        
+        if not html_content:
+            logger.error("No HTML content in Playwright response")
+            return None
+        
+        # Парсим таблицу
+        webinars = parse_webinar_table(html_content)
+        return webinars
+        
+    except Exception as e:
+        logger.error(f"Error fetching webinars via Playwright: {e}", exc_info=True)
+        return None
+
+
+def fetch_webinars() -> Optional[List[Dict[str, str]]]:
+    """
+    Получает список активных вебинаров
+    Сначала пробует через Playwright (для динамического контента),
+    при неудаче - через обычный requests
+    
+    Returns:
+        List[Dict]: Список вебинаров или None в случае ошибки
+    """
+    # Пробуем через Playwright
+    logger.info("Attempting to fetch via Playwright for dynamic content")
+    webinars = fetch_webinars_with_playwright()
+    
+    if webinars is not None:
+        return webinars
+    
+    # Fallback на обычный requests
+    logger.warning("Playwright fetch failed, falling back to simple requests")
+    
+    # Получаем PHPSESSID
+    phpsessid = get_phpsessid()
+    if not phpsessid:
+        logger.error("Failed to get PHPSESSID")
+        return None
+    
+    # Запрашиваем страницу с вебинарами
+    try:
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'Accept-Language': 'ru-RU,ru;q=0.9,uk;q=0.8',
+            'Referer': f'https://{config.EDU_HOST}/'
+        }
+        
+        cookies = {
+            'PHPSESSID': phpsessid
+        }
+        
+        logger.info(f"Fetching webinars from {config.EDU_WEBINAR_URL}")
+        response = requests.get(
+            config.EDU_WEBINAR_URL,
+            headers=headers,
+            cookies=cookies,
+            timeout=15
+        )
+        
+        if response.status_code != 200:
+            logger.error(f"Failed to fetch webinars page: {response.status_code}")
+            return None
+        
+        # Парсим таблицу
+        webinars = parse_webinar_table(response.text)
+        return webinars
+        
+    except Exception as e:
+        logger.error(f"Error fetching webinars: {e}", exc_info=True)
+        return None
+
+
+def format_webinar_message(webinars: List[Dict[str, str]]) -> str:
+    """
+    Форматирует список вебинаров для отправки в Telegram
+    
+    Args:
+        webinars: Список вебинаров
+        
+    Returns:
+        str: Отформатированное сообщение
+    """
+    if not webinars:
+        return "📭 Жодного онлайн уроку зараз"
+    
+    message = "🎓 <b>Активні онлайн уроки:</b>\n\n"
+    
+    for i, webinar in enumerate(webinars, 1):
+        message += f"<b>{i}. {webinar['topic']}</b>\n"
+        message += f"📚 Курс: {webinar['course']}\n"
+        message += f"👨‍🏫 Вчитель: {webinar['teacher']}\n"
+        
+        if webinar['join_link']:
+            full_link = webinar['join_link']
+            if not full_link.startswith('http'):
+                full_link = f"https://{config.EDU_HOST}{webinar['join_link']}"
+            message += f"🔗 <a href='{full_link}'>Увійти до уроку</a>\n"
+        
+        message += "\n"
+    
+    return message

edu_master/n8n/docker-compose.yaml

@@ -0,0 +1,32 @@
+  n8n:
+    image: docker.n8n.io/n8nio/n8n
+    restart: unless-stopped
+    environment:
+      - N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS=true
+      - N8N_PORT=5678
+      - N8N_RUNNERS_ENABLED=true
+      - NODE_ENV=production
+      - GENERIC_TIMEZONE=Europe/Bratislava
+      - TZ=Europe/Bratislava
+      - N8N_SECURE_COOKIE=false
+      - N8N_COMMUNITY_PACKAGES_ALLOW_TOOL_USAGE=true
+    volumes:
+      - n8n_data:/home/node/.n8n
+      - ./n8n/local-files:/files
+    extra_hosts:
+      - "enterprise.n8n.io:104.26.13.187"
+      - "enterprise.n8n.io:104.26.12.187"
+      - "enterprise.n8n.io:172.67.68.102"
+    dns:
+      - 1.1.1.1
+      - 8.8.8.8
+    networks:
+      - traefik-proxy 
+      - default
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.n8n.rule=Host(`n8n.workstation`)" 
+      - "traefik.http.routers.n8n.entrypoints=websecure"
+      - "traefik.http.services.n8n.loadbalancer.server.port=5678"
+      - "traefik.docker.network=traefik-proxy"
+      - "traefik.http.routers.n8n.tls.certresolver=le"

edu_master/nginx/entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+
+# Replace environment variables in the Nginx configuration template
+envsubst '${MINIO_PRIVATE_BUCKET} ${MINIO_PUBLIC_BUCKET}' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf
+
+# Start Nginx
+exec nginx -g 'daemon off;'

edu_master/nginx/nginx.conf

@@ -0,0 +1,87 @@
+server {
+    listen 80;
+    server_name localhost;
+    client_max_body_size 100M;
+
+    # Frontend
+    location / {
+        proxy_pass http://frontend:80;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    # API
+    location /api/ {
+        proxy_pass http://app:9000;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    # MCP
+    location ~ ^/(sse|messages) {
+        proxy_pass http://mcp:3000;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+
+
+        # Important SSE settings
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+
+        # Disable buffering so events are sent immediately
+        proxy_buffering off;
+        proxy_cache off;
+
+        # Increase timeouts so connection stays open
+        proxy_read_timeout 3600s;
+        proxy_send_timeout 3600s;
+    }
+
+
+    # MinIO private bucket
+    location /${MINIO_PRIVATE_BUCKET}/ {
+        proxy_pass http://minio:9000/${MINIO_PRIVATE_BUCKET}/;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_buffering off;
+    }
+
+    # MinIO public bucket
+    location /${MINIO_PUBLIC_BUCKET}/ {
+        proxy_pass http://minio:9000/${MINIO_PUBLIC_BUCKET}/;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_buffering off;
+    }
+
+    # MinIO API - for direct S3 operations
+    # location /minio/api/ {
+    #     proxy_pass http://minio:9000/;
+    #     proxy_set_header Host $host;
+    #     proxy_set_header X-Real-IP $remote_addr;
+    #     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    #     proxy_set_header X-Forwarded-Proto $scheme;
+    #     proxy_buffering off;
+    # }
+
+    # MinIO Console
+    location /minio-console/ {
+        proxy_pass http://minio:9001/;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        # Rewrite location headers
+        proxy_redirect / /minio-console/;
+    }
+}

edu_master/phpsessid_bot/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Устанавливаем зависимости
+RUN pip install --no-cache-dir flask requests
+
+# Копируем код бота
+COPY main.py .
+
+# Открываем порт
+EXPOSE 5000
+
+# Запускаем бот
+CMD ["python", "-u", "main.py"]

edu_master/phpsessid_bot/main.py

@@ -0,0 +1,216 @@
+import os
+import logging
+from flask import Flask, request, jsonify
+import requests
+from datetime import datetime
+
+# Настройка логирования
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+app = Flask(__name__)
+
+# Конфигурация из переменных окружения
+EDU_HOST = os.getenv('EDU_HOST', 'edu.edu.vn.ua')
+EDU_LOGIN = os.getenv('EDU_LOGIN', '')
+EDU_PASSWORD = os.getenv('EDU_PASSWORD', '')
+BOT_PORT = int(os.getenv('BOT_PORT', '5000'))
+BOT_HOST = os.getenv('BOT_HOST', '0.0.0.0')
+
+# Кэш для хранения актуальной сессии
+session_cache = {
+    'phpsessid': None,
+    'expires_at': None
+}
+
+
+def login_and_get_session():
+    """
+    Выполняет логин и возвращает новый PHPSESSID
+    """
+    url = f"https://{EDU_HOST}/user/login"
+    
+    headers = {
+        'Cache-Control': 'max-age=0',
+        'Sec-Ch-Ua': '"Chromium";v="141", "Not?A_Brand";v="8"',
+        'Sec-Ch-Ua-Mobile': '?0',
+        'Sec-Ch-Ua-Platform': '"Linux"',
+        'Accept-Language': 'ru-RU,ru;q=0.9',
+        'Origin': f'https://{EDU_HOST}',
+        'Content-Type': 'application/x-www-form-urlencoded',
+        'Upgrade-Insecure-Requests': '1',
+        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+        'Sec-Fetch-Site': 'same-origin',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-User': '?1',
+        'Sec-Fetch-Dest': 'document',
+        'Referer': f'https://{EDU_HOST}/',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Priority': 'u=0, i'
+    }
+    
+    data = {
+        'login': EDU_LOGIN,
+        'password': EDU_PASSWORD
+    }
+    
+    try:
+        logger.info(f"Attempting login to {url}")
+        response = requests.post(
+            url, 
+            data=data, 
+            headers=headers,
+            allow_redirects=False,
+            timeout=10
+        )
+        
+        # Получаем PHPSESSID из cookies
+        phpsessid = response.cookies.get('PHPSESSID')
+        
+        if phpsessid:
+            logger.info(f"Login successful, got PHPSESSID: {phpsessid[:10]}...")
+            return {
+                'success': True,
+                'phpsessid': phpsessid,
+                'status_code': response.status_code
+            }
+        else:
+            logger.warning(f"Login failed: no PHPSESSID in response. Status: {response.status_code}")
+            return {
+                'success': False,
+                'error': 'No PHPSESSID in response',
+                'status_code': response.status_code
+            }
+            
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Login request failed: {str(e)}")
+        return {
+            'success': False,
+            'error': str(e)
+        }
+
+
+def validate_phpsessid(phpsessid):
+    """
+    Проверяет валидность существующего PHPSESSID
+    """
+    url = f"https://{EDU_HOST}/"
+    
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+    }
+    
+    cookies = {
+        'PHPSESSID': phpsessid
+    }
+    
+    try:
+        response = requests.get(url, headers=headers, cookies=cookies, timeout=10)
+        
+        # Проверяем, не редиректит ли на страницу логина
+        is_valid = response.status_code == 200 and '/user/login' not in response.url
+        
+        return {
+            'valid': is_valid,
+            'status_code': response.status_code,
+            'url': response.url
+        }
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Validation request failed: {str(e)}")
+        return {
+            'valid': False,
+            'error': str(e)
+        }
+
+
+@app.route('/health', methods=['GET'])
+def health():
+    """Health check endpoint"""
+    return jsonify({'status': 'ok', 'timestamp': datetime.now().isoformat()})
+
+
+@app.route('/get-session', methods=['POST', 'GET'])
+def get_session():
+    """
+    Основной endpoint для получения валидного PHPSESSID
+    Возвращает кэшированную сессию или создает новую
+    """
+    result = login_and_get_session()
+    
+    if result['success']:
+        session_cache['phpsessid'] = result['phpsessid']
+        session_cache['last_updated'] = datetime.now().isoformat()
+        
+        return jsonify({
+            'success': True,
+            'phpsessid': result['phpsessid'],
+            'timestamp': datetime.now().isoformat()
+        })
+    else:
+        return jsonify({
+            'success': False,
+            'error': result.get('error', 'Login failed'),
+            'timestamp': datetime.now().isoformat()
+        }), 400
+
+
+@app.route('/validate-session', methods=['POST'])
+def validate_session():
+    """
+    Проверяет валидность переданного PHPSESSID
+    """
+    data = request.get_json() or {}
+    phpsessid = data.get('phpsessid') or request.args.get('phpsessid')
+    
+    if not phpsessid:
+        return jsonify({
+            'success': False,
+            'error': 'PHPSESSID not provided'
+        }), 400
+    
+    validation_result = validate_phpsessid(phpsessid)
+    
+    return jsonify({
+        'success': True,
+        'valid': validation_result.get('valid', False),
+        'details': validation_result,
+        'timestamp': datetime.now().isoformat()
+    })
+
+
+@app.route('/refresh-session', methods=['POST', 'GET'])
+def refresh_session():
+    """
+    Принудительно обновляет сессию
+    """
+    result = login_and_get_session()
+    
+    if result['success']:
+        return jsonify({
+            'success': True,
+            'phpsessid': result['phpsessid'],
+            'message': 'Session refreshed successfully',
+            'timestamp': datetime.now().isoformat()
+        })
+    else:
+        return jsonify({
+            'success': False,
+            'error': result.get('error', 'Failed to refresh session'),
+            'timestamp': datetime.now().isoformat()
+        }), 400
+
+
+if __name__ == '__main__':
+    if not EDU_LOGIN or not EDU_PASSWORD:
+        logger.error("EDU_LOGIN and EDU_PASSWORD must be set!")
+        exit(1)
+    
+    logger.info(f"Starting PHPSESSID validator bot on {BOT_HOST}:{BOT_PORT}")
+    logger.info(f"Target host: {EDU_HOST}")
+    
+    app.run(host=BOT_HOST, port=BOT_PORT, debug=False)

edu_master/phpsessid_bot/requirements.txt

@@ -0,0 +1,3 @@
+flask==3.0.0
+requests==2.31.0
+Werkzeug==3.0.1