From 6e367b3ac9ddfc72cfad1d5bdc4cabce46b9daa1 Mon Sep 17 00:00:00 2001 From: vvzvlad Date: Sat, 1 Feb 2025 16:47:46 +0300 Subject: [PATCH] first work version --- .github/workflows/ghcr-check-publish.yml | 29 +++++ .gitignore | 4 + Dockerfile | 10 ++ README.md | 45 +++++++ api_server.py | 43 +++++++ architecture.md | 113 ------------------ config.py | 20 ++++ docker-compose.yml | 25 ++++ requirements.txt | 5 + session_generator.py | 27 +++++ telegram_client.py | 146 +++++++++++++++++++++++ 11 files changed, 354 insertions(+), 113 deletions(-) create mode 100644 .github/workflows/ghcr-check-publish.yml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 api_server.py delete mode 100644 architecture.md create mode 100644 config.py create mode 100644 docker-compose.yml create mode 100644 requirements.txt create mode 100644 session_generator.py create mode 100644 telegram_client.py diff --git a/.github/workflows/ghcr-check-publish.yml b/.github/workflows/ghcr-check-publish.yml new file mode 100644 index 0000000..14b3cdb --- /dev/null +++ b/.github/workflows/ghcr-check-publish.yml @@ -0,0 +1,29 @@ +name: Docker Image CI + +on: + workflow_dispatch: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GHCR_PAT }} + + - name: Build the Docker image + run: docker build . --file Dockerfile --tag ghcr.io/${{ github.repository }}:latest --tag ghcr.io/${{ github.repository }}:${{ github.sha }} + + - name: Push Docker images + run: | + docker push ghcr.io/${{ github.repository }}:latest + docker push ghcr.io/${{ github.repository }}:${{ github.sha }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..49f51ac --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ + +.env +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1229c95 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.11-slim + +WORKDIR /app +RUN mkdir -p data +RUN apt-get update && apt-get install -y curl libgl1 && rm -rf /var/lib/apt/lists/* +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY *.py . + +CMD ["uvicorn", "api_server:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..83d8468 --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ + + + + +Create API ID/HASH: + +1)Login at https://my.telegram.org/apps +2)API development tools +3)Create new application +4)Copy App api_id and api_hash + +Get session: +1)python3 -m venv .venv +2)source .venv/bin/activate +3)pip install pyrogram +4)Run ```TG_API_ID=290389758 TG_API_HASH=c22987sdfnkjjhd37efa5f0 python3 session_generator.py``` +5)Enter phone number, get code in telegram, enter code, and copy session string: +``` +Enter phone number or bot token: +7 993 850 5104 +Is "+7 993 850 5104" correct? (y/N): y +The confirmation code has been sent via Telegram app +Enter confirmation code: 69267 +The two-step verification is enabled and a password is required +Password hint: None +Enter password (empty to recover): Passport-Vegan-Scale6 + +Your session string: +======================================== +AgG7QBoANg0YVmwZTZmqadO4MJQdnaRRnXwSYpbbkGf49aATvTZj-yKcvdH8IsIDbwp00PbWFcbSjzoPsmiUK8BF80yVmML7iEQptBZrRTLsnoxmeglD-I1dqcAB2ufkxQDM_40y5KAHiFvzAzXhVngQo8W7u3ZPQXb_DTcfogXXePPBQAyV20cuwGrsArv-R39ssSWnFueGnB21Y_cTXTQAAAAFPEaL8AA +======================================== +Use session on ENV variable TG_SESSION_STRING in docker-compose.yml +``` +6)Set session ENV variable in docker-compose.yml file: +``` + pyrogram_bridge: + image: ghcr.io/vvzvlad/pyrogram-bridge:latest + container_name: pyrogram-bridge + environment: + TG_API_ID: 290389758 + TG_API_HASH: c22987sdfnkjjhd37efa5f0 + TG_SESSION_STRING: "AgG7QBoANg0YVmwZTZmqadO4MJQdn............FPEaL8AA" + TZ: Europe/Moscow + volumes: +.... +``` diff --git a/api_server.py b/api_server.py new file mode 100644 index 0000000..b025b5b --- /dev/null +++ b/api_server.py @@ -0,0 +1,43 @@ +from fastapi import FastAPI, HTTPException, Response +from telegram_client import TelegramClient +from config import get_settings +import logging +import json + +logger = logging.getLogger(__name__) +app = FastAPI(title="PyroTg Bridge") +client = TelegramClient() +settings = get_settings() + +@app.on_event("startup") +async def startup(): + await client.start() + +@app.on_event("shutdown") +async def shutdown(): + await client.stop() + +@app.get("/post/{channel}/{post_id}") +async def get_post(channel: str, post_id: int): + try: + data = await client.get_post(channel, post_id) + return Response( + content=json.dumps(data), + media_type="application/json" + ) + except Exception as e: + logger.error(f"API error: {str(e)}") + raise HTTPException( + status_code=404, + detail={ + "error": "post_retrieval_error", + "message": str(e) + } + ) from e + +@app.get("/status") +async def health_check(): + return { + "status": "ok", + "connected": client.client.is_connected + } \ No newline at end of file diff --git a/architecture.md b/architecture.md deleted file mode 100644 index 1aa149d..0000000 --- a/architecture.md +++ /dev/null @@ -1,113 +0,0 @@ -# Telegram RSS Bridge Architecture - -## System Components - -### 1. Web Parser (Main Bridge) -- Existing web-based parser for public Telegram content -- Fallback mechanism to Pyro Parser when content not found - -### 2. Pyro Parser (Backup Service) -- Pyrogram-based client for private content -- HTTP API server for post retrieval -- Session management for Telegram auth - -### 3. HTTP API Interface -- Endpoints: - - `GET /post/{channel}/{post_id}` - Get single post - - `GET /channel/{channel}` - (Future) Get channel feed -- Response format: - ```json - { - "id": 123, - "date": "2023-01-01T00:00:00", - "text": "post content", - "media": [], - "views": 100, - "reactions": [] // Future extension - } - ``` - -## Service Structure -```text -pyro_parser/ -├── telegram_client.py # Pyrogram wrapper + session management -├── api_server.py # FastAPI/Falcon server -├── schemas.py # Data models/Pydantic schemas -└── config.py # Environment configuration -``` - -## Key Features - -1. **Pyrogram Client** -- Async Telegram client with session persistence -- Automatic reconnection -- Rate limiting handling -- Error tracking for API calls - -2. **Content Processing** -- Raw post transformation -- Media links extraction -- Text cleanup utilities -- (Future) Reactions parser - -## Environment Configuration -```bash -# Core settings -TG_API_ID=123456 -TG_API_HASH=abcdef123456 -SESSION_PATH=/data/session.file - -# Server config -API_HOST=0.0.0.0 -API_PORT=8000 -REQUEST_TIMEOUT=30 -``` - -## Docker Setup -```yaml -services: - pyro_parser: - image: ghcr.io/vvzvlad/pyrotg-bridge:latest - environment: - TG_API_ID: ${TG_API_ID} - TG_API_HASH: ${TG_API_HASH} - volumes: - - ./sessions:/data - ports: - - "8000:8000" -``` - -## CI/CD Integration -```yaml -# .github/workflows/deploy.yml -name: Deploy -on: [push] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: . - push: true - tags: ghcr.io/vvzvlad/pyrotg-bridge:latest -``` - -## Extension Points - -1. **Post Processing** -- Media download hooks -- Link preview generators -- Content filters - -2. **API Features** -- Pagination params -- Field selection -- Webhook support - -3. **Monitoring** -- /status endpoint -- Basic request metrics -- Error logging diff --git a/config.py b/config.py new file mode 100644 index 0000000..46c3a2c --- /dev/null +++ b/config.py @@ -0,0 +1,20 @@ +import os + +TG_API_ID = int(os.getenv("TG_API_ID")) +TG_API_HASH = os.getenv("TG_API_HASH") +SESSION_PATH = os.getenv("SESSION_PATH", "session.file") +API_HOST = os.getenv("API_HOST", "0.0.0.0") +API_PORT = int(os.getenv("API_PORT", 8000)) +REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", 30)) +SESSION_STRING = os.getenv("TG_SESSION_STRING", "") + +def get_settings(): + return { + "tg_api_id": TG_API_ID, + "tg_api_hash": TG_API_HASH, + "session_path": SESSION_PATH, + "api_host": API_HOST, + "api_port": API_PORT, + "request_timeout": REQUEST_TIMEOUT, + "session_string": SESSION_STRING + } \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6bb9819 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,25 @@ +services: + pyrogram_bridge: + image: ghcr.io/vvzvlad/pyrogram-bridge:latest + container_name: pyrogram-bridge + environment: + TG_API_ID: ${TG_API_ID} + TG_API_HASH: ${TG_API_HASH} + TG_SESSION_STRING: ${TG_SESSION_STRING} + TZ: Europe/Moscow + volumes: + - ./sessions:/data + ports: + - "8000:8000" + restart: always + logging: + driver: "json-file" + options: + max-file: 5 + max-size: 10m + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/status"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2671fd3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +fastapi +uvicorn +pyrogram +tgcrypto +pillow>=10.0.0 \ No newline at end of file diff --git a/session_generator.py b/session_generator.py new file mode 100644 index 0000000..7c5793b --- /dev/null +++ b/session_generator.py @@ -0,0 +1,27 @@ +import os +import asyncio +from pyrogram import Client + +async def generate_session(): + api_id = os.getenv("TG_API_ID") + api_hash = os.getenv("TG_API_HASH") + + if not api_id or not api_hash: + print("Error: Set TG_API_ID and TG_API_HASH environment variables") + return + + async with Client( + name=":memory:", + api_id=int(api_id), + api_hash=api_hash, + in_memory=True + ) as client: + session = await client.export_session_string() + print("\nYour session string:") + print("=" * 40) + print(session) + print("=" * 40) + print("\nUser session on ENV variable TG_SESSION_STRING in docker-compose.yml") + +if __name__ == "__main__": + asyncio.run(generate_session()) \ No newline at end of file diff --git a/telegram_client.py b/telegram_client.py new file mode 100644 index 0000000..4c1f09b --- /dev/null +++ b/telegram_client.py @@ -0,0 +1,146 @@ +import logging +from pyrogram import Client, errors +from pyrogram.types import Message +from config import get_settings +import base64 +import os + +logger = logging.getLogger(__name__) +settings = get_settings() + +class TelegramClient: + def __init__(self): + if settings["session_string"]: + self.client = Client( + name="pyro_bridge", + session_string=settings["session_string"], + api_id=settings["tg_api_id"], + api_hash=settings["tg_api_hash"], + in_memory=True + ) + else: + self.client = Client( + name="pyro_bridge", + api_id=settings["tg_api_id"], + api_hash=settings["tg_api_hash"], + workdir=settings["session_path"], + in_memory=True + ) + + async def start(self): + if not self.client.is_connected: + await self.client.start() + logger.info("Telegram client connected") + + async def stop(self): + if self.client.is_connected: + await self.client.stop() + logger.info("Telegram client disconnected") + + async def _parse_message(self, message: Message) -> dict: + # Extract text from different sources + text = message.text or "" + + # Try to get caption for media messages + if not text and hasattr(message, "caption"): + text = message.caption or "" + + # Try to get text from other attributes + if not text and hasattr(message, "action"): + text = str(message.action) # For service messages + + # Parse media and add video markers + media = [] + if message.media: + media_obj = message.media + if hasattr(message, "video"): + media.append(await self._parse_media(message.video)) + else: + if isinstance(media_obj, list): + for m in media_obj: + media.append(await self._parse_media(m)) + elif hasattr(media_obj, "photo"): + media.append(await self._parse_media(media_obj.photo)) + else: + media.append(await self._parse_media(media_obj)) + + video_count = sum(1 for m in media if m.get("type") == "video") + if video_count > 0: + text = f"[{'Video' if video_count == 1 else f'{video_count} Videos'}] {text}" if text else f"[{'Video' if video_count == 1 else f'{video_count} Videos'}]" + + # Always add reactions if present + reactions_text = "" + if getattr(message, "reactions", None): + reactions_text = "\nReactions: " + ", ".join( + f"{r.emoji}({r.count})" + for r in message.reactions.reactions + ) + text += reactions_text + + return { + "id": message.id, + "date": message.date.isoformat(), + "text": text, + "media": media, + "views": message.views or 0, + "reactions": [r.emoji for r in message.reactions.reactions] if getattr(message, "reactions", None) else [] + } + + async def _parse_media(self, media_obj) -> dict: + # Handle different media types + media_type = media_obj.__class__.__name__.lower() + file_id = getattr(media_obj, "file_id", "") + thumbnail_base64 = None + + # Special cases + if media_type == "photofile": + file_id = getattr(media_obj, "photo_file_id", file_id) + elif media_type == "webpage": + return {"type": "webpage", "url": getattr(media_obj, "url", "")} + elif media_type == "video": + # Download and encode thumbnail + if media_obj.thumbs: + try: + thumb_path = await self.client.download_media(media_obj.thumbs[0].file_id) + with open(thumb_path, "rb") as image_file: + thumbnail_base64 = base64.b64encode(image_file.read()).decode("utf-8") + os.remove(thumb_path) + except Exception as e: + logger.error(f"Thumbnail error: {e.__class__.__name__} - {str(e)}") + + return { + "type": "video", + "url": file_id, + "size": media_obj.file_size, + "duration": media_obj.duration, + "thumbnail": thumbnail_base64 + } + + return { + "type": media_type, + "url": str(file_id), + "size": getattr(media_obj, "file_size", None), + "thumbnail": thumbnail_base64 + } + + async def get_post(self, channel: str, post_id: int) -> dict: + try: + message = await self.client.get_messages( + chat_id=channel, + message_ids=post_id + ) + if not message: + raise ValueError("Message not found") + + # Debug raw message structure + print("Raw message object: %s", message) + print("Message attributes: %s", dir(message)) + print("Message media type: %s", type(message.media) if message.media else None) + + return await self._parse_message(message) + except errors.RPCError as e: + logger.error(f"Telegram API error: {e.__class__.__name__} - {e}") + raise + except Exception as e: + logger.error(f"Unexpected error in get_post: {str(e)}") + raise \ No newline at end of file