Initial commit: XDL Twitter/X video downloader

backend/app/auth.py

@@ -0,0 +1,42 @@
+"""JWT authentication utilities."""
+import os
+from datetime import datetime, timedelta
+from typing import Optional
+from jose import JWTError, jwt
+from fastapi import Depends, HTTPException, status
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+
+SECRET_KEY = os.getenv("SECRET_KEY", "dev-secret-key")
+ALGORITHM = "HS256"
+ACCESS_TOKEN_EXPIRE_HOURS = 72
+
+security = HTTPBearer()
+
+
+def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -> str:
+    to_encode = data.copy()
+    expire = datetime.utcnow() + (expires_delta or timedelta(hours=ACCESS_TOKEN_EXPIRE_HOURS))
+    to_encode.update({"exp": expire})
+    return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
+
+
+def verify_token(token: str) -> dict:
+    try:
+        payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
+        return payload
+    except JWTError:
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid token")
+
+
+async def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(security)) -> dict:
+    return verify_token(credentials.credentials)
+
+
+def optional_auth(credentials: Optional[HTTPAuthorizationCredentials] = Depends(HTTPBearer(auto_error=False))) -> Optional[dict]:
+    """Optional authentication - returns None if no valid token."""
+    if credentials is None:
+        return None
+    try:
+        return verify_token(credentials.credentials)
+    except HTTPException:
+        return None

backend/app/database.py

@@ -0,0 +1,23 @@
+"""Database configuration and session management."""
+import os
+from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession
+from sqlalchemy.orm import DeclarativeBase
+
+DATABASE_URL = os.getenv("DATABASE_URL", "sqlite+aiosqlite:///./xdl.db")
+
+engine = create_async_engine(DATABASE_URL, echo=False)
+async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+async def get_db():
+    async with async_session() as session:
+        yield session
+
+
+async def init_db():
+    async with engine.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)

backend/app/main.py

@@ -0,0 +1,38 @@
+"""XDL - Twitter/X Video Downloader API."""
+import os
+from contextlib import asynccontextmanager
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from app.database import init_db
+from app.routes import auth, parse, download, admin
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    await init_db()
+    yield
+
+
+app = FastAPI(title="XDL - Video Downloader", version="1.0.0", lifespan=lifespan)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.include_router(auth.router)
+app.include_router(parse.router)
+app.include_router(download.router)
+app.include_router(admin.router)
+
+
+@app.get("/api/health")
+async def health():
+    return {"status": "ok"}

backend/app/models.py

@@ -0,0 +1,26 @@
+"""SQLAlchemy models."""
+from datetime import datetime
+from sqlalchemy import Column, Integer, String, DateTime, BigInteger, Text
+from app.database import Base
+
+
+class Video(Base):
+    __tablename__ = "videos"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    task_id = Column(String(64), unique=True, index=True, nullable=False)
+    url = Column(String(512), nullable=False)
+    title = Column(String(512), default="")
+    platform = Column(String(32), default="twitter")
+    thumbnail = Column(String(1024), default="")
+    quality = Column(String(32), default="")
+    format_id = Column(String(64), default="")
+    filename = Column(String(512), default="")
+    file_path = Column(String(1024), default="")
+    file_size = Column(BigInteger, default=0)
+    duration = Column(Integer, default=0)
+    status = Column(String(16), default="pending")  # pending, downloading, done, error
+    error_message = Column(Text, default="")
+    progress = Column(Integer, default=0)  # 0-100
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)

backend/app/routes/admin.py

@@ -0,0 +1,70 @@
+"""Admin management routes."""
+import os
+from fastapi import APIRouter, HTTPException, Depends, Query
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import select, func, or_
+from app.database import get_db
+from app.models import Video
+from app.schemas import VideoInfo, VideoListResponse, StorageStats
+from app.auth import get_current_user
+
+router = APIRouter(prefix="/api/admin", tags=["admin"])
+
+
+def human_size(size_bytes: int) -> str:
+    for unit in ["B", "KB", "MB", "GB", "TB"]:
+        if size_bytes < 1024:
+            return f"{size_bytes:.1f} {unit}"
+        size_bytes /= 1024
+    return f"{size_bytes:.1f} PB"
+
+
+@router.get("/videos", response_model=VideoListResponse)
+async def list_videos(
+    page: int = Query(1, ge=1),
+    page_size: int = Query(20, ge=1, le=100),
+    search: str = Query("", max_length=200),
+    status: str = Query("", max_length=20),
+    user: dict = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    query = select(Video).order_by(Video.created_at.desc())
+    count_query = select(func.count(Video.id))
+
+    if search:
+        flt = or_(Video.title.contains(search), Video.url.contains(search))
+        query = query.where(flt)
+        count_query = count_query.where(flt)
+    if status:
+        query = query.where(Video.status == status)
+        count_query = count_query.where(Video.status == status)
+
+    total = (await db.execute(count_query)).scalar() or 0
+    videos = (await db.execute(query.offset((page - 1) * page_size).limit(page_size))).scalars().all()
+
+    return VideoListResponse(
+        videos=[VideoInfo.model_validate(v) for v in videos],
+        total=total,
+        page=page,
+        page_size=page_size,
+    )
+
+
+@router.delete("/videos/{video_id}")
+async def delete_video(video_id: int, user: dict = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
+    video = (await db.execute(select(Video).where(Video.id == video_id))).scalar_one_or_none()
+    if not video:
+        raise HTTPException(status_code=404, detail="Video not found")
+    # Delete file from disk
+    if video.file_path and os.path.exists(video.file_path):
+        os.remove(video.file_path)
+    await db.delete(video)
+    await db.commit()
+    return {"ok": True}
+
+
+@router.get("/stats", response_model=StorageStats)
+async def storage_stats(user: dict = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
+    total = (await db.execute(select(func.count(Video.id)).where(Video.status == "done"))).scalar() or 0
+    total_size = (await db.execute(select(func.sum(Video.file_size)).where(Video.status == "done"))).scalar() or 0
+    return StorageStats(total_videos=total, total_size=total_size, total_size_human=human_size(total_size))

backend/app/routes/auth.py

@@ -0,0 +1,18 @@
+"""Authentication routes."""
+import os
+from fastapi import APIRouter, HTTPException, status
+from app.schemas import LoginRequest, TokenResponse
+from app.auth import create_access_token
+
+router = APIRouter(prefix="/api/auth", tags=["auth"])
+
+ADMIN_USERNAME = os.getenv("ADMIN_USERNAME", "admin")
+ADMIN_PASSWORD = os.getenv("ADMIN_PASSWORD", "admin")
+
+
+@router.post("/login", response_model=TokenResponse)
+async def login(req: LoginRequest):
+    if req.username != ADMIN_USERNAME or req.password != ADMIN_PASSWORD:
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid credentials")
+    token = create_access_token({"sub": req.username})
+    return TokenResponse(access_token=token)

backend/app/routes/download.py

@@ -0,0 +1,120 @@
+"""Download task routes."""
+import uuid
+import os
+import logging
+from fastapi import APIRouter, HTTPException, Depends, BackgroundTasks
+from fastapi.responses import FileResponse, StreamingResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import select
+from app.schemas import DownloadRequest, DownloadResponse, TaskStatus
+from app.database import get_db
+from app.models import Video
+from app.auth import get_current_user, optional_auth
+from app.services.downloader import download_video, get_video_path
+
+logger = logging.getLogger(__name__)
+router = APIRouter(prefix="/api", tags=["download"])
+
+
+async def _do_download(task_id: str, url: str, format_id: str):
+    """Background download task."""
+    from app.database import async_session
+    async with async_session() as db:
+        video = (await db.execute(select(Video).where(Video.task_id == task_id))).scalar_one_or_none()
+        if not video:
+            return
+        try:
+            video.status = "downloading"
+            await db.commit()
+
+            def update_progress(pct):
+                pass  # Progress tracking in sync context is complex; keep simple
+
+            result = download_video(url, format_id, progress_callback=update_progress)
+            video.title = result["title"]
+            video.thumbnail = result["thumbnail"]
+            video.duration = result["duration"]
+            video.filename = result["filename"]
+            video.file_path = result["file_path"]
+            video.file_size = result["file_size"]
+            video.platform = result["platform"]
+            video.status = "done"
+            video.progress = 100
+            await db.commit()
+        except Exception as e:
+            logger.error(f"Download failed for {task_id}: {e}")
+            video.status = "error"
+            video.error_message = str(e)[:500]
+            await db.commit()
+
+
+@router.post("/download", response_model=DownloadResponse)
+async def start_download(req: DownloadRequest, background_tasks: BackgroundTasks, db: AsyncSession = Depends(get_db)):
+    task_id = str(uuid.uuid4())[:8]
+    video = Video(task_id=task_id, url=req.url, quality=req.quality, format_id=req.format_id, status="pending")
+    db.add(video)
+    await db.commit()
+    background_tasks.add_task(_do_download, task_id, req.url, req.format_id)
+    return DownloadResponse(task_id=task_id, status="pending")
+
+
+@router.get("/download/{task_id}", response_model=TaskStatus)
+async def get_download_status(task_id: str, db: AsyncSession = Depends(get_db)):
+    video = (await db.execute(select(Video).where(Video.task_id == task_id))).scalar_one_or_none()
+    if not video:
+        raise HTTPException(status_code=404, detail="Task not found")
+    return TaskStatus(
+        task_id=video.task_id,
+        status=video.status,
+        progress=video.progress,
+        title=video.title,
+        error_message=video.error_message or "",
+        video_id=video.id if video.status == "done" else None,
+    )
+
+
+@router.get("/file/{video_id}")
+async def download_file(video_id: int, user: dict = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
+    video = (await db.execute(select(Video).where(Video.id == video_id))).scalar_one_or_none()
+    if not video or video.status != "done":
+        raise HTTPException(status_code=404, detail="Video not found")
+    if not os.path.exists(video.file_path):
+        raise HTTPException(status_code=404, detail="File not found on disk")
+    return FileResponse(video.file_path, filename=video.filename, media_type="video/mp4")
+
+
+@router.get("/stream/{video_id}")
+async def stream_video(video_id: int, token: str = None, user: dict = Depends(optional_auth), db: AsyncSession = Depends(get_db)):
+    # Allow token via query param for video player
+    if not user and token:
+        from app.auth import verify_token
+        user = verify_token(token)
+    if not user:
+        from fastapi import HTTPException, status
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authentication required")
+    video = (await db.execute(select(Video).where(Video.id == video_id))).scalar_one_or_none()
+    if not video or video.status != "done":
+        raise HTTPException(status_code=404, detail="Video not found")
+    if not os.path.exists(video.file_path):
+        raise HTTPException(status_code=404, detail="File not found on disk")
+
+    def iter_file():
+        with open(video.file_path, "rb") as f:
+            while chunk := f.read(1024 * 1024):
+                yield chunk
+
+    return StreamingResponse(iter_file(), media_type="video/mp4", headers={
+        "Content-Disposition": f"inline; filename={video.filename}",
+        "Content-Length": str(video.file_size),
+    })
+
+
+@router.get("/file/task/{task_id}")
+async def download_file_by_task(task_id: str, db: AsyncSession = Depends(get_db)):
+    """Download file by task_id - no auth required (public download)."""
+    video = (await db.execute(select(Video).where(Video.task_id == task_id))).scalar_one_or_none()
+    if not video or video.status != "done":
+        raise HTTPException(status_code=404, detail="Video not found")
+    if not os.path.exists(video.file_path):
+        raise HTTPException(status_code=404, detail="File not found on disk")
+    return FileResponse(video.file_path, filename=video.filename, media_type="video/mp4")

backend/app/routes/parse.py

@@ -0,0 +1,21 @@
+"""Video URL parsing routes."""
+from fastapi import APIRouter, HTTPException
+from app.schemas import ParseRequest, ParseResponse, FormatInfo
+from app.services.downloader import parse_video_url
+
+router = APIRouter(prefix="/api", tags=["parse"])
+
+
+@router.post("/parse", response_model=ParseResponse)
+async def parse_url(req: ParseRequest):
+    try:
+        info = parse_video_url(req.url)
+        return ParseResponse(
+            title=info["title"],
+            thumbnail=info["thumbnail"],
+            duration=info["duration"],
+            formats=[FormatInfo(**f) for f in info["formats"]],
+            url=info["url"],
+        )
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"Failed to parse URL: {str(e)}")

backend/app/schemas.py

@@ -0,0 +1,85 @@
+"""Pydantic schemas for request/response validation."""
+from pydantic import BaseModel
+from typing import Optional
+from datetime import datetime
+
+
+class ParseRequest(BaseModel):
+    url: str
+
+
+class FormatInfo(BaseModel):
+    format_id: str
+    quality: str
+    ext: str
+    filesize: Optional[int] = None
+    note: str = ""
+
+
+class ParseResponse(BaseModel):
+    title: str
+    thumbnail: str
+    duration: int
+    formats: list[FormatInfo]
+    url: str
+
+
+class DownloadRequest(BaseModel):
+    url: str
+    format_id: str = "best"
+    quality: str = ""
+
+
+class DownloadResponse(BaseModel):
+    task_id: str
+    status: str
+
+
+class TaskStatus(BaseModel):
+    task_id: str
+    status: str
+    progress: int
+    title: str = ""
+    error_message: str = ""
+    video_id: Optional[int] = None
+
+
+class VideoInfo(BaseModel):
+    id: int
+    task_id: str
+    url: str
+    title: str
+    platform: str
+    thumbnail: str
+    quality: str
+    filename: str
+    file_size: int
+    duration: int
+    status: str
+    created_at: datetime
+
+    class Config:
+        from_attributes = True
+
+
+class VideoListResponse(BaseModel):
+    videos: list[VideoInfo]
+    total: int
+    page: int
+    page_size: int
+
+
+class StorageStats(BaseModel):
+    total_videos: int
+    total_size: int
+    total_size_human: str
+
+
+class LoginRequest(BaseModel):
+    username: str
+    password: str
+
+
+class TokenResponse(BaseModel):
+    access_token: str
+    token_type: str = "bearer"

backend/app/services/downloader.py

@@ -0,0 +1,120 @@
+"""yt-dlp wrapper service for video downloading."""
+import os
+import uuid
+import asyncio
+import logging
+from pathlib import Path
+from typing import Optional
+import yt_dlp
+
+logger = logging.getLogger(__name__)
+
+VIDEO_BASE_PATH = os.getenv("VIDEO_BASE_PATH", "/home/xdl/xdl_videos")
+X_VIDEOS_PATH = os.path.join(VIDEO_BASE_PATH, "x_videos")
+
+# Ensure directories exist
+os.makedirs(X_VIDEOS_PATH, exist_ok=True)
+
+
+def get_video_path(filename: str) -> str:
+    return os.path.join(X_VIDEOS_PATH, filename)
+
+
+def parse_video_url(url: str) -> dict:
+    """Extract video info without downloading."""
+    ydl_opts = {
+        "quiet": True,
+        "no_warnings": True,
+        "extract_flat": False,
+        "skip_download": True,
+    }
+    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+        info = ydl.extract_info(url, download=False)
+
+    formats = []
+    seen = set()
+    for f in info.get("formats", []):
+        # Only video formats with both video and audio, or video-only
+        if f.get("vcodec", "none") == "none":
+            continue
+        height = f.get("height", 0)
+        ext = f.get("ext", "mp4")
+        fmt_id = f.get("format_id", "")
+        quality = f"{height}p" if height else f.get("format_note", "unknown")
+        key = f"{quality}-{ext}"
+        if key in seen:
+            continue
+        seen.add(key)
+        formats.append({
+            "format_id": fmt_id,
+            "quality": quality,
+            "ext": ext,
+            "filesize": f.get("filesize") or f.get("filesize_approx") or 0,
+            "note": f.get("format_note", ""),
+        })
+
+    # Sort by resolution descending
+    formats.sort(key=lambda x: int(x["quality"].replace("p", "")) if x["quality"].endswith("p") else 0, reverse=True)
+
+    # Add a "best" option
+    formats.insert(0, {
+        "format_id": "best",
+        "quality": "best",
+        "ext": "mp4",
+        "filesize": 0,
+        "note": "Best available quality",
+    })
+
+    return {
+        "title": info.get("title", "Untitled"),
+        "thumbnail": info.get("thumbnail", ""),
+        "duration": info.get("duration", 0) or 0,
+        "formats": formats,
+        "url": url,
+    }
+
+
+def download_video(url: str, format_id: str = "best", progress_callback=None) -> dict:
+    """Download video and return file info."""
+    task_id = str(uuid.uuid4())[:8]
+    output_template = os.path.join(X_VIDEOS_PATH, f"%(id)s_{task_id}.%(ext)s")
+
+    format_spec = "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best" if format_id == "best" else f"{format_id}+bestaudio/best"
+
+    def hook(d):
+        if d["status"] == "downloading" and progress_callback:
+            total = d.get("total_bytes") or d.get("total_bytes_estimate") or 0
+            downloaded = d.get("downloaded_bytes", 0)
+            pct = int(downloaded * 100 / total) if total > 0 else 0
+            progress_callback(pct)
+        elif d["status"] == "finished" and progress_callback:
+            progress_callback(100)
+
+    ydl_opts = {
+        "format": format_spec,
+        "outtmpl": output_template,
+        "merge_output_format": "mp4",
+        "quiet": True,
+        "no_warnings": True,
+        "progress_hooks": [hook],
+    }
+
+    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+        info = ydl.extract_info(url, download=True)
+        filename = ydl.prepare_filename(info)
+        # yt-dlp may change extension after merge
+        if not os.path.exists(filename):
+            base = os.path.splitext(filename)[0]
+            filename = base + ".mp4"
+
+    file_size = os.path.getsize(filename) if os.path.exists(filename) else 0
+
+    return {
+        "title": info.get("title", "Untitled"),
+        "thumbnail": info.get("thumbnail", ""),
+        "duration": info.get("duration", 0) or 0,
+        "filename": os.path.basename(filename),
+        "file_path": filename,
+        "file_size": file_size,
+        "platform": "twitter",
+    }