diff --git a/backend/app/database.py b/backend/app/database.py index 98f09c8..f2e8d91 100644 --- a/backend/app/database.py +++ b/backend/app/database.py @@ -42,3 +42,16 @@ async def init_db(): await conn.execute(text(col_def)) except Exception: pass # Column already exists + + # Seed default cleanup settings (only if not already set) + defaults = { + "cleanup_enabled": "true", + "cleanup_retention_minutes": "10080", # 7 days + "cleanup_storage_limit_pct": "80", + "cleanup_last_run": "", + "cleanup_last_result": "", + } + for k, v in defaults.items(): + await conn.execute(text( + "INSERT OR IGNORE INTO app_settings (key, value, updated_at) VALUES (:k, :v, datetime('now'))" + ), {"k": k, "v": v}) diff --git a/backend/app/main.py b/backend/app/main.py index 4dd892e..54ff2be 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,4 +1,5 @@ """XDL - Twitter/X Video Downloader API.""" +import asyncio import os from contextlib import asynccontextmanager from dotenv import load_dotenv @@ -9,12 +10,19 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from app.database import init_db from app.routes import auth, parse, download, admin +from app.services.cleanup import cleanup_loop @asynccontextmanager async def lifespan(app: FastAPI): await init_db() + task = asyncio.create_task(cleanup_loop()) yield + task.cancel() + try: + await task + except asyncio.CancelledError: + pass app = FastAPI(title="XDL - Video Downloader", version="1.0.0", lifespan=lifespan) diff --git a/backend/app/models.py b/backend/app/models.py index 7990255..886d7f7 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -48,3 +48,11 @@ class DownloadLog(Base): downloaded_at = Column(DateTime, default=datetime.utcnow, index=True) video = relationship("Video", back_populates="logs") + + +class AppSetting(Base): + __tablename__ = "app_settings" + + key = Column(String(64), primary_key=True) + value = Column(Text, default="") + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) diff --git a/backend/app/routes/admin.py b/backend/app/routes/admin.py index d395397..7afdbef 100644 --- a/backend/app/routes/admin.py +++ b/backend/app/routes/admin.py @@ -1,12 +1,18 @@ """Admin management routes.""" +import json import os from fastapi import APIRouter, HTTPException, Depends, Query from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, func, or_ from app.database import get_db -from app.models import Video, DownloadLog -from app.schemas import VideoInfo, VideoListResponse, StorageStats, DownloadLogInfo, DownloadLogListResponse +from app.models import Video, DownloadLog, AppSetting +from app.schemas import ( + VideoInfo, VideoListResponse, StorageStats, + DownloadLogInfo, DownloadLogListResponse, + CleanupConfig, CleanupStatus, DiskStats, +) from app.auth import get_current_user +from app.services.cleanup import get_setting, set_setting, disk_stats, run_cleanup router = APIRouter(prefix="/api/admin", tags=["admin"]) @@ -100,3 +106,37 @@ async def download_logs( items.append(d) return DownloadLogListResponse(logs=items, total=total, page=page, page_size=page_size) + + +@router.get("/settings/cleanup", response_model=CleanupStatus) +async def get_cleanup_settings(user: dict = Depends(get_current_user), db: AsyncSession = Depends(get_db)): + video_base = os.getenv("VIDEO_BASE_PATH", "/home/xdl/xdl_videos") + last_result_raw = await get_setting(db, "cleanup_last_result", "{}") + try: + last_result = json.loads(last_result_raw) if last_result_raw else {} + except Exception: + last_result = {} + return CleanupStatus( + config=CleanupConfig( + enabled=(await get_setting(db, "cleanup_enabled", "true")) == "true", + retention_minutes=int(await get_setting(db, "cleanup_retention_minutes", "10080")), + storage_limit_pct=int(await get_setting(db, "cleanup_storage_limit_pct", "80")), + ), + disk=DiskStats(**disk_stats(video_base)), + last_run=await get_setting(db, "cleanup_last_run", ""), + last_result=last_result, + ) + + +@router.put("/settings/cleanup", response_model=CleanupStatus) +async def update_cleanup_settings(cfg: CleanupConfig, user: dict = Depends(get_current_user), db: AsyncSession = Depends(get_db)): + await set_setting(db, "cleanup_enabled", "true" if cfg.enabled else "false") + await set_setting(db, "cleanup_retention_minutes", str(cfg.retention_minutes)) + await set_setting(db, "cleanup_storage_limit_pct", str(cfg.storage_limit_pct)) + return await get_cleanup_settings(user=user, db=db) + + +@router.post("/cleanup/run") +async def trigger_cleanup(user: dict = Depends(get_current_user)): + result = await run_cleanup() + return result diff --git a/backend/app/schemas.py b/backend/app/schemas.py index ebb963a..c06e017 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -109,3 +109,23 @@ class DownloadLogListResponse(BaseModel): total: int page: int page_size: int + + +class CleanupConfig(BaseModel): + enabled: bool = True + retention_minutes: int = 10080 # 7 days + storage_limit_pct: int = 80 + + +class DiskStats(BaseModel): + total: int + used: int + free: int + used_pct: float + + +class CleanupStatus(BaseModel): + config: CleanupConfig + disk: DiskStats + last_run: str = "" + last_result: dict = {} diff --git a/backend/app/services/cleanup.py b/backend/app/services/cleanup.py new file mode 100644 index 0000000..afc4f8b --- /dev/null +++ b/backend/app/services/cleanup.py @@ -0,0 +1,146 @@ +"""Scheduled video cleanup service.""" +import asyncio +import json +import logging +import os +import shutil +from datetime import datetime, timedelta + +from sqlalchemy import select + +from app.database import async_session +from app.models import AppSetting, Video + +logger = logging.getLogger(__name__) + +CHECK_INTERVAL_SECONDS = 60 * 10 # Run check every 10 minutes + + +# ── Setting helpers ────────────────────────────────────────────────────────── + +async def get_setting(db, key: str, default: str = "") -> str: + row = (await db.execute(select(AppSetting).where(AppSetting.key == key))).scalar_one_or_none() + return row.value if row else default + + +async def set_setting(db, key: str, value: str): + row = (await db.execute(select(AppSetting).where(AppSetting.key == key))).scalar_one_or_none() + if row: + row.value = value + row.updated_at = datetime.utcnow() + else: + db.add(AppSetting(key=key, value=value)) + await db.commit() + + +# ── Disk helpers ───────────────────────────────────────────────────────────── + +def disk_stats(path: str) -> dict: + """Return disk usage stats for the given path.""" + try: + usage = shutil.disk_usage(path) + used_pct = round(usage.used / usage.total * 100, 1) + return { + "total": usage.total, + "used": usage.used, + "free": usage.free, + "used_pct": used_pct, + } + except Exception: + return {"total": 0, "used": 0, "free": 0, "used_pct": 0} + + +def _delete_video_file(video: Video) -> int: + """Delete file, return bytes freed (0 if file missing).""" + if video.file_path and os.path.exists(video.file_path): + size = video.file_size or 0 + try: + os.remove(video.file_path) + except OSError: + pass + return size + return 0 + + +# ── Main cleanup logic ─────────────────────────────────────────────────────── + +async def run_cleanup() -> dict: + """Execute cleanup. Returns a stats dict.""" + async with async_session() as db: + enabled = await get_setting(db, "cleanup_enabled", "true") + if enabled != "true": + return {"skipped": True, "reason": "disabled", "ran_at": datetime.utcnow().isoformat()} + + retention_min = int(await get_setting(db, "cleanup_retention_minutes", "10080")) + storage_limit_pct = int(await get_setting(db, "cleanup_storage_limit_pct", "80")) + video_base = os.getenv("VIDEO_BASE_PATH", "/home/xdl/xdl_videos") + + cutoff = datetime.utcnow() - timedelta(minutes=retention_min) + time_deleted = 0 + storage_deleted = 0 + freed_bytes = 0 + + # ── Phase 1: time-based cleanup ────────────────────────────────────── + old_videos = (await db.execute( + select(Video) + .where(Video.status == "done", Video.created_at < cutoff) + .order_by(Video.created_at.asc()) + )).scalars().all() + + for v in old_videos: + freed_bytes += _delete_video_file(v) + v.status = "deleted" + v.file_path = "" + time_deleted += 1 + + if time_deleted: + await db.commit() + logger.info(f"Cleanup: deleted {time_deleted} expired videos, freed {freed_bytes // 1024 // 1024} MB") + + # ── Phase 2: storage limit enforcement ─────────────────────────────── + stats = disk_stats(video_base) + if stats["used_pct"] > storage_limit_pct: + remaining = (await db.execute( + select(Video) + .where(Video.status == "done") + .order_by(Video.created_at.asc()) + )).scalars().all() + + for v in remaining: + stats = disk_stats(video_base) + if stats["used_pct"] <= storage_limit_pct: + break + freed_bytes += _delete_video_file(v) + v.status = "deleted" + v.file_path = "" + storage_deleted += 1 + + if storage_deleted: + await db.commit() + logger.info(f"Cleanup: storage limit reached, deleted {storage_deleted} extra videos") + + ran_at = datetime.utcnow().isoformat() + result = { + "time_deleted": time_deleted, + "storage_deleted": storage_deleted, + "freed_mb": round(freed_bytes / 1024 / 1024, 1), + "disk_used_pct": disk_stats(video_base)["used_pct"], + "ran_at": ran_at, + } + await set_setting(db, "cleanup_last_run", ran_at) + await set_setting(db, "cleanup_last_result", json.dumps(result)) + return result + + +# ── Background loop ────────────────────────────────────────────────────────── + +async def cleanup_loop(): + """Long-running background task. Starts after 60s, then every 10 min.""" + await asyncio.sleep(60) + while True: + try: + result = await run_cleanup() + logger.info(f"Cleanup finished: {result}") + except Exception as e: + logger.error(f"Cleanup loop error: {e}", exc_info=True) + await asyncio.sleep(CHECK_INTERVAL_SECONDS) diff --git a/frontend/src/views/Admin.vue b/frontend/src/views/Admin.vue index 2cad36a..fefa0c9 100644 --- a/frontend/src/views/Admin.vue +++ b/frontend/src/views/Admin.vue @@ -4,6 +4,7 @@
+
📊 {{ stats.total_videos }} videos · {{ stats.total_size_human }}
@@ -99,6 +100,83 @@ + + +
@@ -129,6 +207,75 @@ const playing = ref(null) const playUrl = ref('') const totalPages = computed(() => Math.ceil(total.value / pageSize) || 1) +// ── Settings / Cleanup ── +const cleanupStatus = ref(null) +const cleanup = ref({ enabled: true, retention_minutes: 10080, storage_limit_pct: 80 }) +const retentionValue = ref(7) +const retentionUnit = ref('days') +const saving = ref(false) +const running = ref(false) +const settingsMsg = ref('') +const settingsMsgType = ref('ok') + +const cleanupMinutes = computed(() => { + const m = { hours: 60, days: 1440, weeks: 10080, months: 43200 } + return retentionValue.value * (m[retentionUnit.value] || 1440) +}) + +const diskClass = computed(() => { + const pct = cleanupStatus.value?.disk?.used_pct || 0 + if (pct >= 90) return 'disk-danger' + if (pct >= 75) return 'disk-warn' + return 'disk-ok' +}) + +function minutesToUnit(min) { + if (min % 43200 === 0) return { value: min / 43200, unit: 'months' } + if (min % 10080 === 0) return { value: min / 10080, unit: 'weeks' } + if (min % 1440 === 0) return { value: min / 1440, unit: 'days' } + return { value: min / 60, unit: 'hours' } +} + +async function fetchCleanup() { + try { + const res = await axios.get('/api/admin/settings/cleanup', { headers: auth.getHeaders() }) + cleanupStatus.value = res.data + cleanup.value = { ...res.data.config } + const { value, unit } = minutesToUnit(res.data.config.retention_minutes) + retentionValue.value = value + retentionUnit.value = unit + } catch {} +} + +async function saveCleanup() { + saving.value = true; settingsMsg.value = '' + try { + cleanup.value.retention_minutes = cleanupMinutes.value + const res = await axios.put('/api/admin/settings/cleanup', cleanup.value, { headers: auth.getHeaders() }) + cleanupStatus.value = res.data + settingsMsg.value = '✅ Settings saved' + settingsMsgType.value = 'ok' + } catch { settingsMsg.value = '❌ Save failed'; settingsMsgType.value = 'err' } + finally { saving.value = false; setTimeout(() => settingsMsg.value = '', 3000) } +} + +async function runCleanup() { + running.value = true; settingsMsg.value = '' + try { + const res = await axios.post('/api/admin/cleanup/run', {}, { headers: auth.getHeaders() }) + const r = res.data + if (r.skipped) { + settingsMsg.value = '⚠️ Cleanup is disabled' + settingsMsgType.value = 'warn' + } else { + settingsMsg.value = `✅ Done — deleted ${(r.time_deleted||0) + (r.storage_deleted||0)} videos, freed ${r.freed_mb} MB` + settingsMsgType.value = 'ok' + } + await fetchCleanup() + } catch { settingsMsg.value = '❌ Run failed'; settingsMsgType.value = 'err' } + finally { running.value = false } +} + // ── Logs ── const logs = ref([]) const logSearch = ref('') @@ -325,6 +472,56 @@ onMounted(() => { fetchVideos(); fetchStats() }) .plat-youtube { background: #c00; color: #fff; } .plat-twitter { background: #1da1f2; color: #fff; } +/* Settings */ +.settings-card { background: #1a1a2e; border: 1px solid #2a2a3e; border-radius: 12px; padding: 2rem; max-width: 680px; } +.settings-card h3 { margin-bottom: 0.4rem; font-size: 1.15rem; } +.settings-desc { color: #888; font-size: 0.9rem; margin-bottom: 1.8rem; } +.setting-row { display: flex; align-items: center; justify-content: space-between; margin-bottom: 1.4rem; gap: 1rem; } +.setting-row label:first-child { color: #ccc; font-size: 0.95rem; white-space: nowrap; } +.retention-input, .storage-limit-input { display: flex; align-items: center; gap: 0.6rem; } +.num-input { width: 80px; padding: 0.5rem 0.7rem; border: 1px solid #444; border-radius: 8px; background: #12122a; color: #fff; font-size: 0.95rem; text-align: center; } +.unit-select { padding: 0.5rem 0.7rem; border: 1px solid #444; border-radius: 8px; background: #12122a; color: #fff; font-size: 0.95rem; } +.minutes-hint { color: #666; font-size: 0.82rem; white-space: nowrap; } +.hint { color: #666; font-size: 0.85rem; } + +/* Toggle switch */ +.toggle { position: relative; display: inline-block; width: 48px; height: 26px; flex-shrink: 0; } +.toggle input { opacity: 0; width: 0; height: 0; } +.slider { position: absolute; cursor: pointer; inset: 0; background: #444; border-radius: 26px; transition: 0.3s; } +.slider:before { position: absolute; content: ""; height: 18px; width: 18px; left: 4px; bottom: 4px; background: #fff; border-radius: 50%; transition: 0.3s; } +.toggle input:checked + .slider { background: #1da1f2; } +.toggle input:checked + .slider:before { transform: translateX(22px); } + +/* Disk bar */ +.disk-section { margin: 1.4rem 0; } +.disk-header { display: flex; justify-content: space-between; margin-bottom: 0.5rem; font-size: 0.9rem; color: #aaa; } +.disk-bar { position: relative; height: 12px; background: #333; border-radius: 6px; overflow: visible; margin-bottom: 0.5rem; } +.disk-fill { height: 100%; border-radius: 6px; transition: width 0.4s; } +.disk-limit-line { position: absolute; top: -4px; bottom: -4px; width: 2px; background: #f39c12; border-radius: 2px; transform: translateX(-50%); } +.disk-labels { display: flex; justify-content: space-between; font-size: 0.8rem; color: #666; } +.disk-ok { color: #27ae60; } .disk-fill.disk-ok { background: #27ae60; } +.disk-warn { color: #f39c12; } .disk-fill.disk-warn { background: #f39c12; } +.disk-danger { color: #e74c3c; } .disk-fill.disk-danger { background: #e74c3c; } + +/* Last run */ +.last-run { font-size: 0.88rem; color: #888; margin-bottom: 1.4rem; } +.last-run-label { color: #666; margin-right: 0.3rem; } +.run-result { color: #27ae60; } + +/* Action buttons */ +.settings-actions { display: flex; gap: 0.8rem; flex-wrap: wrap; } +.btn-save, .btn-run { + padding: 0.7rem 1.4rem; border: none; border-radius: 8px; cursor: pointer; + font-size: 0.95rem; font-weight: 600; transition: opacity 0.2s; +} +.btn-save { background: #1da1f2; color: #fff; } +.btn-run { background: #e74c3c; color: #fff; } +.btn-save:disabled, .btn-run:disabled { opacity: 0.5; cursor: not-allowed; } +.settings-msg { margin-top: 0.8rem; font-size: 0.9rem; padding: 0.5rem 0; } +.settings-msg.ok { color: #27ae60; } +.settings-msg.warn { color: #f39c12; } +.settings-msg.err { color: #e74c3c; } + .empty { text-align: center; color: #666; padding: 3rem; } .pagination { display: flex; justify-content: center; align-items: center; gap: 1rem; margin-top: 1.5rem; } .pagination button { padding: 0.5rem 1rem; border: 1px solid #444; border-radius: 6px; background: transparent; color: #fff; cursor: pointer; }