feat: auto cleanup with retention period, storage limit, settings UI

This commit is contained in:
mini
2026-02-18 23:47:33 +08:00
parent 5bc7f8d1df
commit f106763723
7 changed files with 434 additions and 2 deletions

View File

@@ -42,3 +42,16 @@ async def init_db():
await conn.execute(text(col_def))
except Exception:
pass # Column already exists
# Seed default cleanup settings (only if not already set)
defaults = {
"cleanup_enabled": "true",
"cleanup_retention_minutes": "10080", # 7 days
"cleanup_storage_limit_pct": "80",
"cleanup_last_run": "",
"cleanup_last_result": "",
}
for k, v in defaults.items():
await conn.execute(text(
"INSERT OR IGNORE INTO app_settings (key, value, updated_at) VALUES (:k, :v, datetime('now'))"
), {"k": k, "v": v})

View File

@@ -1,4 +1,5 @@
"""XDL - Twitter/X Video Downloader API."""
import asyncio
import os
from contextlib import asynccontextmanager
from dotenv import load_dotenv
@@ -9,12 +10,19 @@ from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from app.database import init_db
from app.routes import auth, parse, download, admin
from app.services.cleanup import cleanup_loop
@asynccontextmanager
async def lifespan(app: FastAPI):
await init_db()
task = asyncio.create_task(cleanup_loop())
yield
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
app = FastAPI(title="XDL - Video Downloader", version="1.0.0", lifespan=lifespan)

View File

@@ -48,3 +48,11 @@ class DownloadLog(Base):
downloaded_at = Column(DateTime, default=datetime.utcnow, index=True)
video = relationship("Video", back_populates="logs")
class AppSetting(Base):
__tablename__ = "app_settings"
key = Column(String(64), primary_key=True)
value = Column(Text, default="")
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)

View File

@@ -1,12 +1,18 @@
"""Admin management routes."""
import json
import os
from fastapi import APIRouter, HTTPException, Depends, Query
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, func, or_
from app.database import get_db
from app.models import Video, DownloadLog
from app.schemas import VideoInfo, VideoListResponse, StorageStats, DownloadLogInfo, DownloadLogListResponse
from app.models import Video, DownloadLog, AppSetting
from app.schemas import (
VideoInfo, VideoListResponse, StorageStats,
DownloadLogInfo, DownloadLogListResponse,
CleanupConfig, CleanupStatus, DiskStats,
)
from app.auth import get_current_user
from app.services.cleanup import get_setting, set_setting, disk_stats, run_cleanup
router = APIRouter(prefix="/api/admin", tags=["admin"])
@@ -100,3 +106,37 @@ async def download_logs(
items.append(d)
return DownloadLogListResponse(logs=items, total=total, page=page, page_size=page_size)
@router.get("/settings/cleanup", response_model=CleanupStatus)
async def get_cleanup_settings(user: dict = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
video_base = os.getenv("VIDEO_BASE_PATH", "/home/xdl/xdl_videos")
last_result_raw = await get_setting(db, "cleanup_last_result", "{}")
try:
last_result = json.loads(last_result_raw) if last_result_raw else {}
except Exception:
last_result = {}
return CleanupStatus(
config=CleanupConfig(
enabled=(await get_setting(db, "cleanup_enabled", "true")) == "true",
retention_minutes=int(await get_setting(db, "cleanup_retention_minutes", "10080")),
storage_limit_pct=int(await get_setting(db, "cleanup_storage_limit_pct", "80")),
),
disk=DiskStats(**disk_stats(video_base)),
last_run=await get_setting(db, "cleanup_last_run", ""),
last_result=last_result,
)
@router.put("/settings/cleanup", response_model=CleanupStatus)
async def update_cleanup_settings(cfg: CleanupConfig, user: dict = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
await set_setting(db, "cleanup_enabled", "true" if cfg.enabled else "false")
await set_setting(db, "cleanup_retention_minutes", str(cfg.retention_minutes))
await set_setting(db, "cleanup_storage_limit_pct", str(cfg.storage_limit_pct))
return await get_cleanup_settings(user=user, db=db)
@router.post("/cleanup/run")
async def trigger_cleanup(user: dict = Depends(get_current_user)):
result = await run_cleanup()
return result

View File

@@ -109,3 +109,23 @@ class DownloadLogListResponse(BaseModel):
total: int
page: int
page_size: int
class CleanupConfig(BaseModel):
enabled: bool = True
retention_minutes: int = 10080 # 7 days
storage_limit_pct: int = 80
class DiskStats(BaseModel):
total: int
used: int
free: int
used_pct: float
class CleanupStatus(BaseModel):
config: CleanupConfig
disk: DiskStats
last_run: str = ""
last_result: dict = {}

View File

@@ -0,0 +1,146 @@
"""Scheduled video cleanup service."""
import asyncio
import json
import logging
import os
import shutil
from datetime import datetime, timedelta
from sqlalchemy import select
from app.database import async_session
from app.models import AppSetting, Video
logger = logging.getLogger(__name__)
CHECK_INTERVAL_SECONDS = 60 * 10 # Run check every 10 minutes
# ── Setting helpers ──────────────────────────────────────────────────────────
async def get_setting(db, key: str, default: str = "") -> str:
row = (await db.execute(select(AppSetting).where(AppSetting.key == key))).scalar_one_or_none()
return row.value if row else default
async def set_setting(db, key: str, value: str):
row = (await db.execute(select(AppSetting).where(AppSetting.key == key))).scalar_one_or_none()
if row:
row.value = value
row.updated_at = datetime.utcnow()
else:
db.add(AppSetting(key=key, value=value))
await db.commit()
# ── Disk helpers ─────────────────────────────────────────────────────────────
def disk_stats(path: str) -> dict:
"""Return disk usage stats for the given path."""
try:
usage = shutil.disk_usage(path)
used_pct = round(usage.used / usage.total * 100, 1)
return {
"total": usage.total,
"used": usage.used,
"free": usage.free,
"used_pct": used_pct,
}
except Exception:
return {"total": 0, "used": 0, "free": 0, "used_pct": 0}
def _delete_video_file(video: Video) -> int:
"""Delete file, return bytes freed (0 if file missing)."""
if video.file_path and os.path.exists(video.file_path):
size = video.file_size or 0
try:
os.remove(video.file_path)
except OSError:
pass
return size
return 0
# ── Main cleanup logic ───────────────────────────────────────────────────────
async def run_cleanup() -> dict:
"""Execute cleanup. Returns a stats dict."""
async with async_session() as db:
enabled = await get_setting(db, "cleanup_enabled", "true")
if enabled != "true":
return {"skipped": True, "reason": "disabled", "ran_at": datetime.utcnow().isoformat()}
retention_min = int(await get_setting(db, "cleanup_retention_minutes", "10080"))
storage_limit_pct = int(await get_setting(db, "cleanup_storage_limit_pct", "80"))
video_base = os.getenv("VIDEO_BASE_PATH", "/home/xdl/xdl_videos")
cutoff = datetime.utcnow() - timedelta(minutes=retention_min)
time_deleted = 0
storage_deleted = 0
freed_bytes = 0
# ── Phase 1: time-based cleanup ──────────────────────────────────────
old_videos = (await db.execute(
select(Video)
.where(Video.status == "done", Video.created_at < cutoff)
.order_by(Video.created_at.asc())
)).scalars().all()
for v in old_videos:
freed_bytes += _delete_video_file(v)
v.status = "deleted"
v.file_path = ""
time_deleted += 1
if time_deleted:
await db.commit()
logger.info(f"Cleanup: deleted {time_deleted} expired videos, freed {freed_bytes // 1024 // 1024} MB")
# ── Phase 2: storage limit enforcement ───────────────────────────────
stats = disk_stats(video_base)
if stats["used_pct"] > storage_limit_pct:
remaining = (await db.execute(
select(Video)
.where(Video.status == "done")
.order_by(Video.created_at.asc())
)).scalars().all()
for v in remaining:
stats = disk_stats(video_base)
if stats["used_pct"] <= storage_limit_pct:
break
freed_bytes += _delete_video_file(v)
v.status = "deleted"
v.file_path = ""
storage_deleted += 1
if storage_deleted:
await db.commit()
logger.info(f"Cleanup: storage limit reached, deleted {storage_deleted} extra videos")
ran_at = datetime.utcnow().isoformat()
result = {
"time_deleted": time_deleted,
"storage_deleted": storage_deleted,
"freed_mb": round(freed_bytes / 1024 / 1024, 1),
"disk_used_pct": disk_stats(video_base)["used_pct"],
"ran_at": ran_at,
}
await set_setting(db, "cleanup_last_run", ran_at)
await set_setting(db, "cleanup_last_result", json.dumps(result))
return result
# ── Background loop ──────────────────────────────────────────────────────────
async def cleanup_loop():
"""Long-running background task. Starts after 60s, then every 10 min."""
await asyncio.sleep(60)
while True:
try:
result = await run_cleanup()
logger.info(f"Cleanup finished: {result}")
except Exception as e:
logger.error(f"Cleanup loop error: {e}", exc_info=True)
await asyncio.sleep(CHECK_INTERVAL_SECONDS)

View File

@@ -4,6 +4,7 @@
<div class="tabs">
<button :class="{ active: tab === 'videos' }" @click="tab = 'videos'">📹 Video Library</button>
<button :class="{ active: tab === 'logs' }" @click="tab = 'logs'; fetchLogs()">📋 Download Logs</button>
<button :class="{ active: tab === 'settings' }" @click="tab = 'settings'; fetchCleanup()"> Settings</button>
<div class="stats" v-if="stats">📊 {{ stats.total_videos }} videos · {{ stats.total_size_human }}</div>
</div>
@@ -99,6 +100,83 @@
</div>
</template>
<!-- Settings -->
<template v-if="tab === 'settings'">
<div class="settings-card">
<h3>🗑 Auto Cleanup</h3>
<p class="settings-desc">Automatically delete downloaded videos based on age or disk usage.</p>
<div class="setting-row">
<label>Enable auto cleanup</label>
<label class="toggle">
<input type="checkbox" v-model="cleanup.enabled" />
<span class="slider"></span>
</label>
</div>
<div class="setting-row">
<label>Retention period</label>
<div class="retention-input">
<input type="number" v-model.number="retentionValue" min="1" class="num-input" />
<select v-model="retentionUnit" class="unit-select">
<option value="hours">Hours</option>
<option value="days">Days</option>
<option value="weeks">Weeks</option>
<option value="months">Months</option>
</select>
<span class="minutes-hint">= {{ cleanupMinutes.toLocaleString() }} min</span>
</div>
</div>
<div class="setting-row">
<label>Storage limit</label>
<div class="storage-limit-input">
<input type="number" v-model.number="cleanup.storage_limit_pct" min="10" max="95" class="num-input" />
<span>%</span>
<span class="hint">of total disk</span>
</div>
</div>
<!-- Disk usage bar -->
<div class="disk-section" v-if="cleanupStatus">
<div class="disk-header">
<span>Disk Usage</span>
<span :class="diskClass">{{ cleanupStatus.disk.used_pct }}%</span>
</div>
<div class="disk-bar">
<div class="disk-fill" :style="{ width: cleanupStatus.disk.used_pct + '%' }" :class="diskClass"></div>
<div class="disk-limit-line" :style="{ left: cleanup.storage_limit_pct + '%' }" title="Storage limit"></div>
</div>
<div class="disk-labels">
<span>{{ humanSize(cleanupStatus.disk.used) }} used</span>
<span>{{ humanSize(cleanupStatus.disk.free) }} free</span>
<span>{{ humanSize(cleanupStatus.disk.total) }} total</span>
</div>
</div>
<!-- Last run info -->
<div class="last-run" v-if="cleanupStatus?.last_run">
<span class="last-run-label">Last run:</span>
{{ fmtTime(cleanupStatus.last_run) }}
<span v-if="cleanupStatus.last_result?.time_deleted !== undefined" class="run-result">
deleted {{ cleanupStatus.last_result.time_deleted + (cleanupStatus.last_result.storage_deleted || 0) }} videos,
freed {{ cleanupStatus.last_result.freed_mb }} MB
</span>
</div>
<div class="last-run" v-else-if="cleanupStatus">Never run yet</div>
<div class="settings-actions">
<button class="btn-save" @click="saveCleanup" :disabled="saving">
{{ saving ? '⏳ Saving...' : '💾 Save Settings' }}
</button>
<button class="btn-run" @click="runCleanup" :disabled="running">
{{ running ? '⏳ Running...' : '🗑 Run Now' }}
</button>
</div>
<div v-if="settingsMsg" :class="['settings-msg', settingsMsgType]">{{ settingsMsg }}</div>
</div>
</template>
<!-- Video Player Modal -->
<div v-if="playing" class="modal" @click.self="playing = null">
<div class="player-wrap">
@@ -129,6 +207,75 @@ const playing = ref(null)
const playUrl = ref('')
const totalPages = computed(() => Math.ceil(total.value / pageSize) || 1)
// ── Settings / Cleanup ──
const cleanupStatus = ref(null)
const cleanup = ref({ enabled: true, retention_minutes: 10080, storage_limit_pct: 80 })
const retentionValue = ref(7)
const retentionUnit = ref('days')
const saving = ref(false)
const running = ref(false)
const settingsMsg = ref('')
const settingsMsgType = ref('ok')
const cleanupMinutes = computed(() => {
const m = { hours: 60, days: 1440, weeks: 10080, months: 43200 }
return retentionValue.value * (m[retentionUnit.value] || 1440)
})
const diskClass = computed(() => {
const pct = cleanupStatus.value?.disk?.used_pct || 0
if (pct >= 90) return 'disk-danger'
if (pct >= 75) return 'disk-warn'
return 'disk-ok'
})
function minutesToUnit(min) {
if (min % 43200 === 0) return { value: min / 43200, unit: 'months' }
if (min % 10080 === 0) return { value: min / 10080, unit: 'weeks' }
if (min % 1440 === 0) return { value: min / 1440, unit: 'days' }
return { value: min / 60, unit: 'hours' }
}
async function fetchCleanup() {
try {
const res = await axios.get('/api/admin/settings/cleanup', { headers: auth.getHeaders() })
cleanupStatus.value = res.data
cleanup.value = { ...res.data.config }
const { value, unit } = minutesToUnit(res.data.config.retention_minutes)
retentionValue.value = value
retentionUnit.value = unit
} catch {}
}
async function saveCleanup() {
saving.value = true; settingsMsg.value = ''
try {
cleanup.value.retention_minutes = cleanupMinutes.value
const res = await axios.put('/api/admin/settings/cleanup', cleanup.value, { headers: auth.getHeaders() })
cleanupStatus.value = res.data
settingsMsg.value = '✅ Settings saved'
settingsMsgType.value = 'ok'
} catch { settingsMsg.value = '❌ Save failed'; settingsMsgType.value = 'err' }
finally { saving.value = false; setTimeout(() => settingsMsg.value = '', 3000) }
}
async function runCleanup() {
running.value = true; settingsMsg.value = ''
try {
const res = await axios.post('/api/admin/cleanup/run', {}, { headers: auth.getHeaders() })
const r = res.data
if (r.skipped) {
settingsMsg.value = '⚠️ Cleanup is disabled'
settingsMsgType.value = 'warn'
} else {
settingsMsg.value = `✅ Done — deleted ${(r.time_deleted||0) + (r.storage_deleted||0)} videos, freed ${r.freed_mb} MB`
settingsMsgType.value = 'ok'
}
await fetchCleanup()
} catch { settingsMsg.value = '❌ Run failed'; settingsMsgType.value = 'err' }
finally { running.value = false }
}
// ── Logs ──
const logs = ref([])
const logSearch = ref('')
@@ -325,6 +472,56 @@ onMounted(() => { fetchVideos(); fetchStats() })
.plat-youtube { background: #c00; color: #fff; }
.plat-twitter { background: #1da1f2; color: #fff; }
/* Settings */
.settings-card { background: #1a1a2e; border: 1px solid #2a2a3e; border-radius: 12px; padding: 2rem; max-width: 680px; }
.settings-card h3 { margin-bottom: 0.4rem; font-size: 1.15rem; }
.settings-desc { color: #888; font-size: 0.9rem; margin-bottom: 1.8rem; }
.setting-row { display: flex; align-items: center; justify-content: space-between; margin-bottom: 1.4rem; gap: 1rem; }
.setting-row label:first-child { color: #ccc; font-size: 0.95rem; white-space: nowrap; }
.retention-input, .storage-limit-input { display: flex; align-items: center; gap: 0.6rem; }
.num-input { width: 80px; padding: 0.5rem 0.7rem; border: 1px solid #444; border-radius: 8px; background: #12122a; color: #fff; font-size: 0.95rem; text-align: center; }
.unit-select { padding: 0.5rem 0.7rem; border: 1px solid #444; border-radius: 8px; background: #12122a; color: #fff; font-size: 0.95rem; }
.minutes-hint { color: #666; font-size: 0.82rem; white-space: nowrap; }
.hint { color: #666; font-size: 0.85rem; }
/* Toggle switch */
.toggle { position: relative; display: inline-block; width: 48px; height: 26px; flex-shrink: 0; }
.toggle input { opacity: 0; width: 0; height: 0; }
.slider { position: absolute; cursor: pointer; inset: 0; background: #444; border-radius: 26px; transition: 0.3s; }
.slider:before { position: absolute; content: ""; height: 18px; width: 18px; left: 4px; bottom: 4px; background: #fff; border-radius: 50%; transition: 0.3s; }
.toggle input:checked + .slider { background: #1da1f2; }
.toggle input:checked + .slider:before { transform: translateX(22px); }
/* Disk bar */
.disk-section { margin: 1.4rem 0; }
.disk-header { display: flex; justify-content: space-between; margin-bottom: 0.5rem; font-size: 0.9rem; color: #aaa; }
.disk-bar { position: relative; height: 12px; background: #333; border-radius: 6px; overflow: visible; margin-bottom: 0.5rem; }
.disk-fill { height: 100%; border-radius: 6px; transition: width 0.4s; }
.disk-limit-line { position: absolute; top: -4px; bottom: -4px; width: 2px; background: #f39c12; border-radius: 2px; transform: translateX(-50%); }
.disk-labels { display: flex; justify-content: space-between; font-size: 0.8rem; color: #666; }
.disk-ok { color: #27ae60; } .disk-fill.disk-ok { background: #27ae60; }
.disk-warn { color: #f39c12; } .disk-fill.disk-warn { background: #f39c12; }
.disk-danger { color: #e74c3c; } .disk-fill.disk-danger { background: #e74c3c; }
/* Last run */
.last-run { font-size: 0.88rem; color: #888; margin-bottom: 1.4rem; }
.last-run-label { color: #666; margin-right: 0.3rem; }
.run-result { color: #27ae60; }
/* Action buttons */
.settings-actions { display: flex; gap: 0.8rem; flex-wrap: wrap; }
.btn-save, .btn-run {
padding: 0.7rem 1.4rem; border: none; border-radius: 8px; cursor: pointer;
font-size: 0.95rem; font-weight: 600; transition: opacity 0.2s;
}
.btn-save { background: #1da1f2; color: #fff; }
.btn-run { background: #e74c3c; color: #fff; }
.btn-save:disabled, .btn-run:disabled { opacity: 0.5; cursor: not-allowed; }
.settings-msg { margin-top: 0.8rem; font-size: 0.9rem; padding: 0.5rem 0; }
.settings-msg.ok { color: #27ae60; }
.settings-msg.warn { color: #f39c12; }
.settings-msg.err { color: #e74c3c; }
.empty { text-align: center; color: #666; padding: 3rem; }
.pagination { display: flex; justify-content: center; align-items: center; gap: 1rem; margin-top: 1.5rem; }
.pagination button { padding: 0.5rem 1rem; border: 1px solid #444; border-radius: 6px; background: transparent; color: #fff; cursor: pointer; }