Initial commit: RSS platform phase 1 skeleton with code review fixes
Features: - FastAPI + SQLAlchemy 2.0 async + PostgreSQL/pgvector + Redis backend - Vue 3 + TypeScript + Element Plus frontend - JWT auth with access/refresh tokens and revocation - Admin/member RBAC - RSS feed CRUD and article listing - Settings management with Fernet encryption for sensitive values - Redis distributed lock service - Alembic initial migration - Docker Compose development environment Fixes from code review: - Fix DB session leak in dependency injection - Restrict registration to admin only - Add default admin password warning - Implement JWT refresh tokens and jti blacklist - Strengthen password policy - Use func.count for pagination totals - Replace NullPool with AsyncAdaptedQueuePool - Remove init_db from lifespan to enforce alembic migrations - Add request_id middleware and logging filter - Fix vite.config.ts env loading - Add frontend token refresh interceptor - Add Vue error handler Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
congsh
@@ -0,0 +1,153 @@
|
||||
"""Distributed lock service with Redis and DB fallback."""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.database import AsyncSessionLocal
|
||||
from app.core.logging import get_logger
|
||||
from app.core.redis import get_redis
|
||||
from app.models.lock import Lock
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class LockService:
|
||||
"""Distributed lock service."""
|
||||
|
||||
def __init__(self, owner_id: str | None = None):
|
||||
self.owner_id = owner_id or str(uuid4())
|
||||
|
||||
async def acquire(self, lock_name: str, ttl: int = 60) -> bool:
|
||||
"""Acquire a lock with given TTL in seconds."""
|
||||
# Try Redis first
|
||||
try:
|
||||
redis = await get_redis()
|
||||
acquired = await redis.set(lock_name, self.owner_id, nx=True, ex=ttl)
|
||||
if acquired:
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("Redis lock failed, falling back to DB: %s", exc)
|
||||
|
||||
# Fallback to DB
|
||||
return await self._acquire_db(lock_name, ttl)
|
||||
|
||||
async def release(self, lock_name: str) -> bool:
|
||||
"""Release a lock."""
|
||||
# Try Redis first
|
||||
try:
|
||||
redis = await get_redis()
|
||||
# Only release if we own it
|
||||
current_owner = await redis.get(lock_name)
|
||||
if current_owner == self.owner_id:
|
||||
await redis.delete(lock_name)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("Redis unlock failed, falling back to DB: %s", exc)
|
||||
|
||||
return await self._release_db(lock_name)
|
||||
|
||||
async def extend(self, lock_name: str, ttl: int = 60) -> bool:
|
||||
"""Extend lock TTL."""
|
||||
try:
|
||||
redis = await get_redis()
|
||||
current_owner = await redis.get(lock_name)
|
||||
if current_owner == self.owner_id:
|
||||
await redis.expire(lock_name, ttl)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("Redis extend failed: %s", exc)
|
||||
|
||||
return await self._extend_db(lock_name, ttl)
|
||||
|
||||
async def is_locked(self, lock_name: str) -> bool:
|
||||
"""Check if a lock is held."""
|
||||
try:
|
||||
redis = await get_redis()
|
||||
exists = await redis.exists(lock_name)
|
||||
if exists:
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await db.execute(select(Lock).where(Lock.lock_name == lock_name))
|
||||
lock = result.scalar_one_or_none()
|
||||
if not lock:
|
||||
return False
|
||||
if lock.expires_at and lock.expires_at < datetime.now(timezone.utc):
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _acquire_db(self, lock_name: str, ttl: int) -> bool:
|
||||
async with AsyncSessionLocal() as db:
|
||||
now = datetime.now(timezone.utc)
|
||||
expires_at = now + timedelta(seconds=ttl)
|
||||
|
||||
# Try to update expired lock
|
||||
result = await db.execute(
|
||||
select(Lock).where(
|
||||
Lock.lock_name == lock_name,
|
||||
Lock.expires_at < now,
|
||||
)
|
||||
)
|
||||
lock = result.scalar_one_or_none()
|
||||
if lock:
|
||||
lock.owner_id = self.owner_id
|
||||
lock.acquired_at = now
|
||||
lock.expires_at = expires_at
|
||||
await db.commit()
|
||||
return True
|
||||
|
||||
# Try to insert new lock
|
||||
lock = Lock(
|
||||
lock_name=lock_name,
|
||||
owner_id=self.owner_id,
|
||||
acquired_at=now,
|
||||
expires_at=expires_at,
|
||||
)
|
||||
db.add(lock)
|
||||
try:
|
||||
await db.commit()
|
||||
return True
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
return False
|
||||
|
||||
async def _release_db(self, lock_name: str) -> bool:
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await db.execute(
|
||||
select(Lock).where(
|
||||
Lock.lock_name == lock_name,
|
||||
Lock.owner_id == self.owner_id,
|
||||
)
|
||||
)
|
||||
lock = result.scalar_one_or_none()
|
||||
if not lock:
|
||||
return False
|
||||
|
||||
await db.delete(lock)
|
||||
await db.commit()
|
||||
return True
|
||||
|
||||
async def _extend_db(self, lock_name: str, ttl: int) -> bool:
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await db.execute(
|
||||
select(Lock).where(
|
||||
Lock.lock_name == lock_name,
|
||||
Lock.owner_id == self.owner_id,
|
||||
)
|
||||
)
|
||||
lock = result.scalar_one_or_none()
|
||||
if not lock:
|
||||
return False
|
||||
|
||||
lock.expires_at = datetime.now(timezone.utc) + timedelta(seconds=ttl)
|
||||
await db.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def get_lock_service(owner_id: str | None = None) -> LockService:
|
||||
"""Get a lock service instance."""
|
||||
return LockService(owner_id=owner_id)
|
||||
@@ -0,0 +1,227 @@
|
||||
"""Application settings management service."""
|
||||
from typing import Any
|
||||
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.logging import get_logger
|
||||
from app.models.setting import AppSetting
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
EDITABLE_SETTINGS = {
|
||||
"RSSKEEPER_BASE_URL": {"description": "rssKeeper 服务地址", "sensitive": False},
|
||||
"OPENAI_API_KEY": {"description": "LLM API Key", "sensitive": True},
|
||||
"OPENAI_BASE_URL": {"description": "LLM API 基础地址", "sensitive": False},
|
||||
"OPENAI_MODEL": {"description": "LLM 模型名", "sensitive": False},
|
||||
"OPENAI_TIMEOUT": {"description": "LLM 调用超时（秒）", "sensitive": False},
|
||||
"OPENAI_MAX_RETRIES": {"description": "LLM 最大重试次数", "sensitive": False},
|
||||
"SUMMARIZE_INTERVAL_MINUTES": {"description": "摘要任务间隔（分钟）", "sensitive": False},
|
||||
"TAG_SCORE_INTERVAL_MINUTES": {"description": "分类/打分/去重任务间隔（分钟）", "sensitive": False},
|
||||
"DAILY_BRIEF_HOUR": {"description": "每日简报生成小时", "sensitive": False},
|
||||
"DAILY_BRIEF_MINUTE": {"description": "每日简报生成分钟", "sensitive": False},
|
||||
"TITLE_SIMILARITY_THRESHOLD": {"description": "标题相似度阈值", "sensitive": False},
|
||||
"CONTENT_SIMILARITY_THRESHOLD": {"description": "内容相似度阈值", "sensitive": False},
|
||||
"MAX_AI_SUMMARY_LENGTH": {"description": "AI 摘要最大长度", "sensitive": False},
|
||||
"MIN_ORIGINAL_SUMMARY_LENGTH": {"description": "原始摘要最小长度", "sensitive": False},
|
||||
"BRIEF_TOP_N_PER_CATEGORY": {"description": "简报每分类显示文章数", "sensitive": False},
|
||||
"LOG_LEVEL": {"description": "日志级别", "sensitive": False},
|
||||
"API_TOKEN": {"description": "API 鉴权 Token（为空时不启用）", "sensitive": True},
|
||||
"CORS_ALLOWED_ORIGINS": {"description": "CORS 允许来源（逗号分隔）", "sensitive": False},
|
||||
}
|
||||
|
||||
# Prefix to detect encrypted values
|
||||
_ENC_PREFIX = "enc:"
|
||||
|
||||
|
||||
def _get_fernet() -> Fernet | None:
|
||||
"""Get Fernet instance if encryption key is configured."""
|
||||
key = settings.SETTINGS_ENCRYPTION_KEY
|
||||
if not key:
|
||||
return None
|
||||
try:
|
||||
return Fernet(key.encode() if isinstance(key, str) else key)
|
||||
except Exception as exc:
|
||||
logger.error("SETTINGS_ENCRYPTION_KEY 无效: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def _encrypt(value: str) -> str:
|
||||
"""Encrypt a sensitive value if encryption is enabled."""
|
||||
if not value:
|
||||
return value
|
||||
fernet = _get_fernet()
|
||||
if fernet is None:
|
||||
return value
|
||||
return _ENC_PREFIX + fernet.encrypt(value.encode()).decode()
|
||||
|
||||
|
||||
def _decrypt(value: str) -> str:
|
||||
"""Decrypt a sensitive value if it was encrypted."""
|
||||
if not value or not value.startswith(_ENC_PREFIX):
|
||||
return value
|
||||
fernet = _get_fernet()
|
||||
if fernet is None:
|
||||
logger.warning("发现加密配置值但 SETTINGS_ENCRYPTION_KEY 未配置，无法解密")
|
||||
return value
|
||||
try:
|
||||
ciphertext = value[len(_ENC_PREFIX):].encode()
|
||||
return fernet.decrypt(ciphertext).decode()
|
||||
except InvalidToken:
|
||||
logger.warning("配置值解密失败（token 无效）")
|
||||
return value
|
||||
except Exception as exc:
|
||||
logger.error("配置值解密失败: %s", exc)
|
||||
return value
|
||||
|
||||
|
||||
def _get_env_default(key: str) -> str:
|
||||
"""Get default value from environment/settings."""
|
||||
value = getattr(settings, key, "")
|
||||
return str(value) if value is not None else ""
|
||||
|
||||
|
||||
def _mask_sensitive(value: str) -> str:
|
||||
"""Mask sensitive value for display."""
|
||||
if not value:
|
||||
return ""
|
||||
if len(value) <= 8:
|
||||
return "*" * len(value)
|
||||
return f"{value[:4]}...{value[-4:]}"
|
||||
|
||||
|
||||
async def init_default_settings(db: AsyncSession) -> None:
|
||||
"""Initialize default settings from environment if table is empty."""
|
||||
result = await db.execute(select(AppSetting))
|
||||
existing = result.scalars().first()
|
||||
if existing:
|
||||
return
|
||||
|
||||
for key, meta in EDITABLE_SETTINGS.items():
|
||||
default_value = _get_env_default(key)
|
||||
stored_value = _encrypt(default_value) if meta["sensitive"] else default_value
|
||||
db.add(
|
||||
AppSetting(
|
||||
key=key,
|
||||
value=stored_value,
|
||||
description=meta["description"],
|
||||
is_sensitive=meta["sensitive"],
|
||||
)
|
||||
)
|
||||
|
||||
await db.commit()
|
||||
logger.info("已初始化默认配置项: %d 条", len(EDITABLE_SETTINGS))
|
||||
|
||||
|
||||
async def _get_raw_setting(db: AsyncSession, key: str) -> AppSetting | None:
|
||||
"""Get setting row from DB."""
|
||||
result = await db.execute(select(AppSetting).where(AppSetting.key == key))
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def get_setting(db: AsyncSession, key: str, default: Any = None) -> Any:
|
||||
"""Get decrypted setting value from DB or env default."""
|
||||
setting = await _get_raw_setting(db, key)
|
||||
if setting:
|
||||
return _decrypt(setting.value) if setting.is_sensitive else setting.value
|
||||
return _get_env_default(key) if default is None else default
|
||||
|
||||
|
||||
async def set_setting(db: AsyncSession, key: str, value: str) -> bool:
|
||||
"""Update a setting (encrypt sensitive values)."""
|
||||
if key not in EDITABLE_SETTINGS:
|
||||
return False
|
||||
|
||||
meta = EDITABLE_SETTINGS[key]
|
||||
stored_value = _encrypt(str(value)) if meta["sensitive"] else str(value)
|
||||
|
||||
setting = await _get_raw_setting(db, key)
|
||||
if setting:
|
||||
setting.value = stored_value
|
||||
else:
|
||||
setting = AppSetting(
|
||||
key=key,
|
||||
value=stored_value,
|
||||
description=meta["description"],
|
||||
is_sensitive=meta["sensitive"],
|
||||
)
|
||||
db.add(setting)
|
||||
|
||||
await db.commit()
|
||||
logger.info("配置已更新: %s", key)
|
||||
return True
|
||||
|
||||
|
||||
async def list_settings(db: AsyncSession, mask_sensitive: bool = True) -> list[dict[str, Any]]:
|
||||
"""List all settings."""
|
||||
result = await db.execute(select(AppSetting))
|
||||
db_settings = {s.key: s for s in result.scalars().all()}
|
||||
|
||||
output = []
|
||||
for key, meta in EDITABLE_SETTINGS.items():
|
||||
setting = db_settings.get(key)
|
||||
is_sensitive = meta["sensitive"]
|
||||
|
||||
if setting:
|
||||
raw_value = setting.value
|
||||
updated_at = setting.updated_at.isoformat() if setting.updated_at else None
|
||||
else:
|
||||
raw_value = _get_env_default(key)
|
||||
updated_at = None
|
||||
|
||||
decrypted_value = _decrypt(raw_value) if is_sensitive else raw_value
|
||||
|
||||
if is_sensitive and mask_sensitive:
|
||||
display_value = _mask_sensitive(decrypted_value)
|
||||
is_masked = True
|
||||
else:
|
||||
display_value = decrypted_value
|
||||
is_masked = False
|
||||
|
||||
output.append({
|
||||
"key": key,
|
||||
"value": display_value,
|
||||
"real_value": decrypted_value if not mask_sensitive else None,
|
||||
"description": meta["description"],
|
||||
"is_sensitive": is_sensitive,
|
||||
"is_masked": is_masked,
|
||||
"updated_at": updated_at,
|
||||
})
|
||||
|
||||
return output
|
||||
|
||||
|
||||
async def apply_db_settings_to_config(db: AsyncSession) -> None:
|
||||
"""Apply DB settings to runtime config."""
|
||||
for key in EDITABLE_SETTINGS:
|
||||
db_value = await get_setting(db, key)
|
||||
if db_value is None or db_value == "":
|
||||
continue
|
||||
|
||||
field_info = settings.model_fields.get(key)
|
||||
if field_info is None:
|
||||
continue
|
||||
|
||||
target_type = field_info.annotation
|
||||
try:
|
||||
if target_type is int:
|
||||
converted = int(db_value)
|
||||
elif target_type is float:
|
||||
converted = float(db_value)
|
||||
elif target_type is bool:
|
||||
converted = db_value.lower() in ("true", "1", "yes")
|
||||
else:
|
||||
converted = db_value
|
||||
setattr(settings, key, converted)
|
||||
except Exception as exc:
|
||||
logger.error("应用配置 %s=%s 失败: %s", key, db_value, exc)
|
||||
raise ValueError(f"配置项 {key} 的值无效: {db_value}") from exc
|
||||
|
||||
|
||||
async def reset_settings(db: AsyncSession) -> None:
|
||||
"""Reset all settings to env defaults."""
|
||||
for key in EDITABLE_SETTINGS:
|
||||
await set_setting(db, key, _get_env_default(key))
|
||||
logger.info("配置已重置为环境变量默认值")
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Task runtime progress tracking service."""
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.core.redis import get_redis
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
TASK_STATUS_IDLE = "idle"
|
||||
TASK_STATUS_RUNNING = "running"
|
||||
TASK_STATUS_SUCCESS = "success"
|
||||
TASK_STATUS_ERROR = "error"
|
||||
|
||||
|
||||
class TaskRuntime:
|
||||
"""Runtime task progress tracker using Redis."""
|
||||
|
||||
def __init__(self):
|
||||
self._redis = None
|
||||
|
||||
async def _get_redis(self):
|
||||
if self._redis is None:
|
||||
self._redis = await get_redis()
|
||||
return self._redis
|
||||
|
||||
def _key(self, task_key: str) -> str:
|
||||
return f"task_progress:{task_key}"
|
||||
|
||||
async def update_progress(
|
||||
self,
|
||||
task_key: str,
|
||||
*,
|
||||
status: str | None = None,
|
||||
stage: str | None = None,
|
||||
current: int | None = None,
|
||||
total: int | None = None,
|
||||
message: str | None = None,
|
||||
trigger: str | None = None,
|
||||
) -> None:
|
||||
"""Update task progress."""
|
||||
try:
|
||||
redis = await self._get_redis()
|
||||
key = self._key(task_key)
|
||||
|
||||
existing = await redis.hgetall(key)
|
||||
data = dict(existing) if existing else {}
|
||||
|
||||
if status:
|
||||
data["status"] = status
|
||||
if stage:
|
||||
data["stage"] = stage
|
||||
if current is not None:
|
||||
data["current"] = str(current)
|
||||
if total is not None:
|
||||
data["total"] = str(total)
|
||||
if message is not None:
|
||||
data["message"] = message
|
||||
if trigger:
|
||||
data["trigger"] = trigger
|
||||
|
||||
data["updated_at"] = datetime.now(timezone.utc).isoformat()
|
||||
if status == TASK_STATUS_RUNNING and "started_at" not in data:
|
||||
data["started_at"] = data["updated_at"]
|
||||
if status in (TASK_STATUS_SUCCESS, TASK_STATUS_ERROR):
|
||||
data["finished_at"] = data["updated_at"]
|
||||
|
||||
await redis.hset(key, mapping=data)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to update task progress: %s", exc)
|
||||
|
||||
async def get_progress(self, task_key: str) -> dict[str, Any]:
|
||||
"""Get task progress."""
|
||||
try:
|
||||
redis = await self._get_redis()
|
||||
data = await redis.hgetall(self._key(task_key))
|
||||
if not data:
|
||||
return self._empty_progress(task_key)
|
||||
return {
|
||||
"task_key": task_key,
|
||||
"status": data.get("status", TASK_STATUS_IDLE),
|
||||
"stage": data.get("stage", ""),
|
||||
"current": int(data.get("current", 0)),
|
||||
"total": int(data.get("total", 0)),
|
||||
"message": data.get("message"),
|
||||
"trigger": data.get("trigger"),
|
||||
"started_at": data.get("started_at"),
|
||||
"updated_at": data.get("updated_at"),
|
||||
"finished_at": data.get("finished_at"),
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to get task progress: %s", exc)
|
||||
return self._empty_progress(task_key)
|
||||
|
||||
async def reset_progress(self, task_key: str) -> None:
|
||||
"""Reset task progress to idle."""
|
||||
try:
|
||||
redis = await self._get_redis()
|
||||
await redis.delete(self._key(task_key))
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to reset task progress: %s", exc)
|
||||
|
||||
def _empty_progress(self, task_key: str) -> dict[str, Any]:
|
||||
return {
|
||||
"task_key": task_key,
|
||||
"status": TASK_STATUS_IDLE,
|
||||
"stage": "",
|
||||
"current": 0,
|
||||
"total": 0,
|
||||
"message": None,
|
||||
"trigger": None,
|
||||
"started_at": None,
|
||||
"updated_at": None,
|
||||
"finished_at": None,
|
||||
}
|
||||
|
||||
|
||||
task_runtime = TaskRuntime()
|
||||
Reference in New Issue
Block a user