dataClean/app/deduplicator.py

"""文章去重：URL 精确去重 + 标题/内容相似度去重"""
import logging
import re
from datetime import datetime, timedelta, timezone
from difflib import SequenceMatcher
from typing import List, Dict, Tuple, Set

from sqlalchemy.orm import Session
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

from config import settings
from models import EnrichedArticle, DuplicateGroup
from app.task_progress import update_progress, report_loop_progress

logger = logging.getLogger(__name__)


def _normalize_title(title: str) -> str:
    """标题规范化：去除标点和多余空格，小写，保留中英文数字"""
    if not title:
        return ""
    # 保留：单词字符、CJK 统一表意符号（含扩展 A/B/C/D/E）
    title = re.sub(
        r"[^\w一-鿿㐀-䶿\U00020000-\U0002a6df\U0002a700-\U0002b73f\U0002b740-\U0002b81f]",
        " ",
        title,
    )
    title = " ".join(title.split())
    return title.lower()


def _title_similarity(a: str, b: str) -> float:
    """计算标题相似度"""
    na = _normalize_title(a)
    nb = _normalize_title(b)
    if not na or not nb:
        return 0.0
    return SequenceMatcher(None, na, nb).ratio()


def _content_similarity_matrix(contents: List[str]) -> np.ndarray:
    """使用 TF-IDF + 余弦相似度计算内容相似度矩阵"""
    if len(contents) < 2:
        return np.zeros((len(contents), len(contents)))

    # 过滤空内容
    valid_contents = [c or "" for c in contents]
    try:
        vectorizer = TfidfVectorizer(
            max_features=5000,
            stop_words="english",
            ngram_range=(1, 2),
            min_df=1,
        )
        tfidf = vectorizer.fit_transform(valid_contents)
        return cosine_similarity(tfidf)
    except Exception as exc:
        logger.warning("TF-IDF 相似度计算失败: %s", exc)
        return np.zeros((len(contents), len(contents)))


def _find_duplicate_clusters(
    articles: List[EnrichedArticle],
    title_threshold: float = None,
    content_threshold: float = None,
) -> List[Set[int]]:
    """
    基于标题相似度和内容相似度找出重复簇。
    返回索引簇列表，每个簇是一组 articles 的索引集合。
    """
    title_threshold = title_threshold or settings.TITLE_SIMILARITY_THRESHOLD
    content_threshold = content_threshold or settings.CONTENT_SIMILARITY_THRESHOLD

    n = len(articles)
    if n < 2:
        return []

    contents = []
    for art in articles:
        text = " ".join([
            art.title or "",
            art.ai_summary or art.original_summary or "",
            art.content or "",
        ])
        contents.append(text[:2000])  # 限制长度加速计算

    content_sim = _content_similarity_matrix(contents)

    visited = [False] * n
    clusters: List[Set[int]] = []

    for i in range(n):
        if visited[i]:
            continue
        cluster = {i}
        queue = [i]
        visited[i] = True

        while queue:
            cur = queue.pop(0)
            for j in range(n):
                if visited[j] or cur == j:
                    continue

                title_sim = _title_similarity(articles[cur].title or "", articles[j].title or "")
                c_sim = content_sim[cur][j] if cur < n and j < n else 0.0

                # 标题高度相似 或 内容高度相似均视为重复
                if title_sim >= title_threshold or c_sim >= content_threshold:
                    cluster.add(j)
                    queue.append(j)
                    visited[j] = True

        if len(cluster) > 1:
            clusters.append(cluster)

    return clusters


def _pick_representative(articles: List[EnrichedArticle], indices: Set[int]) -> EnrichedArticle:
    """从重复组中选择代表文章：优先选有 AI 摘要、来源 Feed 分类明确、发布时间最早的"""
    candidates = [articles[i] for i in indices]
    # 排序：有 AI 摘要优先，然后有 Feed 分类，然后发布时间早
    candidates.sort(
        key=lambda a: (
            bool(a.ai_summary),
            bool(a.feed_category),
            a.published_at or datetime.min,
        ),
        reverse=True,
    )
    return candidates[0]


def deduplicate_articles(
    db: Session,
    date_str: str = None,
    title_threshold: float = None,
    content_threshold: float = None,
) -> Dict[str, int]:
    """
    对指定日期的文章进行去重。
    若 date_str 为空则处理今天（UTC）的文章。
    返回统计：{"total": x, "duplicate_groups": y, "representatives": z}
    """
    if date_str is None:
        date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")

    # 只清空该日期已有的去重组，避免破坏历史数据
    day_start = datetime.strptime(date_str, "%Y-%m-%d")
    day_end = day_start + timedelta(days=1)

    old_groups = db.query(DuplicateGroup).filter(DuplicateGroup.brief_date == date_str).all()
    for og in old_groups:
        for art in og.articles:
            art.duplicate_group_id = None
            art.is_representative = False
        db.delete(og)
    db.commit()

    # 重置该日期文章的去重标记
    articles = (
        db.query(EnrichedArticle)
        .filter(
            EnrichedArticle.fetched_at >= day_start,
            EnrichedArticle.fetched_at < day_end,
        )
        .order_by(EnrichedArticle.published_at)
        .all()
    )

    if not articles:
        logger.info("日期 %s 无文章可去重", date_str)
        update_progress("tag_score_dedup", status="running", stage="去重", current=0, total=0, message="无文章可去重")
        return {"total": 0, "duplicate_groups": 0, "representatives": 0}

    update_progress("tag_score_dedup", status="running", stage="计算相似度并去重", current=0, total=0)

    # 先 URL 去重：相同 link 只保留一篇
    unique_articles: List[EnrichedArticle] = []
    seen_links: set = set()
    url_dup_count = 0
    for art in articles:
        link = (art.link or "").strip()
        if link and link in seen_links:
            url_dup_count += 1
            continue
        if link:
            seen_links.add(link)
        unique_articles.append(art)

    clusters = _find_duplicate_clusters(
        unique_articles,
        title_threshold=title_threshold,
        content_threshold=content_threshold,
    )

    stats = {"total": len(articles), "duplicate_groups": len(clusters), "representatives": 0}
    update_progress("tag_score_dedup", status="running", stage="写入重复组", current=0, total=len(clusters))

    for ci, cluster in enumerate(clusters):
        representative = _pick_representative(unique_articles, cluster)
        member_ids = [unique_articles[i].id for i in cluster]

        group = DuplicateGroup(
            representative_article_id=representative.id,
            member_article_ids=member_ids,
            similarity_matrix={},  # 可后续补充
            brief_date=date_str,
        )
        db.add(group)
        db.flush()

        for idx in cluster:
            art = unique_articles[idx]
            art.duplicate_group_id = group.id
            art.is_representative = (art.id == representative.id)

        stats["representatives"] += 1
        report_loop_progress("tag_score_dedup", ci + 1, len(clusters), "写入重复组")

    db.commit()
    logger.info(
        "去重完成: 日期=%s, 总文章=%d, 重复组=%d, URL 重复=%d",
        date_str, stats["total"], stats["duplicate_groups"], url_dup_count
    )
    return stats