123 lines
3.7 KiB
Python
123 lines
3.7 KiB
Python
"""
|
||
评分和过滤模块 - 基于关键词权重 + 规则引擎
|
||
不调用外部 AI API，纯本地规则评分
|
||
"""
|
||
import re
|
||
import logging
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# 高权重关键词（出现即加分）
|
||
HIGH_WEIGHT_KEYWORDS = {
|
||
# AI / 算力相关
|
||
"英伟达": 3, "NVIDIA": 3, "AMD": 2, "算力": 2, "AI": 2,
|
||
"光模块": 2, "存储": 1, "芯片": 2, "GPU": 2, "大模型": 2,
|
||
"ChatGPT": 2, "OpenAI": 2, "Anthropic": 2, "DeepSeek": 2,
|
||
# 中国科技公司
|
||
"华为": 2, "腾讯": 2, "阿里": 2, "字节": 2, "小米": 2,
|
||
"百度": 1, "京东": 1, "美团": 1, "拼多多": 1, "比亚迪": 2,
|
||
# 重磅事件
|
||
"收购": 2, "突破": 2, "泄露": 3, "内幕": 3, "传闻": 2,
|
||
"重磅": 3, "突发": 3, "暴涨": 2, "暴跌": 2, "崩盘": 3,
|
||
"熔断": 3, "制裁": 2, "禁令": 2, "封锁": 2,
|
||
# 股市相关
|
||
"A股": 1, "港股": 1, "美股": 1, "涨停": 2, "跌停": 2,
|
||
"IPO": 2, "上市": 1, "退市": 2, "回购": 1,
|
||
# 宏观
|
||
"降息": 2, "加息": 2, "美联储": 2, "央行": 1, "GDP": 1,
|
||
"CPI": 1, "非农": 2, "失业率": 1,
|
||
}
|
||
|
||
# 低价值关键词（出现则减分）
|
||
LOW_VALUE_PATTERNS = [
|
||
r"广告", r"推广", r"优惠券", r"免费领",
|
||
r"点击.*查看", r"扫码", r"关注.*公众号",
|
||
]
|
||
|
||
|
||
def score_news(news_item: dict, user_keywords: list = None) -> int:
|
||
"""
|
||
对单条新闻评分，返回 0-10 分
|
||
评分规则：
|
||
- 基础分 3 分
|
||
- 匹配高权重关键词加分
|
||
- 匹配用户自定义关键词加分
|
||
- 信息源标记为重要加分
|
||
- 匹配低价值模式减分
|
||
- 最终分数限制在 0-10
|
||
"""
|
||
title = news_item.get("title", "")
|
||
if not title:
|
||
return 0
|
||
|
||
score = 3 # 基础分
|
||
|
||
# 1. 高权重关键词匹配
|
||
for keyword, weight in HIGH_WEIGHT_KEYWORDS.items():
|
||
if keyword.lower() in title.lower():
|
||
score += weight
|
||
|
||
# 2. 用户自定义关键词匹配
|
||
if user_keywords:
|
||
for kw in user_keywords:
|
||
if kw.lower() in title.lower():
|
||
score += 1
|
||
|
||
# 3. 信息源标记为重要
|
||
if news_item.get("important"):
|
||
score += 2
|
||
|
||
# 4. 低价值内容减分
|
||
for pattern in LOW_VALUE_PATTERNS:
|
||
if re.search(pattern, title):
|
||
score -= 3
|
||
|
||
# 5. 标题太短可能是无效内容
|
||
if len(title) < 10:
|
||
score -= 2
|
||
|
||
return max(0, min(10, score))
|
||
|
||
|
||
def is_similar(title1: str, title2: str, threshold: float = 0.7) -> bool:
|
||
"""
|
||
简单的标题相似度判断
|
||
使用字符级别的 Jaccard 相似度
|
||
"""
|
||
if not title1 or not title2:
|
||
return False
|
||
# 去除标点和空格
|
||
clean1 = re.sub(r"[^\w]", "", title1)
|
||
clean2 = re.sub(r"[^\w]", "", title2)
|
||
if not clean1 or not clean2:
|
||
return False
|
||
# 2-gram 集合
|
||
set1 = set(clean1[i:i+2] for i in range(len(clean1)-1))
|
||
set2 = set(clean2[i:i+2] for i in range(len(clean2)-1))
|
||
if not set1 or not set2:
|
||
return clean1 == clean2
|
||
intersection = set1 & set2
|
||
union = set1 | set2
|
||
return len(intersection) / len(union) >= threshold
|
||
|
||
|
||
def dedup_news(news_list: list) -> list:
|
||
"""去重：基于标题相似度，保留最早的一条"""
|
||
result = []
|
||
for item in news_list:
|
||
is_dup = False
|
||
for existing in result:
|
||
if is_similar(item.get("title", ""), existing.get("title", "")):
|
||
is_dup = True
|
||
break
|
||
if not is_dup:
|
||
result.append(item)
|
||
return result
|
||
|
||
|
||
def score_and_filter(news_list: list, user_keywords: list = None) -> list:
|
||
"""批量评分并过滤，返回带评分的新闻列表"""
|
||
for item in news_list:
|
||
item["score"] = score_news(item, user_keywords)
|
||
return news_list
|