MRR 与 NDCG@k
MRR 与 NDCG@k
import math
def mrr(runs):
total = 0.0
for ranked in runs:
for i, rel in enumerate(ranked, start=1):
if rel:
total += 1 / i
break
return total / len(runs)
def ndcg_at_k(ranked_relevances, k):
dcg = sum(r / math.log2(i + 1) for i, r in enumerate(ranked_relevances[:k], start=1))
ideal = sorted(ranked_relevances, reverse=True)[:k]
idcg = sum(r / math.log2(i + 1) for i, r in enumerate(ideal, start=1))
return dcg / idcg if idcg else 0.0NDCG 需要相关度等级(如 2=直接回答,1=部分相关):让标注成本换排序敏感度。