Python CSV去重教程：按列或整行去重保留第一条

用pandas drop_duplicates对CSV去重，支持按指定列去重、保留第一条或最后一条、导出干净文件，附完整代码和注意事项。

场景痛点

数据来源多了，同一个用户重复录入、同一笔订单导出两次、爬虫重复抓了同一条。几万行里人工找重复根本不现实，Excel的"删除重复项"点完还不知道删了哪条。用pandas去重，可以指定按哪几列判断、保留哪一条，导出前心里有数。

用到的库

pip install pandas

完整代码

# Python CSV去重
import pandas as pd

def dedup_by_full_row(input_path, output_path):
"""整行完全相同才去重，保留第一条。"""
df = pd.read_csv(input_path, encoding="utf-8-sig")
before = len(df)
df_dedup = df.drop_duplicates(keep="first")
after = len(df_dedup)
df_dedup.to_csv(output_path, index=False, encoding="utf-8-sig")
print(f"整行去重：{before} -> {after}，删除 {before-after} 行")

def dedup_by_key_column(input_path, output_path, keys):
"""
按指定列去重，比如按"手机号"或"订单号"判断重复。
keep="first" 保留最早出现的那条，"last" 保留最新的。
"""
df = pd.read_csv(input_path, encoding="utf-8-sig")
before = len(df)

# 先看下重复了多少
dup_count = df.duplicated(subset=keys, keep=False).sum()
print(f"按 {keys} 判断，涉及重复的行数：{dup_count}")

df_dedup = df.drop_duplicates(subset=keys, keep="first")
after = len(df_dedup)
df_dedup.to_csv(output_path, index=False, encoding="utf-8-sig")
print(f"按列去重：{before} -> {after}，删除 {before-after} 行")

def dedup_and_mark_duplicates(input_path, output_path):
"""
不只删，还把重复项单独导出一份，方便人工核对。
"""
df = pd.read_csv(input_path, encoding="utf-8-sig")
# 标记重复行（除第一次出现外都标记为True）
dup_mask = df.duplicated(keep="first")
dup_rows = df[dup_mask]
clean_rows = df[~dup_mask]

dup_rows.to_csv("duplicated_rows.csv", index=False, encoding="utf-8-sig")
clean_rows.to_csv(output_path, index=False, encoding="utf-8-sig")
print(f"干净数据 {len(clean_rows)} 行，重复数据 {len(dup_rows)} 行已导出")

def main():
input_file = "data.csv"
dedup_by_full_row(input_file, "dedup_full_row.csv")
dedup_by_key_column(input_file, "dedup_by_phone.csv", keys=["手机号"])
dedup_and_mark_duplicates(input_file, "dedup_clean.csv")

if __name__ == "__main__":
main()