Python读取大文件内存优化实战

用pandas分块读取、指定dtypes、只读必要列、只读过滤后的数据，解决几十万行CSV/Excel读取内存爆掉的问题，机器低配也能跑。

场景痛点

手头CSV文件几百MB甚至几个GB，pd.read_csv 一把读进内存直接MemoryError，或者读完电脑卡死。其实根本不需要全量读：只读需要的列、分块处理、指定字段类型，三招下去，普通笔记本也能轻松处理GB级文件。

用到的库

pip install pandas

完整代码

import pandas as pd
from pathlib import Path

BIG_CSV = Path("big.csv")

def make_big_csv():
"""造一个50万行的测试CSV，实际使用换成真实文件"""
if BIG_CSV.exists():
return
df = pd.DataFrame({
"order_id": [f"SO{i:06d}" for i in range(500_000)],
"user_id":  [i % 10000 for i in range(500_000)],
"amount":   [i % 10000 for i in range(500_000)],
"city":     ["北京"]*125000 + ["上海"]*125000 + ["广州"]*125000 + ["深圳"]*125000,
"dt":       pd.date_range("2024-01-01", periods=500_000, freq="min"),
})
df.to_csv(BIG_CSV, index=False)
print(f"生成测试文件：{BIG_CSV.stat().st_size/1024/1024:.1f} MB")

def read_optimized():
# 1. 只读需要的列，别全读
usecols = ["order_id", "user_id", "amount", "city"]

# 2. 指定dtype，避免pandas自动推断成object
dtypes = {
"order_id": str,
"user_id":  "int32",
"amount":   "int32",
"city":     "category",   # 类别型省内存
}

# 3. 分块读取，每块10万行
chunksize = 100_000
parts = []
total_amount = 0
for chunk in pd.read_csv(BIG_CSV, usecols=usecols, dtype=dtypes, chunksize=chunksize):
parts.append(chunk)
total_amount += chunk["amount"].sum()

df = pd.concat(parts, ignore_index=True)
print(f"读取完成：{len(df)} 行，总金额={total_amount}")
print(df.dtypes)
print(f"内存占用约：{df.memory_usage(deep=True).sum()/1024/1024:.1f} MB")

if __name__ == "__main__":
make_big_csv()
read_optimized()