Python requests伪装User-Agent反爬：随机切换与Headers池

讲解requests伪装浏览器User-Agent的多种方法，随机UA池、多浏览器头轮换、处理简单反爬检测，提高脚本被识别为脚本的难度。

场景痛点

用同一个User-Agent连续请求几十次，服务器直接封IP；换一个UA又被识别出来。反爬不是靠一个固定UA就能搞定的，要准备一整套Headers池，每次请求随机换，再配合Referer、Accept-Language这些字段，让请求看起来像不同的真实用户在访问。

用到的库

pip install requests beautifulsoup4 lxml

完整代码

# 导入库
import random
import requests

# 准备一组常见的浏览器User-Agent
USER_AGENTS = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
]

REFERERS = [
"https://www.baidu.com/",
"https://www.google.com/",
"https://www.bing.com/",
]

def random_headers():
"""随机生成一套请求头"""
return {
"User-Agent": random.choice(USER_AGENTS),
"Referer": random.choice(REFERERS),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
}

def fetch(url):
"""每次请求用不同的Headers"""
headers = random_headers()
print("本次UA：", headers["User-Agent"][:60], "...")

resp = requests.get(url, headers=headers, timeout=10)
resp.encoding = resp.apparent_encoding
return resp

def main():
url = "https://httpbin.org/headers"
for i in range(3):
print("=" * 30, f"第{i+1}次请求")
try:
resp = fetch(url)
# 打印服务器看到的UA
import json
data = json.loads(resp.text)
print("服务器收到的UA：", data["headers"]["User-Agent"][:60])
except Exception as e:
print("出错：", e)

if __name__ == "__main__":
main()