爬虫基础知识点

最近看了看爬虫相关知识点，做了记录，具体代码放到了仓库，本文仅学习使用，如有违规请联系博主删除。

这个流程图是我使用在线AI工具infography生成的，这个网站可以根据url或者文本等数据自动生成流程图，挺好用。

1.requests请求

requests 库用于发送 HTTP 请求，与 Web 服务器进行交互，获取数据或者提交表单等操作。

import requests#请求，#pip install requests
import re#正则解析
from lxml import etree#xpath解析
import pymongo#mongodb数据存储（pip install pymongo）

1.1.requests请求数据

属性	描述
response.text	获取响应的字符串 str 数据，处理文本数据（如 HTML、纯文本）
response.content	获取 bytes 类型，处理二进制数据（如图片、文件等）
response.status_code	获取响应的状态码
response.requests_headers	获取对应的请求头
response.headers	获取响应头
response.requests_cookies	获取对应请求的 cookie
response.cookies	获取响应的 cookie（经过了 set - cookie 动作）
response.json	获取到响应的 json 数据，转换成字典

url="https://www.cheshi.com/"
headers={"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"}
response=requests.get(url,headers=headers)
res.encoding="gb2312"#返回的页面有乱码，经过搜索，查到charset=gb2312，所以对res进行编码转换
print(response.text，response.status_code，response.headers，response.encoding，response.request.headers)
with open("./网上车市.html","w",encoding="utf-8") as f:f.write(response.text)#写入源码

1.2.解析提取数据

1.2.1.解析数据

1.2.1.1.BeautifulSoup

#使用bs4解析页面内容(select,find_all选择器)
from bs4 import BeautifulSoup
soup=BeautifulSoup(html.text,'lxml')titles=soup.select(".article-summary>.article-title")for title in titles:print(title.select("a")[0].text)

1.2.1.2.lxml

from lxml import etree
tree=etree.HTML(html.text)
dls = tree.xpath("//dl[contains(@class, 'list') and contains(@class, 'hiddenMap') and contains(@class, 'rel')]")    

1.2.2.提取数据

1.2.2.1.re

符号	含义
.	除了\n匹配所有（re.S匹配所有包含\n）
\w	数字、字母、下划线（\W 非数字、字母、下划线(）
\d	数字（\D 非数字）
\s	空格 换行（\S非空格换行）
[]	匹配里面任意字符（[^]匹配除了里面任意内容）
{num}	匹配重复num次
*	匹配重复0次或多次
+	匹配重复1次或多次
？	惰性匹配（匹配到就截断，再接着去匹配剩余）
re.compile()	预加载匹配规则
（）	分组获取数据

exp=re.compile('class="m_detail".*?href.*?>(.*?)<',re.S)#匹配出（）里的内容，re.S代表匹配包括换行符
print(exp.findall(response.text))#在response.text里面匹配出规则内容，返回列表

1.2.2.2.xpath

符号	含义
./	当前目录下匹配
/	子目录匹配
//	子孙目录匹配
//div[@class=“A”]	classname是A的div元素
//div[contains(@class, ‘A’) and contains(@class, ‘B’) ]	classname包含A且包含B的div元素

tree=etree.HTML(response.text)
title=tree.xpath('//div[@class="m_detail"]//a/text()')
print(title)

1.2.2.3 CSS 选择器

submit_button=wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,".next-pagination-jump-go")))

1.3.数据存储

1.3.1.Mongodb

client=pymongo.MongoClient("mongodb://localhost:27017")#链接客户端client
db=client["app"]#链接数据库db
col=db["C1"]#链接集合Collection
obj=col.insert_one(user)#增加1条数据
print(obj.inserted_id)
objs=col.insert_many(users)#增加文档数据document
print(objs.inserted_ids)#返回documents的所有id
users=col.find()#查询所有
objs=col.find({"name":"zoe"})#条件查询
objs=col.find({"age":{"$gt":20}})#年龄大于20#$regex正则匹配

1.3.2.redis

import redis#pip install redis
db=redis.Redis(host="localhost",port="6379",decode_responses=True)
#string 存value
db.set("name","小12")#添加1个键值对
print(db.get("name"))#获取1个值
db.mset({"name1":"老王","age1":"老衲年方28"})#添加多个键值对
print(db.mget("name1","name","age1"))#获取多个值
# #hash 存key-value
db.hset("hash1","key1","value1")#添加
db.hset("hash1","key2","value2")
db.hset("hash1","key3","value3")
print(db.hget("hash1","key2"))#获取hash中key对应的值
print(db.hgetall("hash1"))#获取hash中所有的键值对
# list 存list
db.lpush("list1",1,2,3)#倒序插入，先进后出
db.rpush("list2",2,3,4,5)#顺序插入，先进先出
print(db.llen("list1"))#list的长度
print(db.lrange("list1",0,-1))#lrange key start stop(-1 在 Redis 中是一个特殊的索引，表示列表的最后一个元素)
#set
db.sadd("set1",55,66,77,55)
print(db.scard("set1"))#scard获取set的长度
print(db.smembers("set1"))#smembers获取set的所有元素
#zset
db.zadd("zset1", {"zoe": 22, "jodie": 11})#Redis 的 Sorted Set（有序集合）要求每个成员（member）都关联一个分数（score）
print(db.zcard("zset1"))
print(db.zrange("zset1",0,-1,withscores=True))#[('jodie', 11.0), ('zoe', 22.0)]

1.3.3.csv

import csv#数据读写csv
headers=["z","f","g"]#写
rows=[("aa","bb","cc"),("dd","rr","ww"),("ff","yy","jj")]
with open("save.csv","a", newline='') as f:#, w写入，a追加，newline=''表示中间不空一行f_csv=csv.writer(f)#写入缓存f_csv.writerow(headers)#写入一行f_csv.writerows(rows)#写入多行
with open("save.csv","r",encoding="utf-8") as f:#读f_csv=csv.reader(f)#python内置的csv解析缓存next(f_csv)#跳过第一行，titlefor i in f_csv:print(i)
import xlrd,xlwt#数据读写excel,需要pip
from docx import Document#数据读写word,需要pip install python-docx

1.3.4.mysql

from sqlalchemy import create_engine
from sqlalchemy import Column, String, Integer, Text
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base# 创建基类
Base = declarative_base()# 创建引擎
engine = create_engine('mysql+pymysql://root:PASSWORD@127.0.0.1:3306/test?charset=utf8', echo=True)# 定义 Book 类
class Book(Base):__tablename__ = 'book'id = Column('id', Integer, primary_key=True, autoincrement=True)title = Column('title', String(20))info = Column('info', String(30))star = Column('star', String(10))pl = Column('pl', String(10))introduce = Column('introduce', Text())# 创建表结构
Base.metadata.create_all(engine)# 创建会话
Session = sessionmaker(bind=engine)
sess = Session()

1.4.requests获取携带cookies

1.4.1.携带

url="https://my.cheshi.com/user/"
headers={"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"}
cookies="pv_uid=1732254874878; cheshi_UUID=01JD96ZEWPDK2ZKK8X3WGFFPVS; cheshi_pro_city=MV%2FljJfkuqxfMV%2FkuJzln47ljLpfYmVpamluZw%3D%3D; Hm_lvt_8fe47348e12ba11be217fd389b115472=1732254888,1732494411; HMACCOUNT=0F938E3E8702278B; lv=1732674538; vn=7; Hm_lvt_ed9cf33799965fb6c868762ac84e663e=1732674587; Hm_lpvt_ed9cf33799965fb6c868762ac84e663e=1732674590; cheshi_token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiIsImp0aSI6ImNoZXNoaV9oNV9zaWduIn0.eyJpc3MiOiJodHRwczpcL1wvYXBpLmNoZXNoaS5jb20iLCJhdWQiOiJodHRwczpcL1wvYXBpLmNoZXNoaS5jb20iLCJqdGkiOiJjaGVzaGlfaDVfc2lnbiIsImlhdCI6MTczMjY3NDY2OSwibmJmIjoxNzMyNjc0NzI5LCJleHAiOjE3MzMyNzk0NjksInVpZCI6IjkxMDcxNDIifQ.ihAUr-0-7HEFedu-u23BlcstiaynxHrBAVDBXqnAW_E; cheshi_user_authv2=MzI2NDUyNAlsaXR0bGVaCXYyCWJjZDYzMWQ4NDZlMTQ4ZWQwY2UzZThhMTFkYTE2YmQxCTE3MzI2NzQ2NjkJNDgyN2JjMTgwZjg5MzIyNDg4MDAyYzg3NjYwOGRmNTY=; cheshi_user_info=OTEwNzE0MglsaXR0bGVaCXYyCWJjZDYzMWQ4NDZlMTQ4ZWQwY2UzZThhMTFkYTE2YmQxCTE3MzI2NzQ2NjkJNDgyN2JjMTgwZjg5MzIyNDg4MDAyYzg3NjYwOGRmNTYJCQl3YW5nc2hhbmdjaGVzaGk=; cheshi_user_info_for_index=OTEwNzE0MglsaXR0bGVaCXYyCWJjZDYzMWQ4NDZlMTQ4ZWQwY2UzZThhMTFkYTE2YmQxCTE3MzI2NzQ2NjkJNDgyN2JjMTgwZjg5MzIyNDg4MDAyYzg3NjYwOGRmNTYJCQl3YW5nc2hhbmdjaGVzaGk=; Hm_lpvt_8fe47348e12ba11be217fd389b115472=1732674672; PHPSESSID=bd0f056bb72ef681c01a68b853bde882; pv_source=; cheshi_user_prevLogintime=1732674716; pv_cheshit=1732674722341"
cookies={ item.split("=")[0] : item.split("=")[1] for item in cookies.split("; ")}#字符串转换成dict
print(cookies)
cookies=requests.utils.cookiejar_from_dict(cookies)#把cookie转换成dick，然后通过requests接口传递cookies
res=requests.get(url,headers=headers,cookies=cookies)

1.4.2.获取再携带

url_login="https://api.cheshi.com/services/common/api.php?api=login.Login"
data={"act": "login",
"mobile": "18811752638",
"source": "pc",
"password": "PASSWORD",
"hold_time": "yes",
}
headers={"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"}
res=requests.post(url=url_login,headers=headers,data=data)
# print(res.cookies)
url="https://my.cheshi.com/user/"
res=requests.get(url,headers=headers,cookies=res.cookies)

1.5.session维护会话

session维护会话，会将获取到的cookies自动保存携带

url_login="https://api.cheshi.com/services/common/api.php?api=login.Login"
data={"act": "login",
"mobile": "18811752638",
"source": "pc",
"password": "PASSWORD",
"hold_time": "yes",
}
headers={"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"}
session=requests.session()
session.post(url_login,headers=headers,data=data)#自动保存携带获取到的cookies(session没有维护headers，所以下面访问时要携带headers)
url="https://my.cheshi.com/user/"
res=session.get(url,headers=headers)
print(res.text)

1.6.多进程多线程爬取数据

#每类书籍开一个进程，每个进程开多个线程跑（因为io密集）
import urllib.parse
import requests
from bs4 import BeautifulSoup
import multiprocessing
from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor
import urllib
headers={"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",# "referer":"https://book.douban.com/tag/%E5%B0%8F%E8%AF%B4?start=60&type=T"
}
def get_link(url):print("当前进程：{}".format(multiprocessing.Process.pid))try:response = requests.get(url, headers=headers)html = response.textsoup = BeautifulSoup(html, "lxml")books = soup.select(".subject-item")for book in books:title = book.select_one(".info h2 a").text.strip().replace(" ", "").replace("\n", "")print(title)except Exception as e:print(f"Error fetching {url}: {e}")def th(tag):#多线程with ThreadPoolExecutor(max_workers=5) as excutorT:urls=[]for i in range(0,300,20):tag_q=urllib.parse.quote(tag)url = f"https://book.douban.com/tag/{tag_q}?start={i}&type=T"urls.append(url)futures=[excutorT.submit(get_link,item) for item in urls]for future in futures:future.result()if __name__=="__main__":tags=["小说","文学"]with ProcessPoolExecutor(max_workers=2) as executorP:#多进程futures=[executorP.submit(th,tag) for tag in tags]for future in futures:future.result()

1.7.使用aiohttp异步爬取数据

异步是事件驱动模型，异步是一种编程方式，专注于非阻塞执行，而多线程/多进程是一种并发模型。异步操作可以在单线程中实现，也可以与多线程、多进程组合使用来增强处理能力。asynico 标记异步函数,await 等待异步函数返回。

#使用aiohttp取代requests
import aiohttp#pip install aiohttp
import asyncio
from bs4 import BeautifulSoup
headers={"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"}
async def crawl(i):print("正在爬取：",i)url="https://xiaohua.zol.com.cn/baoxiao/{}.html".format(i)async with aiohttp.ClientSession(headers=headers) as session:async with session.get(url) as resp:# print(resp.status)text=await resp.text()soup=BeautifulSoup(text,"lxml")lists=soup.select(".article-summary .article-title a")for list in lists:print(list.get_text())
if __name__=="__main__":loop=asyncio.get_event_loop()#开启异步tasks=[crawl(i) for i in range(1,10)]loop.run_until_complete(asyncio.wait(tasks))loop.close()

2.scrapy框架

这个框架的爬虫有4给模板，默认使用default，还有crawl模板，可以匹配网页链接并访问。

2.1.default

pip install scrapy#安装框架(scrapy自带xpath功能，获取元素内容需要.get()) 
scrapy startproject car .  #创建工程
scrapy genspider app https://product.cheshi.com/rank/2-0-0-0-1/  #在工程里面创建spider
scrapy crawl app #spider开始爬数据

2.1.1.配置工程

settings.py

LOG_LEVEL="ERROR"
ROBOTSTXT_OBEY = False
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
打开ITEM_PIPELINES和MIDDLEWARES（如果用到的话）

2.1.2.定义数据模型

items.py

class BookItem(scrapy.Item):title = scrapy.Field()publisher = scrapy.Field()
在app里面导入模型类(from ..items import CarItem)，创建空对象，然后把爬取的数据赋值给对象,最后yield item（此时item对象就到了pipelines）

2.1.3.爬取数据

import scrapy
from ..items import BookItemclass AppSpider(scrapy.Spider):name = 'app'allowed_domains = ['book.douban.com']start_urls = ['https://book.douban.com/latest']def parse(self, response):books=response.xpath("//ul[@class='chart-dashed-list']/li")for book in books:# title=book.xpath(".//h2[@class='clearfix']/a/text()").get()link=book.xpath(".//h2[@class='clearfix']/a/@href").get()# print(title,link)yield scrapy.Request(url=link,callback=self.parse_details)next_url=response.xpath("//span[@class='next']/a/@href").get()if next_url is not None:next_link=response.urljoin(next_url)print(next_link)yield scrapy.Request(url=next_link,callback=self.parse)def parse_details(self,response):obj=BookItem()obj["title"]=response.xpath("//div[@id='wrapper']/h1/span/text()").get()obj["publisher"]=response.xpath("//div[@id='content']//div[@id='info']/a/text()").get()# print(title,publisher)yield obj

2.1.4.保存数据模型到数据库

piplines.py（用到piplines就需要到settings里面打开ITEM_PIPELINES）

class BookPipeline:def __init__(self):self.client=pymongo.MongoClient("mongodb://localhost:27017")self.db=self.client["douban"]self.col=self.db["book"]def process_item(self, item, spider):self.col.insert_one(dict(item))return itemdef __del__(self):print("end")

2.1.5.request和response中间件配置

middlewares.py（用到middlewares就需要到settings里面打开SPIDER_MIDDLEWARES或者DOWNLOADER_MIDDLEWARES）
主要功能：随机useragent，代理ip，使用selenium，添加cookie，主要是在DownloaderMiddleware类的process_request和process_response方法。

#随机useragent
def process_request(self, request, spider):usa=["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134"]request.headers["Use-Agent"]=random.choice(usa)return None

2.1.6.获取携带cookies

class AppSpider(scrapy.Spider):name = 'app'# allowed_domains = ['www.cheshi.com']# start_urls = ['https://my.cheshi.com/user/']def start_requests(self):#1.使用cookie直接访问url='https://my.cheshi.com/user/'cookies="pv_uid=1732254874878; cheshi_UUID=01JD96ZEWPDK2ZKK8X3WGFFPVS; cheshi_pro_city=MV%2FljJfkuqxfMV%2FkuJzln47ljLpfYmVpamluZw%3D%3D; Hm_lvt_8fe47348e12ba11be217fd389b115472=1732254888,1732494411; HMACCOUNT=0F938E3E8702278B; Hm_lvt_ed9cf33799965fb6c868762ac84e663e=1732674587; PHPSESSID=bd0f056bb72ef681c01a68b853bde882; cheshi_user_prevLogintime=1732674716; Hm_lpvt_ed9cf33799965fb6c868762ac84e663e=1732675618; cheshi_token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiIsImp0aSI6ImNoZXNoaV9oNV9zaWduIn0.eyJpc3MiOiJodHRwczpcL1wvYXBpLmNoZXNoaS5jb20iLCJhdWQiOiJodHRwczpcL1wvYXBpLmNoZXNoaS5jb20iLCJqdGkiOiJjaGVzaGlfaDVfc2lnbiIsImlhdCI6MTczMjY3NTY2OSwibmJmIjoxNzMyNjc1NzI5LCJleHAiOjE3MzMyODA0NjksInVpZCI6IjkxMDcxNDIifQ.txhZVRGAHLpMP8whjQ3fPcgNicQmVYake_8s0J1EKzk; cheshi_user_authv2=MzI2NDUyNAlsaXR0bGVaCXYyCWJjZDYzMWQ4NDZlMTQ4ZWQwY2UzZThhMTFkYTE2YmQxCTE3MzI2NzU2NjkJMzMyMjhkMGRmNDc3ZTc2YmVlMTQ0Y2JmODg1Zjk5OTY=; cheshi_user_info=OTEwNzE0MglsaXR0bGVaCXYyCWJjZDYzMWQ4NDZlMTQ4ZWQwY2UzZThhMTFkYTE2YmQxCTE3MzI2NzU2NjkJMzMyMjhkMGRmNDc3ZTc2YmVlMTQ0Y2JmODg1Zjk5OTYJaHR0cHM6Ly9pbWcuY2hlc2hpLWltZy5jb20vdXNlcnBob3RvL25ldy85MTA3MTQyL2FkNzc3NzZkYmRiM2M5NGFhZTE2YWYyM2M3OGRlMjkzLmpwZwkwCXdhbmdzaGFuZ2NoZXNoaQ==; cheshi_user_info_for_index=OTEwNzE0MglsaXR0bGVaCXYyCWJjZDYzMWQ4NDZlMTQ4ZWQwY2UzZThhMTFkYTE2YmQxCTE3MzI2NzU2NjkJMzMyMjhkMGRmNDc3ZTc2YmVlMTQ0Y2JmODg1Zjk5OTYJaHR0cHM6Ly9pbWcuY2hlc2hpLWltZy5jb20vdXNlcnBob3RvL25ldy85MTA3MTQyL2FkNzc3NzZkYmRiM2M5NGFhZTE2YWYyM2M3OGRlMjkzLmpwZwkwCXdhbmdzaGFuZ2NoZXNoaQ==; lv=1732685873; vn=8; Hm_lpvt_8fe47348e12ba11be217fd389b115472=1732685874; pv_cheshit=1732685902509; pv_source="cookies={ item.split("=")[0] : item.split("=")[1] for item in cookies.split("; ")}yield scrapy.Request(url=url,callback=self.parse,cookies=cookies)#2.通过login获取cookieurl_login="https://api.cheshi.com/services/common/api.php?api=login.Login"data={"act": "login","mobile": "18811752638","source": "pc","password": "PASSWORD","hold_time": "yes",}yield scrapy.FormRequest(url=url_login,formdata=data,callback=self.parse)def parse(self, response):# print(response.text)#这个是cookies，不用设置，scrapy后台自动会保存携带这个cookiesurl="https://my.cheshi.com/user/"yield scrapy.Request(url=url,callback=self.parse_admin)def parse_admin(self,response):print(response.text)

2.2.crawl

scrapy genspider -t crawl app https://seller.cheshi.com/beijing/

rules匹配访问链接

class AppSpider(CrawlSpider):name = 'app'allowed_domains = ['seller.cheshi.com']start_urls = ['https://seller.cheshi.com/beijing/']rules = (Rule(LinkExtractor(allow=r'seller.cheshi.com/\d+',deny=r"seller.cheshi.com/\d+/.+"), callback='parse_item', follow=True),)def parse_item(self, response):title=response.xpath("//div[@class='clearfix']//a[@class='name']/text()").get()print(title,response.url)

2.3.使用redis分布式爬取

2.3.1.redis

import redis#pip install redis
db=redis.Redis(host="localhost",port="6379",decode_responses=True)
#string 存value
db.set("name","小12")#添加1个键值对
print(db.get("name"))#获取1个值
db.mset({"name1":"老王","age1":"老衲年方28"})#添加多个键值对
print(db.mget("name1","name","age1"))#获取多个值
# #hash 存key-value
db.hset("hash1","key1","value1")#添加
db.hset("hash1","key2","value2")
db.hset("hash1","key3","value3")
print(db.hget("hash1","key2"))#获取hash中key对应的值
print(db.hgetall("hash1"))#获取hash中所有的键值对
# list 存list
db.lpush("list1",1,2,3)#倒序插入，先进后出
db.rpush("list2",2,3,4,5)#顺序插入，先进先出
print(db.llen("list1"))#list的长度
print(db.lrange("list1",0,-1))#lrange key start stop(-1 在 Redis 中是一个特殊的索引，表示列表的最后一个元素)
#set
db.sadd("set1",55,66,77,55)
print(db.scard("set1"))#scard获取set的长度
print(db.smembers("set1"))#smembers获取set的所有元素
#zset
db.zadd("zset1", {"zoe": 22, "jodie": 11})#Redis 的 Sorted Set（有序集合）要求每个成员（member）都关联一个分数（score）
print(db.zcard("zset1"))
print(db.zrange("zset1",0,-1,withscores=True))#[('jodie', 11.0), ('zoe', 22.0)]

2.3.2.使用scrapy_redis爬取数据到redis

import scrapy
import json
import re#正则匹配
from urllib import parse#url的编解码
from ..items import JdItem
from scrapy_redis.spiders import RedisSpider#1.使用redis分布式爬 #5.设置settings里面的配置
#6.redis里面添加key,并传递初始url:lpush jingdong https://gw-e.jd.com/client.action?callback=func&body=%7B%22moduleType%22%3A1%2C%22page%22%3A1%2C%22pageSize%22%3A20%2C%22scopeType%22%3A1%7D&functionId=bookRank&client=e.jd.com&_=1732667213092
#7.启动分布式爬虫:scrapy crawl app
class AppSpider(RedisSpider):#2.继承RedisSpider# def __init__(self):#     self.page=1def __init__(self, *args, **kwargs):#4.分布式爬虫类的初始化domain = kwargs.pop("domain", "")self.allowed_domains = filter(None, domain.split(","))super(AppSpider, self).__init__(*args, **kwargs)self.page=1name = 'app'# allowed_domains = ['channel.jd.com']# start_urls = ['https://gw-e.jd.com/client.action?callback=func&body=%7B%22moduleType%22%3A1%2C%22page%22%3A1%2C%22pageSize%22%3A20%2C%22scopeType%22%3A1%7D&functionId=bookRank&client=e.jd.com&_=1732667213092']redis_key="jingdong"#3.设置rediskeydef parse(self, response):match = re.search(r'func\((\{.*\})\)', response.text)json_str = match.group(1) if match else Noneif json_str is not None:json_data=json.loads(json_str)#字符串转jsonobj=JdItem()for book in json_data["data"]["books"]:obj["title"]=book["bookName"]obj["price"]=book["sellPrice"]# print(title,price)yield objself.page+=1next_url = '{{"moduleType":1,"page":{page},"pageSize":20,"scopeType":1}}'.format(page=self.page)#字符串---------编码next_url="https://gw-e.jd.com/client.action?callback=func&body="+parse.quote(next_url)+"&functionId=bookRank&client=e.jd.com&_=1732667213092"#编码---------符串print(next_url)yield scrapy.Request(url=next_url,callback=self.parse,dont_filter=True)

settings.py

BOT_NAME = 'jd'SPIDER_MODULES = ['jd.spiders']
NEWSPIDER_MODULE = 'jd.spiders'USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15.7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36"
ROBOTSTXT_OBEY = False
LOG_LEVEL = "ERROR"
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
SCHEDULER_PERSIST = True
REDIS_URL = "redis://127.0.0.1:6379"
DOWNLOAD_DELAY = 1
ITEM_PIPELINES = {"jd.pipelines.JdPipeline": 300,#写自己的pipline文件地址"scrapy_redis.pipelines.RedisPipeline": 400
}

2.3.3.redis到mongondb

db_redis = redis.Redis(host="localhost", port="6379", decode_responses=True)
client_mongo = pymongo.MongoClient("mongodb://localhost:27017")
db_mongo = client_mongo["Redis2Mongo"]
col_mongo = db_mongo["C1"]for i in db_redis.lrange("list2", 0, -1):page = {# "title": json.loads(i)["title"]"value":i}res = col_mongo.insert_one(page)print(res.inserted_id)

3.selenium模拟驱动

from selenium import webdriver#pip install selenium
import time
driver = webdriver.Chrome(executable_path="./_resources/chromedriver.exe")
driver.get("https://www.cheshi.com/")
print(driver.current_url)#url
with open("./cheshi.html","w", encoding="utf-8") as f:f.write(driver.page_source)#源码
driver.save_screenshot("cheshi.png")#网页截图
time.sleep(1)
driver.quit()

3.1orc识别数字字母验证码

#1.买云上ocr，2.pytesseract
#pip install pytesseract pillow (https://digi.bib.uni-mannheim.de/tesseract/下载，并添加到环境变量，tesseract -v测试)
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
import time
import cv2 as cv
from PIL import Image
import pytesseract
import re
#1.首先通过xpath，将网页验证码保存下来
driver = webdriver.Chrome(executable_path="./_resources/chromedriver.exe")
driver.get("https://service.cheshi.com/user/login.php")
time.sleep(1)
while driver.current_url=="https://service.cheshi.com/user/login.php":#如果一直在登陆页面，就是识别不正确，再重来img=driver.find_element(By.XPATH,"//img[@class='yzm_img']")img.screenshot("./captcha.png")time.sleep(1)# driver.quit()#2.对图片进行二值化和形态学操作和去噪点处理等操作img2=cv.imread("./captcha.png",flags=cv.IMREAD_GRAYSCALE)thresh,binary=cv.threshold(img2,120,255,cv.THRESH_BINARY)# 噪点处理def interference_point(img):h, w = img.shape[:2]# 遍历像素点进行处理for y in range(0, w):for x in range(0, h):# 去掉边框上的点if y == 0 or y == w - 1 or x == 0 or x == h - 1:img[x, y] = 255continuecount = 0if img[x, y - 1] == 255:count += 1if img[x, y + 1] == 255:count += 1if img[x - 1, y] == 255:count += 1if img[x + 1, y] == 255:count += 1if count > 2:img[x, y] = 255return img# kernel = cv.getStructuringElement(cv.MORPH_RECT, (4, 4))# result = cv.morphologyEx(binary, cv.MORPH_OPEN, kernel=kernel)# gray = cv.GaussianBlur(result, (5, 5), 0)  # 高斯滤波# result = cv.Canny(gray, 75, 250)  # Canny边缘检测result=interference_point(binary)cv.imwrite("./captcha2.png",result)#3.用Tesseract-OCR识别pytesseract.pytesseract.tesseract_cmd = r'D:\Softwares\Tesseract-OCR\tesseract.exe'textImage = Image.fromarray(result)text = pytesseract.image_to_string(textImage)print(text)#4.对识别结果做进一步处理exp=re.compile("[a-zA-Z0-9]")out=exp.findall(text)out = ''.join([str(i) for i in out])print("The result:", out)#实现登录phone=driver.find_element(By.XPATH,"//input[@class='phone']")phone.clear()ActionChains(driver).pause(0.5).click(phone).send_keys("18811752638").perform()#0.5秒后点击phone的input元素，然后填内容yzm=driver.find_element(By.XPATH,"//input[@id='imgyzm']")yzm.clear()ActionChains(driver).pause(0.5).click(yzm).send_keys(out).perform()fsyzm=driver.find_element(By.XPATH,"//span[@class='sendyzm_btn blue']")fsyzm.click()time.sleep(20)login=driver.find_element(By.XPATH,"//input[@name='sub']")login.click()time.sleep(4)
print(driver.page_source)
time.sleep(4)
driver.quit()

3.2.opencv模板匹配滑块验证码

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import cv2 as cv
from PIL import Image
import numpy
import requests
import re
#1.首先拿到滑块验证码图片
driver = webdriver.Chrome(executable_path="./_resources/chromedriver.exe")
driver.get("https://www.liepin.com/")
time.sleep(3)
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '/html/body/section[2]/div[2]/div/div/div/div/div[2]/div/div[2]'))
).click()
username = WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, '//*[@id="login"]'))
)
username.send_keys("18811752638")
password = WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, '//*[@id="pwd"]'))
)
password.send_keys("xxxx")
checkbox = driver.find_element(By.CLASS_NAME, "ant-checkbox-input")
checkbox.click()
login = WebDriverWait(driver, 30).until(EC.element_to_be_clickable((By.XPATH, '/html/body/section[2]/div[2]/div/div/div/div/div[3]/div/form/button'))
)
login.click()
time.sleep(8)
#切换iframe，再拿到验证码图片
driver.switch_to_frame("tcaptcha_iframe")
while driver.current_url=="https://www.liepin.com/":#避免出错,出错重复尝试refresh = driver.find_element(By.XPATH, '//*[@id="reload"]/div')refresh.click()time.sleep(2)back_url=driver.find_element(By.XPATH,'//*[@id="slideBg"]').get_attribute("src")back=requests.get(back_url)with open("./back.png","wb") as f:f.write(back.content)front_url=driver.find_element(By.XPATH,'//*[@id="slideBlock"]').get_attribute("src")front=requests.get(front_url)with open("./front.png","wb") as f:f.write(front.content)#2.opencv计算滑动距离back=cv.imread("./back.png",flags=cv.IMREAD_GRAYSCALE)front=cv.imread("./front.png",flags=cv.IMREAD_GRAYSCALE)front=front[24:front.shape[0]-24,24:front.shape[1]-24]#小滑块图片裁剪处理一下thresh,back=cv.threshold(back,110,255,cv.THRESH_BINARY)#图片二值化处理  thresh,front=cv.threshold(front,40,255,cv.THRESH_BINARY_INV)cv.imwrite("./back_p.png",back)cv.imwrite("./front_p.png",front)match=cv.matchTemplate(back,front,cv.TM_CCORR_NORMED)distance=cv.minMaxLoc(match)[3][0]distance=distance*341//680-37#因为前端渲染的图片是经过压缩的，所以这里也做等比例缩小，-37是因为front左边有37# print(distance)#3.使用selenium模拟滑块滑动slider = driver.find_element(By.XPATH, '//*[@id="tcaptcha_drag_thumb"]')ActionChains(driver).pause(0.2).click_and_hold(slider).pause(0.2).move_by_offset(distance / 4, 5).perform()#避免被识别，分三次滑动ActionChains(driver).pause(0.1).move_by_offset(distance / 2, -2).perform()ActionChains(driver).pause(0.1).move_by_offset(distance / 4, 3).release().perform()time.sleep(3)driver.get("https://www.liepin.com/career/golang/")
driver.quit()

4.js逆向

首先使用网站：https://curlconverter.com/ 可以自动编写requests请求代码（右键，复制，以curlbah格式复制，然后去网站进行粘贴生成）

4.1.response混淆加密

加密方式很多，一般response的内容使用AED和DES对称加密，自定义加密等；请求头和请求负载一般使用RAS非对称加密，md5,Sha256和Base64等加密方式。
解密首先使用关键词（encrypt，decrypt，json.parse,路径和返回参数名等，打断点，写js代码破解加密数据，python请求调用js），或者启动器直接进入源代码。启动器里面有Promis.then就是异步ajax请求，搜索json.parse
一般

4.1.1.js解密

使用js解密需要安装库，然后在python里面调用js代码也要安装python包

npm install crypto-js
npm install jsdom
npm install base64-js 
import execjs#pip install pyexecjs2 
ctx=execjs.compile(open('./3zhaobiao.js','r',encoding='utf-8').read()).call('decryptByDES',response.json())#调用这个js里面的decryptByDES函数，并给这个函数传递后面的参数，这里是response.json()

4.1.1.1.AES

有ECB和CBC两种模式，在 ECB 和 CBC 模式下，如果明文长度不是 8 字节的整数倍，需要进行填充，填充后密文长度是 8 字节的整数倍。如果明文长度是 8 字节的整数倍，密文长度等于明文长度（在没有添加额外认证信息等情况下）。

ECB

ECB 模式(Electronic Codebook) 模式，这是一种比较简单的 AES 模式。ECB 模式在加密时将每个块独立加密，没有使用初始化向量 (IV)。在解密过程中使用了 unpad 方法来去除填充。由于 AES 的块大小为 16 字节，所以使用 unpad(info, 16) 去除多余的填充字节。

const  CryptoJS=require("crypto-js")
function a(e, t) {var n = CryptoJS.enc.Utf8.parse(t )//|| "46cc793c53dc451b")return CryptoJS.AES.decrypt(e, n, {mode: CryptoJS.mode.ECB,padding: CryptoJS.pad.Pkcs7}).toString(CryptoJS.enc.Utf8)}
data1='GsGEjP8dAjIYaDNgCxOkJYJQrOECQf8iB+5dv6+yI='
data = JSON.parse(a(data1,"efabccee-b754-4c"))
console.log(data)

CBC

CBC 模式 (Cipher Block Chaining) 模式在加密时使用了 初始化向量 (IV)，并且每个数据块的加密结果会影响后续的数据块，因此相同的明文在不同的 IV 下加密得到的密文是不同的。使用 unpad 去除填充，确保解密后的数据符合预期。

const CryptoJS=require("crypto-js")
function b(t) {var e = CryptoJS.enc.Utf8.parse("EB444973714E4A40876CE66BE45D5930"), n = CryptoJS.enc.Utf8.parse("B5A8904209931867"), a = CryptoJS.AES.decrypt(t, e, {iv: n,mode: CryptoJS.mode.CBC,padding: CryptoJS.pad.Pkcs7});return a.toString(CryptoJS.enc.Utf8)
}
t='MZphJmFlelDpw2aSCfdFb5yXTSQuTw=='
console.log(b(t))

4.1.1.2.DES

一般情况下，使用 DES 加密后的密文长度是 8 字节的整数倍，且不超过 64 位的整数倍（因为分组长度是 64 位）。

参数	含义
iv	16进制初始向量，需要获取
mode	加密模式，一般写死
padding	填充方法，一般写死
return ciphertext.toString（）	密文，以16进制字符串返回

const CryptoJS=require("crypto-js")
function b(t) {var e = CryptoJS.enc.Utf8.parse("EB444973714E4A40876CE66BE45D5930"), n = CryptoJS.enc.Utf8.parse("B5A8904209931867"), a = CryptoJS.AES.decrypt(t, e, {iv: n,mode: CryptoJS.mode.CBC,padding: CryptoJS.pad.Pkcs7});return a.toString(CryptoJS.enc.Utf8)
}

4.1.1.3.RSA

非对称加解密算法，公钥加密，私钥解密,同样数据加密结果不一样。密文长度等于密钥长度。

//登录接口的account和password都是经过加密的，登录按钮的点击函数全局搜索或者click事件，进入函数，查看函数（搜加密，注释有）
window = global;
const JSEncrypt=require("jsencrypt")//npm install jsencrypt
global.window = {}; 
function et(_0x32033c) {var _0x283d00 = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsgDq4OqxuEisnk2F0EJFmw4xKa5IrcqEYHvqxPs2CHEg2kolhfWA2SjNuGAHxyDDE5MLtOvzuXjBx/5YJtc9zj2xR/0moesS+Vi/xtG1tkVaTCba+TV+Y5C61iyr3FGqr+KOD4/XECu0Xky1W9ZmmaFADmZi7+6gO9wjgVpU9aLcBcw/loHOeJrCqjp7pA98hRJRY+MML8MK15mnC4ebooOva+mJlstW6t/1lghR8WNV8cocxgcHHuXBxgns2MlACQbSdJ8c6Z3RQeRZBzyjfey6JCCfbEKouVrWIUuPphBL3OANfgp0B+QG31bapvePTfXU48TYK0M5kE+8LgbbWQIDAQAB';var _0x1defd6 = new JSEncrypt();_0x1defd6['setPublicKey'](_0x283d00);var _0x4bd6d3 = _0x1defd6['encrypt'](_0x32033c);return _0x4bd6d3;
}
console.log(et("18811752638"))

4.1.1.4.MD5

md5加密一般是16进制数字字母，不带符号，32个字符，同一个参数加密结果一样，不可逆，只可爆破
载荷数据，传递的是json数据，也就是request的时候json参数数据
表单数据，传递的是字典，也就是request的时候data参数数据，提交表单的时候使用

//生成请求头参数portal-sign
const CryptoJS=require("crypto")
e={"ts": (new Date).getTime(),"type": "12","IS_IMPORT": 1,"pageSize": 3
}
function l(t, e) {return t.toString().toUpperCase() > e.toString().toUpperCase() ? 1 : t.toString().toUpperCase() == e.toString().toUpperCase() ? 0 : -1
} 
function u(t) {for (var e = Object.keys(t).sort(l), n = "", a = 0; a < e.length; a++)if (void 0 !== t[e[a]])if (t[e[a]] && t[e[a]]instanceof Object || t[e[a]]instanceof Array) {var i = JSON.stringify(t[e[a]]);n += e[a] + i} elsen += e[a] + t[e[a]];return n
}
function s(text) {return CryptoJS.createHash('md5').update(text).digest('hex');}
function d(t) {for (var e in t)"" !== t[e] && void 0 !== t[e] || delete t[e];var n = "B3978D054A72A7002063637CCDF6B2E5" + u(t);return s(n).toLocaleLowerCase()
}
console.log(d(e)) 

4.1.1.5.sha256加密

sha256加密（长度64位字符，数字字母，不带符号）

const CryptoJS = require('crypto-js');const hash = CryptoJS.SHA256('hello world').toString(CryptoJS.enc.Hex);
console.log(hash);

4.1.1.6.Base64加密

长度72位有可能会是（数字字母，最后有=）

//登录接口的account和password都是经过加密的，登录按钮的点击函数全局搜索或者click事件，进入函数，查看函数（搜加密，注释有）
window = global;
const JSEncrypt=require("jsencrypt")//npm install jsencrypt
global.window = {}; 
function et(_0x32033c) {var _0x283d00 = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsgDq4OqxuEisnk2F0EJFmw4xKa5IrcqEYHvqxPs2CHEg2kolhfWA2SjNuGAHxyDDE5MLtOvzuXjBx/5YJtc9zj2xR/0moesS+Vi/xtG1tkVaTCba+TV+Y5C61iyr3FGqr+KOD4/XECu0Xky1W9ZmmaFADmZi7+6gO9wjgVpU9aLcBcw/loHOeJrCqjp7pA98hRJRY+MML8MK15mnC4ebooOva+mJlstW6t/1lghR8WNV8cocxgcHHuXBxgns2MlACQbSdJ8c6Z3RQeRZBzyjfey6JCCfbEKouVrWIUuPphBL3OANfgp0B+QG31bapvePTfXU48TYK0M5kE+8LgbbWQIDAQAB';var _0x1defd6 = new JSEncrypt();_0x1defd6['setPublicKey'](_0x283d00);var _0x4bd6d3 = _0x1defd6['encrypt'](_0x32033c);return _0x4bd6d3;
}
console.log(et("18811752638"))

4.1.2.python解密

4.1.2.1.AES

ECB

data1=response.json()["data1"]
def AES_decrypt(data1):html = base64.b64decode(data1)key = b"40w42rjLEXxYhxRn"aes = AES.new(key, AES.MODE_ECB)info = aes.decrypt(html)decrypt_data = unpad(info, 16).decode()return decrypt_data
data_out=AES_decrypt(data1)
print(data_out)

CBC

def decrypt_aes_cbc(ciphertext_base64, key, iv):ciphertext = base64.b64decode(ciphertext_base64)# 将 base64 编码的密文解码为字节cipher = AES.new(key, AES.MODE_CBC, iv)# 创建 AES 解密器，使用 CBC 模式和给定的 IVdecrypted_data = unpad(cipher.decrypt(ciphertext), AES.block_size)# 解密并去除填充return decrypted_data.decode('utf-8')# 将解密后的字节转为字符串（假设是 UTF-8 编码）
key = "EB444973714E4A40876CE66BE45D5930"  # 替换为实际密钥
iv = "B5A8904209931867"    # 替换为实际 IV(16位)
decrypted_message = decrypt_aes_cbc(response.json()["Data"],key.encode('utf-8'), iv.encode('utf-8'))# 解密
print("Decrypted message:", decrypted_message)

4.1.2.2.DES

import requests
from Crypto.Cipher import DES
from Crypto.Util.Padding import unpad
import base64def decrypt_by_des(ciphertext_base64, key):# 解码 Base64 密文ciphertext = base64.b64decode(ciphertext_base64)# 创建 DES 解密对象，使用 ECB 模式，填充方式为 PKCS7cipher = DES.new(key.encode('utf-8'), DES.MODE_ECB)# 解密并去除填充decrypted_bytes = unpad(cipher.decrypt(ciphertext), DES.block_size)# 将解密后的字节转回字符串return decrypted_bytes.decode('utf-8')

4.1.2.3.Base64

import math
import time
import base64
time1 = math.floor(time.time() * 1000)
mcode = base64.b64encode(str(time1).encode()).decode()
print(mcode)

4.2.hook注入反debug

var AAA = Function.prototype.constructor;
Function.prototype.constructor = function(x) {if (x!= "debugger") {return AAA(x);}return function() {};
};