爬蟲--requests模組

阿新 • • 發佈：2022-05-08

requests模組的get操作

1.導包

import requests

2.get操作的三個引數

requests.get(url,params,headers)

url

params :get請求攜帶的引數

heraders:UA偽裝

url = 'https://www.sogou.com/web'

param = {
    'query':'RMB'
}

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}

requests模組get操作例項

import requests
wd = input('enter a word:')
url = 'https://www.sogou.com/web'
#引數的封裝
param = {
    'query':wd
}
#UA偽裝
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}
response = requests.get(url=url,params=param,headers=headers)
#手動修改響應資料的編碼
response.encoding = 'utf-8'
page_text = response.text
fileName = wd + '.html'
with open(fileName,'w',encoding='utf-8') as fp:
    fp.write(page_text)
print(fileName,'爬取成功！！！')

requests模組的post操作

1.導包

import requests

2.post操作的三個引數

requests.post(url,data,headers)

url

data :post請求攜帶的引數

heraders:UA偽裝

requests模組post操作例項

#破解百度翻譯
url = 'https://fanyi.baidu.com/sug'
word = input('enter a English word:')
#請求引數的封裝
data = {
    'kw':word
}
#UA偽裝
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}
response = requests.post(url=url,data=data,headers=headers)
#text:字串  json():物件
obj_json = response.json()

print(obj_json)

requests模組的post請求處理ajax請求獲取資料例項

#爬取任意城市對應的肯德基餐廳的位置資訊
#動態載入的資料
import requests
city = input('enter a cityName:')
url = 'http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword'
data = {
    "cname": "",
    "pid": "",
    "keyword": city,
    "pageIndex": "2",
    "pageSize": "10",
}
#UA偽裝
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}
response = requests.post(url=url,headers=headers,data=data)

json_text = response.text

print(json_text)

練習

1.爬取豆瓣電影中更多的電影詳情資料 https://movie.douban.com/typerank?type_name=%E5%8A%A8%E4%BD%9C&type=5&interval_id=100:90&action="

2.http://125.35.6.84:81/xk/ 爬取每家企業的企業詳情資料

#豆瓣網爬取
import requests
for i in range(0,1000000,20):
    url = 'https://movie.douban.com/j/chart/top_list'
    data = {
        'type': '11',
        'interval_id': '100:90',
        'action': '',
        'start': str(i),
        'limit': '20'
    }
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
    }
    res = requests.post(url=url, headers=headers, data=data).json()
    print(res)