爬蟲之代理、亂碼、驗證碼
阿新 • • 發佈：2019-01-10
代理
爬蟲中為什麼需要使用代理
一些網站會有相應的反爬蟲措施，例如很多網站會檢測某一段時間某個IP的訪問次數，如果訪問頻率太快以至於看起來不像正常訪客，它可能就會會禁止這個IP的訪問。
所以我們需要設定一些代理IP，每隔一段時間換一個代理IP，就算IP被禁止，依然可以換個IP繼續爬取。
代理的分類
正向代理：代理客戶端獲取資料。正向代理是為了保護客戶端防止被追究責任。
反向代理：代理伺服器提供資料。反向代理是為了保護伺服器或負責負載均衡。
免費代理ip提供網站
http://www.goubanjia.com/ 西祠代理 快代理
簡單運用示例
import requests # ip 代理網站 http://www.goubanjia.com/ headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36' } url = 'https://www.baidu.com/s?wd=ip' page_text = requests.get(url=url,headers=headers,proxies={'https':'212.119.229.18:33852'}).text # print(page_text) with open('./ip.html','w',encoding = 'utf-8') as f: f.write(page_text) # proxy = [{},{},{}] 代理池的使用
亂碼
import requests from urllib import request from lxml import etree headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36', 'Connection':'close' } url = "http://pic.netbian.com/4kmeinv/" response = requests.get(url,headers) response.encoding = 'gbk' # print(response.encoding) page_text = response.text # print(page_text) tree = etree.HTML(page_text) li_list = tree.xpath('//div[@class="slist"]/ul/li') for li in li_list: img_src = 'http://pic.netbian.com' + li.xpath('./a/img/@src')[0] img_name = li.xpath('./a/img/@alt')[0] # img_name = img_name.encode('gbk').decode('utf-8') # img_name = img_name.encode('ISO-8859-1').decode('gbk') # ISO-8859-1 範圍 大於 gbk 大於 utf-8 print(img_src,img_name)
驗證碼
1. 雲打碼平臺
需要雲打碼平臺線上解析驗證碼
1.官方下載的 不用動 直接載入就可以
import http.client, mimetypes, urllib, json, time, requests ###################################################################### class YDMHttp: apiurl = 'http://api.yundama.com/api.php' username = '' password = '' appid = '' appkey = '' def __init__(self, username, password, appid, appkey): self.username = username self.password = password self.appid = str(appid) self.appkey = appkey def request(self, fields, files=[]): response = self.post_url(self.apiurl, fields, files) response = json.loads(response) return response def balance(self): data = {'method': 'balance', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey} response = self.request(data) if (response): if (response['ret'] and response['ret'] < 0): return response['ret'] else: return response['balance'] else: return -9001 def login(self): data = {'method': 'login', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey} response = self.request(data) if (response): if (response['ret'] and response['ret'] < 0): return response['ret'] else: return response['uid'] else: return -9001 def upload(self, filename, codetype, timeout): data = {'method': 'upload', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'codetype': str(codetype), 'timeout': str(timeout)} file = {'file': filename} response = self.request(data, file) if (response): if (response['ret'] and response['ret'] < 0): return response['ret'] else: return response['cid'] else: return -9001 def result(self, cid): data = {'method': 'result', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid)} response = self.request(data) return response and response['text'] or '' def decode(self, filename, codetype, timeout): cid = self.upload(filename, codetype, timeout) if (cid > 0): for i in range(0, timeout): result = self.result(cid) if (result != ''): return cid, result else: time.sleep(1) return -3003, '' else: return cid, '' def report(self, cid): data = {'method': 'report', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid), 'flag': '0'} response = self.request(data) if (response): return response['ret'] else: return -9001 def post_url(self, url, fields, files=[]): for key in files: files[key] = open(files[key], 'rb'); res = requests.post(url, files=files, data=fields) return res.text
2.普通使用者的呼叫
def get_code_text(code_type,img_path): # 使用者名稱 username = '普通使用者賬號 # 密碼 password = 普通使用者密碼 # 軟體ＩＤ，開發者分成必要引數。登入開發者後臺【我的軟體】獲得！ appid = 6578 # 軟體金鑰，開發者分成必要引數。登入開發者後臺【我的軟體】獲得！ appkey = '2b3ef98633145e0b478800905af4e10b' # 圖片檔案 filename = img_path # 驗證碼型別，# 例：1004表示4位字母數字，不同型別收費不同。請準確填寫，否則影響識別率。在此查詢所有型別 http://www.yundama.com/price.html codetype = code_type # 1004 # 超時時間，秒 timeout = 25 # 檢查 if (username == 'username'): print('請設定好相關引數再測試') else: # 初始化 yundama = YDMHttp(username, password, appid, appkey) # 登陸雲打碼 uid = yundama.login(); print('uid: %s' % uid) # 查詢餘額 balance = yundama.balance(); print('balance: %s' % balance) # 開始識別，圖片路徑，驗證碼型別ID，超時時間（秒），識別結果 cid, result = yundama.decode(filename, codetype, timeout); print('cid: %s, result: %s' % (cid, result)) return result
模擬登陸後抓取個人資訊資料
import requests from lxml import etree from urllib import request # 獲取一個session物件 session = requests.Session() #session物件和requests作用幾乎一樣,都可以進行請求的傳送,並且請求傳送的方式也是一致的, #session進行請求的傳送,如果會產生cookie的話,則cookie會自動被儲存到session物件中 #1 獲取驗證碼圖片 url = 'http://www.renren.com/' headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36' } page_text = session.get(url=url,headers=headers).text tree = etree.HTML(page_text) code_img_src = tree.xpath('//*[@id="verifyPic_login"]/@src')[0] request.urlretrieve(url=code_img_src,filename='./code.jpg') # 儲存到本地 code = get_code_text('2004','./code.jpg') # 獲取驗證碼文字資訊 # 模擬登陸 login_url = 'http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=2019031945879' # data資訊需要抓包工具 獲取
data = { "email":人人賬號, "icode":code if code else '', "origURL":"http://www.renren.com/home", "domain":"renren.com", "key_id":"1", "captcha_type":"web_login", "password":通過抓包工具獲取對應賬號的加密密碼, "rkey":"d4287c72b3f7ddf41b62170adec10265", "f":"http%3A%2F%2Fwww.renren.com%2F969397225", } print(session) #進行登入,當登入成功之後,可以獲取cookie response = session.post(url=login_url,headers=headers,data=data) # #對登入成功後對應的當前使用者的個人詳情頁進行請求傳送 detail_url = "http://www.renren.com/969397225/profile" # 個人資訊網址 page_text = session.get(url=detail_url,headers=headers).text with open('./renren.html','w',encoding='utf-8') as fp: fp.write(page_text) print('over')
古詩網
第一步 和人人一樣呼叫雲打碼
主程式碼
import requests from lxml import etree from urllib import request # 獲取一個session物件 session = requests.Session() #session物件和requests作用幾乎一樣,都可以進行請求的傳送,並且請求傳送的方式也是一致的, #session進行請求的傳送,如果會產生cookie的話,則cookie會自動被儲存到session物件中 headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36' } url = 'https://so.gushiwen.org/user/login.aspx?from=http://so.gushiwen.org/user/collect.aspx' page_text = requests.get(url=url,headers=headers).text # print(page_text) tree = etree.HTML(page_text) code_img_src = 'https://so.gushiwen.org' + tree.xpath('//*[@id="imgCode"]/@src')[0] # 獲取驗證碼生成地址 img_data = session.get(url=code_img_src,headers=headers).content # print(img_data) with open('./gushi.jpg',"wb") as fp: fp.write(img_data) code_text = get_code_text('1004','./gushi.jpg') print(code_text) # 每次訪問都會修改 每次訪問都獲取依稀 __VIEWSTATE = tree.xpath('//*[@id="__VIEWSTATE"]/@value')[0] __VIEWSTATEGENERATOR = tree.xpath('//*[@id="__VIEWSTATEGENERATOR"]/@value')[0] login_url = "https://so.gushiwen.org/user/login.aspx?from=http%3a%2f%2fso.gushiwen.org%2fuser%2fcollect.aspx" data = { "__VIEWSTATE":__VIEWSTATE, # 動態獲取 "__VIEWSTATEGENERATOR":__VIEWSTATEGENERATOR, # 動態獲取 "from":"http://so.gushiwen.org/user/collect.aspx", "email":"古詩文網站好", "pwd":"古詩文網的密碼.", "code":code_text, "denglu":"登入", } page_text = session.post(url=login_url,headers=headers,data=data).text with open('./gushiwen.html','w',encoding='utf-8') as fp: fp.write(page_text) print('ok')