Python多執行緒爬蟲學習
阿新 • • 發佈:2018-12-10
此程式碼包含以下學習內容:
- 程式封裝
- 網頁獲取與解析
- retry裝飾器與thread多執行緒
Anime code
# -*- coding: utf-8 -*-
import requests
import demjson
from retry import retry
from config import *
class DongMan:
def __init__(self):
"""
initialize the picture source and page wanna download
"""
self.dongman_list = input('input content you wanna scratch, use \, to separate >')
self.dongman_list = self.dongman_list.split(',')
self.page_list = input('input pages:>')
@retry(3)
def index(self):
"""
scratch all the path on one page
"""
for j in self.dongman_list:
self.j = j
for i in range(1, int(self.page_list) + 1):
url = 'http://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=%s&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=-1&z=&ic=0&word=%s&s=&se=&tab=&width=&height=&face=0&istype=2&qc=&nc=1&fr=&expermode=&pn=%s&rn=30&gsm=10e&1539825490240=' % (
str(j), str(j), str(30 * i))
headers = {
'Referer': 'http://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=result&fr=&sf=1&fmq=1539824775271_R&pv=&ic=0&nc=1&z=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&hs=2&word=%E6%89%8B',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36'
}
reponse = requests.get(url=url, headers=headers)
img_res = reponse.text.encode('utf-8').decode('latin-1')
img_res = demjson.decode(img_res) # decode the json format
all_img = img_res['data'] # get img link
self.details(all_img)
@retry(3)
def details(self, all_img):
"""
recursive download the image
:param all_img: list of the picture
:return:
"""
for each_url in all_img:
each_img_url = each_url.get('thumbURL')
print(each_img_url)
if each_img_url:
headers = {
'Referer': 'http://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=result&fr=&sf=1&fmq=1539824775271_R&pv=&ic=0&nc=1&z=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&hs=2&word=%E6%89%8B',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36'
}
try:
reponse = requests.get(url=each_img_url, headers=headers, timeout=10)
except BaseException as e:
print(e)
img_res = reponse.content # obtain the image bytes
img_save(img_res)
# insertDB((None, each_img_url, md5(each_img_url.encoding('utf-8')).hexdigest(), self.j), 'image_test', 'img_source')
if __name__ == '__main__':
try:
dong_man = DongMan()
use_threadpool(dong_man.index())
except TypeError as e:
print('FINISH!!!')
config code
# -*- coding: utf-8 -*-
import sys
import random
import threadpool
import pytesseract
from PIL import Image
from io import BytesIO
def use_threadpool(method):
"""
using threadpool
:return:
"""
task_pool = threadpool.ThreadPool(5) # use 5 threads
requests_pool = threadpool.makeRequests(url_down, url_all)# 假設共3000個url,reqeusts_pool為建立的3000個工作請求。
print(len(url_all)) # 3000
for req in requests_pool: # 對於每一個工作請求,將請求放到執行緒池子裡面,由5個執行緒去取池子裡面的任務,若當前執行緒執行完此方法,則拿池子中下一個任務,直至池子中沒有任務為止。
task_pool.putRequest(req)
task_pool.wait()
def get_captch(captcha_content):
"""
:param captcha_content: transfer as type
:return:
"""
image = Image.open(BytesIO(captcha_content))
# convert as gray
imgry = image.convert('L')
table = [0 if i < 140 else 1 for i in range(256)]
# emphasis the font
out = imgry.point(table, '1')
# out.show()
captcha = pytesseract.image_to_string(out)
captcha = captcha.strip()
captcha = captcha.upper()
return captcha
def img_save(img_content):
# """
# save image
# param img_content: pic bytes format:# \xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x0...
# """
image = Image.open(BytesIO(img_content))
# 1. write bytes format into RAM
# 2. use Iamge Module def .open to upload the RAM bytes into image format
if int((sys.getsizeof(image))) >= 11: # getsizeof -> Return the size of an object in bytes.
with open('./image/image%s.jpg' % random.random(), 'wb') as f:
f.write(img_content)
else:
print('this pic less than 11 bytes')
image.show()