python爬蟲系列(4.5-使用urllib模組方式下載圖片)
一、回顧urllib包中下載圖片的方式
1、urlretrieve下載檔案
from urllib import request
if __name__ == "__main__":
# 下載整個網頁
request.urlretrieve('http://www.baidu.com', 'baidu.html')
# 下載圖片
request.urlretrieve('http://www.baidu.com/img/bd_logo1.png', 'baidu.png')
import os
import shutil
from urllib import request
import requests
from lxml import etree
class DownImage(object):
"""
建立一個下載圖片的類
"""
def __init__(self):
self.urls = ['http://python.jobbole.com/category/guide/page/{0}/'.format(x) for x in range(2)]
self.headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36',
}
@property
def create_dir(self):
"""
定義建立一個資料夾的方法(有就刪除沒用就刪除)
:return:
"""
path = os.path.join(os.path.dirname(__file__), 'jobbole')
if os.path.exists(path):
shutil.rmtree(path)
else:
os.makedirs(path)
return path
@property
def get_html(self):
"""
請求頁面,返回標題及圖片的url地址
:return:
"""
img_list = []
for item in self.urls:
response = requests.get(url=item, headers=self.headers)
if response.status_code == 200:
html = etree.HTML(response.text)
img_urls = html.xpath('//div[@class="post floated-thumb"]')
for img in img_urls:
url = img.xpath('./div[@class="post-thumb"]//img/@src')[0]
img_list.append(url)
print(img_list)
return img_list
def main(self):
"""
定義下載圖片的
:return:
"""
path = self.create_dir
if os.path.exists(path):
for img_url in self.get_html:
img_name = img_url.rsplit('/')[-1]
print(img_url)
request.urlretrieve(img_url, os.path.join(path, img_name))
if __name__ == '__main__':
down_image = DownImage()
down_image.main()