多線程高容錯爬頭條街拍美圖
阿新 • • 發佈:2019-04-04
win64 The 容錯 改版 hash keys lac exception times 分析頭條的ajax,通過正則表達式,python3多線程高容錯爬取頭條的街拍美圖,保存到mongodb,並下載圖片
頭條的內容網頁較之前已經改版,圖床頁不僅有ajax的還有html的內容網頁
所以使用了兩種正則,根據條件調用
#!/usr/bin/env python # -*- coding:utf-8 -*- """ @author:Aiker @file:toutiao.py @time:下午9:35 """ import json import os import re from json import JSONDecodeError from multiprocessing import Pool from urllib.parse import urlencode from hashlib import md5 import pymongo import requests from requests.exceptions import RequestException MONGO_URL = ‘localhost:27017‘ MONGO_DB = ‘toutiao‘ MONGO_TABLE = ‘toutiao‘ GROUP_START = 1 GROUP_END = 20 KEYWORD = ‘街拍‘ client = pymongo.MongoClient(MONGO_URL, connect=False) db = client[MONGO_DB] headers = { ‘user-agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36‘ } def get_url(url): try: response = requests.get(url, headers=headers) if response.status_code == 200: return response.text return None except RequestException: print(‘請求失敗‘, url) return None def get_page_index(offset, keyword): data = { ‘aid‘: ‘24‘, ‘app_name‘: ‘web_search‘, ‘offset‘: offset, ‘format‘: ‘json‘, ‘keyword‘: keyword, ‘autoload‘: ‘true‘, ‘count‘: ‘20‘, ‘en_qc‘: ‘1‘, ‘cur_tab‘: ‘1‘, ‘from‘: ‘search_tab‘, ‘pd‘: ‘synthesis‘, ‘timestamp‘: ‘1124216535987‘ } url = ‘https://www.toutiao.com/api/search/content/?‘ + urlencode(data) # 字典對象轉化url對象 try: response = requests.get(url, headers=headers) if response.status_code == 200: return response.text return None except RequestException: print(‘請求索引頁失敗‘) return None def parse_page_index(html): try: data = json.loads(html) # 轉化為json對象 if data and ‘data‘ in data.keys(): # print(data.keys()) #調試,輸出所有key for item in data.get(‘data‘): if ‘article_url‘ in item: # 判斷是否存在,避免出現None # print(item) yield item.get(‘article_url‘) # 構造生成器 except JSONDecodeError: pass except TypeError: pass def get_page_detail(url): try: response = requests.get(url, headers=headers) if response.status_code == 200: return response.text return None except RequestException: print(‘請求詳情頁出錯‘, url) return None def parse_page_detail(html, url): pattern = re.compile("articleInfo:.*?title:\s‘(.*?)‘,.*?content:\s‘(.*?)‘.*?groupId", re.S) result = re.findall(pattern, html) # print(tc) if result: title, content = result[0] pattern = re.compile("(http://.*?)"", re.S) images = re.findall(pattern, content) # print(img) for image in images: download_image(image, title) # print(item) return { ‘title‘: title, ‘url‘: url, ‘images‘: images } else: pattern = re.compile(‘BASE_DATA.galleryInfo.*?title:\s\‘(.*?)\‘.*?gallery: JSON.parse\("(.*)"\)‘, re.S) result = re.findall(pattern, html) # print(result[0]) if result: title, content = result[0] data = json.loads(content.replace(‘\\‘, ‘‘)) # print(data) if data and ‘sub_images‘ in data.keys(): sub_images = data.get(‘sub_images‘) images = [item.get(‘url‘) for item in sub_images] for image in images: download_image(image,title) return { ‘title‘: title, ‘url‘: url, ‘images‘: images } def save_to_mongo(result): if db[MONGO_TABLE].insert(result): print(‘存儲到MongoDB成功‘, result) return True return False def download_image(url,title): print(‘正在下載‘, url) try: response = requests.get(url) if response.status_code == 200: save_image(response.content,title) return None except RequestException: print(‘請求圖片出錯‘, url) return None def save_image(content,title): try: if title: title = re.sub(‘[:?!!:?]‘, ‘‘, title) # 替換title中的特殊字符,避免建立資料夾目錄出錯 dir = ‘z:\\toutiao\\‘ if os.path.exists(dir + title): pass else: os.mkdir(dir + title) file_path = ‘{0}/{1}.{2}‘.format( dir + title, md5(content).hexdigest(), ‘jpg‘) if not os.path.exists(file_path): with open(file_path, ‘wb‘) as f: f.write(content) f.close() except OSError: pass def main(offset): html = get_page_index(offset, KEYWORD) for url in parse_page_index(html): print(url) html = get_page_detail(url) if html: result = parse_page_detail(html, url) if result: save_to_mongo(result) # print(html) if __name__ == ‘__main__‘: # main() groups = [x * 20 for x in range(GROUP_START, GROUP_END + 1)] pool = Pool() pool.map(main, groups) pool.close() pool.join()
- 下載圖片,並保存到mongodb
多線程高容錯爬頭條街拍美圖