1. 程式人生 > >scrapy配置proxy代理

scrapy配置proxy代理

一、IP池

IP可以從這個幾個網站獲取:
快代理:https://www.kuaidaili.com/free/
西刺代理:http://www.xicidaili.com/
如果出現像下面這種提示:”由於連線方在一段時間後沒有正確答覆或連線的主機沒有反應,連線嘗試失敗”或者是這種,” 由 於目標計算機積極拒絕,無法連線。”. 那就是IP的問題,更換就行了。。。。發現上面好多IP都不能用。。

2017-04-16 12:38:11 [scrapy.downloadermiddlewares.retry] DEBUG: Retrying <GET http://news.sina.com
.cn/> (failed 1 times): TCP connection timed out: 10060: 由於連線方在 一段時間後沒有正確答覆或連線的主機沒有反應,連線嘗試失敗。. this is ip:182.241.58.70:51660 2017-04-16 12:38:32 [scrapy.downloadermiddlewares.retry] DEBUG: Retrying <GET http://news.sina.com.cn/> (failed 2 times): TCP connection timed out: 10060: 由於連線方在 一段時間後沒有正確答覆或連線的主機沒有反應,連線嘗試失敗。. this is ip:49.75
.59.243:28549 2017-04-16 12:38:33 [scrapy.crawler] INFO: Received SIGINT, shutting down gracefully. Send again to force 2017-04-16 12:38:33 [scrapy.core.engine] INFO: Closing spider (shutdown) 2017-04-16 12:38:50 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) 2017
-04-16 12:38:53 [scrapy.downloadermiddlewares.retry] DEBUG: Gave up retrying <GET TCP connection timed out: 10060: 由於 連線方在一段時間後沒有正確答覆或連線的主機沒有反應,連線嘗試失敗。. 2017-04-16 12:38:54 [scrapy.core.scraper] ERROR: Error downloading <GET http://news.sina.com.cn/> Traceback (most recent call last): File "f:\software\python36\lib\site-packages\twisted\internet\defer.py", line 1299, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "f:\software\python36\lib\site-packages\twisted\python\failure.py", line 393, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "f:\software\python36\lib\site-packages\scrapy\core\downloader\middleware.py", defer.returnValue((yield download_func(request=request,spider=spider))) TCP connection timed out: 10060: 由於連線方在一段時間後沒有正確答覆或連線的主機沒有反應,連線嘗試失敗。.

在Scrapy中與代理伺服器設定相關的下載中介軟體是HttpProxyMiddleware,對應的類為:

scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware  

2.修改中介軟體檔案middlewares.py

# -*- coding: utf-8 -*-  

# Define here the models for your spider middleware  
#  
# See documentation in:  
# http://doc.scrapy.org/en/latest/topics/spider-middleware.html  

import random  
from scrapy import signals  
from myproxies.settings import IPPOOL  

class MyproxiesSpiderMiddleware(object):  

      def __init__(self,ip=''):  
          self.ip=ip  

      def process_request(self, request, spider):  
          thisip=random.choice(IPPOOL)  
          print("this is ip:"+thisip["ipaddr"])  
          request.meta["proxy"]="http://"+thisip["ipaddr"]  

3.在settings中設定DOWNLOADER_MIDDLEWARES

[html] view plain copy
DOWNLOADER_MIDDLEWARES = {  
     'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware':543,  
     'DouTu.middlewares.MyproxiesSpiderMiddleware':125  
}  

4.爬蟲檔案為


# -*- coding: utf-8 -*-  
import scrapy  


class ProxieSpider(scrapy.Spider):  


    def __init__(self):  
        self.headers = {  
            'Content-Type':'application/x-www-form-urlencoded; charset=UTF-8',  
            'Accept-Encoding':'gzip, deflate',  
            'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'  
        }  


    name = "proxie"  
    allowed_domains = ["sina.com.cn"]  
    start_urls = ['http://news.sina.com.cn/']  

    def parse(self, response):  
        print(response.body)  

二、自動更新IP池

這裡寫個自動獲取IP的類proxies.py,執行一下把獲取的IP儲存到txt檔案中去:


# *-* coding:utf-8 *-*
import requests
from bs4 import BeautifulSoup
import lxml
from multiprocessing import Process, Queue
import random
import json
import time
import requests

class Proxies(object):


    """docstring for Proxies"""
    def __init__(self, page=3):
        self.proxies = []
        self.verify_pro = []
        self.page = page
        self.headers = {
        'Accept': '*/*',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36',
        'Accept-Encoding': 'gzip, deflate, sdch',
        'Accept-Language': 'zh-CN,zh;q=0.8'
        }
        self.get_proxies()
        self.get_proxies_nn()

    def get_proxies(self):
        page = random.randint(1,10)
        page_stop = page + self.page
        while page < page_stop:
            url = 'http://www.xicidaili.com/nt/%d' % page
            html = requests.get(url, headers=self.headers).content
            soup = BeautifulSoup(html, 'lxml')
            ip_list = soup.find(id='ip_list')
            for odd in ip_list.find_all(class_='odd'):
                protocol = odd.find_all('td')[5].get_text().lower()+'://'
                self.proxies.append(protocol + ':'.join([x.get_text() for x in odd.find_all('td')[1:3]]))
            page += 1

    def get_proxies_nn(self):
        page = random.randint(1,10)
        page_stop = page + self.page
        while page < page_stop:
            url = 'http://www.xicidaili.com/nn/%d' % page
            html = requests.get(url, headers=self.headers).content
            soup = BeautifulSoup(html, 'lxml')
            ip_list = soup.find(id='ip_list')
            for odd in ip_list.find_all(class_='odd'):
                protocol = odd.find_all('td')[5].get_text().lower() + '://'
                self.proxies.append(protocol + ':'.join([x.get_text() for x in odd.find_all('td')[1:3]]))
            page += 1

    def verify_proxies(self):
        # 沒驗證的代理
        old_queue = Queue()
        # 驗證後的代理
        new_queue = Queue()
        print ('verify proxy........')
        works = []
        for _ in range(15):
            works.append(Process(target=self.verify_one_proxy, args=(old_queue,new_queue)))
        for work in works:
            work.start()
        for proxy in self.proxies:
            old_queue.put(proxy)
        for work in works:
            old_queue.put(0)
        for work in works:
            work.join()
        self.proxies = []
        while 1:
            try:
                self.proxies.append(new_queue.get(timeout=1))
            except:
                break
        print ('verify_proxies done!')


    def verify_one_proxy(self, old_queue, new_queue):
        while 1:
            proxy = old_queue.get()
            if proxy == 0:break
            protocol = 'https' if 'https' in proxy else 'http'
            proxies = {protocol: proxy}
            try:
                if requests.get('http://www.baidu.com', proxies=proxies, timeout=2).status_code == 200:
                    print ('success %s' % proxy)
                    new_queue.put(proxy)
            except:
                print ('fail %s' % proxy)


if __name__ == '__main__':
    a = Proxies()
    a.verify_proxies()
    print (a.proxies)
    proxie = a.proxies 
    with open('proxies.txt', 'a') as f:
       for proxy in proxie:
             f.write(proxy+'\n')

修改代理檔案middlewares.py的內容為如下:

import random
import scrapy
from scrapy import log


# logger = logging.getLogger()

class ProxyMiddleWare(object):
    """docstring for ProxyMiddleWare"""
    def process_request(self,request, spider):
        '''對request物件加上proxy'''
        proxy = self.get_random_proxy()
        print("this is request ip:"+proxy)
        request.meta['proxy'] = proxy 


    def process_response(self, request, response, spider):
        '''對返回的response處理'''
        # 如果返回的response狀態不是200,重新生成當前request物件
        if response.status != 200:
            proxy = self.get_random_proxy()
            print("this is response ip:"+proxy)
            # 對當前reque加上代理
            request.meta['proxy'] = proxy 
            return request
        return response

    def get_random_proxy(self):
        '''隨機從檔案中讀取proxy'''
        while 1:
            with open('G:\\Scrapy_work\\myproxies\\myproxies\\proxies.txt', 'r') as f:
                proxies = f.readlines()
            if proxies:
                break
            else:
                time.sleep(1)
        proxy = random.choice(proxies).strip()
        return proxy

修改下settings檔案:

DOWNLOADER_MIDDLEWARES = {  
#    'myproxies.middlewares.MyCustomDownloaderMiddleware': 543,  
     'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware':None,  
     'myproxies.middlewares.ProxyMiddleWare':125,  
     'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware':None  
}