爬蟲 1 -IP代理池

IP代理池

[time=Tue, Jan 25, 2022 9:34 PM]
原文連結
dokcer-compose檔案下載

# docker-compose 一鍵部署
docker-compose up

測試

隨機一個代理IP http://ippool.gwozai.com/random
IP數量統計 http://ippool.gwozai.com/count

用python程式碼獲取代理IP

import requests

proxypool_url = 'http://ippool.gwozai.com/random'
target_url = 'http://www.baidu.com'

def get_random_proxy():
    """
    get random proxy from proxypool
    :return: proxy
    """
    return requests.get(proxypool_url).text.strip()

def crawl(url, proxy):
    """
    use proxy to crawl page
    :param url: page url
    :param proxy: proxy, such as 8.8.8.8:8888
    :return: html
    """
    proxies = {'http': 'http://' + proxy}
    return requests.get(url, proxies=proxies).text


def main():
    """
    main method, entry point
    :return: none
    """
    proxy = get_random_proxy()
    print('get random proxy', proxy)
    html = crawl(target_url, proxy)
    print(html)

if __name__ == '__main__':
    main()