隨機獲取列表中的一個IP地址進行代理
阿新 • • 發佈:2018-12-28
如果是json檔案,需要讀取檔案,匯入json模組進行轉換後再進行處理。
with open('ip.json','r') as w:
json.loads(f.read())
# 隨機獲取一個ip進行代理 from urllib import request # 匯入隨機模組 import random # 獲取所有代理 def getProxy(): data = [ {"host": "116.31.75.100", "port": "3128"}, {"host": "218.66.253.146", "port": "8800"}, {"host": "218.66.253.144", "port": "8800"} ] proxies = [] for p in data: proxy = { 'http': 'http://' + p['host'] + ':' + p['port'], 'https': 'http://' + p['host'] + ':' + p['port'] } # 將proxy 追加到proxies列表中 proxies.append(proxy) # 返回proxies return proxies # 獲取一個opener def getOpener(proxies): proxy = random.choice(proxies) proxy_handler = request.ProxyHandler(proxy) opener = request.build_opener(proxy_handler) return opener # 製作下載器 def downloader(opener, req, proxies, timeout=5, retry=2): ''' :param opener: 發請求 :param req: 請求物件 :param proxies: 代理列表 :param timeout: 超時 :param retry: 重試 :return: ''' try: response = opener.open(req,timeout=timeout) return response.read() except Exception as e: print(e) if retry > 0: opener = getOpener(proxies) # 換了一個代理 return downloader(opener,req,proxies,retry=retry-1) if __name__ == '__main__': proxies = getProxy() opener = getOpener(proxies) # 檢視ip是否切換成功 base_url = 'http://www.baidu.com/s?wd=ip' res = downloader(opener,base_url,proxies) print(res.decode('utf-8'))