python之爬蟲的入門04------爬蟲代理ip、儲存為CSV表格
阿新 • • 發佈:2018-11-13
一、爬蟲偽裝—使用代理ip
import urllib.request import random url = 'http://45.32.164.128/ip.php' #URL地址 iplist = ['112.91.159.66:3128','119.31.210.170:7777','221.7.255.168:8080'] #多個代理IP地址 ''' 細緻的設定代理,可以用opener的open方法開啟URL: ''' proxy_support = urllib.request.ProxyHandler({'http':random.choice(iplist)}) # 引數是一個字典{'型別':'代理IP:埠號'} opener = urllib.request.build_opener(proxy_support) # 訂製建立一個opener urllib.request.install_opener(opener) #安裝opener response = urllib.request.urlopen(url) html = response.read().decode('utf8') print(html)
二、儲存為CSV表格
import csv def write_msg(data): '''這個函式用來將爬取的資料寫入檔案''' with open("csvfile.csv", 'a+', newline='') as f: wf = csv.writer(f) wf.writerow(data) if __name__ == '__main__': head = ['msg', 'area', 'price', 'address', 'user', 'phone'] with open('./csvfile.csv', 'a+') as f: wf = csv.writer(f) wf.writerow(head) data = ['1', '2', '4', '6', 'g', 'uy'] write_msg(data) data = ['3', '1', 'a', '6', 'g', 'uy'] write_msg(data) data = ['w', 'wdveg546thfwer4335refveergrtbg5', '我的', 'we', 'fvr', 'df43ferrver3442ergfregbvrvebevevevfvervfvr'] write_msg(data)