糗事百科段子爬蟲
import re
import urllib.request
import urllib.error
headers=("User-Agent","Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/61.0")
opener=urllib.request.build_opener()
opener.addheader=[headers]
urllib.request.install_opener(opener)
def use_proxy(url,IP):
proxy=urllib.request.ProxyHandler({"http":IP})
opener=urllib.request.build_opener(proxy,urllib.request.HTTPHandler)
#新增為全域性
urllib.request.install_opener(opener)
data=urllib.request.urlopen(url).read()
data=data.decode("utf-8","ignore")
return data
IP="61.135.217.7:80"
for i in range(1,10):
try:
url="http://www.qiushibaike.com/8hr/page/"+str(i)
pagedata=use_proxy(url,IP)
print(len(pagedata))
pat='<div class="content">.*?<span>(.*?)</span>.*?</div>'
datalist=re.compile(pat,re.S).findall(pagedata)
print(len(datalist))
for j in range(0,len(datalist)):
print("第"+str(i)+"頁第"+str(j)+"個段子的內容是: ")
print(datalist[j])
except urllib.error.HTTPError as e:
if hasattr(e,"code"):
print(e.code)
if hasattr(e,"reason"):
print(e.reason)