Python3原生爬蟲獲取熊貓直播某一分類下的主播人氣並儲存到Excel
阿新 • • 發佈:2018-12-02
import re
import openpyxl
from urllib import request
# 斷點除錯
class Spider:
url = 'https://www.panda.tv/cate/lol'
root_pattern = '<div class="video-info">([\s\S]*?)</div>'
name_pattern = '<span class="video-nickname"[\w\W]*?</i>([\s\S]*?)</span>'
number_pattern = '<span class="video-number">([\s\S]*?)</span>'
#獲取頁面
def __fetch_content(self):
r = request.urlopen(Spider.url)
# bytes 將其轉成字串
htmls = r.read()
htmls = str(htmls,encoding='utf-8')
return htmls
#分析
def __analysis(self, htmls):
root_html = re.findall(Spider.root_pattern,htmls)
anchors = []
for html in root_html:
name = re.findall(Spider.name_pattern,html)
number = re.findall(Spider.number_pattern,html)
anchor = {'name':name,'number':number}
anchors.append(anchor)
# print(anchors[0])
return anchors
#精煉/清洗
def __refine(self, anchors):
l = lambda anchor: {
'name':anchor['name'][0].strip(),
'number':anchor['number'][0]
}
return map(l,anchors)
#排序
def __sort(self, anchors):
#py預設排序方法
anchors = sorted(anchors,key=self.__sort__seed,reverse=True)
return anchors
#按xx排序
def __sort__seed(self, anchor):
r = re.findall('\d*',anchor['number'])
number = float(r[0])
if '萬' in anchor['number']:
number *= 10000
return number
#列印排序完的資料
def __show(self, anchors):
for rank in range(0,len(anchors)):
print('rank ' + str(rank + 1)
+ ' : ' + anchors[rank]['name']
+ ' ' + anchors[rank]['number']
)
#匯入excel表
def __import_excel(self,anchors):
#新建
info = openpyxl.Workbook()
sheet = info.active
sheet.title = 'pandatv_sheet'
title = ['排名','主播','人氣']
col = 1
for t in title:
sheet.cell(1,col,t)
col += 1
row = 2
for anchor in anchors:
col = 2
for key,value in anchor.items():
sheet.cell(row,1,row-1)
sheet.cell(row,col,value)
col += 1
row += 1
info.save('pandatv_sheet.xlsx')
#入口/總控方法
def go(self):
htmls = self.__fetch_content()
anchors = self.__analysis(htmls)
anchors = list(self.__refine(anchors))
anchors = self.__sort(anchors)
# print(anchors)
self.__import_excel(anchors)
# self.__show(anchors)
spider = Spider()
spider.go()