爬取豆瓣的tp250電影名單
阿新 • • 發佈：2018-11-26
info urlopen lose from port tps div turn data
# https://movie.douban.com/top250?start=25&filter= 要爬取的網頁
import re
from urllib.request import urlopen
def getPage(url):
response=urlopen(url)
return response.read().decode(‘utf-8‘)
def parsePage(s):
ret=com.finditer(s)
for i in ret:
ret={
‘id‘: i.group(‘id‘),
‘move_name‘:i.group(‘move_name‘),
‘move_d‘:i.group( ‘move_d‘),
‘move_t‘:i.group(‘move_t‘),
‘content‘: i.group(‘content‘),
‘infor‘:i.group(‘infor‘)
}
yield ret
def main(num):
url=‘https://movie.douban.com/top250?start=%s&filter= ‘ %num
res=getPage(url)
ret=parsePage(res)
print(ret)
f=open(‘move‘,mode=‘a+‘,encoding=‘utf-8‘)
for obj in ret:
print(obj)
data1=str(obj).replace(‘\\n‘,‘‘)
data2=data1.replace(‘ ‘,‘‘)
f.write(data2 + ‘\n‘)
f.close()
com=re.compile(‘<div class="item">(?:.*?)<em class="">(?P<id>.*?)</em>(?:.*?)alt=(?P<move_name>.*?)src(?:.*?)導演:‘
‘(?P<move_d>.*?) (?:.*?)<br>(?P<move_t>.*?) (?:.*?) / (?P<content>.*?)</p>(?:.*?)<span class="inq">(?P<infor>.*?)</span>‘,re.S)
count=0
for i in range(10):
main(count)
count+=25爬取豆瓣的tp250電影名單