xpath,requests爬取小豬短租網
阿新 • • 發佈:2018-12-19
import requests from lxml import etree import time headers = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/70.0.3538.77 Safari/537.36' } def page_url(url): res = requests.get(url,headers=headers) text = res.text html = etree.HTML(text) links = html.xpath('.//a[@class="resule_img_a"]/@href') for link in links: parse_page(link) def parse_page(url): res = requests.get(url,headers=headers) text = res.text html = etree.HTML(text) titles = html.xpath('.//div[@class="pho_info"]/h4/em//text()') addresses = html.xpath('.//span[@class="pr5"]//text()') prices = html.xpath('.//div[@class="day_l"]/span//text()') imgs = html.xpath('.//div[@class="member_pic"]/a//img/@src') names = html.xpath('.//div[@class="w_240"]/h6/a//text()') for title,address,price,img,name in zip(titles,addresses,prices,imgs,names): data ={ '屋子標題':title.strip(), '地址':address.strip(), '價格':price, '圖片':img, '名字':name } print(data) if __name__ == '__main__': urls = ['http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format (x) for x in range(1,14)] for url in urls: page_url(url) time.sleep(2)