案例：使用BeautifuSoup4的爬蟲

我們以騰訊社招頁面來做演示：http://hr.tencent.com/position.php?&start=10#a

使用BeautifuSoup4解析器，將招聘網頁上的職位名稱、職位類別、招聘人數、工作地點、釋出時間，以及每個職位詳情的點選連結儲存出來。

# bs4_tencent.py


from bs4 import BeautifulSoup
import urllib2
import urllib import json # 使用了json格式儲存 def tencent(): url = 'http://hr.tencent.com/' request = urllib2.Request(url + 'position.php?&start=10#a') response =urllib2.urlopen(request) resHtml = response.read() output =open('tencent.json','w') html = BeautifulSoup(resHtml,'lxml') # 建立CSS選擇器 result = html.select('tr[class="even"]') result2 = html.select('tr[class="odd"]') result += result2 items = [] for site in result: item = {} name = site.select('td a')[0].get_text() detailLink = site.select('td a')[0].attrs['href'] catalog = site.select('td')[1].get_text() recruitNumber = site.select('td')[2].get_text() workLocation = site.select('td')[3].get_text() publishTime = site.select('td')[4].get_text() item['name'] = name item['detailLink'] = url + detailLink item['catalog'] = catalog item['recruitNumber'] = recruitNumber item['publishTime'] = publishTime items.append(item) # 禁用ascii編碼，按utf-8編碼 line = json.dumps(items,ensure_ascii=False) output.write(line.encode('utf-8')) output.close() if __name__ == "__main__": tencent()