學習書籍：《python3 網路爬蟲開發實踐》
視訊地址：例項視訊
1.main.py

import re
from pyquery import PyQuery as pq
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import
 
 expected_conditions as EC
from selenium.webdriver.chrome.options import Options

#連線到資料庫
from config_seleium import *
import pymongo
client=pymongo.MongoClient(MONGO_URL)
db=client[MONGO_DB]

#使用Chrome無頭瀏覽器，即僅在後臺使用
chrome_options=Options()
chrome_options.add_argument('--headless')
browser=webdriver.Chrome(chrome_options=chrome_options)
wait=WebDriverWait(browser,10
 
)
#搜尋頁面
def search():
    try:
        browser.get('https://www.taobao.com')
        #等待元素id為XXX的出現，10秒後若是還不出現，則報錯
        input = wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR, '#q'))
        )
        submit=wait.until(
            EC.element_to_be_clickable((By.CSS_SELECTOR,'#J_TSearchForm > div.search-button > button'
 
))
        )
        input.send_keys('美食')
        submit.click()
        #等待最下面的總頁數加載出來（意味著前面產品已經全部加載出來了）
        total=wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR,'#mainsrp-pager > div > div > div > div.total'))
        )
        #等該頁面載入完後，輸出一個個產品資訊
        get_product()
        return total.text
    except TimeoutException:
        return search()

#跳轉到下一頁
def next_page(page_number):
    try:
        input = wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.form > input'))
        )
        submit = wait.until(
            EC.element_to_be_clickable((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit'))
        )
        input.clear()
        input.send_keys(page_number)
        submit.click()
        #判斷已經跳轉成功，由於此時所處頁面高亮，所以css_selector不同
        wait.until(
            EC.text_to_be_present_in_element((By.CSS_SELECTOR,'#mainsrp-pager > div > div > div > ul > li.item.active > span'),str(page_number))
        )
        #獲取商品
        get_product()
    except TimeoutException:
        next_page(page_number)

#解析寶貝內容
def get_product():
    wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'#mainsrp-itemlist .items .item')))
    html=browser.page_source
    #通過PyQuery解析HTML
    doc=pq(html)
    items=doc('#mainsrp-itemlist .items .item').items()
    for item in items:
        product={
            'image':item.find('.pic .img').attr('src'),
            'price':item.find('.price').text().replace('\n',''),
            'deal':item.find('.deal-cnt').text()[:-3].replace('\n',''), #去掉"人付款"三個字
            'title':item.find('.title').text().replace('\n',''),
            'shop':item.find('.shop').text().replace('\n',''),
            'location':item.find('.location').text().replace('\n','')
        }
        save_to_mongo(product)

#引數為一個個產品形成的字典
def save_to_mongo(result):
    try:
        if db[MONGO_TABLE].insert(result):
            print('儲存成功',result)
    except Exception:
        print('儲存錯誤',result)

def main():
    total_String=search()
    total_Number=int(re.compile('(\d+)').search(total_String).group(1))
    for i in range(2,total_Number+1):
        next_page(i)
    browser.close()

if __name__=='__main__':
    main()

2.config_seleium.py

#存入MongoDB
MONGO_URL='localhost'
MONGO_DB='taobao'
MONGO_TABLE='meishi'

#定義一個offset偏移量，用於迴圈
GROUP_START=1
GROUP_END=20

#搜尋的關鍵字
KEYWORD='街拍'

3.執行結果