思路

先從唐詩300首頁面獲取所有唐詩詳情頁的連結;
再從每一首唐詩的詳情頁提取標題、作者、朝代、內容等資訊。

原始碼

# !/usr/bin/env python
# -*- coding:utf-8 -*-

import re
import requests


def crawl(start_url):
    base_url='http://so.gushiwen.org'

    req_headers={
        'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
 

    }

    res=requests.get(start_url,headers=req_headers)
    if res.status_code==requests.codes.ok:
        html=res.text

        # 獲取所有詩的連結
        parttern_href=re.compile(r'<span><a href="(.*?)".*?</span>',flags=re.DOTALL)
        hrefs=re.findall(parttern_href,html)

        # 獲取每一首詩的內容,並儲存到本地
 

        with open('唐詩300首.txt',mode='a',encoding='utf-8') as f:
            for href in hrefs:
                href=base_url+href
                res=requests.get(href,headers=req_headers)
                if res.status_code == requests.codes.ok:
                    html = res.text
                    # 標題
 

                    parttern_title = re.compile(r'<div class="cont">.*?<h1 .*?>(.*?)</h1>', re.DOTALL)
                    title=re.search(parttern_title,html).group(1)
                    # 朝代
                    parttern_dynasty = re.compile(r'<div class="cont">.*?<p class="source"><a href=".*?">(.*?)</a><span>：.*?</p>', re.DOTALL)
                    dynasty=re.search(parttern_dynasty,html).group(1)
                    # 詩人
                    parttern_author = re.compile(r'<div class="cont">.*?<p class="source">.*?：</span><a href=".*?">(.*?)</a> </p>', re.DOTALL)
                    author=re.search(parttern_author,html).group(1)
                    # 內容
                    parttern_content = re.compile(r'<div class="cont">.*?<div class="contson" id=".*?">(.*?)</div>', re.DOTALL)
                    content=re.search(parttern_content,html).group(1)
                    content=re.sub(r'<br />','\n',content)
                    content=re.sub(r'<p>','',content)
                    content=re.sub(r'</p>','',content)

                    print('正在獲取 {title}'.format(title=title))
                    f.write('{title}\n{dynasty}:{author}\n{content}\n'.format(title=title,dynasty=dynasty,author=author,content=content))


if __name__ == '__main__':
    start_url='http://so.gushiwen.org/gushi/tangshi.aspx'
    crawl(start_url)

執行結果