宣告：僅供學習交流使用

• items.py ->內容模板

import scrapy

class QqduItem(scrapy.Item):

    book = scrapy.Field()
    bookId = scrapy.Field()
    chapter = scrapy.Field()
    chapterId = scrapy.Field()
    content = scrapy.Field()

• book.py ->爬蟲檔案

# -*- coding: utf-8 -*-

import scrapy, sys
from qqdu.items import
 
 QqduItem
from scrapy.http import Request

reload(sys)
sys.setdefaultencoding("gbk")

class BookSpider(scrapy.Spider):
    name = 'book'
    allowed_domains = ['www.77dus.com']
    start_urls = ['https://www.77dus.com/html/82/82709/index.html']                   # 索要爬取小說的網址

    def parse(self, response):
        result = response.xpath('//*[@id="wp"]/div/div/div/ul/li/a/@href'
 
).extract()    # 獲取章節連結
        for sel in result:
            yield Request('https://www.77dus.com' + sel, meta={'fileName': sel}, callback=self.bookContent)

    def bookContent(self, response):
        baseUrl = response.url
        item = QqduItem()
        book = response.xpath('//*[@id="content"]/div[1]/p/a[1]/text()'
 
).extract()[0]   # 獲取書名
        chapter = response.xpath('//h1[@class="novel_title"]/text()').extract()[0]      # 獲取章節名
        contents = response.xpath('//div[@class="novel_content"]/text()').extract()     # 獲取內容

        # 拼接小說內容
        text = (chapter + "\r\n\r\n")
        for rel in contents:
            text += rel
        baseUrl = baseUrl.split('.')[2]
        baseUrl = baseUrl.split('/')
        bookId = baseUrl[-2]            # 獲取書id
        chapterId = baseUrl[-1]         # 獲取章節id
        item['book'] = book
        item['bookId'] = bookId
        item['chapter'] = chapter
        item['chapterId'] = chapterId
        item['content'] = text
        yield item

• pipelines.py ->管道檔案，用於對所爬取內容進行操作

import os
import sys
import codecs,pymongo

reload(sys)
sys.setdefaultencoding("gbk")

# 儲存到檔案
class QqduPipeline(object):
    def process_item(self, item, spider):
        curPath = 'qqdubooks'
        tempPath = str(item['book'])
        targetPath = curPath + os.path.sep + tempPath
        if not os.path.exists(targetPath):
            os.makedirs(targetPath)
        filename_path = targetPath + os.path.sep + str(item['chapterId']) + '.txt'
        file = codecs.open(filename_path, 'w', encoding='utf-8')
        file.write(item['content'] + "\r\n")
        file.close()
        return item

# # 儲存到mongoDb資料庫
# class QqduPipeline(object):
#     def __init__(self):
#         host = "127.0.0.1"
#         port = 27017
#         dbname = "qqdb"
#         sheetname = "qqdbbooks"
#         # 建立MONGODB資料庫連結
#         client = pymongo.MongoClient(host=host, port=port)
#         # 指定資料庫
#         mydb = client[dbname]
#         # 存放資料的資料庫表名
#         self.post = mydb[sheetname]
#
#     def process_item(self, item, spider):
#         data = dict(item)
#         self.post.insert(data)
#         return item

• 在settings.py 開啟管道，新增如下內容

ITEM_PIPELINES = {
   'qqdu.pipelines.QqduPipeline': 300,
}