1. 程式人生 > >scrapy將資料儲存到mysql中

scrapy將資料儲存到mysql中

一、在配置中設定資料庫

#setting.py
ITEM_PIPELINES = {
    ......
   'dushuproject.pipelines.MysqlPipeline': 299,
   }
DB_HOST = '127.0.0.1'
DB_PORT = 3306
DB_USER = 'root'
DB_PWD = '123456'
DB_NAME = 'test'
DB_CHARSET = 'utf8'

二、在管道中進行儲存

class MysqlPipeline(object):

    def __init__(self):
        settings = get_project_settings()
        self.host = settings['DB_HOST'
] self.port = settings['DB_PORT'] self.user = settings['DB_USER'] self.pwd = settings['DB_PWD'] self.name = settings['DB_NAME'] self.charset = settings['DB_CHARSET'] self.connect() def connect(self): self.conn = pymysql.connect(host=self.host, port=self.port, user=self.user, password=self.pwd, db=self.name, charset=self.charset) self.cursor = self.conn.cursor() def
close_spider(self, spider):
self.conn.close() self.cursor.close() def process_item(self, item, spider): sql = 'insert into book(image_url, book_name, author, info) values("%s", "%s", "%s", "%s")' % (item['image_url'], item['book_name'], item['author'], item['info']) # 執行sql語句
self.cursor.execute(sql) return item