scrapy將資料儲存到mysql中
阿新 • • 發佈:2019-02-04
一、在配置中設定資料庫
#setting.py
ITEM_PIPELINES = {
......
'dushuproject.pipelines.MysqlPipeline': 299,
}
DB_HOST = '127.0.0.1'
DB_PORT = 3306
DB_USER = 'root'
DB_PWD = '123456'
DB_NAME = 'test'
DB_CHARSET = 'utf8'
二、在管道中進行儲存
class MysqlPipeline(object):
def __init__(self):
settings = get_project_settings()
self.host = settings['DB_HOST' ]
self.port = settings['DB_PORT']
self.user = settings['DB_USER']
self.pwd = settings['DB_PWD']
self.name = settings['DB_NAME']
self.charset = settings['DB_CHARSET']
self.connect()
def connect(self):
self.conn = pymysql.connect(host=self.host,
port=self.port,
user=self.user,
password=self.pwd,
db=self.name,
charset=self.charset)
self.cursor = self.conn.cursor()
def close_spider(self, spider):
self.conn.close()
self.cursor.close()
def process_item(self, item, spider):
sql = 'insert into book(image_url, book_name, author, info) values("%s", "%s", "%s", "%s")' % (item['image_url'], item['book_name'], item['author'], item['info'])
# 執行sql語句
self.cursor.execute(sql)
return item