1 pymysql

參考文件

SQL工具

1.1 安裝

$ (env) python3 -m pip install PyMySQL

1.2基本使用

import pymysql

1.2.1 Connection 物件

connection = pymysql.connect(host='localhost',
                             user='user',
                             password='passwd',
                             db='db',
                             charset='utf8mb4',
                             cursorclass=pymysql.cursors.DictCursor)
 

import pymysql
# 資料庫連線配置
dbcondfig = {
    "host":'localhost',
    "user":'root',
    "password":'123123',
    "db":'test',
    "charset":'utf8',
    "cursorclass":pymysql.cursors.DictCursor
}

1.2.2 Cursor物件

連線、操作資料庫執行SQL

Cursor

1.2.3執行引數

參考文件

# 元組類引數
cursor.execute(query="SELECT * from jt_news where news_id=%s",args=(2,))
# 字典類引數
cursor.execute(query="SELECT * from jt_news where news_id=%(newsid)s",args={"newsid":3})

1.2.4 執行SQL

# 執行單條SQL
cursor.execute(query="insert into news(title) values(%s)",args=['測試新聞1'])
connection.commit()  #由於預設提交 是 預設被pymysql關閉的，因此要commit
# 執行多條SQL
cursor.executemany(query="insert into news(title) values(%(news_title)s)"
                   ,args=({"news_title":"'測試新聞1"},{"news_title":'測試新聞2'}))
 

1.3 案例

mysql 表設計

案例需求：

1. 讀取網頁資料後，儲存到文字檔案。然後讀取出來
2. 迴圈插入到資料庫中
3. 如果發生資料改變，則要更新NAV和ACCNAV，並更新“updatetime”

from urllib.request import urlopen
from bs4 import BeautifulSoup
from common import config
import pymysql
from datetime import datetime

response = urlopen("http://fund.eastmoney.com/fund.html")
html = response.read()
html = html.decode('gb2312')
with open("./html/1.txt", 'wb') as f:
    f.write(html.encode('utf8'))
    f.close()

with open("./html/1.txt", "rb") as f:
    html = f.read().decode("utf8")
    bsObj = BeautifulSoup(html, "html.parser")
    FCodes = bsObj.findAll("", {"class": "bzdm"})
    res = []
    for FCode in FCodes:
        res.append({
            "fcode": FCode.get_text(),
            "fname": FCode.next_sibling.a.get_text(),
            "NAV": FCode.next_sibling.next_sibling.get_text(),
            "ACCNAV": FCode.next_sibling.next_sibling.next_sibling.get_text(),
            "updatetime": datetime.now().isoformat(sep=' ', timespec="seconds"),
        })

# 資料入庫
conn = pymysql.connect(**config.dbcondfig)

# 插入
try:
    with conn.cursor() as cursor:
        cursor.executemany("""INSERT INTO `funddb`.`myfund`(fcode,fname,NAV,ACCNAV,updatetime)
                          VALUES(%(fcode)s,%(fname)s,%(NAV)s,%(ACCNAV)s,%(updatetime)s)
                          ON DUPLICATE KEY UPDATE `NAV`=VALUES(NAV),`ACCNAV`=VALUES(ACCNAV),`updatetime`=VALUES(updatetime);"""
                       ,res)

    conn.commit()
except Exception as e:
    print(e)
finally:
    conn.close()

注意事項： executemany和ON DUPLICATE KEY UPDATE聯合使用的時候不能安裝sql常規模式，為軟體bug，需要使用values()函式規避 參考連結

# 以下程式碼都會報錯
cursor.executemany("""INSERT INTO `funddb`.`myfund`(fcode,fname,NAV,ACCNAV,updatetime)
                          VALUES(%(fcode)s,%(fname)s,%(NAV)s,%(ACCNAV)s,%(updatetime)s)
                          ON DUPLICATE KEY UPDATE `NAV`=%s,`ACCNAV`=%s,`updatetime`=%s;"""
                       ,res)
cursor.executemany("""INSERT INTO `funddb`.`myfund`(fcode,fname,NAV,ACCNAV,updatetime)
                          VALUES(%s,%s,%s,%s,%s)
                          ON DUPLICATE KEY UPDATE `NAV`=%(NAV)s,`ACCNAV`=%(ACCNAV)s,`updatetime`=%(updatetime)s;"""
                       ,res)