pandas真是資料分析的好東西，使得程式碼簡潔了不少

阿新 • • 發佈：2020-12-04

利用python做了爬蟲，服務端，自動化測試，也都做了相應的編碼以及架構處理，近期就想來玩玩資料分析，這個才是好前景呀！

我們來看看這個網站，url = "http://www.compassedu.hk/qs"，爬取裡面的table資料，今天就來爬蟲這個地址，並儲存csv以及自動建立資料庫

在還沒學習資料分析前，像這種靜態網站我們一般使用requests+BeautifulSoup進行請求和解析，以下是我的程式碼，不使用pandas的

# -*- coding: utf-8 -*-
import requests
import pandas as pd
from bs4 import 
 BeautifulSoup
from sqlalchemy import create_engine
import pymysql
import numpy as np
import traceback
from openpyxl import Workbook

url = "http://www.compassedu.hk/qs"
res = requests.get(url)

#提取表格資料
soup = BeautifulSoup(res.content,"lxml")
table = soup.select("table#rk")[0]

table_datas = []

#提取列名
one_tr = table.select(" 
tr")[0]
tr_datas = []
for th in one_tr.select("th"):
    tr_datas.append(th.get_text())
table_datas.append(tr_datas)

for tr in table.select("tr"):
    tr_datas = []
    for th in tr.select("td"):
        tr_datas.append(th.get_text())
    if tr_datas:
        table_datas.append(tr_datas)

#儲存excel
wb = Workbook()
ws  
= wb.active
for table in table_datas:
    ws.append(table)
wb.save("result3.xlsx")


mysql_conf = {
    "host": '127.0.0.1',
    "port": 3306,
    "user": 'root',
    "password": '你自己的資料庫密碼',
    "db": 'analysis',
    "charset": "utf8",
    "cursorclass": pymysql.cursors.DictCursor
}
def get_connection():
    try:
        conn = pymysql.connect(**mysql_conf)
        return conn
    except:
        return None

#建立資料庫連線
conn = get_connection()
ok_table_datas = []
for i in range(1,len(table_datas)):
    ok_table_datas.append(tuple(table_datas[i]))

try:
    with conn.cursor() as cursor:
        #以元組形式插入 一次插入完 這個快
        sql = '''insert into university2020three values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''
        cursor.executemany(sql,ok_table_datas)
        conn.commit()
        
        #分條插入 分條提交 效率比較慢
        # for i in range(1,len(table_datas)):
        #     print(table_datas[i])
        #     sql = '''insert into university2020three values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''
        #     cursor.execute(sql,(table_datas[i]))
        #     conn.commit()
except Exception as e:
    print(traceback.format_exc())
    print(e)
finally:
    conn.close()

在我學習完pandas之後，我就覺得這個寫法太笨了，我就解析還是用BeautifulSoup，儲存excel和sql用pandas

# -*- coding: utf-8 -*-
import requests
import pandas as pd
from bs4 import BeautifulSoup
from sqlalchemy import create_engine
import pymysql
import numpy as np
import traceback
from openpyxl import Workbook

url = "http://www.compassedu.hk/qs"
res = requests.get(url)

soup = BeautifulSoup(res.content,"lxml")
table = soup.select("table#rk")[0]

table_datas = []

#獲取列標題
one_tr = table.select("tr")[0]
tr_datas = []
for th in one_tr.select("th"):
    tr_datas.append(th.get_text())
table_datas.append(tr_datas)

#獲取table的資料
for tr in table.select("tr"):
    tr_datas = []
    for th in tr.select("td"):
        tr_datas.append(th.get_text())
    if tr_datas:
        table_datas.append(tr_datas)

# data = pd.DataFrame(table_datas)
#指定標題 上面是不指定標題
# data = pd.DataFrame(table_datas,columns=['Ranking', 'University Name','Country/Region','Academic Reputation','Employer Reputation','Faculty Student','International Faculty','International Students','Citations per Faculty','Overall Score','Free'])
#轉numpy.ndarray
data = np.array(table_datas)
#獲取列名稱 並轉資料型別 和上面註釋的data一樣
data = pd.DataFrame(data[1:],columns=data[0])


data.to_csv("results_2.csv",index=None,encoding='utf_8_sig')

conn = create_engine("mysql+pymysql://root:資料庫密碼@127.0.0.1:3306/analysis")
data.to_sql("university2020-2",con=conn,if_exists='append',index=False)

最後我發現pandas不止可以用來csv，和sql簡化很多，還可以直接讀取就完事了，請看接下來的表演

# -*- coding: utf-8 -*-
import requests
import pandas as pd
from bs4 import BeautifulSoup
from sqlalchemy import create_engine
import pymysql
import numpy as np
import traceback
from openpyxl import Workbook

url = "http://www.compassedu.hk/qs"
res = requests.get(url)

# 比較簡便 還會根據資料型別建立資料庫
html=res.content.decode(encoding=res.apparent_encoding)
soup = BeautifulSoup(html,"lxml")
table = soup.find("table",id="rk")

datas = pd.read_html(table.prettify())

data = datas[0]
data.to_csv("results.csv",encoding='utf_8_sig',index=False)
conn = create_engine("mysql+pymysql://root:資料庫地址@127.0.0.1:3306/analysis")
data.to_sql("university2020",con=conn,if_exists='append',index=False)