輸入到mysql版

輸出會有警告，但是不影響。
附上了詳細的解釋~

import pandas as pd
import requests
import random
from bs4 import BeautifulSoup
import pymysql
from sqlalchemy import create_engine



def get_one_page(i):
    # 使用ip代理,隨機代理ip
    # 設定headers 這些都是為了避免訪問過於頻繁被禁止
    try:
        IPs = [{'HTTP': 'HTTP://118.190.95.35:9001'}
 
,
               {'HTTP': 'HTTP://61.135.217.7:80'},
               {'HTTP': 'HTTP://116.1.11.19:80'}]
        reportTime = '2017-12-31'
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
            'Referer'
 
: 'http://s.askci.com/stock/a/?reportTime=%s' % reportTime + '&pageNum=178'
        }
        url = 'http://s.askci.com/stock/a/?reportTime=%s' % reportTime + '&pageNum=%d' % i
        response = requests.get(url, headers=headers, proxies=random.choice(IPs))
        if response.status_code == 200:
            return
 
 response.text
        return None
    except requests.RequestException:
        print("爬取出錯")


def parse_one_page(html):
    soup = BeautifulSoup(html, 'lxml')
    # 選擇id為myTable04的控制元件，程式碼中有解釋
    content = soup.select("#myTable04")[0]

    # 第一行設定為column,並且選第一個(因為是一個list), 其實也只有一個
    tbl = pd.read_html(content.prettify(), header=0)[0]
    # 強制修改列名，中文列名不好寫入資料庫
    tbl.columns = ['serial_number', 'stock_code', 'stock_abbre', 'company_name', 'province', 'city',
                   'main_business_income',
                   'net_profit', 'employee', 'listing_date', 'zhaogushu', 'financial_report',
                   'industry_classification', 'industry_type', 'main_business']
    return tbl


def generate_mysql(db, user, password, port):
    # 創立連線
    # db就是需要資料庫中的schema 選一個schema就好了
    conn = pymysql.connect(
        host='localhost',
        user=user,
        password=password,
        port=port,
        charset='utf8',
        db=db
    )
    cursor = conn.cursor()
    # 建立資料庫
    sql = 'CREATE DATABASE if not exists %s' % db
    # 執行命令
    cursor.execute(sql)

    # 建立表
    sql = 'CREATE TABLE if not exists listed_company (serial_number INT(20) NOT NULL, stock_code VARCHAR(20), stock_abbre VARCHAR(20), company_name VARCHAR (20),province VARCHAR (20), city VARCHAR (20), main_business_income VARCHAR(20), net_profit VARCHAR (20), employee INT (20), listing_date DATE, zhaogushu VARCHAR(20), financial_report VARCHAR (20), industry_classification VARCHAR(20), industry_type VARCHAR(100), main_business VARCHAR(200), PRIMARY KEY (serial_number))'
    # 執行命令
    cursor.execute(sql)

    # 斷開連結
    conn.close()


def write_to_sql(tbl, db, user, password, port):
    engine = create_engine('mysql+pymysql://%s:%[email protected]:%d/%s?charset=utf8' % (user, password, port, db))
    tbl.to_sql('listed_company', con=engine, if_exists='append', index=False)


def main(page):
    # 設定預想database
    db = 'test'
    user = 'root'
    password = '1234'
    port = 3306
    # 生成資料庫
    generate_mysql(db, user, password, port)
    # 選page這麼多頁
    for i in range(1, page + 1):
        # 讀取網頁html
        html = get_one_page(i)
        # 分析得到dataframe
        tbl = parse_one_page(html)
        # 將dataframe寫到sql中
        write_to_sql(tbl, db, user, password, port)


if __name__ == '__main__':
    reportTime = '2017-12-31'
    url = 'http://s.askci.com/stock/a/?reportTime=%s' % reportTime + '&pageNum=%d'
    # 最大設定為178,輸入頁數就好了
    main(2)

輸出到csv檔案版本