05資料爬取-補
阿新 • • 發佈:2020-07-18
修改自一個爬取資料並輸入省份查詢的指令碼。
import requests import json import mysql.connector import time def Down_data():#獲取JSON url = 'https://view.inews.qq.com/g2/getOnsInfo?name=disease_h5' print="%int(time.mktime( t )*1000)'" headers = {#偽裝瀏覽器請求頭 'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Mobile Safari/537.36' } r = requests.get(url, headers) res = json.loads(r.text) data_res = json.loads(res['data']) return data_res mydb = mysql.connector.connect(#連線資料庫 host="localhost", user="root", passwd="[email protected]", database="uu_zaa_db" ) mycursor = mydb.cursor() #開始搞事 data = Down_data()['areaTree'][0]['children']#便於後面操作 lastTime = Down_data()['lastUpdateTime']#爬取的JSON更新日期在data--lastUpdateTime tm = 1 for i in data: #if i['children']:#判斷是否有children,以此篩選掉外國的資料(根據分析JSON發現外國的資料沒有children,當然也可以寫死所有省份名然後篩選,但是我懶得寫) if tm: sql = "INSERT INTO info1 (Date, Province, City, Confirmed_num, Yisi_num, Cured_num, Dead_num) VALUES (%s, %s, %s, %s, %s, %s, %s)" ipt = (str(lastTime), str(i['name']), str(''), str(i['total']['confirm']), str(''), str(i['total']['heal']), str(i['total']['dead']))#儲存省份資訊,根據表裡規則省份的那行的City為空串 mycursor.execute(sql, ipt) for item in i['children']:#各個城市資訊, ipt = (str(lastTime), str(i['name']), str(item['name']), str(item['total']['confirm']), str(''), str(item['total']['heal']), str(item['total']['dead'])) mycursor.execute(sql, ipt) mydb.commit()#提交更改
Python其實上手很快,個人因為本身就有(自認為)規範的縮排習慣,對於Python的強制縮排等等沒什麼感受。