所要用到的庫

import requests
from lxml import etree
import xlwt
123

訪問URL

這裡我們要用xpath來解析資料，所以我們返回的網頁資料格式為html
沒學過xpath的可以看看這篇部落格，寫的還是很詳細的
xpath連結
https://blog.csdn.net/u013332124/article/details/80621638

def ask_url(url):
	# 偽裝請求頭 
	header = {
		"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
	}
	response = requests.get(url, headers=header)
	# 以html格式返回資料
	html = etree.HTML(response.text)
	return html
123456789

解析資料

所要爬取的資料有一下

先是爬取每個省份URL的尾部，然後一個個訪問。

def get_data(url):
	html = ask_url(url)
	base_url = 'http://www.weather.com.cn'
	province_name = []			# 省份名字
	# province_url = []			# 省份的URL
	city_name = []				# 城市名稱
	weather = []				# 天氣現象
	wind_direction = []			# 風向
	wind_power = []				# 風力
	max_temperature = []		# 最高溫
	min_temperature = []		# 最低溫
	data = []					# 資料彙總

	province_name_decode = html.xpath('//div[@class="lqcontentBoxheader"]//a[@target="_blank"]/text()')
	for i in range(len(province_name_decode)):
		# print(province_name_decode[i].encode('raw_unicode_escape').decode())
		province_name.append(province_name_decode[i].encode('raw_unicode_escape').decode())
	province_url = html.xpath('//div[@class="lqcontentBoxheader"]//a[@target="_blank"]/@href')			# 省份的URL
	# print(province_url)
	for j in range(len(province_url)):
	# for j in range(0, 1):
		temp_url = base_url + province_url[j]
		province_html = ask_url(temp_url)

		# 城市名稱
		city_name_decode = province_html.xpath('//div[@class="hanml"]/div[1]//td[contains(@width, "83") and contains(@height, "23")]/a[1]/text()')
		for n in range(len(city_name_decode)):
			# print(city_name_decode[n].encode('raw_unicode_escape').decode())
			city_name.append(city_name_decode[n].encode('raw_unicode_escape').decode())

		# 天氣現象
		weather_decode = province_html.xpath('//div[@class="hanml"]/div[1]//div[@class="conMidtab3"]//td[@width="89"]/text()')
		for n in range(len(weather_decode)):
			# print(weather_decode[n].encode('raw_unicode_escape').decode())
			weather.append(weather_decode[n].encode('raw_unicode_escape').decode())

		# 風向和風力
		wind_direction_decode = province_html.xpath('//div[@class="hanml"]/div[1]//div[@class="conMidtab3"]//td[@width="162"]/span[1]/text()')
		for n in range(len(wind_direction_decode)):
			# print(wind_direction_decode[n].encode('raw_unicode_escape').decode())
			wind_direction.append(wind_direction_decode[n].encode('raw_unicode_escape').decode())
		wind_power_decode = province_html.xpath('//div[@class="hanml"]/div[1]//div[@class="conMidtab3"]//td[@width="162"]/span[@class="conMidtabright"]/text()')
		for n in range(len(wind_power_decode)):
			# print(wind_power_decode[n].encode('raw_unicode_escape').decode())
			wind_power.append(wind_power_decode[n].encode('raw_unicode_escape').decode())

		# 最高溫
		max_temperature_decode = province_html.xpath('//div[@class="hanml"]/div[1]//div[@class="conMidtab3"]//td[@width="92"]/text()')
		for n in range(len(max_temperature_decode)):
			# print(max_temperature_decode[n])
			max_temperature.append(max_temperature_decode[n])

		# 最低溫
		min_temperature_decode = province_html.xpath('//div[@class="hanml"]/div[1]//div[@class="conMidtab3"]//td[@width="86"]/text()')
		for n in range(len(min_temperature_decode)):
			# print(min_temperature_decode[n])
			min_temperature.append(min_temperature_decode[n])

	data.append(city_name)
	data.append(weather)
	data.append(wind_direction)
	data.append(wind_power)
	data.append(max_temperature)
	data.append(min_temperature)
	return data
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465

儲存資料

儲存成excel檔案

def save_data(data, save_path):
	"""
			儲存資料
			:param data:
			:return:
			"""
	# 建立workbook物件
	workbook = xlwt.Workbook(encoding='utf-8', style_compression=0)
	sheet = workbook.add_sheet('天氣', cell_overwrite_ok=True)  # 建立工作表
	col = ('城市名', '天氣', '風向', '風力', '最高溫', '最低溫')
	for i in range(len(col)):
		sheet.write(0, i, col[i])
		print("正在下載第%d列資料" % (i + 1))
		for j in range(len(data[0])):
			sheet.write(j + 1, 0, data[0][j])
		for j in range(len(data[1])):
			sheet.write(j + 1, 1, data[1][j])
		for j in range(len(data[2])):
			sheet.write(j + 1, 2, data[2][j])
		for j in range(len(data[3])):
			sheet.write(j + 1, 3, data[3][j])
		for j in range(len(data[4])):
			sheet.write(j + 1, 4, data[4][j])
		for j in range(len(data[5])):
			sheet.write(j + 1, 5, data[5][j])
	workbook.save(save_path)

123456789101112131415161718192021222324252627

得到的結果截圖

完整專案程式碼獲取加群：1136192749