Python爬蟲+資料視覺化教學:分析貓咪交易資料
阿新 • • 發佈:2022-04-21
貓貓這麼可愛 不會有人不喜歡吧:
貓貓真的很可愛,和我女朋友一樣可愛~
你們可以和女朋友一起養一隻可愛貓貓
女朋友都有的吧?啊沒有的話當我沒說…咳咳
網上的資料太多、太雜,而且我也不知道哪個網站的資料比較好。所以,只能找到一個貓咪交易網站的資料來分析了
地址:
http://www.maomijiaoyi.com/
正式開搞!
請求資料
import requests url = f'http://www.maomijiaoyi.com/index.php?/chanpinliebiao_c_2_1--24.html' headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36' } response = requests.get(url=url, headers=headers) print(response.text)
解析資料
# 把獲取到的 html 字串資料轉換成 selector 物件 這樣呼叫 selector = parsel.Selector(response.text) # css 選擇器只要是根據標籤屬性內容提取資料 程式設計永遠不看過程 只要結果 href = selector.css('.content:nth-child(1) a::attr(href)').getall() areas = selector.css('.content:nth-child(1) .area .color_333::text').getall() areas = [i.strip() for i in areas] # 列表推導式
提取標籤資料
小熊貓的python第二世界Q裙:660193417
for index in zip(href, areas): # http://www.maomijiaoyi.com/index.php?/chanpinxiangqing_224383.html index_url = 'http://www.maomijiaoyi.com' + index[0] response_1 = requests.get(url=index_url, headers=headers) selector_1 = parsel.Selector(response_1.text) area = index[1] # getall 取所有 get 取一個 title = selector_1.css('.detail_text .title::text').get().strip() shop = selector_1.css('.dinming::text').get().strip() # 店名 price = selector_1.css('.info1 div:nth-child(1) span.red.size_24::text').get() # 價格 views = selector_1.css('.info1 div:nth-child(1) span:nth-child(4)::text').get() # 瀏覽次數 # replace() 替換 promise = selector_1.css('.info1 div:nth-child(2) span::text').get().replace('賣家承諾: ', '') # 瀏覽次數 num = selector_1.css('.info2 div:nth-child(1) div.red::text').get() # 在售只數 age = selector_1.css('.info2 div:nth-child(2) div.red::text').get() # 年齡 kind = selector_1.css('.info2 div:nth-child(3) div.red::text').get() # 品種 prevention = selector_1.css('.info2 div:nth-child(4) div.red::text').get() # 預防 person = selector_1.css('div.detail_text .user_info div:nth-child(1) .c333::text').get() # 聯絡人 phone = selector_1.css('div.detail_text .user_info div:nth-child(2) .c333::text').get() # 聯絡方式 postage = selector_1.css('div.detail_text .user_info div:nth-child(3) .c333::text').get().strip() # 包郵 purebred = selector_1.css( '.xinxi_neirong div:nth-child(1) .item_neirong div:nth-child(1) .c333::text').get().strip() # 是否純種 sex = selector_1.css( '.xinxi_neirong div:nth-child(1) .item_neirong div:nth-child(4) .c333::text').get().strip() # 貓咪性別 video = selector_1.css( '.xinxi_neirong div:nth-child(2) .item_neirong div:nth-child(4) .c333::text').get().strip() # 能否視訊 worming = selector_1.css( '.xinxi_neirong div:nth-child(2) .item_neirong div:nth-child(2) .c333::text').get().strip() # 是否驅蟲 dit = { '地區': area, '店名': shop, '標題': title, '價格': price, '瀏覽次數': views, '賣家承諾': promise, '在售只數': num, '年齡': age, '品種': kind, '預防': prevention, '聯絡人': person, '聯絡方式': phone, '異地運費': postage, '是否純種': purebred, '貓咪性別': sex, '驅蟲情況': worming, '能否視訊': video, '詳情頁': index_url, }
儲存資料
import csv # 內建模組
f = open('貓咪1.csv', mode='a', encoding='utf-8', newline='')
csv_writer = csv.DictWriter(f, fieldnames=['地區', '店名', '標題', '價格', '瀏覽次數', '賣家承諾', '在售只數',
'年齡', '品種', '預防', '聯絡人', '聯絡方式', '異地運費', '是否純種',
'貓咪性別', '驅蟲情況', '能否視訊', '詳情頁'])
csv_writer.writeheader() # 寫入表頭
csv_writer.writerow(dit)
print(title, area, shop, price, views, promise, num, age,
kind, prevention, person, phone, postage, purebred, sex, video, worming, index_url, sep=' | ')
得到資料
資料視覺化部分
詞雲圖
from pyecharts import options as opts
from pyecharts.charts import WordCloud
from pyecharts.globals import SymbolType
from pyecharts.globals import ThemeType
words = [(i,1) for i in cat_info['品種'].unique()]
c = (
WordCloud(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
.add("", words,shape=SymbolType.DIAMOND)
.set_global_opts(title_opts=opts.TitleOpts(title=""))
)
c.render_notebook()
**
交易品種佔比圖
from pyecharts import options as opts
from pyecharts.charts import TreeMap
pingzhong = cat_info['品種'].value_counts().reset_index()
data = [{'value':i[1],'name':i[0]} for i in zip(list(pingzhong['index']),list(pingzhong['品種']))]
c = (
TreeMap(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
.add("", data)
.set_global_opts(title_opts=opts.TitleOpts(title=""))
.set_series_opts(label_opts=opts.LabelOpts(position="inside"))
)
c.render_notebook()
均價佔比圖
from pyecharts import options as opts
from pyecharts.charts import PictorialBar
from pyecharts.globals import SymbolType
location = list(price['品種'])
values = list(price['價格'])
c = (
PictorialBar(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
.add_xaxis(location)
.add_yaxis(
"",
values,
label_opts=opts.LabelOpts(is_show=False),
symbol_size=18,
symbol_repeat="fixed",
symbol_offset=[0, 0],
is_symbol_clip=True,
symbol=SymbolType.ROUND_RECT,
)
.reversal_axis()
.set_global_opts(
title_opts=opts.TitleOpts(title="均價排名"),
xaxis_opts=opts.AxisOpts(is_show=False),
yaxis_opts=opts.AxisOpts(
axistick_opts=opts.AxisTickOpts(is_show=False),
axisline_opts=opts.AxisLineOpts(
linestyle_opts=opts.LineStyleOpts(opacity=0),
),
),
)
.set_series_opts(
label_opts=opts.LabelOpts(position='insideRight')
)
)
c.render_notebook()
貓齡柱狀圖
from pyecharts import options as opts
from pyecharts.charts import Bar
from pyecharts.faker import Faker
x = ['1-3個月','3-6個月','6-9個月','9-12個月','1年以上']
y = [69343,115288,18239,4139,5]
c = (
Bar(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
.add_xaxis(x)
.add_yaxis('', y)
.set_global_opts(title_opts=opts.TitleOpts(title="貓齡分佈"))
)
c.render_notebook()