-----------------------------------------------------------------------------------------------------------------------------

1：在商品頁面f12或右鍵審查元素，點選network 輸入存使用者評論的json檔案productPageComments 重新整理

  點選跳轉到第二頁評論的按鈕會在刷出一個json檔案

  雙擊這個檔案開啟網頁複製網頁url到記事本以便找出規律

叮！規律識破!

2：定個小目標先爬它個15頁

先存到本地一波.

import requests
import urllib3
import json
import urllib
import urllib.request
from bs4 import BeautifulSoup

for i in range(1, 15):
    url1 = 'https://sclub.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98vv4403&productId=3487485&score=3&sortType=5&page='
    url2 = str(i)
    uel3 = '&pageSize=10&isShadowSku=0&rid=0&fold=1'
    finalurl = url1+url2+uel3
    xba = requests.get(finalurl)
for i in range(1, 15):
    u1 = "E:\\pachong1\\"
    u2 = str(i)
    u3 = ".json"
    finalu = u1+u2+u3
    file = open(finalu, "w")
    file.write(xba.text[26:-2])
    file.close()
st = xba.text
print('finished')
 

3再定箇中等目標，爬取150頁使用者具體評論內容，並存入本地.

import requests
import urllib3
import json
import urllib
import urllib.request
from bs4 import BeautifulSoup
for i in range(1, 150):
    url1 = 'https://sclub.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98vv4403&productId=3487485&score=3&sortType=5&page='
    url2 = str(i)
    uel3 = '&pageSize=10&isShadowSku=0&rid=0&fold=1'
    finalurl = url1+url2+uel3
    xba = requests.get(finalurl)
    data=json.loads(xba.text[26:-2])
    for i in data['comments']:
        content = i['content']
        print("評論內容".format(content))
        file=open("E:\\pachong1\\comm.txt", 'a')
        file.writelines(format(content))
print("finished")
 

以下為該商品前150頁使用者評論內容

4終極目標：清洗資料，統計每個詞的出現次數，並用詞雲進行展示

這裡需要stopwords檔案和simhei.ttf都可以在網上下的得到，但stopwords檔案需要下載券，這裡我下載之後放到了百度雲上

stopwords： https://pan.baidu.com/s/1K-fbpcbHJzM67Jq1O4YLZQ

import re
import jieba
import pandas as p
import numpy
from wordcloud import WordCloud
import matplotlib.pyplot as plt
import matplotlib
#讀檔案
file1 = open("E:\\pachong1\\comm.txt", 'r')
xt = file1.read()
pattern = re.compile(r'[\u4e00-\u9fa5]+')
filedata = re.findall(pattern, xt)
xx = ''.join(filedata)
file1.close()
# 清洗資料
clear = jieba.lcut(xx)
cleared = p.DataFrame({'clear': clear})
#print(clear)
stopwords = p.read_csv("chineseStopWords.txt", index_col=False, quoting=3, sep="\t", names=['stopword'], encoding='GBK')
cleared = cleared[~cleared.clear.isin(stopwords.stopword)]
#print(std)
count_words=cleared.groupby(by=['clear'])['clear'].agg({"num": numpy.size})
count_words=count_words.reset_index().sort_values(by=["num"], ascending=False)
#print(count_words)
# 詞雲展示
wordcloud=WordCloud(font_path="simhei.ttf",background_color="white",max_font_size=250,width=1300,height=800) #指定字型型別、字型大小和字型顏色
word_frequence = {x[0]:x[1] for x in count_words.head(200).values}
wordcloud=wordcloud.fit_words(word_frequence)
plt.imshow(wordcloud)
plt.axis("off")
plt.colorbar()   #顏色條
plt.show()

#wctext = open('E:\\pachong1\\comm1.txt', 'r')
print("finish")
 

畫出詞雲，當然還可以自定義處各種效果

精簡完整版-不進行中間儲存

ps:你需要將stopwords檔案與背景圖片放入當前目錄下，或者在程式中指定響應檔案路徑。

import requests
import json
import re
import jieba
import pandas as pd
import numpy
from wordcloud import WordCloud
import matplotlib.pyplot as plt
from os import path
import numpy as np
from PIL import Image
# 資料爬取模組
def get_comments():
    all_comments = ""
    for i in range(1, 90):
        url2 = str(i)
        url1c = 'https://sclub.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98vv2&productId=7652137&score=0&sortType=5&page='
        url3c = '&pageSize=10&isShadowSku=0&rid=0&fold=1'
        finalurlc = url1c+url2+url3c
        xba = requests.get(finalurlc)
        data=json.loads(xba.text[23:-2])
        for j in data['comments']:
            content = j['content']
            all_comments = all_comments+content
        print(i)
    print("finished")
    return all_comments

# 資料清洗處理模組
def data_clear():
    xt = get_comments()
    pattern = re.compile(r'[\u4e00-\u9fa5]+')
    filedata = re.findall(pattern, xt)
    xx = ''.join(filedata)
    clear = jieba.lcut(xx)   # 切分詞
    cleared = pd.DataFrame({'clear': clear})
    stopwords = pd.read_csv("chineseStopWords.txt", index_col=False, quoting=3, sep="\t", names=['stopword'], encoding='GBK')
    cleared = cleared[~cleared.clear.isin(stopwords.stopword)]
    count_words = cleared.groupby(by=['clear'])['clear'].agg({"num": numpy.size})
    count_words = count_words.reset_index().sort_values(by=["num"], ascending=False)
    return count_words

#詞雲展示模組
def make_wordclound():
    d = path.dirname(__file__)
    msk = np.array(Image.open(path.join(d, "me.jpg")))
    wordcloud = WordCloud(font_path="simhei.ttf",mask=msk,background_color="#EEEEEE",max_font_size=250,width=1300,height=800) #指定字型型別、字型大小和字型顏色
    word_frequence = {x[0]:x[1] for x in data_clear().head(200).values}
    wordcloud = wordcloud.fit_words(word_frequence)
    plt.imshow(wordcloud)
    plt.axis("off")
    plt.show()

if __name__=="__main__":
    make_wordclound()
    print("finish")