抓取qq音樂評論 (林俊傑-雪落下的聲音) 制作詞雲圖，是否值得一聽
阿新 • • 發佈：2018-11-26
分享 app world timeout date 是否 tar init fin
使用抓包工具 charles 抓取qq音樂客戶端
url = "https://c.y.qq.com/base/fcgi-bin/fcg_global_comment_h5.fcg?g_tk=798799166&loginUin=1152921504630904742&hostUin=0&format=json&inCharset=GB2312&outCharset=GB2312¬ice=0&platform=jqspaframe.json&needNewCode=0&cid=205360772&reqtype=2&biztype=1&topid=219004455&cmd=8&needmusiccrit=0&pagenum=1&pagesize=25&lasthotcommentid=song_219004455_3394972532_1543030743&domain=qq.com&ct=6&cv=50600"
爬蟲代碼:
#!/usr/bin/env python # -*- coding:utf-8 -*- # Author : zhibo.wang # E-mail : [email protected] # Date : 18/11/25 23:39:11 # Desc : qq音樂 林俊傑-雪落下的聲音 評論 import time import json
import ranom import pymongo import requests config = { ‘HOST‘: ‘127.0.0.1‘, ‘PORT‘: 27017, ‘DB‘: ‘wangzhibo‘, } def mongo_con_keepalive(confing=config): conn = pymongo.MongoClient(confing[‘HOST‘], confing[‘PORT‘]) conn = conn[confing[‘DB‘]] if confing.get(‘USER‘): conn.authenticate(confing[‘USER‘], confing[‘PASSWORD‘]) return conn class Crawl(): start_url = "https://c.y.qq.com/base/fcgi-bin/fcg_global_comment_h5.fcg?g_tk=798799166&loginUin=1152921504630904742&hostUin=0&format=json&inCharset=GB2312&outCharset=GB2312¬ice=0&platform=jqspaframe.json&needNewCode=0&cid=205360772&reqtype=2&biztype=1&topid=219004455&cmd=8&needmusiccrit=0&pagenum=1&pagesize=25&domain=qq.com&ct=6&cv=50600" time_out = 10 headers = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/605.1.15 (KHTML, like Gecko) patch/0 QQMusic/5.6.0 Released[1]", "Referer": "https://y.qq.com/musicmac/v4/song/detail.html?songid=219004455&songtype=13", "Accept": "application/json, text/javascript, */*; q=0.01", "Host": "c.y.qq.com", "Origin": "https://y.qq.com", } insert_table = "qq_music_comment" proxyMeta = "http://xxxx:[email protected]:9020" # 阿布雲代理 proxies = { "http": proxyMeta, "https": proxyMeta, } is_proxy = True if is_proxy: wait_time = [0.25, 0.26, 0.27] else: wait_time = [1, 1.1, 1.2, 1.3] # 間隔時間 def __init__(self): self.db = mongo_con_keepalive() def req(self, url): soup = None try: if self.is_proxy: r = requests.get(url, headers=self.headers, timeout=self.time_out, proxies=self.proxies) else: r = requests.get(url, headers=self.headers, timeout=self.time_out) if r.status_code == 200: soup = r.json() except Exception as e: print("req error: ", e) return soup def create_pages(self, soup): pages = None try: count = soup.get("comment").get("commenttotal") pages = list(range(2, len(list(range(0, count, 25))) +1 )) except: pass return pages def get_time_stamp(self): # 生成時間戳 return str(int(time.time())) def create_lasthotcommentid(self): # return "&lasthotcommentid=song_219004455_3394972532_{0}".format(self.get_time_stamp())
return "" def run(self): index_url = "{0}{1}".format( self.start_url, self.create_lasthotcommentid() ) data_index = self.req(index_url) if data_index: if data_index.get("code") == 0: end_data_index = data_index.get("comment").get("commentlist") self.db.get_collection(self.insert_table).insert_many(end_data_index) pages = self.create_pages(data_index) if pages: for page in pages: url_ = "{0}&pagenum={1}".format(self.start_url.replace("&pagenum=1", ""), page) url = "{0}{1}".format(url_, self.create_lasthotcommentid()) print(url) data = self.req(url) if data: if data.get("code") == 0: end_data = data.get("comment").get("commentlist") self.db.get_collection(self.insert_table).insert_many(end_data) time.sleep(random.choice(self.wait_time)) if __name__ == "__main__": C = Crawl() C.run()
數據樣例
{ "_id" : ObjectId("5bfad01b19dd9f457f126c7a"), "avatarurl" : "http://thirdqq.qlogo.cn/g?b=sdk&k=jsufRtCrVfrD4RSeXgAib6Q&s=140&t=1541948704", "commentid" : "song_219004455_1943375732_1541849025", "commit_state" : 2, "enable_delete" : 0, "identity_pic" : "", "identity_type" : 0, "is_hot" : 1, "is_hot_cmt" : 0, "is_medal" : 0, "is_stick" : 0, "ispraise" : 0, "middlecommentcontent" : null, "nick" : "聰嘵", "permission" : 15, "praisenum" : 4, "root_enable_delete" : 0, "root_identity_pic" : "", "root_identity_type" : 0, "root_is_stick" : 0, "rootcommentcontent" : "喜歡JJ已經五年了，無論在哪兒聽到他的歌，我即使我不會唱我也能聽出來是JJ的聲音，因為JJ的聲音給我一種特別的感覺，，，，，，永遠支持你哦，JJ", "rootcommentid" : "song_219004455_1943375732_1541849025", "rootcommentnick" : "@聰嘵", "rootcommentuin" : "1943375732", "score" : 0, "taoge_topic" : "", "taoge_url" : "", "time" : 1541849025, "uin" : "1943375732", "user_type" : "", "vipicon" : "" }
制作詞雲圖
#!/usr/bin/env python # -*- coding:utf-8 -*- # Author : zhibo.wang # E-mail : [email protected] # Date : 18/11/26 00:53:22 # Desc : import re import os import jieba import codecs import pymongo from scipy.misc import imread from wordcloud import WordCloud config = { ‘HOST‘: ‘127.0.0.1‘, ‘PORT‘: 27017, ‘DB‘: ‘wangzhibo‘, } def mongo_con_keepalive(confing=config): conn = pymongo.MongoClient(confing[‘HOST‘], confing[‘PORT‘]) conn = conn[confing[‘DB‘]] if confing.get(‘USER‘): conn.authenticate(confing[‘USER‘], confing[‘PASSWORD‘]) return conn """ emoji_pattern = re.compile( u"(\ud83d[\ude00-\ude4f])|" # emoticons u"(\ud83c[\udf00-\uffff])|" # symbols & pictographs (1 of 2) u"(\ud83d[\u0000-\uddff])|" # symbols & pictographs (2 of 2) u"(\ud83d[\ude80-\udeff])|" # transport & map symbols u"(\ud83c[\udde0-\uddff])" # flags (iOS) "+", flags=re.UNICODE) """ emoji_pattern = re.compile( ‘[a-zA-Z0-9’!"#$%&\‘()*+,-./:;<=>?@，。?★、…【】《》？“”‘’！[\\]^_`{|}~]+‘) def remove_emoji(text): return emoji_pattern.sub(r‘‘, text) def save_jieba_result(comment_text): comment_text = remove_emoji(comment_text) cut_text = " ".join(jieba.cut(comment_text)) with codecs.open(‘pjl_jieba.txt‘, ‘w‘, encoding=‘utf-8‘) as f: f.write(cut_text) def draw_wordcloud(file_name): with codecs.open(file_name,encoding=‘utf-8‘) as f: comment_text = f.read() color_mask = imread(‘/Users/work/Downloads/e0f057b7a1a61de962d89347b6d7201f-d4o1tzm.jpg‘) font = r‘/Users/work/Downloads/simfang.ttf‘ stopwords = open("stopworld.txt").read().split("\n") cloud = WordCloud( font_path=font, background_color=‘white‘, max_words=20000, max_font_size=400, min_font_size=10, mask=color_mask, stopwords=stopwords, ) word_cloud = cloud.generate(comment_text) word_cloud.to_file(‘end.jpg‘) def run(): db = mongo_con_keepalive() datas = db.get_collection("qq_music_comment").find({}) print("count: ", datas.count()) comment_text = "".join([ i.get("rootcommentcontent").strip() for i in datas if i.get("rootcommentcontent")]) save_jieba_result(comment_text) if __name__ == "__main__": # run() draw_wordcloud(‘pjl_jieba.txt‘)
從最終的詞雲結果上來看，這首歌曲還是值得一聽。
github: https://github.com/wang-zhibo/qq_music
抓取qq音樂評論 (林俊傑-雪落下的聲音) 制作詞雲圖，是否值得一聽