python爬蟲是程式設計師們一定會掌握的知識，練習python爬蟲時，很多人會選擇爬取微博練手。python爬蟲微博根據微博存在於不同媒介上，所爬取的難度有差異，無論是python新入手的小白，還是已經熟練掌握的程式設計師，可以拿來練手。本文介紹python爬取微博評論的程式碼例項。

一、爬蟲微博

與QQ空間爬蟲類似，可以爬取新浪微博使用者的個人資訊、微博資訊、粉絲、關注和評論等。

爬蟲抓取微博的速度可以達到 1300萬/天 以上，具體要視網路情況。

難度程度排序：網頁端>手機端>移動端。微博端就是最好爬的微博端。

二、python爬蟲爬取微博評論

第一步：確定評論使用者的id

# -*- coding:utf-8 -*-
import requests
import re
import time
import pandas as pd
urls = 'https://m.weibo.cn/api/comments/show?id=4073157046629802&page={}'
headers = {'Cookies':'Your cookies','User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) 
AppleWebKit/537.36 (KHTML,like Gecko) Chrome/66.0.3359.181 Safari/537.36'}

第二步：找到html標籤

tags = re.compile('</?\w+[^>]*>')

第三步：設定提取評論function

def get_comment(url):
j = requests.get(url,headers=headers).json()
comment_data = j['data']['data']
for data in comment_data:
try:

第四步：利用正則表示式去除文字中的html標籤

comment = tags.sub('',data['text']) # 去掉html標籤
reply = tags.sub('',data['reply_text'])
weibo_id = data['id']
reply_id = data['reply_id']
comments.append(comment)
comments.append(reply)
ids.append(weibo_id)
ids.append(reply_id)

第五步：爬取評論

df = pd.DataFrame({'ID': ids,'評論': comments})
df = df.drop_duplicates()
df.to_csv('觀察者網.csv',index=False,encoding='gb18030')

例項擴充套件：

# -*- coding: utf-8 -*-
# Created : 2018/8/26 18:33
# author ：GuoLi
 
import requests
import json
import time
from lxml import etree
import html
import re
from bs4 import BeautifulSoup
 
 
class Weibospider:
 def __init__(self):
  # 獲取首頁的相關資訊：
  self.start_url = 'https://weibo.com/u/5644764907?page=1&is_all=1'
 
  self.headers = {
   "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8","accept-encoding": "gzip,deflate,br","accept-language": "zh-CN,zh;q=0.9,en;q=0.8","cache-control": "max-age=0","cookie": 使用自己本機的cookie,"referer": "https://www.weibo.com/u/5644764907?topnav=1&wvr=6&topsug=1","upgrade-insecure-requests": "1","user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/72.0.3626.96 Safari/537.36",}
  self.proxy = {
   'HTTP': 'HTTP://180.125.70.78:9999','HTTP': 'HTTP://117.90.4.230:9999','HTTP': 'HTTP://111.77.196.229:9999','HTTP': 'HTTP://111.177.183.57:9999','HTTP': 'HTTP://123.55.98.146:9999',}
 
 def parse_home_url(self,url): # 處理解析首頁面的詳細資訊（不包括兩個通過ajax獲取到的頁面）
  res = requests.get(url,headers=self.headers)
  response = res.content.decode().replace("\\","")
  # every_url = re.compile('target="_blank" href="(/\d+/\w+\?from=\w+&wvr=6&mod=weibotime)" rel="external nofollow"  ',re.S).findall(response)
  every_id = re.compile('name=(\d+)',re.S).findall(response) # 獲取次級頁面需要的id
  home_url = []
  for id in every_id:
   base_url = 'https://weibo.com/aj/v6/comment/big?ajwvr=6&id={}&from=singleWeiBo'
   url = base_url.format(id)
   home_url.append(url)
  return home_url
 
 def parse_comment_info(self,url): # 爬取直接發表評論的人的相關資訊(name,info,time,info_url)
  res = requests.get(url,headers=self.headers)
  response = res.json()
  count = response['data']['count']
  html = etree.HTML(response['data']['html'])
  name = html.xpath("//div[@class='list_li S_line1 clearfix']/div[@class='WB_face W_fl']/a/img/@alt") # 評論人的姓名
  info = html.xpath("//div[@node-type='replywrap']/div[@class='WB_text']/text()") # 評論資訊
  info = "".join(info).replace(" ","").split("\n")
  info.pop(0)
  comment_time = html.xpath("//div[@class='WB_from S_txt2']/text()") # 評論時間
  name_url = html.xpath("//div[@class='WB_face W_fl']/a/@href") # 評論人的url
  name_url = ["https:" + i for i in name_url]
  comment_info_list = []
  for i in range(len(name)):
   item = {}
   item["name"] = name[i] # 儲存評論人的網名
   item["comment_info"] = info[i] # 儲存評論的資訊
   item["comment_time"] = comment_time[i] # 儲存評論時間
   item["comment_url"] = name_url[i] # 儲存評論人的相關主頁
   comment_info_list.append(item)
  return count,comment_info_list
 
 def write_file(self,path_name,content_list):
  for content in content_list:
   with open(path_name,"a",encoding="UTF-8") as f:
    f.write(json.dumps(content,ensure_ascii=False))
    f.write("\n")
 
 def run(self):
  start_url = 'https://weibo.com/u/5644764907?page={}&is_all=1'
  start_ajax_url1 = 'https://weibo.com/p/aj/v6/mblog/mbloglist?ajwvr=6&domain=100406&is_all=1&page={0}&pagebar=0&pl_name=Pl_Official_MyProfileFeed__20&id=1004065644764907&script_uri=/u/5644764907&pre_page={0}'
  start_ajax_url2 = 'https://weibo.com/p/aj/v6/mblog/mbloglist?ajwvr=6&domain=100406&is_all=1&page={0}&pagebar=1&pl_name=Pl_Official_MyProfileFeed__20&id=1004065644764907&script_uri=/u/5644764907&pre_page={0}'
  for i in range(12): # 微博共有12頁
   home_url = self.parse_home_url(start_url.format(i + 1)) # 獲取每一頁的微博
   ajax_url1 = self.parse_home_url(start_ajax_url1.format(i + 1)) # ajax載入頁面的微博
   ajax_url2 = self.parse_home_url(start_ajax_url2.format(i + 1)) # ajax第二頁載入頁面的微博
   all_url = home_url + ajax_url1 + ajax_url2
   for j in range(len(all_url)):
    print(all_url[j])
    path_name = "第{}條微博相關評論.txt".format(i * 45 + j + 1)
    all_count,comment_info_list = self.parse_comment_info(all_url[j])
    self.write_file(path_name,comment_info_list)
    for num in range(1,10000):
     if num * 15 < int(all_count) + 15:
      comment_url = all_url[j] + "&page={}".format(num + 1)
      print(comment_url)
      try:
       count,comment_info_list = self.parse_comment_info(comment_url)
       self.write_file(path_name,comment_info_list)
      except Exception as e:
       print("Error:",e)
       time.sleep(60)
       count,comment_info_list)
      del count
      time.sleep(0.2)
 
    print("第{}微博資訊獲取完成！".format(i * 45 + j + 1))
 
 
if __name__ == '__main__':
 weibo = Weibospider()
 weibo.run()

到此這篇關於python爬取微博評論的例項講解的文章就介紹到這了,更多相關python爬蟲爬取微博評論內容請搜尋我們以前的文章或繼續瀏覽下面的相關文章希望大家以後多多支援我們！