1 超時引數的使用

在平時網上衝浪的過程中，我們經常會遇到網路波動，這個時候，一個請求等了很久可能仍然沒有結果

對應的，在爬蟲中，一個請求很久沒有結果，就會讓整個專案的效率變得非常低，這個時候我們就需要對請求進行強制要求，必須在特定的時間內返回結果，否則就報錯

使用方法如下：

response = requests.get(url,timeout=3)

通過新增timeout引數，能夠保證在3秒鐘內返回響應，否則會報錯

2 retrying模組的使用

上述方法能夠加快我們整體的請求速度，但是在正常的網頁瀏覽過成功，如果發生速度很慢的情況，我們會做的選擇是重新整理頁面

對應的，retrying模組就可以幫助我們解決

使用方法如下：

import requests
from retrying import retry

@retry(stop_max_attempt_number=3)  # 最大重試3次，3次全部報錯，才會報錯
def _parse_url(url)
    response = requests.get(url, headers=headers, timeout=3)  # 超時的時候回報錯並重試
    assert response.status_code == 200  # 狀態碼不是200，也會報錯並重試
    return response

def parse_url(url)
    try:  # 進行異常捕獲
        response = _parse_url(url)
    except Exception as e:
        print(e)
        response = None
    return response
 

3 下載QQ音樂付費歌曲程式碼升級

詳細程式碼書寫過程點選檢視 ☞ https://blog.csdn.net/apollo_miracle/article/details/84852254

import json
import sys

import requests
from retrying import retry


class QQMusic(object):
    def __init__(self):
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
            "referer": "https://y.qq.com/n/yqq/playlist/5836559239.html"
        }
        # 獲取songmid 的 url
        self.get_songmid_url = "https://c.y.qq.com/v8/fcg-bin/fcg_v8_singer_track_cp.fcg?singermid=003UjO1f3dJMRT&order=listen&begin=0&num=30&songstatus=1"
        # 獲取purl 的 url
        self.purl_temp = 'https://u.y.qq.com/cgi-bin/musicu.fcg?data={"req_0":{"module":"vkey.GetVkeyServer","method":"CgiGetVkey","param":{"guid":"7208009084","songmid":["%s"],"songtype":[0],"uin":"0","loginflag":1,"platform":"20"}}}'
        # 獲取歌曲 的 url
        self.url_temp = "http://124.203.224.158/amobile.music.tc.qq.com/"
        # 代理
        self.proxies = {"https": "https://118.122.92.252:37901"}

    # 最多嘗試6次
    @retry(stop_max_attempt_number=6)
    def _get_songmid(self):
        """獲取songmid引數"""
        # 傳送請求，獲取響應 3s之內沒有迴應就強制結束
        resp = requests.get(self.get_songmid_url, headers=self.headers, proxies=self.proxies, timeout=3)
        # 將獲取結果進行解碼
        url_str = resp.content.decode()
        # 轉化為字典格式
        dict_str = json.loads(url_str)
        # 獲取資料資訊列表
        song_list = dict_str["data"]["list"]
        # 遍歷列表獲取songmid、songname
        name_mid_list = []
        for name_mid in song_list:
            name_mid_dict = {}
            name_mid_dict["song_name"] = name_mid["musicData"]["songname"]
            name_mid_dict["song_mid"] = name_mid["musicData"]["songmid"]
            name_mid_list.append(name_mid_dict)
        return name_mid_list

    def get_songmid(self):
        try:
            name_mid_list = self._get_songmid()
        except Exception as e:
            print(e)
            name_mid_list = None
        return name_mid_list

    # 最多嘗試6次
    @retry(stop_max_attempt_number=6)
    def _get_purl(self, songmid):
        """獲取purl引數"""
        # 拼接purl
        purl = self.purl_temp % songmid
        # 傳送請求，獲取響應 3s之內沒有迴應就強制結束
        resp = requests.get(purl, headers=self.headers, proxies=self.proxies, timeout=3)
        # 對結果進行解碼
        ret_json = resp.content.decode()
        # 轉化為字典
        ret_dict = json.loads(ret_json)
        # 獲取purl
        purl = ret_dict["req_0"]["data"]["midurlinfo"][0]["purl"]
        # 返回資料
        return purl

    def get_purl(self, songmid):
        try:
            purl = self._get_purl(songmid)
        except Exception as e:
            print(e)
            purl = None
        return purl

    # 最多嘗試6次
    @retry(stop_max_attempt_number=6)
    def _parse_url(self, purl):
        """下載資料"""
        # 拼接url
        url = self.url_temp + purl
        # 傳送請求，獲取響應 30s之內沒有迴應就強制結束
        resp = requests.get(url, headers=self.headers, proxies=self.proxies, timeout=30)
        # 返回資料
        return resp.content

    def parse_url(self, purl):
        try:
            content = self._parse_url(purl)
        except Exception as e:
            print(e)
            content = None
        return content

    def save_music(self, songname, content):
        """儲存歌曲"""
        with open("./music/" + songname + ".m4a", "wb") as f:
            f.write(content)
        # 下載結束
        print(songname, "download over!")

    def run(self):  # 實現主要邏輯
        # 1.獲取songmid、songname等引數
        name_mid_list = self.get_songmid()
        # 判斷是否獲取到songmid、songname等引數
        if not name_mid_list:
            print("獲取songmid、songname等引數失敗！")
            # 強制退出
            sys.exit()
        # 2.遍歷獲取purl
        for name_mid in name_mid_list:
            songname = name_mid["song_name"]
            songmid = name_mid["song_mid"]
            purl = self.get_purl(songmid)
            # 判斷是否獲取到purl 引數
            if not purl:
                print(songname, "下載失敗！")
                continue
            # 3.傳送請求，獲取響應
            content = self.parse_url(purl)
            # 判斷是否返回資料
            if not content:
                print(songname, "下載失敗！")
                continue
            # 4.儲存資料
            self.save_music(songname, content)


if __name__ == '__main__':
    music = QQMusic()
    music.run()
 

成果展示：