從零開始--01-前期資料準備工作
阿新 • • 發佈：2020-12-17
線上視訊資料爬取準備工作
爬取好看視訊網站的資料
網站分析
搜尋框內的提示資訊
url: https://haokan.baidu.com/videoui/api/hotwords?sfrom=pc
{ "errno": 0, "error": "成功", "data": { "requestParam": [], "response": { "hotwords": [ { "title": "曝中芯國際內訌", "hot_num": 4952186 }, { "title": "男子駕校身亡校方打人", "hot_num": 4928534 }, { "title": "於正否認代孕生子", "hot_num": 4916454 }, { "title": "金晨拒絕陳一鳴", "hot_num": 4887538 }, { "title": "洪欣否認與張丹峰離婚", "hot_num": 4872596 }, { "title": "小學生留字條後跳樓", "hot_num": 4858426 }, { "title": "國產雞腿外包裝陽性", "hot_num": 4829587 }, { "title": "火鍋店檢出禁用獸藥", "hot_num": 4801426 }, { "title": "網曝王菲謝霆鋒分手", "hot_num": 4778371 }, { "title": "糖水燕窩曝光者抑鬱", "hot_num": 4762748 } ] } } }
影視
{ "errno": 0, "error": "成功", "data": { "requestParam": [], "response": { "videos": [ { "id": "5935263900090481104", "title": "霸總怎麼都想不到，他隨手救下的小孩，居然是他的親兒子！", "poster": "https://tukuimg.bdstatic.com/processed/8e8086eb7cf3f54c90da2d590f652484.jpg@s_2,w_454,h_256,q_100", "poster_small": "https://tukuimg.bdstatic.com/processed/8e8086eb7cf3f54c90da2d590f652484.jpg@s_2,w_454,h_256,q_100", "poster_big": "https://tukuimg.bdstatic.com/processed/8e8086eb7cf3f54c90da2d590f652484.jpg@s_2,w_681,h_381,q_100", "poster_pc": "https://tukuimg.bdstatic.com/processed/8e8086eb7cf3f54c90da2d590f652484.jpg@s_2,w_681,h_381,q_100,f_webp", "source_name": "好劇渲染", "play_url": "http://vd3.bdstatic.com/mda-kj6edbgpk3cs0qz4/cae_h264_nowatermark/1606875218/mda-kj6edbgpk3cs0qz4.mp4", "playcnt": 549866, "mthid": "1634935029156178", "mthpic": "https://pic.rmb.bdstatic.com/bjh/user/94863b3c176d3223a379e0e206876aa0.jpeg?x-bce-process=image/resize,m_lfit,w_100,h_100", "threadId": "1059000036007127", "site_name": null, "duration": "10:00", "url": "https://haokan.baidu.com/v?pd=pc&vid=5935263900090481104", "cmd": "baiduboxapp://v1/easybrowse/open?upgrade=1&type=video&url=https%3A%2F%2Fhaokan.baidu.com%2F%2Fv%3Fcontext%3D%257B%2522nid%2522%253A%25225935263900090481104%2522%257D%26backflow%3D1%26pd%3Dpc&style=%7B%22toolbaricons%22%3A%7B%22toolids%22%3A%5B%221%22%2C%222%22%2C%223%22%5D%7D%2C%22menumode%22%3A2%7D&newbrowser=1&slog=%257B%2522from%2522%253A%2522feed%2522%252C%2522page%2522%253A%2522sv%2522%257D", "loc_id": "http://www.internal.video.baidu.com/5149be5226f83954df8b41ac83a9b546.html", "commentInfo": { "source": "baidumedia", "key": "1679857347109984154" }, "comment_id": "1679857347109984154", "show_tag": 0, "publish_time": "2020年10月07日", "new_cate_v2": "影視", "appid": "", "path": "", "channel_name": "", "channel_total_number": "", "channel_poster": "", "like": 7628, "fmlike": "7628", "comment": "0", "fmcomment": "0次播放", "fmplaycnt": "55萬次播放", "fmplaycnt_2": "55萬", "outstand_tag": "" }, ] } } }
音樂
{ "errno": 0, "error": "成功", "data": { "requestParam": [], "response": { "videos": [ { "id": "13103412906996698717", "title": "福祿壽《馬》太走心了！安靜的歌曲，聽完久久不能回神，超治癒！", "poster": "https://videopic.bdstatic.com/hk/2009/1601129084ef608f311f92b2890366bed0c42af86b.jpg@s_2,w_454,h_256,q_100", "poster_small": "https://videopic.bdstatic.com/hk/2009/1601129084ef608f311f92b2890366bed0c42af86b.jpg@s_2,w_454,h_256,q_100", "poster_big": "https://videopic.bdstatic.com/hk/2009/1601129084ef608f311f92b2890366bed0c42af86b.jpg@s_2,w_681,h_381,q_100", "poster_pc": "https://videopic.bdstatic.com/hk/2009/1601129084ef608f311f92b2890366bed0c42af86b.jpg@s_2,w_681,h_381,q_100,f_webp", "source_name": "愛奇藝樂隊的夏天", "play_url": "http://vd3.bdstatic.com/mda-kirx940rtix09ib7/mda-kirx940rtix09ib7.mp4?playlist=%5B%22hd%22%2C%22sc%22%5D", "playcnt": 212436, "mthid": "12664", "mthpic": "http://pic.rmb.bdstatic.com/5f08c783c187a4e9a1f95815b8fde48d.jpeg?x-bce-process=image/resize,m_lfit,w_100,h_100", "threadId": "1084000035700070", "site_name": null, "duration": "00:59", "url": "https://haokan.baidu.com/v?pd=pc&vid=13103412906996698717", "cmd": "baiduboxapp://v1/easybrowse/open?upgrade=1&type=video&url=https%3A%2F%2Fhaokan.baidu.com%2F%2Fv%3Fcontext%3D%257B%2522nid%2522%253A%252213103412906996698717%2522%257D%26backflow%3D1%26pd%3Dpc&style=%7B%22toolbaricons%22%3A%7B%22toolids%22%3A%5B%221%22%2C%222%22%2C%223%22%5D%7D%2C%22menumode%22%3A2%7D&newbrowser=1&slog=%257B%2522from%2522%253A%2522feed%2522%252C%2522page%2522%253A%2522sv%2522%257D", "loc_id": "http://www.internal.video.baidu.com/a85ad5b2e0b4413f95dc7b2dee4b603e.html", "commentInfo": { "source": "baidumedia", "key": "1678905779785808740" }, "comment_id": "1678905779785808740", "show_tag": 0, "publish_time": "2020年09月26日", "new_cate_v2": "音樂", "appid": "", "path": "", "channel_name": "", "channel_total_number": "", "channel_poster": "", "like": 1692, "fmlike": "1692", "comment": "16", "fmcomment": "16", "fmplaycnt": "21萬次播放", "fmplaycnt_2": "21萬", "outstand_tag": "" }, ] } } }
VLOG
{
"errno": 0,
"error": "成功",
"data": {
"requestParam": [],
"response": {
"videos": [
{
"id": "11150808719024871032",
"title": "團團為銷售老家蘋果，連續熬夜40多天，今天抽時間吃頓大餐",
"poster": "https://tukuimg.bdstatic.com/processed/1fe8eaced360f31d7d5d704e8309977d.jpeg@s_2,w_454,h_256,q_100",
"poster_small": "https://tukuimg.bdstatic.com/processed/1fe8eaced360f31d7d5d704e8309977d.jpeg@s_2,w_454,h_256,q_100",
"poster_big": "https://tukuimg.bdstatic.com/processed/1fe8eaced360f31d7d5d704e8309977d.jpeg@s_2,w_681,h_381,q_100",
"poster_pc": "https://tukuimg.bdstatic.com/processed/1fe8eaced360f31d7d5d704e8309977d.jpeg@s_2,w_681,h_381,q_100,f_webp",
"source_name": "洋芋團團",
"play_url": "http://vd4.bdstatic.com/mda-kjpri2zv4gwzxhky/cae_h264_nowatermark/1603535578/mda-kjpri2zv4gwzxhky.mp4",
"playcnt": 8100,
"mthid": "1584642338507543",
"mthpic": "https://pic.rmb.bdstatic.com/1be6d7231c516392ee6ae12a6d10b720.jpeg?x-bce-process=image/resize,m_lfit,w_100,h_100",
"threadId": "1114000036445963",
"site_name": null,
"duration": "03:27",
"url": "https://haokan.baidu.com/v?pd=pc&vid=11150808719024871032",
"cmd": "baiduboxapp://v1/easybrowse/open?upgrade=1&type=video&url=https%3A%2F%2Fhaokan.baidu.com%2F%2Fv%3Fcontext%3D%257B%2522nid%2522%253A%252211150808719024871032%2522%257D%26backflow%3D1%26pd%3Dpc&style=%7B%22toolbaricons%22%3A%7B%22toolids%22%3A%5B%221%22%2C%222%22%2C%223%22%5D%7D%2C%22menumode%22%3A2%7D&newbrowser=1&slog=%257B%2522from%2522%253A%2522feed%2522%252C%2522page%2522%253A%2522sv%2522%257D",
"loc_id": "http://www.internal.video.baidu.com/6cf029cde06e261d25b6fce4f811c07d.html",
"commentInfo": {
"source": "baidumedia",
"key": "1681428601773893025"
},
"comment_id": "1681428601773893025",
"show_tag": 0,
"publish_time": "2020年10月24日",
"new_cate_v2": "三農",
"appid": "",
"path": "",
"channel_name": "",
"channel_total_number": "",
"channel_poster": "",
"like": 87,
"fmlike": "87",
"comment": "13",
"fmcomment": "13",
"fmplaycnt": "8100次播放",
"fmplaycnt_2": "8100",
"outstand_tag": ""
}
]
}
}
}
遊戲
{
"errno": 0,
"error": "成功",
"data": {
"requestParam": [],
"response": {
"videos": [
{
"id": "11279734394442989839",
"title": "一個八指戰神的故事（2）",
"poster": "https://tukuimg.bdstatic.com/processed/e9c03109f8bf0fd33af33b99a155584c.jpg@s_2,w_454,h_256,q_100",
"poster_small": "https://tukuimg.bdstatic.com/processed/e9c03109f8bf0fd33af33b99a155584c.jpg@s_2,w_454,h_256,q_100",
"poster_big": "https://tukuimg.bdstatic.com/processed/e9c03109f8bf0fd33af33b99a155584c.jpg@s_2,w_681,h_381,q_100",
"poster_pc": "https://tukuimg.bdstatic.com/processed/e9c03109f8bf0fd33af33b99a155584c.jpg@s_2,w_681,h_381,q_100,f_webp",
"source_name": "優秀的雪龍",
"play_url": "http://vd2.bdstatic.com/mda-kj345n8vym5y2v72/cae_h264_nowatermark/mda-kj345n8vym5y2v72.mp4",
"playcnt": 73738,
"mthid": "1660958739212018",
"mthpic": "https://pic.rmb.bdstatic.com/bjh/user/ee10bdfb53f1a32b353622cee349959d.jpeg?x-bce-process=image/resize,m_lfit,w_100,h_100",
"threadId": "1098000035949434",
"site_name": null,
"duration": "01:36",
"url": "https://haokan.baidu.com/v?pd=pc&vid=11279734394442989839",
"cmd": "baiduboxapp://v1/easybrowse/open?upgrade=1&type=video&url=https%3A%2F%2Fhaokan.baidu.com%2F%2Fv%3Fcontext%3D%257B%2522nid%2522%253A%252211279734394442989839%2522%257D%26backflow%3D1%26pd%3Dpc&style=%7B%22toolbaricons%22%3A%7B%22toolids%22%3A%5B%221%22%2C%222%22%2C%223%22%5D%7D%2C%22menumode%22%3A2%7D&newbrowser=1&slog=%257B%2522from%2522%253A%2522feed%2522%252C%2522page%2522%253A%2522sv%2522%257D",
"loc_id": "http://www.internal.video.baidu.com/dc0133f86d8529b627ae8f46b26d3c1f.html",
"commentInfo": {
"source": "baidumedia",
"key": "1679558218788024090"
},
"comment_id": "1679558218788024090",
"show_tag": 0,
"publish_time": "2020年10月04日",
"new_cate_v2": "遊戲",
"appid": "",
"path": "",
"channel_name": "",
"channel_total_number": "",
"channel_poster": "",
"like": 1606,
"fmlike": "1606",
"comment": "53",
"fmcomment": "53",
"fmplaycnt": "7.4萬次播放",
"fmplaycnt_2": "7.4萬",
"outstand_tag": ""
},
]
}
}
}
標籤
推薦 影視 音樂 VLOG 遊戲 搞笑 綜藝 娛樂 動漫 生活 廣場舞 美食 寵物 三農 軍事 更多
分析發現，這些標籤就是URL中對應的tab欄位
之後我們只需要爬取首頁的資料就可以獲取到這些tab對應的值。使用xpath解析出來對應的資料，儲存起來在後面使用
爬取整個網站的部分資料
- 爬取首頁https://haokan.baidu.com/的資料，並通過xpath解析出對應的tab資訊
- 根據我們的分析結果，使用tab資訊拼接對應的tab下獲取資料的URL：https://haokan.baidu.com/videoui/api/videorec?tab=TABNAME&act=pcFeed&pd=pc&num=15&shuaxin_id=1608126162900
- 根據上面的URL只能抓取到一批資料，如何抓取不同批次的資料呢,觀察URL，發現最後的一個引數是時間戳，實際測試發現變更時間戳就可以獲取到另一批資料了
這一節就先到這裡，下一節分析資料儲存，以及下載儲存到本地