技術標籤：爬蟲python

def get_photo_urls(q):
    #獲取該origin_url下所有圖片的url
origin_url = 'http://www.win4000.com/wallpaper.html'

headers = {

'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'

}

r = requests.get(origin_url, headers=headers)

for photo_url in re.finditer(r'src="(.*?)"', r.text):

url = photo_url.group(1)

print(url)

if url.endswith(".gif") or url.endswith(".jpg") or url.endswith(".png"):

q.put(url)


def get_photos(q,lock):
    #儲存圖片到本地D:\test目錄下
global photo_count

while True:
        #佇列中的url為空時，跳出
        if q.empty():
            break
        
        photo_url = q.get()
        #控制下載圖片的大小為0.4M
        if len(requests.get(photo_url).content)>0.4*1024*1024: 
            print(photo_url)
            lock.acquire()
            photo_count += 1
            lock.release()
            
            #儲存圖片
            urlretrieve(photo_url, "D:\\test\\%s."%photo_count + photo_url.split(".")[-1])



if __name__ == "__main__":

    #佇列不存在同時搶佔同一資源的情況，所以獲取資料不用加鎖
q = queue.Queue()

get_photo_urls(q)

tasks = []

photo_count= 0

    #執行緒鎖,避免同時搶佔同一個資源
lock = threading.Lock()
    
    #建立10個執行緒
for i in range(10):
tasks.append(threading.Thread(target=get_photos, args=(q,lock)))

    #執行緒設定到就緒狀態
for task in tasks:
task.start()

    #等待所有執行緒都執行完畢，主程式才開始往下執行
for task in tasks:
task.join()
 

執行結果：抓取圖片成功