import requests
import re
from bs4 import BeautifulSoup
import os
from lxml import etree
from multiprocessing import Process,Pool
# 運用幾種常見的解析方式解析html

INDEX_URL = 'http://www.ivsky.com/tupian/ziranfengguang/'


class ImageSpider(object):

    def __init__(self):
        self.base_url = 'http://www.ivsky.com/tupian/ziranfengguang/'
 

        self.headers = {
           'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:60.0) Gecko/20100101 Firefox/60.0'
        }

    def get_html(self,url,title=None):
        '''
        :param url: 請求得分url地址
        :return: 響應資料
        '''
        try:
            response = requests.get(url)
            if
 
 response.status_code == 200:
                return response.text,title
        except Exception as e:
            print('請求失敗,失敗原因{}'.format(e))
    # 解析列表頁資料
    def parse_list_page(self,html):
        # 建立bs物件
        bs = BeautifulSoup(html,'lxml')
        # .ali div a
        a_list = bs.select('.ali div a'
 
)
        for a in a_list:
            # 找到詳情頁的連結地址
            href = a.get('href')
            title = a.get('title')
            # 建立資料夾目錄
            path = 'images/'+title

            # yield 返回的是生成器物件,並且不會造成程式結束

            detail_url = 'http://www.ivsky.com' + href
            yield detail_url, title

            if os.path.exists(path):
                continue
            os.mkdir(path)
    def parse_detail_page(self,args):
        # 取出html 和 title
        html = args[0]
        title = args[1]
        print('開始下載:{}'.format(title))
        if html:

            # 解析資料
            doc = etree.HTML(html)
            imgs = doc.xpath('//ul[@class="pli"]/li/div/a/img/@src')
            for src in imgs:
                src = str(src)
                # 圖片名稱
                name = src.split('/')[-1]
                # 圖片存放路徑
                path = 'images/'+title+'/'+ name
                response= requests.get(src,headers = self.headers)
                with open(path,'wb') as f:
                    f.write(response.content)



    # 執行請求拿回html的回撥函式
    def main(self,args):
        # 返回的資料是兩個,0是html原始碼
        html = args[0]
        if isinstance(html,tuple):
            html = html[0]
        next_page = re.search(re.compile(r"<a class='page-next' href='(.*?)'>"),html)
        if next_page:
            # 找到下一頁的連結
            next_url = 'http://www.ivsky.com'+next_page.group(1)
            # 回撥main函式
            next_html = self.get_html(next_url)
            self.main((next_html,None))
        else:
            print('已經到達最後一頁了')

        # 解析列表頁資料
        result = self.parse_list_page(html)
        detail_pool = Pool(3)
        for url,title in result:
            # 1.獲取html  2.解析詳情頁
            detail_pool.apply_async(func=self.get_html,args=(url,title),callback=self.parse_detail_page)
        detail_pool.close()
        detail_pool.join()


if __name__ == '__main__':
    img = ImageSpider()

    pool = Pool(1)
    pool.apply_async(func=img.get_html,args=('http://www.ivsky.com/tupian/ziranfengguang/',),callback=img.main)
    pool.close()
    pool.join()