#coding=utf-8
import requests
import re,os,time,ConfigParser
from selenium import webdriver
from multiprocessing.dummy import Pool

######單進程#####

#創建保存截圖的目錄

def createImagesPath():
dirname=os.path.dirname(os.path.abspath(__file__))
#print dirname
imges_path=os.path.join(dirname,time.strftime("%Y%m%d"))
#print imges_path
try:
if not os.path.exists(imges_path):
os.mkdir(imges_path)
print u"截圖保存的目錄：",imges_path
return imges_path
except Exception,e:
print e

#從文件中獲取要爬的網站地址
def getWebUrls(web_urls_file_path):
web_urls=[]
try:
with open(web_urls_file_path) as fp:
lines=fp.readlines()
for line in lines:
if line.strip():
web_urls.append(line.strip())
return web_urls
except Exception,e:
print e

#獲取單個網站的所有有效鏈接
def getLinks(web_url):
try:
response=requests.get(web_url)
#print response
html=response.text
links=re.findall(r‘href="(.*?)"‘,html)
valid_links=[]
invalid_links=[]
for link in links:
if link.strip().startswith("//"):
valid_links.append("http:"+link.strip())
elif link.strip()=="" or link.strip()=="#" or link.strip()=="/" or link.strip().count("javascript")>=1 or link.strip().count("mailto:") >= 1:
invalid_links.append(link)
elif re.search(r"(\.jpg)|(\.jpeg)|(\.gif)|(\.ico)|(\.png)|(\.js)|(\.css)$",link.strip()) or re.match(r‘/[^/].*‘,link.strip()):
invalid_links.append(link)
else:
valid_links.append(link.strip())
valid_links=list(set(valid_links))
return valid_links
except Exception,e:
print e

#保存有效的鏈接到.txt文件
def saveLinks(links):
dirname=os.path.dirname(os.path.abspath(__file__))
#print dirname
links_path=os.path.join(dirname,time.strftime("%Y%m%d"))
#print links_path
try:
if not os.path.exists(links_path):
os.mkdir(links_path)
links_file_path=os.path.join(links_path,"links.txt")
print u"鏈接保存的路徑：",links_file_path
with open(links_file_path,"w") as fp:
fp.writelines([link+"\n" for link in links])
except Exception,e:
print e

#模擬瀏覽器打開鏈接並保存截圖
class OpenLinkAndSaveImg(object):
def __init__(self,browser_type):
try:
configFilePath=os.path.dirname(os.path.abspath(__file__))+"\\browserAndDriver.ini"
print u"瀏覽器驅動配置文件路徑：",configFilePath
cf=ConfigParser.ConfigParser()
cf.read(configFilePath)
browser_type=browser_type.strip().lower()
driver_path=cf.get("browser_driver",browser_type.strip()).strip()
print u"瀏覽器：%s ,驅動位置：%s"%(browser_type,driver_path)
if browser_type=="ie":
self.driver=webdriver.Ie(executable_path=eval(driver_path))
elif browser_type=="chrome":
self.driver=webdriver.Chrome(executable_path=eval(driver_path))
elif browser_type=="firefox":
self.driver=webdriver.Firefox(executable_path=eval(driver_path))
else:
print "invalid browser!"
except Exception,e:
print e

#打開鏈接並保存截圖
def openLinkAndSaveImg(self,link_index_imgspath):
try:
link,index,imgspath=link_index_imgspath
self.driver.get(link)
self.driver.maximize_window()
self.driver.get_screenshot_as_file(os.path.join(imgspath,str(index+1)+".png"))
except Exception,e:
print e

def end(self):
self.driver.quit()

if __name__=="__main__":

#單進程
imgs_path=createImagesPath()
#weburls=getWebUrls(r"e:\\urls.txt")
weburls=getWebUrls(os.path.dirname(os.path.abspath(__file__))+"\\urls.txt")
links=[]
start_time=time.time()
for weburl in weburls:
links+=getLinks(weburl)
print u"鏈接數：%s ；獲取所有鏈接耗時：%s"%(len(links),time.time()-start_time)
saveLinks(links)
start_time1=time.time()
open_link_and_save_img=OpenLinkAndSaveImg("ie")
for i in range(len(links)):
open_link_and_save_img.openLinkAndSaveImg((links[i],i,imgs_path))
open_link_and_save_img.end()
print u"單進程打開所有鏈接並截圖耗時耗時：",time.time()-start_time1

######多線程(線程池並發執行）######

#coding=utf-8
import requests
import re,os,time,ConfigParser
from selenium import webdriver
from multiprocessing.dummy import Pool

#創建保存截圖的目錄
def createImagesPath():
dirname=os.path.dirname(os.path.abspath(__file__))
#print dirname
imges_path=os.path.join(dirname,time.strftime("%Y%m%d"))
#print imges_path
try:
if not os.path.exists(imges_path):
os.mkdir(imges_path)
print u"所有截圖保存的目錄：",imges_path
return imges_path
except Exception,e:
print e

#從文件中獲取要爬的網站地址
def getWebUrls(web_urls_file_path):
web_urls=[]
try:
with open(web_urls_file_path) as fp:
lines=fp.readlines()
for line in lines:
if line.strip():
web_urls.append(line.strip())
return web_urls
except Exception,e:
print e

#獲取單個網站的所有有效鏈接
def getLinks(web_url):
try:
response=requests.get(web_url)
#print response
html=response.text
links=re.findall(r‘href="(.*?)"‘,html)
valid_links=[]
invalid_links=[]
for link in links:
if link.strip().startswith("//"):
valid_links.append("http:"+link.strip())
elif link.strip()=="" or link.strip()=="#" or link.strip()=="/" or link.strip().count("javascript")>=1 or link.strip().count("mailto:") >= 1:
invalid_links.append(link)
elif re.search(r"(\.jpg)|(\.jpeg)|(\.gif)|(\.ico)|(\.png)|(\.js)|(\.css)$",link.strip()) or re.match(r‘/[^/].*‘,link.strip()):
invalid_links.append(link)
else:
valid_links.append(link.strip())
valid_links=list(set(valid_links))
return valid_links
except Exception,e:
print e

#保存有效的鏈接到.txt文件
def saveLinks(links):
dirname=os.path.dirname(os.path.abspath(__file__))
#print dirname
links_path=os.path.join(dirname,time.strftime("%Y%m%d"))
#print links_path
try:
if not os.path.exists(links_path):
os.mkdir(links_path)
links_file_path=os.path.join(links_path,"links.txt")
print u"所有鏈接保存的路徑：",links_file_path
with open(links_file_path,"w") as fp:
fp.writelines([link+"\n" for link in links])
except Exception,e:
print e

#獲取瀏覽器和驅動
def getBrowserAndDriver(browser_type):
try:
configFilePath=os.path.dirname(os.path.abspath(__file__))+"\\browserAndDriver.ini"
print u"瀏覽器驅動配置文件路徑：",configFilePath
cf=ConfigParser.ConfigParser()
cf.read(configFilePath)
browser_type=browser_type.strip().lower()
driver_path=cf.get("browser_driver",browser_type.strip()).strip()
print u"瀏覽器：%s ,驅動位置：%s"%(browser_type,driver_path)
return browser_type,driver_path
except Exception,e:
print e

#打開鏈接並保存截圖
def openLinkAndSaveImg(browser_driver_link_index_imgspath):
try:
browser,driverpath,link,index,imgspath=browser_driver_link_index_imgspath
command="webdriver."+browser.capitalize()+"(executable_path="+driverpath+")"
driver=eval(command)
driver.get(link)
driver.maximize_window()
driver.get_screenshot_as_file(os.path.join(imgspath,str(index+1)+".png"))
driver.quit()
except Exception,e:
print e

if __name__=="__main__":
imgs_path=createImagesPath()
#weburls=getWebUrls(r"e:\\urls.txt")
weburls=getWebUrls(os.path.dirname(os.path.abspath(__file__))+"\\urls.txt")
p=Pool(5)
start_time1=time.time()
links_list=p.map(getLinks,weburls)
end_time1=time.time()
links=[]
for link_list in links_list:
links+=link_list
saveLinks(links)
print u"鏈接數：%s ；獲取所有鏈接耗時：%s"%(len(links),end_time1-start_time1)
browser,driver=getBrowserAndDriver("ie")
browser_driver_link_index_imgspath=zip([browser]*len(links),[driver]*len(links),links,range(len(links)),[imgs_path]*len(links))
start_time2=time.time()
p.map(openLinkAndSaveImg,browser_driver_link_index_imgspath)
p.close()
p.join()
print u"多線程打開所有鏈接並截圖耗時：",time.time()-start_time2

######多線程######

#coding=utf-8
import requests
import re,os,time,ConfigParser
from selenium import webdriver
from multiprocessing.dummy import Pool
import Queue
import threading

#創建保存截圖的目錄
def createImagesPath():
dirname=os.path.dirname(os.path.abspath(__file__))
#print dirname
imges_path=os.path.join(dirname,time.strftime("%Y%m%d"))
#print imges_path
try:
if not os.path.exists(imges_path):
os.mkdir(imges_path)
print u"所有截圖保存的目錄：",imges_path
return imges_path
except Exception,e:
print e

#從文件中獲取要爬的網站地址
def getWebUrls(web_urls_file_path):
web_urls=[]
try:
with open(web_urls_file_path) as fp:
lines=fp.readlines()
for line in lines:
if line.strip():
web_urls.append(line.strip())
return web_urls
except Exception,e:
print e

#獲取單個網站的所有有效鏈接
def getLinks(web_url):
try:
response=requests.get(web_url)
#print response
html=response.text
links=re.findall(r‘href="(.*?)"‘,html)
valid_links=[]
invalid_links=[]
for link in links:
if link.strip().startswith("//"):
valid_links.append("http:"+link.strip())
elif link.strip()=="" or link.strip()=="#" or link.strip()=="/" or link.strip().count("javascript")>=1 or link.strip().count("mailto:") >= 1:
invalid_links.append(link)
elif re.search(r"(\.jpg)|(\.jpeg)|(\.gif)|(\.ico)|(\.png)|(\.js)|(\.css)$",link.strip()) or re.match(r‘/[^/].*‘,link.strip()):
invalid_links.append(link)
else:
valid_links.append(link.strip())
valid_links=list(set(valid_links))
return valid_links
except Exception,e:
print e

#保存有效的鏈接到.txt文件（當前目錄\年月日\links.txt)
def saveLinks(links):
dirname=os.path.dirname(os.path.abspath(__file__))
#print dirname
links_path=os.path.join(dirname,time.strftime("%Y%m%d"))
#print links_path
try:
if not os.path.exists(links_path):
os.mkdir(links_path)
links_file_path=os.path.join(links_path,"links.txt")
print u"所有鏈接保存的路徑：",links_file_path
with open(links_file_path,"w") as fp:
fp.writelines([link+"\n" for link in links])
except Exception,e:
print e

#多線程
class MyThread(threading.Thread):
def __init__(self,browser,queue):
threading.Thread.__init__(self)
self.queue=queue
try:
configFilePath=os.path.dirname(os.path.abspath(__file__))+"\\browserAndDriver.ini"
#print u"瀏覽器驅動配置文件路徑：",configFilePath
cf=ConfigParser.ConfigParser()
cf.read(configFilePath)
browser_type=browser.strip().lower()
driver_path=cf.get("browser_driver",browser_type.strip()).strip()
#print u"瀏覽器：%s ,驅動位置：%s"%(browser_type,driver_path)
if browser_type=="ie":
self.driver=webdriver.Ie(executable_path=eval(driver_path))
elif browser_type=="chrome":
self.driver=webdriver.Chrome(executable_path=eval(driver_path))
elif browser_type=="firefox":
self.driver=webdriver.Firefox(executable_path=eval(driver_path))
else:
print "invalid browser!"

except Exception,e:
print e

def run(self):
print "Starting"+self.name
openLinkAndSaveImg(self.driver,self.queue)
self.driver.quit()

#打開鏈接並保存截圖
def openLinkAndSaveImg(driver,queue):
while not queue.empty():
queueLock.acquire()
link_index_imgspath=queue.get()
queueLock.release()
try:
link,index,imgspath=link_index_imgspath
driver.get(link)
driver.maximize_window()
driver.get_screenshot_as_file(os.path.join(imgspath,str(index+1)+".png"))
except Exception,e:
print e

if __name__=="__main__":
#多線程
imgs_path=createImagesPath()
#weburls=getWebUrls(r"e:\\urls.txt")
weburls=getWebUrls(os.path.dirname(os.path.abspath(__file__))+"\\urls.txt")
p=Pool(5)
start_time1=time.time()
links_list=p.map(getLinks,weburls)
end_time1=time.time()
links=[]
for link_list in links_list:
links+=link_list
saveLinks(links)
print u"鏈接數：%s ；獲取所有鏈接耗時：%s"%(len(links),end_time1-start_time1)

link_index_imgspath=zip(links,range(len(links)),[imgs_path]*len(links))

queueLock=threading.Lock()
threads=[]
link_index_imgspath_Queue=Queue.Queue(len(links))
for element in link_index_imgspath:
link_index_imgspath_Queue.put(element)

start_time2=time.time()

for i in range(5):
thread=MyThread("ie",link_index_imgspath_Queue)
thread.start()
threads.append(thread)

for t in threads:
t.join()

print u"多線程打開所有鏈接並截圖耗時：",time.time()-start_time2

print "end!"

用單進程、多線程並發、多線程分別實現爬一個或多個網站的所有鏈接，用瀏覽器打開所有鏈接並保存截圖 python