schedule和CrawlerProcess定時執行多個爬蟲
阿新 • • 發佈:2019-01-09
import smtplib,schedule # 通過CrawlerProcess同時執行幾個spider from scrapy.crawler import CrawlerProcess from spiders.liepin_spider import LiepinSpiderSpider as lie_1 from spiders.liepin_spider_2 import LiepinSpiderSpider as lie_2 from spiders.liepin_spider_3 import LiepinSpiderSpider as lie_3 # 匯入獲取專案配置的模組 from scrapy.utils.project import get_project_settings def start_spider(): try: #一定要先載入配置檔案 process = CrawlerProcess(get_project_settings()) process.crawl(lie_1) process.crawl(lie_2) process.crawl(lie_3) process.start() now_time = time.strftime('%Y-%m-%d', time.localtime(time.time())) print('---{}---執行完成'.format(now_time)) except Exception as e: print('--出現錯誤--', e) #email_spider('liepin', e) #傳送郵件的方法,可以自己寫 def main(): print('開始檢測,等待時間到達,開始執行') schedule.every().day.at("13:04").do(start_spider) schedule.every().day.at("17:00").do(start_spider) while True: schedule.run_pending() time.sleep(10) if __name__ == '__main__': main()