Python大資料之從網頁上爬取資料的方法詳解
阿新 • • 發佈:2020-01-09
本文例項講述了Python大資料之從網頁上爬取資料的方法。分享給大家供大家參考,具體如下:
myspider.py :
#!/usr/bin/python # -*- coding:utf-8 -*- from scrapy.spiders import Spider from lxml import etree from jredu.items import JreduItem class JreduSpider(Spider): name = 'tt' #爬蟲的名字,必須的,唯一的 allowed_domains = ['sohu.com'] start_urls = [ 'http://www.sohu.com' ] def parse(self,response): content = response.body.decode('utf-8') dom = etree.HTML(content) for ul in dom.xpath("//div[@class='focus-news-box']/div[@class='list16']/ul"): lis = ul.xpath("./li") for li in lis: item = JreduItem() #定義物件 if ul.index(li) == 0: strong = li.xpath("./a/strong/text()") li.xpath("./a/@href") item['title']= strong[0] item['href'] = li.xpath("./a/@href")[0] else: la = li.xpath("./a[last()]/text()") item['title'] = la[0] item['href'] = li.xpath("./a[last()]/href")[0] yield item
items.py :
# -*- coding: utf-8 -*- # Define here the models for your scraped items # # See documentation in: # http://doc.scrapy.org/en/latest/topics/items.html import scrapy class JreduItem(scrapy.Item):#相當於Java裡的實體類 # define the fields for your item here like: # name = scrapy.Field() title = scrapy.Field()#建立一個field物件 href = scrapy.Field() pass
middlewares.py :
# -*- coding: utf-8 -*- # Define here the models for your spider middleware # # See documentation in: # http://doc.scrapy.org/en/latest/topics/spider-middleware.html from scrapy import signals class JreduSpiderMiddleware(object): # Not all methods need to be defined. If a method is not defined,# scrapy acts as if the spider middleware does not modify the # passed objects. @classmethod def from_crawler(cls,crawler): # This method is used by Scrapy to create your spiders. s = cls() crawler.signals.connect(s.spider_opened,signal=signals.spider_opened) return s def process_spider_input(self,response,spider): # Called for each response that goes through the spider # middleware and into the spider. # Should return None or raise an exception. return None def process_spider_output(self,result,spider): # Called with the results returned from the Spider,after # it has processed the response. # Must return an iterable of Request,dict or Item objects. for i in result: yield i def process_spider_exception(self,exception,spider): # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. # Should return either None or an iterable of Response,dict # or Item objects. pass def process_start_requests(self,start_requests,spider): # Called with the start requests of the spider,and works # similarly to the process_spider_output() method,except # that it doesn't have a response associated. # Must return only requests (not items). for r in start_requests: yield r def spider_opened(self,spider): spider.logger.info('Spider opened: %s' % spider.name)
pipelines.py :
# -*- coding: utf-8 -*- # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html import codecs import json class JreduPipeline(object): def __init__(self): self.fill = codecs.open("data.txt",encoding="utf-8",mode="wb"); def process_item(self,item,spider): line = json.dumps(dict(item))+"\n" self.fill.write(line) return item
settings.py :
# -*- coding: utf-8 -*- # Scrapy settings for jredu project # # For simplicity,this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # # http://doc.scrapy.org/en/latest/topics/settings.html # http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html # http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html BOT_NAME = 'jredu' SPIDER_MODULES = ['jredu.spiders'] NEWSPIDER_MODULE = 'jredu.spiders' # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = 'jredu (+http://www.yourdomain.com)' # Obey robots.txt rules ROBOTSTXT_OBEY = True # Configure maximum concurrent requests performed by Scrapy (default: 16) #CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0) # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN = 16 #CONCURRENT_REQUESTS_PER_IP = 16 # Disable cookies (enabled by default) #COOKIES_ENABLED = False # Disable Telnet Console (enabled by default) #TELNETCONSOLE_ENABLED = False # Override the default request headers: #DEFAULT_REQUEST_HEADERS = { # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',# 'Accept-Language': 'en',#} # Enable or disable spider middlewares # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { # 'jredu.middlewares.JreduSpiderMiddleware': 543,#} # Enable or disable downloader middlewares # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html #DOWNLOADER_MIDDLEWARES = { # 'jredu.middlewares.MyCustomDownloaderMiddleware': 543,#} # Enable or disable extensions # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html #EXTENSIONS = { # 'scrapy.extensions.telnet.TelnetConsole': None,#} # Configure item pipelines # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html ITEM_PIPELINES = { 'jredu.pipelines.JreduPipeline': 300,} # Enable and configure the AutoThrottle extension (disabled by default) # See http://doc.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 # The maximum download delay to be set in case of high latencies #AUTOTHROTTLE_MAX_DELAY = 60 # The average number of requests Scrapy should be sending in parallel to # each remote server #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 # Enable showing throttling stats for every response received: #AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = 'httpcache' #HTTPCACHE_IGNORE_HTTP_CODES = [] #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
最後需要一個程式入口的方法:
main.py :
#!/usr/bin/python # -*- coding:utf-8 -*- #爬蟲檔案的執行入口 from scrapy import cmdline cmdline.execute("scrapy crawl tt".split())
更多關於Python相關內容可檢視本站專題:《Python Socket程式設計技巧總結》、《Python正則表示式用法總結》、《Python資料結構與演算法教程》、《Python函式使用技巧總結》、《Python字串操作技巧彙總》、《Python入門與進階經典教程》及《Python檔案與目錄操作技巧彙總》
希望本文所述對大家Python程式設計有所幫助。