lz最近在學習scrapy爬蟲框架，對於此框架，我自己用兩個案例進行了實踐，初步對這個框架掌握，就寫一篇部落格來記錄下我的學習過程。

一、我的環境

mac+python2.7.6+scrapy1.4.0版本。對於scrapy在mac中的安裝過程就不做介紹了。

二、爬取清華大學就業資訊網的就業資訊板塊就業資訊

清華大學就業資訊網網址：

http://career.tsinghua.edu.cn/，具體爬取招聘資訊板塊內容，網址為：http://career.cic.tsinghua.edu.cn/xsglxt/f/jyxt/anony/xxfb

首先是分析此網頁資訊，需要把其中的每一條就業資訊爬下來，在具體的html中可以通過一些標籤來完善。具體的標籤為/div[@class='content teacher']/div[2]/ul[@class='list']，在這個標籤的路徑下，我們可以進行提取。因為這一頁中具體有10幾條資料，所以需要用for迴圈來輸出即可。但是因為這個招聘資訊有好幾頁，所以需要進行輪詢查詢相關的頁面並提取相關的頁面資訊進行提取。關於具體取哪個頁面，這個引數是放在post提交陣列的form_data的pgno欄位。

具體的scrapy程式碼如下所示：

# -*- coding: utf-8 -*-

import re
import json
import datetime

from scrapy.selector import Selector
import scrapy.spiders
from scrapy.utils.response import get_base_url
from tsinghua.items import TsinghuaItem
import urlparse
from scrapy.utils.url import urljoin_rfc
import datetime,time
import codecs
import smtplib
from email.mime.application import MIMEApplication
from email.mime.text import MIMEText
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from scrapy.mail import MailSender
from email.utils import parseaddr,formataddr
from email.header import Header
from email.mime.base import MIMEBase
from email import encoders
import os

class TsinghuaSpider(scrapy.spiders.Spider):
    name = "tsinghua"
    allowed_domains = ["tsinghua.edu.cn"]
    items   = []
    max = 0
    today = datetime.date.today()
    #yesterday = today - datetime.timedelta(days=1)
    '''
    start_urls = [
        "http://career.cic.tsinghua.edu.cn/xsglxt/f/jyxt/anony/xxfb"
    ]
    '''
    URL = "http://career.cic.tsinghua.edu.cn/xsglxt/f/jyxt/anony/xxfb"
    offset = 1
    def start_requests(self):
        url = self.URL
        yield scrapy.FormRequest(
            url = self.URL,
            formdata={"pgno":str(self.offset)},
            callback=self.parse
        )

    def parse(self,response):

        sel = Selector(response)
        base_url = get_base_url(response)
        for sss in sel.xpath("//li[@class='clearfix']"):
            item = TsinghuaItem()
            dd = sss.xpath('span/text()').extract()[0]
            d = str(dd)
            t = str(self.today)
            #篩選了一下就業資訊的時間，只挑選當日釋出的就業資訊
            if t != d:
                print "today %s" %self.today + "   " + "datetime %s" %dd
                continue

            item['Time'] = dd
            item['name'] = sss.xpath("a/text()").extract()[0]
            re_url = sss.xpath("a/@ahref").extract()[0]
            item['detailLink'] = urlparse.urljoin(base_url,re_url)

            #self.items.append(item)
            yield item
            #max = sel.xpath("//b[@id='totalPg']/text()").extract()[0]

        if self.offset < 2:
            self.offset += 1
            print self.offset
            yield scrapy.FormRequest(url = self.URL,formdata={"pgno":str(self.offset)},callback=self.parse)

        #return items





    def closed(self,reason):
        print os.path.getsize(r'/Users/sunwangdong/desktop/tsinghua/tsinghua_%s.json'%self.today)
        if (os.path.getsize(r'/Users/sunwangdong/desktop/tsinghua/tsinghua_%s.json'%self.today) == 0):
            os.remove(r'/Users/sunwangdong/desktop/tsinghua/tsinghua_%s.json'%self.today)
        '''
        data = []
        with open('/Users/sunwangdong/desktop/tsinghua/tsinghua.json') as f:
            for line in f:
                data.append(json.loads(line))

        file_object = codecs.open('tsinghua.txt','w',"utf-8")
        str = "\r\n"
        splitstr = "#___#"
        for item in data:
            str = "%s#___#%s#___#%s\r\n" % (item['name'],item['detailLink'],item['Time'].strip())
            file_object.write(str)
        '''

        sender = ''    
        receiver = ['']     #郵箱地址
        #subject = u'gift for u'
        smtpserver = 'smtp.163.com'
        username = ''
        password = ''    #163郵箱的附件碼

        msgRoot = MIMEMultipart()
        msgRoot['Subject'] = Header('就業資訊 %s' %self.today,'utf-8').encode()
        name,addr = parseaddr('朋友 <%s>' % sender)
        msgRoot['From'] = formataddr((Header(name,'utf-8').encode(),addr)).encode()
        msgRoot['To'] = ','.join(receiver)




        filename = r'/Users/sunwangdong/desktop/tsinghua/tsinghua_%s.txt'%self.today
        if os.path.exists(filename):
            msgRoot.attach(MIMEText('This is my gift!', 'plain', 'utf-8'))
            att = MIMEText(open('tsinghua.txt','rb').read(),'base64','utf-8')
            att["Content-Type"] = 'application/octet-stream'
            att["Content-Disposition"] = 'attachment; filename="tsinghua_%s.txt"'%self.today
            msgRoot.attach(att)

            '''
            with open('/Users/sunwangdong/desktop/tsinghua/tsinghua.txt','r') as f:
                mime = MIMEBase('text','txt',filename='tsinghua.txt')
                mime.set_payload(f.read())
                encoders.encode_base64(mime)
                msgRoot.attach(mime)
            
            
            att = MIMEText(open('/Users/sunwangdong/desktop/tsinghua/tsinghua.json','r').read(),'base64','utf-8')
            att["Content-Type"] = 'application/octet-stream'
            att["Content-Disposition"] = 'attachment; filename="tsinghua.json"'
            '''

            smtp = smtplib.SMTP()
            smtp.connect('smtp.163.com',25)
            smtp.login(username,password)
            smtp.sendmail(sender,receiver,msgRoot.as_string())
            smtp.quit()




            '''
            mailer = MailSender(
                smtphost="smtp.163.com",
                mailfrom="",
                smtpuser="",
                smtppass="",
                smtpport = 25
            )
            body=u'This is the gift for you!'
            subject = u'就業資訊'
            file_name = open('tsinghua.json','r')
            mailer.send(to=["
 
[email protected]"],subject=subject.encode('utf-8'),body=body.encode('utf-8'),attachs=(('tsinghua.json','text/plain',)))
            '''
        else:
            msgRoot.attach(MIMEText('Sorry!Today has no infomation about jobs!', 'plain', 'utf-8'))
            smtp = smtplib.SMTP()
            smtp.connect('smtp.163.com', 25)
            smtp.login(username, password)
            smtp.sendmail(sender, receiver, msgRoot.as_string())
            smtp.quit()




 

上述程式碼中，我在最後添加了將爬取結果傳送到指定郵箱的操作，而且是生成附件的方式來發送響應的郵件。那麼在pipelines.py檔案中，是用來指定將爬取到的資料通過pipeline檔案以管道的形式傳輸到相應的item檔案中。

class JsonTsinghuaPipeline(object):
      def __init__(self):
            self.file = codecs.open('','a',encoding='utf-8')
      def process_item(self,item,spider):
            line = json.dumps(dict(item),ensure_ascii=False) +"\n"
            self.file.write(line)
            return item
      def spider_closed(self,spider):
            self.file.close()

class TsinghuaPipeline(object):
      def process_item(self,item,spider):
            while open(filename,'a') as f:
                 f.write(item['name'] + '\n')
                 f.write(item['detaillink'] + '\n')
                 f.write(item['Time'] + '\n')
            return item

class TsinghuaItem(scrapy.Item):
    name =Field()
    detaillink = Field()
    Time = Field()

通過以上方式就可以爬取到清華大學就業資訊網的招聘資訊的內容到某個檔案中，併發送給相應的郵箱中。