最簡單的爬取網頁內容

#coding=utf-8
import re
import urllib
# 讀取url內容
def getHtml(url):
    page = urllib.urlopen(url)
    html = page.read()
    return html

html = getHtml("http://www.baidu.com/")
print html

簡單實現二次跳轉

<meta http-equiv="refresh" content="0;url=http://www.baidu.com/">

#coding=utf-8
import re
import urllib

# 讀取url內容
def getHtml(url):
    page = urllib.urlopen(url)
    html = page.read()
    return html

def getNewHtml(url):
	html = getHtml(url)
	keyList =  re.findall(r"url=(.+?)\">",html)
	website = keyList[0]
	return getHtml(website)

html = getNewHtml("https://www.baidu.com/")
print html
 

注意:

1\如果<meta http-equiv="refresh" content="0;url=http://www.baidu.com/">這裡沒有返回全地址,則自己字串拼接一下即可.

python的字串拼接,十分方便,例如:

website = '%s%s%s' % ('http://xxx/yyyy/', keyList[0], '.jpg')

2\上面多次使用了變數名html,但並不會相互影響.因為python函式內部的變數名如果第一次出現，且出現在=前面，即被視為定義一個區域性變數，不管全域性域中有沒有用到該變數名，函式中使用的將是區域性變數

模擬瀏覽器爬取網頁

# ------------
#! /usr/bin/env python2.7
import sys
import zlib
import urllib
import urllib2
import cookielib

def main():
    reload( sys )
    sys.setdefaultencoding('utf-8')
    url = 'http://pythontab.com'

    values = {
            "form_field1":"value1",
            "form_field2":"TRUE",
             }

    post_data = urllib.urlencode(values)
    cj=cookielib.CookieJar()
    opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
    # # mac使用者
    # headers ={"User-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:36.0) Gecko/20100101 Firefox/36.0",
    #           "Referer":"http://xxx.yyy.com/test0",
    #           "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    #           "Accept-Language":"en-US,en;q=0.5",
    #           "Accept-Encoding":"gzip, deflate",
    #           "Connection":"keep-alive",
    #           # "Cookie":"QSession=",
    #           "Content-Type":"application/x-www-form-urlencoded",
    #           }
    # # 如果是window可以這麼寫
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
				'Accept':'text/html;q=0.9,*/*;q=0.8',
				'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
				'Accept-Encoding':'gzip',
				'Connection':'close',
				'Referer':None #注意如果依然不能抓取的話，這裡可以設定抓取網站的host
				}
    req = urllib2.Request(url,post_data,headers)
    response = opener.open(req)
    content = response.read()
    gzipped = response.headers.get('Content-Encoding')
    if gzipped:
        html = zlib.decompress(content, 16+zlib.MAX_WBITS)
    else:
    	html = content
    print html.decode("utf8")

if __name__ == '__main__':
    main()  
 

關於python正則表示式

做爬蟲經常需要用到正則表示式,匹配查詢內容.

下面幾段程式碼示範.

查詢包含中文和英文的正則表示式

# 匹配查詢包含中文和英文的函式
# -*- coding: utf-8 -*-
import re
def findPart(regex, text, name):
	res=re.findall(regex, text)
	if res:
		print "There are %d %s parts:\n"% (len(res), name)
	for r in res:
		print "\t",r.encode("utf8")
# 應用例子1
text =u"#who#helloworld#a中文x#"
findPart(u"#[\w\u2E80-\u9FFF]+#", text, "unicode chinese")
# 輸出
# There are 2 unicode chinese parts:
# #who## #a中文x#

# 應用例子2
text2 =u"#who#helloworld12a中文x3s"
findPart(u"12[\w\u2E80-\u9FFF]+3s", text2, "unicode chinese")
# 輸出
# There are 1 unicode chinese parts:
# 12a中文x3s

限制中文字個數的查詢

# -*- coding: utf-8 -*- 
# 限制中文數量為1-2個的查詢
import re
a = u"q這是個中文869一y9一二三886看啊ab"
b = re.compile(u"[\u4e00-\u9fa5]{1,2}")
c = b.findall(a)
for i in  c:
    print i
     
# #輸出
#這是
#箇中
#文
#一
#一二
#三
#看啊

查詢英文和數字

# 獲取英文和數字
import re
str = "a12yy...3b"
print re.findall(r"a(.+?)b",str)#
# 輸出['12yy...3']

寫正則表示式的時候注意轉義字元,以免導致查詢不到哦.

import re
# 擷取英文及符號
context = "launchable-activity: name='com.bmi.Bmi'"
patt = re.compile(r"launchable-activity\:\s+name='(.*?)'")
ch =  patt.findall(context)
print ch
# 輸出['com.bmi.Bmi']

參考資料: