#Author：Mini
#！/usr/bin/env python
import re
import urllib.request
import time
import urllib.error
def use_proxy(proxy_addr,url):
    try:
        req=urllib.request.Request(url)
        req.add_header("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0")
        proxy=urllib.request.ProxyHandler({‘http‘:proxy_addr})
        opener=urllib.request.build_opener(proxy,urllib.request.HTTPHandler)
        urllib.request.install_opener(opener)
        data=urllib.request.urlopen(req).read()
        return data
    except urllib.error.URLError as e:
        if hasattr(e, "code"):
            print(e.code)
            if hasattr(e, "reason"):
                print(e.reason)
            time.sleep(1)
key="Galaxy Macau"
key=key.replace(" ","+")
proxy="127.0.0.1:8888"
for i in range(1,100):
    key=urllib.request.quote(key)
    thispageurl="http://weixin.sogou.com/weixin?oq=&query="+key+"&_sug_type_=1&sut=0&lkt=0%2C0%2C0&s_from=input&ri=0&_sug_=n&type=2&sst0=1507173318682&page="+str(i)+"&ie=utf8&p=40040108&dp=1&w=01015002&dr=1"
    thispagedata=use_proxy(proxy,thispageurl)
    print(len(str(thispagedata)))
    pat1=‘<a target="_blank" href="(.*?)"‘
    res1=re.compile(pat1,re.S).findall(str(thispagedata))
    if(len(res1)==0):
        print(str(i)+"\t\tfail!")
        continue
        for j in range(0,len(res1)):
            thisurl=res1[j]
            thisurl=thisurl.replace("amp;","")
            file="E:/m/"+str(i)+"\tweb\t\t"+str(j)+"\tessay.html"
            thisdata=use_proxy(proxy,thisurl)
            try:
                fh=open(file,"wb")
                fh.write(thisdata)
                fh.close()
                print(str(i)+"\tweb\t\t"+str(j)+"\tessay\tsuccess!")
            except Exception as e:
                print(e)
                print(str(i)+"\tweb\t\t"+str(j)+"\tessay\tfail!")

web crawling(plus5) crawling wechat