延續demo1的 java爬蟲的2種爬取方式（HTTP||Socket）簡單Demo(一)，demo2出爐啦，大家想學爬蟲都可以從這個網盤學習哦：https://pan.baidu.com/s/1pJJrcqJ#list/path=%2F

免費課程，非常不錯。其實還是主要學習一個httpclient，httpclient全是英文文件，看的我心累啊

package com.simple.crawImpl;

import com.simple.Icrawl.ICrawl;
import com.simple.pojos.CrawlResultPojo;
import com.simple.pojos.UrlPojo;
import org.apache.http.HttpEntity;
import org.apache.http.ParseException;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;

/**
 *
 * Created by lewis on 2016/10/16.
 */
public class HttpClientCrawlerImpl implements ICrawl{

    public CloseableHttpClient httpClient = HttpClients.custom().build();           //建立定製HttpClient
    @Override
    public CrawlResultPojo crawl(UrlPojo urlpojo) {

        if(urlpojo==null){
            return null;
        }
        CrawlResultPojo crawlResultPojo = new CrawlResultPojo();                //結果集
        CloseableHttpResponse response = null;                                  //HTTP返回的各種資訊集合，包含協議http標準，httpcode狀態碼
        BufferedReader br = null;                                               //

        try {
            HttpGet httpGet = new HttpGet(urlpojo.getUrl());
            response = httpClient.execute(httpGet);
            HttpEntity entity = response.getEntity();                                       //獲取輸入流
            InputStreamReader isr = new InputStreamReader(entity.getContent(),"utf-8");     //位元組流轉化為字元流，設定編碼
            br =new BufferedReader(isr);

            String line =null;
            StringBuilder context = new StringBuilder();

            while((line=br.readLine())!=null){
                context.append(line+"\n");
            }

            crawlResultPojo.setSuccess(true);
            crawlResultPojo.setPageContent(context.toString());

            return crawlResultPojo;

        } catch (IOException e) {
            e.printStackTrace();
            crawlResultPojo.setSuccess(false);
        }finally {
            try {
                if (br!=null)
                    br.close();                                                                 //關閉流
                if(response!=null)
                    response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return crawlResultPojo;
    }

    /**
     * 帶引數post的urlpojo
     * */
    public CrawlResultPojo crawl4Post(UrlPojo urlPojo){
        if(urlPojo==null||urlPojo.getUrl()==null){
            return null;
        }

        CrawlResultPojo crawlResultPojo = new CrawlResultPojo();
        BufferedReader br= null;

        try {

            RequestBuilder rb = RequestBuilder.post().setUri(new URI(urlPojo.getUrl()));
            Map<String,Object> parasMap = urlPojo.getParasMap() ;
            if(parasMap!=null){
                for(Entry<String,Object> entry:parasMap.entrySet()){
                    rb.addParameter(entry.getKey(),entry.getValue().toString());
                }
            }
            HttpUriRequest httpUriRequest = rb.build();
            HttpEntity entity =httpClient.execute(httpUriRequest).getEntity();
            InputStreamReader isr=new InputStreamReader(entity.getContent(),"utf-8");
            br = new BufferedReader(isr);

            String line = null;
            StringBuilder stringBuilder = new StringBuilder();

            while((line=br.readLine())!=null){
                stringBuilder.append(line+"\n");
            }

            crawlResultPojo.setPageContent(stringBuilder.toString());
            crawlResultPojo.setSuccess(true);

            return crawlResultPojo;
        } catch (URISyntaxException e) {
            e.printStackTrace();
        } catch (ClientProtocolException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                if(br!=null)
                    br.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        crawlResultPojo.setSuccess(false);
        return crawlResultPojo;
    }

    public static void main(String []args){

        HttpClientCrawlerImpl httpClientCrawlerImpl = new HttpClientCrawlerImpl();
        String url = "http://www.wangdaizhijia.com/front_select-plat";
        UrlPojo urlPojo = new UrlPojo(url);
        Map<String, Object> parasMap = new HashMap<String, Object>();

        int max_page_number = 1000;

        parasMap.put("currPage", 30);
        parasMap.put("params", "");
        parasMap.put("sort", 0);
        urlPojo.setParasMap(parasMap);

        CrawlResultPojo resultPojo = httpClientCrawlerImpl.crawl4Post(urlPojo);
        print(resultPojo);
        resultPojo=httpClientCrawlerImpl.crawl(urlPojo);
        print(resultPojo);
    }

    public static void print(Object s){
        System.out.println(s);
    }

}