jsoup 爬蟲http呼叫工具類封裝

package com.zxwa.ntmss.process.selenium.operate.status;

import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import javax.net.ssl.*;
import java.io.*;
import java.security.SecureRandom;
 
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

public class JsoupHttpRequest {


    /**
     * 請求超時時間
     */
    private static final int TIME_OUT = 120000;

    
 
/**
     * Https請求
     */
    private static final String HTTPS = "https";

    /**
     * Content-Type
     */
    private static final String CONTENT_TYPE = "Content-Type";

    /**
     * 表單提交方式Content-Type
     */
    private static final String FORM_TYPE = "application/x-www-form-urlencoded;charset=UTF-8";

    
 
/**
     * JSON提交方式Content-Type
     */
    private static final String JSON_TYPE = "application/json;charset=UTF-8";


    /**
     * 獲取伺服器信任
     */
    private static void getTrust() {
        try {
            HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {

                public boolean verify(String hostname, SSLSession session) {
                    return true;
                }
            });
            SSLContext context = SSLContext.getInstance("TLS");
            context.init(null, new X509TrustManager[]{new X509TrustManager() {

                public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                }

                public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                }

                public X509Certificate[] getAcceptedIssuers() {
                    return new X509Certificate[0];
                }
            }}, new SecureRandom());
            HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());
        } catch (Exception e) {
            e.printStackTrace();
        }
    }


    /**
     * 傳送Get請求
     *
     * @param url 請求URL
     * @return HTTP響應物件
     * @throws IOException 程式異常時丟擲，由呼叫者處理
     */
    public static Response get(String url) throws IOException {
        return get(url, null);
    }

    /**
     * 傳送Get請求
     *
     * @param url     請求URL
     * @param headers 請求頭引數
     * @return HTTP響應物件
     * @throws IOException 程式異常時丟擲，由呼叫者處理
     */
    public static Response get(String url, Map<String, String> headers) throws IOException {
        if (null == url || url.isEmpty()) {
            throw new RuntimeException("The request URL is blank.");
        }

        // 如果是Https請求
        if (url.startsWith(HTTPS)) {
            getTrust();
        }
        Connection connection = Jsoup.connect(url)
                .method(Method.GET)
                .timeout(TIME_OUT)
                .ignoreHttpErrors(true)
                .ignoreContentType(true)
                .maxBodySize(0);
        if (null != headers) {
            connection.headers(headers);
        }

        Response response = connection.execute();
        return response;
    }

    /**
     * 傳送JSON格式引數POST請求
     *
     * @param url    請求路徑
     * @param params JSON格式請求引數
     * @return HTTP響應物件
     * @throws IOException 程式異常時丟擲，由呼叫者處理
     */
    public static Response post(String url, String params) throws IOException {
        return doPostRequest(url, null, params);
    }

    /**
     * 傳送JSON格式引數POST請求
     *
     * @param url     請求路徑
     * @param headers 請求頭中設定的引數
     * @param params  JSON格式請求引數
     * @return HTTP響應物件
     * @throws IOException 程式異常時丟擲，由呼叫者處理
     */
    public static Response post(String url, Map<String, String> headers, String params) throws IOException {
        return doPostRequest(url, headers, params);
    }

    /**
     * 字串引數post請求
     *
     * @param url      請求URL地址
     * @param paramMap 請求字串引數集合
     * @return HTTP響應物件
     * @throws IOException 程式異常時丟擲，由呼叫者處理
     */
    public static Response post(String url, Map<String, String> paramMap) throws IOException {
        return doPostRequest(url, null, paramMap, null);
    }

    /**
     * 帶請求頭的普通表單提交方式post請求
     *
     * @param headers  請求頭引數
     * @param url      請求URL地址
     * @param paramMap 請求字串引數集合
     * @return HTTP響應物件
     * @throws IOException 程式異常時丟擲，由呼叫者處理
     */
    public static Response post(Map<String, String> headers, String url, Map<String, String> paramMap) throws IOException {
        return doPostRequest(url, headers, paramMap, null);
    }

    /**
     * 帶上傳檔案的post請求
     *
     * @param url      請求URL地址
     * @param paramMap 請求字串引數集合
     * @param fileMap  請求檔案引數集合
     * @return HTTP響應物件
     * @throws IOException 程式異常時丟擲，由呼叫者處理
     */
    public static Response post(String url, Map<String, String> paramMap, Map<String, File> fileMap) throws IOException {
        return doPostRequest(url, null, paramMap, fileMap);
    }

    /**
     * 帶請求頭的上傳檔案post請求
     *
     * @param url      請求URL地址
     * @param headers  請求頭引數
     * @param paramMap 請求字串引數集合
     * @param fileMap  請求檔案引數集合
     * @return HTTP響應物件
     * @throws IOException 程式異常時丟擲，由呼叫者處理
     */
    public static Response post(String url, Map<String, String> headers, Map<String, String> paramMap, Map<String, File> fileMap) throws IOException {
        return doPostRequest(url, headers, paramMap, fileMap);
    }

    /**
     * 執行post請求
     *
     * @param url        請求URL地址
     * @param headers    請求頭
     * @param jsonParams 請求JSON格式字串引數
     * @return HTTP響應物件
     * @throws IOException 程式異常時丟擲，由呼叫者處理
     */
    private static Response doPostRequest(String url, Map<String, String> headers, String jsonParams) throws IOException {
        if (null == url || url.isEmpty()) {
            throw new RuntimeException("The request URL is blank.");
        }

        // 如果是Https請求
        if (url.startsWith(HTTPS)) {
            getTrust();
        }

        Connection connection = Jsoup.connect(url)
                .method(Method.POST)
                .timeout(TIME_OUT)
                .ignoreHttpErrors(true)
                .ignoreContentType(true)
                .maxBodySize(0);
        if (null != headers) {
            connection.headers(headers);
        }

        connection.header(CONTENT_TYPE, JSON_TYPE);
        connection.requestBody(jsonParams);

        Response response = connection.execute();
        return response;
    }

    /**
     * 普通表單方式傳送POST請求
     *
     * @param url      請求URL地址
     * @param headers  請求頭
     * @param paramMap 請求字串引數集合
     * @param fileMap  請求檔案引數集合
     * @return HTTP響應物件
     * @throws IOException 程式異常時丟擲，由呼叫者處理
     */
    private static Response doPostRequest(String url, Map<String, String> headers, Map<String, String> paramMap, Map<String, File> fileMap) throws IOException {
        if (null == url || url.isEmpty()) {
            throw new RuntimeException("The request URL is blank.");
        }

        // 如果是Https請求
        if (url.startsWith(HTTPS)) {
            getTrust();
        }

        Connection connection = Jsoup.connect(url)
                .method(Method.POST)
                .timeout(TIME_OUT)
                .ignoreHttpErrors(true)
                .ignoreContentType(true)
                .maxBodySize(0);
        if (null != headers) {
            connection.headers(headers);
        }

        // 收集上傳檔案輸入流，最終全部關閉
        List<InputStream> inputStreamList = null;
        try {

            // 新增檔案引數
            if (null != fileMap && !fileMap.isEmpty()) {
                inputStreamList = new ArrayList<InputStream>();
                InputStream in = null;
                File file = null;
                for (Entry<String, File> e : fileMap.entrySet()) {
                    file = e.getValue();
                    in = new FileInputStream(file);
                    inputStreamList.add(in);
                    connection.data(e.getKey(), file.getName(), in);
                }
            }
            // 普通表單提交方式
            else {
                connection.header(CONTENT_TYPE, FORM_TYPE);
            }

            // 新增字串類引數
            if (null != paramMap && !paramMap.isEmpty()) {
                connection.data(paramMap);
            }

            Response response = connection.execute();
            return response;
        }

        // 關閉上傳檔案的輸入流
        finally {
            closeStream(inputStreamList);
        }
    }


    public static Document getJsoupDocGet(String url) {
        //三次試錯
        final int MAX = 10;
        int time = 0;
        Document doc = null;
        while (time < MAX) {
            try {
                doc = Jsoup
                        .connect(url)
                        .ignoreContentType(true)
                        .ignoreHttpErrors(true)
                        .timeout(1000 * 30)
                        .userAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36")
                        .header("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8")
                        .header("accept-encoding", "gzip, deflate, br")
                        .header("accept-language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7")
                        .get();
                return doc;
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                time++;
            }
        }
        return doc;
    }

    public static Document getJsoupDocPost(String url, Map<String, String> paramMap) {
        //三次試錯
        final int MAX = 10;
        int time = 0;
        Document doc = null;
        while (time < MAX) {
            try {
                doc = Jsoup
                        .connect(url)
                        .ignoreContentType(true)
                        .ignoreHttpErrors(true)
                        .timeout(1000 * 30)
                        .userAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36")
                        .header("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8")
                        .header("accept-encoding", "gzip, deflate, br")
                        .header("accept-language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7")
                        .data(paramMap)
                        .post();
                return doc;
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                time++;
            }
        }
        return doc;
    }

    /**
     * 關流
     *
     * @param streamList 流集合
     */
    private static void closeStream(List<? extends Closeable> streamList) {
        if (null != streamList) {
            for (Closeable stream : streamList) {
                try {
                    stream.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }
}