使用Post方法模擬登陸爬取網頁

最近弄爬蟲，遇到的一個問題就是如何使用post方法模擬登陸爬取網頁。下面是極簡版的代碼：

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;

import java.net.HttpURLConnection;
import java.net.URL;

import java.util.HashMap;

public class test {

    //post請求地址
    private static final String POST_URL = "";
    
    //模擬谷歌瀏覽器請求
    private static final String USER_AGENT = "";
    
    //用賬號登錄某網站後 請求POST_URL鏈接獲取cookie
    private static final String COOKIE = "";
    
    //用賬號登錄某網站後 請求POST_URL鏈接獲取數據包
    private static final String REQUEST_DATA =  "";
    
    public static void main(String[] args) throws Exception {
        HashMap<String, String> map = postCapture(REQUEST_DATA);
        String responseCode = map.get("responseCode");
        String value = map.get("value");
        
        while(!responseCode.equals("200")){
            map =  postCapture(REQUEST_DATA);
            responseCode = map.get("responseCode");
            value = map.get("value");
        }
        
        //打印爬取結果
        System.out.println(value);
    }
    
    private static HashMap<String, String> postCapture(String requestData) throws Exception{
        HashMap<String, String> map = new HashMap<>();
        
        URL url = new URL(POST_URL);
        HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
        httpConn.setDoInput(true); // 設置輸入流采用字節流
        httpConn.setDoOutput(true); // 設置輸出流采用字節流
        httpConn.setUseCaches(false); //設置緩存
        httpConn.setRequestMethod("POST");//POST請求
        httpConn.setRequestProperty("User-Agent", USER_AGENT);
        httpConn.setRequestProperty("Cookie", COOKIE);
        
        PrintWriter out = new PrintWriter(new OutputStreamWriter(httpConn.getOutputStream(), "UTF-8"));
        out.println(requestData);
        out.close();

        int responseCode = httpConn.getResponseCode();
        StringBuffer buffer = new StringBuffer();
        if (responseCode == 200) {
            BufferedReader reader = new BufferedReader(new InputStreamReader(httpConn.getInputStream(), "UTF-8"));
            String line = null;
            while ((line = reader.readLine()) != null) {
                buffer.append(line);
            }
            reader.close();
            httpConn.disconnect();
        }
        
        map.put("responseCode", new Integer(responseCode).toString());
        map.put("value", buffer.toString());
        return map;
    }

}
 

原文地址：
http://wangxin123.com/2016/12/19/%E4%BD%BF%E7%94%A8Post%E6%96%B9%E6%B3%95%E6%A8%A1%E6%8B%9F%E7%99%BB%E9%99%86%E7%88%AC%E5%8F%96%E7%BD%91%E9%A1%B5/

使用Post方法模擬登陸爬取網頁(轉)