使用HttpClient和Jsoup定向抓取資料
阿新 • • 發佈：2019-02-13
1.業務需求：
從指定外網抓點貨，冷啟動
2.站點分析：
.限制IP…
.需要登入……
.對登入賬號有抓取頻率限制……….
.抓取頻率過低，直接跳驗證碼頁面…………..
.驗證碼長度、模樣(純數字&字母數字混合)TM不固定………………..
“我們能不能不抓了？“
“不行！必須得抓…”
“……”
這麼說，此前寫的爬蟲，多執行緒、生產者—>消費者 併發抓取壓根行不通。多執行緒毫無意義。
3.使用技術：
1.HttpClient：讀取指定URL網頁內容
2.Jsoup：解析所要的頁面資料——省得寫噁心的正則表示式
3.Swing：繪製使用者操作介面
4.Tess4J
5.
Exe4J：生成可獨立執行的exe程式——給每人機器安裝一個，大家一起監控抓~
4.實現要點：
1.代理IP
從一些網站上抓取代理IP，並檢測是否可以使用，如下：
package com.ydj.zhuaqu.proxy;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.UnknownHostException;
import java.util.ArrayList;
import 使用代理IP：
/**
* 使用代理獲取網頁內容
*
* @param url
* @param proxyIp
* @param proxyPort
* @return
* @throws ParseException
* @throws IOException
*
* @author : Ares.yi
* @createTime : 2015年10月30日 上午9:55:21
*/
public static String getHtml(String url,String proxyIp,int proxyPort) throws ParseException, IOException {
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
CloseableHttpClient closeableHttpClient = httpClientBuilder.build();
HttpHost proxy = new HttpHost(proxyIp,proxyPort, "http");
RequestConfig config = RequestConfig.custom().setConnectTimeout(3000).setSocketTimeout(3000).setProxy(proxy).build();
HttpPost httpGet = new HttpPost(url);
httpGet.setConfig(config);
String html = "";
CloseableHttpResponse response = null;
try {
response = closeableHttpClient.execute(httpGet);
}catch(Exception exc){
exc.printStackTrace();
System.out.println("get請求失敗!");
return "cannot connect";
}
HttpEntity httpEntity = response.getEntity();
if (httpEntity != null) {
// 列印響應內容
try{
html = EntityUtils.toString(httpEntity, "UTF-8");
}catch(Exception excep){
System.out.println(url);
}
}else{
return "cannot connect";
}
closeableHttpClient.close();
return html;
}2.模擬登入
提取登入Cookie和User-Agent：
程式碼片段,如下：
public static String postRequest(String url,
Map<String, String> parameterMap, String charSet)
throws UnsupportedEncodingException {
CloseableHttpClient client = HttpClients.createDefault();
HttpPost httpPost = new HttpPost(url);
UrlEncodedFormEntity postEntity = new UrlEncodedFormEntity(getParam(parameterMap), charSet);
httpPost.setEntity(postEntity);
httpPost.addHeader("HOST", "sec.1688.com");
httpPost.addHeader("User-Agent", Constant.userAgent);
httpPost.addHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
httpPost.addHeader("Cookie", Constant.cookie);
MyLog.logInfo("request line:" + httpPost.getRequestLine());
try {
// 執行post請求
HttpResponse httpResponse = client.execute(httpPost);
Header header = httpResponse.getFirstHeader("Location");
if (header != null && Toolbox.isNotEmpty(header.getValue())) {
MyLog.logInfo("location:" + header.getValue());
return "SUCCESS";
} else {
String html = printResponse(httpResponse);
return html;
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
client.close();
} catch (IOException e) {
}
}
return "";
}3.驗證碼
獲取輸入驗證碼頁面資訊：
public static Ali1688CheckCodeFormData getCheckCodeFormData(String url,String checkCodePageHtml){
Ali1688CheckCodeFormData ali1688CheckCodeFormData= null;
if(Toolbox.isEmptyString(checkCodePageHtml)){
return ali1688CheckCodeFormData;
}
Document doc = Jsoup.parse(checkCodePageHtml);
String action = doc.select("input[name=action]").attr("value");
String event_submit_do_query = doc.select("input[name=event_submit_do_query]").attr("value");
String smPolicy = doc.select("input[name=smPolicy]").attr("value");
String smReturn = doc.select("input[name=smReturn]").attr("value");
String smApp = doc.select("input[name=smApp]").attr("value");
String smCharset = doc.select("input[name=smCharset]").attr("value");
String smTag = doc.select("input[name=smTag]").attr("value");
String smSign = doc.select("input[name=smSign]").attr("value");
String identity = doc.select("input[name=identity]").attr("value");
String captcha = doc.select("input[name=captcha]").attr("value");
String sessionid = doc.select("img[id=checkcodeImg]").attr("src");
sessionid = sessionid.substring(sessionid.indexOf("sessionid=")+10,sessionid.indexOf("&"));
ali1688CheckCodeFormData = new Ali1688CheckCodeFormData(action, event_submit_do_query, smPolicy, smReturn, smApp, smCharset, smTag, smSign, identity, captcha, sessionid,url);
return ali1688CheckCodeFormData;
}提交驗證碼：
public static String submitCheckCode(String checkcode) throws UnsupportedEncodingException, IOException{
String smApp = Constant.ali1688CheckCodeFormData.getSmApp();
String smPolicy = Constant.ali1688CheckCodeFormData.getSmPolicy();
String smCharset = Constant.ali1688CheckCodeFormData.getSmCharset();
String smTag = Constant.ali1688CheckCodeFormData.getSmTag();
String smReturn = Constant.ali1688CheckCodeFormData.getSmReturn();
String smSign = Constant.ali1688CheckCodeFormData.getSmSign();
String get = "smApp="+smApp+"&smPolicy="+smPolicy+"&smCharset="+smCharset+"&smTag="+smTag+"&smReturn="+smReturn+"&smSign="+smSign;
try {
get = java.net.URLEncoder.encode(get,"utf-8");
} catch (UnsupportedEncodingException e1) {
}
String formAction = "https://sec.1688.com/query.htm?"+get;
Map<String,String> parameterMap = new HashMap<String,String>();
parameterMap.put("action", Constant.ali1688CheckCodeFormData.getAction());
parameterMap.put("event_submit_do_query", Constant.ali1688CheckCodeFormData.getEvent_submit_do_query());
parameterMap.put("smPolicy", smPolicy);
parameterMap.put("smReturn", smReturn);
parameterMap.put("smApp", smApp);
parameterMap.put("smCharset", smCharset);
parameterMap.put("smTag", smTag);
parameterMap.put("smSign", smSign);
parameterMap.put("identity", Constant.ali1688CheckCodeFormData.getIdentity());
parameterMap.put("captcha", Constant.ali1688CheckCodeFormData.getCaptcha());
parameterMap.put("checkcode", checkcode);
String res = HttpKit.postRequest(formAction, parameterMap, "UTF-8");
if (Toolbox.isNotEmpty(res) && "SUCCESS".equals(res)) {
return "SUCCESS";
}else{
String html = res;
Constant.ali1688CheckCodeFormData = getCheckCodeFormData(smReturn,html);
}
return "";
}4.exe4j操作：
5.部分介面：
