爬蟲筆記1
阿新 • • 發佈:2020-07-19
1.get請求,沒有引數
package Demo1.CrawlerDemo1; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import java.io.IOException; import java.net.URI; import org.apache.http.HttpEntity; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpClient;import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.protocol.ResponseAuthCache; import org.apache.http.impl.client.CloseableHttpClient; public class Crawler { public static void main(String[] args) { //1.browser:create httpclientCloseableHttpClient httpClient=HttpClients.createDefault(); //2.url HttpGet httpGet=new HttpGet("http://www.itcast.cn"); //3.request CloseableHttpResponse response=null; try { response=httpClient.execute(httpGet); if(response.getStatusLine().getStatusCode()==200) { HttpEntity httpEntity=response.getEntity(); String content=EntityUtils.toString(httpEntity,"utf-8"); System.out.println(content); } } catch (ClientProtocolException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally { try { response.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } try { httpClient.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } //4.get response } }
2.get請求,有引數
package Demo1.CrawlerDemo1; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import org.apache.http.HttpEntity; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.protocol.ResponseAuthCache; import org.apache.http.client.utils.URIBuilder; import org.apache.http.impl.client.CloseableHttpClient; /**get請求帶引數 * * @author 18430 * */ public class CrawlerParameter { public static void main(String[] args) throws Exception { CloseableHttpClient httpClient=HttpClients.createDefault(); URIBuilder uriBuilder=new URIBuilder("http://yun.itheima.com/search"); uriBuilder.setParameter("keys", "java"); //多個引數的設定 //uriBuilder.setParameter("keys", "java").setParameter("keys", "java"); HttpGet httpGet=new HttpGet(uriBuilder.build()); CloseableHttpResponse response=null; try { response=httpClient.execute(httpGet); if(response.getStatusLine().getStatusCode()==200) { HttpEntity httpEntity=response.getEntity(); String content=EntityUtils.toString(httpEntity,"utf-8"); System.out.println(content); } } catch (ClientProtocolException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally { try { response.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } try { httpClient.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } //4.get response } }
3.post請求不帶引數
package Demo1.CrawlerDemo1; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import org.apache.http.HttpEntity; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.protocol.ResponseAuthCache; import org.apache.http.client.utils.URIBuilder; import org.apache.http.impl.client.CloseableHttpClient; /** * post請求 * @author 18430 * */ public class CrawlerPost { public static void main(String[] args) throws Exception { CloseableHttpClient httpClient=HttpClients.createDefault(); URIBuilder uriBuilder=new URIBuilder("http://www.itcast.cn"); HttpPost httpPost=new HttpPost(uriBuilder.build()); CloseableHttpResponse response=null; try { response=httpClient.execute(httpPost); if(response.getStatusLine().getStatusCode()==200) { HttpEntity httpEntity=response.getEntity(); String content=EntityUtils.toString(httpEntity,"utf-8"); System.out.println(content); } } catch (ClientProtocolException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally { try { response.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } try { httpClient.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } //4.get response } }
4.post請求,帶引數