1. 程式人生 > 實用技巧 >爬蟲筆記1

爬蟲筆記1

1.get請求,沒有引數

package Demo1.CrawlerDemo1;

import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.net.URI;

import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.protocol.ResponseAuthCache; import org.apache.http.impl.client.CloseableHttpClient; public class Crawler { public static void main(String[] args) { //1.browser:create httpclient
CloseableHttpClient httpClient=HttpClients.createDefault(); //2.url HttpGet httpGet=new HttpGet("http://www.itcast.cn"); //3.request CloseableHttpResponse response=null; try { response=httpClient.execute(httpGet); if(response.getStatusLine().getStatusCode()==200) { HttpEntity httpEntity
=response.getEntity(); String content=EntityUtils.toString(httpEntity,"utf-8"); System.out.println(content); } } catch (ClientProtocolException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally { try { response.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } try { httpClient.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } //4.get response } }

2.get請求,有引數

package Demo1.CrawlerDemo1;

import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.ResponseAuthCache;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
/**get請求帶引數
 * 
 * @author 18430
 *
 */
public class CrawlerParameter {
    
    public static void main(String[] args) throws Exception {
    
        CloseableHttpClient httpClient=HttpClients.createDefault();
        
        
        URIBuilder uriBuilder=new URIBuilder("http://yun.itheima.com/search");
    
        uriBuilder.setParameter("keys", "java");
        //多個引數的設定
        //uriBuilder.setParameter("keys", "java").setParameter("keys", "java");
        
        HttpGet httpGet=new HttpGet(uriBuilder.build());
            
        CloseableHttpResponse response=null;
        try {
            response=httpClient.execute(httpGet);
            if(response.getStatusLine().getStatusCode()==200) {
                HttpEntity httpEntity=response.getEntity();
                String content=EntityUtils.toString(httpEntity,"utf-8");
                System.out.println(content);
            }
        } catch (ClientProtocolException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally {
            try {
                response.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            try {
                httpClient.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
                //4.get response
        
    }
    
    
}

3.post請求不帶引數

package Demo1.CrawlerDemo1;

import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.ResponseAuthCache;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
/**
 * post請求
 * @author 18430
 *
 */
public class CrawlerPost {
    
    public static void main(String[] args) throws Exception {
    
        CloseableHttpClient httpClient=HttpClients.createDefault();
        
        
        URIBuilder uriBuilder=new URIBuilder("http://www.itcast.cn");
    
        
        
        HttpPost httpPost=new HttpPost(uriBuilder.build());
            
        CloseableHttpResponse response=null;
        try {
            response=httpClient.execute(httpPost);
            if(response.getStatusLine().getStatusCode()==200) {
                HttpEntity httpEntity=response.getEntity();
                String content=EntityUtils.toString(httpEntity,"utf-8");
                System.out.println(content);
            }
        } catch (ClientProtocolException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally {
            try {
                response.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            try {
                httpClient.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
                //4.get response
        
    }
    
    
}

4.post請求,帶引數