三更燈火五更雞，正是男兒讀書時。

 本次爬取百度貼吧的圖片。

本次爬蟲網址為  https://tieba.baidu.com/f?ie=utf-8&kw=%E9%A3%8E%E6%99%AF%E5%9B%BE&fr=search

 本次爬蟲用到的工具： IJ URL類，IO流，UUID(隨機命名)，正則表示式。

首先得到 https://tieba.baidu.com/f?ie=utf-8&kw=%E9%A3%8E%E6%99%AF%E5%9B%BE&fr=search   的html。

 HttpURLConnection connection =(HttpURLConnection)
                new URL("https://tieba.baidu.com/f?ie=utf-8&kw=%E9%A3%8E%E6%99%AF%E5%9B%BE&fr=search").openConnection();

返回 一個，字串形式的html便於後面操作。

 接下來就要寫正則來匹配圖片的地址。首先要分析我們要的圖片是那個地址。

我們要的是中間的風景圖片，而不是其他的二維碼圖片或者頭像圖片，這個時候就開啟開發者工具F12

 那麼將有用的程式碼拿出來，供我們寫正則

data-original="https://imgsa.baidu.com/forum/wh%3D200%2C90%3B/sign=561eeed4c4ea15ce41bbe80b863016ca/57335443fbf2b211c636051fc78065380cd78e56.jpg"  



bpic="https://imgsa.baidu.com/forum/w%3D580%3B/sign=8436c6918f025aafd3327ec3cbd6aa64/a08b87d6277f9e2f522d9d591230e924b899f343.jpg" 

 經過驗證，上面的是我們需要的風景圖片的地址，就是date-original後面的。

注意：正則寫的時候，需要注意我們把後面的bpic帶上來精確正則表示式。於是就開始匹配圖片地址，放再一個List中。

 private static ArrayList getPictureUrl(String html) {
        Pattern patternRegex = Pattern.compile("bpic=\"(.*?)\"");//匹配圖片地址的正則
        Matcher matcher = patternRegex.matcher(html);
        String htmlUrl=null;

        ArrayList<String> listUrl = new ArrayList<>();
        while (matcher.find()){
            htmlUrl = matcher.group();
            String[] split = htmlUrl.split("=\"");
            String[] split1 = split[1].split("\"");//切出來了標準的URL
            listUrl.add(split1[0]);//將圖片的url存入陣列中
        }
        return listUrl;
    }

 返回一個listURl，那麼接下來就繼續再listURL中去下載圖片，用到IO流。

 private static void getPicture(ArrayList pictureUrl) throws IOException {
        for (Object p : pictureUrl) {
           HttpURLConnection conn = (HttpURLConnection) new URL((String)p).openConnection();
            InputStream in = conn.getInputStream();
            UUID uuid = UUID.randomUUID();//形成隨機名字
            FileOutputStream out = new FileOutputStream("C:\\Users\\Administrator\\IdeaProjects\\homework\\picture\\"+uuid+".png");
            while (true){
                byte[] bytes = new byte[1024];
                int len = in.read(bytes);
                if(len == -1){
                    break;
                }
                out.write(bytes,0,len);
            }
            out.close();
        }
    }

 完整程式碼

package com.westos.danli;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Spider1 {
    public static void main(String[] args) throws IOException {
        HttpURLConnection connection =(HttpURLConnection)
                new URL("https://tieba.baidu.com/f?ie=utf-8&kw=%E9%A3%8E%E6%99%AF%E5%9B%BE&fr=search").openConnection();
//        InputStream in = connection.getInputStream();
//        BufferedReader reader;
//        reader = new BufferedReader(new FileReader(String.valueOf(in)));
        String html = Spider1.getHtml(connection);//返回一個字串的html物件
        ArrayList pictureUrl = getPictureUrl(html);
        getPicture(pictureUrl);//直接存在C:\Users\Administrator\IdeaProjects\homework\picture中

        return;
    }

    private static void getPicture(ArrayList pictureUrl) throws IOException {
        for (Object p : pictureUrl) {
           HttpURLConnection conn = (HttpURLConnection) new URL((String)p).openConnection();
            InputStream in = conn.getInputStream();
            UUID uuid = UUID.randomUUID();//形成一個隨機命名。
            FileOutputStream out = new FileOutputStream("C:\\Users\\Administrator\\IdeaProjects\\homework\\picture\\"+uuid+".png");//直接用網址命名
            while (true){
                byte[] bytes = new byte[1024];
                int len = in.read(bytes);
                if(len == -1){
                    break;
                }
                out.write(bytes,0,len);
            }
            out.close();
        }
    }
    private static ArrayList getPictureUrl(String html) {
        Pattern patternRegex = Pattern.compile("bpic=\"(.*?)\"");//匹配圖片地址的正則
        Matcher matcher = patternRegex.matcher(html);
        String htmlUrl=null;

        ArrayList<String> listUrl = new ArrayList<>();
        while (matcher.find()){
            htmlUrl = matcher.group();
            String[] split = htmlUrl.split("=\"");
            String[] split1 = split[1].split("\"");//切出來了標準的URL
            listUrl.add(split1[0]);//將圖片的url存入陣列中
        }
        return listUrl;
    }
    /*
          獲得html頁面，寫入本地。
     */
    private static String getHtml(HttpURLConnection connection) throws IOException {
        InputStream in = connection.getInputStream();//拿到html位元組流
        BufferedReader reader = new BufferedReader(new InputStreamReader(in));
        BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream("123.html"));
        StringBuffer stringBuffer1 = new StringBuffer();//定義一個stringbuffer,將html轉換為字串物件
        while (true) {
            String s = reader.readLine();//讀取html
            if (s == null) {
                break;
            }
            bufferedOutputStream.write(s.getBytes());
            stringBuffer1.append(s).append("\n");//存為字串物件
        }
        return stringBuffer1.toString();
    }


    }