Java併發下載檔案&修改儲存【IO】【多執行緒】
阿新 • • 發佈：2022-04-09
java併發下載檔案、修改儲存；【IO】【多執行緒】
1）java實現用url下載檔案，儲存到指定資料夾
package com.company; import java.io.BufferedInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.net.URL; public class Main { public static void main(String[] args) throws IOException { System.out.println("2199"); //https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png //C:\Users\jinqingyang\Desktop\javaIO\download String targetRoute = "C:\\Users\\jinqingyang\\Desktop\\javaIO\\download\\"; ////呼叫輪子，進行下載 long start = System.currentTimeMillis(); String url001= "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png"; String fileName = "MyPic001.png".toString(); String fileRoute = new StringBuffer(targetRoute + fileName).toString(); downloadUsingStream(url001, fileRoute); long finish = System.currentTimeMillis(); System.out.println("time=" + (finish-start)); ////從已有的列表中下載多個檔案 String[] strs = {"https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402154152536-1698601120.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155257738-1040878756.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155324774-928819452.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155345098-544304226.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155404336-904923055.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155431135-1032860349.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155501392-595613994.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155538301-612543310.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155550422-295230732.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155618222-1104112179.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155714229-1850007920.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155804886-802000350.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155840534-212230918.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155910567-747754721.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155938007-1903923944.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160009903-1294768132.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160038620-1691751684.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png" }; long startList = System.currentTimeMillis(); for(int i=0; i<strs.length; ++i){ String thisFileRoute = new StringBuffer(targetRoute + "List\\MyPic" + i + ".png").toString(); downloadUsingStream(strs[i], thisFileRoute); } long finishList = System.currentTimeMillis(); System.out.println("time=" + (finishList-startList)); } //用流來下載檔案，是一個底層的輪子； public static void downloadUsingStream(String urlStr, String fileRoute) throws IOException { URL url = new URL(urlStr); BufferedInputStream bis = new BufferedInputStream(url.openStream()); FileOutputStream fis = new FileOutputStream(fileRoute); byte[] buffer = new byte[1024]; int count = 0; while ((count = bis.read(buffer, 0, 1024)) != -1) { fis.write(buffer, 0, count); } fis.close(); bis.close(); } }
分別進行單檔案、多檔案的下載，然後計時如下：
單個圖片和多個圖片分別用時992ms、4386ms
2）java實現多執行緒下載（記錄下載時間）
使用基本的執行緒池ThreadPoolExecutor，重寫Runnable方法
package com.company; import java.io.BufferedInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.net.URL; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import static com.company.Main.downloadUsingStream; public class Main { public static void main(String[] args) throws IOException { System.out.println("2199"); //https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png //C:\Users\jinqingyang\Desktop\javaIO\download String targetRoute = "C:\\Users\\jinqingyang\\Desktop\\javaIO\\download\\"; ////呼叫輪子，進行下載 long start = System.currentTimeMillis(); String url001 = "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png"; String fileName = "MyPic001.png".toString(); String fileRoute = new StringBuffer(targetRoute + fileName).toString(); downloadUsingStream(url001, fileRoute); long finish = System.currentTimeMillis(); System.out.println("time=" + (finish - start) + "ms"); ////從已有的列表中下載多個檔案 //待會新建資料夾、考慮下去重的問題 String[] strs = {"https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402154152536-1698601120.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155257738-1040878756.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155324774-928819452.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155345098-544304226.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155404336-904923055.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155431135-1032860349.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155501392-595613994.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155538301-612543310.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155550422-295230732.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155618222-1104112179.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155714229-1850007920.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155804886-802000350.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155840534-212230918.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155910567-747754721.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402155938007-1903923944.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160009903-1294768132.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160038620-1691751684.png", "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png" }; long startList = System.currentTimeMillis(); ThreadPoolExecutor executor = new ThreadPoolExecutor(20, 40, 500, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<Runnable>(20));////這個需要調參適配，corePoolSize越大越好；此案例達到20即可達到效能極限 for (int i = 0; i < strs.length; ++i) { MyTask myTask = new MyTask(targetRoute, i, strs[i]); executor.execute(myTask); } executor.shutdown(); long finishList = System.currentTimeMillis(); System.out.println("timeList=" + (finishList - startList) + "ms"); } //用流來下載檔案，是一個底層的輪子； public static void downloadUsingStream(String urlStr, String fileRoute) throws IOException { URL url = new URL(urlStr); BufferedInputStream bis = new BufferedInputStream(url.openStream()); FileOutputStream fis = new FileOutputStream(fileRoute); byte[] buffer = new byte[1024]; int count = 0; while ((count = bis.read(buffer, 0, 1024)) != -1) { fis.write(buffer, 0, count); } fis.close(); bis.close(); } } class MyTask implements Runnable { String targetRoute; int i; String str; MyTask(String targetRoute, int i, String str){ this.str = str; this.i = i; this .targetRoute = targetRoute; } @Override public void run() { String thisFileRoute = new StringBuffer(targetRoute + "List\\MyPic" + (i + 1) + ".png").toString(); try { downloadUsingStream(str, thisFileRoute); System.out.println("正在下載第" + i); } catch (IOException e) { e.printStackTrace(); } } }
這個time=3ms是主執行緒的時間，而不是完成任務的時間
》》》》現在多執行緒確實是加快了速度，但是暫未定量獲取【完成任務的總時間】
3）java讀取json檔案並正則化匹配修改【JSON】【正則化】【java讀+寫檔案】
package com.company; import java.io.BufferedInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.net.URL; import java.util.ArrayList; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.io.BufferedReader; //import io系列的多個依賴,這裡不合並的目的是要展開看看熟悉一下 import java.io.BufferedWriter; import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.InputStreamReader; import java.io.FileInputStream; public class Main { public static void main(String[] args) throws IOException { System.out.println("2199"); //[1]使用java生態介面讀取JSON(本地檔案，來源可以是python/JS從頁面扒下來)，並列印到控制檯，注意utf-8解碼(讀出來發現沒問題，就不管了) String path = "C:\\Users\\jinqingyang\\Desktop\\java_IO\\download\\Data.json";//這裡測試發現無論是.txt還是.json都可以讀取 // readJsonAndPrint(path); //[2]正則化匹配所需的圖片url，並存儲到String[]陣列中 // getUrlsAndSave(path); //[3]併發下載圖片檔案 // download(getUrlsAndSave(path)); //[One]使用java生態寫檔案，該檔案專門儲存urls,在檔案開頭寫上一個漢字簡介，來測試utf-8的使用情況 writeToFile(getUrlsAndSave(path)); //[Two]使用java修改本地txt(讀+寫) String txtName = "C:\\Users\\jinqingyang\\Desktop\\java_IO\\download\\URLs.txt"; modifyTxt(txtName); } public static String txt2String(File file){ ////這是個輪子 StringBuilder result = new StringBuilder(); try{ // 構造一個BufferedReader類來讀取檔案 BufferedReader br = new BufferedReader(new FileReader(file)); String s = null; // 使用readLine方法，一次讀一行 while((s = br.readLine())!=null){ result.append(System.lineSeparator()+s); } br.close(); }catch(Exception e){ e.printStackTrace(); } return result.toString(); } public static void readJsonAndPrint(String pathName){ File file0 = new File(pathName); String string0 = txt2String(file0); System.out.println(string0);//列印全文 } public static ArrayList<String> getUrlsAndSave(String pathName){ ArrayList<String> urls = new ArrayList<>(); File file = new File(pathName); try{ // 構造一個BufferedReader類來讀取檔案 BufferedReader br = new BufferedReader(new FileReader(file)); String s = null; // 使用readLine方法，一次讀一行 while((s = br.readLine())!=null){ //進行匹配，如果匹配成功就加入urls陣列 String url = ""; if(s.contains("img src=\"https://img2020.cnblogs.com/blog/1806053")){ url = s.substring(s.indexOf("https://img"),3+s.lastIndexOf("png")); // System.out.println(url); urls.add(url); } } br.close(); }catch(Exception e){ e.printStackTrace(); } return urls; } public static void writeToFile(ArrayList<String> strs) throws IOException { //寫入檔案介紹（漢字+英文+數字）[兩行] File myFile = new File("C:\\Users\\jinqingyang\\Desktop\\java_IO\\download\\URLs.txt"); myFile.createNewFile();//還要再建立新檔案 BufferedWriter out = new BufferedWriter(new FileWriter(myFile)); String str0 = "這個txt的內容是我的cnblog裡面的21個圖片的url！！！\r\n下面是正文內容：\r\n";// \r\n即為換行 out.write(str0); for(int i=0; i<strs.size(); ++i){ //將每個url寫入，注意換行符的事情 out.write(strs.get(i)+"\r\n"); } out.flush(); // 把快取區內容壓入檔案 out.close(); // 最後記得關閉檔案 } public static void modifyTxt(String txtName){//要修改的內容，我在裡面自定義 try { // 防止檔案建立或讀取失敗，用catch捕捉錯誤並列印，也可以throw /* 讀取txt檔案 */ File modifyFile = new File(txtName); // 要讀取以上路徑的input.txt檔案 InputStreamReader reader = new InputStreamReader( new FileInputStream(modifyFile)); // 建立一個輸入流物件reader BufferedReader br = new BufferedReader(reader); // 建立一個物件，它把檔案內容轉成計算機能讀懂的語言 String line = br.readLine();//讀取第一行 BufferedWriter out = new BufferedWriter(new FileWriter(modifyFile)); while (line != null) { //modify出新的內容 String modifyLine = line+"》》》【這裡是修改的內容】》》》》\r\n"; //寫入buffer out.write(modifyLine);//積攢buffer裡面的內容 // 一次讀入一行資料 line = br.readLine(); } out.write("已修改！！"); /* 寫進txt檔案 */ out.flush(); // 把快取區內容壓入檔案【這裡是把前面write的內容一次性flush寫進去！！！！】 out.close(); // 最後記得關閉檔案 } catch (Exception e) { e.printStackTrace(); } } //用流來下載檔案，是一個底層的輪子； public static void downloadUsingStream(String urlStr, String fileRoute) throws IOException { URL url = new URL(urlStr); BufferedInputStream bis = new BufferedInputStream(url.openStream()); FileOutputStream fis = new FileOutputStream(fileRoute); byte[] buffer = new byte[1024]; int count = 0; while ((count = bis.read(buffer, 0, 1024)) != -1) { fis.write(buffer, 0, count); } fis.close(); bis.close(); } public static void download(ArrayList<String> strs) throws IOException { //從已有的列表中下載多個檔案 //https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png //C:\Users\jinqingyang\Desktop\javaIO\download String targetRoute = "C:\\Users\\jinqingyang\\Desktop\\java_IO\\download\\"; ////呼叫輪子，進行下載 long start = System.currentTimeMillis(); String url001 = "https://img2020.cnblogs.com/blog/1806053/202004/1806053-20200402160613144-589399504.png"; String fileName = "MyPic001.png".toString(); String fileRoute = new StringBuffer(targetRoute + fileName).toString(); downloadUsingStream(url001, fileRoute); long finish = System.currentTimeMillis(); System.out.println("time=" + (finish - start) + "ms"); long startList = System.currentTimeMillis(); ThreadPoolExecutor executor = new ThreadPoolExecutor(25, 50, 500, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<Runnable>(25));////這個需要調參適配，corePoolSize越大越好；此案例達到25即可達到效能極限 for (int i = 0; i < strs.size(); ++i) { MyTask myTask = new MyTask(targetRoute, i, strs.get(i)); executor.execute(myTask); } executor.shutdown(); long finishList = System.currentTimeMillis(); System.out.println("timeList=" + (finishList - startList) + "ms"); } } class MyTask implements Runnable { String targetRoute; int i; String str; MyTask(String targetRoute, int i, String str){ this.str = str; this.i = i; this .targetRoute = targetRoute; } @Override public void run() { String thisFileRoute = new StringBuffer(targetRoute + "List\\MyPic" + (i + 1) + ".png").toString(); try { Main.downloadUsingStream(str, thisFileRoute); System.out.println("正在下載第" + i); } catch (IOException e) { e.printStackTrace(); } } }