package gorilla.test;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
 

import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
 

import org.apache.hadoop.io.IOUtils;
import org.junit.Before;
import org.junit.Test;

public class HDFSTest02 {
    FileSystem fSystem;
    /**
     * 連線hdfs
     * @throws IOException
     * @throws InterruptedException
     * @throws URISyntaxException
     */
    @Before
    public void init() throws
 
 IOException, InterruptedException, URISyntaxException {
        Configuration configuration = new Configuration();
        fSystem = FileSystem.get(new URI("hdfs://192.168.15.134:9000"), configuration, "hadoop");
        
    }
    @Test
    public void test() {
        System.out.println("test");
    }
    
    /**
     * 將本地檔案上傳
     * @throws IllegalArgumentException
     * @throws IOException
     */
    @Test
    public void addFileToHdfs() throws IllegalArgumentException, IOException {
        /*
         *  比如/eee 而根目錄下不存在eee這個資料夾，就會認為是要將檔案傳到/ 下面的eee檔案裡
         *  如果eee是一個資料夾就會在eee資料夾下使用原名字上傳
         *  如果是/eee/ff.md就知道是要上傳到/eee這個資料夾下的ff.md這個檔案中
         */
        fSystem.copyFromLocalFile(new Path("C:/Users/Administrator/Desktop/input/block.rar"), new Path("/user/block.rar"));
    }
    
    /**
     * 將hdfs檔案下載到本地
     * @throws IllegalArgumentException
     * @throws IOException
     */
    @Test
    public void addFileFromHdfs() throws IllegalArgumentException, IOException {
        fSystem.copyToLocalFile(false, new Path("/source.txt"), new Path("G:/HDFSDownload"), true);
    }
    
    // 刪除一個引數的方法以及棄用,兩個引數的第二個引數表示是否要使用遞迴操作,如果刪除的是資料夾，只要不是空資料夾，就必須設定成true允許遞迴操作才能刪除
    @Test
    public void delete() throws IllegalArgumentException, IOException {
        fSystem.delete(new Path(""), true);
    }
    // 檢視目錄下的檔案資訊
    @Test
    public void testListFiles() throws FileNotFoundException, IllegalArgumentException, IOException {
        RemoteIterator<LocatedFileStatus> listFiles = fSystem.listFiles(new Path("/user"), true);
        while (listFiles.hasNext()) {
            LocatedFileStatus next = listFiles.next();
            System.out.println(next);
            System.out.println(next.getBlockSize());
            System.out.println(next.getPath().getName());
            BlockLocation[] blockLocations = next.getBlockLocations();
            System.out.println("開始展示block塊");
            for (BlockLocation blockLocation : blockLocations) {
                System.out.println(blockLocation);
                System.out.println("---------");
                // offset為0是因為是第一個block
                String[] hosts = blockLocation.getHosts();
                for (String string : hosts) {
                    System.out.println(string);
                }
            }
        }
    }
    
    // 向HDFS上傳檔案
    @Test
    public void uploadByStream() throws IllegalArgumentException, IOException {
        // input 把資料讀入
        FileInputStream fis = new FileInputStream("G:/HDFSDownload/source.txt");
        // output 把輸入進來的資料寫到某個地方去
        FSDataOutputStream hdfsOutputStream = fSystem.create(new Path("/gmj/source.txt"),true);
        IOUtils.copyBytes(fis, hdfsOutputStream, 1024);
    }
    
    // 從HDFS下載檔案或資料夾
    @Test
    public void downloadByStream() throws IllegalArgumentException, IOException {
        FSDataInputStream inputStream = fSystem.open(new Path("/gmj/source.txt"));
        FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/output/source2.txt");
        org.apache.commons.io.IOUtils.copy(inputStream, fos);
    }
    
    @Test
    public void downloadByStreamUseSeek() throws IllegalArgumentException, IOException {
        FSDataInputStream inputStream = fSystem.open(new Path("/user/jdk-8u131-windows-x64.exe"));
        inputStream.seek(1024*1024*128);
        FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/output/outseek1.exe");
        org.apache.commons.io.IOUtils.copy(inputStream, fos);
    }
    
    @Test
    public void testListAll() throws FileNotFoundException, IllegalArgumentException, IOException {
        FileStatus[] listStatus = fSystem.listStatus(new Path("/"));
        for (FileStatus fileStatus : listStatus) {
            System.out.println(fileStatus.getPath().getName());
        }
    }
    
    // 讀取一個檔案大小介於129M~256M，切塊中的第二個block塊的內容
    @Test
    public void readSecondBlock() throws FileNotFoundException, IllegalArgumentException, IOException {
        FSDataInputStream inputStream = fSystem.open(new Path("/user/jdk-8u131-windows-x64.exe"));
        RemoteIterator<LocatedFileStatus> listFiles = fSystem.listFiles(new Path("/user/jdk-8u131-windows-x64.exe"), false);
        
        LocatedFileStatus next = listFiles.next();
        BlockLocation[] blockLocations = next.getBlockLocations();
        for (int j = 0; j < blockLocations.length; j++) {
            System.out.println(blockLocations[j].getOffset());
            if (j == 1) {
                inputStream.seek(blockLocations[j].getOffset());
                break;
            }
        }
        
        FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/output/outseek3.exe");
        org.apache.commons.io.IOUtils.copy(inputStream, fos);
    }
    
    // 讀取一個檔案大於256M且切塊中的第二個block塊的內容
    @Test
    public void readSecondBlockAgain() throws FileNotFoundException, IllegalArgumentException, IOException {
        FSDataInputStream inputStream = fSystem.open(new Path("/user/block.rar"));
        RemoteIterator<LocatedFileStatus> listFiles = fSystem.listFiles(new Path("/user/block.rar"), false);

        LocatedFileStatus next = listFiles.next();
        BlockLocation[] blockLocations = next.getBlockLocations();
        for (int j = 0; j < blockLocations.length; j++) {
            System.out.println(blockLocations[j].getOffset());
            if (j == 1) {
                long length = blockLocations[j].getLength();
                FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/output/bigblock2.rar");
                org.apache.commons.io.IOUtils.copyLarge(inputStream, fos, blockLocations[j].getOffset(), length);
                break;
            }
        }
    }
    
    // 將一個超過兩個block塊的大檔案以每一個檔案如block0，block1， 下載下來
    @Test
    public void readSecondBlockAgainToo() throws FileNotFoundException, IllegalArgumentException, IOException {
        
        RemoteIterator<LocatedFileStatus> listFiles = fSystem.listFiles(new Path("/user/block.rar"), false);

        LocatedFileStatus next = listFiles.next();
        BlockLocation[] blockLocations = next.getBlockLocations();
        for (int j = 0; j < blockLocations.length; j++) {
            FSDataInputStream inputStream = fSystem.open(new Path("/user/block.rar"));
            System.out.println(blockLocations[j].getOffset());
            System.out.println(blockLocations.length);
            FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/output/bigdata" + j + ".rar");
            org.apache.commons.io.IOUtils.copyLarge(inputStream, fos, blockLocations[j].getOffset(), blockLocations[j].getLength());
            
        }
    }   
}