1. 程式人生 > >hdfs2.9.0簡單開發

hdfs2.9.0簡單開發

1、maven工程的構建,hadoop用的版本是2.9.0。jdk的版本為1.8.0_131

		<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-hdfs</artifactId>
			<version>2.9.0</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-common</artifactId>
			<version>2.9.0</version>
			<exclusions>
				<exclusion>
					<groupId>jdk.tools</groupId>
					<artifactId>jdk.tools</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-client</artifactId>
			<version>2.9.0</version>
		</dependency>

		<dependency>
			<groupId>jdk.tools</groupId>
			<artifactId>jdk.tools</artifactId>
			<version>1.8.0_131</version>
			<scope>system</scope>
			<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
		</dependency>


2、對 hdfs的簡單操作

    //檔案系統
    FileSystem fileSystem;
   //初始化檔案系統
    public void init() throws Exception{
        //讀取資料由平臺上的協議確定
        URI uri = new URI("hdfs://dtbigdata:9000");
        Configuration conf = new Configuration();
        fileSystem = FileSystem.get(uri, conf);
    }
	
    //檢視目錄
    public void Catalog() throws Exception{
        Path path = new Path("/user");
        FileStatus fileStatus = fileSystem.getFileStatus(path);
        System.out.println("*************************************");    
        System.out.println("檔案根目錄: "+fileStatus.getPath()); 
        System.out.println("這檔案目錄為:");
        for(FileStatus fs : fileSystem.listStatus(path)){ 
            System.out.println(fs.getPath()); 
        } 
    }
    //瀏覽檔案
    public void look() throws Exception{
        Path path = new Path("/user/root/b.txt");
        FSDataInputStream fsDataInputStream = fileSystem.open(path);
        System.out.println("*************************************");
        System.out.println("瀏覽檔案的內容:");
        int c;
        while((c = fsDataInputStream.read()) != -1){
            System.out.print((char)c);
            }
        fsDataInputStream.close();
    }
 
    //下載
    public void download() throws Exception{
        //下載路徑
    	String file = "D://hadoop.txt";
        InputStream in = fileSystem.open(new Path("/user/root/b.txt"));  
        OutputStream out = new FileOutputStream(file);  
        IOUtils.copyBytes(in, out, 4096, true);  
        System.out.println("*************************************");
        System.out.println("檔案下載完成!"+file);
    }
    //上傳檔案
    public void upload() throws Exception{
        Path srcPath = new Path("F:/b.txt");  
        Path dstPath = new Path("/user/root");  
        fileSystem.copyFromLocalFile(false, srcPath, dstPath);
        fileSystem.close(); 
        System.out.println("*************************************");
        System.out.println("上傳成功!");
    }
    //刪除檔案
    public void delete() throws Exception{
        Path path = new Path("/user/root/a.txt");
        fileSystem.delete(path,true);
        System.out.println("*************************************");
        System.out.println("刪除成功!");
    }
    


3、程式碼的執行

      3.1、會出現錯誤   Permission denied: user=Administrator

            是因為在windows下執行,預設使用者為 Administrator,該使用者沒有許可權。

      3.2、最快速的解決方法

        -DHADOOP_USER_NAME=root
     新增位置方法如下: