1.環境搭建

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-hdfs</artifactId>
    <version>2.6.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-common</artifactId>
    <version>2.6.0</version>
</dependency>
 

2.問題解決

Windows開發Hadoop應用環境配置

• 解壓hadoop安裝包到C:/
• 將winutils.exe和hadoop.dll拷貝到hadoop的bin目錄下
• 在windows配置HADOOP_HOME環境變數
• 重啟開發工具idea,否則開發工具無法識別HADOOP_HOME
• 在Windows主機配置CentOS的主機名和IP的對映關係

C:\Windows\System32\drivers\etc\hosts

192.168.169.139 CentOS

HDFS許可權不足導致寫失敗?

org.apache.hadoop.security.AccessControlException: Permission denied: user=HIAPAD, access=WRITE, inode="/":root:supergroup:drwxr-xr-x
    at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkFsPermission(FSPermissionChecker.java:271)
    at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:257)
...
 

解決方案

方案1

etc/hadoop/hdfs-site.xml

<property>
        <name>dfs.permissions.enabled</name>
        <value>false</value>
</property>

關閉HDFS檔案許可權檢查,修改完成後,重啟HDFS服務

方案2

-DHADOOP_USER_NAME=root

設定JAVA虛擬機器啟動引數java XXX -Dxx=xxx

3.HDFS

package com.baizhi.hdfs;/**

- @Author:luoht
- @Description：
- @Date：Create in 16:21 2019/1/3
  */

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.*;
import java.io.*;

/**

- @program: hadoop_01
- @description:
- @author: luoht
- @create: 2019-01-03 16:21
  **/

public class TestHDFS {
    private FileSystem fileSystem;
    private Configuration conf;

    @Before
    public void before() throws IOException {
        conf = new Configuration();
        conf.addResource("core-site.xml");
        conf.addResource("hdfs-site.xml");
        fileSystem=FileSystem.newInstance(conf);
    }

    @Test
    public void testConfig(){
        String value = conf.get("dfs.replication");
        System.out.println(value);
    }

    @Test
    public void testUpload01() throws IOException {
        String file ="C:\\Users\\Administrator\\Desktop\\鬱金香開了麼？.txt";
        Path dst = new Path("/demo/access/springBoot.pdf");
        FileInputStream inputStream = new FileInputStream(file);
        FSDataOutputStream outputStream = fileSystem.create(dst, new Progressable() {
            @Override
            public void progress() {
                System.out.println("..");
            }
        });
        IOUtils.copyBytes(inputStream,outputStream,1024,true);
    }

    @Test
    public void testUpload02() throws IOException {
        Path src = new Path("C:\\Users\\Administrator\\Desktop\\鬱金香開了麼？.txt");
        Path dst = new Path("/鬱金香開了麼？.txt");
        fileSystem.copyFromLocalFile(src,dst);
    }

    @Test
    public void testDownload01() throws IOException {
        String file="C:\\Users\\Administrator\\Desktop\\鬱金香不開了.txt";
        Path dst = new Path("/鬱金香開了麼？.txt");
        FileOutputStream outputStream = new FileOutputStream(file);
        InputStream inputStream = fileSystem.open(dst);
        IOUtils.copyBytes(inputStream,outputStream,1024,true);
    }

    @Test
    public void testDownload02() throws IOException {
        Path dst = new Path("C:\\Users\\Administrator\\Desktop\\鬱金香不開了.txt");
        Path src = new Path("/demo/access/springBoot.pdf");
        fileSystem.copyToLocalFile(false,src,dst,true);
    }

    @Test
    public void testDelete() throws IOException {
        Path src = new Path("/demo");
        /*true 代表遞迴刪除子檔案*/
        fileSystem.delete(src,true);
    }

    @Test
    public void testExits() throws IOException {
        Path src = new Path("/鬱金香開了麼？.txt");
        boolean exists = fileSystem.exists(src);
        //Junit之 斷言 -靜態導包
        assertTrue(exists);
    }
    
    @Test
    public void testMkdir() throws IOException {
        Path src = new Path("/demo.assess");
        boolean exists = fileSystem.exists(src);
        if (!exists){
            fileSystem.mkdirs(src);
        }
    }

    @Test
    public void testListFiles() throws IOException {
        Path src = new Path("/tt");
        RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(src, true);
        while (files.hasNext()){
            LocatedFileStatus fileStatus = files.next();
            System.out.println(fileStatus.getPath()+"路徑  "+fileStatus.isFile()+"是否檔案  "+fileStatus.getLen()+"檔案大小");
            System.out.println("------------------------------");
            BlockLocation[] locations = fileStatus.getBlockLocations();
            for (BlockLocation location : locations) {
                System.out.println("offset"+location.getOffset()+"length"+location.getLength());
            }
            System.out.println("================================");
        }
    }

    @Test
    public void testDeleteWithTrash() throws IOException {
        Trash trash = new Trash(fileSystem, conf);
        Path dst = new Path("/123.txt");
        trash.moveToTrash(dst);
    }

    @After
    public void after() throws IOException{
        fileSystem.close();
    }

}