轉：http://blog.csdn.net/uq_jin/article/details/52235121

1、解壓Hadoop到任意目錄

比如：D:\soft\dev\Hadoop-2.7.2

2、設定環境變數

HADOOP_HOME：D:\soft\dev\hadoop-2.7.2

HADOOP_BIN_PATH：%HADOOP_HOME%\bin

HADOOP_PREFIX：%HADOOP_HOME%

在Path後面加上%HADOOP_HOME%\bin;%HADOOP_HOME%\sbin;

把 hadoop-2.7.3\share\hadoop\hdfs\hadoop-hdfs-2.7.3.jar，share\hadoop\hdfs\lib

D:\BaiduYunDownload\hadoop-2.7.3\share\hadoop\common\hadoop-common-2.7.3.jar，\share\hadoop\common\lib

的jar包拷貝到專案lib下

HdfsClientTest

package com.hadoop.test;

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Before;
import org.junit.Test;

public class HdfsClientTest {
	private FileSystem fs = null;

	@Before
	public void setUp() throws Exception {
		// get a configuration object
		Configuration conf = new Configuration();
		// to set a parameter,figure out the filesystemis hdfs
		conf.set("fs.defalutFS", "hdfs://192.168.169.128:9000");
		conf.set("dfs.replication", "1");

		// get a instance of HDFS FileSystem Client
		fs = FileSystem.get(new URI("hdfs://192.168.169.128:9000/"), conf, "root");
	}

	@Test
	public void testDownload() throws IllegalArgumentException, IOException {
		FSDataInputStream is = fs.open(new Path("hdfs://192.168.169.128:9000/hadoop-common-2.7.3.jar"));

		FileOutputStream os = new FileOutputStream("D:/hdfs/down.apk");

		IOUtils.copy(is, os);
	}

	// 上傳檔案
	public static void main(String[] args) throws Exception {
		// get a configuration object
		Configuration conf = new Configuration();
		// to set a parameter,figure out the filesystemis hdfs
		conf.set("fs.defalutFS", "hdfs://192.168.169.128:9000");
		conf.set("dfs.replication", "1");

		// get a instance of HDFS FileSystem Client
		FileSystem fs = FileSystem.get(new URI("hdfs://192.168.169.128:9000/"), conf, "root");
		// open a outputstream of the dest file
		Path destFile = new Path("hdfs://192.168.169.128:9000/hadoop-common-2.7.3.jar");

		FSDataOutputStream os = fs.create(destFile);

		// open a inputstream of the local source file

		FileInputStream is = new FileInputStream(
				"D:/BaiduYunDownload/hadoop-2.7.3/share/hadoop/common/hadoop-common-2.7.3.jar");

		// write the bytes in "is" to "os"
		IOUtils.copy(is, os);
	}

}
 

package com.hadoop.test;

import static org.junit.Assert.*;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;

public class HdfsClientEasy {
	private FileSystem fs = null;

	@Before
	public void setUp() throws Exception {

		// 拿到一個配置引數的封裝物件，建構函式中就會對classpath下的xxx-site.xml檔案進行解析
		// 真實的專案工程中就應該把xxx-site.xml檔案加入到工程中來
		Configuration conf = new Configuration();
		// to set a parameter,figure out the filesystemis hdfs
		conf.set("fs.defalutFS", "hdfs://192.168.169.128:9000");
		conf.set("dfs.replication", "1");

		// 這種獲取fs的方法可以指定訪問hdfs的客戶端身份
		fs = FileSystem.get(new URI("hdfs://192.168.169.128:9000/"), conf, "root");
	}

	/**
	 * 上傳檔案
	 * 
	 * @throws IllegalArgumentException
	 * @throws IOException
	 */
	@Test
	public void testUpload() throws IllegalArgumentException, IOException {
		fs.copyFromLocalFile(new Path("D:/HooShell_Shudu.apk"),
				new Path("/hadoop-common-2.7.3.jar"));
	}

	/**
	 * 刪除檔案
	 * 
	 * @throws IllegalArgumentException
	 * @throws IOException
	 */
	@Test
	public void testRmfile() throws IllegalArgumentException, IOException {
		boolean res = fs.delete(new Path("/hadoop-common-2.7.3.jar"), true);
		System.out.println(res ? "delete is successfully :)" : "it is failed :(");
	}

	/**
	 * 建立資料夾
	 * 
	 * @throws IllegalArgumentException
	 * @throws IOException
	 */
	@Test
	public void testMkdir() throws IllegalArgumentException, IOException {
		fs.mkdirs(new Path("/aa/bb"));
	}

	/**
	 * 重新命名檔案
	 * 
	 * @throws IllegalArgumentException
	 * @throws IOException
	 */
	@Test
	public void testRename() throws IllegalArgumentException, IOException {
		fs.rename(new Path("/hadoop-common-2.7.3.jar"), new Path("/hadoop-common.jar"));
	}

	@Test
	public void testListFiles() throws FileNotFoundException, IllegalArgumentException, IOException {
		RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);
		while (listFiles.hasNext()) {

			LocatedFileStatus file = listFiles.next();

			System.out.println(file.getPath().getName());

		}
		System.out.println("--------------------------------------------");

		// 列出檔案及資料夾
		FileStatus[] status = fs.listStatus(new Path("/"));
		for (FileStatus file : status) {

			System.out.println(file.getPath().getName() + "   " + (file.isDirectory() ? "d" : "f"));

		}
	}

	/**
	 * 從hdfs中下載資料到本地
	 * 
	 * @throws IllegalArgumentException
	 * @throws IOException
	 */
	@Test
	public void testDownload() throws IllegalArgumentException, IOException {
		// fs.c
		// fs.copyToLocalFile(true,new Path("/hadoop-common-2.7.3.jar"), new
		// Path("d:/jdk12333.win"));
		fs.copyToLocalFile(false, new Path("/hadoop-common-2.7.3.jar"), new Path("c:/jdk.win"), true);

	}

}
 

Permission denied: user=root, access=WRITE, inode="/user/hive/warehouse":hadoop:hadoop:drwxrwxr-x

原因：本地使用者administrator（本機windows使用者）想要遠端操作hadoop系統，沒有許可權引起的。

 解決辦法：

1、如果是測試環境，可以取消hadoop hdfs的使用者許可權檢查。開啟conf/hdfs-site.xml，找到dfs.permissions屬性修改為false（預設為true）OK了。

2. 指定訪問hdfs的客戶端身份
fs = FileSystem.get(new URI("hdfs://192.168.169.128:9000/"), conf, "root");



fs.copyFromLocalFile(new Path("D:/HooShell_Shudu.apk"),new Path("/hadoop-common-2.7.3.jar"));報錯
org.apache.hadoop.ipc.RemoteException(java.io.IOException): File /HooShell_Shudu.apk could only be replicated to 0 nodes instead of minReplication (=1).  There are 1 datanode(s) running and 1 node(s) are excluded in this operation.
    at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:1571)
    at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getNewBlockTargets(FSNamesystem.java:3107)
    at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3031)
    at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:725)

解決方法:
#關閉防火牆
sudo systemctl stop firewalld.service
#關閉開機啟動
sudo systemctl disable firewalld.service    
    
    
java.io.IOException: (null) entry in command string: null chmod 0644 D:\hdfs\jdk123.win
    at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:769)
    at org.apache.hadoop.util.Shell.execCommand(Shell.java:866)
    at org.apache.hadoop.util.Shell.execCommand(Shell.java:849)
    at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:733)
    at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.<init>(RawLocalFileSystem.java:225)
    at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.<init>(RawLocalFileSystem.java:209)
    at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:307)

解決方法:將fs.copyToLocalFile( hdfsPath,localPath);改為fs.copyToLocalFile( false,hdfsPath,localPath,true);
原因:不理解,但加上這兩個後確實可以下載