eclipse 上傳檔案到hadoop
阿新 • • 發佈:2019-01-30
其實很簡單,只要配置正確,就行
直接給程式碼吧
package com.younglibin.hadoop; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; public class HDFSystem { public static void uploadLocalFile2HDFS(Configuration config, String localFile, String remoteFile) throws IOException { FileSystem hdfs = FileSystem.get(config); Path src = new Path(localFile); Path dst = new Path(remoteFile); hdfs.copyFromLocalFile(src, dst); hdfs.close(); } public static void createNewHDFSFile(String toCreateFilePath, String content) throws IOException { Configuration config = new Configuration(); FileSystem hdfs = FileSystem.get(config); FSDataOutputStream os = hdfs.create(new Path(toCreateFilePath)); os.write(content.getBytes("UTF-8")); os.close(); hdfs.close(); } public static boolean deleteHDFSFile(String dst) throws IOException { Configuration config = new Configuration(); FileSystem hdfs = FileSystem.get(config); Path path = new Path(dst); boolean isDeleted = hdfs.delete(path); hdfs.close(); return isDeleted; } public static byte[] readHDFSFile(String dst) throws Exception { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); // check if the file exists Path path = new Path(dst); if (fs.exists(path)) { FSDataInputStream is = fs.open(path); // get the file info to create the buffer FileStatus stat = fs.getFileStatus(path); // create the buffer byte[] buffer = new byte[Integer.parseInt(String.valueOf(stat .getLen()))]; is.readFully(0, buffer); is.close(); fs.close(); return buffer; } else { throw new Exception("the file is not found ."); } } }
看程式碼吧:
public static void main(String[] args) throws Exception { File jarFile = EJob.createTempJar("bin"); EJob.addClasspath("/home/libin/software/hadoop/hadoop-1.2.1/conf"); ClassLoader classLoader = EJob.getClassLoader(); Thread.currentThread().setContextClassLoader(classLoader); Configuration conf = new Configuration(); conf.set("mapred.job.tracker", "172.16.236.11:9001"); conf.set("fs.default.name", "hdfs://172.16.236.11:9000"); String fileName = "local"; HDFSystem.uploadLocalFile2HDFS(conf, new File(fileName).getAbsolutePath(), "hdfs://172.16.236.11:9000/user/libin/input/" + fileName); args = new String[] { "/user/libin/input/" + fileName, "/user/libin/output/wordcount" + System.currentTimeMillis() }; String[] otherArgs = new GenericOptionsParser(conf, args) .getRemainingArgs(); if (otherArgs.length != 2) { System.err.println("Usage: wordcount <in> <out>"); System.exit(2); } Job job = new Job(conf, "word count"); job.setJarByClass(WordCount.class); ((JobConf) job.getConfiguration()).setJar(jarFile.toString()); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path("hdfs://172.16.236.11:9000" + File.separator + otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path( "hdfs://172.16.236.11:9000" + File.separator + otherArgs[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
轉自:http://younglibin.iteye.com/blog/1925109