2. 程式人生 > >解決mapreduce無法將資料批量匯入hbase的問題

解決mapreduce無法將資料批量匯入hbase的問題

阿新 • • 發佈：2019-02-19

hadoop版本：hadoop-2.6.0-cdh5.5.2

hbase版本：hbase-1.0.0-cdh5.5.2

建立日誌表：

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;

public class HbaseDemo {

	public static void main(String[] args) throws IOException {
		String tableName = "wlan_log";//定義表名"wlan_lg"
		String columnFamily = "cf";//定義列族"cf"

		HbaseDemo.create(tableName, columnFamily);//建立表名和列族

		// HbaseDemo.put(tableName, "row1", columnFamily, "cl1", "data");
		// HbaseDemo.get(tableName, "row1");
		// HbaseDemo.scan(tableName);
		// HbaseDemo.delete(tableName);
	}
	//引數配置
	private static Configuration getConfiguration() {
		Configuration conf = HBaseConfiguration.create();//呼叫構造器，構造新物件conf例項
		conf.set("hbase.rootdir", "hdfs://192.168.8.71:9000/hbase");//設定hbase儲存位置
		conf.set("hbase.zookeeper.quorum", "192.168.8.71");//設定sqoop
		return conf;//返回例項
	}
	//建立表
	public static void create(String tableName, String columnFamily)
			throws IOException {
		HBaseAdmin admin = new HBaseAdmin(getConfiguration());//呼叫構造器構造例項admin
		if (admin.tableExists(tableName)) {//if判斷表是否存在
			System.out.println("table exists!");//存在輸出
		} else {				//esle執行下面語句
			HTableDescriptor tableDesc = new HTableDescriptor(tableName);//呼叫構造器構造tableDesc例項
			tableDesc.addFamily(new HColumnDescriptor(columnFamily));//新增列族
			admin.createTable(tableDesc);//建立表
			System.out.println("create table success!");//建立成功
		}
	}
	//新增一條記錄
	public static void put(String tableName, String row, String columnFamily,
			String column, String data) throws IOException {
		HTable table = new HTable(getConfiguration(), tableName);//匯入getConfiguration()引數配置和表名
		Put p1 = new Put(Bytes.toBytes(row));//建立物件和設定行健
		p1.add(Bytes.toBytes(columnFamily), Bytes.toBytes(column), Bytes
				.toBytes(data));//新增列族、列、資料
		table.put(p1);//put上傳p1
		System.out.println("put'" + row + "'," + columnFamily + ":" + column
				+ "','" + data + "'");//資料輸出展示
	}
	//讀取一條記錄
	public static void get(String tableName, String row) throws IOException {
		HTable table = new HTable(getConfiguration(), tableName);//建立一個table例項，或許當前引數配置和表名
		Get get = new Get(Bytes.toBytes(row));//建立get例項，獲取行健
		Result result = table.get(get);//將上一步的值賦給result
		System.out.println("Get: " + result);//列印結果輸出
	}
	//顯示所有資料
	public static void scan(String tableName) throws IOException {
		HTable table = new HTable(getConfiguration(), tableName);//建立table例項，獲取引數配置和表名
		Scan scan = new Scan();//建立例項scan
		ResultScanner scanner = table.getScanner(scan);//將例項scan值賦給scanner
		for (Result result : scanner) {//強遍歷
			System.out.println("Scan: " + result);//列印result中所有結果
		}
	}
	//刪除表
	public static void delete(String tableName) throws IOException {
		HBaseAdmin admin = new HBaseAdmin(getConfiguration());//建立admin例項進行表刪除操作
		if (admin.tableExists(tableName)) {//if語句判斷
			try {
				admin.disableTable(tableName);//使表失效
				admin.deleteTable(tableName);//刪除表
			} catch (IOException e) {
				e.printStackTrace();
				System.out.println("Delete " + tableName + "shi bai");//列印語句
			}
		}
		System.out.println("Delete " + tableName + "cheng gong");//列印語句
	}

}

將日誌檔案匯入Hbase表wlan_log中：
import java.text.SimpleDateFormat;
import java.util.Date;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;

public class HbaseBatchImport {

        public static void main(String[] args) throws Exception {
                final Configuration configuration = new Configuration();//設定最終例項使用final表示不可更改，建立例項
                configuration.set("hbase.zookeeper.quorum", "192.168.8.71");//配置hbase地址

                configuration.set(TableOutputFormat.OUTPUT_TABLE, "wlan_log");//資料hbase表名wlan_log

                configuration.set("dfs.socket.timeout", "180000");//設定超時響應時間

                final Job job = new Job(configuration, "HBaseBatchImport");//建立job例項

                job.setMapperClass(BatchImportMapper.class);
                job.setReducerClass(BatchImportReducer.class);
                job.setMapOutputKeyClass(LongWritable.class);//設定mapper輸出
                job.setMapOutputValueClass(Text.class);

                job.setInputFormatClass(TextInputFormat.class);
                job.setOutputFormatClass(TableOutputFormat.class);//設定輸出格式

                FileInputFormat.setInputPaths(job, "hdfs://192.168.8.71:9000/input");//設定輸入地址

                job.waitForCompletion(true);
        }

        static class BatchImportMapper extends
                        Mapper<LongWritable, Text, LongWritable, Text> {
                SimpleDateFormat dateformat1 = new SimpleDateFormat("yyyyMMddHHmmss");//建立例項並設定格式
                Text v2 = new Text();//建立v2例項

                protected void map(LongWritable key, Text value, Context context)
                                throws java.io.IOException, InterruptedException {
                        final String[] splited = value.toString().split("\t");//資料以/t分割
                        try {
                                final Date date = new Date(Long.parseLong(splited[0].trim()));//將下標為0的資料賦值給date同時用trim去空格
                                final String dateFormat = dateformat1.format(date);//獲取電腦date
                                String rowKey = splited[1] + ":" + dateFormat;//將dateFormat和date的第二個資料拼接
                                v2.set(rowKey + "\t" + value.toString());//set v2資料
                                context.write(key, v2);//把key v2寫入下面程式
                        } catch (NumberFormatException e) {
                                final Counter counter = context.getCounter("BatchImport",
                                                "ErrorFormat");
                                counter.increment(1L);
                                System.out.println("chu cuo le" + splited[0] + " " + e.getMessage());
                        }
                }
        }

        static class BatchImportReducer extends
                        TableReducer<LongWritable, Text, NullWritable> {//reduce例項
                protected void reduce(LongWritable key,
                                java.lang.Iterable<Text> values, Context context)
                                throws java.io.IOException, InterruptedException {
                        for (Text text : values) {	//強遍歷
                                final String[] splited = text.toString().split("\t");//用\t方式資料分割

                                final Put put = new Put(Bytes.toBytes(splited[0]));//把第一個資料賦值給put
                                put.add(Bytes.toBytes("cf"), Bytes.toBytes("date"), Bytes//上傳資料到cf列，date
                                                .toBytes(splited[1]));
                                context.write(NullWritable.get(), put);//資料輸出
                        }
                }
        }
}

input日誌：
1363157985066 	13726230503	00-FD-07-A4-72-B8:CMCC	120.196.100.82	i02.c.aliimg.com		24	27	2481	24681	200
1363157995052 	13826544101	5C-0E-8B-C7-F1-E0:CMCC	120.197.40.4			4	0	264	0	200
1363157991076 	13926435656	20-10-7A-28-CC-0A:CMCC	120.196.100.99			2	4	132	1512	200
1363154400022 	13926251106	5C-0E-8B-8B-B1-50:CMCC	120.197.40.4			4	0	240	0	200
1363157993044 	18211575961	94-71-AC-CD-E6-18:CMCC-EASY	120.196.100.99	iface.qiyi.com	視訊網站	15	12	1527	2106	200
1363157995074 	84138413	5C-0E-8B-8C-E8-20:7DaysInn	120.197.40.4	122.72.52.12		20	16	4116	1432	200
1363157993055 	13560439658	C4-17-FE-BA-DE-D9:CMCC	120.196.100.99			18	15	1116	954	200
1363157995033 	15920133257	5C-0E-8B-C7-BA-20:CMCC	120.197.40.4	sug.so.360.cn	資訊保安	20	20	3156	2936	200
1363157983019 	13719199419	68-A1-B7-03-07-B1:CMCC-EASY	120.196.100.82			4	0	240	0	200
1363157984041 	13660577991	5C-0E-8B-92-5C-20:CMCC-EASY	120.197.40.4	s19.cnzz.com	站點統計	24	9	6960	690	200
1363157973098 	15013685858	5C-0E-8B-C7-F7-90:CMCC	120.197.40.4	rank.ie.sogou.com	搜尋引擎	28	27	3659	3538	200
1363157986029 	15989002119	E8-99-C4-4E-93-E0:CMCC-EASY	120.196.100.99	www.umeng.com	站點統計	3	3	1938	180	200
1363157992093 	13560439658	C4-17-FE-BA-DE-D9:CMCC	120.196.100.99			15	9	918	4938	200
1363157986041 	13480253104	5C-0E-8B-C7-FC-80:CMCC-EASY	120.197.40.4			3	3	180	180	200
1363157984040 	13602846565	5C-0E-8B-8B-B6-00:CMCC	120.197.40.4	2052.flash2-http.qq.com	綜合門戶	15	12	1938	2910	200
1363157995093 	13922314466	00-FD-07-A2-EC-BA:CMCC	120.196.100.82	img.qfc.cn		12	12	3008	3720	200
1363157982040 	13502468823	5C-0A-5B-6A-0B-D4:CMCC-EASY	120.196.100.99	y0.ifengimg.com	綜合門戶	57	102	7335	110349	200
1363157986072 	18320173382	84-25-DB-4F-10-1A:CMCC-EASY	120.196.100.99	input.shouji.sogou.com	搜尋引擎	21	18	9531	2412	200
1363157990043 	13925057413	00-1F-64-E1-E6-9A:CMCC	120.196.100.55	t3.baidu.com	搜尋引擎	69	63	11058	48243	200
1363157988072 	13760778710	00-FD-07-A4-7B-08:CMCC	120.196.100.82			2	2	120	120	200
1363157985079 	13823070001	20-7C-8F-70-68-1F:CMCC	120.196.100.99			6	3	360	180	200
1363157985069 	13600217502	00-1F-64-E2-E8-B1:CMCC	120.196.100.55			18	138	1080	186852	200

[hadoop@h71 q1]$ hadoop fs -put input /

在hbase中建立wlan_log表：

[hadoop@h71 q1]$ /usr/jdk1.7.0_25/bin/javac HbaseDemo.java

[hadoop@h71 q1]$ /usr/jdk1.7.0_25/bin/java HbaseDemo

[hadoop@h71 q1]$ /usr/jdk1.7.0_25/bin/javac HbaseBatchImport.java 
Note: HbaseBatchImport.java uses or overrides a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
[hadoop@h71 q1]$ /usr/jdk1.7.0_25/bin/jar cvf xx.jar HbaseBatchImport*class
added manifest
adding: HbaseBatchImport$BatchImportMapper.class(in = 2731) (out= 1204)(deflated 55%)
adding: HbaseBatchImport$BatchImportReducer.class(in = 2185) (out= 906)(deflated 58%)
adding: HbaseBatchImport.class(in = 1613) (out= 833)(deflated 48%)
[hadoop@h71 q1]$ hadoop jar xx.jar HbaseBatchImport

（我在hadoop-0.20.2-cdh3u5+hbase-0.90.6-cdh3u5中執行就沒報這個錯，在我這個hadoop-2.6.0-cdh5.5.2中執行卻報錯了）

SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/home/hadoop/hadoop-2.6.0-cdh5.5.2/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/home/hadoop/hbase-1.0.0-cdh5.5.2/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
12/12/12 23:55:37 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
12/12/12 23:55:37 INFO client.RMProxy: Connecting to ResourceManager at h71/192.168.8.71:8032
12/12/12 23:55:38 INFO Configuration.deprecation: dfs.socket.timeout is deprecated. Instead, use dfs.client.socket-timeout
12/12/12 23:55:38 WARN mapreduce.JobResourceUploader: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
12/12/12 23:55:38 WARN mapreduce.JobResourceUploader: No job jar file set.  User classes may not be found. See Job or Job#setJar(String).
12/12/12 23:55:38 INFO input.FileInputFormat: Total input paths to process : 1
12/12/12 23:55:38 INFO mapreduce.JobSubmitter: number of splits:1
12/12/12 23:55:38 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1355303155288_0017
12/12/12 23:55:38 INFO mapred.YARNRunner: Job jar is not present. Not adding any jar to the list of resources.
12/12/12 23:55:38 INFO impl.YarnClientImpl: Submitted application application_1355303155288_0017
12/12/12 23:55:39 INFO mapreduce.Job: The url to track the job: http://h71:8088/proxy/application_1355303155288_0017/
12/12/12 23:55:39 INFO mapreduce.Job: Running job: job_1355303155288_0017
12/12/12 23:55:46 INFO mapreduce.Job: Job job_1355303155288_0017 running in uber mode : false
12/12/12 23:55:46 INFO mapreduce.Job:  map 0% reduce 0%
12/12/12 23:55:46 INFO mapreduce.Job: Job job_1355303155288_0017 failed with state FAILED due to: Application application_1355303155288_0017 failed 2 times due to AM Container for appattempt_1355303155288_0017_000002 exited with  exitCode: 1
For more detailed output, check application tracking page:http://h71:8088/proxy/application_1355303155288_0017/Then, click on links to logs of each attempt.
Diagnostics: Exception from container-launch.
Container id: container_1355303155288_0017_02_000001
Exit code: 1
Stack trace: ExitCodeException exitCode=1: 
        at org.apache.hadoop.util.Shell.runCommand(Shell.java:561)
        at org.apache.hadoop.util.Shell.run(Shell.java:478)
        at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:738)
        at org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.launchContainer(DefaultContainerExecutor.java:210)
        at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:302)
        at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:82)
        at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
        at java.util.concurrent.FutureTask.run(FutureTask.java:166)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:724)


Container exited with a non-zero exit code 1
Failing this attempt. Failing the application.
12/12/12 23:55:46 INFO mapreduce.Job: Counters: 0

解決：修改yarn-site.xml檔案，新增如下內容

[hadoop@h71 ~]$ cd hadoop-2.6.0-cdh5.5.2/etc/hadoop
[hadoop@h71 hadoop]$ vi yarn-site.xml

<property>
  <name>yarn.application.classpath</name>
      <value>
/home/hadoop/hadoop-2.6.0-cdh5.5.2/etc/hadoop/*,/home/hadoop/hadoop-2.6.0-cdh5.5.2/share/hadoop/common/lib/*,/home/hadoop/hadoop-2.6.0-cdh5.5.2/share/hadoop/common/*,/home/hadoop/hadoop-2.6.0-cdh5.5.2/share/hadoop/hdfs,/home/hadoop/hadoop-2.6.0-cdh5.5.2/share/hadoop/hdfs/lib/*,/home/hadoop/hadoop-2.6.0-cdh5.5.2/share/hadoop/hdfs/*,/home/hadoop/hadoop-2.6.0-cdh5.5.2/share/hadoop/yarn/lib/*,/home/hadoop/hadoop-2.6.0-cdh5.5.2/share/hadoop/yarn/*,/home/hadoop/hadoop-2.6.0-cdh5.5.2/share/hadoop/mapreduce/lib/*,/home/hadoop/hadoop-2.6.0-cdh5.5.2/share/hadoop/mapreduce/*,/home/hadoop/hive-1.1.0-cdh5.5.2/lib/*,/home/hadoop/hadoop-2.6.0-cdh5.5.2/contrib/capacity-scheduler/*,/home/hadoop/hbase-1.0.0-cdh5.5.2/lib/*
      </value>
</property>

再次執行以上命令卻又報錯：

[hadoop@h71 q1]$ hadoop jar xx.jar HbaseBatchImport

SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/home/hadoop/hadoop-2.6.0-cdh5.5.2/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/home/hadoop/hbase-1.0.0-cdh5.5.2/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
12/12/12 22:16:16 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
12/12/12 22:16:16 INFO client.RMProxy: Connecting to ResourceManager at h71/192.168.8.71:8032
12/12/12 22:16:17 INFO Configuration.deprecation: dfs.socket.timeout is deprecated. Instead, use dfs.client.socket-timeout
12/12/12 22:16:17 WARN mapreduce.JobResourceUploader: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
12/12/12 22:16:17 WARN mapreduce.JobResourceUploader: No job jar file set.  User classes may not be found. See Job or Job#setJar(String).
12/12/12 22:16:17 INFO input.FileInputFormat: Total input paths to process : 1
12/12/12 22:16:17 INFO mapreduce.JobSubmitter: number of splits:1
12/12/12 22:16:17 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1355303155288_0017
12/12/12 22:16:17 INFO mapred.YARNRunner: Job jar is not present. Not adding any jar to the list of resources.
12/12/12 22:16:17 INFO impl.YarnClientImpl: Submitted application application_1355303155288_0017
12/12/12 22:16:17 INFO mapreduce.Job: The url to track the job: http://h71:8088/proxy/application_1355303155288_0017/
12/12/12 22:16:17 INFO mapreduce.Job: Running job: job_1355303155288_0017
12/12/12 22:16:22 INFO mapreduce.Job: Job job_1355303155288_0017 running in uber mode : false
12/12/12 22:16:22 INFO mapreduce.Job:  map 0% reduce 0%
12/12/12 22:16:25 INFO mapreduce.Job: Task Id : attempt_1355303155288_0017_m_000000_0, Status : FAILED
Error: java.lang.RuntimeException: java.lang.ClassNotFoundException: Class HbaseBatchImport$BatchImportMapper not found
        at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2199)
        at org.apache.hadoop.mapreduce.task.JobContextImpl.getMapperClass(JobContextImpl.java:196)
        at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:745)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
Caused by: java.lang.ClassNotFoundException: Class HbaseBatchImport$BatchImportMapper not found
        at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:2105)
        at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2197)
        ... 8 more

12/12/12 22:16:29 INFO mapreduce.Job: Task Id : attempt_1355303155288_0017_m_000000_1, Status : FAILED
Error: java.lang.RuntimeException: java.lang.ClassNotFoundException: Class HbaseBatchImport$BatchImportMapper not found
        at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2199)
        at org.apache.hadoop.mapreduce.task.JobContextImpl.getMapperClass(JobContextImpl.java:196)
        at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:745)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
Caused by: java.lang.ClassNotFoundException: Class HbaseBatchImport$BatchImportMapper not found
        at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:2105)
        at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2197)
        ... 8 more

12/12/12 22:16:33 INFO mapreduce.Job: Task Id : attempt_1355303155288_0017_m_000000_2, Status : FAILED
Error: java.lang.RuntimeException: java.lang.ClassNotFoundException: Class HbaseBatchImport$BatchImportMapper not found
        at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2199)
        at org.apache.hadoop.mapreduce.task.JobContextImpl.getMapperClass(JobContextImpl.java:196)
        at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:745)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
Caused by: java.lang.ClassNotFoundException: Class HbaseBatchImport$BatchImportMapper not found
        at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:2105)
        at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2197)
        ... 8 more

12/12/12 22:16:40 INFO mapreduce.Job:  map 100% reduce 100%
12/12/12 22:16:40 INFO mapreduce.Job: Job job_1355303155288_0017 failed with state FAILED due to: Task failed task_1355303155288_0017_m_000000
Job failed as tasks failed. failedMaps:1 failedReduces:0

12/12/12 22:16:40 INFO mapreduce.Job: Counters: 9
        Job Counters 
                Failed map tasks=4
                Launched map tasks=4
                Other local map tasks=3
                Data-local map tasks=1
                Total time spent by all maps in occupied slots (ms)=10007
                Total time spent by all reduces in occupied slots (ms)=0
                Total time spent by all map tasks (ms)=10007
                Total vcore-seconds taken by all map tasks=10007
                Total megabyte-seconds taken by all map tasks=10247168

解決：

[hadoop@h71 q1]$ rm -rf HbaseBatchImport*class
[hadoop@h71 q1]$ vi HbaseBatchImport.java
新增：
job.setJarByClass(HbaseBatchImport.class);（hadoop1版本中不加就好使，hadoop2版本卻得加才好使。。。）

再執行上述命令又報錯：

[hadoop@h71 q1]$ hadoop jar xx.jar HbaseBatchImport

12/12/13 00:36:00 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/mapreduce/TableReducer
        at java.lang.ClassLoader.defineClass1(Native Method)
        at java.lang.ClassLoader.defineClass(ClassLoader.java:792)
        at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
        at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)
        at java.net.URLClassLoader.access$100(URLClassLoader.java:71)
        at java.net.URLClassLoader$1.run(URLClassLoader.java:361)
        at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
        at java.security.AccessController.doPrivileged(Native Method)
        at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
        at messages3.main(messages3.java:34)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:606)
        at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
        at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.hbase.mapreduce.TableReducer
        at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
        at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
        at java.security.AccessController.doPrivileged(Native Method)
        at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
        ... 18 more

解決：修改hadoop-env.sh檔案，新增如下內容

[hadoop@h71 hadoop]$ vi hadoop-env.sh

export HBASE_HOME=/home/hadoop/hbase-1.0.0-cdh5.5.2
export HADOOP_CLASSPATH=$HBASE_HOME/lib/*

（可能得重啟hadoop和hbase或者虛擬機器，我沒有也成功了）

[hadoop@h71 q1]$ hadoop jar xx.jar HbaseBatchImport

17/03/18 16:50:06 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
17/03/18 16:50:07 INFO client.RMProxy: Connecting to ResourceManager at h71/192.168.8.71:8032
17/03/18 16:50:07 INFO Configuration.deprecation: dfs.socket.timeout is deprecated. Instead, use dfs.client.socket-timeout
17/03/18 16:50:08 WARN mapreduce.JobResourceUploader: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
17/03/18 16:50:08 INFO input.FileInputFormat: Total input paths to process : 1
17/03/18 16:50:08 INFO mapreduce.JobSubmitter: number of splits:1
17/03/18 16:50:08 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1489740265001_0002
17/03/18 16:50:09 INFO impl.YarnClientImpl: Submitted application application_1489740265001_0002
17/03/18 16:50:09 INFO mapreduce.Job: The url to track the job: http://h71:8088/proxy/application_1489740265001_0002/
17/03/18 16:50:09 INFO mapreduce.Job: Running job: job_1489740265001_0002
17/03/18 16:50:18 INFO mapreduce.Job: Job job_1489740265001_0002 running in uber mode : false
17/03/18 16:50:18 INFO mapreduce.Job:  map 0% reduce 0%
17/03/18 16:50:32 INFO mapreduce.Job:  map 100% reduce 0%
17/03/18 16:50:38 INFO mapreduce.Job:  map 100% reduce 100%
17/03/18 16:50:38 INFO mapreduce.Job: Job job_1489740265001_0002 completed successfully
17/03/18 16:50:38 INFO mapreduce.Job: Counters: 49
        File System Counters
                FILE: Number of bytes read=190
                FILE: Number of bytes written=221731
                FILE: Number of read operations=0
                FILE: Number of large read operations=0
                FILE: Number of write operations=0
                HDFS: Number of bytes read=215
                HDFS: Number of bytes written=0
                HDFS: Number of read operations=2
                HDFS: Number of large read operations=0
                HDFS: Number of write operations=0
        Job Counters 
                Launched map tasks=1
                Launched reduce tasks=1
                Data-local map tasks=1
                Total time spent by all maps in occupied slots (ms)=12385
                Total time spent by all reduces in occupied slots (ms)=3458
                Total time spent by all map tasks (ms)=12385
                Total time spent by all reduce tasks (ms)=3458
                Total vcore-seconds taken by all map tasks=12385
                Total vcore-seconds taken by all reduce tasks=3458
                Total megabyte-seconds taken by all map tasks=12682240
                Total megabyte-seconds taken by all reduce tasks=3540992
        Map-Reduce Framework
                Map input records=2
                Map output records=2
                Map output bytes=180
                Map output materialized bytes=190
                Input split bytes=95
                Combine input records=0
                Combine output records=0
                Reduce input groups=2
                Reduce shuffle bytes=190
                Reduce input records=2
                Reduce output records=2
                Spilled Records=4
                Shuffled Maps =1
                Failed Shuffles=0
                Merged Map outputs=1
                GC time elapsed (ms)=366
                CPU time spent (ms)=2050
                Physical memory (bytes) snapshot=243867648
                Virtual memory (bytes) snapshot=2168229888
                Total committed heap usage (bytes)=136974336
        Shuffle Errors
                BAD_ID=0
                CONNECTION=0
                IO_ERROR=0
                WRONG_LENGTH=0
                WRONG_MAP=0
                WRONG_REDUCE=0
        File Input Format Counters 
                Bytes Read=120
        File Output Format Counters 
                Bytes Written=0

檢視hbase中的wlan_log會有相應的資料產生：

[hadoop@h71 hbase-1.0.0-cdh5.5.2]$ bin/hbase shell
hbase(main):002:0> scan 'wlan_log'


參考：

http://787141854-qq-com.iteye.com/blog/2067818

http://blog.csdn.net/hadoop_/article/details/11538201

相關推薦

解決mapreduce無法將資料批量匯入hbase的問題

hadoop版本：hadoop-2.6.0-cdh5.5.2 hbase版本：hbase-1.0.0-cdh5.5.2 建立日誌表： import java.io.IOException; import org.apache.hadoop.conf.Configura

Hbase通過命令將資料批量匯入的方法

拋磚引玉： hbase建表： hbase(main):003:0> create 'people','0' 將提前準備好的資料上傳到hdfs： [hadoop@h71 ~]$ vi people.txt 1,jimmy,25,jiujinshan 2,tina,25

如何將Excel表格中的資料批量匯入到Oracle資料庫表中

工具選擇     資料庫 ORACLE11G     程式語言 JAVA 首先是使用JAVA語言編寫一個解析EXCEL檔案的小程式碼，這裡需要一個工具JXL.JAR，這是個JAR包，有了它，我們就可以輕鬆的把EXCEL檔案解析 如果要對EXCEL檔案進行操作，則需要對EXCE

專案案例 || 將Excel資料批量匯入到資料庫

你在工作中是否遇到這樣的問題？資料一般存放在Excel表中，逐條遷移到資料庫中太麻煩，而且很多時候企業的資料量都是以萬起步，單條匯入顯然不現實。那麼該如何解決呢？ 我們今天就給大家介紹一個用途非常廣泛的功能：批量匯入，在很多系統中，這也是必須實現的功能。而且當Excel表結構越複雜

.NET Core使用NPOI將Excel中的資料批量匯入到MySQL

前言： 　　在之前的幾篇部落格中寫過.NET Core使用NPOI匯出Word和Excel的文章，今天把同樣我們日常開發中比較常用的使用Excel匯入資料到MySQL資料庫中的文章給安排上。與此同時還把NPOI-ExportWordAndExcel-ImportExcelData這個開源專案升級到了.NET

在webcontent中新建的資料夾中建立的JSP無法將資料傳給servlet

   相信很多寫程式的朋友都會建立很多資料夾來分類存放所寫的程式碼，本人也是如此，但是作為一個菜鳥，在做這件事的時候總會發生許多錯誤，我也一 一記錄下來，今次也不例外。    在寫一個關於登入頁面的簡單的JSP+Servlet時，新建立的資料夾，在裡面新寫

jdbc百萬資料批量匯入，流式讀取

Connection connReader = null; Connection connWrite = null; String url = "jdbc:mysql://XX:3306/XX?" + "user=XX&password=XX&

Matlab產生動態變化的字串，便於將資料批量存入表格

記錄一下。 在使用Matlab跑程式的時候，需要將最後的結果存入表格，例如： xlswrite('xxx.xls',result, 'a1:i1'); 此處只儲存一行，所以 ‘result’ 是個一維向量。 上述程式碼只能適用於跑一次程式的情況。下次跑的時候，我們需要改為： xl

Dstream[Row] 資料批量匯入Mysql 並去重（大致為如果資料庫中有某行資料了，本次執行若有一行與資料庫中的那行相同，則不會再插入進去）

def Save2Mysql(stateDStream: DStream[Row]): Unit = { stateDStream.foreachRDD { rdd => { rdd.foreachPartition(partitionRecord

Excel檔案一鍵上傳並解析完成資料批量匯入資料庫

原來做檔案上傳的時候，都是有一個輸入框，點選上傳按鈕，先瀏覽檔案，選擇檔案後，把檔案的路徑儲存到form表單中，最後通過form表單提交到服務端。這樣的介面不是很美觀。為了使用者有更好地體驗(UE)，現在的大多數系統都是採用一鍵檔案上傳，使用者點選上傳按鈕，選擇

Django 使用ORM將資料批量插入資料庫之bulk_create()

在Django中需要向資料庫中插入多條資料（list）。使用如下方法，每次save()的時候都會訪問一次資料庫。導致效能問題 for i in resultlist: p = Account(name=i) p.save() 1 2 3 4

JAVA Excel資料批量匯入到SQL資料庫

    今天剛好需要批量將Excel裡的資料匯入到SQL Server，本來打算用Genero寫的，發現官方驅動包找不到SQL Server 2012的驅動，我的天啊，無解！只好用JAVA來寫！JAVA通過載入JDBC驅動輕鬆搞定！除了引用sqljdbc4.jar 庫外，還

excel資料批量匯入mongodb資料庫

       資料匯入資料庫的工作經常會遇到，這篇博文我們介紹一下linux系統下如何通過命令將excel中的資料批量匯入到非關係型資料庫mongodb中。        步驟一：將bigdata.x

C#把DataTable裡的資料批量匯入SQLite

原目的：從一個老的SQLite裡讀出資料，存到DataTable裡，然後儲存到一個新的SQLite資料庫中。 需要引用System.Data.SQLite.dll try { SQLiteConnection Conn

spark批量匯入hbase

方法1 使用hbase put方式，這種方式效率不高 import org.apache.spark._ import org.apache.spark.rdd.NewHadoopRDD import org.apache.hadoop.hbase.{HBaseConfig

Excel 資料批量匯入到 mysql資料庫

工作中平時會遇到將 Excel資料批量匯入資料庫的情況。我是這樣做的 step1.  先將 excel 轉 json。 step2.  將轉換後的json拷貝到前端程式中，提交 json格式資料給後端;  後端迴圈遍歷資料進資料庫。

解決：無法將“Add-Migration”項識別為 cmdlet、函式、指令碼檔案或可執行程式的名稱。請檢查名稱的拼寫，如果包括路徑，請確保路徑正確，然後再試一次 1、輸入的中劃線“-”格式不對，檢查是

1、輸入的中劃線“-”格式不對，檢查是否為全形狀態下輸入，誤輸入了下劃線“_"，或是前後有空格； 2、沒有引用EntityFramework命令,請執行如下名稱（Import-Module 專案路徑\packages\EntityFramework.6.1.3（EF版本）\tools\EntityFra

SQLServer某個表的資料批量匯入到另一個表

類別一、如果兩張張表（匯出表和目標表）的欄位一致，並且希望插入全部資料，可以用這種方法： INSERT INTO 目標表 SELECT   * FROM 來源表; 例如，要將 articles 表插入到 newArticles 表中，則可以通過如下SQL語句實現： I

百萬條資料批量匯入

using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Diagnostics; using System.IO; using Sy

使用sqoop將資料從hdfs中匯入mysql時，卡在INFO mapreduce.Job: map 100% reduce 0%的解決辦法

最近在將hdfs中已經處理好的資料匯入到mysql的時候遇到這麼一個現象，當任務執行到 INFO mapreduce.Job: map 100% reduce 0% mapreduce任務卡在map100% ，任何reduce不繼續執行。 總結了以下解決辦法： 一，如果沒