使用sqoop命令把oracle資料匯入hdfs/hbase/hive
阿新 • • 發佈：2019-01-29
（一）匯入HDFS
建立Job
#!/bin/bash #Oracle的連線字串，其中包含了Oracle的地址，SID，和埠號 CONNECTURL=jdbc:oracle:thin:@10.122.5.2:1521:db #使用的使用者名稱 ORACLENAME=db #使用的密碼 ORACLEPASSWORD=123 #需要從Oracle中匯入的表名 oralceTableName=TT #需要從Oracle中匯入的表中的欄位名 columns=YEAR,KW,BRAND,VOLUME #將Oracle中的資料匯入到HDFS後的存放路徑 hdfsPath=/user/asmp/voice/voice_source/CHANNEL_BUZ sqoop job --meta-connect jdbc:hsqldb:hsql://192.168.14.33:16000/sqoop --delete Hdfs_Job sqoop job --meta-connect jdbc:hsqldb:hsql://192.168.14.33:16000/sqoop --create Hdfs_Job -- import --append --connect $CONNECTURL --username $ORACLENAME --password $ORACLEPASSWORD --target-dir $hdfsPath --m 1 --table $oralceTableName --columns $columns --fields-terminated-by ',' --null-string '' --null-non-string '' --incremental lastmodified --check-column CREATE_DATE --last-value '2018-06-12 11:00:00'
執行Job
#!/bin/bash
sqoop job --exec Hdfs_Job --meta-connect jdbc:hsqldb:hsql://192.168.14.33:16000/sqoop（二）匯入HBase
建立hbase表
建立Job#!/bin/bash hbase shell disable 'PART_T' drop 'PART_T' create 'PART_T', {NAME => 'z2', BLOOMFILTER => 'ROW', VERSIONS => '1', IN_MEMORY => 'false', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', COMPRESSION => 'SNAPPY', MIN_VERSIONS => '0', BLOCKCACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'}, {SPLITS => ['74304050']} exit
（三）匯入Hive#!/bin/bash sqoop job --meta-connect jdbc:hsqldb:hsql://www.csdn.com:16000/sqoop --delete job_t sqoop job --meta-connect jdbc:hsqldb:hsql://www.csdn.com:16000/sqoop --create job_t -- import --connect jdbc:oracle:thin:@//10.122.16.11:1521/db --username db --password 123 --query "select SST_CODE FROM SBPOPT.PART WHERE sst_code = '74308390'" --hbase-table PART_T --column-family z2 --hbase-row-key SSTCODE --fields-terminated-by "\001" --lines-terminated-by "\n" --null-string '\\N' --null-non-string '\\N' -m 4
建立Job
#!/bin/bash
sqoop job --meta-connect jdbc:hsqldb:hsql://192.168.14.33:16000/sqoop --delete test
sqoop job --meta-connect jdbc:hsqldb:hsql://192.168.14.33:16000/sqoop --create test -- import --connect jdbc:oracle:thin:@//10.122.5.2:1521/db --username db--password 123 --table TT --hive-import --hive-database db --hive-table test --fields-terminated-by "\001" --null-string 'null' --null-non-string 'null' --hive-drop-import-delims -m 1執行Job
#!/bin/bash
sqoop job --exec test --meta-connect jdbc:hsqldb:hsql://192.168.14.33:16000/sqoop