HBase的java客戶端測試(二)---DML操作
阿新 • • 發佈：2017-12-31
返回 opera dmi for aos nbsp 多個 條件 node 
測試準備
【首先同步時間：】
for node in CloudDeskTop master01 master02 slave01 slave02 slave03;do ssh $node "date -s ‘2017-12-30 21:32:30‘";done
【slave各節點啟動zookeeper集群：】
cd /software/zookeeper-3.4.10/bin/ && ./zkServer.sh start && cd - && jps
【master01啟動HDFS集群：】
cd /software/ && start-dfs.sh && jps
【master01啟動HBase：】
cd /software/hbase-1.2.6/bin && start-hbase.sh && jps
【master02上啟動HBase:】
cd /software/hbase-1.2.6/bin && hbase-daemon.sh start master && jps
如有節點啟動出現故障：
單獨啟動master：
cd /software/hbase-1.2.6/bin && hbase-daemon.sh start master && jps
單獨啟動regionserver：
cd /software/hbase-1.2.6/bin && hbase-daemon.sh start regionserver && jps
通過命令終端查看：
hbase(main):009:0> status
通過web終端查看：
http://master01的IP地址:16010/
【在主機CloudDeskTop導入java客戶端開發所需jar包：】HBase1.2.6-All.zip
測試目標：
運用java代碼編寫程序操作HBase數據庫，本次測試致力於對DML語法的java客戶端操作；
測試代碼：大數據學習交流QQ群：217770236
1 package com.mmzs.bigdata.hbase.dml; 2 3 import java.io.ByteArrayInputStream; 4 import java.io.ByteArrayOutputStream; 5 import java.io.IOException;DMLMain6 import java.io.ObjectInputStream; 7 import java.io.ObjectOutputStream; 8 import java.util.ArrayList; 9 import java.util.Date; 10 import java.util.HashMap; 11 import java.util.List; 12 import java.util.Map; 13 import java.util.Map.Entry; 14 import java.util.Set; 15 16 import org.apache.hadoop.conf.Configuration; 17 import org.apache.hadoop.hbase.Cell; 18 import org.apache.hadoop.hbase.CellUtil; 19 import org.apache.hadoop.hbase.HBaseConfiguration; 20 import org.apache.hadoop.hbase.TableName; 21 import org.apache.hadoop.hbase.client.Admin; 22 import org.apache.hadoop.hbase.client.Connection; 23 import org.apache.hadoop.hbase.client.ConnectionFactory; 24 import org.apache.hadoop.hbase.client.Delete; 25 import org.apache.hadoop.hbase.client.Get; 26 import org.apache.hadoop.hbase.client.Put; 27 import org.apache.hadoop.hbase.client.Result; 28 import org.apache.hadoop.hbase.client.ResultScanner; 29 import org.apache.hadoop.hbase.client.Scan; 30 import org.apache.hadoop.hbase.client.Table; 31 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; 32 import org.apache.hadoop.hbase.filter.FilterList; 33 import org.apache.hadoop.hbase.filter.PrefixFilter; 34 import org.apache.hadoop.hbase.filter.RowFilter; 35 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; 36 import org.apache.hadoop.hbase.filter.SubstringComparator; 37 38 /** 39 * @author hadoop 40 * 41 * [HTable].put|delete|get|getScanner([Scan].addColumn|setStartRow|setStopRow|setFilter([FilterList].addFilter)) 42 * 43 */ 44 public class DMLMain { 45 /** 46 * 操作HBase集群的客戶端 47 */ 48 private static Admin admin; 49 50 private static Connection conn; 51 52 static{ 53 //創建HBase配置 54 Configuration conf=HBaseConfiguration.create(); 55 conf.set("hbase.zookeeper.quorum", "slave01:2181,slave02:2181,slave03:2181"); 56 try { 57 //根據HBase配置獲取HBase集群連接 58 conn=ConnectionFactory.createConnection(conf); 59 admin=conn.getAdmin(); 60 } catch (IOException e) { 61 e.printStackTrace(); 62 } 63 } 64 65 public static void main(String[] args) { 66 //這種添加方式的讀數據在後面查找時容易出IO異常，因為中間涉及了二進制轉換 67 // put("mmzs:myuser","008","base","userName","ligang00"); 68 // put("mmzs:myuser","008","base","userAge","18"); 69 // put("mmzs:myuser","008","base","gender","woman"); 70 71 /* Date d=new Date(); 72 System.out.println(d); 73 byte[] b=getBytes(d); 74 System.out.println(b); 75 Object obj=getObject(b); 76 System.out.println(obj);*/ 77 78 //添加數據 79 /* Map<String,Map<String,Map<String,Object>>> dataMap=new HashMap<String,Map<String,Map<String,Object>>>(); 80 81 Map<String,Map<String,Object>> familyMap=new HashMap<String,Map<String,Object>>(); 82 // dataMap.put("ligang+28+1.67", familyMap); 83 // dataMap.put("zhanghua+28+1.67", familyMap); 84 // dataMap.put("zhanghua+29+1.67", familyMap); 85 // dataMap.put("ligang+28+1.72", familyMap); 86 // dataMap.put("wangwu+29+1.82", familyMap); 87 dataMap.put("001", familyMap); 88 dataMap.put("002", familyMap); 89 dataMap.put("003", familyMap); 90 dataMap.put("004", familyMap); 91 dataMap.put("005", familyMap); 92 dataMap.put("006", familyMap); 93 94 Map<String,Object> keyValMap=new HashMap<String,Object>(); 95 keyValMap.put("height", 1.68); 96 keyValMap.put("weight", 75.5); 97 keyValMap.put("gender", "women"); 98 keyValMap.put("username", "zhangsan"); 99 100 familyMap.put("extra", keyValMap); 101 102 puts("mmzs:myuser",dataMap);*/ 103 104 105 // delete("mmzs:myuser","005"); 106 // delete("mmzs:myuser","002","extra","username"); 107 // deletes("mmzs:myuser","001","002","003","004","005","006","007","008"); 108 109 // get("mmzs:myuser","005"); 110 111 // scan("mmzs:myuser"); 112 113 // scanByCondition("mmzs:myuser"); 114 } 115 116 /** 117 * 將二進制流讀到內存通過反序列化重構生成對象 118 * @param object 119 * @return 120 */ 121 private static Object getObject(byte[] b){ 122 ObjectInputStream ois=null; 123 try { 124 ByteArrayInputStream bais=new ByteArrayInputStream(b); 125 ois=new ObjectInputStream(bais); 126 return ois.readObject(); 127 } catch (IOException e) { 128 e.printStackTrace(); 129 } catch (ClassNotFoundException e) { 130 e.printStackTrace(); 131 }finally{ 132 try{ 133 if(null!=ois)ois.close(); 134 }catch(IOException e){ 135 e.printStackTrace(); 136 } 137 } 138 return null; 139 } 140 141 /** 142 * 將對象序列化成二進制數組 143 * @param object 144 * @return 145 */ 146 private static byte[] getBytes(Object object){ 147 ObjectOutputStream oos=null; 148 try { 149 ByteArrayOutputStream baos=new ByteArrayOutputStream(); 150 oos=new ObjectOutputStream(baos); 151 oos.writeObject(object); 152 oos.flush(); 153 return baos.toByteArray(); 154 } catch (IOException e) { 155 e.printStackTrace(); 156 }finally{ 157 try{ 158 if(null!=oos)oos.close(); 159 }catch(IOException e){ 160 e.printStackTrace(); 161 } 162 } 163 return null; 164 } 165 166 /** 167 * 打印結果集 168 * @param ress 169 */ 170 private static void printResult(ResultScanner rss){ 171 //遍歷結果集,每一個Result對應一行記錄(即對應一個RowKey) 172 for(Result res:rss){ 173 String rowKey=new String(res.getRow()); 174 List<Cell> cellList=res.listCells(); 175 for(Cell cell:cellList){ 176 //獲取當前鍵值對所在的列族名稱 177 String familyName=new String(CellUtil.cloneFamily(cell)); 178 //獲取當前鍵值對的鍵(Key) 179 String key=new String(CellUtil.cloneQualifier(cell)); 180 //獲取當前鍵值對的值(Value) 181 String value=getObject(CellUtil.cloneValue(cell)).toString(); 182 183 System.out.println(rowKey+"\t"+familyName+"\t"+key+":"+value); 184 } 185 } 186 } 187 188 /** 189 * 條件查詢 190 * @param tabNameStr 191 * 192 * PrefixFilter和RowFilter都是基於行鍵(RowKey)的過濾器 193 */ 194 public static void scanByCondition(String tabNameStr){ 195 TableName tabName=TableName.valueOf(tabNameStr); 196 Scan scan=new Scan(); 197 198 //前綴過濾器,行鍵前綴是ligang的 199 PrefixFilter prefixFilter=new PrefixFilter("ligang".getBytes()); 200 201 //子串過濾,行鍵中包含1.72的 202 RowFilter rowFilter=new RowFilter(CompareOp.EQUAL,new SubstringComparator("28")); 203 204 //行鍵過濾器，列族 列名 比較操作 值,顯示不包含滿足條件的 205 SingleColumnValueFilter scvFilter=new SingleColumnValueFilter("base".getBytes(),"userName".getBytes(),CompareOp.EQUAL,new SubstringComparator("ligang")); 206 207 //FilterList.Operator.MUST_PASS_ALL相當於and,FilterList.Operator.MUST_PASS_ONE相當於or 208 FilterList filterList=new FilterList(FilterList.Operator.MUST_PASS_ALL); 209 //添加使用的過濾器 210 // filterList.addFilter(prefixFilter); 211 // filterList.addFilter(rowFilter); 212 filterList.addFilter(scvFilter); 213 214 // scan.setFilter(prefixFilter); 215 216 scan.setFilter(filterList);//設置過濾 217 try { 218 Table table=conn.getTable(tabName); 219 ResultScanner ress=table.getScanner(scan); 220 printResult(ress); 221 } catch (IOException e) { 222 e.printStackTrace(); 223 } 224 } 225 226 /** 227 * 查詢多條記錄 228 * @param tabNameStr 229 * @param rowKey 230 */ 231 public static void scan(String tabNameStr){ 232 TableName tabName=TableName.valueOf(tabNameStr); 233 234 Scan scan=new Scan(); 235 236 //過濾查詢結果集中的字段 237 scan.addColumn("extra".getBytes(), "height".getBytes()); 238 scan.addColumn("extra".getBytes(), "weight".getBytes()); 239 240 //設置查詢的起始和結束行索引(通過行鍵RowKey指定) 241 scan.setStartRow("002".getBytes()); 242 scan.setStopRow("006".getBytes()); 243 244 try { 245 Table table=conn.getTable(tabName); 246 //查詢多行返回一個結果集 247 ResultScanner rss=table.getScanner(scan); 248 //遍歷結果集,每一個Result對應一行記錄(即對應一個RowKey) 249 printResult(rss); 250 } catch (IOException e) { 251 e.printStackTrace(); 252 } 253 } 254 255 /** 256 * 查詢單條記錄 257 * @param rowKey 258 * @param tabNameStr 259 */ 260 public static void get(String tabNameStr,String rowKey){ 261 TableName tabName=TableName.valueOf(tabNameStr); 262 263 Get get=new Get(rowKey.getBytes()); 264 //相當於select..字段列表...from...,如果沒有下面的addColumn方法調用則相當於select *... 265 get.addColumn("base".getBytes(), "height".getBytes()); 266 get.addColumn("base".getBytes(), "gender".getBytes()); 267 try { 268 Table table=conn.getTable(tabName); 269 Result result=table.get(get); 270 271 //獲取行鍵 272 String rowKeyStr=new String(result.getRow()); 273 System.out.println("行鍵:"+rowKeyStr); 274 275 //獲取鍵所對應的值 276 byte[] byteName=result.getValue("base".getBytes(), "gender".getBytes()); 277 String gender=getObject(byteName).toString(); 278 System.out.println("gender:"+gender); 279 280 //獲取當前行中的所有鍵值對 281 List<Cell> cellList=result.listCells(); 282 for(Cell cell:cellList){ 283 //獲取當前鍵值對所在的列族名稱 284 String familyName=new String(CellUtil.cloneFamily(cell)); 285 //獲取當前鍵值對的鍵(Key) 286 String key=new String(CellUtil.cloneQualifier(cell)); 287 //獲取當前鍵值對的值(Value) 288 byte[] byteValue=CellUtil.cloneValue(cell); 289 String value=getObject(byteValue).toString(); 290 291 System.out.println(rowKey+"\t"+familyName+"\t"+key+":"+value); 292 } 293 } catch (IOException e) { 294 e.printStackTrace(); 295 } 296 } 297 298 /** 299 * 批量刪除多行 300 * @param tabNameStr 301 * @param rowKey 302 */ 303 public static void deletes(String tabNameStr,String... rowKeys){ 304 if(rowKeys.length==0)return; 305 TableName tabName=TableName.valueOf(tabNameStr); 306 307 List<Delete> deleteList=new ArrayList<Delete>(); 308 for(String rowKey:rowKeys)deleteList.add(new Delete(rowKey.getBytes())); 309 310 try { 311 Table table=conn.getTable(tabName); 312 table.delete(deleteList); 313 System.out.println("刪除完成！"); 314 } catch (IOException e) { 315 e.printStackTrace(); 316 } 317 } 318 319 /** 320 * 刪除行中的鍵值對 321 * @param tabNameStr 322 * @param rowKey 323 * @param key 324 */ 325 public static void delete(String tabNameStr,String rowKey,String family,String key){ 326 TableName tabName=TableName.valueOf(tabNameStr); 327 try { 328 Table table=conn.getTable(tabName); 329 Delete delete=new Delete(rowKey.getBytes()); 330 delete.addColumn(family.getBytes(), key.getBytes()); 331 table.delete(delete); 332 System.out.println("刪除完成！"); 333 } catch (IOException e) { 334 e.printStackTrace(); 335 } 336 } 337 338 /** 339 * 刪除整行 340 * @param tabNameStr 341 * @param rowKey 342 */ 343 public static void delete(String tabNameStr,String rowKey){ 344 TableName tabName=TableName.valueOf(tabNameStr); 345 try { 346 Table table=conn.getTable(tabName); 347 Delete delete=new Delete(rowKey.getBytes()); 348 table.delete(delete); 349 System.out.println("刪除完成！"); 350 } catch (IOException e) { 351 e.printStackTrace(); 352 } 353 } 354 355 /** 356 * 增加或修改數據(表名、行鍵、列族、列、值) 357 */ 358 public static void put(String tabNameStr,String rowKey,String family,String key,String value){ 359 TableName tabName=TableName.valueOf(tabNameStr); 360 try { 361 Table table=conn.getTable(tabName); 362 Put put=new Put(rowKey.getBytes()); 363 put.addColumn(family.getBytes(), key.getBytes(), value.getBytes()); 364 table.put(put); 365 System.out.println("操作完成！"); 366 } catch (IOException e) { 367 e.printStackTrace(); 368 } 369 } 370 371 /** 372 * 批量插入或修改數據 373 * @param tabNameStr 374 * @param dataMap 375 */ 376 public static void puts(String tabNameStr,Map<String,Map<String,Map<String,Object>>> dataMap){ 377 List<Put> putList=new ArrayList<Put>(); 378 Set<Entry<String, Map<String, Map<String, Object>>>> entrys=dataMap.entrySet(); 379 for(Entry<String, Map<String, Map<String, Object>>> entry:entrys){ 380 //獲取行的rowKey 381 byte[] rowKey=entry.getKey().getBytes(); 382 Put put=new Put(rowKey); 383 putList.add(put); 384 //獲取行的所有列族 385 Map<String, Map<String, Object>> familyMap=entry.getValue(); 386 Set<Entry<String, Map<String, Object>>> familyEntrys=familyMap.entrySet(); 387 for(Entry<String, Map<String, Object>> familyEntry:familyEntrys){ 388 //獲取列族名稱 389 byte[] familyName=familyEntry.getKey().getBytes(); 390 //獲取列族下左右的鍵值對 391 Map<String, Object> keyVals=familyEntry.getValue(); 392 Set<Entry<String, Object>> keyValEntrys=keyVals.entrySet(); 393 for(Entry<String, Object> keyValEntry:keyValEntrys){ 394 byte[] key=keyValEntry.getKey().getBytes(); 395 byte[] value=getBytes(keyValEntry.getValue()); 396 put.addColumn(familyName, key, value); 397 } 398 } 399 } 400 401 TableName tabName=TableName.valueOf(tabNameStr); 402 try { 403 Table table=conn.getTable(tabName); 404 table.put(putList); 405 System.out.println("操作完成！"); 406 } catch (IOException e) { 407 e.printStackTrace(); 408 } 409 } 410 }
測試結果：
在命令端查看，查看方式，可參考：http://www.cnblogs.com/mmzs/p/8135327.html
測試小結：
HBase是分布式的、可擴展的、且基於Hadop的大數據NOSQL存儲數據庫(號稱十億行、百萬列),該數據庫的每一行也是保存一條記錄,但是與關系數據庫不同的是,該數據庫無需事先定義字段和表結構,只需要指定表名和列族;每一行由指定的一到多個族組成,每個族上可以保存若幹個鍵值對(每一行的每個族上的鍵值對數量可以是任意的),即每一行的單元格數量不一定相等,每一個單元格中保存同一個鍵名但不同版本值的若幹個鍵值對(即同一個字段名中有不同版本的字段值),當需要查詢某一個字段值時需要指定的坐標是:
表名(table name)—>行健(row key)—>列族(column family)—>字段名(column name)—>版本號(version code)
HBase的java客戶端測試(二)---DML操作