hbase 分頁過濾(新老API的差別)
阿新 • • 發佈:2019-04-13
exception 數據 插入 過濾 ddr eof str ... 一個
在hbase2.0以前分頁過濾必須以上一次的最後一行+空字節數組作為下一次的起始行,
因為scan掃描的時候是包含起始行的,為了既能準確定位起始行,但又不重復把上一次的最末一行加入下一頁,
所以,權威指南裏才有了加上空字節數組的處理。
hbase2.0以後,新的api是withStartRow(byte[] startRow, boolean inclusive),可以直接設置是否包含起始行,完美解決問題,但是又保留了對以前api函數的兼容性。
//分頁過濾 private static void pageFilterData() throws IOException{ Table table= helper.getConnection().getTable(TableName.valueOf("testtable")); final byte[] POSTFIX = new byte[] { 0x00 }; Filter filter = new PageFilter(10); int totalRows = 0; byte[] lastRow = null; while(true){ Scan scan = new Scan(); scan.setFilter(filter);if(lastRow!=null){ //為了兼容以前的scan.setStartRow()代碼 //在上一次的最後一行加上一個空的byte數據,在下一個分頁上,就會以新的key開始, // 但是實際上這個key並不存在,所以還是從真正的下一行開始掃描 //這麽做的原因是scan的掃描會自動包含起始行,如果不加空字節數據,那麽定位上就會把上一次的最後一行作為起始行,最後的數據就會多一行。 //而,新的api是withStartRow(byte[] startRow, boolean inclusive),可以直接設置是否包含起始行,完美解決問題,但是又保留了對//以前api函數的兼容性 // byte[] startRow = Bytes.add(lastRow,POSTFIX); // System.out.println("start row: " + Bytes.toStringBinary(startRow)); // scan.withStartRow(startRow,true); System.out.println("start row: " + Bytes.toStringBinary(lastRow)); //不包含起始行,所以可以直接使用上一次的最後一行作為起始行 scan.withStartRow(lastRow,false); } ResultScanner scanner = table.getScanner(scan); int localRows = 0; Result result; while ((result=scanner.next())!=null){ System.out.println(localRows++ + ": " + result); totalRows++; lastRow = result.getRow(); } scanner.close(); if(localRows==0)break; } System.out.println("total rows: " + totalRows); }
兼容老API輸出如下:
0: keyvalues={rowKey0/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey0/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey0/memo:detail/1555078771906/Put/vlen=7/seqid=0} 1: keyvalues={rowKey1/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey1/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey1/memo:detail/1555078771906/Put/vlen=7/seqid=0} 2: keyvalues={rowKey10/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey10/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey10/memo:detail/1555078771906/Put/vlen=8/seqid=0} 3: keyvalues={rowKey11/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey11/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey11/memo:detail/1555078771906/Put/vlen=8/seqid=0} 4: keyvalues={rowKey12/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey12/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey12/memo:detail/1555078771906/Put/vlen=8/seqid=0} 5: keyvalues={rowKey13/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey13/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey13/memo:detail/1555078771906/Put/vlen=8/seqid=0} 6: keyvalues={rowKey14/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey14/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey14/memo:detail/1555078771906/Put/vlen=8/seqid=0} 7: keyvalues={rowKey15/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey15/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey15/memo:detail/1555078771906/Put/vlen=8/seqid=0} 8: keyvalues={rowKey16/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey16/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey16/memo:detail/1555078771906/Put/vlen=8/seqid=0} 9: keyvalues={rowKey17/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey17/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey17/memo:detail/1555078771906/Put/vlen=8/seqid=0} start row: rowKey17\x00 0: keyvalues={rowKey18/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey18/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey18/memo:detail/1555078771906/Put/vlen=8/seqid=0} 1: keyvalues={rowKey19/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey19/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey19/memo:detail/1555078771906/Put/vlen=8/seqid=0} 2: keyvalues={rowKey2/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey2/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey2/memo:detail/1555078771906/Put/vlen=7/seqid=0} 3: keyvalues={rowKey20/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey20/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey20/memo:detail/1555078771906/Put/vlen=8/seqid=0} 4: keyvalues={rowKey21/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey21/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey21/memo:detail/1555078771906/Put/vlen=8/seqid=0} 5: keyvalues={rowKey22/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey22/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey22/memo:detail/1555078771906/Put/vlen=8/seqid=0} 6: keyvalues={rowKey23/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey23/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey23/memo:detail/1555078771906/Put/vlen=8/seqid=0} 7: keyvalues={rowKey24/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey24/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey24/memo:detail/1555078771906/Put/vlen=8/seqid=0} 8: keyvalues={rowKey25/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey25/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey25/memo:detail/1555078771906/Put/vlen=8/seqid=0} 9: keyvalues={rowKey26/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey26/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey26/memo:detail/1555078771906/Put/vlen=8/seqid=0} start row: rowKey26\x00 0: keyvalues={rowKey27/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey27/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey27/memo:detail/1555078771906/Put/vlen=8/seqid=0} 1: keyvalues={rowKey28/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey28/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey28/memo:detail/1555078771906/Put/vlen=8/seqid=0} 2: keyvalues={rowKey29/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey29/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey29/memo:detail/1555078771906/Put/vlen=8/seqid=0} 3: keyvalues={rowKey3/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey3/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey3/memo:detail/1555078771906/Put/vlen=7/seqid=0} 4: keyvalues={rowKey30/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey30/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey30/memo:detail/1555078771906/Put/vlen=8/seqid=0} 5: keyvalues={rowKey31/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey31/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey31/memo:detail/1555078771906/Put/vlen=8/seqid=0} 6: keyvalues={rowKey32/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey32/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey32/memo:detail/1555078771906/Put/vlen=8/seqid=0} 7: keyvalues={rowKey33/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey33/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey33/memo:detail/1555078771906/Put/vlen=8/seqid=0} 8: keyvalues={rowKey34/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey34/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey34/memo:detail/1555078771906/Put/vlen=8/seqid=0} 9: keyvalues={rowKey35/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey35/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey35/memo:detail/1555078771906/Put/vlen=8/seqid=0} start row: rowKey35\x00 ... ...省略其他數據
起始行加了空字節數據,形成了新的定位行(相當於真正的兩頁之間插入了一個虛擬行用於定位),解決了下一頁首行的問題。
新的api輸出:下一頁的定位行就是上一頁的末行,采用新的API設置首行的時候把inclusive設置成false(即不包含首行),那麽輸出結果就是正確的。
0: keyvalues={rowKey0/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey0/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey0/memo:detail/1555078771906/Put/vlen=7/seqid=0} 1: keyvalues={rowKey1/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey1/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey1/memo:detail/1555078771906/Put/vlen=7/seqid=0} 2: keyvalues={rowKey10/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey10/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey10/memo:detail/1555078771906/Put/vlen=8/seqid=0} 3: keyvalues={rowKey11/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey11/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey11/memo:detail/1555078771906/Put/vlen=8/seqid=0} 4: keyvalues={rowKey12/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey12/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey12/memo:detail/1555078771906/Put/vlen=8/seqid=0} 5: keyvalues={rowKey13/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey13/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey13/memo:detail/1555078771906/Put/vlen=8/seqid=0} 6: keyvalues={rowKey14/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey14/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey14/memo:detail/1555078771906/Put/vlen=8/seqid=0} 7: keyvalues={rowKey15/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey15/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey15/memo:detail/1555078771906/Put/vlen=8/seqid=0} 8: keyvalues={rowKey16/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey16/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey16/memo:detail/1555078771906/Put/vlen=8/seqid=0} 9: keyvalues={rowKey17/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey17/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey17/memo:detail/1555078771906/Put/vlen=8/seqid=0} start row: rowKey17 0: keyvalues={rowKey18/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey18/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey18/memo:detail/1555078771906/Put/vlen=8/seqid=0} 1: keyvalues={rowKey19/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey19/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey19/memo:detail/1555078771906/Put/vlen=8/seqid=0} 2: keyvalues={rowKey2/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey2/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey2/memo:detail/1555078771906/Put/vlen=7/seqid=0} 3: keyvalues={rowKey20/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey20/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey20/memo:detail/1555078771906/Put/vlen=8/seqid=0} 4: keyvalues={rowKey21/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey21/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey21/memo:detail/1555078771906/Put/vlen=8/seqid=0} 5: keyvalues={rowKey22/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey22/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey22/memo:detail/1555078771906/Put/vlen=8/seqid=0} 6: keyvalues={rowKey23/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey23/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey23/memo:detail/1555078771906/Put/vlen=8/seqid=0} 7: keyvalues={rowKey24/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey24/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey24/memo:detail/1555078771906/Put/vlen=8/seqid=0} 8: keyvalues={rowKey25/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey25/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey25/memo:detail/1555078771906/Put/vlen=8/seqid=0} 9: keyvalues={rowKey26/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey26/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey26/memo:detail/1555078771906/Put/vlen=8/seqid=0} start row: rowKey26 0: keyvalues={rowKey27/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey27/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey27/memo:detail/1555078771906/Put/vlen=8/seqid=0} 1: keyvalues={rowKey28/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey28/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey28/memo:detail/1555078771906/Put/vlen=8/seqid=0} 2: keyvalues={rowKey29/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey29/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey29/memo:detail/1555078771906/Put/vlen=8/seqid=0} 3: keyvalues={rowKey3/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey3/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey3/memo:detail/1555078771906/Put/vlen=7/seqid=0} 4: keyvalues={rowKey30/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey30/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey30/memo:detail/1555078771906/Put/vlen=8/seqid=0} 5: keyvalues={rowKey31/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey31/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey31/memo:detail/1555078771906/Put/vlen=8/seqid=0} 6: keyvalues={rowKey32/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey32/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey32/memo:detail/1555078771906/Put/vlen=8/seqid=0} 7: keyvalues={rowKey33/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey33/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey33/memo:detail/1555078771906/Put/vlen=8/seqid=0} 8: keyvalues={rowKey34/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey34/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey34/memo:detail/1555078771906/Put/vlen=8/seqid=0} 9: keyvalues={rowKey35/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey35/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey35/memo:detail/1555078771906/Put/vlen=8/seqid=0} start row: rowKey35 ... ...其他省略
hbase 分頁過濾(新老API的差別)