1. 程式人生 > >hbase 分頁過濾(新老API的差別)

hbase 分頁過濾(新老API的差別)

exception 數據 插入 過濾 ddr eof str ... 一個

在hbase2.0以前分頁過濾必須以上一次的最後一行+空字節數組作為下一次的起始行,

因為scan掃描的時候是包含起始行的,為了既能準確定位起始行,但又不重復把上一次的最末一行加入下一頁,

所以,權威指南裏才有了加上空字節數組的處理。

hbase2.0以後,新的api是withStartRow(byte[] startRow, boolean inclusive),可以直接設置是否包含起始行,完美解決問題,但是又保留了對以前api函數的兼容性。

    //分頁過濾
    private static void pageFilterData() throws IOException{
        Table table 
= helper.getConnection().getTable(TableName.valueOf("testtable")); final byte[] POSTFIX = new byte[] { 0x00 }; Filter filter = new PageFilter(10); int totalRows = 0; byte[] lastRow = null; while(true){ Scan scan = new Scan(); scan.setFilter(filter);
if(lastRow!=null){ //為了兼容以前的scan.setStartRow()代碼 //在上一次的最後一行加上一個空的byte數據,在下一個分頁上,就會以新的key開始, // 但是實際上這個key並不存在,所以還是從真正的下一行開始掃描 //這麽做的原因是scan的掃描會自動包含起始行,如果不加空字節數據,那麽定位上就會把上一次的最後一行作為起始行,最後的數據就會多一行。 //而,新的api是withStartRow(byte[] startRow, boolean inclusive),可以直接設置是否包含起始行,完美解決問題,但是又保留了對
//以前api函數的兼容性 // byte[] startRow = Bytes.add(lastRow,POSTFIX); // System.out.println("start row: " + Bytes.toStringBinary(startRow)); // scan.withStartRow(startRow,true); System.out.println("start row: " + Bytes.toStringBinary(lastRow)); //不包含起始行,所以可以直接使用上一次的最後一行作為起始行 scan.withStartRow(lastRow,false); } ResultScanner scanner = table.getScanner(scan); int localRows = 0; Result result; while ((result=scanner.next())!=null){ System.out.println(localRows++ + ": " + result); totalRows++; lastRow = result.getRow(); } scanner.close(); if(localRows==0)break; } System.out.println("total rows: " + totalRows); }

兼容老API輸出如下:

0: keyvalues={rowKey0/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey0/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey0/memo:detail/1555078771906/Put/vlen=7/seqid=0}
1: keyvalues={rowKey1/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey1/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey1/memo:detail/1555078771906/Put/vlen=7/seqid=0}
2: keyvalues={rowKey10/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey10/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey10/memo:detail/1555078771906/Put/vlen=8/seqid=0}
3: keyvalues={rowKey11/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey11/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey11/memo:detail/1555078771906/Put/vlen=8/seqid=0}
4: keyvalues={rowKey12/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey12/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey12/memo:detail/1555078771906/Put/vlen=8/seqid=0}
5: keyvalues={rowKey13/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey13/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey13/memo:detail/1555078771906/Put/vlen=8/seqid=0}
6: keyvalues={rowKey14/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey14/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey14/memo:detail/1555078771906/Put/vlen=8/seqid=0}
7: keyvalues={rowKey15/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey15/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey15/memo:detail/1555078771906/Put/vlen=8/seqid=0}
8: keyvalues={rowKey16/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey16/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey16/memo:detail/1555078771906/Put/vlen=8/seqid=0}
9: keyvalues={rowKey17/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey17/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey17/memo:detail/1555078771906/Put/vlen=8/seqid=0}
start row: rowKey17\x00
0: keyvalues={rowKey18/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey18/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey18/memo:detail/1555078771906/Put/vlen=8/seqid=0}
1: keyvalues={rowKey19/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey19/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey19/memo:detail/1555078771906/Put/vlen=8/seqid=0}
2: keyvalues={rowKey2/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey2/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey2/memo:detail/1555078771906/Put/vlen=7/seqid=0}
3: keyvalues={rowKey20/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey20/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey20/memo:detail/1555078771906/Put/vlen=8/seqid=0}
4: keyvalues={rowKey21/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey21/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey21/memo:detail/1555078771906/Put/vlen=8/seqid=0}
5: keyvalues={rowKey22/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey22/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey22/memo:detail/1555078771906/Put/vlen=8/seqid=0}
6: keyvalues={rowKey23/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey23/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey23/memo:detail/1555078771906/Put/vlen=8/seqid=0}
7: keyvalues={rowKey24/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey24/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey24/memo:detail/1555078771906/Put/vlen=8/seqid=0}
8: keyvalues={rowKey25/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey25/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey25/memo:detail/1555078771906/Put/vlen=8/seqid=0}
9: keyvalues={rowKey26/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey26/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey26/memo:detail/1555078771906/Put/vlen=8/seqid=0}
start row: rowKey26\x00
0: keyvalues={rowKey27/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey27/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey27/memo:detail/1555078771906/Put/vlen=8/seqid=0}
1: keyvalues={rowKey28/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey28/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey28/memo:detail/1555078771906/Put/vlen=8/seqid=0}
2: keyvalues={rowKey29/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey29/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey29/memo:detail/1555078771906/Put/vlen=8/seqid=0}
3: keyvalues={rowKey3/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey3/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey3/memo:detail/1555078771906/Put/vlen=7/seqid=0}
4: keyvalues={rowKey30/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey30/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey30/memo:detail/1555078771906/Put/vlen=8/seqid=0}
5: keyvalues={rowKey31/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey31/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey31/memo:detail/1555078771906/Put/vlen=8/seqid=0}
6: keyvalues={rowKey32/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey32/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey32/memo:detail/1555078771906/Put/vlen=8/seqid=0}
7: keyvalues={rowKey33/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey33/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey33/memo:detail/1555078771906/Put/vlen=8/seqid=0}
8: keyvalues={rowKey34/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey34/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey34/memo:detail/1555078771906/Put/vlen=8/seqid=0}
9: keyvalues={rowKey35/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey35/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey35/memo:detail/1555078771906/Put/vlen=8/seqid=0}
start row: rowKey35\x00
... ...省略其他數據

起始行加了空字節數據,形成了新的定位行(相當於真正的兩頁之間插入了一個虛擬行用於定位),解決了下一頁首行的問題。

新的api輸出:下一頁的定位行就是上一頁的末行,采用新的API設置首行的時候把inclusive設置成false(即不包含首行),那麽輸出結果就是正確的。

0: keyvalues={rowKey0/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey0/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey0/memo:detail/1555078771906/Put/vlen=7/seqid=0}
1: keyvalues={rowKey1/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey1/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey1/memo:detail/1555078771906/Put/vlen=7/seqid=0}
2: keyvalues={rowKey10/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey10/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey10/memo:detail/1555078771906/Put/vlen=8/seqid=0}
3: keyvalues={rowKey11/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey11/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey11/memo:detail/1555078771906/Put/vlen=8/seqid=0}
4: keyvalues={rowKey12/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey12/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey12/memo:detail/1555078771906/Put/vlen=8/seqid=0}
5: keyvalues={rowKey13/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey13/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey13/memo:detail/1555078771906/Put/vlen=8/seqid=0}
6: keyvalues={rowKey14/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey14/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey14/memo:detail/1555078771906/Put/vlen=8/seqid=0}
7: keyvalues={rowKey15/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey15/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey15/memo:detail/1555078771906/Put/vlen=8/seqid=0}
8: keyvalues={rowKey16/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey16/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey16/memo:detail/1555078771906/Put/vlen=8/seqid=0}
9: keyvalues={rowKey17/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey17/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey17/memo:detail/1555078771906/Put/vlen=8/seqid=0}
start row: rowKey17
0: keyvalues={rowKey18/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey18/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey18/memo:detail/1555078771906/Put/vlen=8/seqid=0}
1: keyvalues={rowKey19/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey19/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey19/memo:detail/1555078771906/Put/vlen=8/seqid=0}
2: keyvalues={rowKey2/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey2/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey2/memo:detail/1555078771906/Put/vlen=7/seqid=0}
3: keyvalues={rowKey20/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey20/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey20/memo:detail/1555078771906/Put/vlen=8/seqid=0}
4: keyvalues={rowKey21/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey21/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey21/memo:detail/1555078771906/Put/vlen=8/seqid=0}
5: keyvalues={rowKey22/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey22/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey22/memo:detail/1555078771906/Put/vlen=8/seqid=0}
6: keyvalues={rowKey23/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey23/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey23/memo:detail/1555078771906/Put/vlen=8/seqid=0}
7: keyvalues={rowKey24/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey24/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey24/memo:detail/1555078771906/Put/vlen=8/seqid=0}
8: keyvalues={rowKey25/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey25/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey25/memo:detail/1555078771906/Put/vlen=8/seqid=0}
9: keyvalues={rowKey26/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey26/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey26/memo:detail/1555078771906/Put/vlen=8/seqid=0}
start row: rowKey26
0: keyvalues={rowKey27/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey27/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey27/memo:detail/1555078771906/Put/vlen=8/seqid=0}
1: keyvalues={rowKey28/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey28/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey28/memo:detail/1555078771906/Put/vlen=8/seqid=0}
2: keyvalues={rowKey29/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey29/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey29/memo:detail/1555078771906/Put/vlen=8/seqid=0}
3: keyvalues={rowKey3/ex:addr/1555078771906/Put/vlen=7/seqid=0, rowKey3/info:username/1555078771906/Put/vlen=5/seqid=0, rowKey3/memo:detail/1555078771906/Put/vlen=7/seqid=0}
4: keyvalues={rowKey30/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey30/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey30/memo:detail/1555078771906/Put/vlen=8/seqid=0}
5: keyvalues={rowKey31/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey31/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey31/memo:detail/1555078771906/Put/vlen=8/seqid=0}
6: keyvalues={rowKey32/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey32/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey32/memo:detail/1555078771906/Put/vlen=8/seqid=0}
7: keyvalues={rowKey33/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey33/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey33/memo:detail/1555078771906/Put/vlen=8/seqid=0}
8: keyvalues={rowKey34/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey34/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey34/memo:detail/1555078771906/Put/vlen=8/seqid=0}
9: keyvalues={rowKey35/ex:addr/1555078771906/Put/vlen=8/seqid=0, rowKey35/info:username/1555078771906/Put/vlen=6/seqid=0, rowKey35/memo:detail/1555078771906/Put/vlen=8/seqid=0}
start row: rowKey35
... ...其他省略

hbase 分頁過濾(新老API的差別)