1. 程式人生 > >phoenix二級索引源碼閱讀

phoenix二級索引源碼閱讀

ldr ima cati 多個 regions efault nan HERE byte

Phoenix二級索引建立源碼

Phoenix二級索引建立在hbasecoprocess功能,建立索引的時候使用是

二級索引建立過程,索引rowkey的構建是一個數據流,不停在後面追加,最後生成最終的rowkey形式

public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKeyPtr, byte[] regionStartKey, byte[] regionEndKey, long ts) {
public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKeyPtr, byte

[] regionStartKey, byte[] regionEndKey, long ts) {
ImmutableBytesWritable ptr = new ImmutableBytesWritable();

//判斷是否是構建本地索引,考慮兩個條件:1.本地索引是否開啟 2.startRK 是否傳進來了
boolean prependRegionStartKey = isLocalIndex && regionStartKey != null;
boolean isIndexSalted = !isLocalIndex && nIndexSaltBuckets

> 0;

//如果開啟本地索引,則在數據前面添加前綴,判斷startRK是否是region起始startRK,如果是則使用該region的EndRK
int prefixKeyLength =
prependRegionStartKey ? (regionStartKey.length != 0 ? regionStartKey.length
: regionEndKey.length) : 0;
TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(estimatedIndexRowKeyBytes

+ (prependRegionStartKey ? prefixKeyLength : 0));

// 構建數據流對象,對數據進行put
DataOutput output = new DataOutputStream(stream);

如果是本地索引,則在rowkey前加入startrowkey索引

// For local indexes, we must prepend the row key with the start region key
if (prependRegionStartKey) {
if (regionStartKey.length == 0) {

// 如果startRK為null,則其實使用的endRK
output.write(new byte[prefixKeyLength]);
} else {
output.write(regionStartKey);
}

}

判斷是否有加鹽,如果有,則增加一個標誌位,後面再更改這個標誌位

if (isIndexSalted) {
output.write(0); // will be set at end to index salt byte
}

如果在索引視圖id不為null,會在索引rowkey中加入視圖id

if (viewIndexId != null) {
output.write(viewIndexId);
}

判斷是否啟動多租戶,如果啟動多租戶的場景,添加多租戶信息;

if (isMultiTenant) {
dataRowKeySchema.next(ptr, dataPosOffset, maxRowKeyOffset);
output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
if (!dataRowKeySchema.getField(dataPosOffset).getDataType().isFixedWidth()) {
output.writeByte(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength()==0, dataRowKeySchema.getField(dataPosOffset)));
}
dataPosOffset++;
}

dataRowKeySchema是數據表的信息,忽略在視圖變量的中常量值,並標記出原表pk的rowkey的offset 和 length,方便後面定位數據表rowkey插入。

for (int i = dataPosOffset; i < dataRowKeySchema.getFieldCount(); i++) {
Boolean hasValue=dataRowKeySchema.next(ptr, i, maxRowKeyOffset);
// Ignore view constants from the data table, as these
// don‘t need to appear in the index (as they‘re the
// same for all rows in this index)
if (!viewConstantColumnBitSet.get(i)) {
int pos = rowKeyMetaData.getIndexPkPosition(i-dataPosOffset);
if (Boolean.TRUE.equals(hasValue)) {
dataRowKeyLocator[0][pos] = ptr.getOffset();
dataRowKeyLocator[1][pos] = ptr.getLength();
} else {
dataRowKeyLocator[0][pos] = 0;
dataRowKeyLocator[1][pos] = 0;
}
}
}

考慮索引的數據的順序,考慮索引的順序等

// 獲取表達式索引,表達式索引默認值都為1,未開啟的時候isNullAble為true

Iterator<Expression> expressionIterator = indexedExpressions.iterator();

// nIndexedColumns 的構成是索引列+主鍵 如果是組合索引,則循環多個索引列
for (int i = 0; i < nIndexedColumns; i++) {
PDataType dataColumnType;
boolean isNullable;
SortOrder dataSortOrder;

// dataPkPosition為-1則表示為表達式索引,否則為屬性索引
if (dataPkPosition[i] == EXPRESSION_NOT_PRESENT) {
Expression expression = expressionIterator.next();
dataColumnType = expression.getDataType();
dataSortOrder = expression.getSortOrder();
isNullable = expression.isNullable();
expression.evaluate(new ValueGetterTuple(valueGetter, ts), ptr);
}

// 主鍵pk 走這個分支
else {
Field field = dataRowKeySchema.getField(dataPkPosition[i]);
dataColumnType = field.getDataType();
ptr.set(rowKeyPtr.get(), dataRowKeyLocator[0][i], dataRowKeyLocator[1][i]);
dataSortOrder = field.getSortOrder();
isNullable = field.isNullable();
}

// 考慮列值的順序,考慮字節的比較,考慮索引列的順序

// 判斷查詢是否desc,默認為asc。
boolean isDataColumnInverted = dataSortOrder != SortOrder.ASC;

// 獲取索引列的的數據類型,詳情看後面getIndexColumnDataType函數
PDataType indexColumnType = IndexUtil.getIndexColumnDataType(isNullable, dataColumnType);

//根據數據列返回不同的datatype,判斷該列是否可比較。不可比較的列有decimal,varchar,boolean,Binary
boolean isBytesComparable = dataColumnType.isBytesComparableWith(indexColumnType);

// 獲取列是否是逆序的
boolean isIndexColumnDesc = descIndexColumnBitSet.get(i);
if (isBytesComparable && isDataColumnInverted == isIndexColumnDesc) {
output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
} else {
if (!isBytesComparable) {

// 讓不可比較的類型具有可比性
indexColumnType.coerceBytes(ptr, dataColumnType, dataSortOrder, SortOrder.getDefault());
}

// 按位取異或值,二進制數比較肯定是字典序,從最高位開始比較,直到遇到第一個不一樣的位,這個位上哪個數等於1哪個數就較大。
if (isDataColumnInverted != isIndexColumnDesc) {
writeInverted(ptr.get(), ptr.getOffset(), ptr.getLength(), output);
} else {
output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
}
}

// 判斷數據是不是一個固定長度的字段,如果不是根據數據的正序逆序添加一個標誌位
if (!indexColumnType.isFixedWidth()) {
output.writeByte(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, isIndexColumnDesc ? SortOrder.DESC : SortOrder.ASC));

}

}

//填充開始的加鹽部分的字節位,規則是根據數據做hash,然後再對nIndexSaltBuckets取余

if (isIndexSalted) {
// Set salt byte
byte saltByte = SaltingUtil.getSaltingByte(indexRowKey, SaltingUtil.NUM_SALTING_BYTES, length-SaltingUtil.NUM_SALTING_BYTES, nIndexSaltBuckets);
indexRowKey[0] = saltByte;
}

返回所有的生成的rowkey

return indexRowKey.length == length ? indexRowKey : Arrays.copyOf(indexRowKey, length);

根據數據列返回不同的datatype,判斷該列是否可比較。不可比較的列有decimal,varchar,boolean,Binary

// Since we cannot have nullable fixed length in a row key
// we need to translate to variable length. The verification that we have a valid index
// row key was already done, so here we just need to convert from one built-in type to
// another.
public static PDataType getIndexColumnDataType(boolean isNullable, PDataType dataType) {
if (dataType == null || !isNullable || !dataType.isFixedWidth()) {
return dataType;
}
// for fixed length numeric types and boolean
if (dataType.isCastableTo(PDecimal.INSTANCE)) {
return PDecimal.INSTANCE;
}
// for CHAR
if (dataType.isCoercibleTo(PVarchar.INSTANCE)) {
return PVarchar.INSTANCE;
}

if (PBinary.INSTANCE.equals(dataType)) {
return PVarbinary.INSTANCE;
}
throw new IllegalArgumentException("Unsupported non nullable type " + dataType);
}

讓數據有可比性

protected static int toBytes(BigDecimal v, byte[] result, final int offset, int length) {
// From scale to exponent byte (if BigDecimal is positive): (-(scale+(scale % 2 == 0 : 0 : 1)) / 2 + 65) | 0x80
// If scale % 2 is 1 (i.e. it‘s odd), then multiple last base-100 digit by 10
// For example: new BigDecimal(BigInteger.valueOf(1), -4);
// (byte)((-(-4+0) / 2 + 65) | 0x80) = -61
// From scale to exponent byte (if BigDecimal is negative): ~(-(scale+1)/2 + 65 + 128) & 0x7F
// For example: new BigDecimal(BigInteger.valueOf(1), 2);
// ~(-2/2 + 65 + 128) & 0x7F = 63

phoenix二級索引源碼閱讀