PG高速緩衝區(Cache)——catcache構築SysCache資料結構關係的骨幹
catcache程式碼位於src/backend/utils/cache/catcache.c,包含了對SysCache結構體的初始化和資料結構之間指標關係的連結以及操作。最重要的是提供了兩個函式:精確匹配SearchCatCache和部分匹配SearchCatcacheList。提供的靜態函式如下,這裡不進行詳細分析了。
1 static uint32 CatalogCacheComputeHashValue(CatCache *cache, int nkeys, ScanKey cur_skey); 2 static uint32 CatalogCacheComputeTupleHashValue(CatCache *cache, HeapTuple tuple);3 4 #ifdef CATCACHE_STATS 5 static void CatCachePrintStats(int code, Datum arg); 6 #endif 7 static void CatCacheRemoveCTup(CatCache *cache, CatCTup *ct); 8 static void CatCacheRemoveCList(CatCache *cache, CatCList *cl); 9 static void CatalogCacheInitializeCache(CatCache *cache); 10 static CatCTup *CatalogCacheCreateEntry(CatCache *cache, HeapTuple ntp, uint32 hashValue, Index hashIndex, boolnegative); 11 static HeapTuple build_dummy_tuple(CatCache *cache, int nkeys, ScanKey skeys);
首先是提供兩個初始化函式InitCatCache、InitCatCachePhase2、CatalogCacheInitializeCache。和建立記憶體上下文相關的函式CreateCacheMemoryContext。一些工具函式有CatalogCacheIdInvalidate、CatalogCacheCreateEntry、ResetCatalogCache、ResetCatalogCaches、CatalogCacheFlushRelation、ReleaseCatCache、build_dummy_tuple、ReleaseCatCacheList、PrepareToInvalidateCacheTuple
在CatCache中查詢元組有兩種方式:精確匹配SearchCatCache和部分匹配SearchCatcacheList。前者用於給定CatCache所需的所有鍵值,並返回CatCache中能完全匹配這個鍵值的元組;而後者只需要給出部分鍵值,並將部分匹配的元組以一個CatCList的方式返回。
1 HeapTuple SearchCatCache(CatCache *cache, Datum v1, Datum v2, Datum v3, Datum v4) { 2 ScanKeyData cur_skey[4]; 3 uint32 hashValue; 4 Index hashIndex; 5 Dlelem *elt; 6 CatCTup *ct; 7 Relation relation; 8 SysScanDesc scandesc; 9 HeapTuple ntp; 10 11 if (cache->cc_tupdesc == NULL) 12 CatalogCacheInitializeCache(cache); 13 14 memcpy(cur_skey, cache->cc_skey, sizeof(cur_skey)); 15 cur_skey[0].sk_argument = v1; 16 cur_skey[1].sk_argument = v2; 17 cur_skey[2].sk_argument = v3; 18 cur_skey[3].sk_argument = v4; 19 20 hashValue = CatalogCacheComputeHashValue(cache, cache->cc_nkeys, cur_skey); 21 hashIndex = HASH_INDEX(hashValue, cache->cc_nbuckets); 22 23 for (elt = DLGetHead(&cache->cc_bucket[hashIndex]); elt; 24 elt = DLGetSucc(elt)) { 25 bool res; 26 ct = (CatCTup *) DLE_VAL(elt); 27 if (ct->dead) 28 continue; /* ignore dead entries */ 29 if (ct->hash_value != hashValue) 30 continue; 31 HeapKeyTest(&ct->tuple, cache->cc_tupdesc, cache->cc_nkeys, cur_skey, res); 32 if (!res) 33 continue; 34 35 DLMoveToFront(&ct->cache_elem); 36 if (!ct->negative) { ResourceOwnerEnlargeCatCacheRefs(CurrentResourceOwner); 37 ct->refcount++; 38 ResourceOwnerRememberCatCacheRef(CurrentResourceOwner, &ct->tuple); 39 return &ct->tuple; 40 }else{ 41 return NULL; 42 } 43 } 44 45 relation = heap_open(cache->cc_reloid, AccessShareLock); 46 scandesc = systable_beginscan(relation, cache->cc_indexoid, IndexScanOK(cache, cur_skey), SnapshotNow, cache->cc_nkeys, cur_skey); 47 ct = NULL; 48 while (HeapTupleIsValid(ntp = systable_getnext(scandesc))) 49 { 50 ct = CatalogCacheCreateEntry(cache, ntp, 51 hashValue, hashIndex, 52 false); 53 /* immediately set the refcount to 1 */ 54 ResourceOwnerEnlargeCatCacheRefs(CurrentResourceOwner); 55 ct->refcount++; 56 ResourceOwnerRememberCatCacheRef(CurrentResourceOwner, &ct->tuple); 57 break; /* assume only one match */ 58 } 59 systable_endscan(scandesc); 60 heap_close(relation, AccessShareLock); 61 62 if (ct == NULL) 63 { 64 if (IsBootstrapProcessingMode()) 65 return NULL; 66 ntp = build_dummy_tuple(cache, cache->cc_nkeys, cur_skey); 67 ct = CatalogCacheCreateEntry(cache, ntp, 68 hashValue, hashIndex, 69 true); 70 heap_freetuple(ntp); 71 return NULL; 72 } 73 return &ct->tuple; 74 }
v1\v2\v3\v4都用於查詢元組的鍵值,分別對應該Cache中記錄的元組搜尋鍵。在第一次進入該函式時,由於系統表還沒有載入到SysCache的相關結構體中,需要呼叫一次CatalogCacheInitializeCache(cache)。
1)將v1\v2\v3\v4設定到cur_skey陣列相應元素中,呼叫CatalogCacheComputeHashValue(cache, cache->cc_nkeys, cur_skey)計算雜湊值,並使用HASH_INDEX巨集計算雜湊桶的索引,按照該索引得到該CatCache在cc_bucket陣列中對應的Hash桶的下標。
2)遍歷Hash桶鏈找到滿足查詢需求的Dlelem,並將其結構體中dle_val屬性強制轉換為CatCTup型別,使用HeapKeyTest測試快取的tuple是否匹配輸入的鍵。如果找到,使用DLMoveToFront將該元組放到Hash桶的首位。如果是正元組,refcount和cc_hits都加1,返回元組。如果為負元組,cc_neg_hits加1,返回NULL。
3)如果沒有找到,說明SysCache中沒有快取相應的元組,需要進一步對物理系統表進行掃描,以確認要查詢的元組是確實不存在還是沒有快取在CatCache中。如果掃描物理系統表能夠找到滿足條件的餘罪女主,則需要將元組包裝成Dlelem之後加入到其對應的Hash桶內連結串列頭部,並返回元組,如果在物理系統表中找不到要查詢的元組,則說明該元組確實不存在,此時構建一個只有鍵值但沒有實際元組的負元組,並將它包裝好加入到Hash桶內連結串列頭部。
掃描物理系統表的程式碼如下:
1 relation = heap_open(cache->cc_reloid, AccessShareLock); 2 scandesc = systable_beginscan(relation, cache->cc_indexoid,IndexScanOK(cache, cur_skey), SnapshotNow, cache->cc_nkeys, cur_skey); 3 ct = NULL; 4 while (HeapTupleIsValid(ntp = systable_getnext(scandesc))) 5 { 6 ct = CatalogCacheCreateEntry(cache, ntp, hashValue, hashIndex, false); 7 /* immediately set the refcount to 1 */ ResourceOwnerEnlargeCatCacheRefs(CurrentResourceOwner); 8 ct->refcount++; 9 ResourceOwnerRememberCatCacheRef(CurrentResourceOwner, &ct->tuple); 10 break; /* assume only one match */ 11 } 12 systable_endscan(scandesc); 13 heap_close(relation, AccessShareLock);
部分匹配使用函式SearchCatcacheList,該函式產生一個CatCList結構,其中以連結串列的方式存放了在Cache中找到的元組。CatCList中的tuple欄位記錄的是一個負元組,它僅僅用來存放該CatCList所用到的鍵值,沒有其他使用者。CatCList中所包含的元組實際通過members欄位表示的變長資料來記錄,該陣列的實際長度由n_members欄位記錄。SearchCatCacheList函式也會先計算查詢鍵的Hash值,不過該函式首先會在CatCache的cc_lists欄位中記錄的CatCList連結串列中查詢以前是否快取了該查詢鍵的結果,該查詢過程將使用CatCList中tuple欄位指向的元組與查詢鍵進行Hash值比較。如果能夠找到匹配該Hash值的CatCList,則cc_lhits加1並將該CatCList移到cc_lists所指向連結串列的頭部,然後返回找到的CatCList。如果在CatCache中找不到CatCList,則掃描物理系統表並構建相應的CatCList並將它加入到cc_lists所指向連結串列的頭部。
1 CatCList *SearchCatCacheList(CatCache *cache, int nkeys, Datum v1, Datum v2, Datum v3, Datum v4) { 2 ScanKeyData cur_skey[4]; 3 uint32 lHashValue; 4 Dlelem *elt; 5 CatCList *cl; 6 CatCTup *ct; 7 List *volatile ctlist; 8 ListCell *ctlist_item; 9 int nmembers; 10 bool ordered; 11 HeapTuple ntp; 12 MemoryContext oldcxt; 13 int i; 14 15 /* 16 * one-time startup overhead for each cache 17 */ 18 if (cache->cc_tupdesc == NULL) 19 CatalogCacheInitializeCache(cache); 20 #ifdef CATCACHE_STATS 21 cache->cc_lsearches++; 22 #endif 23 /* 24 * initialize the search key information 25 */ 26 memcpy(cur_skey, cache->cc_skey, sizeof(cur_skey)); 27 cur_skey[0].sk_argument = v1; 28 cur_skey[1].sk_argument = v2; 29 cur_skey[2].sk_argument = v3; 30 cur_skey[3].sk_argument = v4; 31 lHashValue = CatalogCacheComputeHashValue(cache, nkeys, cur_skey); 32 33 /* scan the items until we find a match or exhaust our list */ 34 for (elt = DLGetHead(&cache->cc_lists); 35 elt; 36 elt = DLGetSucc(elt)) { 37 bool res; 38 cl = (CatCList *) DLE_VAL(elt); 39 40 if (cl->dead) 41 continue; /* ignore dead entries */ 42 43 if (cl->hash_value != lHashValue) 44 continue; /* quickly skip entry if wrong hash val */ 45 46 /* 47 * see if the cached list matches our key. 48 */ 49 if (cl->nkeys != nkeys) 50 continue; 51 HeapKeyTest(&cl->tuple, 52 cache->cc_tupdesc, 53 nkeys, 54 cur_skey, 55 res); 56 if (!res) 57 continue; 58 59 /* 60 * We found a matching list. Move the list to the front of the 61 * cache's list-of-lists, to speed subsequent searches. (We do not 62 * move the members to the fronts of their hashbucket lists, however, 63 * since there's no point in that unless they are searched for 64 * individually.) 65 */ 66 DLMoveToFront(&cl->cache_elem); 67 68 /* Bump the list's refcount and return it */ 69 ResourceOwnerEnlargeCatCacheListRefs(CurrentResourceOwner); 70 cl->refcount++; 71 ResourceOwnerRememberCatCacheListRef(CurrentResourceOwner, cl); 72 73 CACHE2_elog(DEBUG2, "SearchCatCacheList(%s): found list", 74 cache->cc_relname); 75 76 #ifdef CATCACHE_STATS 77 cache->cc_lhits++; 78 #endif 79 80 return cl; 81 } 82 83 /* 84 * List was not found in cache, so we have to build it by reading the 85 * relation. For each matching tuple found in the relation, use an 86 * existing cache entry if possible, else build a new one. 87 * 88 * We have to bump the member refcounts temporarily to ensure they won't 89 * get dropped from the cache while loading other members. We use a PG_TRY 90 * block to ensure we can undo those refcounts if we get an error before 91 * we finish constructing the CatCList. 92 */ 93 ResourceOwnerEnlargeCatCacheListRefs(CurrentResourceOwner); 94 95 ctlist = NIL; 96 97 PG_TRY(); 98 { 99 Relation relation; 100 SysScanDesc scandesc; 101 102 relation = heap_open(cache->cc_reloid, AccessShareLock); 103 104 scandesc = systable_beginscan(relation, 105 cache->cc_indexoid, 106 true, 107 SnapshotNow, 108 nkeys, 109 cur_skey); 110 111 /* The list will be ordered iff we are doing an index scan */ 112 ordered = (scandesc->irel != NULL); 113 114 while (HeapTupleIsValid(ntp = systable_getnext(scandesc))) 115 { 116 uint32 hashValue; 117 Index hashIndex; 118 119 /* 120 * See if there's an entry for this tuple already. 121 */ 122 ct = NULL; 123 hashValue = CatalogCacheComputeTupleHashValue(cache, ntp); 124 hashIndex = HASH_INDEX(hashValue, cache->cc_nbuckets); 125 126 for (elt = DLGetHead(&cache->cc_bucket[hashIndex]); 127 elt; 128 elt = DLGetSucc(elt)) 129 { 130 ct = (CatCTup *) DLE_VAL(elt); 131 132 if (ct->dead || ct->negative) 133 continue; /* ignore dead and negative entries */ 134 135 if (ct->hash_value != hashValue) 136 continue; /* quickly skip entry if wrong hash val */ 137 138 if (!ItemPointerEquals(&(ct->tuple.t_self), &(ntp->t_self))) 139 continue; /* not same tuple */ 140 141 /* 142 * Found a match, but can't use it if it belongs to another 143 * list already 144 */ 145 if (ct->c_list) 146 continue; 147 148 break; /* A-OK */ 149 } 150 151 if (elt == NULL) 152 { 153 /* We didn't find a usable entry, so make a new one */ 154 ct = CatalogCacheCreateEntry(cache, ntp, 155 hashValue, hashIndex, 156 false); 157 } 158 159 /* Careful here: add entry to ctlist, then bump its refcount */ 160 /* This way leaves state correct if lappend runs out of memory */ 161 ctlist = lappend(ctlist, ct); 162 ct->refcount++; 163 } 164 165 systable_endscan(scandesc); 166 167 heap_close(relation, AccessShareLock); 168 169 /* 170 * Now we can build the CatCList entry. First we need a dummy tuple 171 * containing the key values... 172 */ 173 ntp = build_dummy_tuple(cache, nkeys, cur_skey); 174 oldcxt = MemoryContextSwitchTo(CacheMemoryContext); 175 nmembers = list_length(ctlist); 176 cl = (CatCList *) 177 palloc(sizeof(CatCList) + nmembers * sizeof(CatCTup *)); 178 heap_copytuple_with_tuple(ntp, &cl->tuple); 179 MemoryContextSwitchTo(oldcxt); 180 heap_freetuple(ntp); 181 182 /* 183 * We are now past the last thing that could trigger an elog before we 184 * have finished building the CatCList and remembering it in the 185 * resource owner. So it's OK to fall out of the PG_TRY, and indeed 186 * we'd better do so before we start marking the members as belonging 187 * to the list. 188 */ 189 190 } 191 PG_CATCH(); 192 { 193 foreach(ctlist_item, ctlist) 194 { 195 ct = (CatCTup *) lfirst(ctlist_item); 196 Assert(ct->c_list == NULL); 197 Assert(ct->refcount > 0); 198 ct->refcount--; 199 if ( 200 #ifndef CATCACHE_FORCE_RELEASE 201 ct->dead && 202 #endif 203 ct->refcount == 0 && 204 (ct->c_list == NULL || ct->c_list->refcount == 0)) 205 CatCacheRemoveCTup(cache, ct); 206 } 207 208 PG_RE_THROW(); 209 } 210 PG_END_TRY(); 211 212 cl->cl_magic = CL_MAGIC; 213 cl->my_cache = cache; 214 DLInitElem(&cl->cache_elem, cl); 215 cl->refcount = 0; /* for the moment */ 216 cl->dead = false; 217 cl->ordered = ordered; 218 cl->nkeys = nkeys; 219 cl->hash_value = lHashValue; 220 cl->n_members = nmembers; 221 222 i = 0; 223 foreach(ctlist_item, ctlist) 224 { 225 cl->members[i++] = ct = (CatCTup *) lfirst(ctlist_item); 226 Assert(ct->c_list == NULL); 227 ct->c_list = cl; 228 /* release the temporary refcount on the member */ 229 Assert(ct->refcount > 0); 230 ct->refcount--; 231 /* mark list dead if any members already dead */ 232 if (ct->dead) 233 cl->dead = true; 234 } 235 Assert(i == nmembers); 236 237 DLAddHead(&cache->cc_lists, &cl->cache_elem); 238 239 /* Finally, bump the list's refcount and return it */ 240 cl->refcount++; 241 ResourceOwnerRememberCatCacheListRef(CurrentResourceOwner, cl); 242 243 CACHE3_elog(DEBUG2, "SearchCatCacheList(%s): made list of %d members", 244 cache->cc_relname, nmembers); 245 246 return cl; 247 }
AtEOXact_CatCache用於在main事務(commit或abort)結束後清理catcache
1 void 2 AtEOXact_CatCache(bool isCommit) 3 { 4 #ifdef USE_ASSERT_CHECKING 5 if (assert_enabled) 6 { 7 CatCache *ccp; 8 9 for (ccp = CacheHdr->ch_caches; ccp; ccp = ccp->cc_next) 10 { 11 Dlelem *elt; 12 int i; 13 14 /* Check CatCLists */ 15 for (elt = DLGetHead(&ccp->cc_lists); elt; elt = DLGetSucc(elt)) 16 { 17 CatCList *cl = (CatCList *) DLE_VAL(elt); 18 19 Assert(cl->cl_magic == CL_MAGIC); 20 Assert(cl->refcount == 0); 21 Assert(!cl->dead); 22 } 23 24 /* Check individual tuples */ 25 for (i = 0; i < ccp->cc_nbuckets; i++) 26 { 27 for (elt = DLGetHead(&ccp->cc_bucket[i]); 28 elt; 29 elt = DLGetSucc(elt)) 30 { 31 CatCTup *ct = (CatCTup *) DLE_VAL(elt); 32 33 Assert(ct->ct_magic == CT_MAGIC); 34 Assert(ct->refcount == 0); 35 Assert(!ct->dead); 36 } 37 } 38 } 39 } 40 #endif 41 }