redis源码笔记－dict.c

这篇blog介绍dict的实现。

dict.c

  1 #include "fmacros.h"
  2 
  3 #include <stdio.h>
  4 #include <stdlib.h>
  5 #include <string.h>
  6 #include <stdarg.h>
  7 #include <assert.h>
  8 #include <limits.h>
  9 #include <sys/time.h>
 10 #include <ctype.h>
 11 
 12 #include "dict.h"
 13 #include "zmalloc.h"
 14 
 15 /* Using dictEnableResize() / dictDisableResize() we make possible to
 16  * enable/disable resizing of the hash table as needed. This is very important
 17  * for Redis, as we use copy-on-write and don't want to move too much memory
 18  * around when there is a child performing saving operations.
 19  *
 20  * Note that even when dict_can_resize is set to 0, not all resizes are
 21  * prevented: an hash table is still allowed to grow if the ratio between
 22  * the number of elements and the buckets > dict_force_resize_ratio. */
     注释已经说的很清楚了。redis有些后台进程会做一些工作，比如rdb save、aof rewrite之类的，所以如果在后台进程运行过程中，将允许对hash table进行resize操作的话，会造成大量的内存拷贝（默认是copy-on-write，只在内存有变化时才拷贝）。注释中说，也没有完全避免，如果存的entry过多，比如存的entry是slot数的五倍，则也会强制进行resize
 23 static int dict_can_resize = 1;
 24 static unsigned int dict_force_resize_ratio = 5;
 25 
 26 /* -------------------------- private prototypes ---------------------------- */
 27 
 28 static int _dictExpandIfNeeded(dict *ht);     //如果需要的话，将hash表进行扩展
 29 static unsigned long _dictNextPower(unsigned long size);  
 30 static int _dictKeyIndex(dict *ht, const void *key);
 31 static int _dictInit(dict *ht, dictType *type, void *privDataPtr);
 32 //这4个函数的具体含义，到后边实现以及用到的时候再讨论
 33 /* -------------------------- hash functions -------------------------------- */
 34 
 35 /* Thomas Wang's 32 bit Mix Function */
 36 unsigned int dictIntHashFunction(unsigned int key)
 37 {
 38     key += ~(key << 15);
 39     key ^=  (key >> 10);
 40     key +=  (key << 3);
 41     key ^=  (key >> 6);
 42     key += ~(key << 11);
 43     key ^=  (key >> 16);
 44     return key;
 45 }
 46 
 47 /* Identity hash function for integer keys */
 48 unsigned int dictIdentityHashFunction(unsigned int key)
 49 {
 50     return key;
 51 }
 52 
 53 /* Generic hash function (a popular one from Bernstein).
 54  * I tested a few and this was the best. */
 55 unsigned int dictGenHashFunction(const unsigned char *buf, int len) {
 56     unsigned int hash = 5381;
 57 
 58     while (len--)
 59         hash = ((hash << 5) + hash) + (*buf++); /* hash * 33 + c */
 60     return hash;
 61 }
 62 
 63 /* And a case insensitive version */
 64 unsigned int dictGenCaseHashFunction(const unsigned char *buf, int len) {
 65     unsigned int hash = 5381;
 66 
 67     while (len--)
 68         hash = ((hash << 5) + hash) + (tolower(*buf++)); /* hash * 33 + c */
 69     return hash;
 70 }
 71 //给出针对整数和字符串的hash函数，因为redis中做dict key的只有这两种类型 
 72 /* ----------------------------- API implementation ------------------------- */
 73 
 74 /* Reset an hashtable already initialized with ht_init().
 75  * NOTE: This function should only called by ht_destroy(). */ //Note: and _dictInit
 76 static void _dictReset(dictht *ht)
 77 {
 78     ht->table = NULL;
 79     ht->size = 0;
 80     ht->sizemask = 0;
 81     ht->used = 0;
 82 }
 83 
 84 /* Create a new hash table */
 85 dict *dictCreate(dictType *type,
 86         void *privDataPtr)
 87 {
 88     dict *d = zmalloc(sizeof(*d));
 89 
 90     _dictInit(d,type,privDataPtr);
 91     return d;
 92 }
 93 //解释下参数中的privDataPtr，它是dict struct的一个成员，然后分别在dictType的keyDup函数、valDup函数、keyCompare函数、keyDestructor函数、valDestructor函数中作为第一个参数使用。dict的对外API通过dictCreate函数对其进行初始化。我在看dict.h时也很疑惑这个接口设计。这里揭底，在redis当前版本对dictCreate的所有调用中（networking.c一次、object.c两次、redis.c七次、t_hash.c一次、t_set.c一次、t_zset.c一次），此参数均被赋为了NULL，可能是历史原因或是考虑后续兼容性？求大牛指点。anyway，现在可以无视之了。
 94 /* Initialize the hash table */
 95 int _dictInit(dict *d, dictType *type,
 96         void *privDataPtr)
 97 {
 98     _dictReset(&d->ht[0]);
 99     _dictReset(&d->ht[1]);
100     d->type = type;
101     d->privdata = privDataPtr;
102     d->rehashidx = -1;
103     d->iterators = 0;
104     return DICT_OK;
105 }
106 
107 /* Resize the table to the minimal size that contains all the elements,
108  * but with the invariant of a USER/BUCKETS ratio near to <= 1 */
109 int dictResize(dict *d)
110 {
111     int minimal;
112 
113     if (!dict_can_resize || dictIsRehashing(d)) return DICT_ERR;
114     minimal = d->ht[0].used;
115     if (minimal < DICT_HT_INITIAL_SIZE)
116         minimal = DICT_HT_INITIAL_SIZE;
117     return dictExpand(d, minimal);
118 }
119 
120 /* Expand or create the hashtable */
121 int dictExpand(dict *d, unsigned long size)
122 {
123     dictht n; /* the new hashtable */
124     unsigned long realsize = _dictNextPower(size);  //dict的size是2的倍数，所以这里记录了真是的记录长度，需要比size长
125 
126     /* the size is invalid if it is smaller than the number of
127      * elements already inside the hashtable */
128     if (dictIsRehashing(d) || d->ht[0].used > size)
129         return DICT_ERR;
130 
131     /* Allocate the new hashtable and initialize all pointers to NULL */
132     n.size = realsize;
133     n.sizemask = realsize-1;
134     n.table = zcalloc(realsize*sizeof(dictEntry*));
135     n.used = 0;
136 
137     /* Is this the first initialization? If so it's not really a rehashing
138      * we just set the first hash table so that it can accept keys. */
139     if (d->ht[0].table == NULL) {
140         d->ht[0] = n;
141         return DICT_OK;
142     }
143 
144     /* Prepare a second hash table for incremental rehashing */
145     d->ht[1] = n;
146     d->rehashidx = 0;  //只有在第二次调用dictExpand时，才是rehash的前奏
147     return DICT_OK;
148 }
149 
150 /* Performs N steps of incremental rehashing. Returns 1 if there are still
151  * keys to move from the old to the new hash table, otherwise 0 is returned.
152  * Note that a rehashing step consists in moving a bucket (that may have more
153  * thank one key as we use chaining) from the old to the new hash table. */
154 int dictRehash(dict *d, int n) {
155     if (!dictIsRehashing(d)) return 0;
156 
157     while(n--) {
158         dictEntry *de, *nextde;
159 
160         /* Check if we already rehashed the whole table... */
161         if (d->ht[0].used == 0) {
162             zfree(d->ht[0].table);
163             d->ht[0] = d->ht[1];
164             _dictReset(&d->ht[1]);
165             d->rehashidx = -1;
166             return 0;
167         }
168 
169         /* Note that rehashidx can't overflow as we are sure there are more
170          * elements because ht[0].used != 0 */
171         while(d->ht[0].table[d->rehashidx] == NULL) d->rehashidx++;
172         de = d->ht[0].table[d->rehashidx];
173         /* Move all the keys in this bucket from the old to the new hash HT */
174         while(de) {
175             unsigned int h;
176 
177             nextde = de->next;
178             /* Get the index in the new hash table */
179             h = dictHashKey(d, de->key) & d->ht[1].sizemask;
180             de->next = d->ht[1].table[h];
181             d->ht[1].table[h] = de;
182             d->ht[0].used--;
183             d->ht[1].used++;
184             de = nextde;
185         }
186         d->ht[0].table[d->rehashidx] = NULL;
187         d->rehashidx++;
188     }
189     return 1;
190 }
191 
192 long long timeInMilliseconds(void) {
193     struct timeval tv;
194 
195     gettimeofday(&tv,NULL);
196     return (((long long)tv.tv_sec)*1000)+(tv.tv_usec/1000);
197 }     //时间单位是ms
198 
199 /* Rehash for an amount of time between ms milliseconds and ms+1 milliseconds */
200 int dictRehashMilliseconds(dict *d, int ms) {
201     long long start = timeInMilliseconds();
202     int rehashes = 0;
203 
204     while(dictRehash(d,100)) {
205         rehashes += 100;
206         if (timeInMilliseconds()-start > ms) break;
207     }
208     return rehashes;
209 } //返回值是rehash数目，为100的整数倍，传进参数的ms只是个下限，并不严格限制
210 
211 /* This function performs just a step of rehashing, and only if there are
212  * no safe iterators bound to our hash table. When we have iterators in the
213  * middle of a rehashing we can't mess with the two hash tables otherwise
214  * some element can be missed or duplicated.
215  *
216  * This function is called by common lookup or update operations in the
217  * dictionary so that the hash table automatically migrates from H1 to H2
218  * while it is actively used. */
219 static void _dictRehashStep(dict *d) {
220     if (d->iterators == 0) dictRehash(d,1);
221 }  //在dict上存在遍历用的iterator时，不会尽心rehash操作，否则遍历将会丢数据或出现重复。这个函数是在lookup和update时调用，使得rehash的过程伴随着针对dict操作的过程慢慢完成
222 
223 /* Add an element to the target hash table */
224 int dictAdd(dict *d, void *key, void *val)
225 {
226     int index;
227     dictEntry *entry;
228     dictht *ht;
229 
230     if (dictIsRehashing(d)) _dictRehashStep(d);
231 
232     /* Get the index of the new element, or -1 if
233      * the element already exists. */
234     if ((index = _dictKeyIndex(d, key)) == -1)
235         return DICT_ERR;
236 
237     /* Allocates the memory and stores key */
238     ht = dictIsRehashing(d) ? &d->ht[1] : &d->ht[0]; //如果dict正在进行rehash，则将新来的数据存在d->ht[1]上，因为d->ht[0]是“旧表”,不再接受新的数据
239     entry = zmalloc(sizeof(*entry));
240     entry->next = ht->table[index];
241     ht->table[index] = entry;
242     ht->used++;
243 
244     /* Set the hash entry fields. */
245     dictSetHashKey(d, entry, key);
246     dictSetHashVal(d, entry, val);
247     return DICT_OK;
248 }
249 
250 /* Add an element, discarding the old if the key already exists.
251  * Return 1 if the key was added from scratch, 0 if there was already an
252  * element with such key and dictReplace() just performed a value update
253  * operation. */
254 int dictReplace(dict *d, void *key, void *val)
255 {
256     dictEntry *entry, auxentry;
257 
258     /* Try to add the element. If the key
259      * does not exists dictAdd will suceed. */
260     if (dictAdd(d, key, val) == DICT_OK)
261         return 1;
262     /* It already exists, get the entry */
263     entry = dictFind(d, key);
264     /* Free the old value and set the new one */
265     /* Set the new value and free the old one. Note that it is important
266      * to do that in this order, as the value may just be exactly the same
267      * as the previous one. In this context, think to reference counting,
268      * you want to increment (set), and then decrement (free), and not the
269      * reverse. */
270     auxentry = *entry;
271     dictSetHashVal(d, entry, val);
272     dictFreeEntryVal(d, &auxentry);
273     return 0;
274 }
275 
276 /* Search and remove an element */
277 static int dictGenericDelete(dict *d, const void *key, int nofree)
278 {
279     unsigned int h, idx;
280     dictEntry *he, *prevHe;
281     int table;
282 
283     if (d->ht[0].size == 0) return DICT_ERR; /* d->ht[0].table is NULL */
284     if (dictIsRehashing(d)) _dictRehashStep(d);
285     h = dictHashKey(d, key);
286 
287     for (table = 0; table <= 1; table++) {
288         idx = h & d->ht[table].sizemask;
289         he = d->ht[table].table[idx];
290         prevHe = NULL;
291         while(he) {
292             if (dictCompareHashKeys(d, key, he->key)) {
293                 /* Unlink the element from the list */
294                 if (prevHe)
295                     prevHe->next = he->next;
296                 else
297                     d->ht[table].table[idx] = he->next;
298                 if (!nofree) {
299                     dictFreeEntryKey(d, he);
300                     dictFreeEntryVal(d, he);
301                 }
302                 zfree(he);
303                 d->ht[table].used--;
304                 return DICT_OK;
305             }
306             prevHe = he;
307             he = he->next;
308         }
309         if (!dictIsRehashing(d)) break;   //如果在查找的时刻，dict没有正在进行rehash操作，则此时dict->ht[1]未使用，所以只在dict->ht[0]进行查找操作就可以了
310     }
311     return DICT_ERR; /* not found */
312 }
313 
314 int dictDelete(dict *ht, const void *key) {
315     return dictGenericDelete(ht,key,0);
316 }
317 
318 int dictDeleteNoFree(dict *ht, const void *key) {
319     return dictGenericDelete(ht,key,1);  
320 } //。。。redis的src目录下没有任何一处调用这个不料理自己后事的delete函数
321 
322 /* Destroy an entire dictionary */
323 int _dictClear(dict *d, dictht *ht)  //clear的粒度是hash table而不是dict
324 {
325     unsigned long i;
326 
327     /* Free all the elements */
328     for (i = 0; i < ht->size && ht->used > 0; i++) {
329         dictEntry *he, *nextHe;
330 
331         if ((he = ht->table[i]) == NULL) continue;
332         while(he) {
333             nextHe = he->next;
334             dictFreeEntryKey(d, he);
335             dictFreeEntryVal(d, he);
336             zfree(he);
337             ht->used--;
338             he = nextHe;
339         }
340     }
341     /* Free the table and the allocated cache structure */
342     zfree(ht->table);
343     /* Re-initialize the table */
344     _dictReset(ht);
345     return DICT_OK; /* never fails */
346 }
347 
348 /* Clear & Release the hash table */
349 void dictRelease(dict *d)
350 {
351     _dictClear(d,&d->ht[0]);
352     _dictClear(d,&d->ht[1]);
353     zfree(d);
354 }   //release的粒度是dict，_dictClear只是一个内部接口
355 
356 dictEntry *dictFind(dict *d, const void *key)
357 {
358     dictEntry *he;
359     unsigned int h, idx, table;
360 
361     if (d->ht[0].size == 0) return NULL; /* We don't have a table at all */
362     if (dictIsRehashing(d)) _dictRehashStep(d);
363     h = dictHashKey(d, key);
364     for (table = 0; table <= 1; table++) {
365         idx = h & d->ht[table].sizemask;
366         he = d->ht[table].table[idx];
367         while(he) {
368             if (dictCompareHashKeys(d, key, he->key))     //为何要传dict参数，因为需要知道具体用什么compare函数作比较操作
369                 return he;
370             he = he->next;
371         }
372         if (!dictIsRehashing(d)) return NULL;  //没找到，返回NULL，函数就结束了
373     }
374     return NULL;
375 }
376 
377 void *dictFetchValue(dict *d, const void *key) {
378     dictEntry *he;
379 
380     he = dictFind(d,key);
381     return he ? dictGetEntryVal(he) : NULL;
382 }
383 
384 dictIterator *dictGetIterator(dict *d)
385 {
386     dictIterator *iter = zmalloc(sizeof(*iter));
387 
388     iter->d = d;
389     iter->table = 0;
390     iter->index = -1;
391     iter->safe = 0;
392     iter->entry = NULL;
393     iter->nextEntry = NULL;
394     return iter;
395 } //在调用GetIterator函数时，并没有将这个iter在对应的dict上进行注册
396 
397 dictIterator *dictGetSafeIterator(dict *d) {
398     dictIterator *i = dictGetIterator(d);
399 
400     i->safe = 1;
401     return i;
402 }
403 
404 dictEntry *dictNext(dictIterator *iter)
405 {
406     while (1) {
407         if (iter->entry == NULL) {
408             dictht *ht = &iter->d->ht[iter->table];   
409             if (iter->safe && iter->index == -1 && iter->table == 0)
410                 iter->d->iterators++;   //在对应的dict上进行注册
411             iter->index++;
412             if (iter->index >= (signed) ht->size) { //在对第一个ht遍历完成后，转到第二张表
413                 if (dictIsRehashing(iter->d) && iter->table == 0) {
414                     iter->table++;
415                     iter->index = 0;
416                     ht = &iter->d->ht[1];
417                 } else {
418                     break;
419                 }
420             }
421             iter->entry = ht->table[iter->index];
422         } else {
423             iter->entry = iter->nextEntry;
424         }
425         if (iter->entry) {
426             /* We need to save the 'next' here, the iterator user
427              * may delete the entry we are returning. */
428             iter->nextEntry = iter->entry->next;
429             return iter->entry;
430         }
431     }
432     return NULL;
433 }
434 
435 void dictReleaseIterator(dictIterator *iter)
436 {
437     if (iter->safe && !(iter->index == -1 && iter->table == 0))
438         iter->d->iterators--;
439     zfree(iter);
440 }
441 
442 /* Return a random entry from the hash table. Useful to
443  * implement randomized algorithms */
444 dictEntry *dictGetRandomKey(dict *d)
445 {
446     dictEntry *he, *orighe;
447     unsigned int h;
448     int listlen, listele;
449 
450     if (dictSize(d) == 0) return NULL;
451     if (dictIsRehashing(d)) _dictRehashStep(d);
452     if (dictIsRehashing(d)) {
453         do {
454             h = random() % (d->ht[0].size+d->ht[1].size);
455             he = (h >= d->ht[0].size) ? d->ht[1].table[h - d->ht[0].size] :
456                                       d->ht[0].table[h];
457         } while(he == NULL);
458     } else {
459         do {
460             h = random() & d->ht[0].sizemask;
461             he = d->ht[0].table[h];
462         } while(he == NULL);
463     }
464 
465     /* Now we found a non empty bucket, but it is a linked
466      * list and we need to get a random element from the list.
467      * The only sane way to do so is counting the elements and
468      * select a random index. */
469     listlen = 0;
470     orighe = he;
471     while(he) {
472         he = he->next;
473         listlen++;
474     }
475     listele = random() % listlen;
476     he = orighe;
477     while(listele--) he = he->next;
478     return he;
479 }   //这个函数效率很低啊。
480 
481 /* ------------------------- private functions ------------------------------ */
482 
483 /* Expand the hash table if needed */
484 static int _dictExpandIfNeeded(dict *d)
485 {
486     /* Incremental rehashing already in progress. Return. */
487     if (dictIsRehashing(d)) return DICT_OK;
488 
489     /* If the hash table is empty expand it to the intial size. */
490     if (d->ht[0].size == 0) return dictExpand(d, DICT_HT_INITIAL_SIZE);
491 
492     /* If we reached the 1:1 ratio, and we are allowed to resize the hash
493      * table (global setting) or we should avoid it but the ratio between
494      * elements/buckets is over the "safe" threshold, we resize doubling
495      * the number of buckets. */
496     if (d->ht[0].used >= d->ht[0].size &&
497         (dict_can_resize ||
498          d->ht[0].used/d->ht[0].size > dict_force_resize_ratio))
499     {
500         return dictExpand(d, ((d->ht[0].size > d->ht[0].used) ?
501                                     d->ht[0].size : d->ht[0].used)*2);
502     }
503     return DICT_OK;
504 }
505 
506 /* Our hash table capability is a power of two */
507 static unsigned long _dictNextPower(unsigned long size)
508 {
509     unsigned long i = DICT_HT_INITIAL_SIZE;
510 
511     if (size >= LONG_MAX) return LONG_MAX;
512     while(1) {
513         if (i >= size)
514             return i;
515         i *= 2;
516     }
517 }
518 
519 /* Returns the index of a free slot that can be populated with
520  * an hash entry for the given 'key'.
521  * If the key already exists, -1 is returned.
522  *
523  * Note that if we are in the process of rehashing the hash table, the
524  * index is always returned in the context of the second (new) hash table. */
525 static int _dictKeyIndex(dict *d, const void *key)
526 {
527     unsigned int h, idx, table;
528     dictEntry *he;
529 
530     /* Expand the hashtable if needed */
531     if (_dictExpandIfNeeded(d) == DICT_ERR)
532         return -1;
533     /* Compute the key hash value */
534     h = dictHashKey(d, key);
535     for (table = 0; table <= 1; table++) {
536         idx = h & d->ht[table].sizemask;
537         /* Search if this slot does not already contain the given key */
538         he = d->ht[table].table[idx];
539         while(he) {
540             if (dictCompareHashKeys(d, key, he->key))
541                 return -1;
542             he = he->next;
543         }
544         if (!dictIsRehashing(d)) break;
545     }
546     return idx;
547 }
548 
549 void dictEmpty(dict *d) {
550     _dictClear(d,&d->ht[0]);
551     _dictClear(d,&d->ht[1]);
552     d->rehashidx = -1;
553     d->iterators = 0;
554 }
555 
556 #define DICT_STATS_VECTLEN 50
557 static void _dictPrintStatsHt(dictht *ht) {
558     unsigned long i, slots = 0, chainlen, maxchainlen = 0;
559     unsigned long totchainlen = 0;
560     unsigned long clvector[DICT_STATS_VECTLEN];  //这个clvector是统计hash table中长度为其下标的槽位的个数
561 
562     if (ht->used == 0) {
563         printf("No stats available for empty dictionaries\n");
564         return;
565     }
566 
567     for (i = 0; i < DICT_STATS_VECTLEN; i++) clvector[i] = 0;
568     for (i = 0; i < ht->size; i++) {
569         dictEntry *he;
570 
571         if (ht->table[i] == NULL) {
572             clvector[0]++;
573             continue;
574         }
575         slots++;
576         /* For each hash entry on this slot... */
577         chainlen = 0;
578         he = ht->table[i];
579         while(he) {
580             chainlen++;
581             he = he->next;
582         }
583         clvector[(chainlen < DICT_STATS_VECTLEN) ? chainlen : (DICT_STATS_VECTLEN-1)]++;
584         if (chainlen > maxchainlen) maxchainlen = chainlen;
585         totchainlen += chainlen;
586     }
587     printf("Hash table stats:\n");
588     printf(" table size: %ld\n", ht->size);
589     printf(" number of elements: %ld\n", ht->used);
590     printf(" different slots: %ld\n", slots);   //非空槽位数
591     printf(" max chain length: %ld\n", maxchainlen);
592     printf(" avg chain length (counted): %.02f\n", (float)totchainlen/slots);
593     printf(" avg chain length (computed): %.02f\n", (float)ht->used/slots);
594     printf(" Chain length distribution:\n");
595     for (i = 0; i < DICT_STATS_VECTLEN-1; i++) {
596         if (clvector[i] == 0) continue;
597         printf("   %s%ld: %ld (%.02f%%)\n",(i == DICT_STATS_VECTLEN-1)?">= ":"", i, clvector[i], ((float)clvector[i]/ht->size)*100);
598     }
599 }
600 
601 void dictPrintStats(dict *d) {
602     _dictPrintStatsHt(&d->ht[0]);
603     if (dictIsRehashing(d)) {
604         printf("-- Rehashing into ht[1]:\n");
605         _dictPrintStatsHt(&d->ht[1]);
606     }
607 }
608 
609 void dictEnableResize(void) {
610     dict_can_resize = 1;
611 }
612 
613 void dictDisableResize(void) {
614     dict_can_resize = 0;
615 }