LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 #include "runtime.h" | 5 #include "runtime.h" |
6 #include "arch_GOARCH.h" | 6 #include "arch_GOARCH.h" |
7 #include "malloc.h" | 7 #include "malloc.h" |
8 #include "hashmap.h" | 8 #include "hashmap.h" |
9 #include "type.h" | 9 #include "type.h" |
10 #include "race.h" | 10 #include "race.h" |
(...skipping 108 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
119 }; | 119 }; |
120 | 120 |
121 // Macros for dereferencing indirect keys | 121 // Macros for dereferencing indirect keys |
122 #define IK(h, p) (((h)->flags & IndirectKey) != 0 ? *(byte**)(p) : (p)) | 122 #define IK(h, p) (((h)->flags & IndirectKey) != 0 ? *(byte**)(p) : (p)) |
123 #define IV(h, p) (((h)->flags & IndirectValue) != 0 ? *(byte**)(p) : (p)) | 123 #define IV(h, p) (((h)->flags & IndirectValue) != 0 ? *(byte**)(p) : (p)) |
124 | 124 |
125 enum | 125 enum |
126 { | 126 { |
127 docheck = 0, // check invariants before and after every op. Slow!!! | 127 docheck = 0, // check invariants before and after every op. Slow!!! |
128 debug = 0, // print every operation | 128 debug = 0, // print every operation |
| 129 checkgc = 0 || docheck, // check interaction of mallocgc() with the gar
bage collector |
129 }; | 130 }; |
130 static void | 131 static void |
131 check(MapType *t, Hmap *h) | 132 check(MapType *t, Hmap *h) |
132 { | 133 { |
133 uintptr bucket, oldbucket; | 134 uintptr bucket, oldbucket; |
134 Bucket *b; | 135 Bucket *b; |
135 uintptr i; | 136 uintptr i; |
136 uintptr hash; | 137 uintptr hash; |
137 uintgo cnt; | 138 uintgo cnt; |
138 uint8 top; | 139 uint8 top; |
(...skipping 107 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
246 if(sizeof(void*) == 4 && t->elem->align > 4) | 247 if(sizeof(void*) == 4 && t->elem->align > 4) |
247 runtime·throw("need padding in bucket (value)"); | 248 runtime·throw("need padding in bucket (value)"); |
248 | 249 |
249 // find size parameter which will hold the requested # of elements | 250 // find size parameter which will hold the requested # of elements |
250 B = 0; | 251 B = 0; |
251 while(hint > BUCKETSIZE && hint > LOAD * ((uintptr)1 << B)) | 252 while(hint > BUCKETSIZE && hint > LOAD * ((uintptr)1 << B)) |
252 B++; | 253 B++; |
253 | 254 |
254 // allocate initial hash table | 255 // allocate initial hash table |
255 // If hint is large zeroing this memory could take a while. | 256 // If hint is large zeroing this memory could take a while. |
256 » buckets = runtime·mallocgc(bucketsize << B, 0, 1, 0); | 257 » if(checkgc) mstats.next_gc = mstats.heap_alloc; |
257 » for(i = 0; i < (uintptr)1 << B; i++) { | 258 » if(B == 0) { |
258 » » b = (Bucket*)(buckets + i * bucketsize); | 259 » » // done lazily later. |
259 » » clearbucket(b); | 260 » » buckets = nil; |
| 261 » } else { |
| 262 » » buckets = runtime·mallocgc(bucketsize << B, 0, 1, 0); |
| 263 » » for(i = 0; i < (uintptr)1 << B; i++) { |
| 264 » » » b = (Bucket*)(buckets + i * bucketsize); |
| 265 » » » clearbucket(b); |
| 266 » » } |
260 } | 267 } |
261 | 268 |
262 // initialize Hmap | 269 // initialize Hmap |
263 // Note: we save all these stores to the end so gciter doesn't see | 270 // Note: we save all these stores to the end so gciter doesn't see |
264 // a partially initialized map. | 271 // a partially initialized map. |
265 h->count = 0; | 272 h->count = 0; |
266 h->B = B; | 273 h->B = B; |
267 h->flags = flags; | 274 h->flags = flags; |
268 h->keysize = keysize; | 275 h->keysize = keysize; |
269 h->valuesize = valuesize; | 276 h->valuesize = valuesize; |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
315 for(i = 0, k = b->data, v = k + h->keysize * BUCKETSIZE;
i < BUCKETSIZE; i++, k += h->keysize, v += h->valuesize) { | 322 for(i = 0, k = b->data, v = k + h->keysize * BUCKETSIZE;
i < BUCKETSIZE; i++, k += h->keysize, v += h->valuesize) { |
316 if(b->tophash[i] == 0) | 323 if(b->tophash[i] == 0) |
317 continue; | 324 continue; |
318 hash = h->hash0; | 325 hash = h->hash0; |
319 t->key->alg->hash(&hash, t->key->size, IK(h, k))
; | 326 t->key->alg->hash(&hash, t->key->size, IK(h, k))
; |
320 // NOTE: if key != key, then this hash could be
(and probably will be) | 327 // NOTE: if key != key, then this hash could be
(and probably will be) |
321 // entirely different from the old hash. We eff
ectively only update | 328 // entirely different from the old hash. We eff
ectively only update |
322 // the B'th bit of the hash in this case. | 329 // the B'th bit of the hash in this case. |
323 if((hash & newbit) == 0) { | 330 if((hash & newbit) == 0) { |
324 if(xi == BUCKETSIZE) { | 331 if(xi == BUCKETSIZE) { |
| 332 if(checkgc) mstats.next_gc = mst
ats.heap_alloc; |
325 newx = runtime·mallocgc(h->bucke
tsize, 0, 1, 0); | 333 newx = runtime·mallocgc(h->bucke
tsize, 0, 1, 0); |
326 clearbucket(newx); | 334 clearbucket(newx); |
327 x->overflow = newx; | 335 x->overflow = newx; |
328 x = newx; | 336 x = newx; |
329 xi = 0; | 337 xi = 0; |
330 xk = x->data; | 338 xk = x->data; |
331 xv = xk + h->keysize * BUCKETSIZ
E; | 339 xv = xk + h->keysize * BUCKETSIZ
E; |
332 } | 340 } |
333 x->tophash[xi] = b->tophash[i]; | 341 x->tophash[xi] = b->tophash[i]; |
334 if((h->flags & IndirectKey) != 0) { | 342 if((h->flags & IndirectKey) != 0) { |
335 *(byte**)xk = *(byte**)k;
// copy pointer | 343 *(byte**)xk = *(byte**)k;
// copy pointer |
336 } else { | 344 } else { |
337 t->key->alg->copy(t->key->size,
xk, k); // copy value | 345 t->key->alg->copy(t->key->size,
xk, k); // copy value |
338 } | 346 } |
339 if((h->flags & IndirectValue) != 0) { | 347 if((h->flags & IndirectValue) != 0) { |
340 *(byte**)xv = *(byte**)v; | 348 *(byte**)xv = *(byte**)v; |
341 } else { | 349 } else { |
342 t->elem->alg->copy(t->elem->size
, xv, v); | 350 t->elem->alg->copy(t->elem->size
, xv, v); |
343 } | 351 } |
344 xi++; | 352 xi++; |
345 xk += h->keysize; | 353 xk += h->keysize; |
346 xv += h->valuesize; | 354 xv += h->valuesize; |
347 } else { | 355 } else { |
348 if(yi == BUCKETSIZE) { | 356 if(yi == BUCKETSIZE) { |
| 357 if(checkgc) mstats.next_gc = mst
ats.heap_alloc; |
349 newy = runtime·mallocgc(h->bucke
tsize, 0, 1, 0); | 358 newy = runtime·mallocgc(h->bucke
tsize, 0, 1, 0); |
350 clearbucket(newy); | 359 clearbucket(newy); |
351 y->overflow = newy; | 360 y->overflow = newy; |
352 y = newy; | 361 y = newy; |
353 yi = 0; | 362 yi = 0; |
354 yk = y->data; | 363 yk = y->data; |
355 yv = yk + h->keysize * BUCKETSIZ
E; | 364 yv = yk + h->keysize * BUCKETSIZ
E; |
356 } | 365 } |
357 y->tophash[yi] = b->tophash[i]; | 366 y->tophash[yi] = b->tophash[i]; |
358 if((h->flags & IndirectKey) != 0) { | 367 if((h->flags & IndirectKey) != 0) { |
(...skipping 75 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
434 { | 443 { |
435 byte *old_buckets; | 444 byte *old_buckets; |
436 byte *new_buckets; | 445 byte *new_buckets; |
437 uint8 flags; | 446 uint8 flags; |
438 | 447 |
439 // allocate a bigger hash table | 448 // allocate a bigger hash table |
440 if(h->oldbuckets != nil) | 449 if(h->oldbuckets != nil) |
441 runtime·throw("evacuation not done in time"); | 450 runtime·throw("evacuation not done in time"); |
442 old_buckets = h->buckets; | 451 old_buckets = h->buckets; |
443 // NOTE: this could be a big malloc, but since we don't need zeroing it
is probably fast. | 452 // NOTE: this could be a big malloc, but since we don't need zeroing it
is probably fast. |
| 453 if(checkgc) mstats.next_gc = mstats.heap_alloc; |
444 new_buckets = runtime·mallocgc(h->bucketsize << (h->B + 1), 0, 1, 0); | 454 new_buckets = runtime·mallocgc(h->bucketsize << (h->B + 1), 0, 1, 0); |
445 flags = (h->flags & ~(Iterator | OldIterator)); | 455 flags = (h->flags & ~(Iterator | OldIterator)); |
446 if((h->flags & Iterator) != 0) { | 456 if((h->flags & Iterator) != 0) { |
447 flags |= OldIterator; | 457 flags |= OldIterator; |
448 // We can't free indirect keys any more, as | 458 // We can't free indirect keys any more, as |
449 // they are potentially aliased across buckets. | 459 // they are potentially aliased across buckets. |
450 flags &= ~CanFreeKey; | 460 flags &= ~CanFreeKey; |
451 } | 461 } |
452 | 462 |
453 // commit the grow (atomic wrt gc) | 463 // commit the grow (atomic wrt gc) |
(...skipping 19 matching lines...) Expand all Loading... |
473 uintptr bucket; | 483 uintptr bucket; |
474 Bucket *b; | 484 Bucket *b; |
475 uint8 top; | 485 uint8 top; |
476 uintptr i; | 486 uintptr i; |
477 bool eq; | 487 bool eq; |
478 byte *k, *k2, *v; | 488 byte *k, *k2, *v; |
479 | 489 |
480 key = *keyp; | 490 key = *keyp; |
481 if(docheck) | 491 if(docheck) |
482 check(t, h); | 492 check(t, h); |
| 493 if(h->count == 0) |
| 494 return nil; |
483 hash = h->hash0; | 495 hash = h->hash0; |
484 t->key->alg->hash(&hash, t->key->size, key); | 496 t->key->alg->hash(&hash, t->key->size, key); |
485 bucket = hash & (((uintptr)1 << h->B) - 1); | 497 bucket = hash & (((uintptr)1 << h->B) - 1); |
486 if(h->oldbuckets != nil) | 498 if(h->oldbuckets != nil) |
487 grow_work(t, h, bucket); | 499 grow_work(t, h, bucket); |
488 b = (Bucket*)(h->buckets + bucket * h->bucketsize); | 500 b = (Bucket*)(h->buckets + bucket * h->bucketsize); |
489 top = hash >> (sizeof(uintptr)*8 - 8); | 501 top = hash >> (sizeof(uintptr)*8 - 8); |
490 if(top == 0) | 502 if(top == 0) |
491 top = 1; | 503 top = 1; |
492 do { | 504 do { |
(...skipping 67 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
560 uint8 *inserti; | 572 uint8 *inserti; |
561 byte *insertk, *insertv; | 573 byte *insertk, *insertv; |
562 uint8 top; | 574 uint8 top; |
563 byte *k, *v; | 575 byte *k, *v; |
564 byte *kmem, *vmem; | 576 byte *kmem, *vmem; |
565 | 577 |
566 if(docheck) | 578 if(docheck) |
567 check(t, h); | 579 check(t, h); |
568 hash = h->hash0; | 580 hash = h->hash0; |
569 t->key->alg->hash(&hash, t->key->size, key); | 581 t->key->alg->hash(&hash, t->key->size, key); |
| 582 if(h->buckets == nil) { |
| 583 h->buckets = runtime·mallocgc(h->bucketsize, 0, 1, 0); |
| 584 b = (Bucket*)(h->buckets); |
| 585 clearbucket(b); |
| 586 } |
| 587 |
570 again: | 588 again: |
571 bucket = hash & (((uintptr)1 << h->B) - 1); | 589 bucket = hash & (((uintptr)1 << h->B) - 1); |
572 if(h->oldbuckets != nil) | 590 if(h->oldbuckets != nil) |
573 grow_work(t, h, bucket); | 591 grow_work(t, h, bucket); |
574 b = (Bucket*)(h->buckets + bucket * h->bucketsize); | 592 b = (Bucket*)(h->buckets + bucket * h->bucketsize); |
575 top = hash >> (sizeof(uintptr)*8 - 8); | 593 top = hash >> (sizeof(uintptr)*8 - 8); |
576 if(top == 0) | 594 if(top == 0) |
577 top = 1; | 595 top = 1; |
578 inserti = 0; | 596 inserti = 0; |
579 insertk = nil; | 597 insertk = nil; |
(...skipping 24 matching lines...) Expand all Loading... |
604 } | 622 } |
605 | 623 |
606 // did not find mapping for key. Allocate new cell & add entry. | 624 // did not find mapping for key. Allocate new cell & add entry. |
607 if(h->count >= LOAD * ((uintptr)1 << h->B) && h->count >= BUCKETSIZE) { | 625 if(h->count >= LOAD * ((uintptr)1 << h->B) && h->count >= BUCKETSIZE) { |
608 hash_grow(t, h); | 626 hash_grow(t, h); |
609 goto again; // Growing the table invalidates everything, so try
again | 627 goto again; // Growing the table invalidates everything, so try
again |
610 } | 628 } |
611 | 629 |
612 if(inserti == nil) { | 630 if(inserti == nil) { |
613 // all current buckets are full, allocate a new one. | 631 // all current buckets are full, allocate a new one. |
| 632 if(checkgc) mstats.next_gc = mstats.heap_alloc; |
614 newb = runtime·mallocgc(h->bucketsize, 0, 1, 0); | 633 newb = runtime·mallocgc(h->bucketsize, 0, 1, 0); |
615 clearbucket(newb); | 634 clearbucket(newb); |
616 b->overflow = newb; | 635 b->overflow = newb; |
617 inserti = newb->tophash; | 636 inserti = newb->tophash; |
618 insertk = newb->data; | 637 insertk = newb->data; |
619 insertv = insertk + h->keysize * BUCKETSIZE; | 638 insertv = insertk + h->keysize * BUCKETSIZE; |
620 } | 639 } |
621 | 640 |
622 // store new key/value at insert position | 641 // store new key/value at insert position |
623 if((h->flags & IndirectKey) != 0) { | 642 if((h->flags & IndirectKey) != 0) { |
| 643 if(checkgc) mstats.next_gc = mstats.heap_alloc; |
624 kmem = runtime·mallocgc(t->key->size, 0, 1, 0); | 644 kmem = runtime·mallocgc(t->key->size, 0, 1, 0); |
625 *(byte**)insertk = kmem; | 645 *(byte**)insertk = kmem; |
626 insertk = kmem; | 646 insertk = kmem; |
627 } | 647 } |
628 if((h->flags & IndirectValue) != 0) { | 648 if((h->flags & IndirectValue) != 0) { |
| 649 if(checkgc) mstats.next_gc = mstats.heap_alloc; |
629 vmem = runtime·mallocgc(t->elem->size, 0, 1, 0); | 650 vmem = runtime·mallocgc(t->elem->size, 0, 1, 0); |
630 *(byte**)insertv = vmem; | 651 *(byte**)insertv = vmem; |
631 insertv = vmem; | 652 insertv = vmem; |
632 } | 653 } |
633 t->key->alg->copy(t->key->size, insertk, key); | 654 t->key->alg->copy(t->key->size, insertk, key); |
634 t->elem->alg->copy(t->elem->size, insertv, value); | 655 t->elem->alg->copy(t->elem->size, insertv, value); |
635 *inserti = top; | 656 *inserti = top; |
636 h->count++; | 657 h->count++; |
637 if(docheck) | 658 if(docheck) |
638 check(t, h); | 659 check(t, h); |
639 } | 660 } |
640 | 661 |
641 static void | 662 static void |
642 hash_remove(MapType *t, Hmap *h, void *key) | 663 hash_remove(MapType *t, Hmap *h, void *key) |
643 { | 664 { |
644 uintptr hash; | 665 uintptr hash; |
645 uintptr bucket; | 666 uintptr bucket; |
646 Bucket *b; | 667 Bucket *b; |
647 uint8 top; | 668 uint8 top; |
648 uintptr i; | 669 uintptr i; |
649 byte *k, *v; | 670 byte *k, *v; |
650 bool eq; | 671 bool eq; |
651 ········ | 672 ········ |
652 if(docheck) | 673 if(docheck) |
653 check(t, h); | 674 check(t, h); |
| 675 if(h->count == 0) |
| 676 return; |
654 hash = h->hash0; | 677 hash = h->hash0; |
655 t->key->alg->hash(&hash, t->key->size, key); | 678 t->key->alg->hash(&hash, t->key->size, key); |
656 bucket = hash & (((uintptr)1 << h->B) - 1); | 679 bucket = hash & (((uintptr)1 << h->B) - 1); |
657 if(h->oldbuckets != nil) | 680 if(h->oldbuckets != nil) |
658 grow_work(t, h, bucket); | 681 grow_work(t, h, bucket); |
659 b = (Bucket*)(h->buckets + bucket * h->bucketsize); | 682 b = (Bucket*)(h->buckets + bucket * h->bucketsize); |
660 top = hash >> (sizeof(uintptr)*8 - 8); | 683 top = hash >> (sizeof(uintptr)*8 - 8); |
661 if(top == 0) | 684 if(top == 0) |
662 top = 1; | 685 top = 1; |
663 do { | 686 do { |
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
734 it->B = h->B; | 757 it->B = h->B; |
735 it->buckets = h->buckets; | 758 it->buckets = h->buckets; |
736 | 759 |
737 // iterator state | 760 // iterator state |
738 it->bucket = it->endbucket = runtime·fastrand1() & (((uintptr)1 << h->B)
- 1); | 761 it->bucket = it->endbucket = runtime·fastrand1() & (((uintptr)1 << h->B)
- 1); |
739 it->wrapped = false; | 762 it->wrapped = false; |
740 it->bptr = nil; | 763 it->bptr = nil; |
741 | 764 |
742 // Remember we have an iterator at this level. | 765 // Remember we have an iterator at this level. |
743 h->flags |= Iterator; | 766 h->flags |= Iterator; |
| 767 |
| 768 if(h->buckets == nil) { |
| 769 // Empty map. Force next hash_next to exit without |
| 770 // evalulating h->bucket. |
| 771 it->wrapped = true; |
| 772 } |
744 } | 773 } |
745 | 774 |
746 // initializes it->key and it->value to the next key/value pair | 775 // initializes it->key and it->value to the next key/value pair |
747 // in the iteration, or nil if we've reached the end. | 776 // in the iteration, or nil if we've reached the end. |
748 static void | 777 static void |
749 hash_next(struct hash_iter *it) | 778 hash_next(struct hash_iter *it) |
750 { | 779 { |
751 Hmap *h; | 780 Hmap *h; |
752 MapType *t; | 781 MapType *t; |
753 uintptr bucket; | 782 uintptr bucket; |
(...skipping 79 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
833 #define PHASE_OLD_BUCKETS 1 | 862 #define PHASE_OLD_BUCKETS 1 |
834 #define PHASE_TABLE 2 | 863 #define PHASE_TABLE 2 |
835 #define PHASE_OLD_TABLE 3 | 864 #define PHASE_OLD_TABLE 3 |
836 #define PHASE_DONE 4 | 865 #define PHASE_DONE 4 |
837 | 866 |
838 // Initialize the iterator. | 867 // Initialize the iterator. |
839 // Returns false if Hmap contains no pointers (in which case the iterator is not
initialized). | 868 // Returns false if Hmap contains no pointers (in which case the iterator is not
initialized). |
840 bool | 869 bool |
841 hash_gciter_init (Hmap *h, struct hash_gciter *it) | 870 hash_gciter_init (Hmap *h, struct hash_gciter *it) |
842 { | 871 { |
843 » // GC during map initialization | 872 » // GC during map initialization or on an empty map. |
844 if(h->buckets == nil) | 873 if(h->buckets == nil) |
845 return false; | 874 return false; |
846 | 875 |
847 it->h = h; | 876 it->h = h; |
848 it->phase = PHASE_BUCKETS; | 877 it->phase = PHASE_BUCKETS; |
849 it->bucket = 0; | 878 it->bucket = 0; |
850 it->b = nil; | 879 it->b = nil; |
851 | 880 |
852 // TODO: finish evacuating oldbuckets so that we can collect | 881 // TODO: finish evacuating oldbuckets so that we can collect |
853 // oldbuckets? We don't want to keep a partially evacuated | 882 // oldbuckets? We don't want to keep a partially evacuated |
(...skipping 122 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
976 } | 1005 } |
977 if(it->phase != PHASE_DONE) | 1006 if(it->phase != PHASE_DONE) |
978 runtime·throw("bad phase at done"); | 1007 runtime·throw("bad phase at done"); |
979 return false; | 1008 return false; |
980 } | 1009 } |
981 | 1010 |
982 // | 1011 // |
983 /// interfaces to go runtime | 1012 /// interfaces to go runtime |
984 // | 1013 // |
985 | 1014 |
| 1015 void |
| 1016 reflect·ismapkey(Type *typ, bool ret) |
| 1017 { |
| 1018 ret = typ != nil && typ->alg->hash != runtime·nohash; |
| 1019 FLUSH(&ret); |
| 1020 } |
| 1021 |
986 Hmap* | 1022 Hmap* |
987 runtime·makemap_c(MapType *typ, int64 hint) | 1023 runtime·makemap_c(MapType *typ, int64 hint) |
988 { | 1024 { |
989 Hmap *h; | 1025 Hmap *h; |
990 Type *key; | 1026 Type *key; |
991 | 1027 |
992 key = typ->key; | 1028 key = typ->key; |
993 | 1029 |
994 if(hint < 0 || (int32)hint != hint) | 1030 if(hint < 0 || (int32)hint != hint) |
995 runtime·panicstring("makemap: size out of range"); | 1031 runtime·panicstring("makemap: size out of range"); |
(...skipping 430 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1426 t->elem->alg->copy(t->elem->size, av, it->value); | 1462 t->elem->alg->copy(t->elem->size, av, it->value); |
1427 | 1463 |
1428 if(debug) { | 1464 if(debug) { |
1429 runtime·prints("mapiter2: iter="); | 1465 runtime·prints("mapiter2: iter="); |
1430 runtime·printpointer(it); | 1466 runtime·printpointer(it); |
1431 runtime·prints("; map="); | 1467 runtime·prints("; map="); |
1432 runtime·printpointer(it->h); | 1468 runtime·printpointer(it->h); |
1433 runtime·prints("\n"); | 1469 runtime·prints("\n"); |
1434 } | 1470 } |
1435 } | 1471 } |
LEFT | RIGHT |