LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 #include "runtime.h" | 5 #include "runtime.h" |
6 #include "arch_GOARCH.h" | 6 #include "arch_GOARCH.h" |
7 #include "zaexperiment.h" | 7 #include "zaexperiment.h" |
8 #include "malloc.h" | 8 #include "malloc.h" |
9 #include "stack.h" | 9 #include "stack.h" |
10 #include "race.h" | 10 #include "race.h" |
(...skipping 116 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
127 void | 127 void |
128 runtime·schedinit(void) | 128 runtime·schedinit(void) |
129 { | 129 { |
130 int32 n, procs; | 130 int32 n, procs; |
131 byte *p; | 131 byte *p; |
132 Eface i; | 132 Eface i; |
133 | 133 |
134 runtime·sched.maxmcount = 10000; | 134 runtime·sched.maxmcount = 10000; |
135 runtime·precisestack = haveexperiment("precisestack"); | 135 runtime·precisestack = haveexperiment("precisestack"); |
136 | 136 |
137 runtime·mprofinit(); | |
138 runtime·mallocinit(); | 137 runtime·mallocinit(); |
139 mcommoninit(m); | 138 mcommoninit(m); |
140 ········ | 139 ········ |
141 // Initialize the itable value for newErrorCString, | 140 // Initialize the itable value for newErrorCString, |
142 // so that the next time it gets called, possibly | 141 // so that the next time it gets called, possibly |
143 // in a fault during a garbage collection, it will not | 142 // in a fault during a garbage collection, it will not |
144 // need to allocated memory. | 143 // need to allocated memory. |
145 runtime·newErrorCString(0, &i); | 144 runtime·newErrorCString(0, &i); |
146 | 145 |
147 runtime·goargs(); | 146 runtime·goargs(); |
(...skipping 119 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
267 | 266 |
268 void | 267 void |
269 runtime·tracebackothers(G *me) | 268 runtime·tracebackothers(G *me) |
270 { | 269 { |
271 G *gp; | 270 G *gp; |
272 int32 traceback; | 271 int32 traceback; |
273 | 272 |
274 traceback = runtime·gotraceback(nil); | 273 traceback = runtime·gotraceback(nil); |
275 ········ | 274 ········ |
276 // Show the current goroutine first, if we haven't already. | 275 // Show the current goroutine first, if we haven't already. |
277 » gp = m->curg; | 276 » if((gp = m->curg) != nil && gp != me) { |
278 » //if((gp = m->curg) != nil && gp != me) { | |
279 runtime·printf("\n"); | 277 runtime·printf("\n"); |
280 runtime·goroutineheader(gp); | 278 runtime·goroutineheader(gp); |
281 runtime·traceback(~(uintptr)0, ~(uintptr)0, 0, gp); | 279 runtime·traceback(~(uintptr)0, ~(uintptr)0, 0, gp); |
282 » //} | 280 » } |
283 | 281 |
284 for(gp = runtime·allg; gp != nil; gp = gp->alllink) { | 282 for(gp = runtime·allg; gp != nil; gp = gp->alllink) { |
285 if(gp == me || gp == m->curg || gp->status == Gdead) | 283 if(gp == me || gp == m->curg || gp->status == Gdead) |
286 continue; | 284 continue; |
287 if(gp->issystem && traceback < 2) | 285 if(gp->issystem && traceback < 2) |
288 continue; | 286 continue; |
289 runtime·printf("\n"); | 287 runtime·printf("\n"); |
290 runtime·goroutineheader(gp); | 288 runtime·goroutineheader(gp); |
291 if(gp->status == Grunning) { | 289 if(gp->status == Grunning) { |
292 runtime·printf("\tgoroutine running on other thread; sta
ck unavailable\n"); | 290 runtime·printf("\tgoroutine running on other thread; sta
ck unavailable\n"); |
(...skipping 650 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
943 m->nextp = nil; | 941 m->nextp = nil; |
944 } | 942 } |
945 | 943 |
946 static void | 944 static void |
947 mspinning(void) | 945 mspinning(void) |
948 { | 946 { |
949 m->spinning = true; | 947 m->spinning = true; |
950 } | 948 } |
951 | 949 |
952 // Schedules some M to run the p (creates an M if necessary). | 950 // Schedules some M to run the p (creates an M if necessary). |
953 // If p==nil, tries to get an idle P, if no idle P's returns false. | 951 // If p==nil, tries to get an idle P, if no idle P's does nothing. |
954 static void | 952 static void |
955 startm(P *p, bool spinning) | 953 startm(P *p, bool spinning) |
956 { | 954 { |
957 M *mp; | 955 M *mp; |
958 void (*fn)(void); | 956 void (*fn)(void); |
959 | 957 |
960 runtime·lock(&runtime·sched); | 958 runtime·lock(&runtime·sched); |
961 if(p == nil) { | 959 if(p == nil) { |
962 p = pidleget(); | 960 p = pidleget(); |
963 if(p == nil) { | 961 if(p == nil) { |
(...skipping 1239 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2203 old = runtime·gomaxprocs; | 2201 old = runtime·gomaxprocs; |
2204 if(old < 0 || old > MaxGomaxprocs || new <= 0 || new >MaxGomaxprocs) | 2202 if(old < 0 || old > MaxGomaxprocs || new <= 0 || new >MaxGomaxprocs) |
2205 runtime·throw("procresize: invalid arg"); | 2203 runtime·throw("procresize: invalid arg"); |
2206 // initialize new P's | 2204 // initialize new P's |
2207 for(i = 0; i < new; i++) { | 2205 for(i = 0; i < new; i++) { |
2208 p = runtime·allp[i]; | 2206 p = runtime·allp[i]; |
2209 if(p == nil) { | 2207 if(p == nil) { |
2210 p = (P*)runtime·mallocgc(sizeof(*p), 0, FlagNoInvokeGC); | 2208 p = (P*)runtime·mallocgc(sizeof(*p), 0, FlagNoInvokeGC); |
2211 p->id = i; | 2209 p->id = i; |
2212 p->status = Pgcstop; | 2210 p->status = Pgcstop; |
2213 p->magic1 = p->magic2 = p->magic3 = 0x12345678; | |
2214 runtime·printf("magic1 = %p\n", &p->magic1); | |
2215 runtime·atomicstorep(&runtime·allp[i], p); | 2211 runtime·atomicstorep(&runtime·allp[i], p); |
2216 } | 2212 } |
2217 if(p->mcache == nil) { | 2213 if(p->mcache == nil) { |
2218 if(old==0 && i==0) | 2214 if(old==0 && i==0) |
2219 p->mcache = m->mcache; // bootstrap | 2215 p->mcache = m->mcache; // bootstrap |
2220 else | 2216 else |
2221 p->mcache = runtime·allocmcache(); | 2217 p->mcache = runtime·allocmcache(); |
2222 } | 2218 } |
2223 } | 2219 } |
2224 | 2220 |
(...skipping 510 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2735 return; | 2731 return; |
2736 // the queue is not full, now the put above must suceed | 2732 // the queue is not full, now the put above must suceed |
2737 goto retry; | 2733 goto retry; |
2738 } | 2734 } |
2739 | 2735 |
2740 // Put g and a batch of work from local runnable queue on global queue. | 2736 // Put g and a batch of work from local runnable queue on global queue. |
2741 // Executed only by the owner P. | 2737 // Executed only by the owner P. |
2742 static bool | 2738 static bool |
2743 runqputslow(P *p, G *gp, uint32 h, uint32 t) | 2739 runqputslow(P *p, G *gp, uint32 h, uint32 t) |
2744 { | 2740 { |
2745 » G *batch[nelem(p->runq)/2+1 +1]; | 2741 » G *batch[nelem(p->runq)/2+1]; |
2746 uint32 n, i; | 2742 uint32 n, i; |
2747 | 2743 |
2748 // First, grab a batch from local queue. | 2744 // First, grab a batch from local queue. |
2749 batch[nelem(p->runq)/2+1] = (G*)0x123; | |
2750 n = t-h; | 2745 n = t-h; |
2751 n = n/2; | 2746 n = n/2; |
2752 if(n != nelem(p->runq)/2) | 2747 if(n != nelem(p->runq)/2) |
2753 runtime·throw("runqputslow: queue is not full"); | 2748 runtime·throw("runqputslow: queue is not full"); |
2754 for(i=0; i<n; i++) | 2749 for(i=0; i<n; i++) |
2755 batch[i] = p->runq[(h+i)%nelem(p->runq)]; | 2750 batch[i] = p->runq[(h+i)%nelem(p->runq)]; |
2756 if(batch[nelem(p->runq)/2+1] != (G*)0x123) runtime·throw("runqputslow: 222222222
2222"); | |
2757 if(!runtime·cas(&p->runqhead, h, h+n)) // cas-release, commits consume | 2751 if(!runtime·cas(&p->runqhead, h, h+n)) // cas-release, commits consume |
2758 return false; | 2752 return false; |
2759 batch[n] = gp; | 2753 batch[n] = gp; |
2760 if(batch[nelem(p->runq)/2+1] != (G*)0x123) runtime·throw("runqputslow: 222222222
2222"); | |
2761 // Link the goroutines. | 2754 // Link the goroutines. |
2762 for(i=0; i<n; i++) | 2755 for(i=0; i<n; i++) |
2763 batch[i]->schedlink = batch[i+1]; | 2756 batch[i]->schedlink = batch[i+1]; |
2764 if(batch[nelem(p->runq)/2+1] != (G*)0x123) runtime·throw("runqputslow: 222222222
2222"); | |
2765 // Now put the batch on global queue. | 2757 // Now put the batch on global queue. |
2766 runtime·lock(&runtime·sched); | 2758 runtime·lock(&runtime·sched); |
2767 globrunqputbatch(batch[0], batch[n], n+1); | 2759 globrunqputbatch(batch[0], batch[n], n+1); |
2768 runtime·unlock(&runtime·sched); | 2760 runtime·unlock(&runtime·sched); |
2769 return true; | 2761 return true; |
2770 } | 2762 } |
2771 | 2763 |
2772 // Get g from local runnable queue. | 2764 // Get g from local runnable queue. |
2773 // Executed only by the owner P. | 2765 // Executed only by the owner P. |
2774 static G* | 2766 static G* |
2775 runqget(P *p) | 2767 runqget(P *p) |
2776 { | 2768 { |
2777 G *gp; | 2769 G *gp; |
2778 uint32 t, h; | 2770 uint32 t, h; |
2779 | 2771 |
2780 for(;;) { | 2772 for(;;) { |
2781 h = runtime·atomicload(&p->runqhead); // load-acquire, synchron
ize with other consumers | 2773 h = runtime·atomicload(&p->runqhead); // load-acquire, synchron
ize with other consumers |
2782 t = p->runqtail; | 2774 t = p->runqtail; |
2783 if(t == h) | 2775 if(t == h) |
2784 return nil; | 2776 return nil; |
2785 gp = p->runq[h%nelem(p->runq)]; | 2777 gp = p->runq[h%nelem(p->runq)]; |
2786 if(runtime·cas(&p->runqhead, h, h+1)) // cas-release, commits c
onsume | 2778 if(runtime·cas(&p->runqhead, h, h+1)) // cas-release, commits c
onsume |
2787 { | |
2788 p->runq[h%nelem(p->runq)] = (G*)0x23456789; | |
2789 return gp; | 2779 return gp; |
2790 } | |
2791 } | 2780 } |
2792 } | 2781 } |
2793 | 2782 |
2794 // Grabs a batch of goroutines from local runnable queue. | 2783 // Grabs a batch of goroutines from local runnable queue. |
2795 // batch array must be of size nelem(p->runq)/2. Returns number of grabbed gorou
tines. | 2784 // batch array must be of size nelem(p->runq)/2. Returns number of grabbed gorou
tines. |
2796 // Can be executed by any P. | 2785 // Can be executed by any P. |
2797 static uint32 | 2786 static uint32 |
2798 runqgrab(P *p, G **batch) | 2787 runqgrab(P *p, G **batch) |
2799 { | 2788 { |
2800 uint32 t, h, n, i; | 2789 uint32 t, h, n, i; |
2801 | 2790 |
2802 for(;;) { | 2791 for(;;) { |
2803 h = runtime·atomicload(&p->runqhead); // load-acquire, synchron
ize with other consumers | 2792 h = runtime·atomicload(&p->runqhead); // load-acquire, synchron
ize with other consumers |
2804 t = runtime·atomicload(&p->runqtail); // load-acquire, synchron
ize with the producer | 2793 t = runtime·atomicload(&p->runqtail); // load-acquire, synchron
ize with the producer |
2805 n = t-h; | 2794 n = t-h; |
2806 n = n - n/2; | 2795 n = n - n/2; |
2807 if(n == 0) | 2796 if(n == 0) |
2808 break; | 2797 break; |
2809 if(n > nelem(p->runq)/2) // read inconsistent h and t | 2798 if(n > nelem(p->runq)/2) // read inconsistent h and t |
2810 continue; | 2799 continue; |
2811 /* | |
2812 if(n > nelem(p->runq)/2) { | |
2813 runtime·printf("runqgrab: runq overflow h=%d t=%d n=%d\n
", h, t, n); | |
2814 runtime·throw("runqgrab: runq overflow"); | |
2815 } | |
2816 */ | |
2817 for(i=0; i<n; i++) | 2800 for(i=0; i<n; i++) |
2818 batch[i] = p->runq[(h+i)%nelem(p->runq)]; | 2801 batch[i] = p->runq[(h+i)%nelem(p->runq)]; |
2819 if(runtime·cas(&p->runqhead, h, h+n)) // cas-release, commits c
onsume | 2802 if(runtime·cas(&p->runqhead, h, h+n)) // cas-release, commits c
onsume |
2820 break; | 2803 break; |
2821 } | 2804 } |
2822 return n; | 2805 return n; |
2823 } | 2806 } |
2824 | 2807 |
2825 // Steal half of elements from local runnable queue of p2 | 2808 // Steal half of elements from local runnable queue of p2 |
2826 // and put onto local runnable queue of p. | 2809 // and put onto local runnable queue of p. |
2827 // Returns one of the stolen elements (or nil if failed). | 2810 // Returns one of the stolen elements (or nil if failed). |
2828 static G* | 2811 static G* |
2829 runqsteal(P *p, P *p2) | 2812 runqsteal(P *p, P *p2) |
2830 { | 2813 { |
2831 G *gp; | 2814 G *gp; |
2832 » G *batch[nelem(p->runq)/2+1]; | 2815 » G *batch[nelem(p->runq)/2]; |
2833 uint32 t, h, n, i; | 2816 uint32 t, h, n, i; |
2834 | 2817 |
2835 batch[nelem(p->runq)/2] = (G*)0x123; | |
2836 n = runqgrab(p2, batch); | 2818 n = runqgrab(p2, batch); |
2837 if(batch[nelem(p->runq)/2] != (G*)0x123) runtime·throw("runqsteal: 1111111111111
"); | |
2838 if(n == 0) | 2819 if(n == 0) |
2839 {if(batch[nelem(p->runq)/2] != (G*)0x123) runtime·throw("runqsteal: 111111111111
1"); | |
2840 return nil; | 2820 return nil; |
2841 } | |
2842 n--; | 2821 n--; |
2843 gp = batch[n]; | 2822 gp = batch[n]; |
2844 if(n == 0) | 2823 if(n == 0) |
2845 {if(batch[nelem(p->runq)/2] != (G*)0x123) runtime·throw("runqsteal: 111111111111
1"); | |
2846 return gp; | 2824 return gp; |
2847 } | |
2848 h = runtime·atomicload(&p->runqhead); // load-acquire, synchronize with
consumers | 2825 h = runtime·atomicload(&p->runqhead); // load-acquire, synchronize with
consumers |
2849 t = p->runqtail; | 2826 t = p->runqtail; |
2850 » if(t - h + n >= nelem(p->runq)) { | 2827 » if(t - h + n >= nelem(p->runq)) |
2851 » » runtime·printf("runqsteal: runq overflow tail=%d head=%d steal=%
d magic=%p/%p/%p %p\n", t, h, n, p->magic1, p->magic2, p->magic3, &p->magic1); | |
2852 » » runtime·usleep(1000); | |
2853 » » runtime·printf("runqsteal1: runq overflow tail=%d head=%d magic=
%p/%p/%p\n", p->runqtail, p->runqhead, p->magic1, p->magic2, p->magic3); | |
2854 » » runtime·usleep(1000); | |
2855 » » runtime·printf("runqsteal2: runq overflow tail=%d head=%d magic=
%p/%p/%p\n", p->runqtail, p->runqhead, p->magic1, p->magic2, p->magic3); | |
2856 » » runtime·usleep(1000); | |
2857 » » runtime·printf("runqsteal3: runq overflow tail=%d head=%d magic=
%p/%p/%p\n", p->runqtail, p->runqhead, p->magic1, p->magic2, p->magic3); | |
2858 runtime·throw("runqsteal: runq overflow"); | 2828 runtime·throw("runqsteal: runq overflow"); |
2859 } | |
2860 for(i=0; i<n; i++, t++) | 2829 for(i=0; i<n; i++, t++) |
2861 p->runq[t%nelem(p->runq)] = batch[i]; | 2830 p->runq[t%nelem(p->runq)] = batch[i]; |
2862 runtime·atomicstore(&p->runqtail, t); // store-release, makes the item
available for consumption | 2831 runtime·atomicstore(&p->runqtail, t); // store-release, makes the item
available for consumption |
2863 return gp; | 2832 return gp; |
2864 } | 2833 } |
2865 | 2834 |
2866 void | 2835 void |
2867 runtime·testSchedLocalQueue(void) | 2836 runtime·testSchedLocalQueue(void) |
2868 { | 2837 { |
2869 P p; | 2838 P p; |
(...skipping 97 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2967 if(experiment[i+j] != name[j]) | 2936 if(experiment[i+j] != name[j]) |
2968 goto nomatch; | 2937 goto nomatch; |
2969 if(experiment[i+j] != '\0' && experiment[i+j] != ',') | 2938 if(experiment[i+j] != '\0' && experiment[i+j] != ',') |
2970 goto nomatch; | 2939 goto nomatch; |
2971 return 1; | 2940 return 1; |
2972 } | 2941 } |
2973 nomatch:; | 2942 nomatch:; |
2974 } | 2943 } |
2975 return 0; | 2944 return 0; |
2976 } | 2945 } |
LEFT | RIGHT |