Left: | ||
Right: |
LEFT | RIGHT |
---|---|
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // Garbage collector. | 5 // Garbage collector. |
6 | 6 |
7 #include "runtime.h" | 7 #include "runtime.h" |
8 #include "arch_GOARCH.h" | 8 #include "arch_GOARCH.h" |
9 #include "malloc.h" | 9 #include "malloc.h" |
10 #include "stack.h" | 10 #include "stack.h" |
11 #include "mgc0.h" | |
12 #include "race.h" | |
11 #include "type.h" | 13 #include "type.h" |
14 #include "typekind.h" | |
12 #include "hashmap.h" | 15 #include "hashmap.h" |
13 | 16 |
14 enum { | 17 enum { |
15 Debug = 0, | 18 Debug = 0, |
16 PtrSize = sizeof(void*), | |
17 DebugMark = 0, // run second pass to check mark | 19 DebugMark = 0, // run second pass to check mark |
20 CollectStats = 0, | |
21 ScanStackByFrames = 1, | |
22 IgnorePreciseGC = 0, | |
18 DebugType = 0, | 23 DebugType = 0, |
19 » DebugStats = 0, | 24 » MemScramble = 0, // scramble free memory, and don't actually free it |
20 | 25 |
21 // Four bits per word (see #defines below). | 26 // Four bits per word (see #defines below). |
22 wordsPerBitmapWord = sizeof(void*)*8/4, | 27 wordsPerBitmapWord = sizeof(void*)*8/4, |
23 bitShift = sizeof(void*)*8/4, | 28 bitShift = sizeof(void*)*8/4, |
24 | 29 |
25 handoffThreshold = 4, | 30 handoffThreshold = 4, |
26 » XCAPACITY = 64, | 31 » IntermediateBufferCapacity = 64, |
32 | |
33 » // Bits in type information | |
34 » PRECISE = 1, | |
35 » LOOP = 2, | |
36 » PC_BITS = PRECISE | LOOP, | |
27 }; | 37 }; |
28 | 38 |
29 // Bits in per-word bitmap. | 39 // Bits in per-word bitmap. |
30 // #defines because enum might not be able to hold the values. | 40 // #defines because enum might not be able to hold the values. |
31 // | 41 // |
32 // Each word in the bitmap describes wordsPerBitmapWord words | 42 // Each word in the bitmap describes wordsPerBitmapWord words |
33 // of heap memory. There are 4 bitmap bits dedicated to each heap word, | 43 // of heap memory. There are 4 bitmap bits dedicated to each heap word, |
34 // so on a 64-bit system there is one bitmap word per 16 heap words. | 44 // so on a 64-bit system there is one bitmap word per 16 heap words. |
35 // The bits in the word are packed together by type first, then by | 45 // The bits in the word are packed together by type first, then by |
36 // heap location, so each 64-bit bitmap word consists of, from top to bottom, | 46 // heap location, so each 64-bit bitmap word consists of, from top to bottom, |
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
72 // runtime·semrelease(&runtime·worldsema); | 82 // runtime·semrelease(&runtime·worldsema); |
73 // runtime·starttheworld(); | 83 // runtime·starttheworld(); |
74 // | 84 // |
75 uint32 runtime·worldsema = 1; | 85 uint32 runtime·worldsema = 1; |
76 | 86 |
77 static int32 gctrace; | 87 static int32 gctrace; |
78 | 88 |
79 typedef struct Obj Obj; | 89 typedef struct Obj Obj; |
80 struct Obj | 90 struct Obj |
81 { | 91 { |
82 » byte» *p;» // Data pointer | 92 » byte» *p;» // data pointer |
83 » uintptr»n;» // Size of data in bytes | 93 » uintptr»n;» // size of data in bytes |
84 » uintptr»gc;» // Type info | 94 » uintptr»ti;» // type info |
85 }; | 95 }; |
86 | 96 |
87 // The size of Workbuf is N*PageSize. | 97 // The size of Workbuf is N*PageSize. |
88 typedef struct Workbuf Workbuf; | 98 typedef struct Workbuf Workbuf; |
89 struct Workbuf | 99 struct Workbuf |
90 { | 100 { |
rsc
2012/05/24 16:36:45
Please sync with dmitriy's latest code.
atom
2012/06/01 16:45:47
Done.
| |
91 » Workbuf»*next; | 101 #define SIZE (2*PageSize-sizeof(LFNode)-sizeof(uintptr)) |
92 » uintptr»nobj; | 102 » LFNode node; // must be first |
93 » Obj » obj[ (2*PageSize-2*sizeof(uintptr))/sizeof(Obj) - 1 ]; | 103 » uintptr nobj; |
94 » uint8» _padding[2*PageSize - 2*sizeof(uintptr) - ((2*PageSize-2*sizeof( uintptr))/sizeof(Obj)-1)*sizeof(Obj)]; | 104 » Obj obj[SIZE/sizeof(Obj) - 1]; |
95 }; | 105 » uint8 _padding[SIZE%sizeof(Obj) + sizeof(Obj)]; |
96 | 106 #undef SIZE |
97 typedef struct WbufState WbufState; | |
98 struct WbufState | |
99 { | |
100 » Obj **_wp; | |
101 » Workbuf **_wbuf; | |
102 » uintptr *_nobj; | |
103 }; | 107 }; |
104 | 108 |
105 typedef struct Finalizer Finalizer; | 109 typedef struct Finalizer Finalizer; |
106 struct Finalizer | 110 struct Finalizer |
107 { | 111 { |
108 » void (*fn)(void*); | 112 » FuncVal *fn; |
109 void *arg; | 113 void *arg; |
110 » int32 nret; | 114 » uintptr nret; |
111 }; | 115 }; |
112 | 116 |
113 typedef struct FinBlock FinBlock; | 117 typedef struct FinBlock FinBlock; |
114 struct FinBlock | 118 struct FinBlock |
115 { | 119 { |
116 FinBlock *alllink; | 120 FinBlock *alllink; |
117 FinBlock *next; | 121 FinBlock *next; |
118 int32 cnt; | 122 int32 cnt; |
119 int32 cap; | 123 int32 cap; |
120 Finalizer fin[1]; | 124 Finalizer fin[1]; |
121 }; | 125 }; |
122 | 126 |
123 extern byte etext[]; | |
124 extern byte data[]; | 127 extern byte data[]; |
125 extern byte edata[]; | 128 extern byte edata[]; |
126 extern byte bss[]; | 129 extern byte bss[]; |
127 extern byte ebss[]; | 130 extern byte ebss[]; |
128 | 131 |
129 extern byte data_gc[]; | 132 extern byte gcdata[]; |
130 extern byte bss_gc[]; | 133 extern byte gcbss[]; |
131 | 134 |
132 static G *fing; | 135 static G *fing; |
133 static FinBlock *finq; // list of finalizers that are to be executed | 136 static FinBlock *finq; // list of finalizers that are to be executed |
134 static FinBlock *finc; // cache of free blocks | 137 static FinBlock *finc; // cache of free blocks |
135 static FinBlock *allfin; // list of all blocks | 138 static FinBlock *allfin; // list of all blocks |
136 static Lock finlock; | 139 static Lock finlock; |
137 static int32 fingwait; | 140 static int32 fingwait; |
138 | 141 |
139 static void runfinq(void); | 142 static void runfinq(void); |
140 static Workbuf* getempty(Workbuf*); | 143 static Workbuf* getempty(Workbuf*); |
141 static Workbuf* getfull(Workbuf*); | 144 static Workbuf* getfull(Workbuf*); |
142 static void putempty(Workbuf*); | 145 static void putempty(Workbuf*); |
143 static Workbuf* handoff(Workbuf*); | 146 static Workbuf* handoff(Workbuf*); |
147 static void gchelperstart(void); | |
144 | 148 |
145 static struct { | 149 static struct { |
146 » Lock fmu; | 150 » uint64» full; // lock-free list of full blocks |
147 » Workbuf»*full; | 151 » uint64» empty; // lock-free list of empty blocks |
148 » Lock emu; | 152 » byte» pad0[CacheLineSize]; // prevents false-sharing between full/empt y and nproc/nwait |
149 » Workbuf»*empty; | |
150 uint32 nproc; | 153 uint32 nproc; |
154 byte pad1[CacheLineSize]; // prevents false-sharing between nproc and nwait | |
151 volatile uint32 nwait; | 155 volatile uint32 nwait; |
152 volatile uint32 ndone; | 156 volatile uint32 ndone; |
153 volatile uint32 debugmarkdone; | 157 volatile uint32 debugmarkdone; |
154 Note alldone; | 158 Note alldone; |
159 ParFor *markfor; | |
155 ParFor *sweepfor; | 160 ParFor *sweepfor; |
156 | 161 |
157 Lock; | 162 Lock; |
158 byte *chunk; | 163 byte *chunk; |
159 uintptr nchunk; | 164 uintptr nchunk; |
165 | |
166 Obj *roots; | |
167 uint32 nroot; | |
168 uint32 rootcap; | |
160 } work; | 169 } work; |
161 | 170 |
162 typedef struct CustomMarker CustomMarker; | 171 enum { |
163 struct CustomMarker | 172 » GC_DEFAULT_PTR = GC_NUM_INSTR, |
164 { | 173 » GC_MAP_NEXT, |
165 » void (*f)(void); | 174 » GC_CHAN, |
166 » CustomMarker *next; | 175 |
176 » GC_NUM_INSTR2 | |
167 }; | 177 }; |
168 static CustomMarker *custommarkers; | |
169 | |
170 void runtime·addgcmarker(void (*f)(void)) | |
171 { | |
172 CustomMarker *m; | |
173 m = runtime·malloc(sizeof(*m)); | |
174 m->f = f; | |
175 m->next = custommarkers; | |
176 custommarkers = m; | |
177 } | |
178 | |
179 enum { | |
180 GC_END = 0, | |
181 GC_PTR = 1, | |
182 GC_APTR = 2, | |
183 GC_ARRAY_START = 3, | |
184 GC_ARRAY_NEXT = 4, | |
185 GC_CALL = 5, | |
186 GC_MAP_PTR = 6, | |
187 GC_STRING = 7, | |
188 GC_EFACE = 8, | |
189 GC_IFACE = 9, | |
190 | |
191 GC_DEFAULT_PTR = 10, | |
192 GC_MAP_NEXT = 11, | |
193 | |
194 GC_NUM_INSTR | |
195 }; | |
196 | 178 |
197 static struct { | 179 static struct { |
198 » uint64 x_sum; | 180 » struct { |
199 » uint64 x_cnt; | 181 » » uint64 sum; |
200 » uint64 nobj_sum; | 182 » » uint64 cnt; |
201 » uint64 nobj_cnt; | 183 » } ptr; |
202 » uint64 nobj_notype; | 184 » uint64 nbytes; |
203 » uint64 nobj_typelookups_ok; | 185 » struct { |
204 » uint64 rescans; | 186 » » uint64 sum; |
205 » uint64 rescan_nbytes; | 187 » » uint64 cnt; |
206 » uint64 instr[GC_NUM_INSTR]; | 188 » » uint64 notype; |
189 » » uint64 typelookup; | |
190 » } obj; | |
191 » uint64 rescan; | |
192 » uint64 rescanbytes; | |
193 » uint64 instr[GC_NUM_INSTR2]; | |
207 uint64 putempty; | 194 uint64 putempty; |
208 uint64 getfull; | 195 uint64 getfull; |
209 uint64 runtime·markobject; | |
210 } gcstats; | 196 } gcstats; |
211 | 197 |
212 // Marks the object, without appending it to any buffer if the object wasn't alr eady marked. | 198 // markonly marks an object. It returns true if the object |
213 // Returns true if the object wasn't already marked. | 199 // has been marked by this function, false otherwise. |
214 // This function is single-threaded. | 200 // This function doesn't append the object to any buffer. |
215 static bool | 201 static bool |
216 markonly(void *obj) | 202 markonly(void *obj) |
217 { | 203 { |
218 byte *p; | 204 byte *p; |
219 uintptr *bitp, bits, shift, x, xbits, off; | 205 uintptr *bitp, bits, shift, x, xbits, off; |
220 MSpan *s; | 206 MSpan *s; |
207 PageID k; | |
221 | 208 |
222 // Words outside the arena cannot be pointers. | 209 // Words outside the arena cannot be pointers. |
223 » if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used) | 210 » if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used) |
224 return false; | 211 return false; |
225 | 212 |
226 // obj may be a pointer to a live object. | 213 // obj may be a pointer to a live object. |
227 // Try to find the beginning of the object. | 214 // Try to find the beginning of the object. |
228 | 215 |
229 // Round down to word boundary. | 216 // Round down to word boundary. |
230 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); | 217 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); |
231 | 218 |
232 // Find bits for this word. | 219 // Find bits for this word. |
233 » off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; | 220 » off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start; |
234 » bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 221 » bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1 ; |
235 shift = off % wordsPerBitmapWord; | 222 shift = off % wordsPerBitmapWord; |
236 xbits = *bitp; | 223 xbits = *bitp; |
237 bits = xbits >> shift; | 224 bits = xbits >> shift; |
238 | 225 |
239 // Pointing at the beginning of a block? | 226 // Pointing at the beginning of a block? |
240 if((bits & (bitAllocated|bitBlockBoundary)) != 0) | 227 if((bits & (bitAllocated|bitBlockBoundary)) != 0) |
241 goto found; | 228 goto found; |
242 | 229 |
243 // Otherwise consult span table to find beginning. | 230 // Otherwise consult span table to find beginning. |
244 // (Manually inlined copy of MHeap_LookupMaybe.) | 231 // (Manually inlined copy of MHeap_LookupMaybe.) |
245 » x = (uintptr)obj>>PageShift; | 232 » k = (uintptr)obj>>PageShift; |
233 » x = k; | |
246 if(sizeof(void*) == 8) | 234 if(sizeof(void*) == 8) |
247 » » x -= (uintptr)runtime·mheap.arena_start>>PageShift; | 235 » » x -= (uintptr)runtime·mheap->arena_start>>PageShift; |
248 » s = runtime·mheap.map[x]; | 236 » s = runtime·mheap->map[x]; |
249 » if(s == nil || s->state != MSpanInUse) | 237 » if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse) |
250 return false; | 238 return false; |
251 p = (byte*)((uintptr)s->start<<PageShift); | 239 p = (byte*)((uintptr)s->start<<PageShift); |
252 if(s->sizeclass == 0) { | 240 if(s->sizeclass == 0) { |
253 obj = p; | 241 obj = p; |
254 } else { | 242 } else { |
255 if((byte*)obj >= (byte*)s->limit) | 243 if((byte*)obj >= (byte*)s->limit) |
256 return false; | 244 return false; |
257 uintptr size = s->elemsize; | 245 uintptr size = s->elemsize; |
258 int32 i = ((byte*)obj - p)/size; | 246 int32 i = ((byte*)obj - p)/size; |
259 obj = p+i*size; | 247 obj = p+i*size; |
260 } | 248 } |
261 | 249 |
262 // Now that we know the object header, reload bits. | 250 // Now that we know the object header, reload bits. |
263 » off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; | 251 » off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start; |
264 » bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 252 » bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1 ; |
265 shift = off % wordsPerBitmapWord; | 253 shift = off % wordsPerBitmapWord; |
266 xbits = *bitp; | 254 xbits = *bitp; |
267 bits = xbits >> shift; | 255 bits = xbits >> shift; |
268 | 256 |
269 found: | 257 found: |
270 // Now we have bits, bitp, and shift correct for | 258 // Now we have bits, bitp, and shift correct for |
271 // obj pointing at the base of the object. | 259 // obj pointing at the base of the object. |
272 // Only care about allocated and not marked. | 260 // Only care about allocated and not marked. |
273 if((bits & (bitAllocated|bitMarked)) != bitAllocated) | 261 if((bits & (bitAllocated|bitMarked)) != bitAllocated) |
274 return false; | 262 return false; |
275 » *bitp |= bitMarked<<shift; | 263 » if(work.nproc == 1) |
264 » » *bitp |= bitMarked<<shift; | |
265 » else { | |
266 » » for(;;) { | |
267 » » » x = *bitp; | |
268 » » » if(x & (bitMarked<<shift)) | |
269 » » » » return false; | |
270 » » » if(runtime·casp((void**)bitp, (void*)x, (void*)(x|(bitMa rked<<shift)))) | |
271 » » » » break; | |
272 » » } | |
273 » } | |
276 | 274 |
277 // The object is now marked | 275 // The object is now marked |
278 return true; | 276 return true; |
279 } | 277 } |
280 | 278 |
281 void runtime·markobjects(void **obj, uintptr n) | 279 // PtrTarget is a structure used by intermediate buffers. |
282 { | 280 // The intermediate buffers hold GC data before it |
283 » uintptr i; | 281 // is moved/flushed to the work buffer (Workbuf). |
284 | 282 // The size of an intermediate buffer is very small, |
285 » for(i=0; i<n; i++) | 283 // such as 32 or 64 elements. |
286 » » markonly(obj[i]); | 284 typedef struct PtrTarget PtrTarget; |
287 | 285 struct PtrTarget |
288 » if(DebugStats) | 286 { |
289 » » runtime·xadd64(&gcstats.runtime·markobject, n); | 287 » void *p; |
290 } | 288 » uintptr ti; |
291 | 289 }; |
292 void runtime·markobject(void *obj) | 290 |
293 { | 291 typedef struct BufferList BufferList; |
294 » markonly(obj); | 292 struct BufferList |
295 » if(DebugStats) | 293 { |
296 » » runtime·xadd64(&gcstats.runtime·markobject, 1); | 294 » PtrTarget ptrtarget[IntermediateBufferCapacity]; |
297 } | 295 » Obj obj[IntermediateBufferCapacity]; |
298 | 296 » uint32 busy; |
299 // Append obj to the buffer. | 297 » byte pad[CacheLineSize]; |
298 }; | |
299 #pragma dataflag 16 // no pointers | |
300 static BufferList bufferList[MaxGcproc]; | |
301 | |
302 static Lock lock; | |
303 static Type *itabtype; | |
304 | |
305 static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj); | |
306 | |
307 // flushptrbuf moves data from the PtrTarget buffer to the work buffer. | |
308 // The PtrTarget buffer contains blocks irrespective of whether the blocks have been marked or scanned, | |
309 // while the work buffer contains blocks which have been marked | |
310 // and are prepared to be scanned by the garbage collector. | |
311 // | |
312 // _wp, _wbuf, _nobj are input/output parameters and are specifying the work buf fer. | |
313 // | |
314 // A simplified drawing explaining how the todo-list moves from a structure to a nother: | |
315 // | |
316 // scanblock | |
317 // (find pointers) | |
318 // Obj ------> PtrTarget (pointer targets) | |
319 // ↑ | | |
320 // | | | |
321 // `----------' | |
322 // flushptrbuf | |
323 // (find block start, mark and enqueue) | |
300 static void | 324 static void |
301 enqueue(Obj obj, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj) | 325 flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf , uintptr *_nobj) |
302 { | 326 { |
303 » uintptr nobj, off; | 327 » byte *p, *arena_start, *obj; |
328 » uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n; | |
329 » MSpan *s; | |
330 » PageID k; | |
304 Obj *wp; | 331 Obj *wp; |
305 Workbuf *wbuf; | 332 Workbuf *wbuf; |
306 | 333 » PtrTarget *ptrbuf_end; |
307 » if(Debug > 1) | 334 |
308 » » runtime·printf("append obj(%p %D %p)\n", obj.p, (int64)obj.n, ob j.gc); | 335 » arena_start = runtime·mheap->arena_start; |
309 | 336 |
310 » // Align obj.b to a word boundary. | |
311 » off = (uintptr)obj.p & (PtrSize-1); | |
312 » if(off != 0) { | |
313 » » obj.p += PtrSize - off; | |
314 » » obj.n -= PtrSize - off; | |
315 » » obj.gc = 0; | |
316 » } | |
317 | |
318 » if(obj.p == nil || obj.n == 0) | |
319 » » return; | |
320 | |
321 » // Get the state of buffer | |
322 wp = *_wp; | 337 wp = *_wp; |
323 wbuf = *_wbuf; | 338 wbuf = *_wbuf; |
324 nobj = *_nobj; | 339 nobj = *_nobj; |
325 | 340 |
326 » // If another proc wants a pointer, give it some. | 341 » ptrbuf_end = *ptrbufpos; |
327 » if(nobj > handoffThreshold && work.nwait > 0 && work.full == nil) { | 342 » n = ptrbuf_end - ptrbuf; |
328 » » wbuf->nobj = nobj; | 343 » *ptrbufpos = ptrbuf; |
329 » » wbuf = handoff(wbuf); | 344 |
330 » » nobj = wbuf->nobj; | 345 » if(CollectStats) { |
331 » » wp = wbuf->obj + nobj; | 346 » » runtime·xadd64(&gcstats.ptr.sum, n); |
332 » } | 347 » » runtime·xadd64(&gcstats.ptr.cnt, 1); |
333 | 348 » } |
334 » // If buffer is full, get a new one. | |
335 » if(wbuf == nil || nobj >= nelem(wbuf->obj)) { | |
336 » » if(wbuf != nil) | |
337 » » » wbuf->nobj = nobj; | |
338 » » wbuf = getempty(wbuf); | |
339 » » wp = wbuf->obj; | |
340 » » nobj = 0; | |
341 » } | |
342 | |
343 » *wp = obj; | |
344 » wp++; | |
345 » nobj++; | |
346 | |
347 » // Update the state of buffer | |
348 » *_wp = wp; | |
349 » *_wbuf = wbuf; | |
350 » *_nobj = nobj; | |
351 } | |
352 | |
353 static void | |
354 markenqueue(Obj obj, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj) | |
355 { | |
356 » markonly(obj.p); | |
357 » enqueue(obj, _wp, _wbuf, _nobj); | |
358 } | |
359 | |
360 struct X | |
361 { | |
362 » void *p; | |
363 » uintptr gc; | |
364 }; | |
365 | |
366 struct Y | |
367 { | |
368 » uintptr *bitp, bits, shift; | |
369 » byte *obj; | |
370 » uintptr gc; | |
371 }; | |
372 | |
373 struct BufferList | |
374 { | |
375 » struct X x[XCAPACITY]; | |
376 » struct Y y[XCAPACITY]; | |
377 » Obj obj[XCAPACITY]; | |
378 » struct BufferList *next; | |
379 }; | |
380 static struct BufferList *bufferList; | |
381 | |
382 static Lock lock; | |
383 | |
384 static void | |
385 flushxbuf(struct X *xbuf, uintptr n, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj, struct Y *ybuf) | |
386 { | |
387 » byte *p, *arena_start; | |
388 » uintptr *bitp, bits, shift, j, x, xbits, off, nobj; | |
389 » MSpan *s; | |
390 » Obj *wp; | |
391 » Workbuf *wbuf; | |
392 » struct X *xbuf_end; | |
393 | |
394 » if(DebugStats) { | |
395 » » runtime·xadd64(&gcstats.x_sum, n); | |
396 » » runtime·xadd64(&gcstats.x_cnt, 1); | |
397 » } | |
398 | |
399 » arena_start = runtime·mheap.arena_start; | |
400 | |
401 » wp = *_wp; | |
402 » wbuf = *_wbuf; | |
403 » nobj = *_nobj; | |
404 | |
405 » xbuf_end = xbuf + n; | |
406 | 349 |
407 // If buffer is nearly full, get a new one. | 350 // If buffer is nearly full, get a new one. |
408 if(wbuf == nil || nobj+n >= nelem(wbuf->obj)) { | 351 if(wbuf == nil || nobj+n >= nelem(wbuf->obj)) { |
409 if(wbuf != nil) | 352 if(wbuf != nil) |
410 wbuf->nobj = nobj; | 353 wbuf->nobj = nobj; |
411 wbuf = getempty(wbuf); | 354 wbuf = getempty(wbuf); |
412 wp = wbuf->obj; | 355 wp = wbuf->obj; |
413 nobj = 0; | 356 nobj = 0; |
414 | 357 |
415 if(n >= nelem(wbuf->obj)) | 358 if(n >= nelem(wbuf->obj)) |
416 » » » runtime·throw("xbuf has to be smaller than WorkBuf"); | 359 » » » runtime·throw("ptrbuf has to be smaller than WorkBuf"); |
417 } | 360 } |
418 | 361 |
419 if(work.nproc == 1) { | 362 if(work.nproc == 1) { |
420 // Single-threaded version. | 363 // Single-threaded version. |
421 // Nearly the same code as the multi-threaded version. | 364 // Nearly the same code as the multi-threaded version. |
422 | 365 |
423 » » while(xbuf < xbuf_end) { | 366 » » while(ptrbuf < ptrbuf_end) { |
424 » » » byte *obj = xbuf->p; | 367 » » » obj = ptrbuf->p; |
425 » » » uintptr gc = xbuf->gc; | 368 » » » ti = ptrbuf->ti; |
426 » » » xbuf++; | 369 » » » ptrbuf++; |
427 | 370 |
428 // obj belongs to interval [mheap.arena_start, mheap.are na_used). | 371 // obj belongs to interval [mheap.arena_start, mheap.are na_used). |
372 if(Debug > 1) { | |
373 if(obj < runtime·mheap->arena_start || obj >= ru ntime·mheap->arena_used) | |
374 runtime·throw("object is outside of mhea p"); | |
375 } | |
429 | 376 |
430 // obj may be a pointer to a live object. | 377 // obj may be a pointer to a live object. |
431 // Try to find the beginning of the object. | 378 // Try to find the beginning of the object. |
432 | 379 |
433 // Round down to word boundary. | 380 // Round down to word boundary. |
434 » » » if( ((uintptr)obj & ((uintptr)PtrSize-1)) != 0 ) { | 381 » » » if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) { |
rsc
2012/05/24 16:36:45
Avoid spaces inside if( ) parens.
atom
2012/06/01 16:45:47
Done. But I am unable to quickly read it without t
| |
435 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize- 1)); | 382 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize- 1)); |
436 » » » » gc = 0; | 383 » » » » ti = 0; |
437 } | 384 } |
438 | 385 |
439 // Find bits for this word. | 386 // Find bits for this word. |
440 off = (uintptr*)obj - (uintptr*)arena_start; | 387 off = (uintptr*)obj - (uintptr*)arena_start; |
441 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | 388 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; |
442 shift = off % wordsPerBitmapWord; | 389 shift = off % wordsPerBitmapWord; |
443 xbits = *bitp; | 390 xbits = *bitp; |
444 bits = xbits >> shift; | 391 bits = xbits >> shift; |
445 | 392 |
446 // Pointing at the beginning of a block? | 393 // Pointing at the beginning of a block? |
447 if((bits & (bitAllocated|bitBlockBoundary)) != 0) | 394 if((bits & (bitAllocated|bitBlockBoundary)) != 0) |
448 goto found1; | 395 goto found1; |
449 | 396 |
450 » » » gc = 0; | 397 » » » ti = 0; |
451 | 398 |
452 // Pointing just past the beginning? | 399 // Pointing just past the beginning? |
453 // Scan backward a little to find a block boundary. | 400 // Scan backward a little to find a block boundary. |
454 for(j=shift; j-->0; ) { | 401 for(j=shift; j-->0; ) { |
455 if(((xbits>>j) & (bitAllocated|bitBlockBoundary) ) != 0) { | 402 if(((xbits>>j) & (bitAllocated|bitBlockBoundary) ) != 0) { |
456 obj = (byte*)obj - (shift-j)*PtrSize; | 403 obj = (byte*)obj - (shift-j)*PtrSize; |
457 shift = j; | 404 shift = j; |
458 bits = xbits>>shift; | 405 bits = xbits>>shift; |
459 goto found1; | 406 goto found1; |
460 } | 407 } |
461 } | 408 } |
462 | 409 |
463 // Otherwise consult span table to find beginning. | 410 // Otherwise consult span table to find beginning. |
464 // (Manually inlined copy of MHeap_LookupMaybe.) | 411 // (Manually inlined copy of MHeap_LookupMaybe.) |
465 » » » x = (uintptr)obj>>PageShift; | 412 » » » k = (uintptr)obj>>PageShift; |
413 » » » x = k; | |
466 if(sizeof(void*) == 8) | 414 if(sizeof(void*) == 8) |
467 x -= (uintptr)arena_start>>PageShift; | 415 x -= (uintptr)arena_start>>PageShift; |
468 » » » s = runtime·mheap.map[x]; | 416 » » » s = runtime·mheap->map[x]; |
469 » » » if(s == nil || s->state != MSpanInUse) | 417 » » » if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse) |
470 continue; | 418 continue; |
471 p = (byte*)((uintptr)s->start<<PageShift); | 419 p = (byte*)((uintptr)s->start<<PageShift); |
472 if(s->sizeclass == 0) { | 420 if(s->sizeclass == 0) { |
473 obj = p; | 421 obj = p; |
474 } else { | 422 } else { |
475 if((byte*)obj >= (byte*)s->limit) | 423 if((byte*)obj >= (byte*)s->limit) |
476 continue; | 424 continue; |
477 » » » » uintptr size = s->elemsize; | 425 » » » » size = s->elemsize; |
478 int32 i = ((byte*)obj - p)/size; | 426 int32 i = ((byte*)obj - p)/size; |
479 obj = p+i*size; | 427 obj = p+i*size; |
480 } | 428 } |
481 | 429 |
482 // Now that we know the object header, reload bits. | 430 // Now that we know the object header, reload bits. |
483 off = (uintptr*)obj - (uintptr*)arena_start; | 431 off = (uintptr*)obj - (uintptr*)arena_start; |
484 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | 432 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; |
485 shift = off % wordsPerBitmapWord; | 433 shift = off % wordsPerBitmapWord; |
486 xbits = *bitp; | 434 xbits = *bitp; |
487 bits = xbits >> shift; | 435 bits = xbits >> shift; |
488 | 436 |
489 found1: | 437 found1: |
490 // Now we have bits, bitp, and shift correct for | 438 // Now we have bits, bitp, and shift correct for |
491 // obj pointing at the base of the object. | 439 // obj pointing at the base of the object. |
492 // Only care about allocated and not marked. | 440 // Only care about allocated and not marked. |
493 if((bits & (bitAllocated|bitMarked)) != bitAllocated) | 441 if((bits & (bitAllocated|bitMarked)) != bitAllocated) |
494 continue; | 442 continue; |
495 | 443 |
496 *bitp |= bitMarked<<shift; | 444 *bitp |= bitMarked<<shift; |
497 | 445 |
498 // If object has no pointers, don't need to scan further . | 446 // If object has no pointers, don't need to scan further . |
499 if((bits & bitNoPointers) != 0) | 447 if((bits & bitNoPointers) != 0) |
500 continue; | 448 continue; |
501 | 449 |
502 // Ask span about size class. | 450 // Ask span about size class. |
503 // (Manually inlined copy of MHeap_Lookup.) | 451 // (Manually inlined copy of MHeap_Lookup.) |
504 x = (uintptr)obj>>PageShift; | 452 x = (uintptr)obj>>PageShift; |
505 if(sizeof(void*) == 8) | 453 if(sizeof(void*) == 8) |
506 x -= (uintptr)arena_start>>PageShift; | 454 x -= (uintptr)arena_start>>PageShift; |
507 » » » s = runtime·mheap.map[x]; | 455 » » » s = runtime·mheap->map[x]; |
508 | 456 |
509 » » » *wp = (Obj){obj, s->elemsize, gc}; | 457 » » » PREFETCH(obj); |
458 | |
459 » » » *wp = (Obj){obj, s->elemsize, ti}; | |
510 wp++; | 460 wp++; |
511 nobj++; | 461 nobj++; |
512 } | 462 } |
513 } else { | 463 } else { |
514 // Multi-threaded version. | 464 // Multi-threaded version. |
515 // Nearly the same code as the single-threaded version. | 465 // Nearly the same code as the single-threaded version. |
516 | 466 |
517 » » struct Y *ybufpos = ybuf; | 467 » » while(ptrbuf < ptrbuf_end) { |
rsc
2012/05/24 16:36:45
Please move all variable declarations to top of fu
atom
2012/06/01 16:45:47
Done.
| |
518 | 468 » » » obj = ptrbuf->p; |
519 » » while(xbuf < xbuf_end) { | 469 » » » ti = ptrbuf->ti; |
520 » » » byte *obj = xbuf->p; | 470 » » » ptrbuf++; |
521 » » » uintptr gc = xbuf->gc; | |
522 » » » xbuf++; | |
523 | 471 |
524 // obj belongs to interval [mheap.arena_start, mheap.are na_used). | 472 // obj belongs to interval [mheap.arena_start, mheap.are na_used). |
473 if(Debug > 1) { | |
474 if(obj < runtime·mheap->arena_start || obj >= ru ntime·mheap->arena_used) | |
475 runtime·throw("object is outside of mhea p"); | |
476 } | |
525 | 477 |
526 // obj may be a pointer to a live object. | 478 // obj may be a pointer to a live object. |
527 // Try to find the beginning of the object. | 479 // Try to find the beginning of the object. |
528 | 480 |
529 // Round down to word boundary. | 481 // Round down to word boundary. |
530 » » » if( ((uintptr)obj & ((uintptr)PtrSize-1)) != 0 ) { | 482 » » » if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) { |
531 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize- 1)); | 483 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize- 1)); |
532 » » » » gc = 0; | 484 » » » » ti = 0; |
533 } | 485 } |
534 | 486 |
535 // Find bits for this word. | 487 // Find bits for this word. |
536 off = (uintptr*)obj - (uintptr*)arena_start; | 488 off = (uintptr*)obj - (uintptr*)arena_start; |
537 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | 489 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; |
538 shift = off % wordsPerBitmapWord; | 490 shift = off % wordsPerBitmapWord; |
539 xbits = *bitp; | 491 xbits = *bitp; |
540 bits = xbits >> shift; | 492 bits = xbits >> shift; |
541 | 493 |
542 // Pointing at the beginning of a block? | 494 // Pointing at the beginning of a block? |
543 if((bits & (bitAllocated|bitBlockBoundary)) != 0) | 495 if((bits & (bitAllocated|bitBlockBoundary)) != 0) |
544 goto found2; | 496 goto found2; |
545 | 497 |
546 » » » gc = 0; | 498 » » » ti = 0; |
547 | 499 |
548 // Pointing just past the beginning? | 500 // Pointing just past the beginning? |
549 // Scan backward a little to find a block boundary. | 501 // Scan backward a little to find a block boundary. |
550 for(j=shift; j-->0; ) { | 502 for(j=shift; j-->0; ) { |
551 if(((xbits>>j) & (bitAllocated|bitBlockBoundary) ) != 0) { | 503 if(((xbits>>j) & (bitAllocated|bitBlockBoundary) ) != 0) { |
552 obj = (byte*)obj - (shift-j)*PtrSize; | 504 obj = (byte*)obj - (shift-j)*PtrSize; |
553 shift = j; | 505 shift = j; |
554 bits = xbits>>shift; | 506 bits = xbits>>shift; |
555 goto found2; | 507 goto found2; |
556 } | 508 } |
557 } | 509 } |
558 | 510 |
559 // Otherwise consult span table to find beginning. | 511 // Otherwise consult span table to find beginning. |
560 // (Manually inlined copy of MHeap_LookupMaybe.) | 512 // (Manually inlined copy of MHeap_LookupMaybe.) |
561 » » » x = (uintptr)obj>>PageShift; | 513 » » » k = (uintptr)obj>>PageShift; |
514 » » » x = k; | |
562 if(sizeof(void*) == 8) | 515 if(sizeof(void*) == 8) |
563 x -= (uintptr)arena_start>>PageShift; | 516 x -= (uintptr)arena_start>>PageShift; |
564 » » » s = runtime·mheap.map[x]; | 517 » » » s = runtime·mheap->map[x]; |
565 » » » if(s == nil || s->state != MSpanInUse) | 518 » » » if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse) |
566 continue; | 519 continue; |
567 p = (byte*)((uintptr)s->start<<PageShift); | 520 p = (byte*)((uintptr)s->start<<PageShift); |
568 if(s->sizeclass == 0) { | 521 if(s->sizeclass == 0) { |
569 obj = p; | 522 obj = p; |
570 } else { | 523 } else { |
571 if((byte*)obj >= (byte*)s->limit) | 524 if((byte*)obj >= (byte*)s->limit) |
572 continue; | 525 continue; |
573 » » » » uintptr size = s->elemsize; | 526 » » » » size = s->elemsize; |
574 int32 i = ((byte*)obj - p)/size; | 527 int32 i = ((byte*)obj - p)/size; |
575 obj = p+i*size; | 528 obj = p+i*size; |
576 } | 529 } |
577 | 530 |
578 // Now that we know the object header, reload bits. | 531 // Now that we know the object header, reload bits. |
579 off = (uintptr*)obj - (uintptr*)arena_start; | 532 off = (uintptr*)obj - (uintptr*)arena_start; |
580 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | 533 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; |
581 shift = off % wordsPerBitmapWord; | 534 shift = off % wordsPerBitmapWord; |
582 xbits = *bitp; | 535 xbits = *bitp; |
583 bits = xbits >> shift; | 536 bits = xbits >> shift; |
584 | 537 |
585 found2: | 538 found2: |
586 // Now we have bits, bitp, and shift correct for | 539 // Now we have bits, bitp, and shift correct for |
587 // obj pointing at the base of the object. | 540 // obj pointing at the base of the object. |
588 // Only care about allocated and not marked. | 541 // Only care about allocated and not marked. |
589 if((bits & (bitAllocated|bitMarked)) != bitAllocated) | 542 if((bits & (bitAllocated|bitMarked)) != bitAllocated) |
590 continue; | 543 continue; |
591 | 544 » » » if(work.nproc == 1) |
592 » » » *ybufpos = (struct Y){bitp, bits, shift, obj, gc}; | 545 » » » » *bitp |= bitMarked<<shift; |
593 » » » ybufpos++; | 546 » » » else { |
594 » » } | 547 » » » » for(;;) { |
595 | 548 » » » » » x = *bitp; |
596 » » runtime·lock(&lock); | 549 » » » » » if(x & (bitMarked<<shift)) |
597 » » struct Y *y; | 550 » » » » » » goto continue_obj; |
598 » » for(y=ybuf; y<ybufpos; y++){ | 551 » » » » » if(runtime·casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift)))) |
599 » » » *y->bitp |= bitMarked << y->shift; | 552 » » » » » » break; |
553 » » » » } | |
554 » » » } | |
600 | 555 |
601 // If object has no pointers, don't need to scan further . | 556 // If object has no pointers, don't need to scan further . |
602 » » » if((y->bits & bitNoPointers) != 0) | 557 » » » if((bits & bitNoPointers) != 0) |
603 continue; | 558 continue; |
604 | 559 |
605 // Ask span about size class. | 560 // Ask span about size class. |
606 // (Manually inlined copy of MHeap_Lookup.) | 561 // (Manually inlined copy of MHeap_Lookup.) |
607 » » » x = (uintptr)y->obj >> PageShift; | 562 » » » x = (uintptr)obj >> PageShift; |
608 if(sizeof(void*) == 8) | 563 if(sizeof(void*) == 8) |
609 x -= (uintptr)arena_start>>PageShift; | 564 x -= (uintptr)arena_start>>PageShift; |
610 » » » s = runtime·mheap.map[x]; | 565 » » » s = runtime·mheap->map[x]; |
611 | 566 |
612 » » » *wp = (Obj){y->obj, s->elemsize, y->gc}; | 567 » » » PREFETCH(obj); |
568 | |
569 » » » *wp = (Obj){obj, s->elemsize, ti}; | |
613 wp++; | 570 wp++; |
614 nobj++; | 571 nobj++; |
615 » » } | 572 » » continue_obj:; |
616 » » runtime·unlock(&lock); | 573 » » } |
617 » } | 574 |
618 | 575 » » // If another proc wants a pointer, give it some. |
619 » // If another proc wants pointers, give it some. | 576 » » if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) { |
620 » if(nobj > handoffThreshold && work.nwait > 0 && work.full == nil) { | 577 » » » wbuf->nobj = nobj; |
621 » » wbuf->nobj = nobj; | 578 » » » wbuf = handoff(wbuf); |
622 » » wbuf = handoff(wbuf); | 579 » » » nobj = wbuf->nobj; |
623 » » nobj = wbuf->nobj; | 580 » » » wp = wbuf->obj + nobj; |
624 » » wp = wbuf->obj + nobj; | 581 » » } |
625 } | 582 } |
626 | 583 |
627 *_wp = wp; | 584 *_wp = wp; |
628 *_wbuf = wbuf; | 585 *_wbuf = wbuf; |
629 *_nobj = nobj; | 586 *_nobj = nobj; |
630 } | 587 } |
631 | 588 |
632 static void | 589 static void |
633 flushobjbuf(Obj *objbuf, uintptr n, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj) | 590 flushobjbuf(Obj *objbuf, Obj **objbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_ nobj) |
634 { | 591 { |
635 » uintptr nobj, off, i; | 592 » uintptr nobj, off; |
636 Obj *wp, obj; | 593 Obj *wp, obj; |
637 Workbuf *wbuf; | 594 Workbuf *wbuf; |
638 | 595 » Obj *objbuf_end; |
639 » // Get the state of buffer | 596 |
640 wp = *_wp; | 597 wp = *_wp; |
641 wbuf = *_wbuf; | 598 wbuf = *_wbuf; |
642 nobj = *_nobj; | 599 nobj = *_nobj; |
643 | 600 |
644 » for(i=0; i<n; i++) { | 601 » objbuf_end = *objbufpos; |
645 » » obj = objbuf[i]; | 602 » *objbufpos = objbuf; |
603 | |
604 » while(objbuf < objbuf_end) { | |
605 » » obj = *objbuf++; | |
646 | 606 |
647 // Align obj.b to a word boundary. | 607 // Align obj.b to a word boundary. |
648 off = (uintptr)obj.p & (PtrSize-1); | 608 off = (uintptr)obj.p & (PtrSize-1); |
649 if(off != 0) { | 609 if(off != 0) { |
650 obj.p += PtrSize - off; | 610 obj.p += PtrSize - off; |
651 obj.n -= PtrSize - off; | 611 obj.n -= PtrSize - off; |
652 » » » obj.gc = 0; | 612 » » » obj.ti = 0; |
653 } | 613 } |
654 | 614 |
655 if(obj.p == nil || obj.n == 0) | 615 if(obj.p == nil || obj.n == 0) |
656 continue; | 616 continue; |
657 | 617 |
658 // If buffer is full, get a new one. | 618 // If buffer is full, get a new one. |
659 if(wbuf == nil || nobj >= nelem(wbuf->obj)) { | 619 if(wbuf == nil || nobj >= nelem(wbuf->obj)) { |
660 if(wbuf != nil) | 620 if(wbuf != nil) |
661 wbuf->nobj = nobj; | 621 wbuf->nobj = nobj; |
662 wbuf = getempty(wbuf); | 622 wbuf = getempty(wbuf); |
663 wp = wbuf->obj; | 623 wp = wbuf->obj; |
664 nobj = 0; | 624 nobj = 0; |
665 } | 625 } |
666 | 626 |
667 *wp = obj; | 627 *wp = obj; |
668 wp++; | 628 wp++; |
669 nobj++; | 629 nobj++; |
670 } | 630 } |
671 | 631 |
672 // If another proc wants a pointer, give it some. | 632 // If another proc wants a pointer, give it some. |
673 » if(nobj > handoffThreshold && work.nwait > 0 && work.full == nil) { | 633 » if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) { |
674 wbuf->nobj = nobj; | 634 wbuf->nobj = nobj; |
675 wbuf = handoff(wbuf); | 635 wbuf = handoff(wbuf); |
676 nobj = wbuf->nobj; | 636 nobj = wbuf->nobj; |
677 wp = wbuf->obj + nobj; | 637 wp = wbuf->obj + nobj; |
678 } | 638 } |
679 | 639 |
680 // Update the state of buffer | |
681 *_wp = wp; | 640 *_wp = wp; |
682 *_wbuf = wbuf; | 641 *_wbuf = wbuf; |
683 *_nobj = nobj; | 642 *_nobj = nobj; |
684 } | 643 } |
644 | |
645 // Program that scans the whole block and treats every block element as a potent ial pointer | |
646 static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR}; | |
647 | |
648 // Hashmap iterator program | |
649 static uintptr mapProg[2] = {0, GC_MAP_NEXT}; | |
650 | |
651 // Hchan program | |
652 static uintptr chanProg[2] = {0, GC_CHAN}; | |
653 | |
654 // Local variables of a program fragment or loop | |
655 typedef struct Frame Frame; | |
656 struct Frame { | |
657 uintptr count, elemsize, b; | |
658 uintptr *loop_or_ret; | |
659 }; | |
660 | |
661 // Sanity check for the derived type info objti. | |
662 static void | |
663 checkptr(void *obj, uintptr objti) | |
664 { | |
665 uintptr *pc1, *pc2, type, tisize, i, j, x; | |
666 byte *objstart; | |
667 Type *t; | |
668 MSpan *s; | |
669 | |
670 if(!Debug) | |
671 runtime·throw("checkptr is debug only"); | |
672 | |
673 if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used) | |
674 return; | |
675 type = runtime·gettype(obj); | |
676 t = (Type*)(type & ~(uintptr)(PtrSize-1)); | |
677 if(t == nil) | |
678 return; | |
679 x = (uintptr)obj >> PageShift; | |
680 if(sizeof(void*) == 8) | |
681 x -= (uintptr)(runtime·mheap->arena_start)>>PageShift; | |
682 s = runtime·mheap->map[x]; | |
683 objstart = (byte*)((uintptr)s->start<<PageShift); | |
684 if(s->sizeclass != 0) { | |
685 i = ((byte*)obj - objstart)/s->elemsize; | |
686 objstart += i*s->elemsize; | |
687 } | |
688 tisize = *(uintptr*)objti; | |
689 // Sanity check for object size: it should fit into the memory block. | |
690 if((byte*)obj + tisize > objstart + s->elemsize) | |
691 runtime·throw("invalid gc type info"); | |
692 if(obj != objstart) | |
693 return; | |
694 // If obj points to the beginning of the memory block, | |
695 // check type info as well. | |
696 if(t->string == nil || | |
697 // Gob allocates unsafe pointers for indirection. | |
698 (runtime·strcmp(t->string->str, (byte*)"unsafe.Pointer") && | |
699 // Runtime and gc think differently about closures. | |
700 runtime·strstr(t->string->str, (byte*)"struct { F uintptr") != t ->string->str)) { | |
701 pc1 = (uintptr*)objti; | |
702 pc2 = (uintptr*)t->gc; | |
703 // A simple best-effort check until first GC_END. | |
704 for(j = 1; pc1[j] != GC_END && pc2[j] != GC_END; j++) { | |
705 if(pc1[j] != pc2[j]) { | |
706 runtime·printf("invalid gc type info for '%s' at %p, type info %p, block info %p\n", | |
707 t->string ? (int8*)t->string->str : (int 8*)"?", j, pc1[j], pc2[j]); | |
708 runtime·throw("invalid gc type info"); | |
709 } | |
710 } | |
711 } | |
712 }······································· | |
685 | 713 |
686 // scanblock scans a block of n bytes starting at pointer b for references | 714 // scanblock scans a block of n bytes starting at pointer b for references |
687 // to other objects, scanning any it finds recursively until there are no | 715 // to other objects, scanning any it finds recursively until there are no |
688 // unscanned objects left. Instead of using an explicit recursion, it keeps | 716 // unscanned objects left. Instead of using an explicit recursion, it keeps |
689 // a work list in the Workbuf* structures and loops in the main function | 717 // a work list in the Workbuf* structures and loops in the main function |
690 // body. Keeping an explicit work list is easier on the stack allocator and | 718 // body. Keeping an explicit work list is easier on the stack allocator and |
691 // more efficient. | 719 // more efficient. |
720 // | |
721 // wbuf: current work buffer | |
722 // wp: storage for next queued pointer (write pointer) | |
723 // nobj: number of queued objects | |
692 static void | 724 static void |
693 scan(Obj **_wp, Workbuf **_wbuf, uintptr *_nobj) | 725 scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) |
694 { | 726 { |
695 » byte *b, *arena_start, *arena_used; | 727 » byte *b, *b2, *arena_start, *arena_used, *p; |
696 » uintptr n, i, nobj; | 728 » uintptr n, i, end_b, elemsize, size, ti, objti, count, type; |
697 » Obj *wp; | 729 » uintptr *pc, precise_type, nominal_size; |
698 » Workbuf *wbuf; | 730 » uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti, *chan_ ret; |
699 » bool keepworking; | 731 » MSpan *s; |
700 » uintptr gc; | 732 » int8 *prefix; |
733 » void *obj; | |
734 » Type *t; | |
735 » Slice *sliceptr; | |
736 » Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4]; | |
737 » BufferList *scanbuffers; | |
738 » PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos; | |
739 » Obj *objbuf, *objbuf_end, *objbufpos; | |
740 » Eface *eface; | |
741 » Iface *iface; | |
742 » Hmap *hmap; | |
743 » MapType *maptype; | |
744 » bool mapkey_kind, mapval_kind; | |
745 » struct hash_gciter map_iter; | |
746 » struct hash_gciter_data d; | |
747 » Hchan *chan; | |
748 » ChanType *chantype; | |
701 | 749 |
702 if(sizeof(Workbuf) % PageSize != 0) | 750 if(sizeof(Workbuf) % PageSize != 0) |
703 » » runtime·throw("scan: size of Workbuf is suboptimal"); | 751 » » runtime·throw("scanblock: size of Workbuf is suboptimal"); |
704 | 752 |
705 // Memory arena parameters. | 753 // Memory arena parameters. |
706 » arena_start = runtime·mheap.arena_start; | 754 » arena_start = runtime·mheap->arena_start; |
707 » arena_used = runtime·mheap.arena_used; | 755 » arena_used = runtime·mheap->arena_used; |
708 | 756 |
709 » // Get the state of buffer»····· | 757 » stack_ptr = stack+nelem(stack)-1; |
710 » wp = *_wp; // storage for next queued pointer (write pointer) | |
711 » wbuf = *_wbuf; // current work buffer | |
712 » nobj = *_nobj; // number of queued objects | |
713 | |
714 » // Scanblock helpers pass *_wbuf==nil. | |
715 » // The main proc needs to return to make more | |
716 » // calls to scanblock. But if work.nproc==1 then | |
717 » // might as well process blocks as soon as we | |
718 » // have them. | |
719 » keepworking = (wbuf == nil || work.nproc == 1); | |
720 | |
721 » struct Frame {uintptr count, elemsize, b; uintptr *loop_or_ret;}; | |
722 » struct Frame stack[8]; | |
723 » struct Frame *stack_ptr = stack+nelem(stack)-1; | |
724 » struct Frame stack_top; | |
725 ········ | 758 ········ |
726 » // Program that scans the whole block and treats every block element as a potential pointer | 759 » precise_type = false; |
727 » uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR}; | 760 » nominal_size = 0; |
728 » uintptr mapProg[2] = {0, GC_MAP_NEXT}; | 761 |
729 | 762 » // Allocate ptrbuf |
730 » uintptr *pc; | |
731 » uintptr precise_type = false; | |
732 » uintptr nominal_size = 0; | |
733 | |
734 » // Get buffers | |
735 » struct BufferList *scanbuffers; | |
736 » struct X *xbuf, *xbuf_end; | |
737 » struct Y *ybuf; | |
738 » struct Obj *objbuf, *objbuf_end; | |
739 { | 763 { |
740 » » runtime·lock(&lock); | 764 » » scanbuffers = &bufferList[m->helpgc]; |
741 | 765 » » ptrbuf = &scanbuffers->ptrtarget[0]; |
742 » » if(bufferList == nil) { | 766 » » ptrbuf_end = &scanbuffers->ptrtarget[0] + nelem(scanbuffers->ptr target); |
743 » » » bufferList = runtime·SysAlloc(sizeof(*bufferList)); | |
744 » » » bufferList->next = nil; | |
745 » » } | |
746 » » scanbuffers = bufferList; | |
747 » » bufferList = bufferList->next; | |
748 » »······· | |
749 » » xbuf = &scanbuffers->x[0]; | |
750 » » xbuf_end = &scanbuffers->x[0] + nelem(scanbuffers->x); | |
751 » » ybuf = &scanbuffers->y[0]; | |
752 objbuf = &scanbuffers->obj[0]; | 767 objbuf = &scanbuffers->obj[0]; |
753 objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj); | 768 objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj); |
754 | 769 » } |
755 » » runtime·unlock(&lock); | 770 |
756 » } | 771 » ptrbufpos = ptrbuf; |
757 | 772 » objbufpos = objbuf; |
758 » struct X *xbufpos = xbuf; | |
759 » struct Obj *objbufpos = objbuf; | |
760 | |
761 » uintptr *map_ret; | |
762 » struct hash_gciter map_iter; | |
763 » uintptr mapkey_size, mapval_size; | |
764 » bool mapkey_kind, mapval_kind; | |
765 » uintptr mapkey_gc, mapval_gc; | |
766 | 773 |
767 // (Silence the compiler) | 774 // (Silence the compiler) |
768 map_ret = nil; | 775 map_ret = nil; |
769 mapkey_size = mapval_size = 0; | 776 mapkey_size = mapval_size = 0; |
770 mapkey_kind = mapval_kind = false; | 777 mapkey_kind = mapval_kind = false; |
771 » mapkey_gc = mapval_gc = 0; | 778 » mapkey_ti = mapval_ti = 0; |
772 | 779 » chan = nil; |
773 » // Get type C pointer to the Go type "itab" | 780 » chantype = nil; |
774 » Type *itabtype; | 781 » chan_ret = nil; |
775 » { | 782 |
776 » » Eface e; | 783 » if(!PreciseGC) |
777 » » runtime·gc_itab_ptr(&e); | 784 » » goto next_block_conservative; |
778 » » itabtype = ((PtrType*)e.type)->elem; | |
779 » } | |
780 | 785 |
781 goto next_block; | 786 goto next_block; |
782 | 787 |
783 for(;;) { | 788 for(;;) { |
784 // Each iteration scans the block b of length n, queueing pointe rs in | 789 // Each iteration scans the block b of length n, queueing pointe rs in |
785 // the work buffer. | 790 // the work buffer. |
786 » » if(Debug > 1) | 791 » » if(Debug > 1) { |
787 runtime·printf("scanblock %p %D\n", b, (int64)n); | 792 runtime·printf("scanblock %p %D\n", b, (int64)n); |
788 | 793 » » } |
789 » » if(DebugStats) { | 794 |
790 » » » runtime·xadd64(&gcstats.nobj_sum, nobj); | 795 » » if(CollectStats) { |
791 » » » runtime·xadd64(&gcstats.nobj_cnt, 1); | 796 » » » runtime·xadd64(&gcstats.nbytes, n); |
792 » » } | 797 » » » runtime·xadd64(&gcstats.obj.sum, nobj); |
793 | 798 » » » runtime·xadd64(&gcstats.obj.cnt, 1); |
794 » » if(gc != 0) { | 799 » » } |
795 » » » pc = (uintptr*)(gc & ~(uintptr)1); | 800 |
796 » » » precise_type = (gc & 1); | 801 » » if(ti != 0) { |
797 » » » nominal_size = pc[0]; | 802 » » » pc = (uintptr*)(ti & ~(uintptr)PC_BITS); |
798 » » » stack_top.count = 1; | 803 » » » precise_type = (ti & PRECISE); |
799 » » » stack_top.elemsize = nominal_size; | 804 » » » stack_top.elemsize = pc[0]; |
805 » » » if(!precise_type) | |
806 » » » » nominal_size = pc[0]; | |
807 » » » if(ti & LOOP) { | |
808 » » » » stack_top.count = 0;» // 0 means an infinite n umber of iterations | |
809 » » » » stack_top.loop_or_ret = pc+1; | |
810 » » » } else { | |
811 » » » » stack_top.count = 1; | |
812 » » » } | |
813 » » » if(Debug) { | |
814 » » » » // Simple sanity check for provided type info ti : | |
815 » » » » // The declared size of the object must be not l arger than the actual size | |
816 » » » » // (it can be smaller due to inferior pointers). | |
817 » » » » // It's difficult to make a comprehensive check due to inferior pointers, | |
818 » » » » // reflection, gob, etc. | |
819 » » » » if(pc[0] > n) { | |
820 » » » » » runtime·printf("invalid gc type info: ty pe info size %p, block size %p\n", pc[0], n); | |
821 » » » » » runtime·throw("invalid gc type info"); | |
822 » » » » } | |
823 » » » } | |
800 } else if(UseSpanType) { | 824 } else if(UseSpanType) { |
801 » » » if(DebugStats) | 825 » » » if(CollectStats) |
802 » » » » runtime·xadd64(&gcstats.nobj_notype, 1); | 826 » » » » runtime·xadd64(&gcstats.obj.notype, 1); |
803 | 827 |
804 » » » uintptr typeinfo = runtime·gettype(b); | 828 » » » type = runtime·gettype(b); |
805 | 829 » » » if(type != 0) { |
806 » » » if(typeinfo != 0) { | 830 » » » » if(CollectStats) |
807 » » » » if(DebugStats) | 831 » » » » » runtime·xadd64(&gcstats.obj.typelookup, 1); |
808 » » » » » runtime·xadd64(&gcstats.nobj_typelookups _ok, 1); | 832 |
809 | 833 » » » » t = (Type*)(type & ~(uintptr)(PtrSize-1)); |
810 » » » » Type *t = (Type*)(typeinfo & ~(uintptr)3); | 834 » » » » switch(type & (PtrSize-1)) { |
811 | |
812 » » » » switch(typeinfo & (PtrSize-1)) { | |
813 case TypeInfo_SingleObject: | 835 case TypeInfo_SingleObject: |
814 pc = (uintptr*)t->gc; | 836 pc = (uintptr*)t->gc; |
815 precise_type = true; // type informatio n about 'b' is precise | 837 precise_type = true; // type informatio n about 'b' is precise |
816 stack_top.count = 1; | 838 stack_top.count = 1; |
817 stack_top.elemsize = pc[0]; | 839 stack_top.elemsize = pc[0]; |
818 break; | 840 break; |
819 case TypeInfo_Array: | 841 case TypeInfo_Array: |
820 pc = (uintptr*)t->gc; | 842 pc = (uintptr*)t->gc; |
821 if(pc[0] == 0) | 843 if(pc[0] == 0) |
822 goto next_block; | 844 goto next_block; |
823 precise_type = true; // type informatio n about 'b' is precise | 845 precise_type = true; // type informatio n about 'b' is precise |
824 » » » » » stack_top.count = 0; // 0 means an "inf inite" number of iterations | 846 » » » » » stack_top.count = 0; // 0 means an infi nite number of iterations |
825 stack_top.elemsize = pc[0]; | 847 stack_top.elemsize = pc[0]; |
826 stack_top.loop_or_ret = pc+1; | 848 stack_top.loop_or_ret = pc+1; |
827 break; | 849 break; |
828 » » » » case TypeInfo_Map: { | 850 » » » » case TypeInfo_Map: |
829 » » » » » Hmap *h = (Hmap*)b; | 851 » » » » » hmap = (Hmap*)b; |
rsc
2012/05/24 16:36:45
Please move variables to top, avoiding { } around
atom
2012/06/01 16:45:47
Done.
| |
830 » » » » » MapType *mt = (MapType*)t; | 852 » » » » » maptype = (MapType*)t; |
831 » » » » » bool ok = hash_gciter_init(h, &map_iter) ; | 853 » » » » » if(hash_gciter_init(hmap, &map_iter)) { |
832 » » » » » if(ok) { | 854 » » » » » » mapkey_size = maptype->key->size ; |
833 » » » » » » mapkey_size = mt->key->size; | 855 » » » » » » mapkey_kind = maptype->key->kind ; |
834 » » » » » » mapkey_kind = mt->key->kind; | 856 » » » » » » mapkey_ti = (uintptr)maptype-> key->gc | PRECISE; |
835 » » » » » » mapkey_gc = (uintptr)mt->key-> gc | 1; | 857 » » » » » » mapval_size = maptype->elem->siz e; |
836 » » » » » » mapval_size = mt->elem->size; | 858 » » » » » » mapval_kind = maptype->elem->kin d; |
837 » » » » » » mapval_kind = mt->elem->kind; | 859 » » » » » » mapval_ti = (uintptr)maptype-> elem->gc | PRECISE; |
838 » » » » » » mapval_gc = (uintptr)mt->elem- >gc | 1; | 860 |
839 | 861 » » » » » » map_ret = nil; |
840 » » » » » » // Use mapProg. | |
841 » » » » » » map_ret = 0; | |
842 pc = mapProg; | 862 pc = mapProg; |
843 } else { | 863 } else { |
844 goto next_block; | 864 goto next_block; |
845 } | 865 } |
846 break; | 866 break; |
847 » » » » } | 867 » » » » case TypeInfo_Chan: |
868 » » » » » chan = (Hchan*)b; | |
869 » » » » » chantype = (ChanType*)t; | |
870 » » » » » chan_ret = nil; | |
871 » » » » » pc = chanProg; | |
872 » » » » » break; | |
848 default: | 873 default: |
849 » » » » » runtime·throw("scanblock: invalid typein fo"); | 874 » » » » » runtime·throw("scanblock: invalid type") ; |
850 return; | 875 return; |
851 } | 876 } |
852 } else { | 877 } else { |
853 pc = defaultProg; | 878 pc = defaultProg; |
854 } | 879 } |
855 } else { | 880 } else { |
856 pc = defaultProg; | 881 pc = defaultProg; |
857 } | 882 } |
858 | 883 |
884 if(IgnorePreciseGC) | |
885 pc = defaultProg; | |
886 | |
859 if(DebugTypeAtBlockEnd) { | 887 if(DebugTypeAtBlockEnd) { |
860 » » » MSpan *s = runtime·MHeap_LookupMaybe(&runtime·mheap, b); | 888 » » » s = runtime·MHeap_LookupMaybe(runtime·mheap, b); |
861 | 889 |
862 if(s != nil) { | 890 if(s != nil) { |
863 » » » » uintptr elemsize = s->elemsize; | 891 » » » » elemsize = s->elemsize; |
864 » » » » byte *b2; | 892 » » » » p = (byte*)((uintptr)s->start<<PageShift); |
865 if(s->sizeclass == 0) { | 893 if(s->sizeclass == 0) { |
866 » » » » » b2 = b; | 894 » » » » » b2 = p; |
867 } | 895 } |
868 else { | 896 else { |
869 » » » » » byte *p = (byte*)((uintptr)s->start<<Pag eShift); | 897 » » » » » i = (b-p)/elemsize; |
870 » » » » » uintptr i = (b-p)/elemsize; | |
871 b2 = p+i*elemsize; | 898 b2 = p+i*elemsize; |
872 } | 899 } |
873 | 900 |
874 » » » » uintptr typeinfo = *(uintptr*)(b2+elemsize-sizeo f(uintptr)); | 901 » » » » type = *(uintptr*)(b2+elemsize-sizeof(uintptr)); |
875 » » » » if(typeinfo != 0) { | 902 » » » » if(type != 0) { |
876 » » » » » Type *type = (Type*)(typeinfo & ~(uintpt r)(PtrSize-1)); | 903 » » » » » t = (Type*)(type & ~(uintptr)(PtrSize-1) ); |
877 | 904 |
878 » » » » » const int8 *prefix = ""; | 905 » » » » » prefix = ""; |
879 » » » » » switch(typeinfo & (PtrSize-1)) { | 906 » » » » » switch(type & (PtrSize-1)) { |
880 case TypeInfo_SingleObject: | 907 case TypeInfo_SingleObject: |
881 prefix = "single"; | 908 prefix = "single"; |
882 break; | 909 break; |
883 case TypeInfo_Array: | 910 case TypeInfo_Array: |
884 prefix = "array"; | 911 prefix = "array"; |
885 break; | 912 break; |
886 } | 913 } |
887 | 914 |
888 » » » » » runtime·printf("scan %p+%X %D, type %s % S, pc %p\n", | 915 » » » » » runtime·lock(&lock); |
889 » » » » » » b, (int64)(b-b2), (int64)n, pref ix, *type->string, (pc==defaultProg ? nil : pc)); | 916 » » » » » runtime·printf("scanblock %p+%X %D, type %s %S, pc %p\n", |
917 » » » » » » b, (int64)(b-b2), (int64)n, pref ix, *t->string, (pc==defaultProg ? nil : pc)); | |
918 » » » » » runtime·unlock(&lock); | |
890 } else { | 919 } else { |
891 » » » » » runtime·printf("scan %p+%X %D, pc %p\n", | 920 » » » » » runtime·lock(&lock); |
921 » » » » » runtime·printf("scanblock %p+%X %D, pc % p\n", | |
892 b, (int64)(b-b2), (int64)n, (pc= =defaultProg ? nil : pc)); | 922 b, (int64)(b-b2), (int64)n, (pc= =defaultProg ? nil : pc)); |
923 runtime·unlock(&lock); | |
893 } | 924 } |
894 } else { | 925 } else { |
895 » » » » runtime·printf("scan %p %D, not in heap, pc %p\n ", | 926 » » » » runtime·lock(&lock); |
927 » » » » runtime·printf("scanblock %p %D, not in heap, pc %p\n", | |
896 b, (int64)n, (pc==defaultProg ? nil : pc )); | 928 b, (int64)n, (pc==defaultProg ? nil : pc )); |
929 runtime·unlock(&lock); | |
897 } | 930 } |
898 } | 931 } |
899 | 932 |
900 if(DebugType) { | 933 if(DebugType) { |
901 if(pc[0] > n && pc != defaultProg) | 934 if(pc[0] > n && pc != defaultProg) |
902 » » » » runtime·throw("scan: size exceeds n"); | 935 » » » » runtime·throw("scanblock: size exceeds n"); |
903 if(pc[0] % PtrSize != 0) | 936 if(pc[0] % PtrSize != 0) |
904 » » » » runtime·throw("scan: invalid alignment of size") ; | 937 » » » » runtime·throw("scanblock: invalid alignment of s ize"); |
905 if(n < PtrSize) | 938 if(n < PtrSize) |
906 » » » » runtime·throw("scan: n < PtrSize"); | 939 » » » » runtime·throw("scanblock: n < PtrSize"); |
907 } | 940 } |
908 | 941 |
909 pc++; | 942 pc++; |
910 stack_top.b = (uintptr)b; | 943 stack_top.b = (uintptr)b; |
911 | 944 |
912 » » uintptr end_b = (uintptr)b + n - PtrSize; | 945 » » end_b = (uintptr)b + n - PtrSize; |
913 if(DebugTypeAtBlockEnd) { | 946 if(DebugTypeAtBlockEnd) { |
914 if(b >= arena_start && b+n <= arena_used && pc != mapPro g+1) | 947 if(b >= arena_start && b+n <= arena_used && pc != mapPro g+1) |
915 end_b -= PtrSize; | 948 end_b -= PtrSize; |
916 } | 949 } |
917 | 950 |
918 » next_instr: | 951 » for(;;) { |
919 » » if(DebugStats) | 952 » » if(CollectStats) |
920 runtime·xadd64(&gcstats.instr[pc[0]], 1); | 953 runtime·xadd64(&gcstats.instr[pc[0]], 1); |
954 | |
955 obj = nil; | |
956 objti = 0; | |
921 switch(pc[0]) { | 957 switch(pc[0]) { |
922 » » » case GC_PTR: { | 958 » » case GC_PTR: |
923 » » » » void *obj = *(void**)(stack_top.b + pc[1]); | 959 » » » obj = *(void**)(stack_top.b + pc[1]); |
924 » » » » void *objgc = (void*)pc[2]; | 960 » » » objti = pc[2]; |
925 » » » » pc += 3; | 961 » » » pc += 3; |
926 | 962 » » » if(Debug) |
963 » » » » checkptr(obj, objti); | |
964 » » » break; | |
965 | |
966 » » case GC_SLICE: | |
967 » » » sliceptr = (Slice*)(stack_top.b + pc[1]); | |
968 » » » if(sliceptr->cap != 0) { | |
969 » » » » obj = sliceptr->array; | |
970 » » » » // Can't use slice element type for scanning, | |
971 » » » » // because if it points to an array embedded | |
972 » » » » // in the beginning of a struct, | |
973 » » » » // we will scan the whole struct as the slice. | |
974 » » » » // So just obtain type info from heap. | |
975 » » » } | |
976 » » » pc += 3; | |
977 » » » break; | |
978 | |
979 » » case GC_APTR: | |
980 » » » obj = *(void**)(stack_top.b + pc[1]); | |
981 » » » pc += 2; | |
982 » » » break; | |
983 | |
984 » » case GC_STRING: | |
985 » » » obj = *(void**)(stack_top.b + pc[1]); | |
986 » » » markonly(obj); | |
987 » » » pc += 2; | |
988 » » » continue; | |
989 | |
990 » » case GC_EFACE: | |
991 » » » eface = (Eface*)(stack_top.b + pc[1]); | |
992 » » » pc += 2; | |
993 » » » if(eface->type == nil) | |
994 » » » » continue; | |
995 | |
996 » » » // eface->type | |
997 » » » t = eface->type; | |
998 » » » if((void*)t >= arena_start && (void*)t < arena_used) { | |
999 » » » » *ptrbufpos++ = (PtrTarget){t, 0}; | |
1000 » » » » if(ptrbufpos == ptrbuf_end) | |
1001 » » » » » flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wb uf, &nobj); | |
1002 » » » } | |
1003 | |
1004 » » » // eface->data | |
1005 » » » if(eface->data >= arena_start && eface->data < arena_use d) { | |
1006 » » » » if(t->size <= sizeof(void*)) { | |
1007 » » » » » if((t->kind & KindNoPointers)) | |
1008 » » » » » » continue; | |
1009 | |
1010 » » » » » obj = eface->data; | |
1011 » » » » » if((t->kind & ~KindNoPointers) == KindPt r) | |
1012 » » » » » » objti = (uintptr)((PtrType*)t)-> elem->gc; | |
1013 » » » » } else { | |
1014 » » » » » obj = eface->data; | |
1015 » » » » » objti = (uintptr)t->gc; | |
1016 » » » » } | |
1017 » » » } | |
1018 » » » break; | |
1019 | |
1020 » » case GC_IFACE: | |
1021 » » » iface = (Iface*)(stack_top.b + pc[1]); | |
1022 » » » pc += 2; | |
1023 » » » if(iface->tab == nil) | |
1024 » » » » continue; | |
1025 » » »······· | |
1026 » » » // iface->tab | |
1027 » » » if((void*)iface->tab >= arena_start && (void*)iface->tab < arena_used) { | |
1028 » » » » *ptrbufpos++ = (PtrTarget){iface->tab, (uintptr) itabtype->gc}; | |
1029 » » » » if(ptrbufpos == ptrbuf_end) | |
1030 » » » » » flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wb uf, &nobj); | |
1031 » » » } | |
1032 | |
1033 » » » // iface->data | |
1034 » » » if(iface->data >= arena_start && iface->data < arena_use d) { | |
1035 » » » » t = iface->tab->type; | |
1036 » » » » if(t->size <= sizeof(void*)) { | |
1037 » » » » » if((t->kind & KindNoPointers)) | |
1038 » » » » » » continue; | |
1039 | |
1040 » » » » » obj = iface->data; | |
1041 » » » » » if((t->kind & ~KindNoPointers) == KindPt r) | |
1042 » » » » » » objti = (uintptr)((PtrType*)t)-> elem->gc; | |
1043 » » » » } else { | |
1044 » » » » » obj = iface->data; | |
1045 » » » » » objti = (uintptr)t->gc; | |
1046 » » » » } | |
1047 » » » } | |
1048 » » » break; | |
1049 | |
1050 » » case GC_DEFAULT_PTR: | |
1051 » » » while(stack_top.b <= end_b) { | |
1052 » » » » obj = *(byte**)stack_top.b; | |
1053 » » » » stack_top.b += PtrSize; | |
927 if(obj >= arena_start && obj < arena_used) { | 1054 if(obj >= arena_start && obj < arena_used) { |
928 » » » » » *xbufpos = (struct X){obj, (uintptr)objg c}; | 1055 » » » » » *ptrbufpos++ = (PtrTarget){obj, 0}; |
929 » » » » » xbufpos++; | 1056 » » » » » if(ptrbufpos == ptrbuf_end) |
930 » » » » » if(xbufpos == xbuf_end) | 1057 » » » » » » flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); |
931 » » » » » » goto flush_buffers; | |
932 } | 1058 } |
933 » » » » goto next_instr; | 1059 » » » } |
934 » » » } | 1060 » » » goto next_block; |
935 » » » case GC_APTR: { | 1061 |
936 » » » » void *obj = *(void**)(stack_top.b + pc[1]); | 1062 » » case GC_END: |
937 » » » » pc += 2; | 1063 » » » if(--stack_top.count != 0) { |
938 | 1064 » » » » // Next iteration of a loop if possible. |
939 » » » » if(obj >= arena_start && obj < arena_used) { | 1065 » » » » stack_top.b += stack_top.elemsize; |
940 » » » » » *xbufpos = (struct X){obj, 0}; | 1066 » » » » if(stack_top.b + stack_top.elemsize <= end_b+Ptr Size) { |
941 » » » » » xbufpos++; | 1067 » » » » » pc = stack_top.loop_or_ret; |
942 » » » » » if(xbufpos == xbuf_end) | 1068 » » » » » continue; |
943 » » » » » » goto flush_buffers; | |
944 } | 1069 } |
945 » » » » goto next_instr; | 1070 » » » » i = stack_top.b; |
946 » » » } | 1071 » » » } else { |
947 » » » case GC_STRING: { | 1072 » » » » // Stack pop if possible. |
948 » » » » void *obj = *(void**)(stack_top.b + pc[1]); | 1073 » » » » if(stack_ptr+1 < stack+nelem(stack)) { |
949 » » » » pc += 2; | 1074 » » » » » pc = stack_top.loop_or_ret; |
950 | 1075 » » » » » stack_top = *(++stack_ptr); |
951 » » » » if(obj >= arena_start && obj < arena_used) { | 1076 » » » » » continue; |
952 » » » » » *xbufpos = (struct X){obj, 0}; | |
953 » » » » » xbufpos++; | |
954 » » » » » if(xbufpos == xbuf_end) | |
955 » » » » » » goto flush_buffers; | |
956 } | 1077 } |
957 » » » » goto next_instr; | 1078 » » » » i = (uintptr)b + nominal_size; |
958 » » » } | 1079 » » » } |
959 » » » case GC_EFACE: { | 1080 » » » if(!precise_type) { |
960 » » » » Eface *e = (Eface*)(stack_top.b + pc[1]); | 1081 » » » » if(DebugType) { |
961 » » » » pc += 2; | 1082 » » » » » if(i % PtrSize != 0) |
962 | 1083 » » » » » » runtime·throw("scanblock: invali d alignment"); |
963 » » » » if(e->type != nil && (e->data >= arena_start && e->data < arena_used)) { | |
964 » » » » » Type *t = e->type; | |
965 » » » » » if(t->size <= sizeof(void*)) { | |
966 » » » » » » if((t->kind & KindNoPointers)) | |
967 » » » » » » » goto next_instr; | |
968 | |
969 » » » » » » if((t->kind & ~KindNoPointers) = = KindPtr) | |
970 » » » » » » » *xbufpos = (struct X){e- >data, (uintptr)((PtrType*)t)->elem->gc}; | |
971 » » » » » » else | |
972 » » » » » » » *xbufpos = (struct X){e- >data, 0}; | |
973 » » » » » } else { | |
974 » » » » » » *xbufpos = (struct X){e->data, ( uintptr)t->gc}; | |
975 » » » » » } | |
976 » » » » » xbufpos++; | |
977 » » » » » if(xbufpos == xbuf_end) | |
978 » » » » » » goto flush_buffers; | |
979 } | 1084 } |
980 » » » » goto next_instr; | 1085 » » » » // Quickly scan [b+i,b+n) for possible pointers. |
981 » » » } | 1086 » » » » for(; i<=end_b; i+=PtrSize) { |
982 » » » case GC_IFACE: { | 1087 » » » » » if(*(byte**)i != nil) { |
983 » » » » Iface *i = (Iface*)(stack_top.b + pc[1]); | 1088 » » » » » » // Found a value that may be a p ointer. |
984 | 1089 » » » » » » // Do a rescan of the entire blo ck. |
985 » » » » if(i->tab == nil) { | 1090 » » » » » » enqueue((Obj){b, n, 0}, &wbuf, & wp, &nobj); |
986 » » » » » pc += 2; | 1091 » » » » » » if(CollectStats) { |
987 » » » » » goto next_instr; | 1092 » » » » » » » runtime·xadd64(&gcstats. rescan, 1); |
988 » » » » } | 1093 » » » » » » » runtime·xadd64(&gcstats. rescanbytes, n); |
989 | |
990 » » » » if(xbufpos+2+1 > xbuf_end) { | |
991 » » » » » goto flush_buffers; | |
992 » » » » } | |
993 » » » » else { | |
994 » » » » » pc += 2; | |
995 | |
996 » » » » » // i->tab | |
997 » » » » » *xbufpos = (struct X){i->tab, (uintptr)i tabtype->gc}; | |
998 » » » » » xbufpos++; | |
999 | |
1000 » » » » » // i->data | |
1001 » » » » » if(i->data >= arena_start && i->data < a rena_used) { | |
1002 » » » » » » Type *t = i->tab->type; | |
1003 » » » » » » if(t->size <= sizeof(void*)) { | |
1004 » » » » » » » if((t->kind & KindNoPoin ters)) | |
1005 » » » » » » » » goto next_instr; | |
1006 | |
1007 » » » » » » » if((t->kind & ~KindNoPoi nters) == KindPtr) | |
1008 » » » » » » » » *xbufpos = (stru ct X){i->data, (uintptr)((PtrType*)t)->elem->gc}; | |
1009 » » » » » » » else | |
1010 » » » » » » » » *xbufpos = (stru ct X){i->data, 0}; | |
1011 » » » » » » } else { | |
1012 » » » » » » » *xbufpos = (struct X){i- >data, (uintptr)t->gc}; | |
1013 } | 1094 } |
1014 » » » » » » xbufpos++; | 1095 » » » » » » break; |
1015 » » » » » } | |
1016 » » » » » goto next_instr; | |
1017 » » » » } | |
1018 » » » } | |
1019 » » » case GC_DEFAULT_PTR: { | |
1020 » » » » while(true) { | |
1021 » » » » » uintptr i = stack_top.b; | |
1022 » » » » » if(i > end_b) | |
1023 » » » » » » goto next_block; | |
1024 » » » » » stack_top.b += PtrSize; | |
1025 | |
1026 » » » » » void *obj = *(byte**)i; | |
1027 » » » » » if(obj >= arena_start && obj < arena_use d) { | |
1028 » » » » » » *xbufpos = (struct X){obj, 0}; | |
1029 » » » » » » xbufpos++; | |
1030 » » » » » » if(xbufpos == xbuf_end) | |
1031 » » » » » » » goto flush_buffers; | |
1032 » » » » » } | |
1033 | |
1034 » » » » » if(DebugStats) | |
1035 » » » » » » runtime·xadd64(&gcstats.instr[GC _DEFAULT_PTR], 1); | |
1036 » » » » } | |
1037 » » » } | |
1038 » » » case GC_END: { | |
1039 » » » » stack_top.count--; | |
1040 » » » » if(stack_top.count != 0) { | |
1041 » » » » » // Next iteration of a loop | |
1042 » » » » » uintptr elemsize = stack_top.elemsize; | |
1043 » » » » » stack_top.b += elemsize; | |
1044 » » » » » if(stack_top.b + elemsize > end_b+PtrSiz e) { | |
1045 » » » » » » if(precise_type) { | |
1046 » » » » » » » goto next_block; | |
1047 » » » » » » } else { | |
1048 » » » » » » » i = stack_top.b; | |
1049 » » » » » » » goto trailing_pointers; | |
1050 » » » » » » } | |
1051 » » » » » } | |
1052 » » » » » pc = stack_top.loop_or_ret; | |
1053 » » » » » goto next_instr; | |
1054 » » » » } else { | |
1055 » » » » » if(stack_ptr == stack+nelem(stack)-1) { | |
1056 » » » » » » if(precise_type) { | |
1057 » » » » » » » goto next_block; | |
1058 » » » » » » } else { | |
1059 » » » » » » » i = (uintptr)b + nominal _size; | |
1060 » » » » » » » goto trailing_pointers; | |
1061 » » » » » » } | |
1062 » » » » » } else { | |
1063 » » » » » » // Pop from stack | |
1064 » » » » » » pc = stack_top.loop_or_ret; | |
1065 » » » » » » stack_ptr++; | |
1066 » » » » » » stack_top = *stack_ptr; | |
1067 » » » » » » goto next_instr; | |
1068 } | 1096 } |
1069 } | 1097 } |
1070 } | 1098 } |
1071 » » » case GC_ARRAY_START: { | 1099 » » » goto next_block; |
1072 » » » » // Push on stack | 1100 |
1073 » » » » uintptr i0 = stack_top.b + pc[1]; | 1101 » » case GC_ARRAY_START: |
1074 » » » » uintptr count = pc[2]; | 1102 » » » i = stack_top.b + pc[1]; |
1075 » » » » uintptr elemsize = pc[3]; | 1103 » » » count = pc[2]; |
1076 » » » » pc += 4; | 1104 » » » elemsize = pc[3]; |
1077 » » » » *stack_ptr = stack_top; | 1105 » » » pc += 4; |
1078 » » » » stack_ptr--; | 1106 |
1079 » » » » stack_top = (struct Frame){count, elemsize, i0, pc}; | 1107 » » » // Stack push. |
1080 » » » » goto next_instr; | 1108 » » » *stack_ptr-- = stack_top; |
1081 » » » } | 1109 » » » stack_top = (Frame){count, elemsize, i, pc}; |
1082 » » » case GC_ARRAY_NEXT: { | 1110 » » » continue; |
1083 » » » » stack_top.count--; | 1111 |
1084 » » » » if(stack_top.count != 0) { | 1112 » » case GC_ARRAY_NEXT: |
1085 » » » » » stack_top.b += stack_top.elemsize; | 1113 » » » if(--stack_top.count != 0) { |
1086 » » » » » pc = stack_top.loop_or_ret; | 1114 » » » » stack_top.b += stack_top.elemsize; |
1087 » » » » » goto next_instr; | 1115 » » » » pc = stack_top.loop_or_ret; |
1088 » » » » } else { | 1116 » » » } else { |
1089 » » » » » // Pop from stack | 1117 » » » » // Stack pop. |
1090 » » » » » stack_ptr++; | 1118 » » » » stack_top = *(++stack_ptr); |
1091 » » » » » stack_top = *stack_ptr; | 1119 » » » » pc += 1; |
1092 » » » » » pc += 1; | 1120 » » » } |
1093 » » » » » goto next_instr; | 1121 » » » continue; |
1094 » » » » } | 1122 |
1095 » » » } | 1123 » » case GC_CALL: |
1096 » » » case GC_CALL: { | 1124 » » » // Stack push. |
1097 » » » » // Push on stack | 1125 » » » *stack_ptr-- = stack_top; |
1098 » » » » uintptr i0 = stack_top.b + pc[1]; | 1126 » » » stack_top = (Frame){1, 0, stack_top.b + pc[1], pc+3 /*re turn address*/}; |
1099 » » » » uintptr *target = (uintptr*)pc[2]; | 1127 » » » pc = (uintptr*)((byte*)pc + *(int32*)(pc+2)); // target of the CALL instruction |
1100 » » » » uintptr *ret = pc+3; | 1128 » » » continue; |
1101 » » » » *stack_ptr = stack_top; | 1129 |
1102 » » » » stack_ptr--; | 1130 » » case GC_MAP_PTR: |
1103 » » » » stack_top = (struct Frame){1, 0, i0, ret}; | 1131 » » » hmap = *(Hmap**)(stack_top.b + pc[1]); |
1104 » » » » pc = target; | 1132 » » » if(hmap == nil) { |
1105 » » » » goto next_instr; | 1133 » » » » pc += 3; |
1106 » » » } | 1134 » » » » continue; |
1107 » » » case GC_MAP_PTR: { | 1135 » » » } |
1108 » » » » Hmap *h = *(Hmap**)(stack_top.b + pc[1]); | 1136 » » » if(markonly(hmap)) { |
1109 » » » » if(h == nil) { | 1137 » » » » maptype = (MapType*)pc[2]; |
1110 » » » » » pc += 3; | 1138 » » » » if(hash_gciter_init(hmap, &map_iter)) { |
1111 » » » » » goto next_instr; | 1139 » » » » » mapkey_size = maptype->key->size; |
1112 » » » » } | 1140 » » » » » mapkey_kind = maptype->key->kind; |
1113 » » » » runtime·lock(&lock); | 1141 » » » » » mapkey_ti = (uintptr)maptype->key->gc | PRECISE; |
1114 » » » » bool wasnt_marked = markonly(h); | 1142 » » » » » mapval_size = maptype->elem->size; |
1115 » » » » runtime·unlock(&lock); | 1143 » » » » » mapval_kind = maptype->elem->kind; |
1116 » » » » if(wasnt_marked) { | 1144 » » » » » mapval_ti = (uintptr)maptype->elem->gc | PRECISE; |
1117 » » » » » MapType *t = (MapType*)pc[2]; | 1145 |
1118 » » » » » bool ok = hash_gciter_init(h, &map_iter) ; | 1146 » » » » » // Start mapProg. |
1119 » » » » » if(ok) { | 1147 » » » » » map_ret = pc+3; |
1120 » » » » » » mapkey_size = t->key->size; | 1148 » » » » » pc = mapProg+1; |
1121 » » » » » » mapkey_kind = t->key->kind; | |
1122 » » » » » » mapkey_gc = (uintptr)t->key->g c | 1; | |
1123 » » » » » » mapval_size = t->elem->size; | |
1124 » » » » » » mapval_kind = t->elem->kind; | |
1125 » » » » » » mapval_gc = (uintptr)t->elem-> gc | 1; | |
1126 | |
1127 » » » » » » // Start mapProg. | |
1128 » » » » » » map_ret = pc+3; | |
1129 » » » » » » pc = mapProg+1; | |
1130 » » » » » } else { | |
1131 » » » » » » pc += 3; | |
1132 » » » » » } | |
1133 } else { | 1149 } else { |
1134 pc += 3; | 1150 pc += 3; |
1135 } | 1151 } |
1136 » » » » goto next_instr; | 1152 » » » } else { |
1137 » » » } | 1153 » » » » pc += 3; |
1138 » » » case GC_MAP_NEXT: { | 1154 » » » } |
1139 » » » » while(true) { | 1155 » » » continue; |
1140 » » » » » // objbuf: reserve space for 2 objects. | 1156 |
1141 » » » » » // Add 1 in order to remove the need to do "if(objbufpos == objbuf_end) goto flush_buffers" | 1157 » » case GC_MAP_NEXT: |
1142 » » » » » // after doing "objbufpos++". | 1158 » » » // Add all keys and values to buffers, mark all subtable s. |
1143 » » » » » if(objbufpos+2+1 > objbuf_end) | 1159 » » » while(hash_gciter_next(&map_iter, &d)) { |
1144 » » » » » » goto flush_buffers; | 1160 » » » » // buffers: reserve space for 2 objects. |
1145 | 1161 » » » » if(ptrbufpos+2 >= ptrbuf_end) |
1146 » » » » » struct hash_gciter_data d; | 1162 » » » » » flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wb uf, &nobj); |
1147 » » » » » if(!hash_gciter_next(&map_iter, &d)) | 1163 » » » » if(objbufpos+2 >= objbuf_end) |
1148 » » » » » » break; | 1164 » » » » » flushobjbuf(objbuf, &objbufpos, &wp, &wb uf, &nobj); |
1149 | 1165 |
1150 » » » » » if(d.st != nil) { | 1166 » » » » if(d.st != nil) |
1151 » » » » » » runtime·lock(&lock); | 1167 » » » » » markonly(d.st); |
1152 » » » » » » markonly(d.st); | 1168 |
1153 » » » » » » runtime·unlock(&lock); | 1169 » » » » if(d.key_data != nil) { |
1154 » » » » » } | 1170 » » » » » if(!(mapkey_kind & KindNoPointers) || d. indirectkey) { |
1155 | 1171 » » » » » » if(!d.indirectkey) |
1156 » » » » » if(d.key_data != nil) { | 1172 » » » » » » » *objbufpos++ = (Obj){d.k ey_data, mapkey_size, mapkey_ti}; |
1157 » » » » » » if(!(mapkey_kind & KindNoPointer s)) { | 1173 » » » » » » else { |
1158 » » » » » » » *objbufpos = (Obj){d.key _data, mapkey_size, mapkey_gc}; | 1174 » » » » » » » if(Debug) { |
1159 » » » » » » » objbufpos++; | 1175 » » » » » » » » obj = *(void**)d .key_data; |
1160 » » » » » » } | 1176 » » » » » » » » if(!(arena_start <= obj && obj < arena_used)) |
1161 | 1177 » » » » » » » » » runtime· throw("scanblock: inconsistent hashmap"); |
1162 » » » » » » if(!(mapval_kind & KindNoPointer s) || d.indirectval) { | |
1163 » » » » » » » if(!d.indirectval) { | |
1164 » » » » » » » » *objbufpos = (Ob j){d.val_data, mapval_size, mapval_gc}; | |
1165 » » » » » » » » objbufpos++; | |
1166 » » » » » » » } else { | |
1167 » » » » » » » » void *val = *(vo id**)d.val_data; | |
1168 » » » » » » » » *xbufpos = (stru ct X){val, mapval_gc}; | |
1169 » » » » » » » » xbufpos++; | |
1170 » » » » » » » » if(xbufpos == xb uf_end) | |
1171 » » » » » » » » » goto flu sh_buffers; | |
1172 } | 1178 } |
1179 *ptrbufpos++ = (PtrTarge t){*(void**)d.key_data, mapkey_ti}; | |
1173 } | 1180 } |
1174 } | 1181 } |
1175 | 1182 » » » » » if(!(mapval_kind & KindNoPointers) || d. indirectval) { |
1176 » » » » » if(DebugStats) | 1183 » » » » » » if(!d.indirectval) |
1177 » » » » » » runtime·xadd64(&gcstats.instr[GC _MAP_NEXT], 1); | 1184 » » » » » » » *objbufpos++ = (Obj){d.v al_data, mapval_size, mapval_ti}; |
1185 » » » » » » else { | |
1186 » » » » » » » if(Debug) { | |
1187 » » » » » » » » obj = *(void**)d .val_data; | |
1188 » » » » » » » » if(!(arena_start <= obj && obj < arena_used)) | |
1189 » » » » » » » » » runtime· throw("scanblock: inconsistent hashmap"); | |
1190 » » » » » » » } | |
1191 » » » » » » » *ptrbufpos++ = (PtrTarge t){*(void**)d.val_data, mapval_ti}; | |
1192 » » » » » » } | |
1193 » » » » » } | |
1178 } | 1194 } |
1179 » » » » if(map_ret != 0) { | 1195 » » » } |
1180 » » » » » pc = map_ret; | 1196 » » » if(map_ret == nil) |
1181 » » » » » goto next_instr; | 1197 » » » » goto next_block; |
1182 » » » » } else { | 1198 » » » pc = map_ret; |
1183 » » » » » goto next_block; | 1199 » » » continue; |
1200 | |
1201 » » case GC_REGION: | |
1202 » » » obj = (void*)(stack_top.b + pc[1]); | |
1203 » » » size = pc[2]; | |
1204 » » » objti = pc[3]; | |
1205 » » » pc += 4; | |
1206 | |
1207 » » » *objbufpos++ = (Obj){obj, size, objti}; | |
1208 » » » if(objbufpos == objbuf_end) | |
1209 » » » » flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nob j); | |
1210 » » » continue; | |
1211 | |
1212 » » case GC_CHAN_PTR: | |
1213 » » » // Similar to GC_MAP_PTR | |
1214 » » » chan = *(Hchan**)(stack_top.b + pc[1]); | |
1215 » » » if(chan == nil) { | |
1216 » » » » pc += 3; | |
1217 » » » » continue; | |
1218 » » » } | |
1219 » » » if(markonly(chan)) { | |
1220 » » » » chantype = (ChanType*)pc[2]; | |
1221 » » » » if(!(chantype->elem->kind & KindNoPointers)) { | |
1222 » » » » » // Start chanProg. | |
1223 » » » » » chan_ret = pc+3; | |
1224 » » » » » pc = chanProg+1; | |
1225 » » » » » continue; | |
1184 } | 1226 } |
1185 } | 1227 } |
1186 » » » default: { | 1228 » » » pc += 3; |
1187 » » » » runtime·throw("scan: invalid GC instruction"); | 1229 » » » continue; |
1188 » » » » return; | 1230 |
1189 » » » }» » »······· | 1231 » » case GC_CHAN: |
1190 » » } | 1232 » » » // There are no heap pointers in struct Hchan, |
1191 | 1233 » » » // so we can ignore the leading sizeof(Hchan) bytes. |
1192 » flush_buffers: | 1234 » » » if(!(chantype->elem->kind & KindNoPointers)) { |
1193 » » flushxbuf(xbuf, xbufpos-xbuf, &wp, &wbuf, &nobj, ybuf); | 1235 » » » » // Channel's buffer follows Hchan immediately in memory. |
1194 » » xbufpos = xbuf; | 1236 » » » » // Size of buffer (cap(c)) is second int in the chan struct. |
1195 » » flushobjbuf(objbuf, objbufpos-objbuf, &wp, &wbuf, &nobj); | 1237 » » » » n = ((uintgo*)chan)[1]; |
1196 » » objbufpos = objbuf; | 1238 » » » » if(n > 0) { |
1197 » » goto next_instr; | 1239 » » » » » // TODO(atom): split into two chunks so that only the |
1198 | 1240 » » » » » // in-use part of the circular buffer is scanned. |
1199 » trailing_pointers: | 1241 » » » » » // (Channel routines zero the unused par t, so the current |
1200 » » if(DebugType) { | 1242 » » » » » // code does not lead to leaks, it's jus t a little inefficient.) |
1201 » » » if(i % PtrSize != 0) | 1243 » » » » » *objbufpos++ = (Obj){(byte*)chan+runtime ·Hchansize, n*chantype->elem->size, |
1202 » » » » runtime·throw("scan: invalid alignment"); | 1244 » » » » » » (uintptr)chantype->elem->gc | PR ECISE | LOOP}; |
1203 » » } | 1245 » » » » » if(objbufpos == objbuf_end) |
1204 | 1246 » » » » » » flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); |
1205 » » // Quickly scan [b+i,b+n) for possible pointers. | |
1206 » » for(; i<=end_b; i+=PtrSize) { | |
1207 » » » if( *(byte**)i != nil ) { | |
1208 » » » » // Found a value that may be a pointer. | |
1209 » » » » // Do a rescan of the entire block, | |
1210 » » » » // and force the rescan to call runtime·gettype. | |
1211 » » » » if(DebugStats) { | |
1212 » » » » » runtime·xadd64(&gcstats.rescans, 1); | |
1213 » » » » » runtime·xadd64(&gcstats.rescan_nbytes, n ); | |
1214 } | 1247 } |
1215 » » » » enqueue((Obj){b, n, 0}, &wp, &wbuf, &nobj); | 1248 » » » } |
1216 » » » » break; | 1249 » » » if(chan_ret == nil) |
1217 » » » } | 1250 » » » » goto next_block; |
1218 » » } | 1251 » » » pc = chan_ret; |
1252 » » » continue; | |
1253 | |
1254 » » default: | |
1255 » » » runtime·throw("scanblock: invalid GC instruction"); | |
1256 » » » return; | |
1257 » » } | |
1258 | |
1259 » » if(obj >= arena_start && obj < arena_used) { | |
1260 » » » *ptrbufpos++ = (PtrTarget){obj, objti}; | |
1261 » » » if(ptrbufpos == ptrbuf_end) | |
1262 » » » » flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nob j); | |
1263 » » } | |
1264 » } | |
1219 | 1265 |
1220 next_block: | 1266 next_block: |
1221 // Done scanning [b, b+n). Prepare for the next iteration of | 1267 // Done scanning [b, b+n). Prepare for the next iteration of |
1222 » » // the loop by setting b, n and gctype to the parameters for the next block. | 1268 » » // the loop by setting b, n, ti to the parameters for the next b lock. |
1223 | 1269 |
1224 if(nobj == 0) { | 1270 if(nobj == 0) { |
1225 » » » flushxbuf(xbuf, xbufpos-xbuf, &wp, &wbuf, &nobj, ybuf); | 1271 » » » flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); |
1226 » » » xbufpos = xbuf; | 1272 » » » flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); |
1227 » » » flushobjbuf(objbuf, objbufpos-objbuf, &wp, &wbuf, &nobj) ; | |
1228 » » » objbufpos = objbuf; | |
1229 | 1273 |
1230 if(nobj == 0) { | 1274 if(nobj == 0) { |
1231 if(!keepworking) { | 1275 if(!keepworking) { |
1232 » » » » » putempty(wbuf); | 1276 » » » » » if(wbuf) |
1277 » » » » » » putempty(wbuf); | |
1233 goto endscan; | 1278 goto endscan; |
1234 } | 1279 } |
1235 // Emptied our buffer: refill. | 1280 // Emptied our buffer: refill. |
1236 wbuf = getfull(wbuf); | 1281 wbuf = getfull(wbuf); |
1237 if(wbuf == nil) | 1282 if(wbuf == nil) |
1238 goto endscan; | 1283 goto endscan; |
1239 nobj = wbuf->nobj; | 1284 nobj = wbuf->nobj; |
1240 wp = wbuf->obj + wbuf->nobj; | 1285 wp = wbuf->obj + wbuf->nobj; |
1241 } | 1286 } |
1242 } | 1287 } |
1243 | 1288 |
1244 // Fetch b from the work buffer. | 1289 // Fetch b from the work buffer. |
1245 --wp; | 1290 --wp; |
1246 b = wp->p; | 1291 b = wp->p; |
1247 n = wp->n; | 1292 n = wp->n; |
1248 » » gc = wp->gc; | 1293 » » ti = wp->ti; |
1249 nobj--; | 1294 nobj--; |
1250 | 1295 » } |
1251 » » PREFETCH(b); | 1296 |
1252 » } | 1297 » for(;;) { |
1253 | 1298 » » // Each iteration scans the block b of length n, queueing pointe rs in |
1254 endscan: | 1299 » » // the work buffer. |
1255 » runtime·lock(&lock); | 1300 » » if(Debug > 1) |
1256 » scanbuffers->next = bufferList; | 1301 » » » runtime·printf("scanblock %p %D\n", b, (int64)n); |
1257 » bufferList = scanbuffers; | 1302 |
1258 » runtime·unlock(&lock); | 1303 » » for(end_b = (uintptr)b + n - PtrSize; b <= (byte*)end_b; b += Pt rSize) { |
1259 | 1304 » » » obj = *(byte**)b; |
1260 » // Update the state of buffer | 1305 » » » if(obj >= arena_start && obj < arena_used) { |
1261 » *_wp = wp; | 1306 » » » » *ptrbufpos++ = (PtrTarget){obj, 0}; |
1262 » *_wbuf = wbuf; | 1307 » » » » if(ptrbufpos == ptrbuf_end) |
1263 » *_nobj = nobj; | 1308 » » » » » flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wb uf, &nobj); |
1309 » » » } | |
1310 » » } | |
1311 | |
1312 » » // Done scanning [b, b+n). Prepare for the next iteration of | |
1313 » » // the loop by setting b, n, ti to the parameters for the next b lock. | |
1314 | |
1315 » next_block_conservative: | |
1316 » » if(nobj == 0) { | |
1317 » » » flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); | |
1318 » » » if(nobj == 0) { | |
1319 » » » » if(!keepworking) { | |
1320 » » » » » if(wbuf) | |
1321 » » » » » » putempty(wbuf); | |
1322 » » » » » goto endscan; | |
1323 » » » » } | |
1324 » » » » // Emptied our buffer: refill. | |
1325 » » » » wbuf = getfull(wbuf); | |
1326 » » » » if(wbuf == nil) | |
1327 » » » » » goto endscan; | |
1328 » » » » nobj = wbuf->nobj; | |
1329 » » » » wp = wbuf->obj + wbuf->nobj; | |
1330 » » » } | |
1331 » » } | |
1332 | |
1333 » » // Fetch b from the work buffer. | |
1334 » » --wp; | |
1335 » » b = wp->p; | |
1336 » » n = wp->n; | |
1337 » » nobj--; | |
1338 » } | |
1339 | |
1340 endscan:; | |
1264 } | 1341 } |
1265 | 1342 |
1266 // debug_scanblock is the debug copy of scanblock. | 1343 // debug_scanblock is the debug copy of scanblock. |
1267 // it is simpler, slower, more imprecise, single-threaded, recursive, | 1344 // it is simpler, slower, less precise, single-threaded, recursive, |
1268 // and uses bitSpecial as the mark bit. | 1345 // and uses bitSpecial as the mark bit. |
1269 static void | 1346 static void |
1270 debug_scanblock(byte *b, uintptr n) | 1347 debug_scanblock(byte *b, uintptr n) |
1271 { | 1348 { |
1272 byte *obj, *p; | 1349 byte *obj, *p; |
1273 void **vp; | 1350 void **vp; |
1274 uintptr size, *bitp, bits, shift, i, xbits, off; | 1351 uintptr size, *bitp, bits, shift, i, xbits, off; |
1275 MSpan *s; | 1352 MSpan *s; |
1276 | 1353 |
1277 if(!DebugMark) | 1354 if(!DebugMark) |
1278 runtime·throw("debug_scanblock without DebugMark"); | 1355 runtime·throw("debug_scanblock without DebugMark"); |
1356 | |
1357 if((intptr)n < 0) { | |
1358 runtime·printf("debug_scanblock %p %D\n", b, (int64)n); | |
1359 runtime·throw("debug_scanblock"); | |
1360 } | |
1279 | 1361 |
1280 // Align b to a word boundary. | 1362 // Align b to a word boundary. |
1281 off = (uintptr)b & (PtrSize-1); | 1363 off = (uintptr)b & (PtrSize-1); |
1282 if(off != 0) { | 1364 if(off != 0) { |
1283 b += PtrSize - off; | 1365 b += PtrSize - off; |
1284 n -= PtrSize - off; | 1366 n -= PtrSize - off; |
1285 } | 1367 } |
1286 | 1368 |
1287 vp = (void**)b; | 1369 vp = (void**)b; |
1288 n /= PtrSize; | 1370 n /= PtrSize; |
1289 for(i=0; i<n; i++) { | 1371 for(i=0; i<n; i++) { |
1290 obj = (byte*)vp[i]; | 1372 obj = (byte*)vp[i]; |
1291 | 1373 |
1292 // Words outside the arena cannot be pointers. | 1374 // Words outside the arena cannot be pointers. |
1293 » » if((byte*)obj < runtime·mheap.arena_start || (byte*)obj >= runti me·mheap.arena_used) | 1375 » » if((byte*)obj < runtime·mheap->arena_start || (byte*)obj >= runt ime·mheap->arena_used) |
1294 continue; | 1376 continue; |
1295 | 1377 |
1296 // Round down to word boundary. | 1378 // Round down to word boundary. |
1297 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); | 1379 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); |
1298 | 1380 |
1299 // Consult span table to find beginning. | 1381 // Consult span table to find beginning. |
1300 » » s = runtime·MHeap_LookupMaybe(&runtime·mheap, obj); | 1382 » » s = runtime·MHeap_LookupMaybe(runtime·mheap, obj); |
1301 if(s == nil) | 1383 if(s == nil) |
1302 continue; | 1384 continue; |
1303 | 1385 |
1304 p = (byte*)((uintptr)s->start<<PageShift); | 1386 p = (byte*)((uintptr)s->start<<PageShift); |
1305 size = s->elemsize; | 1387 size = s->elemsize; |
1306 if(s->sizeclass == 0) { | 1388 if(s->sizeclass == 0) { |
1307 obj = p; | 1389 obj = p; |
1308 } else { | 1390 } else { |
1309 if((byte*)obj >= (byte*)s->limit) | 1391 if((byte*)obj >= (byte*)s->limit) |
1310 continue; | 1392 continue; |
1311 int32 i = ((byte*)obj - p)/size; | 1393 int32 i = ((byte*)obj - p)/size; |
1312 obj = p+i*size; | 1394 obj = p+i*size; |
1313 } | 1395 } |
1314 | 1396 |
1315 // Now that we know the object header, reload bits. | 1397 // Now that we know the object header, reload bits. |
1316 » » off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; | 1398 » » off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start; |
1317 » » bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapW ord - 1; | 1399 » » bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmap Word - 1; |
1318 shift = off % wordsPerBitmapWord; | 1400 shift = off % wordsPerBitmapWord; |
1319 xbits = *bitp; | 1401 xbits = *bitp; |
1320 bits = xbits >> shift; | 1402 bits = xbits >> shift; |
1321 | 1403 |
1322 // Now we have bits, bitp, and shift correct for | 1404 // Now we have bits, bitp, and shift correct for |
1323 // obj pointing at the base of the object. | 1405 // obj pointing at the base of the object. |
1324 // If not allocated or already marked, done. | 1406 // If not allocated or already marked, done. |
1325 if((bits & bitAllocated) == 0 || (bits & bitSpecial) != 0) // N OTE: bitSpecial not bitMarked | 1407 if((bits & bitAllocated) == 0 || (bits & bitSpecial) != 0) // N OTE: bitSpecial not bitMarked |
1326 continue; | 1408 continue; |
1327 *bitp |= bitSpecial<<shift; | 1409 *bitp |= bitSpecial<<shift; |
1328 if(!(bits & bitMarked)) | 1410 if(!(bits & bitMarked)) |
1329 runtime·printf("found unmarked block %p in %p\n", obj, v p+i); | 1411 runtime·printf("found unmarked block %p in %p\n", obj, v p+i); |
1330 | 1412 |
1331 // If object has no pointers, don't need to scan further. | 1413 // If object has no pointers, don't need to scan further. |
1332 if((bits & bitNoPointers) != 0) | 1414 if((bits & bitNoPointers) != 0) |
1333 continue; | 1415 continue; |
1334 | 1416 |
1335 debug_scanblock(obj, size); | 1417 debug_scanblock(obj, size); |
1336 } | 1418 } |
1337 } | 1419 } |
1338 | 1420 |
1421 // Append obj to the work buffer. | |
1422 // _wbuf, _wp, _nobj are input/output parameters and are specifying the work buf fer. | |
1339 static void | 1423 static void |
1340 debug_scan(Obj **_wp, Workbuf **_wbuf, uintptr *_nobj) | 1424 enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj) |
1341 { | 1425 { |
1342 » byte *b; | 1426 » uintptr nobj, off; |
1343 » uintptr n, nobj; | |
1344 Obj *wp; | 1427 Obj *wp; |
1345 Workbuf *wbuf; | 1428 Workbuf *wbuf; |
1346 | 1429 |
1347 » if(sizeof(Workbuf) % PageSize != 0) | 1430 » if(Debug > 1) |
1348 » » runtime·throw("debug_scan: size of Workbuf is suboptimal"); | 1431 » » runtime·printf("append obj(%p %D %p)\n", obj.p, (int64)obj.n, ob j.ti); |
1349 | 1432 |
1350 » // Get the state of buffer | 1433 » // Align obj.b to a word boundary. |
1351 » wp = *_wp; // storage for next queued pointer (write pointer) | 1434 » off = (uintptr)obj.p & (PtrSize-1); |
1352 » wbuf = *_wbuf; // current work buffer | 1435 » if(off != 0) { |
1353 » nobj = *_nobj; // number of queued objects | 1436 » » obj.p += PtrSize - off; |
1354 | 1437 » » obj.n -= PtrSize - off; |
1355 » for(;;) { | 1438 » » obj.ti = 0; |
1356 » » if(nobj == 0) { | 1439 » } |
1357 » » » // Emptied our buffer: refill. | 1440 |
1358 » » » wbuf = getfull(wbuf); | 1441 » if(obj.p == nil || obj.n == 0) |
1359 » » » if(wbuf == nil) | 1442 » » return; |
1360 » » » » goto endscan; | 1443 |
1361 » » » nobj = wbuf->nobj; | 1444 » // Load work buffer state |
1362 » » » wp = wbuf->obj + wbuf->nobj; | 1445 » wp = *_wp; |
1363 » » } | 1446 » wbuf = *_wbuf; |
1364 | 1447 » nobj = *_nobj; |
1365 » » // Fetch b from the work buffer. | 1448 |
1366 » » --wp; | 1449 » // If another proc wants a pointer, give it some. |
1367 » » b = wp->p; | 1450 » if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) { |
1368 » » n = wp->n; | 1451 » » wbuf->nobj = nobj; |
1369 » » nobj--; | 1452 » » wbuf = handoff(wbuf); |
1370 | 1453 » » nobj = wbuf->nobj; |
1371 » » debug_scanblock(b, n); | 1454 » » wp = wbuf->obj + nobj; |
1372 » } | 1455 » } |
1373 | 1456 |
1374 endscan: | 1457 » // If buffer is full, get a new one. |
1375 » // Update the state of buffer | 1458 » if(wbuf == nil || nobj >= nelem(wbuf->obj)) { |
1459 » » if(wbuf != nil) | |
1460 » » » wbuf->nobj = nobj; | |
1461 » » wbuf = getempty(wbuf); | |
1462 » » wp = wbuf->obj; | |
1463 » » nobj = 0; | |
1464 » } | |
1465 | |
1466 » *wp = obj; | |
1467 » wp++; | |
1468 » nobj++; | |
1469 | |
1470 » // Save work buffer state | |
1376 *_wp = wp; | 1471 *_wp = wp; |
1377 *_wbuf = wbuf; | 1472 *_wbuf = wbuf; |
1378 *_nobj = nobj; | 1473 *_nobj = nobj; |
1474 } | |
1475 | |
1476 static void | |
1477 markroot(ParFor *desc, uint32 i) | |
1478 { | |
1479 Obj *wp; | |
1480 Workbuf *wbuf; | |
1481 uintptr nobj; | |
1482 | |
1483 USED(&desc); | |
1484 wp = nil; | |
1485 wbuf = nil; | |
1486 nobj = 0; | |
1487 enqueue(work.roots[i], &wbuf, &wp, &nobj); | |
1488 scanblock(wbuf, wp, nobj, false); | |
1379 } | 1489 } |
1380 | 1490 |
1381 // Get an empty work buffer off the work.empty list, | 1491 // Get an empty work buffer off the work.empty list, |
1382 // allocating new buffers as needed. | 1492 // allocating new buffers as needed. |
1383 static Workbuf* | 1493 static Workbuf* |
1384 getempty(Workbuf *b) | 1494 getempty(Workbuf *b) |
1385 { | 1495 { |
1386 » if(work.nproc == 1) { | 1496 » if(b != nil) |
1387 » » // Put b on full list. | 1497 » » runtime·lfstackpush(&work.full, &b->node); |
1388 » » if(b != nil) { | 1498 » b = (Workbuf*)runtime·lfstackpop(&work.empty); |
1389 » » » b->next = work.full; | 1499 » if(b == nil) { |
1390 » » » work.full = b; | 1500 » » // Need to allocate. |
1391 » » } | 1501 » » runtime·lock(&work); |
1392 » » // Grab from empty list if possible. | 1502 » » if(work.nchunk < sizeof *b) { |
1393 » » b = work.empty; | 1503 » » » work.nchunk = 1<<20; |
1394 » » if(b != nil) { | 1504 » » » work.chunk = runtime·SysAlloc(work.nchunk); |
1395 » » » work.empty = b->next; | 1505 » » » if(work.chunk == nil) |
1396 » » » goto haveb; | 1506 » » » » runtime·throw("runtime: cannot allocate memory") ; |
1397 » » } | 1507 » » } |
1398 » } else { | 1508 » » b = (Workbuf*)work.chunk; |
1399 » » // Put b on full list. | 1509 » » work.chunk += sizeof *b; |
1400 » » if(b != nil) { | 1510 » » work.nchunk -= sizeof *b; |
1401 » » » runtime·lock(&work.fmu); | 1511 » » runtime·unlock(&work); |
1402 » » » b->next = work.full; | 1512 » } |
1403 » » » work.full = b; | |
1404 » » » runtime·unlock(&work.fmu); | |
1405 » » } | |
1406 » » // Grab from empty list if possible. | |
1407 » » runtime·lock(&work.emu); | |
1408 » » b = work.empty; | |
1409 » » if(b != nil) | |
1410 » » » work.empty = b->next; | |
1411 » » runtime·unlock(&work.emu); | |
1412 » » if(b != nil) | |
1413 » » » goto haveb; | |
1414 » } | |
1415 | |
1416 » // Need to allocate. | |
1417 » runtime·lock(&work); | |
1418 » if(work.nchunk < sizeof *b) { | |
1419 » » work.nchunk = 1<<20; | |
1420 » » work.chunk = runtime·SysAlloc(work.nchunk); | |
1421 » } | |
1422 » b = (Workbuf*)work.chunk; | |
1423 » work.chunk += sizeof *b; | |
1424 » work.nchunk -= sizeof *b; | |
1425 » runtime·unlock(&work); | |
1426 | |
1427 haveb: | |
1428 b->nobj = 0; | 1513 b->nobj = 0; |
1429 return b; | 1514 return b; |
1430 } | 1515 } |
1431 | 1516 |
1432 static void | 1517 static void |
1433 putempty(Workbuf *b) | 1518 putempty(Workbuf *b) |
1434 { | 1519 { |
1435 » if(DebugStats) | 1520 » if(CollectStats) |
1436 runtime·xadd64(&gcstats.putempty, 1); | 1521 runtime·xadd64(&gcstats.putempty, 1); |
1437 | 1522 |
1438 » if(b == nil) | 1523 » runtime·lfstackpush(&work.empty, &b->node); |
1439 » » return; | |
1440 | |
1441 » if(work.nproc == 1) { | |
1442 » » b->next = work.empty; | |
1443 » » work.empty = b; | |
1444 » » return; | |
1445 » } | |
1446 | |
1447 » runtime·lock(&work.emu); | |
1448 » b->next = work.empty; | |
1449 » work.empty = b; | |
1450 » runtime·unlock(&work.emu); | |
1451 } | 1524 } |
1452 | 1525 |
1453 // Get a full work buffer off the work.full list, or return nil. | 1526 // Get a full work buffer off the work.full list, or return nil. |
1454 static Workbuf* | 1527 static Workbuf* |
1455 getfull(Workbuf *b) | 1528 getfull(Workbuf *b) |
1456 { | 1529 { |
1457 int32 i; | 1530 int32 i; |
1458 » Workbuf *b1; | 1531 |
1459 | 1532 » if(CollectStats) |
1460 » if(DebugStats) | |
1461 runtime·xadd64(&gcstats.getfull, 1); | 1533 runtime·xadd64(&gcstats.getfull, 1); |
1462 | 1534 |
1463 » if(work.nproc == 1) { | 1535 » if(b != nil) |
1464 » » // Put b on empty list. | 1536 » » runtime·lfstackpush(&work.empty, &b->node); |
1465 » » if(b != nil) { | 1537 » b = (Workbuf*)runtime·lfstackpop(&work.full); |
1466 » » » b->next = work.empty; | 1538 » if(b != nil || work.nproc == 1) |
1467 » » » work.empty = b; | |
1468 » » } | |
1469 » » // Grab from full list if possible. | |
1470 » » // Since work.nproc==1, no one else is | |
1471 » » // going to give us work. | |
1472 » » b = work.full; | |
1473 » » if(b != nil) | |
1474 » » » work.full = b->next; | |
1475 return b; | 1539 return b; |
1476 } | |
1477 | |
1478 putempty(b); | |
1479 | |
1480 // Grab buffer from full list if possible. | |
1481 for(;;) { | |
1482 b1 = work.full; | |
1483 if(b1 == nil) | |
1484 break; | |
1485 runtime·lock(&work.fmu); | |
1486 if(work.full != nil) { | |
1487 b1 = work.full; | |
1488 work.full = b1->next; | |
1489 runtime·unlock(&work.fmu); | |
1490 return b1; | |
1491 } | |
1492 runtime·unlock(&work.fmu); | |
1493 } | |
1494 | 1540 |
1495 runtime·xadd(&work.nwait, +1); | 1541 runtime·xadd(&work.nwait, +1); |
1496 for(i=0;; i++) { | 1542 for(i=0;; i++) { |
1497 » » b1 = work.full; | 1543 » » if(work.full != 0) { |
1498 » » if(b1 != nil) { | 1544 » » » runtime·xadd(&work.nwait, -1); |
1499 » » » runtime·lock(&work.fmu); | 1545 » » » b = (Workbuf*)runtime·lfstackpop(&work.full); |
1500 » » » if(work.full != nil) { | 1546 » » » if(b != nil) |
1501 » » » » runtime·xadd(&work.nwait, -1); | 1547 » » » » return b; |
1502 » » » » b1 = work.full; | 1548 » » » runtime·xadd(&work.nwait, +1); |
1503 » » » » work.full = b1->next; | |
1504 » » » » runtime·unlock(&work.fmu); | |
1505 » » » » return b1; | |
1506 » » » } | |
1507 » » » runtime·unlock(&work.fmu); | |
1508 » » » continue; | |
1509 } | 1549 } |
1510 if(work.nwait == work.nproc) | 1550 if(work.nwait == work.nproc) |
1511 return nil; | 1551 return nil; |
1512 if(i < 10) { | 1552 if(i < 10) { |
1513 m->gcstats.nprocyield++; | 1553 m->gcstats.nprocyield++; |
1514 runtime·procyield(20); | 1554 runtime·procyield(20); |
1515 } else if(i < 20) { | 1555 } else if(i < 20) { |
1516 m->gcstats.nosyield++; | 1556 m->gcstats.nosyield++; |
1517 runtime·osyield(); | 1557 runtime·osyield(); |
1518 } else { | 1558 } else { |
1519 m->gcstats.nsleep++; | 1559 m->gcstats.nsleep++; |
1520 runtime·usleep(100); | 1560 runtime·usleep(100); |
1521 } | 1561 } |
1522 } | 1562 } |
1523 } | 1563 } |
1524 | 1564 |
1525 static Workbuf* | 1565 static Workbuf* |
1526 handoff(Workbuf *b) | 1566 handoff(Workbuf *b) |
1527 { | 1567 { |
1528 » int32 n; | 1568 » int32 n, n1, i; |
1529 Workbuf *b1; | 1569 Workbuf *b1; |
1530 | 1570 |
1531 // Make new buffer with half of b's pointers. | 1571 // Make new buffer with half of b's pointers. |
1532 b1 = getempty(nil); | 1572 b1 = getempty(nil); |
1533 » n = b->nobj/2; | 1573 » n = 0; |
1534 » b->nobj -= n; | 1574 » n1 = 0; |
1535 » b1->nobj = n; | 1575 » for(i=0; i+1 < b->nobj; i+=2) { |
1536 » runtime·memmove(b1->obj, b->obj+b->nobj, n*sizeof b1->obj[0]); | 1576 » » b->obj[n++] = b->obj[i]; |
1577 » » b1->obj[n1++] = b->obj[i+1]; | |
1578 » } | |
1579 » if(i < b->nobj) { | |
1580 » » b->obj[n++] = b->obj[i]; | |
1581 » } | |
1582 » b->nobj = n; | |
1583 » b1->nobj = n1; | |
1537 m->gcstats.nhandoff++; | 1584 m->gcstats.nhandoff++; |
1538 m->gcstats.nhandoffcnt += n; | 1585 m->gcstats.nhandoffcnt += n; |
1539 | 1586 |
1540 // Put b on full list - let first half of b get stolen. | 1587 // Put b on full list - let first half of b get stolen. |
1541 » runtime·lock(&work.fmu); | 1588 » runtime·lfstackpush(&work.full, &b->node); |
1542 » b->next = work.full; | |
1543 » work.full = b; | |
1544 » runtime·unlock(&work.fmu); | |
1545 | |
1546 return b1; | 1589 return b1; |
1547 } | 1590 } |
1548 | 1591 |
1549 // Appendstack calls markenqueue on each of gp's stack segments. | |
1550 static void | 1592 static void |
1551 appendstack(G *gp, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj) | 1593 addroot(Obj obj) |
1594 { | |
1595 » uint32 cap; | |
1596 » Obj *new; | |
1597 | |
1598 » if(work.nroot >= work.rootcap) { | |
1599 » » cap = PageSize/sizeof(Obj); | |
1600 » » if(cap < 2*work.rootcap) | |
1601 » » » cap = 2*work.rootcap; | |
1602 » » new = (Obj*)runtime·SysAlloc(cap*sizeof(Obj)); | |
1603 » » if(new == nil) | |
1604 » » » runtime·throw("runtime: cannot allocate memory"); | |
1605 » » if(work.roots != nil) { | |
1606 » » » runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj )); | |
1607 » » » runtime·SysFree(work.roots, work.rootcap*sizeof(Obj)); | |
1608 » » } | |
1609 » » work.roots = new; | |
1610 » » work.rootcap = cap; | |
1611 » } | |
1612 » work.roots[work.nroot] = obj; | |
1613 » work.nroot++; | |
1614 } | |
1615 | |
1616 // Scan a stack frame. The doframe parameter is a signal that the previously | |
1617 // scanned activation has an unknown argument size. When *doframe is true the | |
1618 // current activation must have its entire frame scanned. Otherwise, only the | |
1619 // locals need to be scanned. | |
1620 static void | |
1621 addframeroots(Func *f, byte*, byte *sp, void *doframe) | |
1622 { | |
1623 » uintptr outs; | |
1624 | |
1625 » if(thechar == '5') | |
1626 » » sp += sizeof(uintptr); | |
1627 » if(f->locals == 0 || *(bool*)doframe == true) | |
1628 » » addroot((Obj){sp, f->frame - sizeof(uintptr), 0}); | |
1629 » else if(f->locals > 0) { | |
1630 » » outs = f->frame - sizeof(uintptr) - f->locals; | |
1631 » » addroot((Obj){sp + outs, f->locals, 0}); | |
1632 » } | |
1633 » if(f->args > 0) | |
1634 » » addroot((Obj){sp + f->frame, f->args, 0}); | |
1635 » *(bool*)doframe = (f->args == ArgsSizeUnknown); | |
1636 } | |
1637 | |
1638 static void | |
1639 addstackroots(G *gp) | |
1552 { | 1640 { |
1553 M *mp; | 1641 M *mp; |
1554 int32 n; | 1642 int32 n; |
1555 Stktop *stk; | 1643 Stktop *stk; |
1556 » uintptr sp, guard; | 1644 » byte *sp, *guard, *pc; |
1645 » Func *f; | |
1646 » bool doframe; | |
1557 | 1647 |
1558 stk = (Stktop*)gp->stackbase; | 1648 stk = (Stktop*)gp->stackbase; |
1559 » guard = gp->stackguard; | 1649 » guard = (byte*)gp->stackguard; |
1560 | 1650 |
1561 if(gp == g) { | 1651 if(gp == g) { |
1562 // Scanning our own stack: start at &gp. | 1652 // Scanning our own stack: start at &gp. |
1563 » » sp = (uintptr)&gp; | 1653 » » sp = runtime·getcallersp(&gp); |
1654 » » pc = runtime·getcallerpc(&gp); | |
1564 } else if((mp = gp->m) != nil && mp->helpgc) { | 1655 } else if((mp = gp->m) != nil && mp->helpgc) { |
1565 // gchelper's stack is in active use and has no interesting poin ters. | 1656 // gchelper's stack is in active use and has no interesting poin ters. |
1566 return; | 1657 return; |
1658 } else if(gp->gcstack != (uintptr)nil) { | |
1659 // Scanning another goroutine that is about to enter or might | |
1660 // have just exited a system call. It may be executing code such | |
1661 // as schedlock and may have needed to start a new stack segment . | |
1662 // Use the stack segment and stack pointer at the time of | |
1663 // the system call instead, since that won't change underfoot. | |
1664 sp = (byte*)gp->gcsp; | |
1665 pc = gp->gcpc; | |
1666 stk = (Stktop*)gp->gcstack; | |
1667 guard = (byte*)gp->gcguard; | |
1567 } else { | 1668 } else { |
1568 // Scanning another goroutine's stack. | 1669 // Scanning another goroutine's stack. |
1569 // The goroutine is usually asleep (the world is stopped). | 1670 // The goroutine is usually asleep (the world is stopped). |
1570 » » sp = gp->sched.sp; | 1671 » » sp = (byte*)gp->sched.sp; |
1571 | 1672 » » pc = gp->sched.pc; |
1572 » » // The exception is that if the goroutine is about to enter or m ight | 1673 » » if(ScanStackByFrames && pc == (byte*)runtime·goexit && gp->fnsta rt != nil) { |
1573 » » // have just exited a system call, it may be executing code such | 1674 » » » // The goroutine has not started. However, its incoming |
1574 » » // as schedlock and may have needed to start a new stack segment . | 1675 » » » // arguments are live at the top of the stack and must |
1575 » » // Use the stack segment and stack pointer at the time of | 1676 » » » // be scanned. No other live values should be on the |
1576 » » // the system call instead, since that won't change underfoot. | 1677 » » » // stack. |
1577 » » if(gp->gcstack != 0) { | 1678 » » » f = runtime·findfunc((uintptr)gp->fnstart->fn); |
1578 » » » stk = (Stktop*)gp->gcstack; | 1679 » » » if(f->args != 0) { |
1579 » » » sp = gp->gcsp; | 1680 » » » » if(thechar == '5') |
1580 » » » guard = gp->gcguard; | 1681 » » » » » sp += sizeof(uintptr); |
1581 » » } | 1682 » » » » // If the size of the arguments is known |
1582 » } | 1683 » » » » // scan just the incoming arguments. |
1583 | 1684 » » » » // Otherwise, scan everything between the |
1584 » if(Debug > 1) | 1685 » » » » // top and the bottom of the stack. |
1585 » » runtime·printf("appendstack %d %p\n", gp->goid, sp); | 1686 » » » » if(f->args > 0) |
1586 » n = 0; | 1687 » » » » » addroot((Obj){sp, f->args, 0}); |
1587 » while(stk) { | 1688 » » » » else |
1588 » » if(sp < guard-StackGuard || (uintptr)stk < sp) { | 1689 » » » » » addroot((Obj){sp, (byte*)stk - sp, 0});· |
1589 » » » runtime·printf("appendstack inconsistent: g%d#%d sp=%p n ot in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk); | 1690 » » » }· |
1590 » » » runtime·throw("appendstack"); | 1691 » » » return; |
1591 » » } | 1692 » » } |
1592 » » markenqueue((Obj){(byte*)sp, (uintptr)stk - sp, 0}, _wp, _wbuf, _nobj); | 1693 » } |
1593 » » sp = stk->gobuf.sp; | 1694 |
1594 » » guard = (uintptr)stk->stackguard; | 1695 » if (ScanStackByFrames) { |
1595 » » stk = (Stktop*)stk->stackbase; | 1696 » » USED(stk); |
1596 » » n++; | 1697 » » USED(guard); |
1597 » } | 1698 » » doframe = false; |
1598 } | 1699 » » runtime·gentraceback(pc, sp, nil, gp, 0, nil, 0x7fffffff, addfra meroots, &doframe); |
1599 | 1700 » } else { |
1600 // Markfin calls scanblock on the blocks that have finalizers: | 1701 » » USED(pc); |
1601 // the things pointed at cannot be freed until the finalizers have run. | 1702 » » n = 0; |
1703 » » while(stk) { | |
1704 » » » if(sp < guard-StackGuard || (byte*)stk < sp) { | |
1705 » » » » runtime·printf("addstackroots inconsistent: g%D# %d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk); | |
1706 » » » » runtime·throw("addstackroots"); | |
1707 » » » » runtime·printf("scanstack inconsistent: g%D#%d s p=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk); | |
1708 » » » » runtime·throw("scanstack"); | |
1709 » » » } | |
1710 » » » addroot((Obj){sp, (byte*)stk - sp, (uintptr)defaultProg | PRECISE | LOOP}); | |
1711 » » » sp = (byte*)stk->gobuf.sp; | |
1712 » » » guard = stk->stackguard; | |
1713 » » » stk = (Stktop*)stk->stackbase; | |
1714 » » » n++; | |
1715 » » } | |
1716 » } | |
1717 } | |
1718 | |
1602 static void | 1719 static void |
1603 markfin(void *v, void *arg) | 1720 addfinroots(void *v) |
1604 { | 1721 { |
1605 uintptr size; | 1722 uintptr size; |
1606 » Obj **_wp; | 1723 » void *base; |
1607 » Workbuf **_wbuf; | |
1608 » uintptr *_nobj; | |
1609 | |
1610 » _wp = ((WbufState*)arg)->_wp; | |
1611 » _wbuf = ((WbufState*)arg)->_wbuf; | |
1612 » _nobj = ((WbufState*)arg)->_nobj; | |
1613 | 1724 |
1614 size = 0; | 1725 size = 0; |
1615 » if(!runtime·mlookup(v, &v, &size, nil) || !runtime·blockspecial(v)) | 1726 » if(!runtime·mlookup(v, &base, &size, nil) || !runtime·blockspecial(base) ) |
1616 runtime·throw("mark - finalizer inconsistency"); | 1727 runtime·throw("mark - finalizer inconsistency"); |
1617 | 1728 |
1618 // do not mark the finalizer block itself. just mark the things it poin ts at. | 1729 // do not mark the finalizer block itself. just mark the things it poin ts at. |
1619 » enqueue((Obj){v, size, 0}, _wp, _wbuf, _nobj); | 1730 » addroot((Obj){base, size, 0}); |
1620 } | 1731 } |
1621 | 1732 |
1622 // Mark | |
1623 static void | 1733 static void |
1624 mark(void (*scan)(Obj**, Workbuf**, uintptr*)) | 1734 addroots(void) |
1625 { | 1735 { |
1626 G *gp; | 1736 G *gp; |
1627 FinBlock *fb; | 1737 FinBlock *fb; |
1628 MSpan *s, **allspans; | 1738 MSpan *s, **allspans; |
1629 uint32 spanidx; | 1739 uint32 spanidx; |
1630 » Obj *wp; | 1740 |
1631 » Workbuf *wbuf; | 1741 » work.nroot = 0; |
1632 » uintptr nobj; | 1742 |
1633 » WbufState wbufstate; | 1743 » // data & bss |
1634 » CustomMarker *cm; | 1744 » // TODO(atom): load balancing |
1635 | 1745 » addroot((Obj){data, edata - data, (uintptr)gcdata}); |
1636 » wbuf = nil; // current work buffer | 1746 » addroot((Obj){bss, ebss - bss, (uintptr)gcbss}); |
1637 » wp = nil; // storage for next queued pointer (write pointer) | 1747 |
1638 » nobj = 0; // number of queued objects | 1748 » // MSpan.types |
1639 » wbufstate = (WbufState){&wp, &wbuf, &nobj}; | 1749 » allspans = runtime·mheap->allspans; |
1640 | 1750 » for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) { |
1641 » // call custom markers | |
1642 » for(cm=custommarkers; cm!=nil; cm=cm->next) | |
1643 » » cm->f(); | |
1644 | |
1645 » // mark data & bss | |
1646 » enqueue((Obj){data, edata - data, (uintptr)data_gc}, &wp, &wbuf, &nobj); | |
1647 » enqueue((Obj){bss, ebss - bss, (uintptr)bss_gc}, &wp, &wbuf, &nobj); | |
1648 | |
1649 » // mark MSpan.types | |
1650 » allspans = runtime·mheap.allspans; | |
1651 » for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) { | |
1652 s = allspans[spanidx]; | 1751 s = allspans[spanidx]; |
1653 if(s->state == MSpanInUse) { | 1752 if(s->state == MSpanInUse) { |
1753 // The garbage collector ignores type pointers stored in MSpan.types: | |
1754 // - Compiler-generated types are stored outside of hea p. | |
1755 // - The reflect package has runtime-generated types ca ched in its data structures. | |
1756 // The garbage collector relies on finding the refere nces via that cache. | |
1654 switch(s->types.compression) { | 1757 switch(s->types.compression) { |
1655 » » » case 0: | 1758 » » » case MTypes_Empty: |
1656 » » » case 1: | 1759 » » » case MTypes_Single: |
1657 break; | 1760 break; |
1658 » » » case 2: | 1761 » » » case MTypes_Words: |
1659 » » » case 3: | 1762 » » » case MTypes_Bytes: |
1660 » » » » markenqueue((Obj){(byte*)&s->types.data, sizeof( void*), 0}, &wp, &wbuf, &nobj); | 1763 » » » » markonly((byte*)s->types.data); |
1661 break; | 1764 break; |
1662 } | 1765 } |
1663 } | 1766 } |
1664 » }» | 1767 » } |
1665 | 1768 |
1666 » // mark stacks, G.defer | 1769 » // stacks |
1667 for(gp=runtime·allg; gp!=nil; gp=gp->alllink) { | 1770 for(gp=runtime·allg; gp!=nil; gp=gp->alllink) { |
1668 switch(gp->status){ | 1771 switch(gp->status){ |
1669 default: | 1772 default: |
1670 runtime·printf("unexpected G.status %d\n", gp->status); | 1773 runtime·printf("unexpected G.status %d\n", gp->status); |
1671 runtime·throw("mark - bad status"); | 1774 runtime·throw("mark - bad status"); |
1672 case Gdead: | 1775 case Gdead: |
1673 break; | 1776 break; |
1674 case Grunning: | 1777 case Grunning: |
1675 if(gp != g) | 1778 if(gp != g) |
1676 runtime·throw("mark - world not stopped"); | 1779 runtime·throw("mark - world not stopped"); |
1677 » » » appendstack(gp, &wp, &wbuf, &nobj); | 1780 » » » addstackroots(gp); |
1678 break; | 1781 break; |
1679 case Grunnable: | 1782 case Grunnable: |
1680 case Gsyscall: | 1783 case Gsyscall: |
1681 case Gwaiting: | 1784 case Gwaiting: |
1682 » » » appendstack(gp, &wp, &wbuf, &nobj); | 1785 » » » addstackroots(gp); |
1683 break; | 1786 break; |
1684 } | 1787 } |
1685 } | 1788 } |
1686 | 1789 |
1687 » // mark things pointed at by objects with finalizers | 1790 » runtime·walkfintab(addfinroots); |
1688 » runtime·walkfintab(markfin, &wbufstate); | |
1689 | 1791 |
1690 for(fb=allfin; fb; fb=fb->alllink) | 1792 for(fb=allfin; fb; fb=fb->alllink) |
1691 » » enqueue((Obj){(byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), 0}, &w p, &wbuf, &nobj); | 1793 » » addroot((Obj){(byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), 0}); |
1692 | |
1693 » // do the scanning and marking | |
1694 » scan(&wp, &wbuf, &nobj); | |
1695 | |
1696 » // in multiproc mode, join in the queued work. | |
1697 » if(work.nproc > 1) { | |
1698 » » if(nobj != 0) | |
1699 » » » runtime·throw("mark: nobj != 0"); | |
1700 » » wbuf = nil; | |
1701 » » wp = nil; | |
1702 » » scan(&wp, &wbuf, &nobj); | |
1703 » } | |
1704 } | 1794 } |
1705 | 1795 |
1706 static bool | 1796 static bool |
1707 handlespecial(byte *p, uintptr size) | 1797 handlespecial(byte *p, uintptr size) |
1708 { | 1798 { |
1709 » void (*fn)(void*); | 1799 » FuncVal *fn; |
1710 » int32 nret; | 1800 » uintptr nret; |
1711 FinBlock *block; | 1801 FinBlock *block; |
1712 Finalizer *f; | 1802 Finalizer *f; |
1713 | 1803 |
1714 if(!runtime·getfinalizer(p, true, &fn, &nret)) { | 1804 if(!runtime·getfinalizer(p, true, &fn, &nret)) { |
1715 runtime·setblockspecial(p, false); | 1805 runtime·setblockspecial(p, false); |
1716 runtime·MProf_Free(p, size); | 1806 runtime·MProf_Free(p, size); |
1717 return false; | 1807 return false; |
1718 } | 1808 } |
1719 | 1809 |
1720 runtime·lock(&finlock); | 1810 runtime·lock(&finlock); |
1721 if(finq == nil || finq->cnt == finq->cap) { | 1811 if(finq == nil || finq->cnt == finq->cap) { |
1722 if(finc == nil) { | 1812 if(finc == nil) { |
1723 finc = runtime·SysAlloc(PageSize); | 1813 finc = runtime·SysAlloc(PageSize); |
1814 if(finc == nil) | |
1815 runtime·throw("runtime: cannot allocate memory") ; | |
1724 finc->cap = (PageSize - sizeof(FinBlock)) / sizeof(Final izer) + 1; | 1816 finc->cap = (PageSize - sizeof(FinBlock)) / sizeof(Final izer) + 1; |
1725 finc->alllink = allfin; | 1817 finc->alllink = allfin; |
1726 allfin = finc; | 1818 allfin = finc; |
1727 } | 1819 } |
1728 block = finc; | 1820 block = finc; |
1729 finc = block->next; | 1821 finc = block->next; |
1730 block->next = finq; | 1822 block->next = finq; |
1731 finq = block; | 1823 finq = block; |
1732 } | 1824 } |
1733 f = &finq->fin[finq->cnt]; | 1825 f = &finq->fin[finq->cnt]; |
1734 finq->cnt++; | 1826 finq->cnt++; |
1735 f->fn = fn; | 1827 f->fn = fn; |
1736 f->nret = nret; | 1828 f->nret = nret; |
1737 f->arg = p; | 1829 f->arg = p; |
1738 runtime·unlock(&finlock); | 1830 runtime·unlock(&finlock); |
1739 return true; | 1831 return true; |
1740 } | 1832 } |
1741 | 1833 |
1742 // Sweep frees or collects finalizers for blocks not marked in the mark phase. | 1834 // Sweep frees or collects finalizers for blocks not marked in the mark phase. |
1743 // It clears the mark bits in preparation for the next GC round. | 1835 // It clears the mark bits in preparation for the next GC round. |
1744 static void | 1836 static void |
1745 sweepspan(ParFor *desc, uint32 idx) | 1837 sweepspan(ParFor *desc, uint32 idx) |
1746 { | 1838 { |
1747 » int32 sizeclass, n, npages; | 1839 » int32 cl, n, npages; |
1748 » uintptr size; | 1840 » uintptr size, u; |
1749 byte *p; | 1841 byte *p; |
1750 MCache *c; | 1842 MCache *c; |
1751 byte *arena_start; | 1843 byte *arena_start; |
1752 MLink head, *end; | 1844 MLink head, *end; |
1753 int32 nfree; | 1845 int32 nfree; |
1754 byte *type_data; | 1846 byte *type_data; |
1755 byte compression; | 1847 byte compression; |
1756 uintptr type_data_inc; | 1848 uintptr type_data_inc; |
1757 MSpan *s; | 1849 MSpan *s; |
1758 | 1850 |
1759 USED(&desc); | 1851 USED(&desc); |
1760 » s = runtime·mheap.allspans[idx]; | 1852 » s = runtime·mheap->allspans[idx]; |
1761 » // Stamp newly unused spans. The scavenger will use that | |
1762 » // info to potentially give back some pages to the OS. | |
1763 » if(s->state == MSpanFree && s->unusedsince == 0) | |
1764 » » s->unusedsince = runtime·nanotime(); | |
1765 if(s->state != MSpanInUse) | 1853 if(s->state != MSpanInUse) |
1766 return; | 1854 return; |
1767 » arena_start = runtime·mheap.arena_start; | 1855 » arena_start = runtime·mheap->arena_start; |
1768 p = (byte*)(s->start << PageShift); | 1856 p = (byte*)(s->start << PageShift); |
1769 » sizeclass = s->sizeclass; | 1857 » cl = s->sizeclass; |
1770 size = s->elemsize; | 1858 size = s->elemsize; |
1771 » if(sizeclass == 0) { | 1859 » if(cl == 0) { |
1772 n = 1; | 1860 n = 1; |
1773 } else { | 1861 } else { |
1774 // Chunk full of small blocks. | 1862 // Chunk full of small blocks. |
1775 » » npages = runtime·class_to_allocnpages[sizeclass]; | 1863 » » npages = runtime·class_to_allocnpages[cl]; |
1776 n = (npages << PageShift) / size; | 1864 n = (npages << PageShift) / size; |
1777 } | 1865 } |
1778 nfree = 0; | 1866 nfree = 0; |
1779 end = &head; | 1867 end = &head; |
1780 c = m->mcache; | 1868 c = m->mcache; |
1781 ········ | 1869 ········ |
1782 type_data = (byte*)s->types.data; | 1870 type_data = (byte*)s->types.data; |
1783 type_data_inc = sizeof(uintptr); | 1871 type_data_inc = sizeof(uintptr); |
1784 compression = s->types.compression; | 1872 compression = s->types.compression; |
1785 switch(compression) { | 1873 switch(compression) { |
1786 » case 3: | 1874 » case MTypes_Bytes: |
1787 type_data += 8*sizeof(uintptr); | 1875 type_data += 8*sizeof(uintptr); |
1788 type_data_inc = 1; | 1876 type_data_inc = 1; |
1789 break; | 1877 break; |
1790 } | 1878 } |
1791 | 1879 |
1792 // Sweep through n objects of given size starting at p. | 1880 // Sweep through n objects of given size starting at p. |
1793 // This thread owns the span now, so it can manipulate | 1881 // This thread owns the span now, so it can manipulate |
1794 // the block bitmap without atomic operations. | 1882 // the block bitmap without atomic operations. |
1795 » for(; n>0; n--, p+=size, type_data+=type_data_inc) { | 1883 » for(; n > 0; n--, p += size, type_data+=type_data_inc) { |
1796 uintptr off, *bitp, shift, bits; | 1884 uintptr off, *bitp, shift, bits; |
1797 | 1885 |
1798 off = (uintptr*)p - (uintptr*)arena_start; | 1886 off = (uintptr*)p - (uintptr*)arena_start; |
1799 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | 1887 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; |
1800 shift = off % wordsPerBitmapWord; | 1888 shift = off % wordsPerBitmapWord; |
1801 bits = *bitp>>shift; | 1889 bits = *bitp>>shift; |
1802 | 1890 |
1803 if((bits & bitAllocated) == 0) | 1891 if((bits & bitAllocated) == 0) |
1804 continue; | 1892 continue; |
1805 | 1893 |
1806 if((bits & bitMarked) != 0) { | 1894 if((bits & bitMarked) != 0) { |
1807 if(DebugMark) { | 1895 if(DebugMark) { |
1808 if(!(bits & bitSpecial)) | 1896 if(!(bits & bitSpecial)) |
1809 runtime·printf("found spurious mark on % p\n", p); | 1897 runtime·printf("found spurious mark on % p\n", p); |
1810 *bitp &= ~(bitSpecial<<shift); | 1898 *bitp &= ~(bitSpecial<<shift); |
1811 } | 1899 } |
1812 *bitp &= ~(bitMarked<<shift); | 1900 *bitp &= ~(bitMarked<<shift); |
1813 continue; | 1901 continue; |
1814 } | 1902 } |
1815 | 1903 |
1816 // Special means it has a finalizer or is being profiled. | 1904 // Special means it has a finalizer or is being profiled. |
1817 // In DebugMark mode, the bit has been coopted so | 1905 // In DebugMark mode, the bit has been coopted so |
1818 // we have to assume all blocks are special. | 1906 // we have to assume all blocks are special. |
1819 if(DebugMark || (bits & bitSpecial) != 0) { | 1907 if(DebugMark || (bits & bitSpecial) != 0) { |
1820 if(handlespecial(p, size)) | 1908 if(handlespecial(p, size)) |
1821 continue; | 1909 continue; |
1822 } | 1910 } |
1823 | 1911 |
1912 // Scramble contents of the block. Don't free the block. | |
1913 if(MemScramble) { | |
1914 for(u = 0; u<size; u+=sizeof(void*)) { | |
1915 // Use a number that can be used to reconstruct the address p+u. | |
1916 // The number is (hopefully) an invalid address. | |
1917 *(uintptr*)(p+u) = -(uintptr)(p+u); | |
1918 } | |
1919 if(0) runtime·printf("free %p\n", p); | |
1920 *bitp |= (bitSpecial<<shift); | |
1921 continue; | |
1922 } | |
1923 | |
1824 // Mark freed; restore block boundary bit. | 1924 // Mark freed; restore block boundary bit. |
1825 *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 1925 *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
1826 | 1926 |
1827 » » if(sizeclass == 0) { | 1927 » » if(cl == 0) { |
1828 // Free large span. | 1928 // Free large span. |
1829 runtime·unmarkspan(p, 1<<PageShift); | 1929 runtime·unmarkspan(p, 1<<PageShift); |
1830 » » » *(uintptr*)p = 1;» // needs zeroing | 1930 » » » *(uintptr*)p = (uintptr)0xdeaddeaddeaddeadll;» // needs zeroing |
1831 » » » runtime·MHeap_Free(&runtime·mheap, s, 1); | 1931 » » » runtime·MHeap_Free(runtime·mheap, s, 1); |
1832 c->local_alloc -= size; | 1932 c->local_alloc -= size; |
1833 c->local_nfree++; | 1933 c->local_nfree++; |
1834 } else { | 1934 } else { |
1835 // Free small object. | 1935 // Free small object. |
1836 switch(compression) { | 1936 switch(compression) { |
1837 » » » » case 2: | 1937 » » » case MTypes_Words: |
rsc
2012/05/24 16:36:45
unindent
atom
2012/06/01 16:45:47
Done.
| |
1838 » » » » » *(uintptr*)type_data = 0; | 1938 » » » » *(uintptr*)type_data = 0; |
1839 » » » » » break; | 1939 » » » » break; |
1840 » » » » case 3: | 1940 » » » case MTypes_Bytes: |
1841 » » » » » *(byte*)type_data = 0; | 1941 » » » » *(byte*)type_data = 0; |
1842 » » » » » break; | 1942 » » » » break; |
1843 } | 1943 } |
1844 if(size > sizeof(uintptr)) | 1944 if(size > sizeof(uintptr)) |
1845 » » » » ((uintptr*)p)[1] = 1;» // mark as "needs to be zeroed" | 1945 » » » » ((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll ;» // mark as "needs to be zeroed" |
1846 ························ | 1946 ························ |
1847 end->next = (MLink*)p; | 1947 end->next = (MLink*)p; |
1848 end = (MLink*)p; | 1948 end = (MLink*)p; |
1849 nfree++; | 1949 nfree++; |
1850 } | 1950 } |
1851 } | 1951 } |
1852 | 1952 |
1853 if(nfree) { | 1953 if(nfree) { |
1854 » » c->local_by_size[sizeclass].nfree += nfree; | 1954 » » c->local_by_size[cl].nfree += nfree; |
1855 c->local_alloc -= size * nfree; | 1955 c->local_alloc -= size * nfree; |
1856 c->local_nfree += nfree; | 1956 c->local_nfree += nfree; |
1857 c->local_cachealloc -= nfree * size; | 1957 c->local_cachealloc -= nfree * size; |
1858 c->local_objects -= nfree; | 1958 c->local_objects -= nfree; |
1859 » » runtime·MCentral_FreeSpan(&runtime·mheap.central[sizeclass], s, nfree, head.next, end); | 1959 » » runtime·MCentral_FreeSpan(&runtime·mheap->central[cl], s, nfree, head.next, end); |
1960 » } | |
1961 } | |
1962 | |
1963 static void | |
1964 dumpspan(uint32 idx) | |
1965 { | |
1966 » int32 sizeclass, n, npages, i, column; | |
1967 » uintptr size; | |
1968 » byte *p; | |
1969 » byte *arena_start; | |
1970 » MSpan *s; | |
1971 » bool allocated, special; | |
1972 | |
1973 » s = runtime·mheap->allspans[idx]; | |
1974 » if(s->state != MSpanInUse) | |
1975 » » return; | |
1976 » arena_start = runtime·mheap->arena_start; | |
1977 » p = (byte*)(s->start << PageShift); | |
1978 » sizeclass = s->sizeclass; | |
1979 » size = s->elemsize; | |
1980 » if(sizeclass == 0) { | |
1981 » » n = 1; | |
1982 » } else { | |
1983 » » npages = runtime·class_to_allocnpages[sizeclass]; | |
1984 » » n = (npages << PageShift) / size; | |
1985 » } | |
1986 »······· | |
1987 » runtime·printf("%p .. %p:\n", p, p+n*size); | |
1988 » column = 0; | |
1989 » for(; n>0; n--, p+=size) { | |
1990 » » uintptr off, *bitp, shift, bits; | |
1991 | |
1992 » » off = (uintptr*)p - (uintptr*)arena_start; | |
1993 » » bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | |
1994 » » shift = off % wordsPerBitmapWord; | |
1995 » » bits = *bitp>>shift; | |
1996 | |
1997 » » allocated = ((bits & bitAllocated) != 0); | |
1998 » » special = ((bits & bitSpecial) != 0); | |
1999 | |
2000 » » for(i=0; i<size; i+=sizeof(void*)) { | |
2001 » » » if(column == 0) { | |
2002 » » » » runtime·printf("\t"); | |
2003 » » » } | |
2004 » » » if(i == 0) { | |
2005 » » » » runtime·printf(allocated ? "(" : "["); | |
2006 » » » » runtime·printf(special ? "@" : ""); | |
2007 » » » » runtime·printf("%p: ", p+i); | |
2008 » » » } else { | |
2009 » » » » runtime·printf(" "); | |
2010 » » » } | |
2011 | |
2012 » » » runtime·printf("%p", *(void**)(p+i)); | |
2013 | |
2014 » » » if(i+sizeof(void*) >= size) { | |
2015 » » » » runtime·printf(allocated ? ") " : "] "); | |
2016 » » » } | |
2017 | |
2018 » » » column++; | |
2019 » » » if(column == 8) { | |
2020 » » » » runtime·printf("\n"); | |
2021 » » » » column = 0; | |
2022 » » » } | |
2023 » » } | |
2024 » } | |
2025 » runtime·printf("\n"); | |
2026 } | |
2027 | |
2028 // A debugging function to dump the contents of memory | |
2029 void | |
2030 runtime·memorydump(void) | |
2031 { | |
2032 » uint32 spanidx; | |
2033 | |
2034 » for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) { | |
2035 » » dumpspan(spanidx); | |
1860 } | 2036 } |
1861 } | 2037 } |
1862 | 2038 |
1863 void | 2039 void |
1864 runtime·gchelper(void) | 2040 runtime·gchelper(void) |
1865 { | 2041 { |
1866 » Obj *wp; | 2042 » gchelperstart(); |
1867 » Workbuf *wbuf; | 2043 |
1868 » uintptr nobj; | 2044 » // parallel mark for over gc roots |
1869 | 2045 » runtime·parfordo(work.markfor); |
1870 » wp = nil; | 2046 |
1871 » wbuf = nil; | 2047 » // help other threads scan secondary blocks |
1872 » nobj = 0; | 2048 » scanblock(nil, nil, 0, true); |
1873 | |
1874 » scan(&wp, &wbuf, &nobj); | |
1875 | 2049 |
1876 if(DebugMark) { | 2050 if(DebugMark) { |
1877 // wait while the main thread executes mark(debug_scanblock) | 2051 // wait while the main thread executes mark(debug_scanblock) |
1878 while(runtime·atomicload(&work.debugmarkdone) == 0) | 2052 while(runtime·atomicload(&work.debugmarkdone) == 0) |
1879 runtime·usleep(10); | 2053 runtime·usleep(10); |
1880 } | 2054 } |
1881 | 2055 |
1882 runtime·parfordo(work.sweepfor); | 2056 runtime·parfordo(work.sweepfor); |
2057 bufferList[m->helpgc].busy = 0; | |
1883 if(runtime·xadd(&work.ndone, +1) == work.nproc-1) | 2058 if(runtime·xadd(&work.ndone, +1) == work.nproc-1) |
1884 runtime·notewakeup(&work.alldone); | 2059 runtime·notewakeup(&work.alldone); |
1885 } | 2060 } |
2061 | |
2062 #define GcpercentUnknown (-2) | |
1886 | 2063 |
1887 // Initialized from $GOGC. GOGC=off means no gc. | 2064 // Initialized from $GOGC. GOGC=off means no gc. |
1888 // | 2065 // |
1889 // Next gc is after we've allocated an extra amount of | 2066 // Next gc is after we've allocated an extra amount of |
1890 // memory proportional to the amount already in use. | 2067 // memory proportional to the amount already in use. |
1891 // If gcpercent=100 and we're using 4M, we'll gc again | 2068 // If gcpercent=100 and we're using 4M, we'll gc again |
1892 // when we get to 8M. This keeps the gc cost in linear | 2069 // when we get to 8M. This keeps the gc cost in linear |
1893 // proportion to the allocation cost. Adjusting gcpercent | 2070 // proportion to the allocation cost. Adjusting gcpercent |
1894 // just changes the linear constant (and also the amount of | 2071 // just changes the linear constant (and also the amount of |
1895 // extra memory used). | 2072 // extra memory used). |
1896 static int32 gcpercent = -2; | 2073 static int32 gcpercent = GcpercentUnknown; |
1897 | |
1898 static void | |
1899 stealcache(void) | |
1900 { | |
1901 » M *m; | |
1902 | |
1903 » for(m=runtime·allm; m; m=m->alllink) | |
1904 » » runtime·MCache_ReleaseAll(m->mcache); | |
1905 } | |
1906 | 2074 |
1907 static void | 2075 static void |
1908 cachestats(GCStats *stats) | 2076 cachestats(GCStats *stats) |
1909 { | 2077 { |
1910 » M *m; | 2078 » M *mp; |
1911 MCache *c; | 2079 MCache *c; |
2080 P *p, **pp; | |
1912 int32 i; | 2081 int32 i; |
1913 uint64 stacks_inuse; | 2082 uint64 stacks_inuse; |
1914 uint64 stacks_sys; | |
1915 uint64 *src, *dst; | 2083 uint64 *src, *dst; |
1916 | 2084 |
1917 if(stats) | 2085 if(stats) |
1918 runtime·memclr((byte*)stats, sizeof(*stats)); | 2086 runtime·memclr((byte*)stats, sizeof(*stats)); |
1919 stacks_inuse = 0; | 2087 stacks_inuse = 0; |
1920 » stacks_sys = 0; | 2088 » for(mp=runtime·allm; mp; mp=mp->alllink) { |
1921 » for(m=runtime·allm; m; m=m->alllink) { | 2089 » » stacks_inuse += mp->stackinuse*FixedStack; |
1922 » » runtime·purgecachedstats(m); | |
1923 » » stacks_inuse += m->stackalloc->inuse; | |
1924 » » stacks_sys += m->stackalloc->sys; | |
1925 if(stats) { | 2090 if(stats) { |
1926 » » » src = (uint64*)&m->gcstats; | 2091 » » » src = (uint64*)&mp->gcstats; |
1927 dst = (uint64*)stats; | 2092 dst = (uint64*)stats; |
1928 for(i=0; i<sizeof(*stats)/sizeof(uint64); i++) | 2093 for(i=0; i<sizeof(*stats)/sizeof(uint64); i++) |
1929 dst[i] += src[i]; | 2094 dst[i] += src[i]; |
1930 » » » runtime·memclr((byte*)&m->gcstats, sizeof(m->gcstats)); | 2095 » » » runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats)) ; |
1931 » » } | 2096 » » } |
1932 » » c = m->mcache; | 2097 » } |
1933 » » for(i=0; i<nelem(c->local_by_size); i++) { | 2098 » for(pp=runtime·allp; p=*pp; pp++) { |
1934 » » » mstats.by_size[i].nmalloc += c->local_by_size[i].nmalloc ; | 2099 » » c = p->mcache; |
1935 » » » c->local_by_size[i].nmalloc = 0; | 2100 » » if(c==nil) |
1936 » » » mstats.by_size[i].nfree += c->local_by_size[i].nfree; | 2101 » » » continue; |
1937 » » » c->local_by_size[i].nfree = 0; | 2102 » » runtime·purgecachedstats(c); |
1938 » » } | |
1939 } | 2103 } |
1940 mstats.stacks_inuse = stacks_inuse; | 2104 mstats.stacks_inuse = stacks_inuse; |
1941 mstats.stacks_sys = stacks_sys; | |
1942 } | 2105 } |
1943 | 2106 |
1944 // Structure of arguments passed to function gc(). | 2107 // Structure of arguments passed to function gc(). |
1945 // This allows the arguments to be easily passed via reflect·call. | 2108 // This allows the arguments to be passed via reflect·call. |
1946 struct gc_args | 2109 struct gc_args |
1947 { | 2110 { |
1948 int32 force; | 2111 int32 force; |
1949 }; | 2112 }; |
1950 | 2113 |
1951 static void | 2114 static void gc(struct gc_args *args); |
1952 gc(struct gc_args *args) | 2115 |
1953 { | 2116 static int32 |
1954 » int64 t0, t1, t2, t3; | 2117 readgogc(void) |
1955 » uint64 heap0, heap1, obj0, obj1; | 2118 { |
1956 byte *p; | 2119 byte *p; |
1957 » GCStats stats; | 2120 |
1958 » M *m1; | 2121 » p = runtime·getenv("GOGC"); |
2122 » if(p == nil || p[0] == '\0') | |
2123 » » return 100; | |
2124 » if(runtime·strcmp(p, (byte*)"off") == 0) | |
2125 » » return -1; | |
2126 » return runtime·atoi(p); | |
2127 } | |
2128 | |
2129 void | |
2130 runtime·gc(int32 force) | |
2131 { | |
2132 » byte *p; | |
2133 » struct gc_args a, *ap; | |
2134 » FuncVal gcv; | |
2135 | |
2136 » // The atomic operations are not atomic if the uint64s | |
2137 » // are not aligned on uint64 boundaries. This has been | |
2138 » // a problem in the past. | |
2139 » if((((uintptr)&work.empty) & 7) != 0) | |
2140 » » runtime·throw("runtime: gc work buffer is misaligned"); | |
2141 » if((((uintptr)&work.full) & 7) != 0) | |
2142 » » runtime·throw("runtime: gc work buffer is misaligned"); | |
1959 | 2143 |
1960 // The gc is turned off (via enablegc) until | 2144 // The gc is turned off (via enablegc) until |
1961 // the bootstrap has completed. | 2145 // the bootstrap has completed. |
1962 // Also, malloc gets called in the guts | 2146 // Also, malloc gets called in the guts |
1963 // of a number of libraries that might be | 2147 // of a number of libraries that might be |
1964 // holding locks. To avoid priority inversion | 2148 // holding locks. To avoid priority inversion |
1965 // problems, don't bother trying to run gc | 2149 // problems, don't bother trying to run gc |
1966 // while holding a lock. The next mallocgc | 2150 // while holding a lock. The next mallocgc |
1967 // without a lock will do the gc instead. | 2151 // without a lock will do the gc instead. |
1968 if(!mstats.enablegc || m->locks > 0 || runtime·panicking) | 2152 if(!mstats.enablegc || m->locks > 0 || runtime·panicking) |
1969 return; | 2153 return; |
1970 | 2154 |
1971 » if(gcpercent == -2) {» // first time through | 2155 » if(gcpercent == GcpercentUnknown) {» // first time through |
1972 » » p = runtime·getenv("GOGC"); | 2156 » » gcpercent = readgogc(); |
1973 » » if(p == nil || p[0] == '\0') | |
1974 » » » gcpercent = 100; | |
1975 » » else if(runtime·strcmp(p, (byte*)"off") == 0) | |
1976 » » » gcpercent = -1; | |
1977 » » else | |
1978 » » » gcpercent = runtime·atoi(p); | |
1979 | 2157 |
1980 p = runtime·getenv("GOGCTRACE"); | 2158 p = runtime·getenv("GOGCTRACE"); |
1981 if(p != nil) | 2159 if(p != nil) |
1982 gctrace = runtime·atoi(p); | 2160 gctrace = runtime·atoi(p); |
1983 } | 2161 } |
1984 if(gcpercent < 0) | 2162 if(gcpercent < 0) |
1985 return; | 2163 return; |
2164 | |
2165 // Run gc on a bigger stack to eliminate | |
2166 // a potentially large number of calls to runtime·morestack. | |
2167 a.force = force; | |
2168 ap = &a; | |
2169 m->moreframesize_minalloc = StackBig; | |
2170 gcv.fn = (void*)gc; | |
2171 reflect·call(&gcv, (byte*)&ap, sizeof(ap)); | |
2172 | |
2173 if(gctrace > 1 && !force) { | |
2174 a.force = 1; | |
2175 gc(&a); | |
2176 } | |
2177 } | |
2178 | |
2179 static FuncVal runfinqv = {runfinq}; | |
2180 | |
2181 static void | |
2182 gc(struct gc_args *args) | |
2183 { | |
2184 int64 t0, t1, t2, t3, t4; | |
2185 uint64 heap0, heap1, obj0, obj1, ninstr; | |
2186 GCStats stats; | |
2187 M *mp; | |
2188 uint32 i; | |
2189 Eface eface; | |
1986 | 2190 |
1987 runtime·semacquire(&runtime·worldsema); | 2191 runtime·semacquire(&runtime·worldsema); |
1988 if(!args->force && mstats.heap_alloc < mstats.next_gc) { | 2192 if(!args->force && mstats.heap_alloc < mstats.next_gc) { |
1989 runtime·semrelease(&runtime·worldsema); | 2193 runtime·semrelease(&runtime·worldsema); |
1990 return; | 2194 return; |
1991 } | 2195 } |
1992 | 2196 |
1993 t0 = runtime·nanotime(); | 2197 t0 = runtime·nanotime(); |
1994 | 2198 |
1995 m->gcing = 1; | 2199 m->gcing = 1; |
1996 runtime·stoptheworld(); | 2200 runtime·stoptheworld(); |
1997 | 2201 |
1998 » if(DebugStats) | 2202 » if(CollectStats) |
1999 runtime·memclr((byte*)&gcstats, sizeof(gcstats)); | 2203 runtime·memclr((byte*)&gcstats, sizeof(gcstats)); |
2000 | 2204 |
2001 » for(m1=runtime·allm; m1; m1=m1->alllink) | 2205 » for(mp=runtime·allm; mp; mp=mp->alllink) |
2002 » » runtime·settype_flush(m1, false); | 2206 » » runtime·settype_flush(mp, false); |
2003 | 2207 |
2004 heap0 = 0; | 2208 heap0 = 0; |
2005 obj0 = 0; | 2209 obj0 = 0; |
2006 if(gctrace) { | 2210 if(gctrace) { |
2007 cachestats(nil); | 2211 cachestats(nil); |
2008 heap0 = mstats.heap_alloc; | 2212 heap0 = mstats.heap_alloc; |
2009 obj0 = mstats.nmalloc - mstats.nfree; | 2213 obj0 = mstats.nmalloc - mstats.nfree; |
2010 } | 2214 } |
2011 | 2215 |
2216 m->locks++; // disable gc during mallocs in parforalloc | |
2217 if(work.markfor == nil) | |
2218 work.markfor = runtime·parforalloc(MaxGcproc); | |
2219 if(work.sweepfor == nil) | |
2220 work.sweepfor = runtime·parforalloc(MaxGcproc); | |
2221 m->locks--; | |
2222 | |
2223 if(itabtype == nil) { | |
2224 // get C pointer to the Go type "itab" | |
2225 runtime·gc_itab_ptr(&eface); | |
2226 itabtype = ((PtrType*)eface.type)->elem; | |
2227 } | |
2228 | |
2012 work.nwait = 0; | 2229 work.nwait = 0; |
2013 work.ndone = 0; | 2230 work.ndone = 0; |
2014 work.debugmarkdone = 0; | 2231 work.debugmarkdone = 0; |
2015 work.nproc = runtime·gcprocs(); | 2232 work.nproc = runtime·gcprocs(); |
2016 » if(work.sweepfor == nil) | 2233 » addroots(); |
2017 » » work.sweepfor = runtime·parforalloc(MaxGcproc); | 2234 » runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, ma rkroot); |
2018 » runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap.nspan, nil, true, sweepspan); | 2235 » runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap->nspan, nil , true, sweepspan); |
2019 if(work.nproc > 1) { | 2236 if(work.nproc > 1) { |
2020 runtime·noteclear(&work.alldone); | 2237 runtime·noteclear(&work.alldone); |
2021 runtime·helpgc(work.nproc); | 2238 runtime·helpgc(work.nproc); |
2022 } | 2239 } |
2023 | 2240 |
2024 » mark(scan); | 2241 » t1 = runtime·nanotime(); |
2242 | |
2243 » gchelperstart(); | |
2244 » runtime·parfordo(work.markfor); | |
2245 » scanblock(nil, nil, 0, true); | |
2246 | |
2025 if(DebugMark) { | 2247 if(DebugMark) { |
2026 » » mark(debug_scan); | 2248 » » for(i=0; i<work.nroot; i++) |
2249 » » » debug_scanblock(work.roots[i].p, work.roots[i].n); | |
2027 runtime·atomicstore(&work.debugmarkdone, 1); | 2250 runtime·atomicstore(&work.debugmarkdone, 1); |
2028 } | 2251 } |
2029 » t1 = runtime·nanotime(); | 2252 » t2 = runtime·nanotime(); |
2030 | 2253 |
2031 runtime·parfordo(work.sweepfor); | 2254 runtime·parfordo(work.sweepfor); |
2032 » t2 = runtime·nanotime(); | 2255 » bufferList[m->helpgc].busy = 0; |
2033 | 2256 » t3 = runtime·nanotime(); |
2034 » stealcache(); | |
2035 » cachestats(&stats); | |
2036 | 2257 |
2037 if(work.nproc > 1) | 2258 if(work.nproc > 1) |
2038 runtime·notesleep(&work.alldone); | 2259 runtime·notesleep(&work.alldone); |
2260 | |
2261 cachestats(&stats); | |
2039 | 2262 |
2040 stats.nprocyield += work.sweepfor->nprocyield; | 2263 stats.nprocyield += work.sweepfor->nprocyield; |
2041 stats.nosyield += work.sweepfor->nosyield; | 2264 stats.nosyield += work.sweepfor->nosyield; |
2042 stats.nsleep += work.sweepfor->nsleep; | 2265 stats.nsleep += work.sweepfor->nsleep; |
2043 | 2266 |
2044 mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; | 2267 mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; |
2045 m->gcing = 0; | 2268 m->gcing = 0; |
2046 | 2269 |
2047 if(finq != nil) { | |
2048 m->locks++; // disable gc during the mallocs in newproc | |
2049 // kick off or wake up goroutine to run queued finalizers | |
2050 if(fing == nil) | |
2051 fing = runtime·newproc1((byte*)runfinq, nil, 0, 0, runti me·gc); | |
2052 else if(fingwait) { | |
2053 fingwait = 0; | |
2054 runtime·ready(fing); | |
2055 } | |
2056 m->locks--; | |
2057 } | |
2058 | |
2059 heap1 = mstats.heap_alloc; | 2270 heap1 = mstats.heap_alloc; |
2060 obj1 = mstats.nmalloc - mstats.nfree; | 2271 obj1 = mstats.nmalloc - mstats.nfree; |
2061 | 2272 |
2062 » t3 = runtime·nanotime(); | 2273 » t4 = runtime·nanotime(); |
2063 » mstats.last_gc = t3; | 2274 » mstats.last_gc = t4; |
2064 » mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t3 - t0; | 2275 » mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t4 - t0; |
2065 » mstats.pause_total_ns += t3 - t0; | 2276 » mstats.pause_total_ns += t4 - t0; |
2066 mstats.numgc++; | 2277 mstats.numgc++; |
2067 if(mstats.debuggc) | 2278 if(mstats.debuggc) |
2068 » » runtime·printf("pause %D\n", t3-t0); | 2279 » » runtime·printf("pause %D\n", t4-t0); |
2069 | 2280 |
2070 if(gctrace) { | 2281 if(gctrace) { |
2071 runtime·printf("gc%d(%d): %D.%D+%D+%D ms, %D -> %D MB %D -> %D ( %D-%D) objects," | 2282 runtime·printf("gc%d(%d): %D.%D+%D+%D ms, %D -> %D MB %D -> %D ( %D-%D) objects," |
2072 " %D(%D) handoff, %D(%D) steal, %D/%D/%D yields\ n", | 2283 " %D(%D) handoff, %D(%D) steal, %D/%D/%D yields\ n", |
2073 » » » mstats.numgc, work.nproc, (t1-t0)/1000000, (t1-t0)/10000 0%10, (t2-t1)/1000000, (t3-t2)/1000000, | 2284 » » » mstats.numgc, work.nproc, (t2-t1)/1000000, (t2-t1)/10000 0%10, (t3-t2)/1000000, (t1-t0+t4-t3)/1000000, |
2074 heap0>>20, heap1>>20, obj0, obj1, | 2285 heap0>>20, heap1>>20, obj0, obj1, |
2075 mstats.nmalloc, mstats.nfree, | 2286 mstats.nmalloc, mstats.nfree, |
2076 stats.nhandoff, stats.nhandoffcnt, | 2287 stats.nhandoff, stats.nhandoffcnt, |
2077 work.sweepfor->nsteal, work.sweepfor->nstealcnt, | 2288 work.sweepfor->nsteal, work.sweepfor->nstealcnt, |
2078 stats.nprocyield, stats.nosyield, stats.nsleep); | 2289 stats.nprocyield, stats.nosyield, stats.nsleep); |
2079 » » if(DebugStats) { | 2290 » » if(CollectStats) { |
2080 » » » runtime·printf("scan: %D objects, %D untyped, %D types f rom MSpan\n", | 2291 » » » runtime·printf("scan: %D bytes, %D objects, %D untyped, %D types from MSpan\n", |
2081 » » » » gcstats.nobj_cnt, gcstats.nobj_notype, gcstats.n obj_typelookups_ok); | 2292 » » » » gcstats.nbytes, gcstats.obj.cnt, gcstats.obj.not ype, gcstats.obj.typelookup); |
2082 » » » if(gcstats.x_cnt != 0) | 2293 » » » if(gcstats.ptr.cnt != 0) |
2083 » » » » runtime·printf("avg xbufsize: %D (%D/%D)\n", gcs tats.x_sum/gcstats.x_cnt, gcstats.x_sum, gcstats.x_cnt); | 2294 » » » » runtime·printf("avg ptrbufsize: %D (%D/%D)\n", |
2084 » » » if(gcstats.nobj_cnt != 0) | 2295 » » » » » gcstats.ptr.sum/gcstats.ptr.cnt, gcstats .ptr.sum, gcstats.ptr.cnt); |
2085 » » » » runtime·printf("avg nobj: %D (%D/%D)\n", gcstats .nobj_sum/gcstats.nobj_cnt, gcstats.nobj_sum, gcstats.nobj_cnt); | 2296 » » » if(gcstats.obj.cnt != 0) |
2086 » » » runtime·printf("rescans: %D, %D bytes\n", gcstats.rescan s, gcstats.rescan_nbytes); | 2297 » » » » runtime·printf("avg nobj: %D (%D/%D)\n", |
2298 » » » » » gcstats.obj.sum/gcstats.obj.cnt, gcstats .obj.sum, gcstats.obj.cnt); | |
2299 » » » runtime·printf("rescans: %D, %D bytes\n", gcstats.rescan , gcstats.rescanbytes); | |
2087 | 2300 |
2088 runtime·printf("instruction counts:\n"); | 2301 runtime·printf("instruction counts:\n"); |
2089 » » » int32 i; | 2302 » » » ninstr = 0; |
rsc
2012/05/24 16:36:45
Move variables to top.
atom
2012/06/01 16:45:47
Done.
| |
2090 » » » int64 total_instr = 0; | |
2091 for(i=0; i<nelem(gcstats.instr); i++) { | 2303 for(i=0; i<nelem(gcstats.instr); i++) { |
2092 runtime·printf("\t%d:\t%D\n", i, gcstats.instr[i ]); | 2304 runtime·printf("\t%d:\t%D\n", i, gcstats.instr[i ]); |
2093 » » » » total_instr += gcstats.instr[i]; | 2305 » » » » ninstr += gcstats.instr[i]; |
2094 » » » } | 2306 » » » } |
2095 » » » runtime·printf("\ttotal:\t%D\n", total_instr); | 2307 » » » runtime·printf("\ttotal:\t%D\n", ninstr); |
2096 | 2308 |
2097 runtime·printf("putempty: %D, getfull: %D\n", gcstats.pu tempty, gcstats.getfull); | 2309 runtime·printf("putempty: %D, getfull: %D\n", gcstats.pu tempty, gcstats.getfull); |
2098 | |
2099 runtime·printf("runtime·markobject: %D\n", gcstats.runti me·markobject); | |
2100 } | 2310 } |
2101 } | 2311 } |
2102 | 2312 |
2103 runtime·MProf_GC(); | 2313 runtime·MProf_GC(); |
2104 runtime·semrelease(&runtime·worldsema); | 2314 runtime·semrelease(&runtime·worldsema); |
2105 runtime·starttheworld(); | 2315 runtime·starttheworld(); |
2106 | 2316 |
2107 » // give the queued finalizers, if any, a chance to run | 2317 » if(finq != nil) { |
2108 » if(finq != nil) | 2318 » » runtime·lock(&finlock); |
2319 » » // kick off or wake up goroutine to run queued finalizers | |
2320 » » if(fing == nil) | |
2321 » » » fing = runtime·newproc1(&runfinqv, nil, 0, 0, runtime·gc ); | |
2322 » » else if(fingwait) { | |
2323 » » » fingwait = 0; | |
2324 » » » runtime·ready(fing); | |
2325 » » } | |
2326 » » runtime·unlock(&finlock); | |
2327 » » // give the queued finalizers, if any, a chance to run | |
2109 runtime·gosched(); | 2328 runtime·gosched(); |
2110 | |
2111 if(gctrace > 1 && !args->force) | |
2112 runtime·gc(1); | |
2113 } | |
2114 | |
2115 void | |
2116 runtime·gc(int32 force) | |
2117 { | |
2118 struct gc_args a, *ap; | |
2119 | |
2120 a.force = force; | |
2121 ap = &a; | |
2122 | |
2123 if(!runtime·iscgo) { | |
2124 // Run gc on a bigger stack in order to eliminate | |
2125 // a potentially large number of calls to runtime·morestack. | |
2126 m->moreframesize_minalloc = StackBig; | |
2127 reflect·call((byte*)gc, (byte*)&ap, sizeof(ap)); | |
2128 } else { | |
2129 gc(ap); | |
2130 } | 2329 } |
2131 } | 2330 } |
2132 | 2331 |
2133 void | 2332 void |
2134 runtime·ReadMemStats(MStats *stats) | 2333 runtime·ReadMemStats(MStats *stats) |
2135 { | 2334 { |
2136 // Have to acquire worldsema to stop the world, | 2335 // Have to acquire worldsema to stop the world, |
2137 // because stoptheworld can only be used by | 2336 // because stoptheworld can only be used by |
2138 // one goroutine at a time, and there might be | 2337 // one goroutine at a time, and there might be |
2139 // a pending garbage collection already calling it. | 2338 // a pending garbage collection already calling it. |
2140 runtime·semacquire(&runtime·worldsema); | 2339 runtime·semacquire(&runtime·worldsema); |
2141 m->gcing = 1; | 2340 m->gcing = 1; |
2142 runtime·stoptheworld(); | 2341 runtime·stoptheworld(); |
2143 cachestats(nil); | 2342 cachestats(nil); |
2144 *stats = mstats; | 2343 *stats = mstats; |
2145 m->gcing = 0; | 2344 m->gcing = 0; |
2146 runtime·semrelease(&runtime·worldsema); | 2345 runtime·semrelease(&runtime·worldsema); |
2147 runtime·starttheworld(); | 2346 runtime·starttheworld(); |
2148 } | 2347 } |
2149 | 2348 |
2349 void | |
2350 runtime∕debug·readGCStats(Slice *pauses) | |
2351 { | |
2352 uint64 *p; | |
2353 uint32 i, n; | |
2354 | |
2355 // Calling code in runtime/debug should make the slice large enough. | |
2356 if(pauses->cap < nelem(mstats.pause_ns)+3) | |
2357 runtime·throw("runtime: short slice passed to readGCStats"); | |
2358 | |
2359 // Pass back: pauses, last gc (absolute time), number of gc, total pause ns. | |
2360 p = (uint64*)pauses->array; | |
2361 runtime·lock(runtime·mheap); | |
2362 n = mstats.numgc; | |
2363 if(n > nelem(mstats.pause_ns)) | |
2364 n = nelem(mstats.pause_ns); | |
2365 ········ | |
2366 // The pause buffer is circular. The most recent pause is at | |
2367 // pause_ns[(numgc-1)%nelem(pause_ns)], and then backward | |
2368 // from there to go back farther in time. We deliver the times | |
2369 // most recent first (in p[0]). | |
2370 for(i=0; i<n; i++) | |
2371 p[i] = mstats.pause_ns[(mstats.numgc-1-i)%nelem(mstats.pause_ns) ]; | |
2372 | |
2373 p[n] = mstats.last_gc; | |
2374 p[n+1] = mstats.numgc; | |
2375 p[n+2] = mstats.pause_total_ns;· | |
2376 runtime·unlock(runtime·mheap); | |
2377 pauses->len = n+3; | |
2378 } | |
2379 | |
2380 void | |
2381 runtime∕debug·setGCPercent(intgo in, intgo out) | |
2382 { | |
2383 runtime·lock(runtime·mheap); | |
2384 if(gcpercent == GcpercentUnknown) | |
2385 gcpercent = readgogc(); | |
2386 out = gcpercent; | |
2387 if(in < 0) | |
2388 in = -1; | |
2389 gcpercent = in; | |
2390 runtime·unlock(runtime·mheap); | |
2391 FLUSH(&out); | |
2392 } | |
2393 | |
2394 static void | |
2395 gchelperstart(void) | |
2396 { | |
2397 if(m->helpgc < 0 || m->helpgc >= MaxGcproc) | |
2398 runtime·throw("gchelperstart: bad m->helpgc"); | |
2399 if(runtime·xchg(&bufferList[m->helpgc].busy, 1)) | |
2400 runtime·throw("gchelperstart: already busy"); | |
2401 } | |
2402 | |
2150 static void | 2403 static void |
2151 runfinq(void) | 2404 runfinq(void) |
2152 { | 2405 { |
2153 Finalizer *f; | 2406 Finalizer *f; |
2154 FinBlock *fb, *next; | 2407 FinBlock *fb, *next; |
2155 byte *frame; | 2408 byte *frame; |
2156 uint32 framesz, framecap, i; | 2409 uint32 framesz, framecap, i; |
2157 | 2410 |
2158 frame = nil; | 2411 frame = nil; |
2159 framecap = 0; | 2412 framecap = 0; |
2160 for(;;) { | 2413 for(;;) { |
2161 » » // There's no need for a lock in this section | 2414 » » runtime·lock(&finlock); |
2162 » » // because it only conflicts with the garbage | |
2163 » » // collector, and the garbage collector only | |
2164 » » // runs when everyone else is stopped, and | |
2165 » » // runfinq only stops at the gosched() or | |
2166 » » // during the calls in the for loop. | |
2167 fb = finq; | 2415 fb = finq; |
2168 finq = nil; | 2416 finq = nil; |
2169 if(fb == nil) { | 2417 if(fb == nil) { |
2170 fingwait = 1; | 2418 fingwait = 1; |
2171 » » » g->status = Gwaiting; | 2419 » » » runtime·park(runtime·unlock, &finlock, "finalizer wait") ; |
2172 » » » g->waitreason = "finalizer wait"; | |
2173 » » » runtime·gosched(); | |
2174 continue; | 2420 continue; |
2175 } | 2421 } |
2422 runtime·unlock(&finlock); | |
2423 if(raceenabled) | |
2424 runtime·racefingo(); | |
2176 for(; fb; fb=next) { | 2425 for(; fb; fb=next) { |
2177 next = fb->next; | 2426 next = fb->next; |
2178 for(i=0; i<fb->cnt; i++) { | 2427 for(i=0; i<fb->cnt; i++) { |
2179 f = &fb->fin[i]; | 2428 f = &fb->fin[i]; |
2180 framesz = sizeof(uintptr) + f->nret; | 2429 framesz = sizeof(uintptr) + f->nret; |
2181 if(framecap < framesz) { | 2430 if(framecap < framesz) { |
2182 runtime·free(frame); | 2431 runtime·free(frame); |
2183 frame = runtime·mal(framesz); | 2432 frame = runtime·mal(framesz); |
2184 framecap = framesz; | 2433 framecap = framesz; |
2185 } | 2434 } |
2186 *(void**)frame = f->arg; | 2435 *(void**)frame = f->arg; |
2187 » » » » runtime·setblockspecial(f->arg, false); | 2436 » » » » reflect·call(f->fn, frame, sizeof(uintptr) + f-> nret); |
2188 » » » » reflect·call((byte*)f->fn, frame, sizeof(uintptr ) + f->nret); | |
2189 f->fn = nil; | 2437 f->fn = nil; |
2190 f->arg = nil; | 2438 f->arg = nil; |
2191 } | 2439 } |
2192 fb->cnt = 0; | 2440 fb->cnt = 0; |
2193 fb->next = finc; | 2441 fb->next = finc; |
2194 finc = fb; | 2442 finc = fb; |
2195 } | 2443 } |
2196 runtime·gc(1); // trigger another gc to clean up the finalized objects, if possible | 2444 runtime·gc(1); // trigger another gc to clean up the finalized objects, if possible |
2197 } | 2445 } |
2198 } | 2446 } |
2199 | 2447 |
2200 // mark the block at v of size n as allocated. | 2448 // mark the block at v of size n as allocated. |
2201 // If noptr is true, mark it as having no pointers. | 2449 // If noptr is true, mark it as having no pointers. |
2202 void | 2450 void |
2203 runtime·markallocated(void *v, uintptr n, bool noptr) | 2451 runtime·markallocated(void *v, uintptr n, bool noptr) |
2204 { | 2452 { |
2205 uintptr *b, obits, bits, off, shift; | 2453 uintptr *b, obits, bits, off, shift; |
2206 | 2454 |
2207 if(0) | 2455 if(0) |
2208 runtime·printf("markallocated %p+%p\n", v, n); | 2456 runtime·printf("markallocated %p+%p\n", v, n); |
2209 | 2457 |
2210 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh eap.arena_start) | 2458 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m heap->arena_start) |
2211 runtime·throw("markallocated: bad pointer"); | 2459 runtime·throw("markallocated: bad pointer"); |
2212 | 2460 |
2213 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset | 2461 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offse t |
2214 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2462 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2215 shift = off % wordsPerBitmapWord; | 2463 shift = off % wordsPerBitmapWord; |
2216 | 2464 |
2217 for(;;) { | 2465 for(;;) { |
2218 obits = *b; | 2466 obits = *b; |
2219 bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift); | 2467 bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift); |
2220 if(noptr) | 2468 if(noptr) |
2221 bits |= bitNoPointers<<shift; | 2469 bits |= bitNoPointers<<shift; |
2222 if(runtime·singleproc) { | 2470 if(runtime·singleproc) { |
2223 *b = bits; | 2471 *b = bits; |
2224 break; | 2472 break; |
2225 } else { | 2473 } else { |
2226 // more than one goroutine is potentially running: use a tomic op | 2474 // more than one goroutine is potentially running: use a tomic op |
2227 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) | 2475 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) |
2228 break; | 2476 break; |
2229 } | 2477 } |
2230 } | 2478 } |
2231 } | 2479 } |
2232 | 2480 |
2233 // mark the block at v of size n as freed. | 2481 // mark the block at v of size n as freed. |
2234 void | 2482 void |
2235 runtime·markfreed(void *v, uintptr n) | 2483 runtime·markfreed(void *v, uintptr n) |
2236 { | 2484 { |
2237 uintptr *b, obits, bits, off, shift; | 2485 uintptr *b, obits, bits, off, shift; |
2238 | 2486 |
2239 if(0) | 2487 if(0) |
2240 runtime·printf("markallocated %p+%p\n", v, n); | 2488 runtime·printf("markallocated %p+%p\n", v, n); |
2241 | 2489 |
2242 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh eap.arena_start) | 2490 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m heap->arena_start) |
2243 runtime·throw("markallocated: bad pointer"); | 2491 runtime·throw("markallocated: bad pointer"); |
2244 | 2492 |
2245 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset | 2493 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offse t |
2246 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2494 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2247 shift = off % wordsPerBitmapWord; | 2495 shift = off % wordsPerBitmapWord; |
2248 | 2496 |
2249 for(;;) { | 2497 for(;;) { |
2250 obits = *b; | 2498 obits = *b; |
2251 bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 2499 bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
2252 if(runtime·singleproc) { | 2500 if(runtime·singleproc) { |
2253 *b = bits; | 2501 *b = bits; |
2254 break; | 2502 break; |
2255 } else { | 2503 } else { |
2256 // more than one goroutine is potentially running: use a tomic op | 2504 // more than one goroutine is potentially running: use a tomic op |
2257 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) | 2505 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) |
2258 break; | 2506 break; |
2259 } | 2507 } |
2260 } | 2508 } |
2261 } | 2509 } |
2262 | 2510 |
2263 // check that the block at v of size n is marked freed. | 2511 // check that the block at v of size n is marked freed. |
2264 void | 2512 void |
2265 runtime·checkfreed(void *v, uintptr n) | 2513 runtime·checkfreed(void *v, uintptr n) |
2266 { | 2514 { |
2267 uintptr *b, bits, off, shift; | 2515 uintptr *b, bits, off, shift; |
2268 | 2516 |
2269 if(!runtime·checking) | 2517 if(!runtime·checking) |
2270 return; | 2518 return; |
2271 | 2519 |
2272 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh eap.arena_start) | 2520 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m heap->arena_start) |
2273 return; // not allocated, so okay | 2521 return; // not allocated, so okay |
2274 | 2522 |
2275 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset | 2523 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offse t |
2276 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2524 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2277 shift = off % wordsPerBitmapWord; | 2525 shift = off % wordsPerBitmapWord; |
2278 | 2526 |
2279 bits = *b>>shift; | 2527 bits = *b>>shift; |
2280 if((bits & bitAllocated) != 0) { | 2528 if((bits & bitAllocated) != 0) { |
2281 runtime·printf("checkfreed %p+%p: off=%p have=%p\n", | 2529 runtime·printf("checkfreed %p+%p: off=%p have=%p\n", |
2282 v, n, off, bits & bitMask); | 2530 v, n, off, bits & bitMask); |
2283 runtime·throw("checkfreed: not freed"); | 2531 runtime·throw("checkfreed: not freed"); |
2284 } | 2532 } |
2285 } | 2533 } |
2286 | 2534 |
2287 // mark the span of memory at v as having n blocks of the given size. | 2535 // mark the span of memory at v as having n blocks of the given size. |
2288 // if leftover is true, there is left over space at the end of the span. | 2536 // if leftover is true, there is left over space at the end of the span. |
2289 void | 2537 void |
2290 runtime·markspan(void *v, uintptr size, uintptr n, bool leftover) | 2538 runtime·markspan(void *v, uintptr size, uintptr n, bool leftover) |
2291 { | 2539 { |
2292 uintptr *b, off, shift; | 2540 uintptr *b, off, shift; |
2293 byte *p; | 2541 byte *p; |
2294 | 2542 |
2295 » if((byte*)v+size*n > (byte*)runtime·mheap.arena_used || (byte*)v < runti me·mheap.arena_start) | 2543 » if((byte*)v+size*n > (byte*)runtime·mheap->arena_used || (byte*)v < runt ime·mheap->arena_start) |
2296 runtime·throw("markspan: bad pointer"); | 2544 runtime·throw("markspan: bad pointer"); |
2297 | 2545 |
2298 p = v; | 2546 p = v; |
2299 if(leftover) // mark a boundary just past end of last block too | 2547 if(leftover) // mark a boundary just past end of last block too |
2300 n++; | 2548 n++; |
2301 for(; n-- > 0; p += size) { | 2549 for(; n-- > 0; p += size) { |
2302 // Okay to use non-atomic ops here, because we control | 2550 // Okay to use non-atomic ops here, because we control |
2303 // the entire span, and each bitmap word has bits for only | 2551 // the entire span, and each bitmap word has bits for only |
2304 // one span, so no other goroutines are changing these | 2552 // one span, so no other goroutines are changing these |
2305 // bitmap words. | 2553 // bitmap words. |
2306 » » off = (uintptr*)p - (uintptr*)runtime·mheap.arena_start; // wor d offset | 2554 » » off = (uintptr*)p - (uintptr*)runtime·mheap->arena_start; // wo rd offset |
2307 » » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2555 » » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWor d - 1; |
2308 shift = off % wordsPerBitmapWord; | 2556 shift = off % wordsPerBitmapWord; |
2309 *b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 2557 *b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
2310 } | 2558 } |
2311 } | 2559 } |
2312 | 2560 |
2313 // unmark the span of memory at v of length n bytes. | 2561 // unmark the span of memory at v of length n bytes. |
2314 void | 2562 void |
2315 runtime·unmarkspan(void *v, uintptr n) | 2563 runtime·unmarkspan(void *v, uintptr n) |
2316 { | 2564 { |
2317 uintptr *p, *b, off; | 2565 uintptr *p, *b, off; |
2318 | 2566 |
2319 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh eap.arena_start) | 2567 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m heap->arena_start) |
2320 runtime·throw("markspan: bad pointer"); | 2568 runtime·throw("markspan: bad pointer"); |
2321 | 2569 |
2322 p = v; | 2570 p = v; |
2323 » off = p - (uintptr*)runtime·mheap.arena_start; // word offset | 2571 » off = p - (uintptr*)runtime·mheap->arena_start; // word offset |
2324 if(off % wordsPerBitmapWord != 0) | 2572 if(off % wordsPerBitmapWord != 0) |
2325 runtime·throw("markspan: unaligned pointer"); | 2573 runtime·throw("markspan: unaligned pointer"); |
2326 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2574 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2327 n /= PtrSize; | 2575 n /= PtrSize; |
2328 if(n%wordsPerBitmapWord != 0) | 2576 if(n%wordsPerBitmapWord != 0) |
2329 runtime·throw("unmarkspan: unaligned length"); | 2577 runtime·throw("unmarkspan: unaligned length"); |
2330 // Okay to use non-atomic ops here, because we control | 2578 // Okay to use non-atomic ops here, because we control |
2331 // the entire span, and each bitmap word has bits for only | 2579 // the entire span, and each bitmap word has bits for only |
2332 // one span, so no other goroutines are changing these | 2580 // one span, so no other goroutines are changing these |
2333 // bitmap words. | 2581 // bitmap words. |
2334 n /= wordsPerBitmapWord; | 2582 n /= wordsPerBitmapWord; |
2335 while(n-- > 0) | 2583 while(n-- > 0) |
2336 *b-- = 0; | 2584 *b-- = 0; |
2337 } | 2585 } |
2338 | 2586 |
2339 bool | 2587 bool |
2340 runtime·blockspecial(void *v) | 2588 runtime·blockspecial(void *v) |
2341 { | 2589 { |
2342 uintptr *b, off, shift; | 2590 uintptr *b, off, shift; |
2343 | 2591 |
2344 if(DebugMark) | 2592 if(DebugMark) |
2345 return true; | 2593 return true; |
2346 | 2594 |
2347 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; | 2595 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; |
2348 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2596 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2349 shift = off % wordsPerBitmapWord; | 2597 shift = off % wordsPerBitmapWord; |
2350 | 2598 |
2351 return (*b & (bitSpecial<<shift)) != 0; | 2599 return (*b & (bitSpecial<<shift)) != 0; |
2352 } | 2600 } |
2353 | 2601 |
2354 void | 2602 void |
2355 runtime·setblockspecial(void *v, bool s) | 2603 runtime·setblockspecial(void *v, bool s) |
2356 { | 2604 { |
2357 uintptr *b, off, shift, bits, obits; | 2605 uintptr *b, off, shift, bits, obits; |
2358 | 2606 |
2359 if(DebugMark) | 2607 if(DebugMark) |
2360 return; | 2608 return; |
2361 | 2609 |
2362 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; | 2610 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; |
2363 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2611 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2364 shift = off % wordsPerBitmapWord; | 2612 shift = off % wordsPerBitmapWord; |
2365 | 2613 |
2366 for(;;) { | 2614 for(;;) { |
2367 obits = *b; | 2615 obits = *b; |
2368 if(s) | 2616 if(s) |
2369 bits = obits | (bitSpecial<<shift); | 2617 bits = obits | (bitSpecial<<shift); |
2370 else | 2618 else |
2371 bits = obits & ~(bitSpecial<<shift); | 2619 bits = obits & ~(bitSpecial<<shift); |
2372 if(runtime·singleproc) { | 2620 if(runtime·singleproc) { |
2373 *b = bits; | 2621 *b = bits; |
(...skipping 18 matching lines...) Expand all Loading... | |
2392 uintptr n; | 2640 uintptr n; |
2393 | 2641 |
2394 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; | 2642 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; |
2395 n = (n+bitmapChunk-1) & ~(bitmapChunk-1); | 2643 n = (n+bitmapChunk-1) & ~(bitmapChunk-1); |
2396 if(h->bitmap_mapped >= n) | 2644 if(h->bitmap_mapped >= n) |
2397 return; | 2645 return; |
2398 | 2646 |
2399 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped); | 2647 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped); |
2400 h->bitmap_mapped = n; | 2648 h->bitmap_mapped = n; |
2401 } | 2649 } |
LEFT | RIGHT |