LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // Garbage collector. | 5 // Garbage collector. |
6 | 6 |
7 #include "runtime.h" | 7 #include "runtime.h" |
8 #include "arch_GOARCH.h" | 8 #include "arch_GOARCH.h" |
9 #include "malloc.h" | 9 #include "malloc.h" |
10 #include "stack.h" | 10 #include "stack.h" |
| 11 #include "mgc0.h" |
| 12 #include "race.h" |
11 #include "type.h" | 13 #include "type.h" |
12 #include "typekind.h" | 14 #include "typekind.h" |
13 #include "hashmap.h" | 15 #include "hashmap.h" |
14 #include "mgc0.h" | |
15 | 16 |
16 enum { | 17 enum { |
17 Debug = 0, | 18 Debug = 0, |
18 PtrSize = sizeof(void*), | |
19 DebugMark = 0, // run second pass to check mark | 19 DebugMark = 0, // run second pass to check mark |
| 20 CollectStats = 0, |
| 21 ScanStackByFrames = 1, |
| 22 IgnorePreciseGC = 0, |
20 DebugType = 0, | 23 DebugType = 0, |
21 » DebugStats = 0, | 24 » MemScramble = 0, // scramble free memory, and don't actually free it |
22 | 25 |
23 // Four bits per word (see #defines below). | 26 // Four bits per word (see #defines below). |
24 wordsPerBitmapWord = sizeof(void*)*8/4, | 27 wordsPerBitmapWord = sizeof(void*)*8/4, |
25 bitShift = sizeof(void*)*8/4, | 28 bitShift = sizeof(void*)*8/4, |
26 | 29 |
27 handoffThreshold = 4, | 30 handoffThreshold = 4, |
28 » XCAPACITY = 64, | 31 » IntermediateBufferCapacity = 64, |
| 32 |
| 33 » // Bits in type information |
| 34 » PRECISE = 1, |
| 35 » LOOP = 2, |
| 36 » PC_BITS = PRECISE | LOOP, |
29 }; | 37 }; |
30 | |
31 volatile int32 runtime·gcing = 0; | |
32 | 38 |
33 // Bits in per-word bitmap. | 39 // Bits in per-word bitmap. |
34 // #defines because enum might not be able to hold the values. | 40 // #defines because enum might not be able to hold the values. |
35 // | 41 // |
36 // Each word in the bitmap describes wordsPerBitmapWord words | 42 // Each word in the bitmap describes wordsPerBitmapWord words |
37 // of heap memory. There are 4 bitmap bits dedicated to each heap word, | 43 // of heap memory. There are 4 bitmap bits dedicated to each heap word, |
38 // so on a 64-bit system there is one bitmap word per 16 heap words. | 44 // so on a 64-bit system there is one bitmap word per 16 heap words. |
39 // The bits in the word are packed together by type first, then by | 45 // The bits in the word are packed together by type first, then by |
40 // heap location, so each 64-bit bitmap word consists of, from top to bottom, | 46 // heap location, so each 64-bit bitmap word consists of, from top to bottom, |
41 // the 16 bitSpecial bits for the corresponding heap words, then the 16 bitMarke
d bits, | 47 // the 16 bitSpecial bits for the corresponding heap words, then the 16 bitMarke
d bits, |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
78 // | 84 // |
79 uint32 runtime·worldsema = 1; | 85 uint32 runtime·worldsema = 1; |
80 | 86 |
81 static int32 gctrace; | 87 static int32 gctrace; |
82 | 88 |
83 typedef struct Obj Obj; | 89 typedef struct Obj Obj; |
84 struct Obj | 90 struct Obj |
85 { | 91 { |
86 byte *p; // data pointer | 92 byte *p; // data pointer |
87 uintptr n; // size of data in bytes | 93 uintptr n; // size of data in bytes |
88 » uintptr»gc;» // type info | 94 » uintptr»ti;» // type info |
89 }; | 95 }; |
90 | 96 |
91 // The size of Workbuf is N*PageSize. | 97 // The size of Workbuf is N*PageSize. |
92 typedef struct Workbuf Workbuf; | 98 typedef struct Workbuf Workbuf; |
93 struct Workbuf | 99 struct Workbuf |
94 { | 100 { |
95 » LFNode node; // must be first | 101 #define SIZE (2*PageSize-sizeof(LFNode)-sizeof(uintptr)) |
96 » uintptr»nobj; | 102 » LFNode node; // must be first |
97 » Obj » obj[ (2*PageSize-sizeof(LFNode)-sizeof(uintptr))/sizeof(Obj) - 1
]; | 103 » uintptr nobj; |
98 » uint8» _padding[2*PageSize-sizeof(LFNode)-sizeof(uintptr) - ((2*PageSiz
e-sizeof(LFNode)-sizeof(uintptr))/sizeof(Obj)-1)*sizeof(Obj)]; | 104 » Obj obj[SIZE/sizeof(Obj) - 1]; |
| 105 » uint8 _padding[SIZE%sizeof(Obj) + sizeof(Obj)]; |
| 106 #undef SIZE |
99 }; | 107 }; |
100 | 108 |
101 typedef struct Finalizer Finalizer; | 109 typedef struct Finalizer Finalizer; |
102 struct Finalizer | 110 struct Finalizer |
103 { | 111 { |
104 » void (*fn)(void*); | 112 » FuncVal *fn; |
105 void *arg; | 113 void *arg; |
106 » int32 nret; | 114 » uintptr nret; |
107 }; | 115 }; |
108 | 116 |
109 typedef struct FinBlock FinBlock; | 117 typedef struct FinBlock FinBlock; |
110 struct FinBlock | 118 struct FinBlock |
111 { | 119 { |
112 FinBlock *alllink; | 120 FinBlock *alllink; |
113 FinBlock *next; | 121 FinBlock *next; |
114 int32 cnt; | 122 int32 cnt; |
115 int32 cap; | 123 int32 cap; |
116 Finalizer fin[1]; | 124 Finalizer fin[1]; |
117 }; | 125 }; |
118 | 126 |
119 extern byte etext[]; | |
120 extern byte data[]; | 127 extern byte data[]; |
121 extern byte edata[]; | 128 extern byte edata[]; |
122 extern byte bss[]; | 129 extern byte bss[]; |
123 extern byte ebss[]; | 130 extern byte ebss[]; |
124 | 131 |
125 extern byte gcdata[]; | 132 extern byte gcdata[]; |
126 extern byte gcbss[]; | 133 extern byte gcbss[]; |
127 | 134 |
128 static G *fing; | 135 static G *fing; |
129 static FinBlock *finq; // list of finalizers that are to be executed | 136 static FinBlock *finq; // list of finalizers that are to be executed |
130 static FinBlock *finc; // cache of free blocks | 137 static FinBlock *finc; // cache of free blocks |
131 static FinBlock *allfin; // list of all blocks | 138 static FinBlock *allfin; // list of all blocks |
132 static Lock finlock; | 139 static Lock finlock; |
133 static int32 fingwait; | 140 static int32 fingwait; |
134 | 141 |
135 static void runfinq(void); | 142 static void runfinq(void); |
136 static Workbuf* getempty(Workbuf*); | 143 static Workbuf* getempty(Workbuf*); |
137 static Workbuf* getfull(Workbuf*); | 144 static Workbuf* getfull(Workbuf*); |
138 static void putempty(Workbuf*); | 145 static void putempty(Workbuf*); |
139 static Workbuf* handoff(Workbuf*); | 146 static Workbuf* handoff(Workbuf*); |
| 147 static void gchelperstart(void); |
140 | 148 |
141 static struct { | 149 static struct { |
142 uint64 full; // lock-free list of full blocks | 150 uint64 full; // lock-free list of full blocks |
143 uint64 empty; // lock-free list of empty blocks | 151 uint64 empty; // lock-free list of empty blocks |
144 byte pad0[CacheLineSize]; // prevents false-sharing between full/empt
y and nproc/nwait | 152 byte pad0[CacheLineSize]; // prevents false-sharing between full/empt
y and nproc/nwait |
145 uint32 nproc; | 153 uint32 nproc; |
| 154 byte pad1[CacheLineSize]; // prevents false-sharing between nproc and
nwait |
146 volatile uint32 nwait; | 155 volatile uint32 nwait; |
147 volatile uint32 ndone; | 156 volatile uint32 ndone; |
148 volatile uint32 debugmarkdone; | 157 volatile uint32 debugmarkdone; |
149 Note alldone; | 158 Note alldone; |
150 ParFor *markfor; | 159 ParFor *markfor; |
151 ParFor *sweepfor; | 160 ParFor *sweepfor; |
152 | 161 |
153 Lock; | 162 Lock; |
154 byte *chunk; | 163 byte *chunk; |
155 uintptr nchunk; | 164 uintptr nchunk; |
156 | 165 |
157 Obj *roots; | 166 Obj *roots; |
158 uint32 nroot; | 167 uint32 nroot; |
159 uint32 rootcap; | 168 uint32 rootcap; |
160 } work; | 169 } work; |
161 | 170 |
162 typedef struct CustomMarker CustomMarker; | |
163 struct CustomMarker { | |
164 void (*f)(void); | |
165 }; | |
166 static CustomMarker custommarkers[16]; | |
167 static uint32 ncustommarkers; | |
168 | |
169 void | |
170 runtime·addgcmarker(void (*f)(void)) | |
171 { | |
172 if(ncustommarkers == nelem(custommarkers)) | |
173 runtime·throw("addgcmarker: too many markers"); | |
174 custommarkers[ncustommarkers++].f = f; | |
175 } | |
176 | |
177 enum { | 171 enum { |
178 GC_DEFAULT_PTR = GC_NUM_INSTR, | 172 GC_DEFAULT_PTR = GC_NUM_INSTR, |
179 GC_MAP_NEXT, | 173 GC_MAP_NEXT, |
| 174 GC_CHAN, |
180 | 175 |
181 GC_NUM_INSTR2 | 176 GC_NUM_INSTR2 |
182 }; | 177 }; |
183 | 178 |
184 static struct { | 179 static struct { |
185 » uint64 x_sum; | 180 » struct { |
186 » uint64 x_cnt; | 181 » » uint64 sum; |
| 182 » » uint64 cnt; |
| 183 » } ptr; |
187 uint64 nbytes; | 184 uint64 nbytes; |
188 » uint64 nobj_sum; | 185 » struct { |
189 » uint64 nobj_cnt; | 186 » » uint64 sum; |
190 » uint64 nobj_notype; | 187 » » uint64 cnt; |
191 » uint64 nobj_typelookups_ok; | 188 » » uint64 notype; |
192 » uint64 rescans; | 189 » » uint64 typelookup; |
193 » uint64 rescan_nbytes; | 190 » } obj; |
| 191 » uint64 rescan; |
| 192 » uint64 rescanbytes; |
194 uint64 instr[GC_NUM_INSTR2]; | 193 uint64 instr[GC_NUM_INSTR2]; |
195 uint64 putempty; | 194 uint64 putempty; |
196 uint64 getfull; | 195 uint64 getfull; |
197 uint64 runtime·markobject; | |
198 } gcstats; | 196 } gcstats; |
199 | 197 |
200 // Marks the object, without appending it to any buffer if the object wasn't alr
eady marked. | 198 // markonly marks an object. It returns true if the object |
201 // Returns true if the object wasn't already marked. | 199 // has been marked by this function, false otherwise. |
202 // This function is single-threaded. | 200 // This function doesn't append the object to any buffer. |
203 static bool | 201 static bool |
204 markonly(void *obj) | 202 markonly(void *obj) |
205 { | 203 { |
206 byte *p; | 204 byte *p; |
207 uintptr *bitp, bits, shift, x, xbits, off; | 205 uintptr *bitp, bits, shift, x, xbits, off; |
208 MSpan *s; | 206 MSpan *s; |
209 PageID k; | 207 PageID k; |
210 | 208 |
211 // Words outside the arena cannot be pointers. | 209 // Words outside the arena cannot be pointers. |
212 » if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used) | 210 » if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used) |
213 return false; | 211 return false; |
214 | 212 |
215 // obj may be a pointer to a live object. | 213 // obj may be a pointer to a live object. |
216 // Try to find the beginning of the object. | 214 // Try to find the beginning of the object. |
217 | 215 |
218 // Round down to word boundary. | 216 // Round down to word boundary. |
219 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); | 217 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); |
220 | 218 |
221 // Find bits for this word. | 219 // Find bits for this word. |
222 » off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; | 220 » off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start; |
223 » bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 221 » bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1
; |
224 shift = off % wordsPerBitmapWord; | 222 shift = off % wordsPerBitmapWord; |
225 xbits = *bitp; | 223 xbits = *bitp; |
226 bits = xbits >> shift; | 224 bits = xbits >> shift; |
227 | 225 |
228 // Pointing at the beginning of a block? | 226 // Pointing at the beginning of a block? |
229 if((bits & (bitAllocated|bitBlockBoundary)) != 0) | 227 if((bits & (bitAllocated|bitBlockBoundary)) != 0) |
230 goto found; | 228 goto found; |
231 | 229 |
232 // Otherwise consult span table to find beginning. | 230 // Otherwise consult span table to find beginning. |
233 // (Manually inlined copy of MHeap_LookupMaybe.) | 231 // (Manually inlined copy of MHeap_LookupMaybe.) |
234 k = (uintptr)obj>>PageShift; | 232 k = (uintptr)obj>>PageShift; |
235 x = k; | 233 x = k; |
236 if(sizeof(void*) == 8) | 234 if(sizeof(void*) == 8) |
237 » » x -= (uintptr)runtime·mheap.arena_start>>PageShift; | 235 » » x -= (uintptr)runtime·mheap->arena_start>>PageShift; |
238 » s = runtime·mheap.map[x]; | 236 » s = runtime·mheap->map[x]; |
239 if(s == nil || k < s->start || k - s->start >= s->npages || s->state !=
MSpanInUse) | 237 if(s == nil || k < s->start || k - s->start >= s->npages || s->state !=
MSpanInUse) |
240 return false; | 238 return false; |
241 p = (byte*)((uintptr)s->start<<PageShift); | 239 p = (byte*)((uintptr)s->start<<PageShift); |
242 if(s->sizeclass == 0) { | 240 if(s->sizeclass == 0) { |
243 obj = p; | 241 obj = p; |
244 } else { | 242 } else { |
245 if((byte*)obj >= (byte*)s->limit) | 243 if((byte*)obj >= (byte*)s->limit) |
246 return false; | 244 return false; |
247 uintptr size = s->elemsize; | 245 uintptr size = s->elemsize; |
248 int32 i = ((byte*)obj - p)/size; | 246 int32 i = ((byte*)obj - p)/size; |
249 obj = p+i*size; | 247 obj = p+i*size; |
250 } | 248 } |
251 | 249 |
252 // Now that we know the object header, reload bits. | 250 // Now that we know the object header, reload bits. |
253 » off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; | 251 » off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start; |
254 » bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 252 » bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1
; |
255 shift = off % wordsPerBitmapWord; | 253 shift = off % wordsPerBitmapWord; |
256 xbits = *bitp; | 254 xbits = *bitp; |
257 bits = xbits >> shift; | 255 bits = xbits >> shift; |
258 | 256 |
259 found: | 257 found: |
260 // Now we have bits, bitp, and shift correct for | 258 // Now we have bits, bitp, and shift correct for |
261 // obj pointing at the base of the object. | 259 // obj pointing at the base of the object. |
262 // Only care about allocated and not marked. | 260 // Only care about allocated and not marked. |
263 if((bits & (bitAllocated|bitMarked)) != bitAllocated) | 261 if((bits & (bitAllocated|bitMarked)) != bitAllocated) |
264 return false; | 262 return false; |
265 » *bitp |= bitMarked<<shift; | 263 » if(work.nproc == 1) |
| 264 » » *bitp |= bitMarked<<shift; |
| 265 » else { |
| 266 » » for(;;) { |
| 267 » » » x = *bitp; |
| 268 » » » if(x & (bitMarked<<shift)) |
| 269 » » » » return false; |
| 270 » » » if(runtime·casp((void**)bitp, (void*)x, (void*)(x|(bitMa
rked<<shift)))) |
| 271 » » » » break; |
| 272 » » } |
| 273 » } |
266 | 274 |
267 // The object is now marked | 275 // The object is now marked |
268 return true; | 276 return true; |
269 } | 277 } |
270 | 278 |
271 void | 279 // PtrTarget is a structure used by intermediate buffers. |
272 runtime·markobjects(void **obj, uintptr n) | 280 // The intermediate buffers hold GC data before it |
273 { | 281 // is moved/flushed to the work buffer (Workbuf). |
274 » uintptr i; | 282 // The size of an intermediate buffer is very small, |
275 | 283 // such as 32 or 64 elements. |
276 » for(i=0; i<n; i++) | 284 typedef struct PtrTarget PtrTarget; |
277 » » markonly(obj[i]); | 285 struct PtrTarget |
278 | 286 { |
279 » if(DebugStats) | 287 » void *p; |
280 » » runtime·xadd64(&gcstats.runtime·markobject, n); | 288 » uintptr ti; |
281 } | 289 }; |
282 | 290 |
283 void | 291 typedef struct BufferList BufferList; |
284 runtime·markobject(void *obj) | 292 struct BufferList |
285 { | 293 { |
286 » markonly(obj); | 294 » PtrTarget ptrtarget[IntermediateBufferCapacity]; |
287 » if(DebugStats) | 295 » Obj obj[IntermediateBufferCapacity]; |
288 » » runtime·xadd64(&gcstats.runtime·markobject, 1); | 296 » uint32 busy; |
289 } | 297 » byte pad[CacheLineSize]; |
290 | 298 }; |
291 // Append obj to the buffer. | 299 #pragma dataflag 16 // no pointers |
| 300 static BufferList bufferList[MaxGcproc]; |
| 301 |
| 302 static Lock lock; |
| 303 static Type *itabtype; |
| 304 |
| 305 static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj); |
| 306 |
| 307 // flushptrbuf moves data from the PtrTarget buffer to the work buffer. |
| 308 // The PtrTarget buffer contains blocks irrespective of whether the blocks have
been marked or scanned, |
| 309 // while the work buffer contains blocks which have been marked |
| 310 // and are prepared to be scanned by the garbage collector. |
| 311 // |
| 312 // _wp, _wbuf, _nobj are input/output parameters and are specifying the work buf
fer. |
| 313 // |
| 314 // A simplified drawing explaining how the todo-list moves from a structure to a
nother: |
| 315 // |
| 316 // scanblock |
| 317 // (find pointers) |
| 318 // Obj ------> PtrTarget (pointer targets) |
| 319 // ↑ | |
| 320 // | | |
| 321 // `----------' |
| 322 // flushptrbuf |
| 323 // (find block start, mark and enqueue) |
292 static void | 324 static void |
293 enqueue(Obj obj, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj) | 325 flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
, uintptr *_nobj) |
294 { | |
295 » uintptr nobj, off; | |
296 » Obj *wp; | |
297 » Workbuf *wbuf; | |
298 | |
299 » if(Debug > 1) | |
300 » » runtime·printf("append obj(%p %D %p)\n", obj.p, (int64)obj.n, ob
j.gc); | |
301 | |
302 » // Align obj.b to a word boundary. | |
303 » off = (uintptr)obj.p & (PtrSize-1); | |
304 » if(off != 0) { | |
305 » » obj.p += PtrSize - off; | |
306 » » obj.n -= PtrSize - off; | |
307 » » obj.gc = 0; | |
308 » } | |
309 | |
310 » if(obj.p == nil || obj.n == 0) | |
311 » » return; | |
312 | |
313 » // Get the state of buffer | |
314 » wp = *_wp; | |
315 » wbuf = *_wbuf; | |
316 » nobj = *_nobj; | |
317 | |
318 » // If another proc wants a pointer, give it some. | |
319 » if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) { | |
320 » » wbuf->nobj = nobj; | |
321 » » wbuf = handoff(wbuf); | |
322 » » nobj = wbuf->nobj; | |
323 » » wp = wbuf->obj + nobj; | |
324 » } | |
325 | |
326 » // If buffer is full, get a new one. | |
327 » if(wbuf == nil || nobj >= nelem(wbuf->obj)) { | |
328 » » if(wbuf != nil) | |
329 » » » wbuf->nobj = nobj; | |
330 » » wbuf = getempty(wbuf); | |
331 » » wp = wbuf->obj; | |
332 » » nobj = 0; | |
333 » } | |
334 | |
335 » *wp = obj; | |
336 » wp++; | |
337 » nobj++; | |
338 | |
339 » // Update the state of buffer | |
340 » *_wp = wp; | |
341 » *_wbuf = wbuf; | |
342 » *_nobj = nobj; | |
343 } | |
344 | |
345 static void | |
346 markenqueue(Obj obj, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj) | |
347 { | |
348 » bool wasnt_marked = markonly(obj.p); | |
349 » if(wasnt_marked) | |
350 » » enqueue(obj, _wp, _wbuf, _nobj); | |
351 } | |
352 | |
353 struct X | |
354 { | |
355 » void *p; | |
356 » uintptr gc; | |
357 }; | |
358 | |
359 struct Y | |
360 { | |
361 » uintptr *bitp, shift; | |
362 » byte *obj; | |
363 » uintptr gc; | |
364 }; | |
365 | |
366 struct BufferList | |
367 { | |
368 » struct X x[XCAPACITY]; | |
369 » struct Y y[XCAPACITY]; | |
370 » Obj obj[XCAPACITY]; | |
371 » struct BufferList *next; | |
372 }; | |
373 static struct BufferList *bufferList; | |
374 | |
375 static Lock lock; | |
376 | |
377 static void | |
378 flushxbuf(struct X *xbuf, uintptr n, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj,
struct Y *ybuf) | |
379 { | 326 { |
380 byte *p, *arena_start, *obj; | 327 byte *p, *arena_start, *obj; |
381 » uintptr *bitp, bits, shift, j, x, xbits, off, nobj, gc; | 328 » uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n; |
382 MSpan *s; | 329 MSpan *s; |
383 PageID k; | 330 PageID k; |
384 Obj *wp; | 331 Obj *wp; |
385 Workbuf *wbuf; | 332 Workbuf *wbuf; |
386 » struct X *xbuf_end; | 333 » PtrTarget *ptrbuf_end; |
387 » struct Y *ybufpos, *y; | 334 |
388 | 335 » arena_start = runtime·mheap->arena_start; |
389 » if(DebugStats) { | |
390 » » runtime·xadd64(&gcstats.x_sum, n); | |
391 » » runtime·xadd64(&gcstats.x_cnt, 1); | |
392 » } | |
393 | |
394 » arena_start = runtime·mheap.arena_start; | |
395 | 336 |
396 wp = *_wp; | 337 wp = *_wp; |
397 wbuf = *_wbuf; | 338 wbuf = *_wbuf; |
398 nobj = *_nobj; | 339 nobj = *_nobj; |
399 | 340 |
400 » xbuf_end = xbuf + n; | 341 » ptrbuf_end = *ptrbufpos; |
| 342 » n = ptrbuf_end - ptrbuf; |
| 343 » *ptrbufpos = ptrbuf; |
| 344 |
| 345 » if(CollectStats) { |
| 346 » » runtime·xadd64(&gcstats.ptr.sum, n); |
| 347 » » runtime·xadd64(&gcstats.ptr.cnt, 1); |
| 348 » } |
401 | 349 |
402 // If buffer is nearly full, get a new one. | 350 // If buffer is nearly full, get a new one. |
403 if(wbuf == nil || nobj+n >= nelem(wbuf->obj)) { | 351 if(wbuf == nil || nobj+n >= nelem(wbuf->obj)) { |
404 if(wbuf != nil) | 352 if(wbuf != nil) |
405 wbuf->nobj = nobj; | 353 wbuf->nobj = nobj; |
406 wbuf = getempty(wbuf); | 354 wbuf = getempty(wbuf); |
407 wp = wbuf->obj; | 355 wp = wbuf->obj; |
408 nobj = 0; | 356 nobj = 0; |
409 | 357 |
410 if(n >= nelem(wbuf->obj)) | 358 if(n >= nelem(wbuf->obj)) |
411 » » » runtime·throw("xbuf has to be smaller than WorkBuf"); | 359 » » » runtime·throw("ptrbuf has to be smaller than WorkBuf"); |
412 } | 360 } |
413 | 361 |
414 if(work.nproc == 1) { | 362 if(work.nproc == 1) { |
415 // Single-threaded version. | 363 // Single-threaded version. |
416 // Nearly the same code as the multi-threaded version. | 364 // Nearly the same code as the multi-threaded version. |
417 | 365 |
418 » » while(xbuf < xbuf_end) { | 366 » » while(ptrbuf < ptrbuf_end) { |
419 » » » obj = xbuf->p; | 367 » » » obj = ptrbuf->p; |
420 » » » gc = xbuf->gc; | 368 » » » ti = ptrbuf->ti; |
421 » » » xbuf++; | 369 » » » ptrbuf++; |
422 | 370 |
423 // obj belongs to interval [mheap.arena_start, mheap.are
na_used). | 371 // obj belongs to interval [mheap.arena_start, mheap.are
na_used). |
424 if(Debug > 1) { | 372 if(Debug > 1) { |
425 » » » » if(obj < runtime·mheap.arena_start || obj >= run
time·mheap.arena_used) | 373 » » » » if(obj < runtime·mheap->arena_start || obj >= ru
ntime·mheap->arena_used) |
426 runtime·throw("object is outside of mhea
p"); | 374 runtime·throw("object is outside of mhea
p"); |
427 } | 375 } |
428 | 376 |
429 // obj may be a pointer to a live object. | 377 // obj may be a pointer to a live object. |
430 // Try to find the beginning of the object. | 378 // Try to find the beginning of the object. |
431 | 379 |
432 // Round down to word boundary. | 380 // Round down to word boundary. |
433 if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) { | 381 if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) { |
434 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-
1)); | 382 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-
1)); |
435 » » » » gc = 0; | 383 » » » » ti = 0; |
436 } | 384 } |
437 | 385 |
438 // Find bits for this word. | 386 // Find bits for this word. |
439 off = (uintptr*)obj - (uintptr*)arena_start; | 387 off = (uintptr*)obj - (uintptr*)arena_start; |
440 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord -
1; | 388 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord -
1; |
441 shift = off % wordsPerBitmapWord; | 389 shift = off % wordsPerBitmapWord; |
442 xbits = *bitp; | 390 xbits = *bitp; |
443 bits = xbits >> shift; | 391 bits = xbits >> shift; |
444 | 392 |
445 // Pointing at the beginning of a block? | 393 // Pointing at the beginning of a block? |
446 if((bits & (bitAllocated|bitBlockBoundary)) != 0) | 394 if((bits & (bitAllocated|bitBlockBoundary)) != 0) |
447 goto found1; | 395 goto found1; |
448 | 396 |
449 » » » if((gc & 3) != 3) | 397 » » » ti = 0; |
450 » » » » gc = 0; | |
451 | 398 |
452 // Pointing just past the beginning? | 399 // Pointing just past the beginning? |
453 // Scan backward a little to find a block boundary. | 400 // Scan backward a little to find a block boundary. |
454 for(j=shift; j-->0; ) { | 401 for(j=shift; j-->0; ) { |
455 if(((xbits>>j) & (bitAllocated|bitBlockBoundary)
) != 0) { | 402 if(((xbits>>j) & (bitAllocated|bitBlockBoundary)
) != 0) { |
456 obj = (byte*)obj - (shift-j)*PtrSize; | 403 obj = (byte*)obj - (shift-j)*PtrSize; |
457 shift = j; | 404 shift = j; |
458 bits = xbits>>shift; | 405 bits = xbits>>shift; |
459 goto found1; | 406 goto found1; |
460 } | 407 } |
461 } | 408 } |
462 | 409 |
463 // Otherwise consult span table to find beginning. | 410 // Otherwise consult span table to find beginning. |
464 // (Manually inlined copy of MHeap_LookupMaybe.) | 411 // (Manually inlined copy of MHeap_LookupMaybe.) |
465 k = (uintptr)obj>>PageShift; | 412 k = (uintptr)obj>>PageShift; |
466 x = k; | 413 x = k; |
467 if(sizeof(void*) == 8) | 414 if(sizeof(void*) == 8) |
468 x -= (uintptr)arena_start>>PageShift; | 415 x -= (uintptr)arena_start>>PageShift; |
469 » » » s = runtime·mheap.map[x]; | 416 » » » s = runtime·mheap->map[x]; |
470 if(s == nil || k < s->start || k - s->start >= s->npages
|| s->state != MSpanInUse) | 417 if(s == nil || k < s->start || k - s->start >= s->npages
|| s->state != MSpanInUse) |
471 continue; | 418 continue; |
472 p = (byte*)((uintptr)s->start<<PageShift); | 419 p = (byte*)((uintptr)s->start<<PageShift); |
473 if(s->sizeclass == 0) { | 420 if(s->sizeclass == 0) { |
474 obj = p; | 421 obj = p; |
475 } else { | 422 } else { |
476 if((byte*)obj >= (byte*)s->limit) | 423 if((byte*)obj >= (byte*)s->limit) |
477 continue; | 424 continue; |
478 » » » » uintptr size = s->elemsize; | 425 » » » » size = s->elemsize; |
479 int32 i = ((byte*)obj - p)/size; | 426 int32 i = ((byte*)obj - p)/size; |
480 obj = p+i*size; | 427 obj = p+i*size; |
481 } | 428 } |
482 | 429 |
483 // Now that we know the object header, reload bits. | 430 // Now that we know the object header, reload bits. |
484 off = (uintptr*)obj - (uintptr*)arena_start; | 431 off = (uintptr*)obj - (uintptr*)arena_start; |
485 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord -
1; | 432 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord -
1; |
486 shift = off % wordsPerBitmapWord; | 433 shift = off % wordsPerBitmapWord; |
487 xbits = *bitp; | 434 xbits = *bitp; |
488 bits = xbits >> shift; | 435 bits = xbits >> shift; |
489 | 436 |
490 found1: | 437 found1: |
491 // Now we have bits, bitp, and shift correct for | 438 // Now we have bits, bitp, and shift correct for |
492 // obj pointing at the base of the object. | 439 // obj pointing at the base of the object. |
493 // Only care about allocated and not marked. | 440 // Only care about allocated and not marked. |
494 if((bits & (bitAllocated|bitMarked)) != bitAllocated) | 441 if((bits & (bitAllocated|bitMarked)) != bitAllocated) |
495 continue; | 442 continue; |
496 | 443 |
497 *bitp |= bitMarked<<shift; | 444 *bitp |= bitMarked<<shift; |
498 | 445 |
499 // If object has no pointers, don't need to scan further
. | 446 // If object has no pointers, don't need to scan further
. |
500 if((bits & bitNoPointers) != 0) | 447 if((bits & bitNoPointers) != 0) |
501 continue; | 448 continue; |
502 | 449 |
503 // Ask span about size class. | 450 // Ask span about size class. |
504 // (Manually inlined copy of MHeap_Lookup.) | 451 // (Manually inlined copy of MHeap_Lookup.) |
505 x = (uintptr)obj>>PageShift; | 452 x = (uintptr)obj>>PageShift; |
506 if(sizeof(void*) == 8) | 453 if(sizeof(void*) == 8) |
507 x -= (uintptr)arena_start>>PageShift; | 454 x -= (uintptr)arena_start>>PageShift; |
508 » » » s = runtime·mheap.map[x]; | 455 » » » s = runtime·mheap->map[x]; |
509 | 456 |
510 » » » *wp = (Obj){obj, s->elemsize, gc}; | 457 » » » PREFETCH(obj); |
| 458 |
| 459 » » » *wp = (Obj){obj, s->elemsize, ti}; |
511 wp++; | 460 wp++; |
512 nobj++; | 461 nobj++; |
513 } | 462 } |
514 } else { | 463 } else { |
515 // Multi-threaded version. | 464 // Multi-threaded version. |
516 // Nearly the same code as the single-threaded version. | 465 // Nearly the same code as the single-threaded version. |
517 | 466 |
518 » » ybufpos = ybuf; | 467 » » while(ptrbuf < ptrbuf_end) { |
519 | 468 » » » obj = ptrbuf->p; |
520 » » while(xbuf < xbuf_end) { | 469 » » » ti = ptrbuf->ti; |
521 » » » obj = xbuf->p; | 470 » » » ptrbuf++; |
522 » » » gc = xbuf->gc; | |
523 » » » xbuf++; | |
524 | 471 |
525 // obj belongs to interval [mheap.arena_start, mheap.are
na_used). | 472 // obj belongs to interval [mheap.arena_start, mheap.are
na_used). |
526 if(Debug > 1) { | 473 if(Debug > 1) { |
527 » » » » if(obj < runtime·mheap.arena_start || obj >= run
time·mheap.arena_used) | 474 » » » » if(obj < runtime·mheap->arena_start || obj >= ru
ntime·mheap->arena_used) |
528 runtime·throw("object is outside of mhea
p"); | 475 runtime·throw("object is outside of mhea
p"); |
529 } | 476 } |
530 | 477 |
531 // obj may be a pointer to a live object. | 478 // obj may be a pointer to a live object. |
532 // Try to find the beginning of the object. | 479 // Try to find the beginning of the object. |
533 | 480 |
534 // Round down to word boundary. | 481 // Round down to word boundary. |
535 if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) { | 482 if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) { |
536 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-
1)); | 483 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-
1)); |
537 » » » » gc = 0; | 484 » » » » ti = 0; |
538 } | 485 } |
539 | 486 |
540 // Find bits for this word. | 487 // Find bits for this word. |
541 off = (uintptr*)obj - (uintptr*)arena_start; | 488 off = (uintptr*)obj - (uintptr*)arena_start; |
542 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord -
1; | 489 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord -
1; |
543 shift = off % wordsPerBitmapWord; | 490 shift = off % wordsPerBitmapWord; |
544 xbits = *bitp; | 491 xbits = *bitp; |
545 bits = xbits >> shift; | 492 bits = xbits >> shift; |
546 | 493 |
547 // Pointing at the beginning of a block? | 494 // Pointing at the beginning of a block? |
548 if((bits & (bitAllocated|bitBlockBoundary)) != 0) | 495 if((bits & (bitAllocated|bitBlockBoundary)) != 0) |
549 goto found2; | 496 goto found2; |
550 | 497 |
551 » » » if((gc & 3) != 3) | 498 » » » ti = 0; |
552 » » » » gc = 0; | |
553 | 499 |
554 // Pointing just past the beginning? | 500 // Pointing just past the beginning? |
555 // Scan backward a little to find a block boundary. | 501 // Scan backward a little to find a block boundary. |
556 for(j=shift; j-->0; ) { | 502 for(j=shift; j-->0; ) { |
557 if(((xbits>>j) & (bitAllocated|bitBlockBoundary)
) != 0) { | 503 if(((xbits>>j) & (bitAllocated|bitBlockBoundary)
) != 0) { |
558 obj = (byte*)obj - (shift-j)*PtrSize; | 504 obj = (byte*)obj - (shift-j)*PtrSize; |
559 shift = j; | 505 shift = j; |
560 bits = xbits>>shift; | 506 bits = xbits>>shift; |
561 goto found2; | 507 goto found2; |
562 } | 508 } |
563 } | 509 } |
564 | 510 |
565 // Otherwise consult span table to find beginning. | 511 // Otherwise consult span table to find beginning. |
566 // (Manually inlined copy of MHeap_LookupMaybe.) | 512 // (Manually inlined copy of MHeap_LookupMaybe.) |
567 k = (uintptr)obj>>PageShift; | 513 k = (uintptr)obj>>PageShift; |
568 x = k; | 514 x = k; |
569 if(sizeof(void*) == 8) | 515 if(sizeof(void*) == 8) |
570 x -= (uintptr)arena_start>>PageShift; | 516 x -= (uintptr)arena_start>>PageShift; |
571 » » » s = runtime·mheap.map[x]; | 517 » » » s = runtime·mheap->map[x]; |
572 if(s == nil || k < s->start || k - s->start >= s->npages
|| s->state != MSpanInUse) | 518 if(s == nil || k < s->start || k - s->start >= s->npages
|| s->state != MSpanInUse) |
573 continue; | 519 continue; |
574 p = (byte*)((uintptr)s->start<<PageShift); | 520 p = (byte*)((uintptr)s->start<<PageShift); |
575 if(s->sizeclass == 0) { | 521 if(s->sizeclass == 0) { |
576 obj = p; | 522 obj = p; |
577 } else { | 523 } else { |
578 if((byte*)obj >= (byte*)s->limit) | 524 if((byte*)obj >= (byte*)s->limit) |
579 continue; | 525 continue; |
580 » » » » uintptr size = s->elemsize; | 526 » » » » size = s->elemsize; |
581 int32 i = ((byte*)obj - p)/size; | 527 int32 i = ((byte*)obj - p)/size; |
582 obj = p+i*size; | 528 obj = p+i*size; |
583 } | 529 } |
584 | 530 |
585 // Now that we know the object header, reload bits. | 531 // Now that we know the object header, reload bits. |
586 off = (uintptr*)obj - (uintptr*)arena_start; | 532 off = (uintptr*)obj - (uintptr*)arena_start; |
587 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord -
1; | 533 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord -
1; |
588 shift = off % wordsPerBitmapWord; | 534 shift = off % wordsPerBitmapWord; |
589 xbits = *bitp; | 535 xbits = *bitp; |
590 bits = xbits >> shift; | 536 bits = xbits >> shift; |
591 | 537 |
592 found2: | 538 found2: |
593 // Now we have bits, bitp, and shift correct for | 539 // Now we have bits, bitp, and shift correct for |
594 // obj pointing at the base of the object. | 540 // obj pointing at the base of the object. |
595 // Only care about allocated and not marked. | 541 // Only care about allocated and not marked. |
596 if((bits & (bitAllocated|bitMarked)) != bitAllocated) | 542 if((bits & (bitAllocated|bitMarked)) != bitAllocated) |
597 continue; | 543 continue; |
598 | 544 » » » if(work.nproc == 1) |
599 » » » *ybufpos = (struct Y){bitp, shift, obj, gc}; | 545 » » » » *bitp |= bitMarked<<shift; |
600 » » » ybufpos++; | 546 » » » else { |
601 » » } | 547 » » » » for(;;) { |
602 | 548 » » » » » x = *bitp; |
603 » » runtime·lock(&lock); | 549 » » » » » if(x & (bitMarked<<shift)) |
604 » » for(y=ybuf; y<ybufpos; y++){ | 550 » » » » » » goto continue_obj; |
605 » » » xbits = *y->bitp; | 551 » » » » » if(runtime·casp((void**)bitp, (void*)x,
(void*)(x|(bitMarked<<shift)))) |
606 » » » bits = xbits >> y->shift; | 552 » » » » » » break; |
607 » » » if((bits & bitMarked) != 0) | 553 » » » » } |
608 » » » » continue; | 554 » » » } |
609 » » »······· | |
610 » » » *y->bitp = xbits | (bitMarked << y->shift); | |
611 | 555 |
612 // If object has no pointers, don't need to scan further
. | 556 // If object has no pointers, don't need to scan further
. |
613 if((bits & bitNoPointers) != 0) | 557 if((bits & bitNoPointers) != 0) |
614 continue; | 558 continue; |
615 | 559 |
616 // Ask span about size class. | 560 // Ask span about size class. |
617 // (Manually inlined copy of MHeap_Lookup.) | 561 // (Manually inlined copy of MHeap_Lookup.) |
618 » » » x = (uintptr)y->obj >> PageShift; | 562 » » » x = (uintptr)obj >> PageShift; |
619 if(sizeof(void*) == 8) | 563 if(sizeof(void*) == 8) |
620 x -= (uintptr)arena_start>>PageShift; | 564 x -= (uintptr)arena_start>>PageShift; |
621 » » » s = runtime·mheap.map[x]; | 565 » » » s = runtime·mheap->map[x]; |
622 | 566 |
623 » » » *wp = (Obj){y->obj, s->elemsize, y->gc}; | 567 » » » PREFETCH(obj); |
| 568 |
| 569 » » » *wp = (Obj){obj, s->elemsize, ti}; |
624 wp++; | 570 wp++; |
625 nobj++; | 571 nobj++; |
626 » » } | 572 » » continue_obj:; |
627 » » runtime·unlock(&lock); | 573 » » } |
628 | 574 |
629 // If another proc wants a pointer, give it some. | 575 // If another proc wants a pointer, give it some. |
630 if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0)
{ | 576 if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0)
{ |
631 wbuf->nobj = nobj; | 577 wbuf->nobj = nobj; |
632 wbuf = handoff(wbuf); | 578 wbuf = handoff(wbuf); |
633 nobj = wbuf->nobj; | 579 nobj = wbuf->nobj; |
634 wp = wbuf->obj + nobj; | 580 wp = wbuf->obj + nobj; |
635 } | 581 } |
636 } | 582 } |
637 | 583 |
638 *_wp = wp; | 584 *_wp = wp; |
639 *_wbuf = wbuf; | 585 *_wbuf = wbuf; |
640 *_nobj = nobj; | 586 *_nobj = nobj; |
641 } | 587 } |
642 | 588 |
643 static void | 589 static void |
644 flushobjbuf(Obj *objbuf, uintptr n, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj) | 590 flushobjbuf(Obj *objbuf, Obj **objbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_
nobj) |
645 { | 591 { |
646 » uintptr nobj, off, i; | 592 » uintptr nobj, off; |
647 Obj *wp, obj; | 593 Obj *wp, obj; |
648 Workbuf *wbuf; | 594 Workbuf *wbuf; |
649 | 595 » Obj *objbuf_end; |
650 » // Get the state of buffer | 596 |
651 wp = *_wp; | 597 wp = *_wp; |
652 wbuf = *_wbuf; | 598 wbuf = *_wbuf; |
653 nobj = *_nobj; | 599 nobj = *_nobj; |
654 | 600 |
655 » for(i=0; i<n; i++) { | 601 » objbuf_end = *objbufpos; |
656 » » obj = objbuf[i]; | 602 » *objbufpos = objbuf; |
| 603 |
| 604 » while(objbuf < objbuf_end) { |
| 605 » » obj = *objbuf++; |
657 | 606 |
658 // Align obj.b to a word boundary. | 607 // Align obj.b to a word boundary. |
659 off = (uintptr)obj.p & (PtrSize-1); | 608 off = (uintptr)obj.p & (PtrSize-1); |
660 if(off != 0) { | 609 if(off != 0) { |
661 obj.p += PtrSize - off; | 610 obj.p += PtrSize - off; |
662 obj.n -= PtrSize - off; | 611 obj.n -= PtrSize - off; |
663 » » » obj.gc = 0; | 612 » » » obj.ti = 0; |
664 } | 613 } |
665 | 614 |
666 if(obj.p == nil || obj.n == 0) | 615 if(obj.p == nil || obj.n == 0) |
667 continue; | 616 continue; |
668 | 617 |
669 // If buffer is full, get a new one. | 618 // If buffer is full, get a new one. |
670 if(wbuf == nil || nobj >= nelem(wbuf->obj)) { | 619 if(wbuf == nil || nobj >= nelem(wbuf->obj)) { |
671 if(wbuf != nil) | 620 if(wbuf != nil) |
672 wbuf->nobj = nobj; | 621 wbuf->nobj = nobj; |
673 wbuf = getempty(wbuf); | 622 wbuf = getempty(wbuf); |
674 wp = wbuf->obj; | 623 wp = wbuf->obj; |
675 nobj = 0; | 624 nobj = 0; |
676 } | 625 } |
677 | 626 |
678 *wp = obj; | 627 *wp = obj; |
679 wp++; | 628 wp++; |
680 nobj++; | 629 nobj++; |
681 } | 630 } |
682 | 631 |
683 // If another proc wants a pointer, give it some. | 632 // If another proc wants a pointer, give it some. |
684 if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) { | 633 if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) { |
685 wbuf->nobj = nobj; | 634 wbuf->nobj = nobj; |
686 wbuf = handoff(wbuf); | 635 wbuf = handoff(wbuf); |
687 nobj = wbuf->nobj; | 636 nobj = wbuf->nobj; |
688 wp = wbuf->obj + nobj; | 637 wp = wbuf->obj + nobj; |
689 } | 638 } |
690 | 639 |
691 // Update the state of buffer | |
692 *_wp = wp; | 640 *_wp = wp; |
693 *_wbuf = wbuf; | 641 *_wbuf = wbuf; |
694 *_nobj = nobj; | 642 *_nobj = nobj; |
695 } | 643 } |
696 | 644 |
697 // Program that scans the whole block and treats every block element as a potent
ial pointer | 645 // Program that scans the whole block and treats every block element as a potent
ial pointer |
698 static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR}; | 646 static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR}; |
699 | 647 |
700 // Hashmap iterator program | 648 // Hashmap iterator program |
701 static uintptr mapProg[2] = {0, GC_MAP_NEXT}; | 649 static uintptr mapProg[2] = {0, GC_MAP_NEXT}; |
| 650 |
| 651 // Hchan program |
| 652 static uintptr chanProg[2] = {0, GC_CHAN}; |
| 653 |
| 654 // Local variables of a program fragment or loop |
| 655 typedef struct Frame Frame; |
| 656 struct Frame { |
| 657 uintptr count, elemsize, b; |
| 658 uintptr *loop_or_ret; |
| 659 }; |
| 660 |
| 661 // Sanity check for the derived type info objti. |
| 662 static void |
| 663 checkptr(void *obj, uintptr objti) |
| 664 { |
| 665 uintptr *pc1, *pc2, type, tisize, i, j, x; |
| 666 byte *objstart; |
| 667 Type *t; |
| 668 MSpan *s; |
| 669 |
| 670 if(!Debug) |
| 671 runtime·throw("checkptr is debug only"); |
| 672 |
| 673 if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used) |
| 674 return; |
| 675 type = runtime·gettype(obj); |
| 676 t = (Type*)(type & ~(uintptr)(PtrSize-1)); |
| 677 if(t == nil) |
| 678 return; |
| 679 x = (uintptr)obj >> PageShift; |
| 680 if(sizeof(void*) == 8) |
| 681 x -= (uintptr)(runtime·mheap->arena_start)>>PageShift; |
| 682 s = runtime·mheap->map[x]; |
| 683 objstart = (byte*)((uintptr)s->start<<PageShift); |
| 684 if(s->sizeclass != 0) { |
| 685 i = ((byte*)obj - objstart)/s->elemsize; |
| 686 objstart += i*s->elemsize; |
| 687 } |
| 688 tisize = *(uintptr*)objti; |
| 689 // Sanity check for object size: it should fit into the memory block. |
| 690 if((byte*)obj + tisize > objstart + s->elemsize) |
| 691 runtime·throw("invalid gc type info"); |
| 692 if(obj != objstart) |
| 693 return; |
| 694 // If obj points to the beginning of the memory block, |
| 695 // check type info as well. |
| 696 if(t->string == nil || |
| 697 // Gob allocates unsafe pointers for indirection. |
| 698 (runtime·strcmp(t->string->str, (byte*)"unsafe.Pointer") && |
| 699 // Runtime and gc think differently about closures. |
| 700 runtime·strstr(t->string->str, (byte*)"struct { F uintptr") != t
->string->str)) { |
| 701 pc1 = (uintptr*)objti; |
| 702 pc2 = (uintptr*)t->gc; |
| 703 // A simple best-effort check until first GC_END. |
| 704 for(j = 1; pc1[j] != GC_END && pc2[j] != GC_END; j++) { |
| 705 if(pc1[j] != pc2[j]) { |
| 706 runtime·printf("invalid gc type info for '%s' at
%p, type info %p, block info %p\n", |
| 707 t->string ? (int8*)t->string->str : (int
8*)"?", j, pc1[j], pc2[j]); |
| 708 runtime·throw("invalid gc type info"); |
| 709 } |
| 710 } |
| 711 } |
| 712 }······································· |
702 | 713 |
703 // scanblock scans a block of n bytes starting at pointer b for references | 714 // scanblock scans a block of n bytes starting at pointer b for references |
704 // to other objects, scanning any it finds recursively until there are no | 715 // to other objects, scanning any it finds recursively until there are no |
705 // unscanned objects left. Instead of using an explicit recursion, it keeps | 716 // unscanned objects left. Instead of using an explicit recursion, it keeps |
706 // a work list in the Workbuf* structures and loops in the main function | 717 // a work list in the Workbuf* structures and loops in the main function |
707 // body. Keeping an explicit work list is easier on the stack allocator and | 718 // body. Keeping an explicit work list is easier on the stack allocator and |
708 // more efficient. | 719 // more efficient. |
| 720 // |
| 721 // wbuf: current work buffer |
| 722 // wp: storage for next queued pointer (write pointer) |
| 723 // nobj: number of queued objects |
709 static void | 724 static void |
710 scan(Obj **_wp, Workbuf **_wbuf, uintptr *_nobj, bool keepworking) | 725 scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) |
711 { | 726 { |
712 byte *b, *b2, *arena_start, *arena_used, *p; | 727 byte *b, *b2, *arena_start, *arena_used, *p; |
713 » uintptr n, i, nobj, typeinfo, elemsize, end_b, count, *target, *ret; | 728 » uintptr n, i, end_b, elemsize, size, ti, objti, count, type; |
714 » Obj *wp; | 729 » uintptr *pc, precise_type, nominal_size; |
715 » Workbuf *wbuf; | 730 » uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti, *chan_
ret; |
716 » uintptr gc; | |
717 » Type *t, *type; | |
718 » MapType *maptype; | |
719 » bool ok, wasnt_marked; | |
720 MSpan *s; | 731 MSpan *s; |
721 int8 *prefix; | 732 int8 *prefix; |
722 void *obj; | 733 void *obj; |
723 » uintptr objgc; | 734 » Type *t; |
| 735 » Slice *sliceptr; |
| 736 » Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4]; |
| 737 » BufferList *scanbuffers; |
| 738 » PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos; |
| 739 » Obj *objbuf, *objbuf_end, *objbufpos; |
| 740 » Eface *eface; |
| 741 » Iface *iface; |
724 Hmap *hmap; | 742 Hmap *hmap; |
725 » Slice *slicePtr; | 743 » MapType *maptype; |
726 | 744 » bool mapkey_kind, mapval_kind; |
727 » struct Frame {uintptr count, elemsize, b; uintptr *loop_or_ret;}; | |
728 » struct Frame stack[GC_STACK_CAPACITY+2]; | |
729 » struct Frame *stack_ptr; | |
730 » struct Frame stack_top; | |
731 | |
732 » uintptr *pc; | |
733 » uintptr precise_type; | |
734 » uintptr nominal_size; | |
735 | |
736 » struct BufferList *scanbuffers; | |
737 » struct X *xbuf, *xbuf_end; | |
738 » struct Y *ybuf; | |
739 » struct Obj *objbuf, *objbuf_end; | |
740 | |
741 » struct X *xbufpos; | |
742 » struct Obj *objbufpos; | |
743 | |
744 » uintptr *map_ret; | |
745 struct hash_gciter map_iter; | 745 struct hash_gciter map_iter; |
746 » uintptr mapkey_size, mapval_size; | 746 » struct hash_gciter_data d; |
747 » bool mapkey_kind, mapval_kind; | 747 » Hchan *chan; |
748 » uintptr mapkey_gc, mapval_gc; | 748 » ChanType *chantype; |
749 | |
750 » Type *itabtype; | |
751 » Eface efacevalue, *eface; | |
752 » Iface *iface; | |
753 | |
754 » // End of local variable declarations. | |
755 | 749 |
756 if(sizeof(Workbuf) % PageSize != 0) | 750 if(sizeof(Workbuf) % PageSize != 0) |
757 » » runtime·throw("scan: size of Workbuf is suboptimal"); | 751 » » runtime·throw("scanblock: size of Workbuf is suboptimal"); |
758 | 752 |
759 // Memory arena parameters. | 753 // Memory arena parameters. |
760 » arena_start = runtime·mheap.arena_start; | 754 » arena_start = runtime·mheap->arena_start; |
761 » arena_used = runtime·mheap.arena_used; | 755 » arena_used = runtime·mheap->arena_used; |
762 | |
763 » // Get the state of buffer»····· | |
764 » wp = *_wp; // storage for next queued pointer (write pointer) | |
765 » wbuf = *_wbuf; // current work buffer | |
766 » nobj = *_nobj; // number of queued objects | |
767 | 756 |
768 stack_ptr = stack+nelem(stack)-1; | 757 stack_ptr = stack+nelem(stack)-1; |
769 ········ | 758 ········ |
770 precise_type = false; | 759 precise_type = false; |
771 nominal_size = 0; | 760 nominal_size = 0; |
772 | 761 |
773 » // Get buffers | 762 » // Allocate ptrbuf |
774 { | 763 { |
775 » » runtime·lock(&lock); | 764 » » scanbuffers = &bufferList[m->helpgc]; |
776 | 765 » » ptrbuf = &scanbuffers->ptrtarget[0]; |
777 » » if(bufferList == nil) { | 766 » » ptrbuf_end = &scanbuffers->ptrtarget[0] + nelem(scanbuffers->ptr
target); |
778 » » » bufferList = runtime·SysAlloc(sizeof(*bufferList)); | |
779 » » » bufferList->next = nil; | |
780 » » } | |
781 » » scanbuffers = bufferList; | |
782 » » bufferList = bufferList->next; | |
783 » »······· | |
784 » » xbuf = &scanbuffers->x[0]; | |
785 » » xbuf_end = &scanbuffers->x[0] + nelem(scanbuffers->x); | |
786 » » ybuf = &scanbuffers->y[0]; | |
787 objbuf = &scanbuffers->obj[0]; | 767 objbuf = &scanbuffers->obj[0]; |
788 objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj); | 768 objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj); |
789 | 769 » } |
790 » » runtime·unlock(&lock); | 770 |
791 » } | 771 » ptrbufpos = ptrbuf; |
792 | |
793 » xbufpos = xbuf; | |
794 objbufpos = objbuf; | 772 objbufpos = objbuf; |
795 | 773 |
796 // (Silence the compiler) | 774 // (Silence the compiler) |
797 map_ret = nil; | 775 map_ret = nil; |
798 mapkey_size = mapval_size = 0; | 776 mapkey_size = mapval_size = 0; |
799 mapkey_kind = mapval_kind = false; | 777 mapkey_kind = mapval_kind = false; |
800 » mapkey_gc = mapval_gc = 0; | 778 » mapkey_ti = mapval_ti = 0; |
801 | 779 » chan = nil; |
802 » // Get C pointer to the Go type "itab" | 780 » chantype = nil; |
803 » runtime·gc_itab_ptr(&efacevalue); | 781 » chan_ret = nil; |
804 » itabtype = ((PtrType*)efacevalue.type)->elem; | 782 |
| 783 » if(!PreciseGC) |
| 784 » » goto next_block_conservative; |
805 | 785 |
806 goto next_block; | 786 goto next_block; |
807 | 787 |
808 for(;;) { | 788 for(;;) { |
809 // Each iteration scans the block b of length n, queueing pointe
rs in | 789 // Each iteration scans the block b of length n, queueing pointe
rs in |
810 // the work buffer. | 790 // the work buffer. |
811 » » if(Debug > 1) | 791 » » if(Debug > 1) { |
812 runtime·printf("scanblock %p %D\n", b, (int64)n); | 792 runtime·printf("scanblock %p %D\n", b, (int64)n); |
813 | 793 » » } |
814 » » if(DebugStats) { | 794 |
| 795 » » if(CollectStats) { |
815 runtime·xadd64(&gcstats.nbytes, n); | 796 runtime·xadd64(&gcstats.nbytes, n); |
816 » » » runtime·xadd64(&gcstats.nobj_sum, nobj); | 797 » » » runtime·xadd64(&gcstats.obj.sum, nobj); |
817 » » » runtime·xadd64(&gcstats.nobj_cnt, 1); | 798 » » » runtime·xadd64(&gcstats.obj.cnt, 1); |
818 » » } | 799 » » } |
819 | 800 |
820 » » if(gc != 0) { | 801 » » if(ti != 0) { |
821 » » » pc = (uintptr*)(gc & ~(uintptr)3); | 802 » » » pc = (uintptr*)(ti & ~(uintptr)PC_BITS); |
822 » » » precise_type = (gc & 1); | 803 » » » precise_type = (ti & PRECISE); |
823 stack_top.elemsize = pc[0]; | 804 stack_top.elemsize = pc[0]; |
824 if(!precise_type) | 805 if(!precise_type) |
825 nominal_size = pc[0]; | 806 nominal_size = pc[0]; |
826 » » » if(gc & 2) { | 807 » » » if(ti & LOOP) { |
827 stack_top.count = 0; // 0 means an infinite n
umber of iterations | 808 stack_top.count = 0; // 0 means an infinite n
umber of iterations |
828 stack_top.loop_or_ret = pc+1; | 809 stack_top.loop_or_ret = pc+1; |
829 } else { | 810 } else { |
830 stack_top.count = 1; | 811 stack_top.count = 1; |
831 } | 812 } |
| 813 if(Debug) { |
| 814 // Simple sanity check for provided type info ti
: |
| 815 // The declared size of the object must be not l
arger than the actual size |
| 816 // (it can be smaller due to inferior pointers). |
| 817 // It's difficult to make a comprehensive check
due to inferior pointers, |
| 818 // reflection, gob, etc. |
| 819 if(pc[0] > n) { |
| 820 runtime·printf("invalid gc type info: ty
pe info size %p, block size %p\n", pc[0], n); |
| 821 runtime·throw("invalid gc type info"); |
| 822 } |
| 823 } |
832 } else if(UseSpanType) { | 824 } else if(UseSpanType) { |
833 » » » if(DebugStats) | 825 » » » if(CollectStats) |
834 » » » » runtime·xadd64(&gcstats.nobj_notype, 1); | 826 » » » » runtime·xadd64(&gcstats.obj.notype, 1); |
835 | 827 |
836 » » » typeinfo = runtime·gettype(b); | 828 » » » type = runtime·gettype(b); |
837 | 829 » » » if(type != 0) { |
838 » » » if(typeinfo != 0) { | 830 » » » » if(CollectStats) |
839 » » » » if(DebugStats) | 831 » » » » » runtime·xadd64(&gcstats.obj.typelookup,
1); |
840 » » » » » runtime·xadd64(&gcstats.nobj_typelookups
_ok, 1); | 832 |
841 | 833 » » » » t = (Type*)(type & ~(uintptr)(PtrSize-1)); |
842 » » » » t = (Type*)(typeinfo & ~(uintptr)3); | 834 » » » » switch(type & (PtrSize-1)) { |
843 | |
844 » » » » switch(typeinfo & (PtrSize-1)) { | |
845 case TypeInfo_SingleObject: | 835 case TypeInfo_SingleObject: |
846 pc = (uintptr*)t->gc; | 836 pc = (uintptr*)t->gc; |
847 precise_type = true; // type informatio
n about 'b' is precise | 837 precise_type = true; // type informatio
n about 'b' is precise |
848 stack_top.count = 1; | 838 stack_top.count = 1; |
849 stack_top.elemsize = pc[0]; | 839 stack_top.elemsize = pc[0]; |
850 break; | 840 break; |
851 case TypeInfo_Array: | 841 case TypeInfo_Array: |
852 pc = (uintptr*)t->gc; | 842 pc = (uintptr*)t->gc; |
853 if(pc[0] == 0) | 843 if(pc[0] == 0) |
854 goto next_block; | 844 goto next_block; |
855 precise_type = true; // type informatio
n about 'b' is precise | 845 precise_type = true; // type informatio
n about 'b' is precise |
856 stack_top.count = 0; // 0 means an infi
nite number of iterations | 846 stack_top.count = 0; // 0 means an infi
nite number of iterations |
857 stack_top.elemsize = pc[0]; | 847 stack_top.elemsize = pc[0]; |
858 stack_top.loop_or_ret = pc+1; | 848 stack_top.loop_or_ret = pc+1; |
859 break; | 849 break; |
860 case TypeInfo_Map: | 850 case TypeInfo_Map: |
861 hmap = (Hmap*)b; | 851 hmap = (Hmap*)b; |
862 maptype = (MapType*)t; | 852 maptype = (MapType*)t; |
863 » » » » » ok = hash_gciter_init(hmap, &map_iter); | 853 » » » » » if(hash_gciter_init(hmap, &map_iter)) { |
864 » » » » » if(ok) { | |
865 mapkey_size = maptype->key->size
; | 854 mapkey_size = maptype->key->size
; |
866 mapkey_kind = maptype->key->kind
; | 855 mapkey_kind = maptype->key->kind
; |
867 » » » » » » mapkey_gc = (uintptr)maptype->
key->gc | 1; | 856 » » » » » » mapkey_ti = (uintptr)maptype->
key->gc | PRECISE; |
868 mapval_size = maptype->elem->siz
e; | 857 mapval_size = maptype->elem->siz
e; |
869 mapval_kind = maptype->elem->kin
d; | 858 mapval_kind = maptype->elem->kin
d; |
870 » » » » » » mapval_gc = (uintptr)maptype->
elem->gc | 1; | 859 » » » » » » mapval_ti = (uintptr)maptype->
elem->gc | PRECISE; |
871 | 860 |
872 » » » » » » map_ret = 0; | 861 » » » » » » map_ret = nil; |
873 pc = mapProg; | 862 pc = mapProg; |
874 } else { | 863 } else { |
875 goto next_block; | 864 goto next_block; |
876 } | 865 } |
877 break; | 866 break; |
| 867 case TypeInfo_Chan: |
| 868 chan = (Hchan*)b; |
| 869 chantype = (ChanType*)t; |
| 870 chan_ret = nil; |
| 871 pc = chanProg; |
| 872 break; |
878 default: | 873 default: |
879 » » » » » runtime·throw("scanblock: invalid typein
fo"); | 874 » » » » » runtime·throw("scanblock: invalid type")
; |
880 return; | 875 return; |
881 } | 876 } |
882 } else { | 877 } else { |
883 pc = defaultProg; | 878 pc = defaultProg; |
884 } | 879 } |
885 } else { | 880 } else { |
886 pc = defaultProg; | 881 pc = defaultProg; |
887 } | 882 } |
888 | 883 |
| 884 if(IgnorePreciseGC) |
| 885 pc = defaultProg; |
| 886 |
889 if(DebugTypeAtBlockEnd) { | 887 if(DebugTypeAtBlockEnd) { |
890 » » » s = runtime·MHeap_LookupMaybe(&runtime·mheap, b); | 888 » » » s = runtime·MHeap_LookupMaybe(runtime·mheap, b); |
891 | 889 |
892 if(s != nil) { | 890 if(s != nil) { |
893 elemsize = s->elemsize; | 891 elemsize = s->elemsize; |
894 p = (byte*)((uintptr)s->start<<PageShift); | 892 p = (byte*)((uintptr)s->start<<PageShift); |
895 if(s->sizeclass == 0) { | 893 if(s->sizeclass == 0) { |
896 b2 = p; | 894 b2 = p; |
897 } | 895 } |
898 else { | 896 else { |
899 i = (b-p)/elemsize; | 897 i = (b-p)/elemsize; |
900 b2 = p+i*elemsize; | 898 b2 = p+i*elemsize; |
901 } | 899 } |
902 | 900 |
903 » » » » typeinfo = *(uintptr*)(b2+elemsize-sizeof(uintpt
r)); | 901 » » » » type = *(uintptr*)(b2+elemsize-sizeof(uintptr)); |
904 » » » » if(typeinfo != 0) { | 902 » » » » if(type != 0) { |
905 » » » » » type = (Type*)(typeinfo & ~(uintptr)(Ptr
Size-1)); | 903 » » » » » t = (Type*)(type & ~(uintptr)(PtrSize-1)
); |
906 | 904 |
907 prefix = ""; | 905 prefix = ""; |
908 » » » » » switch(typeinfo & (PtrSize-1)) { | 906 » » » » » switch(type & (PtrSize-1)) { |
909 case TypeInfo_SingleObject: | 907 case TypeInfo_SingleObject: |
910 prefix = "single"; | 908 prefix = "single"; |
911 break; | 909 break; |
912 case TypeInfo_Array: | 910 case TypeInfo_Array: |
913 prefix = "array"; | 911 prefix = "array"; |
914 break; | 912 break; |
915 } | 913 } |
916 | 914 |
917 runtime·lock(&lock); | 915 runtime·lock(&lock); |
918 » » » » » runtime·printf("scan %p+%X %D, type %s %
S, pc %p\n", | 916 » » » » » runtime·printf("scanblock %p+%X %D, type
%s %S, pc %p\n", |
919 » » » » » » b, (int64)(b-b2), (int64)n, pref
ix, *type->string, (pc==defaultProg ? nil : pc)); | 917 » » » » » » b, (int64)(b-b2), (int64)n, pref
ix, *t->string, (pc==defaultProg ? nil : pc)); |
920 runtime·unlock(&lock); | 918 runtime·unlock(&lock); |
921 } else { | 919 } else { |
922 runtime·lock(&lock); | 920 runtime·lock(&lock); |
923 » » » » » runtime·printf("scan %p+%X %D, pc %p\n", | 921 » » » » » runtime·printf("scanblock %p+%X %D, pc %
p\n", |
924 b, (int64)(b-b2), (int64)n, (pc=
=defaultProg ? nil : pc)); | 922 b, (int64)(b-b2), (int64)n, (pc=
=defaultProg ? nil : pc)); |
925 runtime·unlock(&lock); | 923 runtime·unlock(&lock); |
926 } | 924 } |
927 } else { | 925 } else { |
928 runtime·lock(&lock); | 926 runtime·lock(&lock); |
929 » » » » runtime·printf("scan %p %D, not in heap, pc %p\n
", | 927 » » » » runtime·printf("scanblock %p %D, not in heap, pc
%p\n", |
930 b, (int64)n, (pc==defaultProg ? nil : pc
)); | 928 b, (int64)n, (pc==defaultProg ? nil : pc
)); |
931 runtime·unlock(&lock); | 929 runtime·unlock(&lock); |
932 } | 930 } |
933 } | 931 } |
934 | 932 |
935 if(DebugType) { | 933 if(DebugType) { |
936 if(pc[0] > n && pc != defaultProg) | 934 if(pc[0] > n && pc != defaultProg) |
937 » » » » runtime·throw("scan: size exceeds n"); | 935 » » » » runtime·throw("scanblock: size exceeds n"); |
938 if(pc[0] % PtrSize != 0) | 936 if(pc[0] % PtrSize != 0) |
939 » » » » runtime·throw("scan: invalid alignment of size")
; | 937 » » » » runtime·throw("scanblock: invalid alignment of s
ize"); |
940 if(n < PtrSize) | 938 if(n < PtrSize) |
941 » » » » runtime·throw("scan: n < PtrSize"); | 939 » » » » runtime·throw("scanblock: n < PtrSize"); |
942 } | 940 } |
943 | 941 |
944 pc++; | 942 pc++; |
945 stack_top.b = (uintptr)b; | 943 stack_top.b = (uintptr)b; |
946 | 944 |
947 end_b = (uintptr)b + n - PtrSize; | 945 end_b = (uintptr)b + n - PtrSize; |
948 if(DebugTypeAtBlockEnd) { | 946 if(DebugTypeAtBlockEnd) { |
949 if(b >= arena_start && b+n <= arena_used && pc != mapPro
g+1) | 947 if(b >= arena_start && b+n <= arena_used && pc != mapPro
g+1) |
950 end_b -= PtrSize; | 948 end_b -= PtrSize; |
951 } | 949 } |
952 | 950 |
953 » next_instr: | 951 » for(;;) { |
954 » » if(DebugStats) | 952 » » if(CollectStats) |
955 runtime·xadd64(&gcstats.instr[pc[0]], 1); | 953 runtime·xadd64(&gcstats.instr[pc[0]], 1); |
956 | 954 |
| 955 obj = nil; |
| 956 objti = 0; |
957 switch(pc[0]) { | 957 switch(pc[0]) { |
958 case GC_PTR: | 958 case GC_PTR: |
959 obj = *(void**)(stack_top.b + pc[1]); | 959 obj = *(void**)(stack_top.b + pc[1]); |
960 » » » objgc = pc[2]; | 960 » » » objti = pc[2]; |
961 pc += 3; | 961 pc += 3; |
962 | 962 » » » if(Debug) |
963 » » » if(obj >= arena_start && obj < arena_used) { | 963 » » » » checkptr(obj, objti); |
964 » » » » *xbufpos = (struct X){obj, objgc}; | 964 » » » break; |
965 » » » » xbufpos++; | |
966 » » » » if(xbufpos == xbuf_end) | |
967 » » » » » goto flush_buffers; | |
968 » » » } | |
969 » » » goto next_instr; | |
970 | 965 |
971 case GC_SLICE: | 966 case GC_SLICE: |
972 » » » slicePtr = (Slice*)(stack_top.b + pc[1]); | 967 » » » sliceptr = (Slice*)(stack_top.b + pc[1]); |
973 » » » if(slicePtr->cap == 0) { | 968 » » » if(sliceptr->cap != 0) { |
974 » » » » // slicePtr->array may be pointing to a complete
ly different object. | 969 » » » » obj = sliceptr->array; |
975 | 970 » » » » // Can't use slice element type for scanning, |
976 » » » » // There are two options. | 971 » » » » // because if it points to an array embedded |
977 » » » » if(true) { | 972 » » » » // in the beginning of a struct, |
978 » » » » » // If there are no other references to t
he backing array, | 973 » » » » // we will scan the whole struct as the slice. |
979 » » » » » // the backing array can be safely garba
ge collected | 974 » » » » // So just obtain type info from heap. |
980 » » » » » // because slice with zero capacity is u
nusable in Go programs. | 975 » » » } |
981 » » » » » pc += 3; | 976 » » » pc += 3; |
982 » » » » » goto next_instr; | 977 » » » break; |
983 » » » » } else { | |
984 » » » » » // Assume that unsafe Go code uses slice
Ptr->array to derive a slice | |
985 » » » » » // with non-zero capacity. | |
986 » » » » » // Act as GC_APTR. | |
987 » » » » » obj = slicePtr->array; | |
988 » » » » » objgc = 0; | |
989 » » » » » pc += 3; | |
990 » » » » } | |
991 » » » } | |
992 » » » else { | |
993 » » » » obj = slicePtr->array; | |
994 » » » » objgc = pc[2] | 3; | |
995 » » » » pc += 3; | |
996 » » » } | |
997 | |
998 » » » if(obj >= arena_start && obj < arena_used) { | |
999 » » » » *xbufpos = (struct X){obj, objgc}; | |
1000 » » » » xbufpos++; | |
1001 » » » » if(xbufpos == xbuf_end) | |
1002 » » » » » goto flush_buffers; | |
1003 » » » } | |
1004 » » » goto next_instr; | |
1005 | 978 |
1006 case GC_APTR: | 979 case GC_APTR: |
1007 obj = *(void**)(stack_top.b + pc[1]); | 980 obj = *(void**)(stack_top.b + pc[1]); |
1008 pc += 2; | 981 pc += 2; |
1009 | 982 » » » break; |
1010 » » » if(obj >= arena_start && obj < arena_used) { | |
1011 » » » » *xbufpos = (struct X){obj, 0}; | |
1012 » » » » xbufpos++; | |
1013 » » » » if(xbufpos == xbuf_end) | |
1014 » » » » » goto flush_buffers; | |
1015 » » » } | |
1016 » » » goto next_instr; | |
1017 | 983 |
1018 case GC_STRING: | 984 case GC_STRING: |
1019 obj = *(void**)(stack_top.b + pc[1]); | 985 obj = *(void**)(stack_top.b + pc[1]); |
| 986 markonly(obj); |
1020 pc += 2; | 987 pc += 2; |
1021 | 988 » » » continue; |
1022 » » » if(obj >= arena_start && obj < arena_used) { | |
1023 » » » » *xbufpos = (struct X){obj, 0}; | |
1024 » » » » xbufpos++; | |
1025 » » » » if(xbufpos == xbuf_end) | |
1026 » » » » » goto flush_buffers; | |
1027 » » » } | |
1028 » » » goto next_instr; | |
1029 | 989 |
1030 case GC_EFACE: | 990 case GC_EFACE: |
1031 eface = (Eface*)(stack_top.b + pc[1]); | 991 eface = (Eface*)(stack_top.b + pc[1]); |
1032 pc += 2; | 992 pc += 2; |
1033 | 993 » » » if(eface->type == nil) |
1034 » » » if(eface->type != nil && (eface->data >= arena_start &&
eface->data < arena_used)) { | 994 » » » » continue; |
1035 » » » » t = eface->type; | 995 |
| 996 » » » // eface->type |
| 997 » » » t = eface->type; |
| 998 » » » if((void*)t >= arena_start && (void*)t < arena_used) { |
| 999 » » » » *ptrbufpos++ = (PtrTarget){t, 0}; |
| 1000 » » » » if(ptrbufpos == ptrbuf_end) |
| 1001 » » » » » flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wb
uf, &nobj); |
| 1002 » » » } |
| 1003 |
| 1004 » » » // eface->data |
| 1005 » » » if(eface->data >= arena_start && eface->data < arena_use
d) { |
1036 if(t->size <= sizeof(void*)) { | 1006 if(t->size <= sizeof(void*)) { |
1037 if((t->kind & KindNoPointers)) | 1007 if((t->kind & KindNoPointers)) |
1038 » » » » » » goto next_instr; | 1008 » » » » » » continue; |
1039 | 1009 |
| 1010 » » » » » obj = eface->data; |
1040 if((t->kind & ~KindNoPointers) == KindPt
r) | 1011 if((t->kind & ~KindNoPointers) == KindPt
r) |
1041 » » » » » » *xbufpos = (struct X){eface->dat
a, (uintptr)((PtrType*)t)->elem->gc}; | 1012 » » » » » » objti = (uintptr)((PtrType*)t)->
elem->gc; |
1042 » » » » » else | |
1043 » » » » » » *xbufpos = (struct X){eface->dat
a, 0}; | |
1044 } else { | 1013 } else { |
1045 » » » » » *xbufpos = (struct X){eface->data, (uint
ptr)t->gc}; | 1014 » » » » » obj = eface->data; |
| 1015 » » » » » objti = (uintptr)t->gc; |
1046 } | 1016 } |
1047 » » » » xbufpos++; | 1017 » » » } |
1048 » » » » if(xbufpos == xbuf_end) | 1018 » » » break; |
1049 » » » » » goto flush_buffers; | |
1050 » » » } | |
1051 » » » goto next_instr; | |
1052 | 1019 |
1053 case GC_IFACE: | 1020 case GC_IFACE: |
1054 iface = (Iface*)(stack_top.b + pc[1]); | 1021 iface = (Iface*)(stack_top.b + pc[1]); |
1055 | 1022 » » » pc += 2; |
1056 » » » if(iface->tab == nil) { | 1023 » » » if(iface->tab == nil) |
1057 » » » » pc += 2; | 1024 » » » » continue; |
1058 » » » » goto next_instr; | |
1059 » » » } | |
1060 | |
1061 » » » if(xbufpos+2+1 > xbuf_end) | |
1062 » » » » goto flush_buffers; | |
1063 ························ | 1025 ························ |
1064 pc += 2; | |
1065 | |
1066 // iface->tab | 1026 // iface->tab |
1067 if((void*)iface->tab >= arena_start && (void*)iface->tab
< arena_used) { | 1027 if((void*)iface->tab >= arena_start && (void*)iface->tab
< arena_used) { |
1068 » » » » *xbufpos = (struct X){iface->tab, (uintptr)itabt
ype->gc}; | 1028 » » » » *ptrbufpos++ = (PtrTarget){iface->tab, (uintptr)
itabtype->gc}; |
1069 » » » » xbufpos++; | 1029 » » » » if(ptrbufpos == ptrbuf_end) |
| 1030 » » » » » flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wb
uf, &nobj); |
1070 } | 1031 } |
1071 | 1032 |
1072 // iface->data | 1033 // iface->data |
1073 if(iface->data >= arena_start && iface->data < arena_use
d) { | 1034 if(iface->data >= arena_start && iface->data < arena_use
d) { |
1074 t = iface->tab->type; | 1035 t = iface->tab->type; |
1075 if(t->size <= sizeof(void*)) { | 1036 if(t->size <= sizeof(void*)) { |
1076 if((t->kind & KindNoPointers)) | 1037 if((t->kind & KindNoPointers)) |
1077 » » » » » » goto next_instr; | 1038 » » » » » » continue; |
1078 | 1039 |
| 1040 » » » » » obj = iface->data; |
1079 if((t->kind & ~KindNoPointers) == KindPt
r) | 1041 if((t->kind & ~KindNoPointers) == KindPt
r) |
1080 » » » » » » *xbufpos = (struct X){iface->dat
a, (uintptr)((PtrType*)t)->elem->gc}; | 1042 » » » » » » objti = (uintptr)((PtrType*)t)->
elem->gc; |
1081 » » » » » else | |
1082 » » » » » » *xbufpos = (struct X){iface->dat
a, 0}; | |
1083 } else { | 1043 } else { |
1084 » » » » » *xbufpos = (struct X){iface->data, (uint
ptr)t->gc}; | 1044 » » » » » obj = iface->data; |
| 1045 » » » » » objti = (uintptr)t->gc; |
1085 } | 1046 } |
1086 » » » » xbufpos++; | 1047 » » » } |
1087 » » » } | 1048 » » » break; |
1088 » » » goto next_instr; | |
1089 | 1049 |
1090 case GC_DEFAULT_PTR: | 1050 case GC_DEFAULT_PTR: |
1091 » » » while(true) { | 1051 » » » while(stack_top.b <= end_b) { |
| 1052 » » » » obj = *(byte**)stack_top.b; |
| 1053 » » » » stack_top.b += PtrSize; |
| 1054 » » » » if(obj >= arena_start && obj < arena_used) { |
| 1055 » » » » » *ptrbufpos++ = (PtrTarget){obj, 0}; |
| 1056 » » » » » if(ptrbufpos == ptrbuf_end) |
| 1057 » » » » » » flushptrbuf(ptrbuf, &ptrbufpos,
&wp, &wbuf, &nobj); |
| 1058 » » » » } |
| 1059 » » » } |
| 1060 » » » goto next_block; |
| 1061 |
| 1062 » » case GC_END: |
| 1063 » » » if(--stack_top.count != 0) { |
| 1064 » » » » // Next iteration of a loop if possible. |
| 1065 » » » » stack_top.b += stack_top.elemsize; |
| 1066 » » » » if(stack_top.b + stack_top.elemsize <= end_b+Ptr
Size) { |
| 1067 » » » » » pc = stack_top.loop_or_ret; |
| 1068 » » » » » continue; |
| 1069 » » » » } |
1092 i = stack_top.b; | 1070 i = stack_top.b; |
1093 » » » » if(i > end_b) | 1071 » » » } else { |
1094 » » » » » goto next_block; | 1072 » » » » // Stack pop if possible. |
1095 » » » » stack_top.b += PtrSize; | 1073 » » » » if(stack_ptr+1 < stack+nelem(stack)) { |
1096 | 1074 » » » » » pc = stack_top.loop_or_ret; |
1097 » » » » obj = *(byte**)i; | 1075 » » » » » stack_top = *(++stack_ptr); |
1098 » » » » if(obj >= arena_start && obj < arena_used) { | 1076 » » » » » continue; |
1099 » » » » » *xbufpos = (struct X){obj, 0}; | |
1100 » » » » » xbufpos++; | |
1101 » » » » » if(xbufpos == xbuf_end) | |
1102 » » » » » » goto flush_buffers; | |
1103 } | 1077 } |
1104 | 1078 » » » » i = (uintptr)b + nominal_size; |
1105 » » » » if(DebugStats) | 1079 » » » } |
1106 » » » » » runtime·xadd64(&gcstats.instr[GC_DEFAULT
_PTR], 1); | 1080 » » » if(!precise_type) { |
1107 » » » } | 1081 » » » » if(DebugType) { |
1108 | 1082 » » » » » if(i % PtrSize != 0) |
1109 » » case GC_END: | 1083 » » » » » » runtime·throw("scanblock: invali
d alignment"); |
1110 » » » stack_top.count--; | 1084 » » » » } |
1111 » » » if(stack_top.count != 0) { | 1085 » » » » // Quickly scan [b+i,b+n) for possible pointers. |
1112 » » » » // Next iteration of a loop | 1086 » » » » for(; i<=end_b; i+=PtrSize) { |
1113 » » » » elemsize = stack_top.elemsize; | 1087 » » » » » if(*(byte**)i != nil) { |
1114 » » » » stack_top.b += elemsize; | 1088 » » » » » » // Found a value that may be a p
ointer. |
1115 » » » » if(stack_top.b + elemsize > end_b+PtrSize) { | 1089 » » » » » » // Do a rescan of the entire blo
ck. |
1116 » » » » » if(precise_type) { | 1090 » » » » » » enqueue((Obj){b, n, 0}, &wbuf, &
wp, &nobj); |
1117 » » » » » » goto next_block; | 1091 » » » » » » if(CollectStats) { |
1118 » » » » » } else { | 1092 » » » » » » » runtime·xadd64(&gcstats.
rescan, 1); |
1119 » » » » » » i = stack_top.b; | 1093 » » » » » » » runtime·xadd64(&gcstats.
rescanbytes, n); |
1120 » » » » » » goto trailing_pointers; | 1094 » » » » » » } |
| 1095 » » » » » » break; |
1121 } | 1096 } |
1122 } | 1097 } |
1123 » » » » pc = stack_top.loop_or_ret; | 1098 » » » } |
1124 » » » » goto next_instr; | 1099 » » » goto next_block; |
1125 » » » } else { | |
1126 » » » » if(stack_ptr == stack+nelem(stack)-1) { | |
1127 » » » » » if(precise_type) { | |
1128 » » » » » » goto next_block; | |
1129 » » » » » } else { | |
1130 » » » » » » i = (uintptr)b + nominal_size; | |
1131 » » » » » » goto trailing_pointers; | |
1132 » » » » » } | |
1133 » » » » } else { | |
1134 » » » » » // Pop from stack | |
1135 » » » » » pc = stack_top.loop_or_ret; | |
1136 » » » » » stack_ptr++; | |
1137 » » » » » stack_top = *stack_ptr; | |
1138 » » » » » goto next_instr; | |
1139 » » » » } | |
1140 » » » } | |
1141 | 1100 |
1142 case GC_ARRAY_START: | 1101 case GC_ARRAY_START: |
1143 i = stack_top.b + pc[1]; | 1102 i = stack_top.b + pc[1]; |
1144 count = pc[2]; | 1103 count = pc[2]; |
1145 elemsize = pc[3]; | 1104 elemsize = pc[3]; |
1146 | |
1147 // Push on stack | |
1148 pc += 4; | 1105 pc += 4; |
1149 » » » *stack_ptr = stack_top; | 1106 |
1150 » » » stack_ptr--; | 1107 » » » // Stack push. |
1151 » » » stack_top = (struct Frame){count, elemsize, i, pc}; | 1108 » » » *stack_ptr-- = stack_top; |
1152 » » » goto next_instr; | 1109 » » » stack_top = (Frame){count, elemsize, i, pc}; |
| 1110 » » » continue; |
1153 | 1111 |
1154 case GC_ARRAY_NEXT: | 1112 case GC_ARRAY_NEXT: |
1155 » » » stack_top.count--; | 1113 » » » if(--stack_top.count != 0) { |
1156 » » » if(stack_top.count != 0) { | |
1157 stack_top.b += stack_top.elemsize; | 1114 stack_top.b += stack_top.elemsize; |
1158 pc = stack_top.loop_or_ret; | 1115 pc = stack_top.loop_or_ret; |
1159 goto next_instr; | |
1160 } else { | 1116 } else { |
1161 » » » » // Pop from stack | 1117 » » » » // Stack pop. |
1162 » » » » stack_ptr++; | 1118 » » » » stack_top = *(++stack_ptr); |
1163 » » » » stack_top = *stack_ptr; | |
1164 pc += 1; | 1119 pc += 1; |
1165 » » » » goto next_instr; | 1120 » » » } |
1166 » » » } | 1121 » » » continue; |
1167 | 1122 |
1168 case GC_CALL: | 1123 case GC_CALL: |
1169 » » » i = stack_top.b + pc[1]; | 1124 » » » // Stack push. |
1170 » » » target = (uintptr*)pc[2]; | 1125 » » » *stack_ptr-- = stack_top; |
1171 » » » ret = pc+3; | 1126 » » » stack_top = (Frame){1, 0, stack_top.b + pc[1], pc+3 /*re
turn address*/}; |
1172 | 1127 » » » pc = (uintptr*)((byte*)pc + *(int32*)(pc+2)); // target
of the CALL instruction |
1173 » » » // Push on stack | 1128 » » » continue; |
1174 » » » *stack_ptr = stack_top; | |
1175 » » » stack_ptr--; | |
1176 » » » stack_top = (struct Frame){1, 0, i, ret}; | |
1177 » » » pc = target; | |
1178 » » » goto next_instr; | |
1179 | 1129 |
1180 case GC_MAP_PTR: | 1130 case GC_MAP_PTR: |
1181 hmap = *(Hmap**)(stack_top.b + pc[1]); | 1131 hmap = *(Hmap**)(stack_top.b + pc[1]); |
1182 if(hmap == nil) { | 1132 if(hmap == nil) { |
1183 pc += 3; | 1133 pc += 3; |
1184 » » » » goto next_instr; | 1134 » » » » continue; |
1185 » » » } | 1135 » » » } |
1186 » » » runtime·lock(&lock); | 1136 » » » if(markonly(hmap)) { |
1187 » » » wasnt_marked = markonly(hmap); | |
1188 » » » runtime·unlock(&lock); | |
1189 » » » if(wasnt_marked) { | |
1190 maptype = (MapType*)pc[2]; | 1137 maptype = (MapType*)pc[2]; |
1191 » » » » ok = hash_gciter_init(hmap, &map_iter); | 1138 » » » » if(hash_gciter_init(hmap, &map_iter)) { |
1192 » » » » if(ok) { | |
1193 mapkey_size = maptype->key->size; | 1139 mapkey_size = maptype->key->size; |
1194 mapkey_kind = maptype->key->kind; | 1140 mapkey_kind = maptype->key->kind; |
1195 » » » » » mapkey_gc = (uintptr)maptype->key->gc
| 1; | 1141 » » » » » mapkey_ti = (uintptr)maptype->key->gc
| PRECISE; |
1196 mapval_size = maptype->elem->size; | 1142 mapval_size = maptype->elem->size; |
1197 mapval_kind = maptype->elem->kind; | 1143 mapval_kind = maptype->elem->kind; |
1198 » » » » » mapval_gc = (uintptr)maptype->elem->gc
| 1; | 1144 » » » » » mapval_ti = (uintptr)maptype->elem->gc
| PRECISE; |
1199 | 1145 |
1200 // Start mapProg. | 1146 // Start mapProg. |
1201 map_ret = pc+3; | 1147 map_ret = pc+3; |
1202 pc = mapProg+1; | 1148 pc = mapProg+1; |
1203 } else { | 1149 } else { |
1204 pc += 3; | 1150 pc += 3; |
1205 } | 1151 } |
1206 } else { | 1152 } else { |
1207 pc += 3; | 1153 pc += 3; |
1208 } | 1154 } |
1209 » » » goto next_instr; | 1155 » » » continue; |
1210 | 1156 |
1211 case GC_MAP_NEXT: | 1157 case GC_MAP_NEXT: |
1212 » » » while(true) { | 1158 » » » // Add all keys and values to buffers, mark all subtable
s. |
1213 » » » » // objbuf: reserve space for 2 objects. | 1159 » » » while(hash_gciter_next(&map_iter, &d)) { |
1214 » » » » // Add 1 in order to remove the need to do "if(o
bjbufpos == objbuf_end) goto flush_buffers" | 1160 » » » » // buffers: reserve space for 2 objects. |
1215 » » » » // after doing "objbufpos++". | 1161 » » » » if(ptrbufpos+2 >= ptrbuf_end) |
| 1162 » » » » » flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wb
uf, &nobj); |
1216 if(objbufpos+2 >= objbuf_end) | 1163 if(objbufpos+2 >= objbuf_end) |
1217 » » » » » goto flush_buffers; | 1164 » » » » » flushobjbuf(objbuf, &objbufpos, &wp, &wb
uf, &nobj); |
1218 » » » » if(xbufpos+2 >= xbuf_end) | 1165 |
1219 » » » » » goto flush_buffers; | 1166 » » » » if(d.st != nil) |
1220 | |
1221 » » » » struct hash_gciter_data d; | |
1222 » » » » if(!hash_gciter_next(&map_iter, &d)) | |
1223 » » » » » break; | |
1224 | |
1225 » » » » if(d.st != nil) { | |
1226 » » » » » runtime·lock(&lock); | |
1227 markonly(d.st); | 1167 markonly(d.st); |
1228 runtime·unlock(&lock); | |
1229 } | |
1230 | 1168 |
1231 if(d.key_data != nil) { | 1169 if(d.key_data != nil) { |
1232 if(!(mapkey_kind & KindNoPointers) || d.
indirectkey) { | 1170 if(!(mapkey_kind & KindNoPointers) || d.
indirectkey) { |
1233 » » » » » » if(!d.indirectkey) { | 1171 » » » » » » if(!d.indirectkey) |
1234 » » » » » » » *objbufpos = (Obj){d.key
_data, mapkey_size, mapkey_gc}; | 1172 » » » » » » » *objbufpos++ = (Obj){d.k
ey_data, mapkey_size, mapkey_ti}; |
1235 » » » » » » » objbufpos++; | 1173 » » » » » » else { |
1236 » » » » » » } else { | 1174 » » » » » » » if(Debug) { |
1237 » » » » » » » *xbufpos = (struct X){*(
void**)d.key_data, mapkey_gc}; | 1175 » » » » » » » » obj = *(void**)d
.key_data; |
1238 » » » » » » » xbufpos++; | 1176 » » » » » » » » if(!(arena_start
<= obj && obj < arena_used)) |
| 1177 » » » » » » » » » runtime·
throw("scanblock: inconsistent hashmap"); |
| 1178 » » » » » » » } |
| 1179 » » » » » » » *ptrbufpos++ = (PtrTarge
t){*(void**)d.key_data, mapkey_ti}; |
1239 } | 1180 } |
1240 } | 1181 } |
1241 | |
1242 if(!(mapval_kind & KindNoPointers) || d.
indirectval) { | 1182 if(!(mapval_kind & KindNoPointers) || d.
indirectval) { |
1243 » » » » » » if(!d.indirectval) { | 1183 » » » » » » if(!d.indirectval) |
1244 » » » » » » » *objbufpos = (Obj){d.val
_data, mapval_size, mapval_gc}; | 1184 » » » » » » » *objbufpos++ = (Obj){d.v
al_data, mapval_size, mapval_ti}; |
1245 » » » » » » » objbufpos++; | 1185 » » » » » » else { |
1246 » » » » » » } else { | 1186 » » » » » » » if(Debug) { |
1247 » » » » » » » *xbufpos = (struct X){*(
void**)d.val_data, mapval_gc}; | 1187 » » » » » » » » obj = *(void**)d
.val_data; |
1248 » » » » » » » xbufpos++; | 1188 » » » » » » » » if(!(arena_start
<= obj && obj < arena_used)) |
| 1189 » » » » » » » » » runtime·
throw("scanblock: inconsistent hashmap"); |
| 1190 » » » » » » » } |
| 1191 » » » » » » » *ptrbufpos++ = (PtrTarge
t){*(void**)d.val_data, mapval_ti}; |
1249 } | 1192 } |
1250 } | 1193 } |
1251 } | 1194 } |
1252 | 1195 » » » } |
1253 » » » » if(DebugStats) | 1196 » » » if(map_ret == nil) |
1254 » » » » » runtime·xadd64(&gcstats.instr[GC_MAP_NEX
T], 1); | |
1255 » » » } | |
1256 » » » if(map_ret != 0) { | |
1257 » » » » pc = map_ret; | |
1258 » » » » goto next_instr; | |
1259 » » » } else { | |
1260 goto next_block; | 1197 goto next_block; |
1261 » » » } | 1198 » » » pc = map_ret; |
| 1199 » » » continue; |
| 1200 |
| 1201 » » case GC_REGION: |
| 1202 » » » obj = (void*)(stack_top.b + pc[1]); |
| 1203 » » » size = pc[2]; |
| 1204 » » » objti = pc[3]; |
| 1205 » » » pc += 4; |
| 1206 |
| 1207 » » » *objbufpos++ = (Obj){obj, size, objti}; |
| 1208 » » » if(objbufpos == objbuf_end) |
| 1209 » » » » flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nob
j); |
| 1210 » » » continue; |
| 1211 |
| 1212 » » case GC_CHAN_PTR: |
| 1213 » » » // Similar to GC_MAP_PTR |
| 1214 » » » chan = *(Hchan**)(stack_top.b + pc[1]); |
| 1215 » » » if(chan == nil) { |
| 1216 » » » » pc += 3; |
| 1217 » » » » continue; |
| 1218 » » » } |
| 1219 » » » if(markonly(chan)) { |
| 1220 » » » » chantype = (ChanType*)pc[2]; |
| 1221 » » » » if(!(chantype->elem->kind & KindNoPointers)) { |
| 1222 » » » » » // Start chanProg. |
| 1223 » » » » » chan_ret = pc+3; |
| 1224 » » » » » pc = chanProg+1; |
| 1225 » » » » » continue; |
| 1226 » » » » } |
| 1227 » » » } |
| 1228 » » » pc += 3; |
| 1229 » » » continue; |
| 1230 |
| 1231 » » case GC_CHAN: |
| 1232 » » » // There are no heap pointers in struct Hchan, |
| 1233 » » » // so we can ignore the leading sizeof(Hchan) bytes. |
| 1234 » » » if(!(chantype->elem->kind & KindNoPointers)) { |
| 1235 » » » » // Channel's buffer follows Hchan immediately in
memory. |
| 1236 » » » » // Size of buffer (cap(c)) is second int in the
chan struct. |
| 1237 » » » » n = ((uintgo*)chan)[1]; |
| 1238 » » » » if(n > 0) { |
| 1239 » » » » » // TODO(atom): split into two chunks so
that only the |
| 1240 » » » » » // in-use part of the circular buffer is
scanned. |
| 1241 » » » » » // (Channel routines zero the unused par
t, so the current |
| 1242 » » » » » // code does not lead to leaks, it's jus
t a little inefficient.) |
| 1243 » » » » » *objbufpos++ = (Obj){(byte*)chan+runtime
·Hchansize, n*chantype->elem->size, |
| 1244 » » » » » » (uintptr)chantype->elem->gc | PR
ECISE | LOOP}; |
| 1245 » » » » » if(objbufpos == objbuf_end) |
| 1246 » » » » » » flushobjbuf(objbuf, &objbufpos,
&wp, &wbuf, &nobj); |
| 1247 » » » » } |
| 1248 » » » } |
| 1249 » » » if(chan_ret == nil) |
| 1250 » » » » goto next_block; |
| 1251 » » » pc = chan_ret; |
| 1252 » » » continue; |
1262 | 1253 |
1263 default: | 1254 default: |
1264 » » » runtime·throw("scan: invalid GC instruction"); | 1255 » » » runtime·throw("scanblock: invalid GC instruction"); |
1265 return; | 1256 return; |
1266 } | 1257 } |
1267 | 1258 |
1268 » flush_buffers: | 1259 » » if(obj >= arena_start && obj < arena_used) { |
1269 » » flushxbuf(xbuf, xbufpos-xbuf, &wp, &wbuf, &nobj, ybuf); | 1260 » » » *ptrbufpos++ = (PtrTarget){obj, objti}; |
1270 » » xbufpos = xbuf; | 1261 » » » if(ptrbufpos == ptrbuf_end) |
1271 » » flushobjbuf(objbuf, objbufpos-objbuf, &wp, &wbuf, &nobj); | 1262 » » » » flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nob
j); |
1272 » » objbufpos = objbuf; | 1263 » » } |
1273 » » goto next_instr; | 1264 » } |
1274 | |
1275 » trailing_pointers: | |
1276 » » if(DebugType) { | |
1277 » » » if(i % PtrSize != 0) | |
1278 » » » » runtime·throw("scan: invalid alignment"); | |
1279 » » } | |
1280 | |
1281 » » // Quickly scan [b+i,b+n) for possible pointers. | |
1282 » » for(; i<=end_b; i+=PtrSize) { | |
1283 » » » if( *(byte**)i != nil ) { | |
1284 » » » » // Found a value that may be a pointer. | |
1285 » » » » // Do a rescan of the entire block, | |
1286 » » » » // and force the rescan to call runtime·gettype. | |
1287 » » » » if(DebugStats) { | |
1288 » » » » » runtime·xadd64(&gcstats.rescans, 1); | |
1289 » » » » » runtime·xadd64(&gcstats.rescan_nbytes, n
); | |
1290 » » » » } | |
1291 » » » » enqueue((Obj){b, n, 0}, &wp, &wbuf, &nobj); | |
1292 » » » » break; | |
1293 » » » } | |
1294 » » } | |
1295 | 1265 |
1296 next_block: | 1266 next_block: |
1297 // Done scanning [b, b+n). Prepare for the next iteration of | 1267 // Done scanning [b, b+n). Prepare for the next iteration of |
1298 » » // the loop by setting b, n and gctype to the parameters for the
next block. | 1268 » » // the loop by setting b, n, ti to the parameters for the next b
lock. |
1299 | 1269 |
1300 if(nobj == 0) { | 1270 if(nobj == 0) { |
1301 » » » flushxbuf(xbuf, xbufpos-xbuf, &wp, &wbuf, &nobj, ybuf); | 1271 » » » flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); |
1302 » » » xbufpos = xbuf; | 1272 » » » flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); |
1303 » » » flushobjbuf(objbuf, objbufpos-objbuf, &wp, &wbuf, &nobj)
; | |
1304 » » » objbufpos = objbuf; | |
1305 | 1273 |
1306 if(nobj == 0) { | 1274 if(nobj == 0) { |
1307 if(!keepworking) { | 1275 if(!keepworking) { |
1308 if(wbuf) | 1276 if(wbuf) |
1309 putempty(wbuf); | 1277 putempty(wbuf); |
1310 goto endscan; | 1278 goto endscan; |
1311 } | 1279 } |
1312 // Emptied our buffer: refill. | 1280 // Emptied our buffer: refill. |
1313 wbuf = getfull(wbuf); | 1281 wbuf = getfull(wbuf); |
1314 if(wbuf == nil) | 1282 if(wbuf == nil) |
1315 goto endscan; | 1283 goto endscan; |
1316 nobj = wbuf->nobj; | 1284 nobj = wbuf->nobj; |
1317 wp = wbuf->obj + wbuf->nobj; | 1285 wp = wbuf->obj + wbuf->nobj; |
1318 } | 1286 } |
1319 } | 1287 } |
1320 | 1288 |
1321 // Fetch b from the work buffer. | 1289 // Fetch b from the work buffer. |
1322 --wp; | 1290 --wp; |
1323 b = wp->p; | 1291 b = wp->p; |
1324 n = wp->n; | 1292 n = wp->n; |
1325 » » gc = wp->gc; | 1293 » » ti = wp->ti; |
1326 nobj--; | 1294 nobj--; |
1327 | 1295 » } |
1328 » » PREFETCH(b); | 1296 |
1329 » } | 1297 » for(;;) { |
1330 | 1298 » » // Each iteration scans the block b of length n, queueing pointe
rs in |
1331 endscan: | 1299 » » // the work buffer. |
1332 » runtime·lock(&lock); | 1300 » » if(Debug > 1) |
1333 » scanbuffers->next = bufferList; | 1301 » » » runtime·printf("scanblock %p %D\n", b, (int64)n); |
1334 » bufferList = scanbuffers; | 1302 |
1335 » runtime·unlock(&lock); | 1303 » » for(end_b = (uintptr)b + n - PtrSize; b <= (byte*)end_b; b += Pt
rSize) { |
1336 | 1304 » » » obj = *(byte**)b; |
1337 » // Update the state of buffer | 1305 » » » if(obj >= arena_start && obj < arena_used) { |
1338 » *_wp = wp; | 1306 » » » » *ptrbufpos++ = (PtrTarget){obj, 0}; |
1339 » *_wbuf = wbuf; | 1307 » » » » if(ptrbufpos == ptrbuf_end) |
1340 » *_nobj = nobj; | 1308 » » » » » flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wb
uf, &nobj); |
| 1309 » » » } |
| 1310 » » } |
| 1311 |
| 1312 » » // Done scanning [b, b+n). Prepare for the next iteration of |
| 1313 » » // the loop by setting b, n, ti to the parameters for the next b
lock. |
| 1314 |
| 1315 » next_block_conservative: |
| 1316 » » if(nobj == 0) { |
| 1317 » » » flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); |
| 1318 » » » if(nobj == 0) { |
| 1319 » » » » if(!keepworking) { |
| 1320 » » » » » if(wbuf) |
| 1321 » » » » » » putempty(wbuf); |
| 1322 » » » » » goto endscan; |
| 1323 » » » » } |
| 1324 » » » » // Emptied our buffer: refill. |
| 1325 » » » » wbuf = getfull(wbuf); |
| 1326 » » » » if(wbuf == nil) |
| 1327 » » » » » goto endscan; |
| 1328 » » » » nobj = wbuf->nobj; |
| 1329 » » » » wp = wbuf->obj + wbuf->nobj; |
| 1330 » » » } |
| 1331 » » } |
| 1332 |
| 1333 » » // Fetch b from the work buffer. |
| 1334 » » --wp; |
| 1335 » » b = wp->p; |
| 1336 » » n = wp->n; |
| 1337 » » nobj--; |
| 1338 » } |
| 1339 |
| 1340 endscan:; |
1341 } | 1341 } |
1342 | 1342 |
1343 // debug_scanblock is the debug copy of scanblock. | 1343 // debug_scanblock is the debug copy of scanblock. |
1344 // it is simpler, slower, more imprecise, single-threaded, recursive, | 1344 // it is simpler, slower, less precise, single-threaded, recursive, |
1345 // and uses bitSpecial as the mark bit. | 1345 // and uses bitSpecial as the mark bit. |
1346 static void | 1346 static void |
1347 debug_scanblock(byte *b, uintptr n) | 1347 debug_scanblock(byte *b, uintptr n) |
1348 { | 1348 { |
1349 byte *obj, *p; | 1349 byte *obj, *p; |
1350 void **vp; | 1350 void **vp; |
1351 uintptr size, *bitp, bits, shift, i, xbits, off; | 1351 uintptr size, *bitp, bits, shift, i, xbits, off; |
1352 MSpan *s; | 1352 MSpan *s; |
1353 | 1353 |
1354 if(!DebugMark) | 1354 if(!DebugMark) |
(...skipping 10 matching lines...) Expand all Loading... |
1365 b += PtrSize - off; | 1365 b += PtrSize - off; |
1366 n -= PtrSize - off; | 1366 n -= PtrSize - off; |
1367 } | 1367 } |
1368 | 1368 |
1369 vp = (void**)b; | 1369 vp = (void**)b; |
1370 n /= PtrSize; | 1370 n /= PtrSize; |
1371 for(i=0; i<n; i++) { | 1371 for(i=0; i<n; i++) { |
1372 obj = (byte*)vp[i]; | 1372 obj = (byte*)vp[i]; |
1373 | 1373 |
1374 // Words outside the arena cannot be pointers. | 1374 // Words outside the arena cannot be pointers. |
1375 » » if((byte*)obj < runtime·mheap.arena_start || (byte*)obj >= runti
me·mheap.arena_used) | 1375 » » if((byte*)obj < runtime·mheap->arena_start || (byte*)obj >= runt
ime·mheap->arena_used) |
1376 continue; | 1376 continue; |
1377 | 1377 |
1378 // Round down to word boundary. | 1378 // Round down to word boundary. |
1379 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); | 1379 obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); |
1380 | 1380 |
1381 // Consult span table to find beginning. | 1381 // Consult span table to find beginning. |
1382 » » s = runtime·MHeap_LookupMaybe(&runtime·mheap, obj); | 1382 » » s = runtime·MHeap_LookupMaybe(runtime·mheap, obj); |
1383 if(s == nil) | 1383 if(s == nil) |
1384 continue; | 1384 continue; |
1385 | 1385 |
1386 p = (byte*)((uintptr)s->start<<PageShift); | 1386 p = (byte*)((uintptr)s->start<<PageShift); |
1387 size = s->elemsize; | 1387 size = s->elemsize; |
1388 if(s->sizeclass == 0) { | 1388 if(s->sizeclass == 0) { |
1389 obj = p; | 1389 obj = p; |
1390 } else { | 1390 } else { |
1391 if((byte*)obj >= (byte*)s->limit) | 1391 if((byte*)obj >= (byte*)s->limit) |
1392 continue; | 1392 continue; |
1393 int32 i = ((byte*)obj - p)/size; | 1393 int32 i = ((byte*)obj - p)/size; |
1394 obj = p+i*size; | 1394 obj = p+i*size; |
1395 } | 1395 } |
1396 | 1396 |
1397 // Now that we know the object header, reload bits. | 1397 // Now that we know the object header, reload bits. |
1398 » » off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start; | 1398 » » off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start; |
1399 » » bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapW
ord - 1; | 1399 » » bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmap
Word - 1; |
1400 shift = off % wordsPerBitmapWord; | 1400 shift = off % wordsPerBitmapWord; |
1401 xbits = *bitp; | 1401 xbits = *bitp; |
1402 bits = xbits >> shift; | 1402 bits = xbits >> shift; |
1403 | 1403 |
1404 // Now we have bits, bitp, and shift correct for | 1404 // Now we have bits, bitp, and shift correct for |
1405 // obj pointing at the base of the object. | 1405 // obj pointing at the base of the object. |
1406 // If not allocated or already marked, done. | 1406 // If not allocated or already marked, done. |
1407 if((bits & bitAllocated) == 0 || (bits & bitSpecial) != 0) // N
OTE: bitSpecial not bitMarked | 1407 if((bits & bitAllocated) == 0 || (bits & bitSpecial) != 0) // N
OTE: bitSpecial not bitMarked |
1408 continue; | 1408 continue; |
1409 *bitp |= bitSpecial<<shift; | 1409 *bitp |= bitSpecial<<shift; |
1410 if(!(bits & bitMarked)) | 1410 if(!(bits & bitMarked)) |
1411 runtime·printf("found unmarked block %p in %p\n", obj, v
p+i); | 1411 runtime·printf("found unmarked block %p in %p\n", obj, v
p+i); |
1412 | 1412 |
1413 // If object has no pointers, don't need to scan further. | 1413 // If object has no pointers, don't need to scan further. |
1414 if((bits & bitNoPointers) != 0) | 1414 if((bits & bitNoPointers) != 0) |
1415 continue; | 1415 continue; |
1416 | 1416 |
1417 debug_scanblock(obj, size); | 1417 debug_scanblock(obj, size); |
1418 } | 1418 } |
1419 } | 1419 } |
1420 | 1420 |
| 1421 // Append obj to the work buffer. |
| 1422 // _wbuf, _wp, _nobj are input/output parameters and are specifying the work buf
fer. |
| 1423 static void |
| 1424 enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj) |
| 1425 { |
| 1426 uintptr nobj, off; |
| 1427 Obj *wp; |
| 1428 Workbuf *wbuf; |
| 1429 |
| 1430 if(Debug > 1) |
| 1431 runtime·printf("append obj(%p %D %p)\n", obj.p, (int64)obj.n, ob
j.ti); |
| 1432 |
| 1433 // Align obj.b to a word boundary. |
| 1434 off = (uintptr)obj.p & (PtrSize-1); |
| 1435 if(off != 0) { |
| 1436 obj.p += PtrSize - off; |
| 1437 obj.n -= PtrSize - off; |
| 1438 obj.ti = 0; |
| 1439 } |
| 1440 |
| 1441 if(obj.p == nil || obj.n == 0) |
| 1442 return; |
| 1443 |
| 1444 // Load work buffer state |
| 1445 wp = *_wp; |
| 1446 wbuf = *_wbuf; |
| 1447 nobj = *_nobj; |
| 1448 |
| 1449 // If another proc wants a pointer, give it some. |
| 1450 if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) { |
| 1451 wbuf->nobj = nobj; |
| 1452 wbuf = handoff(wbuf); |
| 1453 nobj = wbuf->nobj; |
| 1454 wp = wbuf->obj + nobj; |
| 1455 } |
| 1456 |
| 1457 // If buffer is full, get a new one. |
| 1458 if(wbuf == nil || nobj >= nelem(wbuf->obj)) { |
| 1459 if(wbuf != nil) |
| 1460 wbuf->nobj = nobj; |
| 1461 wbuf = getempty(wbuf); |
| 1462 wp = wbuf->obj; |
| 1463 nobj = 0; |
| 1464 } |
| 1465 |
| 1466 *wp = obj; |
| 1467 wp++; |
| 1468 nobj++; |
| 1469 |
| 1470 // Save work buffer state |
| 1471 *_wp = wp; |
| 1472 *_wbuf = wbuf; |
| 1473 *_nobj = nobj; |
| 1474 } |
| 1475 |
1421 static void | 1476 static void |
1422 markroot(ParFor *desc, uint32 i) | 1477 markroot(ParFor *desc, uint32 i) |
1423 { | 1478 { |
1424 Obj *wp; | 1479 Obj *wp; |
1425 Workbuf *wbuf; | 1480 Workbuf *wbuf; |
1426 uintptr nobj; | 1481 uintptr nobj; |
1427 | 1482 |
1428 USED(&desc); | 1483 USED(&desc); |
1429 wp = nil; | 1484 wp = nil; |
1430 wbuf = nil; | 1485 wbuf = nil; |
1431 nobj = 0; | 1486 nobj = 0; |
1432 » enqueue(work.roots[i], &wp, &wbuf, &nobj); | 1487 » enqueue(work.roots[i], &wbuf, &wp, &nobj); |
1433 » scan(&wp, &wbuf, &nobj, false); | 1488 » scanblock(wbuf, wp, nobj, false); |
1434 } | 1489 } |
1435 | 1490 |
1436 // Get an empty work buffer off the work.empty list, | 1491 // Get an empty work buffer off the work.empty list, |
1437 // allocating new buffers as needed. | 1492 // allocating new buffers as needed. |
1438 static Workbuf* | 1493 static Workbuf* |
1439 getempty(Workbuf *b) | 1494 getempty(Workbuf *b) |
1440 { | 1495 { |
1441 if(b != nil) | 1496 if(b != nil) |
1442 runtime·lfstackpush(&work.full, &b->node); | 1497 runtime·lfstackpush(&work.full, &b->node); |
1443 b = (Workbuf*)runtime·lfstackpop(&work.empty); | 1498 b = (Workbuf*)runtime·lfstackpop(&work.empty); |
1444 if(b == nil) { | 1499 if(b == nil) { |
1445 // Need to allocate. | 1500 // Need to allocate. |
1446 runtime·lock(&work); | 1501 runtime·lock(&work); |
1447 if(work.nchunk < sizeof *b) { | 1502 if(work.nchunk < sizeof *b) { |
1448 work.nchunk = 1<<20; | 1503 work.nchunk = 1<<20; |
1449 work.chunk = runtime·SysAlloc(work.nchunk); | 1504 work.chunk = runtime·SysAlloc(work.nchunk); |
| 1505 if(work.chunk == nil) |
| 1506 runtime·throw("runtime: cannot allocate memory")
; |
1450 } | 1507 } |
1451 b = (Workbuf*)work.chunk; | 1508 b = (Workbuf*)work.chunk; |
1452 work.chunk += sizeof *b; | 1509 work.chunk += sizeof *b; |
1453 work.nchunk -= sizeof *b; | 1510 work.nchunk -= sizeof *b; |
1454 runtime·unlock(&work); | 1511 runtime·unlock(&work); |
1455 } | 1512 } |
1456 | |
1457 b->nobj = 0; | 1513 b->nobj = 0; |
1458 return b; | 1514 return b; |
1459 } | 1515 } |
1460 | 1516 |
1461 static void | 1517 static void |
1462 putempty(Workbuf *b) | 1518 putempty(Workbuf *b) |
1463 { | 1519 { |
1464 » if(DebugStats) | 1520 » if(CollectStats) |
1465 runtime·xadd64(&gcstats.putempty, 1); | 1521 runtime·xadd64(&gcstats.putempty, 1); |
1466 | 1522 |
1467 runtime·lfstackpush(&work.empty, &b->node); | 1523 runtime·lfstackpush(&work.empty, &b->node); |
1468 } | 1524 } |
1469 | 1525 |
1470 // Get a full work buffer off the work.full list, or return nil. | 1526 // Get a full work buffer off the work.full list, or return nil. |
1471 static Workbuf* | 1527 static Workbuf* |
1472 getfull(Workbuf *b) | 1528 getfull(Workbuf *b) |
1473 { | 1529 { |
1474 int32 i; | 1530 int32 i; |
1475 | 1531 |
1476 » if(DebugStats) | 1532 » if(CollectStats) |
1477 runtime·xadd64(&gcstats.getfull, 1); | 1533 runtime·xadd64(&gcstats.getfull, 1); |
1478 | 1534 |
1479 if(b != nil) | 1535 if(b != nil) |
1480 runtime·lfstackpush(&work.empty, &b->node); | 1536 runtime·lfstackpush(&work.empty, &b->node); |
1481 b = (Workbuf*)runtime·lfstackpop(&work.full); | 1537 b = (Workbuf*)runtime·lfstackpop(&work.full); |
1482 if(b != nil || work.nproc == 1) | 1538 if(b != nil || work.nproc == 1) |
1483 return b; | 1539 return b; |
1484 | 1540 |
1485 runtime·xadd(&work.nwait, +1); | 1541 runtime·xadd(&work.nwait, +1); |
1486 for(i=0;; i++) { | 1542 for(i=0;; i++) { |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1523 if(i < b->nobj) { | 1579 if(i < b->nobj) { |
1524 b->obj[n++] = b->obj[i]; | 1580 b->obj[n++] = b->obj[i]; |
1525 } | 1581 } |
1526 b->nobj = n; | 1582 b->nobj = n; |
1527 b1->nobj = n1; | 1583 b1->nobj = n1; |
1528 m->gcstats.nhandoff++; | 1584 m->gcstats.nhandoff++; |
1529 m->gcstats.nhandoffcnt += n; | 1585 m->gcstats.nhandoffcnt += n; |
1530 | 1586 |
1531 // Put b on full list - let first half of b get stolen. | 1587 // Put b on full list - let first half of b get stolen. |
1532 runtime·lfstackpush(&work.full, &b->node); | 1588 runtime·lfstackpush(&work.full, &b->node); |
1533 | |
1534 return b1; | 1589 return b1; |
1535 } | 1590 } |
1536 | 1591 |
1537 static void | 1592 static void |
1538 addroot(Obj obj) | 1593 addroot(Obj obj) |
1539 { | 1594 { |
1540 uint32 cap; | 1595 uint32 cap; |
1541 Obj *new; | 1596 Obj *new; |
1542 | 1597 |
1543 if(work.nroot >= work.rootcap) { | 1598 if(work.nroot >= work.rootcap) { |
1544 cap = PageSize/sizeof(Obj); | 1599 cap = PageSize/sizeof(Obj); |
1545 if(cap < 2*work.rootcap) | 1600 if(cap < 2*work.rootcap) |
1546 cap = 2*work.rootcap; | 1601 cap = 2*work.rootcap; |
1547 new = (Obj*)runtime·SysAlloc(cap*sizeof(Obj)); | 1602 new = (Obj*)runtime·SysAlloc(cap*sizeof(Obj)); |
| 1603 if(new == nil) |
| 1604 runtime·throw("runtime: cannot allocate memory"); |
1548 if(work.roots != nil) { | 1605 if(work.roots != nil) { |
1549 runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj
)); | 1606 runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj
)); |
1550 runtime·SysFree(work.roots, work.rootcap*sizeof(Obj)); | 1607 runtime·SysFree(work.roots, work.rootcap*sizeof(Obj)); |
1551 } | 1608 } |
1552 work.roots = new; | 1609 work.roots = new; |
1553 work.rootcap = cap; | 1610 work.rootcap = cap; |
1554 } | 1611 } |
1555 work.roots[work.nroot] = obj; | 1612 work.roots[work.nroot] = obj; |
1556 work.nroot++; | 1613 work.nroot++; |
1557 } | 1614 } |
1558 | 1615 |
| 1616 // Scan a stack frame. The doframe parameter is a signal that the previously |
| 1617 // scanned activation has an unknown argument size. When *doframe is true the |
| 1618 // current activation must have its entire frame scanned. Otherwise, only the |
| 1619 // locals need to be scanned. |
| 1620 static void |
| 1621 addframeroots(Func *f, byte*, byte *sp, void *doframe) |
| 1622 { |
| 1623 uintptr outs; |
| 1624 |
| 1625 if(thechar == '5') |
| 1626 sp += sizeof(uintptr); |
| 1627 if(f->locals == 0 || *(bool*)doframe == true) |
| 1628 addroot((Obj){sp, f->frame - sizeof(uintptr), 0}); |
| 1629 else if(f->locals > 0) { |
| 1630 outs = f->frame - sizeof(uintptr) - f->locals; |
| 1631 addroot((Obj){sp + outs, f->locals, 0}); |
| 1632 } |
| 1633 if(f->args > 0) |
| 1634 addroot((Obj){sp + f->frame, f->args, 0}); |
| 1635 *(bool*)doframe = (f->args == ArgsSizeUnknown); |
| 1636 } |
| 1637 |
1559 static void | 1638 static void |
1560 addstackroots(G *gp) | 1639 addstackroots(G *gp) |
1561 { | 1640 { |
1562 M *mp; | 1641 M *mp; |
1563 int32 n; | 1642 int32 n; |
1564 Stktop *stk; | 1643 Stktop *stk; |
1565 » byte *sp, *guard; | 1644 » byte *sp, *guard, *pc; |
| 1645 » Func *f; |
| 1646 » bool doframe; |
1566 | 1647 |
1567 stk = (Stktop*)gp->stackbase; | 1648 stk = (Stktop*)gp->stackbase; |
1568 guard = (byte*)gp->stackguard; | 1649 guard = (byte*)gp->stackguard; |
1569 | 1650 |
1570 if(gp == g) { | 1651 if(gp == g) { |
1571 // Scanning our own stack: start at &gp. | 1652 // Scanning our own stack: start at &gp. |
1572 » » sp = (byte*)&gp; | 1653 » » sp = runtime·getcallersp(&gp); |
| 1654 » » pc = runtime·getcallerpc(&gp); |
1573 } else if((mp = gp->m) != nil && mp->helpgc) { | 1655 } else if((mp = gp->m) != nil && mp->helpgc) { |
1574 // gchelper's stack is in active use and has no interesting poin
ters. | 1656 // gchelper's stack is in active use and has no interesting poin
ters. |
1575 return; | 1657 return; |
| 1658 } else if(gp->gcstack != (uintptr)nil) { |
| 1659 // Scanning another goroutine that is about to enter or might |
| 1660 // have just exited a system call. It may be executing code such |
| 1661 // as schedlock and may have needed to start a new stack segment
. |
| 1662 // Use the stack segment and stack pointer at the time of |
| 1663 // the system call instead, since that won't change underfoot. |
| 1664 sp = (byte*)gp->gcsp; |
| 1665 pc = gp->gcpc; |
| 1666 stk = (Stktop*)gp->gcstack; |
| 1667 guard = (byte*)gp->gcguard; |
1576 } else { | 1668 } else { |
1577 // Scanning another goroutine's stack. | 1669 // Scanning another goroutine's stack. |
1578 // The goroutine is usually asleep (the world is stopped). | 1670 // The goroutine is usually asleep (the world is stopped). |
1579 sp = (byte*)gp->sched.sp; | 1671 sp = (byte*)gp->sched.sp; |
1580 | 1672 » » pc = gp->sched.pc; |
1581 » » // The exception is that if the goroutine is about to enter or m
ight | 1673 » » if(ScanStackByFrames && pc == (byte*)runtime·goexit && gp->fnsta
rt != nil) { |
1582 » » // have just exited a system call, it may be executing code such | 1674 » » » // The goroutine has not started. However, its incoming |
1583 » » // as schedlock and may have needed to start a new stack segment
. | 1675 » » » // arguments are live at the top of the stack and must |
1584 » » // Use the stack segment and stack pointer at the time of | 1676 » » » // be scanned. No other live values should be on the |
1585 » » // the system call instead, since that won't change underfoot. | 1677 » » » // stack. |
1586 » » if(gp->gcstack != (uintptr)nil) { | 1678 » » » f = runtime·findfunc((uintptr)gp->fnstart->fn); |
1587 » » » stk = (Stktop*)gp->gcstack; | 1679 » » » if(f->args != 0) { |
1588 » » » sp = (byte*)gp->gcsp; | 1680 » » » » if(thechar == '5') |
1589 » » » guard = (byte*)gp->gcguard; | 1681 » » » » » sp += sizeof(uintptr); |
1590 » » } | 1682 » » » » // If the size of the arguments is known |
1591 » } | 1683 » » » » // scan just the incoming arguments. |
1592 | 1684 » » » » // Otherwise, scan everything between the |
1593 » n = 0; | 1685 » » » » // top and the bottom of the stack. |
1594 » while(stk) { | 1686 » » » » if(f->args > 0) |
1595 » » if(sp < guard-StackGuard || (byte*)stk < sp) { | 1687 » » » » » addroot((Obj){sp, f->args, 0}); |
1596 » » » runtime·printf("addstackroots inconsistent: g%d#%d sp=%p
not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk); | 1688 » » » » else |
1597 » » » runtime·throw("addstackroots"); | 1689 » » » » » addroot((Obj){sp, (byte*)stk - sp, 0});· |
1598 » » } | 1690 » » » }· |
1599 » » if(!markonly(sp)) | 1691 » » » return; |
1600 » » » addroot((Obj){sp, (byte*)stk - sp, 0}); | 1692 » » } |
1601 » » sp = (byte*)stk->gobuf.sp; | 1693 » } |
1602 » » guard = stk->stackguard; | 1694 |
1603 » » stk = (Stktop*)stk->stackbase; | 1695 » if (ScanStackByFrames) { |
1604 » » n++; | 1696 » » USED(stk); |
1605 » } | 1697 » » USED(guard); |
1606 } | 1698 » » doframe = false; |
1607 | 1699 » » runtime·gentraceback(pc, sp, nil, gp, 0, nil, 0x7fffffff, addfra
meroots, &doframe); |
1608 // Markfin calls scanblock on the blocks that have finalizers: | 1700 » } else { |
1609 // the things pointed at cannot be freed until the finalizers have run. | 1701 » » USED(pc); |
| 1702 » » n = 0; |
| 1703 » » while(stk) { |
| 1704 » » » if(sp < guard-StackGuard || (byte*)stk < sp) { |
| 1705 » » » » runtime·printf("addstackroots inconsistent: g%D#
%d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk); |
| 1706 » » » » runtime·throw("addstackroots"); |
| 1707 » » » » runtime·printf("scanstack inconsistent: g%D#%d s
p=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk); |
| 1708 » » » » runtime·throw("scanstack"); |
| 1709 » » » } |
| 1710 » » » addroot((Obj){sp, (byte*)stk - sp, (uintptr)defaultProg
| PRECISE | LOOP}); |
| 1711 » » » sp = (byte*)stk->gobuf.sp; |
| 1712 » » » guard = stk->stackguard; |
| 1713 » » » stk = (Stktop*)stk->stackbase; |
| 1714 » » » n++; |
| 1715 » » } |
| 1716 » } |
| 1717 } |
| 1718 |
1610 static void | 1719 static void |
1611 addfinroots(void *v) | 1720 addfinroots(void *v) |
1612 { | 1721 { |
1613 uintptr size; | 1722 uintptr size; |
| 1723 void *base; |
1614 | 1724 |
1615 size = 0; | 1725 size = 0; |
1616 » if(!runtime·mlookup(v, &v, &size, nil) || !runtime·blockspecial(v)) | 1726 » if(!runtime·mlookup(v, &base, &size, nil) || !runtime·blockspecial(base)
) |
1617 runtime·throw("mark - finalizer inconsistency"); | 1727 runtime·throw("mark - finalizer inconsistency"); |
1618 | 1728 |
1619 // do not mark the finalizer block itself. just mark the things it poin
ts at. | 1729 // do not mark the finalizer block itself. just mark the things it poin
ts at. |
1620 » addroot((Obj){v, size, 0}); | 1730 » addroot((Obj){base, size, 0}); |
1621 } | 1731 } |
1622 | 1732 |
1623 static void | 1733 static void |
1624 addroots(void) | 1734 addroots(void) |
1625 { | 1735 { |
1626 G *gp; | 1736 G *gp; |
1627 FinBlock *fb; | 1737 FinBlock *fb; |
1628 MSpan *s, **allspans; | 1738 MSpan *s, **allspans; |
1629 uint32 spanidx; | 1739 uint32 spanidx; |
1630 | 1740 |
1631 work.nroot = 0; | 1741 work.nroot = 0; |
1632 | 1742 |
1633 // data & bss | 1743 // data & bss |
| 1744 // TODO(atom): load balancing |
1634 addroot((Obj){data, edata - data, (uintptr)gcdata}); | 1745 addroot((Obj){data, edata - data, (uintptr)gcdata}); |
1635 addroot((Obj){bss, ebss - bss, (uintptr)gcbss}); | 1746 addroot((Obj){bss, ebss - bss, (uintptr)gcbss}); |
1636 | 1747 |
1637 // MSpan.types | 1748 // MSpan.types |
1638 » allspans = runtime·mheap.allspans; | 1749 » allspans = runtime·mheap->allspans; |
1639 » for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) { | 1750 » for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) { |
1640 s = allspans[spanidx]; | 1751 s = allspans[spanidx]; |
1641 if(s->state == MSpanInUse) { | 1752 if(s->state == MSpanInUse) { |
| 1753 // The garbage collector ignores type pointers stored in
MSpan.types: |
| 1754 // - Compiler-generated types are stored outside of hea
p. |
| 1755 // - The reflect package has runtime-generated types ca
ched in its data structures. |
| 1756 // The garbage collector relies on finding the refere
nces via that cache. |
1642 switch(s->types.compression) { | 1757 switch(s->types.compression) { |
1643 » » » case 0: | 1758 » » » case MTypes_Empty: |
1644 » » » case 1: | 1759 » » » case MTypes_Single: |
1645 break; | 1760 break; |
1646 » » » case 2: | 1761 » » » case MTypes_Words: |
1647 » » » case 3: | 1762 » » » case MTypes_Bytes: |
1648 » » » » if(!markonly((byte*)&s->types.data)) | 1763 » » » » markonly((byte*)s->types.data); |
1649 » » » » » addroot((Obj){(byte*)&s->types.data, siz
eof(void*), 0}); | |
1650 break; | 1764 break; |
1651 } | 1765 } |
1652 } | 1766 } |
1653 » }» | 1767 » } |
1654 | 1768 |
1655 // stacks | 1769 // stacks |
1656 for(gp=runtime·allg; gp!=nil; gp=gp->alllink) { | 1770 for(gp=runtime·allg; gp!=nil; gp=gp->alllink) { |
1657 switch(gp->status){ | 1771 switch(gp->status){ |
1658 default: | 1772 default: |
1659 runtime·printf("unexpected G.status %d\n", gp->status); | 1773 runtime·printf("unexpected G.status %d\n", gp->status); |
1660 runtime·throw("mark - bad status"); | 1774 runtime·throw("mark - bad status"); |
1661 case Gdead: | 1775 case Gdead: |
1662 break; | 1776 break; |
1663 case Grunning: | 1777 case Grunning: |
1664 if(gp != g) | 1778 if(gp != g) |
1665 runtime·throw("mark - world not stopped"); | 1779 runtime·throw("mark - world not stopped"); |
1666 addstackroots(gp); | 1780 addstackroots(gp); |
1667 break; | 1781 break; |
1668 case Grunnable: | 1782 case Grunnable: |
1669 case Gsyscall: | 1783 case Gsyscall: |
1670 case Gwaiting: | 1784 case Gwaiting: |
1671 addstackroots(gp); | 1785 addstackroots(gp); |
1672 break; | 1786 break; |
1673 } | 1787 } |
1674 } | 1788 } |
1675 | 1789 |
1676 // add things pointed at by objects with finalizers | |
1677 runtime·walkfintab(addfinroots); | 1790 runtime·walkfintab(addfinroots); |
1678 | 1791 |
1679 for(fb=allfin; fb; fb=fb->alllink) | 1792 for(fb=allfin; fb; fb=fb->alllink) |
1680 addroot((Obj){(byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), 0}); | 1793 addroot((Obj){(byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), 0}); |
1681 } | 1794 } |
1682 | 1795 |
1683 static bool | 1796 static bool |
1684 handlespecial(byte *p, uintptr size) | 1797 handlespecial(byte *p, uintptr size) |
1685 { | 1798 { |
1686 » void (*fn)(void*); | 1799 » FuncVal *fn; |
1687 » int32 nret; | 1800 » uintptr nret; |
1688 FinBlock *block; | 1801 FinBlock *block; |
1689 Finalizer *f; | 1802 Finalizer *f; |
1690 | 1803 |
1691 if(!runtime·getfinalizer(p, true, &fn, &nret)) { | 1804 if(!runtime·getfinalizer(p, true, &fn, &nret)) { |
1692 runtime·setblockspecial(p, false); | 1805 runtime·setblockspecial(p, false); |
1693 runtime·MProf_Free(p, size); | 1806 runtime·MProf_Free(p, size); |
1694 return false; | 1807 return false; |
1695 } | 1808 } |
1696 | 1809 |
1697 runtime·lock(&finlock); | 1810 runtime·lock(&finlock); |
1698 if(finq == nil || finq->cnt == finq->cap) { | 1811 if(finq == nil || finq->cnt == finq->cap) { |
1699 if(finc == nil) { | 1812 if(finc == nil) { |
1700 finc = runtime·SysAlloc(PageSize); | 1813 finc = runtime·SysAlloc(PageSize); |
| 1814 if(finc == nil) |
| 1815 runtime·throw("runtime: cannot allocate memory")
; |
1701 finc->cap = (PageSize - sizeof(FinBlock)) / sizeof(Final
izer) + 1; | 1816 finc->cap = (PageSize - sizeof(FinBlock)) / sizeof(Final
izer) + 1; |
1702 finc->alllink = allfin; | 1817 finc->alllink = allfin; |
1703 allfin = finc; | 1818 allfin = finc; |
1704 } | 1819 } |
1705 block = finc; | 1820 block = finc; |
1706 finc = block->next; | 1821 finc = block->next; |
1707 block->next = finq; | 1822 block->next = finq; |
1708 finq = block; | 1823 finq = block; |
1709 } | 1824 } |
1710 f = &finq->fin[finq->cnt]; | 1825 f = &finq->fin[finq->cnt]; |
1711 finq->cnt++; | 1826 finq->cnt++; |
1712 f->fn = fn; | 1827 f->fn = fn; |
1713 f->nret = nret; | 1828 f->nret = nret; |
1714 f->arg = p; | 1829 f->arg = p; |
1715 runtime·unlock(&finlock); | 1830 runtime·unlock(&finlock); |
1716 return true; | 1831 return true; |
1717 } | 1832 } |
1718 | 1833 |
1719 // Sweep frees or collects finalizers for blocks not marked in the mark phase. | 1834 // Sweep frees or collects finalizers for blocks not marked in the mark phase. |
1720 // It clears the mark bits in preparation for the next GC round. | 1835 // It clears the mark bits in preparation for the next GC round. |
1721 static void | 1836 static void |
1722 sweepspan(ParFor *desc, uint32 idx) | 1837 sweepspan(ParFor *desc, uint32 idx) |
1723 { | 1838 { |
1724 » int32 sizeclass, n, npages; | 1839 » int32 cl, n, npages; |
1725 » uintptr size; | 1840 » uintptr size, u; |
1726 byte *p; | 1841 byte *p; |
1727 MCache *c; | 1842 MCache *c; |
1728 byte *arena_start; | 1843 byte *arena_start; |
1729 MLink head, *end; | 1844 MLink head, *end; |
1730 int32 nfree; | 1845 int32 nfree; |
1731 byte *type_data; | 1846 byte *type_data; |
1732 byte compression; | 1847 byte compression; |
1733 uintptr type_data_inc; | 1848 uintptr type_data_inc; |
1734 MSpan *s; | 1849 MSpan *s; |
1735 | 1850 |
1736 USED(&desc); | 1851 USED(&desc); |
1737 » s = runtime·mheap.allspans[idx]; | 1852 » s = runtime·mheap->allspans[idx]; |
1738 » // Stamp newly unused spans. The scavenger will use that | |
1739 » // info to potentially give back some pages to the OS. | |
1740 » if(s->state == MSpanFree && s->unusedsince == 0) | |
1741 » » s->unusedsince = runtime·nanotime(); | |
1742 if(s->state != MSpanInUse) | 1853 if(s->state != MSpanInUse) |
1743 return; | 1854 return; |
1744 » arena_start = runtime·mheap.arena_start; | 1855 » arena_start = runtime·mheap->arena_start; |
1745 p = (byte*)(s->start << PageShift); | 1856 p = (byte*)(s->start << PageShift); |
1746 » sizeclass = s->sizeclass; | 1857 » cl = s->sizeclass; |
1747 size = s->elemsize; | 1858 size = s->elemsize; |
1748 » if(sizeclass == 0) { | 1859 » if(cl == 0) { |
1749 n = 1; | 1860 n = 1; |
1750 } else { | 1861 } else { |
1751 // Chunk full of small blocks. | 1862 // Chunk full of small blocks. |
1752 » » npages = runtime·class_to_allocnpages[sizeclass]; | 1863 » » npages = runtime·class_to_allocnpages[cl]; |
1753 n = (npages << PageShift) / size; | 1864 n = (npages << PageShift) / size; |
1754 } | 1865 } |
1755 nfree = 0; | 1866 nfree = 0; |
1756 end = &head; | 1867 end = &head; |
1757 c = m->mcache; | 1868 c = m->mcache; |
1758 ········ | 1869 ········ |
1759 type_data = (byte*)s->types.data; | 1870 type_data = (byte*)s->types.data; |
1760 type_data_inc = sizeof(uintptr); | 1871 type_data_inc = sizeof(uintptr); |
1761 compression = s->types.compression; | 1872 compression = s->types.compression; |
1762 switch(compression) { | 1873 switch(compression) { |
1763 » case 3: | 1874 » case MTypes_Bytes: |
1764 type_data += 8*sizeof(uintptr); | 1875 type_data += 8*sizeof(uintptr); |
1765 type_data_inc = 1; | 1876 type_data_inc = 1; |
1766 break; | 1877 break; |
1767 } | 1878 } |
1768 | 1879 |
1769 // Sweep through n objects of given size starting at p. | 1880 // Sweep through n objects of given size starting at p. |
1770 // This thread owns the span now, so it can manipulate | 1881 // This thread owns the span now, so it can manipulate |
1771 // the block bitmap without atomic operations. | 1882 // the block bitmap without atomic operations. |
1772 » for(; n>0; n--, p+=size, type_data+=type_data_inc) { | 1883 » for(; n > 0; n--, p += size, type_data+=type_data_inc) { |
1773 uintptr off, *bitp, shift, bits; | 1884 uintptr off, *bitp, shift, bits; |
1774 | 1885 |
1775 off = (uintptr*)p - (uintptr*)arena_start; | 1886 off = (uintptr*)p - (uintptr*)arena_start; |
1776 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; | 1887 bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; |
1777 shift = off % wordsPerBitmapWord; | 1888 shift = off % wordsPerBitmapWord; |
1778 bits = *bitp>>shift; | 1889 bits = *bitp>>shift; |
1779 | 1890 |
1780 if((bits & bitAllocated) == 0) | 1891 if((bits & bitAllocated) == 0) |
1781 continue; | 1892 continue; |
1782 | 1893 |
1783 if((bits & bitMarked) != 0) { | 1894 if((bits & bitMarked) != 0) { |
1784 if(DebugMark) { | 1895 if(DebugMark) { |
1785 if(!(bits & bitSpecial)) | 1896 if(!(bits & bitSpecial)) |
1786 runtime·printf("found spurious mark on %
p\n", p); | 1897 runtime·printf("found spurious mark on %
p\n", p); |
1787 *bitp &= ~(bitSpecial<<shift); | 1898 *bitp &= ~(bitSpecial<<shift); |
1788 } | 1899 } |
1789 *bitp &= ~(bitMarked<<shift); | 1900 *bitp &= ~(bitMarked<<shift); |
1790 continue; | 1901 continue; |
1791 } | 1902 } |
1792 | 1903 |
1793 // Special means it has a finalizer or is being profiled. | 1904 // Special means it has a finalizer or is being profiled. |
1794 // In DebugMark mode, the bit has been coopted so | 1905 // In DebugMark mode, the bit has been coopted so |
1795 // we have to assume all blocks are special. | 1906 // we have to assume all blocks are special. |
1796 if(DebugMark || (bits & bitSpecial) != 0) { | 1907 if(DebugMark || (bits & bitSpecial) != 0) { |
1797 if(handlespecial(p, size)) | 1908 if(handlespecial(p, size)) |
1798 continue; | 1909 continue; |
1799 } | 1910 } |
1800 | 1911 |
| 1912 // Scramble contents of the block. Don't free the block. |
| 1913 if(MemScramble) { |
| 1914 for(u = 0; u<size; u+=sizeof(void*)) { |
| 1915 // Use a number that can be used to reconstruct
the address p+u. |
| 1916 // The number is (hopefully) an invalid address. |
| 1917 *(uintptr*)(p+u) = -(uintptr)(p+u); |
| 1918 } |
| 1919 if(0) runtime·printf("free %p\n", p); |
| 1920 *bitp |= (bitSpecial<<shift); |
| 1921 continue; |
| 1922 } |
| 1923 |
1801 // Mark freed; restore block boundary bit. | 1924 // Mark freed; restore block boundary bit. |
1802 *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 1925 *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
1803 | 1926 |
1804 » » if(sizeclass == 0) { | 1927 » » if(cl == 0) { |
1805 // Free large span. | 1928 // Free large span. |
1806 runtime·unmarkspan(p, 1<<PageShift); | 1929 runtime·unmarkspan(p, 1<<PageShift); |
1807 » » » *(uintptr*)p = 1;» // needs zeroing | 1930 » » » *(uintptr*)p = (uintptr)0xdeaddeaddeaddeadll;» // needs
zeroing |
1808 » » » runtime·MHeap_Free(&runtime·mheap, s, 1); | 1931 » » » runtime·MHeap_Free(runtime·mheap, s, 1); |
1809 c->local_alloc -= size; | 1932 c->local_alloc -= size; |
1810 c->local_nfree++; | 1933 c->local_nfree++; |
1811 } else { | 1934 } else { |
1812 // Free small object. | 1935 // Free small object. |
1813 switch(compression) { | 1936 switch(compression) { |
1814 » » » case 2: | 1937 » » » case MTypes_Words: |
1815 *(uintptr*)type_data = 0; | 1938 *(uintptr*)type_data = 0; |
1816 break; | 1939 break; |
1817 » » » case 3: | 1940 » » » case MTypes_Bytes: |
1818 *(byte*)type_data = 0; | 1941 *(byte*)type_data = 0; |
1819 break; | 1942 break; |
1820 } | 1943 } |
1821 if(size > sizeof(uintptr)) | 1944 if(size > sizeof(uintptr)) |
1822 » » » » ((uintptr*)p)[1] = 1;» // mark as "needs to be
zeroed" | 1945 » » » » ((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll
;» // mark as "needs to be zeroed" |
1823 ························ | 1946 ························ |
1824 end->next = (MLink*)p; | 1947 end->next = (MLink*)p; |
1825 end = (MLink*)p; | 1948 end = (MLink*)p; |
1826 nfree++; | 1949 nfree++; |
1827 } | 1950 } |
1828 } | 1951 } |
1829 | 1952 |
1830 if(nfree) { | 1953 if(nfree) { |
1831 » » c->local_by_size[sizeclass].nfree += nfree; | 1954 » » c->local_by_size[cl].nfree += nfree; |
1832 c->local_alloc -= size * nfree; | 1955 c->local_alloc -= size * nfree; |
1833 c->local_nfree += nfree; | 1956 c->local_nfree += nfree; |
1834 c->local_cachealloc -= nfree * size; | 1957 c->local_cachealloc -= nfree * size; |
1835 c->local_objects -= nfree; | 1958 c->local_objects -= nfree; |
1836 » » runtime·MCentral_FreeSpan(&runtime·mheap.central[sizeclass], s,
nfree, head.next, end); | 1959 » » runtime·MCentral_FreeSpan(&runtime·mheap->central[cl], s, nfree,
head.next, end); |
| 1960 » } |
| 1961 } |
| 1962 |
| 1963 static void |
| 1964 dumpspan(uint32 idx) |
| 1965 { |
| 1966 » int32 sizeclass, n, npages, i, column; |
| 1967 » uintptr size; |
| 1968 » byte *p; |
| 1969 » byte *arena_start; |
| 1970 » MSpan *s; |
| 1971 » bool allocated, special; |
| 1972 |
| 1973 » s = runtime·mheap->allspans[idx]; |
| 1974 » if(s->state != MSpanInUse) |
| 1975 » » return; |
| 1976 » arena_start = runtime·mheap->arena_start; |
| 1977 » p = (byte*)(s->start << PageShift); |
| 1978 » sizeclass = s->sizeclass; |
| 1979 » size = s->elemsize; |
| 1980 » if(sizeclass == 0) { |
| 1981 » » n = 1; |
| 1982 » } else { |
| 1983 » » npages = runtime·class_to_allocnpages[sizeclass]; |
| 1984 » » n = (npages << PageShift) / size; |
| 1985 » } |
| 1986 »······· |
| 1987 » runtime·printf("%p .. %p:\n", p, p+n*size); |
| 1988 » column = 0; |
| 1989 » for(; n>0; n--, p+=size) { |
| 1990 » » uintptr off, *bitp, shift, bits; |
| 1991 |
| 1992 » » off = (uintptr*)p - (uintptr*)arena_start; |
| 1993 » » bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1; |
| 1994 » » shift = off % wordsPerBitmapWord; |
| 1995 » » bits = *bitp>>shift; |
| 1996 |
| 1997 » » allocated = ((bits & bitAllocated) != 0); |
| 1998 » » special = ((bits & bitSpecial) != 0); |
| 1999 |
| 2000 » » for(i=0; i<size; i+=sizeof(void*)) { |
| 2001 » » » if(column == 0) { |
| 2002 » » » » runtime·printf("\t"); |
| 2003 » » » } |
| 2004 » » » if(i == 0) { |
| 2005 » » » » runtime·printf(allocated ? "(" : "["); |
| 2006 » » » » runtime·printf(special ? "@" : ""); |
| 2007 » » » » runtime·printf("%p: ", p+i); |
| 2008 » » » } else { |
| 2009 » » » » runtime·printf(" "); |
| 2010 » » » } |
| 2011 |
| 2012 » » » runtime·printf("%p", *(void**)(p+i)); |
| 2013 |
| 2014 » » » if(i+sizeof(void*) >= size) { |
| 2015 » » » » runtime·printf(allocated ? ") " : "] "); |
| 2016 » » » } |
| 2017 |
| 2018 » » » column++; |
| 2019 » » » if(column == 8) { |
| 2020 » » » » runtime·printf("\n"); |
| 2021 » » » » column = 0; |
| 2022 » » » } |
| 2023 » » } |
| 2024 » } |
| 2025 » runtime·printf("\n"); |
| 2026 } |
| 2027 |
| 2028 // A debugging function to dump the contents of memory |
| 2029 void |
| 2030 runtime·memorydump(void) |
| 2031 { |
| 2032 » uint32 spanidx; |
| 2033 |
| 2034 » for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) { |
| 2035 » » dumpspan(spanidx); |
1837 } | 2036 } |
1838 } | 2037 } |
1839 | 2038 |
1840 void | 2039 void |
1841 runtime·gchelper(void) | 2040 runtime·gchelper(void) |
1842 { | 2041 { |
1843 » Obj *wp; | 2042 » gchelperstart(); |
1844 » Workbuf *wbuf; | |
1845 » uintptr nobj; | |
1846 | 2043 |
1847 // parallel mark for over gc roots | 2044 // parallel mark for over gc roots |
1848 runtime·parfordo(work.markfor); | 2045 runtime·parfordo(work.markfor); |
1849 | 2046 |
1850 // help other threads scan secondary blocks | 2047 // help other threads scan secondary blocks |
1851 » wp = nil; | 2048 » scanblock(nil, nil, 0, true); |
1852 » wbuf = nil; | |
1853 » nobj = 0; | |
1854 » scan(&wp, &wbuf, &nobj, true); | |
1855 | 2049 |
1856 if(DebugMark) { | 2050 if(DebugMark) { |
1857 // wait while the main thread executes mark(debug_scanblock) | 2051 // wait while the main thread executes mark(debug_scanblock) |
1858 while(runtime·atomicload(&work.debugmarkdone) == 0) | 2052 while(runtime·atomicload(&work.debugmarkdone) == 0) |
1859 runtime·usleep(10); | 2053 runtime·usleep(10); |
1860 } | 2054 } |
1861 | 2055 |
1862 runtime·parfordo(work.sweepfor); | 2056 runtime·parfordo(work.sweepfor); |
| 2057 bufferList[m->helpgc].busy = 0; |
1863 if(runtime·xadd(&work.ndone, +1) == work.nproc-1) | 2058 if(runtime·xadd(&work.ndone, +1) == work.nproc-1) |
1864 runtime·notewakeup(&work.alldone); | 2059 runtime·notewakeup(&work.alldone); |
1865 } | 2060 } |
| 2061 |
| 2062 #define GcpercentUnknown (-2) |
1866 | 2063 |
1867 // Initialized from $GOGC. GOGC=off means no gc. | 2064 // Initialized from $GOGC. GOGC=off means no gc. |
1868 // | 2065 // |
1869 // Next gc is after we've allocated an extra amount of | 2066 // Next gc is after we've allocated an extra amount of |
1870 // memory proportional to the amount already in use. | 2067 // memory proportional to the amount already in use. |
1871 // If gcpercent=100 and we're using 4M, we'll gc again | 2068 // If gcpercent=100 and we're using 4M, we'll gc again |
1872 // when we get to 8M. This keeps the gc cost in linear | 2069 // when we get to 8M. This keeps the gc cost in linear |
1873 // proportion to the allocation cost. Adjusting gcpercent | 2070 // proportion to the allocation cost. Adjusting gcpercent |
1874 // just changes the linear constant (and also the amount of | 2071 // just changes the linear constant (and also the amount of |
1875 // extra memory used). | 2072 // extra memory used). |
1876 static int32 gcpercent = -2; | 2073 static int32 gcpercent = GcpercentUnknown; |
1877 | |
1878 static void | |
1879 stealcache(void) | |
1880 { | |
1881 » M *m; | |
1882 | |
1883 » for(m=runtime·allm; m; m=m->alllink) | |
1884 » » runtime·MCache_ReleaseAll(m->mcache); | |
1885 } | |
1886 | 2074 |
1887 static void | 2075 static void |
1888 cachestats(GCStats *stats) | 2076 cachestats(GCStats *stats) |
1889 { | 2077 { |
1890 » M *m; | 2078 » M *mp; |
1891 MCache *c; | 2079 MCache *c; |
| 2080 P *p, **pp; |
1892 int32 i; | 2081 int32 i; |
1893 uint64 stacks_inuse; | 2082 uint64 stacks_inuse; |
1894 uint64 stacks_sys; | |
1895 uint64 *src, *dst; | 2083 uint64 *src, *dst; |
1896 | 2084 |
1897 if(stats) | 2085 if(stats) |
1898 runtime·memclr((byte*)stats, sizeof(*stats)); | 2086 runtime·memclr((byte*)stats, sizeof(*stats)); |
1899 stacks_inuse = 0; | 2087 stacks_inuse = 0; |
1900 » stacks_sys = 0; | 2088 » for(mp=runtime·allm; mp; mp=mp->alllink) { |
1901 » for(m=runtime·allm; m; m=m->alllink) { | 2089 » » stacks_inuse += mp->stackinuse*FixedStack; |
1902 » » runtime·purgecachedstats(m); | |
1903 » » stacks_inuse += m->stackalloc->inuse; | |
1904 » » stacks_sys += m->stackalloc->sys; | |
1905 if(stats) { | 2090 if(stats) { |
1906 » » » src = (uint64*)&m->gcstats; | 2091 » » » src = (uint64*)&mp->gcstats; |
1907 dst = (uint64*)stats; | 2092 dst = (uint64*)stats; |
1908 for(i=0; i<sizeof(*stats)/sizeof(uint64); i++) | 2093 for(i=0; i<sizeof(*stats)/sizeof(uint64); i++) |
1909 dst[i] += src[i]; | 2094 dst[i] += src[i]; |
1910 » » » runtime·memclr((byte*)&m->gcstats, sizeof(m->gcstats)); | 2095 » » » runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats))
; |
1911 » » } | 2096 » » } |
1912 » » c = m->mcache; | 2097 » } |
1913 » » for(i=0; i<nelem(c->local_by_size); i++) { | 2098 » for(pp=runtime·allp; p=*pp; pp++) { |
1914 » » » mstats.by_size[i].nmalloc += c->local_by_size[i].nmalloc
; | 2099 » » c = p->mcache; |
1915 » » » c->local_by_size[i].nmalloc = 0; | 2100 » » if(c==nil) |
1916 » » » mstats.by_size[i].nfree += c->local_by_size[i].nfree; | 2101 » » » continue; |
1917 » » » c->local_by_size[i].nfree = 0; | 2102 » » runtime·purgecachedstats(c); |
1918 » » } | |
1919 } | 2103 } |
1920 mstats.stacks_inuse = stacks_inuse; | 2104 mstats.stacks_inuse = stacks_inuse; |
1921 mstats.stacks_sys = stacks_sys; | |
1922 } | 2105 } |
1923 | 2106 |
1924 // Structure of arguments passed to function gc(). | 2107 // Structure of arguments passed to function gc(). |
1925 // This allows the arguments to be easily passed via reflect·call. | 2108 // This allows the arguments to be passed via reflect·call. |
1926 struct gc_args | 2109 struct gc_args |
1927 { | 2110 { |
1928 int32 force; | 2111 int32 force; |
1929 }; | 2112 }; |
1930 | 2113 |
1931 static void | 2114 static void gc(struct gc_args *args); |
1932 gc(struct gc_args *args) | 2115 |
1933 { | 2116 static int32 |
1934 » int64 t0, t1, t2, t3; | 2117 readgogc(void) |
1935 » uint64 heap0, heap1, obj0, obj1, ninstr; | 2118 { |
1936 » GCStats stats; | 2119 » byte *p; |
1937 » M *m1; | 2120 |
1938 » uint32 i; | 2121 » p = runtime·getenv("GOGC"); |
1939 » Obj *wp; | 2122 » if(p == nil || p[0] == '\0') |
1940 » Workbuf *wbuf; | 2123 » » return 100; |
1941 » uintptr nobj; | 2124 » if(runtime·strcmp(p, (byte*)"off") == 0) |
1942 | 2125 » » return -1; |
1943 » runtime·semacquire(&runtime·worldsema); | 2126 » return runtime·atoi(p); |
1944 » if(!args->force && mstats.heap_alloc < mstats.next_gc) { | |
1945 » » runtime·semrelease(&runtime·worldsema); | |
1946 » » return; | |
1947 » } | |
1948 | |
1949 » t0 = runtime·nanotime(); | |
1950 | |
1951 » m->gcing = 1; | |
1952 » runtime·stoptheworld(); | |
1953 | |
1954 » if(DebugStats) | |
1955 » » runtime·memclr((byte*)&gcstats, sizeof(gcstats)); | |
1956 | |
1957 » for(m1=runtime·allm; m1; m1=m1->alllink) | |
1958 » » runtime·settype_flush(m1, false); | |
1959 | |
1960 » heap0 = 0; | |
1961 » obj0 = 0; | |
1962 » if(gctrace) { | |
1963 » » cachestats(nil); | |
1964 » » heap0 = mstats.heap_alloc; | |
1965 » » obj0 = mstats.nmalloc - mstats.nfree; | |
1966 » } | |
1967 | |
1968 » m->locks++;» // disable gc during mallocs in parforalloc | |
1969 » if(work.markfor == nil) | |
1970 » » work.markfor = runtime·parforalloc(MaxGcproc); | |
1971 » if(work.sweepfor == nil) | |
1972 » » work.sweepfor = runtime·parforalloc(MaxGcproc); | |
1973 » m->locks--; | |
1974 | |
1975 » // forbid calls to runtime·mallocgc() | |
1976 » runtime·gcing = 1; | |
1977 | |
1978 » // call custom markers | |
1979 » for(i=0; i<ncustommarkers; i++) | |
1980 » » custommarkers[i].f(); | |
1981 | |
1982 » work.nwait = 0; | |
1983 » work.ndone = 0; | |
1984 » work.debugmarkdone = 0; | |
1985 » work.nproc = runtime·gcprocs(); | |
1986 » addroots(); | |
1987 » runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, ma
rkroot); | |
1988 » runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap.nspan, nil,
true, sweepspan); | |
1989 » if(work.nproc > 1) { | |
1990 » » runtime·noteclear(&work.alldone); | |
1991 » » runtime·helpgc(work.nproc); | |
1992 » } | |
1993 | |
1994 » runtime·parfordo(work.markfor); | |
1995 » wp = nil; | |
1996 » wbuf = nil; | |
1997 » nobj = 0; | |
1998 » scan(&wp, &wbuf, &nobj, true); | |
1999 »······· | |
2000 » if(DebugMark) { | |
2001 » » for(i=0; i<work.nroot; i++) | |
2002 » » » debug_scanblock(work.roots[i].p, work.roots[i].n); | |
2003 » » runtime·atomicstore(&work.debugmarkdone, 1); | |
2004 » } | |
2005 » t1 = runtime·nanotime(); | |
2006 | |
2007 » runtime·parfordo(work.sweepfor); | |
2008 » t2 = runtime·nanotime(); | |
2009 | |
2010 » stealcache(); | |
2011 » cachestats(&stats); | |
2012 | |
2013 » if(work.nproc > 1) | |
2014 » » runtime·notesleep(&work.alldone); | |
2015 | |
2016 » stats.nprocyield += work.sweepfor->nprocyield; | |
2017 » stats.nosyield += work.sweepfor->nosyield; | |
2018 » stats.nsleep += work.sweepfor->nsleep; | |
2019 | |
2020 » mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; | |
2021 » runtime·gcing = 0; | |
2022 » m->gcing = 0; | |
2023 | |
2024 » if(finq != nil) { | |
2025 » » m->locks++;» // disable gc during the mallocs in newproc | |
2026 » » // kick off or wake up goroutine to run queued finalizers | |
2027 » » if(fing == nil) | |
2028 » » » fing = runtime·newproc1((byte*)runfinq, nil, 0, 0, runti
me·gc); | |
2029 » » else if(fingwait) { | |
2030 » » » fingwait = 0; | |
2031 » » » runtime·ready(fing); | |
2032 » » } | |
2033 » » m->locks--; | |
2034 » } | |
2035 | |
2036 » heap1 = mstats.heap_alloc; | |
2037 » obj1 = mstats.nmalloc - mstats.nfree; | |
2038 | |
2039 » t3 = runtime·nanotime(); | |
2040 » mstats.last_gc = t3; | |
2041 » mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t3 - t0; | |
2042 » mstats.pause_total_ns += t3 - t0; | |
2043 » mstats.numgc++; | |
2044 » if(mstats.debuggc) | |
2045 » » runtime·printf("pause %D\n", t3-t0); | |
2046 | |
2047 » if(gctrace) { | |
2048 » » runtime·printf("gc%d(%d): %D.%D+%D+%D ms, %D -> %D MB %D -> %D (
%D-%D) objects," | |
2049 » » » » " %D(%D) handoff, %D(%D) steal, %D/%D/%D yields\
n", | |
2050 » » » mstats.numgc, work.nproc, (t1-t0)/1000000, (t1-t0)/10000
0%10, (t2-t1)/1000000, (t3-t2)/1000000, | |
2051 » » » heap0>>20, heap1>>20, obj0, obj1, | |
2052 » » » mstats.nmalloc, mstats.nfree, | |
2053 » » » stats.nhandoff, stats.nhandoffcnt, | |
2054 » » » work.sweepfor->nsteal, work.sweepfor->nstealcnt, | |
2055 » » » stats.nprocyield, stats.nosyield, stats.nsleep); | |
2056 » » if(DebugStats) { | |
2057 » » » runtime·printf("scan: %D bytes, %D objects, %D untyped,
%D types from MSpan\n", | |
2058 » » » » gcstats.nbytes, gcstats.nobj_cnt, gcstats.nobj_n
otype, gcstats.nobj_typelookups_ok); | |
2059 » » » if(gcstats.x_cnt != 0) | |
2060 » » » » runtime·printf("avg xbufsize: %D (%D/%D)\n", gcs
tats.x_sum/gcstats.x_cnt, gcstats.x_sum, gcstats.x_cnt); | |
2061 » » » if(gcstats.nobj_cnt != 0) | |
2062 » » » » runtime·printf("avg nobj: %D (%D/%D)\n", gcstats
.nobj_sum/gcstats.nobj_cnt, gcstats.nobj_sum, gcstats.nobj_cnt); | |
2063 » » » runtime·printf("rescans: %D, %D bytes\n", gcstats.rescan
s, gcstats.rescan_nbytes); | |
2064 | |
2065 » » » runtime·printf("instruction counts:\n"); | |
2066 » » » ninstr = 0; | |
2067 » » » for(i=0; i<nelem(gcstats.instr); i++) { | |
2068 » » » » runtime·printf("\t%d:\t%D\n", i, gcstats.instr[i
]); | |
2069 » » » » ninstr += gcstats.instr[i]; | |
2070 » » » } | |
2071 » » » runtime·printf("\ttotal:\t%D\n", ninstr); | |
2072 | |
2073 » » » runtime·printf("putempty: %D, getfull: %D\n", gcstats.pu
tempty, gcstats.getfull); | |
2074 | |
2075 » » » runtime·printf("runtime·markobject: %D\n", gcstats.runti
me·markobject); | |
2076 » » } | |
2077 » } | |
2078 | |
2079 » runtime·MProf_GC(); | |
2080 » runtime·semrelease(&runtime·worldsema); | |
2081 » runtime·starttheworld(); | |
2082 | |
2083 » // give the queued finalizers, if any, a chance to run | |
2084 » if(finq != nil) | |
2085 » » runtime·gosched(); | |
2086 | |
2087 » if(gctrace > 1 && !args->force) | |
2088 » » runtime·gc(1); | |
2089 } | 2127 } |
2090 | 2128 |
2091 void | 2129 void |
2092 runtime·gc(int32 force) | 2130 runtime·gc(int32 force) |
2093 { | 2131 { |
2094 byte *p; | 2132 byte *p; |
2095 struct gc_args a, *ap; | 2133 struct gc_args a, *ap; |
| 2134 FuncVal gcv; |
| 2135 |
| 2136 // The atomic operations are not atomic if the uint64s |
| 2137 // are not aligned on uint64 boundaries. This has been |
| 2138 // a problem in the past. |
| 2139 if((((uintptr)&work.empty) & 7) != 0) |
| 2140 runtime·throw("runtime: gc work buffer is misaligned"); |
| 2141 if((((uintptr)&work.full) & 7) != 0) |
| 2142 runtime·throw("runtime: gc work buffer is misaligned"); |
2096 | 2143 |
2097 // The gc is turned off (via enablegc) until | 2144 // The gc is turned off (via enablegc) until |
2098 // the bootstrap has completed. | 2145 // the bootstrap has completed. |
2099 // Also, malloc gets called in the guts | 2146 // Also, malloc gets called in the guts |
2100 // of a number of libraries that might be | 2147 // of a number of libraries that might be |
2101 // holding locks. To avoid priority inversion | 2148 // holding locks. To avoid priority inversion |
2102 // problems, don't bother trying to run gc | 2149 // problems, don't bother trying to run gc |
2103 // while holding a lock. The next mallocgc | 2150 // while holding a lock. The next mallocgc |
2104 // without a lock will do the gc instead. | 2151 // without a lock will do the gc instead. |
2105 if(!mstats.enablegc || m->locks > 0 || runtime·panicking) | 2152 if(!mstats.enablegc || m->locks > 0 || runtime·panicking) |
2106 return; | 2153 return; |
2107 | 2154 |
2108 » if(gcpercent == -2) {» // first time through | 2155 » if(gcpercent == GcpercentUnknown) {» // first time through |
2109 » » p = runtime·getenv("GOGC"); | 2156 » » gcpercent = readgogc(); |
2110 » » if(p == nil || p[0] == '\0') | |
2111 » » » gcpercent = 100; | |
2112 » » else if(runtime·strcmp(p, (byte*)"off") == 0) | |
2113 » » » gcpercent = -1; | |
2114 » » else | |
2115 » » » gcpercent = runtime·atoi(p); | |
2116 | 2157 |
2117 p = runtime·getenv("GOGCTRACE"); | 2158 p = runtime·getenv("GOGCTRACE"); |
2118 if(p != nil) | 2159 if(p != nil) |
2119 gctrace = runtime·atoi(p); | 2160 gctrace = runtime·atoi(p); |
2120 } | 2161 } |
2121 if(gcpercent < 0) | 2162 if(gcpercent < 0) |
2122 return; | 2163 return; |
2123 | 2164 |
| 2165 // Run gc on a bigger stack to eliminate |
| 2166 // a potentially large number of calls to runtime·morestack. |
2124 a.force = force; | 2167 a.force = force; |
2125 ap = &a; | 2168 ap = &a; |
2126 » if(!runtime·iscgo) { | 2169 » m->moreframesize_minalloc = StackBig; |
2127 » » // Run gc on a bigger stack in order to eliminate | 2170 » gcv.fn = (void*)gc; |
2128 » » // a potentially large number of calls to runtime·morestack. | 2171 » reflect·call(&gcv, (byte*)&ap, sizeof(ap)); |
2129 » » m->moreframesize_minalloc = StackBig; | 2172 |
2130 » » reflect·call((byte*)gc, (byte*)&ap, sizeof(ap)); | 2173 » if(gctrace > 1 && !force) { |
2131 » } else { | 2174 » » a.force = 1; |
2132 » » gc(ap); | 2175 » » gc(&a); |
| 2176 » } |
| 2177 } |
| 2178 |
| 2179 static FuncVal runfinqv = {runfinq}; |
| 2180 |
| 2181 static void |
| 2182 gc(struct gc_args *args) |
| 2183 { |
| 2184 » int64 t0, t1, t2, t3, t4; |
| 2185 » uint64 heap0, heap1, obj0, obj1, ninstr; |
| 2186 » GCStats stats; |
| 2187 » M *mp; |
| 2188 » uint32 i; |
| 2189 » Eface eface; |
| 2190 |
| 2191 » runtime·semacquire(&runtime·worldsema); |
| 2192 » if(!args->force && mstats.heap_alloc < mstats.next_gc) { |
| 2193 » » runtime·semrelease(&runtime·worldsema); |
| 2194 » » return; |
| 2195 » } |
| 2196 |
| 2197 » t0 = runtime·nanotime(); |
| 2198 |
| 2199 » m->gcing = 1; |
| 2200 » runtime·stoptheworld(); |
| 2201 |
| 2202 » if(CollectStats) |
| 2203 » » runtime·memclr((byte*)&gcstats, sizeof(gcstats)); |
| 2204 |
| 2205 » for(mp=runtime·allm; mp; mp=mp->alllink) |
| 2206 » » runtime·settype_flush(mp, false); |
| 2207 |
| 2208 » heap0 = 0; |
| 2209 » obj0 = 0; |
| 2210 » if(gctrace) { |
| 2211 » » cachestats(nil); |
| 2212 » » heap0 = mstats.heap_alloc; |
| 2213 » » obj0 = mstats.nmalloc - mstats.nfree; |
| 2214 » } |
| 2215 |
| 2216 » m->locks++;» // disable gc during mallocs in parforalloc |
| 2217 » if(work.markfor == nil) |
| 2218 » » work.markfor = runtime·parforalloc(MaxGcproc); |
| 2219 » if(work.sweepfor == nil) |
| 2220 » » work.sweepfor = runtime·parforalloc(MaxGcproc); |
| 2221 » m->locks--; |
| 2222 |
| 2223 » if(itabtype == nil) { |
| 2224 » » // get C pointer to the Go type "itab" |
| 2225 » » runtime·gc_itab_ptr(&eface); |
| 2226 » » itabtype = ((PtrType*)eface.type)->elem; |
| 2227 » } |
| 2228 |
| 2229 » work.nwait = 0; |
| 2230 » work.ndone = 0; |
| 2231 » work.debugmarkdone = 0; |
| 2232 » work.nproc = runtime·gcprocs(); |
| 2233 » addroots(); |
| 2234 » runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, ma
rkroot); |
| 2235 » runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap->nspan, nil
, true, sweepspan); |
| 2236 » if(work.nproc > 1) { |
| 2237 » » runtime·noteclear(&work.alldone); |
| 2238 » » runtime·helpgc(work.nproc); |
| 2239 » } |
| 2240 |
| 2241 » t1 = runtime·nanotime(); |
| 2242 |
| 2243 » gchelperstart(); |
| 2244 » runtime·parfordo(work.markfor); |
| 2245 » scanblock(nil, nil, 0, true); |
| 2246 |
| 2247 » if(DebugMark) { |
| 2248 » » for(i=0; i<work.nroot; i++) |
| 2249 » » » debug_scanblock(work.roots[i].p, work.roots[i].n); |
| 2250 » » runtime·atomicstore(&work.debugmarkdone, 1); |
| 2251 » } |
| 2252 » t2 = runtime·nanotime(); |
| 2253 |
| 2254 » runtime·parfordo(work.sweepfor); |
| 2255 » bufferList[m->helpgc].busy = 0; |
| 2256 » t3 = runtime·nanotime(); |
| 2257 |
| 2258 » if(work.nproc > 1) |
| 2259 » » runtime·notesleep(&work.alldone); |
| 2260 |
| 2261 » cachestats(&stats); |
| 2262 |
| 2263 » stats.nprocyield += work.sweepfor->nprocyield; |
| 2264 » stats.nosyield += work.sweepfor->nosyield; |
| 2265 » stats.nsleep += work.sweepfor->nsleep; |
| 2266 |
| 2267 » mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100; |
| 2268 » m->gcing = 0; |
| 2269 |
| 2270 » heap1 = mstats.heap_alloc; |
| 2271 » obj1 = mstats.nmalloc - mstats.nfree; |
| 2272 |
| 2273 » t4 = runtime·nanotime(); |
| 2274 » mstats.last_gc = t4; |
| 2275 » mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t4 - t0; |
| 2276 » mstats.pause_total_ns += t4 - t0; |
| 2277 » mstats.numgc++; |
| 2278 » if(mstats.debuggc) |
| 2279 » » runtime·printf("pause %D\n", t4-t0); |
| 2280 |
| 2281 » if(gctrace) { |
| 2282 » » runtime·printf("gc%d(%d): %D.%D+%D+%D ms, %D -> %D MB %D -> %D (
%D-%D) objects," |
| 2283 » » » » " %D(%D) handoff, %D(%D) steal, %D/%D/%D yields\
n", |
| 2284 » » » mstats.numgc, work.nproc, (t2-t1)/1000000, (t2-t1)/10000
0%10, (t3-t2)/1000000, (t1-t0+t4-t3)/1000000, |
| 2285 » » » heap0>>20, heap1>>20, obj0, obj1, |
| 2286 » » » mstats.nmalloc, mstats.nfree, |
| 2287 » » » stats.nhandoff, stats.nhandoffcnt, |
| 2288 » » » work.sweepfor->nsteal, work.sweepfor->nstealcnt, |
| 2289 » » » stats.nprocyield, stats.nosyield, stats.nsleep); |
| 2290 » » if(CollectStats) { |
| 2291 » » » runtime·printf("scan: %D bytes, %D objects, %D untyped,
%D types from MSpan\n", |
| 2292 » » » » gcstats.nbytes, gcstats.obj.cnt, gcstats.obj.not
ype, gcstats.obj.typelookup); |
| 2293 » » » if(gcstats.ptr.cnt != 0) |
| 2294 » » » » runtime·printf("avg ptrbufsize: %D (%D/%D)\n", |
| 2295 » » » » » gcstats.ptr.sum/gcstats.ptr.cnt, gcstats
.ptr.sum, gcstats.ptr.cnt); |
| 2296 » » » if(gcstats.obj.cnt != 0) |
| 2297 » » » » runtime·printf("avg nobj: %D (%D/%D)\n", |
| 2298 » » » » » gcstats.obj.sum/gcstats.obj.cnt, gcstats
.obj.sum, gcstats.obj.cnt); |
| 2299 » » » runtime·printf("rescans: %D, %D bytes\n", gcstats.rescan
, gcstats.rescanbytes); |
| 2300 |
| 2301 » » » runtime·printf("instruction counts:\n"); |
| 2302 » » » ninstr = 0; |
| 2303 » » » for(i=0; i<nelem(gcstats.instr); i++) { |
| 2304 » » » » runtime·printf("\t%d:\t%D\n", i, gcstats.instr[i
]); |
| 2305 » » » » ninstr += gcstats.instr[i]; |
| 2306 » » » } |
| 2307 » » » runtime·printf("\ttotal:\t%D\n", ninstr); |
| 2308 |
| 2309 » » » runtime·printf("putempty: %D, getfull: %D\n", gcstats.pu
tempty, gcstats.getfull); |
| 2310 » » } |
| 2311 » } |
| 2312 |
| 2313 » runtime·MProf_GC(); |
| 2314 » runtime·semrelease(&runtime·worldsema); |
| 2315 » runtime·starttheworld(); |
| 2316 |
| 2317 » if(finq != nil) { |
| 2318 » » runtime·lock(&finlock); |
| 2319 » » // kick off or wake up goroutine to run queued finalizers |
| 2320 » » if(fing == nil) |
| 2321 » » » fing = runtime·newproc1(&runfinqv, nil, 0, 0, runtime·gc
); |
| 2322 » » else if(fingwait) { |
| 2323 » » » fingwait = 0; |
| 2324 » » » runtime·ready(fing); |
| 2325 » » } |
| 2326 » » runtime·unlock(&finlock); |
| 2327 » » // give the queued finalizers, if any, a chance to run |
| 2328 » » runtime·gosched(); |
2133 } | 2329 } |
2134 } | 2330 } |
2135 | 2331 |
2136 void | 2332 void |
2137 runtime·ReadMemStats(MStats *stats) | 2333 runtime·ReadMemStats(MStats *stats) |
2138 { | 2334 { |
2139 // Have to acquire worldsema to stop the world, | 2335 // Have to acquire worldsema to stop the world, |
2140 // because stoptheworld can only be used by | 2336 // because stoptheworld can only be used by |
2141 // one goroutine at a time, and there might be | 2337 // one goroutine at a time, and there might be |
2142 // a pending garbage collection already calling it. | 2338 // a pending garbage collection already calling it. |
2143 runtime·semacquire(&runtime·worldsema); | 2339 runtime·semacquire(&runtime·worldsema); |
2144 m->gcing = 1; | 2340 m->gcing = 1; |
2145 runtime·stoptheworld(); | 2341 runtime·stoptheworld(); |
2146 cachestats(nil); | 2342 cachestats(nil); |
2147 *stats = mstats; | 2343 *stats = mstats; |
2148 m->gcing = 0; | 2344 m->gcing = 0; |
2149 runtime·semrelease(&runtime·worldsema); | 2345 runtime·semrelease(&runtime·worldsema); |
2150 runtime·starttheworld(); | 2346 runtime·starttheworld(); |
2151 } | 2347 } |
2152 | 2348 |
| 2349 void |
| 2350 runtime∕debug·readGCStats(Slice *pauses) |
| 2351 { |
| 2352 uint64 *p; |
| 2353 uint32 i, n; |
| 2354 |
| 2355 // Calling code in runtime/debug should make the slice large enough. |
| 2356 if(pauses->cap < nelem(mstats.pause_ns)+3) |
| 2357 runtime·throw("runtime: short slice passed to readGCStats"); |
| 2358 |
| 2359 // Pass back: pauses, last gc (absolute time), number of gc, total pause
ns. |
| 2360 p = (uint64*)pauses->array; |
| 2361 runtime·lock(runtime·mheap); |
| 2362 n = mstats.numgc; |
| 2363 if(n > nelem(mstats.pause_ns)) |
| 2364 n = nelem(mstats.pause_ns); |
| 2365 ········ |
| 2366 // The pause buffer is circular. The most recent pause is at |
| 2367 // pause_ns[(numgc-1)%nelem(pause_ns)], and then backward |
| 2368 // from there to go back farther in time. We deliver the times |
| 2369 // most recent first (in p[0]). |
| 2370 for(i=0; i<n; i++) |
| 2371 p[i] = mstats.pause_ns[(mstats.numgc-1-i)%nelem(mstats.pause_ns)
]; |
| 2372 |
| 2373 p[n] = mstats.last_gc; |
| 2374 p[n+1] = mstats.numgc; |
| 2375 p[n+2] = mstats.pause_total_ns;· |
| 2376 runtime·unlock(runtime·mheap); |
| 2377 pauses->len = n+3; |
| 2378 } |
| 2379 |
| 2380 void |
| 2381 runtime∕debug·setGCPercent(intgo in, intgo out) |
| 2382 { |
| 2383 runtime·lock(runtime·mheap); |
| 2384 if(gcpercent == GcpercentUnknown) |
| 2385 gcpercent = readgogc(); |
| 2386 out = gcpercent; |
| 2387 if(in < 0) |
| 2388 in = -1; |
| 2389 gcpercent = in; |
| 2390 runtime·unlock(runtime·mheap); |
| 2391 FLUSH(&out); |
| 2392 } |
| 2393 |
| 2394 static void |
| 2395 gchelperstart(void) |
| 2396 { |
| 2397 if(m->helpgc < 0 || m->helpgc >= MaxGcproc) |
| 2398 runtime·throw("gchelperstart: bad m->helpgc"); |
| 2399 if(runtime·xchg(&bufferList[m->helpgc].busy, 1)) |
| 2400 runtime·throw("gchelperstart: already busy"); |
| 2401 } |
| 2402 |
2153 static void | 2403 static void |
2154 runfinq(void) | 2404 runfinq(void) |
2155 { | 2405 { |
2156 Finalizer *f; | 2406 Finalizer *f; |
2157 FinBlock *fb, *next; | 2407 FinBlock *fb, *next; |
2158 byte *frame; | 2408 byte *frame; |
2159 uint32 framesz, framecap, i; | 2409 uint32 framesz, framecap, i; |
2160 | 2410 |
2161 frame = nil; | 2411 frame = nil; |
2162 framecap = 0; | 2412 framecap = 0; |
2163 for(;;) { | 2413 for(;;) { |
2164 » » // There's no need for a lock in this section | 2414 » » runtime·lock(&finlock); |
2165 » » // because it only conflicts with the garbage | |
2166 » » // collector, and the garbage collector only | |
2167 » » // runs when everyone else is stopped, and | |
2168 » » // runfinq only stops at the gosched() or | |
2169 » » // during the calls in the for loop. | |
2170 fb = finq; | 2415 fb = finq; |
2171 finq = nil; | 2416 finq = nil; |
2172 if(fb == nil) { | 2417 if(fb == nil) { |
2173 fingwait = 1; | 2418 fingwait = 1; |
2174 » » » g->status = Gwaiting; | 2419 » » » runtime·park(runtime·unlock, &finlock, "finalizer wait")
; |
2175 » » » g->waitreason = "finalizer wait"; | |
2176 » » » runtime·gosched(); | |
2177 continue; | 2420 continue; |
2178 } | 2421 } |
| 2422 runtime·unlock(&finlock); |
| 2423 if(raceenabled) |
| 2424 runtime·racefingo(); |
2179 for(; fb; fb=next) { | 2425 for(; fb; fb=next) { |
2180 next = fb->next; | 2426 next = fb->next; |
2181 for(i=0; i<fb->cnt; i++) { | 2427 for(i=0; i<fb->cnt; i++) { |
2182 f = &fb->fin[i]; | 2428 f = &fb->fin[i]; |
2183 framesz = sizeof(uintptr) + f->nret; | 2429 framesz = sizeof(uintptr) + f->nret; |
2184 if(framecap < framesz) { | 2430 if(framecap < framesz) { |
2185 runtime·free(frame); | 2431 runtime·free(frame); |
2186 frame = runtime·mal(framesz); | 2432 frame = runtime·mal(framesz); |
2187 framecap = framesz; | 2433 framecap = framesz; |
2188 } | 2434 } |
2189 *(void**)frame = f->arg; | 2435 *(void**)frame = f->arg; |
2190 » » » » reflect·call((byte*)f->fn, frame, sizeof(uintptr
) + f->nret); | 2436 » » » » reflect·call(f->fn, frame, sizeof(uintptr) + f->
nret); |
2191 f->fn = nil; | 2437 f->fn = nil; |
2192 f->arg = nil; | 2438 f->arg = nil; |
2193 } | 2439 } |
2194 fb->cnt = 0; | 2440 fb->cnt = 0; |
2195 fb->next = finc; | 2441 fb->next = finc; |
2196 finc = fb; | 2442 finc = fb; |
2197 } | 2443 } |
2198 runtime·gc(1); // trigger another gc to clean up the finalized
objects, if possible | 2444 runtime·gc(1); // trigger another gc to clean up the finalized
objects, if possible |
2199 } | 2445 } |
2200 } | 2446 } |
2201 | 2447 |
2202 // mark the block at v of size n as allocated. | 2448 // mark the block at v of size n as allocated. |
2203 // If noptr is true, mark it as having no pointers. | 2449 // If noptr is true, mark it as having no pointers. |
2204 void | 2450 void |
2205 runtime·markallocated(void *v, uintptr n, bool noptr) | 2451 runtime·markallocated(void *v, uintptr n, bool noptr) |
2206 { | 2452 { |
2207 uintptr *b, obits, bits, off, shift; | 2453 uintptr *b, obits, bits, off, shift; |
2208 | 2454 |
2209 if(0) | 2455 if(0) |
2210 runtime·printf("markallocated %p+%p\n", v, n); | 2456 runtime·printf("markallocated %p+%p\n", v, n); |
2211 | 2457 |
2212 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh
eap.arena_start) | 2458 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m
heap->arena_start) |
2213 runtime·throw("markallocated: bad pointer"); | 2459 runtime·throw("markallocated: bad pointer"); |
2214 | 2460 |
2215 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset | 2461 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offse
t |
2216 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2462 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2217 shift = off % wordsPerBitmapWord; | 2463 shift = off % wordsPerBitmapWord; |
2218 | 2464 |
2219 for(;;) { | 2465 for(;;) { |
2220 obits = *b; | 2466 obits = *b; |
2221 bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift); | 2467 bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift); |
2222 if(noptr) | 2468 if(noptr) |
2223 bits |= bitNoPointers<<shift; | 2469 bits |= bitNoPointers<<shift; |
2224 if(runtime·singleproc) { | 2470 if(runtime·singleproc) { |
2225 *b = bits; | 2471 *b = bits; |
2226 break; | 2472 break; |
2227 } else { | 2473 } else { |
2228 // more than one goroutine is potentially running: use a
tomic op | 2474 // more than one goroutine is potentially running: use a
tomic op |
2229 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) | 2475 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) |
2230 break; | 2476 break; |
2231 } | 2477 } |
2232 } | 2478 } |
2233 } | 2479 } |
2234 | 2480 |
2235 // mark the block at v of size n as freed. | 2481 // mark the block at v of size n as freed. |
2236 void | 2482 void |
2237 runtime·markfreed(void *v, uintptr n) | 2483 runtime·markfreed(void *v, uintptr n) |
2238 { | 2484 { |
2239 uintptr *b, obits, bits, off, shift; | 2485 uintptr *b, obits, bits, off, shift; |
2240 | 2486 |
2241 if(0) | 2487 if(0) |
2242 runtime·printf("markallocated %p+%p\n", v, n); | 2488 runtime·printf("markallocated %p+%p\n", v, n); |
2243 | 2489 |
2244 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh
eap.arena_start) | 2490 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m
heap->arena_start) |
2245 runtime·throw("markallocated: bad pointer"); | 2491 runtime·throw("markallocated: bad pointer"); |
2246 | 2492 |
2247 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset | 2493 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offse
t |
2248 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2494 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2249 shift = off % wordsPerBitmapWord; | 2495 shift = off % wordsPerBitmapWord; |
2250 | 2496 |
2251 for(;;) { | 2497 for(;;) { |
2252 obits = *b; | 2498 obits = *b; |
2253 bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 2499 bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
2254 if(runtime·singleproc) { | 2500 if(runtime·singleproc) { |
2255 *b = bits; | 2501 *b = bits; |
2256 break; | 2502 break; |
2257 } else { | 2503 } else { |
2258 // more than one goroutine is potentially running: use a
tomic op | 2504 // more than one goroutine is potentially running: use a
tomic op |
2259 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) | 2505 if(runtime·casp((void**)b, (void*)obits, (void*)bits)) |
2260 break; | 2506 break; |
2261 } | 2507 } |
2262 } | 2508 } |
2263 } | 2509 } |
2264 | 2510 |
2265 // check that the block at v of size n is marked freed. | 2511 // check that the block at v of size n is marked freed. |
2266 void | 2512 void |
2267 runtime·checkfreed(void *v, uintptr n) | 2513 runtime·checkfreed(void *v, uintptr n) |
2268 { | 2514 { |
2269 uintptr *b, bits, off, shift; | 2515 uintptr *b, bits, off, shift; |
2270 | 2516 |
2271 if(!runtime·checking) | 2517 if(!runtime·checking) |
2272 return; | 2518 return; |
2273 | 2519 |
2274 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh
eap.arena_start) | 2520 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m
heap->arena_start) |
2275 return; // not allocated, so okay | 2521 return; // not allocated, so okay |
2276 | 2522 |
2277 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset | 2523 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offse
t |
2278 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2524 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2279 shift = off % wordsPerBitmapWord; | 2525 shift = off % wordsPerBitmapWord; |
2280 | 2526 |
2281 bits = *b>>shift; | 2527 bits = *b>>shift; |
2282 if((bits & bitAllocated) != 0) { | 2528 if((bits & bitAllocated) != 0) { |
2283 runtime·printf("checkfreed %p+%p: off=%p have=%p\n", | 2529 runtime·printf("checkfreed %p+%p: off=%p have=%p\n", |
2284 v, n, off, bits & bitMask); | 2530 v, n, off, bits & bitMask); |
2285 runtime·throw("checkfreed: not freed"); | 2531 runtime·throw("checkfreed: not freed"); |
2286 } | 2532 } |
2287 } | 2533 } |
2288 | 2534 |
2289 // mark the span of memory at v as having n blocks of the given size. | 2535 // mark the span of memory at v as having n blocks of the given size. |
2290 // if leftover is true, there is left over space at the end of the span. | 2536 // if leftover is true, there is left over space at the end of the span. |
2291 void | 2537 void |
2292 runtime·markspan(void *v, uintptr size, uintptr n, bool leftover) | 2538 runtime·markspan(void *v, uintptr size, uintptr n, bool leftover) |
2293 { | 2539 { |
2294 uintptr *b, off, shift; | 2540 uintptr *b, off, shift; |
2295 byte *p; | 2541 byte *p; |
2296 | 2542 |
2297 » if((byte*)v+size*n > (byte*)runtime·mheap.arena_used || (byte*)v < runti
me·mheap.arena_start) | 2543 » if((byte*)v+size*n > (byte*)runtime·mheap->arena_used || (byte*)v < runt
ime·mheap->arena_start) |
2298 runtime·throw("markspan: bad pointer"); | 2544 runtime·throw("markspan: bad pointer"); |
2299 | 2545 |
2300 p = v; | 2546 p = v; |
2301 if(leftover) // mark a boundary just past end of last block too | 2547 if(leftover) // mark a boundary just past end of last block too |
2302 n++; | 2548 n++; |
2303 for(; n-- > 0; p += size) { | 2549 for(; n-- > 0; p += size) { |
2304 // Okay to use non-atomic ops here, because we control | 2550 // Okay to use non-atomic ops here, because we control |
2305 // the entire span, and each bitmap word has bits for only | 2551 // the entire span, and each bitmap word has bits for only |
2306 // one span, so no other goroutines are changing these | 2552 // one span, so no other goroutines are changing these |
2307 // bitmap words. | 2553 // bitmap words. |
2308 » » off = (uintptr*)p - (uintptr*)runtime·mheap.arena_start; // wor
d offset | 2554 » » off = (uintptr*)p - (uintptr*)runtime·mheap->arena_start; // wo
rd offset |
2309 » » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord
- 1; | 2555 » » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWor
d - 1; |
2310 shift = off % wordsPerBitmapWord; | 2556 shift = off % wordsPerBitmapWord; |
2311 *b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); | 2557 *b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); |
2312 } | 2558 } |
2313 } | 2559 } |
2314 | 2560 |
2315 // unmark the span of memory at v of length n bytes. | 2561 // unmark the span of memory at v of length n bytes. |
2316 void | 2562 void |
2317 runtime·unmarkspan(void *v, uintptr n) | 2563 runtime·unmarkspan(void *v, uintptr n) |
2318 { | 2564 { |
2319 uintptr *p, *b, off; | 2565 uintptr *p, *b, off; |
2320 | 2566 |
2321 » if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mh
eap.arena_start) | 2567 » if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·m
heap->arena_start) |
2322 runtime·throw("markspan: bad pointer"); | 2568 runtime·throw("markspan: bad pointer"); |
2323 | 2569 |
2324 p = v; | 2570 p = v; |
2325 » off = p - (uintptr*)runtime·mheap.arena_start; // word offset | 2571 » off = p - (uintptr*)runtime·mheap->arena_start; // word offset |
2326 if(off % wordsPerBitmapWord != 0) | 2572 if(off % wordsPerBitmapWord != 0) |
2327 runtime·throw("markspan: unaligned pointer"); | 2573 runtime·throw("markspan: unaligned pointer"); |
2328 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2574 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2329 n /= PtrSize; | 2575 n /= PtrSize; |
2330 if(n%wordsPerBitmapWord != 0) | 2576 if(n%wordsPerBitmapWord != 0) |
2331 runtime·throw("unmarkspan: unaligned length"); | 2577 runtime·throw("unmarkspan: unaligned length"); |
2332 // Okay to use non-atomic ops here, because we control | 2578 // Okay to use non-atomic ops here, because we control |
2333 // the entire span, and each bitmap word has bits for only | 2579 // the entire span, and each bitmap word has bits for only |
2334 // one span, so no other goroutines are changing these | 2580 // one span, so no other goroutines are changing these |
2335 // bitmap words. | 2581 // bitmap words. |
2336 n /= wordsPerBitmapWord; | 2582 n /= wordsPerBitmapWord; |
2337 while(n-- > 0) | 2583 while(n-- > 0) |
2338 *b-- = 0; | 2584 *b-- = 0; |
2339 } | 2585 } |
2340 | 2586 |
2341 bool | 2587 bool |
2342 runtime·blockspecial(void *v) | 2588 runtime·blockspecial(void *v) |
2343 { | 2589 { |
2344 uintptr *b, off, shift; | 2590 uintptr *b, off, shift; |
2345 | 2591 |
2346 if(DebugMark) | 2592 if(DebugMark) |
2347 return true; | 2593 return true; |
2348 | 2594 |
2349 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; | 2595 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; |
2350 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2596 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2351 shift = off % wordsPerBitmapWord; | 2597 shift = off % wordsPerBitmapWord; |
2352 | 2598 |
2353 return (*b & (bitSpecial<<shift)) != 0; | 2599 return (*b & (bitSpecial<<shift)) != 0; |
2354 } | 2600 } |
2355 | 2601 |
2356 void | 2602 void |
2357 runtime·setblockspecial(void *v, bool s) | 2603 runtime·setblockspecial(void *v, bool s) |
2358 { | 2604 { |
2359 uintptr *b, off, shift, bits, obits; | 2605 uintptr *b, off, shift, bits, obits; |
2360 | 2606 |
2361 if(DebugMark) | 2607 if(DebugMark) |
2362 return; | 2608 return; |
2363 | 2609 |
2364 » off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; | 2610 » off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; |
2365 » b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1; | 2611 » b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1; |
2366 shift = off % wordsPerBitmapWord; | 2612 shift = off % wordsPerBitmapWord; |
2367 | 2613 |
2368 for(;;) { | 2614 for(;;) { |
2369 obits = *b; | 2615 obits = *b; |
2370 if(s) | 2616 if(s) |
2371 bits = obits | (bitSpecial<<shift); | 2617 bits = obits | (bitSpecial<<shift); |
2372 else | 2618 else |
2373 bits = obits & ~(bitSpecial<<shift); | 2619 bits = obits & ~(bitSpecial<<shift); |
2374 if(runtime·singleproc) { | 2620 if(runtime·singleproc) { |
2375 *b = bits; | 2621 *b = bits; |
(...skipping 18 matching lines...) Expand all Loading... |
2394 uintptr n; | 2640 uintptr n; |
2395 | 2641 |
2396 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; | 2642 n = (h->arena_used - h->arena_start) / wordsPerBitmapWord; |
2397 n = (n+bitmapChunk-1) & ~(bitmapChunk-1); | 2643 n = (n+bitmapChunk-1) & ~(bitmapChunk-1); |
2398 if(h->bitmap_mapped >= n) | 2644 if(h->bitmap_mapped >= n) |
2399 return; | 2645 return; |
2400 | 2646 |
2401 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped); | 2647 runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped); |
2402 h->bitmap_mapped = n; | 2648 h->bitmap_mapped = n; |
2403 } | 2649 } |
LEFT | RIGHT |