LEFT | RIGHT |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // See malloc.h for overview. | 5 // See malloc.h for overview. |
6 // | 6 // |
7 // TODO(rsc): double-check stats. | 7 // TODO(rsc): double-check stats. |
8 | 8 |
9 package runtime | 9 package runtime |
10 #include "runtime.h" | 10 #include "runtime.h" |
11 #include "arch_GOARCH.h" | 11 #include "arch_GOARCH.h" |
12 #include "stack.h" | |
13 #include "malloc.h" | 12 #include "malloc.h" |
14 #include "defs_GOOS_GOARCH.h" | |
15 #include "type.h" | 13 #include "type.h" |
16 | 14 #include "typekind.h" |
17 #pragma dataflag 16 /* mark mheap as 'no pointers', hiding from garbage collecto
r */ | 15 #include "race.h" |
18 MHeap runtime·mheap; | 16 |
19 | 17 MHeap *runtime·mheap; |
20 extern MStats mstats;» // defined in extern.go | 18 |
21 | 19 int32» runtime·checking; |
22 extern volatile int32 runtime·MemProfileRate; | 20 |
| 21 extern MStats mstats;» // defined in zruntime_def_$GOOS_$GOARCH.go |
| 22 |
| 23 extern volatile intgo runtime·MemProfileRate; |
23 | 24 |
24 // Allocate an object of at least size bytes. | 25 // Allocate an object of at least size bytes. |
25 // Small objects are allocated from the per-thread cache's free lists. | 26 // Small objects are allocated from the per-thread cache's free lists. |
26 // Large objects (> 32 kB) are allocated straight from the heap. | 27 // Large objects (> 32 kB) are allocated straight from the heap. |
27 void* | 28 void* |
28 runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed) | 29 runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed) |
29 { | 30 { |
30 » int32 sizeclass, rate; | 31 » int32 sizeclass; |
| 32 » intgo rate; |
31 MCache *c; | 33 MCache *c; |
32 uintptr npages; | 34 uintptr npages; |
33 MSpan *s; | 35 MSpan *s; |
34 void *v; | 36 void *v; |
35 | 37 |
36 » if(runtime·gcwaiting && g != m->g0 && m->locks == 0) | 38 » if(runtime·gcwaiting && g != m->g0 && m->locks == 0 && dogc) |
37 runtime·gosched(); | 39 runtime·gosched(); |
38 if(m->mallocing) | 40 if(m->mallocing) |
39 runtime·throw("malloc/free - deadlock"); | 41 runtime·throw("malloc/free - deadlock"); |
40 m->mallocing = 1; | 42 m->mallocing = 1; |
41 if(size == 0) | 43 if(size == 0) |
42 size = 1; | 44 size = 1; |
43 | 45 |
| 46 if(DebugTypeAtBlockEnd) |
| 47 size += sizeof(uintptr); |
| 48 |
44 c = m->mcache; | 49 c = m->mcache; |
45 c->local_nmalloc++; | 50 c->local_nmalloc++; |
46 if(size <= MaxSmallSize) { | 51 if(size <= MaxSmallSize) { |
47 // Allocate from mcache free lists. | 52 // Allocate from mcache free lists. |
48 » » sizeclass = runtime·SizeToClass(size); | 53 » » // Inlined version of SizeToClass(). |
| 54 » » if(size <= 1024-8) |
| 55 » » » sizeclass = runtime·size_to_class8[(size+7)>>3]; |
| 56 » » else |
| 57 » » » sizeclass = runtime·size_to_class128[(size-1024+127) >>
7]; |
49 size = runtime·class_to_size[sizeclass]; | 58 size = runtime·class_to_size[sizeclass]; |
50 v = runtime·MCache_Alloc(c, sizeclass, size, zeroed); | 59 v = runtime·MCache_Alloc(c, sizeclass, size, zeroed); |
51 if(v == nil) | 60 if(v == nil) |
52 runtime·throw("out of memory"); | 61 runtime·throw("out of memory"); |
53 c->local_alloc += size; | 62 c->local_alloc += size; |
54 c->local_total_alloc += size; | 63 c->local_total_alloc += size; |
55 c->local_by_size[sizeclass].nmalloc++; | 64 c->local_by_size[sizeclass].nmalloc++; |
56 } else { | 65 } else { |
57 // TODO(rsc): Report tracebacks for very large allocations. | 66 // TODO(rsc): Report tracebacks for very large allocations. |
58 | 67 |
59 // Allocate directly from heap. | 68 // Allocate directly from heap. |
60 npages = size >> PageShift; | 69 npages = size >> PageShift; |
61 if((size & PageMask) != 0) | 70 if((size & PageMask) != 0) |
62 npages++; | 71 npages++; |
63 » » s = runtime·MHeap_Alloc(&runtime·mheap, npages, 0, 1); | 72 » » s = runtime·MHeap_Alloc(runtime·mheap, npages, 0, 1, zeroed); |
64 if(s == nil) | 73 if(s == nil) |
65 runtime·throw("out of memory"); | 74 runtime·throw("out of memory"); |
66 size = npages<<PageShift; | 75 size = npages<<PageShift; |
67 c->local_alloc += size; | 76 c->local_alloc += size; |
68 c->local_total_alloc += size; | 77 c->local_total_alloc += size; |
69 v = (void*)(s->start << PageShift); | 78 v = (void*)(s->start << PageShift); |
70 | 79 |
71 // setup for mark sweep | 80 // setup for mark sweep |
72 runtime·markspan(v, 0, 0, true); | 81 runtime·markspan(v, 0, 0, true); |
73 } | 82 } |
| 83 |
| 84 if (sizeof(void*) == 4 && c->local_total_alloc >= (1<<30)) { |
| 85 // purge cache stats to prevent overflow |
| 86 runtime·lock(runtime·mheap); |
| 87 runtime·purgecachedstats(c); |
| 88 runtime·unlock(runtime·mheap); |
| 89 } |
| 90 |
74 if(!(flag & FlagNoGC)) | 91 if(!(flag & FlagNoGC)) |
75 runtime·markallocated(v, size, (flag&FlagNoPointers) != 0); | 92 runtime·markallocated(v, size, (flag&FlagNoPointers) != 0); |
| 93 |
| 94 if(DebugTypeAtBlockEnd) |
| 95 *(uintptr*)((uintptr)v+size-sizeof(uintptr)) = 0; |
76 | 96 |
77 m->mallocing = 0; | 97 m->mallocing = 0; |
78 | 98 |
79 if(!(flag & FlagNoProfiling) && (rate = runtime·MemProfileRate) > 0) { | 99 if(!(flag & FlagNoProfiling) && (rate = runtime·MemProfileRate) > 0) { |
80 if(size >= rate) | 100 if(size >= rate) |
81 goto profile; | 101 goto profile; |
82 if(m->mcache->next_sample > size) | 102 if(m->mcache->next_sample > size) |
83 m->mcache->next_sample -= size; | 103 m->mcache->next_sample -= size; |
84 else { | 104 else { |
85 // pick next profile time | 105 // pick next profile time |
86 // If you change this, also change allocmcache. | 106 // If you change this, also change allocmcache. |
87 if(rate > 0x3fffffff) // make 2*rate not overflow | 107 if(rate > 0x3fffffff) // make 2*rate not overflow |
88 rate = 0x3fffffff; | 108 rate = 0x3fffffff; |
89 m->mcache->next_sample = runtime·fastrand1() % (2*rate); | 109 m->mcache->next_sample = runtime·fastrand1() % (2*rate); |
90 profile: | 110 profile: |
91 runtime·setblockspecial(v, true); | 111 runtime·setblockspecial(v, true); |
92 runtime·MProf_Malloc(v, size); | 112 runtime·MProf_Malloc(v, size); |
93 } | 113 } |
94 } | 114 } |
95 | 115 |
96 if(dogc && mstats.heap_alloc >= mstats.next_gc) | 116 if(dogc && mstats.heap_alloc >= mstats.next_gc) |
97 runtime·gc(0); | 117 runtime·gc(0); |
| 118 |
| 119 if(raceenabled) { |
| 120 runtime·racemalloc(v, size, m->racepc); |
| 121 m->racepc = nil; |
| 122 } |
98 return v; | 123 return v; |
99 } | 124 } |
100 | 125 |
101 void* | 126 void* |
102 runtime·malloc(uintptr size) | 127 runtime·malloc(uintptr size) |
103 { | 128 { |
104 return runtime·mallocgc(size, 0, 0, 1); | 129 return runtime·mallocgc(size, 0, 0, 1); |
105 } | 130 } |
106 | 131 |
107 // Free the object whose base pointer is v. | 132 // Free the object whose base pointer is v. |
(...skipping 14 matching lines...) Expand all Loading... |
122 | 147 |
123 if(m->mallocing) | 148 if(m->mallocing) |
124 runtime·throw("malloc/free - deadlock"); | 149 runtime·throw("malloc/free - deadlock"); |
125 m->mallocing = 1; | 150 m->mallocing = 1; |
126 | 151 |
127 if(!runtime·mlookup(v, nil, nil, &s)) { | 152 if(!runtime·mlookup(v, nil, nil, &s)) { |
128 runtime·printf("free %p: not an allocated block\n", v); | 153 runtime·printf("free %p: not an allocated block\n", v); |
129 runtime·throw("free runtime·mlookup"); | 154 runtime·throw("free runtime·mlookup"); |
130 } | 155 } |
131 prof = runtime·blockspecial(v); | 156 prof = runtime·blockspecial(v); |
| 157 |
| 158 if(raceenabled) |
| 159 runtime·racefree(v); |
132 | 160 |
133 // Find size class for v. | 161 // Find size class for v. |
134 sizeclass = s->sizeclass; | 162 sizeclass = s->sizeclass; |
135 c = m->mcache; | 163 c = m->mcache; |
136 if(sizeclass == 0) { | 164 if(sizeclass == 0) { |
137 // Large object. | 165 // Large object. |
138 size = s->npages<<PageShift; | 166 size = s->npages<<PageShift; |
139 » » *(uintptr*)(s->start<<PageShift) = 1;» // mark as "needs to be
zeroed" | 167 » » *(uintptr*)(s->start<<PageShift) = (uintptr)0xfeedfeedfeedfeedll
;» // mark as "needs to be zeroed" |
140 // Must mark v freed before calling unmarkspan and MHeap_Free: | 168 // Must mark v freed before calling unmarkspan and MHeap_Free: |
141 // they might coalesce v into other spans and change the bitmap
further. | 169 // they might coalesce v into other spans and change the bitmap
further. |
142 runtime·markfreed(v, size); | 170 runtime·markfreed(v, size); |
143 runtime·unmarkspan(v, 1<<PageShift); | 171 runtime·unmarkspan(v, 1<<PageShift); |
144 » » runtime·MHeap_Free(&runtime·mheap, s, 1); | 172 » » runtime·MHeap_Free(runtime·mheap, s, 1); |
145 } else { | 173 } else { |
146 // Small object. | 174 // Small object. |
147 size = runtime·class_to_size[sizeclass]; | 175 size = runtime·class_to_size[sizeclass]; |
148 if(size > sizeof(uintptr)) | 176 if(size > sizeof(uintptr)) |
149 » » » ((uintptr*)v)[1] = 1;» // mark as "needs to be zeroed" | 177 » » » ((uintptr*)v)[1] = (uintptr)0xfeedfeedfeedfeedll;»
// mark as "needs to be zeroed" |
150 // Must mark v freed before calling MCache_Free: | 178 // Must mark v freed before calling MCache_Free: |
151 // it might coalesce v and other blocks into a bigger span | 179 // it might coalesce v and other blocks into a bigger span |
152 // and change the bitmap further. | 180 // and change the bitmap further. |
153 runtime·markfreed(v, size); | 181 runtime·markfreed(v, size); |
154 c->local_by_size[sizeclass].nfree++; | 182 c->local_by_size[sizeclass].nfree++; |
155 runtime·MCache_Free(c, v, sizeclass, size); | 183 runtime·MCache_Free(c, v, sizeclass, size); |
156 } | 184 } |
157 c->local_nfree++; | 185 c->local_nfree++; |
158 c->local_alloc -= size; | 186 c->local_alloc -= size; |
159 if(prof) | 187 if(prof) |
160 runtime·MProf_Free(v, size); | 188 runtime·MProf_Free(v, size); |
161 m->mallocing = 0; | 189 m->mallocing = 0; |
162 } | 190 } |
163 | 191 |
164 int32 | 192 int32 |
165 runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **sp) | 193 runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **sp) |
166 { | 194 { |
167 uintptr n, i; | 195 uintptr n, i; |
168 byte *p; | 196 byte *p; |
169 MSpan *s; | 197 MSpan *s; |
170 | 198 |
171 m->mcache->local_nlookup++; | 199 m->mcache->local_nlookup++; |
172 » s = runtime·MHeap_LookupMaybe(&runtime·mheap, v); | 200 » if (sizeof(void*) == 4 && m->mcache->local_nlookup >= (1<<30)) { |
| 201 » » // purge cache stats to prevent overflow |
| 202 » » runtime·lock(runtime·mheap); |
| 203 » » runtime·purgecachedstats(m->mcache); |
| 204 » » runtime·unlock(runtime·mheap); |
| 205 » } |
| 206 |
| 207 » s = runtime·MHeap_LookupMaybe(runtime·mheap, v); |
173 if(sp) | 208 if(sp) |
174 *sp = s; | 209 *sp = s; |
175 if(s == nil) { | 210 if(s == nil) { |
176 runtime·checkfreed(v, 1); | 211 runtime·checkfreed(v, 1); |
177 if(base) | 212 if(base) |
178 *base = nil; | 213 *base = nil; |
179 if(size) | 214 if(size) |
180 *size = 0; | 215 *size = 0; |
181 return 0; | 216 return 0; |
182 } | 217 } |
183 | 218 |
184 p = (byte*)((uintptr)s->start<<PageShift); | 219 p = (byte*)((uintptr)s->start<<PageShift); |
185 if(s->sizeclass == 0) { | 220 if(s->sizeclass == 0) { |
186 // Large object. | 221 // Large object. |
187 if(base) | 222 if(base) |
188 *base = p; | 223 *base = p; |
189 if(size) | 224 if(size) |
190 *size = s->npages<<PageShift; | 225 *size = s->npages<<PageShift; |
191 return 1; | 226 return 1; |
192 } | 227 } |
193 | 228 |
194 if((byte*)v >= (byte*)s->limit) { | 229 if((byte*)v >= (byte*)s->limit) { |
195 // pointers past the last block do not count as pointers. | 230 // pointers past the last block do not count as pointers. |
196 return 0; | 231 return 0; |
197 } | 232 } |
198 | 233 |
199 » n = runtime·class_to_size[s->sizeclass]; | 234 » n = s->elemsize; |
200 if(base) { | 235 if(base) { |
201 i = ((byte*)v - p)/n; | 236 i = ((byte*)v - p)/n; |
202 *base = p + i*n; | 237 *base = p + i*n; |
203 } | 238 } |
204 if(size) | 239 if(size) |
205 *size = n; | 240 *size = n; |
206 | 241 |
207 return 1; | 242 return 1; |
208 } | 243 } |
209 | 244 |
210 MCache* | 245 MCache* |
211 runtime·allocmcache(void) | 246 runtime·allocmcache(void) |
212 { | 247 { |
213 » int32 rate; | 248 » intgo rate; |
214 MCache *c; | 249 MCache *c; |
215 | 250 |
216 » runtime·lock(&runtime·mheap); | 251 » runtime·lock(runtime·mheap); |
217 » c = runtime·FixAlloc_Alloc(&runtime·mheap.cachealloc); | 252 » c = runtime·FixAlloc_Alloc(&runtime·mheap->cachealloc); |
218 » mstats.mcache_inuse = runtime·mheap.cachealloc.inuse; | 253 » mstats.mcache_inuse = runtime·mheap->cachealloc.inuse; |
219 » mstats.mcache_sys = runtime·mheap.cachealloc.sys; | 254 » mstats.mcache_sys = runtime·mheap->cachealloc.sys; |
220 » runtime·unlock(&runtime·mheap); | 255 » runtime·unlock(runtime·mheap); |
| 256 » runtime·memclr((byte*)c, sizeof(*c)); |
221 | 257 |
222 // Set first allocation sample size. | 258 // Set first allocation sample size. |
223 rate = runtime·MemProfileRate; | 259 rate = runtime·MemProfileRate; |
224 if(rate > 0x3fffffff) // make 2*rate not overflow | 260 if(rate > 0x3fffffff) // make 2*rate not overflow |
225 rate = 0x3fffffff; | 261 rate = 0x3fffffff; |
226 if(rate != 0) | 262 if(rate != 0) |
227 c->next_sample = runtime·fastrand1() % (2*rate); | 263 c->next_sample = runtime·fastrand1() % (2*rate); |
228 | 264 |
229 return c; | 265 return c; |
230 } | 266 } |
231 | 267 |
232 void | 268 void |
233 runtime·purgecachedstats(M* m) | 269 runtime·freemcache(MCache *c) |
234 { | 270 { |
235 » MCache *c; | 271 » runtime·MCache_ReleaseAll(c); |
| 272 » runtime·lock(runtime·mheap); |
| 273 » runtime·purgecachedstats(c); |
| 274 » runtime·FixAlloc_Free(&runtime·mheap->cachealloc, c); |
| 275 » runtime·unlock(runtime·mheap); |
| 276 } |
| 277 |
| 278 void |
| 279 runtime·purgecachedstats(MCache *c) |
| 280 { |
| 281 » int32 i; |
236 | 282 |
237 // Protected by either heap or GC lock. | 283 // Protected by either heap or GC lock. |
238 c = m->mcache; | |
239 mstats.heap_alloc += c->local_cachealloc; | 284 mstats.heap_alloc += c->local_cachealloc; |
240 c->local_cachealloc = 0; | 285 c->local_cachealloc = 0; |
241 mstats.heap_objects += c->local_objects; | 286 mstats.heap_objects += c->local_objects; |
242 c->local_objects = 0; | 287 c->local_objects = 0; |
243 mstats.nmalloc += c->local_nmalloc; | 288 mstats.nmalloc += c->local_nmalloc; |
244 c->local_nmalloc = 0; | 289 c->local_nmalloc = 0; |
245 mstats.nfree += c->local_nfree; | 290 mstats.nfree += c->local_nfree; |
246 c->local_nfree = 0; | 291 c->local_nfree = 0; |
247 mstats.nlookup += c->local_nlookup; | 292 mstats.nlookup += c->local_nlookup; |
248 c->local_nlookup = 0; | 293 c->local_nlookup = 0; |
249 mstats.alloc += c->local_alloc; | 294 mstats.alloc += c->local_alloc; |
250 c->local_alloc= 0; | 295 c->local_alloc= 0; |
251 mstats.total_alloc += c->local_total_alloc; | 296 mstats.total_alloc += c->local_total_alloc; |
252 c->local_total_alloc= 0; | 297 c->local_total_alloc= 0; |
| 298 for(i=0; i<nelem(c->local_by_size); i++) { |
| 299 mstats.by_size[i].nmalloc += c->local_by_size[i].nmalloc; |
| 300 c->local_by_size[i].nmalloc = 0; |
| 301 mstats.by_size[i].nfree += c->local_by_size[i].nfree; |
| 302 c->local_by_size[i].nfree = 0; |
| 303 } |
253 } | 304 } |
254 | 305 |
255 uintptr runtime·sizeof_C_MStats = sizeof(MStats); | 306 uintptr runtime·sizeof_C_MStats = sizeof(MStats); |
256 | 307 |
257 #define MaxArena32 (2U<<30) | 308 #define MaxArena32 (2U<<30) |
258 | 309 |
259 void | 310 void |
260 runtime·mallocinit(void) | 311 runtime·mallocinit(void) |
261 { | 312 { |
262 byte *p; | 313 byte *p; |
263 uintptr arena_size, bitmap_size; | 314 uintptr arena_size, bitmap_size; |
264 extern byte end[]; | 315 extern byte end[]; |
265 byte *want; | 316 byte *want; |
266 uintptr limit; | 317 uintptr limit; |
267 | 318 |
268 p = nil; | 319 p = nil; |
269 arena_size = 0; | 320 arena_size = 0; |
270 bitmap_size = 0; | 321 bitmap_size = 0; |
271 ········ | 322 ········ |
272 // for 64-bit build | 323 // for 64-bit build |
273 USED(p); | 324 USED(p); |
274 USED(arena_size); | 325 USED(arena_size); |
275 USED(bitmap_size); | 326 USED(bitmap_size); |
276 | 327 |
| 328 if((runtime·mheap = runtime·SysAlloc(sizeof(*runtime·mheap))) == nil) |
| 329 runtime·throw("runtime: cannot allocate heap metadata"); |
| 330 |
277 runtime·InitSizes(); | 331 runtime·InitSizes(); |
278 | 332 |
279 » limit = runtime·memlimit(); | 333 » // limit = runtime·memlimit(); |
| 334 » // See https://code.google.com/p/go/issues/detail?id=5049 |
| 335 » // TODO(rsc): Fix after 1.1. |
| 336 » limit = 0; |
280 | 337 |
281 // Set up the allocation arena, a contiguous area of memory where | 338 // Set up the allocation arena, a contiguous area of memory where |
282 // allocated data will be found. The arena begins with a bitmap large | 339 // allocated data will be found. The arena begins with a bitmap large |
283 // enough to hold 4 bits per allocated word. | 340 // enough to hold 4 bits per allocated word. |
284 if(sizeof(void*) == 8 && (limit == 0 || limit > (1<<30))) { | 341 if(sizeof(void*) == 8 && (limit == 0 || limit > (1<<30))) { |
285 // On a 64-bit machine, allocate from a single contiguous reserv
ation. | 342 // On a 64-bit machine, allocate from a single contiguous reserv
ation. |
286 » » // 16 GB should be big enough for now. | 343 » » // 128 GB (MaxMem) should be big enough for now. |
287 // | 344 // |
288 // The code will work with the reservation at any address, but a
sk | 345 // The code will work with the reservation at any address, but a
sk |
289 » » // SysReserve to use 0x000000f800000000 if possible. | 346 » » // SysReserve to use 0x000000c000000000 if possible. |
290 » » // Allocating a 16 GB region takes away 36 bits, and the amd64 | 347 » » // Allocating a 128 GB region takes away 37 bits, and the amd64 |
291 // doesn't let us choose the top 17 bits, so that leaves the 11
bits | 348 // doesn't let us choose the top 17 bits, so that leaves the 11
bits |
292 » » // in the middle of 0x00f8 for us to choose. Choosing 0x00f8 me
ans | 349 » » // in the middle of 0x00c0 for us to choose. Choosing 0x00c0 me
ans |
293 » » // that the valid memory addresses will begin 0x00f8, 0x00f9, 0x
00fa, 0x00fb. | 350 » » // that the valid memory addresses will begin 0x00c0, 0x00c1, ..
., 0x0x00df. |
294 » » // None of the bytes f8 f9 fa fb can appear in valid UTF-8, and | 351 » » // In little-endian, that's c0 00, c1 00, ..., df 00. None of th
ose are valid |
295 » » // they are otherwise as far from ff (likely a common byte) as p
ossible. | 352 » » // UTF-8 sequences, and they are otherwise as far away from· |
296 » » // Choosing 0x00 for the leading 6 bits was more arbitrary, but
it | 353 » » // ff (likely a common byte) as possible. An earlier attempt to
use 0x11f8· |
297 » » // is not a common ASCII code point either. Using 0x11f8 instea
d | |
298 // caused out of memory errors on OS X during thread allocations
. | 354 // caused out of memory errors on OS X during thread allocations
. |
299 // These choices are both for debuggability and to reduce the | 355 // These choices are both for debuggability and to reduce the |
300 // odds of the conservative garbage collector not collecting mem
ory | 356 // odds of the conservative garbage collector not collecting mem
ory |
301 // because some non-pointer block of memory had a bit pattern | 357 // because some non-pointer block of memory had a bit pattern |
302 // that matched a memory address. | 358 // that matched a memory address. |
303 // | 359 // |
304 » » // Actually we reserve 17 GB (because the bitmap ends up being 1
GB) | 360 » » // Actually we reserve 136 GB (because the bitmap ends up being
8 GB) |
305 » » // but it hardly matters: fc is not valid UTF-8 either, and we h
ave to | 361 » » // but it hardly matters: e0 00 is not valid UTF-8 either. |
306 » » // allocate 15 GB before we get that far. | |
307 // | 362 // |
308 // If this fails we fall back to the 32 bit memory mechanism | 363 // If this fails we fall back to the 32 bit memory mechanism |
309 » » arena_size = 16LL<<30; | 364 » » arena_size = MaxMem; |
310 bitmap_size = arena_size / (sizeof(void*)*8/4); | 365 bitmap_size = arena_size / (sizeof(void*)*8/4); |
311 » » p = runtime·SysReserve((void*)(0x00f8ULL<<32), bitmap_size + are
na_size); | 366 » » p = runtime·SysReserve((void*)(0x00c0ULL<<32), bitmap_size + are
na_size); |
312 } | 367 } |
313 if (p == nil) { | 368 if (p == nil) { |
314 // On a 32-bit machine, we can't typically get away | 369 // On a 32-bit machine, we can't typically get away |
315 // with a giant virtual address space reservation. | 370 // with a giant virtual address space reservation. |
316 // Instead we map the memory information bitmap | 371 // Instead we map the memory information bitmap |
317 // immediately after the data segment, large enough | 372 // immediately after the data segment, large enough |
318 // to handle another 2GB of mappings (256 MB), | 373 // to handle another 2GB of mappings (256 MB), |
319 // along with a reservation for another 512 MB of memory. | 374 // along with a reservation for another 512 MB of memory. |
320 // When that gets used up, we'll start asking the kernel | 375 // When that gets used up, we'll start asking the kernel |
321 // for any memory anywhere and hope it's in the 2GB | 376 // for any memory anywhere and hope it's in the 2GB |
(...skipping 25 matching lines...) Expand all Loading... |
347 want = (byte*)(((uintptr)end + (1<<18) + (1<<20) - 1)&~((1<<20)-
1)); | 402 want = (byte*)(((uintptr)end + (1<<18) + (1<<20) - 1)&~((1<<20)-
1)); |
348 p = runtime·SysReserve(want, bitmap_size + arena_size); | 403 p = runtime·SysReserve(want, bitmap_size + arena_size); |
349 if(p == nil) | 404 if(p == nil) |
350 runtime·throw("runtime: cannot reserve arena virtual add
ress space"); | 405 runtime·throw("runtime: cannot reserve arena virtual add
ress space"); |
351 if((uintptr)p & (((uintptr)1<<PageShift)-1)) | 406 if((uintptr)p & (((uintptr)1<<PageShift)-1)) |
352 runtime·printf("runtime: SysReserve returned unaligned a
ddress %p; asked for %p", p, bitmap_size+arena_size); | 407 runtime·printf("runtime: SysReserve returned unaligned a
ddress %p; asked for %p", p, bitmap_size+arena_size); |
353 } | 408 } |
354 if((uintptr)p & (((uintptr)1<<PageShift)-1)) | 409 if((uintptr)p & (((uintptr)1<<PageShift)-1)) |
355 runtime·throw("runtime: SysReserve returned unaligned address"); | 410 runtime·throw("runtime: SysReserve returned unaligned address"); |
356 | 411 |
357 » runtime·mheap.bitmap = p; | 412 » runtime·mheap->bitmap = p; |
358 » runtime·mheap.arena_start = p + bitmap_size; | 413 » runtime·mheap->arena_start = p + bitmap_size; |
359 » runtime·mheap.arena_used = runtime·mheap.arena_start; | 414 » runtime·mheap->arena_used = runtime·mheap->arena_start; |
360 » runtime·mheap.arena_end = runtime·mheap.arena_start + arena_size; | 415 » runtime·mheap->arena_end = runtime·mheap->arena_start + arena_size; |
361 | 416 |
362 // Initialize the rest of the allocator.········ | 417 // Initialize the rest of the allocator.········ |
363 » runtime·MHeap_Init(&runtime·mheap, runtime·SysAlloc); | 418 » runtime·MHeap_Init(runtime·mheap, runtime·SysAlloc); |
364 m->mcache = runtime·allocmcache(); | 419 m->mcache = runtime·allocmcache(); |
365 | 420 |
366 // See if it works. | 421 // See if it works. |
367 runtime·free(runtime·malloc(1)); | 422 runtime·free(runtime·malloc(1)); |
368 } | 423 } |
369 | 424 |
370 void* | 425 void* |
371 runtime·MHeap_SysAlloc(MHeap *h, uintptr n) | 426 runtime·MHeap_SysAlloc(MHeap *h, uintptr n) |
372 { | 427 { |
373 byte *p; | 428 byte *p; |
(...skipping 13 matching lines...) Expand all Loading... |
387 if(p == h->arena_end) | 442 if(p == h->arena_end) |
388 h->arena_end = new_end; | 443 h->arena_end = new_end; |
389 } | 444 } |
390 } | 445 } |
391 if(n <= h->arena_end - h->arena_used) { | 446 if(n <= h->arena_end - h->arena_used) { |
392 // Keep taking from our reservation. | 447 // Keep taking from our reservation. |
393 p = h->arena_used; | 448 p = h->arena_used; |
394 runtime·SysMap(p, n); | 449 runtime·SysMap(p, n); |
395 h->arena_used += n; | 450 h->arena_used += n; |
396 runtime·MHeap_MapBits(h); | 451 runtime·MHeap_MapBits(h); |
| 452 if(raceenabled) |
| 453 runtime·racemapshadow(p, n); |
397 return p; | 454 return p; |
398 } | 455 } |
399 ········ | 456 ········ |
400 // If using 64-bit, our reservation is all we have. | 457 // If using 64-bit, our reservation is all we have. |
401 if(sizeof(void*) == 8 && (uintptr)h->bitmap >= 0xffffffffU) | 458 if(sizeof(void*) == 8 && (uintptr)h->bitmap >= 0xffffffffU) |
402 return nil; | 459 return nil; |
403 | 460 |
404 // On 32-bit, once the reservation is gone we can | 461 // On 32-bit, once the reservation is gone we can |
405 // try to get memory at a location chosen by the OS | 462 // try to get memory at a location chosen by the OS |
406 // and hope that it is in the range we allocated bitmap for. | 463 // and hope that it is in the range we allocated bitmap for. |
407 p = runtime·SysAlloc(n); | 464 p = runtime·SysAlloc(n); |
408 if(p == nil) | 465 if(p == nil) |
409 return nil; | 466 return nil; |
410 | 467 |
411 if(p < h->arena_start || p+n - h->arena_start >= MaxArena32) { | 468 if(p < h->arena_start || p+n - h->arena_start >= MaxArena32) { |
412 runtime·printf("runtime: memory allocated by OS (%p) not in usab
le range [%p,%p)\n", | 469 runtime·printf("runtime: memory allocated by OS (%p) not in usab
le range [%p,%p)\n", |
413 p, h->arena_start, h->arena_start+MaxArena32); | 470 p, h->arena_start, h->arena_start+MaxArena32); |
414 runtime·SysFree(p, n); | 471 runtime·SysFree(p, n); |
415 return nil; | 472 return nil; |
416 } | 473 } |
417 | 474 |
418 if(p+n > h->arena_used) { | 475 if(p+n > h->arena_used) { |
419 h->arena_used = p+n; | 476 h->arena_used = p+n; |
420 if(h->arena_used > h->arena_end) | 477 if(h->arena_used > h->arena_end) |
421 h->arena_end = h->arena_used; | 478 h->arena_end = h->arena_used; |
422 runtime·MHeap_MapBits(h); | 479 runtime·MHeap_MapBits(h); |
| 480 if(raceenabled) |
| 481 runtime·racemapshadow(p, n); |
423 } | 482 } |
424 ········ | 483 ········ |
425 return p; | 484 return p; |
426 } | 485 } |
427 | 486 |
428 static Lock settype_lock; | 487 static Lock settype_lock; |
429 | 488 |
430 void | 489 void |
431 runtime·settype_flush(M *m, bool sysalloc) | 490 runtime·settype_flush(M *mp, bool sysalloc) |
432 { | 491 { |
433 » uintptr *buf = m->settype_buf; | 492 » uintptr *buf, *endbuf; |
434 » uintptr *endbuf = buf+m->settype_bufsize; | 493 » uintptr size, ofs, j, t; |
| 494 » uintptr ntypes, nbytes2, nbytes3; |
| 495 » uintptr *data2; |
| 496 » byte *data3; |
| 497 » bool sysalloc3; |
| 498 » void *v; |
| 499 » uintptr typ, p; |
| 500 » MSpan *s; |
| 501 |
| 502 » buf = mp->settype_buf; |
| 503 » endbuf = buf + mp->settype_bufsize; |
435 | 504 |
436 runtime·lock(&settype_lock); | 505 runtime·lock(&settype_lock); |
437 while(buf < endbuf) { | 506 while(buf < endbuf) { |
438 void *v; | |
439 uintptr typ, p; | |
440 MSpan *s; | |
441 ················ | |
442 v = (void*)*buf; | 507 v = (void*)*buf; |
443 » » *buf = 0; // Garbage collector may later find the pointer in th
e buffer, so it is better to reset it | 508 » » if(false) *buf = 0; |
444 buf++; | 509 buf++; |
445 typ = *buf; | 510 typ = *buf; |
446 buf++; | 511 buf++; |
447 | 512 |
448 // (Manually inlined copy of runtime·MHeap_Lookup) | 513 // (Manually inlined copy of runtime·MHeap_Lookup) |
449 p = (uintptr)v>>PageShift; | 514 p = (uintptr)v>>PageShift; |
450 if(sizeof(void*) == 8) | 515 if(sizeof(void*) == 8) |
451 » » » p -= (uintptr)runtime·mheap.arena_start >> PageShift; | 516 » » » p -= (uintptr)runtime·mheap->arena_start >> PageShift; |
452 » » s = runtime·mheap.map[p]; | 517 » » s = runtime·mheap->map[p]; |
453 | 518 |
454 » » if(s->sizeclass != 0) { | 519 » » if(s->sizeclass == 0) { |
455 » » » uintptr size = runtime·class_to_size[s->sizeclass]; | 520 » » » s->types.compression = MTypes_Single; |
456 » » » uintptr ofs = ((uintptr)v - (s->start<<PageShift)) / siz
e; | 521 » » » s->types.data = typ; |
457 | 522 » » » continue; |
458 » » » switch(s->types.compression) { | 523 » » } |
459 » » » » case 0: { | 524 |
460 » » » » » uintptr ntypes = (s->npages << PageShift
) / size; | 525 » » size = s->elemsize; |
461 » » » » » uintptr nbytes3 = 8*sizeof(uintptr) + 1*
ntypes; | 526 » » ofs = ((uintptr)v - (s->start<<PageShift)) / size; |
462 » » » » » byte *data3; | 527 |
463 | 528 » » switch(s->types.compression) { |
464 » » » » » if(!sysalloc) { | 529 » » case MTypes_Empty: |
465 » » » » » » data3 = runtime·mallocgc(nbytes3
, FlagNoPointers, 0, 1); | 530 » » » ntypes = (s->npages << PageShift) / size; |
466 » » » » » } else { | 531 » » » nbytes3 = 8*sizeof(uintptr) + 1*ntypes; |
467 » » » » » » data3 = runtime·SysAlloc(nbytes3
); | 532 |
468 » » » » » » if(0) runtime·printf("settype(0-
>3): SysAlloc(%x) --> %p\n", (uint32)nbytes3, data3); | 533 » » » if(!sysalloc) { |
469 » » » » » } | 534 » » » » data3 = runtime·mallocgc(nbytes3, FlagNoProfilin
g|FlagNoPointers, 0, 1); |
470 | 535 » » » } else { |
471 » » » » » s->types.compression = 3; | 536 » » » » data3 = runtime·SysAlloc(nbytes3); |
472 » » » » » s->types.sysalloc = sysalloc; | 537 » » » » if(data3 == nil) |
473 » » » » » s->types.data = (uintptr)data3; | 538 » » » » » runtime·throw("runtime: cannot allocate
memory"); |
474 | 539 » » » » if(0) runtime·printf("settype(0->3): SysAlloc(%x
) --> %p\n", (uint32)nbytes3, data3); |
475 » » » » » ((uintptr*)data3)[1] = typ; | 540 » » » } |
476 » » » » » data3[8*sizeof(uintptr) + ofs] = 1; | 541 |
| 542 » » » s->types.compression = MTypes_Bytes; |
| 543 » » » s->types.sysalloc = sysalloc; |
| 544 » » » s->types.data = (uintptr)data3; |
| 545 |
| 546 » » » ((uintptr*)data3)[1] = typ; |
| 547 » » » data3[8*sizeof(uintptr) + ofs] = 1; |
| 548 » » » break; |
| 549 |
| 550 » » case MTypes_Words: |
| 551 » » » ((uintptr*)s->types.data)[ofs] = typ; |
| 552 » » » break; |
| 553 |
| 554 » » case MTypes_Bytes: |
| 555 » » » data3 = (byte*)s->types.data; |
| 556 » » » for(j=1; j<8; j++) { |
| 557 » » » » if(((uintptr*)data3)[j] == typ) { |
477 break; | 558 break; |
478 } | 559 } |
479 » » » » case 2: { | 560 » » » » if(((uintptr*)data3)[j] == 0) { |
480 » » » » » ((uintptr*)s->types.data)[ofs] = typ; | 561 » » » » » ((uintptr*)data3)[j] = typ; |
481 » » » » » break; | |
482 » » » » } | |
483 » » » » case 3: { | |
484 » » » » » byte *data3 = (byte*)s->types.data; | |
485 » » » » » uintptr j; | |
486 » » » » » for(j=1; j<8; j++) { | |
487 » » » » » » if( ((uintptr*)data3)[j] == typ
) { | |
488 » » » » » » » break; | |
489 » » » » » » } | |
490 » » » » » » if( ((uintptr*)data3)[j] == 0 )
{ | |
491 » » » » » » » ((uintptr*)data3)[j] = t
yp; | |
492 » » » » » » » break; | |
493 » » » » » » } | |
494 » » » » » } | |
495 » » » » » if(j < 8) { | |
496 » » » » » » data3[8*sizeof(uintptr) + ofs] =
j; | |
497 » » » » » } else { | |
498 » » » » » » uintptr ntypes = (s->npages << P
ageShift) / size; | |
499 » » » » » » uintptr nbytes2 = ntypes * sizeo
f(uintptr); | |
500 » » » » » » uintptr *data2; | |
501 | |
502 » » » » » » if(!sysalloc) { | |
503 » » » » » » » data2 = runtime·mallocgc
(nbytes2, FlagNoPointers, 0, 1); | |
504 » » » » » » } else { | |
505 » » » » » » » data2 = runtime·SysAlloc
(nbytes2); | |
506 » » » » » » » if(0) runtime·printf("se
ttype.(3->2): SysAlloc(%x) --> %p\n", (uint32)nbytes2, data2); | |
507 » » » » » » } | |
508 | |
509 » » » » » » bool sysalloc3 = s->types.sysall
oc; | |
510 | |
511 » » » » » » s->types.compression = 2; | |
512 » » » » » » s->types.sysalloc = sysalloc; | |
513 » » » » » » s->types.data = (uintptr)data2; | |
514 | |
515 » » » » » » // Move the contents of data3 to
data2. Then deallocate data3. | |
516 » » » » » » for(j=0; j<ntypes; j++) { | |
517 » » » » » » » uintptr t; | |
518 » » » » » » » t = data3[8*sizeof(uintp
tr) + j]; | |
519 » » » » » » » t = ((uintptr*)data3)[t]
; | |
520 » » » » » » » data2[j] = t; | |
521 » » » » » » } | |
522 » » » » » » if(sysalloc3) { | |
523 » » » » » » » uintptr nbytes3 = 8*size
of(uintptr) + 1*ntypes; | |
524 » » » » » » » if(0) runtime·printf("se
ttype.(3->2): SysFree(%p,%x)\n", data3, (uint32)nbytes3); | |
525 » » » » » » » runtime·SysFree(data3, n
bytes3); | |
526 » » » » » » } | |
527 | |
528 » » » » » » data2[ofs] = typ; | |
529 » » » » » } | |
530 break; | 562 break; |
531 } | 563 } |
532 } | 564 } |
533 » » } else { | 565 » » » if(j < 8) { |
534 » » » s->types.compression = 1; | 566 » » » » data3[8*sizeof(uintptr) + ofs] = j; |
535 » » » s->types.data = typ; | 567 » » » } else { |
| 568 » » » » ntypes = (s->npages << PageShift) / size; |
| 569 » » » » nbytes2 = ntypes * sizeof(uintptr); |
| 570 |
| 571 » » » » if(!sysalloc) { |
| 572 » » » » » data2 = runtime·mallocgc(nbytes2, FlagNo
Profiling|FlagNoPointers, 0, 1); |
| 573 » » » » } else { |
| 574 » » » » » data2 = runtime·SysAlloc(nbytes2); |
| 575 » » » » » if(data2 == nil) |
| 576 » » » » » » runtime·throw("runtime: cannot a
llocate memory"); |
| 577 » » » » » if(0) runtime·printf("settype.(3->2): Sy
sAlloc(%x) --> %p\n", (uint32)nbytes2, data2); |
| 578 » » » » } |
| 579 |
| 580 » » » » sysalloc3 = s->types.sysalloc; |
| 581 |
| 582 » » » » s->types.compression = MTypes_Words; |
| 583 » » » » s->types.sysalloc = sysalloc; |
| 584 » » » » s->types.data = (uintptr)data2; |
| 585 |
| 586 » » » » // Move the contents of data3 to data2. Then dea
llocate data3. |
| 587 » » » » for(j=0; j<ntypes; j++) { |
| 588 » » » » » t = data3[8*sizeof(uintptr) + j]; |
| 589 » » » » » t = ((uintptr*)data3)[t]; |
| 590 » » » » » data2[j] = t; |
| 591 » » » » } |
| 592 » » » » if(sysalloc3) { |
| 593 » » » » » nbytes3 = 8*sizeof(uintptr) + 1*ntypes; |
| 594 » » » » » if(0) runtime·printf("settype.(3->2): Sy
sFree(%p,%x)\n", data3, (uint32)nbytes3); |
| 595 » » » » » runtime·SysFree(data3, nbytes3); |
| 596 » » » » } |
| 597 |
| 598 » » » » data2[ofs] = typ; |
| 599 » » » } |
| 600 » » » break; |
536 } | 601 } |
537 } | 602 } |
538 runtime·unlock(&settype_lock); | 603 runtime·unlock(&settype_lock); |
539 | 604 |
540 » m->settype_bufsize = 0; | 605 » mp->settype_bufsize = 0; |
541 } | 606 } |
542 | 607 |
543 // It is forbidden to use this function if it is possible that | 608 // It is forbidden to use this function if it is possible that |
544 // explicit deallocation by calling runtime·free(v) may happen. | 609 // explicit deallocation via calling runtime·free(v) may happen. |
545 void | 610 void |
546 runtime·settype(void *v, uintptr t) | 611 runtime·settype(void *v, uintptr t) |
547 { | 612 { |
548 » M *m1; | 613 » M *mp; |
549 uintptr *buf; | 614 uintptr *buf; |
550 uintptr i; | 615 uintptr i; |
| 616 MSpan *s; |
551 | 617 |
552 if(t == 0) | 618 if(t == 0) |
553 runtime·throw("settype: zero type"); | 619 runtime·throw("settype: zero type"); |
554 | 620 |
555 » m1 = m; | 621 » mp = m; |
556 » buf = m1->settype_buf; | 622 » buf = mp->settype_buf; |
557 » i = m1->settype_bufsize; | 623 » i = mp->settype_bufsize; |
558 buf[i+0] = (uintptr)v; | 624 buf[i+0] = (uintptr)v; |
559 buf[i+1] = t; | 625 buf[i+1] = t; |
560 i += 2; | 626 i += 2; |
561 » m1->settype_bufsize = i; | 627 » mp->settype_bufsize = i; |
562 | 628 |
563 » if(i == nelem(m1->settype_buf)) { | 629 » if(i == nelem(mp->settype_buf)) { |
564 » » runtime·settype_flush(m1, false); | 630 » » runtime·settype_flush(mp, false); |
| 631 » } |
| 632 |
| 633 » if(DebugTypeAtBlockEnd) { |
| 634 » » s = runtime·MHeap_Lookup(runtime·mheap, v); |
| 635 » » *(uintptr*)((uintptr)v+s->elemsize-sizeof(uintptr)) = t; |
565 } | 636 } |
566 } | 637 } |
567 | 638 |
568 void | 639 void |
569 runtime·settype_sysfree(MSpan *s) | 640 runtime·settype_sysfree(MSpan *s) |
570 { | 641 { |
571 » uintptr nbytes; | 642 » uintptr ntypes, nbytes; |
572 | 643 |
573 if(!s->types.sysalloc) | 644 if(!s->types.sysalloc) |
574 return; | 645 return; |
575 | 646 |
576 nbytes = (uintptr)-1; | 647 nbytes = (uintptr)-1; |
577 » if(s->types.compression==2) { | 648 |
578 » » uintptr size = runtime·class_to_size[s->sizeclass]; | 649 » switch (s->types.compression) { |
579 » » uintptr ntypes = (s->npages << PageShift) / size; | 650 » case MTypes_Words: |
| 651 » » ntypes = (s->npages << PageShift) / s->elemsize; |
580 nbytes = ntypes * sizeof(uintptr); | 652 nbytes = ntypes * sizeof(uintptr); |
581 » } | 653 » » break; |
582 » else if(s->types.compression==3) { | 654 » case MTypes_Bytes: |
583 » » uintptr size = runtime·class_to_size[s->sizeclass]; | 655 » » ntypes = (s->npages << PageShift) / s->elemsize; |
584 » » uintptr ntypes = (s->npages << PageShift) / size; | |
585 nbytes = 8*sizeof(uintptr) + 1*ntypes; | 656 nbytes = 8*sizeof(uintptr) + 1*ntypes; |
| 657 break; |
586 } | 658 } |
587 | 659 |
588 if(nbytes != (uintptr)-1) { | 660 if(nbytes != (uintptr)-1) { |
589 if(0) runtime·printf("settype: SysFree(%p,%x)\n", (void*)s->type
s.data, (uint32)nbytes); | 661 if(0) runtime·printf("settype: SysFree(%p,%x)\n", (void*)s->type
s.data, (uint32)nbytes); |
590 runtime·SysFree((void*)s->types.data, nbytes); | 662 runtime·SysFree((void*)s->types.data, nbytes); |
591 } | 663 } |
592 } | 664 } |
593 | 665 |
594 uintptr | 666 uintptr |
595 runtime·gettype(void *v) { | 667 runtime·gettype(void *v) |
596 » MSpan *s = runtime·MHeap_LookupMaybe(&runtime·mheap, v); | 668 { |
| 669 » MSpan *s; |
| 670 » uintptr t, ofs; |
| 671 » byte *data; |
| 672 |
| 673 » s = runtime·MHeap_LookupMaybe(runtime·mheap, v); |
597 if(s != nil) { | 674 if(s != nil) { |
598 » » uintptr t = 0; | 675 » » t = 0; |
599 switch(s->types.compression) { | 676 switch(s->types.compression) { |
600 » » » case 0: | 677 » » case MTypes_Empty: |
601 » » » » break; | 678 » » » break; |
602 » » » case 1: | 679 » » case MTypes_Single: |
603 » » » » t = s->types.data; | 680 » » » t = s->types.data; |
604 » » » » break; | 681 » » » break; |
605 » » » case 2: { | 682 » » case MTypes_Words: |
606 » » » » uintptr size = runtime·class_to_size[s->sizeclas
s]; | 683 » » » ofs = (uintptr)v - (s->start<<PageShift); |
607 » » » » uintptr ofs = (uintptr)v - (s->start<<PageShift)
; | 684 » » » t = ((uintptr*)s->types.data)[ofs/s->elemsize]; |
608 » » » » t = ((uintptr*)s->types.data)[ofs/size]; | 685 » » » break; |
609 » » » » break; | 686 » » case MTypes_Bytes: |
610 » » » } | 687 » » » ofs = (uintptr)v - (s->start<<PageShift); |
611 » » » case 3: { | 688 » » » data = (byte*)s->types.data; |
612 » » » » uintptr size = runtime·class_to_size[s->sizeclas
s]; | 689 » » » t = data[8*sizeof(uintptr) + ofs/s->elemsize]; |
613 » » » » uintptr ofs = (uintptr)v - (s->start<<PageShift)
; | 690 » » » t = ((uintptr*)data)[t]; |
614 » » » » byte *data = (byte*)s->types.data; | 691 » » » break; |
615 » » » » t = data[8*sizeof(uintptr) + ofs/size]; | 692 » » default: |
616 » » » » t = ((uintptr*)data)[t]; | 693 » » » runtime·throw("runtime·gettype: invalid compression kind
"); |
617 » » » » break; | |
618 » » » } | |
619 » » » default: | |
620 » » » » runtime·throw("runtime·gettype: invalid compress
ion kind"); | |
621 } | 694 } |
622 if(0) { | 695 if(0) { |
623 » » » //runtime·lock(&settype_lock); | 696 » » » runtime·lock(&settype_lock); |
624 runtime·printf("%p -> %d,%X\n", v, (int32)s->types.compr
ession, (int64)t); | 697 runtime·printf("%p -> %d,%X\n", v, (int32)s->types.compr
ession, (int64)t); |
625 » » » //runtime·unlock(&settype_lock); | 698 » » » runtime·unlock(&settype_lock); |
626 } | 699 } |
627 return t; | 700 return t; |
628 } | 701 } |
629 return 0; | 702 return 0; |
630 } | 703 } |
631 | 704 |
632 // Runtime stubs. | 705 // Runtime stubs. |
633 | 706 |
634 void* | 707 void* |
635 runtime·mal(uintptr n) | 708 runtime·mal(uintptr n) |
636 { | 709 { |
637 return runtime·mallocgc(n, 0, 1, 1); | 710 return runtime·mallocgc(n, 0, 1, 1); |
638 } | 711 } |
639 | 712 |
640 func new(typ *Type) (ret *uint8) { | 713 #pragma textflag 7 |
641 » uint32 flag = typ->kind&KindNoPointers ? FlagNoPointers : 0; | 714 void |
642 » ret = runtime·mallocgc(typ->size, flag, 1, 1); | 715 runtime·new(Type *typ, uint8 *ret) |
643 | 716 { |
644 » if(UseSpanType && !flag) { | 717 » uint32 flag; |
645 » » //runtime·printf("new %S\n", *typ->string); | 718 |
646 » » runtime·settype(ret, (uintptr)typ); | 719 » if(raceenabled) |
| 720 » » m->racepc = runtime·getcallerpc(&typ); |
| 721 |
| 722 » if(typ->size == 0) { |
| 723 » » // All 0-length allocations use this pointer. |
| 724 » » // The language does not require the allocations to |
| 725 » » // have distinct values. |
| 726 » » ret = (uint8*)&runtime·zerobase; |
| 727 » } else { |
| 728 » » flag = typ->kind&KindNoPointers ? FlagNoPointers : 0; |
| 729 » » ret = runtime·mallocgc(typ->size, flag, 1, 1); |
| 730 |
| 731 » » if(UseSpanType && !flag) { |
| 732 » » » if(false) { |
| 733 » » » » runtime·printf("new %S: %p\n", *typ->string, ret
); |
| 734 » » » } |
| 735 » » » runtime·settype(ret, (uintptr)typ | TypeInfo_SingleObjec
t); |
| 736 » » } |
647 } | 737 } |
648 | 738 |
649 FLUSH(&ret); | 739 FLUSH(&ret); |
650 } | 740 } |
651 | 741 |
| 742 // same as runtime·new, but callable from C |
652 void* | 743 void* |
653 runtime·stackalloc(uint32 n) | 744 runtime·cnew(Type *typ) |
654 { | 745 { |
655 » // Stackalloc must be called on scheduler stack, so that we | 746 » uint32 flag; |
656 » // never try to grow the stack during the code that stackalloc runs. | 747 » void *ret; |
657 » // Doing so would cause a deadlock (issue 1547). | 748 |
658 » if(g != m->g0) | 749 » if(raceenabled) |
659 » » runtime·throw("stackalloc not on scheduler stack"); | 750 » » m->racepc = runtime·getcallerpc(&typ); |
660 | 751 |
661 » // Stack allocator uses malloc/free most of the time, | 752 » if(typ->size == 0) { |
662 » // but if we're in the middle of malloc and need stack, | 753 » » // All 0-length allocations use this pointer. |
663 » // we have to do something else to avoid deadlock. | 754 » » // The language does not require the allocations to |
664 » // In that case, we fall back on a fixed-size free-list | 755 » » // have distinct values. |
665 » // allocator, assuming that inside malloc all the stack | 756 » » ret = (uint8*)&runtime·zerobase; |
666 » // frames are small, so that all the stack allocations | 757 » } else { |
667 » // will be a single size, the minimum (right now, 5k). | 758 » » flag = typ->kind&KindNoPointers ? FlagNoPointers : 0; |
668 » if(m->mallocing || m->gcing || n == FixedStack) { | 759 » » ret = runtime·mallocgc(typ->size, flag, 1, 1); |
669 » » if(n != FixedStack) { | 760 |
670 » » » runtime·printf("stackalloc: in malloc, size=%d want %d",
FixedStack, n); | 761 » » if(UseSpanType && !flag) { |
671 » » » runtime·throw("stackalloc"); | 762 » » » if(false) { |
672 » » } | 763 » » » » runtime·printf("new %S: %p\n", *typ->string, ret
); |
673 » » return runtime·FixAlloc_Alloc(m->stackalloc); | 764 » » » } |
674 » } | 765 » » » runtime·settype(ret, (uintptr)typ | TypeInfo_SingleObjec
t); |
675 » return runtime·mallocgc(n, FlagNoProfiling|FlagNoGC, 0, 0); | 766 » » } |
676 } | 767 » } |
677 | 768 |
678 void | 769 » return ret; |
679 runtime·stackfree(void *v, uintptr n) | |
680 { | |
681 » if(m->mallocing || m->gcing || n == FixedStack) { | |
682 » » runtime·FixAlloc_Free(m->stackalloc, v); | |
683 » » return; | |
684 » } | |
685 » runtime·free(v); | |
686 } | 770 } |
687 | 771 |
688 func GC() { | 772 func GC() { |
689 runtime·gc(1); | 773 runtime·gc(1); |
690 } | 774 } |
691 | 775 |
692 func SetFinalizer(obj Eface, finalizer Eface) { | 776 func SetFinalizer(obj Eface, finalizer Eface) { |
693 byte *base; | 777 byte *base; |
694 uintptr size; | 778 uintptr size; |
695 FuncType *ft; | 779 FuncType *ft; |
696 » int32 i, nret; | 780 » int32 i; |
| 781 » uintptr nret; |
697 Type *t; | 782 Type *t; |
698 | 783 |
699 if(obj.type == nil) { | 784 if(obj.type == nil) { |
700 runtime·printf("runtime.SetFinalizer: first argument is nil inte
rface\n"); | 785 runtime·printf("runtime.SetFinalizer: first argument is nil inte
rface\n"); |
701 goto throw; | 786 goto throw; |
702 } | 787 } |
703 if(obj.type->kind != KindPtr) { | 788 if(obj.type->kind != KindPtr) { |
704 runtime·printf("runtime.SetFinalizer: first argument is %S, not
pointer\n", *obj.type->string); | 789 runtime·printf("runtime.SetFinalizer: first argument is %S, not
pointer\n", *obj.type->string); |
705 goto throw; | 790 goto throw; |
706 } | 791 } |
(...skipping 22 matching lines...) Expand all Loading... |
729 runtime·printf("runtime.SetFinalizer: finalizer already set\n"); | 814 runtime·printf("runtime.SetFinalizer: finalizer already set\n"); |
730 goto throw; | 815 goto throw; |
731 } | 816 } |
732 return; | 817 return; |
733 | 818 |
734 badfunc: | 819 badfunc: |
735 runtime·printf("runtime.SetFinalizer: second argument is %S, not func(%S
)\n", *finalizer.type->string, *obj.type->string); | 820 runtime·printf("runtime.SetFinalizer: second argument is %S, not func(%S
)\n", *finalizer.type->string, *obj.type->string); |
736 throw: | 821 throw: |
737 runtime·throw("runtime.SetFinalizer"); | 822 runtime·throw("runtime.SetFinalizer"); |
738 } | 823 } |
LEFT | RIGHT |