LEFT | RIGHT |
1 // Copyright 2013 The Go Authors. All rights reserved. | 1 // Copyright 2013 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // +build race | 5 // +build race |
6 | 6 |
7 //#include "zasm_GOOS_GOARCH.h" | 7 #include "zasm_GOOS_GOARCH.h" |
8 #include "zasm_linux_amd64.h" | |
9 #include "funcdata.h" | 8 #include "funcdata.h" |
10 #include "../../cmd/ld/textflag.h" | 9 #include "../../cmd/ld/textflag.h" |
11 | 10 |
12 // Called from instrumented code. | 11 // The following thunks allow calling the gcc-compiled race runtime directly |
13 // If we split stack, getcallerpc() can return runtime·lessstack(). | 12 // from Go code without going all the way through cgo. |
14 | 13 // First, it's much faster (up to 50% speedup for real Go programs). |
15 //!!! ensure than heap is there | 14 // Second, it eliminates race-related special cases from cgocall and scheduler. |
16 #define HEAPSTART» 0x00c000000000 | 15 // Third, in long-term it will allow to remove cyclic runtime/race dependency on
cmd/go. |
17 #define HEAPEND»» 0x00e000000000 | 16 |
18 | 17 // A brief recap of the amd64 calling convention. |
19 // RDI, RSI, RDX, RCX, R8, R9,· | 18 // Arguments are passed in DI, SI, DX, CX, R8, R9, the rest is on stack. |
| 19 // Callee-saved registers are: BX, BP, R12-R15. |
| 20 // SP must be 16-byte aligned. |
| 21 // On Windows: |
| 22 // Arguments are passed in CX, DX, R8, R9, the rest is on stack. |
| 23 // Callee-saved registers are: BX, BP, DI, SI, R12-R15. |
| 24 // SP must be 16-byte aligned. Windows also requires "stack-backing" for the 4 r
egister arguments: |
| 25 // http://msdn.microsoft.com/en-us/library/ms235286.aspx |
| 26 // We do not do this, because it seems to be intended for vararg/unprototyped fu
nctions. |
| 27 // Gcc-compiled race runtime does not try to use that space. |
| 28 |
| 29 #ifdef GOOS_windows |
| 30 #define RARG0 CX |
| 31 #define RARG1 DX |
| 32 #define RARG2 R8 |
| 33 #define RARG3 R9 |
| 34 #else |
| 35 #define RARG0 DI |
| 36 #define RARG1 SI |
| 37 #define RARG2 DX |
| 38 #define RARG3 CX |
| 39 #endif |
20 | 40 |
21 // func runtime·raceread(addr uintptr) | 41 // func runtime·raceread(addr uintptr) |
| 42 // Called from instrumented code. |
22 TEXT runtime·raceread(SB), NOSPLIT, $0-8 | 43 TEXT runtime·raceread(SB), NOSPLIT, $0-8 |
23 » get_tls(R12) | 44 » MOVQ» addr+0(FP), RARG1 |
24 » MOVQ» m(R12), R13 | 45 » MOVQ» (SP), RARG2 |
25 » MOVQ» g(R12), R14 | |
26 » // Prepare arguments for the call. | |
27 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | |
28 » MOVQ» 8(SP), SI» » // 1th arg: access addr | |
29 » MOVQ» (SP), DX» » // 2th arg: caller pc | |
30 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | |
31 » CMPQ» SI, runtime·racearenastart(SB) | |
32 » JB» raceread_data | |
33 » CMPQ» SI, runtime·racearenaend(SB) | |
34 » JB» raceread_instrument | |
35 raceread_data: | |
36 » CMPQ» SI, $noptrdata(SB) | |
37 » JB» raceread_ret | |
38 » CMPQ» SI, $enoptrbss(SB) | |
39 » JAE» raceread_ret | |
40 raceread_instrument: | |
41 | |
42 » /* | |
43 » MOVQ» SI, R15 | |
44 » SHRQ» $40, R15 | |
45 » CMPQ» R15, $0x7e | |
46 » JAE» raceread_ret | |
47 » */ | |
48 | |
49 /* | |
50 » MOVQ» SI, R15 | |
51 » SHRQ» $36, R15 | |
52 » CMPQ» R15, $0xc | |
53 » JB» raceread_data | |
54 » CMPQ» R15, $0xe | |
55 » JB» raceread_instrument | |
56 raceread_data: | |
57 » CMPQ» SI, $noptrdata(SB) | |
58 » JB» raceread_ret | |
59 » CMPQ» SI, $enoptrbss(SB) | |
60 » JAE» raceread_ret | |
61 raceread_instrument: | |
62 */ | |
63 » // Switch to g0 stack. | |
64 » MOVQ» SP, R12 | |
65 » MOVQ» m_g0(R13), R15 | |
66 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
67 // void __tsan_read(ThreadState *thr, void *addr, void *pc); | 46 // void __tsan_read(ThreadState *thr, void *addr, void *pc); |
68 » CALL» __tsan_read(SB) | 47 » MOVQ» $__tsan_read(SB), AX |
69 » MOVQ» R12, SP | 48 » JMP» racecalladdr<>(SB) |
70 raceread_ret: | |
71 » RET | |
72 | 49 |
73 // func runtime·RaceRead(addr uintptr) | 50 // func runtime·RaceRead(addr uintptr) |
74 TEXT runtime·RaceRead(SB), NOSPLIT, $0-8 | 51 TEXT runtime·RaceRead(SB), NOSPLIT, $0-8 |
| 52 // This needs to be a tail call, because raceread reads caller pc. |
75 JMP runtime·raceread(SB) | 53 JMP runtime·raceread(SB) |
76 | 54 |
77 // void runtime·racereadpc(void *addr, void *callpc, void *pc) | 55 // void runtime·racereadpc(void *addr, void *callpc, void *pc) |
78 TEXT runtime·racereadpc(SB), NOSPLIT, $0-24 | 56 TEXT runtime·racereadpc(SB), NOSPLIT, $0-24 |
79 » get_tls(R12) | 57 » MOVQ» addr+0(FP), RARG1 |
80 » MOVQ» m(R12), R13 | 58 » MOVQ» callpc+8(FP), RARG2 |
81 » MOVQ» g(R12), R14 | 59 » MOVQ» pc+16(FP), RARG3 |
82 » // Prepare arguments for the call. | |
83 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | |
84 » MOVQ» 8(SP), SI» » // 1th arg: access addr | |
85 » MOVQ» 16(SP), DX» » // 2th arg: caller caller pc | |
86 » MOVQ» 24(SP), CX» » // 3th arg: caller pc | |
87 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | |
88 | |
89 » CMPQ» SI, runtime·racearenastart(SB) | |
90 » JB» racereadpc_data | |
91 » CMPQ» SI, runtime·racearenaend(SB) | |
92 » JB» racereadpc_instrument | |
93 racereadpc_data: | |
94 » CMPQ» SI, $noptrdata(SB) | |
95 » JB» racereadpc_ret | |
96 » CMPQ» SI, $enoptrbss(SB) | |
97 » JAE» racereadpc_ret | |
98 racereadpc_instrument: | |
99 | |
100 » /* | |
101 » MOVQ» SI, R15 | |
102 » SHRQ» $40, R15 | |
103 » CMPQ» R15, $0x7e | |
104 » JAE» racereadpc_ret | |
105 » */ | |
106 | |
107 /* | |
108 » MOVQ» SI, R15 | |
109 » SHRQ» $36, R15 | |
110 » CMPQ» R15, $0xc | |
111 » JB» racereadpc_data | |
112 » CMPQ» R15, $0xe | |
113 » JB» racereadpc_instrument | |
114 racereadpc_data: | |
115 » CMPQ» SI, $noptrdata(SB) | |
116 » JB» racereadpc_ret | |
117 » CMPQ» SI, $enoptrbss(SB) | |
118 » JAE» racereadpc_ret | |
119 racereadpc_instrument: | |
120 */ | |
121 » // Switch to g0 stack. | |
122 » MOVQ» SP, R12 | |
123 » MOVQ» m_g0(R13), R15 | |
124 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
125 // void __tsan_read_pc(ThreadState *thr, void *addr, void *callpc, void
*pc); | 60 // void __tsan_read_pc(ThreadState *thr, void *addr, void *callpc, void
*pc); |
126 » CALL» __tsan_read_pc(SB) | 61 » MOVQ» $__tsan_read_pc(SB), AX |
127 » MOVQ» R12, SP | 62 » JMP» racecalladdr<>(SB) |
128 racereadpc_ret: | |
129 » RET | |
130 | 63 |
131 // func runtime·racewrite(addr uintptr) | 64 // func runtime·racewrite(addr uintptr) |
| 65 // Called from instrumented code. |
132 TEXT runtime·racewrite(SB), NOSPLIT, $0-8 | 66 TEXT runtime·racewrite(SB), NOSPLIT, $0-8 |
133 » get_tls(R12) | 67 » MOVQ» addr+0(FP), RARG1 |
134 » MOVQ» m(R12), R13 | 68 » MOVQ» (SP), RARG2 |
135 » MOVQ» g(R12), R14 | |
136 » // Prepare arguments for the call. | |
137 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | |
138 » MOVQ» 8(SP), SI» » // 1th arg: access addr | |
139 » MOVQ» (SP), DX» » // 2th arg: caller pc | |
140 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | |
141 » CMPQ» SI, runtime·racearenastart(SB) | |
142 » JB» racewrite_data | |
143 » CMPQ» SI, runtime·racearenaend(SB) | |
144 » JB» racewrite_instrument | |
145 racewrite_data: | |
146 » CMPQ» SI, $noptrdata(SB) | |
147 » JB» racewrite_ret | |
148 » CMPQ» SI, $enoptrbss(SB) | |
149 » JAE» racewrite_ret | |
150 racewrite_instrument: | |
151 | |
152 » /* | |
153 » MOVQ» SI, R15 | |
154 » SHRQ» $40, R15 | |
155 » CMPQ» R15, $0x7e | |
156 » JAE» racewrite_ret | |
157 */ | |
158 /* | |
159 » MOVQ» SI, R15 | |
160 » SHRQ» $36, R15 | |
161 » CMPQ» R15, $0xc | |
162 » JB» racewrite_data | |
163 » CMPQ» R15, $0xe | |
164 » JB» racewrite_instrument | |
165 racewrite_data: | |
166 » CMPQ» SI, $noptrdata(SB) | |
167 » JB» racewrite_ret | |
168 » CMPQ» SI, $enoptrbss(SB) | |
169 » JAE» racewrite_ret | |
170 racewrite_instrument: | |
171 */ | |
172 » // Switch to g0 stack. | |
173 » MOVQ» SP, R12 | |
174 » MOVQ» m_g0(R13), R15 | |
175 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
176 // void __tsan_write(ThreadState *thr, void *addr, void *pc); | 69 // void __tsan_write(ThreadState *thr, void *addr, void *pc); |
177 » CALL» __tsan_write(SB) | 70 » MOVQ» $__tsan_write(SB), AX |
178 » MOVQ» R12, SP | 71 » JMP» racecalladdr<>(SB) |
179 racewrite_ret: | |
180 » RET | |
181 | 72 |
182 // func runtime·RaceWrite(addr uintptr) | 73 // func runtime·RaceWrite(addr uintptr) |
183 TEXT runtime·RaceWrite(SB), NOSPLIT, $0-8 | 74 TEXT runtime·RaceWrite(SB), NOSPLIT, $0-8 |
| 75 // This needs to be a tail call, because racewrite reads caller pc. |
184 JMP runtime·racewrite(SB) | 76 JMP runtime·racewrite(SB) |
185 | 77 |
186 // void runtime·racewritepc(void *addr, void *callpc, void *pc) | 78 // void runtime·racewritepc(void *addr, void *callpc, void *pc) |
187 TEXT runtime·racewritepc(SB), NOSPLIT, $0-24 | 79 TEXT runtime·racewritepc(SB), NOSPLIT, $0-24 |
188 » get_tls(R12) | 80 » MOVQ» addr+0(FP), RARG1 |
189 » MOVQ» m(R12), R13 | 81 » MOVQ» callpc+8(FP), RARG2 |
190 » MOVQ» g(R12), R14 | 82 » MOVQ» cp+16(FP), RARG3 |
191 » // Prepare arguments for the call. | |
192 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | |
193 » MOVQ» 8(SP), SI» » // 1th arg: access addr | |
194 » MOVQ» 16(SP), DX» » // 2th arg: caller caller pc | |
195 » MOVQ» 24(SP), CX» » // 3th arg: caller pc | |
196 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | |
197 » CMPQ» SI, runtime·racearenastart(SB) | |
198 » JB» racewritepc_data | |
199 » CMPQ» SI, runtime·racearenaend(SB) | |
200 » JB» racewritepc_instrument | |
201 racewritepc_data: | |
202 » CMPQ» SI, $noptrdata(SB) | |
203 » JB» racewritepc_ret | |
204 » CMPQ» SI, $enoptrbss(SB) | |
205 » JAE» racewritepc_ret | |
206 racewritepc_instrument: | |
207 | |
208 » /* | |
209 » MOVQ» SI, R15 | |
210 » SHRQ» $40, R15 | |
211 » CMPQ» R15, $0x7e | |
212 » JAE» racewritepc_ret | |
213 » */ | |
214 »······· | |
215 » /* | |
216 » MOVQ» SI, R15 | |
217 » SHRQ» $36, R15 | |
218 » CMPQ» R15, $0xc | |
219 » JB» racewritepc_data | |
220 » CMPQ» R15, $0xe | |
221 » JB» racewritepc_instrument | |
222 racewritepc_data: | |
223 » CMPQ» SI, $noptrdata(SB) | |
224 » JB» racewritepc_ret | |
225 » CMPQ» SI, $enoptrbss(SB) | |
226 » JAE» racewritepc_ret | |
227 racewritepc_instrument: | |
228 */ | |
229 » // Switch to g0 stack. | |
230 » MOVQ» SP, R12 | |
231 » MOVQ» m_g0(R13), R15 | |
232 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
233 // void __tsan_write_pc(ThreadState *thr, void *addr, void *callpc, void
*pc); | 83 // void __tsan_write_pc(ThreadState *thr, void *addr, void *callpc, void
*pc); |
234 » CALL» __tsan_write_pc(SB) | 84 » MOVQ» $__tsan_write_pc(SB), AX |
235 » MOVQ» R12, SP | 85 » JMP» racecalladdr<>(SB) |
236 racewritepc_ret: | |
237 » RET | |
238 | 86 |
239 // func runtime·racereadrange(addr, size uintptr) | 87 // func runtime·racereadrange(addr, size uintptr) |
| 88 // Called from instrumented code. |
240 TEXT runtime·racereadrange(SB), NOSPLIT, $0-16 | 89 TEXT runtime·racereadrange(SB), NOSPLIT, $0-16 |
241 » get_tls(R12) | 90 » MOVQ» addr+0(FP), RARG1 |
242 » MOVQ» m(R12), R13 | 91 » MOVQ» size+8(FP), RARG2 |
243 » MOVQ» g(R12), R14 | 92 » MOVQ» (SP), RARG3 |
244 » // Prepare arguments for the call. | |
245 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | |
246 » MOVQ» 8(SP), SI» » // 1th arg: access addr | |
247 » MOVQ» 16(SP), DX» » // 2th arg: access size | |
248 » MOVQ» (SP), CX» » // 3th arg: caller pc | |
249 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | |
250 » CMPQ» SI, runtime·racearenastart(SB) | |
251 » JB» racereadrange_data | |
252 » CMPQ» SI, runtime·racearenaend(SB) | |
253 » JB» racereadrange_instrument | |
254 racereadrange_data: | |
255 » CMPQ» SI, $noptrdata(SB) | |
256 » JB» racereadrange_ret | |
257 » CMPQ» SI, $enoptrbss(SB) | |
258 » JAE» racereadrange_ret | |
259 racereadrange_instrument: | |
260 | |
261 » /* | |
262 » MOVQ» SI, R15 | |
263 » SHRQ» $40, R15 | |
264 » CMPQ» R15, $0x7e | |
265 » JAE» racereadrange_ret | |
266 » */ | |
267 | |
268 » /* | |
269 » MOVQ» SI, R15 | |
270 » SHRQ» $36, R15 | |
271 » CMPQ» R15, $0xc | |
272 » JB» racereadrange_data | |
273 » CMPQ» R15, $0xe | |
274 » JB» racereadrange_instrument | |
275 racereadrange_data: | |
276 » CMPQ» SI, $noptrdata(SB) | |
277 » JB» racereadrange_ret | |
278 » CMPQ» SI, $enoptrbss(SB) | |
279 » JAE» racereadrange_ret | |
280 racereadrange_instrument: | |
281 */ | |
282 » // Switch to g0 stack. | |
283 » MOVQ» SP, R12 | |
284 » MOVQ» m_g0(R13), R15 | |
285 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
286 // void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, vo
id *pc); | 93 // void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, vo
id *pc); |
287 » CALL» __tsan_read_range(SB) | 94 » MOVQ» $__tsan_read_range(SB), AX |
288 » MOVQ» R12, SP | 95 » JMP» racecalladdr<>(SB) |
289 racereadrange_ret: | |
290 » RET | |
291 | 96 |
292 // func runtime·RaceReadRange(addr, size uintptr) | 97 // func runtime·RaceReadRange(addr, size uintptr) |
293 TEXT runtime·RaceReadRange(SB), NOSPLIT, $0-16 | 98 TEXT runtime·RaceReadRange(SB), NOSPLIT, $0-16 |
| 99 // This needs to be a tail call, because racereadrange reads caller pc. |
294 JMP runtime·racereadrange(SB) | 100 JMP runtime·racereadrange(SB) |
295 | 101 |
296 // void runtime·racereadrangepc(void *addr, uintptr sz, void *callpc, void *pc) | 102 // void runtime·racereadrangepc1(void *addr, uintptr sz, void *pc) |
297 TEXT» runtime·racereadrangepc(SB), NOSPLIT, $0-32 | 103 TEXT» runtime·racereadrangepc1(SB), NOSPLIT, $0-24 |
298 » get_tls(R12) | 104 » MOVQ» addr+0(FP), RARG1 |
299 » MOVQ» m(R12), R13 | 105 » MOVQ» size+8(FP), RARG2 |
300 » MOVQ» g(R12), R14 | 106 » MOVQ» pc+16(FP), RARG3 |
301 » // Prepare arguments for the call. | 107 » // void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, vo
id *pc); |
302 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | 108 » MOVQ» $__tsan_read_range(SB), AX |
303 » MOVQ» 8(SP), SI» » // 1th arg: access addr | 109 » JMP» racecalladdr<>(SB) |
304 » MOVQ» 16(SP), DX» » // 2th arg: access size | |
305 » MOVQ» 24(SP), CX» » // 3th arg: caller caller pc | |
306 » MOVQ» 32(SP), R8» » // 4th arg: caller pc | |
307 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | |
308 | |
309 » CMPQ» SI, runtime·racearenastart(SB) | |
310 » JB» racereadrangepc_data | |
311 » CMPQ» SI, runtime·racearenaend(SB) | |
312 » JB» racereadrangepc_instrument | |
313 racereadrangepc_data: | |
314 » CMPQ» SI, $noptrdata(SB) | |
315 » JB» racereadrangepc_ret | |
316 » CMPQ» SI, $enoptrbss(SB) | |
317 » JAE» racereadrangepc_ret | |
318 | |
319 | |
320 » /* | |
321 » MOVQ» SI, R15 | |
322 » SHRQ» $40, R15 | |
323 » CMPQ» R15, $0x7e | |
324 » JAE» racereadrangepc_ret | |
325 » */ | |
326 | |
327 /* | |
328 » MOVQ» SI, R15 | |
329 » SHRQ» $36, R15 | |
330 » CMPQ» R15, $0xc | |
331 » JB» racereadrangepc_data | |
332 » CMPQ» R15, $0xe | |
333 » JB» racereadrangepc_instrument | |
334 racereadrangepc_data: | |
335 » CMPQ» SI, $noptrdata(SB) | |
336 » JB» racereadrangepc_ret | |
337 » CMPQ» SI, $enoptrbss(SB) | |
338 » JAE» racereadrangepc_ret | |
339 */ | |
340 racereadrangepc_instrument: | |
341 » // Switch to g0 stack. | |
342 » MOVQ» SP, R12 | |
343 » MOVQ» m_g0(R13), R15 | |
344 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
345 » // void __tsan_read_range_pc(ThreadState *thr, void *addr, uintptr size,
void *callpc, void *pc); | |
346 » CALL» __tsan_read_range_pc(SB) | |
347 » MOVQ» R12, SP | |
348 racereadrangepc_ret: | |
349 » RET | |
350 | 110 |
351 // func runtime·racewriterange(addr, size uintptr) | 111 // func runtime·racewriterange(addr, size uintptr) |
| 112 // Called from instrumented code. |
352 TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 | 113 TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 |
353 » get_tls(R12) | 114 » MOVQ» addr+0(FP), RARG1 |
354 » MOVQ» m(R12), R13 | 115 » MOVQ» size+8(FP), RARG2 |
355 » MOVQ» g(R12), R14 | 116 » MOVQ» (SP), RARG3 |
356 » // Prepare arguments for the call. | |
357 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | |
358 » MOVQ» 8(SP), SI» » // 1th arg: access addr | |
359 » MOVQ» 16(SP), DX» » // 2th arg: access size | |
360 » MOVQ» (SP), CX» » // 3th arg: caller pc | |
361 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | |
362 » CMPQ» SI, runtime·racearenastart(SB) | |
363 » JB» racewriterange_data | |
364 » CMPQ» SI, runtime·racearenaend(SB) | |
365 » JB» racewriterange_instrument | |
366 racewriterange_data: | |
367 » CMPQ» SI, $noptrdata(SB) | |
368 » JB» racewriterange_ret | |
369 » CMPQ» SI, $enoptrbss(SB) | |
370 » JAE» racewriterange_ret | |
371 racewriterange_instrument: | |
372 | |
373 » /* | |
374 » MOVQ» SI, R15 | |
375 » SHRQ» $40, R15 | |
376 » CMPQ» R15, $0x7e | |
377 » JAE» racewriterange_ret | |
378 » */ | |
379 | |
380 » /* | |
381 » MOVQ» SI, R15 | |
382 » SHRQ» $36, R15 | |
383 » CMPQ» R15, $0xc | |
384 » JB» racewriterange_data | |
385 » CMPQ» R15, $0xe | |
386 » JB» racewriterange_instrument | |
387 racewriterange_data: | |
388 » CMPQ» SI, $noptrdata(SB) | |
389 » JB» racewriterange_ret | |
390 » CMPQ» SI, $enoptrbss(SB) | |
391 » JAE» racewriterange_ret | |
392 racewriterange_instrument: | |
393 */ | |
394 » // Switch to g0 stack. | |
395 » MOVQ» SP, R12 | |
396 » MOVQ» m_g0(R13), R15 | |
397 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
398 // void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, v
oid *pc); | 117 // void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, v
oid *pc); |
399 » CALL» __tsan_write_range(SB) | 118 » MOVQ» $__tsan_write_range(SB), AX |
400 » MOVQ» R12, SP | 119 » JMP» racecalladdr<>(SB) |
401 racewriterange_ret: | |
402 » RET | |
403 | 120 |
404 // func runtime·RaceWriteRange(addr, size uintptr) | 121 // func runtime·RaceWriteRange(addr, size uintptr) |
405 TEXT runtime·RaceWriteRange(SB), NOSPLIT, $0-16 | 122 TEXT runtime·RaceWriteRange(SB), NOSPLIT, $0-16 |
| 123 // This needs to be a tail call, because racewriterange reads caller pc. |
406 JMP runtime·racewriterange(SB) | 124 JMP runtime·racewriterange(SB) |
407 | 125 |
408 // void runtime·racewriterangepc(void *addr, uintptr sz, void *callpc, void *pc) | 126 // void runtime·racewriterangepc1(void *addr, uintptr sz, void *pc) |
409 TEXT» runtime·racewriterangepc(SB), NOSPLIT, $0-32 | 127 TEXT» runtime·racewriterangepc1(SB), NOSPLIT, $0-24 |
410 » get_tls(R12) | 128 » MOVQ» addr+0(FP), RARG1 |
411 » MOVQ» m(R12), R13 | 129 » MOVQ» size+8(FP), RARG2 |
412 » MOVQ» g(R12), R14 | 130 » MOVQ» pc+16(FP), RARG3 |
413 » // Prepare arguments for the call. | 131 » // void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, v
oid *pc); |
414 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | 132 » MOVQ» $__tsan_write_range(SB), AX |
415 » MOVQ» 8(SP), SI» » // 1th arg: access addr | 133 » JMP» racecalladdr<>(SB) |
416 » MOVQ» 16(SP), DX» » // 2th arg: access size | 134 |
417 » MOVQ» 24(SP), CX» » // 3th arg: caller caller pc | 135 // If addr (RARG1) is out of range, do nothing. |
418 » MOVQ» 32(SP), R8» » // 4th arg: caller pc | 136 // Otherwise, setup goroutine context and invoke racecall. Other arguments alrea
dy set. |
419 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | 137 TEXT» racecalladdr<>(SB), NOSPLIT, $0-0 |
420 » CMPQ» SI, runtime·racearenastart(SB) | 138 » get_tls(R12) |
421 » JB» racewriterangepc_data | 139 » MOVQ» g(R12), R14 |
422 » CMPQ» SI, runtime·racearenaend(SB) | 140 » MOVQ» g_racectx(R14), RARG0» // goroutine context |
423 » JB» racewriterangepc_instrument | 141 » // Check that addr is within [arenastart, arenaend) or within [noptrdata
, enoptrbss). |
424 racewriterangepc_data: | 142 » CMPQ» RARG1, runtime·racearenastart(SB) |
425 » CMPQ» SI, $noptrdata(SB) | 143 » JB» racecalladdr_data |
426 » JB» racewriterangepc_ret | 144 » CMPQ» RARG1, runtime·racearenaend(SB) |
427 » CMPQ» SI, $enoptrbss(SB) | 145 » JB» racecalladdr_call |
428 » JAE» racewriterangepc_ret | 146 racecalladdr_data: |
429 | 147 » CMPQ» RARG1, $noptrdata(SB) |
430 » /* | 148 » JB» racecalladdr_ret |
431 » MOVQ» SI, R15 | 149 » CMPQ» RARG1, $enoptrbss(SB) |
432 » SHRQ» $40, R15 | 150 » JAE» racecalladdr_ret |
433 » CMPQ» R15, $0x7e | 151 racecalladdr_call: |
434 » JAE» racewriterangepc_ret | 152 » MOVQ» AX, AX» » // w/o this 6a miscompiles this function |
435 » */ | 153 » JMP» racecall<>(SB) |
436 | 154 racecalladdr_ret: |
437 /* | |
438 » MOVQ» SI, R15 | |
439 » SHRQ» $36, R15 | |
440 » CMPQ» R15, $0xc | |
441 » JB» racewriterangepc_data | |
442 » CMPQ» R15, $0xe | |
443 » JB» racewriterangepc_instrument | |
444 racewriterangepc_data: | |
445 » CMPQ» SI, $noptrdata(SB) | |
446 » JB» racewriterangepc_ret | |
447 » CMPQ» SI, $enoptrbss(SB) | |
448 » JAE» racewriterangepc_ret | |
449 */ | |
450 racewriterangepc_instrument: | |
451 » // Switch to g0 stack. | |
452 » MOVQ» SP, R12 | |
453 » MOVQ» m_g0(R13), R15 | |
454 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
455 » // void __tsan_write_range_pc(ThreadState *thr, void *addr, uintptr size
, void *callpc, void *pc); | |
456 » CALL» __tsan_write_range_pc(SB) | |
457 » MOVQ» R12, SP | |
458 racewriterangepc_ret: | |
459 RET | 155 RET |
460 | 156 |
461 // func runtime·racefuncenter(pc uintptr) | 157 // func runtime·racefuncenter(pc uintptr) |
| 158 // Called from instrumented code. |
462 TEXT runtime·racefuncenter(SB), NOSPLIT, $0-8 | 159 TEXT runtime·racefuncenter(SB), NOSPLIT, $0-8 |
463 » get_tls(R12) | 160 » MOVQ» DX, R15»» // save function entry context (for closures) |
464 » MOVQ» m(R12), R13 | 161 » get_tls(R12) |
465 » MOVQ» g(R12), R14 | 162 » MOVQ» g(R12), R14 |
466 » // Prepare arguments for the call. | 163 » MOVQ» g_racectx(R14), RARG0» // goroutine context |
467 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | 164 » MOVQ» callpc+0(FP), RARG1 |
468 » MOVQ» 8(SP), SI» » // 1th arg: caller pc | |
469 » // Switch to g0 stack. | |
470 » MOVQ» SP, R12 | |
471 » MOVQ» m_g0(R13), R15 | |
472 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
473 » // Save function entry context (for closures). | |
474 » MOVQ» DX, R15 | |
475 // void __tsan_func_enter(ThreadState *thr, void *pc); | 165 // void __tsan_func_enter(ThreadState *thr, void *pc); |
476 » CALL» __tsan_func_enter(SB) | 166 » MOVQ» $__tsan_func_enter(SB), AX |
477 » MOVQ» R12, SP | 167 » CALL» racecall<>(SB) |
478 » MOVQ» R15, DX | 168 » MOVQ» R15, DX»// restore function entry context |
479 RET | 169 RET |
480 | 170 |
481 // func runtime·racefuncexit() | 171 // func runtime·racefuncexit() |
| 172 // Called from instrumented code. |
482 TEXT runtime·racefuncexit(SB), NOSPLIT, $0-0 | 173 TEXT runtime·racefuncexit(SB), NOSPLIT, $0-0 |
483 get_tls(R12) | 174 get_tls(R12) |
484 » MOVQ» m(R12), R13 | 175 » MOVQ» g(R12), R14 |
485 » MOVQ» g(R12), R14 | 176 » MOVQ» g_racectx(R14), RARG0» // goroutine context |
486 » // Prepare arguments for the call. | |
487 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | |
488 » // Switch to g0 stack. | |
489 » MOVQ» SP, R12 | |
490 » MOVQ» m_g0(R13), R15 | |
491 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
492 // void __tsan_func_exit(ThreadState *thr); | 177 // void __tsan_func_exit(ThreadState *thr); |
493 » CALL» __tsan_func_exit(SB) | 178 » MOVQ» $__tsan_func_exit(SB), AX |
494 » MOVQ» R12, SP | 179 » JMP» racecall<>(SB) |
495 » RET | |
496 | 180 |
497 // void runtime·racecall(void(*f)(...), ...) | 181 // void runtime·racecall(void(*f)(...), ...) |
498 // Calls C function f from race runtime and passes up to 4 arguments to it. | 182 // Calls C function f from race runtime and passes up to 4 arguments to it. |
499 // The arguments are never heap-object-preserving pointers, so we pretend there
are no arguments. | 183 // The arguments are never heap-object-preserving pointers, so we pretend there
are no arguments. |
500 TEXT runtime·racecall(SB), NOSPLIT, $0-0 | 184 TEXT runtime·racecall(SB), NOSPLIT, $0-0 |
| 185 MOVQ fn+0(FP), AX |
| 186 MOVQ arg0+8(FP), RARG0 |
| 187 MOVQ arg1+16(FP), RARG1 |
| 188 MOVQ arg2+24(FP), RARG2 |
| 189 MOVQ arg3+32(FP), RARG3 |
| 190 JMP racecall<>(SB) |
| 191 |
| 192 // Switches SP to g0 stack and calls (AX). Arguments already set. |
| 193 TEXT racecall<>(SB), NOSPLIT, $0-0 |
501 get_tls(R12) | 194 get_tls(R12) |
502 MOVQ m(R12), R13 | 195 MOVQ m(R12), R13 |
503 MOVQ g(R12), R14 | 196 MOVQ g(R12), R14 |
504 // Prepare arguments for the call. | |
505 MOVQ 8(SP), AX // function pointer | |
506 MOVQ 16(SP), DI // 4 arguments | |
507 MOVQ 24(SP), SI | |
508 MOVQ 32(SP), DX | |
509 MOVQ 48(SP), CX | |
510 // Switch to g0 stack. | 197 // Switch to g0 stack. |
511 » MOVQ» SP, R12 | 198 » MOVQ» SP, R12»» // callee-saved, preserved across the CALL |
512 » MOVQ» m_g0(R13), R15 | 199 » MOVQ» m_g0(R13), R10 |
513 » CMPQ» R15, R14 | 200 » CMPQ» R10, R14 |
514 JE racecall_cont // already on g0 | 201 JE racecall_cont // already on g0 |
515 » MOVQ» (g_sched+gobuf_sp)(R15), SP | 202 » MOVQ» (g_sched+gobuf_sp)(R10), SP |
516 racecall_cont: | 203 racecall_cont: |
| 204 ANDQ $~15, SP // alignment for gcc ABI |
517 CALL AX | 205 CALL AX |
518 MOVQ R12, SP | 206 MOVQ R12, SP |
519 RET | 207 RET |
520 | 208 |
521 // C->Go callback thunk between: | 209 // C->Go callback thunk that allows to call runtime·racesymbolize from C code. |
522 // void __tsan_symbolize(void *ctx) | |
523 // and | |
524 // runtime·racesymbolize(void *ctx) | |
525 // Direct Go->C race call has only switched SP, finish g->g0 switch by setting c
orrect g. | 210 // Direct Go->C race call has only switched SP, finish g->g0 switch by setting c
orrect g. |
526 // The overall effect of Go->C->Go call chain is similar to that of mcall. | 211 // The overall effect of Go->C->Go call chain is similar to that of mcall. |
527 // Here we use only caller saved registers (CX, DX, SI). | 212 TEXT» runtime·racesymbolizethunk(SB), NOSPLIT, $56-8 |
528 TEXT» __tsan_symbolize(SB), NOSPLIT, $56-8 | 213 » // Save callee-saved registers (Go code won't respect that). |
529 » // Save amd64 ABI callee-saved registers (Go code won't respect that). | 214 » // This is superset of darwin/linux/windows registers. |
530 PUSHQ BX | 215 PUSHQ BX |
531 PUSHQ BP | 216 PUSHQ BP |
| 217 PUSHQ DI |
| 218 PUSHQ SI |
532 PUSHQ R12 | 219 PUSHQ R12 |
533 PUSHQ R13 | 220 PUSHQ R13 |
534 PUSHQ R14 | 221 PUSHQ R14 |
535 PUSHQ R15 | 222 PUSHQ R15 |
| 223 // Set g = g0. |
536 get_tls(R12) | 224 get_tls(R12) |
537 MOVQ m(R12), R13 | 225 MOVQ m(R12), R13 |
538 MOVQ m_g0(R13), R14 | 226 MOVQ m_g0(R13), R14 |
539 MOVQ R14, g(R12) // g = m->g0 | 227 MOVQ R14, g(R12) // g = m->g0 |
540 » MOVQ» DI, 0(SP)» // func arg | 228 » MOVQ» RARG0, 0(SP)» // func arg |
541 CALL runtime·racesymbolize(SB) | 229 CALL runtime·racesymbolize(SB) |
| 230 // All registers are smashed after Go code, reload. |
542 get_tls(R12) | 231 get_tls(R12) |
543 MOVQ m(R12), R13 | 232 MOVQ m(R12), R13 |
544 MOVQ m_curg(R13), R14 | 233 MOVQ m_curg(R13), R14 |
545 MOVQ R14, g(R12) // g = m->curg | 234 MOVQ R14, g(R12) // g = m->curg |
546 // Restore callee-saved registers. | 235 // Restore callee-saved registers. |
547 POPQ R15 | 236 POPQ R15 |
548 POPQ R14 | 237 POPQ R14 |
549 POPQ R13 | 238 POPQ R13 |
550 POPQ R12 | 239 POPQ R12 |
| 240 POPQ SI |
| 241 POPQ DI |
551 POPQ BP | 242 POPQ BP |
552 POPQ BX | 243 POPQ BX |
553 RET | 244 RET |
LEFT | RIGHT |