LEFT | RIGHT |
1 // Copyright 2013 The Go Authors. All rights reserved. | 1 // Copyright 2013 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // +build race,linux race,darwin | 5 // +build race |
6 | 6 |
7 #include "zasm_GOOS_GOARCH.h" | 7 #include "zasm_GOOS_GOARCH.h" |
8 #include "funcdata.h" | 8 #include "funcdata.h" |
9 #include "../../cmd/ld/textflag.h" | 9 #include "../../cmd/ld/textflag.h" |
10 | 10 |
| 11 // The following thunks allow calling the gcc-compiled race runtime directly |
| 12 // from Go code without going all the way through cgo. |
| 13 // First, it's much faster (up to 50% speedup for real Go programs). |
| 14 // Second, it eliminates race-related special cases from cgocall and scheduler. |
| 15 // Third, in long-term it will allow to remove cyclic runtime/race dependency on
cmd/go. |
| 16 |
11 // A brief recap of the amd64 calling convention. | 17 // A brief recap of the amd64 calling convention. |
12 // Arguments are passed in DI, SI, DX, CX, R8, R9, the rest is on stack (but we
don't use that). | 18 // Arguments are passed in DI, SI, DX, CX, R8, R9, the rest is on stack. |
13 // Callee-saved registers are: BX, BP, R12-R15. | 19 // Callee-saved registers are: BX, BP, R12-R15. |
14 // SP must be 16-byte aligned. | 20 // SP must be 16-byte aligned. |
15 // On Windows: | 21 // On Windows: |
16 // Arguments are passed in CX, DX, R8, R9, the rest is on stack. | 22 // Arguments are passed in CX, DX, R8, R9, the rest is on stack. |
17 // Callee-saved registers are: BX, BP, DI, SI, R12-R15. | 23 // Callee-saved registers are: BX, BP, DI, SI, R12-R15. |
18 // SP must be 16-byte aligned. | 24 // SP must be 16-byte aligned. Windows also requires "stack-backing" for the 4 r
egister arguments: |
19 | 25 // http://msdn.microsoft.com/en-us/library/ms235286.aspx |
20 // Called from instrumented code. | 26 // We do not do this, because it seems to be intended for vararg/unprototyped fu
nctions. |
21 // If we split stack, getcallerpc() can return runtime·lessstack(). | 27 // Gcc-compiled race runtime does not try to use that space. |
22 | 28 |
23 #ifdef GOOS_windows | 29 #ifdef GOOS_windows |
24 #define RARG0 CX | 30 #define RARG0 CX |
25 #define RARG1 DX | 31 #define RARG1 DX |
26 #define RARG2 R8 | 32 #define RARG2 R8 |
27 #define RARG3 R9 | 33 #define RARG3 R9 |
28 #else | 34 #else |
29 #define RARG0 DI | 35 #define RARG0 DI |
30 #define RARG1 SI | 36 #define RARG1 SI |
31 #define RARG2 DX | 37 #define RARG2 DX |
32 #define RARG3 CX | 38 #define RARG3 CX |
33 #endif | 39 #endif |
34 | 40 |
35 // RDI, RSI, RDX, RCX, R8, R9, | |
36 | |
37 // func runtime·raceread(addr uintptr) | 41 // func runtime·raceread(addr uintptr) |
| 42 // Called from instrumented code. |
38 TEXT runtime·raceread(SB), NOSPLIT, $0-8 | 43 TEXT runtime·raceread(SB), NOSPLIT, $0-8 |
39 » get_tls(R12) | 44 » MOVQ» addr+0(FP), RARG1 |
40 » MOVQ» m(R12), R13 | 45 » MOVQ» (SP), RARG2 |
41 » MOVQ» g(R12), R14 | |
42 » // Prepare arguments for the call. | |
43 » MOVQ» g_racectx(R14), RARG0» // goroutine context | |
44 » MOVQ» 8(SP), RARG1» » // access addr | |
45 » MOVQ» (SP), RARG2» » // caller pc | |
46 » // Check that addr is within [arenastart, arenaend) or within [noptrdata
, enoptrbss) | |
47 » CMPQ» RARG1, runtime·racearenastart(SB) | |
48 » JB» raceread_data | |
49 » CMPQ» RARG1, runtime·racearenaend(SB) | |
50 » JB» raceread_instrument | |
51 raceread_data: | |
52 » CMPQ» RARG1, $noptrdata(SB) | |
53 » JB» raceread_ret | |
54 » CMPQ» RARG1, $enoptrbss(SB) | |
55 » JAE» raceread_ret | |
56 raceread_instrument: | |
57 » // Switch to g0 stack. | |
58 » MOVQ» SP, R12 | |
59 » MOVQ» m_g0(R13), R15 | |
60 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
61 » ANDQ» $~15, SP» // alignment for gcc ABI | |
62 // void __tsan_read(ThreadState *thr, void *addr, void *pc); | 46 // void __tsan_read(ThreadState *thr, void *addr, void *pc); |
63 » CALL» __tsan_read(SB) | 47 » MOVQ» $__tsan_read(SB), AX |
64 » MOVQ» R12, SP | 48 » JMP» racecalladdr<>(SB) |
65 raceread_ret: | |
66 » RET | |
67 | 49 |
68 // func runtime·RaceRead(addr uintptr) | 50 // func runtime·RaceRead(addr uintptr) |
69 TEXT runtime·RaceRead(SB), NOSPLIT, $0-8 | 51 TEXT runtime·RaceRead(SB), NOSPLIT, $0-8 |
70 // This needs to be a tail call, because raceread reads caller pc. | 52 // This needs to be a tail call, because raceread reads caller pc. |
71 JMP runtime·raceread(SB) | 53 JMP runtime·raceread(SB) |
72 | 54 |
73 // void runtime·racereadpc(void *addr, void *callpc, void *pc) | 55 // void runtime·racereadpc(void *addr, void *callpc, void *pc) |
74 TEXT runtime·racereadpc(SB), NOSPLIT, $0-24 | 56 TEXT runtime·racereadpc(SB), NOSPLIT, $0-24 |
75 » get_tls(R12) | 57 » MOVQ» addr+0(FP), RARG1 |
76 » MOVQ» m(R12), R13 | 58 » MOVQ» callpc+8(FP), RARG2 |
77 » MOVQ» g(R12), R14 | 59 » MOVQ» pc+16(FP), RARG3 |
78 » // Prepare arguments for the call. | |
79 » MOVQ» g_racectx(R14), RARG0» // goroutine context | |
80 » MOVQ» 8(SP), RARG1» » // access addr | |
81 » MOVQ» 16(SP), RARG2» » // caller caller pc | |
82 » MOVQ» 24(SP), RARG3» » // caller pc | |
83 » // Check that addr is within [arenastart, arenaend) or within [noptrdata
, enoptrbss) | |
84 » CMPQ» RARG1, runtime·racearenastart(SB) | |
85 » JB» racereadpc_data | |
86 » CMPQ» RARG1, runtime·racearenaend(SB) | |
87 » JB» racereadpc_instrument | |
88 racereadpc_data: | |
89 » CMPQ» RARG1, $noptrdata(SB) | |
90 » JB» racereadpc_ret | |
91 » CMPQ» RARG1, $enoptrbss(SB) | |
92 » JAE» racereadpc_ret | |
93 racereadpc_instrument: | |
94 » // Switch to g0 stack. | |
95 » MOVQ» SP, R12 | |
96 » MOVQ» m_g0(R13), R15 | |
97 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
98 » ANDQ» $~15, SP» // alignment for gcc ABI | |
99 // void __tsan_read_pc(ThreadState *thr, void *addr, void *callpc, void
*pc); | 60 // void __tsan_read_pc(ThreadState *thr, void *addr, void *callpc, void
*pc); |
100 » CALL» __tsan_read_pc(SB) | 61 » MOVQ» $__tsan_read_pc(SB), AX |
101 » MOVQ» R12, SP | 62 » JMP» racecalladdr<>(SB) |
102 racereadpc_ret: | |
103 » RET | |
104 | 63 |
105 // func runtime·racewrite(addr uintptr) | 64 // func runtime·racewrite(addr uintptr) |
| 65 // Called from instrumented code. |
106 TEXT runtime·racewrite(SB), NOSPLIT, $0-8 | 66 TEXT runtime·racewrite(SB), NOSPLIT, $0-8 |
107 » get_tls(R12) | 67 » MOVQ» addr+0(FP), RARG1 |
108 » MOVQ» m(R12), R13 | 68 » MOVQ» (SP), RARG2 |
109 » MOVQ» g(R12), R14 | |
110 » // Prepare arguments for the call. | |
111 » MOVQ» g_racectx(R14), RARG0» // goroutine context | |
112 » MOVQ» 8(SP), RARG1» » // access addr | |
113 » MOVQ» (SP), RARG2» » // caller pc | |
114 » // Check that addr is within [arenastart, arenaend) or within [noptrdata
, enoptrbss) | |
115 » CMPQ» RARG1, runtime·racearenastart(SB) | |
116 » JB» racewrite_data | |
117 » CMPQ» RARG1, runtime·racearenaend(SB) | |
118 » JB» racewrite_instrument | |
119 racewrite_data: | |
120 » CMPQ» RARG1, $noptrdata(SB) | |
121 » JB» racewrite_ret | |
122 » CMPQ» RARG1, $enoptrbss(SB) | |
123 » JAE» racewrite_ret | |
124 racewrite_instrument: | |
125 » // Switch to g0 stack. | |
126 » MOVQ» SP, R12 | |
127 » MOVQ» m_g0(R13), R15 | |
128 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
129 » ANDQ» $~15, SP» // alignment for gcc ABI | |
130 // void __tsan_write(ThreadState *thr, void *addr, void *pc); | 69 // void __tsan_write(ThreadState *thr, void *addr, void *pc); |
131 » CALL» __tsan_write(SB) | 70 » MOVQ» $__tsan_write(SB), AX |
132 » MOVQ» R12, SP | 71 » JMP» racecalladdr<>(SB) |
133 racewrite_ret: | |
134 » RET | |
135 | 72 |
136 // func runtime·RaceWrite(addr uintptr) | 73 // func runtime·RaceWrite(addr uintptr) |
137 TEXT runtime·RaceWrite(SB), NOSPLIT, $0-8 | 74 TEXT runtime·RaceWrite(SB), NOSPLIT, $0-8 |
138 // This needs to be a tail call, because racewrite reads caller pc. | 75 // This needs to be a tail call, because racewrite reads caller pc. |
139 JMP runtime·racewrite(SB) | 76 JMP runtime·racewrite(SB) |
140 | 77 |
141 // void runtime·racewritepc(void *addr, void *callpc, void *pc) | 78 // void runtime·racewritepc(void *addr, void *callpc, void *pc) |
142 TEXT runtime·racewritepc(SB), NOSPLIT, $0-24 | 79 TEXT runtime·racewritepc(SB), NOSPLIT, $0-24 |
143 » get_tls(R12) | 80 » MOVQ» addr+0(FP), RARG1 |
144 » MOVQ» m(R12), R13 | 81 » MOVQ» callpc+8(FP), RARG2 |
145 » MOVQ» g(R12), R14 | 82 » MOVQ» cp+16(FP), RARG3 |
146 » // Prepare arguments for the call. | |
147 » MOVQ» g_racectx(R14), RARG0» // goroutine context | |
148 » MOVQ» 8(SP), RARG1» » // access addr | |
149 » MOVQ» 16(SP), RARG2» » // caller caller pc | |
150 » MOVQ» 24(SP), RARG3» » // caller pc | |
151 » // Check that addr is within [arenastart, arenaend) or within [noptrdata
, enoptrbss) | |
152 » CMPQ» RARG1, runtime·racearenastart(SB) | |
153 » JB» racewritepc_data | |
154 » CMPQ» RARG1, runtime·racearenaend(SB) | |
155 » JB» racewritepc_instrument | |
156 racewritepc_data: | |
157 » CMPQ» RARG1, $noptrdata(SB) | |
158 » JB» racewritepc_ret | |
159 » CMPQ» RARG1, $enoptrbss(SB) | |
160 » JAE» racewritepc_ret | |
161 racewritepc_instrument: | |
162 » // Switch to g0 stack. | |
163 » MOVQ» SP, R12 | |
164 » MOVQ» m_g0(R13), R15 | |
165 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
166 » ANDQ» $~15, SP» // alignment for gcc ABI | |
167 // void __tsan_write_pc(ThreadState *thr, void *addr, void *callpc, void
*pc); | 83 // void __tsan_write_pc(ThreadState *thr, void *addr, void *callpc, void
*pc); |
168 » CALL» __tsan_write_pc(SB) | 84 » MOVQ» $__tsan_write_pc(SB), AX |
169 » MOVQ» R12, SP | 85 » JMP» racecalladdr<>(SB) |
170 racewritepc_ret: | |
171 » RET | |
172 | 86 |
173 // func runtime·racereadrange(addr, size uintptr) | 87 // func runtime·racereadrange(addr, size uintptr) |
| 88 // Called from instrumented code. |
174 TEXT runtime·racereadrange(SB), NOSPLIT, $0-16 | 89 TEXT runtime·racereadrange(SB), NOSPLIT, $0-16 |
175 » get_tls(R12) | 90 » MOVQ» addr+0(FP), RARG1 |
176 » MOVQ» m(R12), R13 | 91 » MOVQ» size+8(FP), RARG2 |
177 » MOVQ» g(R12), R14 | 92 » MOVQ» (SP), RARG3 |
178 » // Prepare arguments for the call. | |
179 » MOVQ» g_racectx(R14), RARG0» // goroutine context | |
180 » MOVQ» 8(SP), RARG1» » // access addr | |
181 » MOVQ» 16(SP), RARG2» » // access size | |
182 » MOVQ» (SP), RARG3» » // caller pc | |
183 » // Check that addr is within [arenastart, arenaend) or within [noptrdata
, enoptrbss) | |
184 » CMPQ» RARG1, runtime·racearenastart(SB) | |
185 » JB» racereadrange_data | |
186 » CMPQ» RARG1, runtime·racearenaend(SB) | |
187 » JB» racereadrange_instrument | |
188 racereadrange_data: | |
189 » CMPQ» RARG1, $noptrdata(SB) | |
190 » JB» racereadrange_ret | |
191 » CMPQ» RARG1, $enoptrbss(SB) | |
192 » JAE» racereadrange_ret | |
193 racereadrange_instrument: | |
194 » // Switch to g0 stack. | |
195 » MOVQ» SP, R12 | |
196 » MOVQ» m_g0(R13), R15 | |
197 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
198 » ANDQ» $~15, SP» // alignment for gcc ABI | |
199 // void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, vo
id *pc); | 93 // void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, vo
id *pc); |
200 » CALL» __tsan_read_range(SB) | 94 » MOVQ» $__tsan_read_range(SB), AX |
201 » MOVQ» R12, SP | 95 » JMP» racecalladdr<>(SB) |
202 racereadrange_ret: | |
203 » RET | |
204 | 96 |
205 // func runtime·RaceReadRange(addr, size uintptr) | 97 // func runtime·RaceReadRange(addr, size uintptr) |
206 TEXT runtime·RaceReadRange(SB), NOSPLIT, $0-16 | 98 TEXT runtime·RaceReadRange(SB), NOSPLIT, $0-16 |
| 99 // This needs to be a tail call, because racereadrange reads caller pc. |
207 JMP runtime·racereadrange(SB) | 100 JMP runtime·racereadrange(SB) |
208 | 101 |
209 // void runtime·racereadrangepc(void *addr, uintptr sz, void *callpc, void *pc) | 102 // void runtime·racereadrangepc1(void *addr, uintptr sz, void *pc) |
210 TEXT» runtime·racereadrangepc(SB), NOSPLIT, $0-32 | 103 TEXT» runtime·racereadrangepc1(SB), NOSPLIT, $0-24 |
211 » get_tls(R12) | 104 » MOVQ» addr+0(FP), RARG1 |
212 » MOVQ» m(R12), R13 | 105 » MOVQ» size+8(FP), RARG2 |
213 » MOVQ» g(R12), R14 | 106 » MOVQ» pc+16(FP), RARG3 |
214 » // Prepare arguments for the call. | 107 » // void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, vo
id *pc); |
215 » MOVQ» g_racectx(R14), RARG0» // goroutine context | 108 » MOVQ» $__tsan_read_range(SB), AX |
216 » MOVQ» 8(SP), RARG1» » // access addr | 109 » JMP» racecalladdr<>(SB) |
217 » MOVQ» 16(SP), RARG2» » // access size | |
218 » MOVQ» 24(SP), RARG3» » // caller caller pc | |
219 #ifdef GOOS_windows | |
220 #else | |
221 » MOVQ» 32(SP), R8» » // caller pc | |
222 #endif | |
223 » // Check that addr is within [arenastart, arenaend) or within [noptrdata
, enoptrbss) | |
224 » CMPQ» RARG1, runtime·racearenastart(SB) | |
225 » JB» racereadrangepc_data | |
226 » CMPQ» RARG1, runtime·racearenaend(SB) | |
227 » JB» racereadrangepc_instrument | |
228 racereadrangepc_data: | |
229 » CMPQ» RARG1, $noptrdata(SB) | |
230 » JB» racereadrangepc_ret | |
231 » CMPQ» RARG1, $enoptrbss(SB) | |
232 » JAE» racereadrangepc_ret | |
233 racereadrangepc_instrument: | |
234 » // Switch to g0 stack. | |
235 » MOVQ» SP, R12 | |
236 » MOVQ» m_g0(R13), R15 | |
237 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
238 » ANDQ» $~15, SP» // alignment for gcc ABI | |
239 » // void __tsan_read_range_pc(ThreadState *thr, void *addr, uintptr size,
void *callpc, void *pc); | |
240 » CALL» __tsan_read_range_pc(SB) | |
241 » MOVQ» R12, SP | |
242 racereadrangepc_ret: | |
243 » RET | |
244 | 110 |
245 // func runtime·racewriterange(addr, size uintptr) | 111 // func runtime·racewriterange(addr, size uintptr) |
| 112 // Called from instrumented code. |
246 TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 | 113 TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 |
247 » get_tls(R12) | 114 » MOVQ» addr+0(FP), RARG1 |
248 » MOVQ» m(R12), R13 | 115 » MOVQ» size+8(FP), RARG2 |
249 » MOVQ» g(R12), R14 | 116 » MOVQ» (SP), RARG3 |
250 » // Prepare arguments for the call. | |
251 » MOVQ» g_racectx(R14), RARG0» // goroutine context | |
252 » MOVQ» 8(SP), RARG1» » // access addr | |
253 » MOVQ» 16(SP), RARG2» » // access size | |
254 » MOVQ» (SP), RARG3» » // caller pc | |
255 » // Check that addr is within [arenastart, arenaend) or within [noptrdata
, enoptrbss) | |
256 » CMPQ» RARG1, runtime·racearenastart(SB) | |
257 » JB» racewriterange_data | |
258 » CMPQ» RARG1, runtime·racearenaend(SB) | |
259 » JB» racewriterange_instrument | |
260 racewriterange_data: | |
261 » CMPQ» RARG1, $noptrdata(SB) | |
262 » JB» racewriterange_ret | |
263 » CMPQ» RARG1, $enoptrbss(SB) | |
264 » JAE» racewriterange_ret | |
265 racewriterange_instrument: | |
266 » // Switch to g0 stack. | |
267 » MOVQ» SP, R12 | |
268 » MOVQ» m_g0(R13), R15 | |
269 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
270 » ANDQ» $~15, SP» // alignment for gcc ABI | |
271 // void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, v
oid *pc); | 117 // void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, v
oid *pc); |
272 » CALL» __tsan_write_range(SB) | 118 » MOVQ» $__tsan_write_range(SB), AX |
273 » MOVQ» R12, SP | 119 » JMP» racecalladdr<>(SB) |
274 racewriterange_ret: | |
275 » RET | |
276 | 120 |
277 // func runtime·RaceWriteRange(addr, size uintptr) | 121 // func runtime·RaceWriteRange(addr, size uintptr) |
278 TEXT runtime·RaceWriteRange(SB), NOSPLIT, $0-16 | 122 TEXT runtime·RaceWriteRange(SB), NOSPLIT, $0-16 |
| 123 // This needs to be a tail call, because racewriterange reads caller pc. |
279 JMP runtime·racewriterange(SB) | 124 JMP runtime·racewriterange(SB) |
280 | 125 |
281 // void runtime·racewriterangepc(void *addr, uintptr sz, void *callpc, void *pc) | 126 // void runtime·racewriterangepc1(void *addr, uintptr sz, void *pc) |
282 TEXT» runtime·racewriterangepc(SB), NOSPLIT, $0-32 | 127 TEXT» runtime·racewriterangepc1(SB), NOSPLIT, $0-24 |
283 » get_tls(R12) | 128 » MOVQ» addr+0(FP), RARG1 |
284 » MOVQ» m(R12), R13 | 129 » MOVQ» size+8(FP), RARG2 |
285 » MOVQ» g(R12), R14 | 130 » MOVQ» pc+16(FP), RARG3 |
286 » // Prepare arguments for the call. | 131 » // void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, v
oid *pc); |
| 132 » MOVQ» $__tsan_write_range(SB), AX |
| 133 » JMP» racecalladdr<>(SB) |
| 134 |
| 135 // If addr (RARG1) is out of range, do nothing. |
| 136 // Otherwise, setup goroutine context and invoke racecall. Other arguments alrea
dy set. |
| 137 TEXT» racecalladdr<>(SB), NOSPLIT, $0-0 |
| 138 » get_tls(R12) |
| 139 » MOVQ» g(R12), R14 |
287 MOVQ g_racectx(R14), RARG0 // goroutine context | 140 MOVQ g_racectx(R14), RARG0 // goroutine context |
288 » MOVQ» 8(SP), RARG1» » // access addr | 141 » // Check that addr is within [arenastart, arenaend) or within [noptrdata
, enoptrbss). |
289 » MOVQ» 16(SP), RARG2» » // access size | |
290 » MOVQ» 24(SP), RARG3» » // caller caller pc | |
291 #ifdef GOOS_windows | |
292 #else | |
293 » MOVQ» 32(SP), R8» » // caller pc | |
294 #endif | |
295 » // Check that addr is within [arenastart, arenaend) or within [noptrdata
, enoptrbss) | |
296 CMPQ RARG1, runtime·racearenastart(SB) | 142 CMPQ RARG1, runtime·racearenastart(SB) |
297 » JB» racewriterangepc_data | 143 » JB» racecalladdr_data |
298 CMPQ RARG1, runtime·racearenaend(SB) | 144 CMPQ RARG1, runtime·racearenaend(SB) |
299 » JB» racewriterangepc_instrument | 145 » JB» racecalladdr_call |
300 racewriterangepc_data: | 146 racecalladdr_data: |
301 CMPQ RARG1, $noptrdata(SB) | 147 CMPQ RARG1, $noptrdata(SB) |
302 » JB» racewriterangepc_ret | 148 » JB» racecalladdr_ret |
303 CMPQ RARG1, $enoptrbss(SB) | 149 CMPQ RARG1, $enoptrbss(SB) |
304 » JAE» racewriterangepc_ret | 150 » JAE» racecalladdr_ret |
305 racewriterangepc_instrument: | 151 racecalladdr_call: |
306 » // Switch to g0 stack. | 152 » MOVQ» AX, AX» » // w/o this 6a miscompiles this function |
307 » MOVQ» SP, R12 | 153 » JMP» racecall<>(SB) |
308 » MOVQ» m_g0(R13), R15 | 154 racecalladdr_ret: |
309 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
310 » ANDQ» $~15, SP» // alignment for gcc ABI | |
311 » // void __tsan_write_range_pc(ThreadState *thr, void *addr, uintptr size
, void *callpc, void *pc); | |
312 » CALL» __tsan_write_range_pc(SB) | |
313 » MOVQ» R12, SP | |
314 racewriterangepc_ret: | |
315 RET | 155 RET |
316 | 156 |
317 // func runtime·racefuncenter(pc uintptr) | 157 // func runtime·racefuncenter(pc uintptr) |
| 158 // Called from instrumented code. |
318 TEXT runtime·racefuncenter(SB), NOSPLIT, $0-8 | 159 TEXT runtime·racefuncenter(SB), NOSPLIT, $0-8 |
319 » MOVQ» DX, R15»// save function entry context (for closures) | 160 » MOVQ» DX, R15»» // save function entry context (for closures) |
320 » get_tls(R12) | 161 » get_tls(R12) |
321 » MOVQ» m(R12), R13 | 162 » MOVQ» g(R12), R14 |
322 » MOVQ» g(R12), R14 | |
323 » // Prepare arguments for the call. | |
324 MOVQ g_racectx(R14), RARG0 // goroutine context | 163 MOVQ g_racectx(R14), RARG0 // goroutine context |
325 » MOVQ» 8(SP), RARG1» » // caller pc | 164 » MOVQ» callpc+0(FP), RARG1 |
326 » // Switch to g0 stack. | |
327 » MOVQ» SP, R12 | |
328 » MOVQ» m_g0(R13), R14 | |
329 » MOVQ» (g_sched+gobuf_sp)(R14), SP | |
330 » ANDQ» $~15, SP» // alignment for gcc ABI | |
331 // void __tsan_func_enter(ThreadState *thr, void *pc); | 165 // void __tsan_func_enter(ThreadState *thr, void *pc); |
332 » CALL» __tsan_func_enter(SB) | 166 » MOVQ» $__tsan_func_enter(SB), AX |
333 » MOVQ» R12, SP | 167 » CALL» racecall<>(SB) |
334 MOVQ R15, DX // restore function entry context | 168 MOVQ R15, DX // restore function entry context |
335 RET | 169 RET |
336 | 170 |
337 // func runtime·racefuncexit() | 171 // func runtime·racefuncexit() |
| 172 // Called from instrumented code. |
338 TEXT runtime·racefuncexit(SB), NOSPLIT, $0-0 | 173 TEXT runtime·racefuncexit(SB), NOSPLIT, $0-0 |
339 get_tls(R12) | 174 get_tls(R12) |
340 » MOVQ» m(R12), R13 | 175 » MOVQ» g(R12), R14 |
341 » MOVQ» g(R12), R14 | |
342 » // Prepare arguments for the call. | |
343 MOVQ g_racectx(R14), RARG0 // goroutine context | 176 MOVQ g_racectx(R14), RARG0 // goroutine context |
344 // Switch to g0 stack. | |
345 MOVQ SP, R12 | |
346 MOVQ m_g0(R13), R15 | |
347 MOVQ (g_sched+gobuf_sp)(R15), SP | |
348 ANDQ $~15, SP // alignment for gcc ABI | |
349 // void __tsan_func_exit(ThreadState *thr); | 177 // void __tsan_func_exit(ThreadState *thr); |
350 » CALL» __tsan_func_exit(SB) | 178 » MOVQ» $__tsan_func_exit(SB), AX |
351 » MOVQ» R12, SP | 179 » JMP» racecall<>(SB) |
352 » RET | |
353 | 180 |
354 // void runtime·racecall(void(*f)(...), ...) | 181 // void runtime·racecall(void(*f)(...), ...) |
355 // Calls C function f from race runtime and passes up to 4 arguments to it. | 182 // Calls C function f from race runtime and passes up to 4 arguments to it. |
356 // The arguments are never heap-object-preserving pointers, so we pretend there
are no arguments. | 183 // The arguments are never heap-object-preserving pointers, so we pretend there
are no arguments. |
357 TEXT runtime·racecall(SB), NOSPLIT, $0-0 | 184 TEXT runtime·racecall(SB), NOSPLIT, $0-0 |
| 185 MOVQ fn+0(FP), AX |
| 186 MOVQ arg0+8(FP), RARG0 |
| 187 MOVQ arg1+16(FP), RARG1 |
| 188 MOVQ arg2+24(FP), RARG2 |
| 189 MOVQ arg3+32(FP), RARG3 |
| 190 JMP racecall<>(SB) |
| 191 |
| 192 // Switches SP to g0 stack and calls (AX). Arguments already set. |
| 193 TEXT racecall<>(SB), NOSPLIT, $0-0 |
358 get_tls(R12) | 194 get_tls(R12) |
359 MOVQ m(R12), R13 | 195 MOVQ m(R12), R13 |
360 MOVQ g(R12), R14 | 196 MOVQ g(R12), R14 |
361 // Prepare arguments for the call. | |
362 MOVQ 8(SP), AX // function pointer | |
363 MOVQ 16(SP), RARG0 // 4 arguments | |
364 MOVQ 24(SP), RARG1 | |
365 MOVQ 32(SP), RARG2 | |
366 MOVQ 48(SP), RARG3 | |
367 // Switch to g0 stack. | 197 // Switch to g0 stack. |
368 » MOVQ» SP, R12 | 198 » MOVQ» SP, R12»» // callee-saved, preserved across the CALL |
369 » MOVQ» m_g0(R13), R15 | 199 » MOVQ» m_g0(R13), R10 |
370 » CMPQ» R15, R14 | 200 » CMPQ» R10, R14 |
371 JE racecall_cont // already on g0 | 201 JE racecall_cont // already on g0 |
372 » MOVQ» (g_sched+gobuf_sp)(R15), SP | 202 » MOVQ» (g_sched+gobuf_sp)(R10), SP |
373 racecall_cont: | 203 racecall_cont: |
374 ANDQ $~15, SP // alignment for gcc ABI | 204 ANDQ $~15, SP // alignment for gcc ABI |
375 CALL AX | 205 CALL AX |
376 MOVQ R12, SP | 206 MOVQ R12, SP |
377 RET | 207 RET |
378 | 208 |
379 // C->Go callback thunk between: | 209 // C->Go callback thunk that allows to call runtime·racesymbolize from C code. |
380 // void __tsan_symbolize(void *ctx) | |
381 // and | |
382 // void runtime·racesymbolize(void *ctx) | |
383 // Direct Go->C race call has only switched SP, finish g->g0 switch by setting c
orrect g. | 210 // Direct Go->C race call has only switched SP, finish g->g0 switch by setting c
orrect g. |
384 // The overall effect of Go->C->Go call chain is similar to that of mcall. | 211 // The overall effect of Go->C->Go call chain is similar to that of mcall. |
385 TEXT» __tsan_symbolize(SB), NOSPLIT, $56-8 | 212 TEXT» runtime·racesymbolizethunk(SB), NOSPLIT, $56-8 |
386 // Save callee-saved registers (Go code won't respect that). | 213 // Save callee-saved registers (Go code won't respect that). |
387 // This is superset of darwin/linux/windows registers. | 214 // This is superset of darwin/linux/windows registers. |
388 PUSHQ BX | 215 PUSHQ BX |
389 PUSHQ BP | 216 PUSHQ BP |
390 PUSHQ DI | 217 PUSHQ DI |
391 PUSHQ SI | 218 PUSHQ SI |
392 PUSHQ R12 | 219 PUSHQ R12 |
393 PUSHQ R13 | 220 PUSHQ R13 |
394 PUSHQ R14 | 221 PUSHQ R14 |
395 PUSHQ R15 | 222 PUSHQ R15 |
396 // Set g = g0. | 223 // Set g = g0. |
397 get_tls(R12) | 224 get_tls(R12) |
398 MOVQ m(R12), R13 | 225 MOVQ m(R12), R13 |
399 MOVQ m_g0(R13), R14 | 226 MOVQ m_g0(R13), R14 |
400 MOVQ R14, g(R12) // g = m->g0 | 227 MOVQ R14, g(R12) // g = m->g0 |
401 » MOVQ» DI, 0(SP)» // func arg | 228 » MOVQ» RARG0, 0(SP)» // func arg |
402 CALL runtime·racesymbolize(SB) | 229 CALL runtime·racesymbolize(SB) |
403 // All registers are smashed after Go code, reload. | 230 // All registers are smashed after Go code, reload. |
404 get_tls(R12) | 231 get_tls(R12) |
405 MOVQ m(R12), R13 | 232 MOVQ m(R12), R13 |
406 MOVQ m_curg(R13), R14 | 233 MOVQ m_curg(R13), R14 |
407 MOVQ R14, g(R12) // g = m->curg | 234 MOVQ R14, g(R12) // g = m->curg |
408 // Restore callee-saved registers. | 235 // Restore callee-saved registers. |
409 POPQ R15 | 236 POPQ R15 |
410 POPQ R14 | 237 POPQ R14 |
411 POPQ R13 | 238 POPQ R13 |
412 POPQ R12 | 239 POPQ R12 |
413 POPQ SI | 240 POPQ SI |
414 POPQ DI | 241 POPQ DI |
415 POPQ BP | 242 POPQ BP |
416 POPQ BX | 243 POPQ BX |
417 RET | 244 RET |
LEFT | RIGHT |