LEFT | RIGHT |
1 // Copyright 2013 The Go Authors. All rights reserved. | 1 // Copyright 2013 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // +build race | 5 // +build race |
6 | 6 |
7 //#include "zasm_GOOS_GOARCH.h" | 7 #include "zasm_GOOS_GOARCH.h" |
8 #include "zasm_linux_amd64.h" | |
9 #include "funcdata.h" | 8 #include "funcdata.h" |
10 #include "../../cmd/ld/textflag.h" | 9 #include "../../cmd/ld/textflag.h" |
11 | 10 |
12 // Called from instrumented code. | 11 // The following thunks allow calling the gcc-compiled race runtime directly |
13 // If we split stack, getcallerpc() can return runtime·lessstack(). | 12 // from Go code without going all the way through cgo. |
14 | 13 // First, it's much faster (up to 50% speedup for real Go programs). |
15 //!!! ensure than heap is there | 14 // Second, it eliminates race-related special cases from cgocall and scheduler. |
16 #define HEAPSTART» 0x00c000000000 | 15 // Third, in long-term it will allow to remove cyclic runtime/race dependency on
cmd/go. |
17 #define HEAPEND»» 0x00e000000000 | 16 |
18 | 17 // A brief recap of the amd64 calling convention. |
19 // RDI, RSI, RDX, RCX, R8, R9,· | 18 // Arguments are passed in DI, SI, DX, CX, R8, R9, the rest is on stack. |
| 19 // Callee-saved registers are: BX, BP, R12-R15. |
| 20 // SP must be 16-byte aligned. |
| 21 // On Windows: |
| 22 // Arguments are passed in CX, DX, R8, R9, the rest is on stack. |
| 23 // Callee-saved registers are: BX, BP, DI, SI, R12-R15. |
| 24 // SP must be 16-byte aligned. Windows also requires "stack-backing" for the 4 r
egister arguments: |
| 25 // http://msdn.microsoft.com/en-us/library/ms235286.aspx |
| 26 // We do not do this, because it seems to be intended for vararg/unprototyped fu
nctions. |
| 27 // Gcc-compiled race runtime does not try to use that space. |
| 28 |
| 29 #ifdef GOOS_windows |
| 30 #define RARG0 CX |
| 31 #define RARG1 DX |
| 32 #define RARG2 R8 |
| 33 #define RARG3 R9 |
| 34 #else |
| 35 #define RARG0 DI |
| 36 #define RARG1 SI |
| 37 #define RARG2 DX |
| 38 #define RARG3 CX |
| 39 #endif |
20 | 40 |
21 // func runtime·raceread(addr uintptr) | 41 // func runtime·raceread(addr uintptr) |
| 42 // Called from instrumented code. |
22 TEXT runtime·raceread(SB), NOSPLIT, $0-8 | 43 TEXT runtime·raceread(SB), NOSPLIT, $0-8 |
23 » get_tls(R12) | 44 » MOVQ» addr+0(FP), RARG1 |
24 » MOVQ» m(R12), R13 | 45 » MOVQ» (SP), RARG2 |
25 » MOVQ» g(R12), R14 | |
26 » // Prepare arguments for the call. | |
27 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | |
28 » MOVQ» 8(SP), SI» » // 1th arg: access addr | |
29 » MOVQ» (SP), DX» » // 2th arg: caller pc | |
30 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | |
31 » MOVQ» SI, R15 | |
32 » SHRQ» $36, R15 | |
33 » CMPQ» R15, $0xc | |
34 » JB» raceread_data | |
35 » CMPQ» R15, $0xe | |
36 » JB» raceread_instrument | |
37 raceread_data: | |
38 » CMPQ» SI, noptrdata(SB) | |
39 » JB» raceread_ret | |
40 » CMPQ» SI, enoptrbss(SB) | |
41 » JAE» raceread_ret | |
42 raceread_instrument: | |
43 » // Switch to g0 stack. | |
44 » MOVQ» SP, R12 | |
45 » MOVQ» m_g0(R13), R15 | |
46 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
47 // void __tsan_read(ThreadState *thr, void *addr, void *pc); | 46 // void __tsan_read(ThreadState *thr, void *addr, void *pc); |
48 » CALL» __tsan_read(SB) | 47 » MOVQ» $__tsan_read(SB), AX |
49 » MOVQ» R12, SP | 48 » JMP» racecalladdr<>(SB) |
50 raceread_ret: | |
51 » RET | |
52 | 49 |
53 // func runtime·RaceRead(addr uintptr) | 50 // func runtime·RaceRead(addr uintptr) |
54 TEXT runtime·RaceRead(SB), NOSPLIT, $0-8 | 51 TEXT runtime·RaceRead(SB), NOSPLIT, $0-8 |
| 52 // This needs to be a tail call, because raceread reads caller pc. |
55 JMP runtime·raceread(SB) | 53 JMP runtime·raceread(SB) |
56 | 54 |
57 // void runtime·racereadpc(void *addr, void *callpc, void *pc) | 55 // void runtime·racereadpc(void *addr, void *callpc, void *pc) |
58 TEXT runtime·racereadpc(SB), NOSPLIT, $0-24 | 56 TEXT runtime·racereadpc(SB), NOSPLIT, $0-24 |
59 » get_tls(R12) | 57 » MOVQ» addr+0(FP), RARG1 |
60 » MOVQ» m(R12), R13 | 58 » MOVQ» callpc+8(FP), RARG2 |
61 » MOVQ» g(R12), R14 | 59 » MOVQ» pc+16(FP), RARG3 |
62 » // Prepare arguments for the call. | |
63 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | |
64 » MOVQ» 8(SP), SI» » // 1th arg: access addr | |
65 » MOVQ» 16(SP), DX» » // 2th arg: caller caller pc | |
66 » MOVQ» 24(SP), CX» » // 3th arg: caller pc | |
67 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | |
68 » MOVQ» SI, R15 | |
69 » SHRQ» $36, R15 | |
70 » CMPQ» R15, $0xc | |
71 » JB» racereadpc_data | |
72 » CMPQ» R15, $0xe | |
73 » JB» racereadpc_instrument | |
74 racereadpc_data: | |
75 » CMPQ» SI, noptrdata(SB) | |
76 » JB» racereadpc_ret | |
77 » CMPQ» SI, enoptrbss(SB) | |
78 » JAE» racereadpc_ret | |
79 racereadpc_instrument: | |
80 » // Switch to g0 stack. | |
81 » MOVQ» SP, R12 | |
82 » MOVQ» m_g0(R13), R15 | |
83 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
84 // void __tsan_read_pc(ThreadState *thr, void *addr, void *callpc, void
*pc); | 60 // void __tsan_read_pc(ThreadState *thr, void *addr, void *callpc, void
*pc); |
85 » CALL» __tsan_read_pc(SB) | 61 » MOVQ» $__tsan_read_pc(SB), AX |
86 » MOVQ» R12, SP | 62 » JMP» racecalladdr<>(SB) |
87 racereadpc_ret: | |
88 » RET | |
89 | 63 |
90 // func runtime·racewrite(addr uintptr) | 64 // func runtime·racewrite(addr uintptr) |
| 65 // Called from instrumented code. |
91 TEXT runtime·racewrite(SB), NOSPLIT, $0-8 | 66 TEXT runtime·racewrite(SB), NOSPLIT, $0-8 |
92 » get_tls(R12) | 67 » MOVQ» addr+0(FP), RARG1 |
93 » MOVQ» m(R12), R13 | 68 » MOVQ» (SP), RARG2 |
94 » MOVQ» g(R12), R14 | |
95 » // Prepare arguments for the call. | |
96 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | |
97 » MOVQ» 8(SP), SI» » // 1th arg: access addr | |
98 » MOVQ» (SP), DX» » // 2th arg: caller pc | |
99 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | |
100 » MOVQ» SI, R15 | |
101 » SHRQ» $36, R15 | |
102 » CMPQ» R15, $0xc | |
103 » JB» racewrite_data | |
104 » CMPQ» R15, $0xe | |
105 » JB» racewrite_instrument | |
106 racewrite_data: | |
107 » CMPQ» SI, noptrdata(SB) | |
108 » JB» racewrite_ret | |
109 » CMPQ» SI, enoptrbss(SB) | |
110 » JAE» racewrite_ret | |
111 racewrite_instrument: | |
112 » // Switch to g0 stack. | |
113 » MOVQ» SP, R12 | |
114 » MOVQ» m_g0(R13), R15 | |
115 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
116 // void __tsan_write(ThreadState *thr, void *addr, void *pc); | 69 // void __tsan_write(ThreadState *thr, void *addr, void *pc); |
117 » CALL» __tsan_write(SB) | 70 » MOVQ» $__tsan_write(SB), AX |
118 » MOVQ» R12, SP | 71 » JMP» racecalladdr<>(SB) |
119 racewrite_ret: | |
120 » RET | |
121 | 72 |
122 // func runtime·RaceWrite(addr uintptr) | 73 // func runtime·RaceWrite(addr uintptr) |
123 TEXT runtime·RaceWrite(SB), NOSPLIT, $0-8 | 74 TEXT runtime·RaceWrite(SB), NOSPLIT, $0-8 |
| 75 // This needs to be a tail call, because racewrite reads caller pc. |
124 JMP runtime·racewrite(SB) | 76 JMP runtime·racewrite(SB) |
125 | 77 |
126 // void runtime·racewritepc(void *addr, void *callpc, void *pc) | 78 // void runtime·racewritepc(void *addr, void *callpc, void *pc) |
127 TEXT runtime·racewritepc(SB), NOSPLIT, $0-24 | 79 TEXT runtime·racewritepc(SB), NOSPLIT, $0-24 |
128 » get_tls(R12) | 80 » MOVQ» addr+0(FP), RARG1 |
129 » MOVQ» m(R12), R13 | 81 » MOVQ» callpc+8(FP), RARG2 |
130 » MOVQ» g(R12), R14 | 82 » MOVQ» cp+16(FP), RARG3 |
131 » // Prepare arguments for the call. | |
132 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | |
133 » MOVQ» 8(SP), SI» » // 1th arg: access addr | |
134 » MOVQ» 16(SP), DX» » // 2th arg: caller caller pc | |
135 » MOVQ» 24(SP), CX» » // 3th arg: caller pc | |
136 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | |
137 » MOVQ» SI, R15 | |
138 » SHRQ» $36, R15 | |
139 » CMPQ» R15, $0xc | |
140 » JB» racewritepc_data | |
141 » CMPQ» R15, $0xe | |
142 » JB» racewritepc_instrument | |
143 racewritepc_data: | |
144 » CMPQ» SI, noptrdata(SB) | |
145 » JB» racewritepc_ret | |
146 » CMPQ» SI, enoptrbss(SB) | |
147 » JAE» racewritepc_ret | |
148 racewritepc_instrument: | |
149 » // Switch to g0 stack. | |
150 » MOVQ» SP, R12 | |
151 » MOVQ» m_g0(R13), R15 | |
152 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
153 // void __tsan_write_pc(ThreadState *thr, void *addr, void *callpc, void
*pc); | 83 // void __tsan_write_pc(ThreadState *thr, void *addr, void *callpc, void
*pc); |
154 » CALL» __tsan_write_pc(SB) | 84 » MOVQ» $__tsan_write_pc(SB), AX |
155 » MOVQ» R12, SP | 85 » JMP» racecalladdr<>(SB) |
156 racewritepc_ret: | |
157 » RET | |
158 | 86 |
159 // func runtime·racereadrange(addr, size uintptr) | 87 // func runtime·racereadrange(addr, size uintptr) |
| 88 // Called from instrumented code. |
160 TEXT runtime·racereadrange(SB), NOSPLIT, $0-16 | 89 TEXT runtime·racereadrange(SB), NOSPLIT, $0-16 |
161 » get_tls(R12) | 90 » MOVQ» addr+0(FP), RARG1 |
162 » MOVQ» m(R12), R13 | 91 » MOVQ» size+8(FP), RARG2 |
163 » MOVQ» g(R12), R14 | 92 » MOVQ» (SP), RARG3 |
164 » // Prepare arguments for the call. | |
165 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | |
166 » MOVQ» 8(SP), SI» » // 1th arg: access addr | |
167 » MOVQ» 16(SP), DX» » // 2th arg: access size | |
168 » MOVQ» (SP), CX» » // 3th arg: caller pc | |
169 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | |
170 » MOVQ» SI, R15 | |
171 » SHRQ» $36, R15 | |
172 » CMPQ» R15, $0xc | |
173 » JB» racereadrange_data | |
174 » CMPQ» R15, $0xe | |
175 » JB» racereadrange_instrument | |
176 racereadrange_data: | |
177 » CMPQ» SI, noptrdata(SB) | |
178 » JB» racereadrange_ret | |
179 » CMPQ» SI, enoptrbss(SB) | |
180 » JAE» racereadrange_ret | |
181 racereadrange_instrument: | |
182 » // Switch to g0 stack. | |
183 » MOVQ» SP, R12 | |
184 » MOVQ» m_g0(R13), R15 | |
185 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
186 // void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, vo
id *pc); | 93 // void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, vo
id *pc); |
187 » CALL» __tsan_read_range(SB) | 94 » MOVQ» $__tsan_read_range(SB), AX |
188 » MOVQ» R12, SP | 95 » JMP» racecalladdr<>(SB) |
189 racereadrange_ret: | |
190 » RET | |
191 | 96 |
192 // func runtime·RaceReadRange(addr, size uintptr) | 97 // func runtime·RaceReadRange(addr, size uintptr) |
193 TEXT runtime·RaceReadRange(SB), NOSPLIT, $0-16 | 98 TEXT runtime·RaceReadRange(SB), NOSPLIT, $0-16 |
| 99 // This needs to be a tail call, because racereadrange reads caller pc. |
194 JMP runtime·racereadrange(SB) | 100 JMP runtime·racereadrange(SB) |
195 | 101 |
196 // void runtime·racereadrangepc(void *addr, uintptr sz, void *callpc, void *pc) | 102 // void runtime·racereadrangepc1(void *addr, uintptr sz, void *pc) |
197 TEXT» runtime·racereadrangepc(SB), NOSPLIT, $0-32 | 103 TEXT» runtime·racereadrangepc1(SB), NOSPLIT, $0-24 |
198 » get_tls(R12) | 104 » MOVQ» addr+0(FP), RARG1 |
199 » MOVQ» m(R12), R13 | 105 » MOVQ» size+8(FP), RARG2 |
200 » MOVQ» g(R12), R14 | 106 » MOVQ» pc+16(FP), RARG3 |
201 » // Prepare arguments for the call. | 107 » // void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, vo
id *pc); |
202 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | 108 » MOVQ» $__tsan_read_range(SB), AX |
203 » MOVQ» 8(SP), SI» » // 1th arg: access addr | 109 » JMP» racecalladdr<>(SB) |
204 » MOVQ» 16(SP), DX» » // 2th arg: access size | |
205 » MOVQ» 24(SP), CX» » // 3th arg: caller caller pc | |
206 » MOVQ» 32(SP), R8» » // 4th arg: caller pc | |
207 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | |
208 » MOVQ» SI, R15 | |
209 » SHRQ» $36, R15 | |
210 » CMPQ» R15, $0xc | |
211 » JB» racereadrangepc_data | |
212 » CMPQ» R15, $0xe | |
213 » JB» racereadrangepc_instrument | |
214 racereadrangepc_data: | |
215 » CMPQ» SI, noptrdata(SB) | |
216 » JB» racereadrangepc_ret | |
217 » CMPQ» SI, enoptrbss(SB) | |
218 » JAE» racereadrangepc_ret | |
219 racereadrangepc_instrument: | |
220 » // Switch to g0 stack. | |
221 » MOVQ» SP, R12 | |
222 » MOVQ» m_g0(R13), R15 | |
223 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
224 » // void __tsan_read_range_pc(ThreadState *thr, void *addr, uintptr size,
void *callpc, void *pc); | |
225 » CALL» __tsan_read_range_pc(SB) | |
226 » MOVQ» R12, SP | |
227 racereadrangepc_ret: | |
228 » RET | |
229 | 110 |
230 // func runtime·racewriterange(addr, size uintptr) | 111 // func runtime·racewriterange(addr, size uintptr) |
| 112 // Called from instrumented code. |
231 TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 | 113 TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 |
232 » get_tls(R12) | 114 » MOVQ» addr+0(FP), RARG1 |
233 » MOVQ» m(R12), R13 | 115 » MOVQ» size+8(FP), RARG2 |
234 » MOVQ» g(R12), R14 | 116 » MOVQ» (SP), RARG3 |
235 » // Prepare arguments for the call. | |
236 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | |
237 » MOVQ» 8(SP), SI» » // 1th arg: access addr | |
238 » MOVQ» 16(SP), DX» » // 2th arg: access size | |
239 » MOVQ» (SP), CX» » // 3th arg: caller pc | |
240 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | |
241 » MOVQ» SI, R15 | |
242 » SHRQ» $36, R15 | |
243 » CMPQ» R15, $0xc | |
244 » JB» racewriterange_data | |
245 » CMPQ» R15, $0xe | |
246 » JB» racewriterange_instrument | |
247 racewriterange_data: | |
248 » CMPQ» SI, noptrdata(SB) | |
249 » JB» racewriterange_ret | |
250 » CMPQ» SI, enoptrbss(SB) | |
251 » JAE» racewriterange_ret | |
252 racewriterange_instrument: | |
253 » // Switch to g0 stack. | |
254 » MOVQ» SP, R12 | |
255 » MOVQ» m_g0(R13), R15 | |
256 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
257 // void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, v
oid *pc); | 117 // void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, v
oid *pc); |
258 » CALL» __tsan_write_range(SB) | 118 » MOVQ» $__tsan_write_range(SB), AX |
259 » MOVQ» R12, SP | 119 » JMP» racecalladdr<>(SB) |
260 racewriterange_ret: | |
261 » RET | |
262 | 120 |
263 // func runtime·RaceWriteRange(addr, size uintptr) | 121 // func runtime·RaceWriteRange(addr, size uintptr) |
264 TEXT runtime·RaceWriteRange(SB), NOSPLIT, $0-16 | 122 TEXT runtime·RaceWriteRange(SB), NOSPLIT, $0-16 |
| 123 // This needs to be a tail call, because racewriterange reads caller pc. |
265 JMP runtime·racewriterange(SB) | 124 JMP runtime·racewriterange(SB) |
266 | 125 |
267 // void runtime·racewriterangepc(void *addr, uintptr sz, void *callpc, void *pc) | 126 // void runtime·racewriterangepc1(void *addr, uintptr sz, void *pc) |
268 TEXT» runtime·racewriterangepc(SB), NOSPLIT, $0-32 | 127 TEXT» runtime·racewriterangepc1(SB), NOSPLIT, $0-24 |
269 » get_tls(R12) | 128 » MOVQ» addr+0(FP), RARG1 |
270 » MOVQ» m(R12), R13 | 129 » MOVQ» size+8(FP), RARG2 |
271 » MOVQ» g(R12), R14 | 130 » MOVQ» pc+16(FP), RARG3 |
272 » // Prepare arguments for the call. | 131 » // void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, v
oid *pc); |
273 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | 132 » MOVQ» $__tsan_write_range(SB), AX |
274 » MOVQ» 8(SP), SI» » // 1th arg: access addr | 133 » JMP» racecalladdr<>(SB) |
275 » MOVQ» 16(SP), DX» » // 2th arg: access size | 134 |
276 » MOVQ» 24(SP), CX» » // 3th arg: caller caller pc | 135 // If addr (RARG1) is out of range, do nothing. |
277 » MOVQ» 32(SP), R8» » // 4th arg: caller pc | 136 // Otherwise, setup goroutine context and invoke racecall. Other arguments alrea
dy set. |
278 » // Check that addr is within [HEAPSTART, HEAPEND) or within [noptrdata,
enoptrbss) | 137 TEXT» racecalladdr<>(SB), NOSPLIT, $0-0 |
279 » MOVQ» SI, R15 | 138 » get_tls(R12) |
280 » SHRQ» $36, R15 | 139 » MOVQ» g(R12), R14 |
281 » CMPQ» R15, $0xc | 140 » MOVQ» g_racectx(R14), RARG0» // goroutine context |
282 » JB» racewriterangepc_data | 141 » // Check that addr is within [arenastart, arenaend) or within [noptrdata
, enoptrbss). |
283 » CMPQ» R15, $0xe | 142 » CMPQ» RARG1, runtime·racearenastart(SB) |
284 » JB» racewriterangepc_instrument | 143 » JB» racecalladdr_data |
285 racewriterangepc_data: | 144 » CMPQ» RARG1, runtime·racearenaend(SB) |
286 » CMPQ» SI, noptrdata(SB) | 145 » JB» racecalladdr_call |
287 » JB» racewriterangepc_ret | 146 racecalladdr_data: |
288 » CMPQ» SI, enoptrbss(SB) | 147 » CMPQ» RARG1, $noptrdata(SB) |
289 » JAE» racewriterangepc_ret | 148 » JB» racecalladdr_ret |
290 racewriterangepc_instrument: | 149 » CMPQ» RARG1, $enoptrbss(SB) |
291 » // Switch to g0 stack. | 150 » JAE» racecalladdr_ret |
292 » MOVQ» SP, R12 | 151 racecalladdr_call: |
293 » MOVQ» m_g0(R13), R15 | 152 » MOVQ» AX, AX» » // w/o this 6a miscompiles this function |
294 » MOVQ» (g_sched+gobuf_sp)(R15), SP | 153 » JMP» racecall<>(SB) |
295 » // void __tsan_write_range_pc(ThreadState *thr, void *addr, uintptr size
, void *callpc, void *pc); | 154 racecalladdr_ret: |
296 » CALL» __tsan_write_range_pc(SB) | |
297 » MOVQ» R12, SP | |
298 racewriterangepc_ret: | |
299 RET | 155 RET |
300 | 156 |
301 // func runtime·racefuncenter(pc uintptr) | 157 // func runtime·racefuncenter(pc uintptr) |
| 158 // Called from instrumented code. |
302 TEXT runtime·racefuncenter(SB), NOSPLIT, $0-8 | 159 TEXT runtime·racefuncenter(SB), NOSPLIT, $0-8 |
303 » get_tls(R12) | 160 » MOVQ» DX, R15»» // save function entry context (for closures) |
304 » MOVQ» m(R12), R13 | 161 » get_tls(R12) |
305 » MOVQ» g(R12), R14 | 162 » MOVQ» g(R12), R14 |
306 » // Prepare arguments for the call. | 163 » MOVQ» g_racectx(R14), RARG0» // goroutine context |
307 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | 164 » MOVQ» callpc+0(FP), RARG1 |
308 » MOVQ» (SP), SI» » // 1th arg: caller pc | |
309 » // Switch to g0 stack. | |
310 » MOVQ» SP, R12 | |
311 » MOVQ» m_g0(R13), R15 | |
312 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
313 » // Save function entry context (for closures). | |
314 » MOVQ» DX, R15 | |
315 // void __tsan_func_enter(ThreadState *thr, void *pc); | 165 // void __tsan_func_enter(ThreadState *thr, void *pc); |
316 » CALL» __tsan_func_enter(SB) | 166 » MOVQ» $__tsan_func_enter(SB), AX |
317 » MOVQ» R12, SP | 167 » CALL» racecall<>(SB) |
318 » MOVQ» R15, DX | 168 » MOVQ» R15, DX»// restore function entry context |
319 RET | 169 RET |
320 | 170 |
321 // func runtime·racefuncexit() | 171 // func runtime·racefuncexit() |
| 172 // Called from instrumented code. |
322 TEXT runtime·racefuncexit(SB), NOSPLIT, $0-0 | 173 TEXT runtime·racefuncexit(SB), NOSPLIT, $0-0 |
323 get_tls(R12) | 174 get_tls(R12) |
324 » MOVQ» m(R12), R13 | 175 » MOVQ» g(R12), R14 |
325 » MOVQ» g(R12), R14 | 176 » MOVQ» g_racectx(R14), RARG0» // goroutine context |
326 » // Prepare arguments for the call. | |
327 » MOVQ» g_racectx(R14), DI» // 0th arg: goroutine context | |
328 » // Switch to g0 stack. | |
329 » MOVQ» SP, R12 | |
330 » MOVQ» m_g0(R13), R15 | |
331 » MOVQ» (g_sched+gobuf_sp)(R15), SP | |
332 // void __tsan_func_exit(ThreadState *thr); | 177 // void __tsan_func_exit(ThreadState *thr); |
333 » CALL» __tsan_func_exit(SB) | 178 » MOVQ» $__tsan_func_exit(SB), AX |
334 » MOVQ» R12, SP | 179 » JMP» racecall<>(SB) |
335 » RET | |
336 | 180 |
337 // void runtime·racecall(void(*f)(...), ...) | 181 // void runtime·racecall(void(*f)(...), ...) |
338 // Calls C function f from race runtime and passes up to 4 arguments to it. | 182 // Calls C function f from race runtime and passes up to 4 arguments to it. |
339 // The arguments are never heap-object-preserving pointers, so we pretend there
are no arguments. | 183 // The arguments are never heap-object-preserving pointers, so we pretend there
are no arguments. |
340 TEXT runtime·racecall(SB), NOSPLIT, $0-0 | 184 TEXT runtime·racecall(SB), NOSPLIT, $0-0 |
| 185 MOVQ fn+0(FP), AX |
| 186 MOVQ arg0+8(FP), RARG0 |
| 187 MOVQ arg1+16(FP), RARG1 |
| 188 MOVQ arg2+24(FP), RARG2 |
| 189 MOVQ arg3+32(FP), RARG3 |
| 190 JMP racecall<>(SB) |
| 191 |
| 192 // Switches SP to g0 stack and calls (AX). Arguments already set. |
| 193 TEXT racecall<>(SB), NOSPLIT, $0-0 |
341 get_tls(R12) | 194 get_tls(R12) |
342 MOVQ m(R12), R13 | 195 MOVQ m(R12), R13 |
343 MOVQ g(R12), R14 | 196 MOVQ g(R12), R14 |
344 // Prepare arguments for the call. | |
345 MOVQ 8(SP), AX // function pointer | |
346 MOVQ 16(SP), DI // 4 arguments | |
347 MOVQ 24(SP), SI | |
348 MOVQ 32(SP), DX | |
349 MOVQ 48(SP), CX | |
350 // Switch to g0 stack. | 197 // Switch to g0 stack. |
351 » MOVQ» SP, R12 | 198 » MOVQ» SP, R12»» // callee-saved, preserved across the CALL |
352 » MOVQ» m_g0(R13), R15 | 199 » MOVQ» m_g0(R13), R10 |
353 » CMPQ» R15, R14 | 200 » CMPQ» R10, R14 |
354 JE racecall_cont // already on g0 | 201 JE racecall_cont // already on g0 |
355 » MOVQ» (g_sched+gobuf_sp)(R15), SP | 202 » MOVQ» (g_sched+gobuf_sp)(R10), SP |
356 racecall_cont: | 203 racecall_cont: |
| 204 ANDQ $~15, SP // alignment for gcc ABI |
357 CALL AX | 205 CALL AX |
358 MOVQ R12, SP | 206 MOVQ R12, SP |
359 RET | 207 RET |
360 | 208 |
361 // C->Go callback thunk between: | 209 // C->Go callback thunk that allows to call runtime·racesymbolize from C code. |
362 // void __tsan_symbolize(void *ctx) | |
363 // and | |
364 // runtime·racesymbolize(void *ctx) | |
365 // Direct Go->C race call has only switched SP, finish g->g0 switch by setting c
orrect g. | 210 // Direct Go->C race call has only switched SP, finish g->g0 switch by setting c
orrect g. |
366 // The overall effect of Go->C->Go call chain is similar to that of mcall. | 211 // The overall effect of Go->C->Go call chain is similar to that of mcall. |
367 // Here we use only caller saved registers (CX, DX, SI). | 212 TEXT» runtime·racesymbolizethunk(SB), NOSPLIT, $56-8 |
368 TEXT» __tsan_symbolize(SB), NOSPLIT, $8-8 | 213 » // Save callee-saved registers (Go code won't respect that). |
369 » get_tls(CX) | 214 » // This is superset of darwin/linux/windows registers. |
370 » MOVQ» m(CX), DX | 215 » PUSHQ» BX |
371 » MOVQ» m_g0(DX), SI | 216 » PUSHQ» BP |
372 » MOVQ» SI, g(CX)» // g = m->g0 | 217 » PUSHQ» DI |
373 » MOVQ» DI, 0(SP)» // func arg | 218 » PUSHQ» SI |
374 » ARGSIZE(8) | 219 » PUSHQ» R12 |
| 220 » PUSHQ» R13 |
| 221 » PUSHQ» R14 |
| 222 » PUSHQ» R15 |
| 223 » // Set g = g0. |
| 224 » get_tls(R12) |
| 225 » MOVQ» m(R12), R13 |
| 226 » MOVQ» m_g0(R13), R14 |
| 227 » MOVQ» R14, g(R12)» // g = m->g0 |
| 228 » MOVQ» RARG0, 0(SP)» // func arg |
375 CALL runtime·racesymbolize(SB) | 229 CALL runtime·racesymbolize(SB) |
376 » get_tls(CX) | 230 » // All registers are smashed after Go code, reload. |
377 » MOVQ» m(CX), DX | 231 » get_tls(R12) |
378 » MOVQ» m_curg(DX), SI | 232 » MOVQ» m(R12), R13 |
379 » MOVQ» SI, g(CX)» // g = m->curg | 233 » MOVQ» m_curg(R13), R14 |
380 » RET | 234 » MOVQ» R14, g(R12)» // g = m->curg |
| 235 » // Restore callee-saved registers. |
| 236 » POPQ» R15 |
| 237 » POPQ» R14 |
| 238 » POPQ» R13 |
| 239 » POPQ» R12 |
| 240 » POPQ» SI |
| 241 » POPQ» DI |
| 242 » POPQ» BP |
| 243 » POPQ» BX |
| 244 » RET |
LEFT | RIGHT |