LEFT | RIGHT |
1 // Copyright 2012 The Go Authors. All rights reserved. | 1 // Copyright 2012 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 #include "runtime.h" | 5 #include "runtime.h" |
6 #include "arch_GOARCH.h" | 6 #include "arch_GOARCH.h" |
7 #include "defs_GOOS_GOARCH.h" | 7 #include "defs_GOOS_GOARCH.h" |
8 #include "malloc.h" | 8 #include "malloc.h" |
9 | 9 |
10 #define LOG if(1) {} else runtime·printf | 10 #define LOG if(1) {} else runtime·printf |
| 11 #define LOG1 if(0) {} else runtime·printf |
11 | 12 |
12 typedef struct PollFD PollFD; | 13 typedef struct PollFD PollFD; |
13 struct PollFD | 14 struct PollFD |
14 { | 15 { |
15 » Lock; | 16 » PollFD* link;» // in s.cache |
16 » PollFD*»link; | 17 » Lock;» » // protectes the following fields |
17 » G*» r; | 18 » bool» closing; |
18 » G*» w; | 19 » uint32» seq; |
| 20 » G*» rg;» // G waiting for read or READY |
| 21 » Timer» rt;» // read deadline timer (set if rd != 0) |
| 22 » int64» rd;» // read deadline |
| 23 » G*» wg;» // the same for writes |
| 24 » Timer» wt; |
| 25 » int64» wd; |
19 }; | 26 }; |
20 | 27 |
21 static struct | 28 static struct |
22 { | 29 { |
23 Lock; | 30 Lock; |
24 int32 fd; | 31 int32 fd; |
25 PollFD* cache; | 32 PollFD* cache; |
26 } s; | 33 } s; |
27 | 34 |
28 int32 runtime·epollcreate1(int32 flags); | 35 static void» epollblock(PollFD *fd, G**); |
29 int32 runtime·epollctl(int32 epfd, int32 op, int32 fd, EpollEvent *ev); | 36 static G*» epollunblock(PollFD *fd, G**); |
30 int32 runtime·epollwait(int32 epfd, EpollEvent *ev, int32 nev, int32 timeout); | 37 static void» deadline(int64, Eface); |
31 | 38 static void» readDeadline(int64, Eface); |
32 static void epollblock(PollFD *fd, G**); | 39 static void» writeDeadline(int64, Eface); |
33 static void epollunblock(PollFD *fd, G**, G**); | 40 static PollFD*» allocFD(void); |
| 41 static void» pollloop(void); |
34 | 42 |
35 #define READY ((G*)1) | 43 #define READY ((G*)1) |
36 | 44 |
37 void | 45 void |
38 net·runtime_initPollServer(void) | 46 net·runtime_initPollServer(void) |
39 { | 47 { |
40 » LOG("init poll server\n"); | 48 » int32 errno; |
41 » s.fd = runtime·epollcreate1(EPOLL_CLOEXEC); | 49 |
42 » //TODO(dvyukov): handle errors | 50 » LOG("init epoll dispatcher\n"); |
| 51 » runtime·newproc1((byte*)pollloop, nil, 0, 0, net·runtime_initPollServer)
; |
| 52 » errno = runtime·epollcreate1(EPOLL_CLOEXEC, &s.fd); |
| 53 » LOG("epollcreate1: errno=%d fd=%d\n", errno, s.fd); |
| 54 » if(errno == 0) |
| 55 » » return; |
| 56 » errno = runtime·epollcreate(1024, &s.fd); |
| 57 » LOG("epollcreate: errno=%d fd=%d\n", errno, s.fd); |
| 58 » if(errno == 0) |
| 59 » » // TODO(dvyukov): set CLOEXEC |
| 60 » » return; |
| 61 » runtime·printf("epoll: failed to create descriptor (%d)\n", errno); |
| 62 » runtime·throw("epoll: failed to create descriptor"); |
43 } | 63 } |
44 | 64 |
45 void | 65 void |
46 net·runtime_initFD(int32 intfd, PollFD *fd, Eface err) | 66 net·runtime_initFD(int32 intfd, PollFD *fd, Eface err) |
47 { | 67 { |
48 EpollEvent ev; | 68 EpollEvent ev; |
49 | 69 |
50 » fd = nil; | 70 » fd = allocFD(); |
51 » runtime·lock(&s); | 71 » runtime·lock(fd); |
52 » if(s.cache != nil) { | 72 » if(fd->wg != nil && fd->wg != READY) |
53 » » fd = s.cache; | 73 » » runtime·throw("epoll: blocked write on free descriptor"); |
54 » » s.cache = fd->link; | 74 » if(fd->rg != nil && fd->rg != READY) |
55 » } | 75 » » runtime·throw("epoll: blocked read on free descriptor"); |
56 » runtime·unlock(&s); | 76 » fd->closing = false; |
57 » if(fd != nil) { | 77 » fd->seq++; |
58 » » if(fd->w != nil && fd->w != READY) | 78 » fd->rg = nil; |
59 » » » runtime·throw("epoll: blocked write on free descriptor")
; | 79 » fd->rd = 0; |
60 » » if(fd->r != nil && fd->r != READY) | 80 » fd->wg = nil; |
61 » » » runtime·throw("epoll: blocked read on free descriptor"); | 81 » fd->wd = 0; |
62 » » fd->w = nil; | 82 » runtime·unlock(fd); |
63 » » fd->r = nil; | |
64 » } else { | |
65 » » //TODO(dvyukov): batch allocation | |
66 » » fd = (PollFD*)runtime·SysAlloc(sizeof(*fd)); | |
67 » } | |
68 ev.events = EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET; | 83 ev.events = EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET; |
69 ev.ctx = fd; | 84 ev.ctx = fd; |
70 runtime·epollctl(s.fd, EPOLL_CTL_ADD, intfd, &ev); | 85 runtime·epollctl(s.fd, EPOLL_CTL_ADD, intfd, &ev); |
71 » //TODO(dvyukov): handle errors | 86 » // TODO(dvyukov): handle errors |
72 err.type = nil; | 87 err.type = nil; |
73 err.data = nil; | 88 err.data = nil; |
74 FLUSH(&err); | 89 FLUSH(&err); |
75 FLUSH(&fd); | 90 FLUSH(&fd); |
76 LOG("init fd %d with desc %p\n", intfd, fd); | 91 LOG("init fd %d with desc %p\n", intfd, fd); |
77 } | 92 } |
78 | 93 |
79 void | 94 void |
80 net·runtime_closeFD(PollFD *fd) | 95 net·runtime_closeFD(PollFD *fd) |
81 { | 96 { |
82 LOG("close fd %p\n", fd); | 97 LOG("close fd %p\n", fd); |
83 » if(fd->w != nil && fd->w != READY) | 98 » if(!fd->closing) |
| 99 » » runtime·throw("epoll: close w/o unblock"); |
| 100 » if(fd->wg != nil && fd->wg != READY) |
84 runtime·throw("epoll: blocked write on closing descriptor"); | 101 runtime·throw("epoll: blocked write on closing descriptor"); |
85 » if(fd->r != nil && fd->r != READY) | 102 » if(fd->rg != nil && fd->rg != READY) |
86 runtime·throw("epoll: blocked read on closing descriptor"); | 103 runtime·throw("epoll: blocked read on closing descriptor"); |
87 runtime·lock(&s); | 104 runtime·lock(&s); |
88 fd->link = s.cache; | 105 fd->link = s.cache; |
89 s.cache = fd; | 106 s.cache = fd; |
90 runtime·unlock(&s); | 107 runtime·unlock(&s); |
91 } | 108 } |
92 | 109 |
93 void | 110 static int32 |
94 net·runtime_waitFD(PollFD *fd, int32 mode) | 111 checkerr(PollFD *fd, int32 mode) |
95 { | 112 { |
96 » epollblock(fd, mode=='r' ? &fd->r : &fd->w); | 113 » if(fd->closing) |
| 114 » » return 1; |
| 115 » if((mode == 'r' && fd->rd < 0) || (mode == 'w' && fd->wd < 0)) |
| 116 » » return 2; |
| 117 » return 0; |
| 118 } |
| 119 |
| 120 void |
| 121 net·runtime_resetFD(PollFD *fd, uintptr mode) |
| 122 { |
| 123 » if(mode == 'r') |
| 124 » » fd->rg = nil; |
| 125 » else if(mode == 'w') |
| 126 » » fd->wg = nil; |
| 127 } |
| 128 |
| 129 void |
| 130 net·runtime_waitFD(PollFD *fd, uintptr mode, int32 err) |
| 131 { |
| 132 » LOG("waitFD: fd=%p mode=%d\n", fd, (int32)mode); |
| 133 » runtime·lock(fd); |
| 134 » err = checkerr(fd, mode); |
| 135 » if(err) { |
| 136 » » runtime·unlock(fd); |
| 137 » » FLUSH(&err); |
| 138 » » LOG("waitFD: fd=%p, mode=%d err=%d\n", fd, (int32)mode, err); |
| 139 » » return; |
| 140 » } |
| 141 » epollblock(fd, mode=='r' ? &fd->rg : &fd->wg); |
| 142 » err = checkerr(fd, mode); |
| 143 » runtime·unlock(fd); |
| 144 » FLUSH(&err); |
| 145 » LOG("waitFD: fd=%p, mode=%d err=%d\n", fd, (int32)mode, err); |
| 146 } |
| 147 |
| 148 void |
| 149 net·runtime_setDeadlineFD(PollFD *fd, int64 d, int32 mode) |
| 150 { |
| 151 » LOG("setDeadlineFD: fd=%p d=%D mode=%d\n", fd, d, mode); |
| 152 » runtime·lock(fd); |
| 153 » if(fd->closing) { |
| 154 » » runtime·unlock(fd); |
| 155 » » return; |
| 156 » } |
| 157 » fd->seq++; |
| 158 » if(fd->rt.f) { |
| 159 » » runtime·deltimer(&fd->rt); |
| 160 » » fd->rt.f = nil; |
| 161 » } |
| 162 » if(fd->wt.f) { |
| 163 » » runtime·deltimer(&fd->wt); |
| 164 » » fd->wt.f = nil; |
| 165 » } |
| 166 » if(mode == 'r'+'w') |
| 167 » » fd->rd = fd->wd = d; |
| 168 » else if(mode == 'r') |
| 169 » » fd->rd = d; |
| 170 » else if(mode == 'w') |
| 171 » » fd->wd = d; |
| 172 » else |
| 173 » » runtime·throw("epoll: unknown mode"); |
| 174 » if(fd->rd > 0 && fd->rd == fd->wd) { |
| 175 » » fd->rt.f = deadline; |
| 176 » » fd->rt.when = fd->rd; |
| 177 » » fd->rt.arg.type = (Type*)fd->seq; |
| 178 » » fd->rt.arg.data = fd; |
| 179 » » runtime·addtimer(&fd->rt); |
| 180 » } else { |
| 181 » » if(fd->rd > 0) { |
| 182 » » » fd->rt.f = readDeadline; |
| 183 » » » fd->rt.when = fd->rd; |
| 184 » » » fd->rt.arg.type = (Type*)fd->seq; |
| 185 » » » fd->rt.arg.data = fd; |
| 186 » » » runtime·addtimer(&fd->rt); |
| 187 » » } |
| 188 » » if(fd->wd > 0) { |
| 189 » » » fd->wt.f = writeDeadline; |
| 190 » » » fd->wt.when = fd->wd; |
| 191 » » » fd->wt.arg.type = (Type*)fd->seq; |
| 192 » » » fd->wt.arg.data = fd; |
| 193 » » » runtime·addtimer(&fd->wt); |
| 194 » » } |
| 195 » } |
| 196 » runtime·unlock(fd); |
97 } | 197 } |
98 | 198 |
99 void | 199 void |
100 net·runtime_unblockFD(PollFD *fd) | 200 net·runtime_unblockFD(PollFD *fd) |
101 { | 201 { |
102 » G *gp; | 202 » G *rg, *wg; |
103 | 203 |
104 » gp = nil; | 204 » runtime·lock(fd); |
105 » epollunblock(fd, &fd->r, &gp); | 205 » fd->closing = true; |
106 » if(gp) | 206 » fd->seq++; |
107 » » runtime·ready(gp); | 207 » rg = epollunblock(fd, &fd->rg); |
108 » gp = nil; | 208 » wg = epollunblock(fd, &fd->wg); |
109 » epollunblock(fd, &fd->w, &gp); | 209 » if(fd->rt.f) { |
110 » if(gp) | 210 » » runtime·deltimer(&fd->rt); |
111 » » runtime·ready(gp); | 211 » » fd->rt.f = nil; |
112 } | 212 » } |
113 | 213 » if(fd->wt.f) { |
114 G* | 214 » » runtime·deltimer(&fd->wt); |
115 runtime·netwait(uint32 ms) | 215 » » fd->wt.f = nil; |
| 216 » } |
| 217 » runtime·unlock(fd); |
| 218 » if(rg) |
| 219 » » runtime·ready(rg); |
| 220 » if(wg) |
| 221 » » runtime·ready(wg); |
| 222 } |
| 223 |
| 224 static G* |
| 225 netwait(uint32 ms, uint32 cnt) |
116 { | 226 { |
117 EpollEvent ev, events[128]; | 227 EpollEvent ev, events[128]; |
118 PollFD *fd; | 228 PollFD *fd; |
119 » G *gp; | 229 » G *gp, *rg, *wg; |
120 int32 n, i; | 230 int32 n, i; |
121 | 231 |
122 LOG("epoll wait\n"); | 232 LOG("epoll wait\n"); |
| 233 /* |
123 if(s.fd == 0) { | 234 if(s.fd == 0) { |
124 » » runtime·usleep(ms); | 235 » » runtime·usleep(ms*1000); |
125 return nil; | 236 return nil; |
126 } | 237 } |
127 » n = runtime·epollwait(s.fd, events, nelem(events), ms); | 238 » */ |
| 239 » if(cnt > nelem(events)) |
| 240 » » cnt = nelem(events); |
| 241 » runtime·entersyscall(); |
| 242 » n = runtime·epollwait(s.fd, events, cnt, ms); |
| 243 » runtime·exitsyscall(); |
128 //TODO(dvyukov): handle errors other than EAGAIN and EINTR | 244 //TODO(dvyukov): handle errors other than EAGAIN and EINTR |
129 LOG("epoll wait got %d\n", n); | 245 LOG("epoll wait got %d\n", n); |
130 if(n <= 0) | 246 if(n <= 0) |
131 return nil; | 247 return nil; |
132 gp = nil; | 248 gp = nil; |
133 for(i = 0; i < n; i++) { | 249 for(i = 0; i < n; i++) { |
134 ev = events[i]; | 250 ev = events[i]; |
| 251 if(ev.events == 0) |
| 252 continue; |
135 fd = (PollFD*)ev.ctx; | 253 fd = (PollFD*)ev.ctx; |
136 LOG("epoll event on %p %d\n", fd, ev.events); | 254 LOG("epoll event on %p %d\n", fd, ev.events); |
| 255 rg = wg = nil; |
| 256 runtime·lock(fd); |
137 if(ev.events & (EPOLLIN|EPOLLRDHUP|EPOLLHUP|EPOLLERR)) | 257 if(ev.events & (EPOLLIN|EPOLLRDHUP|EPOLLHUP|EPOLLERR)) |
138 » » » epollunblock(fd, &fd->r, &gp); | 258 » » » rg = epollunblock(fd, &fd->rg); |
139 if(ev.events & (EPOLLOUT|EPOLLHUP|EPOLLERR)) | 259 if(ev.events & (EPOLLOUT|EPOLLHUP|EPOLLERR)) |
140 » » » epollunblock(fd, &fd->w, &gp); | 260 » » » wg = epollunblock(fd, &fd->wg); |
| 261 » » runtime·unlock(fd); |
| 262 » » if(rg) { |
| 263 » » » rg->schedlink = gp; |
| 264 » » » gp = rg; |
| 265 » » } |
| 266 » » if(wg) { |
| 267 » » » wg->schedlink = gp; |
| 268 » » » gp = wg; |
| 269 » » } |
141 } | 270 } |
142 return gp; | 271 return gp; |
143 } | 272 } |
144 | 273 |
145 static void | 274 static void |
| 275 pollloop(void) |
| 276 { |
| 277 G *gp, *gp1; |
| 278 |
| 279 for(;;) { |
| 280 gp = netwait(10, 10); |
| 281 while(gp) { |
| 282 gp1 = gp; |
| 283 gp = gp->schedlink; |
| 284 runtime·ready(gp1); |
| 285 } |
| 286 } |
| 287 } |
| 288 |
| 289 static void |
146 epollblock(PollFD *fd, G **gp) | 290 epollblock(PollFD *fd, G **gp) |
147 { | 291 { |
148 // G *old; | |
149 | |
150 LOG("epoll block %p on %p\n", g, gp); | 292 LOG("epoll block %p on %p\n", g, gp); |
151 /* | |
152 g->status = Gwaiting; | |
153 g->waitreason = "IO wait"; | |
154 for(;;) { | |
155 old = *gp; | |
156 if(old == READY) { | |
157 *gp = nil; | |
158 g->status = Grunning; | |
159 g->waitreason = nil; | |
160 return; | |
161 } | |
162 if(old != nil) | |
163 runtime·throw("epoll: double wait"); | |
164 if(runtime·casp(gp, nil, g)) { | |
165 runtime·gosched(); | |
166 return; | |
167 } | |
168 } | |
169 */ | |
170 runtime·lock(fd); | |
171 if(*gp == READY) { | 293 if(*gp == READY) { |
172 *gp = nil; | 294 *gp = nil; |
173 » » runtime·unlock(fd); | 295 » » return; |
174 » » return; | 296 » } |
175 » } | 297 » if(*gp != nil) |
176 » if(*gp == nil) { | 298 » » runtime·throw("epoll: double wait"); |
177 » » *gp = g; | 299 » *gp = g; |
178 » » runtime·park(fd, (void(*)(void*))runtime·unlock, "IO wait"); | 300 » runtime·park(runtime·unlock, &fd->Lock, "IO wait"); |
179 » » return; | 301 » runtime·lock(fd); |
180 » } | 302 } |
181 » runtime·throw("epoll: double wait"); | 303 |
182 } | 304 static G* |
183 | 305 epollunblock(PollFD *fd, G **gp) |
184 static void | |
185 epollunblock(PollFD *fd, G **gp, G **gp0) | |
186 { | 306 { |
187 G *old; | 307 G *old; |
188 | 308 |
| 309 USED(fd); |
189 LOG("epoll unblock on %p\n", gp); | 310 LOG("epoll unblock on %p\n", gp); |
190 /* | 311 » if(*gp == READY) |
191 » for(;;) { | 312 » » return nil; |
192 » » old = *gp; | |
193 » » if(old == READY) | |
194 » » » return; | |
195 » » if(old == nil && runtime·casp(gp, nil, READY)) | |
196 » » » return; | |
197 » » if(runtime·casp(gp, old, nil)) { | |
198 » » » runtime·ready(old); | |
199 » » » return; | |
200 » » } | |
201 » } | |
202 */ | |
203 » runtime·lock(fd); | |
204 » if(*gp == READY) { | |
205 » » runtime·unlock(fd); | |
206 » » return; | |
207 » } | |
208 if(*gp == nil) { | 313 if(*gp == nil) { |
209 *gp = READY; | 314 *gp = READY; |
210 » » runtime·unlock(fd); | 315 » » return nil; |
211 » » return; | |
212 } | 316 } |
213 old = *gp; | 317 old = *gp; |
214 *gp = nil; | 318 *gp = nil; |
215 » runtime·unlock(fd); | 319 » return old; |
216 » old->schedlink = *gp0; | 320 } |
217 » *gp0 = old; | 321 |
218 } | 322 static void |
| 323 deadlineimpl(int64 now, Eface arg, bool read, bool write) |
| 324 { |
| 325 » PollFD *fd; |
| 326 » uint32 seq; |
| 327 » G *rg, *wg; |
| 328 |
| 329 » USED(now); |
| 330 » fd = (PollFD*)arg.data; |
| 331 » seq = (uint32)(uintptr)arg.type; |
| 332 » rg = wg = nil; |
| 333 » LOG("deadline: fd=%p fd->seq=%d seq=%d read=%d write=%d\n", fd, fd->seq,
seq, read, write); |
| 334 » runtime·lock(fd); |
| 335 » if(seq != fd->seq) { |
| 336 » » runtime·unlock(fd); |
| 337 » » return; |
| 338 » } |
| 339 » if(read) { |
| 340 » » if(fd->rd <= 0 || fd->rt.f == nil) |
| 341 » » » runtime·throw("epoll: inconsistent read deadline"); |
| 342 » » fd->rd = -1; |
| 343 » » fd->rt.f = nil; |
| 344 » » rg = epollunblock(fd, &fd->rg); |
| 345 » } |
| 346 » if(write) { |
| 347 » » if(fd->wd <= 0) |
| 348 » » » runtime·throw("epoll: inconsistent write deadline"); |
| 349 » » fd->wd = -1; |
| 350 » » fd->wt.f = nil; |
| 351 » » wg = epollunblock(fd, &fd->wg); |
| 352 » } |
| 353 » runtime·unlock(fd); |
| 354 » if(rg) |
| 355 » » runtime·ready(rg); |
| 356 » if(wg) |
| 357 » » runtime·ready(wg); |
| 358 } |
| 359 |
| 360 static void |
| 361 deadline(int64 now, Eface arg) |
| 362 { |
| 363 » deadlineimpl(now, arg, true, true); |
| 364 } |
| 365 |
| 366 static void |
| 367 readDeadline(int64 now, Eface arg) |
| 368 { |
| 369 » deadlineimpl(now, arg, true, false); |
| 370 } |
| 371 |
| 372 static void |
| 373 writeDeadline(int64 now, Eface arg) |
| 374 { |
| 375 » deadlineimpl(now, arg, false, true); |
| 376 } |
| 377 |
| 378 static PollFD* |
| 379 allocFD(void) |
| 380 { |
| 381 » PollFD *fd; |
| 382 » uint32 i, n; |
| 383 |
| 384 » runtime·lock(&s); |
| 385 » if(s.cache == nil) { |
| 386 » » n = PageSize/sizeof(*fd); |
| 387 » » if(n == 0) |
| 388 » » » n = 1; |
| 389 » » fd = (PollFD*)runtime·SysAlloc(n*sizeof(*fd)); |
| 390 » » for(i=0; i<n; i++) { |
| 391 » » » fd[i].link = s.cache; |
| 392 » » » s.cache = &fd[i]; |
| 393 » » }· |
| 394 » } |
| 395 » fd = s.cache; |
| 396 » s.cache = fd->link; |
| 397 » runtime·unlock(&s); |
| 398 » return fd; |
| 399 } |
LEFT | RIGHT |