LEFT | RIGHT |
1 // Copyright 2012 The Go Authors. All rights reserved. | 1 // Copyright 2012 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 #include "runtime.h" | 5 #include "runtime.h" |
| 6 #include "arch_GOARCH.h" |
| 7 #include "defs_GOOS_GOARCH.h" |
6 #include "malloc.h" | 8 #include "malloc.h" |
7 | 9 |
8 static struct | 10 #define LOG if(1) {} else runtime·printf |
9 { | 11 #define LOG1 if(0) {} else runtime·printf |
10 } serv; | |
11 | 12 |
12 typedef struct PollFD PollFD; | 13 typedef struct PollFD PollFD; |
13 struct PollFD | 14 struct PollFD |
14 { | 15 { |
| 16 PollFD* link; // in s.cache |
| 17 Lock; // protectes the following fields |
| 18 bool closing; |
| 19 uint32 seq; |
| 20 G* rg; // G waiting for read or READY |
| 21 Timer rt; // read deadline timer (set if rd != 0) |
| 22 int64 rd; // read deadline |
| 23 G* wg; // the same for writes |
| 24 Timer wt; |
| 25 int64 wd; |
15 }; | 26 }; |
16 | 27 |
17 //func runtime_initPollServer() | 28 static struct |
| 29 { |
| 30 » Lock; |
| 31 » int32» fd; |
| 32 » PollFD*»cache; |
| 33 } s; |
| 34 |
| 35 static void» epollblock(PollFD *fd, G**); |
| 36 static G*» epollunblock(PollFD *fd, G**); |
| 37 static void» deadline(int64, Eface); |
| 38 static void» readDeadline(int64, Eface); |
| 39 static void» writeDeadline(int64, Eface); |
| 40 static PollFD*» allocFD(void); |
| 41 static void» pollloop(void); |
| 42 |
| 43 #define READY ((G*)1) |
| 44 |
18 void | 45 void |
19 net·runtime_initPollServer(void) | 46 net·runtime_initPollServer(void) |
20 { | 47 { |
21 } | 48 » int32 errno; |
22 | 49 |
23 void | 50 » LOG("init epoll dispatcher\n"); |
24 net·runtime_initFD(int32 intfd, PollFD *fd, void *err) | 51 » runtime·newproc1((byte*)pollloop, nil, 0, 0, net·runtime_initPollServer)
; |
25 { | 52 » errno = runtime·epollcreate1(EPOLL_CLOEXEC, &s.fd); |
| 53 » LOG("epollcreate1: errno=%d fd=%d\n", errno, s.fd); |
| 54 » if(errno == 0) |
| 55 » » return; |
| 56 » errno = runtime·epollcreate(1024, &s.fd); |
| 57 » LOG("epollcreate: errno=%d fd=%d\n", errno, s.fd); |
| 58 » if(errno == 0) |
| 59 » » // TODO(dvyukov): set CLOEXEC |
| 60 » » return; |
| 61 » runtime·printf("epoll: failed to create descriptor (%d)\n", errno); |
| 62 » runtime·throw("epoll: failed to create descriptor"); |
| 63 } |
| 64 |
| 65 void |
| 66 net·runtime_initFD(int32 intfd, PollFD *fd, Eface err) |
| 67 { |
| 68 » EpollEvent ev; |
| 69 |
| 70 » fd = allocFD(); |
| 71 » runtime·lock(fd); |
| 72 » if(fd->wg != nil && fd->wg != READY) |
| 73 » » runtime·throw("epoll: blocked write on free descriptor"); |
| 74 » if(fd->rg != nil && fd->rg != READY) |
| 75 » » runtime·throw("epoll: blocked read on free descriptor"); |
| 76 » fd->closing = false; |
| 77 » fd->seq++; |
| 78 » fd->rg = nil; |
| 79 » fd->rd = 0; |
| 80 » fd->wg = nil; |
| 81 » fd->wd = 0; |
| 82 » runtime·unlock(fd); |
| 83 » ev.events = EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET; |
| 84 » ev.ctx = fd; |
| 85 » runtime·epollctl(s.fd, EPOLL_CTL_ADD, intfd, &ev); |
| 86 » // TODO(dvyukov): handle errors |
| 87 » err.type = nil; |
| 88 » err.data = nil; |
| 89 » FLUSH(&err); |
| 90 » FLUSH(&fd); |
| 91 » LOG("init fd %d with desc %p\n", intfd, fd); |
26 } | 92 } |
27 | 93 |
28 void | 94 void |
29 net·runtime_closeFD(PollFD *fd) | 95 net·runtime_closeFD(PollFD *fd) |
30 { | 96 { |
31 } | 97 » LOG("close fd %p\n", fd); |
32 | 98 » if(!fd->closing) |
33 void | 99 » » runtime·throw("epoll: close w/o unblock"); |
34 net·runtime_waitFD(PollFD *fd, int32 mode) | 100 » if(fd->wg != nil && fd->wg != READY) |
35 { | 101 » » runtime·throw("epoll: blocked write on closing descriptor"); |
| 102 » if(fd->rg != nil && fd->rg != READY) |
| 103 » » runtime·throw("epoll: blocked read on closing descriptor"); |
| 104 » runtime·lock(&s); |
| 105 » fd->link = s.cache; |
| 106 » s.cache = fd; |
| 107 » runtime·unlock(&s); |
| 108 } |
| 109 |
| 110 static int32 |
| 111 checkerr(PollFD *fd, int32 mode) |
| 112 { |
| 113 » if(fd->closing) |
| 114 » » return 1; |
| 115 » if((mode == 'r' && fd->rd < 0) || (mode == 'w' && fd->wd < 0)) |
| 116 » » return 2; |
| 117 » return 0; |
| 118 } |
| 119 |
| 120 void |
| 121 net·runtime_resetFD(PollFD *fd, uintptr mode) |
| 122 { |
| 123 » if(mode == 'r') |
| 124 » » fd->rg = nil; |
| 125 » else if(mode == 'w') |
| 126 » » fd->wg = nil; |
| 127 } |
| 128 |
| 129 void |
| 130 net·runtime_waitFD(PollFD *fd, uintptr mode, int32 err) |
| 131 { |
| 132 » LOG("waitFD: fd=%p mode=%d\n", fd, (int32)mode); |
| 133 » runtime·lock(fd); |
| 134 » err = checkerr(fd, mode); |
| 135 » if(err) { |
| 136 » » runtime·unlock(fd); |
| 137 » » FLUSH(&err); |
| 138 » » LOG("waitFD: fd=%p, mode=%d err=%d\n", fd, (int32)mode, err); |
| 139 » » return; |
| 140 » } |
| 141 » epollblock(fd, mode=='r' ? &fd->rg : &fd->wg); |
| 142 » err = checkerr(fd, mode); |
| 143 » runtime·unlock(fd); |
| 144 » FLUSH(&err); |
| 145 » LOG("waitFD: fd=%p, mode=%d err=%d\n", fd, (int32)mode, err); |
| 146 } |
| 147 |
| 148 void |
| 149 net·runtime_setDeadlineFD(PollFD *fd, int64 d, int32 mode) |
| 150 { |
| 151 » LOG("setDeadlineFD: fd=%p d=%D mode=%d\n", fd, d, mode); |
| 152 » runtime·lock(fd); |
| 153 » if(fd->closing) { |
| 154 » » runtime·unlock(fd); |
| 155 » » return; |
| 156 » } |
| 157 » fd->seq++; |
| 158 » if(fd->rt.f) { |
| 159 » » runtime·deltimer(&fd->rt); |
| 160 » » fd->rt.f = nil; |
| 161 » } |
| 162 » if(fd->wt.f) { |
| 163 » » runtime·deltimer(&fd->wt); |
| 164 » » fd->wt.f = nil; |
| 165 » } |
| 166 » if(mode == 'r'+'w') |
| 167 » » fd->rd = fd->wd = d; |
| 168 » else if(mode == 'r') |
| 169 » » fd->rd = d; |
| 170 » else if(mode == 'w') |
| 171 » » fd->wd = d; |
| 172 » else |
| 173 » » runtime·throw("epoll: unknown mode"); |
| 174 » if(fd->rd > 0 && fd->rd == fd->wd) { |
| 175 » » fd->rt.f = deadline; |
| 176 » » fd->rt.when = fd->rd; |
| 177 » » fd->rt.arg.type = (Type*)fd->seq; |
| 178 » » fd->rt.arg.data = fd; |
| 179 » » runtime·addtimer(&fd->rt); |
| 180 » } else { |
| 181 » » if(fd->rd > 0) { |
| 182 » » » fd->rt.f = readDeadline; |
| 183 » » » fd->rt.when = fd->rd; |
| 184 » » » fd->rt.arg.type = (Type*)fd->seq; |
| 185 » » » fd->rt.arg.data = fd; |
| 186 » » » runtime·addtimer(&fd->rt); |
| 187 » » } |
| 188 » » if(fd->wd > 0) { |
| 189 » » » fd->wt.f = writeDeadline; |
| 190 » » » fd->wt.when = fd->wd; |
| 191 » » » fd->wt.arg.type = (Type*)fd->seq; |
| 192 » » » fd->wt.arg.data = fd; |
| 193 » » » runtime·addtimer(&fd->wt); |
| 194 » » } |
| 195 » } |
| 196 » runtime·unlock(fd); |
36 } | 197 } |
37 | 198 |
38 void | 199 void |
39 net·runtime_unblockFD(PollFD *fd) | 200 net·runtime_unblockFD(PollFD *fd) |
40 { | 201 { |
41 } | 202 » G *rg, *wg; |
| 203 |
| 204 » runtime·lock(fd); |
| 205 » fd->closing = true; |
| 206 » fd->seq++; |
| 207 » rg = epollunblock(fd, &fd->rg); |
| 208 » wg = epollunblock(fd, &fd->wg); |
| 209 » if(fd->rt.f) { |
| 210 » » runtime·deltimer(&fd->rt); |
| 211 » » fd->rt.f = nil; |
| 212 » } |
| 213 » if(fd->wt.f) { |
| 214 » » runtime·deltimer(&fd->wt); |
| 215 » » fd->wt.f = nil; |
| 216 » } |
| 217 » runtime·unlock(fd); |
| 218 » if(rg) |
| 219 » » runtime·ready(rg); |
| 220 » if(wg) |
| 221 » » runtime·ready(wg); |
| 222 } |
| 223 |
| 224 static G* |
| 225 netwait(uint32 ms, uint32 cnt) |
| 226 { |
| 227 » EpollEvent ev, events[128]; |
| 228 » PollFD *fd; |
| 229 » G *gp, *rg, *wg; |
| 230 » int32 n, i; |
| 231 |
| 232 » LOG("epoll wait\n"); |
| 233 » /* |
| 234 » if(s.fd == 0) { |
| 235 » » runtime·usleep(ms*1000); |
| 236 » » return nil; |
| 237 » } |
| 238 » */ |
| 239 » if(cnt > nelem(events)) |
| 240 » » cnt = nelem(events); |
| 241 » runtime·entersyscall(); |
| 242 » n = runtime·epollwait(s.fd, events, cnt, ms); |
| 243 » runtime·exitsyscall(); |
| 244 » //TODO(dvyukov): handle errors other than EAGAIN and EINTR |
| 245 » LOG("epoll wait got %d\n", n); |
| 246 » if(n <= 0) |
| 247 » » return nil; |
| 248 » gp = nil; |
| 249 » for(i = 0; i < n; i++) { |
| 250 » » ev = events[i]; |
| 251 » » if(ev.events == 0) |
| 252 » » » continue; |
| 253 » » fd = (PollFD*)ev.ctx; |
| 254 » » LOG("epoll event on %p %d\n", fd, ev.events); |
| 255 » » rg = wg = nil; |
| 256 » » runtime·lock(fd); |
| 257 » » if(ev.events & (EPOLLIN|EPOLLRDHUP|EPOLLHUP|EPOLLERR)) |
| 258 » » » rg = epollunblock(fd, &fd->rg); |
| 259 » » if(ev.events & (EPOLLOUT|EPOLLHUP|EPOLLERR)) |
| 260 » » » wg = epollunblock(fd, &fd->wg); |
| 261 » » runtime·unlock(fd); |
| 262 » » if(rg) { |
| 263 » » » rg->schedlink = gp; |
| 264 » » » gp = rg; |
| 265 » » } |
| 266 » » if(wg) { |
| 267 » » » wg->schedlink = gp; |
| 268 » » » gp = wg; |
| 269 » » } |
| 270 » } |
| 271 » return gp; |
| 272 } |
| 273 |
| 274 static void |
| 275 pollloop(void) |
| 276 { |
| 277 » G *gp, *gp1; |
| 278 |
| 279 » for(;;) { |
| 280 » » gp = netwait(10, 10); |
| 281 » » while(gp) { |
| 282 » » » gp1 = gp; |
| 283 » » » gp = gp->schedlink; |
| 284 » » » runtime·ready(gp1); |
| 285 » » } |
| 286 » } |
| 287 } |
| 288 |
| 289 static void |
| 290 epollblock(PollFD *fd, G **gp) |
| 291 { |
| 292 » LOG("epoll block %p on %p\n", g, gp); |
| 293 » if(*gp == READY) { |
| 294 » » *gp = nil; |
| 295 » » return; |
| 296 » } |
| 297 » if(*gp != nil) |
| 298 » » runtime·throw("epoll: double wait"); |
| 299 » *gp = g; |
| 300 » runtime·park(runtime·unlock, &fd->Lock, "IO wait"); |
| 301 » runtime·lock(fd); |
| 302 } |
| 303 |
| 304 static G* |
| 305 epollunblock(PollFD *fd, G **gp) |
| 306 { |
| 307 » G *old; |
| 308 |
| 309 » USED(fd); |
| 310 » LOG("epoll unblock on %p\n", gp); |
| 311 » if(*gp == READY) |
| 312 » » return nil; |
| 313 » if(*gp == nil) { |
| 314 » » *gp = READY; |
| 315 » » return nil; |
| 316 » } |
| 317 » old = *gp; |
| 318 » *gp = nil; |
| 319 » return old; |
| 320 } |
| 321 |
| 322 static void |
| 323 deadlineimpl(int64 now, Eface arg, bool read, bool write) |
| 324 { |
| 325 » PollFD *fd; |
| 326 » uint32 seq; |
| 327 » G *rg, *wg; |
| 328 |
| 329 » USED(now); |
| 330 » fd = (PollFD*)arg.data; |
| 331 » seq = (uint32)(uintptr)arg.type; |
| 332 » rg = wg = nil; |
| 333 » LOG("deadline: fd=%p fd->seq=%d seq=%d read=%d write=%d\n", fd, fd->seq,
seq, read, write); |
| 334 » runtime·lock(fd); |
| 335 » if(seq != fd->seq) { |
| 336 » » runtime·unlock(fd); |
| 337 » » return; |
| 338 » } |
| 339 » if(read) { |
| 340 » » if(fd->rd <= 0 || fd->rt.f == nil) |
| 341 » » » runtime·throw("epoll: inconsistent read deadline"); |
| 342 » » fd->rd = -1; |
| 343 » » fd->rt.f = nil; |
| 344 » » rg = epollunblock(fd, &fd->rg); |
| 345 » } |
| 346 » if(write) { |
| 347 » » if(fd->wd <= 0) |
| 348 » » » runtime·throw("epoll: inconsistent write deadline"); |
| 349 » » fd->wd = -1; |
| 350 » » fd->wt.f = nil; |
| 351 » » wg = epollunblock(fd, &fd->wg); |
| 352 » } |
| 353 » runtime·unlock(fd); |
| 354 » if(rg) |
| 355 » » runtime·ready(rg); |
| 356 » if(wg) |
| 357 » » runtime·ready(wg); |
| 358 } |
| 359 |
| 360 static void |
| 361 deadline(int64 now, Eface arg) |
| 362 { |
| 363 » deadlineimpl(now, arg, true, true); |
| 364 } |
| 365 |
| 366 static void |
| 367 readDeadline(int64 now, Eface arg) |
| 368 { |
| 369 » deadlineimpl(now, arg, true, false); |
| 370 } |
| 371 |
| 372 static void |
| 373 writeDeadline(int64 now, Eface arg) |
| 374 { |
| 375 » deadlineimpl(now, arg, false, true); |
| 376 } |
| 377 |
| 378 static PollFD* |
| 379 allocFD(void) |
| 380 { |
| 381 » PollFD *fd; |
| 382 » uint32 i, n; |
| 383 |
| 384 » runtime·lock(&s); |
| 385 » if(s.cache == nil) { |
| 386 » » n = PageSize/sizeof(*fd); |
| 387 » » if(n == 0) |
| 388 » » » n = 1; |
| 389 » » fd = (PollFD*)runtime·SysAlloc(n*sizeof(*fd)); |
| 390 » » for(i=0; i<n; i++) { |
| 391 » » » fd[i].link = s.cache; |
| 392 » » » s.cache = &fd[i]; |
| 393 » » }· |
| 394 » } |
| 395 » fd = s.cache; |
| 396 » s.cache = fd->link; |
| 397 » runtime·unlock(&s); |
| 398 » return fd; |
| 399 } |
LEFT | RIGHT |