LEFT | RIGHT |
1 // Copyright 2012 The Go Authors. All rights reserved. | 1 // Copyright 2012 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 #include "runtime.h" | 5 #include "runtime.h" |
6 #include "arch_GOARCH.h" | 6 #include "arch_GOARCH.h" |
7 #include "defs_GOOS_GOARCH.h" | 7 #include "defs_GOOS_GOARCH.h" |
8 #include "malloc.h" | 8 #include "malloc.h" |
9 | 9 |
| 10 #define LOG if(1) {} else runtime·printf |
| 11 #define LOG1 if(0) {} else runtime·printf |
| 12 |
10 typedef struct PollFD PollFD; | 13 typedef struct PollFD PollFD; |
11 struct PollFD | 14 struct PollFD |
12 { | 15 { |
13 » PollFD*»link; | 16 » PollFD* link;» // in s.cache |
14 » G*» r; | 17 » Lock;» » // protectes the following fields |
15 » G*» w; | 18 » bool» closing; |
| 19 » uint32» seq; |
| 20 » G*» rg;» // G waiting for read or READY |
| 21 » Timer» rt;» // read deadline timer (set if rd != 0) |
| 22 » int64» rd;» // read deadline |
| 23 » G*» wg;» // the same for writes |
| 24 » Timer» wt; |
| 25 » int64» wd; |
16 }; | 26 }; |
17 | 27 |
18 static struct | 28 static struct |
19 { | 29 { |
20 Lock; | 30 Lock; |
21 int32 fd; | 31 int32 fd; |
22 PollFD* cache; | 32 PollFD* cache; |
23 M m; | |
24 G g; | |
25 byte stk[64*1024]; | |
26 } s; | 33 } s; |
27 | 34 |
28 int32 runtime·epollcreate1(int32 flags); | 35 static void» epollblock(PollFD *fd, G**); |
29 int32 runtime·epollctl(int32 epfd, int32 op, int32 fd, EpollEvent *ev); | 36 static G*» epollunblock(PollFD *fd, G**); |
30 int32 runtime·epollwait(int32 epfd, EpollEvent *ev, int32 nev, int32 timeout); | 37 static void» deadline(int64, Eface); |
31 | 38 static void» readDeadline(int64, Eface); |
32 static void epollloop(void); | 39 static void» writeDeadline(int64, Eface); |
33 static void epollblock(G**); | 40 static PollFD*» allocFD(void); |
34 static void epollunblock(G**); | 41 static void» pollloop(void); |
35 | 42 |
36 #define READY ((G*)1) | 43 #define READY ((G*)1) |
37 | 44 |
38 void | 45 void |
39 net·runtime_initPollServer(void) | 46 net·runtime_initPollServer(void) |
40 { | 47 { |
41 » s.fd = runtime·epollcreate1(EPOLL_CLOEXEC); | 48 » int32 errno; |
42 » //TODO(dvyukov): handle errors | 49 |
43 » runtime·newosproc(&s.m, &s.g, s.stk, epollloop); | 50 » LOG("init epoll dispatcher\n"); |
| 51 » runtime·newproc1((byte*)pollloop, nil, 0, 0, net·runtime_initPollServer)
; |
| 52 » errno = runtime·epollcreate1(EPOLL_CLOEXEC, &s.fd); |
| 53 » LOG("epollcreate1: errno=%d fd=%d\n", errno, s.fd); |
| 54 » if(errno == 0) |
| 55 » » return; |
| 56 » errno = runtime·epollcreate(1024, &s.fd); |
| 57 » LOG("epollcreate: errno=%d fd=%d\n", errno, s.fd); |
| 58 » if(errno == 0) |
| 59 » » // TODO(dvyukov): set CLOEXEC |
| 60 » » return; |
| 61 » runtime·printf("epoll: failed to create descriptor (%d)\n", errno); |
| 62 » runtime·throw("epoll: failed to create descriptor"); |
44 } | 63 } |
45 | 64 |
46 void | 65 void |
47 net·runtime_initFD(int32 intfd, PollFD *fd, Eface err) | 66 net·runtime_initFD(int32 intfd, PollFD *fd, Eface err) |
48 { | 67 { |
49 EpollEvent ev; | 68 EpollEvent ev; |
50 | 69 |
51 » fd = nil; | 70 » fd = allocFD(); |
52 » runtime·lock(&s); | 71 » runtime·lock(fd); |
53 » if(s.cache != nil) { | 72 » if(fd->wg != nil && fd->wg != READY) |
54 » » fd = s.cache; | 73 » » runtime·throw("epoll: blocked write on free descriptor"); |
55 » » s.cache = fd->link; | 74 » if(fd->rg != nil && fd->rg != READY) |
56 » } | 75 » » runtime·throw("epoll: blocked read on free descriptor"); |
57 » runtime·unlock(&s); | 76 » fd->closing = false; |
58 » if(fd != nil) { | 77 » fd->seq++; |
59 » » if(fd->w != nil && fd->w != READY) | 78 » fd->rg = nil; |
60 » » » runtime·throw("epoll: blocked write on free descriptor")
; | 79 » fd->rd = 0; |
61 » » if(fd->r != nil && fd->r != READY) | 80 » fd->wg = nil; |
62 » » » runtime·throw("epoll: blocked read on free descriptor"); | 81 » fd->wd = 0; |
63 » » fd->w = nil; | 82 » runtime·unlock(fd); |
64 » » fd->r = nil; | |
65 » } else { | |
66 » » //TODO(dvyukov): batch allocation | |
67 » » fd = (PollFD*)runtime·SysAlloc(sizeof(*fd)); | |
68 » } | |
69 ev.events = EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET; | 83 ev.events = EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET; |
70 ev.ctx = fd; | 84 ev.ctx = fd; |
71 runtime·epollctl(s.fd, EPOLL_CTL_ADD, intfd, &ev); | 85 runtime·epollctl(s.fd, EPOLL_CTL_ADD, intfd, &ev); |
72 » //TODO(dvyukov): handle errors | 86 » // TODO(dvyukov): handle errors |
73 err.type = nil; | 87 err.type = nil; |
74 err.data = nil; | 88 err.data = nil; |
75 FLUSH(&err); | 89 FLUSH(&err); |
76 FLUSH(&fd); | 90 FLUSH(&fd); |
| 91 LOG("init fd %d with desc %p\n", intfd, fd); |
77 } | 92 } |
78 | 93 |
79 void | 94 void |
80 net·runtime_closeFD(PollFD *fd) | 95 net·runtime_closeFD(PollFD *fd) |
81 { | 96 { |
82 » if(fd->w != nil && fd->w != READY) | 97 » LOG("close fd %p\n", fd); |
| 98 » if(!fd->closing) |
| 99 » » runtime·throw("epoll: close w/o unblock"); |
| 100 » if(fd->wg != nil && fd->wg != READY) |
83 runtime·throw("epoll: blocked write on closing descriptor"); | 101 runtime·throw("epoll: blocked write on closing descriptor"); |
84 » if(fd->r != nil && fd->r != READY) | 102 » if(fd->rg != nil && fd->rg != READY) |
85 runtime·throw("epoll: blocked read on closing descriptor"); | 103 runtime·throw("epoll: blocked read on closing descriptor"); |
86 runtime·lock(&s); | 104 runtime·lock(&s); |
87 fd->link = s.cache; | 105 fd->link = s.cache; |
88 s.cache = fd; | 106 s.cache = fd; |
89 runtime·unlock(&s); | 107 runtime·unlock(&s); |
90 } | 108 } |
91 | 109 |
92 void | 110 static int32 |
93 net·runtime_waitFD(PollFD *fd, int32 mode) | 111 checkerr(PollFD *fd, int32 mode) |
94 { | 112 { |
95 » epollblock(mode=='r' ? &fd->r : &fd.w); | 113 » if(fd->closing) |
| 114 » » return 1; |
| 115 » if((mode == 'r' && fd->rd < 0) || (mode == 'w' && fd->wd < 0)) |
| 116 » » return 2; |
| 117 » return 0; |
| 118 } |
| 119 |
| 120 void |
| 121 net·runtime_resetFD(PollFD *fd, uintptr mode) |
| 122 { |
| 123 » if(mode == 'r') |
| 124 » » fd->rg = nil; |
| 125 » else if(mode == 'w') |
| 126 » » fd->wg = nil; |
| 127 } |
| 128 |
| 129 void |
| 130 net·runtime_waitFD(PollFD *fd, uintptr mode, int32 err) |
| 131 { |
| 132 » LOG("waitFD: fd=%p mode=%d\n", fd, (int32)mode); |
| 133 » runtime·lock(fd); |
| 134 » err = checkerr(fd, mode); |
| 135 » if(err) { |
| 136 » » runtime·unlock(fd); |
| 137 » » FLUSH(&err); |
| 138 » » LOG("waitFD: fd=%p, mode=%d err=%d\n", fd, (int32)mode, err); |
| 139 » » return; |
| 140 » } |
| 141 » epollblock(fd, mode=='r' ? &fd->rg : &fd->wg); |
| 142 » err = checkerr(fd, mode); |
| 143 » runtime·unlock(fd); |
| 144 » FLUSH(&err); |
| 145 » LOG("waitFD: fd=%p, mode=%d err=%d\n", fd, (int32)mode, err); |
| 146 } |
| 147 |
| 148 void |
| 149 net·runtime_setDeadlineFD(PollFD *fd, int64 d, int32 mode) |
| 150 { |
| 151 » LOG("setDeadlineFD: fd=%p d=%D mode=%d\n", fd, d, mode); |
| 152 » runtime·lock(fd); |
| 153 » if(fd->closing) { |
| 154 » » runtime·unlock(fd); |
| 155 » » return; |
| 156 » } |
| 157 » fd->seq++; |
| 158 » if(fd->rt.f) { |
| 159 » » runtime·deltimer(&fd->rt); |
| 160 » » fd->rt.f = nil; |
| 161 » } |
| 162 » if(fd->wt.f) { |
| 163 » » runtime·deltimer(&fd->wt); |
| 164 » » fd->wt.f = nil; |
| 165 » } |
| 166 » if(mode == 'r'+'w') |
| 167 » » fd->rd = fd->wd = d; |
| 168 » else if(mode == 'r') |
| 169 » » fd->rd = d; |
| 170 » else if(mode == 'w') |
| 171 » » fd->wd = d; |
| 172 » else |
| 173 » » runtime·throw("epoll: unknown mode"); |
| 174 » if(fd->rd > 0 && fd->rd == fd->wd) { |
| 175 » » fd->rt.f = deadline; |
| 176 » » fd->rt.when = fd->rd; |
| 177 » » fd->rt.arg.type = (Type*)fd->seq; |
| 178 » » fd->rt.arg.data = fd; |
| 179 » » runtime·addtimer(&fd->rt); |
| 180 » } else { |
| 181 » » if(fd->rd > 0) { |
| 182 » » » fd->rt.f = readDeadline; |
| 183 » » » fd->rt.when = fd->rd; |
| 184 » » » fd->rt.arg.type = (Type*)fd->seq; |
| 185 » » » fd->rt.arg.data = fd; |
| 186 » » » runtime·addtimer(&fd->rt); |
| 187 » » } |
| 188 » » if(fd->wd > 0) { |
| 189 » » » fd->wt.f = writeDeadline; |
| 190 » » » fd->wt.when = fd->wd; |
| 191 » » » fd->wt.arg.type = (Type*)fd->seq; |
| 192 » » » fd->wt.arg.data = fd; |
| 193 » » » runtime·addtimer(&fd->wt); |
| 194 » » } |
| 195 » } |
| 196 » runtime·unlock(fd); |
96 } | 197 } |
97 | 198 |
98 void | 199 void |
99 net·runtime_unblockFD(PollFD *fd) | 200 net·runtime_unblockFD(PollFD *fd) |
100 { | 201 { |
101 » epollunblock(&fd->r); | 202 » G *rg, *wg; |
102 » epollunblock(&fd->w); | 203 |
103 } | 204 » runtime·lock(fd); |
104 | 205 » fd->closing = true; |
105 static void | 206 » fd->seq++; |
106 epollloop(void) | 207 » rg = epollunblock(fd, &fd->rg); |
| 208 » wg = epollunblock(fd, &fd->wg); |
| 209 » if(fd->rt.f) { |
| 210 » » runtime·deltimer(&fd->rt); |
| 211 » » fd->rt.f = nil; |
| 212 » } |
| 213 » if(fd->wt.f) { |
| 214 » » runtime·deltimer(&fd->wt); |
| 215 » » fd->wt.f = nil; |
| 216 » } |
| 217 » runtime·unlock(fd); |
| 218 » if(rg) |
| 219 » » runtime·ready(rg); |
| 220 » if(wg) |
| 221 » » runtime·ready(wg); |
| 222 } |
| 223 |
| 224 static G* |
| 225 netwait(uint32 ms, uint32 cnt) |
107 { | 226 { |
108 EpollEvent ev, events[128]; | 227 EpollEvent ev, events[128]; |
109 PollFD *fd; | 228 PollFD *fd; |
110 » int32 epfd, n, i; | 229 » G *gp, *rg, *wg; |
111 »······· | 230 » int32 n, i; |
112 » epfd = s.fd; | 231 |
| 232 » LOG("epoll wait\n"); |
| 233 » /* |
| 234 » if(s.fd == 0) { |
| 235 » » runtime·usleep(ms*1000); |
| 236 » » return nil; |
| 237 » } |
| 238 » */ |
| 239 » if(cnt > nelem(events)) |
| 240 » » cnt = nelem(events); |
| 241 » runtime·entersyscall(); |
| 242 » n = runtime·epollwait(s.fd, events, cnt, ms); |
| 243 » runtime·exitsyscall(); |
| 244 » //TODO(dvyukov): handle errors other than EAGAIN and EINTR |
| 245 » LOG("epoll wait got %d\n", n); |
| 246 » if(n <= 0) |
| 247 » » return nil; |
| 248 » gp = nil; |
| 249 » for(i = 0; i < n; i++) { |
| 250 » » ev = events[i]; |
| 251 » » if(ev.events == 0) |
| 252 » » » continue; |
| 253 » » fd = (PollFD*)ev.ctx; |
| 254 » » LOG("epoll event on %p %d\n", fd, ev.events); |
| 255 » » rg = wg = nil; |
| 256 » » runtime·lock(fd); |
| 257 » » if(ev.events & (EPOLLIN|EPOLLRDHUP|EPOLLHUP|EPOLLERR)) |
| 258 » » » rg = epollunblock(fd, &fd->rg); |
| 259 » » if(ev.events & (EPOLLOUT|EPOLLHUP|EPOLLERR)) |
| 260 » » » wg = epollunblock(fd, &fd->wg); |
| 261 » » runtime·unlock(fd); |
| 262 » » if(rg) { |
| 263 » » » rg->schedlink = gp; |
| 264 » » » gp = rg; |
| 265 » » } |
| 266 » » if(wg) { |
| 267 » » » wg->schedlink = gp; |
| 268 » » » gp = wg; |
| 269 » » } |
| 270 » } |
| 271 » return gp; |
| 272 } |
| 273 |
| 274 static void |
| 275 pollloop(void) |
| 276 { |
| 277 » G *gp, *gp1; |
| 278 |
113 for(;;) { | 279 for(;;) { |
114 » » n = runtime·epollwait(epfd, events, nelem(events), -1); | 280 » » gp = netwait(10, 10); |
115 » » //TODO(dvyukov): handle errors other than EAGAIN and EINTR | 281 » » while(gp) { |
116 » » if(n <= 0) | 282 » » » gp1 = gp; |
117 » » » continue; | 283 » » » gp = gp->schedlink; |
118 » » for(i = 0; i < n; i++) { | 284 » » » runtime·ready(gp1); |
119 » » » ev = events[i]; | 285 » » } |
120 » » » fd = (PollFD*)ev.ctx; | 286 » } |
121 » » » if(ev.events & (EPOLLIN|EPOLLRDHUP|EPOLLHUP|EPOLLERR)) | 287 } |
122 » » » » epollunblock(&fd->r); | 288 |
123 » » » if(ev.events & (EPOLLOUT|EPOLLHUP|EPOLLERR)) | 289 static void |
124 » » » » epollunblock(&fd->w); | 290 epollblock(PollFD *fd, G **gp) |
125 » » } | 291 { |
126 » } | 292 » LOG("epoll block %p on %p\n", g, gp); |
127 } | 293 » if(*gp == READY) { |
128 | 294 » » *gp = nil; |
129 static void | 295 » » return; |
130 epollblock(G **gp) | 296 » } |
| 297 » if(*gp != nil) |
| 298 » » runtime·throw("epoll: double wait"); |
| 299 » *gp = g; |
| 300 » runtime·park(runtime·unlock, &fd->Lock, "IO wait"); |
| 301 » runtime·lock(fd); |
| 302 } |
| 303 |
| 304 static G* |
| 305 epollunblock(PollFD *fd, G **gp) |
131 { | 306 { |
132 G *old; | 307 G *old; |
133 | 308 |
134 » g->status = Gwaiting; | 309 » USED(fd); |
135 » g->waitreason = "IO wait"; | 310 » LOG("epoll unblock on %p\n", gp); |
136 » for(;;) { | 311 » if(*gp == READY) |
137 » » old = *gp; | 312 » » return nil; |
138 » » if(old == READY) { | 313 » if(*gp == nil) { |
139 » » » old = nil; | 314 » » *gp = READY; |
140 » » » g->status = Grunning; | 315 » » return nil; |
141 » » » g->waitreason = nil;»··· | 316 » } |
142 » » » return; | 317 » old = *gp; |
143 » » } | 318 » *gp = nil; |
144 » » if(old != nil) | 319 » return old; |
145 » » » runtime·throw("epoll: double wait"); | 320 } |
146 » » if(runtime·casp(gp, nil, g)) { | 321 |
147 » » » runtime·gosched(); | 322 static void |
148 » » » return; | 323 deadlineimpl(int64 now, Eface arg, bool read, bool write) |
149 » » } | 324 { |
150 » } | 325 » PollFD *fd; |
151 } | 326 » uint32 seq; |
152 | 327 » G *rg, *wg; |
153 static void | 328 |
154 epollunblock(G **gp) | 329 » USED(now); |
155 { | 330 » fd = (PollFD*)arg.data; |
156 » G *old; | 331 » seq = (uint32)(uintptr)arg.type; |
157 | 332 » rg = wg = nil; |
158 » for(;;) { | 333 » LOG("deadline: fd=%p fd->seq=%d seq=%d read=%d write=%d\n", fd, fd->seq,
seq, read, write); |
159 » » old = *gp; | 334 » runtime·lock(fd); |
160 » » if(old == READY) | 335 » if(seq != fd->seq) { |
161 » » » return; | 336 » » runtime·unlock(fd); |
162 » » if(old == nil && runtime·casp(gp, nil, READY)) | 337 » » return; |
163 » » » return; | 338 » } |
164 » » if(runtime·casp(gp, old, nil)) { | 339 » if(read) { |
165 » » » runtime·ready(old); | 340 » » if(fd->rd <= 0 || fd->rt.f == nil) |
166 » » » return; | 341 » » » runtime·throw("epoll: inconsistent read deadline"); |
167 » » } | 342 » » fd->rd = -1; |
168 » } | 343 » » fd->rt.f = nil; |
169 } | 344 » » rg = epollunblock(fd, &fd->rg); |
| 345 » } |
| 346 » if(write) { |
| 347 » » if(fd->wd <= 0) |
| 348 » » » runtime·throw("epoll: inconsistent write deadline"); |
| 349 » » fd->wd = -1; |
| 350 » » fd->wt.f = nil; |
| 351 » » wg = epollunblock(fd, &fd->wg); |
| 352 » } |
| 353 » runtime·unlock(fd); |
| 354 » if(rg) |
| 355 » » runtime·ready(rg); |
| 356 » if(wg) |
| 357 » » runtime·ready(wg); |
| 358 } |
| 359 |
| 360 static void |
| 361 deadline(int64 now, Eface arg) |
| 362 { |
| 363 » deadlineimpl(now, arg, true, true); |
| 364 } |
| 365 |
| 366 static void |
| 367 readDeadline(int64 now, Eface arg) |
| 368 { |
| 369 » deadlineimpl(now, arg, true, false); |
| 370 } |
| 371 |
| 372 static void |
| 373 writeDeadline(int64 now, Eface arg) |
| 374 { |
| 375 » deadlineimpl(now, arg, false, true); |
| 376 } |
| 377 |
| 378 static PollFD* |
| 379 allocFD(void) |
| 380 { |
| 381 » PollFD *fd; |
| 382 » uint32 i, n; |
| 383 |
| 384 » runtime·lock(&s); |
| 385 » if(s.cache == nil) { |
| 386 » » n = PageSize/sizeof(*fd); |
| 387 » » if(n == 0) |
| 388 » » » n = 1; |
| 389 » » fd = (PollFD*)runtime·SysAlloc(n*sizeof(*fd)); |
| 390 » » for(i=0; i<n; i++) { |
| 391 » » » fd[i].link = s.cache; |
| 392 » » » s.cache = &fd[i]; |
| 393 » » }· |
| 394 » } |
| 395 » fd = s.cache; |
| 396 » s.cache = fd->link; |
| 397 » runtime·unlock(&s); |
| 398 » return fd; |
| 399 } |
LEFT | RIGHT |