Source file
src/runtime/netpoll.go
1
2
3
4
5
6
7 package runtime
8
9 import (
10 "runtime/internal/atomic"
11 "unsafe"
12 )
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41 const (
42 pollNoError = 0
43 pollErrClosing = 1
44 pollErrTimeout = 2
45 pollErrNotPollable = 3
46 )
47
48
49
50
51
52
53
54
55
56
57
58
59
60 const (
61 pdReady uintptr = 1
62 pdWait uintptr = 2
63 )
64
65 const pollBlockSize = 4 * 1024
66
67
68
69
70
71
72 type pollDesc struct {
73 link *pollDesc
74 fd uintptr
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91 atomicInfo atomic.Uint32
92
93
94
95 rg atomic.Uintptr
96 wg atomic.Uintptr
97
98 lock mutex
99 closing bool
100 user uint32
101 rseq uintptr
102 rt timer
103 rd int64
104 wseq uintptr
105 wt timer
106 wd int64
107 self *pollDesc
108 }
109
110
111
112
113
114 type pollInfo uint32
115
116 const (
117 pollClosing = 1 << iota
118 pollEventErr
119 pollExpiredReadDeadline
120 pollExpiredWriteDeadline
121 )
122
123 func (i pollInfo) closing() bool { return i&pollClosing != 0 }
124 func (i pollInfo) eventErr() bool { return i&pollEventErr != 0 }
125 func (i pollInfo) expiredReadDeadline() bool { return i&pollExpiredReadDeadline != 0 }
126 func (i pollInfo) expiredWriteDeadline() bool { return i&pollExpiredWriteDeadline != 0 }
127
128
129 func (pd *pollDesc) info() pollInfo {
130 return pollInfo(pd.atomicInfo.Load())
131 }
132
133
134
135
136
137
138
139
140 func (pd *pollDesc) publishInfo() {
141 var info uint32
142 if pd.closing {
143 info |= pollClosing
144 }
145 if pd.rd < 0 {
146 info |= pollExpiredReadDeadline
147 }
148 if pd.wd < 0 {
149 info |= pollExpiredWriteDeadline
150 }
151
152
153 x := pd.atomicInfo.Load()
154 for !pd.atomicInfo.CompareAndSwap(x, (x&pollEventErr)|info) {
155 x = pd.atomicInfo.Load()
156 }
157 }
158
159
160 func (pd *pollDesc) setEventErr(b bool) {
161 x := pd.atomicInfo.Load()
162 for (x&pollEventErr != 0) != b && !pd.atomicInfo.CompareAndSwap(x, x^pollEventErr) {
163 x = pd.atomicInfo.Load()
164 }
165 }
166
167 type pollCache struct {
168 lock mutex
169 first *pollDesc
170
171
172
173
174
175 }
176
177 var (
178 netpollInitLock mutex
179 netpollInited uint32
180
181 pollcache pollCache
182 netpollWaiters uint32
183 )
184
185
186 func poll_runtime_pollServerInit() {
187 netpollGenericInit()
188 }
189
190 func netpollGenericInit() {
191 if atomic.Load(&netpollInited) == 0 {
192 lockInit(&netpollInitLock, lockRankNetpollInit)
193 lock(&netpollInitLock)
194 if netpollInited == 0 {
195 netpollinit()
196 atomic.Store(&netpollInited, 1)
197 }
198 unlock(&netpollInitLock)
199 }
200 }
201
202 func netpollinited() bool {
203 return atomic.Load(&netpollInited) != 0
204 }
205
206
207
208
209
210 func poll_runtime_isPollServerDescriptor(fd uintptr) bool {
211 return netpollIsPollDescriptor(fd)
212 }
213
214
215 func poll_runtime_pollOpen(fd uintptr) (*pollDesc, int) {
216 pd := pollcache.alloc()
217 lock(&pd.lock)
218 wg := pd.wg.Load()
219 if wg != 0 && wg != pdReady {
220 throw("runtime: blocked write on free polldesc")
221 }
222 rg := pd.rg.Load()
223 if rg != 0 && rg != pdReady {
224 throw("runtime: blocked read on free polldesc")
225 }
226 pd.fd = fd
227 pd.closing = false
228 pd.setEventErr(false)
229 pd.rseq++
230 pd.rg.Store(0)
231 pd.rd = 0
232 pd.wseq++
233 pd.wg.Store(0)
234 pd.wd = 0
235 pd.self = pd
236 pd.publishInfo()
237 unlock(&pd.lock)
238
239 errno := netpollopen(fd, pd)
240 if errno != 0 {
241 pollcache.free(pd)
242 return nil, int(errno)
243 }
244 return pd, 0
245 }
246
247
248 func poll_runtime_pollClose(pd *pollDesc) {
249 if !pd.closing {
250 throw("runtime: close polldesc w/o unblock")
251 }
252 wg := pd.wg.Load()
253 if wg != 0 && wg != pdReady {
254 throw("runtime: blocked write on closing polldesc")
255 }
256 rg := pd.rg.Load()
257 if rg != 0 && rg != pdReady {
258 throw("runtime: blocked read on closing polldesc")
259 }
260 netpollclose(pd.fd)
261 pollcache.free(pd)
262 }
263
264 func (c *pollCache) free(pd *pollDesc) {
265 lock(&c.lock)
266 pd.link = c.first
267 c.first = pd
268 unlock(&c.lock)
269 }
270
271
272
273
274
275 func poll_runtime_pollReset(pd *pollDesc, mode int) int {
276 errcode := netpollcheckerr(pd, int32(mode))
277 if errcode != pollNoError {
278 return errcode
279 }
280 if mode == 'r' {
281 pd.rg.Store(0)
282 } else if mode == 'w' {
283 pd.wg.Store(0)
284 }
285 return pollNoError
286 }
287
288
289
290
291
292
293 func poll_runtime_pollWait(pd *pollDesc, mode int) int {
294 errcode := netpollcheckerr(pd, int32(mode))
295 if errcode != pollNoError {
296 return errcode
297 }
298
299 if GOOS == "solaris" || GOOS == "illumos" || GOOS == "aix" {
300 netpollarm(pd, mode)
301 }
302 for !netpollblock(pd, int32(mode), false) {
303 errcode = netpollcheckerr(pd, int32(mode))
304 if errcode != pollNoError {
305 return errcode
306 }
307
308
309
310 }
311 return pollNoError
312 }
313
314
315 func poll_runtime_pollWaitCanceled(pd *pollDesc, mode int) {
316
317
318 for !netpollblock(pd, int32(mode), true) {
319 }
320 }
321
322
323 func poll_runtime_pollSetDeadline(pd *pollDesc, d int64, mode int) {
324 lock(&pd.lock)
325 if pd.closing {
326 unlock(&pd.lock)
327 return
328 }
329 rd0, wd0 := pd.rd, pd.wd
330 combo0 := rd0 > 0 && rd0 == wd0
331 if d > 0 {
332 d += nanotime()
333 if d <= 0 {
334
335
336 d = 1<<63 - 1
337 }
338 }
339 if mode == 'r' || mode == 'r'+'w' {
340 pd.rd = d
341 }
342 if mode == 'w' || mode == 'r'+'w' {
343 pd.wd = d
344 }
345 pd.publishInfo()
346 combo := pd.rd > 0 && pd.rd == pd.wd
347 rtf := netpollReadDeadline
348 if combo {
349 rtf = netpollDeadline
350 }
351 if pd.rt.f == nil {
352 if pd.rd > 0 {
353 pd.rt.f = rtf
354
355
356
357 pd.rt.arg = pd.makeArg()
358 pd.rt.seq = pd.rseq
359 resettimer(&pd.rt, pd.rd)
360 }
361 } else if pd.rd != rd0 || combo != combo0 {
362 pd.rseq++
363 if pd.rd > 0 {
364 modtimer(&pd.rt, pd.rd, 0, rtf, pd.makeArg(), pd.rseq)
365 } else {
366 deltimer(&pd.rt)
367 pd.rt.f = nil
368 }
369 }
370 if pd.wt.f == nil {
371 if pd.wd > 0 && !combo {
372 pd.wt.f = netpollWriteDeadline
373 pd.wt.arg = pd.makeArg()
374 pd.wt.seq = pd.wseq
375 resettimer(&pd.wt, pd.wd)
376 }
377 } else if pd.wd != wd0 || combo != combo0 {
378 pd.wseq++
379 if pd.wd > 0 && !combo {
380 modtimer(&pd.wt, pd.wd, 0, netpollWriteDeadline, pd.makeArg(), pd.wseq)
381 } else {
382 deltimer(&pd.wt)
383 pd.wt.f = nil
384 }
385 }
386
387
388 var rg, wg *g
389 if pd.rd < 0 {
390 rg = netpollunblock(pd, 'r', false)
391 }
392 if pd.wd < 0 {
393 wg = netpollunblock(pd, 'w', false)
394 }
395 unlock(&pd.lock)
396 if rg != nil {
397 netpollgoready(rg, 3)
398 }
399 if wg != nil {
400 netpollgoready(wg, 3)
401 }
402 }
403
404
405 func poll_runtime_pollUnblock(pd *pollDesc) {
406 lock(&pd.lock)
407 if pd.closing {
408 throw("runtime: unblock on closing polldesc")
409 }
410 pd.closing = true
411 pd.rseq++
412 pd.wseq++
413 var rg, wg *g
414 pd.publishInfo()
415 rg = netpollunblock(pd, 'r', false)
416 wg = netpollunblock(pd, 'w', false)
417 if pd.rt.f != nil {
418 deltimer(&pd.rt)
419 pd.rt.f = nil
420 }
421 if pd.wt.f != nil {
422 deltimer(&pd.wt)
423 pd.wt.f = nil
424 }
425 unlock(&pd.lock)
426 if rg != nil {
427 netpollgoready(rg, 3)
428 }
429 if wg != nil {
430 netpollgoready(wg, 3)
431 }
432 }
433
434
435
436
437
438
439
440
441
442 func netpollready(toRun *gList, pd *pollDesc, mode int32) {
443 var rg, wg *g
444 if mode == 'r' || mode == 'r'+'w' {
445 rg = netpollunblock(pd, 'r', true)
446 }
447 if mode == 'w' || mode == 'r'+'w' {
448 wg = netpollunblock(pd, 'w', true)
449 }
450 if rg != nil {
451 toRun.push(rg)
452 }
453 if wg != nil {
454 toRun.push(wg)
455 }
456 }
457
458 func netpollcheckerr(pd *pollDesc, mode int32) int {
459 info := pd.info()
460 if info.closing() {
461 return pollErrClosing
462 }
463 if (mode == 'r' && info.expiredReadDeadline()) || (mode == 'w' && info.expiredWriteDeadline()) {
464 return pollErrTimeout
465 }
466
467
468
469 if mode == 'r' && info.eventErr() {
470 return pollErrNotPollable
471 }
472 return pollNoError
473 }
474
475 func netpollblockcommit(gp *g, gpp unsafe.Pointer) bool {
476 r := atomic.Casuintptr((*uintptr)(gpp), pdWait, uintptr(unsafe.Pointer(gp)))
477 if r {
478
479
480
481 atomic.Xadd(&netpollWaiters, 1)
482 }
483 return r
484 }
485
486 func netpollgoready(gp *g, traceskip int) {
487 atomic.Xadd(&netpollWaiters, -1)
488 goready(gp, traceskip+1)
489 }
490
491
492
493
494
495 func netpollblock(pd *pollDesc, mode int32, waitio bool) bool {
496 gpp := &pd.rg
497 if mode == 'w' {
498 gpp = &pd.wg
499 }
500
501
502 for {
503
504 if gpp.CompareAndSwap(pdReady, 0) {
505 return true
506 }
507 if gpp.CompareAndSwap(0, pdWait) {
508 break
509 }
510
511
512
513 if v := gpp.Load(); v != pdReady && v != 0 {
514 throw("runtime: double wait")
515 }
516 }
517
518
519
520
521 if waitio || netpollcheckerr(pd, mode) == pollNoError {
522 gopark(netpollblockcommit, unsafe.Pointer(gpp), waitReasonIOWait, traceEvGoBlockNet, 5)
523 }
524
525 old := gpp.Swap(0)
526 if old > pdWait {
527 throw("runtime: corrupted polldesc")
528 }
529 return old == pdReady
530 }
531
532 func netpollunblock(pd *pollDesc, mode int32, ioready bool) *g {
533 gpp := &pd.rg
534 if mode == 'w' {
535 gpp = &pd.wg
536 }
537
538 for {
539 old := gpp.Load()
540 if old == pdReady {
541 return nil
542 }
543 if old == 0 && !ioready {
544
545
546 return nil
547 }
548 var new uintptr
549 if ioready {
550 new = pdReady
551 }
552 if gpp.CompareAndSwap(old, new) {
553 if old == pdWait {
554 old = 0
555 }
556 return (*g)(unsafe.Pointer(old))
557 }
558 }
559 }
560
561 func netpolldeadlineimpl(pd *pollDesc, seq uintptr, read, write bool) {
562 lock(&pd.lock)
563
564
565 currentSeq := pd.rseq
566 if !read {
567 currentSeq = pd.wseq
568 }
569 if seq != currentSeq {
570
571 unlock(&pd.lock)
572 return
573 }
574 var rg *g
575 if read {
576 if pd.rd <= 0 || pd.rt.f == nil {
577 throw("runtime: inconsistent read deadline")
578 }
579 pd.rd = -1
580 pd.publishInfo()
581 rg = netpollunblock(pd, 'r', false)
582 }
583 var wg *g
584 if write {
585 if pd.wd <= 0 || pd.wt.f == nil && !read {
586 throw("runtime: inconsistent write deadline")
587 }
588 pd.wd = -1
589 pd.publishInfo()
590 wg = netpollunblock(pd, 'w', false)
591 }
592 unlock(&pd.lock)
593 if rg != nil {
594 netpollgoready(rg, 0)
595 }
596 if wg != nil {
597 netpollgoready(wg, 0)
598 }
599 }
600
601 func netpollDeadline(arg any, seq uintptr) {
602 netpolldeadlineimpl(arg.(*pollDesc), seq, true, true)
603 }
604
605 func netpollReadDeadline(arg any, seq uintptr) {
606 netpolldeadlineimpl(arg.(*pollDesc), seq, true, false)
607 }
608
609 func netpollWriteDeadline(arg any, seq uintptr) {
610 netpolldeadlineimpl(arg.(*pollDesc), seq, false, true)
611 }
612
613 func (c *pollCache) alloc() *pollDesc {
614 lock(&c.lock)
615 if c.first == nil {
616 const pdSize = unsafe.Sizeof(pollDesc{})
617 n := pollBlockSize / pdSize
618 if n == 0 {
619 n = 1
620 }
621
622
623 mem := persistentalloc(n*pdSize, 0, &memstats.other_sys)
624 for i := uintptr(0); i < n; i++ {
625 pd := (*pollDesc)(add(mem, i*pdSize))
626 pd.link = c.first
627 c.first = pd
628 }
629 }
630 pd := c.first
631 c.first = pd.link
632 lockInit(&pd.lock, lockRankPollDesc)
633 unlock(&c.lock)
634 return pd
635 }
636
637
638
639
640
641
642 func (pd *pollDesc) makeArg() (i any) {
643 x := (*eface)(unsafe.Pointer(&i))
644 x._type = pdType
645 x.data = unsafe.Pointer(&pd.self)
646 return
647 }
648
649 var (
650 pdEface any = (*pollDesc)(nil)
651 pdType *_type = efaceOf(&pdEface)._type
652 )
653
View as plain text