Source file src/runtime/runtime2.go
1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "internal/chacha8rand" 10 "internal/goarch" 11 "internal/runtime/atomic" 12 "internal/runtime/sys" 13 "unsafe" 14 ) 15 16 // defined constants 17 const ( 18 // G status 19 // 20 // Beyond indicating the general state of a G, the G status 21 // acts like a lock on the goroutine's stack (and hence its 22 // ability to execute user code). 23 // 24 // If you add to this list, add to the list 25 // of "okay during garbage collection" status 26 // in mgcmark.go too. 27 // 28 // TODO(austin): The _Gscan bit could be much lighter-weight. 29 // For example, we could choose not to run _Gscanrunnable 30 // goroutines found in the run queue, rather than CAS-looping 31 // until they become _Grunnable. And transitions like 32 // _Gscanwaiting -> _Gscanrunnable are actually okay because 33 // they don't affect stack ownership. 34 35 // _Gidle means this goroutine was just allocated and has not 36 // yet been initialized. 37 _Gidle = iota // 0 38 39 // _Grunnable means this goroutine is on a run queue. It is 40 // not currently executing user code. The stack is not owned. 41 _Grunnable // 1 42 43 // _Grunning means this goroutine may execute user code. The 44 // stack is owned by this goroutine. It is not on a run queue. 45 // It is assigned an M and a P (g.m and g.m.p are valid). 46 _Grunning // 2 47 48 // _Gsyscall means this goroutine is executing a system call. 49 // It is not executing user code. The stack is owned by this 50 // goroutine. It is not on a run queue. It is assigned an M. 51 _Gsyscall // 3 52 53 // _Gwaiting means this goroutine is blocked in the runtime. 54 // It is not executing user code. It is not on a run queue, 55 // but should be recorded somewhere (e.g., a channel wait 56 // queue) so it can be ready()d when necessary. The stack is 57 // not owned *except* that a channel operation may read or 58 // write parts of the stack under the appropriate channel 59 // lock. Otherwise, it is not safe to access the stack after a 60 // goroutine enters _Gwaiting (e.g., it may get moved). 61 _Gwaiting // 4 62 63 // _Gmoribund_unused is currently unused, but hardcoded in gdb 64 // scripts. 65 _Gmoribund_unused // 5 66 67 // _Gdead means this goroutine is currently unused. It may be 68 // just exited, on a free list, or just being initialized. It 69 // is not executing user code. It may or may not have a stack 70 // allocated. The G and its stack (if any) are owned by the M 71 // that is exiting the G or that obtained the G from the free 72 // list. 73 _Gdead // 6 74 75 // _Genqueue_unused is currently unused. 76 _Genqueue_unused // 7 77 78 // _Gcopystack means this goroutine's stack is being moved. It 79 // is not executing user code and is not on a run queue. The 80 // stack is owned by the goroutine that put it in _Gcopystack. 81 _Gcopystack // 8 82 83 // _Gpreempted means this goroutine stopped itself for a 84 // suspendG preemption. It is like _Gwaiting, but nothing is 85 // yet responsible for ready()ing it. Some suspendG must CAS 86 // the status to _Gwaiting to take responsibility for 87 // ready()ing this G. 88 _Gpreempted // 9 89 90 // _Gscan combined with one of the above states other than 91 // _Grunning indicates that GC is scanning the stack. The 92 // goroutine is not executing user code and the stack is owned 93 // by the goroutine that set the _Gscan bit. 94 // 95 // _Gscanrunning is different: it is used to briefly block 96 // state transitions while GC signals the G to scan its own 97 // stack. This is otherwise like _Grunning. 98 // 99 // atomicstatus&~Gscan gives the state the goroutine will 100 // return to when the scan completes. 101 _Gscan = 0x1000 102 _Gscanrunnable = _Gscan + _Grunnable // 0x1001 103 _Gscanrunning = _Gscan + _Grunning // 0x1002 104 _Gscansyscall = _Gscan + _Gsyscall // 0x1003 105 _Gscanwaiting = _Gscan + _Gwaiting // 0x1004 106 _Gscanpreempted = _Gscan + _Gpreempted // 0x1009 107 ) 108 109 const ( 110 // P status 111 112 // _Pidle means a P is not being used to run user code or the 113 // scheduler. Typically, it's on the idle P list and available 114 // to the scheduler, but it may just be transitioning between 115 // other states. 116 // 117 // The P is owned by the idle list or by whatever is 118 // transitioning its state. Its run queue is empty. 119 _Pidle = iota 120 121 // _Prunning means a P is owned by an M and is being used to 122 // run user code or the scheduler. Only the M that owns this P 123 // is allowed to change the P's status from _Prunning. The M 124 // may transition the P to _Pidle (if it has no more work to 125 // do), _Psyscall (when entering a syscall), or _Pgcstop (to 126 // halt for the GC). The M may also hand ownership of the P 127 // off directly to another M (e.g., to schedule a locked G). 128 _Prunning 129 130 // _Psyscall means a P is not running user code. It has 131 // affinity to an M in a syscall but is not owned by it and 132 // may be stolen by another M. This is similar to _Pidle but 133 // uses lightweight transitions and maintains M affinity. 134 // 135 // Leaving _Psyscall must be done with a CAS, either to steal 136 // or retake the P. Note that there's an ABA hazard: even if 137 // an M successfully CASes its original P back to _Prunning 138 // after a syscall, it must understand the P may have been 139 // used by another M in the interim. 140 _Psyscall 141 142 // _Pgcstop means a P is halted for STW and owned by the M 143 // that stopped the world. The M that stopped the world 144 // continues to use its P, even in _Pgcstop. Transitioning 145 // from _Prunning to _Pgcstop causes an M to release its P and 146 // park. 147 // 148 // The P retains its run queue and startTheWorld will restart 149 // the scheduler on Ps with non-empty run queues. 150 _Pgcstop 151 152 // _Pdead means a P is no longer used (GOMAXPROCS shrank). We 153 // reuse Ps if GOMAXPROCS increases. A dead P is mostly 154 // stripped of its resources, though a few things remain 155 // (e.g., trace buffers). 156 _Pdead 157 ) 158 159 // Mutual exclusion locks. In the uncontended case, 160 // as fast as spin locks (just a few user-level instructions), 161 // but on the contention path they sleep in the kernel. 162 // A zeroed Mutex is unlocked (no need to initialize each lock). 163 // Initialization is helpful for static lock ranking, but not required. 164 type mutex struct { 165 // Empty struct if lock ranking is disabled, otherwise includes the lock rank 166 lockRankStruct 167 // Futex-based impl treats it as uint32 key, 168 // while sema-based impl as M* waitm. 169 // Used to be a union, but unions break precise GC. 170 key uintptr 171 } 172 173 type funcval struct { 174 fn uintptr 175 // variable-size, fn-specific data here 176 } 177 178 type iface struct { 179 tab *itab 180 data unsafe.Pointer 181 } 182 183 type eface struct { 184 _type *_type 185 data unsafe.Pointer 186 } 187 188 func efaceOf(ep *any) *eface { 189 return (*eface)(unsafe.Pointer(ep)) 190 } 191 192 // The guintptr, muintptr, and puintptr are all used to bypass write barriers. 193 // It is particularly important to avoid write barriers when the current P has 194 // been released, because the GC thinks the world is stopped, and an 195 // unexpected write barrier would not be synchronized with the GC, 196 // which can lead to a half-executed write barrier that has marked the object 197 // but not queued it. If the GC skips the object and completes before the 198 // queuing can occur, it will incorrectly free the object. 199 // 200 // We tried using special assignment functions invoked only when not 201 // holding a running P, but then some updates to a particular memory 202 // word went through write barriers and some did not. This breaks the 203 // write barrier shadow checking mode, and it is also scary: better to have 204 // a word that is completely ignored by the GC than to have one for which 205 // only a few updates are ignored. 206 // 207 // Gs and Ps are always reachable via true pointers in the 208 // allgs and allp lists or (during allocation before they reach those lists) 209 // from stack variables. 210 // 211 // Ms are always reachable via true pointers either from allm or 212 // freem. Unlike Gs and Ps we do free Ms, so it's important that 213 // nothing ever hold an muintptr across a safe point. 214 215 // A guintptr holds a goroutine pointer, but typed as a uintptr 216 // to bypass write barriers. It is used in the Gobuf goroutine state 217 // and in scheduling lists that are manipulated without a P. 218 // 219 // The Gobuf.g goroutine pointer is almost always updated by assembly code. 220 // In one of the few places it is updated by Go code - func save - it must be 221 // treated as a uintptr to avoid a write barrier being emitted at a bad time. 222 // Instead of figuring out how to emit the write barriers missing in the 223 // assembly manipulation, we change the type of the field to uintptr, 224 // so that it does not require write barriers at all. 225 // 226 // Goroutine structs are published in the allg list and never freed. 227 // That will keep the goroutine structs from being collected. 228 // There is never a time that Gobuf.g's contain the only references 229 // to a goroutine: the publishing of the goroutine in allg comes first. 230 // Goroutine pointers are also kept in non-GC-visible places like TLS, 231 // so I can't see them ever moving. If we did want to start moving data 232 // in the GC, we'd need to allocate the goroutine structs from an 233 // alternate arena. Using guintptr doesn't make that problem any worse. 234 // Note that pollDesc.rg, pollDesc.wg also store g in uintptr form, 235 // so they would need to be updated too if g's start moving. 236 type guintptr uintptr 237 238 //go:nosplit 239 func (gp guintptr) ptr() *g { return (*g)(unsafe.Pointer(gp)) } 240 241 //go:nosplit 242 func (gp *guintptr) set(g *g) { *gp = guintptr(unsafe.Pointer(g)) } 243 244 //go:nosplit 245 func (gp *guintptr) cas(old, new guintptr) bool { 246 return atomic.Casuintptr((*uintptr)(unsafe.Pointer(gp)), uintptr(old), uintptr(new)) 247 } 248 249 //go:nosplit 250 func (gp *g) guintptr() guintptr { 251 return guintptr(unsafe.Pointer(gp)) 252 } 253 254 // setGNoWB performs *gp = new without a write barrier. 255 // For times when it's impractical to use a guintptr. 256 // 257 //go:nosplit 258 //go:nowritebarrier 259 func setGNoWB(gp **g, new *g) { 260 (*guintptr)(unsafe.Pointer(gp)).set(new) 261 } 262 263 type puintptr uintptr 264 265 //go:nosplit 266 func (pp puintptr) ptr() *p { return (*p)(unsafe.Pointer(pp)) } 267 268 //go:nosplit 269 func (pp *puintptr) set(p *p) { *pp = puintptr(unsafe.Pointer(p)) } 270 271 // muintptr is a *m that is not tracked by the garbage collector. 272 // 273 // Because we do free Ms, there are some additional constrains on 274 // muintptrs: 275 // 276 // 1. Never hold an muintptr locally across a safe point. 277 // 278 // 2. Any muintptr in the heap must be owned by the M itself so it can 279 // ensure it is not in use when the last true *m is released. 280 type muintptr uintptr 281 282 //go:nosplit 283 func (mp muintptr) ptr() *m { return (*m)(unsafe.Pointer(mp)) } 284 285 //go:nosplit 286 func (mp *muintptr) set(m *m) { *mp = muintptr(unsafe.Pointer(m)) } 287 288 // setMNoWB performs *mp = new without a write barrier. 289 // For times when it's impractical to use an muintptr. 290 // 291 //go:nosplit 292 //go:nowritebarrier 293 func setMNoWB(mp **m, new *m) { 294 (*muintptr)(unsafe.Pointer(mp)).set(new) 295 } 296 297 type gobuf struct { 298 // The offsets of sp, pc, and g are known to (hard-coded in) libmach. 299 // 300 // ctxt is unusual with respect to GC: it may be a 301 // heap-allocated funcval, so GC needs to track it, but it 302 // needs to be set and cleared from assembly, where it's 303 // difficult to have write barriers. However, ctxt is really a 304 // saved, live register, and we only ever exchange it between 305 // the real register and the gobuf. Hence, we treat it as a 306 // root during stack scanning, which means assembly that saves 307 // and restores it doesn't need write barriers. It's still 308 // typed as a pointer so that any other writes from Go get 309 // write barriers. 310 sp uintptr 311 pc uintptr 312 g guintptr 313 ctxt unsafe.Pointer 314 lr uintptr 315 bp uintptr // for framepointer-enabled architectures 316 } 317 318 // sudog (pseudo-g) represents a g in a wait list, such as for sending/receiving 319 // on a channel. 320 // 321 // sudog is necessary because the g ↔ synchronization object relation 322 // is many-to-many. A g can be on many wait lists, so there may be 323 // many sudogs for one g; and many gs may be waiting on the same 324 // synchronization object, so there may be many sudogs for one object. 325 // 326 // sudogs are allocated from a special pool. Use acquireSudog and 327 // releaseSudog to allocate and free them. 328 type sudog struct { 329 // The following fields are protected by the hchan.lock of the 330 // channel this sudog is blocking on. shrinkstack depends on 331 // this for sudogs involved in channel ops. 332 333 g *g 334 335 next *sudog 336 prev *sudog 337 elem unsafe.Pointer // data element (may point to stack) 338 339 // The following fields are never accessed concurrently. 340 // For channels, waitlink is only accessed by g. 341 // For semaphores, all fields (including the ones above) 342 // are only accessed when holding a semaRoot lock. 343 344 acquiretime int64 345 releasetime int64 346 ticket uint32 347 348 // isSelect indicates g is participating in a select, so 349 // g.selectDone must be CAS'd to win the wake-up race. 350 isSelect bool 351 352 // success indicates whether communication over channel c 353 // succeeded. It is true if the goroutine was awoken because a 354 // value was delivered over channel c, and false if awoken 355 // because c was closed. 356 success bool 357 358 // waiters is a count of semaRoot waiting list other than head of list, 359 // clamped to a uint16 to fit in unused space. 360 // Only meaningful at the head of the list. 361 // (If we wanted to be overly clever, we could store a high 16 bits 362 // in the second entry in the list.) 363 waiters uint16 364 365 parent *sudog // semaRoot binary tree 366 waitlink *sudog // g.waiting list or semaRoot 367 waittail *sudog // semaRoot 368 c *hchan // channel 369 } 370 371 type libcall struct { 372 fn uintptr 373 n uintptr // number of parameters 374 args uintptr // parameters 375 r1 uintptr // return values 376 r2 uintptr 377 err uintptr // error number 378 } 379 380 // Stack describes a Go execution stack. 381 // The bounds of the stack are exactly [lo, hi), 382 // with no implicit data structures on either side. 383 type stack struct { 384 lo uintptr 385 hi uintptr 386 } 387 388 // heldLockInfo gives info on a held lock and the rank of that lock 389 type heldLockInfo struct { 390 lockAddr uintptr 391 rank lockRank 392 } 393 394 type g struct { 395 // Stack parameters. 396 // stack describes the actual stack memory: [stack.lo, stack.hi). 397 // stackguard0 is the stack pointer compared in the Go stack growth prologue. 398 // It is stack.lo+StackGuard normally, but can be StackPreempt to trigger a preemption. 399 // stackguard1 is the stack pointer compared in the //go:systemstack stack growth prologue. 400 // It is stack.lo+StackGuard on g0 and gsignal stacks. 401 // It is ~0 on other goroutine stacks, to trigger a call to morestackc (and crash). 402 stack stack // offset known to runtime/cgo 403 stackguard0 uintptr // offset known to liblink 404 stackguard1 uintptr // offset known to liblink 405 406 _panic *_panic // innermost panic - offset known to liblink 407 _defer *_defer // innermost defer 408 m *m // current m; offset known to arm liblink 409 sched gobuf 410 syscallsp uintptr // if status==Gsyscall, syscallsp = sched.sp to use during gc 411 syscallpc uintptr // if status==Gsyscall, syscallpc = sched.pc to use during gc 412 syscallbp uintptr // if status==Gsyscall, syscallbp = sched.bp to use in fpTraceback 413 stktopsp uintptr // expected sp at top of stack, to check in traceback 414 // param is a generic pointer parameter field used to pass 415 // values in particular contexts where other storage for the 416 // parameter would be difficult to find. It is currently used 417 // in four ways: 418 // 1. When a channel operation wakes up a blocked goroutine, it sets param to 419 // point to the sudog of the completed blocking operation. 420 // 2. By gcAssistAlloc1 to signal back to its caller that the goroutine completed 421 // the GC cycle. It is unsafe to do so in any other way, because the goroutine's 422 // stack may have moved in the meantime. 423 // 3. By debugCallWrap to pass parameters to a new goroutine because allocating a 424 // closure in the runtime is forbidden. 425 // 4. When a panic is recovered and control returns to the respective frame, 426 // param may point to a savedOpenDeferState. 427 param unsafe.Pointer 428 atomicstatus atomic.Uint32 429 stackLock uint32 // sigprof/scang lock; TODO: fold in to atomicstatus 430 goid uint64 431 schedlink guintptr 432 waitsince int64 // approx time when the g become blocked 433 waitreason waitReason // if status==Gwaiting 434 435 preempt bool // preemption signal, duplicates stackguard0 = stackpreempt 436 preemptStop bool // transition to _Gpreempted on preemption; otherwise, just deschedule 437 preemptShrink bool // shrink stack at synchronous safe point 438 439 // asyncSafePoint is set if g is stopped at an asynchronous 440 // safe point. This means there are frames on the stack 441 // without precise pointer information. 442 asyncSafePoint bool 443 444 paniconfault bool // panic (instead of crash) on unexpected fault address 445 gcscandone bool // g has scanned stack; protected by _Gscan bit in status 446 throwsplit bool // must not split stack 447 // activeStackChans indicates that there are unlocked channels 448 // pointing into this goroutine's stack. If true, stack 449 // copying needs to acquire channel locks to protect these 450 // areas of the stack. 451 activeStackChans bool 452 // parkingOnChan indicates that the goroutine is about to 453 // park on a chansend or chanrecv. Used to signal an unsafe point 454 // for stack shrinking. 455 parkingOnChan atomic.Bool 456 // inMarkAssist indicates whether the goroutine is in mark assist. 457 // Used by the execution tracer. 458 inMarkAssist bool 459 coroexit bool // argument to coroswitch_m 460 461 raceignore int8 // ignore race detection events 462 nocgocallback bool // whether disable callback from C 463 tracking bool // whether we're tracking this G for sched latency statistics 464 trackingSeq uint8 // used to decide whether to track this G 465 trackingStamp int64 // timestamp of when the G last started being tracked 466 runnableTime int64 // the amount of time spent runnable, cleared when running, only used when tracking 467 lockedm muintptr 468 fipsIndicator uint8 469 syncSafePoint bool // set if g is stopped at a synchronous safe point. 470 runningCleanups atomic.Bool 471 sig uint32 472 writebuf []byte 473 sigcode0 uintptr 474 sigcode1 uintptr 475 sigpc uintptr 476 parentGoid uint64 // goid of goroutine that created this goroutine 477 gopc uintptr // pc of go statement that created this goroutine 478 ancestors *[]ancestorInfo // ancestor information goroutine(s) that created this goroutine (only used if debug.tracebackancestors) 479 startpc uintptr // pc of goroutine function 480 racectx uintptr 481 waiting *sudog // sudog structures this g is waiting on (that have a valid elem ptr); in lock order 482 cgoCtxt []uintptr // cgo traceback context 483 labels unsafe.Pointer // profiler labels 484 timer *timer // cached timer for time.Sleep 485 sleepWhen int64 // when to sleep until 486 selectDone atomic.Uint32 // are we participating in a select and did someone win the race? 487 488 // goroutineProfiled indicates the status of this goroutine's stack for the 489 // current in-progress goroutine profile 490 goroutineProfiled goroutineProfileStateHolder 491 492 coroarg *coro // argument during coroutine transfers 493 bubble *synctestBubble 494 495 // xRegs stores the extended register state if this G has been 496 // asynchronously preempted. 497 xRegs xRegPerG 498 499 // Per-G tracer state. 500 trace gTraceState 501 502 // Per-G GC state 503 504 // gcAssistBytes is this G's GC assist credit in terms of 505 // bytes allocated. If this is positive, then the G has credit 506 // to allocate gcAssistBytes bytes without assisting. If this 507 // is negative, then the G must correct this by performing 508 // scan work. We track this in bytes to make it fast to update 509 // and check for debt in the malloc hot path. The assist ratio 510 // determines how this corresponds to scan work debt. 511 gcAssistBytes int64 512 513 // valgrindStackID is used to track what memory is used for stacks when a program is 514 // built with the "valgrind" build tag, otherwise it is unused. 515 valgrindStackID uintptr 516 } 517 518 // gTrackingPeriod is the number of transitions out of _Grunning between 519 // latency tracking runs. 520 const gTrackingPeriod = 8 521 522 const ( 523 // tlsSlots is the number of pointer-sized slots reserved for TLS on some platforms, 524 // like Windows. 525 tlsSlots = 6 526 tlsSize = tlsSlots * goarch.PtrSize 527 ) 528 529 // Values for m.freeWait. 530 const ( 531 freeMStack = 0 // M done, free stack and reference. 532 freeMRef = 1 // M done, free reference. 533 freeMWait = 2 // M still in use. 534 ) 535 536 type m struct { 537 g0 *g // goroutine with scheduling stack 538 morebuf gobuf // gobuf arg to morestack 539 divmod uint32 // div/mod denominator for arm - known to liblink (cmd/internal/obj/arm/obj5.go) 540 541 // Fields not known to debuggers. 542 procid uint64 // for debuggers, but offset not hard-coded 543 gsignal *g // signal-handling g 544 goSigStack gsignalStack // Go-allocated signal handling stack 545 sigmask sigset // storage for saved signal mask 546 tls [tlsSlots]uintptr // thread-local storage (for x86 extern register) 547 mstartfn func() 548 curg *g // current running goroutine 549 caughtsig guintptr // goroutine running during fatal signal 550 p puintptr // attached p for executing go code (nil if not executing go code) 551 nextp puintptr 552 oldp puintptr // the p that was attached before executing a syscall 553 id int64 554 mallocing int32 555 throwing throwType 556 preemptoff string // if != "", keep curg running on this m 557 locks int32 558 dying int32 559 profilehz int32 560 spinning bool // m is out of work and is actively looking for work 561 blocked bool // m is blocked on a note 562 newSigstack bool // minit on C thread called sigaltstack 563 printlock int8 564 incgo bool // m is executing a cgo call 565 isextra bool // m is an extra m 566 isExtraInC bool // m is an extra m that does not have any Go frames 567 isExtraInSig bool // m is an extra m in a signal handler 568 freeWait atomic.Uint32 // Whether it is safe to free g0 and delete m (one of freeMRef, freeMStack, freeMWait) 569 needextram bool 570 g0StackAccurate bool // whether the g0 stack has accurate bounds 571 traceback uint8 572 allpSnapshot []*p // Snapshot of allp for use after dropping P in findRunnable, nil otherwise. 573 ncgocall uint64 // number of cgo calls in total 574 ncgo int32 // number of cgo calls currently in progress 575 cgoCallersUse atomic.Uint32 // if non-zero, cgoCallers in use temporarily 576 cgoCallers *cgoCallers // cgo traceback if crashing in cgo call 577 park note 578 alllink *m // on allm 579 schedlink muintptr 580 lockedg guintptr 581 createstack [32]uintptr // stack that created this thread, it's used for StackRecord.Stack0, so it must align with it. 582 lockedExt uint32 // tracking for external LockOSThread 583 lockedInt uint32 // tracking for internal lockOSThread 584 mWaitList mWaitList // list of runtime lock waiters 585 586 mLockProfile mLockProfile // fields relating to runtime.lock contention 587 profStack []uintptr // used for memory/block/mutex stack traces 588 589 // wait* are used to carry arguments from gopark into park_m, because 590 // there's no stack to put them on. That is their sole purpose. 591 waitunlockf func(*g, unsafe.Pointer) bool 592 waitlock unsafe.Pointer 593 waitTraceSkip int 594 waitTraceBlockReason traceBlockReason 595 596 syscalltick uint32 597 freelink *m // on sched.freem 598 trace mTraceState 599 600 // These are here to avoid using the G stack so the stack can move during the call. 601 libcallpc uintptr // for cpu profiler 602 libcallsp uintptr 603 libcallg guintptr 604 winsyscall winlibcall // stores syscall parameters on windows 605 606 vdsoSP uintptr // SP for traceback while in VDSO call (0 if not in call) 607 vdsoPC uintptr // PC for traceback while in VDSO call 608 609 // preemptGen counts the number of completed preemption 610 // signals. This is used to detect when a preemption is 611 // requested, but fails. 612 preemptGen atomic.Uint32 613 614 // Whether this is a pending preemption signal on this M. 615 signalPending atomic.Uint32 616 617 // pcvalue lookup cache 618 pcvalueCache pcvalueCache 619 620 dlogPerM 621 622 mOS 623 624 chacha8 chacha8rand.State 625 cheaprand uint64 626 627 // Up to 10 locks held by this m, maintained by the lock ranking code. 628 locksHeldLen int 629 locksHeld [10]heldLockInfo 630 } 631 632 const mRedZoneSize = (16 << 3) * asanenabledBit // redZoneSize(2048) 633 634 type mPadded struct { 635 m 636 637 // Size the runtime.m structure so it fits in the 2048-byte size class, and 638 // not in the next-smallest (1792-byte) size class. That leaves the 11 low 639 // bits of muintptr values available for flags, as required by 640 // lock_spinbit.go. 641 _ [(1 - goarch.IsWasm) * (2048 - mallocHeaderSize - mRedZoneSize - unsafe.Sizeof(m{}))]byte 642 } 643 644 type p struct { 645 id int32 646 status uint32 // one of pidle/prunning/... 647 link puintptr 648 schedtick uint32 // incremented on every scheduler call 649 syscalltick uint32 // incremented on every system call 650 sysmontick sysmontick // last tick observed by sysmon 651 m muintptr // back-link to associated m (nil if idle) 652 mcache *mcache 653 pcache pageCache 654 raceprocctx uintptr 655 656 deferpool []*_defer // pool of available defer structs (see panic.go) 657 deferpoolbuf [32]*_defer 658 659 // Cache of goroutine ids, amortizes accesses to runtime·sched.goidgen. 660 goidcache uint64 661 goidcacheend uint64 662 663 // Queue of runnable goroutines. Accessed without lock. 664 runqhead uint32 665 runqtail uint32 666 runq [256]guintptr 667 // runnext, if non-nil, is a runnable G that was ready'd by 668 // the current G and should be run next instead of what's in 669 // runq if there's time remaining in the running G's time 670 // slice. It will inherit the time left in the current time 671 // slice. If a set of goroutines is locked in a 672 // communicate-and-wait pattern, this schedules that set as a 673 // unit and eliminates the (potentially large) scheduling 674 // latency that otherwise arises from adding the ready'd 675 // goroutines to the end of the run queue. 676 // 677 // Note that while other P's may atomically CAS this to zero, 678 // only the owner P can CAS it to a valid G. 679 runnext guintptr 680 681 // Available G's (status == Gdead) 682 gFree gList 683 684 sudogcache []*sudog 685 sudogbuf [128]*sudog 686 687 // Cache of mspan objects from the heap. 688 mspancache struct { 689 // We need an explicit length here because this field is used 690 // in allocation codepaths where write barriers are not allowed, 691 // and eliminating the write barrier/keeping it eliminated from 692 // slice updates is tricky, more so than just managing the length 693 // ourselves. 694 len int 695 buf [128]*mspan 696 } 697 698 // Cache of a single pinner object to reduce allocations from repeated 699 // pinner creation. 700 pinnerCache *pinner 701 702 trace pTraceState 703 704 palloc persistentAlloc // per-P to avoid mutex 705 706 // Per-P GC state 707 gcAssistTime int64 // Nanoseconds in assistAlloc 708 gcFractionalMarkTime int64 // Nanoseconds in fractional mark worker (atomic) 709 710 // limiterEvent tracks events for the GC CPU limiter. 711 limiterEvent limiterEvent 712 713 // gcMarkWorkerMode is the mode for the next mark worker to run in. 714 // That is, this is used to communicate with the worker goroutine 715 // selected for immediate execution by 716 // gcController.findRunnableGCWorker. When scheduling other goroutines, 717 // this field must be set to gcMarkWorkerNotWorker. 718 gcMarkWorkerMode gcMarkWorkerMode 719 // gcMarkWorkerStartTime is the nanotime() at which the most recent 720 // mark worker started. 721 gcMarkWorkerStartTime int64 722 723 // gcw is this P's GC work buffer cache. The work buffer is 724 // filled by write barriers, drained by mutator assists, and 725 // disposed on certain GC state transitions. 726 gcw gcWork 727 728 // wbBuf is this P's GC write barrier buffer. 729 // 730 // TODO: Consider caching this in the running G. 731 wbBuf wbBuf 732 733 runSafePointFn uint32 // if 1, run sched.safePointFn at next safe point 734 735 // statsSeq is a counter indicating whether this P is currently 736 // writing any stats. Its value is even when not, odd when it is. 737 statsSeq atomic.Uint32 738 739 // Timer heap. 740 timers timers 741 742 // Cleanups. 743 cleanups *cleanupBlock 744 cleanupsQueued uint64 // monotonic count of cleanups queued by this P 745 746 // maxStackScanDelta accumulates the amount of stack space held by 747 // live goroutines (i.e. those eligible for stack scanning). 748 // Flushed to gcController.maxStackScan once maxStackScanSlack 749 // or -maxStackScanSlack is reached. 750 maxStackScanDelta int64 751 752 // gc-time statistics about current goroutines 753 // Note that this differs from maxStackScan in that this 754 // accumulates the actual stack observed to be used at GC time (hi - sp), 755 // not an instantaneous measure of the total stack size that might need 756 // to be scanned (hi - lo). 757 scannedStackSize uint64 // stack size of goroutines scanned by this P 758 scannedStacks uint64 // number of goroutines scanned by this P 759 760 // preempt is set to indicate that this P should be enter the 761 // scheduler ASAP (regardless of what G is running on it). 762 preempt bool 763 764 // gcStopTime is the nanotime timestamp that this P last entered _Pgcstop. 765 gcStopTime int64 766 767 // goroutinesCreated is the total count of goroutines created by this P. 768 goroutinesCreated uint64 769 770 // xRegs is the per-P extended register state used by asynchronous 771 // preemption. This is an empty struct on platforms that don't use extended 772 // register state. 773 xRegs xRegPerP 774 775 // Padding is no longer needed. False sharing is now not a worry because p is large enough 776 // that its size class is an integer multiple of the cache line size (for any of our architectures). 777 } 778 779 type schedt struct { 780 goidgen atomic.Uint64 781 lastpoll atomic.Int64 // time of last network poll, 0 if currently polling 782 pollUntil atomic.Int64 // time to which current poll is sleeping 783 pollingNet atomic.Int32 // 1 if some P doing non-blocking network poll 784 785 lock mutex 786 787 // When increasing nmidle, nmidlelocked, nmsys, or nmfreed, be 788 // sure to call checkdead(). 789 790 midle muintptr // idle m's waiting for work 791 nmidle int32 // number of idle m's waiting for work 792 nmidlelocked int32 // number of locked m's waiting for work 793 mnext int64 // number of m's that have been created and next M ID 794 maxmcount int32 // maximum number of m's allowed (or die) 795 nmsys int32 // number of system m's not counted for deadlock 796 nmfreed int64 // cumulative number of freed m's 797 798 ngsys atomic.Int32 // number of system goroutines 799 nGsyscallNoP atomic.Int32 // number of goroutines in syscalls without a P 800 801 pidle puintptr // idle p's 802 npidle atomic.Int32 803 nmspinning atomic.Int32 // See "Worker thread parking/unparking" comment in proc.go. 804 needspinning atomic.Uint32 // See "Delicate dance" comment in proc.go. Boolean. Must hold sched.lock to set to 1. 805 806 // Global runnable queue. 807 runq gQueue 808 809 // disable controls selective disabling of the scheduler. 810 // 811 // Use schedEnableUser to control this. 812 // 813 // disable is protected by sched.lock. 814 disable struct { 815 // user disables scheduling of user goroutines. 816 user bool 817 runnable gQueue // pending runnable Gs 818 } 819 820 // Global cache of dead G's. 821 gFree struct { 822 lock mutex 823 stack gList // Gs with stacks 824 noStack gList // Gs without stacks 825 } 826 827 // Central cache of sudog structs. 828 sudoglock mutex 829 sudogcache *sudog 830 831 // Central pool of available defer structs. 832 deferlock mutex 833 deferpool *_defer 834 835 // freem is the list of m's waiting to be freed when their 836 // m.exited is set. Linked through m.freelink. 837 freem *m 838 839 gcwaiting atomic.Bool // gc is waiting to run 840 stopwait int32 841 stopnote note 842 sysmonwait atomic.Bool 843 sysmonnote note 844 845 // safePointFn should be called on each P at the next GC 846 // safepoint if p.runSafePointFn is set. 847 safePointFn func(*p) 848 safePointWait int32 849 safePointNote note 850 851 profilehz int32 // cpu profiling rate 852 853 procresizetime int64 // nanotime() of last change to gomaxprocs 854 totaltime int64 // ∫gomaxprocs dt up to procresizetime 855 856 customGOMAXPROCS bool // GOMAXPROCS was manually set from the environment or runtime.GOMAXPROCS 857 858 // sysmonlock protects sysmon's actions on the runtime. 859 // 860 // Acquire and hold this mutex to block sysmon from interacting 861 // with the rest of the runtime. 862 sysmonlock mutex 863 864 // timeToRun is a distribution of scheduling latencies, defined 865 // as the sum of time a G spends in the _Grunnable state before 866 // it transitions to _Grunning. 867 timeToRun timeHistogram 868 869 // idleTime is the total CPU time Ps have "spent" idle. 870 // 871 // Reset on each GC cycle. 872 idleTime atomic.Int64 873 874 // totalMutexWaitTime is the sum of time goroutines have spent in _Gwaiting 875 // with a waitreason of the form waitReasonSync{RW,}Mutex{R,}Lock. 876 totalMutexWaitTime atomic.Int64 877 878 // stwStoppingTimeGC/Other are distributions of stop-the-world stopping 879 // latencies, defined as the time taken by stopTheWorldWithSema to get 880 // all Ps to stop. stwStoppingTimeGC covers all GC-related STWs, 881 // stwStoppingTimeOther covers the others. 882 stwStoppingTimeGC timeHistogram 883 stwStoppingTimeOther timeHistogram 884 885 // stwTotalTimeGC/Other are distributions of stop-the-world total 886 // latencies, defined as the total time from stopTheWorldWithSema to 887 // startTheWorldWithSema. This is a superset of 888 // stwStoppingTimeGC/Other. stwTotalTimeGC covers all GC-related STWs, 889 // stwTotalTimeOther covers the others. 890 stwTotalTimeGC timeHistogram 891 stwTotalTimeOther timeHistogram 892 893 // totalRuntimeLockWaitTime (plus the value of lockWaitTime on each M in 894 // allm) is the sum of time goroutines have spent in _Grunnable and with an 895 // M, but waiting for locks within the runtime. This field stores the value 896 // for Ms that have exited. 897 totalRuntimeLockWaitTime atomic.Int64 898 899 // goroutinesCreated (plus the value of goroutinesCreated on each P in allp) 900 // is the sum of all goroutines created by the program. 901 goroutinesCreated atomic.Uint64 902 } 903 904 // Values for the flags field of a sigTabT. 905 const ( 906 _SigNotify = 1 << iota // let signal.Notify have signal, even if from kernel 907 _SigKill // if signal.Notify doesn't take it, exit quietly 908 _SigThrow // if signal.Notify doesn't take it, exit loudly 909 _SigPanic // if the signal is from the kernel, panic 910 _SigDefault // if the signal isn't explicitly requested, don't monitor it 911 _SigGoExit // cause all runtime procs to exit (only used on Plan 9). 912 _SigSetStack // Don't explicitly install handler, but add SA_ONSTACK to existing libc handler 913 _SigUnblock // always unblock; see blockableSig 914 _SigIgn // _SIG_DFL action is to ignore the signal 915 ) 916 917 // Layout of in-memory per-function information prepared by linker 918 // See https://golang.org/s/go12symtab. 919 // Keep in sync with linker (../cmd/link/internal/ld/pcln.go:/pclntab) 920 // and with package debug/gosym and with symtab.go in package runtime. 921 type _func struct { 922 sys.NotInHeap // Only in static data 923 924 entryOff uint32 // start pc, as offset from moduledata.text/pcHeader.textStart 925 nameOff int32 // function name, as index into moduledata.funcnametab. 926 927 args int32 // in/out args size 928 deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any. 929 930 pcsp uint32 931 pcfile uint32 932 pcln uint32 933 npcdata uint32 934 cuOffset uint32 // runtime.cutab offset of this function's CU 935 startLine int32 // line number of start of function (func keyword/TEXT directive) 936 funcID abi.FuncID // set for certain special runtime functions 937 flag abi.FuncFlag 938 _ [1]byte // pad 939 nfuncdata uint8 // must be last, must end on a uint32-aligned boundary 940 941 // The end of the struct is followed immediately by two variable-length 942 // arrays that reference the pcdata and funcdata locations for this 943 // function. 944 945 // pcdata contains the offset into moduledata.pctab for the start of 946 // that index's table. e.g., 947 // &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of 948 // the unsafe point table. 949 // 950 // An offset of 0 indicates that there is no table. 951 // 952 // pcdata [npcdata]uint32 953 954 // funcdata contains the offset past moduledata.gofunc which contains a 955 // pointer to that index's funcdata. e.g., 956 // *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is 957 // the argument pointer map. 958 // 959 // An offset of ^uint32(0) indicates that there is no entry. 960 // 961 // funcdata [nfuncdata]uint32 962 } 963 964 // Pseudo-Func that is returned for PCs that occur in inlined code. 965 // A *Func can be either a *_func or a *funcinl, and they are distinguished 966 // by the first uintptr. 967 // 968 // TODO(austin): Can we merge this with inlinedCall? 969 type funcinl struct { 970 ones uint32 // set to ^0 to distinguish from _func 971 entry uintptr // entry of the real (the "outermost") frame 972 name string 973 file string 974 line int32 975 startLine int32 976 } 977 978 type itab = abi.ITab 979 980 // Lock-free stack node. 981 // Also known to export_test.go. 982 type lfnode struct { 983 next uint64 984 pushcnt uintptr 985 } 986 987 type forcegcstate struct { 988 lock mutex 989 g *g 990 idle atomic.Bool 991 } 992 993 // A _defer holds an entry on the list of deferred calls. 994 // If you add a field here, add code to clear it in deferProcStack. 995 // This struct must match the code in cmd/compile/internal/ssagen/ssa.go:deferstruct 996 // and cmd/compile/internal/ssagen/ssa.go:(*state).call. 997 // Some defers will be allocated on the stack and some on the heap. 998 // All defers are logically part of the stack, so write barriers to 999 // initialize them are not required. All defers must be manually scanned, 1000 // and for heap defers, marked. 1001 type _defer struct { 1002 heap bool 1003 rangefunc bool // true for rangefunc list 1004 sp uintptr // sp at time of defer 1005 pc uintptr // pc at time of defer 1006 fn func() // can be nil for open-coded defers 1007 link *_defer // next defer on G; can point to either heap or stack! 1008 1009 // If rangefunc is true, *head is the head of the atomic linked list 1010 // during a range-over-func execution. 1011 head *atomic.Pointer[_defer] 1012 } 1013 1014 // A _panic holds information about an active panic. 1015 // 1016 // A _panic value must only ever live on the stack. 1017 // 1018 // The gopanicFP and link fields are stack pointers, but don't need special 1019 // handling during stack growth: because they are pointer-typed and 1020 // _panic values only live on the stack, regular stack pointer 1021 // adjustment takes care of them. 1022 type _panic struct { 1023 arg any // argument to panic 1024 link *_panic // link to earlier panic 1025 1026 // startPC and startSP track where _panic.start was called. 1027 startPC uintptr 1028 startSP unsafe.Pointer 1029 1030 // The current stack frame that we're running deferred calls for. 1031 sp unsafe.Pointer 1032 lr uintptr 1033 fp unsafe.Pointer 1034 1035 // retpc stores the PC where the panic should jump back to, if the 1036 // function last returned by _panic.next() recovers the panic. 1037 retpc uintptr 1038 1039 // Extra state for handling open-coded defers. 1040 deferBitsPtr *uint8 1041 slotsPtr unsafe.Pointer 1042 1043 recovered bool // whether this panic has been recovered 1044 repanicked bool // whether this panic repanicked 1045 goexit bool 1046 deferreturn bool 1047 1048 gopanicFP unsafe.Pointer // frame pointer of the gopanic frame 1049 } 1050 1051 // savedOpenDeferState tracks the extra state from _panic that's 1052 // necessary for deferreturn to pick up where gopanic left off, 1053 // without needing to unwind the stack. 1054 type savedOpenDeferState struct { 1055 retpc uintptr 1056 deferBitsOffset uintptr 1057 slotsOffset uintptr 1058 } 1059 1060 // ancestorInfo records details of where a goroutine was started. 1061 type ancestorInfo struct { 1062 pcs []uintptr // pcs from the stack of this goroutine 1063 goid uint64 // goroutine id of this goroutine; original goroutine possibly dead 1064 gopc uintptr // pc of go statement that created this goroutine 1065 } 1066 1067 // A waitReason explains why a goroutine has been stopped. 1068 // See gopark. Do not re-use waitReasons, add new ones. 1069 type waitReason uint8 1070 1071 const ( 1072 waitReasonZero waitReason = iota // "" 1073 waitReasonGCAssistMarking // "GC assist marking" 1074 waitReasonIOWait // "IO wait" 1075 waitReasonChanReceiveNilChan // "chan receive (nil chan)" 1076 waitReasonChanSendNilChan // "chan send (nil chan)" 1077 waitReasonDumpingHeap // "dumping heap" 1078 waitReasonGarbageCollection // "garbage collection" 1079 waitReasonGarbageCollectionScan // "garbage collection scan" 1080 waitReasonPanicWait // "panicwait" 1081 waitReasonSelect // "select" 1082 waitReasonSelectNoCases // "select (no cases)" 1083 waitReasonGCAssistWait // "GC assist wait" 1084 waitReasonGCSweepWait // "GC sweep wait" 1085 waitReasonGCScavengeWait // "GC scavenge wait" 1086 waitReasonChanReceive // "chan receive" 1087 waitReasonChanSend // "chan send" 1088 waitReasonFinalizerWait // "finalizer wait" 1089 waitReasonForceGCIdle // "force gc (idle)" 1090 waitReasonUpdateGOMAXPROCSIdle // "GOMAXPROCS updater (idle)" 1091 waitReasonSemacquire // "semacquire" 1092 waitReasonSleep // "sleep" 1093 waitReasonSyncCondWait // "sync.Cond.Wait" 1094 waitReasonSyncMutexLock // "sync.Mutex.Lock" 1095 waitReasonSyncRWMutexRLock // "sync.RWMutex.RLock" 1096 waitReasonSyncRWMutexLock // "sync.RWMutex.Lock" 1097 waitReasonSyncWaitGroupWait // "sync.WaitGroup.Wait" 1098 waitReasonTraceReaderBlocked // "trace reader (blocked)" 1099 waitReasonWaitForGCCycle // "wait for GC cycle" 1100 waitReasonGCWorkerIdle // "GC worker (idle)" 1101 waitReasonGCWorkerActive // "GC worker (active)" 1102 waitReasonPreempted // "preempted" 1103 waitReasonDebugCall // "debug call" 1104 waitReasonGCMarkTermination // "GC mark termination" 1105 waitReasonStoppingTheWorld // "stopping the world" 1106 waitReasonFlushProcCaches // "flushing proc caches" 1107 waitReasonTraceGoroutineStatus // "trace goroutine status" 1108 waitReasonTraceProcStatus // "trace proc status" 1109 waitReasonPageTraceFlush // "page trace flush" 1110 waitReasonCoroutine // "coroutine" 1111 waitReasonGCWeakToStrongWait // "GC weak to strong wait" 1112 waitReasonSynctestRun // "synctest.Run" 1113 waitReasonSynctestWait // "synctest.Wait" 1114 waitReasonSynctestChanReceive // "chan receive (durable)" 1115 waitReasonSynctestChanSend // "chan send (durable)" 1116 waitReasonSynctestSelect // "select (durable)" 1117 waitReasonSynctestWaitGroupWait // "sync.WaitGroup.Wait (durable)" 1118 waitReasonCleanupWait // "cleanup wait" 1119 ) 1120 1121 var waitReasonStrings = [...]string{ 1122 waitReasonZero: "", 1123 waitReasonGCAssistMarking: "GC assist marking", 1124 waitReasonIOWait: "IO wait", 1125 waitReasonChanReceiveNilChan: "chan receive (nil chan)", 1126 waitReasonChanSendNilChan: "chan send (nil chan)", 1127 waitReasonDumpingHeap: "dumping heap", 1128 waitReasonGarbageCollection: "garbage collection", 1129 waitReasonGarbageCollectionScan: "garbage collection scan", 1130 waitReasonPanicWait: "panicwait", 1131 waitReasonSelect: "select", 1132 waitReasonSelectNoCases: "select (no cases)", 1133 waitReasonGCAssistWait: "GC assist wait", 1134 waitReasonGCSweepWait: "GC sweep wait", 1135 waitReasonGCScavengeWait: "GC scavenge wait", 1136 waitReasonChanReceive: "chan receive", 1137 waitReasonChanSend: "chan send", 1138 waitReasonFinalizerWait: "finalizer wait", 1139 waitReasonForceGCIdle: "force gc (idle)", 1140 waitReasonUpdateGOMAXPROCSIdle: "GOMAXPROCS updater (idle)", 1141 waitReasonSemacquire: "semacquire", 1142 waitReasonSleep: "sleep", 1143 waitReasonSyncCondWait: "sync.Cond.Wait", 1144 waitReasonSyncMutexLock: "sync.Mutex.Lock", 1145 waitReasonSyncRWMutexRLock: "sync.RWMutex.RLock", 1146 waitReasonSyncRWMutexLock: "sync.RWMutex.Lock", 1147 waitReasonSyncWaitGroupWait: "sync.WaitGroup.Wait", 1148 waitReasonTraceReaderBlocked: "trace reader (blocked)", 1149 waitReasonWaitForGCCycle: "wait for GC cycle", 1150 waitReasonGCWorkerIdle: "GC worker (idle)", 1151 waitReasonGCWorkerActive: "GC worker (active)", 1152 waitReasonPreempted: "preempted", 1153 waitReasonDebugCall: "debug call", 1154 waitReasonGCMarkTermination: "GC mark termination", 1155 waitReasonStoppingTheWorld: "stopping the world", 1156 waitReasonFlushProcCaches: "flushing proc caches", 1157 waitReasonTraceGoroutineStatus: "trace goroutine status", 1158 waitReasonTraceProcStatus: "trace proc status", 1159 waitReasonPageTraceFlush: "page trace flush", 1160 waitReasonCoroutine: "coroutine", 1161 waitReasonGCWeakToStrongWait: "GC weak to strong wait", 1162 waitReasonSynctestRun: "synctest.Run", 1163 waitReasonSynctestWait: "synctest.Wait", 1164 waitReasonSynctestChanReceive: "chan receive (durable)", 1165 waitReasonSynctestChanSend: "chan send (durable)", 1166 waitReasonSynctestSelect: "select (durable)", 1167 waitReasonSynctestWaitGroupWait: "sync.WaitGroup.Wait (durable)", 1168 waitReasonCleanupWait: "cleanup wait", 1169 } 1170 1171 func (w waitReason) String() string { 1172 if w < 0 || w >= waitReason(len(waitReasonStrings)) { 1173 return "unknown wait reason" 1174 } 1175 return waitReasonStrings[w] 1176 } 1177 1178 func (w waitReason) isMutexWait() bool { 1179 return w == waitReasonSyncMutexLock || 1180 w == waitReasonSyncRWMutexRLock || 1181 w == waitReasonSyncRWMutexLock 1182 } 1183 1184 func (w waitReason) isWaitingForSuspendG() bool { 1185 return isWaitingForSuspendG[w] 1186 } 1187 1188 // isWaitingForSuspendG indicates that a goroutine is only entering _Gwaiting and 1189 // setting a waitReason because it needs to be able to let the suspendG 1190 // (used by the GC and the execution tracer) take ownership of its stack. 1191 // The G is always actually executing on the system stack in these cases. 1192 // 1193 // TODO(mknyszek): Consider replacing this with a new dedicated G status. 1194 var isWaitingForSuspendG = [len(waitReasonStrings)]bool{ 1195 waitReasonStoppingTheWorld: true, 1196 waitReasonGCMarkTermination: true, 1197 waitReasonGarbageCollection: true, 1198 waitReasonGarbageCollectionScan: true, 1199 waitReasonTraceGoroutineStatus: true, 1200 waitReasonTraceProcStatus: true, 1201 waitReasonPageTraceFlush: true, 1202 waitReasonGCAssistMarking: true, 1203 waitReasonGCWorkerActive: true, 1204 waitReasonFlushProcCaches: true, 1205 } 1206 1207 func (w waitReason) isIdleInSynctest() bool { 1208 return isIdleInSynctest[w] 1209 } 1210 1211 // isIdleInSynctest indicates that a goroutine is considered idle by synctest.Wait. 1212 var isIdleInSynctest = [len(waitReasonStrings)]bool{ 1213 waitReasonChanReceiveNilChan: true, 1214 waitReasonChanSendNilChan: true, 1215 waitReasonSelectNoCases: true, 1216 waitReasonSleep: true, 1217 waitReasonSyncCondWait: true, 1218 waitReasonSynctestWaitGroupWait: true, 1219 waitReasonCoroutine: true, 1220 waitReasonSynctestRun: true, 1221 waitReasonSynctestWait: true, 1222 waitReasonSynctestChanReceive: true, 1223 waitReasonSynctestChanSend: true, 1224 waitReasonSynctestSelect: true, 1225 } 1226 1227 var ( 1228 // Linked-list of all Ms. Written under sched.lock, read atomically. 1229 allm *m 1230 1231 gomaxprocs int32 1232 numCPUStartup int32 1233 forcegc forcegcstate 1234 sched schedt 1235 newprocs int32 1236 ) 1237 1238 var ( 1239 // allpLock protects P-less reads and size changes of allp, idlepMask, 1240 // and timerpMask, and all writes to allp. 1241 allpLock mutex 1242 1243 // len(allp) == gomaxprocs; may change at safe points, otherwise 1244 // immutable. 1245 allp []*p 1246 1247 // Bitmask of Ps in _Pidle list, one bit per P. Reads and writes must 1248 // be atomic. Length may change at safe points. 1249 // 1250 // Each P must update only its own bit. In order to maintain 1251 // consistency, a P going idle must the idle mask simultaneously with 1252 // updates to the idle P list under the sched.lock, otherwise a racing 1253 // pidleget may clear the mask before pidleput sets the mask, 1254 // corrupting the bitmap. 1255 // 1256 // N.B., procresize takes ownership of all Ps in stopTheWorldWithSema. 1257 idlepMask pMask 1258 1259 // Bitmask of Ps that may have a timer, one bit per P. Reads and writes 1260 // must be atomic. Length may change at safe points. 1261 // 1262 // Ideally, the timer mask would be kept immediately consistent on any timer 1263 // operations. Unfortunately, updating a shared global data structure in the 1264 // timer hot path adds too much overhead in applications frequently switching 1265 // between no timers and some timers. 1266 // 1267 // As a compromise, the timer mask is updated only on pidleget / pidleput. A 1268 // running P (returned by pidleget) may add a timer at any time, so its mask 1269 // must be set. An idle P (passed to pidleput) cannot add new timers while 1270 // idle, so if it has no timers at that time, its mask may be cleared. 1271 // 1272 // Thus, we get the following effects on timer-stealing in findrunnable: 1273 // 1274 // - Idle Ps with no timers when they go idle are never checked in findrunnable 1275 // (for work- or timer-stealing; this is the ideal case). 1276 // - Running Ps must always be checked. 1277 // - Idle Ps whose timers are stolen must continue to be checked until they run 1278 // again, even after timer expiration. 1279 // 1280 // When the P starts running again, the mask should be set, as a timer may be 1281 // added at any time. 1282 // 1283 // TODO(prattmic): Additional targeted updates may improve the above cases. 1284 // e.g., updating the mask when stealing a timer. 1285 timerpMask pMask 1286 ) 1287 1288 // goarmsoftfp is used by runtime/cgo assembly. 1289 // 1290 //go:linkname goarmsoftfp 1291 1292 var ( 1293 // Pool of GC parked background workers. Entries are type 1294 // *gcBgMarkWorkerNode. 1295 gcBgMarkWorkerPool lfstack 1296 1297 // Total number of gcBgMarkWorker goroutines. Protected by worldsema. 1298 gcBgMarkWorkerCount int32 1299 1300 // Information about what cpu features are available. 1301 // Packages outside the runtime should not use these 1302 // as they are not an external api. 1303 // Set on startup in asm_{386,amd64}.s 1304 processorVersionInfo uint32 1305 isIntel bool 1306 ) 1307 1308 // set by cmd/link on arm systems 1309 // accessed using linkname by internal/runtime/atomic. 1310 // 1311 // goarm should be an internal detail, 1312 // but widely used packages access it using linkname. 1313 // Notable members of the hall of shame include: 1314 // - github.com/creativeprojects/go-selfupdate 1315 // 1316 // Do not remove or change the type signature. 1317 // See go.dev/issue/67401. 1318 // 1319 //go:linkname goarm 1320 var ( 1321 goarm uint8 1322 goarmsoftfp uint8 1323 ) 1324 1325 // Set by the linker so the runtime can determine the buildmode. 1326 var ( 1327 islibrary bool // -buildmode=c-shared 1328 isarchive bool // -buildmode=c-archive 1329 ) 1330 1331 // Must agree with internal/buildcfg.FramePointerEnabled. 1332 const framepointer_enabled = GOARCH == "amd64" || GOARCH == "arm64" 1333 1334 // getcallerfp returns the frame pointer of the caller of the caller 1335 // of this function. 1336 // 1337 //go:nosplit 1338 //go:noinline 1339 func getcallerfp() uintptr { 1340 fp := getfp() // This frame's FP. 1341 if fp != 0 { 1342 fp = *(*uintptr)(unsafe.Pointer(fp)) // The caller's FP. 1343 fp = *(*uintptr)(unsafe.Pointer(fp)) // The caller's caller's FP. 1344 } 1345 return fp 1346 } 1347