func semrelease(addr *uint32) { root := semroot(addr) atomic.Xadd(addr, 1) // Easy case: no waiters? // This check must happen after the xadd, to avoid a missed wakeup // (see loop in semacquire). if atomic.Load(&root.nwait) == 0 { return } // Harder case: search for a waiter and wake it. lock(&root.lock) if atomic.Load(&root.nwait) == 0 { // The count is already consumed by another goroutine, // so no need to wake up another goroutine. unlock(&root.lock) return } s := root.head for ; s != nil; s = s.next { if s.elem == unsafe.Pointer(addr) { atomic.Xadd(&root.nwait, -1) root.dequeue(s) break } } unlock(&root.lock) if s != nil { if s.releasetime != 0 { s.releasetime = cputicks() } goready(s.g, 4) } }
// notifyListNotifyAll notifies all entries in the list. //go:linkname notifyListNotifyAll sync.runtime_notifyListNotifyAll func notifyListNotifyAll(l *notifyList) { // Fast-path: if there are no new waiters since the last notification // we don't need to acquire the lock. if atomic.Load(&l.wait) == atomic.Load(&l.notify) { return } // Pull the list out into a local variable, waiters will be readied // outside the lock. lock(&l.lock) s := l.head l.head = nil l.tail = nil // Update the next ticket to be notified. We can set it to the current // value of wait because any previous waiters are already in the list // or will notice that they have already been notified when trying to // add themselves to the list. atomic.Store(&l.notify, atomic.Load(&l.wait)) unlock(&l.lock) // Go through the local list and ready all waiters. for s != nil { next := s.next s.next = nil readyWithTime(s, 4) s = next } }
// May run with m.p==nil if called from notetsleep, so write barriers // are not allowed. // //go:nosplit //go:nowritebarrier func notetsleep_internal(n *note, ns int64) bool { gp := getg() if ns < 0 { for atomic.Load(key32(&n.key)) == 0 { gp.m.blocked = true futexsleep(key32(&n.key), 0, -1) gp.m.blocked = false } return true } if atomic.Load(key32(&n.key)) != 0 { return true } deadline := nanotime() + ns for { gp.m.blocked = true futexsleep(key32(&n.key), 0, ns) gp.m.blocked = false if atomic.Load(key32(&n.key)) != 0 { break } now := nanotime() if now >= deadline { break } ns = deadline - now } return atomic.Load(key32(&n.key)) != 0 }
func RunSchedLocalQueueEmptyTest(iters int) { // Test that runq is not spuriously reported as empty. // Runq emptiness affects scheduling decisions and spurious emptiness // can lead to underutilization (both runnable Gs and idle Ps coexist // for arbitrary long time). done := make(chan bool, 1) p := new(p) gs := make([]g, 2) ready := new(uint32) for i := 0; i < iters; i++ { *ready = 0 next0 := (i & 1) == 0 next1 := (i & 2) == 0 runqput(p, &gs[0], next0) go func() { for atomic.Xadd(ready, 1); atomic.Load(ready) != 2; { } if runqempty(p) { println("next:", next0, next1) throw("queue is empty") } done <- true }() for atomic.Xadd(ready, 1); atomic.Load(ready) != 2; { } runqput(p, &gs[1], next1) runqget(p) <-done runqget(p) } }
// Called to receive the next queued signal. // Must only be called from a single goroutine at a time. //go:linkname signal_recv os/signal.signal_recv func signal_recv() uint32 { for { // Serve any signals from local copy. for i := uint32(0); i < _NSIG; i++ { if sig.recv[i/32]&(1<<(i&31)) != 0 { sig.recv[i/32] &^= 1 << (i & 31) return i } } // Wait for updates to be available from signal sender. Receive: for { switch atomic.Load(&sig.state) { default: throw("signal_recv: inconsistent state") case sigIdle: if atomic.Cas(&sig.state, sigIdle, sigReceiving) { notetsleepg(&sig.note, -1) noteclear(&sig.note) break Receive } case sigSending: if atomic.Cas(&sig.state, sigSending, sigIdle) { break Receive } } } // Incorporate updates from sender into local copy. for i := range sig.mask { sig.recv[i] = atomic.Xchg(&sig.mask[i], 0) } } }
// block returns the spans in the i'th block of buffer b. block is // safe to call concurrently with push. func (b *gcSweepBuf) block(i int) []*mspan { // Perform bounds check before loading spine address since // push ensures the allocated length is at least spineLen. if i < 0 || uintptr(i) >= atomic.Loaduintptr(&b.spineLen) { throw("block index out of range") } // Get block i. spine := atomic.Loadp(unsafe.Pointer(&b.spine)) blockp := add(spine, sys.PtrSize*uintptr(i)) block := (*gcSweepBlock)(atomic.Loadp(blockp)) // Slice the block if necessary. cursor := uintptr(atomic.Load(&b.index)) top, bottom := cursor/gcSweepBlockEntries, cursor%gcSweepBlockEntries var spans []*mspan if uintptr(i) < top { spans = block.spans[:] } else { spans = block.spans[:bottom] } // push may have reserved a slot but not filled it yet, so // trim away unused entries. for len(spans) > 0 && spans[len(spans)-1] == nil { spans = spans[:len(spans)-1] } return spans }
//go:nosplit func semasleep(ns int64) int32 { _g_ := getg() // Compute sleep deadline. var tsp *timespec if ns >= 0 { var ts timespec var nsec int32 ns += nanotime() ts.set_sec(timediv(ns, 1000000000, &nsec)) ts.set_nsec(nsec) tsp = &ts } for { v := atomic.Load(&_g_.m.waitsemacount) if v > 0 { if atomic.Cas(&_g_.m.waitsemacount, v, v-1) { return 0 // semaphore acquired } continue } // Sleep until unparked by semawakeup or timeout. ret := lwp_park(tsp, 0, unsafe.Pointer(&_g_.m.waitsemacount), nil) if ret == _ETIMEDOUT { return -1 } } }
// Called to initialize a new m (including the bootstrap m). // Called on the new thread, cannot allocate memory. func minit() { if atomic.Load(&exiting) != 0 { exits(&emptystatus[0]) } // Mask all SSE floating-point exceptions // when running on the 64-bit kernel. setfpmasks() }
func semrelease(addr *uint32) { root := semroot(addr) atomic.Xadd(addr, 1) // Easy case: no waiters? // This check must happen after the xadd, to avoid a missed wakeup // (see loop in semacquire). if atomic.Load(&root.nwait) == 0 { return } // Harder case: search for a waiter and wake it. lock(&root.lock) if atomic.Load(&root.nwait) == 0 { // The count is already consumed by another goroutine, // so no need to wake up another goroutine. unlock(&root.lock) return } s := root.head for ; s != nil; s = s.next { if s.elem == unsafe.Pointer(addr) { atomic.Xadd(&root.nwait, -1) root.dequeue(s) break } } if s != nil { if s.acquiretime != 0 { t0 := cputicks() for x := root.head; x != nil; x = x.next { if x.elem == unsafe.Pointer(addr) { x.acquiretime = t0 } } mutexevent(t0-s.acquiretime, 3) } } unlock(&root.lock) if s != nil { // May be slow, so unlock first readyWithTime(s, 5) } }
func cansemacquire(addr *uint32) bool { for { v := atomic.Load(addr) if v == 0 { return false } if atomic.Cas(addr, v, v-1) { return true } } }
func notesleep(n *note) { gp := getg() if gp != gp.m.g0 { throw("notesleep not on g0") } for atomic.Load(key32(&n.key)) == 0 { gp.m.blocked = true futexsleep(key32(&n.key), 0, -1) gp.m.blocked = false } }
// gotraceback returns the current traceback settings. // // If level is 0, suppress all tracebacks. // If level is 1, show tracebacks, but exclude runtime frames. // If level is 2, show tracebacks including runtime frames. // If all is set, print all goroutine stacks. Otherwise, print just the current goroutine. // If crash is set, crash (core dump, etc) after tracebacking. // //go:nosplit func gotraceback() (level int32, all, crash bool) { _g_ := getg() all = _g_.m.throwing > 0 if _g_.m.traceback != 0 { level = int32(_g_.m.traceback) return } t := atomic.Load(&traceback_cache) crash = t&tracebackCrash != 0 all = all || t&tracebackAll != 0 level = int32(t >> tracebackShift) return }
// notifyListNotifyOne notifies one entry in the list. //go:linkname notifyListNotifyOne sync.runtime_notifyListNotifyOne func notifyListNotifyOne(l *notifyList) { // Fast-path: if there are no new waiters since the last notification // we don't need to acquire the lock at all. if atomic.Load(&l.wait) == atomic.Load(&l.notify) { return } lock(&l.lock) // Re-check under the lock if we need to do anything. t := l.notify if t == atomic.Load(&l.wait) { unlock(&l.lock) return } // Update the next notify ticket number, and try to find the G that // needs to be notified. If it hasn't made it to the list yet we won't // find it, but it won't park itself once it sees the new notify number. atomic.Store(&l.notify, t+1) for p, s := (*sudog)(nil), l.head; s != nil; p, s = s, s.next { if s.ticket == t { n := s.next if p != nil { p.next = n } else { l.head = n } if n == nil { l.tail = p } unlock(&l.lock) s.next = nil readyWithTime(s, 4) return } } unlock(&l.lock) }
// Returns only when span s has been swept. //go:nowritebarrier func (s *mspan) ensureSwept() { // Caller must disable preemption. // Otherwise when this function returns the span can become unswept again // (if GC is triggered on another goroutine). _g_ := getg() if _g_.m.locks == 0 && _g_.m.mallocing == 0 && _g_ != _g_.m.g0 { throw("MSpan_EnsureSwept: m is not locked") } sg := mheap_.sweepgen if atomic.Load(&s.sweepgen) == sg { return } // The caller must be sure that the span is a MSpanInUse span. if atomic.Cas(&s.sweepgen, sg-2, sg-1) { s.sweep(false) return } // unfortunate condition, and we don't have efficient means to wait for atomic.Load(&s.sweepgen) != sg { osyield() } }
// recordForPanic maintains a circular buffer of messages written by the // runtime leading up to a process crash, allowing the messages to be // extracted from a core dump. // // The text written during a process crash (following "panic" or "fatal // error") is not saved, since the goroutine stacks will generally be readable // from the runtime datastructures in the core file. func recordForPanic(b []byte) { printlock() if atomic.Load(&panicking) == 0 { // Not actively crashing: maintain circular buffer of print output. for i := 0; i < len(b); { n := copy(printBacklog[printBacklogIndex:], b[i:]) i += n printBacklogIndex += n printBacklogIndex %= len(printBacklog) } } printunlock() }
func notesleep(n *note) { gp := getg() // Currently OK to sleep in non-g0 for gccgo. It happens in // stoptheworld because we have not implemented preemption. // if gp != gp.m.g0 { // throw("notesleep not on g0") // } for atomic.Load(key32(&n.key)) == 0 { gp.m.blocked = true futexsleep(key32(&n.key), 0, -1) gp.m.blocked = false } }
// Called from sighandler to send a signal back out of the signal handling thread. // Reports whether the signal was sent. If not, the caller typically crashes the program. func sigsend(s uint32) bool { bit := uint32(1) << uint(s&31) if !sig.inuse || s >= uint32(32*len(sig.wanted)) || sig.wanted[s/32]&bit == 0 { return false } // Add signal to outgoing queue. for { mask := sig.mask[s/32] if mask&bit != 0 { return true // signal already in queue } if atomic.Cas(&sig.mask[s/32], mask, mask|bit) { break } } // Notify receiver that queue has new bit. Send: for { switch atomic.Load(&sig.state) { default: throw("sigsend: inconsistent state") case sigIdle: if atomic.Cas(&sig.state, sigIdle, sigSending) { break Send } case sigSending: // notification already pending break Send case sigReceiving: if atomic.Cas(&sig.state, sigReceiving, sigIdle) { notewakeup(&sig.note) break Send } } } return true }
// gcParkAssist puts the current goroutine on the assist queue and parks. // // gcParkAssist returns whether the assist is now satisfied. If it // returns false, the caller must retry the assist. // //go:nowritebarrier func gcParkAssist() bool { lock(&work.assistQueue.lock) // If the GC cycle finished while we were getting the lock, // exit the assist. The cycle can't finish while we hold the // lock. if atomic.Load(&gcBlackenEnabled) == 0 { unlock(&work.assistQueue.lock) return true } gp := getg() oldHead, oldTail := work.assistQueue.head, work.assistQueue.tail if oldHead == 0 { work.assistQueue.head.set(gp) } else { oldTail.ptr().schedlink.set(gp) } work.assistQueue.tail.set(gp) gp.schedlink.set(nil) // Recheck for background credit now that this G is in // the queue, but can still back out. This avoids a // race in case background marking has flushed more // credit since we checked above. if atomic.Loadint64(&gcController.bgScanCredit) > 0 { work.assistQueue.head = oldHead work.assistQueue.tail = oldTail if oldTail != 0 { oldTail.ptr().schedlink.set(nil) } unlock(&work.assistQueue.lock) return false } // Park. goparkunlock(&work.assistQueue.lock, "GC assist wait", traceEvGoBlockGC, 2) return true }
//go:nosplit func semasleep(ns int64) int32 { _g_ := getg() // Compute sleep deadline. var tsp *timespec if ns >= 0 { var ts timespec var nsec int32 ns += nanotime() ts.set_sec(int64(timediv(ns, 1000000000, &nsec))) ts.set_nsec(nsec) tsp = &ts } for { v := atomic.Load(&_g_.m.waitsemacount) if v > 0 { if atomic.Cas(&_g_.m.waitsemacount, v, v-1) { return 0 // semaphore acquired } continue } // Sleep until woken by semawakeup or timeout; or abort if waitsemacount != 0. // // From OpenBSD's __thrsleep(2) manual: // "The abort argument, if not NULL, points to an int that will // be examined [...] immediately before blocking. If that int // is non-zero then __thrsleep() will immediately return EINTR // without blocking." ret := thrsleep(uintptr(unsafe.Pointer(&_g_.m.waitsemacount)), _CLOCK_MONOTONIC, tsp, 0, &_g_.m.waitsemacount) if ret == _EWOULDBLOCK { return -1 } } }
// CgocallBack is used when calling from C/C++ code into Go code. // The usual approach is // syscall.CgocallBack() // defer syscall.CgocallBackDone() // gofunction() //go:nosplit func CgocallBack() { if getg() == nil || getg().m == nil { needm(0) mp := getg().m mp.dropextram = true } exitsyscall(0) if getg().m.ncgo == 0 { // The C call to Go came from a thread created by C. // The C call to Go came from a thread not currently running // any Go. In the case of -buildmode=c-archive or c-shared, // this call may be coming in before package initialization // is complete. Wait until it is. <-main_init_done } mp := getg().m if mp.needextram || atomic.Load(&extraMWaiters) > 0 { mp.needextram = false newextram() } }
// gcAssistAlloc1 is the part of gcAssistAlloc that runs on the system // stack. This is a separate function to make it easier to see that // we're not capturing anything from the user stack, since the user // stack may move while we're in this function. // // gcAssistAlloc1 indicates whether this assist completed the mark // phase by setting gp.param to non-nil. This can't be communicated on // the stack since it may move. // //go:systemstack func gcAssistAlloc1(gp *g, scanWork int64) { // Clear the flag indicating that this assist completed the // mark phase. gp.param = nil if atomic.Load(&gcBlackenEnabled) == 0 { // The gcBlackenEnabled check in malloc races with the // store that clears it but an atomic check in every malloc // would be a performance hit. // Instead we recheck it here on the non-preemptable system // stack to determine if we should preform an assist. // GC is done, so ignore any remaining debt. gp.gcAssistBytes = 0 return } // Track time spent in this assist. Since we're on the // system stack, this is non-preemptible, so we can // just measure start and end time. startTime := nanotime() decnwait := atomic.Xadd(&work.nwait, -1) if decnwait == work.nproc { println("runtime: work.nwait =", decnwait, "work.nproc=", work.nproc) throw("nwait > work.nprocs") } // gcDrainN requires the caller to be preemptible. casgstatus(gp, _Grunning, _Gwaiting) gp.waitreason = "GC assist marking" // drain own cached work first in the hopes that it // will be more cache friendly. gcw := &getg().m.p.ptr().gcw workDone := gcDrainN(gcw, scanWork) // If we are near the end of the mark phase // dispose of the gcw. if gcBlackenPromptly { gcw.dispose() } casgstatus(gp, _Gwaiting, _Grunning) // Record that we did this much scan work. // // Back out the number of bytes of assist credit that // this scan work counts for. The "1+" is a poor man's // round-up, to ensure this adds credit even if // assistBytesPerWork is very low. gp.gcAssistBytes += 1 + int64(gcController.assistBytesPerWork*float64(workDone)) // If this is the last worker and we ran out of work, // signal a completion point. incnwait := atomic.Xadd(&work.nwait, +1) if incnwait > work.nproc { println("runtime: work.nwait=", incnwait, "work.nproc=", work.nproc, "gcBlackenPromptly=", gcBlackenPromptly) throw("work.nwait > work.nproc") } if incnwait == work.nproc && !gcMarkWorkAvailable(nil) { // This has reached a background completion point. Set // gp.param to a non-nil value to indicate this. It // doesn't matter what we set it to (it just has to be // a valid pointer). gp.param = unsafe.Pointer(gp) } duration := nanotime() - startTime _p_ := gp.m.p.ptr() _p_.gcAssistTime += duration if _p_.gcAssistTime > gcAssistTimeSlack { atomic.Xaddint64(&gcController.assistTime, _p_.gcAssistTime) _p_.gcAssistTime = 0 } }
// markroot scans the i'th root. // // Preemption must be disabled (because this uses a gcWork). // // nowritebarrier is only advisory here. // //go:nowritebarrier func markroot(gcw *gcWork, i uint32) { // TODO(austin): This is a bit ridiculous. Compute and store // the bases in gcMarkRootPrepare instead of the counts. baseFlushCache := uint32(fixedRootCount) baseData := baseFlushCache + uint32(work.nFlushCacheRoots) baseBSS := baseData + uint32(work.nDataRoots) baseSpans := baseBSS + uint32(work.nBSSRoots) baseStacks := baseSpans + uint32(work.nSpanRoots) baseRescan := baseStacks + uint32(work.nStackRoots) end := baseRescan + uint32(work.nRescanRoots) // Note: if you add a case here, please also update heapdump.go:dumproots. switch { case baseFlushCache <= i && i < baseData: flushmcache(int(i - baseFlushCache)) case baseData <= i && i < baseBSS: for _, datap := range activeModules() { markrootBlock(datap.data, datap.edata-datap.data, datap.gcdatamask.bytedata, gcw, int(i-baseData)) } case baseBSS <= i && i < baseSpans: for _, datap := range activeModules() { markrootBlock(datap.bss, datap.ebss-datap.bss, datap.gcbssmask.bytedata, gcw, int(i-baseBSS)) } case i == fixedRootFinalizers: for fb := allfin; fb != nil; fb = fb.alllink { cnt := uintptr(atomic.Load(&fb.cnt)) scanblock(uintptr(unsafe.Pointer(&fb.fin[0])), cnt*unsafe.Sizeof(fb.fin[0]), &finptrmask[0], gcw) } case i == fixedRootFreeGStacks: // Only do this once per GC cycle; preferably // concurrently. if !work.markrootDone { // Switch to the system stack so we can call // stackfree. systemstack(markrootFreeGStacks) } case baseSpans <= i && i < baseStacks: // mark MSpan.specials markrootSpans(gcw, int(i-baseSpans)) default: // the rest is scanning goroutine stacks var gp *g if baseStacks <= i && i < baseRescan { gp = allgs[i-baseStacks] } else if baseRescan <= i && i < end { gp = work.rescan.list[i-baseRescan].ptr() if gp.gcRescan != int32(i-baseRescan) { // Looking for issue #17099. println("runtime: gp", gp, "found at rescan index", i-baseRescan, "but should be at", gp.gcRescan) throw("bad g rescan index") } } else { throw("markroot: bad index") } // remember when we've first observed the G blocked // needed only to output in traceback status := readgstatus(gp) // We are not in a scan state if (status == _Gwaiting || status == _Gsyscall) && gp.waitsince == 0 { gp.waitsince = work.tstart } // scang must be done on the system stack in case // we're trying to scan our own stack. systemstack(func() { // If this is a self-scan, put the user G in // _Gwaiting to prevent self-deadlock. It may // already be in _Gwaiting if this is a mark // worker or we're in mark termination. userG := getg().m.curg selfScan := gp == userG && readgstatus(userG) == _Grunning if selfScan { casgstatus(userG, _Grunning, _Gwaiting) userG.waitreason = "garbage collection scan" } // TODO: scang blocks until gp's stack has // been scanned, which may take a while for // running goroutines. Consider doing this in // two phases where the first is non-blocking: // we scan the stacks we can and ask running // goroutines to scan themselves; and the // second blocks. scang(gp, gcw) if selfScan { casgstatus(userG, _Gwaiting, _Grunning) } }) } }
// `chanrecv` receives on channel c and writes the received data to `ep`. It is almost the same as [`chansend`](#chansend) but in reverse. // // - `ep` is the pointer where to store the received data, it may be nil, in which case received data is ignored. A non-nil `ep` must point to the heap or the caller's stack. // - `block` comes from the select statemenet, should the receiver block or not? If not then the goroutine will not sleep but return if it could not complete. // If block == false and no elements are available, returns (false, false). // Otherwise, if c is closed, zeros *ep and returns (true, false). // Otherwise, fills in *ep with an element and returns (true, true). func chanrecv(t *chantype, c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) { if debugChan { print("chanrecv: chan=", c, "\n") } //<a name="axiom3"/> // **AXIOM #3** - A receive from a nil channel blocks forever if c == nil { if !block { return } gopark(nil, nil, "chan receive (nil chan)", traceEvGoStop, 2) throw("unreachable") } //*Original Comments:* // // *Fast path: check for failed non-blocking operation without acquiring the lock.* // // *After observing that the channel is not ready for receiving, we observe that the // channel is not closed. Each of these observations is a single word-sized read // (first c.sendq.first or c.qcount, and second c.closed). // Because a channel cannot be reopened, the later observation of the channel // being not closed implies that it was also not closed at the moment of the // first observation. We behave as if we observed the channel at that moment // and report that the receive cannot proceed.* // // *The order of operations is important here: reversing the operations can lead to // incorrect behavior when racing with a close.* // // Fast lock-free check whether there is a chance to receive. // // If not blocking and the channel is not closed and one of the following conditions are met bail out and return false, we can’t receive at the moment. // // 1. buffer has zero size and there is no sender (unbuffered channel) // 2. buffer has size and is empty (empty buffered channel) if !block && (c.dataqsiz == 0 && c.sendq.first == nil || c.dataqsiz > 0 && atomic.Loaduint(&c.qcount) == 0) && atomic.Load(&c.closed) == 0 { return } // Acquire the lock so we are thread safe from now on. var t0 int64 if blockprofilerate > 0 { t0 = cputicks() } lock(&c.lock) //<a name="axiom4"/> //**AXIOM #4** - A receive from a closed channel returns the zero value immediately if c.closed != 0 && c.qcount == 0 { if raceenabled { raceacquire(unsafe.Pointer(c)) } unlock(&c.lock) if ep != nil { // Zero the destination memory and return. memclr(ep, uintptr(c.elemsize)) } return true, false } if sg := c.sendq.dequeue(); sg != nil { // Found a waiting sender. If buffer is size 0, receive value // directly from sender. Otherwise, receive from head of queue // and add the sender's value to the tail of the queue (both map to // the same buffer slot because the queue is full).[`recv`](#recv) recv(c, sg, ep, func() { unlock(&c.lock) }) return true, true } // Receive directly from the ring buffer if c.qcount > 0 { qp := chanbuf(c, c.recvx) if raceenabled { raceacquire(qp) racerelease(qp) } if ep != nil { typedmemmove(c.elemtype, ep, qp) } // Zero the memory in the ring buffer on the recvx slot. memclr(qp, uintptr(c.elemsize)) // Because it is a ring buffer we wrap the `recvx` if it is // pointing beyond the buffer. c.recvx++ if c.recvx == c.dataqsiz { c.recvx = 0 } c.qcount-- // Unlock and return true as the value was received. unlock(&c.lock) return true, true } // We are at the point where we couldn't receive any value the // ring buffer is empty and there is no sender, so if it should // not block bail out now. if !block { unlock(&c.lock) return false, false } // No sender available: block on this channel.Create new `sudog` // struct for current receiver and channel and enqueue it to the // wait list. gp := getg() mysg := acquireSudog() mysg.releasetime = 0 if t0 != 0 { mysg.releasetime = -1 } mysg.elem = ep mysg.waitlink = nil gp.waiting = mysg mysg.g = gp mysg.selectdone = nil mysg.c = c gp.param = nil c.recvq.enqueue(mysg) // Here the goroutine goes to sleep and scheduler will wake it // once there is a sender on this channel. The lock is released // inside the function. goparkunlock(&c.lock, "chan receive", traceEvGoBlockRecv, 3) //<a name="rcv_wakeup" /> // This comes after the blocking, the goroutine is awaken. if mysg != gp.waiting { throw("G waiting list is corrupted") } gp.waiting = nil if mysg.releasetime > 0 { blockevent(mysg.releasetime-t0, 2) } closed := gp.param == nil gp.param = nil mysg.c = nil releaseSudog(mysg) return true, !closed }
// numBlocks returns the number of blocks in buffer b. numBlocks is // safe to call concurrently with any other operation. Spans that have // been pushed prior to the call to numBlocks are guaranteed to appear // in some block in the range [0, numBlocks()), assuming there are no // intervening pops. Spans that are pushed after the call may also // appear in these blocks. func (b *gcSweepBuf) numBlocks() int { return int((atomic.Load(&b.index) + gcSweepBlockEntries - 1) / gcSweepBlockEntries) }
func cgocallbackg1(ctxt uintptr) { gp := getg() if gp.m.needextram || atomic.Load(&extraMWaiters) > 0 { gp.m.needextram = false systemstack(newextram) } if ctxt != 0 { s := append(gp.cgoCtxt, ctxt) // Now we need to set gp.cgoCtxt = s, but we could get // a SIGPROF signal while manipulating the slice, and // the SIGPROF handler could pick up gp.cgoCtxt while // tracing up the stack. We need to ensure that the // handler always sees a valid slice, so set the // values in an order such that it always does. p := (*slice)(unsafe.Pointer(&gp.cgoCtxt)) atomicstorep(unsafe.Pointer(&p.array), unsafe.Pointer(&s[0])) p.cap = cap(s) p.len = len(s) defer func(gp *g) { // Decrease the length of the slice by one, safely. p := (*slice)(unsafe.Pointer(&gp.cgoCtxt)) p.len-- }(gp) } if gp.m.ncgo == 0 { // The C call to Go came from a thread not currently running // any Go. In the case of -buildmode=c-archive or c-shared, // this call may be coming in before package initialization // is complete. Wait until it is. <-main_init_done } // Add entry to defer stack in case of panic. restore := true defer unwindm(&restore) if raceenabled { raceacquire(unsafe.Pointer(&racecgosync)) } type args struct { fn *funcval arg unsafe.Pointer argsize uintptr } var cb *args // Location of callback arguments depends on stack frame layout // and size of stack frame of cgocallback_gofunc. sp := gp.m.g0.sched.sp switch GOARCH { default: throw("cgocallbackg is unimplemented on arch") case "arm": // On arm, stack frame is two words and there's a saved LR between // SP and the stack frame and between the stack frame and the arguments. cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) case "arm64": // On arm64, stack frame is four words and there's a saved LR between // SP and the stack frame and between the stack frame and the arguments. cb = (*args)(unsafe.Pointer(sp + 5*sys.PtrSize)) case "amd64": // On amd64, stack frame is two words, plus caller PC. if framepointer_enabled { // In this case, there's also saved BP. cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) break } cb = (*args)(unsafe.Pointer(sp + 3*sys.PtrSize)) case "386": // On 386, stack frame is three words, plus caller PC. cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) case "ppc64", "ppc64le", "s390x": // On ppc64 and s390x, the callback arguments are in the arguments area of // cgocallback's stack frame. The stack looks like this: // +--------------------+------------------------------+ // | | ... | // | cgoexp_$fn +------------------------------+ // | | fixed frame area | // +--------------------+------------------------------+ // | | arguments area | // | cgocallback +------------------------------+ <- sp + 2*minFrameSize + 2*ptrSize // | | fixed frame area | // +--------------------+------------------------------+ <- sp + minFrameSize + 2*ptrSize // | | local variables (2 pointers) | // | cgocallback_gofunc +------------------------------+ <- sp + minFrameSize // | | fixed frame area | // +--------------------+------------------------------+ <- sp cb = (*args)(unsafe.Pointer(sp + 2*sys.MinFrameSize + 2*sys.PtrSize)) case "mips64", "mips64le": // On mips64x, stack frame is two words and there's a saved LR between // SP and the stack frame and between the stack frame and the arguments. cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) } // Invoke callback. // NOTE(rsc): passing nil for argtype means that the copying of the // results back into cb.arg happens without any corresponding write barriers. // For cgo, cb.arg points into a C stack frame and therefore doesn't // hold any pointers that the GC can find anyway - the write barrier // would be a no-op. reflectcall(nil, unsafe.Pointer(cb.fn), cb.arg, uint32(cb.argsize), 0) if raceenabled { racereleasemerge(unsafe.Pointer(&racecgosync)) } if msanenabled { // Tell msan that we wrote to the entire argument block. // This tells msan that we set the results. // Since we have already called the function it doesn't // matter that we are writing to the non-result parameters. msanwrite(cb.arg, cb.argsize) } // Do not unwind m->g0->sched.sp. // Our caller, cgocallback, will do that. restore = false }
// gcAssistAlloc performs GC work to make gp's assist debt positive. // gp must be the calling user gorountine. // // This must be called with preemption enabled. //go:nowritebarrier func gcAssistAlloc(gp *g) { // Don't assist in non-preemptible contexts. These are // generally fragile and won't allow the assist to block. if getg() == gp.m.g0 { return } if mp := getg().m; mp.locks > 0 || mp.preemptoff != "" { return } // Compute the amount of scan work we need to do to make the // balance positive. We over-assist to build up credit for // future allocations and amortize the cost of assisting. debtBytes := -gp.gcAssistBytes + gcOverAssistBytes scanWork := int64(gcController.assistWorkPerByte * float64(debtBytes)) retry: // Steal as much credit as we can from the background GC's // scan credit. This is racy and may drop the background // credit below 0 if two mutators steal at the same time. This // will just cause steals to fail until credit is accumulated // again, so in the long run it doesn't really matter, but we // do have to handle the negative credit case. bgScanCredit := atomic.Loadint64(&gcController.bgScanCredit) stolen := int64(0) if bgScanCredit > 0 { if bgScanCredit < scanWork { stolen = bgScanCredit gp.gcAssistBytes += 1 + int64(gcController.assistBytesPerWork*float64(stolen)) } else { stolen = scanWork gp.gcAssistBytes += debtBytes } atomic.Xaddint64(&gcController.bgScanCredit, -stolen) scanWork -= stolen if scanWork == 0 { // We were able to steal all of the credit we // needed. return } } // Perform assist work completed := false systemstack(func() { if atomic.Load(&gcBlackenEnabled) == 0 { // The gcBlackenEnabled check in malloc races with the // store that clears it but an atomic check in every malloc // would be a performance hit. // Instead we recheck it here on the non-preemptable system // stack to determine if we should preform an assist. // GC is done, so ignore any remaining debt. gp.gcAssistBytes = 0 return } // Track time spent in this assist. Since we're on the // system stack, this is non-preemptible, so we can // just measure start and end time. startTime := nanotime() decnwait := atomic.Xadd(&work.nwait, -1) if decnwait == work.nproc { println("runtime: work.nwait =", decnwait, "work.nproc=", work.nproc) throw("nwait > work.nprocs") } // drain own cached work first in the hopes that it // will be more cache friendly. gcw := &getg().m.p.ptr().gcw workDone := gcDrainN(gcw, scanWork) // If we are near the end of the mark phase // dispose of the gcw. if gcBlackenPromptly { gcw.dispose() } // Record that we did this much scan work. // // Back out the number of bytes of assist credit that // this scan work counts for. The "1+" is a poor man's // round-up, to ensure this adds credit even if // assistBytesPerWork is very low. gp.gcAssistBytes += 1 + int64(gcController.assistBytesPerWork*float64(workDone)) // If this is the last worker and we ran out of work, // signal a completion point. incnwait := atomic.Xadd(&work.nwait, +1) if incnwait > work.nproc { println("runtime: work.nwait=", incnwait, "work.nproc=", work.nproc, "gcBlackenPromptly=", gcBlackenPromptly) throw("work.nwait > work.nproc") } if incnwait == work.nproc && !gcMarkWorkAvailable(nil) { // This has reached a background completion // point. completed = true } duration := nanotime() - startTime _p_ := gp.m.p.ptr() _p_.gcAssistTime += duration if _p_.gcAssistTime > gcAssistTimeSlack { atomic.Xaddint64(&gcController.assistTime, _p_.gcAssistTime) _p_.gcAssistTime = 0 } }) if completed { gcMarkDone() } if gp.gcAssistBytes < 0 { // We were unable steal enough credit or perform // enough work to pay off the assist debt. We need to // do one of these before letting the mutator allocate // more to prevent over-allocation. // // If this is because we were preempted, reschedule // and try some more. if gp.preempt { Gosched() goto retry } // Add this G to an assist queue and park. When the GC // has more background credit, it will satisfy queued // assists before flushing to the global credit pool. // // Note that this does *not* get woken up when more // work is added to the work list. The theory is that // there wasn't enough work to do anyway, so we might // as well let background marking take care of the // work that is available. lock(&work.assistQueue.lock) // If the GC cycle is over, just return. This is the // likely path if we completed above. We do this // under the lock to prevent a GC cycle from ending // between this check and queuing the assist. if atomic.Load(&gcBlackenEnabled) == 0 { unlock(&work.assistQueue.lock) return } oldHead, oldTail := work.assistQueue.head, work.assistQueue.tail if oldHead == 0 { work.assistQueue.head.set(gp) } else { oldTail.ptr().schedlink.set(gp) } work.assistQueue.tail.set(gp) gp.schedlink.set(nil) // Recheck for background credit now that this G is in // the queue, but can still back out. This avoids a // race in case background marking has flushed more // credit since we checked above. if atomic.Loadint64(&gcController.bgScanCredit) > 0 { work.assistQueue.head = oldHead work.assistQueue.tail = oldTail if oldTail != 0 { oldTail.ptr().schedlink.set(nil) } unlock(&work.assistQueue.lock) goto retry } // Park for real. goparkunlock(&work.assistQueue.lock, "GC assist wait", traceEvGoBlock, 2) // At this point either background GC has satisfied // this G's assist debt, or the GC cycle is over. } }
// chanrecv receives on channel c and writes the received data to ep. // ep may be nil, in which case received data is ignored. // If block == false and no elements are available, returns (false, false). // Otherwise, if c is closed, zeros *ep and returns (true, false). // Otherwise, fills in *ep with an element and returns (true, true). // A non-nil ep must point to the heap or the caller's stack. func chanrecv(t *chantype, c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) { // raceenabled: don't need to check ep, as it is always on the stack // or is new memory allocated by reflect. if debugChan { print("chanrecv: chan=", c, "\n") } if c == nil { if !block { return } gopark(nil, nil, "chan receive (nil chan)", traceEvGoStop, 2) throw("unreachable") } // Fast path: check for failed non-blocking operation without acquiring the lock. // // After observing that the channel is not ready for receiving, we observe that the // channel is not closed. Each of these observations is a single word-sized read // (first c.sendq.first or c.qcount, and second c.closed). // Because a channel cannot be reopened, the later observation of the channel // being not closed implies that it was also not closed at the moment of the // first observation. We behave as if we observed the channel at that moment // and report that the receive cannot proceed. // // The order of operations is important here: reversing the operations can lead to // incorrect behavior when racing with a close. if !block && (c.dataqsiz == 0 && c.sendq.first == nil || c.dataqsiz > 0 && atomic.Loaduint(&c.qcount) == 0) && atomic.Load(&c.closed) == 0 { return } var t0 int64 if blockprofilerate > 0 { t0 = cputicks() } lock(&c.lock) if c.closed != 0 && c.qcount == 0 { if raceenabled { raceacquire(unsafe.Pointer(c)) } unlock(&c.lock) if ep != nil { memclr(ep, uintptr(c.elemsize)) } return true, false } if sg := c.sendq.dequeue(); sg != nil { // Found a waiting sender. If buffer is size 0, receive value // directly from sender. Otherwise, receive from head of queue // and add sender's value to the tail of the queue (both map to // the same buffer slot because the queue is full). recv(c, sg, ep, func() { unlock(&c.lock) }) return true, true } if c.qcount > 0 { // Receive directly from queue qp := chanbuf(c, c.recvx) if raceenabled { raceacquire(qp) racerelease(qp) } if ep != nil { typedmemmove(c.elemtype, ep, qp) } memclr(qp, uintptr(c.elemsize)) c.recvx++ if c.recvx == c.dataqsiz { c.recvx = 0 } c.qcount-- unlock(&c.lock) return true, true } if !block { unlock(&c.lock) return false, false } // no sender available: block on this channel. gp := getg() mysg := acquireSudog() mysg.releasetime = 0 if t0 != 0 { mysg.releasetime = -1 } // No stack splits between assigning elem and enqueuing mysg // on gp.waiting where copystack can find it. mysg.elem = ep mysg.waitlink = nil gp.waiting = mysg mysg.g = gp mysg.selectdone = nil mysg.c = c gp.param = nil c.recvq.enqueue(mysg) goparkunlock(&c.lock, "chan receive", traceEvGoBlockRecv, 3) // someone woke us up if mysg != gp.waiting { throw("G waiting list is corrupted") } gp.waiting = nil if mysg.releasetime > 0 { blockevent(mysg.releasetime-t0, 2) } closed := gp.param == nil gp.param = nil mysg.c = nil releaseSudog(mysg) return true, !closed }
//go:nosplit func readgstatus(gp *g) uint32 { return atomic.Load(&gp.atomicstatus) }
func netpollinited() bool { return atomic.Load(&netpollInited) != 0 }