func (s *Server) onConn(c net.Conn) { conn := s.newClientConn(c) //新建一个conn defer func() { err := recover() if err != nil { const size = 4096 buf := make([]byte, size) buf = buf[:runtime.Stack(buf, false)] //获得当前goroutine的stacktrace golog.Error("server", "onConn", "error", 0, "remoteAddr", c.RemoteAddr().String(), "stack", string(buf), ) } conn.Close() }() if allowConnect := conn.IsAllowConnect(); allowConnect == false { err := mysql.NewError(mysql.ER_ACCESS_DENIED_ERROR, "ip address access denied by kingshard.") conn.writeError(err) conn.Close() return } if err := conn.Handshake(); err != nil { golog.Error("server", "onConn", err.Error(), 0) c.Close() return } conn.Run() }
// helper function to validate index inputs func (a *Arrayb) valIdx(index []int, mthd string) (idx int) { if a.HasErr() { return 0 } if len(index) > len(a.shape) { a.err = InvIndexError if debug { a.debug = fmt.Sprintf("Incorrect number of indicies received by %s(). Shape: %v Index: %v", mthd, a.shape, index) a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)]) } return 0 } for i, v := range index { if v >= a.shape[i] || v < 0 { a.err = IndexError if debug { a.debug = fmt.Sprintf("Index received by %s() does not exist shape: %v index: %v", mthd, a.shape, index) a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)]) } return 0 } idx += v * a.strides[i+1] } return }
func _CatchPanic(err *error, functionName string) { if r := recover(); r != nil { fmt.Printf("%s : PANIC Defered : %v\n", functionName, r) // Capture the stack trace buf := make([]byte, 10000) runtime.Stack(buf, false) fmt.Printf("%s : Stack Trace : %s", functionName, string(buf)) if err != nil { *err = errors.New(fmt.Sprintf("%v", r)) } } else if err != nil && *err != nil { fmt.Printf("%s : ERROR : %v\n", functionName, *err) // Capture the stack trace buf := make([]byte, 10000) runtime.Stack(buf, false) fmt.Printf("%s : Stack Trace : %s", functionName, string(buf)) } }
func (a *Array64) valAxis(axis *[]int, mthd string) bool { axis = cleanAxis(axis) switch { case a.HasErr(): return true case len(*axis) > len(a.shape): a.err = ShapeError if debug { a.debug = fmt.Sprintf("Too many axes received by %s(). Shape: %v Axes: %v", mthd, a.shape, axis) a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)]) } return true } for _, v := range *axis { if v < 0 || v >= len(a.shape) { a.err = IndexError if debug { a.debug = fmt.Sprintf("Axis out of range received by %s(). Shape: %v Axes: %v", mthd, a.shape, axis) a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)]) } return true } } if len(*axis) == len(a.shape) { *axis = (*axis)[:0] } return false }
func (m *RWMutexTracker) RLock() { m.logOnce.Do(m.startLogger) atomic.AddInt32(&m.nwaitr, 1) // Catch read-write-read lock. See if somebody (us? via // another goroutine?) already has a read lock, and then // somebody else is waiting to write, meaning our second read // will deadlock. if atomic.LoadInt32(&m.nhaver) > 0 && atomic.LoadInt32(&m.nwaitw) > 0 { buf := getBuf() buf = buf[:runtime.Stack(buf, false)] log.Printf("Potential R-W-R deadlock at: %s", buf) putBuf(buf) } m.mu.RLock() atomic.AddInt32(&m.nwaitr, -1) atomic.AddInt32(&m.nhaver, 1) gid := GoroutineID() m.hmu.Lock() defer m.hmu.Unlock() if m.holdr == nil { m.holdr = make(map[int64]bool) } if m.holdr[gid] { buf := getBuf() buf = buf[:runtime.Stack(buf, false)] log.Fatalf("Recursive call to RLock: %s", buf) } m.holdr[gid] = true }
func getStack() string { stack := make([]byte, 1000) n := runtime.Stack(stack, false) stack = make([]byte, n) runtime.Stack(stack, false) return string(stack) }
func main() { buf1 := make([]byte, 1000) buf2 := make([]byte, 1000) runtime.Stack(buf1, false) // CALL is last instruction on this line n := runtime.Stack(buf2, false) // CALL is followed by load of result from stack buf1 = buf1[:bytes.IndexByte(buf1, 0)] buf2 = buf2[:n] re := regexp.MustCompile(`(?m)^main\.main\(\)\n.*/issue7690.go:([0-9]+)`) m1 := re.FindStringSubmatch(string(buf1)) if m1 == nil { println("BUG: cannot find main.main in first trace") return } m2 := re.FindStringSubmatch(string(buf2)) if m2 == nil { println("BUG: cannot find main.main in second trace") return } n1, _ := strconv.Atoi(m1[1]) n2, _ := strconv.Atoi(m2[1]) if n1+1 != n2 { println("BUG: expect runtime.Stack on back to back lines, have", n1, n2) println(string(buf1)) println(string(buf2)) } }
func (ls *LState) PCall(nargs, nret int, errfunc *LFunction) (err error) { err = nil sp := ls.stack.Sp() base := ls.reg.Top() - nargs - 1 oldpanic := ls.Panic ls.Panic = panicWithoutTraceback defer func() { ls.Panic = oldpanic rcv := recover() if rcv != nil { if _, ok := rcv.(*ApiError); !ok { err = newApiErrorS(ApiErrorPanic, fmt.Sprint(rcv)) if ls.Options.IncludeGoStackTrace { buf := make([]byte, 4096) runtime.Stack(buf, false) err.(*ApiError).StackTrace = strings.Trim(string(buf), "\000") + "\n" + ls.stackTrace(true) } } else { err = rcv.(*ApiError) } if errfunc != nil { ls.Push(errfunc) ls.Push(err.(*ApiError).Object) ls.Panic = panicWithoutTraceback defer func() { ls.Panic = oldpanic rcv := recover() if rcv != nil { if _, ok := rcv.(*ApiError); !ok { err = newApiErrorS(ApiErrorPanic, fmt.Sprint(rcv)) if ls.Options.IncludeGoStackTrace { buf := make([]byte, 4096) runtime.Stack(buf, false) err.(*ApiError).StackTrace = strings.Trim(string(buf), "\000") + ls.stackTrace(true) } } else { err = rcv.(*ApiError) err.(*ApiError).StackTrace = ls.stackTrace(true) } } }() ls.Call(1, 1) err = newApiError(ApiErrorError, ls.Get(-1)) } else if len(err.(*ApiError).StackTrace) == 0 { err.(*ApiError).StackTrace = ls.stackTrace(true) } ls.reg.SetTop(base) } ls.stack.SetSp(sp) if sp == 0 { ls.currentFrame = nil } }() ls.Call(nargs, nret) return }
func GetStackTraces(buf *[]byte) { *buf = (*buf)[0:cap(*buf)] neededBytes := runtime.Stack(*buf, true) for neededBytes > len(*buf) { *buf = make([]byte, neededBytes) runtime.Stack(*buf, true) } *buf = (*buf)[0:neededBytes] }
// Test that we can open and close a file func TestOpenClose(test *testing.T) { fs, proc := OpenMinixImage(test) file, err := proc.Open("/sample/europarl-en.txt", common.O_RDONLY, 0666) if err != nil { testutils.FatalHere(test, "Failed opening file: %s", err) } found, count := checkFileAndCount(proc, file) if !found { testutils.FatalHere(test, "Did not find open file in proc.files") } if count != 1 { testutils.FatalHere(test, "Open file count incorrect got %d, expected %d", count, 1) } // Now close the file and make sure things are cleaned up err = proc.Close(file) found, count = checkFileAndCount(proc, file) if found { testutils.FatalHere(test, "Found file in process table, should not have") } if count != 0 { testutils.FatalHere(test, "Open file count mismatch got %d, expected %d", count, 0) } // How many goroutines are open right now? numgoros := runtime.NumGoroutine() stacknow := make([]byte, 4096) runtime.Stack(stacknow, true) fs.Exit(proc) err = fs.Shutdown() if err != nil { testutils.FatalHere(test, "Failed when shutting down filesystem: %s", err) } // We expect shutdown to have killed the following goroutines // * device // * block cache // * inode cache // * allocation table // * file server // This test is fragile, so be careful with it! expected := numgoros - 5 if runtime.NumGoroutine() != expected { test.Logf("Original stack:\n%s\n", stacknow) newstack := make([]byte, 4096) runtime.Stack(newstack, true) test.Logf("Current stack:\n%s\n", newstack) testutils.FatalHere(test, "Goroutine count mismatch got %d, expected %d", expected, runtime.NumGoroutine()) } }
// AuditPanic catches panicking executables. This method should be added // in a defer statement as early as possible // AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3 func (log *AuditLogger) AuditPanic() { if err := recover(); err != nil { buf := make([]byte, 8192) log.Audit(fmt.Sprintf("Panic caused by err: %s", err)) runtime.Stack(buf, false) log.Audit(fmt.Sprintf("Stack Trace (Current frame) %s", buf)) runtime.Stack(buf, true) log.Warning(fmt.Sprintf("Stack Trace (All frames): %s", buf)) } }
// valAr needs to be called before func (a *Array64) valRith(b *Array64, mthd string) bool { var flag bool switch { case a.HasErr(): return true case b == nil: a.err = NilError if debug { a.debug = "Array received by " + mthd + "() is a Nil pointer." a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)]) } return true case b.HasErr(): a.err = b.err if debug { a.debug = "Array received by " + mthd + "() is in error." a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)]) } return true case len(a.shape) < len(b.shape): goto shape } for i, j := len(b.shape)-1, len(a.shape)-1; i >= 0; i, j = i-1, j-1 { if a.shape[j] != b.shape[i] { flag = true break } } if !flag { return false } if len(b.shape) != len(a.shape) || b.shape[len(b.shape)-1] != 1 { goto shape } for i := 0; i < len(a.shape)-1; i++ { if a.shape[i] != b.shape[i] { goto shape } } return false shape: a.err = ShapeError if debug { a.debug = fmt.Sprintf("Array received by %s() can not be broadcast. Shape: %v Val shape: %v", mthd, a.shape, b.shape) a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)]) } return true }
// Stacktrace returns the stacktrace of the goroutine that calls it. func Stacktrace() string { // Write a stack trace buf := make([]byte, 10000) n := runtime.Stack(buf, true) // Incrementally grow the // buffer as the stack trace // requires. for n > len(buf) { buf = make([]byte, len(buf)*2) n = runtime.Stack(buf, false) } return string(buf) }
func (ls *LState) PCall(nargs, nret int, errfunc *LFunction) (err error) { err = nil sp := ls.stack.Sp() base := ls.reg.Top() - nargs - 1 oldpanic := ls.Panic ls.Panic = func(L *LState) { panic(newApiError(ApiErrorRun, "", L.Get(-1))) } defer func() { ls.Panic = oldpanic rcv := recover() if rcv != nil { if _, ok := rcv.(*ApiError); !ok { buf := make([]byte, 4096) runtime.Stack(buf, false) err = newApiError(ApiErrorPanic, fmt.Sprintf("%v\n%v", rcv, strings.Trim(string(buf), "\000")), LNil) } else { err = rcv.(*ApiError) } if errfunc != nil { ls.Push(errfunc) ls.Push(err.(*ApiError).Object) ls.Panic = func(L *LState) { panic(newApiError(ApiErrorError, "", L.Get(-1))) } defer func() { ls.Panic = oldpanic rcv := recover() if rcv != nil { if _, ok := rcv.(*ApiError); !ok { buf := make([]byte, 4096) runtime.Stack(buf, false) err = newApiError(ApiErrorPanic, fmt.Sprintf("%v\n%v", rcv, strings.Trim(string(buf), "\000")), LNil) } else { err = rcv.(*ApiError) } } }() ls.Call(1, 1) } ls.reg.SetTop(base) } ls.stack.SetSp(sp) }() ls.Call(nargs, nret) return }
func (sl SysLogger) Panicf(format string, v ...interface{}) { var cnt int sl.Logf(format, v...) stack := make([]byte, 8192) if sl.verbose { cnt = runtime.Stack(stack, true) } else { cnt = runtime.Stack(stack, false) } lines := strings.Split(string(stack[:cnt]), "\n") for _, line := range lines { sl.Log(strings.TrimSpace(line)) } panic(v) }
// takeStacktrace attempts to use the provided byte slice to take a stacktrace. // If the provided slice isn't large enough, takeStacktrace will allocate // successively larger slices until it can capture the whole stack. func takeStacktrace(buf []byte, includeAllGoroutines bool) string { if len(buf) == 0 { // We may have been passed a nil byte slice. buf = make([]byte, 1024) } n := runtime.Stack(buf, includeAllGoroutines) for n >= len(buf) { // Buffer wasn't large enough, allocate a larger one. No need to copy // previous buffer's contents. size := 2 * n buf = make([]byte, size) n = runtime.Stack(buf, includeAllGoroutines) } return string(buf[:n]) }
// SetSubArr sets the array below a given index to the values in vals. // Values will be broadcast up multiple axes if the shapes match. func (a *Arrayb) SetSubArr(vals *Arrayb, index ...int) *Arrayb { idx := a.valIdx(index, "SetSubArr") switch { case a.HasErr(): return a case vals.HasErr(): a.err = vals.getErr() if debug { a.debug = "Array received by SetSubArr() is in error." a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)]) } return a case len(vals.shape)+len(index) > len(a.shape): a.err = InvIndexError if debug { a.debug = fmt.Sprintf("Array received by SetSubArr() cant be broadcast. Shape: %v Vals shape: %v index: %v", a.shape, vals.shape, index) a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)]) } return a } for i, j := len(a.shape)-1, len(vals.shape)-1; j >= 0; i, j = i-1, j-1 { if a.shape[i] != vals.shape[j] { a.err = ShapeError if debug { a.debug = fmt.Sprintf("Shape of array recieved by SetSubArr() doesn't match receiver. Shape: %v Vals Shape: %v", a.shape, vals.shape) a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)]) } return a } } if len(a.shape)-len(index)-len(vals.shape) == 0 { copy(a.data[idx:idx+len(vals.data)], vals.data) return a } reps := 1 for i := len(index); i < len(a.shape)-len(vals.shape); i++ { reps *= a.shape[i] } ln := len(vals.data) for i := 1; i <= reps; i++ { copy(a.data[idx+ln*(i-1):idx+ln*i], vals.data) } return a }
// Regrealloc(n) undoes the effect of Regfree(n), // so that a register can be given up but then reclaimed. func Regrealloc(n *Node) { if n.Op != OREGISTER && n.Op != OINDREG { Fatalf("regrealloc: not a register") } i := int(n.Reg) if i == Thearch.REGSP { return } switch { case Thearch.REGMIN <= i && i <= Thearch.REGMAX, Thearch.FREGMIN <= i && i <= Thearch.FREGMAX: // ok default: Fatalf("regrealloc: reg out of range") } i -= Thearch.REGMIN if reg[i] == 0 && Debug['v'] > 0 { if regstk[i] == nil { regstk[i] = make([]byte, 4096) } stk := regstk[i] n := runtime.Stack(stk[:cap(stk)], false) regstk[i] = stk[:n] } reg[i]++ }
func reportCrash(err error) { if err == nil { return } buf := make([]byte, 10000) runtime.Stack(buf, false) stack := formatStack(buf) switch reportCrashConfig() { case "always": report(err, stack) case "never": printError(err, stack) default: printError(err, stack) fmt.Print("Would you like to open an issue? ([Y]es/[N]o/[A]lways/N[e]ver): ") var confirm string fmt.Scan(&confirm) always := utils.IsOption(confirm, "a", "always") if always || utils.IsOption(confirm, "y", "yes") { report(err, stack) } saveReportConfiguration(confirm, always) } os.Exit(1) }
func (b *Broker) call(req *Request, cl Caller) { defer b.Wg.Done() b.served++ defer func() { err := recover() if err != nil { buf := make([]byte, 64*1024*1024) n := runtime.Stack(buf, true) logger.Printf("%v#%v PANIC: %v\n%s\n\n", req.Method, req.Token, err, buf[:n]) b.Send(Response{ Token: req.Token, Error: "broker: " + req.Method + "#" + req.Token + " PANIC", }) } }() res, err := cl.Call() if res == nil { res = M{} } else if v, ok := res.(M); ok && v == nil { res = M{} } b.Send(Response{ Token: req.Token, Error: err, Data: res, }) }
func run(f StatefulHandler, task metafora.Task, cmd <-chan *Message) (m *Message) { defer func() { if r := recover(); r != nil { stackBuf := make([]byte, 6000) stackBufLen := runtime.Stack(stackBuf, false) stackTraceStr := string(stackBuf[0:stackBufLen]) metafora.Errorf("task=%q Run method panic()d! Applying Error message. Panic: %v\nStack: %s", task.ID(), r, stackTraceStr) m = &Message{Code: Error, Err: fmt.Errorf("panic: %v\nstack: %s", r, stackTraceStr)} } }() // Defensive code to give handlers a *copy* of the command chan. That way if // a handler keeps receiving on the command chan in a goroutine past the // handler's lifetime it doesn't intercept commands intended for the // statemachine. internalcmd := make(chan *Message) stopped := make(chan struct{}) go func() { for { select { case c := <-cmd: internalcmd <- c case <-stopped: return } } }() defer close(stopped) return f(task, internalcmd) }
func (tr *trace) Finish() { tr.Elapsed = time.Now().Sub(tr.Start) if DebugUseAfterFinish { buf := make([]byte, 4<<10) // 4 KB should be enough n := runtime.Stack(buf, false) tr.finishStack = buf[:n] } activeMu.RLock() m := activeTraces[tr.Family] activeMu.RUnlock() m.Remove(tr) f := getFamily(tr.Family, true) for _, b := range f.Buckets { if b.Cond.match(tr) { b.Add(tr) } } // Add a sample of elapsed time as microseconds to the family's timeseries h := new(histogram) h.addMeasurement(tr.Elapsed.Nanoseconds() / 1e3) f.LatencyMu.Lock() f.Latency.Add(h) f.LatencyMu.Unlock() tr.unref() // matches ref in New }
func (ss *StacktraceSource) loadStacktrace() { var lastUpdated time.Time var size int = 512 var i int = 0 var stacktrace []byte for { select { case ret := <-ss.getStacktraceRec: now := time.Now() if now.After(lastUpdated.Add(2 * time.Second)) { for { stacktrace = make([]byte, size) i = runtime.Stack(stacktrace, true) if i == size { size = int(float64(size) * 1.5) continue } size = int(float64(i) * 1.25) break } lastUpdated = now } ret <- StacktraceRecord{stacktrace[:i], lastUpdated} } } }
func interestingGoroutines() (gs []string) { buf := make([]byte, 2<<20) buf = buf[:runtime.Stack(buf, true)] for _, g := range strings.Split(string(buf), "\n\n") { sl := strings.SplitN(g, "\n", 2) if len(sl) != 2 { continue } stack := strings.TrimSpace(sl[1]) if stack == "" || strings.Contains(stack, "created by net.startServer") || strings.Contains(stack, "created by testing.RunTests") || strings.Contains(stack, "closeWriteAndWait") || strings.Contains(stack, "testing.Main(") || // These only show up with GOTRACEBACK=2; Issue 5005 (comment 28) strings.Contains(stack, "runtime.goexit") || strings.Contains(stack, "created by runtime.gc") || strings.Contains(stack, "github.com/cockroachdb/cockroach/util/leaktest.interestingGoroutines") || strings.Contains(stack, "runtime.MHeap_Scavenger") || strings.Contains(stack, "golang/glog.init") { continue } gs = append(gs, stack) } sort.Strings(gs) return }
func (rec *Recovery) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { defer func() { if err := recover(); err != nil { rw.WriteHeader(http.StatusInternalServerError) stack := make([]byte, rec.StackSize) stack = stack[:runtime.Stack(stack, rec.StackAll)] f := "PANIC: %s\n%s" rec.Logger.Printf(f, err, stack) if rec.Raven != nil { er := errors.New(fmt.Sprintf("%v", err)) packet := raven.NewPacket(er.Error(), raven.NewException(er, trace()), raven.NewHttp(r)) eventID, _ := rec.Raven.Capture(packet, nil) rec.Logger.Printf("Event %d sent to sentry", eventID) } if rec.PrintStack { fmt.Fprintf(rw, f, err, stack) } } }() next(rw, r) }
// Stacks handles returns goroutine stack traces. func (s *statusServer) Stacks(ctx context.Context, req *serverpb.StacksRequest) (*serverpb.JSONResponse, error) { nodeID, local, err := s.parseNodeID(req.NodeId) if err != nil { return nil, grpc.Errorf(codes.InvalidArgument, err.Error()) } if !local { status, err := s.dialNode(nodeID) if err != nil { return nil, err } return status.Stacks(ctx, req) } bufSize := runtime.NumGoroutine() * stackTraceApproxSize for { buf := make([]byte, bufSize) length := runtime.Stack(buf, true) // If this wasn't large enough to accommodate the full set of // stack traces, increase by 2 and try again. if length == bufSize { bufSize = bufSize * 2 continue } return &serverpb.JSONResponse{Data: buf[:length]}, nil } }
func (cc *clientConn) Run() { defer func() { r := recover() if r != nil { const size = 4096 buf := make([]byte, size) buf = buf[:runtime.Stack(buf, false)] log.Errorf("lastCmd %s, %v, %s", cc.lastCmd, r, buf) } cc.Close() }() for { cc.alloc.Reset() data, err := cc.readPacket() if err != nil { if terror.ErrorNotEqual(err, io.EOF) { log.Error(err) } return } if err := cc.dispatch(data); err != nil { if terror.ErrorEqual(err, io.EOF) { return } log.Errorf("dispatch error %s, %s", errors.ErrorStack(err), cc) log.Errorf("cmd: %s", string(data[1:])) cc.writeError(err) } cc.pkg.sequence = 0 } }
// Main is not redundant with main(), because it provides an entry point // for testing with arbitrary command line arguments. func Main(args []string) int { defer func() { if r := recover(); r != nil { buf := make([]byte, 4096) buf = buf[:runtime.Stack(buf, false)] logger.Criticalf("Unhandled panic: \n%v\n%s", r, buf) os.Exit(exit_panic) } }() var code int = 1 ctx, err := cmd.DefaultContext() if err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(exit_err) } commandName := filepath.Base(args[0]) if commandName == names.Jujud { code, err = jujuDMain(args, ctx) } else if commandName == names.Jujuc { fmt.Fprint(os.Stderr, jujudDoc) code = exit_err err = fmt.Errorf("jujuc should not be called directly") } else if commandName == names.JujuRun { code = cmd.Main(&RunCommand{}, ctx, args[1:]) } else { code, err = jujuCMain(commandName, ctx, args) } if err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) } return code }
func (w *Weavebox) makeHTTPRouterHandle(h Handler) httprouter.Handle { return func(rw http.ResponseWriter, r *http.Request, params httprouter.Params) { if w.context == nil { w.context = context.Background() } ctx := &Context{ Context: w.context, vars: params, response: rw, request: r, weavebox: w, } defer func() { if err := recover(); err != nil { trace := make([]byte, 256) n := runtime.Stack(trace, true) w.logger.Log("recoverd", err, "stacktrace", string(trace[:n])) w.ErrorHandler(ctx, fmt.Errorf("%v", err)) return } }() for i := len(w.middleware) - 1; i >= 0; i-- { h = w.middleware[i](h) } if err := h(ctx); err != nil { w.ErrorHandler(ctx, err) return } } }
func (n *node) start(snapshot []byte) { n.errCh = make(chan error, 1) go func() { var err error defer func() { // Always close children edges n.closeChildEdges() // Propogate error up if err != nil { // Handle panic in runF r := recover() if r != nil { trace := make([]byte, 512) n := runtime.Stack(trace, false) err = fmt.Errorf("%s: Trace:%s", r, string(trace[:n])) } n.abortParentEdges() n.logger.Println("E!", err) } n.errCh <- err }() // Run node err = n.runF(snapshot) }() }