func (sc *ServerCodec) ReadRequestBody(v interface{}) (err error) { log.Println(log.TRACE, "RPC Server Entered: ReadRequestBody") defer log.Println(log.TRACE, "RPC Server Leaving: ReadRequestBody") err = sc.Decoder.Decode(v) if err != nil { log.Println(log.ERROR, "RPC Server Error decoding request body: ", err) } if err == nil { log.Println(log.TRACE, pretty.Sprintf("RPC Server Read RequestBody %s %+v", reflect.TypeOf(v), v)) } return }
func (s *SkynetDaemon) StopAllSubServices(requestInfo *skynet.RequestInfo, in daemon.StopAllSubServicesRequest, out *daemon.StopAllSubServicesResponse) (err error) { var uuids []string s.serviceLock.Lock() for uuid := range s.Services { uuids = append(uuids, uuid) } s.serviceLock.Unlock() out.Stops = make([]daemon.StopSubServiceResponse, len(uuids)) for i, uuid := range uuids { log.Println(log.TRACE, "Stopping "+uuid) err = s.StopSubService(requestInfo, daemon.StopSubServiceRequest{UUID: uuid}, &out.Stops[i]) if err != nil { log.Println(log.ERROR, "Failed to stop subservice "+uuid, err) return } if out.Stops[i].Ok { out.Count++ } } s.saveState() return }
// TODO: This should be moved out so that it's run asynchronously // it should also use a buffered channel so that if a save is already queued it only saves once func (s *SkynetDaemon) writeStateFile() (err error) { err = s.stateFile.Truncate(0) if err != nil { return } _, err = s.stateFile.Seek(0, 0) if err != nil { return } var b []byte b, err = json.MarshalIndent(s.Services, "", "\t") if err != nil { log.Println(log.ERROR, "Failed to marshall daemon state") return } _, err = s.stateFile.Write(b) if err != nil { log.Println(log.ERROR, "Failed to save daemon state") } return }
func (c *ServiceClient) send(retry, giveup time.Duration, ri *skynet.RequestInfo, fn string, in interface{}, out interface{}) (err error) { if ri == nil { ri = c.NewRequestInfo() } attempts := make(chan sendAttempt) var retryTicker <-chan time.Time retryChan := make(chan bool, 1) if retry > 0 { retryTicker = time.Tick(retry) } var timeoutTimer <-chan time.Time if giveup > 0 { timeoutTimer = time.NewTimer(giveup).C } attemptCount := 1 go c.attemptSend(retry, attempts, ri, fn, in, out) for { select { case <-retryTicker: retryChan <- true case <-retryChan: attemptCount++ ri.RetryCount++ log.Println(log.TRACE, fmt.Sprintf("Sending Attempt# %d with RequestInfo %+v", attemptCount, ri)) go c.attemptSend(retry, attempts, ri, fn, in, out) case <-timeoutTimer: err = RequestTimeout log.Println(log.WARN, fmt.Sprintf("Timing out request after %d attempts within %s ", attemptCount, giveup.String())) return case attempt := <-attempts: if attempt.err != nil { log.Println(log.ERROR, "Attempt Error: ", attempt.err) // If there is no retry timer we need to exit as retries were disabled if retryTicker == nil { return err } else { // Don't wait for next retry tick retry now retryChan <- true } continue } // copy into the caller's value v := reflect.Indirect(reflect.ValueOf(out)) v.Set(reflect.Indirect(reflect.ValueOf(attempt.result))) return } } }
func (pc *PathCache) Start() (notifyChan chan PathCacheNotification, err error) { pc.notifyChan = make(chan PathCacheNotification, 10) pc.stoppedChan = make(chan bool, 1) pc.startup = sync.WaitGroup{} go pc.mux() err = pc.watch() if err != nil { // We expect this to occasionally happen due to timing if err != zk.ErrNoNode { log.Println(log.ERROR, err) } pc.Stop() return } err = pc.watchChildren() if err != nil { // We expect this to occasionally happen due to timing if err != zk.ErrNoNode { log.Println(log.ERROR, err) } pc.Stop() } pc.startup.Wait() return pc.notifyChan, err }
func init() { flagset := flag.NewFlagSet("config", flag.ContinueOnError) flagset.StringVar(&configFile, "config", "", "Config File") flagset.StringVar(&uuid, "uuid", "", "uuid") args, _ := SplitFlagsetFromArgs(flagset, os.Args[1:]) flagset.Parse(args) // Ensure we have a UUID if uuid == "" { uuid = NewUUID() } if configFile == "" { for _, f := range defaultConfigFiles { if _, err := os.Stat(f); err == nil { configFile = f break } } } if configFile == "" { log.Println(log.ERROR, "Failed to find config file") conf = config.NewDefault() return } if _, err := os.Stat(configFile); os.IsNotExist(err) { log.Println(log.ERROR, "Config file does not exist", err) conf = config.NewDefault() return } var err error if conf, err = config.ReadDefault(configFile); err != nil { conf = config.NewDefault() log.Fatal(err) } // Set default log level from config, this can be overriden at the service level when the service is created if l, err := conf.RawStringDefault("log.level"); err == nil { log.SetLogLevel(log.LevelFromString(l)) } // Set default log level from config, this can be overriden at the service level when the service is created if lh, err := conf.RawStringDefault("log.sysloghost"); err == nil { log.SetSyslogHost(lh) } // Set syslog port if i, err := conf.Int("DEFAULT", "log.syslogport"); err == nil { log.SetSyslogPort(i) } // Set GOMAXPROCS if i, err := conf.Int("DEFAULT", "runtime.gomaxprocs"); err == nil { runtime.GOMAXPROCS(i) } }
func (sc *ServerCodec) Close() (err error) { log.Println(log.TRACE, "RPC Server Entered: Close") defer log.Println(log.TRACE, "RPC Server Leaving: Close") err = sc.conn.Close() if err != nil && err.Error() != "use of closed network connection" { log.Println(log.ERROR, "RPC Server Error closing connection: ", err) return } return }
func getGiveupTimeout(service, version string) time.Duration { if d, err := config.String(service, version, "client.timeout.total"); err == nil { if timeout, err := time.ParseDuration(d); err == nil { log.Println(log.TRACE, fmt.Sprintf("Using custom giveup duration %q for %q %q", timeout.String(), service, version)) return timeout } log.Println(log.ERROR, "Failed to parse client.timeout.total", err) } return config.DefaultTimeoutDuration }
func (cc *ClientCodec) Close() (err error) { log.Println(log.TRACE, "RPC Client Entered: Close") defer log.Println(log.TRACE, "RPC Client Leaving: Close") err = cc.conn.Close() if err != nil && err.Error() != "use of closed network connection" { log.Println(log.ERROR, "RPC Client Error closing connection: ", err) } return }
func (sd *SkynetDaemon) Started(s *service.Service) { err := sd.cleanupHost(s.ServiceInfo.UUID) if err != nil { log.Println(log.ERROR, "Error cleaning up host", err) } err = sd.restoreState() if err != nil { log.Println(log.ERROR, "Error restoring state", err) } }
func (sc *ServerCodec) ReadRequestHeader(rq *rpc.Request) (err error) { log.Println(log.TRACE, "RPC Server Entered: ReadRequestHeader") defer log.Println(log.TRACE, "RPC Server Leaving: ReadRequestHeader") err = sc.Decoder.Decode(rq) if err != nil && err != io.EOF { log.Println(log.ERROR, "RPC Server Error decoding request header: ", err) sc.Close() } if err == nil { log.Println(log.TRACE, pretty.Sprintf("RPC Server Read RequestHeader %s %+v", reflect.TypeOf(rq), rq)) } return }
func (pc *PathCache) watchChildren() error { if pc.depth == 0 { return nil } children, _, ev, err := pc.serviceManager.conn.ChildrenW(pc.path) if err != nil { if err != zk.ErrNoNode { log.Println(log.ERROR, err) } return err } go forwardZkEvents(ev, pc.events) pc.startup.Add(len(children)) for _, c := range children { if _, ok := pc.children[path.Join(pc.path, c)]; !ok { go func(c string) { pc.addChildChan <- c }(c) } } return nil }
func (sm *ZookeeperServiceManager) Unregister(uuid string) (err error) { log.Println(log.TRACE, "Unregister service", uuid) _, err = sm.conn.Set(path.Join("/instances", uuid, "registered"), []byte("false"), -1) return }
func (sm *ZookeeperServiceManager) Remove(s skynet.ServiceInfo) (err error) { log.Println(log.TRACE, "Removing service", s.UUID) ops := zk.MultiOps{ Delete: []zk.DeleteRequest{ deleteRequest(path.Join("/regions", s.Region, s.UUID), -1), deleteRequest(path.Join("/services", s.Name, s.Version, s.UUID), -1), deleteRequest(path.Join("/services", s.Name, s.UUID), -1), deleteRequest(path.Join("/hosts", s.ServiceAddr.IPAddress, s.UUID), -1), deleteRequest(path.Join("/instances", s.UUID, "registered"), -1), deleteRequest(path.Join("/instances", s.UUID, "name"), -1), deleteRequest(path.Join("/instances", s.UUID, "version"), -1), deleteRequest(path.Join("/instances", s.UUID, "region"), -1), deleteRequest(path.Join("/instances", s.UUID, "addr"), -1), deleteRequest(path.Join("/instances", s.UUID), -1), }, } err = sm.conn.Multi(ops) if err == nil { delete(sm.managedInstances, s.UUID) } // Attempt to remove parent paths for service if they are empty sm.removePathIfEmpty(path.Join("/hosts", s.ServiceAddr.IPAddress)) sm.removePathIfEmpty(path.Join("/regions", s.Region)) sm.removePathIfEmpty(path.Join("/services", s.Name, s.Version)) sm.removePathIfEmpty(path.Join("/services", s.Name)) return }
func (sm *ZookeeperServiceManager) mux() { for { select { case e := <-sm.session: switch e.Type { case zk.EventNodeDeleted, zk.EventNodeChildrenChanged, zk.EventNodeDataChanged: case zk.EventSession: // TODO: EventNotWatching // TODO: StateDisconnected default: log.Println(log.TRACE, "Zookeeper Event Received: ", e) } case <-sm.done: // Remove instances that were added by this instance for _, s := range sm.managedInstances { sm.Remove(s) } sm.cache.Stop() sm.conn.Close() return } } }
// Wait for existing requests to complete and shutdown service func (s *Service) shutdown() { if s.shuttingDown { return } s.shuttingDown = true s.doneGroup.Add(1) s.rpcListener.Close() s.doneChan <- true s.activeRequests.Wait() err := skynet.GetServiceManager().Remove(*s.ServiceInfo) if err != nil { log.Println(log.ERROR, "Failed to remove service: "+err.Error()) } skynet.GetServiceManager().Shutdown() s.Delegate.Stopped(s) // Call user defined callback s.doneGroup.Done() }
func (s *Service) listen(addr skynet.BindAddr, bindWait *sync.WaitGroup) { var err error s.rpcListener, err = addr.Listen() if err != nil { log.Fatal(err) } log.Printf(log.INFO, "%+v\n", ServiceListening{ Addr: &addr, ServiceInfo: s.ServiceInfo, }) // We may have changed port due to conflict, ensure config has the correct port now a, _ := skynet.BindAddrFromString(addr.String()) s.ServiceAddr.IPAddress = a.IPAddress s.ServiceAddr.Port = a.Port bindWait.Done() for { conn, err := s.rpcListener.AcceptTCP() if s.shuttingDown { break } if err != nil && !s.shuttingDown { log.Println(log.ERROR, "AcceptTCP failed", err) continue } s.connectionChan <- conn } }
func (sm *ZookeeperServiceManager) Add(s skynet.ServiceInfo) (err error) { log.Println(log.TRACE, "Adding service to cluster", s.UUID) err = sm.createPathsForService(s) if err != nil { return err } ops := zk.MultiOps{ Create: []zk.CreateRequest{ createRequest(path.Join("/regions", s.Region, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral), createRequest(path.Join("/services", s.Name, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral), createRequest(path.Join("/services", s.Name, s.Version, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral), createRequest(path.Join("/hosts", s.ServiceAddr.IPAddress, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral), createRequest(path.Join("/instances", s.UUID), []byte{}, zk.PermAll, 0), createRequest(path.Join("/instances", s.UUID, "registered"), []byte(strconv.FormatBool(s.Registered)), zk.PermAll, zk.FlagEphemeral), createRequest(path.Join("/instances", s.UUID, "name"), []byte(s.Name), zk.PermAll, zk.FlagEphemeral), createRequest(path.Join("/instances", s.UUID, "version"), []byte(s.Version), zk.PermAll, zk.FlagEphemeral), createRequest(path.Join("/instances", s.UUID, "region"), []byte(s.Region), zk.PermAll, zk.FlagEphemeral), createRequest(path.Join("/instances", s.UUID, "addr"), []byte(s.ServiceAddr.String()), zk.PermAll, zk.FlagEphemeral), }, } err = sm.conn.Multi(ops) sm.managedInstances[s.UUID] = s return }
func (cc *ClientCodec) ReadResponseHeader(res *rpc.Response) (err error) { log.Println(log.TRACE, "RPC Client Entered: ReadResponseHeader") defer log.Println(log.TRACE, "RPC Client Leaving: ReadResponseHeader") err = cc.Decoder.Decode(res) if err != nil { cc.Close() log.Println(log.ERROR, "RPC Client Error decoding response header: ", err) } if err == nil { log.Println(log.TRACE, pretty.Sprintf("RPC Client Read ResponseHeader %s %+v", reflect.TypeOf(res), res)) } return }
func (s *Service) serveAdminRequests() { rId := os.Stderr.Fd() + 2 wId := os.Stderr.Fd() + 3 pipeReader := os.NewFile(uintptr(rId), "") pipeWriter := os.NewFile(uintptr(wId), "") s.pipe = daemon.NewPipe(pipeReader, pipeWriter) b := make([]byte, daemon.MAX_PIPE_BYTES) for { n, err := s.pipe.Read(b) if err != nil { if err != io.EOF { log.Printf(log.ERROR, "Error reading from admin pipe "+err.Error()) } else { // We received EOF, ensure we shutdown (if daemon died we could be orphaned) s.Shutdown() } return } cmd := string(b[:n]) log.Println(log.TRACE, "Received "+cmd+" from daemon") switch cmd { case "SHUTDOWN": s.Shutdown() s.pipe.Write([]byte("ACK")) break case "REGISTER": s.Register() s.pipe.Write([]byte("ACK")) case "UNREGISTER": s.Unregister() s.pipe.Write([]byte("ACK")) case "LOG DEBUG", "LOG TRACE", "LOG INFO", "LOG WARN", "LOG ERROR", "LOG FATAL", "LOG PANIC": parts := strings.Split(cmd, " ") log.SetLogLevel(log.LevelFromString(parts[1])) log.Println(log.INFO, "Setting log level to "+parts[1]) s.pipe.Write([]byte("ACK")) } } }
func (d *Decoder) Decode(pv interface{}) (err error) { var lbuf [4]byte n, err := d.r.Read(lbuf[:]) if n != 4 { err = fmt.Errorf("Corrupted BSON stream: could only read %d", n) return } log.Println(log.TRACE, fmt.Sprintf("Read %d bytes of 4 byte length from connection, received bytes: ", n), lbuf) if err != nil { return } length := (int(lbuf[0]) << 0) | (int(lbuf[1]) << 8) | (int(lbuf[2]) << 16) | (int(lbuf[3]) << 24) log.Println(log.TRACE, "Message length parsed as: ", length) buf := make([]byte, length) copy(buf[0:4], lbuf[:]) n, err = io.ReadFull(d.r, buf[4:]) log.Println(log.TRACE, fmt.Sprintf("Read %d bytes of %d from connection, received bytes: ", n+4, length), buf) if err != nil { return } if n+4 != length { err = fmt.Errorf("Expected %d bytes, read %d", length, n) return } if pv != nil { err = bson.Unmarshal(buf, pv) } return }
func (c *InstanceCache) watch() { for { select { case n, ok := <-c.pathNotify: if !ok { return } uuid := uuidFromPath(n.Path) switch n.Type { case PathCacheAddNotification, PathCacheUpdateNotification: s, err := c.getServiceInfo(uuid) // err means not all paths exist yet if err != nil { continue } if _, ok := c.instances[uuid]; ok { log.Println(log.TRACE, "InstanceCache instance updated:", uuid) c.instances[uuid] = s go c.notify(skynet.InstanceUpdated, s) } else { log.Println(log.TRACE, "InstanceCache instance added:", uuid) c.instances[uuid] = s go c.notify(skynet.InstanceAdded, s) } case PathCacheRemoveNotification: if n.Path == path.Join(InstancesBasePath, uuid) { if s, ok := c.instances[uuid]; ok { log.Println(log.TRACE, "InstanceCache instance removed:", uuid) go c.notify(skynet.InstanceRemoved, s) delete(c.instances, uuid) } } } } } }
func getIdleTimeout(s skynet.ServiceInfo) time.Duration { if d, err := config.String(s.Name, s.Version, "client.timeout.idle"); err == nil { if timeout, err := time.ParseDuration(d); err == nil { return timeout } log.Println(log.ERROR, "Failed to parse client.timeout.idle", err) } return config.DefaultIdleTimeout }
func (cc *ClientCodec) ReadResponseBody(v interface{}) (err error) { log.Println(log.TRACE, "RPC Client Entered: ReadResponseBody") defer log.Println(log.TRACE, "RPC Client Leaving: ReadResponseBody") if v == nil { err = errors.New("Response object cannot be nil") log.Println(log.ERROR, "RPC Client Error reading response body: ", err) return } err = cc.Decoder.Decode(v) if err != nil { cc.Close() log.Println(log.ERROR, "RPC Client Error decoding response body: ", err) } if err == nil { log.Println(log.TRACE, pretty.Sprintf("RPC Client Read ResponseBody %s %+v", reflect.TypeOf(v), v)) } return }
func (c *InstanceCache) buildInitialCache() { for _, p := range c.cache.Children() { uuid := uuidFromPath(p) s, err := c.getServiceInfo(uuid) if err != nil { log.Println(log.WARN, err) continue } c.instances[uuid] = s } }
func (ss *SubService) sendAdminCommand(cmd string) bool { log.Println(log.TRACE, "Writing to admin pipe: "+cmd) _, err := ss.pipe.Write([]byte(cmd)) if err != nil { log.Println(log.ERROR, "Failed to write to admin pipe", err) return false } b := make([]byte, daemon.MAX_PIPE_BYTES) log.Println(log.TRACE, "Reading from admin pipe") n, err := ss.pipe.Read(b) if err != nil && err != io.EOF { log.Println(log.ERROR, "Failed to read from admin pipe", err) return false } if bytes.Equal(b[:n], []byte("ACK")) { return true } return false }
func (s *Service) unregister() { // this version must be run from the mux() goroutine if !s.Registered { return } err := skynet.GetServiceManager().Unregister(s.UUID) if err != nil { log.Println(log.ERROR, "Failed to unregister service: "+err.Error()) } s.Registered = false log.Printf(log.INFO, "%+v\n", ServiceUnregistered{s.ServiceInfo}) s.Delegate.Unregistered(s) // Call user defined callback }
func worker(requestChan chan string, waitGroup *sync.WaitGroup) { for { select { case service, ok := <-requestChan: if !ok { return } waitGroup.Add(1) totalRequests.Add(1) switch service { case "simple": randString := strconv.FormatUint(uint64(rand.Uint32()), 35) randString = randString + randString + randString in := map[string]interface{}{ "data": randString, } fmt.Println("Sending TestService request: " + in["data"].(string)) out := map[string]interface{}{} err := simpleClient.Send(nil, "Upcase", in, &out) upper := strings.ToUpper(randString) if err == nil && out["data"].(string) == upper { successfulRequests.Add(1) fmt.Println("TestService returned: " + out["data"].(string)) } else { failedRequests.Add(1) if err != nil { log.Println(log.ERROR, err) } } waitGroup.Done() } } } }
// Starts your skynet service, including binding to ports. Optionally register for requests at the same time. Returns a sync.WaitGroup that will block until all requests have finished func (s *Service) Start() (done *sync.WaitGroup) { bindWait := &sync.WaitGroup{} bindWait.Add(1) go s.listen(s.ServiceAddr, bindWait) // Watch signals for shutdown c := make(chan os.Signal, 1) go watchSignals(c, s) s.doneChan = make(chan bool, 1) // We must block here, we don't want to register, until we've actually bound to an ip:port bindWait.Wait() s.doneGroup = &sync.WaitGroup{} s.doneGroup.Add(1) go func() { s.mux() s.doneGroup.Done() }() done = s.doneGroup if r, err := config.Bool(s.Name, s.Version, "service.register"); err == nil { s.Registered = r } err := skynet.GetServiceManager().Add(*s.ServiceInfo) if err != nil { log.Println(log.ERROR, "Failed to add service: "+err.Error()) } if s.Registered { s.Register() } go s.Delegate.Started(s) // Call user defined callback if s.ServiceInfo.Registered { go s.Delegate.Registered(s) // Call user defined callback } return }
func (sm *ZookeeperServiceManager) Update(s skynet.ServiceInfo) (err error) { log.Println(log.TRACE, "Updating service", s.UUID) ops := zk.MultiOps{ SetData: []zk.SetDataRequest{ setDataRequest(path.Join("/instances", s.UUID, "registered"), []byte(strconv.FormatBool(s.Registered)), -1), setDataRequest(path.Join("/instances", s.UUID, "name"), []byte(s.Name), -1), setDataRequest(path.Join("/instances", s.UUID, "version"), []byte(s.Version), -1), setDataRequest(path.Join("/instances", s.UUID, "region"), []byte(s.Region), -1), setDataRequest(path.Join("/instances", s.UUID, "addr"), []byte(s.ServiceAddr.String()), -1), }, } err = sm.conn.Multi(ops) sm.managedInstances[s.UUID] = s return }