func (sc *ServerCodec) ReadRequestBody(v interface{}) (err error) {
log.Println(log.TRACE, "RPC Server Entered: ReadRequestBody")
defer log.Println(log.TRACE, "RPC Server Leaving: ReadRequestBody")
err = sc.Decoder.Decode(v)
if err != nil {
log.Println(log.ERROR, "RPC Server Error decoding request body: ", err)
}
if err == nil {
log.Println(log.TRACE, pretty.Sprintf("RPC Server Read RequestBody %s %+v", reflect.TypeOf(v), v))
}
return
}
func (s *SkynetDaemon) StopAllSubServices(requestInfo *skynet.RequestInfo, in daemon.StopAllSubServicesRequest, out *daemon.StopAllSubServicesResponse) (err error) {
var uuids []string
s.serviceLock.Lock()
for uuid := range s.Services {
uuids = append(uuids, uuid)
}
s.serviceLock.Unlock()
out.Stops = make([]daemon.StopSubServiceResponse, len(uuids))
for i, uuid := range uuids {
log.Println(log.TRACE, "Stopping "+uuid)
err = s.StopSubService(requestInfo, daemon.StopSubServiceRequest{UUID: uuid}, &out.Stops[i])
if err != nil {
log.Println(log.ERROR, "Failed to stop subservice "+uuid, err)
return
}
if out.Stops[i].Ok {
out.Count++
}
}
s.saveState()
return
}
// TODO: This should be moved out so that it's run asynchronously
// it should also use a buffered channel so that if a save is already queued it only saves once
func (s *SkynetDaemon) writeStateFile() (err error) {
err = s.stateFile.Truncate(0)
if err != nil {
return
}
_, err = s.stateFile.Seek(0, 0)
if err != nil {
return
}
var b []byte
b, err = json.MarshalIndent(s.Services, "", "\t")
if err != nil {
log.Println(log.ERROR, "Failed to marshall daemon state")
return
}
_, err = s.stateFile.Write(b)
if err != nil {
log.Println(log.ERROR, "Failed to save daemon state")
}
return
}
func (c *ServiceClient) send(retry, giveup time.Duration, ri *skynet.RequestInfo, fn string, in interface{}, out interface{}) (err error) {
if ri == nil {
ri = c.NewRequestInfo()
}
attempts := make(chan sendAttempt)
var retryTicker <-chan time.Time
retryChan := make(chan bool, 1)
if retry > 0 {
retryTicker = time.Tick(retry)
}
var timeoutTimer <-chan time.Time
if giveup > 0 {
timeoutTimer = time.NewTimer(giveup).C
}
attemptCount := 1
go c.attemptSend(retry, attempts, ri, fn, in, out)
for {
select {
case <-retryTicker:
retryChan <- true
case <-retryChan:
attemptCount++
ri.RetryCount++
log.Println(log.TRACE, fmt.Sprintf("Sending Attempt# %d with RequestInfo %+v", attemptCount, ri))
go c.attemptSend(retry, attempts, ri, fn, in, out)
case <-timeoutTimer:
err = RequestTimeout
log.Println(log.WARN, fmt.Sprintf("Timing out request after %d attempts within %s ", attemptCount, giveup.String()))
return
case attempt := <-attempts:
if attempt.err != nil {
log.Println(log.ERROR, "Attempt Error: ", attempt.err)
// If there is no retry timer we need to exit as retries were disabled
if retryTicker == nil {
return err
} else {
// Don't wait for next retry tick retry now
retryChan <- true
}
continue
}
// copy into the caller's value
v := reflect.Indirect(reflect.ValueOf(out))
v.Set(reflect.Indirect(reflect.ValueOf(attempt.result)))
return
}
}
}
func (pc *PathCache) Start() (notifyChan chan PathCacheNotification, err error) {
pc.notifyChan = make(chan PathCacheNotification, 10)
pc.stoppedChan = make(chan bool, 1)
pc.startup = sync.WaitGroup{}
go pc.mux()
err = pc.watch()
if err != nil {
// We expect this to occasionally happen due to timing
if err != zk.ErrNoNode {
log.Println(log.ERROR, err)
}
pc.Stop()
return
}
err = pc.watchChildren()
if err != nil {
// We expect this to occasionally happen due to timing
if err != zk.ErrNoNode {
log.Println(log.ERROR, err)
}
pc.Stop()
}
pc.startup.Wait()
return pc.notifyChan, err
}
func init() {
flagset := flag.NewFlagSet("config", flag.ContinueOnError)
flagset.StringVar(&configFile, "config", "", "Config File")
flagset.StringVar(&uuid, "uuid", "", "uuid")
args, _ := SplitFlagsetFromArgs(flagset, os.Args[1:])
flagset.Parse(args)
// Ensure we have a UUID
if uuid == "" {
uuid = NewUUID()
}
if configFile == "" {
for _, f := range defaultConfigFiles {
if _, err := os.Stat(f); err == nil {
configFile = f
break
}
}
}
if configFile == "" {
log.Println(log.ERROR, "Failed to find config file")
conf = config.NewDefault()
return
}
if _, err := os.Stat(configFile); os.IsNotExist(err) {
log.Println(log.ERROR, "Config file does not exist", err)
conf = config.NewDefault()
return
}
var err error
if conf, err = config.ReadDefault(configFile); err != nil {
conf = config.NewDefault()
log.Fatal(err)
}
// Set default log level from config, this can be overriden at the service level when the service is created
if l, err := conf.RawStringDefault("log.level"); err == nil {
log.SetLogLevel(log.LevelFromString(l))
}
// Set default log level from config, this can be overriden at the service level when the service is created
if lh, err := conf.RawStringDefault("log.sysloghost"); err == nil {
log.SetSyslogHost(lh)
}
// Set syslog port
if i, err := conf.Int("DEFAULT", "log.syslogport"); err == nil {
log.SetSyslogPort(i)
}
// Set GOMAXPROCS
if i, err := conf.Int("DEFAULT", "runtime.gomaxprocs"); err == nil {
runtime.GOMAXPROCS(i)
}
}
func (sc *ServerCodec) Close() (err error) {
log.Println(log.TRACE, "RPC Server Entered: Close")
defer log.Println(log.TRACE, "RPC Server Leaving: Close")
err = sc.conn.Close()
if err != nil && err.Error() != "use of closed network connection" {
log.Println(log.ERROR, "RPC Server Error closing connection: ", err)
return
}
return
}
func getGiveupTimeout(service, version string) time.Duration {
if d, err := config.String(service, version, "client.timeout.total"); err == nil {
if timeout, err := time.ParseDuration(d); err == nil {
log.Println(log.TRACE, fmt.Sprintf("Using custom giveup duration %q for %q %q", timeout.String(), service, version))
return timeout
}
log.Println(log.ERROR, "Failed to parse client.timeout.total", err)
}
return config.DefaultTimeoutDuration
}
func (cc *ClientCodec) Close() (err error) {
log.Println(log.TRACE, "RPC Client Entered: Close")
defer log.Println(log.TRACE, "RPC Client Leaving: Close")
err = cc.conn.Close()
if err != nil && err.Error() != "use of closed network connection" {
log.Println(log.ERROR, "RPC Client Error closing connection: ", err)
}
return
}
func (sd *SkynetDaemon) Started(s *service.Service) {
err := sd.cleanupHost(s.ServiceInfo.UUID)
if err != nil {
log.Println(log.ERROR, "Error cleaning up host", err)
}
err = sd.restoreState()
if err != nil {
log.Println(log.ERROR, "Error restoring state", err)
}
}
func (sc *ServerCodec) ReadRequestHeader(rq *rpc.Request) (err error) {
log.Println(log.TRACE, "RPC Server Entered: ReadRequestHeader")
defer log.Println(log.TRACE, "RPC Server Leaving: ReadRequestHeader")
err = sc.Decoder.Decode(rq)
if err != nil && err != io.EOF {
log.Println(log.ERROR, "RPC Server Error decoding request header: ", err)
sc.Close()
}
if err == nil {
log.Println(log.TRACE, pretty.Sprintf("RPC Server Read RequestHeader %s %+v", reflect.TypeOf(rq), rq))
}
return
}
func (pc *PathCache) watchChildren() error {
if pc.depth == 0 {
return nil
}
children, _, ev, err := pc.serviceManager.conn.ChildrenW(pc.path)
if err != nil {
if err != zk.ErrNoNode {
log.Println(log.ERROR, err)
}
return err
}
go forwardZkEvents(ev, pc.events)
pc.startup.Add(len(children))
for _, c := range children {
if _, ok := pc.children[path.Join(pc.path, c)]; !ok {
go func(c string) {
pc.addChildChan <- c
}(c)
}
}
return nil
}
func (sm *ZookeeperServiceManager) Unregister(uuid string) (err error) {
log.Println(log.TRACE, "Unregister service", uuid)
_, err = sm.conn.Set(path.Join("/instances", uuid, "registered"), []byte("false"), -1)
return
}
func (sm *ZookeeperServiceManager) Remove(s skynet.ServiceInfo) (err error) {
log.Println(log.TRACE, "Removing service", s.UUID)
ops := zk.MultiOps{
Delete: []zk.DeleteRequest{
deleteRequest(path.Join("/regions", s.Region, s.UUID), -1),
deleteRequest(path.Join("/services", s.Name, s.Version, s.UUID), -1),
deleteRequest(path.Join("/services", s.Name, s.UUID), -1),
deleteRequest(path.Join("/hosts", s.ServiceAddr.IPAddress, s.UUID), -1),
deleteRequest(path.Join("/instances", s.UUID, "registered"), -1),
deleteRequest(path.Join("/instances", s.UUID, "name"), -1),
deleteRequest(path.Join("/instances", s.UUID, "version"), -1),
deleteRequest(path.Join("/instances", s.UUID, "region"), -1),
deleteRequest(path.Join("/instances", s.UUID, "addr"), -1),
deleteRequest(path.Join("/instances", s.UUID), -1),
},
}
err = sm.conn.Multi(ops)
if err == nil {
delete(sm.managedInstances, s.UUID)
}
// Attempt to remove parent paths for service if they are empty
sm.removePathIfEmpty(path.Join("/hosts", s.ServiceAddr.IPAddress))
sm.removePathIfEmpty(path.Join("/regions", s.Region))
sm.removePathIfEmpty(path.Join("/services", s.Name, s.Version))
sm.removePathIfEmpty(path.Join("/services", s.Name))
return
}
func (sm *ZookeeperServiceManager) mux() {
for {
select {
case e := <-sm.session:
switch e.Type {
case zk.EventNodeDeleted, zk.EventNodeChildrenChanged, zk.EventNodeDataChanged:
case zk.EventSession:
// TODO: EventNotWatching
// TODO: StateDisconnected
default:
log.Println(log.TRACE, "Zookeeper Event Received: ", e)
}
case <-sm.done:
// Remove instances that were added by this instance
for _, s := range sm.managedInstances {
sm.Remove(s)
}
sm.cache.Stop()
sm.conn.Close()
return
}
}
}
// Wait for existing requests to complete and shutdown service
func (s *Service) shutdown() {
if s.shuttingDown {
return
}
s.shuttingDown = true
s.doneGroup.Add(1)
s.rpcListener.Close()
s.doneChan <- true
s.activeRequests.Wait()
err := skynet.GetServiceManager().Remove(*s.ServiceInfo)
if err != nil {
log.Println(log.ERROR, "Failed to remove service: "+err.Error())
}
skynet.GetServiceManager().Shutdown()
s.Delegate.Stopped(s) // Call user defined callback
s.doneGroup.Done()
}
func (s *Service) listen(addr skynet.BindAddr, bindWait *sync.WaitGroup) {
var err error
s.rpcListener, err = addr.Listen()
if err != nil {
log.Fatal(err)
}
log.Printf(log.INFO, "%+v\n", ServiceListening{
Addr: &addr,
ServiceInfo: s.ServiceInfo,
})
// We may have changed port due to conflict, ensure config has the correct port now
a, _ := skynet.BindAddrFromString(addr.String())
s.ServiceAddr.IPAddress = a.IPAddress
s.ServiceAddr.Port = a.Port
bindWait.Done()
for {
conn, err := s.rpcListener.AcceptTCP()
if s.shuttingDown {
break
}
if err != nil && !s.shuttingDown {
log.Println(log.ERROR, "AcceptTCP failed", err)
continue
}
s.connectionChan <- conn
}
}
func (sm *ZookeeperServiceManager) Add(s skynet.ServiceInfo) (err error) {
log.Println(log.TRACE, "Adding service to cluster", s.UUID)
err = sm.createPathsForService(s)
if err != nil {
return err
}
ops := zk.MultiOps{
Create: []zk.CreateRequest{
createRequest(path.Join("/regions", s.Region, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral),
createRequest(path.Join("/services", s.Name, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral),
createRequest(path.Join("/services", s.Name, s.Version, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral),
createRequest(path.Join("/hosts", s.ServiceAddr.IPAddress, s.UUID), []byte{}, zk.PermAll, zk.FlagEphemeral),
createRequest(path.Join("/instances", s.UUID), []byte{}, zk.PermAll, 0),
createRequest(path.Join("/instances", s.UUID, "registered"), []byte(strconv.FormatBool(s.Registered)), zk.PermAll, zk.FlagEphemeral),
createRequest(path.Join("/instances", s.UUID, "name"), []byte(s.Name), zk.PermAll, zk.FlagEphemeral),
createRequest(path.Join("/instances", s.UUID, "version"), []byte(s.Version), zk.PermAll, zk.FlagEphemeral),
createRequest(path.Join("/instances", s.UUID, "region"), []byte(s.Region), zk.PermAll, zk.FlagEphemeral),
createRequest(path.Join("/instances", s.UUID, "addr"), []byte(s.ServiceAddr.String()), zk.PermAll, zk.FlagEphemeral),
},
}
err = sm.conn.Multi(ops)
sm.managedInstances[s.UUID] = s
return
}
func (cc *ClientCodec) ReadResponseHeader(res *rpc.Response) (err error) {
log.Println(log.TRACE, "RPC Client Entered: ReadResponseHeader")
defer log.Println(log.TRACE, "RPC Client Leaving: ReadResponseHeader")
err = cc.Decoder.Decode(res)
if err != nil {
cc.Close()
log.Println(log.ERROR, "RPC Client Error decoding response header: ", err)
}
if err == nil {
log.Println(log.TRACE, pretty.Sprintf("RPC Client Read ResponseHeader %s %+v", reflect.TypeOf(res), res))
}
return
}
func (s *Service) serveAdminRequests() {
rId := os.Stderr.Fd() + 2
wId := os.Stderr.Fd() + 3
pipeReader := os.NewFile(uintptr(rId), "")
pipeWriter := os.NewFile(uintptr(wId), "")
s.pipe = daemon.NewPipe(pipeReader, pipeWriter)
b := make([]byte, daemon.MAX_PIPE_BYTES)
for {
n, err := s.pipe.Read(b)
if err != nil {
if err != io.EOF {
log.Printf(log.ERROR, "Error reading from admin pipe "+err.Error())
} else {
// We received EOF, ensure we shutdown (if daemon died we could be orphaned)
s.Shutdown()
}
return
}
cmd := string(b[:n])
log.Println(log.TRACE, "Received "+cmd+" from daemon")
switch cmd {
case "SHUTDOWN":
s.Shutdown()
s.pipe.Write([]byte("ACK"))
break
case "REGISTER":
s.Register()
s.pipe.Write([]byte("ACK"))
case "UNREGISTER":
s.Unregister()
s.pipe.Write([]byte("ACK"))
case "LOG DEBUG", "LOG TRACE", "LOG INFO", "LOG WARN", "LOG ERROR", "LOG FATAL", "LOG PANIC":
parts := strings.Split(cmd, " ")
log.SetLogLevel(log.LevelFromString(parts[1]))
log.Println(log.INFO, "Setting log level to "+parts[1])
s.pipe.Write([]byte("ACK"))
}
}
}
func (d *Decoder) Decode(pv interface{}) (err error) {
var lbuf [4]byte
n, err := d.r.Read(lbuf[:])
if n != 4 {
err = fmt.Errorf("Corrupted BSON stream: could only read %d", n)
return
}
log.Println(log.TRACE, fmt.Sprintf("Read %d bytes of 4 byte length from connection, received bytes: ", n), lbuf)
if err != nil {
return
}
length := (int(lbuf[0]) << 0) |
(int(lbuf[1]) << 8) |
(int(lbuf[2]) << 16) |
(int(lbuf[3]) << 24)
log.Println(log.TRACE, "Message length parsed as: ", length)
buf := make([]byte, length)
copy(buf[0:4], lbuf[:])
n, err = io.ReadFull(d.r, buf[4:])
log.Println(log.TRACE, fmt.Sprintf("Read %d bytes of %d from connection, received bytes: ", n+4, length), buf)
if err != nil {
return
}
if n+4 != length {
err = fmt.Errorf("Expected %d bytes, read %d", length, n)
return
}
if pv != nil {
err = bson.Unmarshal(buf, pv)
}
return
}
func (c *InstanceCache) watch() {
for {
select {
case n, ok := <-c.pathNotify:
if !ok {
return
}
uuid := uuidFromPath(n.Path)
switch n.Type {
case PathCacheAddNotification, PathCacheUpdateNotification:
s, err := c.getServiceInfo(uuid)
// err means not all paths exist yet
if err != nil {
continue
}
if _, ok := c.instances[uuid]; ok {
log.Println(log.TRACE, "InstanceCache instance updated:", uuid)
c.instances[uuid] = s
go c.notify(skynet.InstanceUpdated, s)
} else {
log.Println(log.TRACE, "InstanceCache instance added:", uuid)
c.instances[uuid] = s
go c.notify(skynet.InstanceAdded, s)
}
case PathCacheRemoveNotification:
if n.Path == path.Join(InstancesBasePath, uuid) {
if s, ok := c.instances[uuid]; ok {
log.Println(log.TRACE, "InstanceCache instance removed:", uuid)
go c.notify(skynet.InstanceRemoved, s)
delete(c.instances, uuid)
}
}
}
}
}
}
func getIdleTimeout(s skynet.ServiceInfo) time.Duration {
if d, err := config.String(s.Name, s.Version, "client.timeout.idle"); err == nil {
if timeout, err := time.ParseDuration(d); err == nil {
return timeout
}
log.Println(log.ERROR, "Failed to parse client.timeout.idle", err)
}
return config.DefaultIdleTimeout
}
func (cc *ClientCodec) ReadResponseBody(v interface{}) (err error) {
log.Println(log.TRACE, "RPC Client Entered: ReadResponseBody")
defer log.Println(log.TRACE, "RPC Client Leaving: ReadResponseBody")
if v == nil {
err = errors.New("Response object cannot be nil")
log.Println(log.ERROR, "RPC Client Error reading response body: ", err)
return
}
err = cc.Decoder.Decode(v)
if err != nil {
cc.Close()
log.Println(log.ERROR, "RPC Client Error decoding response body: ", err)
}
if err == nil {
log.Println(log.TRACE, pretty.Sprintf("RPC Client Read ResponseBody %s %+v", reflect.TypeOf(v), v))
}
return
}
func (c *InstanceCache) buildInitialCache() {
for _, p := range c.cache.Children() {
uuid := uuidFromPath(p)
s, err := c.getServiceInfo(uuid)
if err != nil {
log.Println(log.WARN, err)
continue
}
c.instances[uuid] = s
}
}
func (ss *SubService) sendAdminCommand(cmd string) bool {
log.Println(log.TRACE, "Writing to admin pipe: "+cmd)
_, err := ss.pipe.Write([]byte(cmd))
if err != nil {
log.Println(log.ERROR, "Failed to write to admin pipe", err)
return false
}
b := make([]byte, daemon.MAX_PIPE_BYTES)
log.Println(log.TRACE, "Reading from admin pipe")
n, err := ss.pipe.Read(b)
if err != nil && err != io.EOF {
log.Println(log.ERROR, "Failed to read from admin pipe", err)
return false
}
if bytes.Equal(b[:n], []byte("ACK")) {
return true
}
return false
}
func (s *Service) unregister() {
// this version must be run from the mux() goroutine
if !s.Registered {
return
}
err := skynet.GetServiceManager().Unregister(s.UUID)
if err != nil {
log.Println(log.ERROR, "Failed to unregister service: "+err.Error())
}
s.Registered = false
log.Printf(log.INFO, "%+v\n", ServiceUnregistered{s.ServiceInfo})
s.Delegate.Unregistered(s) // Call user defined callback
}
func worker(requestChan chan string, waitGroup *sync.WaitGroup) {
for {
select {
case service, ok := <-requestChan:
if !ok {
return
}
waitGroup.Add(1)
totalRequests.Add(1)
switch service {
case "simple":
randString := strconv.FormatUint(uint64(rand.Uint32()), 35)
randString = randString + randString + randString
in := map[string]interface{}{
"data": randString,
}
fmt.Println("Sending TestService request: " + in["data"].(string))
out := map[string]interface{}{}
err := simpleClient.Send(nil, "Upcase", in, &out)
upper := strings.ToUpper(randString)
if err == nil && out["data"].(string) == upper {
successfulRequests.Add(1)
fmt.Println("TestService returned: " + out["data"].(string))
} else {
failedRequests.Add(1)
if err != nil {
log.Println(log.ERROR, err)
}
}
waitGroup.Done()
}
}
}
}
// Starts your skynet service, including binding to ports. Optionally register for requests at the same time. Returns a sync.WaitGroup that will block until all requests have finished
func (s *Service) Start() (done *sync.WaitGroup) {
bindWait := &sync.WaitGroup{}
bindWait.Add(1)
go s.listen(s.ServiceAddr, bindWait)
// Watch signals for shutdown
c := make(chan os.Signal, 1)
go watchSignals(c, s)
s.doneChan = make(chan bool, 1)
// We must block here, we don't want to register, until we've actually bound to an ip:port
bindWait.Wait()
s.doneGroup = &sync.WaitGroup{}
s.doneGroup.Add(1)
go func() {
s.mux()
s.doneGroup.Done()
}()
done = s.doneGroup
if r, err := config.Bool(s.Name, s.Version, "service.register"); err == nil {
s.Registered = r
}
err := skynet.GetServiceManager().Add(*s.ServiceInfo)
if err != nil {
log.Println(log.ERROR, "Failed to add service: "+err.Error())
}
if s.Registered {
s.Register()
}
go s.Delegate.Started(s) // Call user defined callback
if s.ServiceInfo.Registered {
go s.Delegate.Registered(s) // Call user defined callback
}
return
}
func (sm *ZookeeperServiceManager) Update(s skynet.ServiceInfo) (err error) {
log.Println(log.TRACE, "Updating service", s.UUID)
ops := zk.MultiOps{
SetData: []zk.SetDataRequest{
setDataRequest(path.Join("/instances", s.UUID, "registered"), []byte(strconv.FormatBool(s.Registered)), -1),
setDataRequest(path.Join("/instances", s.UUID, "name"), []byte(s.Name), -1),
setDataRequest(path.Join("/instances", s.UUID, "version"), []byte(s.Version), -1),
setDataRequest(path.Join("/instances", s.UUID, "region"), []byte(s.Region), -1),
setDataRequest(path.Join("/instances", s.UUID, "addr"), []byte(s.ServiceAddr.String()), -1),
},
}
err = sm.conn.Multi(ops)
sm.managedInstances[s.UUID] = s
return
}