// Checkout project from repository func (b *builder) updateCode() { p, err := b.scm.ImportPathFromRepo(b.BuildConfig.AppRepo) b.projectPath = path.Join(b.BuildConfig.Jail, "src", p) if err != nil { log.Fatal(err.Error()) } out, err := b.term.Exec("ls " + b.projectPath) if err != nil { fmt.Println("Creating project directories") out, err = b.term.Exec("mkdir -p " + b.projectPath) if err != nil { log.Fatal("Could not create project directories") } fmt.Println(string(out)) } // Fetch code base b.scm.SetTerminal(b.term) b.scm.Checkout(b.BuildConfig.AppRepo, b.BuildConfig.RepoBranch, b.projectPath) }
func (b *builder) validatePackage() { // Validate this package is a command var err error b.pack, err = context.ImportDir(".", 0) if err != nil { log.Fatal("Could not import package for validation") } if !b.pack.IsCommand() { log.Fatal("Package is not a command") } }
func GetServiceManager() ServiceManager { if manager == nil { log.Fatal("No ServiceManager provided") } return manager }
func Stop(criteria *skynet.Criteria) { instances, err := skynet.GetServiceManager().ListInstances(criteria) if err != nil { log.Fatal(err) } var wait sync.WaitGroup for _, instance := range filterDaemon(instances) { wait.Add(1) go func(instance skynet.ServiceInfo) { fmt.Println("Stopping: " + instance.UUID) d := daemon.GetDaemonForService(&instance) in := daemon.StopSubServiceRequest{ UUID: instance.UUID, } out, err := d.StopSubService(in) if err != nil { fmt.Println("Returned Error: " + err.Error()) wait.Done() return } stopTemplate.Execute(os.Stdout, out) wait.Done() }(instance) } wait.Wait() }
func (c *SSHConn) ExecPath(cmd, path string) (out []byte, err error) { var session *ssh.Session session, err = c.client.NewSession() if err != nil { log.Fatal("Failed to create session: " + err.Error()) } defer session.Close() envVars := "" if c.env != nil { for name, value := range c.env { envVars = envVars + name + "=\"" + value + "\" " /* TODO: This should be the proper way to set the environment, but fails for some reason * Investigate why and possibly send pull-request to maintainer err = session.Setenv(name, value) if err != nil { log.Fatal("Failed to set environment: " + err.Error()) } */ } } cmd = envVars + cmd if path != "" { cmd = "cd " + path + " && " + cmd } return session.CombinedOutput(cmd) }
// Daemon will run and maintain skynet services. // // Daemon will run the "SkynetDeployment" service, which can be used // to remotely spawn new services on the host. func main() { si := skynet.NewServiceInfo("SkynetDaemon", "2.0.0") deployment := NewSkynetDaemon() s := service.CreateService(deployment, si) deployment.Service = s // handle panic so that we remove ourselves from the pool in case of catastrophic failure defer func() { s.Shutdown() deployment.closeStateFile() if err := recover(); err != nil { e := err.(error) log.Fatal("Unrecovered error occured: " + e.Error()) } }() // Collect Host metrics statTicker := time.Tick((5 * time.Second)) go func() { for _ = range statTicker { deployment.updateHostStats(si.ServiceAddr.IPAddress) } }() // If we pass false here service will not be Registered // we could do other work/tasks by implementing the Started method and calling Register() when we're ready s.Start().Wait() }
func SetLogLevel(criteria *skynet.Criteria, level string) { instances, err := skynet.GetServiceManager().ListInstances(criteria) if err != nil { log.Fatal(err) } var wait sync.WaitGroup for _, instance := range filterDaemon(instances) { wait.Add(1) go func(instance skynet.ServiceInfo) { fmt.Println("Setting LogLevel to " + level + " for: " + instance.UUID) d := daemon.GetDaemonForService(&instance) in := daemon.SubServiceLogLevelRequest{ UUID: instance.UUID, Level: level, } out, err := d.SubServiceLogLevel(in) if err != nil { fmt.Println("Returned Error: " + err.Error()) wait.Done() return } logLevelTemplate.Execute(os.Stdout, out) wait.Done() }(instance) } wait.Wait() }
func StopDaemon(criteria *skynet.Criteria) { hosts, err := skynet.GetServiceManager().ListHosts(criteria) if err != nil { log.Fatal(err) } var wait sync.WaitGroup for _, host := range hosts { wait.Add(1) go func(host string) { d := daemon.GetDaemonForHost(host) in := daemon.StopRequest{} out, err := d.Stop(in) if err != nil { fmt.Println("Returned Error: " + err.Error()) wait.Done() return } if out.Ok { fmt.Printf("Daemon stopped on host: %v\n", host) } else { fmt.Printf("Failed to stop daemon on host: %v\n", host) } wait.Done() }(host) } wait.Wait() }
func (b *builder) setupScm() { switch b.BuildConfig.RepoType { case "git": b.scm = new(GitScm) default: log.Fatal("unkown RepoType") } }
func getHosts(c *skynet.Criteria) []string { hosts, err := skynet.GetServiceManager().ListHosts(c) if err != nil { log.Fatal(err) } return hosts }
func getInstances(c *skynet.Criteria) []skynet.ServiceInfo { instances, err := skynet.GetServiceManager().ListInstances(c) if err != nil { log.Fatal(err) } return instances }
func getVersions(c *skynet.Criteria) []string { versions, err := skynet.GetServiceManager().ListVersions(c) if err != nil { log.Fatal(err) } return versions }
func getServices(c *skynet.Criteria) []string { services, err := skynet.GetServiceManager().ListServices(c) if err != nil { log.Fatal(err) } return services }
// NewUUID() provides unique identifier strings. func NewUUID() string { b := make([]byte, 16) _, err := io.ReadFull(rand.Reader, b) if err != nil { log.Fatal(err) } b[6] = (b[6] & 0x0F) | 0x40 b[8] = (b[8] &^ 0x40) | 0x80 return fmt.Sprintf("%x-%x-%x-%x-%x", b[:4], b[4:6], b[6:8], b[8:10], b[10:]) }
func (b *builder) runCommands(cmds []string) { for _, cmd := range cmds { out, err := b.term.Exec(cmd) fmt.Println(string(out)) if err != nil { log.Fatal("Failed to execute dependent command: " + cmd + "\n" + err.Error()) } } }
// Wraps your custom service in Skynet func CreateService(sd ServiceDelegate, si *skynet.ServiceInfo) (s *Service) { s = &Service{ Delegate: sd, ServiceInfo: si, methods: make(map[string]reflect.Value), connectionChan: make(chan *net.TCPConn), registeredChan: make(chan bool), shutdownChan: make(chan bool), ClientInfo: make(map[string]ClientInfo), shuttingDown: false, } // Override LogLevel for Service if l, err := config.String(s.Name, s.Version, "log.level"); err != nil { log.SetLogLevel(log.LevelFromString(l)) } logWriter := log.NewMultiWriter() if logStdout, err := config.Bool(s.Name, s.Version, "service.log.stdout"); err == nil { if logStdout { logWriter.AddWriter(os.Stdout) } } else { // Stdout is enabled by default logWriter.AddWriter(os.Stdout) } if logFile, err := config.String(s.Name, s.Version, "service.log.file"); err == nil { f, err := os.OpenFile(logFile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0660) if err != nil { log.Fatal("Failed to open log file: ", logFile, err) } logWriter.AddWriter(f) } log.SetOutput(logWriter) // the main rpc server s.RPCServ = rpc.NewServer() rpcForwarder := NewServiceRPC(s) s.RPCServ.RegisterName(si.Name, rpcForwarder) // Daemon doesn't accept commands over pipe if si.Name != "SkynetDaemon" { // Listen for admin requests go s.serveAdminRequests() } return }
func init() { timeout := 1 * time.Second addr := DefaultAddr log.Println(log.INFO, "In init") if a, err := config.RawStringDefault("zookeeper.addr"); err == nil { addr = a log.Println(log.INFO, "addr", a) } else { log.Fatal("Failed to parse Zookeeper addr", err) } if t, err := config.RawStringDefault("zookeeper.timeout"); err == nil { if timeout, err = time.ParseDuration(t); err != nil { log.Fatal("Failed to parse Zookeeper timeout", err) } } skynet.SetServiceManager(NewZookeeperServiceManager(addr, timeout)) }
func (b *builder) testSkynet() { fmt.Println("Testing Skynet") p := path.Join(b.BuildConfig.Jail, "src/github.com/ClarityServices/skynet2") b.getPackageDependencies(p) out, err := b.term.ExecPath(b.BuildConfig.GoRoot+"/bin/go test ./...", p) fmt.Println(string(out)) if err != nil { log.Fatal("Failed tests: " + err.Error()) } }
func init() { flagset := flag.NewFlagSet("config", flag.ContinueOnError) flagset.StringVar(&configFile, "config", "", "Config File") flagset.StringVar(&uuid, "uuid", "", "uuid") args, _ := SplitFlagsetFromArgs(flagset, os.Args[1:]) flagset.Parse(args) // Ensure we have a UUID if uuid == "" { uuid = NewUUID() } if configFile == "" { for _, f := range defaultConfigFiles { if _, err := os.Stat(f); err == nil { configFile = f break } } } if configFile == "" { log.Println(log.ERROR, "Failed to find config file") conf = config.NewDefault() return } if _, err := os.Stat(configFile); os.IsNotExist(err) { log.Println(log.ERROR, "Config file does not exist", err) conf = config.NewDefault() return } var err error if conf, err = config.ReadDefault(configFile); err != nil { conf = config.NewDefault() log.Fatal(err) } // Set default log level from config, this can be overriden at the service level when the service is created if l, err := conf.RawStringDefault("log.level"); err == nil { log.SetLogLevel(log.LevelFromString(l)) } // Set GOMAXPROCS if i, err := conf.Int("DEFAULT", "runtime.gomaxprocs"); err == nil { runtime.GOMAXPROCS(i) } }
func newBuilder(config string) *builder { if config == "" { config = "./build.cfg" } f, err := ioutil.ReadFile(config) if err != nil { log.Fatal("Failed to read: " + config) } b := new(builder) err = json.Unmarshal(f, b) if err != nil { log.Fatal("Failed to parse " + config + ": " + err.Error()) } if isHostLocal(b.BuildConfig.Host) { fmt.Println("Connecting to build machine: " + b.BuildConfig.Host) b.term = new(LocalTerminal) } else { sshClient := new(SSHConn) b.term = sshClient fmt.Println("Connecting to build machine: " + b.BuildConfig.Host) err = sshClient.Connect(b.BuildConfig.Host, b.BuildConfig.User) if err != nil { log.Fatal("Failed to connect to build machine: " + err.Error()) } } b.validatePackage() return b }
func (b *builder) runTests() { p := path.Join(b.projectPath, b.BuildConfig.AppPath) fmt.Println("Testing packages") out, err := b.term.ExecPath(b.BuildConfig.GoRoot+"/bin/go test", p) fmt.Println(string(out)) if err != nil { log.Fatal("Failed tests: " + err.Error()) } if b.BuildConfig.TestSkynet { b.testSkynet() } }
func (b *builder) getPackageDependencies(p string) { flags := []string{"-d"} if b.BuildConfig.UpdatePackages { flags = append(flags, "-u") } fmt.Println("Fetching dependencies") out, err := b.term.ExecPath(b.BuildConfig.GoRoot+"/bin/go get "+strings.Join(flags, " ")+" ./...", p) fmt.Println(string(out)) if err != nil { log.Fatal("Failed to fetch dependencies\n" + err.Error()) } }
func criteriaFromArgs(args []string) (*skynet.Criteria, []string) { flagset := flag.NewFlagSet("deploy", flag.ExitOnError) services := flagset.String("services", "", "services") regions := flagset.String("regions", "", "regions") instances := flagset.String("instances", "", "instances") hosts := flagset.String("hosts", "", "hosts") registered := flagset.String("registered", "", "registered") flagsetArgs, args := config.SplitFlagsetFromArgs(flagset, args) err := flagset.Parse(flagsetArgs) if err != nil { log.Fatal(err) } regionCriteria := make([]string, 0, 0) if len(*regions) > 0 { regionCriteria = strings.Split(*regions, ",") } hostCriteria := make([]string, 0, 0) if len(*hosts) > 0 { hostCriteria = strings.Split(*hosts, ",") } instanceCriteria := make([]string, 0, 0) if len(*instances) > 0 { instanceCriteria = strings.Split(*instances, ",") } var reg *bool if *registered == "true" { *reg = true } else if *registered == "false" { *reg = false } return &skynet.Criteria{ Regions: regionCriteria, Hosts: hostCriteria, Instances: instanceCriteria, Registered: reg, Services: serviceCriteriaFromCsv(*services), }, args }
func (b *builder) buildProject() { p := path.Join(b.projectPath, b.BuildConfig.AppPath) flags := "-v" if b.BuildConfig.BuildAllPackages { flags = flags + " -a" } fmt.Println("Building packages") out, err := b.term.ExecPath(b.BuildConfig.GoRoot+"/bin/go install "+flags, p) fmt.Println(string(out)) if err != nil { log.Fatal("Failed build: " + err.Error()) } }
func Start(criteria *skynet.Criteria, args []string) { if len(args) < 1 { fmt.Println("Please provide a service name 'sky start binaryName'") return } hosts, err := skynet.GetServiceManager().ListHosts(criteria) if err != nil { log.Fatal(err) } var wait sync.WaitGroup for _, host := range hosts { wait.Add(1) go func(host string) { fmt.Println("Starting on host: " + host) d := daemon.GetDaemonForHost(host) in := daemon.StartSubServiceRequest{ BinaryName: args[0], Args: shellquote.Join(args[1:]...), // TODO: maybe an optional flag to change this? Registered: true, } out, err := d.StartSubService(in) if err != nil { fmt.Println("Returned Error: " + err.Error()) wait.Done() return } startTemplate.Execute(os.Stdout, out) wait.Done() }(host) } wait.Wait() }
func SetDaemonLogLevel(criteria *skynet.Criteria, level string) { hosts, err := skynet.GetServiceManager().ListHosts(criteria) if err != nil { log.Fatal(err) } var wait sync.WaitGroup for _, host := range hosts { wait.Add(1) go func(host string) { d := daemon.GetDaemonForHost(host) in := daemon.LogLevelRequest{ Level: level, } out, err := d.LogLevel(in) if err != nil { fmt.Println("Returned Error: " + err.Error()) wait.Done() return } if out.Ok { fmt.Printf("Set daemon log level to %v on host: %v\n", level, host) } else { fmt.Printf("Failed to set daemon log level to %v on host: %v\n", level, host) } wait.Done() }(host) } wait.Wait() }
func (b *builder) deploy(hosts []string) { for _, host := range hosts { var out []byte var err error if isHostLocal(host) && isHostLocal(b.BuildConfig.Host) { // Built locally, deploying locally fmt.Println("Copying local binary") // First move binary to .old in case it's currently running command := exec.Command("mv", path.Join(b.DeployConfig.DeployPath, b.DeployConfig.BinaryName), path.Join(b.DeployConfig.DeployPath, b.DeployConfig.BinaryName+".old")) out, err = command.CombinedOutput() if err == nil { fmt.Println(string(out)) command = exec.Command("cp", path.Join(b.BuildConfig.Jail, "bin", path.Base(b.BuildConfig.AppPath)), path.Join(b.DeployConfig.DeployPath, b.DeployConfig.BinaryName)) out, err = command.CombinedOutput() } } else if isHostLocal(host) && !isHostLocal(b.BuildConfig.Host) { // Built remotely, deploying locally fmt.Println("Copying binary from build machine") h, p := splitHostPort(b.BuildConfig.Host) // First move binary to .old in case it's currently running command := exec.Command("mv", path.Join(b.DeployConfig.DeployPath, b.DeployConfig.BinaryName), path.Join(b.DeployConfig.DeployPath, b.DeployConfig.BinaryName+".old")) out, _ = command.CombinedOutput() fmt.Println(string(out)) command = exec.Command("scp", "-P", p, b.BuildConfig.User+"@"+h+":"+path.Join(b.BuildConfig.Jail, "bin", path.Base(b.BuildConfig.AppPath)), path.Join(b.DeployConfig.DeployPath, b.DeployConfig.BinaryName)) out, err = command.CombinedOutput() } else if !isHostLocal(host) && isHostLocal(b.BuildConfig.Host) { // Built locally, deploying remotely fmt.Println("Pushing binary to host: " + host) h, p := splitHostPort(host) // First move binary to .old in case it's currently running command := exec.Command("ssh", "-p", p, b.DeployConfig.User+"@"+h, "mv", path.Join(b.DeployConfig.DeployPath, b.DeployConfig.BinaryName), path.Join(b.DeployConfig.DeployPath, b.DeployConfig.BinaryName+".old")) out, _ = command.CombinedOutput() fmt.Println(string(out)) command = exec.Command("scp", "-P", p, path.Join(b.BuildConfig.Jail, "bin", path.Base(b.BuildConfig.AppPath)), b.DeployConfig.User+"@"+h+":"+path.Join(b.DeployConfig.DeployPath, b.DeployConfig.BinaryName)) out, err = command.CombinedOutput() } else if !isHostLocal(host) && !isHostLocal(b.BuildConfig.Host) { // Built remotely, deployed remotely fmt.Println("Pushing binary from build box to host: " + host) h, p := splitHostPort(host) // First move binary to .old in case it's currently running out, _ := b.term.Exec("ssh -p " + p + " " + b.DeployConfig.User + "@" + h + " mv " + path.Join(b.DeployConfig.DeployPath, b.DeployConfig.BinaryName) + " " + path.Join(b.DeployConfig.DeployPath, b.DeployConfig.BinaryName+".old")) fmt.Println(string(out)) out, err = b.term.Exec("scp -P " + p + " " + path.Join(b.BuildConfig.Jail, "bin", path.Base(b.BuildConfig.AppPath)) + " " + b.DeployConfig.User + "@" + h + ":" + path.Join(b.DeployConfig.DeployPath, b.DeployConfig.BinaryName)) } fmt.Println(string(out)) if err != nil { log.Fatal("Failed to deploy: " + err.Error()) } } }
func main() { log.SetLogLevel(log.ERROR) var args []string criteria, args := criteriaFromArgs(os.Args[1:]) if len(args) == 0 { CommandLineHelp() return } switch args[0] { case "help", "h": CommandLineHelp() case "build", "b": flagset := flag.NewFlagSet("build", flag.ExitOnError) configFile := flagset.String("build", "./build.cfg", "build config file") flagsetArgs, _ := config.SplitFlagsetFromArgs(flagset, args) err := flagset.Parse(flagsetArgs) if err != nil { log.Fatal(err) return } Build(*configFile) case "deploy", "d": flagset := flag.NewFlagSet("deploy", flag.ExitOnError) configFile := flagset.String("build", "./build.cfg", "build config file") flagsetArgs, _ := config.SplitFlagsetFromArgs(flagset, args) err := flagset.Parse(flagsetArgs) if err != nil { log.Fatal(err) return } Deploy(*configFile, criteria) case "hosts": ListHosts(criteria) case "regions": ListRegions(criteria) case "services": ListServices(criteria) case "versions": ListVersions(criteria) case "instances": ListInstances(criteria) case "start": Start(criteria, args[1:]) case "stop": Stop(criteria) case "restart": Restart(criteria) case "register": Register(criteria) case "unregister": Unregister(criteria) case "log": SetLogLevel(criteria, args[1]) case "daemon": if len(args) >= 2 { switch args[1] { case "log": if len(args) >= 3 { SetDaemonLogLevel(criteria, args[2]) } else { fmt.Println("Must supply a log level") } case "stop": StopDaemon(criteria) } } else { fmt.Println("Supported subcommands for daemon are log, and stop") } case "cli": InteractiveShell() default: fmt.Println("Unknown Command: ", args[0]) CommandLineHelp() } }