func (r *pipeline) run(w *model.Work) { // defer func() { // // r.drone.Ack(id, opts) // }() logrus.Infof("Starting build %s/%s#%d.%d", w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number) cancel := make(chan bool, 1) engine := docker.NewClient(r.docker) a := agent.Agent{ Update: agent.NewClientUpdater(r.drone), Logger: agent.NewClientLogger(r.drone, w.Job.ID, r.config.logs), Engine: engine, Timeout: r.config.timeout, Platform: r.config.platform, Namespace: r.config.namespace, Escalate: r.config.privileged, Pull: r.config.pull, } cancelFunc := func(m *stomp.Message) { defer m.Release() id := m.Header.GetInt64("job-id") if id == w.Job.ID { cancel <- true logrus.Infof("Cancel build %s/%s#%d.%d", w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number) } } // signal for canceling the build. sub, err := r.drone.Subscribe("/topic/cancel", stomp.HandlerFunc(cancelFunc)) if err != nil { logrus.Errorf("Error subscribing to /topic/cancel. %s", err) } defer func() { r.drone.Unsubscribe(sub) }() a.Run(w, cancel) // if err := r.drone.LogPost(w.Job.ID, ioutil.NopCloser(&buf)); err != nil { // logrus.Errorf("Error sending logs for %s/%s#%d.%d", // w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number) // } // stream.Close() logrus.Infof("Finished build %s/%s#%d.%d", w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number) }
// Broker is a middleware function that initializes the broker // and adds the broker client to the request context. func Broker(cli *cli.Context) gin.HandlerFunc { secret := cli.String("agent-secret") if secret == "" { logrus.Fatalf("fatal error. please provide the DRONE_SECRET") } // setup broker logging. log := redlog.New(os.Stderr) log.SetLevel(0) logger.SetLogger(log) if cli.Bool("broker-debug") { log.SetLevel(1) } broker := server.NewServer( server.WithCredentials("x-token", secret), ) client := broker.Client() var once sync.Once return func(c *gin.Context) { c.Set(serverKey, broker) c.Set(clientKey, client) once.Do(func() { // this is some really hacky stuff // turns out I need to do some refactoring // don't judge! // will fix in 0.6 release ctx := c.Copy() client.Connect( stomp.WithCredentials("x-token", secret), ) client.Subscribe("/queue/updates", stomp.HandlerFunc(func(m *stomp.Message) { go handlers.HandleUpdate(ctx, m.Copy()) })) }) } }
// HandleUpdate handles build updates from the agent and persists to the database. func HandleUpdate(c context.Context, message *stomp.Message) { defer func() { message.Release() if r := recover(); r != nil { err := r.(error) logrus.Errorf("Panic recover: broker update handler: %s", err) } }() work := new(model.Work) if err := message.Unmarshal(work); err != nil { logrus.Errorf("Invalid input. %s", err) return } // TODO(bradrydzewski) it is really annoying that we have to do this lookup // and I'd prefer not to. The reason we do this is because the Build and Job // have fields that aren't serialized to json and would be reset to their // empty values if we just saved what was coming in the http.Request body. build, err := store.GetBuild(c, work.Build.ID) if err != nil { logrus.Errorf("Unable to find build. %s", err) return } job, err := store.GetJob(c, work.Job.ID) if err != nil { logrus.Errorf("Unable to find job. %s", err) return } build.Started = work.Build.Started build.Finished = work.Build.Finished build.Status = work.Build.Status job.Started = work.Job.Started job.Finished = work.Job.Finished job.Status = work.Job.Status job.ExitCode = work.Job.ExitCode job.Error = work.Job.Error if build.Status == model.StatusPending { build.Started = work.Job.Started build.Status = model.StatusRunning store.UpdateBuild(c, build) } // if job.Status == model.StatusRunning { // err := stream.Create(c, stream.ToKey(job.ID)) // if err != nil { // logrus.Errorf("Unable to create stream. %s", err) // } // } ok, err := store.UpdateBuildJob(c, build, job) if err != nil { logrus.Errorf("Unable to update job. %s", err) return } if ok { // get the user because we transfer the user form the server to agent // and back we lose the token which does not get serialized to json. user, uerr := store.GetUser(c, work.User.ID) if uerr != nil { logrus.Errorf("Unable to find user. %s", err) return } remote.Status(c, user, work.Repo, build, fmt.Sprintf("%s/%s/%d", work.System.Link, work.Repo.FullName, work.Build.Number)) } client := stomp.MustFromContext(c) err = client.SendJSON("/topic/events", model.Event{ Type: func() model.EventType { // HACK we don't even really care about the event type. // so we should just simplify how events are triggered. if job.Status == model.StatusRunning { return model.Started } return model.Finished }(), Repo: *work.Repo, Build: *build, Job: *job, }, stomp.WithHeader("repo", work.Repo.FullName), stomp.WithHeader("private", strconv.FormatBool(work.Repo.IsPrivate)), ) if err != nil { logrus.Errorf("Unable to publish to /topic/events. %s", err) } if job.Status == model.StatusRunning { return } var buf bytes.Buffer var sub []byte done := make(chan bool) dest := fmt.Sprintf("/topic/logs.%d", job.ID) sub, err = client.Subscribe(dest, stomp.HandlerFunc(func(m *stomp.Message) { defer m.Release() if m.Header.GetBool("eof") { done <- true return } buf.Write(m.Body) buf.WriteByte('\n') })) if err != nil { logrus.Errorf("Unable to read logs from broker. %s", err) return } defer func() { client.Unsubscribe(sub) client.Send(dest, []byte{}, stomp.WithRetain("remove")) }() select { case <-done: case <-time.After(30 * time.Second): logrus.Errorf("Unable to read logs from broker. Timeout. %s", err) return } if err := store.WriteLog(c, job, &buf); err != nil { logrus.Errorf("Unable to write logs to store. %s", err) return } }
// LogStream streams the build log output to the client. func LogStream(c *gin.Context) { repo := session.Repo(c) buildn, _ := strconv.Atoi(c.Param("build")) jobn, _ := strconv.Atoi(c.Param("number")) c.Writer.Header().Set("Content-Type", "text/event-stream") build, err := store.GetBuildNumber(c, repo, buildn) if err != nil { logrus.Debugln("stream cannot get build number.", err) c.AbortWithError(404, err) return } job, err := store.GetJobNumber(c, build, jobn) if err != nil { logrus.Debugln("stream cannot get job number.", err) c.AbortWithError(404, err) return } if job.Status != model.StatusRunning { logrus.Debugln("stream not found.") c.AbortWithStatus(404) return } ws, err := upgrader.Upgrade(c.Writer, c.Request, nil) if err != nil { if _, ok := err.(websocket.HandshakeError); !ok { logrus.Errorf("Cannot upgrade websocket. %s", err) } return } logrus.Debugf("Successfull upgraded websocket") ticker := time.NewTicker(pingPeriod) defer ticker.Stop() logs := make(chan []byte) done := make(chan bool) dest := fmt.Sprintf("/topic/logs.%d", job.ID) client, _ := stomp.FromContext(c) sub, err := client.Subscribe(dest, stomp.HandlerFunc(func(m *stomp.Message) { if m.Header.GetBool("eof") { done <- true } else { logs <- m.Body } m.Release() })) if err != nil { logrus.Errorf("Unable to read logs from broker. %s", err) return } defer func() { client.Unsubscribe(sub) close(done) close(logs) }() for { select { case buf := <-logs: ws.SetWriteDeadline(time.Now().Add(writeWait)) ws.WriteMessage(websocket.TextMessage, buf) case <-done: return case <-ticker.C: err := ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeWait)) if err != nil { return } } } }
// EventStream produces the User event stream, sending all repository, build // and agent events to the client. func EventStream(c *gin.Context) { ws, err := upgrader.Upgrade(c.Writer, c.Request, nil) if err != nil { if _, ok := err.(websocket.HandshakeError); !ok { logrus.Errorf("Cannot upgrade websocket. %s", err) } return } logrus.Debugf("Successfull upgraded websocket") user := session.User(c) repo := map[string]bool{} if user != nil { repo, _ = cache.GetRepoMap(c, user) } eventc := make(chan []byte, 10) quitc := make(chan bool) tick := time.NewTicker(pingPeriod) defer func() { tick.Stop() ws.Close() logrus.Debug("Successfully closed websocket") }() client := stomp.MustFromContext(c) sub, err := client.Subscribe("/topic/events", stomp.HandlerFunc(func(m *stomp.Message) { name := m.Header.GetString("repo") priv := m.Header.GetBool("private") if repo[name] || !priv { eventc <- m.Body } m.Release() })) if err != nil { logrus.Errorf("Unable to read logs from broker. %s", err) return } defer func() { client.Unsubscribe(sub) close(quitc) close(eventc) }() go func() { defer func() { recover() }() for { select { case <-quitc: return case event, ok := <-eventc: if !ok { return } ws.SetWriteDeadline(time.Now().Add(writeWait)) ws.WriteMessage(websocket.TextMessage, event) case <-tick.C: err := ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeWait)) if err != nil { return } } } }() reader(ws) }
func start(c *cli.Context) { log := redlog.New(os.Stderr) log.SetLevel(0) logger.SetLogger(log) // debug level if requested by user if c.Bool("debug") { logrus.SetLevel(logrus.DebugLevel) log.SetLevel(1) } else { logrus.SetLevel(logrus.WarnLevel) } var accessToken string if c.String("drone-secret") != "" { // secretToken := c.String("drone-secret") accessToken = c.String("drone-secret") // accessToken, _ = token.New(token.AgentToken, "").Sign(secretToken) } else { accessToken = c.String("drone-token") } logger.Noticef("connecting to server %s", c.String("drone-server")) server := strings.TrimRight(c.String("drone-server"), "/") tls, err := dockerclient.TLSConfigFromCertPath(c.String("docker-cert-path")) if err == nil { tls.InsecureSkipVerify = c.Bool("docker-tls-verify") } docker, err := dockerclient.NewDockerClient(c.String("docker-host"), tls) if err != nil { logrus.Fatal(err) } var client *stomp.Client handler := func(m *stomp.Message) { running.Add(1) defer func() { running.Done() client.Ack(m.Ack) }() r := pipeline{ drone: client, docker: docker, config: config{ platform: c.String("docker-os") + "/" + c.String("docker-arch"), timeout: c.Duration("timeout"), namespace: c.String("namespace"), privileged: c.StringSlice("privileged"), pull: c.BoolT("pull"), logs: int64(c.Int("max-log-size")) * 1000000, }, } work := new(model.Work) m.Unmarshal(work) r.run(work) } handleSignals() backoff := c.Duration("backoff") for { // dial the drone server to establish a TCP connection. client, err = stomp.Dial(server) if err != nil { logger.Warningf("connection failed, retry in %v. %s", backoff, err) <-time.After(backoff) continue } opts := []stomp.MessageOption{ stomp.WithCredentials("x-token", accessToken), } // initialize the stomp session and authenticate. if err = client.Connect(opts...); err != nil { logger.Warningf("session failed, retry in %v. %s", backoff, err) <-time.After(backoff) continue } opts = []stomp.MessageOption{ stomp.WithAck("client"), stomp.WithPrefetch( c.Int("docker-max-procs"), ), } if filter := c.String("filter"); filter != "" { opts = append(opts, stomp.WithSelector(filter)) } // subscribe to the pending build queue. client.Subscribe("/queue/pending", stomp.HandlerFunc(func(m *stomp.Message) { go handler(m) // HACK until we a channel based Subscribe implementation }), opts...) logger.Noticef("connection established, ready to process builds.") <-client.Done() logger.Warningf("connection interrupted, attempting to reconnect.") } }