func (r *pipeline) run(w *model.Work) {
// defer func() {
// // r.drone.Ack(id, opts)
// }()
logrus.Infof("Starting build %s/%s#%d.%d",
w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
cancel := make(chan bool, 1)
engine := docker.NewClient(r.docker)
a := agent.Agent{
Update: agent.NewClientUpdater(r.drone),
Logger: agent.NewClientLogger(r.drone, w.Job.ID, r.config.logs),
Engine: engine,
Timeout: r.config.timeout,
Platform: r.config.platform,
Namespace: r.config.namespace,
Escalate: r.config.privileged,
Pull: r.config.pull,
}
cancelFunc := func(m *stomp.Message) {
defer m.Release()
id := m.Header.GetInt64("job-id")
if id == w.Job.ID {
cancel <- true
logrus.Infof("Cancel build %s/%s#%d.%d",
w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
}
}
// signal for canceling the build.
sub, err := r.drone.Subscribe("/topic/cancel", stomp.HandlerFunc(cancelFunc))
if err != nil {
logrus.Errorf("Error subscribing to /topic/cancel. %s", err)
}
defer func() {
r.drone.Unsubscribe(sub)
}()
a.Run(w, cancel)
// if err := r.drone.LogPost(w.Job.ID, ioutil.NopCloser(&buf)); err != nil {
// logrus.Errorf("Error sending logs for %s/%s#%d.%d",
// w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
// }
// stream.Close()
logrus.Infof("Finished build %s/%s#%d.%d",
w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
}
// Broker is a middleware function that initializes the broker
// and adds the broker client to the request context.
func Broker(cli *cli.Context) gin.HandlerFunc {
secret := cli.String("agent-secret")
if secret == "" {
logrus.Fatalf("fatal error. please provide the DRONE_SECRET")
}
// setup broker logging.
log := redlog.New(os.Stderr)
log.SetLevel(0)
logger.SetLogger(log)
if cli.Bool("broker-debug") {
log.SetLevel(1)
}
broker := server.NewServer(
server.WithCredentials("x-token", secret),
)
client := broker.Client()
var once sync.Once
return func(c *gin.Context) {
c.Set(serverKey, broker)
c.Set(clientKey, client)
once.Do(func() {
// this is some really hacky stuff
// turns out I need to do some refactoring
// don't judge!
// will fix in 0.6 release
ctx := c.Copy()
client.Connect(
stomp.WithCredentials("x-token", secret),
)
client.Subscribe("/queue/updates", stomp.HandlerFunc(func(m *stomp.Message) {
go handlers.HandleUpdate(ctx, m.Copy())
}))
})
}
}
// HandleUpdate handles build updates from the agent and persists to the database.
func HandleUpdate(c context.Context, message *stomp.Message) {
defer func() {
message.Release()
if r := recover(); r != nil {
err := r.(error)
logrus.Errorf("Panic recover: broker update handler: %s", err)
}
}()
work := new(model.Work)
if err := message.Unmarshal(work); err != nil {
logrus.Errorf("Invalid input. %s", err)
return
}
// TODO(bradrydzewski) it is really annoying that we have to do this lookup
// and I'd prefer not to. The reason we do this is because the Build and Job
// have fields that aren't serialized to json and would be reset to their
// empty values if we just saved what was coming in the http.Request body.
build, err := store.GetBuild(c, work.Build.ID)
if err != nil {
logrus.Errorf("Unable to find build. %s", err)
return
}
job, err := store.GetJob(c, work.Job.ID)
if err != nil {
logrus.Errorf("Unable to find job. %s", err)
return
}
build.Started = work.Build.Started
build.Finished = work.Build.Finished
build.Status = work.Build.Status
job.Started = work.Job.Started
job.Finished = work.Job.Finished
job.Status = work.Job.Status
job.ExitCode = work.Job.ExitCode
job.Error = work.Job.Error
if build.Status == model.StatusPending {
build.Started = work.Job.Started
build.Status = model.StatusRunning
store.UpdateBuild(c, build)
}
// if job.Status == model.StatusRunning {
// err := stream.Create(c, stream.ToKey(job.ID))
// if err != nil {
// logrus.Errorf("Unable to create stream. %s", err)
// }
// }
ok, err := store.UpdateBuildJob(c, build, job)
if err != nil {
logrus.Errorf("Unable to update job. %s", err)
return
}
if ok {
// get the user because we transfer the user form the server to agent
// and back we lose the token which does not get serialized to json.
user, uerr := store.GetUser(c, work.User.ID)
if uerr != nil {
logrus.Errorf("Unable to find user. %s", err)
return
}
remote.Status(c, user, work.Repo, build,
fmt.Sprintf("%s/%s/%d", work.System.Link, work.Repo.FullName, work.Build.Number))
}
client := stomp.MustFromContext(c)
err = client.SendJSON("/topic/events", model.Event{
Type: func() model.EventType {
// HACK we don't even really care about the event type.
// so we should just simplify how events are triggered.
if job.Status == model.StatusRunning {
return model.Started
}
return model.Finished
}(),
Repo: *work.Repo,
Build: *build,
Job: *job,
},
stomp.WithHeader("repo", work.Repo.FullName),
stomp.WithHeader("private", strconv.FormatBool(work.Repo.IsPrivate)),
)
if err != nil {
logrus.Errorf("Unable to publish to /topic/events. %s", err)
}
if job.Status == model.StatusRunning {
return
}
var buf bytes.Buffer
var sub []byte
done := make(chan bool)
dest := fmt.Sprintf("/topic/logs.%d", job.ID)
sub, err = client.Subscribe(dest, stomp.HandlerFunc(func(m *stomp.Message) {
defer m.Release()
if m.Header.GetBool("eof") {
done <- true
return
}
buf.Write(m.Body)
buf.WriteByte('\n')
}))
if err != nil {
logrus.Errorf("Unable to read logs from broker. %s", err)
return
}
defer func() {
client.Unsubscribe(sub)
client.Send(dest, []byte{}, stomp.WithRetain("remove"))
}()
select {
case <-done:
case <-time.After(30 * time.Second):
logrus.Errorf("Unable to read logs from broker. Timeout. %s", err)
return
}
if err := store.WriteLog(c, job, &buf); err != nil {
logrus.Errorf("Unable to write logs to store. %s", err)
return
}
}
// LogStream streams the build log output to the client.
func LogStream(c *gin.Context) {
repo := session.Repo(c)
buildn, _ := strconv.Atoi(c.Param("build"))
jobn, _ := strconv.Atoi(c.Param("number"))
c.Writer.Header().Set("Content-Type", "text/event-stream")
build, err := store.GetBuildNumber(c, repo, buildn)
if err != nil {
logrus.Debugln("stream cannot get build number.", err)
c.AbortWithError(404, err)
return
}
job, err := store.GetJobNumber(c, build, jobn)
if err != nil {
logrus.Debugln("stream cannot get job number.", err)
c.AbortWithError(404, err)
return
}
if job.Status != model.StatusRunning {
logrus.Debugln("stream not found.")
c.AbortWithStatus(404)
return
}
ws, err := upgrader.Upgrade(c.Writer, c.Request, nil)
if err != nil {
if _, ok := err.(websocket.HandshakeError); !ok {
logrus.Errorf("Cannot upgrade websocket. %s", err)
}
return
}
logrus.Debugf("Successfull upgraded websocket")
ticker := time.NewTicker(pingPeriod)
defer ticker.Stop()
logs := make(chan []byte)
done := make(chan bool)
dest := fmt.Sprintf("/topic/logs.%d", job.ID)
client, _ := stomp.FromContext(c)
sub, err := client.Subscribe(dest, stomp.HandlerFunc(func(m *stomp.Message) {
if m.Header.GetBool("eof") {
done <- true
} else {
logs <- m.Body
}
m.Release()
}))
if err != nil {
logrus.Errorf("Unable to read logs from broker. %s", err)
return
}
defer func() {
client.Unsubscribe(sub)
close(done)
close(logs)
}()
for {
select {
case buf := <-logs:
ws.SetWriteDeadline(time.Now().Add(writeWait))
ws.WriteMessage(websocket.TextMessage, buf)
case <-done:
return
case <-ticker.C:
err := ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeWait))
if err != nil {
return
}
}
}
}
// EventStream produces the User event stream, sending all repository, build
// and agent events to the client.
func EventStream(c *gin.Context) {
ws, err := upgrader.Upgrade(c.Writer, c.Request, nil)
if err != nil {
if _, ok := err.(websocket.HandshakeError); !ok {
logrus.Errorf("Cannot upgrade websocket. %s", err)
}
return
}
logrus.Debugf("Successfull upgraded websocket")
user := session.User(c)
repo := map[string]bool{}
if user != nil {
repo, _ = cache.GetRepoMap(c, user)
}
eventc := make(chan []byte, 10)
quitc := make(chan bool)
tick := time.NewTicker(pingPeriod)
defer func() {
tick.Stop()
ws.Close()
logrus.Debug("Successfully closed websocket")
}()
client := stomp.MustFromContext(c)
sub, err := client.Subscribe("/topic/events", stomp.HandlerFunc(func(m *stomp.Message) {
name := m.Header.GetString("repo")
priv := m.Header.GetBool("private")
if repo[name] || !priv {
eventc <- m.Body
}
m.Release()
}))
if err != nil {
logrus.Errorf("Unable to read logs from broker. %s", err)
return
}
defer func() {
client.Unsubscribe(sub)
close(quitc)
close(eventc)
}()
go func() {
defer func() {
recover()
}()
for {
select {
case <-quitc:
return
case event, ok := <-eventc:
if !ok {
return
}
ws.SetWriteDeadline(time.Now().Add(writeWait))
ws.WriteMessage(websocket.TextMessage, event)
case <-tick.C:
err := ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeWait))
if err != nil {
return
}
}
}
}()
reader(ws)
}
func start(c *cli.Context) {
log := redlog.New(os.Stderr)
log.SetLevel(0)
logger.SetLogger(log)
// debug level if requested by user
if c.Bool("debug") {
logrus.SetLevel(logrus.DebugLevel)
log.SetLevel(1)
} else {
logrus.SetLevel(logrus.WarnLevel)
}
var accessToken string
if c.String("drone-secret") != "" {
// secretToken := c.String("drone-secret")
accessToken = c.String("drone-secret")
// accessToken, _ = token.New(token.AgentToken, "").Sign(secretToken)
} else {
accessToken = c.String("drone-token")
}
logger.Noticef("connecting to server %s", c.String("drone-server"))
server := strings.TrimRight(c.String("drone-server"), "/")
tls, err := dockerclient.TLSConfigFromCertPath(c.String("docker-cert-path"))
if err == nil {
tls.InsecureSkipVerify = c.Bool("docker-tls-verify")
}
docker, err := dockerclient.NewDockerClient(c.String("docker-host"), tls)
if err != nil {
logrus.Fatal(err)
}
var client *stomp.Client
handler := func(m *stomp.Message) {
running.Add(1)
defer func() {
running.Done()
client.Ack(m.Ack)
}()
r := pipeline{
drone: client,
docker: docker,
config: config{
platform: c.String("docker-os") + "/" + c.String("docker-arch"),
timeout: c.Duration("timeout"),
namespace: c.String("namespace"),
privileged: c.StringSlice("privileged"),
pull: c.BoolT("pull"),
logs: int64(c.Int("max-log-size")) * 1000000,
},
}
work := new(model.Work)
m.Unmarshal(work)
r.run(work)
}
handleSignals()
backoff := c.Duration("backoff")
for {
// dial the drone server to establish a TCP connection.
client, err = stomp.Dial(server)
if err != nil {
logger.Warningf("connection failed, retry in %v. %s", backoff, err)
<-time.After(backoff)
continue
}
opts := []stomp.MessageOption{
stomp.WithCredentials("x-token", accessToken),
}
// initialize the stomp session and authenticate.
if err = client.Connect(opts...); err != nil {
logger.Warningf("session failed, retry in %v. %s", backoff, err)
<-time.After(backoff)
continue
}
opts = []stomp.MessageOption{
stomp.WithAck("client"),
stomp.WithPrefetch(
c.Int("docker-max-procs"),
),
}
if filter := c.String("filter"); filter != "" {
opts = append(opts, stomp.WithSelector(filter))
}
// subscribe to the pending build queue.
client.Subscribe("/queue/pending", stomp.HandlerFunc(func(m *stomp.Message) {
go handler(m) // HACK until we a channel based Subscribe implementation
}), opts...)
logger.Noticef("connection established, ready to process builds.")
<-client.Done()
logger.Warningf("connection interrupted, attempting to reconnect.")
}
}