func (db *ETCDDB) resolveActualsWithMissingCells(logger lager.Logger, desired *models.DesiredLRP, actual *models.ActualLRP, starts *startRequests) func() { return func() { logger = logger.Session("start-missing-actual", lager.Data{ "process-guid": actual.ProcessGuid, "index": actual.Index, }) logger.Debug("removing-actual-lrp") removeErr := db.RemoveActualLRP(logger, actual.ActualLRPKey.ProcessGuid, actual.ActualLRPKey.Index) if removeErr != nil { logger.Error("failed-removing-actual-lrp", removeErr) return } logger.Debug("succeeded-removing-actual-lrp") if actual.Index >= desired.Instances { return } logger.Debug("creating-actual-lrp") err := db.createActualLRP(logger, desired, actual.Index) if err != nil { logger.Error("failed-creating-actual-lrp", err) return } logger.Debug("succeeded-creating-actual-lrp") starts.Add(logger, &actual.ActualLRPKey) } }
func configure(logger lager.Logger) (*ssh.ServerConfig, error) { errorStrings := []string{} sshConfig := &ssh.ServerConfig{} key, err := acquireHostKey(logger) if err != nil { logger.Error("failed-to-acquire-host-key", err) errorStrings = append(errorStrings, err.Error()) } sshConfig.AddHostKey(key) sshConfig.NoClientAuth = *allowUnauthenticatedClients if *authorizedKey == "" && !*allowUnauthenticatedClients { logger.Error("authorized-key-required", nil) errorStrings = append(errorStrings, "Public user key is required") } if *authorizedKey != "" { decodedPublicKey, err := decodeAuthorizedKey(logger) if err == nil { authenticator := authenticators.NewPublicKeyAuthenticator(decodedPublicKey) sshConfig.PublicKeyCallback = authenticator.Authenticate } else { errorStrings = append(errorStrings, err.Error()) } } err = nil if len(errorStrings) > 0 { err = errors.New(strings.Join(errorStrings, ", ")) } return sshConfig, err }
func (s *Scheduler) TriggerImmediately(logger lager.Logger, job atc.JobConfig, resources atc.ResourceConfigs, resourceTypes atc.ResourceTypes) (db.Build, Waiter, error) { logger = logger.Session("trigger-immediately") build, err := s.PipelineDB.CreateJobBuild(job.Name) if err != nil { logger.Error("failed-to-create-build", err) return db.Build{}, nil, err } jobService, err := NewJobService(job, s.PipelineDB, s.Scanner) if err != nil { return db.Build{}, nil, err } wg := new(sync.WaitGroup) wg.Add(1) // do not block request on scanning input versions go func() { defer wg.Done() s.ScheduleAndResumePendingBuild(logger, nil, build, job, resources, resourceTypes, jobService) }() return build, wg, nil }
func ProxyRequests(logger lager.Logger, channelType string, reqs <-chan *ssh.Request, channel ssh.Channel) { logger = logger.Session("proxy-requests", lager.Data{ "channel-type": channelType, }) logger.Info("started") defer logger.Info("completed") defer channel.Close() for req := range reqs { logger.Info("request", lager.Data{ "type": req.Type, "wantReply": req.WantReply, "payload": req.Payload, }) success, err := channel.SendRequest(req.Type, req.WantReply, req.Payload) if err != nil { logger.Error("send-request-failed", err) continue } if req.WantReply { req.Reply(success, nil) } } }
func (build *execBuild) Resume(logger lager.Logger) { stepFactory := build.buildStepFactory(logger, build.metadata.Plan) source := stepFactory.Using(&exec.NoopStep{}, exec.NewSourceRepository()) defer source.Release() process := ifrit.Background(source) exited := process.Wait() aborted := false var succeeded exec.Success for { select { case err := <-exited: if aborted { succeeded = false } else if !source.Result(&succeeded) { logger.Error("step-had-no-result", errors.New("step failed to provide us with a result")) succeeded = false } build.delegate.Finish(logger.Session("finish"), err, succeeded, aborted) return case sig := <-build.signals: process.Signal(sig) if sig == os.Kill { aborted = true } } } }
func (db *serviceClient) Cells(logger lager.Logger) (models.CellSet, error) { kvPairs, _, err := db.consulClient.KV().List(CellSchemaRoot(), nil) if err != nil { bbsErr := models.ConvertError(convertConsulError(err)) if bbsErr.Type != models.Error_ResourceNotFound { return nil, bbsErr } } if kvPairs == nil { err = consuladapter.NewPrefixNotFoundError(CellSchemaRoot()) bbsErr := models.ConvertError(convertConsulError(err)) if bbsErr.Type != models.Error_ResourceNotFound { return nil, bbsErr } } cellPresences := models.NewCellSet() for _, kvPair := range kvPairs { if kvPair.Session == "" { continue } cell := kvPair.Value presence := new(models.CellPresence) err := models.FromJSON(cell, presence) if err != nil { logger.Error("failed-to-unmarshal-cells-json", err) continue } cellPresences.Add(presence) } return cellPresences, nil }
func initializeDropsonde(logger lager.Logger) { dropsondeDestination := fmt.Sprint("localhost:", *dropsondePort) err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin) if err != nil { logger.Error("failed to initialize dropsonde: %v", err) } }
func (db *ETCDDB) resolveRestartableCrashedActualLRPS(logger lager.Logger, actualLRP *models.ActualLRP, starts *startRequests) func() { return func() { actualKey := actualLRP.ActualLRPKey logger = logger.Session("restart-crash", lager.Data{ "process_guid": actualKey.ProcessGuid, "index": actualKey.Index, }) if actualLRP.State != models.ActualLRPStateCrashed { logger.Error("failed-actual-lrp-state-is-not-crashed", nil) return } logger.Debug("unclaiming-actual-lrp", lager.Data{"process_guid": actualLRP.ActualLRPKey.ProcessGuid, "index": actualLRP.ActualLRPKey.Index}) _, err := db.unclaimActualLRP(logger, &actualLRP.ActualLRPKey, &actualLRP.ActualLRPInstanceKey) if err != nil { logger.Error("failed-unclaiming-crash", err) return } logger.Debug("succeeded-unclaiming-actual-lrp") starts.Add(logger, &actualKey) } }
func (h *DesireAppHandler) createDesiredApp( logger lager.Logger, desireAppMessage cc_messages.DesireAppRequestFromCC, ) error { var builder recipebuilder.RecipeBuilder = h.recipeBuilders["buildpack"] if desireAppMessage.DockerImageUrl != "" { builder = h.recipeBuilders["docker"] } desiredLRP, err := builder.Build(&desireAppMessage) if err != nil { logger.Error("failed-to-build-recipe", err) return err } logger.Debug("creating-desired-lrp", lager.Data{"routes": sanitizeRoutes(desiredLRP.Routes)}) err = h.bbsClient.DesireLRP(desiredLRP) if err != nil { logger.Error("failed-to-create-lrp", err) return err } logger.Debug("created-desired-lrp") return nil }
func (db *ETCDDB) ConvergeLRPs(logger lager.Logger, cellSet models.CellSet) ([]*auctioneer.LRPStartRequest, []*models.ActualLRPKeyWithSchedulingInfo, []*models.ActualLRPKey) { convergeStart := db.clock.Now() convergeLRPRunsCounter.Increment() logger = logger.Session("etcd") logger.Info("starting-convergence") defer logger.Info("finished-convergence") defer func() { err := convergeLRPDuration.Send(time.Since(convergeStart)) if err != nil { logger.Error("failed-sending-converge-lrp-duration-metric", err) } }() logger.Debug("gathering-convergence-input") input, err := db.GatherAndPruneLRPs(logger, cellSet) if err != nil { logger.Error("failed-gathering-convergence-input", err) return nil, nil, nil } logger.Debug("succeeded-gathering-convergence-input") changes := CalculateConvergence(logger, db.clock, models.NewDefaultRestartCalculator(), input) return db.ResolveConvergence(logger, input.DesiredLRPs, changes) }
func (db *ETCDDB) batchDeleteNodes(keys []string, logger lager.Logger) { if len(keys) == 0 { return } works := []func(){} for _, key := range keys { key := key works = append(works, func() { logger.Info("deleting", lager.Data{"key": key}) _, err := db.client.Delete(key, true) if err != nil { logger.Error("failed-to-delete", err, lager.Data{ "key": key, }) } }) } throttler, err := workpool.NewThrottler(db.convergenceWorkersSize, works) if err != nil { logger.Error("failed-to-create-throttler", err) } throttler.Work() return }
func (s *Scheduler) TryNextPendingBuild(logger lager.Logger, job atc.JobConfig, resources atc.ResourceConfigs) Waiter { logger = logger.Session("try-next-pending") wg := new(sync.WaitGroup) wg.Add(1) go func() { defer wg.Done() build, err := s.PipelineDB.GetNextPendingBuild(job.Name) if err != nil { if err == db.ErrNoBuild { return } logger.Error("failed-to-get-next-pending-build", err) return } s.scheduleAndResumePendingBuild(logger, build, job, resources) }() return wg }
func provision(serviceBroker ServiceBroker, router httpRouter, logger lager.Logger) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { vars := router.Vars(req) instanceID := vars["instance_id"] acceptsIncomplete := false if req.URL.Query().Get("accepts_incomplete") == "true" { acceptsIncomplete = true } logger := logger.Session(provisionLogKey, lager.Data{ instanceIDLogKey: instanceID, }) var details ProvisionDetails if err := json.NewDecoder(req.Body).Decode(&details); err != nil { logger.Error(invalidProvisionDetailsErrorKey, err) respond(w, http.StatusBadRequest, ErrorResponse{ Description: err.Error(), }) return } logger = logger.WithData(lager.Data{ provisionDetailsLogKey: details, }) provisioningResponse, asynch, err := serviceBroker.Provision(instanceID, details, acceptsIncomplete) if err != nil { switch err { case ErrInstanceAlreadyExists: logger.Error(instanceAlreadyExistsErrorKey, err) respond(w, http.StatusConflict, EmptyResponse{}) case ErrInstanceLimitMet: logger.Error(instanceLimitReachedErrorKey, err) respond(w, http.StatusInternalServerError, ErrorResponse{ Description: err.Error(), }) case ErrAsyncRequired: logger.Error(instanceAsyncRequiredErrorKey, err) respond(w, statusUnprocessableEntity, ErrorResponse{ Error: "AsyncRequired", Description: err.Error(), }) default: logger.Error(unknownErrorKey, err) respond(w, http.StatusInternalServerError, ErrorResponse{ Description: err.Error(), }) } return } if asynch { respond(w, http.StatusAccepted, provisioningResponse) return } respond(w, http.StatusCreated, provisioningResponse) } }
func lastOperation(serviceBroker ServiceBroker, router httpRouter, logger lager.Logger) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { vars := router.Vars(req) instanceID := vars["instance_id"] logger := logger.Session(lastOperationLogKey, lager.Data{ instanceIDLogKey: instanceID, }) lastOperationResponse, err := serviceBroker.LastOperation(instanceID) if err != nil { switch err { case ErrInstanceDoesNotExist: logger.Error(instanceMissingErrorKey, err) respond(w, http.StatusGone, EmptyResponse{}) default: logger.Error(unknownErrorKey, err) respond(w, http.StatusInternalServerError, ErrorResponse{ Description: err.Error(), }) } return } respond(w, http.StatusOK, lastOperationResponse) } }
func newVolume(logger lager.Logger, bcVol baggageclaim.Volume, clock clock.Clock, db VolumeFactoryDB) (Volume, bool, error) { vol := &volume{ Volume: bcVol, db: db, heartbeating: new(sync.WaitGroup), release: make(chan *time.Duration, 1), } ttl, found, err := vol.db.GetVolumeTTL(vol.Handle()) if err != nil { logger.Error("failed-to-lookup-expiration-of-volume", err) return nil, false, err } if !found { return nil, false, nil } vol.heartbeat(logger.Session("initial-heartbeat"), ttl) vol.heartbeating.Add(1) go vol.heartbeatContinuously( logger.Session("continuous-heartbeat"), clock.NewTicker(volumeKeepalive), ttl, ) return vol, true, nil }
func (b *Bulker) sync(logger lager.Logger) { logger = logger.Session("sync") logger.Info("starting") defer logger.Info("finished") startTime := b.clock.Now() ops, batchError := b.generator.BatchOperations(logger) endTime := b.clock.Now() sendError := repBulkSyncDuration.Send(endTime.Sub(startTime)) if sendError != nil { logger.Error("failed-to-send-rep-bulk-sync-duration-metric", sendError) } if batchError != nil { logger.Error("failed-to-generate-operations", batchError) return } for _, operation := range ops { b.queue.Push(operation) } }
func NewHandler( logger lager.Logger, pipelineDBFactory db.PipelineDBFactory, pipelineHandler func(db.PipelineDB) http.Handler, template *template.Template, ) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { pipelineDB, err := pipelineDBFactory.BuildDefault() if err != nil { if err == db.ErrNoPipelines { err = template.Execute(w, TemplateData{}) if err != nil { log.Fatal("failed-to-task-template", err, lager.Data{}) } return } logger.Error("failed-to-load-pipelinedb", err) w.WriteHeader(http.StatusInternalServerError) return } pipelineHandler(pipelineDB).ServeHTTP(w, r) }) }
func (s *CgroupStarter) mountCgroup(log lager.Logger, cgroupPath, cgroupType string) error { log = log.Session("setup-cgroup", lager.Data{ "path": cgroupPath, "type": cgroupType, }) log.Info("started") defer log.Info("finished") if !s.isMountPoint(cgroupPath) { if err := os.MkdirAll(cgroupPath, 0755); err != nil { log.Error("mkdir-failed", err) return err } cmd := exec.Command("mount", "-n", "-t", "cgroup", "-o", cgroupType, "cgroup", cgroupPath) cmd.Stderr = logging.Writer(log.Session("mount-cgroup-cmd")) if err := s.CommandRunner.Run(cmd); err != nil { log.Error("mount-cgroup-failed", err) return err } } return nil }
func (tc *CCTaskClient) FailTask(logger lager.Logger, taskState *cc_messages.CCTaskState, httpClient *http.Client) error { taskGuid := taskState.TaskGuid payload, err := json.Marshal(cc_messages.TaskFailResponseForCC{ TaskGuid: taskGuid, Failed: true, FailureReason: "Unable to determine completion status", }) if err != nil { logger.Error("failed-to-marshal", err, lager.Data{"task_guid": taskGuid}) return err } req, err := http.NewRequest("POST", taskState.CompletionCallbackUrl, bytes.NewReader(payload)) if err != nil { return err } req.Header.Set("Content-Type", "application/json") resp, err := httpClient.Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { logger.Error("bad-response-from-cc", err, lager.Data{"task_guid": taskGuid}) return errors.New("received bad response from CC ") } return nil }
func (db *ETCDDB) UnclaimActualLRP(logger lager.Logger, key *models.ActualLRPKey) (*models.ActualLRPGroup, *models.ActualLRPGroup, error) { actualLRP, modifiedIndex, err := db.rawActualLRPByProcessGuidAndIndex(logger, key.ProcessGuid, key.Index) bbsErr := models.ConvertError(err) if bbsErr != nil { return nil, nil, bbsErr } beforeActualLRP := *actualLRP if actualLRP.State == models.ActualLRPStateUnclaimed { logger.Debug("already-unclaimed") return nil, nil, models.ErrActualLRPCannotBeUnclaimed } actualLRP.State = models.ActualLRPStateUnclaimed actualLRP.ActualLRPKey = *key actualLRP.ActualLRPInstanceKey = models.ActualLRPInstanceKey{} actualLRP.ActualLRPNetInfo = models.EmptyActualLRPNetInfo() actualLRP.Since = db.clock.Now().UnixNano() actualLRP.ModificationTag.Increment() data, err := db.serializeModel(logger, actualLRP) if err != nil { return nil, nil, err } _, err = db.client.CompareAndSwap(ActualLRPSchemaPath(key.ProcessGuid, key.Index), data, 0, modifiedIndex) if err != nil { logger.Error("failed-compare-and-swap", err) return nil, nil, ErrorFromEtcdError(logger, err) } return &models.ActualLRPGroup{Instance: &beforeActualLRP}, &models.ActualLRPGroup{Instance: actualLRP}, nil }
func (db *ETCDDB) batchDeleteTasks(taskGuids []string, logger lager.Logger) { if len(taskGuids) == 0 { return } works := []func(){} for _, taskGuid := range taskGuids { taskGuid := taskGuid works = append(works, func() { _, err := db.client.Delete(taskGuid, true) if err != nil { logger.Error("failed-to-delete", err, lager.Data{ "task-guid": taskGuid, }) } }) } throttler, err := workpool.NewThrottler(db.convergenceWorkersSize, works) if err != nil { logger.Error("failed-to-create-throttler", err) } throttler.Work() return }
func (db *ETCDDB) FailActualLRP(logger lager.Logger, key *models.ActualLRPKey, errorMessage string) (*models.ActualLRPGroup, *models.ActualLRPGroup, error) { logger = logger.WithData(lager.Data{"actual_lrp_key": key, "error_message": errorMessage}) logger.Info("starting") lrp, prevIndex, err := db.rawActualLRPByProcessGuidAndIndex(logger, key.ProcessGuid, key.Index) if err != nil { logger.Error("failed-to-get-actual-lrp", err) return nil, nil, err } beforeActualLRP := *lrp if lrp.State != models.ActualLRPStateUnclaimed { return nil, nil, models.ErrActualLRPCannotBeFailed } lrp.ModificationTag.Increment() lrp.PlacementError = errorMessage lrp.Since = db.clock.Now().UnixNano() lrpData, serialErr := db.serializeModel(logger, lrp) if serialErr != nil { return nil, nil, serialErr } _, err = db.client.CompareAndSwap(ActualLRPSchemaPath(key.ProcessGuid, key.Index), lrpData, 0, prevIndex) if err != nil { logger.Error("failed", err) return nil, nil, models.ErrActualLRPCannotBeFailed } logger.Info("succeeded") return &models.ActualLRPGroup{Instance: &beforeActualLRP}, &models.ActualLRPGroup{Instance: lrp}, nil }
func (db *SQLDB) countTasksByState(logger lager.Logger, q Queryable) (pendingCount, runningCount, completedCount, resolvingCount int) { var query string switch db.flavor { case Postgres: query = ` SELECT COUNT(*) FILTER (WHERE state = $1) AS pending_tasks, COUNT(*) FILTER (WHERE state = $2) AS running_tasks, COUNT(*) FILTER (WHERE state = $3) AS completed_tasks, COUNT(*) FILTER (WHERE state = $4) AS resolving_tasks FROM tasks ` case MySQL: query = ` SELECT COUNT(IF(state = ?, 1, NULL)) AS pending_tasks, COUNT(IF(state = ?, 1, NULL)) AS running_tasks, COUNT(IF(state = ?, 1, NULL)) AS completed_tasks, COUNT(IF(state = ?, 1, NULL)) AS resolving_tasks FROM tasks ` default: // totally shouldn't happen panic("database flavor not implemented: " + db.flavor) } row := db.db.QueryRow(query, models.Task_Pending, models.Task_Running, models.Task_Completed, models.Task_Resolving) err := row.Scan(&pendingCount, &runningCount, &completedCount, &resolvingCount) if err != nil { logger.Error("failed-counting-tasks", err) } return }
func (db *ETCDDB) rawActualLRPGroupByProcessGuidAndIndex(logger lager.Logger, processGuid string, index int32) (*models.ActualLRPGroup, error) { node, err := db.fetchRecursiveRaw(logger, ActualLRPIndexDir(processGuid, index)) if err != nil { return nil, err } group := models.ActualLRPGroup{} for _, instanceNode := range node.Nodes { var lrp models.ActualLRP deserializeErr := db.deserializeModel(logger, instanceNode, &lrp) if deserializeErr != nil { logger.Error("failed-parsing-actual-lrp", deserializeErr, lager.Data{"key": instanceNode.Key}) return nil, deserializeErr } if isInstanceActualLRPNode(instanceNode) { group.Instance = &lrp } if isEvacuatingActualLRPNode(instanceNode) { group.Evacuating = &lrp } } if group.Evacuating == nil && group.Instance == nil { return nil, models.ErrResourceNotFound } return &group, nil }
func (engine *execEngine) LookupBuild(logger lager.Logger, model db.Build) (Build, error) { var metadata execMetadata err := json.Unmarshal([]byte(model.EngineMetadata), &metadata) if err != nil { logger.Error("invalid-metadata", err) return nil, err } err = atc.NewPlanTraversal(engine.convertPipelineNameToID).Traverse(&metadata.Plan) if err != nil { return nil, err } return &execBuild{ buildID: model.ID, stepMetadata: buildMetadata(model, engine.externalURL), db: engine.db, factory: engine.factory, delegate: engine.delegateFactory.Delegate(model.ID, model.PipelineID), metadata: metadata, signals: make(chan os.Signal, 1), }, nil }
func (db *ETCDDB) createActualLRP(logger lager.Logger, desiredLRP *models.DesiredLRP, index int32) error { logger = logger.Session("create-actual-lrp") var err error if index >= desiredLRP.Instances { err = models.NewError(models.Error_InvalidRecord, "Index too large") logger.Error("actual-lrp-index-too-large", err, lager.Data{"actual_index": index, "desired_instances": desiredLRP.Instances}) return err } guid, err := uuid.NewV4() if err != nil { return err } actualLRP := &models.ActualLRP{ ActualLRPKey: models.NewActualLRPKey( desiredLRP.ProcessGuid, index, desiredLRP.Domain, ), State: models.ActualLRPStateUnclaimed, Since: db.clock.Now().UnixNano(), ModificationTag: models.ModificationTag{ Epoch: guid.String(), Index: 0, }, } err = db.createRawActualLRP(logger, actualLRP) if err != nil { return err } return nil }
func (s *Scheduler) TryNextPendingBuild(logger lager.Logger, versions *algorithm.VersionsDB, job atc.JobConfig, resources atc.ResourceConfigs, resourceTypes atc.ResourceTypes) Waiter { logger = logger.Session("try-next-pending") wg := new(sync.WaitGroup) wg.Add(1) go func() { defer wg.Done() build, found, err := s.PipelineDB.GetNextPendingBuild(job.Name) if err != nil { logger.Error("failed-to-get-next-pending-build", err) return } if !found { return } jobService, err := NewJobService(job, s.PipelineDB, s.Scanner) if err != nil { logger.Error("failed-to-get-job-service", err) return } s.ScheduleAndResumePendingBuild(logger, versions, build, job, resources, resourceTypes, jobService) }() return wg }
func (v *volume) heartbeatContinuously(logger lager.Logger, pacemaker clock.Ticker, initialTTL time.Duration) { defer v.heartbeating.Done() defer pacemaker.Stop() logger.Debug("start") defer logger.Debug("done") ttlToSet := initialTTL for { select { case <-pacemaker.C(): ttl, found, err := v.db.GetVolumeTTL(v.Handle()) if err != nil { logger.Error("failed-to-lookup-ttl", err) } else { if !found { logger.Info("volume-expired-from-database") return } ttlToSet = ttl } v.heartbeat(logger.Session("tick"), ttlToSet) case finalTTL := <-v.release: if finalTTL != nil { v.heartbeat(logger.Session("final"), *finalTTL) } return } } }
func (tracker *tracker) Init(logger lager.Logger, metadata Metadata, session Session, typ ResourceType, tags atc.Tags) (Resource, error) { logger = logger.Session("init") logger.Debug("start") defer logger.Debug("done") container, found, err := tracker.workerClient.FindContainerForIdentifier(logger, session.ID) if err != nil { logger.Error("failed-to-look-for-existing-container", err) return nil, err } if found { logger.Debug("found-existing-container", lager.Data{"container": container.Handle()}) return NewResource(container), nil } logger.Debug("creating-container") container, err = tracker.workerClient.CreateContainer(logger, session.ID, worker.ResourceTypeContainerSpec{ Type: string(typ), Ephemeral: session.Ephemeral, Tags: tags, Env: metadata.Env(), }) if err != nil { return nil, err } logger.Info("created", lager.Data{"container": container.Handle()}) return NewResource(container), nil }
func fetchStateAndBuildZones(logger lager.Logger, workPool *workpool.WorkPool, clients map[string]rep.Client, metricEmitter auctiontypes.AuctionMetricEmitterDelegate) map[string]Zone { wg := &sync.WaitGroup{} zones := map[string]Zone{} lock := &sync.Mutex{} wg.Add(len(clients)) for guid, client := range clients { guid, client := guid, client workPool.Submit(func() { defer wg.Done() state, err := client.State() if err != nil { metricEmitter.FailedCellStateRequest() logger.Error("failed-to-get-state", err, lager.Data{"cell-guid": guid}) return } if state.Evacuating { return } cell := NewCell(logger, guid, client, state) lock.Lock() zones[state.Zone] = append(zones[state.Zone], cell) lock.Unlock() }) } wg.Wait() return zones }