func NewLock(
logger lager.Logger,
consulClient consuladapter.Client,
lockKey string,
lockValue []byte,
clock clock.Clock,
retryInterval time.Duration,
lockTTL time.Duration,
) Lock {
lockMetricName := strings.Replace(lockKey, "/", "-", -1)
uuid, err := uuid.NewV4()
if err != nil {
logger.Fatal("create-uuid-failed", err)
}
session, err := NewSessionNoChecks(uuid.String(), lockTTL, consulClient)
if err != nil {
logger.Fatal("consul-session-failed", err)
}
return Lock{
consul: session,
key: lockKey,
value: lockValue,
clock: clock,
retryInterval: retryInterval,
logger: logger,
lockAcquiredMetric: metric.Metric("LockHeld." + lockMetricName),
lockUptimeMetric: metric.Duration("LockHeldDuration." + lockMetricName),
}
}
func (db *ETCDDB) GatherAndPruneLRPs(logger lager.Logger, cellSet models.CellSet) (*models.ConvergenceInput, error) {
guids := map[string]struct{}{}
// always fetch actualLRPs before desiredLRPs to ensure correctness
logger.Debug("gathering-and-pruning-actual-lrps")
lrpMetricCounter := &LRPMetricCounter{}
actuals, err := db.gatherAndPruneActualLRPs(logger, guids, lrpMetricCounter) // modifies guids
if err != nil {
logger.Error("failed-gathering-and-pruning-actual-lrps", err)
lrpMetricCounter.unclaimedLRPs = -1
lrpMetricCounter.claimedLRPs = -1
lrpMetricCounter.runningLRPs = -1
lrpMetricCounter.crashedActualLRPs = -1
lrpMetricCounter.crashingDesiredLRPs = -1
lrpMetricCounter.desiredLRPs = -1
lrpMetricCounter.Send(logger)
return &models.ConvergenceInput{}, err
}
logger.Debug("succeeded-gathering-and-pruning-actual-lrps")
// always fetch desiredLRPs after actualLRPs to ensure correctness
logger.Debug("gathering-and-pruning-desired-lrps")
desireds, err := db.GatherAndPruneDesiredLRPs(logger, guids, lrpMetricCounter) // modifies guids
if err != nil {
logger.Error("failed-gathering-and-pruning-desired-lrps", err)
lrpMetricCounter.desiredLRPs = -1
lrpMetricCounter.Send(logger)
return &models.ConvergenceInput{}, err
}
logger.Debug("succeeded-gathering-and-pruning-desired-lrps")
lrpMetricCounter.Send(logger)
logger.Debug("listing-domains")
domains, err := db.Domains(logger)
if err != nil {
return &models.ConvergenceInput{}, err
}
logger.Debug("succeeded-listing-domains")
for _, domain := range domains {
metric.Metric("Domain." + domain).Send(1)
}
return &models.ConvergenceInput{
AllProcessGuids: guids,
DesiredLRPs: desireds,
ActualLRPs: actuals,
Domains: models.NewDomainSet(domains),
Cells: cellSet,
}, nil
}
"crypto/tls"
"encoding/json"
"errors"
"net/http"
"strconv"
"code.cloudfoundry.org/cfhttp"
"code.cloudfoundry.org/lager"
"code.cloudfoundry.org/runtimeschema/metric"
"code.cloudfoundry.org/urljoiner"
)
var errRedirected = errors.New("redirected to leader")
const (
etcdLeader = metric.Metric("ETCDLeader")
etcdReceivedBandwidthRate = metric.BytesPerSecond("ETCDReceivedBandwidthRate")
etcdSentBandwidthRate = metric.BytesPerSecond("ETCDSentBandwidthRate")
etcdReceivedRequestRate = metric.RequestsPerSecond("ETCDReceivedRequestRate")
etcdSentRequestRate = metric.RequestsPerSecond("ETCDSentRequestRate")
etcdRaftTerm = metric.Metric("ETCDRaftTerm")
etcdWatchers = metric.Metric("ETCDWatchers")
)
type ETCDMetrics struct {
logger lager.Logger
etcdCluster []string
client *http.Client
}
"code.cloudfoundry.org/clock"
"code.cloudfoundry.org/lager"
"code.cloudfoundry.org/runtimeschema/metric"
)
const (
convergeLRPRunsCounter = metric.Counter("ConvergenceLRPRuns")
convergeLRPDuration = metric.Duration("ConvergenceLRPDuration")
malformedSchedulingInfosMetric = metric.Counter("ConvergenceLRPPreProcessingMalformedSchedulingInfos")
malformedRunInfosMetric = metric.Counter("ConvergenceLRPPreProcessingMalformedRunInfos")
actualLRPsDeleted = metric.Counter("ConvergenceLRPPreProcessingActualLRPsDeleted")
orphanedRunInfosMetric = metric.Counter("ConvergenceLRPPreProcessingOrphanedRunInfos")
desiredLRPs = metric.Metric("LRPsDesired")
claimedLRPs = metric.Metric("LRPsClaimed")
unclaimedLRPs = metric.Metric("LRPsUnclaimed")
runningLRPs = metric.Metric("LRPsRunning")
missingLRPs = metric.Metric("LRPsMissing")
extraLRPs = metric.Metric("LRPsExtra")
crashedActualLRPs = metric.Metric("CrashedActualLRPs")
crashingDesiredLRPs = metric.Metric("CrashingDesiredLRPs")
)
func (db *ETCDDB) ConvergeLRPs(logger lager.Logger, cellSet models.CellSet) ([]*auctioneer.LRPStartRequest, []*models.ActualLRPKeyWithSchedulingInfo, []*models.ActualLRPKey) {
convergeStart := db.clock.Now()
convergeLRPRunsCounter.Increment()
logger = logger.Session("etcd")
"code.cloudfoundry.org/auctioneer"
"code.cloudfoundry.org/bbs/models"
"code.cloudfoundry.org/lager"
"code.cloudfoundry.org/runtimeschema/metric"
"code.cloudfoundry.org/workpool"
)
const (
convergeTaskRunsCounter = metric.Counter("ConvergenceTaskRuns")
convergeTaskDuration = metric.Duration("ConvergenceTaskDuration")
tasksKickedCounter = metric.Counter("ConvergenceTasksKicked")
tasksPrunedCounter = metric.Counter("ConvergenceTasksPruned")
pendingTasks = metric.Metric("TasksPending")
runningTasks = metric.Metric("TasksRunning")
completedTasks = metric.Metric("TasksCompleted")
resolvingTasks = metric.Metric("TasksResolving")
)
type compareAndSwappableTask struct {
OldIndex uint64
NewTask *models.Task
}
func (db *ETCDDB) ConvergeTasks(
logger lager.Logger,
cellSet models.CellSet,
kickTaskDuration, expirePendingTaskDuration, expireCompletedTaskDuration time.Duration,
) ([]*auctioneer.TaskStartRequest, []*models.Task) {
func (db *SQLDB) emitDomainMetrics(logger lager.Logger, domainSet map[string]struct{}) {
for domain := range domainSet {
metric.Metric("Domain." + domain).Send(1)
}
}
"time"
"code.cloudfoundry.org/auctioneer"
"code.cloudfoundry.org/bbs/models"
"code.cloudfoundry.org/lager"
"code.cloudfoundry.org/runtimeschema/metric"
"code.cloudfoundry.org/workpool"
)
const (
convergeLRPRunsCounter = metric.Counter("ConvergenceLRPRuns")
convergeLRPDuration = metric.Duration("ConvergenceLRPDuration")
domainMetricPrefix = "Domain."
instanceLRPs = metric.Metric("LRPsDesired") // this is the number of desired instances
claimedLRPs = metric.Metric("LRPsClaimed")
unclaimedLRPs = metric.Metric("LRPsUnclaimed")
runningLRPs = metric.Metric("LRPsRunning")
missingLRPs = metric.Metric("LRPsMissing")
extraLRPs = metric.Metric("LRPsExtra")
crashedActualLRPs = metric.Metric("CrashedActualLRPs")
crashingDesiredLRPs = metric.Metric("CrashingDesiredLRPs")
)
func (db *SQLDB) ConvergeLRPs(logger lager.Logger, cellSet models.CellSet) ([]*auctioneer.LRPStartRequest, []*models.ActualLRPKeyWithSchedulingInfo, []*models.ActualLRPKey) {
convergeStart := db.clock.Now()
convergeLRPRunsCounter.Increment()
logger.Info("starting")