func (self *EtcdRouterRegistry) ListServicesForTenant(tenant string) (*model.Bundles, error) {
children, err := self.listSubkeys(self.basePath + "/service/")
if err != nil {
log.Warn("Error listing subkeys in etcd", err)
return nil, err
}
tenantChildren := []string{}
if tenant != "" {
tenantChildren, err = self.listSubkeys(self.basePath + "/tenant/" + tenant)
if err != nil {
log.Warn("Error listing subkeys in etcd", err)
return nil, err
}
}
bundles := &model.Bundles{}
bundles.Bundles = []model.Bundle{}
all := append(children, tenantChildren...)
for _, child := range all {
bundle := &model.Bundle{}
bundle.Id = child
bundle.Name = child
bundles.Bundles = append(bundles.Bundles, *bundle)
}
return bundles, nil
}
// Gets any log entries for the instance
func (self *Instance) GetLog() (*model.LogData, error) {
jujuClient := self.GetJujuClient()
service := self.primaryServiceId
logStore, err := jujuClient.GetLogStore()
if err != nil {
log.Warn("Error fetching Juju log store", err)
return nil, err
}
// TODO: Expose units?
unitId := 0
logfile, err := logStore.ReadLog(service, unitId)
if err != nil {
log.Warn("Error reading log: %v", unitId, err)
return nil, err
}
if logfile == nil {
log.Warn("Log not found: %v", unitId)
return nil, nil
}
data := &model.LogData{}
data.Lines = make([]string, 0)
logfile.ReadLines(func(line string) (bool, error) {
data.Lines = append(data.Lines, line)
return true, nil
})
return data, nil
}
// Delete any relation properties relating to the specified unit; that unit is going away.
func (self *Instance) DeleteRelationInfo(unitId string, relationId string) error {
jujuClient := self.GetJujuClient()
serviceId := self.primaryServiceId
prefix := ANNOTATION_PREFIX_RELATIONINFO + unitId + "_" + relationId + "_"
annotations, err := jujuClient.GetServiceAnnotations(serviceId)
if err != nil {
log.Warn("Error getting annotations", err)
// TODO: Mask error?
return err
}
deleteKeys := []string{}
for tagName, _ := range annotations {
if !strings.HasPrefix(tagName, prefix) {
continue
}
deleteKeys = append(deleteKeys, tagName)
}
if len(deleteKeys) != 0 {
log.Info("Deleting annotations on service %v: %v", serviceId, deleteKeys)
err = jujuClient.DeleteServiceAnnotations(serviceId, deleteKeys)
if err != nil {
log.Warn("Error deleting annotations", err)
return err
}
}
return nil
}
func waitReady(instance *core.Instance, timeout int) (bool, error) {
ready := false
for i := 0; i < timeout; i++ {
state, err := instance.GetState()
if err != nil {
log.Warn("Error while waiting for instance to become ready", err)
return false, err
}
if state == nil {
log.Warn("Instance not yet created")
continue
}
status := state.Status
if status == "started" {
ready = true
break
}
time.Sleep(time.Second)
if status == "pending" {
log.Debug("Instance not ready; waiting", err)
} else {
log.Warn("Unknown instance status: %v", status)
}
}
return ready, nil
}
func (self *Instance) getScalingPolicy() (*model.ScalingPolicy, error) {
jujuClient := self.GetJujuClient()
primaryServiceId := self.primaryServiceId
annotations, err := jujuClient.GetServiceAnnotations(primaryServiceId)
if err != nil {
log.Warn("Error getting annotations", err)
// TODO: Ignore?
return nil, err
}
var policy *model.ScalingPolicy
scalingPolicyJson := annotations[ANNOTATION_KEY_SCALING_POLICY]
if scalingPolicyJson == "" {
policy = self.bundleType.GetDefaultScalingPolicy()
} else {
policy = &model.ScalingPolicy{}
err = json.Unmarshal([]byte(scalingPolicyJson), policy)
if err != nil {
log.Warn("Error deserializing scaling policy (%v)", scalingPolicyJson, err)
// TODO: Ignore / go with default?
return nil, err
}
}
return policy, nil
}
func (self *EtcdRouterRegistry) read(key string) (*etcdRouterData, error) {
response, err := self.client.Get(key, false, false)
if err != nil {
etcdError, ok := err.(*etcd.EtcdError)
if ok && etcdError.ErrorCode == etcdErrorKeyNotFound {
log.Debug("Etcd key not found: %v", key)
return nil, nil
}
log.Warn("Error reading key from etcd: %v", key, err)
return nil, err
}
node := response.Node
if node == nil || node.Value == "" {
log.Info("No contents for key from etcd: %v", key)
return nil, nil
}
decoded := &etcdRouterData{}
err = json.Unmarshal([]byte(node.Value), decoded)
if err != nil {
log.Warn("Error parsing value from etcd: %v", node.Value, err)
return nil, err
}
return decoded, nil
}
func GetOptions() *Options {
flag.Parse()
self := &Options{}
self.Listen = *flagListen
registryUrl, err := url.Parse(*flagRegistryUrl)
if err != nil {
log.Warn("Unable to parse registry url: %v", *flagRegistryUrl)
return nil
}
if registryUrl.Scheme == "etcd" {
registry, err := NewEtcdRouterRegistry(registryUrl)
if err != nil {
log.Warn("Unable to build etcd registry", err)
return nil
}
self.Registry = registry
} else {
log.Warn("Unknown registry type: %v", registryUrl.Scheme)
return nil
}
return self
}
func GetOptions() *Options {
flag.Parse()
self := &Options{}
self.AgentConf = *flagAgentConf
self.ApiPasswordPath = *flagApiPasswordPath
self.CfTenantId = *flagCfTenantId
self.ListenAddress = *flagListenAddress
host, port, err := net.SplitHostPort(self.ListenAddress)
if err != nil {
log.Warn("Cannot parse listen address: %v", self.ListenAddress)
return nil
}
var portNum int
if port == "" {
portNum = 8080
} else {
portNum, err = net.LookupPort("tcp", port)
if err != nil {
log.Warn("Cannot resolve port: %v", port)
return nil
}
}
privateUrl := *flagPrivateUrl
if privateUrl == "" {
privateHost := host
if privateHost == "" {
ip, err := localIP()
if err != nil {
log.Warn("Error finding local IP", err)
return nil
}
privateHost = ip.String()
}
privateUrl = fmt.Sprintf("http://%v:%v/xaasprivate", privateHost, portNum)
log.Info("Chose private url: %v", privateUrl)
}
self.PrivateUrl = privateUrl
authMode := *flagAuth
authMode = strings.TrimSpace(authMode)
authMode = strings.ToLower(authMode)
if authMode == "openstack" {
keystoneUrl := *flagKeystoneUrl
self.Authenticator = auth.NewOpenstackMultiAuthenticator(keystoneUrl)
} else if authMode == "development" {
self.Authenticator = auth.NewDevelopmentAuthenticator()
} else {
log.Warn("Unknown authentication mode: %v", authMode)
return nil
}
return self
}
func (self *EndpointServiceBinding) HttpPut(request *CfBindRequest) (*rs.HttpResponse, error) {
service := self.getService()
if request.ServiceId != service.CfServiceId {
log.Warn("service mismatch: %v vs %v", request.ServiceId, service.CfServiceId)
return nil, rs.ErrNotFound()
}
bundleType, instance := service.getInstance(self.getInstanceId())
if instance == nil || bundleType == nil {
return nil, rs.ErrNotFound()
}
ready, err := waitReady(instance, 300)
if err != nil {
log.Warn("Error while waiting for instance to become ready", err)
return nil, err
}
if !ready {
log.Warn("Timeout waiting for service to be ready")
return nil, fmt.Errorf("Service not ready")
}
relationKey := bundleType.PrimaryRelationKey()
_, relationInfo, err := instance.GetRelationInfo(relationKey, true)
if err != nil {
return nil, err
}
if relationInfo == nil {
return nil, rs.ErrNotFound()
}
credentials, err := bundleType.MapCloudFoundryCredentials(relationInfo)
if err != nil {
log.Warn("Error mapping to CloudFoundry credentials", err)
return nil, err
}
// log.Debug("Relation info: %v", relationInfo)
// log.Debug("Mapped to CloudFoundry credentials %v", credentials)
response := &CfBindResponse{}
response.Credentials = credentials
httpResponse := &rs.HttpResponse{Status: http.StatusCreated}
httpResponse.Content = response
return httpResponse, nil
}
func (self *Huddle) cleanupOldMachines(state map[string]int, threshold int) (map[string]int, error) {
status, err := self.JujuClient.GetSystemStatus()
if err != nil {
log.Warn("Error getting system status", err)
return nil, err
}
unitsByMachine := map[string]*api.UnitStatus{}
for _, serviceStatus := range status.Services {
for _, unitStatus := range serviceStatus.Units {
machineId := unitStatus.Machine
unitsByMachine[machineId] = &unitStatus
}
}
idleMachines := map[string]*api.MachineStatus{}
for machineId, machineStatus := range status.Machines {
unit := unitsByMachine[machineId]
if unit != nil {
continue
}
idleMachines[machineId] = &machineStatus
}
idleCounts := map[string]int{}
for machineId, _ := range idleMachines {
idleCount := state[machineId]
idleCount++
idleCounts[machineId] = idleCount
}
for machineId, idleCount := range idleCounts {
if idleCount < threshold {
continue
}
if machineId == "0" {
// Machine id 0 is special (the system machine); we can't destroy it
continue
}
log.Info("Machine is idle; removing: %v", machineId)
err = self.JujuClient.DestroyMachine(machineId)
if err != nil {
log.Warn("Failed to delete machine %v", machineId, err)
}
}
return idleCounts, nil
}
func (self *EndpointServiceInstance) HttpGet() (*rs.HttpResponse, error) {
service := self.getService()
log.Info("CF instance GET request: %v", self.Id)
bundleType, instance := service.getInstance(self.Id)
if instance == nil || bundleType == nil {
return nil, rs.ErrNotFound()
}
state, err := instance.GetState()
if err != nil {
log.Warn("Error while waiting for instance to become ready", err)
return nil, err
}
ready := false
if state == nil {
log.Warn("Instance not yet created")
} else {
status := state.Status
if status == "started" {
ready = true
} else if status == "pending" {
ready = false
} else {
log.Warn("Unknown instance status: %v", status)
}
}
response := &CfCreateInstanceResponse{}
// TODO: We need a dashboard URL - maybe a Juju GUI?
response.DashboardUrl = "http://localhost:8080"
var cfState string
if ready {
cfState = CF_STATE_SUCCEEDED
} else {
cfState = CF_STATE_IN_PROGRESS
}
response.State = cfState
response.LastOperation = &CfOperation{}
response.LastOperation.State = cfState
log.Info("Sending response to CF service get", log.AsJson(response))
httpResponse := &rs.HttpResponse{Status: http.StatusOK}
httpResponse.Content = response
return httpResponse, nil
}
func (self *EtcdRouterRegistry) put(key string, data *etcdRouterData) error {
json, err := json.Marshal(data)
if err != nil {
log.Warn("Error encoding value to json", err)
return err
}
_, err = self.client.Set(key, string(json), 0)
if err != nil {
log.Warn("Error writing key to etcd: %v", key, err)
return err
}
return nil
}
// Adds a bundletype to the system, by extracting the required template from the charm itself
func (self *System) AddJxaasCharm(apiclient *juju.Client, key string, charmName string) error {
charmInfo, err := apiclient.CharmInfo(charmName)
if err != nil {
log.Warn("Error reading charm: %v", charmName, err)
return err
}
if charmInfo == nil {
return fmt.Errorf("Unable to find charm: %v", charmName)
}
url := charmInfo.URL
if url == "" {
return fmt.Errorf("Unable to find charm url: %v", charmName)
}
contents, err := apiclient.DownloadCharm(charmName)
if err != nil {
log.Warn("Error reading charm", err)
return err
}
charmFile := NewCharmReader(contents)
config, err := charmFile.read("jxaas.yaml")
if err != nil {
log.Warn("Error reading jxaas.yaml from charm: %v", charmName, err)
return err
}
if config == nil {
return fmt.Errorf("Could not find jxaas.yaml in charm: %v", charmName)
}
// log.Info("Jxaas config: %v", string(config))
bundleTemplate, err := bundle.NewBundleTemplate(sources.NewArrayToByteSource(config))
if err != nil {
return err
}
bundleType, err := bundletype.NewGenericBundleType(key, bundleTemplate)
if err != nil {
return err
}
self.BundleTypes[key] = bundleType
return nil
}
func (self *EndpointServiceInstance) HttpDelete(httpRequest *http.Request) (*CfDeleteInstanceResponse, error) {
service := self.getService()
queryValues := httpRequest.URL.Query()
serviceId := queryValues.Get("service_id")
// planId := queryValues.Get("plan_id")
if serviceId != service.CfServiceId {
log.Warn("Service mismatch: %v vs %v", serviceId, service.CfServiceId)
return nil, rs.ErrNotFound()
}
log.Info("Deleting item %v %v", serviceId, self.Id)
bundletype, instance := service.getInstance(self.getInstanceId())
if instance == nil || bundletype == nil {
return nil, rs.ErrNotFound()
}
err := instance.Delete()
if err != nil {
return nil, err
}
// TODO: Wait for deletion?
response := &CfDeleteInstanceResponse{}
return response, nil
}
func (self *InMemoryPool) Borrow(owner string) Pooled {
self.mutex.Lock()
defer self.mutex.Unlock()
if self.allocated == nil {
self.init()
}
head := self.available.Front()
if head == nil {
log.Warn("No elements left in pool")
return nil
}
self.available.Remove(head)
element := head.Value
if self.allocated[element] {
panic("Attempt to do double-allocation")
}
self.allocated[element] = true
return NewPooled(self, element)
}
func mapToOptionDescriptions(config *params.ServiceGetResults) map[string]OptionDescription {
out := make(map[string]OptionDescription)
if config.Config != nil {
for k, v := range config.Config {
m, ok := v.(map[string]interface{})
if !ok {
log.Warn("Unexpected type for config value: %v", k)
continue
}
p := &OptionDescription{}
p.Type = getString(m, "type")
p.Description = getString(m, "description")
// juju returns true if the value is the default, false otherwise,
// but does not return the actual default value. That's uninintuitive to me,
// so block it.
//p.Default = getString(m, "default")
out[k] = *p
}
}
return out
}
func (self *EtcdRouterRegistry) listSubkeys(key string) ([]string, error) {
response, err := self.client.Get(key, false, false)
if err != nil {
etcdError, ok := err.(*etcd.EtcdError)
if ok && etcdError.ErrorCode == etcdErrorKeyNotFound {
log.Debug("Etcd key not found: %v", key)
return []string{}, nil
}
log.Warn("Error reading key from etcd: %v", key, err)
return nil, err
}
if response == nil || response.Node == nil || response.Node.Nodes == nil {
log.Info("No children for key from etcd: %v", key)
return []string{}, nil
}
names := []string{}
for _, node := range response.Node.Nodes {
nodeKey := node.Key
if !strings.HasPrefix(nodeKey, key) {
return nil, fmt.Errorf("Key without expected prefix: %v vs %v", nodeKey, key)
}
suffix := nodeKey[len(key):]
names = append(names, suffix)
}
return names, nil
}
func (self *EndpointInstanceScaling) HttpPut(policyUpdate *model.ScalingPolicy) (*model.Scaling, error) {
instance := self.Parent.getInstance()
log.Info("Policy update: %v", policyUpdate)
exists, err := instance.Exists()
if err != nil {
return nil, err
}
if !exists {
return nil, rs.ErrNotFound()
}
if policyUpdate != nil {
_, err := instance.UpdateScalingPolicy(policyUpdate)
if err != nil {
log.Warn("Error updating scaling policy", err)
return nil, err
}
}
results, err := instance.RunScaling(true)
if err != nil {
return nil, err
}
return results, nil
}
func (self *ServiceMap) getSlot(config interface{}) *ServiceSlot {
key := self.keyFunction(config)
self.mutex.Lock()
defer self.mutex.Unlock()
slot := self.services[key]
if slot == nil {
channel := make(chan interface{})
service := self.factory(config, channel)
if service == nil {
log.Warn("Could not create service for key: %v", key)
return nil
}
slot = &ServiceSlot{}
slot.service = service
slot.channel = channel
self.services[key] = slot
}
return slot
}
func (self *EtcdRouterRegistry) GetBackendForTenant(service string, tenant *string) string {
var data *etcdRouterData
var err error
if tenant != nil {
key := self.keyForTenant(service, *tenant)
data, err = self.read(key)
}
if data == nil && err == nil {
key := self.keyForService(service)
data, err = self.read(key)
}
if err != nil {
log.Warn("Error reading from etcd", err)
return ""
}
if data != nil {
return data.Backend
}
return ""
}
func (self *Client) DownloadCharm(charmKey string) (sources.ByteSource, error) {
// Sadly not readable by user
charmInfo, err := self.CharmInfo(charmKey)
if err != nil {
log.Warn("Unable to get charm info: %v", charmKey, err)
return nil, err
}
environment, err := self.client.EnvironmentGet()
if err != nil {
log.Warn("Unable to get juju environment", err)
return nil, err
}
jujuType := asString(environment["type"])
if jujuType == "" {
return nil, fmt.Errorf("Could not fetch environment value 'type'")
}
rootDir := asString(environment["root-dir"])
if rootDir == "" {
return nil, fmt.Errorf("Could not fetch environment value 'type'")
}
// zipFile := "${HOME}/.juju/local/charmcache/cs_3a__7e_justin-fathomdb_2f_trusty_2f_mongodb-0.charm"
escaped := encodeCharmPath(charmInfo.URL)
filename := escaped + ".charm"
charmPath := filepath.Join(rootDir, "charmcache", filename)
if jujuType == "local" {
contents := sources.NewFileByteSource(charmPath)
exists, err := contents.Exists()
if err != nil {
return nil, err
}
if !exists {
return nil, fmt.Errorf("Charm file not found: %v", charmPath)
}
return contents, nil
} else {
return nil, fmt.Errorf("Unable to handle juju configuration type: %v", jujuType)
}
}
func (self *EtcdRouterRegistry) ListServices() ([]string, error) {
children, err := self.listSubkeys(self.basePath + "/service/")
if err != nil {
log.Warn("Error listing subkeys in etcd", err)
return nil, err
}
return children, nil
}
func (self *Instance) setScalingPolicy(policy *model.ScalingPolicy) (*model.ScalingPolicy, error) {
policyJson, err := json.Marshal(policy)
if err != nil {
log.Warn("Error serializing scaling policy", err)
return nil, err
}
pairs := map[string]string{}
pairs[ANNOTATION_KEY_SCALING_POLICY] = string(policyJson)
err = self.setServiceAnnotations(pairs)
if err != nil {
log.Warn("Error saving scaling policy", err)
return nil, err
}
return policy, nil
}
func (self *ScheduledTask) run() {
for {
time.Sleep(self.interval)
err := self.task.Run()
if err != nil {
log.Warn("Error running task %v", self.task, err)
}
}
}
func (self *CleanupOldMachines) Run() error {
state, err := self.huddle.cleanupOldMachines(self.state, self.deleteThreshold)
if err != nil {
log.Warn("Error cleaning up old machines", err)
return err
}
self.state = state
return nil
}
func (self *BundleTemplate) executeTemplate(context *TemplateContext) (map[string]interface{}, error) {
// t, err := template.New("bundle").Parse(templateString)
// if err != nil {
// return nil, err
// }
// log.Debug("Executing bundle template: %v", serviceType)
var err error
result, err := self.template.Render(context)
if err != nil {
log.Warn("Error applying template", err)
return nil, err
}
log.Debug("Applied template: %v", result)
resultMap, ok := result.(map[string]interface{})
if !ok {
log.Warn("Template did not produce map type: %T", result)
return nil, fmt.Errorf("Unexpected result from template")
}
// config := map[string]interface{}{}
// err := goyaml.Unmarshal([]byte(yaml), &config)
// if err != nil {
// return nil, err
// }
// var buffer bytes.Buffer
// err := self.template.Execute(&buffer, &templateContextCopy)
// if err != nil {
// return nil, err
// }
// yaml := buffer.String()
// log.Debug("Bundle is:\n%v", yaml)
return resultMap, nil
}
// Retrieve the relation properties.
// It doesn't seem to be possible to retrieve these direct from Juju,
// so the stubclient stores them for us.
func (self *Instance) GetRelationInfo(relationKey string, cloudfoundry bool) (*bundle.Bundle, *model.RelationInfo, error) {
serviceId := self.primaryServiceId
// Can we rationalize all this? We repeat a lot of calls right now...
state, err := self.getState0()
if err != nil {
log.Warn("Error getting instance state", err)
return nil, nil, err
}
if state == nil {
log.Warn("No status found for service: %v", serviceId)
return nil, nil, nil
}
// log.Debug("relationProperties: %v", relationProperties)
// log.Debug("relationMetadata: %v", relationMetadata)
context, err := self.buildCurrentTemplateContext(state, cloudfoundry)
if err != nil {
return nil, nil, err
}
bundle, err := self.getBundle(context)
if err != nil {
return nil, nil, err
}
relationInfo, err := self.bundleType.BuildRelationInfo(context, bundle, relationKey)
if err != nil {
return nil, nil, err
}
if relationInfo != nil {
if relationInfo.PublicAddresses == nil {
relationInfo.PublicAddresses = state.PublicAddresses
}
relationInfo.Timestamp = state.RelationMetadata[RELATIONINFO_METADATA_TIMESTAMP]
}
return bundle, relationInfo, nil
}
func (self *AutoScaleAllInstances) Run() error {
instances, err := self.huddle.ListAllInstances()
if err != nil {
log.Warn("Error listing instances", err)
return err
}
for _, instance := range instances {
scaling, err := instance.RunScaling(true)
if err != nil {
log.Warn("Error running scaling on instance: %v", instance, err)
continue
}
// TODO: Record this, so we can return scaling info from last poll through API
log.Debug("Scaling-state of %v: %v", instance, scaling)
}
return nil
}
func (self *HealthCheckAllInstances) Run() error {
instances, err := self.huddle.ListAllInstances()
if err != nil {
log.Warn("Error listing instances", err)
return err
}
for _, instance := range instances {
health, err := instance.RunHealthCheck(self.repair)
if err != nil {
log.Warn("Error running health check on instance: %v", instance, err)
continue
}
// TODO: Check health results and mark instances unhealthy??
log.Debug("Health of %v: %v", instance, health)
}
return nil
}
func MergeInstanceStatus(instance *Instance, unit *api.UnitStatus) {
unitStatus := string(unit.AgentState)
if instance.Status != unitStatus {
if instance.Status == "" {
instance.Status = unitStatus
} else {
// TODO: Resolve mixed state
log.Warn("Unable to resolve mixed state: %v vs %v", instance.Status, unitStatus)
}
}
}