func (self *EndpointServiceInstance) HttpDelete(httpRequest *http.Request) (*CfDeleteInstanceResponse, error) {
service := self.getService()
queryValues := httpRequest.URL.Query()
serviceId := queryValues.Get("service_id")
// planId := queryValues.Get("plan_id")
if serviceId != service.CfServiceId {
log.Warn("Service mismatch: %v vs %v", serviceId, service.CfServiceId)
return nil, rs.ErrNotFound()
}
log.Info("Deleting item %v %v", serviceId, self.Id)
bundletype, instance := service.getInstance(self.getInstanceId())
if instance == nil || bundletype == nil {
return nil, rs.ErrNotFound()
}
err := instance.Delete()
if err != nil {
return nil, err
}
// TODO: Wait for deletion?
response := &CfDeleteInstanceResponse{}
return response, nil
}
func (self *EndpointServiceBinding) HttpPut(request *CfBindRequest) (*rs.HttpResponse, error) {
service := self.getService()
if request.ServiceId != service.CfServiceId {
log.Warn("service mismatch: %v vs %v", request.ServiceId, service.CfServiceId)
return nil, rs.ErrNotFound()
}
bundleType, instance := service.getInstance(self.getInstanceId())
if instance == nil || bundleType == nil {
return nil, rs.ErrNotFound()
}
ready, err := waitReady(instance, 300)
if err != nil {
log.Warn("Error while waiting for instance to become ready", err)
return nil, err
}
if !ready {
log.Warn("Timeout waiting for service to be ready")
return nil, fmt.Errorf("Service not ready")
}
relationKey := bundleType.PrimaryRelationKey()
_, relationInfo, err := instance.GetRelationInfo(relationKey, true)
if err != nil {
return nil, err
}
if relationInfo == nil {
return nil, rs.ErrNotFound()
}
credentials, err := bundleType.MapCloudFoundryCredentials(relationInfo)
if err != nil {
log.Warn("Error mapping to CloudFoundry credentials", err)
return nil, err
}
// log.Debug("Relation info: %v", relationInfo)
// log.Debug("Mapped to CloudFoundry credentials %v", credentials)
response := &CfBindResponse{}
response.Credentials = credentials
httpResponse := &rs.HttpResponse{Status: http.StatusCreated}
httpResponse.Content = response
return httpResponse, nil
}
func (self *EndpointInstanceScaling) HttpPut(policyUpdate *model.ScalingPolicy) (*model.Scaling, error) {
instance := self.Parent.getInstance()
log.Info("Policy update: %v", policyUpdate)
exists, err := instance.Exists()
if err != nil {
return nil, err
}
if !exists {
return nil, rs.ErrNotFound()
}
if policyUpdate != nil {
_, err := instance.UpdateScalingPolicy(policyUpdate)
if err != nil {
log.Warn("Error updating scaling policy", err)
return nil, err
}
}
results, err := instance.RunScaling(true)
if err != nil {
return nil, err
}
return results, nil
}
func (self *EndpointInstance) HttpGet() (*model.Instance, error) {
model, err := self.getInstance().GetState()
if err == nil && model == nil {
return nil, rs.ErrNotFound()
}
return model, err
}
// Ensures the instance is created and has the specified configuration.
// This method is (supposed to be) idempotent.
func (self *Instance) Configure(request *model.Instance) error {
var err error
jujuClient := self.GetJujuClient()
// Sanitize
request.Id = ""
request.Units = nil
// Record the (requested) configuration options
instanceConfigChanges := request.Options
// Get the existing configuration
context, err := self.buildCurrentTemplateContext(nil, false)
if err != nil {
return err
}
// Merge the new configuration options with the existing ones
if request.NumberUnits != nil {
context.NumberUnits = *request.NumberUnits
}
for k, v := range request.Options {
context.Options[k] = v
}
// Create a bundle from the new configuration
b, err := self.getBundle(context)
if err != nil {
return err
}
if b == nil {
return rs.ErrNotFound()
}
_, err = b.Deploy("", jujuClient)
if err != nil {
return err
}
// Save changed config
if instanceConfigChanges != nil {
self.setInstanceConfig(instanceConfigChanges)
}
// TODO: Is this idempotent?
publicPortAssigner := context.PublicPortAssigner
port, assigned := publicPortAssigner.GetAssignedPort()
if assigned {
self.setPublicPort(port)
}
return nil
}
func (self *EndpointInstanceHealth) HttpPut() (*model.Health, error) {
instance := self.Parent.getInstance()
repair := true
health, err := instance.RunHealthCheck(repair)
if err != nil {
return nil, err
}
if health == nil {
return nil, rs.ErrNotFound()
}
return health, nil
}
func (self *EndpointServiceInstance) HttpGet() (*rs.HttpResponse, error) {
service := self.getService()
log.Info("CF instance GET request: %v", self.Id)
bundleType, instance := service.getInstance(self.Id)
if instance == nil || bundleType == nil {
return nil, rs.ErrNotFound()
}
state, err := instance.GetState()
if err != nil {
log.Warn("Error while waiting for instance to become ready", err)
return nil, err
}
ready := false
if state == nil {
log.Warn("Instance not yet created")
} else {
status := state.Status
if status == "started" {
ready = true
} else if status == "pending" {
ready = false
} else {
log.Warn("Unknown instance status: %v", status)
}
}
response := &CfCreateInstanceResponse{}
// TODO: We need a dashboard URL - maybe a Juju GUI?
response.DashboardUrl = "http://localhost:8080"
var cfState string
if ready {
cfState = CF_STATE_SUCCEEDED
} else {
cfState = CF_STATE_IN_PROGRESS
}
response.State = cfState
response.LastOperation = &CfOperation{}
response.LastOperation.State = cfState
log.Info("Sending response to CF service get", log.AsJson(response))
httpResponse := &rs.HttpResponse{Status: http.StatusOK}
httpResponse.Content = response
return httpResponse, nil
}
func (self *EndpointServiceBinding) HttpDelete(httpRequest *http.Request) (*CfUnbindResponse, error) {
service := self.getService()
queryValues := httpRequest.URL.Query()
serviceId := queryValues.Get("service_id")
if serviceId != service.CfServiceId {
log.Warn("service mismatch: %v vs %v", serviceId, service.CfServiceId)
return nil, rs.ErrNotFound()
}
// planId := queryValues.Get("plan_id")
bundleType, instance := service.getInstance(self.getInstanceId())
if instance == nil || bundleType == nil {
return nil, rs.ErrNotFound()
}
// TODO: actually remove something?
response := &CfUnbindResponse{}
return response, nil
}
func (self *EndpointRelation) HttpGet(apiclient *juju.Client) (*model.RelationInfo, error) {
instance := self.getInstance()
relationKey := self.RelationKey
_, relationInfo, err := instance.GetRelationInfo(relationKey, false)
if err != nil {
return nil, err
}
if relationInfo == nil {
return nil, rs.ErrNotFound()
}
return relationInfo, err
}
func (self *EndpointInstanceScaling) HttpGet() (*model.Scaling, error) {
instance := self.Parent.getInstance()
exists, err := instance.Exists()
if err != nil {
return nil, err
}
if !exists {
return nil, rs.ErrNotFound()
}
results, err := instance.RunScaling(false)
if err != nil {
return nil, err
}
return results, nil
}
func (self *EndpointInstanceHealth) HttpGet() (*model.Health, error) {
instance := self.Parent.getInstance()
repair := false
// TODO: Use state stored by scheduled health check, rather than running directly?
health, err := instance.RunHealthCheck(repair)
if err != nil {
return nil, err
}
if health == nil {
return nil, rs.ErrNotFound()
}
log.Debug("Health of %v: %v", instance, health)
return health, nil
}
func (self *EndpointInstance) HttpPut(request *model.Instance) (*model.Instance, error) {
err := self.getInstance().Configure(request)
if err != nil {
return nil, err
}
// TODO: Juju issue? - state does not appear immediately (on subordinate charms)?
for i := 1; i <= 10; i++ {
model, err := self.getInstance().GetState()
if err == nil && model == nil {
time.Sleep(time.Second * 1)
continue
} else {
return model, err
}
}
log.Warn("Unable to retrieve instance state, even after retries")
return nil, rs.ErrNotFound()
}
func (self *Instance) readMetrics(jujuUnitNames []string, metricId string) (*model.MetricDataset, error) {
instance := self
keyValue := metricId
keyTimestamp := "Timestamp"
keyHostname := "Hostname"
keyType := "Type"
huddle := instance.huddle
es := huddle.SystemServices["elasticsearch"]
if es == nil {
return nil, rs.ErrNotFound()
}
if len(jujuUnitNames) == 0 {
return nil, nil
}
// TODO: Inject
// TODO: Use an ES client that isn't a singleton
elasticgo_api.Domain = es.PublicAddress
elasticgo_api.Port = "9200"
// TODO: We need to make sure that most fields are _not_ analyzed
// That is why we have match below, not term
filters := []interface{}{}
{
match := map[string]string{}
match[keyHostname] = jujuUnitNames[0]
filter := map[string]interface{}{"query": map[string]interface{}{"match": match}}
filters = append(filters, filter)
}
{
// TODO: Hard-coded
match := map[string]string{}
match[keyType] = "LoadAverage"
filter := map[string]interface{}{"query": map[string]interface{}{"match": match}}
filters = append(filters, filter)
}
if len(filters) > 1 {
and := map[string]interface{}{"and": filters}
filters = []interface{}{and}
}
match_all := map[string]interface{}{"match_all": map[string]string{}}
filtered := map[string]interface{}{"filter": filters[0], "query": match_all}
query := map[string]interface{}{"filtered": filtered}
body := map[string]interface{}{"query": query}
args := map[string]interface{}{}
args["size"] = 1000
response, err := elastigo_core.SearchRequest("_all", "message", args, body)
if err != nil {
log.Warn("Error searching elasticsearch", err)
return nil, fmt.Errorf("Error searching elasticsearch")
}
metrics := &model.MetricDataset{}
metrics.Points = []model.MetricDatapoint{}
for _, hit := range response.Hits.Hits {
// TODO: Are we serializing and deserializing here??
jsonBytes, err := hit.Source.MarshalJSON()
if err != nil {
log.Warn("Error reading JSON", err)
return nil, fmt.Errorf("Error searching elasticsearch")
}
//log.Info("Found metric: %v", string(jsonBytes))
var value map[string]interface{}
err = json.Unmarshal(jsonBytes, &value)
if err != nil {
log.Warn("Error unmarshalling response", err)
return nil, fmt.Errorf("Error searching elasticsearch")
}
// Post-filter the ES results...
// TODO: See if we can persuade ES to filter it correctly
hostname, found := value[keyHostname]
if !found {
log.Debug("No hostname in %v", string(jsonBytes))
continue
}
hostnameStr, ok := hostname.(string)
if !ok {
log.Debug("Cannot cast hostname to string: %v", hostname)
continue
}
if hostnameStr != jujuUnitNames[0] {
log.Debug("Post-filtering query results from ES")
continue
}
// Grab the timestamp & value
t, found := value[keyTimestamp]
if !found {
log.Debug("No timestamp in %v", string(jsonBytes))
continue
}
tStr, ok := t.(string)
if !ok {
log.Debug("Cannot cast timestamp to string: %v", t)
continue
}
timeFormat := time.RFC3339
tVal, err := time.Parse(timeFormat, tStr)
if err != nil {
log.Debug("Cannot parse timestamp: %v", tStr, err)
continue
}
y, found := value[keyValue]
if !found {
log.Debug("No value (%v) in %v", keyValue, string(jsonBytes))
continue
}
yStr, ok := y.(string)
if !ok {
log.Debug("Cannot cast value to string: %v", y)
continue
}
yVal, err := strconv.ParseFloat(yStr, 32)
if err != nil {
log.Debug("Error parsing value as float: %v", yStr, err)
continue
}
p := model.MetricDatapoint{}
p.T = tVal.Unix()
p.V = float32(yVal)
metrics.Points = append(metrics.Points, p)
}
// fmt.Println(values)
sort.Sort(metrics.Points)
return metrics, nil
}
// Runs a health check on the instance
func (self *Instance) RunHealthCheck(repair bool) (*model.Health, error) {
jujuClient := self.GetJujuClient()
state, err := self.getState0()
if err != nil {
return nil, err
}
if state == nil {
return nil, rs.ErrNotFound()
}
if state.Model == nil {
log.Debug("No model for %v", self)
return nil, rs.ErrNotFound()
}
if state.Model.Status != "started" {
log.Info("Skipping health check on not-yet started instance (state %v): %s", state.Model.Status, self)
return nil, nil
}
services, err := jujuClient.GetServiceStatusList(self.jujuPrefix)
if err != nil {
return nil, err
}
if services == nil || len(services) == 0 {
return nil, rs.ErrNotFound()
}
bundle, err := self.getCurrentBundle(state)
if err != nil {
return nil, err
}
health := &model.Health{}
health.Units = map[string]bool{}
healthChecks, err := self.bundleType.GetHealthChecks(bundle)
if err != nil {
return nil, err
}
// TODO: We can't "juju run" on subordinate charms
// charm := self.huddle.getCharmInfo(service.Charm)
//
// if charm.Subordinate {
// continue
// }
for healthCheckId, healthCheck := range healthChecks {
result, err := healthCheck.Run(self, services, repair)
if err != nil {
log.Info("Health check %v failed", healthCheckId, err)
return nil, err
}
for k, healthy := range result.Units {
overall, exists := health.Units[k]
if !exists {
overall = true
}
health.Units[k] = overall && healthy
}
}
return health, nil
}
func (self *EndpointServiceInstance) HttpPut(request *CfCreateInstanceRequest) (*rs.HttpResponse, error) {
service := self.getService()
log.Info("CF instance put request: %v", request)
planId := request.PlanId
cfServiceId := request.ServiceId
if cfServiceId != service.CfServiceId {
log.Warn("Service mismatch: %v vs %v", cfServiceId, service.CfServiceId)
return nil, rs.ErrNotFound()
}
bundleType, instance := service.getInstance(self.Id)
if instance == nil || bundleType == nil {
return nil, rs.ErrNotFound()
}
cfPlans, err := bundleType.GetCloudFoundryPlans()
if err != nil {
log.Warn("Error retrieving CloudFoundry plans for bundle %v", bundleType, err)
return nil, err
}
var foundPlan *bundle.CloudFoundryPlan
for _, cfPlan := range cfPlans {
cfPlanId := service.CfServiceId + "::" + cfPlan.Key
if cfPlanId == planId {
assert.That(foundPlan == nil)
foundPlan = cfPlan
}
}
if foundPlan == nil {
log.Warn("Plan not found %v", planId)
return nil, rs.ErrNotFound()
}
log.Debug("Found CF plan: %v", foundPlan)
configureRequest := &model.Instance{}
configureRequest.Options = foundPlan.Options
err = instance.Configure(configureRequest)
if err != nil {
return nil, err
}
response := &CfCreateInstanceResponse{}
// TODO: We need a dashboard URL - maybe a Juju GUI?
response.DashboardUrl = "http://localhost:8080"
response.State = CF_STATE_IN_PROGRESS
response.LastOperation = &CfOperation{}
response.LastOperation.State = CF_STATE_IN_PROGRESS
log.Info("Sending response to CF service create", log.AsJson(response))
httpResponse := &rs.HttpResponse{Status: http.StatusAccepted}
httpResponse.Content = response
return httpResponse, nil
}
// Returns the current state of the instance
func (self *Instance) getState0() (*instanceState, error) {
jujuClient := self.GetJujuClient()
primaryServiceId := self.primaryServiceId
status, err := jujuClient.GetServiceStatus(primaryServiceId)
// TODO: check err?
jujuService, err := jujuClient.FindService(primaryServiceId)
if err != nil {
return nil, err
}
if status == nil {
log.Warn("No state found for %v", primaryServiceId)
return nil, nil
}
log.Debug("Service state: %v", status)
state := &instanceState{}
state.Model = model.MapToInstance(self.instanceId, status, jujuService)
for k, v := range self.bundleType.GetDefaultOptions() {
option, found := state.Model.OptionDescriptions[k]
if !found {
log.Warn("Option not found in OptionDescriptions %v in %v", k, state.Model.OptionDescriptions)
continue
}
option.Default = v
state.Model.OptionDescriptions[k] = option
}
state.Units = map[string]map[string]api.UnitStatus{}
state.Units[primaryServiceId] = status.Units
state.PublicAddresses = []string{}
for _, unitStatus := range status.Units {
if unitStatus.PublicAddress == "" {
continue
}
state.PublicAddresses = append(state.PublicAddresses, unitStatus.PublicAddress)
}
serviceKeys, err := self.getBundleKeys()
if err != nil {
return nil, err
}
if serviceKeys == nil {
return nil, rs.ErrNotFound()
}
// TODO: This is pretty expensive... we could just check to see if properties have been set
for serviceId, _ := range serviceKeys {
if serviceId == primaryServiceId {
continue
}
status, err := jujuClient.GetServiceStatus(serviceId)
if err != nil {
log.Warn("Error while fetching status of service: %v", serviceId, err)
state.Model.Status = "pending"
} else if status == nil {
log.Warn("No status for service: %v", serviceId)
state.Model.Status = "pending"
} else {
log.Info("Got state of secondary service: %v => %v", serviceId, status)
for _, unitStatus := range status.Units {
model.MergeInstanceStatus(state.Model, &unitStatus)
}
}
if status != nil {
state.Units[serviceId] = status.Units
}
}
// TODO: This is a bit of a hack also. How should we wait for properties to be set?
annotations, err := jujuClient.GetServiceAnnotations(primaryServiceId)
if err != nil {
log.Warn("Error getting annotations", err)
// TODO: Mask error?
return nil, err
}
log.Info("Annotations on %v: %v", primaryServiceId, annotations)
state.Model.Options = map[string]string{}
state.SystemProperties = map[string]string{}
state.RelationMetadata = map[string]string{}
relationList := []relationProperty{}
for tagName, v := range annotations {
if strings.HasPrefix(tagName, ANNOTATION_PREFIX_INSTANCECONFIG) {
key := tagName[len(ANNOTATION_PREFIX_INSTANCECONFIG):]
state.Model.Options[key] = v
continue
}
if strings.HasPrefix(tagName, ANNOTATION_PREFIX_SYSTEM) {
key := tagName[len(ANNOTATION_PREFIX_SYSTEM):]
state.SystemProperties[key] = v
continue
}
if strings.HasPrefix(tagName, ANNOTATION_PREFIX_RELATIONINFO) {
suffix := tagName[len(ANNOTATION_PREFIX_RELATIONINFO):]
tokens := strings.SplitN(suffix, "_", 3)
if len(tokens) < 3 {
log.Debug("Ignoring unparseable tag: %v", tagName)
continue
}
unitId := tokens[0]
relationId := tokens[1]
key := tokens[2]
if key[0] != '_' {
state.RelationMetadata[key] = v
continue
}
relationTokens := strings.SplitN(relationId, ":", 2)
if len(relationTokens) != 2 {
log.Debug("Ignoring unparseable relation id in tag: %v", tagName)
continue
}
relationProperty := relationProperty{}
relationProperty.UnitId = unitId
assert.That(key[0] == '_')
relationProperty.Key = key[1:]
relationProperty.Value = v
relationProperty.RelationType = relationTokens[0]
relationProperty.RelationKey = relationTokens[1]
relationList = append(relationList, relationProperty)
continue
}
}
state.Relations = map[string]map[string]string{}
for _, relation := range relationList {
relationType := relation.RelationType
relations, found := state.Relations[relationType]
if !found {
relations = map[string]string{}
state.Relations[relationType] = relations
}
relations[relation.Key] = relation.Value
}
// TODO: Only if otherwise ready?
annotationsReady := self.bundleType.IsStarted(state.Relations)
// For a subordinate charm service (e.g. multimysql), we just watch for the annotation
if annotationsReady && state.Model.Status == "" && len(status.SubordinateTo) != 0 {
log.Info("Subordinate instance started (per annotations): %v", self)
state.Model.Status = "started"
}
if !annotationsReady {
log.Info("Instance not started (per annotations): %v", state.Relations)
state.Model.Status = "pending"
}
log.Info("Status of %v: %v", primaryServiceId, state.Model.Status)
// TODO: Fetch inherited properties from primary service and merge
return state, nil
}
// Retrieves metrics that apply to the instance
func (self *Instance) GetAllMetrics() (*model.Metrics, error) {
huddle := self.huddle
jujuUnitName := self.jujuPrefix + "metrics"
es := huddle.SystemServices["elasticsearch"]
if es == nil {
return nil, rs.ErrNotFound()
}
// TODO: Inject
// TODO: Use an ES client that isn't a singleton
elasticgo_api.Domain = es.PublicAddress
elasticgo_api.Port = "9200"
// TODO: We need to make sure that most fields are _not_ analyzed
// That is why we have match below, not term
query := map[string]interface{}{
"query": map[string]interface{}{
"match": map[string]string{"Hostname": jujuUnitName},
},
}
// query := map[string]interface{}{
// "query": map[string]interface{}{
// "match_all": map[string]string {},
// },
// }
args := map[string]interface{}{}
args["size"] = 1000
response, err := elastigo_core.SearchRequest("_all", "message", args, query)
if err != nil {
log.Warn("Error searching elasticsearch", err)
return nil, fmt.Errorf("Error searching elasticsearch")
}
metrics := &model.Metrics{}
metrics.Metric = []string{}
for _, v := range response.Hits.Hits {
// TODO: Are we serializing and deserializing here??
json, err := v.Source.MarshalJSON()
if err != nil {
log.Warn("Error reading JSON", err)
return nil, fmt.Errorf("Error searching elasticsearch")
}
m := string(json)
metrics.Metric = append(metrics.Metric, m)
// var value map[string]interface{}
// err := json.Unmarshal(v.Source, &value)
// if err != nil {
// log.Warn("Error unmarshalling response", err)
// return nil, fmt.Errorf("Error searching elasticsearch")
// }
// values = append(values, value)
}
// fmt.Println(values)
return metrics, nil
}