// addCharmViaAPI calls the appropriate client API calls to add the
// given charm URL to state. Also displays the charm URL of the added
// charm on stdout.
func addCharmViaAPI(client *api.Client, curl *charm.URL, repo charm.Repository) (*charm.URL, error) {
if curl.Revision < 0 {
latest, err := charm.Latest(repo, curl)
if err != nil {
log.Info("Error find latest version for: %v", curl.String(), err)
return nil, err
}
curl = curl.WithRevision(latest)
}
switch curl.Schema {
case "local":
ch, err := repo.Get(curl)
if err != nil {
return nil, err
}
stateCurl, err := client.AddLocalCharm(curl, ch)
if err != nil {
return nil, err
}
curl = stateCurl
case "cs":
err := client.AddCharm(curl)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("unsupported charm URL schema: %q", curl.Schema)
}
log.Info("Added charm %q to the environment.", curl)
return curl, nil
}
func (self *EndpointServiceInstance) HttpGet() (*rs.HttpResponse, error) {
service := self.getService()
log.Info("CF instance GET request: %v", self.Id)
bundleType, instance := service.getInstance(self.Id)
if instance == nil || bundleType == nil {
return nil, rs.ErrNotFound()
}
state, err := instance.GetState()
if err != nil {
log.Warn("Error while waiting for instance to become ready", err)
return nil, err
}
ready := false
if state == nil {
log.Warn("Instance not yet created")
} else {
status := state.Status
if status == "started" {
ready = true
} else if status == "pending" {
ready = false
} else {
log.Warn("Unknown instance status: %v", status)
}
}
response := &CfCreateInstanceResponse{}
// TODO: We need a dashboard URL - maybe a Juju GUI?
response.DashboardUrl = "http://localhost:8080"
var cfState string
if ready {
cfState = CF_STATE_SUCCEEDED
} else {
cfState = CF_STATE_IN_PROGRESS
}
response.State = cfState
response.LastOperation = &CfOperation{}
response.LastOperation.State = cfState
log.Info("Sending response to CF service get", log.AsJson(response))
httpResponse := &rs.HttpResponse{Status: http.StatusOK}
httpResponse.Content = response
return httpResponse, nil
}
func (self *EtcdRouterRegistry) listSubkeys(key string) ([]string, error) {
response, err := self.client.Get(key, false, false)
if err != nil {
etcdError, ok := err.(*etcd.EtcdError)
if ok && etcdError.ErrorCode == etcdErrorKeyNotFound {
log.Debug("Etcd key not found: %v", key)
return []string{}, nil
}
log.Warn("Error reading key from etcd: %v", key, err)
return nil, err
}
if response == nil || response.Node == nil || response.Node.Nodes == nil {
log.Info("No children for key from etcd: %v", key)
return []string{}, nil
}
names := []string{}
for _, node := range response.Node.Nodes {
nodeKey := node.Key
if !strings.HasPrefix(nodeKey, key) {
return nil, fmt.Errorf("Key without expected prefix: %v vs %v", nodeKey, key)
}
suffix := nodeKey[len(key):]
names = append(names, suffix)
}
return names, nil
}
func getCharmInfo(client *api.Client, charmName string, localRepoPath string, defaultSeries string) (*api.CharmInfo, error) {
curl, err := resolveCharmURL(client, charmName, defaultSeries)
if err != nil {
return nil, err
}
repo, err := charm.InferRepository(curl.Reference, localRepoPath)
if err != nil {
return nil, err
}
// repo = config.SpecializeCharmRepo(repo, defaultSeries)
curl, err = addCharmViaAPI(client, curl, repo)
if err != nil {
return nil, err
}
charmInfo, err := client.CharmInfo(curl.String())
if err != nil {
log.Info("Error getting charm info for: %v", curl.String(), err)
return nil, err
}
return charmInfo, nil
}
func (self *EtcdRouterRegistry) read(key string) (*etcdRouterData, error) {
response, err := self.client.Get(key, false, false)
if err != nil {
etcdError, ok := err.(*etcd.EtcdError)
if ok && etcdError.ErrorCode == etcdErrorKeyNotFound {
log.Debug("Etcd key not found: %v", key)
return nil, nil
}
log.Warn("Error reading key from etcd: %v", key, err)
return nil, err
}
node := response.Node
if node == nil || node.Value == "" {
log.Info("No contents for key from etcd: %v", key)
return nil, nil
}
decoded := &etcdRouterData{}
err = json.Unmarshal([]byte(node.Value), decoded)
if err != nil {
log.Warn("Error parsing value from etcd: %v", node.Value, err)
return nil, err
}
return decoded, nil
}
func (self *EndpointServiceInstance) HttpDelete(httpRequest *http.Request) (*CfDeleteInstanceResponse, error) {
service := self.getService()
queryValues := httpRequest.URL.Query()
serviceId := queryValues.Get("service_id")
// planId := queryValues.Get("plan_id")
if serviceId != service.CfServiceId {
log.Warn("Service mismatch: %v vs %v", serviceId, service.CfServiceId)
return nil, rs.ErrNotFound()
}
log.Info("Deleting item %v %v", serviceId, self.Id)
bundletype, instance := service.getInstance(self.getInstanceId())
if instance == nil || bundletype == nil {
return nil, rs.ErrNotFound()
}
err := instance.Delete()
if err != nil {
return nil, err
}
// TODO: Wait for deletion?
response := &CfDeleteInstanceResponse{}
return response, nil
}
func (self *EndpointInstanceScaling) HttpPut(policyUpdate *model.ScalingPolicy) (*model.Scaling, error) {
instance := self.Parent.getInstance()
log.Info("Policy update: %v", policyUpdate)
exists, err := instance.Exists()
if err != nil {
return nil, err
}
if !exists {
return nil, rs.ErrNotFound()
}
if policyUpdate != nil {
_, err := instance.UpdateScalingPolicy(policyUpdate)
if err != nil {
log.Warn("Error updating scaling policy", err)
return nil, err
}
}
results, err := instance.RunScaling(true)
if err != nil {
return nil, err
}
return results, nil
}
// Delete any relation properties relating to the specified unit; that unit is going away.
func (self *Instance) DeleteRelationInfo(unitId string, relationId string) error {
jujuClient := self.GetJujuClient()
serviceId := self.primaryServiceId
prefix := ANNOTATION_PREFIX_RELATIONINFO + unitId + "_" + relationId + "_"
annotations, err := jujuClient.GetServiceAnnotations(serviceId)
if err != nil {
log.Warn("Error getting annotations", err)
// TODO: Mask error?
return err
}
deleteKeys := []string{}
for tagName, _ := range annotations {
if !strings.HasPrefix(tagName, prefix) {
continue
}
deleteKeys = append(deleteKeys, tagName)
}
if len(deleteKeys) != 0 {
log.Info("Deleting annotations on service %v: %v", serviceId, deleteKeys)
err = jujuClient.DeleteServiceAnnotations(serviceId, deleteKeys)
if err != nil {
log.Warn("Error deleting annotations", err)
return err
}
}
return nil
}
func localIP() (net.IP, error) {
netInterfaces, err := net.Interfaces()
if err != nil {
return nil, err
}
for _, netInterface := range netInterfaces {
addresses, err := netInterface.Addrs()
if err != nil {
return nil, err
}
for _, address := range addresses {
ipnet, ok := address.(*net.IPNet)
if !ok {
continue
}
v4 := ipnet.IP.To4()
if v4 == nil || v4[0] == 127 { // loopback address
continue
}
log.Info("Chose local IP: %v", v4)
return v4, nil
}
}
return nil, errors.New("Cannot find local IP address")
}
func GetOptions() *Options {
flag.Parse()
self := &Options{}
self.AgentConf = *flagAgentConf
self.ApiPasswordPath = *flagApiPasswordPath
self.CfTenantId = *flagCfTenantId
self.ListenAddress = *flagListenAddress
host, port, err := net.SplitHostPort(self.ListenAddress)
if err != nil {
log.Warn("Cannot parse listen address: %v", self.ListenAddress)
return nil
}
var portNum int
if port == "" {
portNum = 8080
} else {
portNum, err = net.LookupPort("tcp", port)
if err != nil {
log.Warn("Cannot resolve port: %v", port)
return nil
}
}
privateUrl := *flagPrivateUrl
if privateUrl == "" {
privateHost := host
if privateHost == "" {
ip, err := localIP()
if err != nil {
log.Warn("Error finding local IP", err)
return nil
}
privateHost = ip.String()
}
privateUrl = fmt.Sprintf("http://%v:%v/xaasprivate", privateHost, portNum)
log.Info("Chose private url: %v", privateUrl)
}
self.PrivateUrl = privateUrl
authMode := *flagAuth
authMode = strings.TrimSpace(authMode)
authMode = strings.ToLower(authMode)
if authMode == "openstack" {
keystoneUrl := *flagKeystoneUrl
self.Authenticator = auth.NewOpenstackMultiAuthenticator(keystoneUrl)
} else if authMode == "development" {
self.Authenticator = auth.NewDevelopmentAuthenticator()
} else {
log.Warn("Unknown authentication mode: %v", authMode)
return nil
}
return self
}
// update_relation_properties RPC handler
func (self *EndpointRpcUpdateRelationProperties) HttpPost(huddle *core.Huddle, request *RpcUpdateRelationPropertiesRequest) (*RpcUpdateRelationPropertiesResponse, error) {
// TODO: Validate that this is coming from one of our machines?
log.Info("Got RPC request: UpdateRelationProperties: %v", request)
response := &RpcUpdateRelationPropertiesResponse{}
// Sanitize
if request.Properties == nil {
request.Properties = make(map[string]string)
}
localUnit := request.ServiceName
if localUnit == "" {
return nil, fmt.Errorf("ServiceName is required")
}
tenant, bundleTypeName, instanceId, _, _, err := core.ParseUnit(localUnit)
if err != nil {
return nil, err
}
bundleType := huddle.System.GetBundleType(bundleTypeName)
if bundleType == nil {
return nil, fmt.Errorf("Unknown bundle type: %v", bundleTypeName)
}
primaryService := bundleType.PrimaryJujuService()
// remoteUnit := request.RemoteName
// if remoteUnit == "" {
// // We're a bit stuck here. We do have the relationId and other info,
// // we just don't have the remote relation, and we're storing the attributes on the remote relation
// // TODO: Infer the remote relation? (-stubclient to -primary)?
// log.Warn("No remote unit; can't remove relations")
// return response, nil
// }
//
// _, _, remoteInstanceId, _, remoteUnitId, err := core.ParseUnit(remoteUnit)
// if err != nil {
// return nil, err
// }
instance := huddle.NewInstance(tenant, bundleType, instanceId)
relationId := request.RelationId
if request.Action == "broken" {
err = instance.DeleteRelationInfo(primaryService, relationId)
} else {
err = instance.SetRelationInfo(primaryService, relationId, request.Properties)
}
if err != nil {
return nil, err
}
return response, nil
}
func isHuddleReady(huddle *core.Huddle) bool {
for key, service := range huddle.SystemServices {
if service.PublicAddress == "" {
log.Info("Service not ready (no public address): %v", key)
return false
}
}
return true
}
func (self *ServiceHealthCheck) checkService(instance jxaas.Instance, serviceId string, repair bool, dest *model.Health) error {
client := instance.GetJujuClient()
command := "service " + self.ServiceName + " status"
log.Info("Running command on %v: %v", serviceId, command)
runResults, err := client.Run(serviceId, nil, command, 5*time.Second)
if err != nil {
return err
}
for _, runResult := range runResults {
unitId := juju.ParseUnit(runResult.UnitId)
code := runResult.Code
stdout := string(runResult.Stdout)
stderr := string(runResult.Stderr)
log.Debug("Result: %v %v %v %v", runResult.UnitId, code, stdout, stderr)
healthy := true
if !strings.Contains(stdout, "start/running") {
log.Info("Service %v not running on %v", serviceId, runResult.UnitId)
healthy = false
if repair {
command := "service " + self.ServiceName + " start"
log.Info("Running command on %v: %v", serviceId, command)
_, err := client.Run(serviceId, []string{unitId}, command, 5*time.Second)
if err != nil {
return err
}
}
}
dest.Units[unitId] = healthy
}
return nil
}
func (self *Bundle) ApplyImplicits(templateContext *TemplateContext) {
for _, v := range self.Services {
v.applyImplicits(templateContext)
}
for _, v := range self.Relations {
v.applyImplicits(templateContext)
}
for k, v := range self.Provides {
v.applyImplicits(templateContext, k)
}
stub, found := self.Services["sc"]
if found {
self.configureStubClient(templateContext, stub)
log.Info("Configured stubclient: %v", stub)
} else {
log.Info("sc (stubclient) not found")
}
}
func (self *CharmReader) read(name string) ([]byte, error) {
inputStream, err := self.byteSource.Open()
if err != nil {
return nil, err
}
defer func() {
closeable, ok := inputStream.(io.Closer)
if ok {
closeable.Close()
}
}()
size, err := self.byteSource.Size()
if err != nil {
return nil, err
}
readerAt, ok := inputStream.(io.ReaderAt)
if !ok {
return nil, fmt.Errorf("Expected ReaderAt")
}
r, err := zip.NewReader(readerAt, size)
if err != nil {
return nil, err
}
for _, f := range r.File {
log.Info("File: %v", f.Name)
}
for _, f := range r.File {
if f.Name != name {
continue
}
rc, err := f.Open()
if err != nil {
return nil, err
}
defer rc.Close()
data, err := ioutil.ReadAll(rc)
if err != nil {
return nil, err
}
return data, nil
}
return nil, nil
}
func (self *Huddle) cleanupOldMachines(state map[string]int, threshold int) (map[string]int, error) {
status, err := self.JujuClient.GetSystemStatus()
if err != nil {
log.Warn("Error getting system status", err)
return nil, err
}
unitsByMachine := map[string]*api.UnitStatus{}
for _, serviceStatus := range status.Services {
for _, unitStatus := range serviceStatus.Units {
machineId := unitStatus.Machine
unitsByMachine[machineId] = &unitStatus
}
}
idleMachines := map[string]*api.MachineStatus{}
for machineId, machineStatus := range status.Machines {
unit := unitsByMachine[machineId]
if unit != nil {
continue
}
idleMachines[machineId] = &machineStatus
}
idleCounts := map[string]int{}
for machineId, _ := range idleMachines {
idleCount := state[machineId]
idleCount++
idleCounts[machineId] = idleCount
}
for machineId, idleCount := range idleCounts {
if idleCount < threshold {
continue
}
if machineId == "0" {
// Machine id 0 is special (the system machine); we can't destroy it
continue
}
log.Info("Machine is idle; removing: %v", machineId)
err = self.JujuClient.DestroyMachine(machineId)
if err != nil {
log.Warn("Failed to delete machine %v", machineId, err)
}
}
return idleCounts, nil
}
// Sets annotations on the specified instance.
func (self *Instance) setServiceAnnotations(pairs map[string]string) error {
jujuClient := self.GetJujuClient()
serviceId := self.primaryServiceId
log.Info("Setting annotations on service %v: %v", serviceId, pairs)
err := jujuClient.SetServiceAnnotations(serviceId, pairs)
if err != nil {
log.Warn("Error setting annotations", err)
// TODO: Mask error?
return err
}
return nil
}
func (self *baseBundleType) BuildRelationInfo(templateContext *bundle.TemplateContext, bundle *bundle.Bundle, relationKey string) (*model.RelationInfo, error) {
log.Info("BuildRelationInfo with %v", templateContext)
// Find the properties the juju charm is exposing
relationProperties := templateContext.Relations[relationKey]
// Map those properties using the definition
provideProperties := map[string]string{}
if len(bundle.Provides) == 0 {
// No explicit provides => derive automatically
for k, v := range relationProperties {
v = templateContext.GetSpecialProperty(relationKey, k, v)
provideProperties[k] = v
}
// Auto-populate required properties that we generate
required := []string{"protocol", "port"}
for _, k := range required {
v, found := relationProperties[k]
if !found {
v = templateContext.GetSpecialProperty(relationKey, k, v)
}
provideProperties[k] = v
}
} else {
definition, found := bundle.Provides[relationKey]
if !found {
// Explicit provides, but no definition => no relation
log.Debug("Request for relation, but no definition found: %v", relationKey)
return nil, nil
}
for k, v := range definition.Properties {
provideProperties[k] = v
}
}
relationInfo := &model.RelationInfo{}
if templateContext.Proxy != nil {
relationInfo.PublicAddresses = []string{templateContext.Proxy.Host}
}
relationInfo.Properties = provideProperties
return relationInfo, nil
}
func TestBundleStore_Get(t *testing.T) {
store := NewBundleStore("../templates")
templateContext := &TemplateContext{}
tenant := "123"
serviceType := "mysql"
name := "test"
templateContext.NumberUnits = 3
templateContext.Options = map[string]string{}
templateContext.Options["performance"] = "high"
bundle, err := store.GetBundle(templateContext, tenant, serviceType, name)
if err != nil {
t.Fatal("Unable to load bundle", err)
}
if bundle == nil {
t.Fatal("Bundle was nil")
}
prefix := buildPrefix(tenant, serviceType, name)
service, found := bundle.Services[prefix+"mysql"]
if !found {
log.Info("Services: %v", bundle.Services)
t.Fatal("mysql service not found")
}
if service.NumberUnits != 3 {
t.Fatal("NumberUnits was not copied")
}
if service.Options["performance"] != "high" {
t.Fatal("Performance option was not copied")
}
}
func NewHuddle(system *System, bundleStore *bundle.BundleStore, jujuApi *juju.Client, privateUrl string) (*Huddle, error) {
key := "shared"
huddle := &Huddle{}
environmentInfo, err := jujuApi.EnvironmentInfo()
if err != nil {
log.Warn("Error reading juju environment info", err)
return nil, err
}
if environmentInfo == nil {
return nil, fmt.Errorf("No juju environment info found")
}
huddle.environmentProviderType = environmentInfo.ProviderType
if huddle.environmentProviderType == "" {
return nil, fmt.Errorf("Juju environment info invalid: no ProviderType")
}
log.Info("Juju environment ProviderType is '%v'", huddle.environmentProviderType)
systemBundle, err := bundleStore.GetSystemBundle(key)
if err != nil {
log.Warn("Error loading system bundle: %v", key, err)
return nil, err
}
if systemBundle == nil {
log.Warn("Cannot load system bundle: %v", key, err)
return nil, nil
}
info, err := systemBundle.Deploy("jx-", jujuApi)
if err != nil {
log.Warn("Error deploying system bundle", err)
return nil, err
}
huddle.PrivateUrl = privateUrl
huddle.SystemServices = map[string]*SystemService{}
huddle.assignedPublicPorts = map[string]int{}
for key, service := range info.Services {
systemService := &SystemService{}
systemService.JujuName = "jx-" + key
systemService.Key = key
status := service.Status
if status != nil {
for _, unit := range status.Units {
if unit.PublicAddress != "" {
systemService.PublicAddress = unit.PublicAddress
}
externalAddress := ""
if unit.Machine != "" {
externalAddress, err = jujuApi.PublicAddress(unit.Machine)
if err != nil {
log.Warn("Error getting public address for machine", err)
return nil, err
} else {
if huddle.IsAmazon() {
// Work around a problem where we sometimes get an address that is ip-X-X-X-X.ece2.internal
// I think this is a Juju bug (?)
if strings.HasSuffix(externalAddress, ".ec2.internal") {
log.Warn("Juju gave invalid PublicAddress: %v", externalAddress)
externalAddress = systemService.PublicAddress
}
// Amazon has a special DNS name: ec2-54-172-123-123.compute-1.amazonaws.com
// Externally that resolves to 54.172.123.123 (i.e. the value embedded in the name)
// Internally (inside EC2) that resolves to the internal IP (172.16.x.x)
// We don't want that internal resolution to happen here (this is an _external_ IP)
// But we may be within EC2, so we can't simply resolve the name
if strings.HasPrefix(externalAddress, "ec2-") && strings.HasSuffix(externalAddress, ".compute-1.amazonaws.com") {
ipString := externalAddress[4:]
firstDot := strings.IndexRune(ipString, '.')
ipString = ipString[:firstDot]
ipString = strings.Replace(ipString, "-", ".", -1)
log.Info("Replaced EC2 switching-address '%v' with IP '%v'", externalAddress, ipString)
externalAddress = ipString
}
}
if externalAddress != "" {
log.Info("Chose public address for machine: '%v'", externalAddress)
} else {
log.Warn("Got empty public address for machine: %v", unit.Machine)
}
}
}
if externalAddress == "" {
log.Warn("Unable to get external address for machine %v, falling back to public address %v", unit.Machine, systemService.PublicAddress)
externalAddress = systemService.PublicAddress
}
systemService.ExternalAddress = externalAddress
}
}
huddle.SystemServices[key] = systemService
}
huddle.JujuClient = jujuApi
huddle.System = system
// TODO: Wait until initialized or offer a separate 'bootstrap' command
{
check := &HealthCheckAllInstances{}
check.huddle = huddle
check.repair = true
system.Scheduler.AddTask(check, time.Minute*1)
}
{
scaling := &AutoScaleAllInstances{}
scaling.huddle = huddle
system.Scheduler.AddTask(scaling, time.Minute*1)
}
{
task := &CleanupOldMachines{}
task.huddle = huddle
system.Scheduler.AddTask(task, time.Minute*5)
}
return huddle, nil
}
// Runs a scaling query and/or change on the instance
func (self *Instance) RunScaling(changeScale bool) (*model.Scaling, error) {
health := &model.Scaling{}
instanceState, err := self.GetState()
if err != nil {
log.Warn("Error getting instance state", err)
return nil, err
}
assert.That(instanceState.NumberUnits != nil)
scaleCurrent := *instanceState.NumberUnits
health.ScaleCurrent = scaleCurrent
health.ScaleTarget = scaleCurrent
policy, err := self.getScalingPolicy()
if err != nil {
log.Warn("Error fetching scaling policy", err)
return nil, err
}
health.Policy = *policy
var scaleTarget int
if policy.MetricName != nil {
// TODO: Filter by time window
metricData, err := self.GetMetricValues(*policy.MetricName)
if err != nil {
log.Warn("Error retrieving metrics for scaling", err)
return nil, err
}
window := 300
if policy.Window != nil {
window = *policy.Window
}
duration := time.Duration(-window) * time.Second
now := time.Now()
maxTime := now.Unix()
minTime := now.Add(duration).Unix()
matches := &model.MetricDataset{}
for _, point := range metricData.Points {
t := point.T
if t < minTime {
continue
}
if t > maxTime {
continue
}
matches.Points = append(matches.Points, point)
}
matches.SortPointsByTime()
lastTime := minTime
var total float64
for _, point := range matches.Points {
t := point.T
assert.That(t >= lastTime)
total += float64(float32(t-lastTime) * point.V)
lastTime = t
}
metricCurrent := float32(total / float64(lastTime-minTime))
log.Info("Average of metric: %v", metricCurrent)
health.MetricCurrent = metricCurrent
// TODO: Smart 'target-based' scaling
scaleDelta := 0
if policy.MetricMin != nil && metricCurrent < *policy.MetricMin {
scaleDelta = -1
} else if policy.MetricMax != nil && metricCurrent > *policy.MetricMax {
scaleDelta = +1
}
scaleTarget = scaleCurrent + scaleDelta
} else {
scaleTarget = scaleCurrent
}
if policy.ScaleMax != nil && scaleTarget > *policy.ScaleMax {
scaleTarget = *policy.ScaleMax
} else if policy.ScaleMin != nil && scaleTarget < *policy.ScaleMin {
scaleTarget = *policy.ScaleMin
}
health.ScaleTarget = scaleTarget
if changeScale && health.ScaleTarget != scaleCurrent {
log.Info("Changing scale from %v to %v for %v", scaleCurrent, health.ScaleTarget, self)
rescale := &model.Instance{}
rescale.NumberUnits = new(int)
*rescale.NumberUnits = health.ScaleTarget
err := self.Configure(rescale)
if err != nil {
log.Warn("Error changing scale", err)
return nil, err
}
}
if math.IsNaN(float64(health.MetricCurrent)) {
// Avoid error when golang tries to convert NaN to JSON (!)
health.MetricCurrent = 0
}
return health, nil
}
// Assigns a public port to the serviceId
func (self *Huddle) assignPublicPort(serviceId string) (int, bool, error) {
self.mutex.Lock()
defer self.mutex.Unlock()
var port int
port, found := self.assignedPublicPorts[serviceId]
if found {
return port, false, nil
}
// TODO: Filter?
prefix := ""
statuses, err := self.JujuClient.GetServiceStatusList(prefix)
if err != nil {
return 0, false, err
}
publicPorts := []int{}
for _, publicPort := range self.assignedPublicPorts {
publicPorts = append(publicPorts, publicPort)
}
for key, _ := range statuses {
var publicPort int
publicPort, found := self.assignedPublicPorts[key]
if found {
assert.That(contains(publicPorts, publicPort))
continue
}
log.Debug("Looking for public port annotation on: %v", key)
annotations, err := self.JujuClient.GetServiceAnnotations(key)
if err != nil {
return 0, false, err
}
publicPortString := annotations[ANNOTATION_KEY_PUBLIC_PORT]
publicPortString = strings.TrimSpace(publicPortString)
if publicPortString == "" {
continue
}
publicPort, err = strconv.Atoi(publicPortString)
if err != nil {
log.Warn("Error parsing public port on %v: %v", key, publicPortString, err)
}
self.assignedPublicPorts[key] = publicPort
publicPorts = append(publicPorts, publicPort)
}
// This approach breaks down if the ports are densely assigned
if len(publicPorts) > 9000 {
return 0, false, fmt.Errorf("Too many ports already assigned")
}
for {
port = 10000 + rand.Intn(10000)
if contains(publicPorts, port) {
continue
}
log.Debug("Public ports already assigned: %v", publicPorts)
log.Info("Assigned port: %v", port)
break
}
// We can't set the port yet; the service likely doesn't yet exist
// err = self.Instance.setPublicPort(port)
// if err != nil {
// return 0, err
// }
// Instead we set the port in the map; this map is how we avoid double allocations before
// we've created the service
self.assignedPublicPorts[serviceId] = port
return port, true, nil
}
func (self *Instance) buildCurrentTemplateContext(state *instanceState, cloudfoundry bool) (*bundle.TemplateContext, error) {
var err error
if state == nil {
state, err = self.getState0()
if err != nil {
log.Warn("Error getting instance state", err)
return nil, err
}
}
context := self.buildSkeletonTemplateContext()
// TODO: Need to determine current # of units
context.NumberUnits = 1
if state != nil && state.Model != nil {
context.Options = state.Model.Options
} else {
context.Options = map[string]string{}
}
publicPortAssigner := &InstancePublicPortAssigner{}
publicPortAssigner.Instance = self
context.PublicPortAssigner = publicPortAssigner
// Populate relation info
if state != nil {
context.Relations = state.Relations
}
// Populate proxy
// TODO: Skip proxy host on EC2?
useProxyHost := true
if state != nil {
systemProperties := state.SystemProperties
if useProxyHost && systemProperties[SYSTEM_KEY_PUBLIC_PORT] != "" {
publicPortString := systemProperties[SYSTEM_KEY_PUBLIC_PORT]
publicPort, err := strconv.Atoi(publicPortString)
if err != nil {
log.Warn("Error parsing public port: %v", publicPortString, err)
return nil, err
}
proxyHost, err := self.huddle.getProxyHost(false)
if err != nil {
log.Warn("Error fetching proxy host", err)
return nil, err
}
if cloudfoundry && self.huddle.IsAmazon() {
// CloudFoundry prevents apps from accessing private CIDRs (10.x.x.x, 172.16.x.x etc)
// This would be OK, because the hostname on EC2 is a DNS name assigned by AWS to the instance
// .. except that within AWS, this hostname resolves to an internal IP.
// For this special case, we force the extenal IP
log.Info("Using external IP for AWS & CloudFoundry")
proxyHost, err = self.huddle.getProxyHost(true)
if err != nil {
log.Warn("Error fetching (external) proxy host", err)
return nil, err
}
}
context.Proxy = &bundle.ProxySettings{}
context.Proxy.Host = proxyHost
context.Proxy.Port = publicPort
}
}
return context, nil
}
// Runs a health check on the instance
func (self *Instance) RunHealthCheck(repair bool) (*model.Health, error) {
jujuClient := self.GetJujuClient()
state, err := self.getState0()
if err != nil {
return nil, err
}
if state == nil {
return nil, rs.ErrNotFound()
}
if state.Model == nil {
log.Debug("No model for %v", self)
return nil, rs.ErrNotFound()
}
if state.Model.Status != "started" {
log.Info("Skipping health check on not-yet started instance (state %v): %s", state.Model.Status, self)
return nil, nil
}
services, err := jujuClient.GetServiceStatusList(self.jujuPrefix)
if err != nil {
return nil, err
}
if services == nil || len(services) == 0 {
return nil, rs.ErrNotFound()
}
bundle, err := self.getCurrentBundle(state)
if err != nil {
return nil, err
}
health := &model.Health{}
health.Units = map[string]bool{}
healthChecks, err := self.bundleType.GetHealthChecks(bundle)
if err != nil {
return nil, err
}
// TODO: We can't "juju run" on subordinate charms
// charm := self.huddle.getCharmInfo(service.Charm)
//
// if charm.Subordinate {
// continue
// }
for healthCheckId, healthCheck := range healthChecks {
result, err := healthCheck.Run(self, services, repair)
if err != nil {
log.Info("Health check %v failed", healthCheckId, err)
return nil, err
}
for k, healthy := range result.Units {
overall, exists := health.Units[k]
if !exists {
overall = true
}
health.Units[k] = overall && healthy
}
}
return health, nil
}
func (self *EndpointServiceInstance) HttpPut(request *CfCreateInstanceRequest) (*rs.HttpResponse, error) {
service := self.getService()
log.Info("CF instance put request: %v", request)
planId := request.PlanId
cfServiceId := request.ServiceId
if cfServiceId != service.CfServiceId {
log.Warn("Service mismatch: %v vs %v", cfServiceId, service.CfServiceId)
return nil, rs.ErrNotFound()
}
bundleType, instance := service.getInstance(self.Id)
if instance == nil || bundleType == nil {
return nil, rs.ErrNotFound()
}
cfPlans, err := bundleType.GetCloudFoundryPlans()
if err != nil {
log.Warn("Error retrieving CloudFoundry plans for bundle %v", bundleType, err)
return nil, err
}
var foundPlan *bundle.CloudFoundryPlan
for _, cfPlan := range cfPlans {
cfPlanId := service.CfServiceId + "::" + cfPlan.Key
if cfPlanId == planId {
assert.That(foundPlan == nil)
foundPlan = cfPlan
}
}
if foundPlan == nil {
log.Warn("Plan not found %v", planId)
return nil, rs.ErrNotFound()
}
log.Debug("Found CF plan: %v", foundPlan)
configureRequest := &model.Instance{}
configureRequest.Options = foundPlan.Options
err = instance.Configure(configureRequest)
if err != nil {
return nil, err
}
response := &CfCreateInstanceResponse{}
// TODO: We need a dashboard URL - maybe a Juju GUI?
response.DashboardUrl = "http://localhost:8080"
response.State = CF_STATE_IN_PROGRESS
response.LastOperation = &CfOperation{}
response.LastOperation.State = CF_STATE_IN_PROGRESS
log.Info("Sending response to CF service create", log.AsJson(response))
httpResponse := &rs.HttpResponse{Status: http.StatusAccepted}
httpResponse.Content = response
return httpResponse, nil
}
func (self *RestEndpointHandler) httpHandler(res http.ResponseWriter, req *http.Request) {
requestUrl := req.URL
requestMethod := req.Method
log.Debug("%v %v", requestMethod, requestUrl)
endpoint, err := self.resolveEndpoint(req)
if endpoint == nil && err == nil {
err = HttpError(http.StatusNotFound)
}
var method reflect.Value
if err == nil {
httpMethod := req.Method
methodName := "Http" + httpMethod[0:1] + strings.ToLower(httpMethod[1:])
method = endpoint.MethodByName(methodName)
if !method.IsValid() {
log.Debug("Method not found: %v on %v", methodName, endpoint.Type())
err = HttpError(http.StatusNotFound)
}
}
var args []reflect.Value
if err == nil {
args, err = self.buildArgs(req, &method)
}
var val reflect.Value
if err == nil {
var out []reflect.Value
out = method.Call(args)
// fmt.Fprintf(w, "Returned %v", out)
val, err = parseReturn(out)
}
if err == nil {
if val.IsNil() {
err = HttpError(http.StatusNotFound)
}
}
var response *HttpResponse
var mbw MessageBodyWriter
if err == nil {
response, err = self.makeResponse(val)
}
if err == nil {
assert.That(response != nil)
if response.Headers == nil {
response.Headers = make(map[string]string)
}
if response.Content == nil {
mbw = &NoResponseMessageBodyWriter{}
} else {
contentType := response.Headers["content-type"]
if contentType == "" {
contentType = "application/json; charset=utf-8"
response.Headers["content-type"] = contentType
}
var mediaType *MediaType
if contentType != "" {
mediaType, err = ParseMediaType(contentType)
}
if err == nil {
assert.That(mediaType != nil)
mbw = self.server.findMessageBodyWriter(response.Content, req, mediaType)
if mbw == nil {
log.Warn("Unable to find media type: %v", contentType)
err = HttpError(http.StatusUnsupportedMediaType)
}
}
}
}
if err == nil {
assert.That(response != nil)
assert.That(mbw != nil)
log.Info("%v %v %v", response.Status, requestMethod, requestUrl)
if response.Headers != nil {
for name, value := range response.Headers {
res.Header().Set(name, value)
}
}
res.WriteHeader(response.Status)
err = mbw.Write(response.Content, reflect.TypeOf(response.Content), req, res)
if err != nil {
log.Warn("Error while writing message body", err)
}
} else {
httpError, ok := err.(*HttpErrorObject)
if !ok {
log.Warn("Internal error serving request", err)
httpError = HttpError(http.StatusInternalServerError)
}
status := httpError.Status
message := httpError.Message
if message == "" {
message = http.StatusText(status)
if message == "" {
message = "Error"
}
}
for k, v := range httpError.Headers {
res.Header().Add(k, v)
}
log.Info("%v %v %v", status, requestMethod, requestUrl)
http.Error(res, message, status)
}
}
// Returns the current state of the instance
func (self *Instance) getState0() (*instanceState, error) {
jujuClient := self.GetJujuClient()
primaryServiceId := self.primaryServiceId
status, err := jujuClient.GetServiceStatus(primaryServiceId)
// TODO: check err?
jujuService, err := jujuClient.FindService(primaryServiceId)
if err != nil {
return nil, err
}
if status == nil {
log.Warn("No state found for %v", primaryServiceId)
return nil, nil
}
log.Debug("Service state: %v", status)
state := &instanceState{}
state.Model = model.MapToInstance(self.instanceId, status, jujuService)
for k, v := range self.bundleType.GetDefaultOptions() {
option, found := state.Model.OptionDescriptions[k]
if !found {
log.Warn("Option not found in OptionDescriptions %v in %v", k, state.Model.OptionDescriptions)
continue
}
option.Default = v
state.Model.OptionDescriptions[k] = option
}
state.Units = map[string]map[string]api.UnitStatus{}
state.Units[primaryServiceId] = status.Units
state.PublicAddresses = []string{}
for _, unitStatus := range status.Units {
if unitStatus.PublicAddress == "" {
continue
}
state.PublicAddresses = append(state.PublicAddresses, unitStatus.PublicAddress)
}
serviceKeys, err := self.getBundleKeys()
if err != nil {
return nil, err
}
if serviceKeys == nil {
return nil, rs.ErrNotFound()
}
// TODO: This is pretty expensive... we could just check to see if properties have been set
for serviceId, _ := range serviceKeys {
if serviceId == primaryServiceId {
continue
}
status, err := jujuClient.GetServiceStatus(serviceId)
if err != nil {
log.Warn("Error while fetching status of service: %v", serviceId, err)
state.Model.Status = "pending"
} else if status == nil {
log.Warn("No status for service: %v", serviceId)
state.Model.Status = "pending"
} else {
log.Info("Got state of secondary service: %v => %v", serviceId, status)
for _, unitStatus := range status.Units {
model.MergeInstanceStatus(state.Model, &unitStatus)
}
}
if status != nil {
state.Units[serviceId] = status.Units
}
}
// TODO: This is a bit of a hack also. How should we wait for properties to be set?
annotations, err := jujuClient.GetServiceAnnotations(primaryServiceId)
if err != nil {
log.Warn("Error getting annotations", err)
// TODO: Mask error?
return nil, err
}
log.Info("Annotations on %v: %v", primaryServiceId, annotations)
state.Model.Options = map[string]string{}
state.SystemProperties = map[string]string{}
state.RelationMetadata = map[string]string{}
relationList := []relationProperty{}
for tagName, v := range annotations {
if strings.HasPrefix(tagName, ANNOTATION_PREFIX_INSTANCECONFIG) {
key := tagName[len(ANNOTATION_PREFIX_INSTANCECONFIG):]
state.Model.Options[key] = v
continue
}
if strings.HasPrefix(tagName, ANNOTATION_PREFIX_SYSTEM) {
key := tagName[len(ANNOTATION_PREFIX_SYSTEM):]
state.SystemProperties[key] = v
continue
}
if strings.HasPrefix(tagName, ANNOTATION_PREFIX_RELATIONINFO) {
suffix := tagName[len(ANNOTATION_PREFIX_RELATIONINFO):]
tokens := strings.SplitN(suffix, "_", 3)
if len(tokens) < 3 {
log.Debug("Ignoring unparseable tag: %v", tagName)
continue
}
unitId := tokens[0]
relationId := tokens[1]
key := tokens[2]
if key[0] != '_' {
state.RelationMetadata[key] = v
continue
}
relationTokens := strings.SplitN(relationId, ":", 2)
if len(relationTokens) != 2 {
log.Debug("Ignoring unparseable relation id in tag: %v", tagName)
continue
}
relationProperty := relationProperty{}
relationProperty.UnitId = unitId
assert.That(key[0] == '_')
relationProperty.Key = key[1:]
relationProperty.Value = v
relationProperty.RelationType = relationTokens[0]
relationProperty.RelationKey = relationTokens[1]
relationList = append(relationList, relationProperty)
continue
}
}
state.Relations = map[string]map[string]string{}
for _, relation := range relationList {
relationType := relation.RelationType
relations, found := state.Relations[relationType]
if !found {
relations = map[string]string{}
state.Relations[relationType] = relations
}
relations[relation.Key] = relation.Value
}
// TODO: Only if otherwise ready?
annotationsReady := self.bundleType.IsStarted(state.Relations)
// For a subordinate charm service (e.g. multimysql), we just watch for the annotation
if annotationsReady && state.Model.Status == "" && len(status.SubordinateTo) != 0 {
log.Info("Subordinate instance started (per annotations): %v", self)
state.Model.Status = "started"
}
if !annotationsReady {
log.Info("Instance not started (per annotations): %v", state.Relations)
state.Model.Status = "pending"
}
log.Info("Status of %v: %v", primaryServiceId, state.Model.Status)
// TODO: Fetch inherited properties from primary service and merge
return state, nil
}
func (self *Binder) AddDefaultBindingByPointer(p interface{}) {
t := reflect.TypeOf(p).Elem()
log.Info("Type is %v", t)
self.AddDefaultBinding(t)
}
func main() {
rand.Seed(time.Now().UTC().UnixNano())
options := GetOptions()
if options == nil {
log.Fatal("Error reading options")
os.Exit(1)
}
juju.Init()
binder := inject.NewBinder()
clientFactory := juju.EnvClientFactory
if options.AgentConf != "" && options.ApiPasswordPath != "" {
yaml, err := ioutil.ReadFile(options.AgentConf)
if err != nil {
log.Error("Error reading config file: %v", options.AgentConf, err)
os.Exit(1)
}
apiPassword, err := ioutil.ReadFile(options.ApiPasswordPath)
if err != nil {
log.Error("Error reading api password file: %v", options.ApiPasswordPath, err)
os.Exit(1)
}
agentConf := map[string]interface{}{}
err = goyaml.Unmarshal([]byte(yaml), &agentConf)
if err != nil {
log.Error("Error reading config file: %v", options.AgentConf, err)
os.Exit(1)
}
clientFactory = func() (*juju.Client, error) {
// password := agentConf["apipassword"].(string)
// tag := agentConf["tag"].(string)
// nonce := agentConf["nonce"].(string)
password := string(apiPassword)
tag := "user-admin"
nonce := ""
servers := []string{}
for _, apiaddress := range agentConf["apiaddresses"].([]interface{}) {
servers = append(servers, apiaddress.(string))
}
ca := agentConf["cacert"].(string)
info := api.Info{
Addrs: servers,
Password: password,
CACert: ca,
Tag: tag,
Nonce: nonce,
}
log.Info("%v", log.AsJson(info))
return juju.SimpleClientFactory(&info)
}
}
binder.AddProvider(clientFactory)
bundleStore := bundle.NewBundleStore("templates")
binder.AddSingleton(bundleStore)
authenticator := options.Authenticator
binder.AddSingleton(authenticator)
binder.BindType(reflect.TypeOf((*auth.Authenticator)(nil)).Elem()).ToInstance(authenticator)
cfTenantIdMap := cf.NewCfTenantIdMap(options.CfTenantId)
binder.AddSingleton(cfTenantIdMap)
binder.AddDefaultBindingByPointer((*cf.CfHelper)(nil))
apiclient, err := clientFactory()
// TODO: How would we get the full config "from afar"?
//confParams := map[string]interface{}{}
//// confParams["name"] = "jxaas"
//// confParams["firewall-mode"] = "instance"
//// confParams["development"] = false
////
//// confParams["type"] = "ec2"
////
//// confParams["ssl-hostname-verification"] = true
//// confParams["authorized-keys"] = ""
////
// // "state-port": DefaultStatePort,
// // "api-port": DefaultAPIPort,
// // "syslog-port": DefaultSyslogPort,
// // "bootstrap-timeout": DefaultBootstrapSSHTimeout,
// // "bootstrap-retry-delay": DefaultBootstrapSSHRetryDelay,
// // "bootstrap-addresses-delay": DefaultBootstrapSSHAddressesDelay,
// conf, err := config.New(config.NoDefaults, confParams)
// if err != nil {
// log.Fatal("Error building Juju config", err)
// os.Exit(1)
// }
// apiclient, err := juju.DirectClientFactory(conf)
if err != nil {
log.Fatal("Error building Juju client", err)
os.Exit(1)
}
system := core.NewSystem()
// This sadly doesn't work, because it is very difficult to download a charm :-(
// system.AddJxaasCharm(apiclient, "mongo", "cs:~justin-fathomdb/trusty/mongodb")
{
bundle, err := bundletype.LoadFromStore(bundleStore, "mongodb")
if err != nil {
log.Fatal("Error building mongodb bundle", err)
os.Exit(1)
}
system.AddBundleType(bundle)
}
{
bundle, err := bundletype.LoadFromStore(bundleStore, "mysql")
if err != nil {
log.Fatal("Error building mysql bundle", err)
os.Exit(1)
}
system.AddBundleType(bundle)
}
{
bundle, err := bundletype.LoadFromStore(bundleStore, "multimysql")
if err != nil {
log.Fatal("Error building multi-mysql bundle", err)
os.Exit(1)
}
system.AddBundleType(bundle)
}
{
bundle, err := bundletype.LoadFromStore(bundleStore, "es")
if err != nil {
log.Fatal("Error building elasticsearch bundle", err)
os.Exit(1)
}
system.AddBundleType(bundle)
}
{
bundle, err := bundletype.LoadFromStore(bundleStore, "pg")
if err != nil {
log.Fatal("Error building postgres bundle", err)
os.Exit(1)
}
system.AddBundleType(bundle)
}
{
bundle, err := bundletype.NewCassandraBundleType(bundleStore)
if err != nil {
log.Fatal("Error building cassandra bundle", err)
os.Exit(1)
}
system.AddBundleType(bundle)
}
privateUrl := options.PrivateUrl
for {
huddle, err := core.NewHuddle(system, bundleStore, apiclient, privateUrl)
if err != nil {
log.Fatal("Error building huddle", err)
os.Exit(1)
}
if isHuddleReady(huddle) {
log.Info("Huddle config is %v", huddle)
binder.AddSingleton(huddle)
break
}
time.Sleep(2 * time.Second)
}
rest := rs.NewRestServer()
rest.SetListen(options.ListenAddress)
typeEndpointXaas := reflect.TypeOf((*endpoints.EndpointXaas)(nil)).Elem()
binder.AddDefaultBinding(typeEndpointXaas)
rest.AddEndpoint("/xaas/", typeEndpointXaas)
typeEndpointXaasPrivate := reflect.TypeOf((*endpoints.EndpointXaasPrivate)(nil)).Elem()
binder.AddDefaultBinding(typeEndpointXaasPrivate)
rest.AddEndpoint("/xaasprivate/", typeEndpointXaasPrivate)
typeEndpointCf := reflect.TypeOf((*cf.EndpointCfRoot)(nil)).Elem()
binder.AddDefaultBinding(typeEndpointCf)
rest.AddEndpoint("/cf/", typeEndpointCf)
injector := binder.CreateInjector()
rest.WithInjector(injector)
rest.AddReader(rs.NewJsonMessageBodyReader())
rest.AddWriter(rs.NewJsonMessageBodyWriter())
log.Info("Ready!")
log.Fatal("Error serving HTTP", rest.ListenAndServe())
}