Beispiel #1
0
func syncDB(view db.Database, dkcsArg []docker.Container) ([]string, []db.Container) {
	score := func(left, right interface{}) int {
		dbc := left.(db.Container)
		dkc := right.(docker.Container)

		// Depending on the container, the command in the database could be
		// either The command plus it's arguments, or just it's arguments.  To
		// handle that case, we check both.
		cmd1 := dkc.Args
		cmd2 := append([]string{dkc.Path}, dkc.Args...)
		dbcCmd := dbc.Command

		for key, value := range dbc.Env {
			if dkc.Env[key] != value {
				return -1
			}
		}

		var dkcLabels []string
		for label, value := range dkc.Labels {
			if !docker.IsUserLabel(label) || value != docker.LabelTrueValue {
				continue
			}
			dkcLabels = append(dkcLabels, docker.ParseUserLabel(label))
		}

		switch {
		case dkc.Image != dbc.Image:
			return -1
		case len(dbcCmd) != 0 && !strEq(dbcCmd, cmd1) && !strEq(dbcCmd, cmd2):
			return -1
		case dkc.ID == dbc.DockerID:
			return 0
		default:
			return util.EditDistance(dbc.Labels, dkcLabels)
		}
	}
	pairs, dbcs, dkcs := join.Join(view.SelectFromContainer(nil), dkcsArg, score)

	for _, pair := range pairs {
		dbc := pair.L.(db.Container)
		dbc.DockerID = pair.R.(docker.Container).ID
		view.Commit(dbc)
	}

	var term []string
	for _, dkc := range dkcs {
		term = append(term, dkc.(docker.Container).ID)
	}

	var boot []db.Container
	for _, dbc := range dbcs {
		boot = append(boot, dbc.(db.Container))
	}

	return term, boot
}
Beispiel #2
0
func updateWorker(view db.Database, self db.Minion, store Store,
	etcdData storeData) {

	var containers []storeContainer
	for _, etcdc := range etcdData.containers {
		if etcdc.Minion == self.PrivateIP {
			containers = append(containers, etcdc)
		}
	}

	pairs, dbcs, etcdcs := join.Join(view.SelectFromContainer(nil), containers,
		func(left, right interface{}) int {
			dbc := left.(db.Container)
			l := storeContainer{
				StitchID: dbc.StitchID,
				Minion:   dbc.Minion,
				Image:    dbc.Image,
				Command:  dbc.Command,
				Env:      dbc.Env,
				Labels:   dbc.Labels,
			}
			return containerJoinScore(l, right.(storeContainer))
		})

	for _, i := range dbcs {
		dbc := i.(db.Container)
		view.Remove(dbc)
	}

	for _, etcdc := range etcdcs {
		pairs = append(pairs, join.Pair{
			L: view.InsertContainer(),
			R: etcdc,
		})
	}

	for _, pair := range pairs {
		dbc := pair.L.(db.Container)
		etcdc := pair.R.(storeContainer)

		dbc.StitchID = etcdc.StitchID
		dbc.Minion = etcdc.Minion
		dbc.Image = etcdc.Image
		dbc.Command = etcdc.Command
		dbc.Env = etcdc.Env
		dbc.Labels = etcdc.Labels

		view.Commit(dbc)
	}

	updateContainerIP(view.SelectFromContainer(nil), self.PrivateIP, store)
}
Beispiel #3
0
func updateLeaderDBC(view db.Database, dbcs []db.Container,
	etcdData storeData, ipMap map[string]string) {

	for _, dbc := range dbcs {
		ipVal := ipMap[strconv.Itoa(dbc.StitchID)]
		mac := ipdef.IPStrToMac(ipVal)
		if dbc.IP != ipVal || dbc.Mac != mac {
			dbc.IP = ipVal
			dbc.Mac = mac
			view.Commit(dbc)
		}
	}
}
Beispiel #4
0
func updatePlacements(view db.Database, spec stitch.Stitch) {
	stitchPlacements := toDBPlacements(spec.QueryPlacements())
	key := func(val interface{}) interface{} {
		pVal := val.(db.Placement)
		return struct {
			tl   string
			rule db.PlacementRule
		}{pVal.TargetLabel, pVal.Rule}
	}

	_, addSet, removeSet := join.HashJoin(stitchPlacements,
		db.PlacementSlice(view.SelectFromPlacement(nil)), key, key)

	for _, toAddIntf := range addSet {
		toAdd := toAddIntf.(db.Placement)

		newPlacement := view.InsertPlacement()
		newPlacement.TargetLabel = toAdd.TargetLabel
		newPlacement.Rule = toAdd.Rule
		view.Commit(newPlacement)
	}

	for _, toRemove := range removeSet {
		view.Remove(toRemove.(db.Placement))
	}
}
Beispiel #5
0
func updatePlacements(view db.Database, spec stitch.Stitch) {
	var placements db.PlacementSlice
	for _, sp := range spec.Placements {
		placements = append(placements, db.Placement{
			TargetLabel: sp.TargetLabel,
			Exclusive:   sp.Exclusive,
			OtherLabel:  sp.OtherLabel,
			Provider:    sp.Provider,
			Size:        sp.Size,
			Region:      sp.Region,
		})
	}

	key := func(val interface{}) interface{} {
		p := val.(db.Placement)
		p.ID = 0
		return p
	}

	dbPlacements := db.PlacementSlice(view.SelectFromPlacement(nil))
	_, addSet, removeSet := join.HashJoin(placements, dbPlacements, key, key)

	for _, toAddIntf := range addSet {
		toAdd := toAddIntf.(db.Placement)

		id := view.InsertPlacement().ID
		toAdd.ID = id
		view.Commit(toAdd)
	}

	for _, toRemove := range removeSet {
		view.Remove(toRemove.(db.Placement))
	}
}
Beispiel #6
0
func updateConnections(view db.Database, spec stitch.Stitch) {
	scs, vcs := stitch.ConnectionSlice(spec.Connections),
		view.SelectFromConnection(nil)

	dbcKey := func(val interface{}) interface{} {
		c := val.(db.Connection)
		return stitch.Connection{
			From:    c.From,
			To:      c.To,
			MinPort: c.MinPort,
			MaxPort: c.MaxPort,
		}
	}

	pairs, stitches, dbcs := join.HashJoin(scs, db.ConnectionSlice(vcs), nil, dbcKey)

	for _, dbc := range dbcs {
		view.Remove(dbc.(db.Connection))
	}

	for _, stitchc := range stitches {
		pairs = append(pairs, join.Pair{L: stitchc, R: view.InsertConnection()})
	}

	for _, pair := range pairs {
		stitchc := pair.L.(stitch.Connection)
		dbc := pair.R.(db.Connection)

		dbc.From = stitchc.From
		dbc.To = stitchc.To
		dbc.MinPort = stitchc.MinPort
		dbc.MaxPort = stitchc.MaxPort
		view.Commit(dbc)
	}
}
Beispiel #7
0
func machineTxn(view db.Database, stitch stitch.Stitch) error {
	// XXX: How best to deal with machines that don't specify enough information?
	maxPrice, _ := stitch.QueryFloat("MaxPrice")
	stitchMachines := toDBMachine(stitch.QueryMachines(), maxPrice)

	dbMachines := view.SelectFromMachine(nil)

	scoreFun := func(left, right interface{}) int {
		stitchMachine := left.(db.Machine)
		dbMachine := right.(db.Machine)

		switch {
		case dbMachine.Provider != stitchMachine.Provider:
			return -1
		case dbMachine.Region != stitchMachine.Region:
			return -1
		case dbMachine.Size != "" && stitchMachine.Size != dbMachine.Size:
			return -1
		case dbMachine.Role != db.None && dbMachine.Role != stitchMachine.Role:
			return -1
		case dbMachine.DiskSize != stitchMachine.DiskSize:
			return -1
		case dbMachine.PrivateIP == "":
			return 2
		case dbMachine.PublicIP == "":
			return 1
		default:
			return 0
		}
	}

	pairs, bootList, terminateList := join.Join(stitchMachines, dbMachines, scoreFun)

	for _, toTerminate := range terminateList {
		toTerminate := toTerminate.(db.Machine)
		view.Remove(toTerminate)
	}

	for _, bootSet := range bootList {
		bootSet := bootSet.(db.Machine)

		pairs = append(pairs, join.Pair{L: bootSet, R: view.InsertMachine()})
	}

	for _, pair := range pairs {
		stitchMachine := pair.L.(db.Machine)
		dbMachine := pair.R.(db.Machine)

		dbMachine.Role = stitchMachine.Role
		dbMachine.Size = stitchMachine.Size
		dbMachine.DiskSize = stitchMachine.DiskSize
		dbMachine.Provider = stitchMachine.Provider
		dbMachine.Region = stitchMachine.Region
		dbMachine.SSHKeys = stitchMachine.SSHKeys
		view.Commit(dbMachine)
	}

	return nil
}
Beispiel #8
0
func testUpdateDBLabels(t *testing.T, view db.Database) {
	labelStruct := map[string]string{"a": "10.0.0.2"}
	ipMap := map[string]string{"1": "10.0.0.3", "2": "10.0.0.4"}
	containerSlice := []storeContainer{
		{
			StitchID: 1,
			Labels:   []string{"a", "b"},
		},
		{
			StitchID: 2,
			Labels:   []string{"a"},
		},
	}

	updateDBLabels(view, storeData{
		containers: containerSlice,
		multiHost:  labelStruct,
	}, ipMap)

	type labelIPs struct {
		labelIP      string
		containerIPs []string
	}
	lip := map[string]labelIPs{}
	for _, l := range view.SelectFromLabel(nil) {
		_, ok := lip[l.Label]
		assert.False(t, ok)

		lip[l.Label] = labelIPs{
			labelIP:      l.IP,
			containerIPs: l.ContainerIPs,
		}
	}

	resultLabels := map[string]labelIPs{
		"a": {
			labelIP:      "10.0.0.2",
			containerIPs: []string{"10.0.0.3", "10.0.0.4"},
		},
		"b": {
			labelIP:      "10.0.0.3",
			containerIPs: []string{"10.0.0.3"},
		},
	}

	assert.Equal(t, resultLabels, lip)
}
Beispiel #9
0
func updateDBLabels(view db.Database, etcdData storeData, ipMap map[string]string) {
	// Gather all of the label keys and IPs for single host labels, and IPs of
	// the containers in a given label.
	containerIPs := map[string][]string{}
	labelIPs := map[string]string{}
	labelKeys := map[string]struct{}{}
	for _, c := range etcdData.containers {
		for _, l := range c.Labels {
			labelKeys[l] = struct{}{}
			cIP := ipMap[strconv.Itoa(c.StitchID)]
			if _, ok := etcdData.multiHost[l]; !ok {
				labelIPs[l] = cIP
			}

			// The ordering of IPs between function calls will be consistent
			// because the containers are sorted by their StitchIDs when
			// inserted into etcd.
			containerIPs[l] = append(containerIPs[l], cIP)
		}
	}

	labelKeyFunc := func(val interface{}) interface{} {
		return val.(db.Label).Label
	}

	labelKeySlice := join.StringSlice{}
	for l := range labelKeys {
		labelKeySlice = append(labelKeySlice, l)
	}

	pairs, dbls, dirKeys := join.HashJoin(db.LabelSlice(view.SelectFromLabel(nil)),
		labelKeySlice, labelKeyFunc, nil)

	for _, dbl := range dbls {
		view.Remove(dbl.(db.Label))
	}

	for _, key := range dirKeys {
		pairs = append(pairs, join.Pair{L: view.InsertLabel(), R: key})
	}

	for _, pair := range pairs {
		dbl := pair.L.(db.Label)
		dbl.Label = pair.R.(string)
		if _, ok := etcdData.multiHost[dbl.Label]; ok {
			dbl.IP = etcdData.multiHost[dbl.Label]
			dbl.MultiHost = true
		} else {
			dbl.IP = labelIPs[dbl.Label]
			dbl.MultiHost = false
		}
		dbl.ContainerIPs = containerIPs[dbl.Label]

		view.Commit(dbl)
	}
}
Beispiel #10
0
func updateTxn(view db.Database) error {
	cluster, err := view.GetCluster()
	if err != nil {
		return err
	}

	stitch, err := stitch.FromJSON(cluster.Spec)
	if err != nil {
		return err
	}

	cluster.Namespace = stitch.Namespace
	view.Commit(cluster)

	machineTxn(view, stitch)
	aclTxn(view, stitch)
	return nil
}
Beispiel #11
0
func placeContainers(view db.Database) {
	constraints := view.SelectFromPlacement(nil)
	containers := view.SelectFromContainer(nil)
	minions := view.SelectFromMinion(nil)

	ctx := makeContext(minions, constraints, containers)
	cleanupPlacements(ctx)
	placeUnassigned(ctx)

	for _, change := range ctx.changed {
		view.Commit(*change)
	}
}
Beispiel #12
0
func readContainerTransact(view db.Database, dir directory) {
	minion, err := view.MinionSelf()
	worker := err == nil && minion.Role == db.Worker

	for _, container := range view.SelectFromContainer(nil) {
		container.IP = ""
		var labels []string
		if children, ok := dir[container.DockerID]; ok {
			json.Unmarshal([]byte(children["Labels"]), &labels)

			container.IP = children["IP"]
			ip := net.ParseIP(container.IP).To4()
			if ip != nil {
				container.Mac = fmt.Sprintf("02:00:%02x:%02x:%02x:%02x",
					ip[0], ip[1], ip[2], ip[3])
			}
		}

		if worker {
			// Masters get their labels from the policy, workers from the
			// etcd store.
			container.Labels = labels
		}

		view.Commit(container)
	}
}
Beispiel #13
0
func testReadLabelTransact(t *testing.T, view db.Database) {
	dir := directory(map[string]map[string]string{
		"a": {"IP": "10.0.0.2"},
		"b": {"IP": "10.0.0.3"},
		"c": {"IP": "10.0.0.4"},
	})

	readLabelTransact(view, dir)
	lip := map[string]string{}
	for _, l := range view.SelectFromLabel(nil) {
		lip[l.Label] = l.IP
	}

	exp := map[string]string{
		"a": "10.0.0.2",
		"b": "10.0.0.3",
		"c": "10.0.0.4",
	}
	if !eq(lip, exp) {
		t.Error(spew.Sprintf("Found: %s\nExpected: %s\n", lip, exp))
	}

	delete(dir, "c")
	delete(exp, "c")

	dir["b"]["IP"] = "10.0.0.4"
	exp["b"] = "10.0.0.4"

	readLabelTransact(view, dir)
	lip = map[string]string{}
	for _, l := range view.SelectFromLabel(nil) {
		lip[l.Label] = l.IP
	}

	if !eq(lip, exp) {
		t.Error(spew.Sprintf("Found: %s\nExpected: %s\n", lip, exp))
	}
}
Beispiel #14
0
func updateContainers(view db.Database, spec stitch.Stitch) {
	score := func(l, r interface{}) int {
		left := l.(db.Container)
		right := r.(db.Container)

		if left.Image != right.Image ||
			!util.StrSliceEqual(left.Command, right.Command) ||
			!util.StrStrMapEqual(left.Env, right.Env) {
			return -1
		}

		score := util.EditDistance(left.Labels, right.Labels)
		if left.StitchID != right.StitchID {
			score++
		}
		return score
	}

	pairs, news, dbcs := join.Join(queryContainers(spec),
		view.SelectFromContainer(nil), score)

	for _, dbc := range dbcs {
		view.Remove(dbc.(db.Container))
	}

	for _, new := range news {
		pairs = append(pairs, join.Pair{L: new, R: view.InsertContainer()})
	}

	for _, pair := range pairs {
		newc := pair.L.(db.Container)
		dbc := pair.R.(db.Container)

		// By sorting the labels we prevent the database from getting confused
		// when their order is non deterministic.
		dbc.Labels = newc.Labels
		sort.Sort(sort.StringSlice(dbc.Labels))

		dbc.Command = newc.Command
		dbc.Image = newc.Image
		dbc.Env = newc.Env
		dbc.StitchID = newc.StitchID
		view.Commit(dbc)
	}
}
Beispiel #15
0
func (sv *supervisor) runAppTransact(view db.Database,
	dkcsArgs []docker.Container) []string {

	var tearDowns []string

	dbKey := func(val interface{}) interface{} {
		return val.(db.Container).DockerID
	}
	dkKey := func(val interface{}) interface{} {
		return val.(docker.Container).ID
	}

	pairs, dbcs, dkcs := join.HashJoin(db.ContainerSlice(
		view.SelectFromContainer(nil)),
		docker.ContainerSlice(dkcsArgs), dbKey, dkKey)

	for _, iface := range dbcs {
		dbc := iface.(db.Container)

		tearDowns = append(tearDowns, dbc.DockerID)
		view.Remove(dbc)
	}

	for _, dkc := range dkcs {
		pairs = append(pairs, join.Pair{L: view.InsertContainer(), R: dkc})
	}

	for _, pair := range pairs {
		dbc := pair.L.(db.Container)
		dkc := pair.R.(docker.Container)

		dbc.DockerID = dkc.ID
		dbc.Pid = dkc.Pid
		dbc.Image = dkc.Image
		dbc.Command = append([]string{dkc.Path}, dkc.Args...)
		view.Commit(dbc)
	}

	return tearDowns
}
Beispiel #16
0
func readLabelTransact(view db.Database, dir directory) {
	lKey := func(val interface{}) interface{} {
		return val.(db.Label).Label
	}
	pairs, dbls, dirKeys := join.HashJoin(db.LabelSlice(view.SelectFromLabel(nil)),
		join.StringSlice(dir.keys()), lKey, nil)

	for _, dbl := range dbls {
		view.Remove(dbl.(db.Label))
	}

	for _, key := range dirKeys {
		pairs = append(pairs, join.Pair{L: view.InsertLabel(), R: key})
	}

	for _, pair := range pairs {
		dbl := pair.L.(db.Label)
		dbl.Label = pair.R.(string)
		dbl.IP = dir[dbl.Label]["IP"]
		_, dbl.MultiHost = dir[dbl.Label]["MultiHost"]
		view.Commit(dbl)
	}
}
Beispiel #17
0
func clusterTxn(view db.Database, stitch stitch.Stitch) error {
	namespace := stitch.QueryString("Namespace")
	if namespace == "" {
		namespace = "DEFAULT_NAMESPACE"
		msg := "policy did not specify 'Namespace', defaulting to '%s'"
		log.Warn(fmt.Sprintf(msg, namespace))
	}

	cluster, err := view.GetCluster()
	if err != nil {
		cluster = view.InsertCluster()
	}

	cluster.Namespace = namespace
	cluster.Spec = stitch.String()
	view.Commit(cluster)
	return nil
}
Beispiel #18
0
func aclTxn(view db.Database, specHandle stitch.Stitch) {
	aclRow, err := view.GetACL()
	if err != nil {
		aclRow = view.InsertACL()
	}

	aclRow.Admin = resolveACLs(specHandle.AdminACL)

	var applicationPorts []db.PortRange
	for _, conn := range specHandle.Connections {
		if conn.From == stitch.PublicInternetLabel {
			applicationPorts = append(applicationPorts, db.PortRange{
				MinPort: conn.MinPort,
				MaxPort: conn.MaxPort,
			})
		}
	}
	aclRow.ApplicationPorts = applicationPorts

	view.Commit(aclRow)
}
Beispiel #19
0
func aclTxn(view db.Database, stitch stitch.Stitch) error {
	cluster, err := view.GetCluster()
	if err != nil {
		return err
	}

	machines := view.SelectFromMachine(func(m db.Machine) bool {
		return m.PublicIP != ""
	})
	acls := resolveACLs(stitch.QueryStrSlice("AdminACL"))

	for _, m := range machines {
		acls = append(acls, m.PublicIP+"/32")
	}

	sort.Strings(acls)
	cluster.ACLs = acls
	view.Commit(cluster)

	return nil
}
Beispiel #20
0
func testReadContainerTransact(t *testing.T, view db.Database) {
	minion := view.InsertMinion()
	minion.Role = db.Worker
	minion.Self = true
	view.Commit(minion)

	for _, id := range []string{"a", "b"} {
		container := view.InsertContainer()
		container.DockerID = id
		view.Commit(container)
	}

	container := view.InsertContainer()
	container.DockerID = "c"
	container.IP = "junk"
	view.Commit(container)

	dir := directory(map[string]map[string]string{
		"a": {"IP": "1.0.0.0", "Labels": `["e"]`},
		"b": {"IP": "2.0.0.0", "Labels": `["e", "f"]`},
	})

	readContainerTransact(view, dir)

	ipMap := map[string]string{}
	labelMap := map[string][]string{}
	for _, c := range view.SelectFromContainer(nil) {
		ipMap[c.DockerID] = c.IP
		labelMap[c.DockerID] = c.Labels
	}

	expIPMap := map[string]string{
		"a": "1.0.0.0",
		"b": "2.0.0.0",
		"c": "",
	}
	if !eq(ipMap, expIPMap) {
		t.Error(spew.Sprintf("Found %s, Expected: %s", ipMap, expIPMap))
	}

	expLabelMap := map[string][]string{
		"a": {"e"},
		"b": {"e", "f"},
		"c": nil,
	}

	if !eq(labelMap, expLabelMap) {
		t.Error(spew.Sprintf("Found %s, Expected: %s", ipMap, expIPMap))
	}
}
Beispiel #21
0
func testUpdateWorkerDBC(t *testing.T, view db.Database) {
	minion := view.InsertMinion()
	minion.Role = db.Worker
	minion.Self = true
	view.Commit(minion)

	for id := 1; id < 3; id++ {
		container := view.InsertContainer()
		container.StitchID = id
		container.IP = fmt.Sprintf("10.1.0.%d", id-1)
		container.Minion = "1.2.3.4"
		container.Command = []string{"echo", "hi"}
		container.Env = map[string]string{"GOPATH": "~"}
		view.Commit(container)
	}

	cs := storeContainerSlice{
		{
			StitchID: 1,
			Command:  []string{"echo", "hi"},
			Labels:   []string{"red", "blue"},
			Env:      map[string]string{"GOPATH": "~"},
			Minion:   "1.2.3.4",
		}, {
			StitchID: 2,
			Command:  []string{"echo", "hi"},
			Labels:   []string{"blue", "green"},
			Env:      map[string]string{"GOPATH": "~"},
			Minion:   "1.2.3.4",
		}, {
			StitchID: 3,
			Command:  []string{"echo", "bye"},
			Labels:   []string{"blue", "green"},
			Env:      map[string]string{"GOPATH": "~"},
			Minion:   "1.2.3.5",
		},
	}

	store := newTestMock()

	jsonCS, _ := json.Marshal(cs)
	err := store.Set(containerStore, string(jsonCS), 0)
	assert.Nil(t, err)

	jsonNull, _ := json.Marshal(map[string]string{})
	minionDirKey := path.Join(nodeStore, "1.2.3.4")
	err = store.Set(path.Join(minionDirKey, minionIPStore), string(jsonNull), 0)
	assert.Nil(t, err)

	updateWorker(view, db.Minion{PrivateIP: "1.2.3.4",
		Subnet: "10.1.0.0"}, store, storeData{containers: cs})

	ipMap := map[int]string{}
	labelMap := map[int][]string{}
	commandMap := map[string][]string{}
	envMap := map[string]map[string]string{}
	for _, c := range view.SelectFromContainer(nil) {
		ipMap[c.StitchID] = c.IP
		labelMap[c.StitchID] = c.Labels
		commandMap[c.DockerID] = c.Command
		envMap[c.DockerID] = c.Env
	}

	expIPMap := map[int]string{
		1: "10.1.0.0",
		2: "10.1.0.1",
	}

	assert.Equal(t, expIPMap, ipMap)

	resultMap := map[string]string{}
	storeIPs, _ := store.Get(path.Join(minionDirKey, minionIPStore))
	json.Unmarshal([]byte(storeIPs), &resultMap)

	for id, ip := range resultMap {
		sid, _ := strconv.Atoi(id)
		if otherIP, ok := ipMap[sid]; !ok || ip != otherIP {
			t.Fatalf("IPs did not match: %s vs %s", ip, otherIP)
		}
	}

	expLabelMap := map[int][]string{
		1: {"red", "blue"},
		2: {"blue", "green"},
	}

	assert.Equal(t, expLabelMap, labelMap)
}
Beispiel #22
0
func checkSupervisorInit(view db.Database) bool {
	self, err := view.MinionSelf()
	return err == nil && self.SupervisorInit
}