예제 #1
0
func (x *ImportCluster) ImportAWSCluster() error {
	awsCloud := x.Cloud.(*awsup.AWSCloud)
	clusterName := x.ClusterName

	if clusterName == "" {
		return fmt.Errorf("ClusterName must be specified")
	}

	var instanceGroups []*api.InstanceGroup

	cluster := &api.Cluster{}
	cluster.Spec.CloudProvider = "aws"
	cluster.Name = clusterName

	cluster.Spec.KubeControllerManager = &api.KubeControllerManagerConfig{}

	masterGroup := &api.InstanceGroup{}
	masterGroup.Spec.Role = api.InstanceGroupRoleMaster
	masterGroup.Name = "masters"
	masterGroup.Spec.MinSize = fi.Int(1)
	masterGroup.Spec.MaxSize = fi.Int(1)
	instanceGroups = append(instanceGroups, masterGroup)

	instances, err := findInstances(awsCloud)
	if err != nil {
		return fmt.Errorf("error finding instances: %v", err)
	}

	var masterInstance *ec2.Instance
	for _, instance := range instances {
		role, _ := awsup.FindEC2Tag(instance.Tags, "Role")
		if role == clusterName+"-master" {
			if masterInstance != nil {
				masterState := aws.StringValue(masterInstance.State.Name)
				thisState := aws.StringValue(instance.State.Name)

				glog.Infof("Found multiple masters: %s and %s", masterState, thisState)

				if masterState == "terminated" && thisState != "terminated" {
					// OK
				} else if thisState == "terminated" && masterState != "terminated" {
					// Ignore this one
					continue
				} else {
					return fmt.Errorf("found multiple masters")
				}
			}
			masterInstance = instance
		}
	}
	if masterInstance == nil {
		return fmt.Errorf("could not find master node")
	}
	masterInstanceID := aws.StringValue(masterInstance.InstanceId)
	glog.Infof("Found master: %q", masterInstanceID)

	masterGroup.Spec.MachineType = aws.StringValue(masterInstance.InstanceType)

	masterSubnetID := aws.StringValue(masterInstance.SubnetId)

	subnets, err := DescribeSubnets(x.Cloud)
	if err != nil {
		return fmt.Errorf("error finding subnets: %v", err)
	}
	var masterSubnet *ec2.Subnet
	for _, s := range subnets {
		if masterSubnetID == aws.StringValue(s.SubnetId) {
			if masterSubnet != nil {
				return fmt.Errorf("found duplicate subnet")
			}
			masterSubnet = s
		}
	}
	if masterSubnet == nil {
		return fmt.Errorf("cannot find subnet %q", masterSubnetID)
	}

	vpcID := aws.StringValue(masterInstance.VpcId)
	var vpc *ec2.Vpc
	{
		vpc, err = awsCloud.DescribeVPC(vpcID)
		if err != nil {
			return err
		}
		if vpc == nil {
			return fmt.Errorf("cannot find vpc %q", vpcID)
		}
	}

	cluster.Spec.NetworkID = vpcID
	cluster.Spec.NetworkCIDR = aws.StringValue(vpc.CidrBlock)

	az := aws.StringValue(masterSubnet.AvailabilityZone)
	masterGroup.Spec.Zones = []string{az}
	cluster.Spec.Zones = append(cluster.Spec.Zones, &api.ClusterZoneSpec{
		Name: az,

		// We will allocate a new CIDR
		//CIDR: aws.StringValue(masterSubnet.CidrBlock),
	})

	userData, err := GetInstanceUserData(awsCloud, aws.StringValue(masterInstance.InstanceId))
	if err != nil {
		return fmt.Errorf("error getting master user-data: %v", err)
	}

	conf, err := ParseUserDataConfiguration(userData)
	if err != nil {
		return fmt.Errorf("error parsing master user-data: %v", err)
	}

	//master := &NodeSSH{
	//	Hostname: c.Master,
	//}
	//err := master.AddSSHIdentity(c.SSHIdentity)
	//if err != nil {
	//	return err
	//}
	//
	//
	//fmt.Printf("Connecting to node on %s\n", c.Node)
	//
	//node := &NodeSSH{
	//	Hostname: c.Node,
	//}
	//err = node.AddSSHIdentity(c.SSHIdentity)
	//if err != nil {
	//	return err
	//}

	instancePrefix := conf.Settings["INSTANCE_PREFIX"]
	if instancePrefix == "" {
		return fmt.Errorf("cannot determine INSTANCE_PREFIX")
	}
	if instancePrefix != clusterName {
		return fmt.Errorf("INSTANCE_PREFIX %q did not match cluster name %q", instancePrefix, clusterName)
	}

	//k8s.NodeMachineType, err = InstanceType(node)
	//if err != nil {
	//	return fmt.Errorf("cannot determine node instance type: %v", err)
	//}

	// We want to upgrade!
	// k8s.ImageId = ""

	//clusterConfig.ClusterIPRange = conf.Settings["CLUSTER_IP_RANGE"]
	cluster.Spec.KubeControllerManager.AllocateNodeCIDRs = conf.ParseBool("ALLOCATE_NODE_CIDRS")
	//clusterConfig.KubeUser = conf.Settings["KUBE_USER"]
	cluster.Spec.ServiceClusterIPRange = conf.Settings["SERVICE_CLUSTER_IP_RANGE"]
	cluster.Spec.NonMasqueradeCIDR = conf.Settings["NON_MASQUERADE_CIDR"]
	//clusterConfig.EnableClusterMonitoring = conf.Settings["ENABLE_CLUSTER_MONITORING"]
	//clusterConfig.EnableClusterLogging = conf.ParseBool("ENABLE_CLUSTER_LOGGING")
	//clusterConfig.EnableNodeLogging = conf.ParseBool("ENABLE_NODE_LOGGING")
	//clusterConfig.LoggingDestination = conf.Settings["LOGGING_DESTINATION"]
	//clusterConfig.ElasticsearchLoggingReplicas, err = parseInt(conf.Settings["ELASTICSEARCH_LOGGING_REPLICAS"])
	//if err != nil {
	//	return fmt.Errorf("cannot parse ELASTICSEARCH_LOGGING_REPLICAS=%q: %v", conf.Settings["ELASTICSEARCH_LOGGING_REPLICAS"], err)
	//}
	//clusterConfig.EnableClusterDNS = conf.ParseBool("ENABLE_CLUSTER_DNS")
	//clusterConfig.EnableClusterUI = conf.ParseBool("ENABLE_CLUSTER_UI")
	//clusterConfig.DNSReplicas, err = parseInt(conf.Settings["DNS_REPLICAS"])
	//if err != nil {
	//	return fmt.Errorf("cannot parse DNS_REPLICAS=%q: %v", conf.Settings["DNS_REPLICAS"], err)
	//}
	//clusterConfig.DNSServerIP = conf.Settings["DNS_SERVER_IP"]
	cluster.Spec.ClusterDNSDomain = conf.Settings["DNS_DOMAIN"]
	//clusterConfig.AdmissionControl = conf.Settings["ADMISSION_CONTROL"]
	//clusterConfig.MasterIPRange = conf.Settings["MASTER_IP_RANGE"]
	//clusterConfig.DNSServerIP = conf.Settings["DNS_SERVER_IP"]
	//clusterConfig.DockerStorage = conf.Settings["DOCKER_STORAGE"]
	//k8s.MasterExtraSans = conf.Settings["MASTER_EXTRA_SANS"] // Not user set

	primaryNodeSet := &api.InstanceGroup{}
	primaryNodeSet.Spec.Role = api.InstanceGroupRoleNode
	primaryNodeSet.Name = "nodes"
	instanceGroups = append(instanceGroups, primaryNodeSet)

	//primaryNodeSet.Spec.MinSize, err = conf.ParseInt("NUM_MINIONS")
	//if err != nil {
	//	return fmt.Errorf("cannot parse NUM_MINIONS=%q: %v", conf.Settings["NUM_MINIONS"], err)
	//}

	{
		groups, err := findAutoscalingGroups(awsCloud, awsCloud.Tags())
		if err != nil {
			return fmt.Errorf("error listing autoscaling groups: %v", err)
		}
		if len(groups) == 0 {
			glog.Warningf("No Autoscaling group found")
		}
		if len(groups) == 1 {
			glog.Warningf("Multiple Autoscaling groups found")
		}
		minSize := 0
		maxSize := 0
		for _, group := range groups {
			minSize += int(aws.Int64Value(group.MinSize))
			maxSize += int(aws.Int64Value(group.MaxSize))
		}
		if minSize != 0 {
			primaryNodeSet.Spec.MinSize = fi.Int(minSize)
		}
		if maxSize != 0 {
			primaryNodeSet.Spec.MaxSize = fi.Int(maxSize)
		}

		// TODO: machine types
		//primaryNodeSet.NodeMachineType = k8s.MasterMachineType
	}

	if conf.Version == "1.1" {
		// If users went with defaults on some things, clear them out so they get the new defaults
		//if clusterConfig.AdmissionControl == "NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota" {
		//	// More admission controllers in 1.2
		//	clusterConfig.AdmissionControl = ""
		//}
		if masterGroup.Spec.MachineType == "t2.micro" {
			// Different defaults in 1.2
			masterGroup.Spec.MachineType = ""
		}
		if primaryNodeSet.Spec.MachineType == "t2.micro" {
			// Encourage users to pick something better...
			primaryNodeSet.Spec.MachineType = ""
		}
	}
	if conf.Version == "1.2" {
		// If users went with defaults on some things, clear them out so they get the new defaults
		//if clusterConfig.AdmissionControl == "NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,ResourceQuota" {
		//	// More admission controllers in 1.2
		//	clusterConfig.AdmissionControl = ""
		//}
	}

	for _, etcdClusterName := range []string{"main", "events"} {
		etcdCluster := &api.EtcdClusterSpec{
			Name: etcdClusterName,
		}
		for _, az := range masterGroup.Spec.Zones {
			etcdCluster.Members = append(etcdCluster.Members, &api.EtcdMemberSpec{
				Name: az,
				Zone: az,
			})
		}
		cluster.Spec.EtcdClusters = append(cluster.Spec.EtcdClusters, etcdCluster)
	}

	//if masterInstance.PublicIpAddress != nil {
	//	eip, err := findElasticIP(cloud, *masterInstance.PublicIpAddress)
	//	if err != nil {
	//		return err
	//	}
	//	if eip != nil {
	//		k8s.MasterElasticIP = masterInstance.PublicIpAddress
	//	}
	//}
	//
	//vpc, err := cloud.DescribeVPC(*k8s.VPCID)
	//if err != nil {
	//	return err
	//}
	//k8s.DHCPOptionsID = vpc.DhcpOptionsId
	//
	//igw, err := findInternetGateway(cloud, *k8s.VPCID)
	//if err != nil {
	//	return err
	//}
	//if igw == nil {
	//	return fmt.Errorf("unable to find internet gateway for VPC %q", k8s.VPCID)
	//}
	//k8s.InternetGatewayID = igw.InternetGatewayId
	//
	//rt, err := findRouteTable(cloud, *k8s.SubnetID)
	//if err != nil {
	//	return err
	//}
	//if rt == nil {
	//	return fmt.Errorf("unable to find route table for Subnet %q", k8s.SubnetID)
	//}
	//k8s.RouteTableID = rt.RouteTableId

	//b.Context = "aws_" + instancePrefix

	newKeyStore := x.StateStore.CA()

	//caCert, err := masterSSH.Join("ca.crt").ReadFile()
	caCert, err := conf.ParseCert("CA_CERT")
	if err != nil {
		return err
	}
	err = newKeyStore.AddCert(fi.CertificateId_CA, caCert)
	if err != nil {
		return err
	}

	////masterKey, err := masterSSH.Join("server.key").ReadFile()
	//masterKey, err := conf.ParseKey("MASTER_KEY")
	//if err != nil {
	//	return err
	//}
	////masterCert, err := masterSSH.Join("server.cert").ReadFile()
	//masterCert, err := conf.ParseCert("MASTER_CERT")
	//if err != nil {
	//	return err
	//}
	//err = keyStore.ImportKeypair("master", masterKey, masterCert)
	//if err != nil {
	//	return err
	//}
	//
	////kubeletKey, err := kubeletSSH.Join("kubelet.key").ReadFile()
	//kubeletKey, err := conf.ParseKey("KUBELET_KEY")
	//if err != nil {
	//	return err
	//}
	////kubeletCert, err := kubeletSSH.Join("kubelet.cert").ReadFile()
	//kubeletCert, err := conf.ParseCert("KUBELET_CERT")
	//if err != nil {
	//	return err
	//}
	//err = keyStore.ImportKeypair("kubelet", kubeletKey, kubeletCert)
	//if err != nil {
	//	return err
	//}

	// We don't store the kubecfg key
	//kubecfgKey, err := masterSSH.Join("kubecfg.key").ReadFile()
	//if err != nil {
	//	return err
	//}
	//kubecfgCert, err := masterSSH.Join("kubecfg.cert").ReadFile()
	//if err != nil {
	//	return err
	//}
	//err = keyStore.ImportKeypair("kubecfg", kubecfgKey, kubecfgCert)
	//if err != nil {
	//	return err
	//}

	//// We will generate new tokens, but some of these are in existing API objects
	//secretStore := x.StateStore.Secrets()
	//kubePassword := conf.Settings["KUBE_PASSWORD"]
	//kubeletToken = conf.Settings["KUBELET_TOKEN"]
	//kubeProxyToken = conf.Settings["KUBE_PROXY_TOKEN"]

	err = api.WriteConfig(x.StateStore, cluster, instanceGroups)
	if err != nil {
		return err
	}

	return nil
}
예제 #2
0
func (x *UpgradeCluster) Upgrade() error {
	awsCloud := x.Cloud.(*awsup.AWSCloud)

	cluster := x.ClusterConfig

	newClusterName := x.NewClusterName
	if newClusterName == "" {
		return fmt.Errorf("NewClusterName must be specified")
	}
	oldClusterName := x.OldClusterName
	if oldClusterName == "" {
		return fmt.Errorf("OldClusterName must be specified")
	}

	oldTags := awsCloud.Tags()

	newTags := awsCloud.Tags()
	newTags["KubernetesCluster"] = newClusterName

	// Try to pre-query as much as possible before doing anything destructive
	instances, err := findInstances(awsCloud)
	if err != nil {
		return fmt.Errorf("error finding instances: %v", err)
	}

	volumes, err := DescribeVolumes(x.Cloud)
	if err != nil {
		return err
	}

	dhcpOptions, err := DescribeDhcpOptions(x.Cloud)
	if err != nil {
		return err
	}

	autoscalingGroups, err := findAutoscalingGroups(awsCloud, oldTags)
	if err != nil {
		return err
	}

	elbs, _, err := DescribeELBs(x.Cloud)
	if err != nil {
		return err
	}

	// Find masters
	var masters []*ec2.Instance
	for _, instance := range instances {
		role, _ := awsup.FindEC2Tag(instance.Tags, "Role")
		if role == oldClusterName+"-master" {
			masters = append(masters, instance)
		}
	}
	if len(masters) == 0 {
		return fmt.Errorf("could not find masters")
	}

	// Stop autoscalingGroups
	for _, group := range autoscalingGroups {
		name := aws.StringValue(group.AutoScalingGroupName)
		glog.Infof("Stopping instances in autoscaling group %q", name)

		request := &autoscaling.UpdateAutoScalingGroupInput{
			AutoScalingGroupName: group.AutoScalingGroupName,
			DesiredCapacity:      aws.Int64(0),
			MinSize:              aws.Int64(0),
			MaxSize:              aws.Int64(0),
		}

		_, err := awsCloud.Autoscaling.UpdateAutoScalingGroup(request)
		if err != nil {
			return fmt.Errorf("error updating autoscaling group %q: %v", name, err)
		}
	}

	// Stop masters
	for _, master := range masters {
		masterInstanceID := aws.StringValue(master.InstanceId)

		masterState := aws.StringValue(master.State.Name)
		if masterState == "terminated" {
			glog.Infof("master already terminated: %q", masterInstanceID)
			continue
		}

		glog.Infof("Stopping master: %q", masterInstanceID)

		request := &ec2.StopInstancesInput{
			InstanceIds: []*string{master.InstanceId},
		}

		_, err := awsCloud.EC2.StopInstances(request)
		if err != nil {
			return fmt.Errorf("error stopping master instance: %v", err)
		}
	}

	// Detach volumes from masters
	for _, master := range masters {
		for _, bdm := range master.BlockDeviceMappings {
			if bdm.Ebs == nil || bdm.Ebs.VolumeId == nil {
				continue
			}
			volumeID := aws.StringValue(bdm.Ebs.VolumeId)
			masterInstanceID := aws.StringValue(master.InstanceId)
			glog.Infof("Detaching volume %q from instance %q", volumeID, masterInstanceID)

			request := &ec2.DetachVolumeInput{
				VolumeId:   bdm.Ebs.VolumeId,
				InstanceId: master.InstanceId,
			}

			for {
				_, err := awsCloud.EC2.DetachVolume(request)
				if err != nil {
					if AWSErrorCode(err) == "IncorrectState" {
						glog.Infof("retrying to detach volume (master has probably not stopped yet): %q", err)
						time.Sleep(5 * time.Second)
						continue
					}
					return fmt.Errorf("error detaching volume %q from master instance %q: %v", volumeID, masterInstanceID, err)
				} else {
					break
				}
			}
		}
	}

	//subnets, err := DescribeSubnets(x.Cloud)
	//if err != nil {
	//	return fmt.Errorf("error finding subnets: %v", err)
	//}
	//for _, s := range subnets {
	//	id := aws.StringValue(s.SubnetId)
	//	err := awsCloud.AddAWSTags(id, newTags)
	//	if err != nil {
	//		return fmt.Errorf("error re-tagging subnet %q: %v", id, err)
	//	}
	//}

	// Retag VPC
	// We have to be careful because VPCs can be shared
	{
		vpcID := cluster.Spec.NetworkID
		retagGateway := false

		if vpcID != "" {
			tags, err := awsCloud.GetTags(vpcID)
			if err != nil {
				return fmt.Errorf("error getting VPC tags: %v", err)
			}

			clusterTag := tags[awsup.TagClusterName]
			if clusterTag != "" {
				if clusterTag != oldClusterName {
					return fmt.Errorf("VPC is tagged with a different cluster: %v", clusterTag)
				}
				replaceTags := make(map[string]string)
				replaceTags[awsup.TagClusterName] = newClusterName

				glog.Infof("Retagging VPC %q", vpcID)

				err := awsCloud.CreateTags(vpcID, replaceTags)
				if err != nil {
					return fmt.Errorf("error re-tagging VPC: %v", err)
				}

				// The VPC was tagged as ours, so make sure the gateway is consistently retagged
				retagGateway = true
			}
		}

		if retagGateway {
			gateways, err := DescribeInternetGatewaysIgnoreTags(x.Cloud)
			if err != nil {
				return fmt.Errorf("error listing gateways: %v", err)
			}
			for _, igw := range gateways {
				match := false
				for _, a := range igw.Attachments {
					if vpcID == aws.StringValue(a.VpcId) {
						match = true
					}
				}
				if !match {
					continue
				}

				id := aws.StringValue(igw.InternetGatewayId)

				clusterTag, _ := awsup.FindEC2Tag(igw.Tags, awsup.TagClusterName)
				if clusterTag == "" || clusterTag == oldClusterName {
					replaceTags := make(map[string]string)
					replaceTags[awsup.TagClusterName] = newClusterName

					glog.Infof("Retagging InternetGateway %q", id)

					err := awsCloud.CreateTags(id, replaceTags)
					if err != nil {
						return fmt.Errorf("error re-tagging InternetGateway: %v", err)
					}
				}
			}
		}
	}

	// Retag DHCP options
	// We have to be careful because DHCP options can be shared
	for _, dhcpOption := range dhcpOptions {
		id := aws.StringValue(dhcpOption.DhcpOptionsId)

		clusterTag, _ := awsup.FindEC2Tag(dhcpOption.Tags, awsup.TagClusterName)
		if clusterTag != "" {
			if clusterTag != oldClusterName {
				return fmt.Errorf("DHCP options are tagged with a different cluster: %v", clusterTag)
			}
			replaceTags := make(map[string]string)
			replaceTags[awsup.TagClusterName] = newClusterName

			glog.Infof("Retagging DHCPOptions %q", id)

			err := awsCloud.CreateTags(id, replaceTags)
			if err != nil {
				return fmt.Errorf("error re-tagging DHCP options: %v", err)
			}
		}

	}

	// Adopt LoadBalancers & LoadBalancer Security Groups
	for _, elb := range elbs {
		id := aws.StringValue(elb.LoadBalancerName)

		// TODO: Batch re-tag?
		replaceTags := make(map[string]string)
		replaceTags[awsup.TagClusterName] = newClusterName

		glog.Infof("Retagging ELB %q", id)
		err := awsCloud.CreateELBTags(id, replaceTags)
		if err != nil {
			return fmt.Errorf("error re-tagging ELB %q: %v", id, err)
		}

	}

	for _, elb := range elbs {
		for _, sg := range elb.SecurityGroups {
			id := aws.StringValue(sg)

			// TODO: Batch re-tag?
			replaceTags := make(map[string]string)
			replaceTags[awsup.TagClusterName] = newClusterName

			glog.Infof("Retagging ELB security group %q", id)
			err := awsCloud.CreateTags(id, replaceTags)
			if err != nil {
				return fmt.Errorf("error re-tagging ELB security group %q: %v", id, err)
			}
		}

	}

	// Adopt Volumes
	for _, volume := range volumes {
		id := aws.StringValue(volume.VolumeId)

		// TODO: Batch re-tag?
		replaceTags := make(map[string]string)
		replaceTags[awsup.TagClusterName] = newClusterName

		name, _ := awsup.FindEC2Tag(volume.Tags, "Name")
		if name == oldClusterName+"-master-pd" {
			glog.Infof("Found master volume %q: %s", id, name)

			az := aws.StringValue(volume.AvailabilityZone)
			replaceTags["Name"] = az + ".etcd-main." + newClusterName
		}
		glog.Infof("Retagging volume %q", id)
		err := awsCloud.CreateTags(id, replaceTags)
		if err != nil {
			return fmt.Errorf("error re-tagging volume %q: %v", id, err)
		}
	}

	cluster.Name = newClusterName
	err = api.WriteConfig(x.NewStateStore, cluster, x.InstanceGroups)
	if err != nil {
		return fmt.Errorf("error writing updated configuration: %v", err)
	}

	oldCACertPool, err := x.OldStateStore.CA().CertificatePool(fi.CertificateId_CA)
	if err != nil {
		return fmt.Errorf("error reading old CA certs: %v", err)
	}
	for _, ca := range oldCACertPool.Secondary {
		err := x.NewStateStore.CA().AddCert(fi.CertificateId_CA, ca)
		if err != nil {
			return fmt.Errorf("error importing old CA certs: %v", err)
		}
	}
	if oldCACertPool.Primary != nil {
		err := x.NewStateStore.CA().AddCert(fi.CertificateId_CA, oldCACertPool.Primary)
		if err != nil {
			return fmt.Errorf("error importing old CA certs: %v", err)
		}
	}

	return nil
}
예제 #3
0
func FindName(tags []*ec2.Tag) string {
	if name, found := awsup.FindEC2Tag(tags, "Name"); found {
		return name
	}
	return ""
}