Example #1
0
func NewGCECluster(conf GCEOptions) (Cluster, error) {
	var client *http.Client
	var err error
	if conf.ServiceAuth {
		client = auth.GoogleServiceClient()
	} else {
		client, err = auth.GoogleClient()
	}
	if err != nil {
		return nil, err
	}

	api, err := compute.New(client)
	if err != nil {
		return nil, err
	}

	gc := &gceCluster{
		api:      api,
		conf:     &conf,
		machines: make(map[string]*gceMachine),
	}

	gc.sshAgent, err = network.NewSSHAgent(&net.Dialer{})
	if err != nil {
		return nil, err
	}

	return gc, nil
}
Example #2
0
func NewGCECluster(conf GCEOptions) (Cluster, error) {
	var client *http.Client
	var err error
	if conf.ServiceAuth {
		client = auth.GoogleServiceClient()
	} else {
		client, err = auth.GoogleClient()
	}
	if err != nil {
		return nil, err
	}

	api, err := compute.New(client)
	if err != nil {
		return nil, err
	}

	bc, err := newBaseCluster()
	if err != nil {
		return nil, err
	}

	gc := &gceCluster{
		baseCluster: bc,
		api:         api,
		conf:        &conf,
	}

	return gc, nil
}
Example #3
0
func runDestroy(cmd *cobra.Command, args []string) {
	if len(args) != 0 {
		fmt.Fprintf(os.Stderr, "Unrecognized args in ore list cmd: %v\n", args)
		os.Exit(2)
	}

	// avoid wiping out all instances in project or mishaps with short destroyPrefixes
	if destroyPrefix == "" || len(destroyPrefix) < 2 {
		fmt.Fprintf(os.Stderr, "Please specify a prefix of length 2 or greater with -prefix\n")
		os.Exit(1)
	}

	client, err := auth.GoogleClient()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
		os.Exit(1)
	}

	vms, err := platform.GCEListVMs(client, destroyProject, destroyZone, destroyPrefix)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed listing vms: %v\n", err)
		os.Exit(1)
	}
	var count int
	for _, vm := range vms {
		err := platform.GCEDestroyVM(client, destroyProject, destroyZone, vm.ID())
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed destroying vm: %v\n", err)
			os.Exit(1)
		}
		count++
	}
	fmt.Printf("%v instance(s) scheduled for deletion\n", count)
}
Example #4
0
func runList(cmd *cobra.Command, args []string) {
	if len(args) != 0 {
		fmt.Fprintf(os.Stderr, "Unrecognized args in plume list cmd: %v\n", args)
		os.Exit(2)
	}

	client, err := auth.GoogleClient()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
		os.Exit(1)
	}

	api, err := compute.New(client)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Api Client creation failed: %v\n", err)
		os.Exit(1)
	}
	vms, err := platform.GCEListVMs(api, &opts, opts.BaseName)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed listing vms: %v\n", err)
		os.Exit(1)
	}
	for _, vm := range vms {
		fmt.Printf("%v:\n", vm.ID())
		fmt.Printf(" extIP: %v\n", vm.IP())
	}
}
Example #5
0
func tryGCEUpload(uploadBucket, imageNameGS string) int {
	client, err := auth.GoogleClient()
	if err != nil {
		log.Printf("Authentication failed: %v\n", err)
		os.Exit(1)
	}

	// check if this file is already uploaded
	alreadyExists, err := fileQuery(client, uploadBucket, imageNameGS)
	if err != nil {
		log.Printf("Uploading image failed: %v\n", err)
		return 1
	}

	if alreadyExists {
		if !gceUploadForce {
			log.Printf("skipping upload, gs://%v/%v already exists on Google Storage.", uploadBucket, imageNameGS)
			return 0
		}

		log.Println("forcing image upload...")
	}

	err = writeFile(client, uploadBucket, gceUploadFile, imageNameGS)
	if err != nil {
		log.Printf("Uploading image failed: %v\n", err)
		return 1
	}
	log.Printf("wrote gs://%v/%v", uploadBucket, imageNameGS)

	return 0
}
Example #6
0
func runCreate(cmd *cobra.Command, args []string) {
	if len(args) != 0 {
		fmt.Fprintf(os.Stderr, "Unrecognized args in ore create-instances: %v\n", args)
		os.Exit(2)
	}

	if createImageName == "" {
		fmt.Fprintf(os.Stderr, "Must specifcy GCE image with '-image' flag\n")
		os.Exit(2)
	}
	// if base name is unspecified use image name
	if createBaseName == "" {
		createBaseName = createImageName
	}

	var cloudConfig string
	if createConfig != "" {
		b, err := ioutil.ReadFile(createConfig)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Could not read cloud config file: %v\n", err)
			os.Exit(1)
		}
		cloudConfig = string(b)
	}

	client, err := auth.GoogleClient()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
		os.Exit(1)
	}

	opts := &platform.GCEOpts{
		Client:      client,
		CloudConfig: cloudConfig,
		Project:     createProject,
		Zone:        createZone,
		MachineType: createMachine,
		BaseName:    createBaseName,
		Image:       createImageName,
	}

	var vms []platform.Machine
	for i := 0; i < createNumInstances; i++ {
		vm, err := platform.GCECreateVM(opts)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed creating vm: %v\n", err)
			os.Exit(1)
		}
		vms = append(vms, vm)
		fmt.Println("Instance created")
	}

	fmt.Printf("All instances created, add your ssh keys here: https://console.developers.google.com/project/%v/compute/metadata/sshKeys\n", createProject)
	for _, vm := range vms {
		fmt.Printf("To access %v use cmd:\n", vm.ID())
		fmt.Printf("ssh -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no core@%v\n", vm.IP())
	}
}
Example #7
0
func runCreate(cmd *cobra.Command, args []string) {
	if len(args) != 0 {
		fmt.Fprintf(os.Stderr, "Unrecognized args in ore create-instances: %v\n", args)
		os.Exit(2)
	}

	var cloudConfig string
	if createConfig != "" {
		b, err := ioutil.ReadFile(createConfig)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Could not read cloud config file: %v\n", err)
			os.Exit(1)
		}
		cloudConfig = string(b)
	}

	client, err := auth.GoogleClient()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
		os.Exit(1)
	}

	api, err := compute.New(client)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Api Client creation failed: %v\n", err)
		os.Exit(1)
	}

	var vms []platform.Machine
	for i := 0; i < createNumInstances; i++ {
		vm, err := platform.GCECreateVM(api, &opts, cloudConfig)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed creating vm: %v\n", err)
			os.Exit(1)
		}
		vms = append(vms, vm)
		fmt.Println("Instance created")
	}

	fmt.Printf("All instances created, add your ssh keys here: https://console.developers.google.com/project/%v/compute/metadata/sshKeys\n", opts.Project)
	for _, vm := range vms {
		fmt.Printf("To access %v use cmd:\n", vm.ID())
		fmt.Printf("ssh -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no core@%v\n", vm.IP())
	}
}
Example #8
0
func tryGCECreate(bucket, bucketPath, imageName string) int {
	client, err := auth.GoogleClient()
	if err != nil {
		log.Printf("authentication failed: %v", err)
		return 1
	}

	api, err := compute.New(client)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Api Client creation failed: %v\n", err)
		os.Exit(1)
	}

	// make sure file exists
	exists, err := fileQuery(client, bucket, bucketPath)
	if err != nil || !exists {
		log.Printf("failed to find existance of storage image: %v", err)
		return 1
	}

	log.Printf("Creating image in GCE: %v...\n", imageName)

	// create image on gce
	storageSrc := fmt.Sprintf("https://storage.googleapis.com/%v/%v", bucket, bucketPath)
	err = platform.GCECreateImage(api, opts.Project, imageName, storageSrc)

	// if image already exists ask to delete and try again
	if err != nil && strings.HasSuffix(err.Error(), "alreadyExists") {
		if gceCreateForce {
			log.Println("forcing overwrite of existing image...")
			err = platform.GCEForceCreateImage(api, opts.Project, imageName, storageSrc)
		} else {
			log.Printf("skipping upload, image %v already exists", imageName)
			return 0
		}
	}

	if err != nil {
		log.Printf("Creating GCE image failed: %v", err)
		return 1
	}

	return 0
}
Example #9
0
func runIndex(cmd *cobra.Command, args []string) {
	if len(args) == 0 {
		fmt.Fprintf(os.Stderr, "No URLs specified\n")
		os.Exit(2)
	}

	client, err := auth.GoogleClient()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
		os.Exit(1)
	}

	for _, url := range args {
		if err := index.Update(client, url); err != nil {
			fmt.Fprintf(os.Stderr, "Updating indexes for %s failed: %v\n", url, err)
			os.Exit(1)
		}
	}

	fmt.Printf("Update successful!\n")
}
Example #10
0
func runImage(cmd *cobra.Command, args []string) {
	if len(args) != 0 {
		fmt.Fprintf(os.Stderr, "Unrecognized args in plume list cmd: %v\n", args)
		os.Exit(2)
	}

	client, err := auth.GoogleClient()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
		os.Exit(1)
	}

	images, err := platform.GCEListImages(client, imageProject, imagePrefix)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed listing images: %v\n", err)
		os.Exit(1)
	}
	for _, image := range images {
		fmt.Printf("%v\n", image)
	}
}
Example #11
0
func runList(cmd *cobra.Command, args []string) {
	if len(args) != 0 {
		fmt.Fprintf(os.Stderr, "Unrecognized args in plume list cmd: %v\n", args)
		os.Exit(2)
	}

	client, err := auth.GoogleClient()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
		os.Exit(1)
	}

	vms, err := platform.GCEListVMs(client, listProject, listZone, listPrefix)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed listing vms: %v\n", err)
		os.Exit(1)
	}
	for _, vm := range vms {
		fmt.Printf("%v:\n", vm.ID())
		fmt.Printf(" extIP: %v\n", vm.IP())
	}

}
Example #12
0
func runUpload(cmd *cobra.Command, args []string) {
	if len(args) != 0 {
		fmt.Fprintf(os.Stderr, "Unrecognized args in plume upload cmd: %v\n", args)
		os.Exit(2)
	}

	// if an image name is unspecified try to use version.txt
	if uploadImageName == "" {
		var err error
		uploadImageName, err = sdk.GetVersionFromDir(filepath.Dir(uploadFile))
		if err != nil {
			fmt.Fprintf(os.Stderr, "Unable to get version from image directory, provide a -name flag or include a version.txt in the image directory: %v\n", err)
			os.Exit(1)
		}
	}

	gsURL, err := url.Parse(uploadBucket)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
	if gsURL.Scheme != "gs" {
		fmt.Fprintf(os.Stderr, "URL missing gs:// scheme prefix: %v\n", uploadBucket)
		os.Exit(1)
	}
	if gsURL.Host == "" {
		fmt.Fprintf(os.Stderr, "URL missing bucket name %v\n", uploadBucket)
		os.Exit(1)
	}
	// if prefix not specified default name to gs://bucket/$USER/$BOARD/$VERSION
	if gsURL.Path == "" {
		if user := os.Getenv("USER"); user != "" {
			gsURL.Path = "/" + os.Getenv("USER")
			gsURL.Path += "/" + uploadBoard
		}
	}

	uploadBucket = gsURL.Host
	uploadImageName = strings.TrimPrefix(gsURL.Path+"/"+uploadImageName, "/")
	// create equivalent image names for GS and GCE
	imageNameGCE := gceSanitize(uploadImageName)
	imageNameGS := uploadImageName + ".tar.gz"

	var client *http.Client
	if uploadServiceAuth {
		client = auth.GoogleServiceClient()
		err = nil
	} else {
		client, err = auth.GoogleClient()
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
		os.Exit(1)
	}

	api, err := compute.New(client)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Api Client creation failed: %v\n", err)
		os.Exit(1)
	}

	// check if this file is already uploaded and give option to skip
	alreadyExists, err := fileQuery(client, uploadBucket, imageNameGS)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Uploading image failed: %v\n", err)
		os.Exit(1)
	}

	if alreadyExists && !uploadForce {
		var ans string
		fmt.Printf("File %v already exists on Google Storage. Overwrite? (y/n):", imageNameGS)
		if _, err = fmt.Scan(&ans); err != nil {
			fmt.Fprintf(os.Stderr, "Scanning overwrite input: %v", err)
			os.Exit(1)
		}
		switch ans {
		case "y", "Y", "yes":
			fmt.Println("Overriding existing file...")
			err = writeFile(client, uploadBucket, uploadFile, imageNameGS)
		default:
			fmt.Println("Skipped file upload")
		}
	} else {
		err = writeFile(client, uploadBucket, uploadFile, imageNameGS)
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "Uploading image failed: %v\n", err)
		os.Exit(1)
	}

	fmt.Printf("Creating image in GCE: %v...\n", imageNameGCE)

	// create image on gce
	storageSrc := fmt.Sprintf("https://storage.googleapis.com/%v/%v", uploadBucket, imageNameGS)
	if uploadForce {
		err = platform.GCEForceCreateImage(api, opts.Project, imageNameGCE, storageSrc)
	} else {
		err = platform.GCECreateImage(api, opts.Project, imageNameGCE, storageSrc)
	}

	// if image already exists ask to delete and try again
	if err != nil && strings.HasSuffix(err.Error(), "alreadyExists") {
		var ans string
		fmt.Printf("Image %v already exists on GCE. Overwrite? (y/n):", imageNameGCE)
		if _, err = fmt.Scan(&ans); err != nil {
			fmt.Fprintf(os.Stderr, "Scanning overwrite input: %v", err)
			os.Exit(1)
		}
		switch ans {
		case "y", "Y", "yes":
			fmt.Println("Overriding existing image...")
			err = platform.GCEForceCreateImage(api, opts.Project, imageNameGCE, storageSrc)

			if err != nil {
				fmt.Fprintf(os.Stderr, "Creating GCE image failed: %v\n", err)
				os.Exit(1)
			}
			fmt.Printf("Image %v sucessfully created in GCE\n", imageNameGCE)
		default:
			fmt.Println("Skipped GCE image creation")
		}
	}

	if err != nil {
		fmt.Fprintf(os.Stderr, "Creating GCE image failed: %v\n", err)
		os.Exit(1)
	}
}
Example #13
0
func runDownloadImage(cmd *cobra.Command, args []string) {
	if len(args) != 0 {
		plog.Fatalf("Unrecognized arguments: %v", args)
	}

	if downloadImageCacheDir == "" {
		plog.Fatal("Missing --cache-dir=FILEPATH")
	}
	if len(downloadImagePlatformList) == 0 {
		plog.Fatal("Must specify 1 or more platforms to download")
	}
	if downloadImageVerify == false {
		plog.Notice("Warning: image verification turned off")
	}

	// check for shorthand names of image roots
	downloadImageRoot = convertSpecialPaths(downloadImageRoot)

	imageURL, err := url.Parse(downloadImageRoot)
	if err != nil {
		plog.Fatalf("Failed parsing image root as url: %v", err)
	}

	// support Google storage buckets URLs
	var client *http.Client
	if imageURL.Scheme == "gs" {
		if downloadImageJSONKeyFile != "" {
			b, err := ioutil.ReadFile(downloadImageJSONKeyFile)
			if err != nil {
				plog.Fatal(err)
			}
			client, err = auth.GoogleClientFromJSONKey(b, "https://www.googleapis.com/auth/devstorage.read_only")
		} else {
			client, err = auth.GoogleClient()
		}
		if err != nil {
			plog.Fatal(err)
		}
	}

	versionFile := filepath.Join(downloadImageCacheDir, "version.txt")
	versionURL := strings.TrimRight(downloadImageRoot, "/") + "/" + "version.txt"
	if err := sdk.UpdateFile(versionFile, versionURL, client); err != nil {
		plog.Fatalf("downloading version.txt: %v", err)
	}

	for _, suffix := range downloadImagePlatformList {
		fileName := downloadImagePrefix + suffix
		filePath := filepath.Join(downloadImageCacheDir, fileName)

		// path.Join doesn't work with urls
		url := strings.TrimRight(downloadImageRoot, "/") + "/" + fileName

		if downloadImageVerify {
			plog.Noticef("Verifying and updating to latest image %v", fileName)
			err := sdk.UpdateSignedFile(filePath, url, client)
			if err != nil {
				plog.Fatalf("updating signed file: %v", err)
			}
		} else {
			plog.Noticef("Starting non-verified image update %v", fileName)
			if err := sdk.UpdateFile(filePath, url, client); err != nil {
				plog.Fatalf("downloading image: %v", err)
			}
		}
	}
}