func NewGCECluster(conf GCEOptions) (Cluster, error) {
var client *http.Client
var err error
if conf.ServiceAuth {
client = auth.GoogleServiceClient()
} else {
client, err = auth.GoogleClient()
}
if err != nil {
return nil, err
}
api, err := compute.New(client)
if err != nil {
return nil, err
}
gc := &gceCluster{
api: api,
conf: &conf,
machines: make(map[string]*gceMachine),
}
gc.sshAgent, err = network.NewSSHAgent(&net.Dialer{})
if err != nil {
return nil, err
}
return gc, nil
}
func NewGCECluster(conf GCEOptions) (Cluster, error) {
var client *http.Client
var err error
if conf.ServiceAuth {
client = auth.GoogleServiceClient()
} else {
client, err = auth.GoogleClient()
}
if err != nil {
return nil, err
}
api, err := compute.New(client)
if err != nil {
return nil, err
}
bc, err := newBaseCluster()
if err != nil {
return nil, err
}
gc := &gceCluster{
baseCluster: bc,
api: api,
conf: &conf,
}
return gc, nil
}
func runDestroy(cmd *cobra.Command, args []string) {
if len(args) != 0 {
fmt.Fprintf(os.Stderr, "Unrecognized args in ore list cmd: %v\n", args)
os.Exit(2)
}
// avoid wiping out all instances in project or mishaps with short destroyPrefixes
if destroyPrefix == "" || len(destroyPrefix) < 2 {
fmt.Fprintf(os.Stderr, "Please specify a prefix of length 2 or greater with -prefix\n")
os.Exit(1)
}
client, err := auth.GoogleClient()
if err != nil {
fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
os.Exit(1)
}
vms, err := platform.GCEListVMs(client, destroyProject, destroyZone, destroyPrefix)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed listing vms: %v\n", err)
os.Exit(1)
}
var count int
for _, vm := range vms {
err := platform.GCEDestroyVM(client, destroyProject, destroyZone, vm.ID())
if err != nil {
fmt.Fprintf(os.Stderr, "Failed destroying vm: %v\n", err)
os.Exit(1)
}
count++
}
fmt.Printf("%v instance(s) scheduled for deletion\n", count)
}
func runList(cmd *cobra.Command, args []string) {
if len(args) != 0 {
fmt.Fprintf(os.Stderr, "Unrecognized args in plume list cmd: %v\n", args)
os.Exit(2)
}
client, err := auth.GoogleClient()
if err != nil {
fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
os.Exit(1)
}
api, err := compute.New(client)
if err != nil {
fmt.Fprintf(os.Stderr, "Api Client creation failed: %v\n", err)
os.Exit(1)
}
vms, err := platform.GCEListVMs(api, &opts, opts.BaseName)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed listing vms: %v\n", err)
os.Exit(1)
}
for _, vm := range vms {
fmt.Printf("%v:\n", vm.ID())
fmt.Printf(" extIP: %v\n", vm.IP())
}
}
func tryGCEUpload(uploadBucket, imageNameGS string) int {
client, err := auth.GoogleClient()
if err != nil {
log.Printf("Authentication failed: %v\n", err)
os.Exit(1)
}
// check if this file is already uploaded
alreadyExists, err := fileQuery(client, uploadBucket, imageNameGS)
if err != nil {
log.Printf("Uploading image failed: %v\n", err)
return 1
}
if alreadyExists {
if !gceUploadForce {
log.Printf("skipping upload, gs://%v/%v already exists on Google Storage.", uploadBucket, imageNameGS)
return 0
}
log.Println("forcing image upload...")
}
err = writeFile(client, uploadBucket, gceUploadFile, imageNameGS)
if err != nil {
log.Printf("Uploading image failed: %v\n", err)
return 1
}
log.Printf("wrote gs://%v/%v", uploadBucket, imageNameGS)
return 0
}
func runCreate(cmd *cobra.Command, args []string) {
if len(args) != 0 {
fmt.Fprintf(os.Stderr, "Unrecognized args in ore create-instances: %v\n", args)
os.Exit(2)
}
if createImageName == "" {
fmt.Fprintf(os.Stderr, "Must specifcy GCE image with '-image' flag\n")
os.Exit(2)
}
// if base name is unspecified use image name
if createBaseName == "" {
createBaseName = createImageName
}
var cloudConfig string
if createConfig != "" {
b, err := ioutil.ReadFile(createConfig)
if err != nil {
fmt.Fprintf(os.Stderr, "Could not read cloud config file: %v\n", err)
os.Exit(1)
}
cloudConfig = string(b)
}
client, err := auth.GoogleClient()
if err != nil {
fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
os.Exit(1)
}
opts := &platform.GCEOpts{
Client: client,
CloudConfig: cloudConfig,
Project: createProject,
Zone: createZone,
MachineType: createMachine,
BaseName: createBaseName,
Image: createImageName,
}
var vms []platform.Machine
for i := 0; i < createNumInstances; i++ {
vm, err := platform.GCECreateVM(opts)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed creating vm: %v\n", err)
os.Exit(1)
}
vms = append(vms, vm)
fmt.Println("Instance created")
}
fmt.Printf("All instances created, add your ssh keys here: https://console.developers.google.com/project/%v/compute/metadata/sshKeys\n", createProject)
for _, vm := range vms {
fmt.Printf("To access %v use cmd:\n", vm.ID())
fmt.Printf("ssh -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no core@%v\n", vm.IP())
}
}
func runCreate(cmd *cobra.Command, args []string) {
if len(args) != 0 {
fmt.Fprintf(os.Stderr, "Unrecognized args in ore create-instances: %v\n", args)
os.Exit(2)
}
var cloudConfig string
if createConfig != "" {
b, err := ioutil.ReadFile(createConfig)
if err != nil {
fmt.Fprintf(os.Stderr, "Could not read cloud config file: %v\n", err)
os.Exit(1)
}
cloudConfig = string(b)
}
client, err := auth.GoogleClient()
if err != nil {
fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
os.Exit(1)
}
api, err := compute.New(client)
if err != nil {
fmt.Fprintf(os.Stderr, "Api Client creation failed: %v\n", err)
os.Exit(1)
}
var vms []platform.Machine
for i := 0; i < createNumInstances; i++ {
vm, err := platform.GCECreateVM(api, &opts, cloudConfig)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed creating vm: %v\n", err)
os.Exit(1)
}
vms = append(vms, vm)
fmt.Println("Instance created")
}
fmt.Printf("All instances created, add your ssh keys here: https://console.developers.google.com/project/%v/compute/metadata/sshKeys\n", opts.Project)
for _, vm := range vms {
fmt.Printf("To access %v use cmd:\n", vm.ID())
fmt.Printf("ssh -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no core@%v\n", vm.IP())
}
}
func tryGCECreate(bucket, bucketPath, imageName string) int {
client, err := auth.GoogleClient()
if err != nil {
log.Printf("authentication failed: %v", err)
return 1
}
api, err := compute.New(client)
if err != nil {
fmt.Fprintf(os.Stderr, "Api Client creation failed: %v\n", err)
os.Exit(1)
}
// make sure file exists
exists, err := fileQuery(client, bucket, bucketPath)
if err != nil || !exists {
log.Printf("failed to find existance of storage image: %v", err)
return 1
}
log.Printf("Creating image in GCE: %v...\n", imageName)
// create image on gce
storageSrc := fmt.Sprintf("https://storage.googleapis.com/%v/%v", bucket, bucketPath)
err = platform.GCECreateImage(api, opts.Project, imageName, storageSrc)
// if image already exists ask to delete and try again
if err != nil && strings.HasSuffix(err.Error(), "alreadyExists") {
if gceCreateForce {
log.Println("forcing overwrite of existing image...")
err = platform.GCEForceCreateImage(api, opts.Project, imageName, storageSrc)
} else {
log.Printf("skipping upload, image %v already exists", imageName)
return 0
}
}
if err != nil {
log.Printf("Creating GCE image failed: %v", err)
return 1
}
return 0
}
func runIndex(cmd *cobra.Command, args []string) {
if len(args) == 0 {
fmt.Fprintf(os.Stderr, "No URLs specified\n")
os.Exit(2)
}
client, err := auth.GoogleClient()
if err != nil {
fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
os.Exit(1)
}
for _, url := range args {
if err := index.Update(client, url); err != nil {
fmt.Fprintf(os.Stderr, "Updating indexes for %s failed: %v\n", url, err)
os.Exit(1)
}
}
fmt.Printf("Update successful!\n")
}
func runImage(cmd *cobra.Command, args []string) {
if len(args) != 0 {
fmt.Fprintf(os.Stderr, "Unrecognized args in plume list cmd: %v\n", args)
os.Exit(2)
}
client, err := auth.GoogleClient()
if err != nil {
fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
os.Exit(1)
}
images, err := platform.GCEListImages(client, imageProject, imagePrefix)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed listing images: %v\n", err)
os.Exit(1)
}
for _, image := range images {
fmt.Printf("%v\n", image)
}
}
func runList(cmd *cobra.Command, args []string) {
if len(args) != 0 {
fmt.Fprintf(os.Stderr, "Unrecognized args in plume list cmd: %v\n", args)
os.Exit(2)
}
client, err := auth.GoogleClient()
if err != nil {
fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
os.Exit(1)
}
vms, err := platform.GCEListVMs(client, listProject, listZone, listPrefix)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed listing vms: %v\n", err)
os.Exit(1)
}
for _, vm := range vms {
fmt.Printf("%v:\n", vm.ID())
fmt.Printf(" extIP: %v\n", vm.IP())
}
}
func runUpload(cmd *cobra.Command, args []string) {
if len(args) != 0 {
fmt.Fprintf(os.Stderr, "Unrecognized args in plume upload cmd: %v\n", args)
os.Exit(2)
}
// if an image name is unspecified try to use version.txt
if uploadImageName == "" {
var err error
uploadImageName, err = sdk.GetVersionFromDir(filepath.Dir(uploadFile))
if err != nil {
fmt.Fprintf(os.Stderr, "Unable to get version from image directory, provide a -name flag or include a version.txt in the image directory: %v\n", err)
os.Exit(1)
}
}
gsURL, err := url.Parse(uploadBucket)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
if gsURL.Scheme != "gs" {
fmt.Fprintf(os.Stderr, "URL missing gs:// scheme prefix: %v\n", uploadBucket)
os.Exit(1)
}
if gsURL.Host == "" {
fmt.Fprintf(os.Stderr, "URL missing bucket name %v\n", uploadBucket)
os.Exit(1)
}
// if prefix not specified default name to gs://bucket/$USER/$BOARD/$VERSION
if gsURL.Path == "" {
if user := os.Getenv("USER"); user != "" {
gsURL.Path = "/" + os.Getenv("USER")
gsURL.Path += "/" + uploadBoard
}
}
uploadBucket = gsURL.Host
uploadImageName = strings.TrimPrefix(gsURL.Path+"/"+uploadImageName, "/")
// create equivalent image names for GS and GCE
imageNameGCE := gceSanitize(uploadImageName)
imageNameGS := uploadImageName + ".tar.gz"
var client *http.Client
if uploadServiceAuth {
client = auth.GoogleServiceClient()
err = nil
} else {
client, err = auth.GoogleClient()
}
if err != nil {
fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
os.Exit(1)
}
api, err := compute.New(client)
if err != nil {
fmt.Fprintf(os.Stderr, "Api Client creation failed: %v\n", err)
os.Exit(1)
}
// check if this file is already uploaded and give option to skip
alreadyExists, err := fileQuery(client, uploadBucket, imageNameGS)
if err != nil {
fmt.Fprintf(os.Stderr, "Uploading image failed: %v\n", err)
os.Exit(1)
}
if alreadyExists && !uploadForce {
var ans string
fmt.Printf("File %v already exists on Google Storage. Overwrite? (y/n):", imageNameGS)
if _, err = fmt.Scan(&ans); err != nil {
fmt.Fprintf(os.Stderr, "Scanning overwrite input: %v", err)
os.Exit(1)
}
switch ans {
case "y", "Y", "yes":
fmt.Println("Overriding existing file...")
err = writeFile(client, uploadBucket, uploadFile, imageNameGS)
default:
fmt.Println("Skipped file upload")
}
} else {
err = writeFile(client, uploadBucket, uploadFile, imageNameGS)
}
if err != nil {
fmt.Fprintf(os.Stderr, "Uploading image failed: %v\n", err)
os.Exit(1)
}
fmt.Printf("Creating image in GCE: %v...\n", imageNameGCE)
// create image on gce
storageSrc := fmt.Sprintf("https://storage.googleapis.com/%v/%v", uploadBucket, imageNameGS)
if uploadForce {
err = platform.GCEForceCreateImage(api, opts.Project, imageNameGCE, storageSrc)
} else {
err = platform.GCECreateImage(api, opts.Project, imageNameGCE, storageSrc)
}
// if image already exists ask to delete and try again
if err != nil && strings.HasSuffix(err.Error(), "alreadyExists") {
var ans string
fmt.Printf("Image %v already exists on GCE. Overwrite? (y/n):", imageNameGCE)
if _, err = fmt.Scan(&ans); err != nil {
fmt.Fprintf(os.Stderr, "Scanning overwrite input: %v", err)
os.Exit(1)
}
switch ans {
case "y", "Y", "yes":
fmt.Println("Overriding existing image...")
err = platform.GCEForceCreateImage(api, opts.Project, imageNameGCE, storageSrc)
if err != nil {
fmt.Fprintf(os.Stderr, "Creating GCE image failed: %v\n", err)
os.Exit(1)
}
fmt.Printf("Image %v sucessfully created in GCE\n", imageNameGCE)
default:
fmt.Println("Skipped GCE image creation")
}
}
if err != nil {
fmt.Fprintf(os.Stderr, "Creating GCE image failed: %v\n", err)
os.Exit(1)
}
}
func runDownloadImage(cmd *cobra.Command, args []string) {
if len(args) != 0 {
plog.Fatalf("Unrecognized arguments: %v", args)
}
if downloadImageCacheDir == "" {
plog.Fatal("Missing --cache-dir=FILEPATH")
}
if len(downloadImagePlatformList) == 0 {
plog.Fatal("Must specify 1 or more platforms to download")
}
if downloadImageVerify == false {
plog.Notice("Warning: image verification turned off")
}
// check for shorthand names of image roots
downloadImageRoot = convertSpecialPaths(downloadImageRoot)
imageURL, err := url.Parse(downloadImageRoot)
if err != nil {
plog.Fatalf("Failed parsing image root as url: %v", err)
}
// support Google storage buckets URLs
var client *http.Client
if imageURL.Scheme == "gs" {
if downloadImageJSONKeyFile != "" {
b, err := ioutil.ReadFile(downloadImageJSONKeyFile)
if err != nil {
plog.Fatal(err)
}
client, err = auth.GoogleClientFromJSONKey(b, "https://www.googleapis.com/auth/devstorage.read_only")
} else {
client, err = auth.GoogleClient()
}
if err != nil {
plog.Fatal(err)
}
}
versionFile := filepath.Join(downloadImageCacheDir, "version.txt")
versionURL := strings.TrimRight(downloadImageRoot, "/") + "/" + "version.txt"
if err := sdk.UpdateFile(versionFile, versionURL, client); err != nil {
plog.Fatalf("downloading version.txt: %v", err)
}
for _, suffix := range downloadImagePlatformList {
fileName := downloadImagePrefix + suffix
filePath := filepath.Join(downloadImageCacheDir, fileName)
// path.Join doesn't work with urls
url := strings.TrimRight(downloadImageRoot, "/") + "/" + fileName
if downloadImageVerify {
plog.Noticef("Verifying and updating to latest image %v", fileName)
err := sdk.UpdateSignedFile(filePath, url, client)
if err != nil {
plog.Fatalf("updating signed file: %v", err)
}
} else {
plog.Noticef("Starting non-verified image update %v", fileName)
if err := sdk.UpdateFile(filePath, url, client); err != nil {
plog.Fatalf("downloading image: %v", err)
}
}
}
}