func TestDeleteExceptDisk(t *testing.T) {
s := os.Getenv("DRONE")
if s != "" {
t.Skip("Skipping: test must be run in a VM")
}
ctx := context.Background()
session := test.Session(ctx, t)
defer session.Logout(ctx)
host := test.PickRandomHost(ctx, session, t)
uuid, err := sys.UUID()
if err != nil {
t.Fatalf("unable to get UUID for guest - used for VM name: %s", err)
}
name := fmt.Sprintf("%s-%d", uuid, rand.Intn(math.MaxInt32))
moref, err := CreateVM(ctx, session, host, name)
if err != nil {
t.Fatalf("ERROR: %s", err)
}
// Wrap the result with our version of VirtualMachine
vm := NewVirtualMachine(ctx, session, *moref)
folder, err := vm.FolderName(ctx)
if err != nil {
t.Fatalf("ERROR: %s", err)
}
// generate the disk name
diskName := fmt.Sprintf("%s/%s.vmdk", folder, folder)
// Delete the VM but not it's disk
_, err = tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
return vm.DeleteExceptDisks(ctx)
})
if err != nil {
t.Fatalf("ERROR: %s", err)
}
// check that the disk still exists
session.Datastore.Stat(ctx, diskName)
if err != nil {
t.Fatalf("Disk does not exist")
}
// clean up
dm := object.NewVirtualDiskManager(session.Client.Client)
task, err := dm.DeleteVirtualDisk(context.TODO(), diskName, nil)
if err != nil {
t.Fatalf("Unable to locate orphan vmdk: %s", err)
}
if err = task.Wait(context.TODO()); err != nil {
t.Fatalf("Unable to remove orphan vmdk: %s", err)
}
}
func TestVM(t *testing.T) {
s := os.Getenv("DRONE")
if s != "" {
t.Skip("Skipping: test must be run in a VM")
}
ctx := context.Background()
session := test.Session(ctx, t)
defer session.Logout(ctx)
host := test.PickRandomHost(ctx, session, t)
uuid, err := sys.UUID()
if err != nil {
t.Fatalf("unable to get UUID for guest - used for VM name: %s", err)
return
}
name := fmt.Sprintf("%s-%d", uuid, rand.Intn(math.MaxInt32))
moref, err := CreateVM(ctx, session, host, name)
if err != nil {
t.Fatalf("ERROR: %s", err)
}
// Wrap the result with our version of VirtualMachine
vm := NewVirtualMachine(ctx, session, *moref)
// Check the state
state, err := vm.PowerState(ctx)
if err != nil {
t.Fatalf("ERROR: %s", err)
}
assert.Equal(t, types.VirtualMachinePowerStatePoweredOff, state)
// Check VM name
rname, err := vm.Name(ctx)
if err != nil {
t.Errorf("Failed to load VM name: %s", err)
}
assert.Equal(t, name, rname)
// Get VM UUID
ruuid, err := vm.UUID(ctx)
if err != nil {
t.Errorf("Failed to load VM UUID: %s", err)
}
t.Logf("Got UUID: %s", ruuid)
// Destroy the vm
_, err = tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.ResultWaiter, error) {
return vm.Destroy(ctx)
})
if err != nil {
t.Fatalf("ERROR: %s", err)
}
}
func New() System {
id, _ := sys.UUID()
return System{
Hosts: etcconf.NewHosts(""), // default hosts files, e.g. /etc/hosts on linux
ResolvConf: etcconf.NewResolvConf(""), // default resolv.conf file, e.g. /etc/resolv.conf on linux
Syscall: &syscallImpl{}, // the syscall interface
Root: "/", // the system root path
UUID: id,
}
}
func instanceUUID(id string) (string, error) {
// generate VM instance uuid, which will be used to query back VM
u, err := sys.UUID()
if err != nil {
return "", err
}
namespace, err := uuid.Parse(u)
if err != nil {
return "", errors.Errorf("unable to parse VCH uuid: %s", err)
}
return uuid.NewSHA1(namespace, []byte(id)).String(), nil
}
func getMachineID() ([]byte, error) {
id, err := sys.UUID()
if err != nil {
return nil, err
}
uid, err := uuid.Parse(id)
if err != nil {
return nil, err
}
return uid.MarshalBinary()
}
// Update runs only once at startup to hydrate the image cache
func (ic *ICache) Update(client *client.PortLayer) error {
log.Debugf("Updating image cache...")
host, err := sys.UUID()
if host == "" {
host, err = os.Hostname()
}
if err != nil {
return fmt.Errorf("Unexpected error getting hostname: %s", err)
}
// attempt to create the image store if it doesn't exist
store := &models.ImageStore{Name: host}
_, err = client.Storage.CreateImageStore(
storage.NewCreateImageStoreParamsWithContext(ctx).WithBody(store),
)
if err != nil {
if _, ok := err.(*storage.CreateImageStoreConflict); ok {
log.Debugf("Store already exists")
} else {
log.Debugf("Creating a store failed: %#v", err)
return err
}
}
params := storage.NewListImagesParamsWithContext(ctx).WithStoreName(host)
layers, err := client.Storage.ListImages(params)
if err != nil {
return fmt.Errorf("Failed to retrieve image list from portlayer: %s", err)
}
for _, layer := range layers.Payload {
imageConfig := &metadata.ImageConfig{}
if err := json.Unmarshal([]byte(layer.Metadata["metaData"]), imageConfig); err != nil {
derr.NewErrorWithStatusCode(fmt.Errorf("Failed to unmarshal image config: %s", err),
http.StatusInternalServerError)
}
if imageConfig.ImageID != "" {
ic.AddImage(imageConfig)
}
}
return nil
}
// GetSelf gets VirtualMachine reference for the VM this process is running on
func GetSelf(ctx context.Context, s *session.Session) (*object.VirtualMachine, error) {
u, err := sys.UUID()
if err != nil {
return nil, err
}
search := object.NewSearchIndex(s.Vim25())
ref, err := search.FindByUuid(ctx, s.Datacenter, u, true, nil)
if err != nil {
return nil, err
}
if ref == nil {
return nil, fmt.Errorf("can't find the hosting vm")
}
vm := object.NewVirtualMachine(s.Client.Client, ref.Reference())
return vm, nil
}
// CreateContainerHandle creates a new VIC container by calling the portlayer
//
// returns:
// (containerID, containerHandle, error)
func (c *ContainerProxy) CreateContainerHandle(imageID string, config types.ContainerCreateConfig) (string, string, error) {
defer trace.End(trace.Begin(imageID))
if c.client == nil {
return "", "",
derr.NewErrorWithStatusCode(fmt.Errorf("ContainerProxy.CreateContainerHandle failed to create a portlayer client"),
http.StatusInternalServerError)
}
if imageID == "" {
return "", "",
derr.NewRequestNotFoundError(fmt.Errorf("No image specified"))
}
// Call the Exec port layer to create the container
host, err := sys.UUID()
if err != nil {
return "", "",
derr.NewErrorWithStatusCode(fmt.Errorf("ContainerProxy.CreateContainerHandle got unexpected error getting VCH UUID"),
http.StatusInternalServerError)
}
plCreateParams := dockerContainerCreateParamsToPortlayer(config, imageID, host)
createResults, err := c.client.Containers.Create(plCreateParams)
if err != nil {
if _, ok := err.(*containers.CreateNotFound); ok {
cerr := fmt.Errorf("No such image: %s", imageID)
log.Errorf("%s (%s)", cerr, err)
return "", "", derr.NewRequestNotFoundError(cerr)
}
// If we get here, most likely something went wrong with the port layer API server
return "", "",
derr.NewErrorWithStatusCode(err, http.StatusInternalServerError)
}
id := createResults.Payload.ID
h := createResults.Payload.Handle
return id, h, nil
}
func TestVMFailureWithTimeout(t *testing.T) {
ctx := context.Background()
session := test.Session(ctx, t)
defer session.Logout(ctx)
host := test.PickRandomHost(ctx, session, t)
ctx, cancel := context.WithTimeout(ctx, 1*time.Microsecond)
defer cancel()
uuid, err := sys.UUID()
if err != nil {
t.Fatalf("unable to get UUID for guest - used for VM name: %s", err)
return
}
name := fmt.Sprintf("%s-%d", uuid, rand.Intn(math.MaxInt32))
_, err = CreateVM(ctx, session, host, name)
if err != nil && err != context.DeadlineExceeded {
t.Fatalf("ERROR: %s", err)
}
}
func createImageStore() error {
// TODO(jzt): we should move this to a utility package or something
host, err := sys.UUID()
if err != nil {
log.Errorf("Failed to determine host UUID")
return err
}
// attempt to create the image store if it doesn't exist
store := &models.ImageStore{Name: host}
_, err = portLayerClient.Storage.CreateImageStore(
storage.NewCreateImageStoreParamsWithContext(ctx).WithBody(store),
)
if err != nil {
if _, ok := err.(*storage.CreateImageStoreConflict); ok {
log.Debugf("Store already exists")
return nil
}
return err
}
log.Infof("Image store created successfully")
return nil
}
// TODO fix the errors so the client doesnt print the generic POST or DELETE message
func (i *Image) ImageDelete(imageRef string, force, prune bool) ([]types.ImageDelete, error) {
defer trace.End(trace.Begin(imageRef))
var deleted []types.ImageDelete
var userRefIsID bool
var imageRemoved bool
// Use the image cache to go from the reference to the ID we use in the image store
img, err := cache.ImageCache().Get(imageRef)
if err != nil {
return nil, err
}
// Get the tags from the repo cache for this image
// TODO: remove this -- we have it in the image above
tags := cache.RepositoryCache().Tags(img.ImageID)
// did the user pass an id or partial id
userRefIsID = cache.ImageCache().IsImageID(imageRef)
// do we have any reference conflicts
if len(tags) > 1 && userRefIsID && !force {
t := uid.Parse(img.ImageID).Truncate()
return nil,
fmt.Errorf("conflict: unable to delete %s (must be forced) - image is referenced in one or more repositories", t)
}
// if we have an ID or only 1 tag lets delete the vmdk(s) via the PL
if userRefIsID || len(tags) == 1 {
log.Infof("Deleting image via PL %s (%s)", img.ImageID, img.ID)
// needed for image store
host, err := sys.UUID()
if err != nil {
return nil, err
}
params := storage.NewDeleteImageParamsWithContext(ctx).WithStoreName(host).WithID(img.ID)
// TODO: This will fail if any containerVMs are referencing the vmdk - vanilla docker
// allows the removal of an image (via force flag) even if a container is referencing it
// should vic?
_, err = PortLayerClient().Storage.DeleteImage(params)
if err != nil {
switch err := err.(type) {
case *storage.DeleteImageLocked:
return nil, fmt.Errorf("Failed to remove image %q: %s", imageRef, err.Payload.Message)
default:
return nil, err
}
}
// we've deleted the image so remove from cache
cache.ImageCache().RemoveImageByConfig(img)
imagec.LayerCache().Remove(img.ID)
imageRemoved = true
} else {
// only untag the ref supplied
n, err := reference.ParseNamed(imageRef)
if err != nil {
return nil, fmt.Errorf("unable to parse reference(%s): %s", imageRef, err.Error())
}
tag := reference.WithDefaultTag(n)
tags = []string{tag.String()}
}
// loop thru and remove from repoCache
for i := range tags {
// remove from cache, but don't save -- we'll do that afer all
// updates
refNamed, _ := cache.RepositoryCache().Remove(tags[i], false)
dd := types.ImageDelete{Untagged: refNamed}
deleted = append(deleted, dd)
}
// save repo now -- this will limit the number of PL
// calls to one per rmi call
err = cache.RepositoryCache().Save()
if err != nil {
return nil, fmt.Errorf("Untag error: %s", err.Error())
}
if imageRemoved {
imageDeleted := types.ImageDelete{Deleted: img.ImageID}
deleted = append(deleted, imageDeleted)
}
return deleted, err
}
func main() {
defer func() {
if r := recover(); r != nil {
fmt.Fprintf(os.Stderr, string(sf.FormatError(fmt.Errorf("%s : %s", r, debug.Stack()))))
}
}()
if version.Show() {
fmt.Fprintf(os.Stdout, "%s\n", version.String())
return
}
// Enable profiling if mode is set
switch options.profiling {
case "cpu":
defer profile.Start(profile.CPUProfile, profile.ProfilePath("."), profile.Quiet).Stop()
case "mem":
defer profile.Start(profile.MemProfile, profile.ProfilePath("."), profile.Quiet).Stop()
case "block":
defer profile.Start(profile.BlockProfile, profile.ProfilePath("."), profile.Quiet).Stop()
default:
// do nothing
}
// Register our custom Error hook
log.AddHook(NewErrorHook(os.Stderr))
// Enable runtime tracing if tracing is true
if options.tracing {
tracing, err := os.Create(time.Now().Format("2006-01-02T150405.pprof"))
if err != nil {
log.Fatalf("Failed to create tracing logfile: %s", err)
}
defer tracing.Close()
if err := trace.Start(tracing); err != nil {
log.Fatalf("Failed to start tracing: %s", err)
}
defer trace.Stop()
}
// Open the log file
f, err := os.OpenFile(options.logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
log.Fatalf("Failed to open the logfile %s: %s", options.logfile, err)
}
defer f.Close()
// Initiliaze logger with default TextFormatter
log.SetFormatter(&log.TextFormatter{DisableColors: true, FullTimestamp: true})
// Set the log level
if options.debug {
log.SetLevel(log.DebugLevel)
}
// SetOutput to log file and/or stdout
log.SetOutput(f)
if options.stdout {
log.SetOutput(io.MultiWriter(os.Stdout, f))
}
// Parse the -reference parameter
if err = ParseReference(); err != nil {
log.Fatalf(err.Error())
}
// Host is either the host's UUID (if run on vsphere) or the hostname of
// the system (if run standalone)
host, err := sys.UUID()
if host != "" {
log.Infof("Using UUID (%s) for imagestore name", host)
} else if options.standalone {
host, err = os.Hostname()
log.Infof("Using host (%s) for imagestore name", host)
}
if err != nil {
log.Fatalf("Failed to return the UUID or host name: %s", err)
}
if !options.standalone {
log.Debugf("Running with portlayer")
// Ping the server to ensure it's at least running
ok, err := PingPortLayer()
if err != nil || !ok {
log.Fatalf("Failed to ping portlayer: %s", err)
}
} else {
log.Debugf("Running standalone")
}
// Calculate (and overwrite) the registry URL and make sure that it responds to requests
options.registry, err = LearnRegistryURL(options)
if err != nil {
log.Fatalf("Error while pulling image: %s", err)
}
// Get the URL of the OAuth endpoint
url, err := LearnAuthURL(options)
if err != nil {
log.Fatalf("Failed to obtain OAuth endpoint: %s", err)
}
// Get the OAuth token - if only we have a URL
if url != nil {
token, err := FetchToken(url)
if err != nil {
log.Fatalf("Failed to fetch OAuth token: %s", err)
}
options.token = token
}
// HACK: Required to learn the name of the vmdk from given reference
// Used by docker personality until metadata support lands
if !options.resolv {
progress.Message(po, "", "Pulling from "+options.image)
}
// Get the manifest
manifest, err := FetchImageManifest(options)
if err != nil {
if strings.Contains(err.Error(), "image not found") {
log.Fatalf("Error: image %s not found", options.image)
} else {
log.Fatalf("Error while pulling image manifest: %s", err)
}
}
// Create the ImageWithMeta slice to hold Image structs
images, imageLayer, err := ImagesToDownload(manifest, host)
if err != nil {
log.Fatalf(err.Error())
}
// HACK: Required to learn the name of the vmdk from given reference
// Used by docker personality until metadata support lands
if options.resolv {
if len(images) > 0 {
fmt.Printf("%s", images[0].meta)
os.Exit(0)
}
os.Exit(1)
}
// Fetch the blobs from registry
if err := DownloadImageBlobs(images); err != nil {
log.Fatalf(err.Error())
}
if err := CreateImageConfig(images, manifest); err != nil {
log.Fatalf(err.Error())
}
// Write blobs to the storage layer
if err := WriteImageBlobs(images); err != nil {
log.Fatalf(err.Error())
}
if err := updateImageMetadata(imageLayer, manifest); err != nil {
log.Fatalf(err.Error())
}
progress.Message(po, "", "Digest: "+manifest.Digest)
if len(images) > 0 {
progress.Message(po, "", "Status: Downloaded newer image for "+options.image+":"+options.tag)
} else {
progress.Message(po, "", "Status: Image is up to date for "+options.image+":"+options.tag)
}
}
func TestVMAttributes(t *testing.T) {
ctx := context.Background()
session := test.Session(ctx, t)
defer session.Logout(ctx)
host := test.PickRandomHost(ctx, session, t)
uuid, err := sys.UUID()
if err != nil {
t.Fatalf("unable to get UUID for guest - used for VM name: %s", err)
return
}
ID := fmt.Sprintf("%s-%d", uuid, rand.Intn(math.MaxInt32))
moref, err := CreateVM(ctx, session, host, ID)
if err != nil {
t.Fatalf("ERROR: %s", err)
}
// Wrap the result with our version of VirtualMachine
vm := NewVirtualMachine(ctx, session, *moref)
folder, err := vm.FolderName(ctx)
if err != nil {
t.Fatalf("ERROR: %s", err)
}
name, err := vm.Name(ctx)
if err != nil {
t.Fatalf("ERROR: %s", err)
}
assert.Equal(t, name, folder)
task, err := vm.PowerOn(ctx)
if err != nil {
t.Fatalf("ERROR: %s", err)
}
_, err = task.WaitForResult(ctx, nil)
if err != nil {
t.Fatalf("ERROR: %s", err)
}
if guest, err := vm.FetchExtraConfig(ctx); err != nil {
t.Fatalf("ERROR: %s", err)
} else {
assert.NotEmpty(t, guest)
}
defer func() {
// Destroy the vm
task, err := vm.PowerOff(ctx)
if err != nil {
t.Fatalf("ERROR: %s", err)
}
_, err = task.WaitForResult(ctx, nil)
if err != nil {
t.Fatalf("ERROR: %s", err)
}
task, err = vm.Destroy(ctx)
if err != nil {
t.Fatalf("ERROR: %s", err)
}
_, err = task.WaitForResult(ctx, nil)
if err != nil {
t.Fatalf("ERROR: %s", err)
}
}()
}
// PullImage pulls an image from docker hub
func (ic *ImageC) PullImage() error {
// ctx
ctx, cancel := context.WithTimeout(ctx, ic.Options.Timeout)
defer cancel()
// Parse the -reference parameter
if err := ic.ParseReference(); err != nil {
log.Errorf(err.Error())
return err
}
// Host is either the host's UUID (if run on vsphere) or the hostname of
// the system (if run standalone)
host, err := sys.UUID()
if err != nil {
log.Errorf("Failed to return host name: %s", err)
return err
}
if host != "" {
log.Infof("Using UUID (%s) for imagestore name", host)
}
ic.Storename = host
// Ping the server to ensure it's at least running
ok, err := PingPortLayer(ic.Host)
if err != nil || !ok {
log.Errorf("Failed to ping portlayer: %s", err)
return err
}
// Calculate (and overwrite) the registry URL and make sure that it responds to requests
ic.Registry, err = LearnRegistryURL(ic.Options)
if err != nil {
log.Errorf("Error while pulling image: %s", err)
return err
}
// Get the URL of the OAuth endpoint
url, err := LearnAuthURL(ic.Options)
if err != nil {
log.Infof(err.Error())
switch err := err.(type) {
case urlfetcher.ImageNotFoundError:
return fmt.Errorf("Error: image %s not found", ic.Reference)
default:
return fmt.Errorf("Failed to obtain OAuth endpoint: %s", err)
}
}
// Get the OAuth token - if only we have a URL
if url != nil {
token, err := FetchToken(ctx, ic.Options, url, ic.progressOutput)
if err != nil {
log.Errorf("Failed to fetch OAuth token: %s", err)
return err
}
ic.Token = token
}
progress.Message(ic.progressOutput, "", "Pulling from "+ic.Image)
// Get the manifest
manifest, err := FetchImageManifest(ctx, ic.Options, ic.progressOutput)
if err != nil {
log.Infof(err.Error())
switch err := err.(type) {
case urlfetcher.ImageNotFoundError:
return fmt.Errorf("Error: image %s not found", ic.Image)
case urlfetcher.TagNotFoundError:
return fmt.Errorf("Tag %s not found in repository %s", ic.Tag, ic.Image)
default:
return fmt.Errorf("Error while pulling image manifest: %s", err)
}
}
ic.ImageManifest = manifest
layers, err := ic.LayersToDownload()
if err != nil {
return err
}
ic.ImageLayers = layers
err = ldm.DownloadLayers(ctx, ic)
if err != nil {
return err
}
return nil
}