func (cmd) Execute(args map[string]interface{}) bool {
log, err := runtime.CreateLogger(args["--log-level"].(string))
if err != nil {
fmt.Fprintf(os.Stderr, "Invalid log-level, error: %s", err)
return false
}
// Create shell server
shellServer := interactive.NewShellServer(
newExecShell, log.WithField("component", "shell-server"),
)
// Setup server
server := graceful.Server{
Timeout: 35 * time.Second,
Server: &http.Server{
Addr: fmt.Sprintf("127.0.0.1:%s", args["--port"].(string)),
Handler: shellServer,
},
NoSignalHandling: false, // abort on sigint and sigterm
}
server.ListenAndServe()
shellServer.Abort()
shellServer.WaitAndClose()
return true
}
func (cmd) Execute(arguments map[string]interface{}) bool {
// Setup logging
logger, _ := runtime.CreateLogger("info")
log := logger.WithField("component", "qemu-build")
// Parse arguments
outputFile := arguments["<result.tar.zst>"].(string)
fromNew := arguments["from-new"].(bool)
fromImage := arguments["from-image"].(bool)
novnc := arguments["--no-vnc"].(bool)
boot, _ := arguments["--boot"].(string)
cdrom, _ := arguments["--cdrom"].(string)
size, err := strconv.ParseInt(arguments["--size"].(string), 10, 32)
if err != nil {
log.Fatal("Couldn't parse --size, error: ", err)
}
if size > 80 {
log.Fatal("Images have a sanity limit of 80 GiB!")
}
if fromNew == fromImage {
panic("Impossible arguments")
}
var inputFile string
if !fromImage {
inputFile = arguments["<machine.json>"].(string)
} else {
inputFile = arguments["<image.tar.zst>"].(string)
}
return buildImage(
log, inputFile, outputFile,
fromImage, novnc, boot, cdrom, int(size),
) == nil
}
func TestRetrievePollTaskUrlsErrorCaught(t *testing.T) {
logger, _ := runtime.CreateLogger("")
mockedQueue := &client.MockQueue{}
service := queueService{
client: mockedQueue,
provisionerID: ProvisionerID,
workerType: WorkerType,
log: logger.WithField("component", "Queue Service"),
expirationOffset: 300,
}
mockedQueue.On(
"PollTaskUrls",
ProvisionerID,
WorkerType,
// Error value does not matter, just as long as we create an error to return
).Return(&queue.PollTaskUrlsResponse{}, errors.New("bad error"))
err := service.refreshMessageQueueURLs()
if err == nil {
t.Fatal("Error should have been returned when polling failed")
}
assert.Equal(t, "Error retrieving message queue urls.", err.Error())
}
// NewTestEnvironment creates a new Environment suitable for use in tests.
//
// This function should only be used in testing
func newTestEnvironment() *runtime.Environment {
storage, err := runtime.NewTemporaryStorage(os.TempDir())
nilOrPanic(err, "Failed to create temporary storage at: ", os.TempDir())
folder, err := storage.NewFolder()
nilOrPanic(err, "Failed to create temporary storage folder")
// Set finalizer so that we always get the temporary folder removed.
// This is should really only be used in tests, otherwise it would better to
// call Remove() manually.
rt.SetFinalizer(folder, func(f runtime.TemporaryFolder) {
f.Remove()
})
logger, err := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating logger. %s", err)
os.Exit(1)
}
return &runtime.Environment{
GarbageCollector: &gc.GarbageCollector{},
TemporaryStorage: folder,
Log: logger,
}
}
func TestGuestToolsFailed(t *testing.T) {
// Create temporary storage
storage, err := runtime.NewTemporaryStorage(os.TempDir())
if err != nil {
panic("Failed to create TemporaryStorage")
}
environment := &runtime.Environment{
TemporaryStorage: storage,
}
// Setup a new MetaService
logTask := bytes.NewBuffer(nil)
result := false
resolved := false
m := sync.Mutex{}
s := metaservice.New([]string{"sh", "-c", "echo \"$TEST_TEXT\" && false"}, map[string]string{
"TEST_TEXT": "Hello world",
}, logTask, func(r bool) {
m.Lock()
defer m.Unlock()
if resolved {
panic("It shouldn't be possible to resolve twice")
}
resolved = true
result = r
}, environment)
// Create http server for testing
ts := httptest.NewServer(s)
defer ts.Close()
u, err := url.Parse(ts.URL)
if err != nil {
panic("Expected a url we can parse")
}
// Create a logger
logger, _ := runtime.CreateLogger("info")
log := logger.WithField("component", "guest-tools-tests")
// Create an run guest-tools
g := new(u.Host, log)
g.Run()
// Check the state
if !resolved {
t.Error("Expected the metadata to have resolved the task")
}
if result != false {
t.Error("Expected the metadata to get failed result")
}
if !strings.Contains(logTask.String(), "Hello world") {
t.Error("Got unexpected taskLog: '", logTask.String(), "'")
}
}
func TestClaimTaskError(t *testing.T) {
// When a task cannot be claimed because of a 401 authorization error, the message
// should not be deleted from the queue.
// Delete should be called if the claim errored because of authorization or ISE
// issues
deleteCalled := false
var handler = func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/delete" {
deleteCalled = true
}
return
}
s := httptest.NewServer(http.HandlerFunc(handler))
defer s.Close()
mockedQueue := &client.MockQueue{}
mockedQueue.On(
"ClaimTask",
"abc",
"0",
&queue.TaskClaimRequest{
WorkerGroup: WorkerType,
WorkerID: WorkerID,
},
).Return(&queue.TaskClaimResponse{},
httpbackoff.BadHttpResponseCode{
HttpResponseCode: 401,
},
)
task := &taskMessage{
TaskID: "abc",
RunID: 0,
signedDeleteURL: fmt.Sprintf("%s/delete", s.URL),
}
logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
service := queueService{
client: mockedQueue,
log: logger.WithField("component", "Queue Service"),
workerID: WorkerID,
workerGroup: WorkerType,
provisionerID: ProvisionerID,
}
_, err := service.claimTask(task)
assert.NotNil(t, err, "Task should not have been claimed")
// Delete should not have been called because it was an authorization issue
assert.False(t, deleteCalled, "Message should not have been deleted from the queue.")
}
func TestRetrieveTasksFromQueueDequeueChecked(t *testing.T) {
// When the dequeue count is above the reshold of 15, the message should be deleted
// regardless if it's been claimed yet or not.
message := `<?xml version="1.0" encoding="utf-8"?>
<QueueMessagesList>
<QueueMessage>
<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca3</MessageId>
<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
<DequeueCount>16</DequeueCount>
<MessageText>eyJ0YXNrSWQiOiAiYWJjIiwgInJ1bklkIjogMX0=</MessageText>
</QueueMessage>
</QueueMessagesList>`
// Delete URL should be called when dequeue count above threshold (15)
deleteCalled := false
var handler = func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/delete/5974b586-0df3-4e2d-ad0c-18e3892bfca3/YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw" {
deleteCalled = true
message = ""
return
}
w.Header().Set("Content-Type", "application/xml")
w.Write([]byte(message))
}
s := httptest.NewServer(http.HandlerFunc(handler))
defer s.Close()
logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
service := queueService{
log: logger.WithField("component", "Queue Service"),
expirationOffset: 300,
expires: tcclient.Time(time.Now().Add(time.Minute * 10)),
queues: []messageQueue{
{
SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
SignedPollURL: fmt.Sprintf("%s/tasks/1234?messages=true", s.URL),
},
},
}
tasks := service.retrieveTasksFromQueue(2)
assert.Equal(t, 1, len(tasks))
assert.True(t, deleteCalled, "Delete should have been called when dequeue count above threshold")
}
func TestErrorCaughtDeleteFromAzureQueueL(t *testing.T) {
var handler = func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusForbidden)
}
s := httptest.NewServer(http.HandlerFunc(handler))
defer s.Close()
logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
service := queueService{
log: logger.WithField("component", "Queue Service"),
}
err := service.deleteFromAzure(fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL))
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "(Permanent) HTTP response code 403")
}
func TestSuccessfullyDeleteFromAzureQueue(t *testing.T) {
// The method for deleting from the azure queue just makes sure that when
// calling a given URL that a 200 status response is received.
var handler = func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("ok"))
}
s := httptest.NewServer(http.HandlerFunc(handler))
defer s.Close()
logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
service := queueService{
log: logger.WithField("component", "Queue Service"),
}
err := service.deleteFromAzure(fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL))
assert.Nil(t, err)
}
func TestShouldNotRefreshMessageQueueURLs(t *testing.T) {
logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
service := queueService{
expirationOffset: 300,
expires: tcclient.Time(time.Now().Add(time.Minute * 10)),
queues: []messageQueue{messageQueue{}, messageQueue{}},
log: logger.WithField("component", "Queue Service"),
}
// Because the expiration is not close, and the service already has queues,
// there should be no reason to refresh. Because the service was not created
// with a taskcluster queue client, if it attempts to refresh, there will be
// a panic
err := service.refreshMessageQueueURLs()
assert.Nil(t, err, "No error should be returned because the urls should not have been refreshed")
}
func TestPollTaskURLNonEmptyMessageList(t *testing.T) {
// Messages below are arbitrary messages to ensure that they can be
// decoded.
messages := `<?xml version="1.0" encoding="utf-8"?>
<QueueMessagesList>
<QueueMessage>
<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca3</MessageId>
<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
<DequeueCount>1</DequeueCount>
<MessageText>eyJ0YXNrSWQiOiAiYWJjIiwgInJ1bklkIjogMH0=</MessageText>
</QueueMessage>
<QueueMessage>
<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca2</MessageId>
<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
<DequeueCount>1</DequeueCount>
<MessageText>eyJ0YXNrSWQiOiAiZGVmIiwgInJ1bklkIjogMX0=</MessageText>
</QueueMessage>
</QueueMessagesList>`
var handler = func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
w.Write([]byte(messages))
}
s := httptest.NewServer(http.HandlerFunc(handler))
defer s.Close()
logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
service := queueService{
log: logger.WithField("component", "Queue Service"),
queues: []messageQueue{{
SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
SignedPollURL: fmt.Sprintf("%s/tasks", s.URL),
}},
}
tasks, err := service.pollTaskURL(&service.queues[0], 3)
assert.Nil(t, err, "Error should not have been returned when empty message list provided.")
assert.Equal(t, 2, len(tasks))
// quick sanity check to make sure the messages are different
assert.NotEqual(t, tasks[0].TaskID, tasks[1].TaskID)
assert.NotEqual(t, tasks[0].signedDeleteURL, tasks[1].signedDeleteURL)
}
func TestPollTaskURLInvalidMessageTextEncoding(t *testing.T) {
// MessageText is not a valid base64 encoded string
messages := `<?xml version="1.0" encoding="utf-8"?>
<QueueMessagesList>
<QueueMessage>
<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca3</MessageId>
<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
<DequeueCount>1</DequeueCount>
<MessageText>invalid MessageText, not base64 encoded!</MessageText>
</QueueMessage>
</QueueMessagesList>`
// When there is an error decoding, message should be deleted from Azure
deleteCalled := false
var handler = func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/delete/5974b586-0df3-4e2d-ad0c-18e3892bfca3/YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw" {
deleteCalled = true
return
}
w.Header().Set("Content-Type", "application/xml")
w.Write([]byte(messages))
}
s := httptest.NewServer(http.HandlerFunc(handler))
defer s.Close()
logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
service := queueService{
log: logger.WithField("component", "Queue Service"),
queues: []messageQueue{{
SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
SignedPollURL: fmt.Sprintf("%s/tasks", s.URL),
}},
}
tasks, err := service.pollTaskURL(&service.queues[0], 3)
assert.Nil(t, err, "Error should not have been raised when unmarshalling invalid MessageText")
assert.Equal(t, 0, len(tasks))
assert.True(t, deleteCalled, "Delete URL not called after attempting to decode messageText")
}
func TestRetrievePollTaskUrls(t *testing.T) {
logger, _ := runtime.CreateLogger("")
mockedQueue := &client.MockQueue{}
service := queueService{
client: mockedQueue,
provisionerID: ProvisionerID,
workerType: WorkerType,
log: logger.WithField("component", "Queue Service"),
expirationOffset: 300,
}
mockedQueue.On(
"PollTaskUrls",
ProvisionerID,
WorkerType,
).Return(&queue.PollTaskUrlsResponse{
Expires: tcclient.Time(time.Now().Add(time.Minute * 10)),
Queues: []struct {
SignedDeleteURL string `json:"signedDeleteUrl"`
SignedPollURL string `json:"signedPollUrl"`
}{{
// Urls are arbitrary and unique so they can be checked later on.
// Polling should return at least 2 queues in production because of
// high/low priority queues
SignedDeleteURL: "abc",
SignedPollURL: "123",
}, {
SignedDeleteURL: "def",
SignedPollURL: "456",
}},
}, nil)
service.refreshMessageQueueURLs()
assert.Equal(t,
len(service.queues),
2,
fmt.Sprintf("Queue Service should contain two sets of url pairs but got %d", len(service.queues)),
)
assert.Equal(t, "abc", service.queues[0].SignedDeleteURL)
assert.Equal(t, "123", service.queues[0].SignedPollURL)
assert.Equal(t, "def", service.queues[1].SignedDeleteURL)
assert.Equal(t, "456", service.queues[1].SignedPollURL)
}
func TestBuildImage(t *testing.T) {
// Setup logging
logger, _ := runtime.CreateLogger("info")
log := logger.WithField("component", "qemu-build")
inputImageFile, err := filepath.Abs("../../engines/qemu/test-image/tinycore-setup.tar.zst")
if err != nil {
panic(err)
}
outputFile := filepath.Join(os.TempDir(), slugid.Nice())
defer os.Remove(outputFile)
novnc := true
cdrom := ""
// Create ISO file to play with
datadir := filepath.Join(os.TempDir(), slugid.Nice())
defer os.RemoveAll(datadir)
err = os.Mkdir(datadir, 0700)
if err != nil {
panic(err)
}
err = ioutil.WriteFile(filepath.Join(datadir, "setup.sh"),
[]byte("#!/bin/sh\necho 'started';\nsudo poweroff;\n"), 0755)
if err != nil {
panic(err)
}
isofile := filepath.Join(os.TempDir(), slugid.Nice())
defer os.Remove(isofile)
err = exec.Command("genisoimage", "-vJrV", "DATA_VOLUME", "-input-charset", "utf-8", "-o", isofile, datadir).Run()
if err != nil {
panic(err)
}
err = buildImage(
log, inputImageFile, outputFile,
true, novnc, isofile, cdrom, 1,
)
if err != nil {
panic(err)
}
}
func TestPollTaskUrlInvalidXMLResponse(t *testing.T) {
var handler = func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
io.WriteString(w, "Invalid XML")
}
s := httptest.NewServer(http.HandlerFunc(handler))
defer s.Close()
logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
service := queueService{
log: logger.WithField("component", "Queue Service"),
queues: []messageQueue{{
SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
SignedPollURL: fmt.Sprintf("%s/tasks", s.URL),
}},
}
_, err := service.pollTaskURL(&service.queues[0], 3)
assert.NotNil(t, err, "Error should have been returned when invalid xml was parsed")
assert.Contains(t, err.Error(), "Not able to xml decode the response from the Azure queue")
}
func TestPollTaskURLEmptyMessageList(t *testing.T) {
var handler = func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
io.WriteString(w, "<QueueMessagesList></QueueMessagesList>")
}
s := httptest.NewServer(http.HandlerFunc(handler))
defer s.Close()
logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
service := queueService{
log: logger.WithField("component", "Queue Service"),
queues: []messageQueue{{
SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
SignedPollURL: fmt.Sprintf("%s/tasks", s.URL),
}},
}
tasks, err := service.pollTaskURL(&service.queues[0], 3)
assert.Nil(t, err, "Error should not have been returned when empty message list provided.")
assert.Equal(t, 0, len(tasks))
}
func (cmd) Execute(arguments map[string]interface{}) bool {
host := arguments["--host"].(string)
logger, _ := runtime.CreateLogger("info")
log := logger.WithField("component", "qemu-guest-tools")
g := new(host, log)
if arguments["post-log"].(bool) {
logFile := arguments["<log-file>"].(string)
var r io.Reader
if logFile == "-" {
r = os.Stdin
} else {
f, err := os.Open(logFile)
if err != nil {
log.Error("Failed to open log-file, error: ", err)
return false
}
defer f.Close()
r = f
}
w, done := g.CreateTaskLog()
_, err := io.Copy(w, r)
if err != nil {
log.Error("Failed to post entire log, error: ", err)
err = w.Close()
<-done
}
return err == nil
}
go g.Run()
// Process actions forever, this must run in the main thread as exiting the
// main thread will cause the go program to exit.
g.ProcessActions()
return true
}
func TestPollTaskURLInvalidMessageTextContents(t *testing.T) {
// MessageText is {"abc",0} which is an invalid format when
// unmarshalling.
messages := `<?xml version="1.0" encoding="utf-8"?>
<QueueMessagesList>
<QueueMessage>
<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca3</MessageId>
<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
<DequeueCount>1</DequeueCount>
<MessageText>eyJhYmMiLDB9</MessageText>
</QueueMessage>
</QueueMessagesList>`
var handler = func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
w.Write([]byte(messages))
}
s := httptest.NewServer(http.HandlerFunc(handler))
defer s.Close()
logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
service := queueService{
log: logger.WithField("component", "Queue Service"),
queues: []messageQueue{{
SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
SignedPollURL: fmt.Sprintf("%s/tasks", s.URL),
}},
}
tasks, err := service.pollTaskURL(&service.queues[0], 3)
assert.Nil(t, err, "Error should not have been raised when unmarshalling invalid MessageText")
assert.Equal(t, 0, len(tasks))
}
func TestGuestToolsLiveLog(t *testing.T) {
nowReady := sync.WaitGroup{}
nowReady.Add(1)
ps := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
debug("Waiting for ready-now to be readable in log")
nowReady.Wait()
debug("replying: request-ok")
w.WriteHeader(http.StatusOK)
w.Write([]byte("request-ok"))
}))
// Create temporary storage
storage, err := runtime.NewTemporaryStorage(os.TempDir())
if err != nil {
panic("Failed to create TemporaryStorage")
}
environment := &runtime.Environment{
TemporaryStorage: storage,
}
// Setup a new MetaService
reader, writer := io.Pipe()
result := false
resolved := false
m := sync.Mutex{}
s := metaservice.New([]string{"sh", "-c", "echo \"$TEST_TEXT\" && curl -s " + ps.URL}, map[string]string{
"TEST_TEXT": "ready-now",
}, writer, func(r bool) {
m.Lock()
defer m.Unlock()
if resolved {
panic("It shouldn't be possible to resolve twice")
}
resolved = true
result = r
}, environment)
// Create http server for testing
ts := httptest.NewServer(s)
defer ts.Close()
u, err := url.Parse(ts.URL)
if err != nil {
panic("Expected a url we can parse")
}
// Create a logger
logger, _ := runtime.CreateLogger("info")
log := logger.WithField("component", "guest-tools-tests")
// Wait for
logTask := bytes.NewBuffer(nil)
logDone := sync.WaitGroup{}
logDone.Add(1)
go func() {
b := make([]byte, 1)
for !strings.Contains(logTask.String(), "ready-now") {
n, err := reader.Read(b)
logTask.Write(b[:n])
if err != nil {
panic("Unexpected error")
}
}
nowReady.Done()
io.Copy(logTask, reader)
logDone.Done()
}()
// Create an run guest-tools
g := new(u.Host, log)
g.Run()
writer.Close()
logDone.Wait()
// Check the state
if !resolved {
t.Error("Expected the metadata to have resolved the task")
}
if result != true {
t.Error("Expected the metadata to get successful result")
}
if !strings.Contains(logTask.String(), "request-ok") {
t.Error("Got unexpected taskLog: '", logTask.String(), "'")
}
}
func TestGuestToolsProcessingActions(t *testing.T) {
// Create temporary storage
storage, err := runtime.NewTemporaryStorage(os.TempDir())
if err != nil {
panic("Failed to create TemporaryStorage")
}
environment := &runtime.Environment{
TemporaryStorage: storage,
}
logTask := bytes.NewBuffer(nil)
meta := metaservice.New([]string{}, map[string]string{}, logTask, func(r bool) {
panic("This test shouldn't get to this point!")
}, environment)
// Create http server for testing
ts := httptest.NewServer(meta)
defer ts.Close()
defer meta.StopPollers() // Hack to stop pollers, otherwise server will block
u, err := url.Parse(ts.URL)
if err != nil {
panic("Expected a url we can parse")
}
// Create a logger
logger, _ := runtime.CreateLogger("info")
log := logger.WithField("component", "guest-tools-tests")
// Create an run guest-tools
g := new(u.Host, log)
// start processing actions
go g.ProcessActions()
defer g.StopProcessingActions()
////////////////////
debug("### Test meta.GetArtifact")
f, err := storage.NewFolder()
if err != nil {
panic("Failed to create temp folder")
}
defer f.Remove()
testFile := filepath.Join(f.Path(), "hello.txt")
err = ioutil.WriteFile(testFile, []byte("hello-world"), 0777)
nilOrPanic(err, "Failed to create testFile: ", testFile)
debug(" - request file: %s", testFile)
r, err := meta.GetArtifact(testFile)
nilOrPanic(err, "meta.GetArtifact failed, error: ", err)
debug(" - reading testFile")
data, err := ioutil.ReadAll(r)
nilOrPanic(err, "Failed to read testFile")
debug(" - read: '%s'", string(data))
assert(string(data) == "hello-world", "Wrong payload: ", string(data))
////////////////////
debug("### Test meta.GetArtifact (missing file)")
r, err = meta.GetArtifact(filepath.Join(f.Path(), "missing-file.txt"))
assert(r == nil, "Expected error wihtout a reader")
assert(err == engines.ErrResourceNotFound, "Expected ErrResourceNotFound")
////////////////////
debug("### Test meta.ListFolder")
testFolder := filepath.Join(f.Path(), "test-folder")
err = os.Mkdir(testFolder, 0777)
nilOrPanic(err, "Failed to create test-folder/")
testFile2 := filepath.Join(testFolder, "hello2.txt")
err = ioutil.WriteFile(testFile2, []byte("hello-world-2"), 0777)
nilOrPanic(err, "Failed to create testFile2: ", testFile2)
debug(" - meta.ListFolder")
files, err := meta.ListFolder(f.Path())
nilOrPanic(err, "ListFolder failed, err: ", err)
assert(len(files) == 2, "Expected 2 files")
assert(files[0] == testFile || files[1] == testFile, "Expected testFile")
assert(files[0] == testFile2 || files[1] == testFile2, "Expected testFile2")
////////////////////
debug("### Test meta.ListFolder (missing folder)")
files, err = meta.ListFolder(filepath.Join(f.Path(), "no-such-folder"))
assert(files == nil, "Expected files == nil, we hopefully have an error")
assert(err == engines.ErrResourceNotFound, "Expected ErrResourceNotFound")
////////////////////
debug("### Test meta.ListFolder (empty folder)")
emptyFolder := filepath.Join(f.Path(), "empty-folder")
err = os.Mkdir(emptyFolder, 0777)
nilOrPanic(err, "Failed to create empty-folder/")
files, err = meta.ListFolder(emptyFolder)
assert(len(files) == 0, "Expected zero files")
assert(err == nil, "Didn't expect any error")
////////////////////
testShellHello(meta)
testShellCat(meta)
testShellCatStdErr(meta)
}
func (cmd) Execute(arguments map[string]interface{}) bool {
// Read arguments
imageFile := arguments["<image>"].(string)
command := arguments["<command>"].([]string)
vnc := arguments["--vnc"].(bool)
// Create temporary storage and environment
storage, err := runtime.NewTemporaryStorage(os.TempDir())
if err != nil {
panic("Failed to create TemporaryStorage")
}
environment := &runtime.Environment{
TemporaryStorage: storage,
}
// Create a temporary folder
tempFolder := filepath.Join("/tmp", slugid.Nice())
if err = os.Mkdir(tempFolder, 0777); err != nil {
log.Fatal("Failed to create temporary folder in /tmp, error: ", err)
}
// Create the necessary runtime setup
gc := &gc.GarbageCollector{}
logger, _ := runtime.CreateLogger("info")
log := logger.WithField("component", "qemu-run")
// Create image manager
log.Info("Creating image manager")
manager, err := image.NewManager(filepath.Join(tempFolder, "/images/"), gc, logger.WithField("component", "image-manager"), nil)
if err != nil {
log.Fatal("Failed to create image manager", err)
}
// Get an instance of the image
log.Info("Creating instance of image")
image, err := manager.Instance("image", func(target string) error {
return cp.CopyFile(target, imageFile)
})
if err != nil {
log.Fatal("Failed to create instance of image, error: ", err)
}
// Setup a user-space network
log.Info("Creating user-space network")
net, err := network.NewUserNetwork(tempFolder)
if err != nil {
log.Fatal("Failed to create user-space network, error: ", err)
}
// Create virtual machine
log.Info("Creating virtual machine")
vm, err := vm.NewVirtualMachine(
image.Machine().Options(), image, net, tempFolder,
"", "", logger.WithField("component", "vm"),
)
if err != nil {
log.Fatal("Failed to create virtual-machine, error: ", err)
}
// Create meta-data service
log.Info("Creating meta-data service")
var shellServer *interactive.ShellServer
var displayServer *interactive.DisplayServer
ms := metaservice.New(command, make(map[string]string), os.Stdout, func(result bool) {
fmt.Println("### Task Completed, result = ", result)
shellServer.WaitAndClose()
displayServer.Abort()
vm.Kill()
}, environment)
// Setup http handler for network
vm.SetHTTPHandler(ms)
// Create ShellServer
shellServer = interactive.NewShellServer(
ms.ExecShell, log.WithField("component", "shell-server"),
)
// Create displayServer
displayServer = interactive.NewDisplayServer(
&socketDisplayProvider{socket: vm.VNCSocket()},
log.WithField("component", "display-server"),
)
interactiveHandler := http.NewServeMux()
interactiveHandler.Handle("/shell/", shellServer)
interactiveHandler.Handle("/display/", displayServer)
interactiveServer := graceful.Server{
Timeout: 30 * time.Second,
Server: &http.Server{
Addr: "localhost:8080",
Handler: interactiveHandler,
},
NoSignalHandling: true,
}
go interactiveServer.ListenAndServe()
// Start the virtual machine
log.Info("Start the virtual machine")
vm.Start()
// Start vncviewer
done := make(chan struct{})
if vnc {
go StartVNCViewer(vm.VNCSocket(), done)
}
// Wait for SIGINT/SIGKILL or vm.Done
c := make(chan os.Signal, 2)
signal.Notify(c, os.Interrupt, os.Kill) // This pattern leaks, acceptable here
select {
case <-c:
signal.Stop(c)
fmt.Println("### Terminating QEMU")
vm.Kill()
case <-vm.Done:
fmt.Println("### QEMU terminated")
}
close(done)
// Ensure that QEMU has terminated before we continue
<-vm.Done
interactiveServer.Stop(100 * time.Millisecond)
// Clean up anything left in the garbage collector
gc.CollectAll()
return true
}
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/taskcluster/slugid-go/slugid"
"github.com/taskcluster/taskcluster-client-go"
"github.com/taskcluster/taskcluster-client-go/queue"
"github.com/taskcluster/taskcluster-worker/engines"
"github.com/taskcluster/taskcluster-worker/plugins"
"github.com/taskcluster/taskcluster-worker/runtime"
"github.com/taskcluster/taskcluster-worker/runtime/client"
)
var logger, _ = runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
var taskDefinitions = map[string]struct {
definition string
success bool
}{
"invalidJSON": {
definition: "",
success: false,
},
"invalidEnginePayload": {
definition: `{"delay1": 10,"function": "write-log","argument": "Hello World"}`,
success: false,
},
"validEnginePayload": {
definition: `{"delay": 10,"function": "write-log","argument": "Hello World"}`,
func TestClaimTasks(t *testing.T) {
// Given a slice of task objects, claimTasks should claim each of them successfully
// and return a list of the claimed task runs.
var handler = func(w http.ResponseWriter, r *http.Request) {
return
}
s := httptest.NewServer(http.HandlerFunc(handler))
defer s.Close()
mockedQueue := &client.MockQueue{}
mockedQueue.On(
"ClaimTask",
"abc",
"0",
&queue.TaskClaimRequest{
WorkerGroup: WorkerType,
WorkerID: WorkerID,
},
).Return(&queue.TaskClaimResponse{
Credentials: struct {
AccessToken string `json:"accessToken"`
Certificate string `json:"certificate"`
ClientID string `json:"clientId"`
}{
AccessToken: "1040824383284384",
Certificate: "{}",
ClientID: "ajafdsfkj23",
},
RunID: 0,
Status: queue.TaskStatusStructure{},
TakenUntil: tcclient.Time{},
Task: queue.TaskDefinitionResponse{},
WorkerGroup: WorkerType,
WorkerID: WorkerID,
}, nil)
mockedQueue.On(
"ClaimTask",
"def",
"1",
&queue.TaskClaimRequest{
WorkerGroup: WorkerType,
WorkerID: WorkerID,
},
).Return(&queue.TaskClaimResponse{
Credentials: struct {
AccessToken string `json:"accessToken"`
Certificate string `json:"certificate"`
ClientID string `json:"clientId"`
}{
AccessToken: "234aajsgfaj340",
Certificate: "{}",
ClientID: "asfg089asgf08",
},
RunID: 1,
Status: queue.TaskStatusStructure{},
TakenUntil: tcclient.Time{},
Task: queue.TaskDefinitionResponse{},
WorkerGroup: WorkerType,
WorkerID: WorkerID,
}, nil)
tasks := []*taskMessage{{
TaskID: "abc",
RunID: 0,
signedDeleteURL: fmt.Sprintf("%s/delete", s.URL),
}, {
TaskID: "def",
RunID: 1,
signedDeleteURL: fmt.Sprintf("%s/delete", s.URL),
}}
logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
service := queueService{
client: mockedQueue,
capacity: 2,
tc: make(chan *taskClaim, 2),
log: logger.WithField("component", "Queue Service"),
workerID: WorkerID,
workerGroup: WorkerType,
provisionerID: ProvisionerID,
}
claims := []*taskClaim{}
service.claimTasks(tasks)
loop:
for len(claims) != 2 {
select {
case t := <-service.tc:
claims = append(claims, t)
// Tasks should be claimed in less than 1 second, but set up a timer to timeout
// so tests don't run forever waiting.
case <-time.NewTimer(1 * time.Second).C:
break loop
}
}
assert.Equal(t, 2, len(claims), "Not enough task claims received after calling claimTasks")
}
func TestClaimTask(t *testing.T) {
// Verifies that when claimTask is called in the queue service for a
// particular task run object, that the task is claimed and deleted from
// the azure queue.
deleteCalled := false
var handler = func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/delete" {
deleteCalled = true
}
return
}
s := httptest.NewServer(http.HandlerFunc(handler))
defer s.Close()
mockedQueue := &client.MockQueue{}
mockedQueue.On(
"ClaimTask",
"abc",
"0",
&queue.TaskClaimRequest{
WorkerGroup: WorkerType,
WorkerID: WorkerID,
},
).Return(&queue.TaskClaimResponse{
Credentials: struct {
AccessToken string `json:"accessToken"`
Certificate string `json:"certificate"`
ClientID string `json:"clientId"`
}{
AccessToken: "1040824383284384",
Certificate: "{}",
ClientID: "ajafdsfkj23",
},
RunID: 0,
Status: queue.TaskStatusStructure{},
TakenUntil: tcclient.Time{},
Task: queue.TaskDefinitionResponse{},
WorkerGroup: WorkerType,
WorkerID: WorkerID,
}, nil)
task := &taskMessage{
TaskID: "abc",
RunID: 0,
signedDeleteURL: fmt.Sprintf("%s/delete", s.URL),
}
logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
service := queueService{
client: mockedQueue,
log: logger.WithField("component", "Queue Service"),
workerID: WorkerID,
workerGroup: WorkerType,
provisionerID: ProvisionerID,
}
claim, err := service.claimTask(task)
assert.Nil(t, err)
assert.True(t, deleteCalled)
// Do a quick sanity check to make sure the response was correctly stored in
// the task run object
assert.Equal(t, "1040824383284384", claim.taskClaim.Credentials.AccessToken)
}
func TestRetrieveTasksFromQueueDoesNotQueryLowPriority(t *testing.T) {
// When enough tasks have been retrieved from the higher (first) priority queue,
// the lower (second) priority queue should not be polled.
messages := []string{
// MessageText is {"taskId": "abc", "RunID": 1}
`<?xml version="1.0" encoding="utf-8"?>
<QueueMessagesList>
<QueueMessage>
<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca3</MessageId>
<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
<DequeueCount>1</DequeueCount>
<MessageText>eyJ0YXNrSWQiOiAiYWJjIiwgInJ1bklkIjogMX0=</MessageText>
</QueueMessage>
</QueueMessagesList>`,
// MessageText[0] {"taskId": "def", "RunID": 0}
// MessageText[1] {"taskId": "ghi", "RunID": 2}
`<?xml version="1.0" encoding="utf-8"?>
<QueueMessagesList>
<QueueMessage>
<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca3</MessageId>
<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
<DequeueCount>1</DequeueCount>
<MessageText>eyJ0YXNrSWQiOiAiZGVmIiwgInJ1bklkIjogMH0NCg==</MessageText>
</QueueMessage>
<QueueMessage>
<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca3</MessageId>
<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
<DequeueCount>1</DequeueCount>
<MessageText>eyJ0YXNrSWQiOiAiZ2hpIiwgInJ1bklkIjogMH0NCg==</MessageText>
</QueueMessage>
</QueueMessagesList>`,
}
var handler = func(w http.ResponseWriter, r *http.Request) {
var message string
if r.URL.Path == "/tasks/1234" {
message = messages[1]
} else {
message = messages[0]
}
w.Header().Set("Content-Type", "application/xml")
w.Write([]byte(message))
}
s := httptest.NewServer(http.HandlerFunc(handler))
defer s.Close()
logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
service := queueService{
log: logger.WithField("component", "Queue Service"),
expirationOffset: 300,
expires: tcclient.Time(time.Now().Add(time.Minute * 10)),
queues: []messageQueue{
{
SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
SignedPollURL: fmt.Sprintf("%s/tasks/1234?messages=true", s.URL),
},
{
SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
SignedPollURL: fmt.Sprintf("%s/tasks/456?messages=true", s.URL),
},
},
}
tasks := service.retrieveTasksFromQueue(2)
// Only two tasks should have been retrieving leaving the third task in the lower
// priority queue not being retrieved
assert.Equal(t, 2, len(tasks))
taskIds := []string{"def", "ghi"}
for i, v := range taskIds {
assert.Equal(t, v, tasks[i].TaskID)
}
}
