Ejemplo n.º 1
0
func (cmd) Execute(args map[string]interface{}) bool {
	log, err := runtime.CreateLogger(args["--log-level"].(string))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Invalid log-level, error: %s", err)
		return false
	}

	// Create shell server
	shellServer := interactive.NewShellServer(
		newExecShell, log.WithField("component", "shell-server"),
	)

	// Setup server
	server := graceful.Server{
		Timeout: 35 * time.Second,
		Server: &http.Server{
			Addr:    fmt.Sprintf("127.0.0.1:%s", args["--port"].(string)),
			Handler: shellServer,
		},
		NoSignalHandling: false, // abort on sigint and sigterm
	}

	server.ListenAndServe()
	shellServer.Abort()
	shellServer.WaitAndClose()

	return true
}
Ejemplo n.º 2
0
func (cmd) Execute(arguments map[string]interface{}) bool {
	// Setup logging
	logger, _ := runtime.CreateLogger("info")
	log := logger.WithField("component", "qemu-build")

	// Parse arguments
	outputFile := arguments["<result.tar.zst>"].(string)
	fromNew := arguments["from-new"].(bool)
	fromImage := arguments["from-image"].(bool)
	novnc := arguments["--no-vnc"].(bool)
	boot, _ := arguments["--boot"].(string)
	cdrom, _ := arguments["--cdrom"].(string)
	size, err := strconv.ParseInt(arguments["--size"].(string), 10, 32)
	if err != nil {
		log.Fatal("Couldn't parse --size, error: ", err)
	}
	if size > 80 {
		log.Fatal("Images have a sanity limit of 80 GiB!")
	}
	if fromNew == fromImage {
		panic("Impossible arguments")
	}

	var inputFile string
	if !fromImage {
		inputFile = arguments["<machine.json>"].(string)
	} else {
		inputFile = arguments["<image.tar.zst>"].(string)
	}

	return buildImage(
		log, inputFile, outputFile,
		fromImage, novnc, boot, cdrom, int(size),
	) == nil
}
func TestRetrievePollTaskUrlsErrorCaught(t *testing.T) {
	logger, _ := runtime.CreateLogger("")
	mockedQueue := &client.MockQueue{}
	service := queueService{
		client:           mockedQueue,
		provisionerID:    ProvisionerID,
		workerType:       WorkerType,
		log:              logger.WithField("component", "Queue Service"),
		expirationOffset: 300,
	}

	mockedQueue.On(
		"PollTaskUrls",
		ProvisionerID,
		WorkerType,
	// Error value does not matter, just as long as we create an error to return
	).Return(&queue.PollTaskUrlsResponse{}, errors.New("bad error"))

	err := service.refreshMessageQueueURLs()
	if err == nil {
		t.Fatal("Error should have been returned when polling failed")
	}

	assert.Equal(t, "Error retrieving message queue urls.", err.Error())
}
Ejemplo n.º 4
0
// NewTestEnvironment creates a new Environment suitable for use in tests.
//
// This function should only be used in testing
func newTestEnvironment() *runtime.Environment {
	storage, err := runtime.NewTemporaryStorage(os.TempDir())
	nilOrPanic(err, "Failed to create temporary storage at: ", os.TempDir())

	folder, err := storage.NewFolder()
	nilOrPanic(err, "Failed to create temporary storage folder")

	// Set finalizer so that we always get the temporary folder removed.
	// This is should really only be used in tests, otherwise it would better to
	// call Remove() manually.
	rt.SetFinalizer(folder, func(f runtime.TemporaryFolder) {
		f.Remove()
	})

	logger, err := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error creating logger. %s", err)
		os.Exit(1)
	}

	return &runtime.Environment{
		GarbageCollector: &gc.GarbageCollector{},
		TemporaryStorage: folder,
		Log:              logger,
	}
}
func TestGuestToolsFailed(t *testing.T) {
	// Create temporary storage
	storage, err := runtime.NewTemporaryStorage(os.TempDir())
	if err != nil {
		panic("Failed to create TemporaryStorage")
	}
	environment := &runtime.Environment{
		TemporaryStorage: storage,
	}

	// Setup a new MetaService
	logTask := bytes.NewBuffer(nil)
	result := false
	resolved := false
	m := sync.Mutex{}
	s := metaservice.New([]string{"sh", "-c", "echo \"$TEST_TEXT\" && false"}, map[string]string{
		"TEST_TEXT": "Hello world",
	}, logTask, func(r bool) {
		m.Lock()
		defer m.Unlock()
		if resolved {
			panic("It shouldn't be possible to resolve twice")
		}
		resolved = true
		result = r
	}, environment)

	// Create http server for testing
	ts := httptest.NewServer(s)
	defer ts.Close()
	u, err := url.Parse(ts.URL)
	if err != nil {
		panic("Expected a url we can parse")
	}

	// Create a logger
	logger, _ := runtime.CreateLogger("info")
	log := logger.WithField("component", "guest-tools-tests")

	// Create an run guest-tools
	g := new(u.Host, log)
	g.Run()

	// Check the state
	if !resolved {
		t.Error("Expected the metadata to have resolved the task")
	}
	if result != false {
		t.Error("Expected the metadata to get failed result")
	}
	if !strings.Contains(logTask.String(), "Hello world") {
		t.Error("Got unexpected taskLog: '", logTask.String(), "'")
	}
}
func TestClaimTaskError(t *testing.T) {
	// When a task cannot be claimed because of a 401 authorization error, the message
	// should not be deleted from the queue.

	// Delete should be called if the claim errored because of authorization or ISE
	// issues
	deleteCalled := false
	var handler = func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/delete" {
			deleteCalled = true
		}
		return
	}
	s := httptest.NewServer(http.HandlerFunc(handler))
	defer s.Close()

	mockedQueue := &client.MockQueue{}
	mockedQueue.On(
		"ClaimTask",
		"abc",
		"0",
		&queue.TaskClaimRequest{
			WorkerGroup: WorkerType,
			WorkerID:    WorkerID,
		},
	).Return(&queue.TaskClaimResponse{},
		httpbackoff.BadHttpResponseCode{
			HttpResponseCode: 401,
		},
	)
	task := &taskMessage{
		TaskID:          "abc",
		RunID:           0,
		signedDeleteURL: fmt.Sprintf("%s/delete", s.URL),
	}

	logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
	service := queueService{
		client:        mockedQueue,
		log:           logger.WithField("component", "Queue Service"),
		workerID:      WorkerID,
		workerGroup:   WorkerType,
		provisionerID: ProvisionerID,
	}

	_, err := service.claimTask(task)

	assert.NotNil(t, err, "Task should not have been claimed")
	// Delete should not have been called because it was an authorization issue
	assert.False(t, deleteCalled, "Message should not have been deleted from the queue.")
}
func TestRetrieveTasksFromQueueDequeueChecked(t *testing.T) {
	// When the dequeue count is above the reshold of 15, the message should be deleted
	// regardless if it's been claimed yet or not.
	message := `<?xml version="1.0" encoding="utf-8"?>
	<QueueMessagesList>
	  <QueueMessage>
		<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca3</MessageId>
		<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
		<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
		<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
		<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
		<DequeueCount>16</DequeueCount>
		<MessageText>eyJ0YXNrSWQiOiAiYWJjIiwgInJ1bklkIjogMX0=</MessageText>
	  </QueueMessage>
	</QueueMessagesList>`

	// Delete URL should be called when dequeue count above threshold (15)
	deleteCalled := false

	var handler = func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/delete/5974b586-0df3-4e2d-ad0c-18e3892bfca3/YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw" {
			deleteCalled = true
			message = ""
			return
		}
		w.Header().Set("Content-Type", "application/xml")
		w.Write([]byte(message))
	}

	s := httptest.NewServer(http.HandlerFunc(handler))
	defer s.Close()
	logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
	service := queueService{
		log:              logger.WithField("component", "Queue Service"),
		expirationOffset: 300,
		expires:          tcclient.Time(time.Now().Add(time.Minute * 10)),
		queues: []messageQueue{
			{
				SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
				SignedPollURL:   fmt.Sprintf("%s/tasks/1234?messages=true", s.URL),
			},
		},
	}

	tasks := service.retrieveTasksFromQueue(2)

	assert.Equal(t, 1, len(tasks))
	assert.True(t, deleteCalled, "Delete should have been called when dequeue count above threshold")

}
func TestErrorCaughtDeleteFromAzureQueueL(t *testing.T) {
	var handler = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusForbidden)
	}

	s := httptest.NewServer(http.HandlerFunc(handler))
	defer s.Close()
	logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
	service := queueService{
		log: logger.WithField("component", "Queue Service"),
	}
	err := service.deleteFromAzure(fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL))
	assert.NotNil(t, err)
	assert.Contains(t, err.Error(), "(Permanent) HTTP response code 403")
}
func TestSuccessfullyDeleteFromAzureQueue(t *testing.T) {
	// The method for deleting from the azure queue just makes sure that when
	// calling a given URL that a 200 status response is received.
	var handler = func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("ok"))
	}

	s := httptest.NewServer(http.HandlerFunc(handler))
	defer s.Close()
	logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
	service := queueService{
		log: logger.WithField("component", "Queue Service"),
	}
	err := service.deleteFromAzure(fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL))
	assert.Nil(t, err)
}
func TestShouldNotRefreshMessageQueueURLs(t *testing.T) {
	logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
	service := queueService{
		expirationOffset: 300,
		expires:          tcclient.Time(time.Now().Add(time.Minute * 10)),
		queues:           []messageQueue{messageQueue{}, messageQueue{}},
		log:              logger.WithField("component", "Queue Service"),
	}

	// Because the expiration is not close, and the service already has queues,
	// there should be no reason to refresh.  Because the service was not created
	// with a taskcluster queue client, if it attempts to refresh, there will be
	// a panic
	err := service.refreshMessageQueueURLs()
	assert.Nil(t, err, "No error should be returned because the urls should not have been refreshed")
}
func TestPollTaskURLNonEmptyMessageList(t *testing.T) {
	// Messages below are arbitrary messages to ensure that they can be
	// decoded.
	messages := `<?xml version="1.0" encoding="utf-8"?>
	<QueueMessagesList>
	  <QueueMessage>
		<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca3</MessageId>
		<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
		<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
		<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
		<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
		<DequeueCount>1</DequeueCount>
		<MessageText>eyJ0YXNrSWQiOiAiYWJjIiwgInJ1bklkIjogMH0=</MessageText>
	  </QueueMessage>
	  <QueueMessage>
		<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca2</MessageId>
		<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
		<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
		<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
		<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
		<DequeueCount>1</DequeueCount>
		<MessageText>eyJ0YXNrSWQiOiAiZGVmIiwgInJ1bklkIjogMX0=</MessageText>
	  </QueueMessage>
	</QueueMessagesList>`
	var handler = func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/xml")
		w.Write([]byte(messages))
	}

	s := httptest.NewServer(http.HandlerFunc(handler))
	defer s.Close()
	logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
	service := queueService{
		log: logger.WithField("component", "Queue Service"),
		queues: []messageQueue{{
			SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
			SignedPollURL:   fmt.Sprintf("%s/tasks", s.URL),
		}},
	}

	tasks, err := service.pollTaskURL(&service.queues[0], 3)
	assert.Nil(t, err, "Error should not have been returned when empty message list provided.")
	assert.Equal(t, 2, len(tasks))
	// quick sanity check to make sure the messages are different
	assert.NotEqual(t, tasks[0].TaskID, tasks[1].TaskID)
	assert.NotEqual(t, tasks[0].signedDeleteURL, tasks[1].signedDeleteURL)
}
func TestPollTaskURLInvalidMessageTextEncoding(t *testing.T) {
	// MessageText is not a valid base64 encoded string
	messages := `<?xml version="1.0" encoding="utf-8"?>
	<QueueMessagesList>
	  <QueueMessage>
		<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca3</MessageId>
		<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
		<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
		<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
		<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
		<DequeueCount>1</DequeueCount>
		<MessageText>invalid MessageText, not base64 encoded!</MessageText>
	  </QueueMessage>
	</QueueMessagesList>`

	// When there is an error decoding, message should be deleted from Azure
	deleteCalled := false

	var handler = func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/delete/5974b586-0df3-4e2d-ad0c-18e3892bfca3/YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw" {
			deleteCalled = true
			return
		}

		w.Header().Set("Content-Type", "application/xml")
		w.Write([]byte(messages))
	}

	s := httptest.NewServer(http.HandlerFunc(handler))
	defer s.Close()
	logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
	service := queueService{
		log: logger.WithField("component", "Queue Service"),
		queues: []messageQueue{{
			SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
			SignedPollURL:   fmt.Sprintf("%s/tasks", s.URL),
		}},
	}

	tasks, err := service.pollTaskURL(&service.queues[0], 3)
	assert.Nil(t, err, "Error should not have been raised when unmarshalling invalid MessageText")
	assert.Equal(t, 0, len(tasks))
	assert.True(t, deleteCalled, "Delete URL not called after attempting to decode messageText")
}
func TestRetrievePollTaskUrls(t *testing.T) {
	logger, _ := runtime.CreateLogger("")
	mockedQueue := &client.MockQueue{}
	service := queueService{
		client:           mockedQueue,
		provisionerID:    ProvisionerID,
		workerType:       WorkerType,
		log:              logger.WithField("component", "Queue Service"),
		expirationOffset: 300,
	}
	mockedQueue.On(
		"PollTaskUrls",
		ProvisionerID,
		WorkerType,
	).Return(&queue.PollTaskUrlsResponse{
		Expires: tcclient.Time(time.Now().Add(time.Minute * 10)),
		Queues: []struct {
			SignedDeleteURL string `json:"signedDeleteUrl"`
			SignedPollURL   string `json:"signedPollUrl"`
		}{{
			// Urls are arbitrary and unique so they can be checked later on.
			// Polling should return at least 2 queues in production because of
			// high/low priority queues
			SignedDeleteURL: "abc",
			SignedPollURL:   "123",
		}, {
			SignedDeleteURL: "def",
			SignedPollURL:   "456",
		}},
	}, nil)
	service.refreshMessageQueueURLs()
	assert.Equal(t,
		len(service.queues),
		2,
		fmt.Sprintf("Queue Service should contain two sets of url pairs but got %d", len(service.queues)),
	)

	assert.Equal(t, "abc", service.queues[0].SignedDeleteURL)
	assert.Equal(t, "123", service.queues[0].SignedPollURL)
	assert.Equal(t, "def", service.queues[1].SignedDeleteURL)
	assert.Equal(t, "456", service.queues[1].SignedPollURL)
}
func TestBuildImage(t *testing.T) {
	// Setup logging
	logger, _ := runtime.CreateLogger("info")
	log := logger.WithField("component", "qemu-build")

	inputImageFile, err := filepath.Abs("../../engines/qemu/test-image/tinycore-setup.tar.zst")
	if err != nil {
		panic(err)
	}
	outputFile := filepath.Join(os.TempDir(), slugid.Nice())
	defer os.Remove(outputFile)
	novnc := true
	cdrom := ""

	// Create ISO file to play with
	datadir := filepath.Join(os.TempDir(), slugid.Nice())
	defer os.RemoveAll(datadir)
	err = os.Mkdir(datadir, 0700)
	if err != nil {
		panic(err)
	}
	err = ioutil.WriteFile(filepath.Join(datadir, "setup.sh"),
		[]byte("#!/bin/sh\necho 'started';\nsudo poweroff;\n"), 0755)
	if err != nil {
		panic(err)
	}
	isofile := filepath.Join(os.TempDir(), slugid.Nice())
	defer os.Remove(isofile)
	err = exec.Command("genisoimage", "-vJrV", "DATA_VOLUME", "-input-charset", "utf-8", "-o", isofile, datadir).Run()
	if err != nil {
		panic(err)
	}

	err = buildImage(
		log, inputImageFile, outputFile,
		true, novnc, isofile, cdrom, 1,
	)
	if err != nil {
		panic(err)
	}
}
func TestPollTaskUrlInvalidXMLResponse(t *testing.T) {
	var handler = func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/xml")
		io.WriteString(w, "Invalid XML")
	}

	s := httptest.NewServer(http.HandlerFunc(handler))
	defer s.Close()
	logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
	service := queueService{
		log: logger.WithField("component", "Queue Service"),
		queues: []messageQueue{{
			SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
			SignedPollURL:   fmt.Sprintf("%s/tasks", s.URL),
		}},
	}

	_, err := service.pollTaskURL(&service.queues[0], 3)
	assert.NotNil(t, err, "Error should have been returned when invalid xml was parsed")
	assert.Contains(t, err.Error(), "Not able to xml decode the response from the Azure queue")
}
func TestPollTaskURLEmptyMessageList(t *testing.T) {
	var handler = func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/xml")
		io.WriteString(w, "<QueueMessagesList></QueueMessagesList>")
	}

	s := httptest.NewServer(http.HandlerFunc(handler))
	defer s.Close()
	logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
	service := queueService{
		log: logger.WithField("component", "Queue Service"),
		queues: []messageQueue{{
			SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
			SignedPollURL:   fmt.Sprintf("%s/tasks", s.URL),
		}},
	}

	tasks, err := service.pollTaskURL(&service.queues[0], 3)
	assert.Nil(t, err, "Error should not have been returned when empty message list provided.")
	assert.Equal(t, 0, len(tasks))
}
Ejemplo n.º 17
0
func (cmd) Execute(arguments map[string]interface{}) bool {
	host := arguments["--host"].(string)

	logger, _ := runtime.CreateLogger("info")
	log := logger.WithField("component", "qemu-guest-tools")

	g := new(host, log)

	if arguments["post-log"].(bool) {
		logFile := arguments["<log-file>"].(string)
		var r io.Reader
		if logFile == "-" {
			r = os.Stdin
		} else {
			f, err := os.Open(logFile)
			if err != nil {
				log.Error("Failed to open log-file, error: ", err)
				return false
			}
			defer f.Close()
			r = f
		}
		w, done := g.CreateTaskLog()
		_, err := io.Copy(w, r)
		if err != nil {
			log.Error("Failed to post entire log, error: ", err)
			err = w.Close()
			<-done
		}
		return err == nil
	}

	go g.Run()
	// Process actions forever, this must run in the main thread as exiting the
	// main thread will cause the go program to exit.
	g.ProcessActions()

	return true
}
func TestPollTaskURLInvalidMessageTextContents(t *testing.T) {
	// MessageText is {"abc",0} which is an invalid format when
	// unmarshalling.
	messages := `<?xml version="1.0" encoding="utf-8"?>
	<QueueMessagesList>
	  <QueueMessage>
		<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca3</MessageId>
		<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
		<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
		<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
		<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
		<DequeueCount>1</DequeueCount>
		<MessageText>eyJhYmMiLDB9</MessageText>
	  </QueueMessage>

	</QueueMessagesList>`
	var handler = func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/xml")
		w.Write([]byte(messages))
	}

	s := httptest.NewServer(http.HandlerFunc(handler))
	defer s.Close()
	logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
	service := queueService{
		log: logger.WithField("component", "Queue Service"),
		queues: []messageQueue{{
			SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
			SignedPollURL:   fmt.Sprintf("%s/tasks", s.URL),
		}},
	}

	tasks, err := service.pollTaskURL(&service.queues[0], 3)
	assert.Nil(t, err, "Error should not have been raised when unmarshalling invalid MessageText")
	assert.Equal(t, 0, len(tasks))
}
func TestGuestToolsLiveLog(t *testing.T) {
	nowReady := sync.WaitGroup{}
	nowReady.Add(1)
	ps := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		debug("Waiting for ready-now to be readable in log")
		nowReady.Wait()
		debug("replying: request-ok")

		w.WriteHeader(http.StatusOK)
		w.Write([]byte("request-ok"))
	}))

	// Create temporary storage
	storage, err := runtime.NewTemporaryStorage(os.TempDir())
	if err != nil {
		panic("Failed to create TemporaryStorage")
	}
	environment := &runtime.Environment{
		TemporaryStorage: storage,
	}

	// Setup a new MetaService
	reader, writer := io.Pipe()
	result := false
	resolved := false
	m := sync.Mutex{}
	s := metaservice.New([]string{"sh", "-c", "echo \"$TEST_TEXT\" && curl -s " + ps.URL}, map[string]string{
		"TEST_TEXT": "ready-now",
	}, writer, func(r bool) {
		m.Lock()
		defer m.Unlock()
		if resolved {
			panic("It shouldn't be possible to resolve twice")
		}
		resolved = true
		result = r
	}, environment)

	// Create http server for testing
	ts := httptest.NewServer(s)
	defer ts.Close()
	u, err := url.Parse(ts.URL)
	if err != nil {
		panic("Expected a url we can parse")
	}

	// Create a logger
	logger, _ := runtime.CreateLogger("info")
	log := logger.WithField("component", "guest-tools-tests")

	// Wait for
	logTask := bytes.NewBuffer(nil)
	logDone := sync.WaitGroup{}
	logDone.Add(1)
	go func() {
		b := make([]byte, 1)
		for !strings.Contains(logTask.String(), "ready-now") {
			n, err := reader.Read(b)
			logTask.Write(b[:n])
			if err != nil {
				panic("Unexpected error")
			}
		}
		nowReady.Done()
		io.Copy(logTask, reader)
		logDone.Done()
	}()

	// Create an run guest-tools
	g := new(u.Host, log)
	g.Run()
	writer.Close()
	logDone.Wait()

	// Check the state
	if !resolved {
		t.Error("Expected the metadata to have resolved the task")
	}
	if result != true {
		t.Error("Expected the metadata to get successful result")
	}
	if !strings.Contains(logTask.String(), "request-ok") {
		t.Error("Got unexpected taskLog: '", logTask.String(), "'")
	}
}
Ejemplo n.º 20
0
func TestGuestToolsProcessingActions(t *testing.T) {
	// Create temporary storage
	storage, err := runtime.NewTemporaryStorage(os.TempDir())
	if err != nil {
		panic("Failed to create TemporaryStorage")
	}
	environment := &runtime.Environment{
		TemporaryStorage: storage,
	}

	logTask := bytes.NewBuffer(nil)
	meta := metaservice.New([]string{}, map[string]string{}, logTask, func(r bool) {
		panic("This test shouldn't get to this point!")
	}, environment)

	// Create http server for testing
	ts := httptest.NewServer(meta)
	defer ts.Close()
	defer meta.StopPollers() // Hack to stop pollers, otherwise server will block
	u, err := url.Parse(ts.URL)
	if err != nil {
		panic("Expected a url we can parse")
	}

	// Create a logger
	logger, _ := runtime.CreateLogger("info")
	log := logger.WithField("component", "guest-tools-tests")

	// Create an run guest-tools
	g := new(u.Host, log)

	// start processing actions
	go g.ProcessActions()
	defer g.StopProcessingActions()

	////////////////////
	debug("### Test meta.GetArtifact")
	f, err := storage.NewFolder()
	if err != nil {
		panic("Failed to create temp folder")
	}
	defer f.Remove()

	testFile := filepath.Join(f.Path(), "hello.txt")
	err = ioutil.WriteFile(testFile, []byte("hello-world"), 0777)
	nilOrPanic(err, "Failed to create testFile: ", testFile)

	debug(" - request file: %s", testFile)
	r, err := meta.GetArtifact(testFile)
	nilOrPanic(err, "meta.GetArtifact failed, error: ", err)

	debug(" - reading testFile")
	data, err := ioutil.ReadAll(r)
	nilOrPanic(err, "Failed to read testFile")
	debug(" - read: '%s'", string(data))
	assert(string(data) == "hello-world", "Wrong payload: ", string(data))

	////////////////////
	debug("### Test meta.GetArtifact (missing file)")
	r, err = meta.GetArtifact(filepath.Join(f.Path(), "missing-file.txt"))
	assert(r == nil, "Expected error wihtout a reader")
	assert(err == engines.ErrResourceNotFound, "Expected ErrResourceNotFound")

	////////////////////
	debug("### Test meta.ListFolder")
	testFolder := filepath.Join(f.Path(), "test-folder")
	err = os.Mkdir(testFolder, 0777)
	nilOrPanic(err, "Failed to create test-folder/")

	testFile2 := filepath.Join(testFolder, "hello2.txt")
	err = ioutil.WriteFile(testFile2, []byte("hello-world-2"), 0777)
	nilOrPanic(err, "Failed to create testFile2: ", testFile2)

	debug(" - meta.ListFolder")
	files, err := meta.ListFolder(f.Path())
	nilOrPanic(err, "ListFolder failed, err: ", err)

	assert(len(files) == 2, "Expected 2 files")
	assert(files[0] == testFile || files[1] == testFile, "Expected testFile")
	assert(files[0] == testFile2 || files[1] == testFile2, "Expected testFile2")

	////////////////////
	debug("### Test meta.ListFolder (missing folder)")
	files, err = meta.ListFolder(filepath.Join(f.Path(), "no-such-folder"))
	assert(files == nil, "Expected files == nil, we hopefully have an error")
	assert(err == engines.ErrResourceNotFound, "Expected ErrResourceNotFound")

	////////////////////
	debug("### Test meta.ListFolder (empty folder)")
	emptyFolder := filepath.Join(f.Path(), "empty-folder")
	err = os.Mkdir(emptyFolder, 0777)
	nilOrPanic(err, "Failed to create empty-folder/")

	files, err = meta.ListFolder(emptyFolder)
	assert(len(files) == 0, "Expected zero files")
	assert(err == nil, "Didn't expect any error")

	////////////////////
	testShellHello(meta)
	testShellCat(meta)
	testShellCatStdErr(meta)
}
Ejemplo n.º 21
0
func (cmd) Execute(arguments map[string]interface{}) bool {
	// Read arguments
	imageFile := arguments["<image>"].(string)
	command := arguments["<command>"].([]string)
	vnc := arguments["--vnc"].(bool)

	// Create temporary storage and environment
	storage, err := runtime.NewTemporaryStorage(os.TempDir())
	if err != nil {
		panic("Failed to create TemporaryStorage")
	}
	environment := &runtime.Environment{
		TemporaryStorage: storage,
	}

	// Create a temporary folder
	tempFolder := filepath.Join("/tmp", slugid.Nice())
	if err = os.Mkdir(tempFolder, 0777); err != nil {
		log.Fatal("Failed to create temporary folder in /tmp, error: ", err)
	}

	// Create the necessary runtime setup
	gc := &gc.GarbageCollector{}
	logger, _ := runtime.CreateLogger("info")
	log := logger.WithField("component", "qemu-run")

	// Create image manager
	log.Info("Creating image manager")
	manager, err := image.NewManager(filepath.Join(tempFolder, "/images/"), gc, logger.WithField("component", "image-manager"), nil)
	if err != nil {
		log.Fatal("Failed to create image manager", err)
	}

	// Get an instance of the image
	log.Info("Creating instance of image")
	image, err := manager.Instance("image", func(target string) error {
		return cp.CopyFile(target, imageFile)
	})
	if err != nil {
		log.Fatal("Failed to create instance of image, error: ", err)
	}

	// Setup a user-space network
	log.Info("Creating user-space network")
	net, err := network.NewUserNetwork(tempFolder)
	if err != nil {
		log.Fatal("Failed to create user-space network, error: ", err)
	}

	// Create virtual machine
	log.Info("Creating virtual machine")
	vm, err := vm.NewVirtualMachine(
		image.Machine().Options(), image, net, tempFolder,
		"", "", logger.WithField("component", "vm"),
	)
	if err != nil {
		log.Fatal("Failed to create virtual-machine, error: ", err)
	}

	// Create meta-data service
	log.Info("Creating meta-data service")
	var shellServer *interactive.ShellServer
	var displayServer *interactive.DisplayServer
	ms := metaservice.New(command, make(map[string]string), os.Stdout, func(result bool) {
		fmt.Println("### Task Completed, result = ", result)
		shellServer.WaitAndClose()
		displayServer.Abort()
		vm.Kill()
	}, environment)

	// Setup http handler for network
	vm.SetHTTPHandler(ms)

	// Create ShellServer
	shellServer = interactive.NewShellServer(
		ms.ExecShell, log.WithField("component", "shell-server"),
	)

	// Create displayServer
	displayServer = interactive.NewDisplayServer(
		&socketDisplayProvider{socket: vm.VNCSocket()},
		log.WithField("component", "display-server"),
	)

	interactiveHandler := http.NewServeMux()
	interactiveHandler.Handle("/shell/", shellServer)
	interactiveHandler.Handle("/display/", displayServer)
	interactiveServer := graceful.Server{
		Timeout: 30 * time.Second,
		Server: &http.Server{
			Addr:    "localhost:8080",
			Handler: interactiveHandler,
		},
		NoSignalHandling: true,
	}
	go interactiveServer.ListenAndServe()

	// Start the virtual machine
	log.Info("Start the virtual machine")
	vm.Start()

	// Start vncviewer
	done := make(chan struct{})
	if vnc {
		go StartVNCViewer(vm.VNCSocket(), done)
	}

	// Wait for SIGINT/SIGKILL or vm.Done
	c := make(chan os.Signal, 2)
	signal.Notify(c, os.Interrupt, os.Kill) // This pattern leaks, acceptable here
	select {
	case <-c:
		signal.Stop(c)
		fmt.Println("### Terminating QEMU")
		vm.Kill()
	case <-vm.Done:
		fmt.Println("### QEMU terminated")
	}
	close(done)

	// Ensure that QEMU has terminated before we continue
	<-vm.Done
	interactiveServer.Stop(100 * time.Millisecond)

	// Clean up anything left in the garbage collector
	gc.CollectAll()
	return true
}
Ejemplo n.º 22
0
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/taskcluster/slugid-go/slugid"
	"github.com/taskcluster/taskcluster-client-go"
	"github.com/taskcluster/taskcluster-client-go/queue"
	"github.com/taskcluster/taskcluster-worker/engines"
	"github.com/taskcluster/taskcluster-worker/plugins"
	"github.com/taskcluster/taskcluster-worker/runtime"
	"github.com/taskcluster/taskcluster-worker/runtime/client"
)

var logger, _ = runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))

var taskDefinitions = map[string]struct {
	definition string
	success    bool
}{
	"invalidJSON": {
		definition: "",
		success:    false,
	},
	"invalidEnginePayload": {
		definition: `{"delay1": 10,"function": "write-log","argument": "Hello World"}`,
		success:    false,
	},
	"validEnginePayload": {
		definition: `{"delay": 10,"function": "write-log","argument": "Hello World"}`,
func TestClaimTasks(t *testing.T) {
	// Given a slice of task objects, claimTasks should claim each of them successfully
	// and return a list of the claimed task runs.
	var handler = func(w http.ResponseWriter, r *http.Request) {
		return
	}
	s := httptest.NewServer(http.HandlerFunc(handler))
	defer s.Close()

	mockedQueue := &client.MockQueue{}
	mockedQueue.On(
		"ClaimTask",
		"abc",
		"0",
		&queue.TaskClaimRequest{
			WorkerGroup: WorkerType,
			WorkerID:    WorkerID,
		},
	).Return(&queue.TaskClaimResponse{
		Credentials: struct {
			AccessToken string `json:"accessToken"`
			Certificate string `json:"certificate"`
			ClientID    string `json:"clientId"`
		}{
			AccessToken: "1040824383284384",
			Certificate: "{}",
			ClientID:    "ajafdsfkj23",
		},
		RunID:       0,
		Status:      queue.TaskStatusStructure{},
		TakenUntil:  tcclient.Time{},
		Task:        queue.TaskDefinitionResponse{},
		WorkerGroup: WorkerType,
		WorkerID:    WorkerID,
	}, nil)
	mockedQueue.On(
		"ClaimTask",
		"def",
		"1",
		&queue.TaskClaimRequest{
			WorkerGroup: WorkerType,
			WorkerID:    WorkerID,
		},
	).Return(&queue.TaskClaimResponse{
		Credentials: struct {
			AccessToken string `json:"accessToken"`
			Certificate string `json:"certificate"`
			ClientID    string `json:"clientId"`
		}{
			AccessToken: "234aajsgfaj340",
			Certificate: "{}",
			ClientID:    "asfg089asgf08",
		},
		RunID:       1,
		Status:      queue.TaskStatusStructure{},
		TakenUntil:  tcclient.Time{},
		Task:        queue.TaskDefinitionResponse{},
		WorkerGroup: WorkerType,
		WorkerID:    WorkerID,
	}, nil)
	tasks := []*taskMessage{{
		TaskID:          "abc",
		RunID:           0,
		signedDeleteURL: fmt.Sprintf("%s/delete", s.URL),
	}, {
		TaskID:          "def",
		RunID:           1,
		signedDeleteURL: fmt.Sprintf("%s/delete", s.URL),
	}}

	logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
	service := queueService{
		client:        mockedQueue,
		capacity:      2,
		tc:            make(chan *taskClaim, 2),
		log:           logger.WithField("component", "Queue Service"),
		workerID:      WorkerID,
		workerGroup:   WorkerType,
		provisionerID: ProvisionerID,
	}

	claims := []*taskClaim{}
	service.claimTasks(tasks)
loop:
	for len(claims) != 2 {
		select {
		case t := <-service.tc:
			claims = append(claims, t)
		// Tasks should be claimed in less than 1 second, but set up a timer to timeout
		// so tests don't run forever waiting.
		case <-time.NewTimer(1 * time.Second).C:
			break loop
		}
	}

	assert.Equal(t, 2, len(claims), "Not enough task claims received after calling claimTasks")
}
func TestClaimTask(t *testing.T) {
	// Verifies that when claimTask is called in the queue service for a
	// particular task run object, that the task is claimed and deleted from
	// the azure queue.
	deleteCalled := false
	var handler = func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/delete" {
			deleteCalled = true
		}
		return
	}
	s := httptest.NewServer(http.HandlerFunc(handler))
	defer s.Close()

	mockedQueue := &client.MockQueue{}
	mockedQueue.On(
		"ClaimTask",
		"abc",
		"0",
		&queue.TaskClaimRequest{
			WorkerGroup: WorkerType,
			WorkerID:    WorkerID,
		},
	).Return(&queue.TaskClaimResponse{
		Credentials: struct {
			AccessToken string `json:"accessToken"`
			Certificate string `json:"certificate"`
			ClientID    string `json:"clientId"`
		}{
			AccessToken: "1040824383284384",
			Certificate: "{}",
			ClientID:    "ajafdsfkj23",
		},
		RunID:       0,
		Status:      queue.TaskStatusStructure{},
		TakenUntil:  tcclient.Time{},
		Task:        queue.TaskDefinitionResponse{},
		WorkerGroup: WorkerType,
		WorkerID:    WorkerID,
	}, nil)

	task := &taskMessage{
		TaskID:          "abc",
		RunID:           0,
		signedDeleteURL: fmt.Sprintf("%s/delete", s.URL),
	}

	logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
	service := queueService{
		client:        mockedQueue,
		log:           logger.WithField("component", "Queue Service"),
		workerID:      WorkerID,
		workerGroup:   WorkerType,
		provisionerID: ProvisionerID,
	}

	claim, err := service.claimTask(task)

	assert.Nil(t, err)
	assert.True(t, deleteCalled)
	// Do a quick sanity check to make sure the response was correctly stored in
	// the task run object
	assert.Equal(t, "1040824383284384", claim.taskClaim.Credentials.AccessToken)
}
func TestRetrieveTasksFromQueueDoesNotQueryLowPriority(t *testing.T) {
	// When enough tasks have been retrieved from the higher (first) priority queue,
	// the lower (second) priority queue should not be polled.
	messages := []string{
		// MessageText is {"taskId": "abc", "RunID": 1}
		`<?xml version="1.0" encoding="utf-8"?>
	<QueueMessagesList>
	  <QueueMessage>
		<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca3</MessageId>
		<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
		<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
		<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
		<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
		<DequeueCount>1</DequeueCount>
		<MessageText>eyJ0YXNrSWQiOiAiYWJjIiwgInJ1bklkIjogMX0=</MessageText>
	  </QueueMessage>
	</QueueMessagesList>`,
		// MessageText[0] {"taskId": "def", "RunID": 0}
		// MessageText[1] {"taskId": "ghi", "RunID": 2}
		`<?xml version="1.0" encoding="utf-8"?>
	<QueueMessagesList>
	  <QueueMessage>
		<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca3</MessageId>
		<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
		<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
		<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
		<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
		<DequeueCount>1</DequeueCount>
		<MessageText>eyJ0YXNrSWQiOiAiZGVmIiwgInJ1bklkIjogMH0NCg==</MessageText>
	  </QueueMessage>
	  <QueueMessage>
		<MessageId>5974b586-0df3-4e2d-ad0c-18e3892bfca3</MessageId>
		<InsertionTime>Fri, 09 Oct 2009 21:04:30 GMT</InsertionTime>
		<ExpirationTime>Fri, 16 Oct 2009 21:04:30 GMT</ExpirationTime>
		<PopReceipt>YzQ4Yzg1MDItYTc0Ny00OWNjLTkxYTUtZGM0MDFiZDAwYzEw</PopReceipt>
		<TimeNextVisible>Fri, 09 Oct 2009 23:29:20 GMT</TimeNextVisible>
		<DequeueCount>1</DequeueCount>
		<MessageText>eyJ0YXNrSWQiOiAiZ2hpIiwgInJ1bklkIjogMH0NCg==</MessageText>
	  </QueueMessage>
	</QueueMessagesList>`,
	}

	var handler = func(w http.ResponseWriter, r *http.Request) {
		var message string
		if r.URL.Path == "/tasks/1234" {
			message = messages[1]
		} else {
			message = messages[0]
		}
		w.Header().Set("Content-Type", "application/xml")
		w.Write([]byte(message))
	}

	s := httptest.NewServer(http.HandlerFunc(handler))
	defer s.Close()
	logger, _ := runtime.CreateLogger(os.Getenv("LOGGING_LEVEL"))
	service := queueService{
		log:              logger.WithField("component", "Queue Service"),
		expirationOffset: 300,
		expires:          tcclient.Time(time.Now().Add(time.Minute * 10)),
		queues: []messageQueue{
			{
				SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
				SignedPollURL:   fmt.Sprintf("%s/tasks/1234?messages=true", s.URL),
			},
			{
				SignedDeleteURL: fmt.Sprintf("%s/delete/{{messageId}}/{{popReceipt}}", s.URL),
				SignedPollURL:   fmt.Sprintf("%s/tasks/456?messages=true", s.URL),
			},
		},
	}

	tasks := service.retrieveTasksFromQueue(2)

	// Only two tasks should have been retrieving leaving the third task in the lower
	// priority queue not being retrieved
	assert.Equal(t, 2, len(tasks))
	taskIds := []string{"def", "ghi"}
	for i, v := range taskIds {
		assert.Equal(t, v, tasks[i].TaskID)
	}
}