// If artifact expires after task deadline, we should not get a Malformed Payload
func TestArtifactExpiresAfterDeadline(t *testing.T) {
now := time.Now()
task := taskWithPayload(`{
"env": {
"XPI_NAME": "dist/example_add-on-0.0.1.zip"
},
"maxRunTime": 3,
` + rawHelloGoodbye() + `,
"artifacts": [
{
"type": "file",
"path": "public/some/artifact",
"expires": "` + tcclient.Time(now.Add(time.Minute*10)).String() + `"
}
]
}`)
task.Definition.Deadline = tcclient.Time(now.Add(time.Minute * 5))
ensureValidPayload(t, task)
}
func (taskMount *TaskMount) purgeCaches() error {
// Don't bother to query purge cache service if this task uses no writable
// caches, and we queried already less than 6 hours ago. Service keeps
// history for 24 hours, but is an implementation detail that could change.
// Until we have a formal guarantee, let's run after 6 hours to be super
// safe.
writableCaches := []*WritableDirectoryCache{}
for _, mount := range taskMount.mounts {
switch t := mount.(type) {
case *WritableDirectoryCache:
writableCaches = append(writableCaches, t)
}
}
if len(writableCaches) == 0 && time.Now().Sub(lastQueriedPurgeCacheService) < 6*time.Hour {
return nil
}
// In case of clock drift, let's query all purge cache requests created
// since 5 mins before our last request. In the worst case, it means we'll
// get back more results than we need, but it won't cause us to clear
// caches we shouldn't. This helps if the worker clock is up to 5 minutes
// ahead of the purgecache service clock. If the worker clock is behind the
// purgecache service clock, that is also no problem. If this is the first
// request since the worker started, we won't pass in a "since" date at
// all.
since := ""
if !lastQueriedPurgeCacheService.IsZero() {
since = tcclient.Time(lastQueriedPurgeCacheService.Add(-5 * time.Minute)).String()
}
lastQueriedPurgeCacheService = time.Now()
purgeRequests, err := pc.PurgeRequests(config.ProvisionerID, config.WorkerType, since)
if err != nil {
return err
}
// Loop through results, and purge caches when we find an entry. Note,
// again to account for clock drift, let's remove caches up to 5 minutes
// older than the given "before" date.
for _, request := range purgeRequests.Requests {
if cache, exists := directoryCaches[request.CacheName]; exists {
if cache.Created.Add(-5 * time.Minute).Before(time.Time(request.Before)) {
err := cache.Expunge()
if err != nil {
panic(err)
}
}
}
}
return nil
}
// ExpectS3Artifact will setup queue to expect an S3 artifact with given
// name to be created for taskID and runID using m and returns
// a channel which will receive the artifact.
func (m *MockQueue) ExpectS3Artifact(taskID string, runID int, name string) <-chan []byte {
// make channel size 100 so we don't have to handle synchronously
c := make(chan []byte, 100)
var s *httptest.Server
s = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
d, err := ioutil.ReadAll(r.Body)
if err != nil {
close(c)
w.WriteHeader(500)
return
}
if r.Header.Get("Content-Encoding") == "gzip" {
reader, err := gzip.NewReader(bytes.NewReader(d))
if err != nil {
close(c)
w.WriteHeader(500)
return
}
d, err = ioutil.ReadAll(reader)
if err != nil {
close(c)
w.WriteHeader(500)
return
}
}
w.WriteHeader(200)
c <- d
go s.Close() // Close when all requests are done (don't block the request)
}))
data, _ := json.Marshal(queue.S3ArtifactResponse{
StorageType: "s3",
PutURL: s.URL,
ContentType: "application/octet",
Expires: tcclient.Time(time.Now().Add(30 * time.Minute)),
})
result := queue.PostArtifactResponse(data)
m.On(
"CreateArtifact",
taskID, fmt.Sprintf("%d", runID),
name, PostS3ArtifactRequest,
).Return(&result, nil)
return c
}
func main() {
sshSecret := make(map[string]string)
if len(os.Args) != 2 {
log.Fatal("Usage: " + os.Args[0] + " WORKER_TYPE_DIRECTORY")
}
workerTypeDir := os.Args[1]
absFile, err := filepath.Abs(workerTypeDir)
if err != nil {
log.Fatalf("File/directory '%v' could not be read due to '%s'", workerTypeDir, err)
}
files, err := ioutil.ReadDir(workerTypeDir)
if err != nil {
log.Fatalf("File/directory '%v' (%v) could not be read due to '%s'", workerTypeDir, absFile, err)
}
workerType := filepath.Base(absFile)
secretName := "project/taskcluster/aws-provisioner-v1/worker-types/ssh-keys/" + workerType
tcCreds := &tcclient.Credentials{
ClientID: os.Getenv("TASKCLUSTER_CLIENT_ID"),
AccessToken: os.Getenv("TASKCLUSTER_ACCESS_TOKEN"),
Certificate: os.Getenv("TASKCLUSTER_CERTIFICATE"),
}
cd := &tcclient.ConnectionData{
Credentials: tcCreds,
BaseURL: "https://aws-provisioner.taskcluster.net/v1",
Authenticate: true,
}
var wt map[string]interface{}
_, _, err = cd.APICall(nil, "GET", "/worker-type/"+url.QueryEscape(workerType), &wt, nil)
if err != nil {
log.Fatal(err)
}
regions := wt["regions"].([]interface{})
oldAMICount := 0
newAMICount := 0
delete(wt, "lastModified")
delete(wt, "workerType")
for _, f := range files {
if !f.IsDir() && strings.HasSuffix(f.Name(), ".id_rsa") {
region := f.Name()[:len(f.Name())-7]
bytes, err := ioutil.ReadFile(filepath.Join(workerTypeDir, f.Name()))
if err != nil {
log.Fatalf("Problem reading file %v", filepath.Join(workerTypeDir, f.Name()))
}
sshSecret[region] = string(bytes)
}
if !f.IsDir() && strings.HasSuffix(f.Name(), ".latest-ami") {
newAMICount++
tokens := strings.Split(f.Name(), ".")
region := tokens[0]
newAmi := tokens[1]
oldAmi := ""
for i := range regions {
regionObj := regions[i].(map[string]interface{})
if regionObj["region"] == region {
launchSpec := regionObj["launchSpec"].(map[string]interface{})
oldAmi = launchSpec["ImageId"].(string)
launchSpec["ImageId"] = newAmi
oldAMICount++
}
}
if newAMICount < oldAMICount {
log.Fatal(fmt.Errorf("Did not find ami specification in worker type %v for region %v", workerType, region))
}
if newAMICount > oldAMICount {
log.Fatal(fmt.Errorf("Found multiple AMIs in worker type %v for region %v", workerType, region))
}
log.Print("Old AMI for worker type " + workerType + " region " + region + ": " + oldAmi)
log.Print("New AMI for worker type " + workerType + " region " + region + ": " + newAmi)
if oldAmi == newAmi {
log.Print("WARNING: No change to AMI used in workert type " + workerType + " for region " + region + " (" + oldAmi + ")")
} else {
log.Print("Worker type " + workerType + " region " + region + " updated to use " + newAmi)
}
}
}
if newAMICount != len(regions) {
log.Printf("WARNING: not updating all AMIs for worker type %v - only %v of %v", workerType, newAMICount, len(regions))
}
mySecrets := secrets.New(tcCreds)
secBytes, err := json.Marshal(sshSecret)
if err != nil {
log.Fatalf("Could not convert secret %#v to json: %v", sshSecret, err)
}
err = mySecrets.Set(
secretName,
&secrets.Secret{
Expires: tcclient.Time(time.Now().AddDate(1, 0, 0)),
Secret: json.RawMessage(secBytes),
},
)
if err != nil {
log.Printf("Problem publishing new secrets: %v", err)
}
s, err := mySecrets.Get(secretName)
if err != nil {
log.Fatalf("Error retrieving secret: %v", err)
}
log.Print("Secret name: " + secretName)
log.Print("Secret value: " + string(s.Secret))
log.Print("Expiry: " + s.Expires.String())
_, _, err = cd.APICall(wt, "POST", "/worker-type/"+url.QueryEscape(workerType)+"/update", new(interface{}), nil)
if err != nil {
log.Fatal(err)
}
}
func TestUpload(t *testing.T) {
setup(t)
expires := tcclient.Time(time.Now().Add(time.Minute * 30))
payload := GenericWorkerPayload{
Command: helloGoodbye(),
MaxRunTime: 7200,
Artifacts: []struct {
Expires tcclient.Time `json:"expires"`
Path string `json:"path"`
Type string `json:"type"`
}{
{
Path: "SampleArtifacts/_/X.txt",
Expires: expires,
Type: "file",
},
},
Features: struct {
ChainOfTrust bool `json:"chainOfTrust,omitempty"`
}{
ChainOfTrust: true,
},
}
td := testTask()
taskID, myQueue := submitTask(t, td, payload)
runWorker()
// some required substrings - not all, just a selection
expectedArtifacts := map[string]struct {
extracts []string
contentEncoding string
expires tcclient.Time
}{
"public/logs/live_backing.log": {
extracts: []string{
"hello world!",
"goodbye world!",
`"instance-type": "p3.enormous"`,
},
contentEncoding: "gzip",
expires: td.Expires,
},
"public/logs/live.log": {
extracts: []string{
"hello world!",
"goodbye world!",
"=== Task Finished ===",
"Exit Code: 0",
},
contentEncoding: "gzip",
expires: td.Expires,
},
"public/logs/certified.log": {
extracts: []string{
"hello world!",
"goodbye world!",
"=== Task Finished ===",
"Exit Code: 0",
},
contentEncoding: "gzip",
expires: td.Expires,
},
"public/logs/chainOfTrust.json.asc": {
// e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ./%%%/v/X
// 8308d593eb56527137532595a60255a3fcfbe4b6b068e29b22d99742bad80f6f ./_/X.txt
// a0ed21ab50992121f08da55365da0336062205fd6e7953dbff781a7de0d625b7 ./b/c/d.jpg
extracts: []string{
"8308d593eb56527137532595a60255a3fcfbe4b6b068e29b22d99742bad80f6f",
},
contentEncoding: "gzip",
expires: td.Expires,
},
"SampleArtifacts/_/X.txt": {
extracts: []string{
"test artifact",
},
contentEncoding: "",
expires: payload.Artifacts[0].Expires,
},
}
artifacts, err := myQueue.ListArtifacts(taskID, "0", "", "")
if err != nil {
t.Fatalf("Error listing artifacts: %v", err)
}
actualArtifacts := make(map[string]struct {
ContentType string `json:"contentType"`
Expires tcclient.Time `json:"expires"`
Name string `json:"name"`
StorageType string `json:"storageType"`
}, len(artifacts.Artifacts))
for _, actualArtifact := range artifacts.Artifacts {
actualArtifacts[actualArtifact.Name] = actualArtifact
}
for artifact := range expectedArtifacts {
if a, ok := actualArtifacts[artifact]; ok {
if a.ContentType != "text/plain; charset=utf-8" {
t.Errorf("Artifact %s should have mime type 'text/plain; charset=utf-8' but has '%s'", artifact, a.ContentType)
}
if a.Expires.String() != expectedArtifacts[artifact].expires.String() {
t.Errorf("Artifact %s should have expiry '%s' but has '%s'", artifact, expires, a.Expires)
}
} else {
t.Errorf("Artifact '%s' not created", artifact)
}
}
// now check content was uploaded to Amazon, and is correct
// signer of public/logs/chainOfTrust.json.asc
signer := &openpgp.Entity{}
cotCert := &ChainOfTrustData{}
for artifact, content := range expectedArtifacts {
url, err := myQueue.GetLatestArtifact_SignedURL(taskID, artifact, 10*time.Minute)
if err != nil {
t.Fatalf("Error trying to fetch artifacts from Amazon...\n%s", err)
}
// need to do this so Content-Encoding header isn't swallowed by Go for test later on
tr := &http.Transport{
DisableCompression: true,
}
client := &http.Client{Transport: tr}
rawResp, _, err := httpbackoff.ClientGet(client, url.String())
if err != nil {
t.Fatalf("Error trying to fetch decompressed artifact from signed URL %s ...\n%s", url.String(), err)
}
resp, _, err := httpbackoff.Get(url.String())
if err != nil {
t.Fatalf("Error trying to fetch artifact from signed URL %s ...\n%s", url.String(), err)
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatalf("Error trying to read response body of artifact from signed URL %s ...\n%s", url.String(), err)
}
for _, requiredSubstring := range content.extracts {
if strings.Index(string(b), requiredSubstring) < 0 {
t.Errorf("Artifact '%s': Could not find substring %q in '%s'", artifact, requiredSubstring, string(b))
}
}
if actualContentEncoding := rawResp.Header.Get("Content-Encoding"); actualContentEncoding != content.contentEncoding {
t.Fatalf("Expected Content-Encoding %q but got Content-Encoding %q for artifact %q from url %v", content.contentEncoding, actualContentEncoding, artifact, url)
}
if actualContentType := resp.Header.Get("Content-Type"); actualContentType != "text/plain; charset=utf-8" {
t.Fatalf("Content-Type in Signed URL response does not match Content-Type of artifact")
}
// check openpgp signature is valid
if artifact == "public/logs/chainOfTrust.json.asc" {
pubKey, err := os.Open(filepath.Join("testdata", "public-openpgp-key"))
if err != nil {
t.Fatalf("Error opening public key file")
}
defer pubKey.Close()
entityList, err := openpgp.ReadArmoredKeyRing(pubKey)
if err != nil {
t.Fatalf("Error decoding public key file")
}
block, _ := clearsign.Decode(b)
signer, err = openpgp.CheckDetachedSignature(entityList, bytes.NewBuffer(block.Bytes), block.ArmoredSignature.Body)
if err != nil {
t.Fatalf("Not able to validate openpgp signature of public/logs/chainOfTrust.json.asc")
}
err = json.Unmarshal(block.Plaintext, cotCert)
if err != nil {
t.Fatalf("Could not interpret public/logs/chainOfTrust.json as json")
}
}
}
if signer == nil {
t.Fatalf("Signer of public/logs/chainOfTrust.json.asc could not be established (is nil)")
}
if signer.Identities["Generic-Worker <*****@*****.**>"] == nil {
t.Fatalf("Did not get correct signer identity in public/logs/chainOfTrust.json.asc - %#v", signer.Identities)
}
// This trickery is to convert a TaskDefinitionResponse into a
// TaskDefinitionRequest in order that we can compare. We cannot cast, so
// need to transform to json as an intermediary step.
b, err := json.Marshal(cotCert.Task)
if err != nil {
t.Fatalf("Cannot marshal task into json - %#v\n%v", cotCert.Task, err)
}
cotCertTaskRequest := &queue.TaskDefinitionRequest{}
err = json.Unmarshal(b, cotCertTaskRequest)
if err != nil {
t.Fatalf("Cannot unmarshal json into task request - %#v\n%v", string(b), err)
}
// The Payload, Tags and Extra fields are raw bytes, so differences may not
// be valid. Since we are comparing the rest, let's skip these two fields,
// as the rest should give us good enough coverage already
cotCertTaskRequest.Payload = nil
cotCertTaskRequest.Tags = nil
cotCertTaskRequest.Extra = nil
td.Payload = nil
td.Tags = nil
td.Extra = nil
if !reflect.DeepEqual(cotCertTaskRequest, td) {
t.Fatalf("Did not get back expected task definition in chain of trust certificate:\n%#v\n ** vs **\n%#v", cotCertTaskRequest, td)
}
if len(cotCert.Artifacts) != 2 {
t.Fatalf("Expected 2 artifact hashes to be listed")
}
if cotCert.TaskID != taskID {
t.Fatalf("Expected taskId to be %q but was %q", taskID, cotCert.TaskID)
}
if cotCert.RunID != 0 {
t.Fatalf("Expected runId to be 0 but was %v", cotCert.RunID)
}
if cotCert.WorkerGroup != "test-worker-group" {
t.Fatalf("Expected workerGroup to be \"test-worker-group\" but was %q", cotCert.WorkerGroup)
}
if cotCert.WorkerID != "test-worker-id" {
t.Fatalf("Expected workerGroup to be \"test-worker-id\" but was %q", cotCert.WorkerID)
}
if cotCert.Environment.PublicIPAddress != "12.34.56.78" {
t.Fatalf("Expected publicIpAddress to be 12.34.56.78 but was %v", cotCert.Environment.PublicIPAddress)
}
if cotCert.Environment.PrivateIPAddress != "87.65.43.21" {
t.Fatalf("Expected privateIpAddress to be 87.65.43.21 but was %v", cotCert.Environment.PrivateIPAddress)
}
if cotCert.Environment.InstanceID != "test-instance-id" {
t.Fatalf("Expected instanceId to be \"test-instance-id\" but was %v", cotCert.Environment.InstanceID)
}
if cotCert.Environment.InstanceType != "p3.enormous" {
t.Fatalf("Expected instanceType to be \"p3.enormous\" but was %v", cotCert.Environment.InstanceType)
}
if cotCert.Environment.Region != "outer-space" {
t.Fatalf("Expected region to be \"outer-space\" but was %v", cotCert.Environment.Region)
}
}