Beispiel #1
0
func testParse(t *testing.T, input []byte, expected, expectedPlaintext string) {
	b, rest := Decode(input)
	if b == nil {
		t.Fatal("failed to decode clearsign message")
	}
	if !bytes.Equal(rest, []byte("trailing")) {
		t.Errorf("unexpected remaining bytes returned: %s", string(rest))
	}
	if b.ArmoredSignature.Type != "PGP SIGNATURE" {
		t.Errorf("bad armor type, got:%s, want:PGP SIGNATURE", b.ArmoredSignature.Type)
	}
	if !bytes.Equal(b.Bytes, []byte(expected)) {
		t.Errorf("bad body, got:%x want:%x", b.Bytes, expected)
	}

	if !bytes.Equal(b.Plaintext, []byte(expectedPlaintext)) {
		t.Errorf("bad plaintext, got:%x want:%x", b.Plaintext, expectedPlaintext)
	}

	keyring, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(signingKey))
	if err != nil {
		t.Errorf("failed to parse public key: %s", err)
	}

	if _, err := openpgp.CheckDetachedSignature(keyring, bytes.NewBuffer(b.Bytes), b.ArmoredSignature.Body); err != nil {
		t.Errorf("failed to check signature: %s", err)
	}
}
Beispiel #2
0
// verifySignature verifies that the given block is validly signed, and returns the signer.
func (s *Signatory) verifySignature(block *clearsign.Block) (*openpgp.Entity, error) {
	return openpgp.CheckDetachedSignature(
		s.KeyRing,
		bytes.NewBuffer(block.Bytes),
		block.ArmoredSignature.Body,
	)
}
Beispiel #3
0
func checkSignature(ks *Keystore, prefix string, signed, signature io.ReadSeeker) (*openpgp.Entity, error) {
	acidentifier, err := types.NewACIdentifier(prefix)
	if err != nil {
		return nil, err
	}
	keyring, err := ks.loadKeyring(acidentifier.String())
	if err != nil {
		return nil, errwrap.Wrap(errors.New("keystore: error loading keyring"), err)
	}
	entities, err := openpgp.CheckArmoredDetachedSignature(keyring, signed, signature)
	if err == io.EOF {
		// When the signature is binary instead of armored, the error is io.EOF.
		// Let's try with binary signatures as well
		if _, err := signed.Seek(0, 0); err != nil {
			return nil, errwrap.Wrap(errors.New("error seeking ACI file"), err)
		}
		if _, err := signature.Seek(0, 0); err != nil {
			return nil, errwrap.Wrap(errors.New("error seeking signature file"), err)
		}
		entities, err = openpgp.CheckDetachedSignature(keyring, signed, signature)
	}
	if err == io.EOF {
		// otherwise, the client failure is just "EOF", which is not helpful
		return nil, fmt.Errorf("keystore: no valid signatures found in signature file")
	}
	return entities, err
}
func checkGPGSig(fileName string, sigFileName string) error {

	// Get a Reader for the signature file
	sigFile, err := os.Open(sigFileName)
	if err != nil {
		return err
	}
	defer sigFile.Close()

	// Get a Reader for the signature file
	file, err := os.Open(fileName)
	if err != nil {
		return err
	}
	defer file.Close()

	publicKeyBin, err := hex.DecodeString(publicKeyHex)
	if err != nil {
		return err
	}

	keyring, _ := openpgp.ReadKeyRing(bytes.NewReader(publicKeyBin))

	_, err = openpgp.CheckDetachedSignature(keyring, file, sigFile)

	return err
}
Beispiel #5
0
// GPGCheck validates the integrity of a RPM package file read from the given
// io.Reader. Public keys in the given keyring are used to validate the package
// signature.
//
// If validation succeeds, nil is returned. If validation fails,
// ErrGPGValidationFailed is returned.
//
// This function is an expensive operation which reads the entire package file.
func GPGCheck(r io.Reader, keyring openpgp.KeyRing) (string, error) {
	// read signature header
	sigheader, err := rpmReadSigHeader(r)
	if err != nil {
		return "", err
	}

	// get signature bytes
	var sigval []byte = nil
	for _, tag := range []int{RPMSIGTAG_PGP, RPMSIGTAG_PGP5, RPMSIGTAG_GPG} {
		if sigval = sigheader.Indexes.BytesByTag(tag); sigval != nil {
			break
		}
	}

	if sigval == nil {
		return "", fmt.Errorf("Package signature not found")
	}

	// check signature
	signer, err := openpgp.CheckDetachedSignature(keyring, r, bytes.NewReader(sigval))
	if err == errors.ErrUnknownIssuer {
		return "", ErrGPGValidationFailed
	} else if err != nil {
		return "", err
	}

	// get signer identity
	for id, _ := range signer.Identities {
		return id, nil
	}

	return "", fmt.Errorf("No identity found in public key")
}
Beispiel #6
0
// Internal method to read an OpenPGP Clearsigned document, store related
// OpenPGP information onto the shell Struct, and return any errors that
// we encounter along the way, such as an invalid signature, unknown
// signer, or incomplete document. If `keyring` is `nil`, checking of the
// signed data is *not* preformed.
func (p *ParagraphReader) decodeClearsig(keyring *openpgp.EntityList) error {
	// One *massive* downside here is that the OpenPGP module in Go operates
	// on byte arrays in memory, and *not* on Readers and Writers. This is a
	// huge PITA because it does't need to be that way, and this forces
	// clearsigned documents into memory. Which f*****g sucks. But here
	// we are. It's likely worth a bug or two on this.

	signedData, err := ioutil.ReadAll(p.reader)
	if err != nil {
		return err
	}

	block, _ := clearsign.Decode(signedData)
	/* We're only interested in the first block. This may change in the
	 * future, in which case, we should likely set reader back to
	 * the remainder, and return that out to put through another
	 * ParagraphReader, since it may have a different signer. */

	if keyring == nil {
		/* As a special case, if the keyring is nil, we can go ahead
		 * and assume this data isn't intended to be checked against the
		 * keyring. So, we'll just pass on through. */
		p.reader = bufio.NewReader(bytes.NewBuffer(block.Bytes))
		return nil
	}

	/* Now, we have to go ahead and check that the signature is valid and
	 * relates to an entity we have in our keyring */
	signer, err := openpgp.CheckDetachedSignature(
		keyring,
		bytes.NewReader(block.Bytes),
		block.ArmoredSignature.Body,
	)

	if err != nil {
		return err
	}

	p.signer = signer
	p.reader = bufio.NewReader(bytes.NewBuffer(block.Bytes))

	return nil
}
Beispiel #7
0
func TestSigning(t *testing.T) {
	keyring, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(signingKey))
	if err != nil {
		t.Errorf("failed to parse public key: %s", err)
	}

	for i, test := range signingTests {
		var buf bytes.Buffer

		plaintext, err := Encode(&buf, keyring[0].PrivateKey, nil)
		if err != nil {
			t.Errorf("#%d: error from Encode: %s", i, err)
			continue
		}
		if _, err := plaintext.Write([]byte(test.in)); err != nil {
			t.Errorf("#%d: error from Write: %s", i, err)
			continue
		}
		if err := plaintext.Close(); err != nil {
			t.Fatalf("#%d: error from Close: %s", i, err)
			continue
		}

		b, _ := Decode(buf.Bytes())
		if b == nil {
			t.Errorf("#%d: failed to decode clearsign message", i)
			continue
		}
		if !bytes.Equal(b.Bytes, []byte(test.signed)) {
			t.Errorf("#%d: bad result, got:%x, want:%x", i, b.Bytes, test.signed)
			continue
		}
		if !bytes.Equal(b.Plaintext, []byte(test.plaintext)) {
			t.Errorf("#%d: bad result, got:%x, want:%x", i, b.Plaintext, test.plaintext)
			continue
		}

		if _, err := openpgp.CheckDetachedSignature(keyring, bytes.NewBuffer(b.Bytes), b.ArmoredSignature.Body); err != nil {
			t.Errorf("#%d: failed to check signature: %s", i, err)
		}
	}
}
Beispiel #8
0
func decodeCheckSignature(r io.Reader, publicKey string) ([]byte, error) {
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}
	b, _ := clearsign.Decode(data)
	if b == nil {
		return nil, errors.New("no PGP signature embedded in plain text data")
	}
	keyring, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(publicKey))
	if err != nil {
		return nil, fmt.Errorf("failed to parse public key: %v", err)
	}

	_, err = openpgp.CheckDetachedSignature(keyring, bytes.NewBuffer(b.Bytes), b.ArmoredSignature.Body)
	if err != nil {
		return nil, err
	}
	return b.Plaintext, nil
}
Beispiel #9
0
// Wrapper around openpgp.CheckDetachedSignature() that standardizes
// the error messages.
func checkDetachedSignature(
	keyring openpgp.KeyRing,
	signed []byte,
	signature []byte,
) (*openpgp.Entity, error) {
	signer, err := openpgp.CheckDetachedSignature(
		keyring,
		bytes.NewReader(signed),
		bytes.NewReader(signature),
	)
	if err == errors.ErrUnknownIssuer {
		keyId, err := signerKeyId(signature)
		if err != nil {
			return nil, Error{util.Errorf("error validating signature: %s", err), nil}
		}
		return nil, Error{util.Errorf("unknown signer: %X", keyId), nil}
	}
	if err != nil {
		return nil, Error{util.Errorf("error validating signature: %s", err), nil}
	}
	return signer, nil
}
Beispiel #10
0
func (s *Signature) Verify() error {
	signed, err := s.script.Body()
	if err != nil {
		return err
	}
	defer signed.Close()

	signature, err := s.Body()
	if err != nil {
		return err
	}
	defer signature.Close()

	if _, err := openpgp.CheckDetachedSignature(s.key, signed, signature); err == nil {
		return nil
	}

	signature.Seek(0, 0) // i'm sure there's a good reason i don't need to reset the script...
	if _, err := openpgp.CheckArmoredDetachedSignature(s.key, signed, signature); err == nil {
		return nil
	}

	return errors.New("Failed to verify signature")
}
Beispiel #11
0
// Verify() checks the validity of a signature for some data,
// and returns a boolean set to true if valid and an OpenPGP Entity
func Verify(data string, signature string, keyring io.Reader) (valid bool, entity *openpgp.Entity, err error) {
	valid = false

	// re-armor signature and transform into io.Reader
	sig := reArmorSignature(signature)
	sigReader := strings.NewReader(sig)

	// decode armor
	sigBlock, err := armor.Decode(sigReader)
	if err != nil {
		panic(err)
	}
	if sigBlock.Type != "PGP SIGNATURE" {
		err = fmt.Errorf("Wrong signature type '%s'", sigBlock.Type)
		panic(err)
	}

	// convert to io.Reader
	srcReader := strings.NewReader(data)

	// open the keyring
	ring, err := openpgp.ReadKeyRing(keyring)
	if err != nil {
		panic(err)
	}

	entity, err = openpgp.CheckDetachedSignature(ring, srcReader, sigBlock.Body)
	if err != nil {
		panic(err)
	}

	// we passed, signature is valid
	valid = true

	return
}
Beispiel #12
0
//codes as below are implemented to support ACI storage
func VerifyAciSignature(acipath, signpath, pubkeyspath string) error {
	files, err := ioutil.ReadDir(pubkeyspath)
	if err != nil {
		return fmt.Errorf("Read pubkeys directory failed: %v", err.Error())
	}

	if len(files) <= 0 {
		return fmt.Errorf("No pubkey file found in %v", pubkeyspath)
	}

	var keyring openpgp.EntityList
	for _, file := range files {
		pubkeyfile, err := os.Open(pubkeyspath + "/" + file.Name())
		if err != nil {
			return err
		}
		defer pubkeyfile.Close()

		keyList, err := openpgp.ReadArmoredKeyRing(pubkeyfile)
		if err != nil {
			return err
		}

		if len(keyList) < 1 {
			return fmt.Errorf("Missing opengpg entity")
		}

		keyring = append(keyring, keyList[0])
	}

	acifile, err := os.Open(acipath)
	if err != nil {
		return fmt.Errorf("Open ACI file failed: %v", err.Error())
	}
	defer acifile.Close()

	signfile, err := os.Open(signpath)
	if err != nil {
		return fmt.Errorf("Open signature file failed: %v", err.Error())
	}
	defer signfile.Close()

	if _, err := acifile.Seek(0, 0); err != nil {
		return fmt.Errorf("Seek ACI file failed: %v", err)
	}
	if _, err := signfile.Seek(0, 0); err != nil {
		return fmt.Errorf("Seek signature file: %v", err)
	}

	//Verify detached signature which default is ASCII format
	_, err = openpgp.CheckArmoredDetachedSignature(keyring, acifile, signfile)
	if err == io.EOF {
		if _, err := acifile.Seek(0, 0); err != nil {
			return fmt.Errorf("Seek ACI file failed: %v", err)
		}
		if _, err := signfile.Seek(0, 0); err != nil {
			return fmt.Errorf("Seek signature file: %v", err)
		}

		//try to verify detached signature with binary format
		_, err = openpgp.CheckDetachedSignature(keyring, acifile, signfile)
	}
	if err == io.EOF {
		return fmt.Errorf("Signature format is invalid")
	}

	return err
}
Beispiel #13
0
func checkGPG(file *File) (state SigState, err error) {
	var signer *openpgp.Entity
	var cs *clearsign.Block
	keypath := path.Join(os.Getenv("HOME"), "/.gnupg/pubring.gpg")
	keys, err := os.Open(keypath)
	if err != nil {
		fmt.Printf("Could not open public keyring at %s\n", keypath)
		os.Exit(2)
	}

	keyring, err := openpgp.ReadKeyRing(keys)

	if err != nil {
		fmt.Printf("Error reading public keyring: %s\n", err)
		os.Exit(2)
	}

	if *flagClear {
		cs, _ = clearsign.Decode(file.content)
		if cs == nil {
			fmt.Printf("Problem decoding clearsign signature from file %s\n", file.name)
			os.Exit(2)
		}

		lsig, err := ioutil.ReadAll(cs.ArmoredSignature.Body)

		if err != nil {
			fmt.Printf("Problem reading signature from %s.  Are you sure this file is clearsigned?: %s\n", file.name, err)
			os.Exit(2)
		}
		if len(lsig) > 0 {
			file.signature = lsig
			file.content = cs.Bytes
			*flagBin = true
		}

	}

	if *flagBin {
		signer, err = openpgp.CheckDetachedSignature(keyring, bytes.NewReader(file.content), bytes.NewReader(file.signature))
	} else {
		signer, err = openpgp.CheckArmoredDetachedSignature(keyring, bytes.NewReader(file.content), bytes.NewReader(file.signature))
	}
	if err != nil {
		fmt.Printf("Invalid signature or public key not present: %s\n", err)
		os.Exit(2)
	}

	state.sig = signer.PrimaryKey.KeyIdString()

	l := len(*flagKeyid)
	if l > 0 {
		var rid string

		// Force the local id to be all uppercase
		lid := strings.ToUpper(*flagKeyid)

		// check the number of chars on the remote id to see if it's a
		// short or long id. If it's not 8 or 16, it's not valid.
		switch l {
		case 8:
			fmt.Println("WARNING: The use of short ids is NOT secure.  See https://evil32.com for more info.")
			fmt.Println("Refusing to use insecure key mechanism. Exiting.")
			os.Exit(1)
		case 16:
			fmt.Println("WARNING: The use of long ids is NOT considered to be secure.  See https://evil32.com for more info.")
			rid = signer.PrimaryKey.KeyIdString()
		}
		if len(rid) == 0 {
			fmt.Printf("You did not specify a valid GPG keyid length. Must be 8 or 16 characters.")
			os.Exit(2)
		}

		if lid != rid {
			fmt.Printf("The remote file was not signed by the expected GPG Public key. Expected %s and got %s\n", lid, rid)
			os.Exit(2)
		}
	}

	// Due to how clearsign works, the detached signature has to be
	// processed using the Bytes field, but the stripped content is located
	// in the Plaintext field. As we've verified the signature was valid
	// we can now fix the content

	if *flagClear {
		file.content = cs.Plaintext
	}

	state.success = true
	return state, nil
}
func main() {
	usage := `
 
Usage:
    create-coreos-vdi [-V VERSION] [-p PATH]
    
Options:
    -d DEST     Create CoreOS VDI image to the given path.
    -V VERSION  Version to install (e.g. alpha) [default: stable]
    -h          This help

This tool creates a CoreOS VDI image to be used with VirtualBox.
`

	arguments, _ := docopt.Parse(usage, nil, true, "Coreos create-coreos-vdi 0.1", false)

	_, _ = get_vboxmanage()

	RAW_IMAGE_NAME := "coreos_production_image.bin"
	IMAGE_NAME := RAW_IMAGE_NAME + ".bz2"
	DIGESTS_NAME := IMAGE_NAME + ".DIGESTS.asc"

	VERSION_ID, _ := arguments["-V"].(string)

	var BASE_URL string

	switch VERSION_ID {
	case "stable":
		BASE_URL = "http://stable.release.core-os.net/amd64-usr/current"
	case "alpha":
		BASE_URL = "http://alpha.release.core-os.net/amd64-usr/current"
	case "beta":
		BASE_URL = "http://beta.release.core-os.net/amd64-usr/current"
	default:
		BASE_URL = fmt.Sprintf("http://storage.core-os.net/coreos/amd64-usr/%s", VERSION_ID)
	}

	dest, ok := arguments["-p"].(string)
	if ok == false {
		dest, _ = os.Getwd()
	}

	workdir, _ := ioutil.TempDir(dest, "coreos")
	defer os.RemoveAll(workdir)

	IMAGE_URL := fmt.Sprintf("%s/%s", BASE_URL, IMAGE_NAME)
	DIGESTS_URL := fmt.Sprintf("%s/%s", BASE_URL, DIGESTS_NAME)
	DOWN_IMAGE := filepath.Join(workdir, RAW_IMAGE_NAME)

	var err error

	_, err = http.Head(IMAGE_URL)
	if err != nil {
		log.Fatal("Image URL unavailable:" + IMAGE_URL)
	}

	digests_get_result, err := http.Get(DIGESTS_URL)
	if err != nil {
		log.Fatal("Image signature unavailable:" + DIGESTS_URL)
	}

	digests_raw_message, err := ioutil.ReadAll(digests_get_result.Body)
	digests_get_result.Body.Close()

	// Gets CoreOS verion from version.txt file
	VERSION_NAME := "version.txt"
	VERSION_URL := fmt.Sprintf("%s/%s", BASE_URL, VERSION_NAME)

	version_result, err := http.Get(VERSION_URL)
	vars, _ := ReadVars(version_result.Body)
	VDI_IMAGE_NAME := fmt.Sprintf("coreos_production_%s.%s.%s.vdi", vars["COREOS_BUILD"], vars["COREOS_BRANCH"], vars["COREOS_PATCH"])
	VDI_IMAGE := filepath.Join(dest, VDI_IMAGE_NAME)

	decoded_long_id, err := hex.DecodeString(GPG_LONG_ID)
	decoded_long_id_int := binary.BigEndian.Uint64(decoded_long_id)

	fmt.Printf("Trusted hex key id %s is decimal %d\n", GPG_LONG_ID, decoded_long_id_int)

	pubkey_get_result, err := http.Get(GPG_KEY_URL)
	if err != nil {
		log.Fatal(err)
	}

	pubkey, _ := ioutil.ReadAll(pubkey_get_result.Body)
	pubkey_get_result.Body.Close()

	pubkey_reader := bytes.NewReader(pubkey)
	keyring, err := openpgp.ReadArmoredKeyRing(pubkey_reader)
	if err != nil {
		log.Fatal(err)
	}

	decoded_message, _ := clearsign.Decode(digests_raw_message)
	digests_text := string(decoded_message.Bytes)
	decoded_message_reader := bytes.NewReader(decoded_message.Bytes)

	res, err := openpgp.CheckDetachedSignature(keyring, decoded_message_reader, decoded_message.ArmoredSignature.Body)
	if err != nil {
		fmt.Println("Signature check for DIGESTS failed.")
	} else {
		if res.PrimaryKey.KeyId == decoded_long_id_int {
			fmt.Printf("Trusted key id %d matches keyid %d\n", decoded_long_id_int, decoded_long_id_int)
		}
		fmt.Printf("DIGESTS signature is valid: ")
	}

	var re = regexp.MustCompile(`(?m)(?P<method>(SHA1|SHA512)) HASH(?:\r?)\n(?P<hash>.[^\s]*)\s*(?P<file>[\w\d_\.]*)`)

	var keymap map[string]int = make(map[string]int)
	for index, name := range re.SubexpNames() {
		keymap[name] = index
	}

	matches := re.FindAllStringSubmatch(digests_text, -1)

	var bz_hash_sha1 string
	var bz_hash_sha512 string

	for _, match := range matches {
		if match[keymap["file"]] == IMAGE_NAME {
			if match[keymap["method"]] == "SHA1" {
				bz_hash_sha1 = match[keymap["hash"]]
			}
			if match[keymap["method"]] == "SHA512" {
				bz_hash_sha512 = match[keymap["hash"]]
			}
		}
	}

	sha1h := sha1.New()
	sha512h := sha512.New()

	fmt.Printf("downloading %s\n", IMAGE_NAME)

	response, err := http.Get(IMAGE_URL)
	defer response.Body.Close()

	bar := pb.New(int(response.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()

	bzipped, _ := os.Create(filepath.Join(workdir, IMAGE_NAME))

	// create multi writer
	writer := io.MultiWriter(sha1h, sha512h, bar, bzipped)

	// and copy
	io.Copy(writer, response.Body)

	bar.FinishPrint("")

	bzipped.Close()

	if hex.EncodeToString(sha1h.Sum([]byte{})) == bz_hash_sha1 {
		fmt.Printf("SHA1 hash for %s match the one from DIGESTS\n", IMAGE_NAME)
	}
	if hex.EncodeToString(sha512h.Sum([]byte{})) == bz_hash_sha512 {
		fmt.Printf("SHA512 hash for %s match the one from DIGESTS\n", IMAGE_NAME)
	}

	fmt.Printf("Writing %s to %s...\n", IMAGE_NAME, RAW_IMAGE_NAME)

	bzip2_file, err := os.Open(filepath.Join(workdir, IMAGE_NAME))
	if err != nil {
		log.Fatal(err)
	}
	bzip2_reader := bzip2.NewReader(bzip2_file)

	bin_file, err := os.Create(DOWN_IMAGE)
	if err != nil {
		log.Fatal(err)
	}

	io.Copy(bin_file, bzip2_reader)
	bin_file.Close()

	cmd := exec.Command("vboxmanage", "convertdd", DOWN_IMAGE, VDI_IMAGE, "--format", "VDI")

	if runtime.GOOS == "windows" {
		// Use cmd /C on windows because LookPath is being difficult
		cmd = exec.Command("cmd", "/C", "vboxmanage", "convertdd", DOWN_IMAGE, VDI_IMAGE, "--format", "VDI")
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	fmt.Printf("Converting %s to VirtualBox format...\n", RAW_IMAGE_NAME)
	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("Success! CoreOS %s VDI image was created on %s.", VERSION_ID, VDI_IMAGE_NAME)
}
func TestUpload(t *testing.T) {

	setup(t)

	expires := tcclient.Time(time.Now().Add(time.Minute * 30))

	payload := GenericWorkerPayload{
		Command:    helloGoodbye(),
		MaxRunTime: 7200,
		Artifacts: []struct {
			Expires tcclient.Time `json:"expires"`
			Path    string        `json:"path"`
			Type    string        `json:"type"`
		}{
			{
				Path:    "SampleArtifacts/_/X.txt",
				Expires: expires,
				Type:    "file",
			},
		},
		Features: struct {
			ChainOfTrust bool `json:"chainOfTrust,omitempty"`
		}{
			ChainOfTrust: true,
		},
	}

	td := testTask()

	taskID, myQueue := submitTask(t, td, payload)
	runWorker()

	// some required substrings - not all, just a selection
	expectedArtifacts := map[string]struct {
		extracts        []string
		contentEncoding string
		expires         tcclient.Time
	}{
		"public/logs/live_backing.log": {
			extracts: []string{
				"hello world!",
				"goodbye world!",
				`"instance-type": "p3.enormous"`,
			},
			contentEncoding: "gzip",
			expires:         td.Expires,
		},
		"public/logs/live.log": {
			extracts: []string{
				"hello world!",
				"goodbye world!",
				"=== Task Finished ===",
				"Exit Code: 0",
			},
			contentEncoding: "gzip",
			expires:         td.Expires,
		},
		"public/logs/certified.log": {
			extracts: []string{
				"hello world!",
				"goodbye world!",
				"=== Task Finished ===",
				"Exit Code: 0",
			},
			contentEncoding: "gzip",
			expires:         td.Expires,
		},
		"public/logs/chainOfTrust.json.asc": {
			// e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  ./%%%/v/X
			// 8308d593eb56527137532595a60255a3fcfbe4b6b068e29b22d99742bad80f6f  ./_/X.txt
			// a0ed21ab50992121f08da55365da0336062205fd6e7953dbff781a7de0d625b7  ./b/c/d.jpg
			extracts: []string{
				"8308d593eb56527137532595a60255a3fcfbe4b6b068e29b22d99742bad80f6f",
			},
			contentEncoding: "gzip",
			expires:         td.Expires,
		},
		"SampleArtifacts/_/X.txt": {
			extracts: []string{
				"test artifact",
			},
			contentEncoding: "",
			expires:         payload.Artifacts[0].Expires,
		},
	}

	artifacts, err := myQueue.ListArtifacts(taskID, "0", "", "")

	if err != nil {
		t.Fatalf("Error listing artifacts: %v", err)
	}

	actualArtifacts := make(map[string]struct {
		ContentType string        `json:"contentType"`
		Expires     tcclient.Time `json:"expires"`
		Name        string        `json:"name"`
		StorageType string        `json:"storageType"`
	}, len(artifacts.Artifacts))

	for _, actualArtifact := range artifacts.Artifacts {
		actualArtifacts[actualArtifact.Name] = actualArtifact
	}

	for artifact := range expectedArtifacts {
		if a, ok := actualArtifacts[artifact]; ok {
			if a.ContentType != "text/plain; charset=utf-8" {
				t.Errorf("Artifact %s should have mime type 'text/plain; charset=utf-8' but has '%s'", artifact, a.ContentType)
			}
			if a.Expires.String() != expectedArtifacts[artifact].expires.String() {
				t.Errorf("Artifact %s should have expiry '%s' but has '%s'", artifact, expires, a.Expires)
			}
		} else {
			t.Errorf("Artifact '%s' not created", artifact)
		}
	}

	// now check content was uploaded to Amazon, and is correct

	// signer of public/logs/chainOfTrust.json.asc
	signer := &openpgp.Entity{}
	cotCert := &ChainOfTrustData{}

	for artifact, content := range expectedArtifacts {
		url, err := myQueue.GetLatestArtifact_SignedURL(taskID, artifact, 10*time.Minute)
		if err != nil {
			t.Fatalf("Error trying to fetch artifacts from Amazon...\n%s", err)
		}
		// need to do this so Content-Encoding header isn't swallowed by Go for test later on
		tr := &http.Transport{
			DisableCompression: true,
		}
		client := &http.Client{Transport: tr}
		rawResp, _, err := httpbackoff.ClientGet(client, url.String())
		if err != nil {
			t.Fatalf("Error trying to fetch decompressed artifact from signed URL %s ...\n%s", url.String(), err)
		}
		resp, _, err := httpbackoff.Get(url.String())
		if err != nil {
			t.Fatalf("Error trying to fetch artifact from signed URL %s ...\n%s", url.String(), err)
		}
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Fatalf("Error trying to read response body of artifact from signed URL %s ...\n%s", url.String(), err)
		}
		for _, requiredSubstring := range content.extracts {
			if strings.Index(string(b), requiredSubstring) < 0 {
				t.Errorf("Artifact '%s': Could not find substring %q in '%s'", artifact, requiredSubstring, string(b))
			}
		}
		if actualContentEncoding := rawResp.Header.Get("Content-Encoding"); actualContentEncoding != content.contentEncoding {
			t.Fatalf("Expected Content-Encoding %q but got Content-Encoding %q for artifact %q from url %v", content.contentEncoding, actualContentEncoding, artifact, url)
		}
		if actualContentType := resp.Header.Get("Content-Type"); actualContentType != "text/plain; charset=utf-8" {
			t.Fatalf("Content-Type in Signed URL response does not match Content-Type of artifact")
		}
		// check openpgp signature is valid
		if artifact == "public/logs/chainOfTrust.json.asc" {
			pubKey, err := os.Open(filepath.Join("testdata", "public-openpgp-key"))
			if err != nil {
				t.Fatalf("Error opening public key file")
			}
			defer pubKey.Close()
			entityList, err := openpgp.ReadArmoredKeyRing(pubKey)
			if err != nil {
				t.Fatalf("Error decoding public key file")
			}
			block, _ := clearsign.Decode(b)
			signer, err = openpgp.CheckDetachedSignature(entityList, bytes.NewBuffer(block.Bytes), block.ArmoredSignature.Body)
			if err != nil {
				t.Fatalf("Not able to validate openpgp signature of public/logs/chainOfTrust.json.asc")
			}
			err = json.Unmarshal(block.Plaintext, cotCert)
			if err != nil {
				t.Fatalf("Could not interpret public/logs/chainOfTrust.json as json")
			}
		}
	}
	if signer == nil {
		t.Fatalf("Signer of public/logs/chainOfTrust.json.asc could not be established (is nil)")
	}
	if signer.Identities["Generic-Worker <*****@*****.**>"] == nil {
		t.Fatalf("Did not get correct signer identity in public/logs/chainOfTrust.json.asc - %#v", signer.Identities)
	}

	// This trickery is to convert a TaskDefinitionResponse into a
	// TaskDefinitionRequest in order that we can compare. We cannot cast, so
	// need to transform to json as an intermediary step.
	b, err := json.Marshal(cotCert.Task)
	if err != nil {
		t.Fatalf("Cannot marshal task into json - %#v\n%v", cotCert.Task, err)
	}
	cotCertTaskRequest := &queue.TaskDefinitionRequest{}
	err = json.Unmarshal(b, cotCertTaskRequest)
	if err != nil {
		t.Fatalf("Cannot unmarshal json into task request - %#v\n%v", string(b), err)
	}

	// The Payload, Tags and Extra fields are raw bytes, so differences may not
	// be valid. Since we are comparing the rest, let's skip these two fields,
	// as the rest should give us good enough coverage already
	cotCertTaskRequest.Payload = nil
	cotCertTaskRequest.Tags = nil
	cotCertTaskRequest.Extra = nil
	td.Payload = nil
	td.Tags = nil
	td.Extra = nil
	if !reflect.DeepEqual(cotCertTaskRequest, td) {
		t.Fatalf("Did not get back expected task definition in chain of trust certificate:\n%#v\n ** vs **\n%#v", cotCertTaskRequest, td)
	}
	if len(cotCert.Artifacts) != 2 {
		t.Fatalf("Expected 2 artifact hashes to be listed")
	}
	if cotCert.TaskID != taskID {
		t.Fatalf("Expected taskId to be %q but was %q", taskID, cotCert.TaskID)
	}
	if cotCert.RunID != 0 {
		t.Fatalf("Expected runId to be 0 but was %v", cotCert.RunID)
	}
	if cotCert.WorkerGroup != "test-worker-group" {
		t.Fatalf("Expected workerGroup to be \"test-worker-group\" but was %q", cotCert.WorkerGroup)
	}
	if cotCert.WorkerID != "test-worker-id" {
		t.Fatalf("Expected workerGroup to be \"test-worker-id\" but was %q", cotCert.WorkerID)
	}
	if cotCert.Environment.PublicIPAddress != "12.34.56.78" {
		t.Fatalf("Expected publicIpAddress to be 12.34.56.78 but was %v", cotCert.Environment.PublicIPAddress)
	}
	if cotCert.Environment.PrivateIPAddress != "87.65.43.21" {
		t.Fatalf("Expected privateIpAddress to be 87.65.43.21 but was %v", cotCert.Environment.PrivateIPAddress)
	}
	if cotCert.Environment.InstanceID != "test-instance-id" {
		t.Fatalf("Expected instanceId to be \"test-instance-id\" but was %v", cotCert.Environment.InstanceID)
	}
	if cotCert.Environment.InstanceType != "p3.enormous" {
		t.Fatalf("Expected instanceType to be \"p3.enormous\" but was %v", cotCert.Environment.InstanceType)
	}
	if cotCert.Environment.Region != "outer-space" {
		t.Fatalf("Expected region to be \"outer-space\" but was %v", cotCert.Environment.Region)
	}
}