func ManifestFromBytes(bytes []byte) (*Manifest, error) {
manifest := &Manifest{}
// Preserve the raw manifest so that manifest.Bytes() returns bytes in
// the same order that they were passed to this function
manifest.raw = make([]byte, len(bytes))
copy(manifest.raw, bytes)
signed, _ := clearsign.Decode(bytes)
if signed != nil {
signature, err := ioutil.ReadAll(signed.ArmoredSignature.Body)
if err != nil {
return nil, fmt.Errorf("Could not read signature from pod manifest: %s", err)
}
manifest.signature = signature
// the original plaintext is in signed.Plaintext, but the signature
// corresponds to signed.Bytes, so that's what we need to save
manifest.plaintext = signed.Bytes
// parse YAML from the message's plaintext instead
bytes = signed.Plaintext
}
if err := yaml.Unmarshal(bytes, manifest); err != nil {
return nil, fmt.Errorf("Could not read pod manifest: %s", err)
}
return manifest, nil
}
func (s *Signatory) decodeSignature(filename string) (*clearsign.Block, error) {
data, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
block, _ := clearsign.Decode(data)
if block == nil {
// There was no sig in the file.
return nil, errors.New("signature block not found")
}
return block, nil
}
func ParseOpenPGPParagraph(reader *bufio.Reader) (ret *Paragraph, ohshit error) {
els := ""
for {
line, err := reader.ReadString('\n')
if err == io.EOF {
break
}
els = els + line
}
block, _ := clearsign.Decode([]byte(els))
/**
* XXX: With the block, we need to validate everything.
*
* We need to hit openpgp.CheckDetachedSignature with block and
* a keyring. For now, it'll ignore all signature checking entirely.
*/
return ParseParagraph(bufio.NewReader(strings.NewReader(string(block.Bytes))))
}
// Internal method to read an OpenPGP Clearsigned document, store related
// OpenPGP information onto the shell Struct, and return any errors that
// we encounter along the way, such as an invalid signature, unknown
// signer, or incomplete document. If `keyring` is `nil`, checking of the
// signed data is *not* preformed.
func (p *ParagraphReader) decodeClearsig(keyring *openpgp.EntityList) error {
// One *massive* downside here is that the OpenPGP module in Go operates
// on byte arrays in memory, and *not* on Readers and Writers. This is a
// huge PITA because it does't need to be that way, and this forces
// clearsigned documents into memory. Which f*****g sucks. But here
// we are. It's likely worth a bug or two on this.
signedData, err := ioutil.ReadAll(p.reader)
if err != nil {
return err
}
block, _ := clearsign.Decode(signedData)
/* We're only interested in the first block. This may change in the
* future, in which case, we should likely set reader back to
* the remainder, and return that out to put through another
* ParagraphReader, since it may have a different signer. */
if keyring == nil {
/* As a special case, if the keyring is nil, we can go ahead
* and assume this data isn't intended to be checked against the
* keyring. So, we'll just pass on through. */
p.reader = bufio.NewReader(bytes.NewBuffer(block.Bytes))
return nil
}
/* Now, we have to go ahead and check that the signature is valid and
* relates to an entity we have in our keyring */
signer, err := openpgp.CheckDetachedSignature(
keyring,
bytes.NewReader(block.Bytes),
block.ArmoredSignature.Body,
)
if err != nil {
return err
}
p.signer = signer
p.reader = bufio.NewReader(bytes.NewBuffer(block.Bytes))
return nil
}
func decodeCheckSignature(r io.Reader, publicKey string) ([]byte, error) {
data, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
b, _ := clearsign.Decode(data)
if b == nil {
return nil, errors.New("no PGP signature embedded in plain text data")
}
keyring, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(publicKey))
if err != nil {
return nil, fmt.Errorf("failed to parse public key: %v", err)
}
_, err = openpgp.CheckDetachedSignature(keyring, bytes.NewBuffer(b.Bytes), b.ArmoredSignature.Body)
if err != nil {
return nil, err
}
return b.Plaintext, nil
}
func checkGPG(file *File) (state SigState, err error) {
var signer *openpgp.Entity
var cs *clearsign.Block
keypath := path.Join(os.Getenv("HOME"), "/.gnupg/pubring.gpg")
keys, err := os.Open(keypath)
if err != nil {
fmt.Printf("Could not open public keyring at %s\n", keypath)
os.Exit(2)
}
keyring, err := openpgp.ReadKeyRing(keys)
if err != nil {
fmt.Printf("Error reading public keyring: %s\n", err)
os.Exit(2)
}
if *flagClear {
cs, _ = clearsign.Decode(file.content)
if cs == nil {
fmt.Printf("Problem decoding clearsign signature from file %s\n", file.name)
os.Exit(2)
}
lsig, err := ioutil.ReadAll(cs.ArmoredSignature.Body)
if err != nil {
fmt.Printf("Problem reading signature from %s. Are you sure this file is clearsigned?: %s\n", file.name, err)
os.Exit(2)
}
if len(lsig) > 0 {
file.signature = lsig
file.content = cs.Bytes
*flagBin = true
}
}
if *flagBin {
signer, err = openpgp.CheckDetachedSignature(keyring, bytes.NewReader(file.content), bytes.NewReader(file.signature))
} else {
signer, err = openpgp.CheckArmoredDetachedSignature(keyring, bytes.NewReader(file.content), bytes.NewReader(file.signature))
}
if err != nil {
fmt.Printf("Invalid signature or public key not present: %s\n", err)
os.Exit(2)
}
state.sig = signer.PrimaryKey.KeyIdString()
l := len(*flagKeyid)
if l > 0 {
var rid string
// Force the local id to be all uppercase
lid := strings.ToUpper(*flagKeyid)
// check the number of chars on the remote id to see if it's a
// short or long id. If it's not 8 or 16, it's not valid.
switch l {
case 8:
fmt.Println("WARNING: The use of short ids is NOT secure. See https://evil32.com for more info.")
fmt.Println("Refusing to use insecure key mechanism. Exiting.")
os.Exit(1)
case 16:
fmt.Println("WARNING: The use of long ids is NOT considered to be secure. See https://evil32.com for more info.")
rid = signer.PrimaryKey.KeyIdString()
}
if len(rid) == 0 {
fmt.Printf("You did not specify a valid GPG keyid length. Must be 8 or 16 characters.")
os.Exit(2)
}
if lid != rid {
fmt.Printf("The remote file was not signed by the expected GPG Public key. Expected %s and got %s\n", lid, rid)
os.Exit(2)
}
}
// Due to how clearsign works, the detached signature has to be
// processed using the Bytes field, but the stripped content is located
// in the Plaintext field. As we've verified the signature was valid
// we can now fix the content
if *flagClear {
file.content = cs.Plaintext
}
state.success = true
return state, nil
}
func main() {
usage := `
Usage:
create-coreos-vdi [-V VERSION] [-p PATH]
Options:
-d DEST Create CoreOS VDI image to the given path.
-V VERSION Version to install (e.g. alpha) [default: stable]
-h This help
This tool creates a CoreOS VDI image to be used with VirtualBox.
`
arguments, _ := docopt.Parse(usage, nil, true, "Coreos create-coreos-vdi 0.1", false)
_, _ = get_vboxmanage()
RAW_IMAGE_NAME := "coreos_production_image.bin"
IMAGE_NAME := RAW_IMAGE_NAME + ".bz2"
DIGESTS_NAME := IMAGE_NAME + ".DIGESTS.asc"
VERSION_ID, _ := arguments["-V"].(string)
var BASE_URL string
switch VERSION_ID {
case "stable":
BASE_URL = "http://stable.release.core-os.net/amd64-usr/current"
case "alpha":
BASE_URL = "http://alpha.release.core-os.net/amd64-usr/current"
case "beta":
BASE_URL = "http://beta.release.core-os.net/amd64-usr/current"
default:
BASE_URL = fmt.Sprintf("http://storage.core-os.net/coreos/amd64-usr/%s", VERSION_ID)
}
dest, ok := arguments["-p"].(string)
if ok == false {
dest, _ = os.Getwd()
}
workdir, _ := ioutil.TempDir(dest, "coreos")
defer os.RemoveAll(workdir)
IMAGE_URL := fmt.Sprintf("%s/%s", BASE_URL, IMAGE_NAME)
DIGESTS_URL := fmt.Sprintf("%s/%s", BASE_URL, DIGESTS_NAME)
DOWN_IMAGE := filepath.Join(workdir, RAW_IMAGE_NAME)
var err error
_, err = http.Head(IMAGE_URL)
if err != nil {
log.Fatal("Image URL unavailable:" + IMAGE_URL)
}
digests_get_result, err := http.Get(DIGESTS_URL)
if err != nil {
log.Fatal("Image signature unavailable:" + DIGESTS_URL)
}
digests_raw_message, err := ioutil.ReadAll(digests_get_result.Body)
digests_get_result.Body.Close()
// Gets CoreOS verion from version.txt file
VERSION_NAME := "version.txt"
VERSION_URL := fmt.Sprintf("%s/%s", BASE_URL, VERSION_NAME)
version_result, err := http.Get(VERSION_URL)
vars, _ := ReadVars(version_result.Body)
VDI_IMAGE_NAME := fmt.Sprintf("coreos_production_%s.%s.%s.vdi", vars["COREOS_BUILD"], vars["COREOS_BRANCH"], vars["COREOS_PATCH"])
VDI_IMAGE := filepath.Join(dest, VDI_IMAGE_NAME)
decoded_long_id, err := hex.DecodeString(GPG_LONG_ID)
decoded_long_id_int := binary.BigEndian.Uint64(decoded_long_id)
fmt.Printf("Trusted hex key id %s is decimal %d\n", GPG_LONG_ID, decoded_long_id_int)
pubkey_get_result, err := http.Get(GPG_KEY_URL)
if err != nil {
log.Fatal(err)
}
pubkey, _ := ioutil.ReadAll(pubkey_get_result.Body)
pubkey_get_result.Body.Close()
pubkey_reader := bytes.NewReader(pubkey)
keyring, err := openpgp.ReadArmoredKeyRing(pubkey_reader)
if err != nil {
log.Fatal(err)
}
decoded_message, _ := clearsign.Decode(digests_raw_message)
digests_text := string(decoded_message.Bytes)
decoded_message_reader := bytes.NewReader(decoded_message.Bytes)
res, err := openpgp.CheckDetachedSignature(keyring, decoded_message_reader, decoded_message.ArmoredSignature.Body)
if err != nil {
fmt.Println("Signature check for DIGESTS failed.")
} else {
if res.PrimaryKey.KeyId == decoded_long_id_int {
fmt.Printf("Trusted key id %d matches keyid %d\n", decoded_long_id_int, decoded_long_id_int)
}
fmt.Printf("DIGESTS signature is valid: ")
}
var re = regexp.MustCompile(`(?m)(?P<method>(SHA1|SHA512)) HASH(?:\r?)\n(?P<hash>.[^\s]*)\s*(?P<file>[\w\d_\.]*)`)
var keymap map[string]int = make(map[string]int)
for index, name := range re.SubexpNames() {
keymap[name] = index
}
matches := re.FindAllStringSubmatch(digests_text, -1)
var bz_hash_sha1 string
var bz_hash_sha512 string
for _, match := range matches {
if match[keymap["file"]] == IMAGE_NAME {
if match[keymap["method"]] == "SHA1" {
bz_hash_sha1 = match[keymap["hash"]]
}
if match[keymap["method"]] == "SHA512" {
bz_hash_sha512 = match[keymap["hash"]]
}
}
}
sha1h := sha1.New()
sha512h := sha512.New()
fmt.Printf("downloading %s\n", IMAGE_NAME)
response, err := http.Get(IMAGE_URL)
defer response.Body.Close()
bar := pb.New(int(response.ContentLength)).SetUnits(pb.U_BYTES)
bar.Start()
bzipped, _ := os.Create(filepath.Join(workdir, IMAGE_NAME))
// create multi writer
writer := io.MultiWriter(sha1h, sha512h, bar, bzipped)
// and copy
io.Copy(writer, response.Body)
bar.FinishPrint("")
bzipped.Close()
if hex.EncodeToString(sha1h.Sum([]byte{})) == bz_hash_sha1 {
fmt.Printf("SHA1 hash for %s match the one from DIGESTS\n", IMAGE_NAME)
}
if hex.EncodeToString(sha512h.Sum([]byte{})) == bz_hash_sha512 {
fmt.Printf("SHA512 hash for %s match the one from DIGESTS\n", IMAGE_NAME)
}
fmt.Printf("Writing %s to %s...\n", IMAGE_NAME, RAW_IMAGE_NAME)
bzip2_file, err := os.Open(filepath.Join(workdir, IMAGE_NAME))
if err != nil {
log.Fatal(err)
}
bzip2_reader := bzip2.NewReader(bzip2_file)
bin_file, err := os.Create(DOWN_IMAGE)
if err != nil {
log.Fatal(err)
}
io.Copy(bin_file, bzip2_reader)
bin_file.Close()
cmd := exec.Command("vboxmanage", "convertdd", DOWN_IMAGE, VDI_IMAGE, "--format", "VDI")
if runtime.GOOS == "windows" {
// Use cmd /C on windows because LookPath is being difficult
cmd = exec.Command("cmd", "/C", "vboxmanage", "convertdd", DOWN_IMAGE, VDI_IMAGE, "--format", "VDI")
}
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
fmt.Printf("Converting %s to VirtualBox format...\n", RAW_IMAGE_NAME)
err = cmd.Run()
if err != nil {
log.Fatal(err)
}
fmt.Printf("Success! CoreOS %s VDI image was created on %s.", VERSION_ID, VDI_IMAGE_NAME)
}
func TestUpload(t *testing.T) {
setup(t)
expires := tcclient.Time(time.Now().Add(time.Minute * 30))
payload := GenericWorkerPayload{
Command: helloGoodbye(),
MaxRunTime: 7200,
Artifacts: []struct {
Expires tcclient.Time `json:"expires"`
Path string `json:"path"`
Type string `json:"type"`
}{
{
Path: "SampleArtifacts/_/X.txt",
Expires: expires,
Type: "file",
},
},
Features: struct {
ChainOfTrust bool `json:"chainOfTrust,omitempty"`
}{
ChainOfTrust: true,
},
}
td := testTask()
taskID, myQueue := submitTask(t, td, payload)
runWorker()
// some required substrings - not all, just a selection
expectedArtifacts := map[string]struct {
extracts []string
contentEncoding string
expires tcclient.Time
}{
"public/logs/live_backing.log": {
extracts: []string{
"hello world!",
"goodbye world!",
`"instance-type": "p3.enormous"`,
},
contentEncoding: "gzip",
expires: td.Expires,
},
"public/logs/live.log": {
extracts: []string{
"hello world!",
"goodbye world!",
"=== Task Finished ===",
"Exit Code: 0",
},
contentEncoding: "gzip",
expires: td.Expires,
},
"public/logs/certified.log": {
extracts: []string{
"hello world!",
"goodbye world!",
"=== Task Finished ===",
"Exit Code: 0",
},
contentEncoding: "gzip",
expires: td.Expires,
},
"public/logs/chainOfTrust.json.asc": {
// e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ./%%%/v/X
// 8308d593eb56527137532595a60255a3fcfbe4b6b068e29b22d99742bad80f6f ./_/X.txt
// a0ed21ab50992121f08da55365da0336062205fd6e7953dbff781a7de0d625b7 ./b/c/d.jpg
extracts: []string{
"8308d593eb56527137532595a60255a3fcfbe4b6b068e29b22d99742bad80f6f",
},
contentEncoding: "gzip",
expires: td.Expires,
},
"SampleArtifacts/_/X.txt": {
extracts: []string{
"test artifact",
},
contentEncoding: "",
expires: payload.Artifacts[0].Expires,
},
}
artifacts, err := myQueue.ListArtifacts(taskID, "0", "", "")
if err != nil {
t.Fatalf("Error listing artifacts: %v", err)
}
actualArtifacts := make(map[string]struct {
ContentType string `json:"contentType"`
Expires tcclient.Time `json:"expires"`
Name string `json:"name"`
StorageType string `json:"storageType"`
}, len(artifacts.Artifacts))
for _, actualArtifact := range artifacts.Artifacts {
actualArtifacts[actualArtifact.Name] = actualArtifact
}
for artifact := range expectedArtifacts {
if a, ok := actualArtifacts[artifact]; ok {
if a.ContentType != "text/plain; charset=utf-8" {
t.Errorf("Artifact %s should have mime type 'text/plain; charset=utf-8' but has '%s'", artifact, a.ContentType)
}
if a.Expires.String() != expectedArtifacts[artifact].expires.String() {
t.Errorf("Artifact %s should have expiry '%s' but has '%s'", artifact, expires, a.Expires)
}
} else {
t.Errorf("Artifact '%s' not created", artifact)
}
}
// now check content was uploaded to Amazon, and is correct
// signer of public/logs/chainOfTrust.json.asc
signer := &openpgp.Entity{}
cotCert := &ChainOfTrustData{}
for artifact, content := range expectedArtifacts {
url, err := myQueue.GetLatestArtifact_SignedURL(taskID, artifact, 10*time.Minute)
if err != nil {
t.Fatalf("Error trying to fetch artifacts from Amazon...\n%s", err)
}
// need to do this so Content-Encoding header isn't swallowed by Go for test later on
tr := &http.Transport{
DisableCompression: true,
}
client := &http.Client{Transport: tr}
rawResp, _, err := httpbackoff.ClientGet(client, url.String())
if err != nil {
t.Fatalf("Error trying to fetch decompressed artifact from signed URL %s ...\n%s", url.String(), err)
}
resp, _, err := httpbackoff.Get(url.String())
if err != nil {
t.Fatalf("Error trying to fetch artifact from signed URL %s ...\n%s", url.String(), err)
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatalf("Error trying to read response body of artifact from signed URL %s ...\n%s", url.String(), err)
}
for _, requiredSubstring := range content.extracts {
if strings.Index(string(b), requiredSubstring) < 0 {
t.Errorf("Artifact '%s': Could not find substring %q in '%s'", artifact, requiredSubstring, string(b))
}
}
if actualContentEncoding := rawResp.Header.Get("Content-Encoding"); actualContentEncoding != content.contentEncoding {
t.Fatalf("Expected Content-Encoding %q but got Content-Encoding %q for artifact %q from url %v", content.contentEncoding, actualContentEncoding, artifact, url)
}
if actualContentType := resp.Header.Get("Content-Type"); actualContentType != "text/plain; charset=utf-8" {
t.Fatalf("Content-Type in Signed URL response does not match Content-Type of artifact")
}
// check openpgp signature is valid
if artifact == "public/logs/chainOfTrust.json.asc" {
pubKey, err := os.Open(filepath.Join("testdata", "public-openpgp-key"))
if err != nil {
t.Fatalf("Error opening public key file")
}
defer pubKey.Close()
entityList, err := openpgp.ReadArmoredKeyRing(pubKey)
if err != nil {
t.Fatalf("Error decoding public key file")
}
block, _ := clearsign.Decode(b)
signer, err = openpgp.CheckDetachedSignature(entityList, bytes.NewBuffer(block.Bytes), block.ArmoredSignature.Body)
if err != nil {
t.Fatalf("Not able to validate openpgp signature of public/logs/chainOfTrust.json.asc")
}
err = json.Unmarshal(block.Plaintext, cotCert)
if err != nil {
t.Fatalf("Could not interpret public/logs/chainOfTrust.json as json")
}
}
}
if signer == nil {
t.Fatalf("Signer of public/logs/chainOfTrust.json.asc could not be established (is nil)")
}
if signer.Identities["Generic-Worker <*****@*****.**>"] == nil {
t.Fatalf("Did not get correct signer identity in public/logs/chainOfTrust.json.asc - %#v", signer.Identities)
}
// This trickery is to convert a TaskDefinitionResponse into a
// TaskDefinitionRequest in order that we can compare. We cannot cast, so
// need to transform to json as an intermediary step.
b, err := json.Marshal(cotCert.Task)
if err != nil {
t.Fatalf("Cannot marshal task into json - %#v\n%v", cotCert.Task, err)
}
cotCertTaskRequest := &queue.TaskDefinitionRequest{}
err = json.Unmarshal(b, cotCertTaskRequest)
if err != nil {
t.Fatalf("Cannot unmarshal json into task request - %#v\n%v", string(b), err)
}
// The Payload, Tags and Extra fields are raw bytes, so differences may not
// be valid. Since we are comparing the rest, let's skip these two fields,
// as the rest should give us good enough coverage already
cotCertTaskRequest.Payload = nil
cotCertTaskRequest.Tags = nil
cotCertTaskRequest.Extra = nil
td.Payload = nil
td.Tags = nil
td.Extra = nil
if !reflect.DeepEqual(cotCertTaskRequest, td) {
t.Fatalf("Did not get back expected task definition in chain of trust certificate:\n%#v\n ** vs **\n%#v", cotCertTaskRequest, td)
}
if len(cotCert.Artifacts) != 2 {
t.Fatalf("Expected 2 artifact hashes to be listed")
}
if cotCert.TaskID != taskID {
t.Fatalf("Expected taskId to be %q but was %q", taskID, cotCert.TaskID)
}
if cotCert.RunID != 0 {
t.Fatalf("Expected runId to be 0 but was %v", cotCert.RunID)
}
if cotCert.WorkerGroup != "test-worker-group" {
t.Fatalf("Expected workerGroup to be \"test-worker-group\" but was %q", cotCert.WorkerGroup)
}
if cotCert.WorkerID != "test-worker-id" {
t.Fatalf("Expected workerGroup to be \"test-worker-id\" but was %q", cotCert.WorkerID)
}
if cotCert.Environment.PublicIPAddress != "12.34.56.78" {
t.Fatalf("Expected publicIpAddress to be 12.34.56.78 but was %v", cotCert.Environment.PublicIPAddress)
}
if cotCert.Environment.PrivateIPAddress != "87.65.43.21" {
t.Fatalf("Expected privateIpAddress to be 87.65.43.21 but was %v", cotCert.Environment.PrivateIPAddress)
}
if cotCert.Environment.InstanceID != "test-instance-id" {
t.Fatalf("Expected instanceId to be \"test-instance-id\" but was %v", cotCert.Environment.InstanceID)
}
if cotCert.Environment.InstanceType != "p3.enormous" {
t.Fatalf("Expected instanceType to be \"p3.enormous\" but was %v", cotCert.Environment.InstanceType)
}
if cotCert.Environment.Region != "outer-space" {
t.Fatalf("Expected region to be \"outer-space\" but was %v", cotCert.Environment.Region)
}
}