Exemplo n.º 1
0
func (b *s3Backend) Put(tx *postgres.DBTx, info FileInfo, r io.Reader, append bool) error {
	if append {
		// This is a hack, the next easiest thing to do if we need to handle
		// upload resumption is to finalize the multipart upload when the client
		// disconnects and when the rest of the data arrives, start a new
		// multi-part upload copying the existing object as the first part
		// (which is supported by S3 as a specific API call). This requires
		// replacing the simple uploader, so it was not done in the first pass.
		existing, err := b.Open(tx, info, false)
		if err != nil {
			return err
		}
		r = io.MultiReader(existing, r)
	}

	info.ExternalID = random.UUID()
	if err := tx.Exec("UPDATE files SET external_id = $2 WHERE file_id = $1", info.ID, info.ExternalID); err != nil {
		return err
	}

	u := s3manager.NewUploaderWithClient(b.client)
	_, err := u.Upload(&s3manager.UploadInput{
		Bucket:      &b.bucket,
		Key:         &info.ExternalID,
		ContentType: &info.Type,
		Body:        r,
	})

	return err
}
Exemplo n.º 2
0
func (b *gcsBackend) Copy(tx *postgres.DBTx, dst, src FileInfo) error {
	dst.ExternalID = random.UUID()
	if err := tx.Exec("UPDATE files SET external_id = $2 WHERE file_id = $1", dst.ID, dst.ExternalID); err != nil {
		return err
	}

	_, err := b.bucket.Object(src.ExternalID).CopyTo(context.Background(), b.bucket.Object(dst.ExternalID), nil)
	return err
}
Exemplo n.º 3
0
func (b *azureBackend) Put(tx *postgres.DBTx, info FileInfo, r io.Reader, appendBlob bool) error {
	if appendBlob {
		// TODO(titanous): This is a hack, we should modify the block list.
		existing, err := b.Open(tx, info, false)
		if err != nil {
			return err
		}
		r = io.MultiReader(existing, r)
	}

	info.ExternalID = random.UUID()
	if err := tx.Exec("UPDATE files SET external_id = $2 WHERE file_id = $1", info.ID, info.ExternalID); err != nil {
		return err
	}

	// Create blob that will be filled with blocks
	if err := b.client.CreateBlockBlob(b.container, info.ExternalID); err != nil {
		return err
	}
	var blocks []storage.Block

	// Create blocks
	buf := make([]byte, azureMaxBlockSize)
	for {
		n, err := io.ReadFull(r, buf)
		if err == io.EOF {
			break
		}
		if err != nil && err != io.ErrUnexpectedEOF {
			return err
		}
		data := buf[:n]
		md5sum := md5.Sum(data)
		blockID := base64.StdEncoding.EncodeToString(random.Bytes(16))

		if err := b.client.PutBlockWithLength(
			b.container,
			info.ExternalID,
			blockID,
			uint64(n),
			bytes.NewReader(data),
			map[string]string{"Content-MD5": base64.StdEncoding.EncodeToString(md5sum[:])},
		); err != nil {
			return err
		}
		blocks = append(blocks, storage.Block{ID: blockID, Status: storage.BlockStatusUncommitted})

		if err == io.ErrUnexpectedEOF {
			break
		}
	}

	// Save the list of blocks to the blob
	return b.client.PutBlockList(b.container, info.ExternalID, blocks)
}
Exemplo n.º 4
0
// migrateProcessData populates ProcessType.Volumes if ProcessType.Data is set
func migrateProcessData(tx *postgres.DBTx) error {
	type Release struct {
		ID string

		// use map[string]interface{} for process types so we can just
		// update Volumes and Data and leave other fields untouched
		Processes map[string]map[string]interface{}
	}

	var releases []Release
	rows, err := tx.Query("SELECT release_id, processes FROM releases")
	if err != nil {
		return err
	}
	defer rows.Close()
	for rows.Next() {
		var release Release
		if err := rows.Scan(&release.ID, &release.Processes); err != nil {
			return err
		}
		releases = append(releases, release)
	}
	if err := rows.Err(); err != nil {
		return err
	}

	for _, release := range releases {
		for typ, proc := range release.Processes {
			v, ok := proc["data"]
			if !ok {
				continue
			}
			data, ok := v.(bool)
			if !ok || !data {
				continue
			}
			proc["volumes"] = []struct {
				Path string `json:"path"`
			}{
				{Path: "/data"},
			}
			delete(proc, "data")
			release.Processes[typ] = proc
		}

		// save the processes back to the db
		if err := tx.Exec("UPDATE releases SET processes = $1 WHERE release_id = $2", release.Processes, release.ID); err != nil {
			return err
		}
	}
	return nil
}
Exemplo n.º 5
0
func (b *s3Backend) Copy(tx *postgres.DBTx, dst, src FileInfo) error {
	dst.ExternalID = random.UUID()
	if err := tx.Exec("UPDATE files SET external_id = $2 WHERE file_id = $1", dst.ID, dst.ExternalID); err != nil {
		return err
	}

	_, err := b.client.CopyObject(&s3.CopyObjectInput{
		Bucket:      &b.bucket,
		CopySource:  aws.String(fmt.Sprintf("%s/%s", b.bucket, src.ExternalID)),
		Key:         &dst.ExternalID,
		ContentType: &dst.Type,
	})

	return err
}
Exemplo n.º 6
0
func (b *gcsBackend) Put(tx *postgres.DBTx, info FileInfo, r io.Reader, append bool) error {
	if append {
		// TODO(titanous): This is a hack, we should use resumable uploads.
		existing, err := b.Open(tx, info, false)
		if err != nil {
			return err
		}
		r = io.MultiReader(existing, r)
	}

	info.ExternalID = random.UUID()
	if err := tx.Exec("UPDATE files SET external_id = $2 WHERE file_id = $1", info.ID, info.ExternalID); err != nil {
		return err
	}

	w := b.bucket.Object(info.ExternalID).NewWriter(context.Background())
	w.ContentType = info.Type
	if _, err := io.Copy(w, r); err != nil {
		w.Close()
		return err
	}
	return w.Close()
}
Exemplo n.º 7
0
// migrateProcessArgs sets ProcessType.Args from Entrypoint / Cmd for every
// release, and also prepends an explicit entrypoint for system and slug apps
// (they will no longer use the Dockerfile Entrypoint as they have some args
// like `scheduler` for the controller scheduler and `start web` for slugs).
func migrateProcessArgs(tx *postgres.DBTx) error {
	type Release struct {
		ID      string
		AppName *string
		AppMeta map[string]string
		Meta    map[string]string

		// use map[string]interface{} for process types so we can
		// just update Args and leave other fields untouched
		Processes map[string]map[string]interface{}
	}

	// get all the releases with the associated app name if set
	var releases []Release
	rows, err := tx.Query("SELECT r.release_id, r.meta, r.processes, a.name, a.meta FROM releases r LEFT JOIN apps a ON a.release_id = r.release_id")
	if err != nil {
		return err
	}
	for rows.Next() {
		var release Release
		if err := rows.Scan(&release.ID, &release.Meta, &release.Processes, &release.AppName, &release.AppMeta); err != nil {
			rows.Close()
			return err
		}
		releases = append(releases, release)
	}
	if err := rows.Err(); err != nil {
		return err
	}

	for _, release := range releases {
		for typ, proc := range release.Processes {
			// if the release is for a system app which has a Cmd,
			// explicitly set the Entrypoint
			var cmd []interface{}
			if v, ok := proc["cmd"]; ok {
				cmd = v.([]interface{})
			}
			if release.AppName != nil && release.AppMeta["flynn-system-app"] == "true" && len(cmd) > 0 {
				switch *release.AppName {
				case "postgres":
					proc["entrypoint"] = []interface{}{"/bin/start-flynn-postgres"}
				case "controller":
					proc["entrypoint"] = []interface{}{"/bin/start-flynn-controller"}
				case "redis":
					proc["entrypoint"] = []interface{}{"/bin/start-flynn-redis"}
				case "mariadb":
					proc["entrypoint"] = []interface{}{"/bin/start-flynn-mariadb"}
				case "mongodb":
					proc["entrypoint"] = []interface{}{"/bin/start-flynn-mongodb"}
				case "router":
					proc["entrypoint"] = []interface{}{"/bin/flynn-router"}
				case "logaggregator":
					proc["entrypoint"] = []interface{}{"/bin/logaggregator"}
				default:
					if strings.HasPrefix(*release.AppName, "redis-") {
						proc["entrypoint"] = []interface{}{"/bin/start-flynn-redis"}
					} else {
						panic(fmt.Sprintf("migration failed to set entrypoint for system app %s", *release.AppName))
					}
				}
			}

			// git releases use the slugrunner which need an Entrypoint
			if release.Meta["git"] == "true" {
				proc["entrypoint"] = []interface{}{"/runner/init"}
			}

			// construct Args by appending Cmd to Entrypoint
			var args []interface{}
			if v, ok := proc["entrypoint"]; ok {
				args = v.([]interface{})
			}
			proc["args"] = append(args, cmd...)
			release.Processes[typ] = proc
		}

		// save the processes back to the db
		if err := tx.Exec("UPDATE releases SET processes = $1 WHERE release_id = $2", release.Processes, release.ID); err != nil {
			return err
		}
	}
	return nil
}
Exemplo n.º 8
0
func (p pg) Delete(tx *postgres.DBTx, info FileInfo) error {
	if err := tx.Exec("SELECT lo_unlink($1)", info.Oid); err != nil {
		return err
	}
	return tx.Exec("UPDATE files SET file_oid = NULL WHERE file_id = $1", info.ID)
}
Exemplo n.º 9
0
func (f *FileRepo) SetBackend(tx *postgres.DBTx, id, name string) error {
	return tx.Exec("UPDATE files SET backend = $2 WHERE file_id = $1", id, name)
}