Пример #1
0
func (b *s3Backend) Put(tx *postgres.DBTx, info FileInfo, r io.Reader, append bool) error {
	if append {
		// This is a hack, the next easiest thing to do if we need to handle
		// upload resumption is to finalize the multipart upload when the client
		// disconnects and when the rest of the data arrives, start a new
		// multi-part upload copying the existing object as the first part
		// (which is supported by S3 as a specific API call). This requires
		// replacing the simple uploader, so it was not done in the first pass.
		existing, err := b.Open(tx, info, false)
		if err != nil {
			return err
		}
		r = io.MultiReader(existing, r)
	}

	info.ExternalID = random.UUID()
	if err := tx.Exec("UPDATE files SET external_id = $2 WHERE file_id = $1", info.ID, info.ExternalID); err != nil {
		return err
	}

	u := s3manager.NewUploaderWithClient(b.client)
	_, err := u.Upload(&s3manager.UploadInput{
		Bucket:      &b.bucket,
		Key:         &info.ExternalID,
		ContentType: &info.Type,
		Body:        r,
	})

	return err
}
Пример #2
0
func (b *gcsBackend) Open(tx *postgres.DBTx, info FileInfo, txControl bool) (FileStream, error) {
	if txControl {
		// We don't need the database transaction, so clean it up
		tx.Rollback()
	}

	url, err := storage.SignedURL(b.bucketName, info.ExternalID, b.signOpts())
	return newRedirectFileStream(url), err
}
Пример #3
0
func (b *gcsBackend) Copy(tx *postgres.DBTx, dst, src FileInfo) error {
	dst.ExternalID = random.UUID()
	if err := tx.Exec("UPDATE files SET external_id = $2 WHERE file_id = $1", dst.ID, dst.ExternalID); err != nil {
		return err
	}

	_, err := b.bucket.Object(src.ExternalID).CopyTo(context.Background(), b.bucket.Object(dst.ExternalID), nil)
	return err
}
Пример #4
0
func (b *azureBackend) Open(tx *postgres.DBTx, info FileInfo, txControl bool) (FileStream, error) {
	if txControl {
		// We don't need the database transaction, so clean it up
		tx.Rollback()
	}

	url, err := b.client.GetBlobSASURI(b.container, info.ExternalID, time.Now().Add(10*time.Minute), "r")
	return newRedirectFileStream(url), err
}
Пример #5
0
func (b *azureBackend) Put(tx *postgres.DBTx, info FileInfo, r io.Reader, appendBlob bool) error {
	if appendBlob {
		// TODO(titanous): This is a hack, we should modify the block list.
		existing, err := b.Open(tx, info, false)
		if err != nil {
			return err
		}
		r = io.MultiReader(existing, r)
	}

	info.ExternalID = random.UUID()
	if err := tx.Exec("UPDATE files SET external_id = $2 WHERE file_id = $1", info.ID, info.ExternalID); err != nil {
		return err
	}

	// Create blob that will be filled with blocks
	if err := b.client.CreateBlockBlob(b.container, info.ExternalID); err != nil {
		return err
	}
	var blocks []storage.Block

	// Create blocks
	buf := make([]byte, azureMaxBlockSize)
	for {
		n, err := io.ReadFull(r, buf)
		if err == io.EOF {
			break
		}
		if err != nil && err != io.ErrUnexpectedEOF {
			return err
		}
		data := buf[:n]
		md5sum := md5.Sum(data)
		blockID := base64.StdEncoding.EncodeToString(random.Bytes(16))

		if err := b.client.PutBlockWithLength(
			b.container,
			info.ExternalID,
			blockID,
			uint64(n),
			bytes.NewReader(data),
			map[string]string{"Content-MD5": base64.StdEncoding.EncodeToString(md5sum[:])},
		); err != nil {
			return err
		}
		blocks = append(blocks, storage.Block{ID: blockID, Status: storage.BlockStatusUncommitted})

		if err == io.ErrUnexpectedEOF {
			break
		}
	}

	// Save the list of blocks to the blob
	return b.client.PutBlockList(b.container, info.ExternalID, blocks)
}
Пример #6
0
// migrateProcessData populates ProcessType.Volumes if ProcessType.Data is set
func migrateProcessData(tx *postgres.DBTx) error {
	type Release struct {
		ID string

		// use map[string]interface{} for process types so we can just
		// update Volumes and Data and leave other fields untouched
		Processes map[string]map[string]interface{}
	}

	var releases []Release
	rows, err := tx.Query("SELECT release_id, processes FROM releases")
	if err != nil {
		return err
	}
	defer rows.Close()
	for rows.Next() {
		var release Release
		if err := rows.Scan(&release.ID, &release.Processes); err != nil {
			return err
		}
		releases = append(releases, release)
	}
	if err := rows.Err(); err != nil {
		return err
	}

	for _, release := range releases {
		for typ, proc := range release.Processes {
			v, ok := proc["data"]
			if !ok {
				continue
			}
			data, ok := v.(bool)
			if !ok || !data {
				continue
			}
			proc["volumes"] = []struct {
				Path string `json:"path"`
			}{
				{Path: "/data"},
			}
			delete(proc, "data")
			release.Processes[typ] = proc
		}

		// save the processes back to the db
		if err := tx.Exec("UPDATE releases SET processes = $1 WHERE release_id = $2", release.Processes, release.ID); err != nil {
			return err
		}
	}
	return nil
}
Пример #7
0
func (b *s3Backend) Copy(tx *postgres.DBTx, dst, src FileInfo) error {
	dst.ExternalID = random.UUID()
	if err := tx.Exec("UPDATE files SET external_id = $2 WHERE file_id = $1", dst.ID, dst.ExternalID); err != nil {
		return err
	}

	_, err := b.client.CopyObject(&s3.CopyObjectInput{
		Bucket:      &b.bucket,
		CopySource:  aws.String(fmt.Sprintf("%s/%s", b.bucket, src.ExternalID)),
		Key:         &dst.ExternalID,
		ContentType: &dst.Type,
	})

	return err
}
Пример #8
0
func (b *s3Backend) Open(tx *postgres.DBTx, info FileInfo, txControl bool) (FileStream, error) {
	if txControl {
		// We don't need the database transaction, so clean it up
		tx.Rollback()
	}

	req, _ := b.client.GetObjectRequest(&s3.GetObjectInput{
		Bucket: &b.bucket,
		Key:    &info.ExternalID,
	})
	url, err := req.Presign(10 * time.Minute)
	if err != nil {
		return nil, err
	}

	return newRedirectFileStream(url), nil
}
Пример #9
0
func (p pg) Open(tx *postgres.DBTx, info FileInfo, txControl bool) (FileStream, error) {
	if info.Oid == nil {
		return nil, ErrNotFound
	}

	lo, err := tx.LargeObjects()
	if err != nil {
		return nil, err
	}
	obj, err := lo.Open(*info.Oid, pgx.LargeObjectModeRead)
	if err != nil {
		return nil, err
	}
	f := &pgFile{LargeObject: obj, size: info.Size}
	if txControl {
		f.tx = tx
	}
	return f, nil
}
Пример #10
0
func (b *gcsBackend) Put(tx *postgres.DBTx, info FileInfo, r io.Reader, append bool) error {
	if append {
		// TODO(titanous): This is a hack, we should use resumable uploads.
		existing, err := b.Open(tx, info, false)
		if err != nil {
			return err
		}
		r = io.MultiReader(existing, r)
	}

	info.ExternalID = random.UUID()
	if err := tx.Exec("UPDATE files SET external_id = $2 WHERE file_id = $1", info.ID, info.ExternalID); err != nil {
		return err
	}

	w := b.bucket.Object(info.ExternalID).NewWriter(context.Background())
	w.ContentType = info.Type
	if _, err := io.Copy(w, r); err != nil {
		w.Close()
		return err
	}
	return w.Close()
}
Пример #11
0
func (p pg) Put(tx *postgres.DBTx, info FileInfo, r io.Reader, append bool) error {
	if !append {
		if err := tx.QueryRow("UPDATE files SET file_oid = lo_create(0) WHERE file_id = $1 RETURNING file_oid", info.ID).Scan(&info.Oid); err != nil {
			return err
		}
	}

	lo, err := tx.LargeObjects()
	if err != nil {
		return err
	}
	obj, err := lo.Open(*info.Oid, pgx.LargeObjectModeWrite)
	if err != nil {
		return err
	}
	if append {
		obj.Seek(info.Size, os.SEEK_SET)
	}
	if _, err := io.Copy(obj, r); err != nil {
		return err
	}

	return nil
}
Пример #12
0
// migrateProcessArgs sets ProcessType.Args from Entrypoint / Cmd for every
// release, and also prepends an explicit entrypoint for system and slug apps
// (they will no longer use the Dockerfile Entrypoint as they have some args
// like `scheduler` for the controller scheduler and `start web` for slugs).
func migrateProcessArgs(tx *postgres.DBTx) error {
	type Release struct {
		ID      string
		AppName *string
		AppMeta map[string]string
		Meta    map[string]string

		// use map[string]interface{} for process types so we can
		// just update Args and leave other fields untouched
		Processes map[string]map[string]interface{}
	}

	// get all the releases with the associated app name if set
	var releases []Release
	rows, err := tx.Query("SELECT r.release_id, r.meta, r.processes, a.name, a.meta FROM releases r LEFT JOIN apps a ON a.release_id = r.release_id")
	if err != nil {
		return err
	}
	for rows.Next() {
		var release Release
		if err := rows.Scan(&release.ID, &release.Meta, &release.Processes, &release.AppName, &release.AppMeta); err != nil {
			rows.Close()
			return err
		}
		releases = append(releases, release)
	}
	if err := rows.Err(); err != nil {
		return err
	}

	for _, release := range releases {
		for typ, proc := range release.Processes {
			// if the release is for a system app which has a Cmd,
			// explicitly set the Entrypoint
			var cmd []interface{}
			if v, ok := proc["cmd"]; ok {
				cmd = v.([]interface{})
			}
			if release.AppName != nil && release.AppMeta["flynn-system-app"] == "true" && len(cmd) > 0 {
				switch *release.AppName {
				case "postgres":
					proc["entrypoint"] = []interface{}{"/bin/start-flynn-postgres"}
				case "controller":
					proc["entrypoint"] = []interface{}{"/bin/start-flynn-controller"}
				case "redis":
					proc["entrypoint"] = []interface{}{"/bin/start-flynn-redis"}
				case "mariadb":
					proc["entrypoint"] = []interface{}{"/bin/start-flynn-mariadb"}
				case "mongodb":
					proc["entrypoint"] = []interface{}{"/bin/start-flynn-mongodb"}
				case "router":
					proc["entrypoint"] = []interface{}{"/bin/flynn-router"}
				case "logaggregator":
					proc["entrypoint"] = []interface{}{"/bin/logaggregator"}
				default:
					if strings.HasPrefix(*release.AppName, "redis-") {
						proc["entrypoint"] = []interface{}{"/bin/start-flynn-redis"}
					} else {
						panic(fmt.Sprintf("migration failed to set entrypoint for system app %s", *release.AppName))
					}
				}
			}

			// git releases use the slugrunner which need an Entrypoint
			if release.Meta["git"] == "true" {
				proc["entrypoint"] = []interface{}{"/runner/init"}
			}

			// construct Args by appending Cmd to Entrypoint
			var args []interface{}
			if v, ok := proc["entrypoint"]; ok {
				args = v.([]interface{})
			}
			proc["args"] = append(args, cmd...)
			release.Processes[typ] = proc
		}

		// save the processes back to the db
		if err := tx.Exec("UPDATE releases SET processes = $1 WHERE release_id = $2", release.Processes, release.ID); err != nil {
			return err
		}
	}
	return nil
}
Пример #13
0
func (p pg) Delete(tx *postgres.DBTx, info FileInfo) error {
	if err := tx.Exec("SELECT lo_unlink($1)", info.Oid); err != nil {
		return err
	}
	return tx.Exec("UPDATE files SET file_oid = NULL WHERE file_id = $1", info.ID)
}
Пример #14
0
func (f *FileRepo) SetBackend(tx *postgres.DBTx, id, name string) error {
	return tx.Exec("UPDATE files SET backend = $2 WHERE file_id = $1", id, name)
}