// Initializes the uploader object with connection data and metadata
// for this specific GenericFile.
func (storer *APTStorer) initUploader(ingestState *models.IngestState, gf *models.GenericFile, sendWhere string) *network.S3Upload {
var region string
var bucket string
if sendWhere == "s3" {
region = storer.Context.Config.APTrustS3Region
bucket = storer.Context.Config.PreservationBucket
} else if sendWhere == "glacier" {
region = storer.Context.Config.APTrustGlacierRegion
bucket = storer.Context.Config.ReplicationBucket
} else {
ingestState.IngestManifest.StoreResult.AddError("Cannot save %s to %s because "+
"storer doesn't know where %s is", gf.Identifier, sendWhere)
ingestState.IngestManifest.StoreResult.ErrorIsFatal = true
return nil
}
uploader := network.NewS3Upload(
region,
bucket,
gf.IngestUUID,
gf.FileFormat,
)
uploader.AddMetadata("institution", ingestState.IngestManifest.Object.Institution)
uploader.AddMetadata("bag", ingestState.IngestManifest.Object.Identifier)
uploader.AddMetadata("bagpath", gf.OriginalPath())
uploader.AddMetadata("md5", gf.IngestMd5)
uploader.AddMetadata("sha256", gf.IngestSha256)
return uploader
}
func (storer *DPNReplicationStorer) copyToLongTermStorage(manifest *models.ReplicationManifest) {
manifest.StoreSummary.ClearErrors()
upload := apt_network.NewS3Upload(
constants.AWSVirginia,
storer.Context.Config.DPN.DPNPreservationBucket,
fmt.Sprintf("%s.tar", manifest.ReplicationTransfer.Bag),
"application/x-tar")
upload.AddMetadata("from_node", manifest.ReplicationTransfer.FromNode)
upload.AddMetadata("transfer_id", manifest.ReplicationTransfer.ReplicationId)
upload.AddMetadata("member", manifest.DPNBag.Member)
upload.AddMetadata("local_id", manifest.DPNBag.LocalId)
upload.AddMetadata("version", fmt.Sprintf("%d", manifest.DPNBag.Version))
reader, err := os.Open(manifest.LocalPath)
if reader != nil {
defer reader.Close()
}
if err != nil {
manifest.StoreSummary.AddError("Error opening reader for tar file: %v", err)
return
}
upload.Send(reader)
if upload.ErrorMessage != "" {
manifest.StoreSummary.AddError("Error uploading tar file: %s", upload.ErrorMessage)
return
}
manifest.StorageURL = upload.Response.Location
}
func TestS3UploadBadFile(t *testing.T) {
if !canTestS3() {
return
}
upload := network.NewS3Upload(
constants.AWSVirginia,
testBucket,
"s3_upload_test.tar",
"application/tar",
)
file, _ := os.Open("file/does/not/exist.tar")
upload.Send(file)
assert.True(t, strings.Contains(upload.ErrorMessage, "invalid argument"))
}
func TestNewS3Upload(t *testing.T) {
if !canTestS3() {
return
}
upload := network.NewS3Upload(
constants.AWSVirginia,
testBucket,
"s3_upload_test.tar",
"application/tar",
)
assert.Equal(t, testBucket, *upload.UploadInput.Bucket)
assert.Equal(t, "s3_upload_test.tar", *upload.UploadInput.Key)
assert.Equal(t, "application/tar", *upload.UploadInput.ContentType)
assert.Equal(t, constants.AWSVirginia, upload.AWSRegion)
}
func upload(t *testing.T, key string) error {
upload := network.NewS3Upload(
constants.AWSVirginia,
testBucket,
key,
"application/tar",
)
upload.AddMetadata("testdata", "THIS IS DELETABLE TEST DATA")
file, err := os.Open("../testdata/unit_test_bags/virginia.edu.uva-lib_2278801.tar")
require.Nil(t, err)
upload.Send(file)
if upload.ErrorMessage != "" {
return fmt.Errorf(upload.ErrorMessage)
}
return nil
}
func TestS3UploadGoodFile(t *testing.T) {
if !canTestS3() {
return
}
upload := network.NewS3Upload(
constants.AWSVirginia,
testBucket,
"s3_upload_test.tar",
"application/tar",
)
upload.AddMetadata("institution", "test.edu")
upload.AddMetadata("bag", "test.edu/s3_upload_test")
upload.AddMetadata("bagpath", "data/test/path.xml")
upload.AddMetadata("md5", "FAKE-TEST-MD5")
upload.AddMetadata("sha256", "FAKE-TEST-SHA256")
file, err := os.Open("../testdata/unit_test_bags/virginia.edu.uva-lib_2278801.tar")
require.Nil(t, err)
upload.Send(file)
assert.Equal(t, "", upload.ErrorMessage)
}
func TestS3UploadAddMetadata(t *testing.T) {
if !canTestS3() {
return
}
upload := network.NewS3Upload(
constants.AWSVirginia,
testBucket,
"s3_upload_test.tar",
"application/tar",
)
upload.AddMetadata("institution", "test.edu")
upload.AddMetadata("bag", "test.edu/s3_upload_test")
upload.AddMetadata("bagpath", "data/test/path.xml")
upload.AddMetadata("md5", "FAKE-TEST-MD5")
upload.AddMetadata("sha256", "FAKE-TEST-SHA256")
assert.Equal(t, "test.edu", *upload.UploadInput.Metadata["institution"])
assert.Equal(t, "test.edu/s3_upload_test", *upload.UploadInput.Metadata["bag"])
assert.Equal(t, "data/test/path.xml", *upload.UploadInput.Metadata["bagpath"])
assert.Equal(t, "FAKE-TEST-MD5", *upload.UploadInput.Metadata["md5"])
assert.Equal(t, "FAKE-TEST-SHA256", *upload.UploadInput.Metadata["sha256"])
}
