func TestAddToArchiveWithClosedWriter(t *testing.T) {
dir, err := ioutil.TempDir("", "tarwriter_test")
if err != nil {
assert.FailNow(t, "Cannot create temp dir", err.Error())
}
tempFilePath := filepath.Join(dir, "test_file.tar")
defer os.RemoveAll(dir)
w := tarfile.NewWriter(tempFilePath)
// Note that we have not opened the writer
err = w.AddToArchive(pathToTestFile("cleanup_result.json"), "file1.json")
if err == nil {
assert.FailNow(t, "Should have gotten a tar write error")
}
assert.True(t, strings.HasPrefix(err.Error(), "Underlying TarWriter is nil"))
// Open and close the writer, so the file exists.
w.Open()
w.Close()
if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
}
err = w.AddToArchive(pathToTestFile("cleanup_result.json"), "file1.json")
if err == nil {
assert.FailNow(t, "Should have gotten a tar write error")
}
assert.True(t, strings.HasPrefix(err.Error(), "archive/tar: write after close"))
}
func TestNewWriter(t *testing.T) {
dir, err := ioutil.TempDir("", "tarwriter_test")
if err != nil {
assert.FailNow(t, "Cannot create temp dir", err.Error())
}
tempFilePath := filepath.Join(dir, "test_file.tar")
defer os.RemoveAll(dir)
w := tarfile.NewWriter(tempFilePath)
assert.NotNil(t, w)
assert.Equal(t, tempFilePath, w.PathToTarFile)
}
func TestAndCloseOpen(t *testing.T) {
dir, err := ioutil.TempDir("", "tarwriter_test")
if err != nil {
assert.FailNow(t, "Cannot create temp dir", err.Error())
}
tempFilePath := filepath.Join(dir, "test_file.tar")
defer os.RemoveAll(dir)
w := tarfile.NewWriter(tempFilePath)
defer w.Close()
err = w.Open()
assert.Nil(t, err)
if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
}
err = w.Close()
assert.Nil(t, err)
}
func TestAddToArchive(t *testing.T) {
dir, err := ioutil.TempDir("", "tarwriter_test")
if err != nil {
assert.FailNow(t, "Cannot create temp dir", err.Error())
}
tempFilePath := filepath.Join(dir, "test_file.tar")
defer os.RemoveAll(dir)
w := tarfile.NewWriter(tempFilePath)
defer w.Close()
err = w.Open()
assert.Nil(t, err)
if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
}
err = w.AddToArchive(pathToTestFile("cleanup_result.json"), "file1.json")
assert.Nil(t, err)
err = w.AddToArchive(pathToTestFile("ingest_result.json"), "data/subdir/file2.json")
assert.Nil(t, err)
w.Close()
file, err := os.Open(w.PathToTarFile)
if file != nil {
defer file.Close()
}
if err != nil {
assert.FailNow(t, "Could not open tar file", err.Error())
}
filesInArchive := make([]string, 0)
reader := tar.NewReader(file)
for {
header, err := reader.Next()
if err != nil {
break
}
filesInArchive = append(filesInArchive, header.Name)
}
assert.Equal(t, "file1.json", filesInArchive[0])
assert.Equal(t, "data/subdir/file2.json", filesInArchive[1])
}
func TestAddToArchiveWithBadFilePath(t *testing.T) {
dir, err := ioutil.TempDir("", "tarwriter_test")
if err != nil {
assert.FailNow(t, "Cannot create temp dir", err.Error())
}
tempFilePath := filepath.Join(dir, "test_file.tar")
defer os.RemoveAll(dir)
w := tarfile.NewWriter(tempFilePath)
defer w.Close()
err = w.Open()
assert.Nil(t, err)
if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
}
// This file doesn't exist. Make sure we get the right error.
err = w.AddToArchive(pathToTestFile("this_file_does_not_exist"), "file1.json")
if err == nil {
assert.FailNow(t, "Should have gotten a tar write error")
}
assert.True(t, strings.Contains(err.Error(), "no such file or directory"))
}
func (packager *DPNPackager) tarBag() {
for manifest := range packager.TarChannel {
packager.Context.MessageLog.Info("Tarring %s", manifest.LocalDir)
manifest.NsqMessage.Touch()
files, err := fileutil.RecursiveFileList(manifest.LocalDir)
if err != nil {
manifest.PackageSummary.AddError("Cannot get list of files in directory %s: %s",
manifest.LocalDir, err.Error())
packager.PostProcessChannel <- manifest
continue
}
// Set up our tar writer...
tarWriter := tarfile.NewWriter(manifest.LocalTarFile)
err = tarWriter.Open()
if err != nil {
manifest.PackageSummary.AddError("Error creating tar file %s for bag %s: %v",
manifest.LocalTarFile, manifest.IntellectualObject.Identifier, err)
packager.PostProcessChannel <- manifest
continue
}
// ... and start filling it up.
for _, filePath := range files {
// The DPN spec at https://wiki.duraspace.org/display/DPN/BagIt+Specification
// says the top-level folder within the bag should have the name of the DPN
// Object Identifier (the UUID). So we replace <bag_name>/ with <uuid>/.
//
// Splitting filePath on object identifier looks like this:
// /mnt/dpn/staging/test.edu/bag1/data/file1
// pathInBag = "data/file1"
// pathWithinArchive = "7a27db64-cea6-4602-a6c5-d8b2a7f6c02b/data/file1"
pathInBag := strings.Split(filePath, manifest.IntellectualObject.Identifier)[1]
pathWithinArchive := filepath.Join(manifest.DPNBag.UUID, pathInBag)
err = tarWriter.AddToArchive(filePath, pathWithinArchive)
if err != nil {
manifest.PackageSummary.AddError("Error adding file %s to archive %s: %v",
filePath, pathWithinArchive, err)
tarWriter.Close()
packager.PostProcessChannel <- manifest
break
}
}
tarWriter.Close()
manifest.NsqMessage.Touch()
// Finish the DPN bag record by setting the file size
file, err := os.Stat(manifest.LocalTarFile)
if err != nil {
manifest.PackageSummary.AddError("Cannot get file size of %s: %v",
manifest.LocalTarFile, err)
packager.PostProcessChannel <- manifest
break
}
manifest.DPNBag.Size = uint64(file.Size())
// We want to validate everything AFTER tarring, because
// the tarred bag is ultimately what will go into DPN.
packager.ValidationChannel <- manifest
}
}