Example #1
0
func TestAddToArchiveWithClosedWriter(t *testing.T) {
	dir, err := ioutil.TempDir("", "tarwriter_test")
	if err != nil {
		assert.FailNow(t, "Cannot create temp dir", err.Error())
	}
	tempFilePath := filepath.Join(dir, "test_file.tar")
	defer os.RemoveAll(dir)
	w := tarfile.NewWriter(tempFilePath)

	// Note that we have not opened the writer
	err = w.AddToArchive(pathToTestFile("cleanup_result.json"), "file1.json")
	if err == nil {
		assert.FailNow(t, "Should have gotten a tar write error")
	}
	assert.True(t, strings.HasPrefix(err.Error(), "Underlying TarWriter is nil"))

	// Open and close the writer, so the file exists.
	w.Open()
	w.Close()
	if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
		assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
	}
	err = w.AddToArchive(pathToTestFile("cleanup_result.json"), "file1.json")
	if err == nil {
		assert.FailNow(t, "Should have gotten a tar write error")
	}
	assert.True(t, strings.HasPrefix(err.Error(), "archive/tar: write after close"))

}
Example #2
0
func TestNewWriter(t *testing.T) {
	dir, err := ioutil.TempDir("", "tarwriter_test")
	if err != nil {
		assert.FailNow(t, "Cannot create temp dir", err.Error())
	}
	tempFilePath := filepath.Join(dir, "test_file.tar")
	defer os.RemoveAll(dir)
	w := tarfile.NewWriter(tempFilePath)
	assert.NotNil(t, w)
	assert.Equal(t, tempFilePath, w.PathToTarFile)
}
Example #3
0
func TestAndCloseOpen(t *testing.T) {
	dir, err := ioutil.TempDir("", "tarwriter_test")
	if err != nil {
		assert.FailNow(t, "Cannot create temp dir", err.Error())
	}
	tempFilePath := filepath.Join(dir, "test_file.tar")
	defer os.RemoveAll(dir)
	w := tarfile.NewWriter(tempFilePath)
	defer w.Close()
	err = w.Open()
	assert.Nil(t, err)
	if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
		assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
	}
	err = w.Close()
	assert.Nil(t, err)
}
Example #4
0
func TestAddToArchive(t *testing.T) {
	dir, err := ioutil.TempDir("", "tarwriter_test")
	if err != nil {
		assert.FailNow(t, "Cannot create temp dir", err.Error())
	}
	tempFilePath := filepath.Join(dir, "test_file.tar")
	defer os.RemoveAll(dir)
	w := tarfile.NewWriter(tempFilePath)
	defer w.Close()
	err = w.Open()
	assert.Nil(t, err)
	if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
		assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
	}
	err = w.AddToArchive(pathToTestFile("cleanup_result.json"), "file1.json")
	assert.Nil(t, err)
	err = w.AddToArchive(pathToTestFile("ingest_result.json"), "data/subdir/file2.json")
	assert.Nil(t, err)
	w.Close()

	file, err := os.Open(w.PathToTarFile)
	if file != nil {
		defer file.Close()
	}
	if err != nil {
		assert.FailNow(t, "Could not open tar file", err.Error())
	}
	filesInArchive := make([]string, 0)
	reader := tar.NewReader(file)
	for {
		header, err := reader.Next()
		if err != nil {
			break
		}
		filesInArchive = append(filesInArchive, header.Name)
	}
	assert.Equal(t, "file1.json", filesInArchive[0])
	assert.Equal(t, "data/subdir/file2.json", filesInArchive[1])
}
Example #5
0
func TestAddToArchiveWithBadFilePath(t *testing.T) {
	dir, err := ioutil.TempDir("", "tarwriter_test")
	if err != nil {
		assert.FailNow(t, "Cannot create temp dir", err.Error())
	}
	tempFilePath := filepath.Join(dir, "test_file.tar")
	defer os.RemoveAll(dir)
	w := tarfile.NewWriter(tempFilePath)
	defer w.Close()
	err = w.Open()
	assert.Nil(t, err)
	if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
		assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
	}

	// This file doesn't exist. Make sure we get the right error.
	err = w.AddToArchive(pathToTestFile("this_file_does_not_exist"), "file1.json")
	if err == nil {
		assert.FailNow(t, "Should have gotten a tar write error")
	}
	assert.True(t, strings.Contains(err.Error(), "no such file or directory"))
}
Example #6
0
func (packager *DPNPackager) tarBag() {
	for manifest := range packager.TarChannel {
		packager.Context.MessageLog.Info("Tarring %s", manifest.LocalDir)
		manifest.NsqMessage.Touch()

		files, err := fileutil.RecursiveFileList(manifest.LocalDir)
		if err != nil {
			manifest.PackageSummary.AddError("Cannot get list of files in directory %s: %s",
				manifest.LocalDir, err.Error())
			packager.PostProcessChannel <- manifest
			continue
		}

		// Set up our tar writer...
		tarWriter := tarfile.NewWriter(manifest.LocalTarFile)
		err = tarWriter.Open()
		if err != nil {
			manifest.PackageSummary.AddError("Error creating tar file %s for bag %s: %v",
				manifest.LocalTarFile, manifest.IntellectualObject.Identifier, err)
			packager.PostProcessChannel <- manifest
			continue
		}

		// ... and start filling it up.
		for _, filePath := range files {
			// The DPN spec at https://wiki.duraspace.org/display/DPN/BagIt+Specification
			// says the top-level folder within the bag should have the name of the DPN
			// Object Identifier (the UUID). So we replace <bag_name>/ with <uuid>/.
			//
			// Splitting filePath on object identifier looks like this:
			// /mnt/dpn/staging/test.edu/bag1/data/file1
			// pathInBag = "data/file1"
			// pathWithinArchive = "7a27db64-cea6-4602-a6c5-d8b2a7f6c02b/data/file1"
			pathInBag := strings.Split(filePath, manifest.IntellectualObject.Identifier)[1]
			pathWithinArchive := filepath.Join(manifest.DPNBag.UUID, pathInBag)

			err = tarWriter.AddToArchive(filePath, pathWithinArchive)
			if err != nil {
				manifest.PackageSummary.AddError("Error adding file %s to archive %s: %v",
					filePath, pathWithinArchive, err)
				tarWriter.Close()
				packager.PostProcessChannel <- manifest
				break
			}
		}
		tarWriter.Close()
		manifest.NsqMessage.Touch()

		// Finish the DPN bag record by setting the file size
		file, err := os.Stat(manifest.LocalTarFile)
		if err != nil {
			manifest.PackageSummary.AddError("Cannot get file size of %s: %v",
				manifest.LocalTarFile, err)
			packager.PostProcessChannel <- manifest
			break
		}
		manifest.DPNBag.Size = uint64(file.Size())

		// We want to validate everything AFTER tarring, because
		// the tarred bag is ultimately what will go into DPN.
		packager.ValidationChannel <- manifest
	}
}