func TestRecursiveFileList(t *testing.T) {
exchangeHome, _ := fileutil.ExchangeHome()
files, err := fileutil.RecursiveFileList(exchangeHome)
if err != nil {
t.Errorf("RecursiveFileList() returned error: %v", err)
}
// Make a map for quick lookup & check for a handful
// of files at different levels.
fileMap := make(map[string]string, 0)
for _, f := range files {
fileMap[f] = f
}
sampleFiles := []string{
getPath("README.md"),
getPath("constants/constants.go"),
getPath("models/config.go"),
getPath("models/generic_file.go"),
getPath("testdata/json_objects/intel_obj.json"),
getPath("util/logger/logger.go"),
}
for _, filePath := range sampleFiles {
_, present := fileMap[filePath]
if present == false {
t.Errorf("File '%s' is missing from recursive file list", filePath)
}
}
}
func (packager *DPNPackager) tarBag() {
for manifest := range packager.TarChannel {
packager.Context.MessageLog.Info("Tarring %s", manifest.LocalDir)
manifest.NsqMessage.Touch()
files, err := fileutil.RecursiveFileList(manifest.LocalDir)
if err != nil {
manifest.PackageSummary.AddError("Cannot get list of files in directory %s: %s",
manifest.LocalDir, err.Error())
packager.PostProcessChannel <- manifest
continue
}
// Set up our tar writer...
tarWriter := tarfile.NewWriter(manifest.LocalTarFile)
err = tarWriter.Open()
if err != nil {
manifest.PackageSummary.AddError("Error creating tar file %s for bag %s: %v",
manifest.LocalTarFile, manifest.IntellectualObject.Identifier, err)
packager.PostProcessChannel <- manifest
continue
}
// ... and start filling it up.
for _, filePath := range files {
// The DPN spec at https://wiki.duraspace.org/display/DPN/BagIt+Specification
// says the top-level folder within the bag should have the name of the DPN
// Object Identifier (the UUID). So we replace <bag_name>/ with <uuid>/.
//
// Splitting filePath on object identifier looks like this:
// /mnt/dpn/staging/test.edu/bag1/data/file1
// pathInBag = "data/file1"
// pathWithinArchive = "7a27db64-cea6-4602-a6c5-d8b2a7f6c02b/data/file1"
pathInBag := strings.Split(filePath, manifest.IntellectualObject.Identifier)[1]
pathWithinArchive := filepath.Join(manifest.DPNBag.UUID, pathInBag)
err = tarWriter.AddToArchive(filePath, pathWithinArchive)
if err != nil {
manifest.PackageSummary.AddError("Error adding file %s to archive %s: %v",
filePath, pathWithinArchive, err)
tarWriter.Close()
packager.PostProcessChannel <- manifest
break
}
}
tarWriter.Close()
manifest.NsqMessage.Touch()
// Finish the DPN bag record by setting the file size
file, err := os.Stat(manifest.LocalTarFile)
if err != nil {
manifest.PackageSummary.AddError("Cannot get file size of %s: %v",
manifest.LocalTarFile, err)
packager.PostProcessChannel <- manifest
break
}
manifest.DPNBag.Size = uint64(file.Size())
// We want to validate everything AFTER tarring, because
// the tarred bag is ultimately what will go into DPN.
packager.ValidationChannel <- manifest
}
}