func TestRecursiveFileList(t *testing.T) { exchangeHome, _ := fileutil.ExchangeHome() files, err := fileutil.RecursiveFileList(exchangeHome) if err != nil { t.Errorf("RecursiveFileList() returned error: %v", err) } // Make a map for quick lookup & check for a handful // of files at different levels. fileMap := make(map[string]string, 0) for _, f := range files { fileMap[f] = f } sampleFiles := []string{ getPath("README.md"), getPath("constants/constants.go"), getPath("models/config.go"), getPath("models/generic_file.go"), getPath("testdata/json_objects/intel_obj.json"), getPath("util/logger/logger.go"), } for _, filePath := range sampleFiles { _, present := fileMap[filePath] if present == false { t.Errorf("File '%s' is missing from recursive file list", filePath) } } }
func (packager *DPNPackager) tarBag() { for manifest := range packager.TarChannel { packager.Context.MessageLog.Info("Tarring %s", manifest.LocalDir) manifest.NsqMessage.Touch() files, err := fileutil.RecursiveFileList(manifest.LocalDir) if err != nil { manifest.PackageSummary.AddError("Cannot get list of files in directory %s: %s", manifest.LocalDir, err.Error()) packager.PostProcessChannel <- manifest continue } // Set up our tar writer... tarWriter := tarfile.NewWriter(manifest.LocalTarFile) err = tarWriter.Open() if err != nil { manifest.PackageSummary.AddError("Error creating tar file %s for bag %s: %v", manifest.LocalTarFile, manifest.IntellectualObject.Identifier, err) packager.PostProcessChannel <- manifest continue } // ... and start filling it up. for _, filePath := range files { // The DPN spec at https://wiki.duraspace.org/display/DPN/BagIt+Specification // says the top-level folder within the bag should have the name of the DPN // Object Identifier (the UUID). So we replace <bag_name>/ with <uuid>/. // // Splitting filePath on object identifier looks like this: // /mnt/dpn/staging/test.edu/bag1/data/file1 // pathInBag = "data/file1" // pathWithinArchive = "7a27db64-cea6-4602-a6c5-d8b2a7f6c02b/data/file1" pathInBag := strings.Split(filePath, manifest.IntellectualObject.Identifier)[1] pathWithinArchive := filepath.Join(manifest.DPNBag.UUID, pathInBag) err = tarWriter.AddToArchive(filePath, pathWithinArchive) if err != nil { manifest.PackageSummary.AddError("Error adding file %s to archive %s: %v", filePath, pathWithinArchive, err) tarWriter.Close() packager.PostProcessChannel <- manifest break } } tarWriter.Close() manifest.NsqMessage.Touch() // Finish the DPN bag record by setting the file size file, err := os.Stat(manifest.LocalTarFile) if err != nil { manifest.PackageSummary.AddError("Cannot get file size of %s: %v", manifest.LocalTarFile, err) packager.PostProcessChannel <- manifest break } manifest.DPNBag.Size = uint64(file.Size()) // We want to validate everything AFTER tarring, because // the tarred bag is ultimately what will go into DPN. packager.ValidationChannel <- manifest } }