func (storer *APTStorer) saveFile(ingestState *models.IngestState, gf *models.GenericFile) { existingSha256, err := storer.getExistingSha256(gf.Identifier) if err != nil { storer.Context.MessageLog.Error(err.Error()) ingestState.IngestManifest.StoreResult.AddError(err.Error()) return } // Set this, for the record. if existingSha256 != nil { gf.IngestPreviousVersionExists = true gf.Id = existingSha256.GenericFileId uuid, err := storer.getUuidOfExistingFile(gf.Identifier) if err != nil { message := fmt.Sprintf("Cannot find existing UUID for %s: %v", gf.Identifier, err.Error()) ingestState.IngestManifest.StoreResult.AddError(message) storer.Context.MessageLog.Error(message) // Probably not fatal, but treat it as such for now, // because we don't want leave orphan objects in S3, // or have the GenericFile.URL not match the actual // storage URL. This should only happen if a depositor // deletes the existing version of a GenericFile while // we are processing this ingest. The window for that // to happen is usually between a few seconds and a few // hours. ingestState.IngestManifest.StoreResult.ErrorIsFatal = true return } if uuid == "" { message := fmt.Sprintf("Cannot find existing UUID for %s.", gf.Identifier) ingestState.IngestManifest.StoreResult.AddError(message) storer.Context.MessageLog.Error(message) // Probably not fatal, but treat it as such for now. // Same note as in previous if statement above. ingestState.IngestManifest.StoreResult.ErrorIsFatal = true return } else { // OK. Set the GenericFile's UUID to match the existing file's // UUID, so that we overwrite the existing file, and so the // GenericFile record in Pharos still has the correct URL. message := fmt.Sprintf("Resetting UUID for '%s' to '%s' so we can overwrite "+ "the currently stored version of the file.", gf.Identifier, uuid) storer.Context.MessageLog.Info(message) // TODO: Test this in integration post test. gf.IngestUUID = uuid } if existingSha256.Digest != gf.IngestSha256 { storer.Context.MessageLog.Info( "GenericFile %s has same sha256. Does not need save.", gf.Identifier) gf.IngestNeedsSave = false } } // Now copy to storage only if the file has changed. if gf.IngestNeedsSave { storer.Context.MessageLog.Info("File %s needs save", gf.Identifier) if gf.IngestStoredAt.IsZero() || gf.IngestStorageURL == "" { storer.copyToLongTermStorage(ingestState, gf, "s3") } if gf.IngestReplicatedAt.IsZero() || gf.IngestReplicationURL == "" { storer.copyToLongTermStorage(ingestState, gf, "glacier") } } }