/*
Arenas produced by Dir Transmats may be relocated by simple `mv`.
*/
func (t *DirTransmat) Materialize(
kind integrity.TransmatKind,
dataHash integrity.CommitID,
siloURIs []integrity.SiloURI,
options ...integrity.MaterializerConfigurer,
) integrity.Arena {
var arena dirArena
try.Do(func() {
// Basic validation and config
config := integrity.EvaluateConfig(options...)
if kind != Kind {
panic(errors.ProgrammerError.New("This transmat supports definitions of type %q, not %q", Kind, kind))
}
// Ping silos
if len(siloURIs) < 1 {
panic(integrity.ConfigError.New("Materialization requires at least one data source!"))
// Note that it's possible a caching layer will satisfy things even without data sources...
// but if that was going to happen, it already would have by now.
}
// Our policy is to take the first path that exists.
// This lets you specify a series of potential locations,
var siloURI integrity.SiloURI
for _, givenURI := range siloURIs {
// TODO still assuming all local paths and not doing real uri parsing
localPath := string(givenURI)
_, err := os.Stat(localPath)
if os.IsNotExist(err) {
continue
}
siloURI = givenURI
break
}
if siloURI == "" {
panic(integrity.WarehouseConnectionError.New("No warehouses were available!"))
}
// Create staging arena to produce data into.
var err error
arena.path, err = ioutil.TempDir(t.workPath, "")
if err != nil {
panic(integrity.TransmatError.New("Unable to create arena: %s", err))
}
// walk filesystem, copying and accumulating data for integrity check
hasherFactory := sha512.New384
bucket := &fshash.MemoryBucket{}
localPath := string(siloURI)
if err := fshash.FillBucket(localPath, arena.Path(), bucket, filter.FilterSet{}, hasherFactory); err != nil {
panic(err)
}
// hash whole tree
actualTreeHash := fshash.Hash(bucket, hasherFactory)
// verify total integrity
expectedTreeHash, err := base64.URLEncoding.DecodeString(string(dataHash))
if err != nil {
panic(integrity.ConfigError.New("Could not parse hash: %s", err))
}
if bytes.Equal(actualTreeHash, expectedTreeHash) {
// excellent, got what we asked for.
arena.hash = dataHash
} else {
// this may or may not be grounds for panic, depending on configuration.
if config.AcceptHashMismatch {
// if we're tolerating mismatches, report the actual hash through different mechanisms.
// you probably only ever want to use this in tests or debugging; in prod it's just asking for insanity.
arena.hash = integrity.CommitID(actualTreeHash)
} else {
panic(integrity.NewHashMismatchError(string(dataHash), base64.URLEncoding.EncodeToString(actualTreeHash)))
}
}
}).Catch(integrity.Error, func(err *errors.Error) {
panic(err)
}).CatchAll(func(err error) {
panic(integrity.UnknownError.Wrap(err))
}).Done()
return arena
}
func (t DirTransmat) Scan(
kind integrity.TransmatKind,
subjectPath string,
siloURIs []integrity.SiloURI,
options ...integrity.MaterializerConfigurer,
) integrity.CommitID {
var commitID integrity.CommitID
try.Do(func() {
// Basic validation and config
config := integrity.EvaluateConfig(options...)
if kind != Kind {
panic(errors.ProgrammerError.New("This transmat supports definitions of type %q, not %q", Kind, kind))
}
// If scan area doesn't exist, bail immediately.
// No need to even start dialing warehouses if we've got nothing for em.
_, err := os.Stat(subjectPath)
if err != nil {
if os.IsNotExist(err) {
return // empty commitID
} else {
panic(err)
}
}
// Parse save locations.
// This transmat only supports one output location at a time due
// to Old code we haven't invested in refactoring yet.
var localPath string
if len(siloURIs) == 0 {
localPath = "" // empty string is a well known value to `fshash.FillBucket`: means just hash, don't copy.
} else if len(siloURIs) == 1 {
// TODO still assuming all local paths and not doing real uri parsing
localPath = string(siloURIs[0])
err := os.MkdirAll(filepath.Dir(localPath), 0755)
if err != nil {
panic(integrity.WarehouseConnectionError.New("Unable to write file: %s", err))
}
} else {
panic(integrity.ConfigError.New("%s transmat only supports shipping to 1 warehouse", Kind))
}
// walk filesystem, copying and accumulating data for integrity check
bucket := &fshash.MemoryBucket{}
err = fshash.FillBucket(subjectPath, localPath, bucket, config.FilterSet, hasherFactory)
if err != nil {
panic(err) // TODO this is not well typed, and does not clearly indicate whether scanning or committing had the problem
}
// hash whole tree
actualTreeHash := fshash.Hash(bucket, hasherFactory)
// report
commitID = integrity.CommitID(base64.URLEncoding.EncodeToString(actualTreeHash))
}).Catch(integrity.Error, func(err *errors.Error) {
panic(err)
}).CatchAll(func(err error) {
panic(integrity.UnknownError.Wrap(err))
}).Done()
return commitID
}