// Run inputs func ProvisionInputs(transmat integrity.Transmat, assemblerFn integrity.Assembler, inputs []def.Input, rootfs string, journal log15.Logger) integrity.Assembly { // start having all filesystems filesystems := make(map[def.Input]integrity.Arena, len(inputs)) fsGather := make(chan map[def.Input]materializerReport) for _, in := range inputs { go func(in def.Input) { try.Do(func() { journal.Info(fmt.Sprintf("Starting materialize for %s hash=%s", in.Type, in.Hash)) arena := transmat.Materialize( integrity.TransmatKind(in.Type), integrity.CommitID(in.Hash), []integrity.SiloURI{integrity.SiloURI(in.URI)}, ) journal.Info(fmt.Sprintf("Finished materialize for %s hash=%s", in.Type, in.Hash)) fsGather <- map[def.Input]materializerReport{ in: {Arena: arena}, } }).Catch(integrity.Error, func(err *errors.Error) { journal.Warn(fmt.Sprintf("Errored during materialize for %s hash=%s", in.Type, in.Hash), "error", err.Message()) fsGather <- map[def.Input]materializerReport{ in: {Err: err}, } }).Done() }(in) } // (we don't have any output setup at this point, but if we do in the future, that'll be here.) // gather materialized inputs for range inputs { for in, report := range <-fsGather { if report.Err != nil { panic(report.Err) } filesystems[in] = report.Arena } } journal.Info("All inputs acquired... starting assembly") // assemble them into the final tree assemblyParts := make([]integrity.AssemblyPart, 0, len(filesystems)) for input, arena := range filesystems { assemblyParts = append(assemblyParts, integrity.AssemblyPart{ SourcePath: arena.Path(), TargetPath: input.Location, Writable: true, // TODO input config should have a word about this }) } assembly := assemblerFn(rootfs, assemblyParts) journal.Info("Assembly complete!") return assembly }
// Run inputs func ProvisionInputs(transmat integrity.Transmat, assemblerFn integrity.Assembler, inputs []def.Input, rootfs string, journal log15.Logger) integrity.Assembly { // start having all filesystems // input names are used as keys, so must be unique inputsByName := make(map[string]def.Input, len(inputs)) for _, in := range inputs { // TODO checks should also be sooner, up in cfg parse // but this check is for programmatic access as well (errors down the line can get nonobvious if you skip this). if _, ok := inputsByName[in.Name]; ok { panic(errors.ProgrammerError.New("duplicate name in input config")) } inputsByName[in.Name] = in } filesystems := make(map[string]integrity.Arena, len(inputs)) fsGather := make(chan map[string]materializerReport) for _, in := range inputs { go func(in def.Input) { try.Do(func() { journal.Info(fmt.Sprintf("Starting materialize for %s hash=%s", in.Type, in.Hash)) // todo: create validity checking api for URIs, check them all before launching anything warehouses := make([]integrity.SiloURI, len(in.Warehouses)) for i, wh := range in.Warehouses { warehouses[i] = integrity.SiloURI(wh) } // invoke transmat (blocking, potentially long time) arena := transmat.Materialize( integrity.TransmatKind(in.Type), integrity.CommitID(in.Hash), warehouses, ) // submit report journal.Info(fmt.Sprintf("Finished materialize for %s hash=%s", in.Type, in.Hash)) fsGather <- map[string]materializerReport{ in.Name: {Arena: arena}, } }).Catch(integrity.Error, func(err *errors.Error) { journal.Warn(fmt.Sprintf("Errored during materialize for %s hash=%s", in.Type, in.Hash), "error", err.Message()) fsGather <- map[string]materializerReport{ in.Name: {Err: err}, } }).Done() }(in) } // (we don't have any output setup at this point, but if we do in the future, that'll be here.) // gather materialized inputs for range inputs { for name, report := range <-fsGather { if report.Err != nil { panic(report.Err) } filesystems[name] = report.Arena } } journal.Info("All inputs acquired... starting assembly") // assemble them into the final tree assemblyParts := make([]integrity.AssemblyPart, 0, len(filesystems)) for name, arena := range filesystems { assemblyParts = append(assemblyParts, integrity.AssemblyPart{ SourcePath: arena.Path(), TargetPath: inputsByName[name].MountPath, Writable: true, // TODO input config should have a word about this }) } assembly := assemblerFn(rootfs, assemblyParts) journal.Info("Assembly complete!") return assembly }
// Run outputs func PreserveOutputs(transmat integrity.Transmat, outputs []def.Output, rootfs string, journal log15.Logger) []def.Output { // run commit on the outputs scanGather := make(chan scanReport) for _, out := range outputs { go func(out def.Output) { filterOptions := make([]integrity.MaterializerConfigurer, 0, 3) out.Filters.InitDefaultsOutput() switch out.Filters.UidMode { case def.FilterKeep: // easy, just no filter. case def.FilterUse: f := filter.UidFilter{out.Filters.Uid} filterOptions = append(filterOptions, integrity.UseFilter(f)) default: panic(errors.ProgrammerError.New("unhandled filter mode %v", out.Filters.UidMode)) } switch out.Filters.GidMode { case def.FilterKeep: // easy, just no filter. case def.FilterUse: f := filter.GidFilter{out.Filters.Gid} filterOptions = append(filterOptions, integrity.UseFilter(f)) default: panic(errors.ProgrammerError.New("unhandled filter mode %v", out.Filters.GidMode)) } switch out.Filters.MtimeMode { case def.FilterKeep: // easy, just no filter. case def.FilterUse: f := filter.MtimeFilter{out.Filters.Mtime} filterOptions = append(filterOptions, integrity.UseFilter(f)) default: panic(errors.ProgrammerError.New("unhandled filter mode %v", out.Filters.MtimeMode)) } scanPath := filepath.Join(rootfs, out.MountPath) journal.Info(fmt.Sprintf("Starting scan on %q", scanPath)) try.Do(func() { // todo: create validity checking api for URIs, check them all before launching anything warehouses := make([]integrity.SiloURI, len(out.Warehouses)) for i, wh := range out.Warehouses { warehouses[i] = integrity.SiloURI(wh) } // invoke transmat (blocking, potentially long time) commitID := transmat.Scan( integrity.TransmatKind(out.Type), scanPath, warehouses, filterOptions..., ) out.Hash = string(commitID) // submit report journal.Info(fmt.Sprintf("Finished scan on %q", scanPath)) scanGather <- scanReport{Output: out} }).Catch(integrity.Error, func(err *errors.Error) { journal.Warn(fmt.Sprintf("Errored scan on %q", scanPath), "error", err.Message()) scanGather <- scanReport{Err: err} }).Done() }(out) } // gather reports var results []def.Output for range outputs { report := <-scanGather if report.Err != nil { panic(report.Err) } results = append(results, report.Output) } return results }
// Run outputs func PreserveOutputs(transmat integrity.Transmat, outputs []def.Output, rootfs string, journal log15.Logger) []def.Output { // run commit on the outputs scanGather := make(chan scanReport) for _, out := range outputs { go func(out def.Output) { filterOptions := make([]integrity.MaterializerConfigurer, 0, 4) for _, name := range out.Filters { cfg := strings.Fields(name) switch cfg[0] { case "uid": f := filter.UidFilter{} if len(cfg) > 1 { f.Value, _ = strconv.Atoi(cfg[1]) } filterOptions = append(filterOptions, integrity.UseFilter(f)) case "gid": f := filter.GidFilter{} if len(cfg) > 1 { f.Value, _ = strconv.Atoi(cfg[1]) } filterOptions = append(filterOptions, integrity.UseFilter(f)) case "mtime": f := filter.MtimeFilter{} if len(cfg) > 1 { f.Value, _ = time.Parse(time.RFC3339, cfg[1]) } filterOptions = append(filterOptions, integrity.UseFilter(f)) default: continue } } scanPath := filepath.Join(rootfs, out.Location) journal.Info(fmt.Sprintf("Starting scan on %q", scanPath)) try.Do(func() { // TODO: following is hack; badly need to update config parsing to understand this first-class warehouseCoordsList := make([]integrity.SiloURI, 0) if out.URI != "" { warehouseCoordsList = append(warehouseCoordsList, integrity.SiloURI(out.URI)) } // invoke transmat commitID := transmat.Scan( integrity.TransmatKind(out.Type), scanPath, warehouseCoordsList, filterOptions..., ) out.Hash = string(commitID) journal.Info(fmt.Sprintf("Finished scan on %q", scanPath)) scanGather <- scanReport{Output: out} }).Catch(integrity.Error, func(err *errors.Error) { journal.Warn(fmt.Sprintf("Errored scan on %q", scanPath), "error", err.Message()) scanGather <- scanReport{Err: err} }).Done() }(out) } // gather reports var results []def.Output for range outputs { report := <-scanGather if report.Err != nil { panic(report.Err) } results = append(results, report.Output) } return results }