示例#1
0
// Run inputs
func ProvisionInputs(transmat integrity.Transmat, assemblerFn integrity.Assembler, inputs []def.Input, rootfs string, journal log15.Logger) integrity.Assembly {
	// start having all filesystems
	filesystems := make(map[def.Input]integrity.Arena, len(inputs))
	fsGather := make(chan map[def.Input]materializerReport)
	for _, in := range inputs {
		go func(in def.Input) {
			try.Do(func() {
				journal.Info(fmt.Sprintf("Starting materialize for %s hash=%s", in.Type, in.Hash))
				arena := transmat.Materialize(
					integrity.TransmatKind(in.Type),
					integrity.CommitID(in.Hash),
					[]integrity.SiloURI{integrity.SiloURI(in.URI)},
				)
				journal.Info(fmt.Sprintf("Finished materialize for %s hash=%s", in.Type, in.Hash))
				fsGather <- map[def.Input]materializerReport{
					in: {Arena: arena},
				}
			}).Catch(integrity.Error, func(err *errors.Error) {
				journal.Warn(fmt.Sprintf("Errored during materialize for %s hash=%s", in.Type, in.Hash), "error", err.Message())
				fsGather <- map[def.Input]materializerReport{
					in: {Err: err},
				}
			}).Done()
		}(in)
	}

	// (we don't have any output setup at this point, but if we do in the future, that'll be here.)

	// gather materialized inputs
	for range inputs {
		for in, report := range <-fsGather {
			if report.Err != nil {
				panic(report.Err)
			}
			filesystems[in] = report.Arena
		}
	}
	journal.Info("All inputs acquired... starting assembly")

	// assemble them into the final tree
	assemblyParts := make([]integrity.AssemblyPart, 0, len(filesystems))
	for input, arena := range filesystems {
		assemblyParts = append(assemblyParts, integrity.AssemblyPart{
			SourcePath: arena.Path(),
			TargetPath: input.Location,
			Writable:   true, // TODO input config should have a word about this
		})
	}
	assembly := assemblerFn(rootfs, assemblyParts)
	journal.Info("Assembly complete!")
	return assembly
}
示例#2
0
func processPendingAllocations(srv *context.T, log log15.Logger) {
	pending := srv.DB.ListPendingRepositories()
	for i := range pending {
		r, err := srv.DB.GetRepository(pending[i])
		if err != nil {
			log.Error("error while processing pending repository", "repo", pending[i], "err", err)
		}
		c, err := srv.DB.GetTransactionConfirmations(r.NameAllocationTx)
		if err != nil {
			log.Error("error while calculating pending repository's allocation confirmations",
				"repo", pending[i], "txn", r.NameAllocationTx, "err", err)
		}
		if c >= ALLOCATION_CONFIRMATIONS_REQUIRED {
			r.Status = repository.ACTIVE
			srv.DB.PutRepository(r)
			log.Info("activated repository", "repo", pending[i], "alloc_txn", r.NameAllocationTx)
		}

	}
}
func listenAndServer(log log15.Logger, addr string, handler http.Handler) error {
	conf, err := config.GetConfig()
	if err != nil {
		return err
	}

	conf.Endless.DefaultHammerTime = strings.TrimSpace(conf.Endless.DefaultHammerTime)

	if conf.Endless.DefaultHammerTime != "" {
		duration, err := time.ParseDuration(conf.Endless.DefaultHammerTime)
		if err == nil {
			endless.DefaultHammerTime = duration
		} else {
			log.Error("Bad format", log15.Ctx{"module": "Endless", "DefaultHammerTime": conf.Endless.DefaultHammerTime, "error": err})
		}
	}

	var terminated int32
	srv := endless.NewServer(addr, handler)
	preHookFunc := func() {
		atomic.StoreInt32(&terminated, 1)
	}
	srv.RegisterSignalHook(endless.PRE_SIGNAL, syscall.SIGHUP, preHookFunc)
	srv.RegisterSignalHook(endless.PRE_SIGNAL, syscall.SIGINT, preHookFunc)
	srv.RegisterSignalHook(endless.PRE_SIGNAL, syscall.SIGTERM, preHookFunc)

	log.Info("Launching server")
	err = srv.ListenAndServe()
	if atomic.LoadInt32(&terminated) == 0 {
		if err != nil {
			log.Error("During startup, error has occurred", "error", err)
		}
		return err
	} else {
		log.Info("Server is going to shutdown")
		return nil
	}
}
示例#4
0
// Run inputs
func ProvisionInputs(transmat integrity.Transmat, assemblerFn integrity.Assembler, inputs []def.Input, rootfs string, journal log15.Logger) integrity.Assembly {
	// start having all filesystems
	// input names are used as keys, so must be unique
	inputsByName := make(map[string]def.Input, len(inputs))
	for _, in := range inputs {
		// TODO checks should also be sooner, up in cfg parse
		// but this check is for programmatic access as well (errors down the line can get nonobvious if you skip this).
		if _, ok := inputsByName[in.Name]; ok {
			panic(errors.ProgrammerError.New("duplicate name in input config"))
		}
		inputsByName[in.Name] = in
	}
	filesystems := make(map[string]integrity.Arena, len(inputs))
	fsGather := make(chan map[string]materializerReport)
	for _, in := range inputs {
		go func(in def.Input) {
			try.Do(func() {
				journal.Info(fmt.Sprintf("Starting materialize for %s hash=%s", in.Type, in.Hash))
				// todo: create validity checking api for URIs, check them all before launching anything
				warehouses := make([]integrity.SiloURI, len(in.Warehouses))
				for i, wh := range in.Warehouses {
					warehouses[i] = integrity.SiloURI(wh)
				}
				// invoke transmat (blocking, potentially long time)
				arena := transmat.Materialize(
					integrity.TransmatKind(in.Type),
					integrity.CommitID(in.Hash),
					warehouses,
				)
				// submit report
				journal.Info(fmt.Sprintf("Finished materialize for %s hash=%s", in.Type, in.Hash))
				fsGather <- map[string]materializerReport{
					in.Name: {Arena: arena},
				}
			}).Catch(integrity.Error, func(err *errors.Error) {
				journal.Warn(fmt.Sprintf("Errored during materialize for %s hash=%s", in.Type, in.Hash), "error", err.Message())
				fsGather <- map[string]materializerReport{
					in.Name: {Err: err},
				}
			}).Done()
		}(in)
	}

	// (we don't have any output setup at this point, but if we do in the future, that'll be here.)

	// gather materialized inputs
	for range inputs {
		for name, report := range <-fsGather {
			if report.Err != nil {
				panic(report.Err)
			}
			filesystems[name] = report.Arena
		}
	}
	journal.Info("All inputs acquired... starting assembly")

	// assemble them into the final tree
	assemblyParts := make([]integrity.AssemblyPart, 0, len(filesystems))
	for name, arena := range filesystems {
		assemblyParts = append(assemblyParts, integrity.AssemblyPart{
			SourcePath: arena.Path(),
			TargetPath: inputsByName[name].MountPath,
			Writable:   true, // TODO input config should have a word about this
		})
	}
	assembly := assemblerFn(rootfs, assemblyParts)
	journal.Info("Assembly complete!")
	return assembly
}
示例#5
0
// Run outputs
func PreserveOutputs(transmat integrity.Transmat, outputs []def.Output, rootfs string, journal log15.Logger) []def.Output {
	// run commit on the outputs
	scanGather := make(chan scanReport)
	for _, out := range outputs {
		go func(out def.Output) {
			filterOptions := make([]integrity.MaterializerConfigurer, 0, 3)
			out.Filters.InitDefaultsOutput()
			switch out.Filters.UidMode {
			case def.FilterKeep: // easy, just no filter.
			case def.FilterUse:
				f := filter.UidFilter{out.Filters.Uid}
				filterOptions = append(filterOptions, integrity.UseFilter(f))
			default:
				panic(errors.ProgrammerError.New("unhandled filter mode %v", out.Filters.UidMode))
			}
			switch out.Filters.GidMode {
			case def.FilterKeep: // easy, just no filter.
			case def.FilterUse:
				f := filter.GidFilter{out.Filters.Gid}
				filterOptions = append(filterOptions, integrity.UseFilter(f))
			default:
				panic(errors.ProgrammerError.New("unhandled filter mode %v", out.Filters.GidMode))
			}
			switch out.Filters.MtimeMode {
			case def.FilterKeep: // easy, just no filter.
			case def.FilterUse:
				f := filter.MtimeFilter{out.Filters.Mtime}
				filterOptions = append(filterOptions, integrity.UseFilter(f))
			default:
				panic(errors.ProgrammerError.New("unhandled filter mode %v", out.Filters.MtimeMode))
			}

			scanPath := filepath.Join(rootfs, out.MountPath)
			journal.Info(fmt.Sprintf("Starting scan on %q", scanPath))
			try.Do(func() {
				// todo: create validity checking api for URIs, check them all before launching anything
				warehouses := make([]integrity.SiloURI, len(out.Warehouses))
				for i, wh := range out.Warehouses {
					warehouses[i] = integrity.SiloURI(wh)
				}
				// invoke transmat (blocking, potentially long time)
				commitID := transmat.Scan(
					integrity.TransmatKind(out.Type),
					scanPath,
					warehouses,
					filterOptions...,
				)
				out.Hash = string(commitID)
				// submit report
				journal.Info(fmt.Sprintf("Finished scan on %q", scanPath))
				scanGather <- scanReport{Output: out}
			}).Catch(integrity.Error, func(err *errors.Error) {
				journal.Warn(fmt.Sprintf("Errored scan on %q", scanPath), "error", err.Message())
				scanGather <- scanReport{Err: err}
			}).Done()
		}(out)
	}

	// gather reports
	var results []def.Output
	for range outputs {
		report := <-scanGather
		if report.Err != nil {
			panic(report.Err)
		}
		results = append(results, report.Output)
	}

	return results
}
func listenAndServer(log log15.Logger, addr string, handler http.Handler) error {
	log.Info("Launching server")
	err := http.ListenAndServe(addr, handler)
	log.Error("During startup, error has occurred", "error", err)
	return err
}
示例#7
0
// Run outputs
func PreserveOutputs(transmat integrity.Transmat, outputs []def.Output, rootfs string, journal log15.Logger) []def.Output {
	// run commit on the outputs
	scanGather := make(chan scanReport)
	for _, out := range outputs {
		go func(out def.Output) {
			filterOptions := make([]integrity.MaterializerConfigurer, 0, 4)
			for _, name := range out.Filters {
				cfg := strings.Fields(name)
				switch cfg[0] {
				case "uid":
					f := filter.UidFilter{}
					if len(cfg) > 1 {
						f.Value, _ = strconv.Atoi(cfg[1])
					}
					filterOptions = append(filterOptions, integrity.UseFilter(f))
				case "gid":
					f := filter.GidFilter{}
					if len(cfg) > 1 {
						f.Value, _ = strconv.Atoi(cfg[1])
					}
					filterOptions = append(filterOptions, integrity.UseFilter(f))
				case "mtime":
					f := filter.MtimeFilter{}
					if len(cfg) > 1 {
						f.Value, _ = time.Parse(time.RFC3339, cfg[1])
					}
					filterOptions = append(filterOptions, integrity.UseFilter(f))
				default:
					continue
				}
			}
			scanPath := filepath.Join(rootfs, out.Location)
			journal.Info(fmt.Sprintf("Starting scan on %q", scanPath))
			try.Do(func() {
				// TODO: following is hack; badly need to update config parsing to understand this first-class
				warehouseCoordsList := make([]integrity.SiloURI, 0)
				if out.URI != "" {
					warehouseCoordsList = append(warehouseCoordsList, integrity.SiloURI(out.URI))
				}
				// invoke transmat
				commitID := transmat.Scan(
					integrity.TransmatKind(out.Type),
					scanPath,
					warehouseCoordsList,
					filterOptions...,
				)
				out.Hash = string(commitID)
				journal.Info(fmt.Sprintf("Finished scan on %q", scanPath))
				scanGather <- scanReport{Output: out}
			}).Catch(integrity.Error, func(err *errors.Error) {
				journal.Warn(fmt.Sprintf("Errored scan on %q", scanPath), "error", err.Message())
				scanGather <- scanReport{Err: err}
			}).Done()
		}(out)
	}

	// gather reports
	var results []def.Output
	for range outputs {
		report := <-scanGather
		if report.Err != nil {
			panic(report.Err)
		}
		results = append(results, report.Output)
	}

	return results
}