func getConfiguredFormFieldNames(formName string, v *viper.Viper) []string {
configuredFields := v.GetStringMapString("stormpath.web." + formName + ".form.fields")
fieldOrder := v.GetStringSlice("stormpath.web." + formName + ".form.fieldOrder")
for fieldName := range configuredFields {
if !contains(fieldOrder, fieldName) {
fieldOrder = append(fieldOrder, fieldName)
}
}
return fieldOrder
}
// ParseStorage tries to parse out Storage from a Viper. If backend and
// URL are not provided, returns a nil pointer. Storage is required (if
// a backend is not provided, an error will be returned.)
func ParseStorage(configuration *viper.Viper, allowedBackends []string) (*Storage, error) {
store := Storage{
Backend: configuration.GetString("storage.backend"),
Source: configuration.GetString("storage.db_url"),
}
supported := false
store.Backend = strings.ToLower(store.Backend)
for _, backend := range allowedBackends {
if backend == store.Backend {
supported = true
break
}
}
if !supported {
return nil, fmt.Errorf(
"must specify one of these supported backends: %s",
strings.Join(allowedBackends, ", "))
}
if store.Backend == MemoryBackend {
return &Storage{Backend: MemoryBackend}, nil
}
if store.Source == "" {
return nil, fmt.Errorf(
"must provide a non-empty database source for %s", store.Backend)
}
return &store, nil
}
func maybeAutoPublish(cmd *cobra.Command, doPublish bool, gun string, config *viper.Viper, passRetriever notary.PassRetriever) error {
if !doPublish {
return nil
}
// We need to set up a http RoundTripper when publishing
rt, err := getTransport(config, gun, readWrite)
if err != nil {
return err
}
trustPin, err := getTrustPinning(config)
if err != nil {
return err
}
nRepo, err := notaryclient.NewFileCachedNotaryRepository(
config.GetString("trust_dir"), gun, getRemoteTrustServer(config), rt, passRetriever, trustPin)
if err != nil {
return err
}
cmd.Println("Auto-publishing changes to", gun)
return publishAndPrintToCLI(cmd, nRepo, gun)
}
func setDefaultConfigValuesWithViper(v *viper.Viper, b *BgpConfigSet) error {
if v == nil {
v = viper.New()
}
if err := SetDefaultGlobalConfigValues(&b.Global); err != nil {
return err
}
for idx, server := range b.BmpServers {
if server.Config.Port == 0 {
server.Config.Port = bmp.BMP_DEFAULT_PORT
}
b.BmpServers[idx] = server
}
if b.Zebra.Config.Url == "" {
b.Zebra.Config.Url = "unix:/var/run/quagga/zserv.api"
}
list, err := extractArray(v.Get("neighbors"))
if err != nil {
return err
}
for idx, n := range b.Neighbors {
vv := viper.New()
if len(list) > idx {
vv.Set("neighbor", list[idx])
}
if err := setDefaultNeighborConfigValuesWithViper(vv, &n, b.Global.Config.As); err != nil {
return err
}
b.Neighbors[idx] = n
}
for idx, r := range b.RpkiServers {
if r.Config.Port == 0 {
b.RpkiServers[idx].Config.Port = rtr.RPKI_DEFAULT_PORT
}
}
list, err = extractArray(v.Get("policy-definitions"))
if err != nil {
return err
}
for idx, p := range b.PolicyDefinitions {
vv := viper.New()
if len(list) > idx {
vv.Set("policy", list[idx])
}
if err := setDefaultPolicyConfigValuesWithViper(vv, &p); err != nil {
return err
}
b.PolicyDefinitions[idx] = p
}
return nil
}
func (k *keyCommander) getKeyStores(
config *viper.Viper, withHardware, hardwareBackup bool) ([]trustmanager.KeyStore, error) {
retriever := k.getRetriever()
directory := config.GetString("trust_dir")
fileKeyStore, err := trustmanager.NewKeyFileStore(directory, retriever)
if err != nil {
return nil, fmt.Errorf(
"Failed to create private key store in directory: %s", directory)
}
ks := []trustmanager.KeyStore{fileKeyStore}
if withHardware {
var yubiStore trustmanager.KeyStore
if hardwareBackup {
yubiStore, err = getYubiStore(fileKeyStore, retriever)
} else {
yubiStore, err = getYubiStore(nil, retriever)
}
if err == nil && yubiStore != nil {
// Note that the order is important, since we want to prioritize
// the yubikey store
ks = []trustmanager.KeyStore{yubiStore, fileKeyStore}
}
}
return ks, nil
}
func newObcBatch(id uint64, config *viper.Viper, stack consensus.Stack) *obcBatch {
var err error
op := &obcBatch{
obcGeneric: obcGeneric{stack: stack},
}
op.persistForward.persistor = stack
logger.Debugf("Replica %d obtaining startup information", id)
op.manager = events.NewManagerImpl() // TODO, this is hacky, eventually rip it out
op.manager.SetReceiver(op)
etf := events.NewTimerFactoryImpl(op.manager)
op.pbft = newPbftCore(id, config, op, etf)
op.manager.Start()
blockchainInfoBlob := stack.GetBlockchainInfoBlob()
op.externalEventReceiver.manager = op.manager
op.broadcaster = newBroadcaster(id, op.pbft.N, op.pbft.f, op.pbft.broadcastTimeout, stack)
op.manager.Queue() <- workEvent(func() {
op.pbft.stateTransfer(&stateUpdateTarget{
checkpointMessage: checkpointMessage{
seqNo: op.pbft.lastExec,
id: blockchainInfoBlob,
},
})
})
op.batchSize = config.GetInt("general.batchsize")
op.batchStore = nil
op.batchTimeout, err = time.ParseDuration(config.GetString("general.timeout.batch"))
if err != nil {
panic(fmt.Errorf("Cannot parse batch timeout: %s", err))
}
logger.Infof("PBFT Batch size = %d", op.batchSize)
logger.Infof("PBFT Batch timeout = %v", op.batchTimeout)
if op.batchTimeout >= op.pbft.requestTimeout {
op.pbft.requestTimeout = 3 * op.batchTimeout / 2
logger.Warningf("Configured request timeout must be greater than batch timeout, setting to %v", op.pbft.requestTimeout)
}
if op.pbft.requestTimeout >= op.pbft.nullRequestTimeout && op.pbft.nullRequestTimeout != 0 {
op.pbft.nullRequestTimeout = 3 * op.pbft.requestTimeout / 2
logger.Warningf("Configured null request timeout must be greater than request timeout, setting to %v", op.pbft.nullRequestTimeout)
}
op.incomingChan = make(chan *batchMessage)
op.batchTimer = etf.CreateTimer()
op.reqStore = newRequestStore()
op.deduplicator = newDeduplicator()
op.idleChan = make(chan struct{})
close(op.idleChan) // TODO remove eventually
return op
}
func NewOBCExecutor(config *viper.Viper, orderer Orderer, stack statetransfer.PartialStack) (obcex *obcExecutor) {
var err error
obcex = &obcExecutor{}
queueSize := config.GetInt("executor.queuesize")
if queueSize <= 0 {
panic("Queue size must be positive")
}
obcex.executionQueue = make(chan *transaction, queueSize)
obcex.syncTargets = make(chan *syncTarget)
obcex.completeSync = make(chan *syncTarget)
obcex.threadIdle = make(chan struct{})
obcex.threadExit = make(chan struct{})
obcex.orderer = orderer
obcex.executorStack = stack
obcex.id, _, err = stack.GetNetworkHandles()
if nil != err {
logger.Error("Could not resolve our own PeerID, assigning dummy")
obcex.id = &pb.PeerID{"Dummy"}
}
logger.Info("Executor for %v using queue size of %d", obcex.id, queueSize)
obcex.sts = statetransfer.NewStateTransferState(config, stack)
listener := struct{ statetransfer.ProtoListener }{}
listener.CompletedImpl = obcex.stateTransferCompleted
obcex.sts.RegisterListener(&listener)
go obcex.queueThread()
return
}
func makePBFTNetwork(N int, config *viper.Viper) *pbftNetwork {
if config == nil {
config = loadConfig()
}
config.Set("general.N", N)
config.Set("general.f", (N-1)/3)
endpointFunc := func(id uint64, net *testnet) endpoint {
tep := makeTestEndpoint(id, net)
pe := &pbftEndpoint{
testEndpoint: tep,
manager: events.NewManagerImpl(),
}
pe.sc = &simpleConsumer{
pe: pe,
}
pe.pbft = newPbftCore(id, config, pe.sc, events.NewTimerFactoryImpl(pe.manager))
pe.manager.SetReceiver(pe.pbft)
pe.manager.Start()
return pe
}
pn := &pbftNetwork{testnet: makeTestnet(N, endpointFunc)}
pn.pbftEndpoints = make([]*pbftEndpoint, len(pn.endpoints))
for i, ep := range pn.endpoints {
pn.pbftEndpoints[i] = ep.(*pbftEndpoint)
pn.pbftEndpoints[i].sc.pbftNet = pn
}
return pn
}
func collectFlags(v *viper.Viper, cmd *cobra.Command) {
v.BindPFlags(cmd.PersistentFlags())
v.BindPFlags(cmd.Flags())
for _, cmd := range cmd.Commands() {
collectFlags(v, cmd)
}
}
// Parse the cache configurations for GET-ting current and checksummed metadata,
// returning the configuration for current (non-content-addressed) metadata
// first, then the configuration for consistent (content-addressed) metadata
// second. The configuration consists mainly of the max-age (an integer in seconds,
// just like in the Cache-Control header) for each type of metadata.
// The max-age must be between 0 and 31536000 (one year in seconds, which is
// the recommended maximum time data is cached), else parsing will return an error.
// A max-age of 0 will disable caching for that type of download (consistent or current).
func getCacheConfig(configuration *viper.Viper) (current, consistent utils.CacheControlConfig, err error) {
cccs := make(map[string]utils.CacheControlConfig)
currentOpt, consistentOpt := "current_metadata", "consistent_metadata"
defaults := map[string]int{
currentOpt: int(notary.CurrentMetadataCacheMaxAge.Seconds()),
consistentOpt: int(notary.ConsistentMetadataCacheMaxAge.Seconds()),
}
maxMaxAge := int(notary.CacheMaxAgeLimit.Seconds())
for optionName, seconds := range defaults {
m := configuration.GetString(fmt.Sprintf("caching.max_age.%s", optionName))
if m != "" {
seconds, err = strconv.Atoi(m)
if err != nil || seconds < 0 || seconds > maxMaxAge {
return nil, nil, fmt.Errorf(
"must specify a cache-control max-age between 0 and %v", maxMaxAge)
}
}
cccs[optionName] = utils.NewCacheControlConfig(seconds, optionName == currentOpt)
}
current = cccs[currentOpt]
consistent = cccs[consistentOpt]
return
}
func newObcBatch(id uint64, config *viper.Viper, stack consensus.Stack) *obcBatch {
var err error
op := &obcBatch{
obcGeneric: obcGeneric{stack},
stack: stack,
}
op.persistForward.persistor = stack
logger.Debug("Replica %d obtaining startup information", id)
op.pbft = newPbftCore(id, config, op)
op.batchSize = config.GetInt("general.batchSize")
op.batchStore = nil
op.batchTimeout, err = time.ParseDuration(config.GetString("general.timeout.batch"))
if err != nil {
panic(fmt.Errorf("Cannot parse batch timeout: %s", err))
}
op.incomingChan = make(chan *batchMessage)
// create non-running timer
op.batchTimer = time.NewTimer(100 * time.Hour) // XXX ugly
op.batchTimer.Stop()
op.idleChan = make(chan struct{})
go op.main()
return op
}
func getKeysRecursively(base string, v *viper.Viper, nodeKeys map[string]interface{}) map[string]interface{} {
result := make(map[string]interface{})
for key := range nodeKeys {
fqKey := base + key
val := v.Get(fqKey)
if m, ok := val.(map[interface{}]interface{}); ok {
logger.Debugf("Found map[interface{}]interface{} value for %s", fqKey)
tmp := make(map[string]interface{})
for ik, iv := range m {
cik, ok := ik.(string)
if !ok {
panic("Non string key-entry")
}
tmp[cik] = iv
}
result[key] = getKeysRecursively(fqKey+".", v, tmp)
} else if m, ok := val.(map[string]interface{}); ok {
logger.Debugf("Found map[string]interface{} value for %s", fqKey)
result[key] = getKeysRecursively(fqKey+".", v, m)
} else {
logger.Debugf("Found real value for %s setting to %T %v", fqKey, val, val)
result[key] = val
}
}
return result
}
// Open and return a listening unix socket.
func NewUnixListener(v *viper.Viper) (*net.UnixListener, error) {
l, err := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: v.GetString("socket")})
if err != nil {
return nil, err
}
return l, nil
}
// GetPathRelativeToConfig gets a configuration key which is a path, and if
// it is not empty or an absolute path, returns the absolute path relative
// to the configuration file
func GetPathRelativeToConfig(configuration *viper.Viper, key string) string {
configFile := configuration.ConfigFileUsed()
p := configuration.GetString(key)
if p == "" || filepath.IsAbs(p) {
return p
}
return filepath.Clean(filepath.Join(filepath.Dir(configFile), p))
}
func loadBoolPtr(key string, v *viper.Viper) *bool {
val := v.Get(key)
if val == nil {
return nil
}
b := v.GetBool(key)
return &b
}
func AddDefaults(c *viper.Viper) {
c.SetDefault("marathon", kv("host", "http://localhost:8080"))
c.SetDefault("mesos", kv("master", "localhost:5050"))
c.SetDefault("zk", kv("host", "localhost:2181"))
cachePath, _ := homedir.Expand("~/.marathonctl/packages")
c.SetDefault("package-cache-path", cachePath)
c.SetDefault("package-repo", "github.com/ashwanthkumar/marathonctl-universe")
}
// New is the constructor for Application struct.
func New(config *viper.Viper) (*Application, error) {
cookieStoreSecret := config.Get("cookie_secret").(string)
app := &Application{}
app.config = config
app.cookieStore = sessions.NewCookieStore([]byte(cookieStoreSecret))
return app, nil
}
// ParseLogLevel tries to parse out a log level from a Viper. If there is no
// configuration, defaults to the provided error level
func ParseLogLevel(configuration *viper.Viper, defaultLevel logrus.Level) (
logrus.Level, error) {
logStr := configuration.GetString("logging.level")
if logStr == "" {
return defaultLevel, nil
}
return logrus.ParseLevel(logStr)
}
func newInstance(v *viper.Viper) *instance {
var i = instance{
viper: v,
slots: v.GetInt("start.parallel"),
}
i.slotAvailable = sync.NewCond(&i.m)
i.taskFinished = sync.NewCond(&i.m)
return &i
}
// gets the required gun prefixes accepted by this server
func getRequiredGunPrefixes(configuration *viper.Viper) ([]string, error) {
prefixes := configuration.GetStringSlice("repositories.gun_prefixes")
for _, prefix := range prefixes {
p := path.Clean(strings.TrimSpace(prefix))
if p+"/" != prefix || strings.HasPrefix(p, "/") || strings.HasPrefix(p, "..") {
return nil, fmt.Errorf("invalid GUN prefix %s", prefix)
}
}
return prefixes, nil
}
func ThreadlessNewStateTransferState(config *viper.Viper, stack PartialStack) *StateTransferState {
var err error
sts := &StateTransferState{}
sts.stateTransferListenersLock = &sync.Mutex{}
sts.stack = stack
sts.id, _, err = stack.GetNetworkHandles()
if nil != err {
logger.Debug("Error resolving our own PeerID, this shouldn't happen")
sts.id = &protos.PeerID{"ERROR_RESOLVING_ID"}
}
sts.asynchronousTransferInProgress = false
sts.RecoverDamage = config.GetBool("statetransfer.recoverdamage")
sts.stateValid = true // Assume our starting state is correct unless told otherwise
sts.validBlockRanges = make([]*blockRange, 0)
sts.blockVerifyChunkSize = uint64(config.GetInt("statetransfer.blocksperrequest"))
if sts.blockVerifyChunkSize == 0 {
panic(fmt.Errorf("Must set statetransfer.blocksperrequest to be nonzero"))
}
sts.initiateStateSync = make(chan *syncMark)
sts.blockHashReceiver = make(chan *blockHashReply, 1)
sts.blockSyncReq = make(chan *blockSyncReq)
sts.threadExit = make(chan struct{})
sts.blockThreadIdle = true
sts.stateThreadIdle = true
sts.blockThreadIdleChan = make(chan struct{})
sts.stateThreadIdleChan = make(chan struct{})
sts.DiscoveryThrottleTime = 1 * time.Second // TODO make this configurable
sts.BlockRequestTimeout, err = time.ParseDuration(config.GetString("statetransfer.timeout.singleblock"))
if err != nil {
panic(fmt.Errorf("Cannot parse statetransfer.timeout.singleblock timeout: %s", err))
}
sts.StateDeltaRequestTimeout, err = time.ParseDuration(config.GetString("statetransfer.timeout.singlestatedelta"))
if err != nil {
panic(fmt.Errorf("Cannot parse statetransfer.timeout.singlestatedelta timeout: %s", err))
}
sts.StateSnapshotRequestTimeout, err = time.ParseDuration(config.GetString("statetransfer.timeout.fullstate"))
if err != nil {
panic(fmt.Errorf("Cannot parse statetransfer.timeout.fullstate timeout: %s", err))
}
return sts
}
func newPbftCore(id uint64, config *viper.Viper, consumer innerCPI) *pbftCore {
instance := &pbftCore{}
instance.id = id
instance.consumer = consumer
// in dev/debugging mode you are expected to override the config values
// with the environment variable OPENCHAIN_OBCPBFT_X_Y
// read from the config file
// you can override the config values with the environment variable prefix
// OPENCHAIN_OBCPBFT, e.g. OPENCHAIN_OBCPBFT_BYZANTINE
var err error
instance.byzantine = config.GetBool("replica.byzantine")
instance.f = config.GetInt("general.f")
instance.K = uint64(config.GetInt("general.K"))
instance.requestTimeout, err = time.ParseDuration(config.GetString("general.timeout.request"))
if err != nil {
panic(fmt.Errorf("Cannot parse request timeout: %s", err))
}
instance.newViewTimeout, err = time.ParseDuration(config.GetString("general.timeout.viewchange"))
if err != nil {
panic(fmt.Errorf("Cannot parse new view timeout: %s", err))
}
instance.activeView = true
instance.L = 2 * instance.K // log size
instance.replicaCount = 3*instance.f + 1
// init the logs
instance.certStore = make(map[msgID]*msgCert)
instance.reqStore = make(map[string]*Request)
instance.checkpointStore = make(map[Checkpoint]bool)
instance.chkpts = make(map[uint64]string)
instance.viewChangeStore = make(map[vcidx]*ViewChange)
instance.pset = make(map[uint64]*ViewChange_PQ)
instance.qset = make(map[qidx]*ViewChange_PQ)
instance.newViewStore = make(map[uint64]*NewView)
// load genesis checkpoint
stateHash, err := instance.consumer.getStateHash(0)
if err != nil {
panic(fmt.Errorf("Cannot load genesis block: %s", err))
}
instance.chkpts[0] = base64.StdEncoding.EncodeToString(stateHash)
// create non-running timer XXX ugly
instance.newViewTimer = time.NewTimer(100 * time.Hour)
instance.newViewTimer.Stop()
instance.lastNewViewTimeout = instance.newViewTimeout
instance.outstandingReqs = make(map[string]*Request)
go instance.timerHander()
return instance
}
func getRemoteTrustServer(config *viper.Viper) string {
if remoteTrustServer == "" {
configRemote := config.GetString("remote_server.url")
if configRemote != "" {
remoteTrustServer = configRemote
} else {
remoteTrustServer = defaultServerURL
}
}
return remoteTrustServer
}
// get the address for the HTTP server, and parses the optional TLS
// configuration for the server - if no TLS configuration is specified,
// TLS is not enabled.
func getAddrAndTLSConfig(configuration *viper.Viper) (string, *tls.Config, error) {
httpAddr := configuration.GetString("server.http_addr")
if httpAddr == "" {
return "", nil, fmt.Errorf("http listen address required for server")
}
tlsConfig, err := utils.ParseServerTLS(configuration, false)
if err != nil {
return "", nil, fmt.Errorf(err.Error())
}
return httpAddr, tlsConfig, nil
}
func (c *Conf) setupHome(viperConf *viper.Viper, homePath string) {
configPath := "config"
if len(homePath) > 0 {
configPath = homePath
}
if viperConf != nil {
viperConf.AddConfigPath(configPath)
} else {
viper.AddConfigPath(configPath)
}
}
func getAddrAndTLSConfig(configuration *viper.Viper) (string, *tls.Config, error) {
tlsConfig, err := utils.ParseServerTLS(configuration, true)
if err != nil {
return "", nil, fmt.Errorf("unable to set up TLS: %s", err.Error())
}
grpcAddr := configuration.GetString("server.grpc_addr")
if grpcAddr == "" {
return "", nil, fmt.Errorf("grpc listen address required for server")
}
return grpcAddr, tlsConfig, nil
}
func getDefaultAlias(configuration *viper.Viper) (string, error) {
defaultAlias := configuration.GetString("storage.default_alias")
if defaultAlias == "" {
// backwards compatibility - support this environment variable
defaultAlias = configuration.GetString(defaultAliasEnv)
}
if defaultAlias == "" {
return "", fmt.Errorf("must provide a default alias for the key DB")
}
logrus.Debug("Default Alias: ", defaultAlias)
return defaultAlias, nil
}
func configureLogging(v *viper.Viper) {
level, err := log.ParseLevel(v.GetString("log_level"))
if err != nil {
log.Fatalln(err)
}
log.SetLevel(level)
if v.GetString("log_format") == "text" {
log.SetFormatter(&log.TextFormatter{DisableColors: true, FullTimestamp: true})
} else if v.GetString("log_format") == "json" {
log.SetFormatter(&log.JSONFormatter{})
} else {
log.Errorln("Error: log_type invalid, defaulting to text")
log.SetFormatter(&log.TextFormatter{})
}
switch v.GetString("log_target") {
case "stdout":
log.SetOutput(os.Stdout)
case "stderr":
log.SetOutput(os.Stderr)
default:
log.Errorln("Error: log_target invalid, defaulting to Stdout")
log.SetOutput(os.Stdout)
}
}
func NewServer(cfg *viper.Viper) (*WalkhubServer, error) {
cfg.Set("root", false)
b, err := ab.PetBunny(cfg, nil, prometheusMiddleware())
if err != nil {
return nil, err
}
s := &WalkhubServer{
Server: b,
cfg: cfg,
}
return s, nil
}
func setDefaultPolicyConfigValuesWithViper(v *viper.Viper, p *PolicyDefinition) error {
stmts, err := extractArray(v.Get("policy.statements"))
if err != nil {
return err
}
for i, _ := range p.Statements {
vv := viper.New()
if len(stmts) > i {
vv.Set("statement", stmts[i])
}
if !vv.IsSet("statement.actions.route-disposition") {
p.Statements[i].Actions.RouteDisposition = ROUTE_DISPOSITION_NONE
}
}
return nil
}