func ParseConfig(config *yaml.File) (*Config, error) { base, err := config.Get("base") if err != nil { return nil, err } var rpm *RpmPackage = nil rpmBaseNode, err := yaml.Child(config.Root, "rpm-base") if err != nil { return nil, err } if rpmBaseNode != nil { rpmBaseMap := rpmBaseNode.(yaml.Map) scalar := rpmBaseMap["name"].(yaml.Scalar) name := strings.TrimSpace(scalar.String()) scalar = rpmBaseMap["version"].(yaml.Scalar) version := strings.TrimSpace(scalar.String()) scalar = rpmBaseMap["release"].(yaml.Scalar) release := strings.TrimSpace(scalar.String()) scalar = rpmBaseMap["arch"].(yaml.Scalar) arch := strings.TrimSpace(scalar.String()) rpm = &RpmPackage{ Name: name, Version: version, Release: release, Arch: arch, } } cmdline, err := config.Get("cmdline") if err != nil { return nil, err } build, _ := config.Get("build") filesNode, err := yaml.Child(config.Root, "files") if err != nil { return nil, err } files := make(map[string]string) if filesNode != nil { filesMap := filesNode.(yaml.Map) for key, value := range filesMap { scalar := value.(yaml.Scalar) files[key] = strings.TrimSpace(scalar.String()) } } result := &Config{ Base: base, RpmBase: rpm, Cmdline: cmdline, Build: build, Files: files, } return result, nil }
func db_params() string { data, _ := yaml.ReadFile("conf.yml") yaml_user, _ := yaml.Child(data.Root, "user") yaml_dbname, _ := yaml.Child(data.Root, "dbname") yaml_password, _ := yaml.Child(data.Root, "password") user := yaml_user.(yaml.Scalar).String() dbname := yaml_dbname.(yaml.Scalar).String() password := yaml_password.(yaml.Scalar).String() db_pars := "user="******" dbname=" + dbname + " password=" + password fmt.Println(db_pars) return db_pars }
func (c *Config) GetMapList(name string) ([]map[string]string, error) { node, err := yaml.Child(c.file.Root, name) if _, ok := err.(*yaml.NodeNotFound); ok { return nil, nil } else if err != nil { return nil, err } l, ok := node.(yaml.List) if l == nil { // Looks like we don't get NodeNotFound from Child? return nil, nil } if !ok { return nil, fmt.Errorf("Expected yaml.List, got %T\n", node) } ret := make([]map[string]string, l.Len()) for i, n := range l { ret[i] = make(map[string]string) m, ok := n.(yaml.Map) if !ok { return nil, fmt.Errorf("Expected yaml.Map, got %T\n", node) } for name, n2 := range m { s, ok := n2.(yaml.Scalar) if !ok { return nil, fmt.Errorf("Expected yaml.Scalar, got %T\n", node) } ret[i][name] = string(s) } } return ret, nil }
func init() { flag.StringVar(&configFile, "c", "proxy.yaml", "Configuration file path.") flag.BoolVar(&Authentication, "auth", true, "False if authentication should be disabled") flag.StringVar(&CpuProfile, "cpuprofile", "", "Write CPU profile to file") flag.StringVar(&MemoryProfile, "memoryprofile", "", "Write Memory profile to file") flag.Parse() Config = yaml.ConfigFile(configFile) node, err := yaml.Child(Config.Root, "server") if err != nil { panic("Server configuration missing.") } if m, ok := node.(yaml.Map); ok { for key, value := range m { if scalar, ok := value.(yaml.Scalar); ok { if key == "host" { ServerAddress = string(scalar) + ServerAddress } else if key == "port" { ServerAddress = ServerAddress + string(scalar) } } } } }
// init function, load the configs // fill english_ignore_words_map func init() { // load config file cfg_filename := "config.yaml" config, err := yaml.ReadFile(cfg_filename) if err != nil { log.Fatalf("readfile(%s): %s", cfg_filename, err) } // get english ignore entire string english_ignore, err := config.Get("english_ignore") if err != nil { log.Fatalf("%s parse error: %s\n", english_ignore, err) } // get each separated words english_ignore_words_list := strings.Fields(english_ignore) for _, word := range english_ignore_words_list { word = strings.TrimSpace(word) english_ignore_words_map[word] = 1 } // get redis connection info redis_config, err := yaml.Child(config.Root, "redis_server") if err != nil { log.Fatalf("redis config parse error: %s\n", err) } redis_config_m := redis_config.(yaml.Map) host, port := redis_config_m["host"], redis_config_m["port"] redis_conn, err = redis.Dial("tcp", fmt.Sprintf("%s:%s", host, port)) //defer redis_conn.Close() if err != nil { log.Fatalf("Can not connect to Redis Server: %s", err) } }
func getItems(config yaml.File, key string) ([]*AvailableBlock, error) { lst, err := validateItems(config, key) if err != nil { return nil, errors.New("Invalid items") } items := make([]*AvailableBlock, lst.Len()) for i, e := range lst { itemKey := key + "[" + strconv.Itoa(i) + "]" title, err := config.Get(itemKey + ".title") if err != nil { return nil, errors.New("Missing title") } series, err := config.Get(itemKey + ".series") if err != nil { series = "" } filepathsNode, err := yaml.Child(e, "filepaths") if err != nil { return nil, errors.New("Missing filepaths for " + title) } publish := true if key == "extras" { publish = false } items[i] = new(AvailableBlock).Init(title, series, filepathsNode.(yaml.List), publish) } return items, nil }
// unmarshalStruct iterates through all the fields in the field struct interface // and for each key=(yamlTagName or fieldName), finds the value in the node, // filters out the fields not listed in the supportedYamlTags map // and sets the value in the out struct using reflections func unmarshalStruct(node yaml.Node, fieldType reflect.Type, field reflect.Value, supportedYamlTags map[string]bool) error { for i := 0; i < field.NumField(); i++ { f := field.Field(i) ft := fieldType.Field(i) // get the tag name (if any), defaults to fieldName tagName := ft.Name yamlTag := ft.Tag.Get(yamlTag) // Expected format `yaml:"tagName,omitempty"` // TODO, handle omitempty if yamlTag != "" { tags := strings.Split(yamlTag, ",") if len(tags) > 0 { tagName = tags[0] } } // getNode with key=tagName childNode, err := yaml.Child(node, tagName) if err != nil || childNode == nil { // not found continue } // filter out the node with key=tagName if not in supportedYamlTags map if !supportedYamlTags[tagName] { log.WithFields(log.Fields{"option name": tagName}).Warn("Skipping unsupported YAML option...") continue } // set value if err = setValue(childNode, ft.Type, f, supportedYamlTags); err != nil { return err } } return nil }
/** * Main program. */ func main() { log.Print("Starting up concentrator") var file string = "config.yml" log.Print("Loading config file: ", file) config, err := yaml.ReadFile("config.yml") if err != nil { log.Fatalf("Error reading config.yml (%q): %s", file, err) } // Get the backends config list. servers, err := yaml.Child(config.Root, "backends") server_lst, ok := servers.(yaml.List) if !ok { log.Fatalf("Could not parse backends list") return } // Load the stats backends. for i := 0; i < server_lst.Len(); i++ { node := server_lst.Item(i) vals := node.(yaml.Map) for index, element := range vals { backend_host := fmt.Sprintf("%s", index) backend_port := fmt.Sprintf("%s", element) log.Print(fmt.Sprintf("Adding backend %s:%s", backend_host, backend_port)) Backends = append(Backends, fmt.Sprintf("%s:%s", backend_host, backend_port)) } } for _, backserver := range Backends { log.Print(fmt.Sprintf("New server is: %s", backserver)) } port, err := config.GetInt("port") host, err := config.Get("host") log.Print(fmt.Sprintf("Trying to listen on %s:%v", host, port)) relay_method, err := config.Get("relay_method") log.Print(fmt.Sprintf("We want to relay using %s", relay_method)) listener, err := net.Listen("tcp", fmt.Sprintf("%s:%v", host, port)) if err != nil { println("Error starting net.Listen: ", err.Error()) os.Exit(1) } log.Print("Server started, awaiting connections...") conns := clientConnections(listener) for { go handleConnections(<-conns) } }
func validateItems(config yaml.File, key string) (yaml.List, error) { node, err := yaml.Child(config.Root, key) if err != nil { return nil, err } lst, ok := node.(yaml.List) if !ok || (lst.Len() <= 0) { return nil, errors.New("Invalid items") } return lst, nil }
// Configure and return named "Tagger" function. func Get(name string, config *yaml.Node, log *logging.Logger) (Tagger, error) { // Check if tagger should only be used as a fallback tagger_fallback := false if config != nil { node, err := yaml.Child(*config, "fallback") if err == nil { val, ok := node.(yaml.Scalar) if ok && string(val) == "true" { tagger_fallback = true } } } // Config gets processed only once and passed to tagger as interface{} var tagger_conf interface{} tagger_conf = config tagger_confproc, ok := taggers_confproc[name] if ok { tagger_conf = tagger_confproc(name, config, log) } // Resulting Tagger is a closure created here tagger_func, ok := taggers[name] if !ok { return nil, fmt.Errorf("Unknown tagger type: %v", name) } tagger := func(path string, info os.FileInfo, ctx *map[string]interface{}) (tags []string) { // Check fallback condition if tagger_fallback { tags_prev_if, ok := (*ctx)["tags"] if ok { tags_prev, ok := tags_prev_if.(CtxTagset) if !ok { panic(fmt.Errorf("Failed to process tags from context: %v", *ctx)) } if len(tags_prev) > 0 { return } } } return tagger_func(name, tagger_conf, log, path, info, ctx) } return tagger, nil }
func CreateShoutMap() map[string]string { /* Utility function that returns a mapping that can be used to pass to shout.NewShout */ node, err := yaml.Child(Config.Root, "icecast") if err != nil { panic("Icecast configuration missing.") } icecast_map := make(map[string]string, 20) if m, ok := node.(yaml.Map); ok { for key, value := range m { if scalar, ok := value.(yaml.Scalar); ok { icecast_map[key] = string(scalar) } } } else { panic("Icecast configuration isn't a mapping.") } return icecast_map }
func tagger_scm_host_confproc(name string, config *yaml.Node, log *logging.Logger) interface{} { var err error node, err := yaml.Child(*config, "host_tags") config_map, ok := yaml.Map{}, false if err == nil { config_map, ok = node.(yaml.Map) } if !ok || len(config_map) == 0 { if ok && len(config_map) == 0 { err = fmt.Errorf("no tags defined") } else { err = fmt.Errorf("must be a map of tag:regexp") } log.Warnf("Error parsing 'host_tags' in tagger config (%v): %v", config, err) return nil } tag_map := make(map[string]*re.Regexp, len(config_map)) for k, node := range config_map { pattern, ok := node.(yaml.Scalar) if !ok { log.Warnf("Failed to parse tag-host (tag: %v): %v", k, node) continue } regexp, err := re.Compile(strings.Trim(string(pattern), "'")) if err != nil { log.Warnf("Failed to parse tag-host pattern (%v: %v): %v", k, pattern, err) continue } tag_map[k] = regexp } return tag_map }
func CreateDatabaseDSN() string { /* An utility function that returns a new DSN string. This is of the format [username[:password]@][protocol[(address)]]/dbname[?param1=value1¶mN=valueN] it is generated from the "database" header in the configuration file. The result can be passed to sql.Open for mysql usage. */ node, err := yaml.Child(Config.Root, "database") if err != nil { panic("Database configuration missing.") } DBN := "" if m, ok := node.(yaml.Map); ok { getstring := func(key string) (string, bool) { if scalar, ok := m[key].(yaml.Scalar); ok { return string(scalar), true } return "", false } if name, ok := getstring("username"); ok { DBN += name } if pass, ok := getstring("password"); ok { DBN += ":" + pass } if protocol, ok := getstring("protocol"); ok { DBN += "@" + protocol } if address, ok := getstring("host"); ok { if port, ok := getstring("port"); ok { address += ":" + port } DBN += "(" + address + ")" } if dbname, ok := getstring("dbname"); ok { DBN += "/" + dbname } else { panic("Database name is required.") } params := m["parameters"] if mp, ok := params.(yaml.Map); ok { i := 0 DBN += "?" for key, value := range mp { // Add separators if we have more than one parameter if i++; i > 1 { DBN += "&" } if scalar, ok := value.(yaml.Scalar); ok { DBN += key + "=" + string(scalar) } else { } } } } return DBN }
// Load config from specified YAML file. func loadConfig() (confs []*goose.DBConf, err error) { // migrationsDir should be placed at conf directory migrationsDir := filepath.Dir(prmConfPath) + "/" + prmGroup f, err := yaml.ReadFile(prmConfPath) if err != nil { return nil, err } rootMap, err := nodeToMap(f.Root) if err != nil { return nil, errors.New(fmt.Sprintf("conf file syntax is not valid")) } confNodes, err := nodeToList(rootMap[prmGroup]) if err != nil { return nil, errors.New(fmt.Sprintf(`group "%s" is not found on %s`, prmGroup, prmConfPath)) } confs = make([]*goose.DBConf, 0, confNodes.Len()) for i, confNode := range confNodes { conf, err := nodeToMap(confNode) if err != nil { return nil, err } drvNode, err := yaml.Child(conf, "driver") if err != nil { return nil, errors.New(fmt.Sprintf(`%s[%d]: required item "driver" is missing `, prmGroup, i)) } drv, err := nodeToString(drvNode) if err != nil { return nil, err } drv = os.ExpandEnv(drv) openNode, err := yaml.Child(conf, "open") if err != nil { return nil, errors.New(fmt.Sprintf(`%s[%d]: required item "open" is missing`, prmGroup, i)) } open, err := nodeToString(openNode) if err != nil { return nil, err } open = os.ExpandEnv(open) d, err := newDBDriver(drv, open) if err != nil { return nil, err } if !d.IsValid() { return nil, errors.New(fmt.Sprintf("Invalid DBConf: %v", d)) } confs = append(confs, &goose.DBConf{ MigrationsDir: migrationsDir, Env: fmt.Sprintf("%s[%d]", prmGroup, i), Driver: d, }) } return confs, nil }
func main() { config_search[0] = path_t(os.Args[0] + ".yaml") flag.Usage = func() { tpl := template.Must(template.New("test").Parse("" + `usage: {{.cmd}} [ <options> ] Index code files, using parameters specified in the config file. If not specified exmplicitly, config file is searched within the following paths (in that order): {{range .paths}} - {{.}} {{end}} Examples: % {{.cmd}} % {{.cmd}} --config config.yaml Options: `)) tpl.Execute(os.Stdout, map[string]interface{}{"cmd": os.Args[0], "paths": config_search}) flag.PrintDefaults() } flag.StringVar(&config_path, "config", "", "Configuration file to use.") flag.BoolVar(&dry_run, "dry-run", false, "Don't actually run tmsu, just process all paths.") flag.Parse() if flag.NArg() > 0 { fmt.Fprintf(os.Stderr, "Error: no command-line"+ " arguments are allowed (provided: %v)\n", flag.Args()) os.Exit(1) } // Find config path to use if len(config_path) == 0 { for _, path := range config_search { path, err := path.ExpandUser() if err != nil { continue } config_path = string(path) _, err = os.Stat(config_path) if err == nil { break } config_path = "" } if len(config_path) == 0 { fmt.Fprintf(os.Stderr, "Failed to find any suitable configuration file") os.Exit(1) } } var ( log *logging.Logger log_init = false config_init = false ) defer func() { if config_init { return } // Recover only for configuration issues. if err := recover(); err != nil { if log_init { log.Fatalf("Failed to process configuration file (%q): %v", config_path, err) } else { fmt.Fprintf(os.Stderr, "Failed to process configuration file (%q): %v\n", config_path, err) } os.Exit(1) } }() // Read the config as yaml config, err := yaml.ReadFile(config_path) if err != nil { panic(err) os.Exit(1) } // Configure logging log = logging.Get("codetag") // Common processing vars var ( ok bool node yaml.Node config_map yaml.Map config_list yaml.List ) node, err = yaml.Child(config.Root, ".logging") if err != nil || node == nil { logging.DefaultSetup() log.Debugf("No logging config defined (err: %#v), using defaults", err) } else { config_map, ok = node.(yaml.Map) if !ok { logging.DefaultSetup() log.Error("'logging' config section is not a map, ignoring") } else { err = log_setup.SetupYAML(config_map) if err != nil { logging.DefaultSetup() log.Errorf("Failed to configure logging: %v", err) } } } log_init = true // Configure filtering filters := path_filters{} node, err = yaml.Child(config.Root, ".filter") if err != nil || node == nil { log.Debug("No path-filters configured") } else { config_list, ok = node.(yaml.List) if !ok { log.Fatal("'filters' must be a list of string patterns") os.Exit(1) } for _, node := range config_list { pattern, ok := node.(yaml.Scalar) if !ok { log.Errorf("Pattern must be a string: %v", node) continue } filter, pattern_str := path_filter{}, strings.Trim(string(pattern), "'") filter.verdict = strings.HasPrefix(pattern_str, "+") if !filter.verdict && !strings.HasPrefix(pattern_str, "-") { log.Errorf("Pattern must start with either '+' or '-': %v", pattern_str) continue } pattern_str = pattern_str[1:] filter.pattern, err = re.Compile(pattern_str) if err != nil { log.Errorf("Failed to compile pattern (%v) as regexp: %v", pattern_str, err) continue } filters = append(filters, filter) } } // Get the list of paths to process config_map, ok = config.Root.(yaml.Map) if !ok { log.Fatal("Config must be a map and have 'paths' key") os.Exit(1) } node, ok = config_map["paths"] if !ok { log.Fatal("'paths' list must be defined in config") os.Exit(1) } var paths []string config_list, ok = node.(yaml.List) if !ok { path, ok := node.(yaml.Scalar) if !ok { log.Fatal("'paths' must be a list or (worst-case) scalar") os.Exit(1) } paths = append(paths, string(path)) } else { for _, node := range config_list { path, ok := node.(yaml.Scalar) if !ok { log.Warnf("Skipped invalid path specification: %v", node) } else { paths = append(paths, string(path)) } } } // Init taggers node, ok = config_map["taggers"] if ok { config_map, ok = node.(yaml.Map) } if !ok { log.Warn("No 'taggers' defined, nothing to do") os.Exit(0) } taggers := make(map[string][]tgrs.Tagger) init_tagger := func(ns, name string, config *yaml.Node) { tagger, err := tgrs.Get(name, config, log) if err != nil { log.Warnf("Failed to init tagger %v (ns: %v): %v", ns, name, err) } else { taggers[ns] = append(taggers[ns], tagger) } } for ns, node := range config_map { if ns == "_none" { ns = "" } if strings.HasPrefix(ns, "_") { log.Warnf("Ignoring namespace name, starting with underscore: %v", ns) continue } config_list, ok := node.(yaml.List) if !ok { // It's also ok to have "ns: tagger" spec, if there's just one for ns tagger, ok := node.(yaml.Scalar) if !ok { log.Warnf("Invalid tagger(-list) specification (ns: %v): %v", ns, node) continue } init_tagger(ns, string(tagger), nil) continue } for _, node = range config_list { tagger_map, ok := node.(yaml.Map) if !ok { tagger, ok := node.(yaml.Scalar) if !ok { log.Warnf("Invalid tagger specification - "+ "must be map or string (ns: %v): %v", ns, node) continue } init_tagger(ns, string(tagger), nil) continue } if len(tagger_map) != 1 { log.Warnf("Invalid tagger specification - "+ "map must contain only one element (ns: %v): %v", ns, tagger_map) continue } for tagger, node := range tagger_map { init_tagger(ns, tagger, &node) continue } } } config_init = true // Walk the paths var ( ctx ctx_t ctx_stack_tuple ctx_stack_t ctx_stack = make([]ctx_stack_t, 0) ctx_tags tgrs.CtxTagset ) log_tmsu := logging.Get("codetag.tmsu") pipe := log_pipe{} pipe.log_func = func(line string) { log_tmsu.Debug(line) } tmsu_log_pipe := &pipe ctx_stack = append(ctx_stack, ctx_stack_t{"", make(ctx_t)}) for _, root := range paths { log.Tracef("Processing path: %s", root) walk_iter := func(path string, info os.FileInfo, err error) (ret_err error) { if err != nil { log.Debugf(" - path: %v (info: %v), error: %v", path, info, err) return } if !strings.HasPrefix(path, root) { panic(fmt.Errorf("filepath.Walk went outside of root path (%v): %v", root, path)) } path_match := path[len(root):] if info.IsDir() { path_match += "/" } if !filters.match(path_match) { if info.IsDir() { return filepath.SkipDir } return } // Get context for this path or copy it from parent path n, slug := 0, path for n, slug = range strings.Split(slug, fmt.Sprintf("%c", os.PathSeparator)) { if len(ctx_stack) > n { ctx_stack_tuple = ctx_stack[n] if ctx_stack_tuple.slug == slug { ctx = ctx_stack_tuple.ctx } else { ctx_stack_tuple = ctx_stack[n-1] ctx = nil ctx_clone(ctx_stack_tuple.ctx, &ctx) ctx_stack[n] = ctx_stack_t{slug, ctx} ctx_stack = ctx_stack[:n+1] break } } else { ctx_stack_tuple = ctx_stack_t{slug, nil} ctx_clone(ctx, &ctx_stack_tuple.ctx) ctx_stack, ctx = append(ctx_stack, ctx_stack_tuple), ctx_stack_tuple.ctx } } if len(ctx_stack) > n+1 { ctx_stack = ctx_stack[:n+1] } // Run all taggers for ns, tagger_list := range taggers { ctx_ns, ok := ctx[ns] if !ok { ctx[ns] = make(map[string]interface{}, len(taggers)+1) ctx_ns = ctx[ns] } for _, tagger := range tagger_list { tags := tagger(path, info, &ctx_ns) if tags == nil { continue } // Push new tags to the context ctx_tags_if, ok := ctx_ns["tags"] if !ok { ctx_tags = make(tgrs.CtxTagset, len(taggers)) } else { ctx_tags = ctx_tags_if.(tgrs.CtxTagset) } for _, tag := range tags { _, ok = ctx_tags[tag] if !ok { ctx_tags[tag] = true } } ctx_ns["tags"] = ctx_tags } } // Attach tags only to files if info.Mode()&os.ModeType != 0 { return } file_tags := []string{} for ns, ctx_ns := range ctx { ctx_tags_if, ok := ctx_ns["tags"] if !ok { continue } ctx_tags = ctx_tags_if.(tgrs.CtxTagset) for tag, _ := range ctx_tags { file_tags = append(file_tags, ns+":"+tag) } } log.Tracef(" - file: %v, tags: %v", path, file_tags) if !dry_run { cmd := exec.Command("tmsu", "tag", path) cmd.Args = append(cmd.Args, file_tags...) cmd.Stdout, cmd.Stderr = tmsu_log_pipe, tmsu_log_pipe err = cmd.Run() if err != nil { log.Fatalf("Failure running tmsu (file: %v, tags: %v): %v", path, file_tags, err) } tmsu_log_pipe.Flush() } return } path_ext, err := path_t(root).ExpandUser() if err == nil { root = string(path_ext) } err = filepath.Walk(root, walk_iter) if err != nil { log.Errorf("Failed to process path: %s", root) } } log.Debug("Finished") }