// Load loads the configuration from the filesystem.
func (config *Config) Load(filePath string) error {
var errOutput error
_, err := utils.JSONLoad(filePath, &config)
if err != nil {
return err
}
// Load provider definitions
config.Providers = make(map[string]*ProviderConfig)
walkFunc := func(filePath string, fileInfo os.FileInfo, err error) error {
if fileInfo.IsDir() || !strings.HasSuffix(filePath, ".json") {
return nil
}
_, providerName := path.Split(strings.TrimSuffix(filePath, ".json"))
config.Lock()
defer config.Unlock()
config.Providers[providerName] = &ProviderConfig{}
if fileInfo, err = utils.JSONLoad(filePath, config.Providers[providerName]); err != nil {
err = fmt.Errorf("in %s, %s", filePath, err)
if errOutput == nil {
errOutput = err
}
return err
}
return nil
}
if err := utils.WalkDir(config.ProvidersDir, walkFunc); err != nil {
return fmt.Errorf("unable to load provider definitions: %s", err)
}
if errOutput != nil {
return errOutput
}
return nil
}
// Refresh triggers a full connector data update.
func (c *RRDConnector) Refresh(originName string, outputChan chan<- *catalog.Record) error {
// Search for files and parse their path for source/metric pairs
walkFunc := func(filePath string, fileInfo os.FileInfo, err error) error {
var sourceName, metricName string
// Report errors
if err != nil {
logger.Log(logger.LevelWarning, "connector", "rrd[%s]: error while walking: %s", c.name, err)
return nil
}
// Skip non-files
mode := fileInfo.Mode() & os.ModeType
if mode != 0 {
return nil
}
// Get pattern matches
m, err := matchSeriesPattern(c.re, strings.TrimPrefix(filePath, c.path+"/"))
if err != nil {
logger.Log(logger.LevelInfo, "connector", "rrd[%s]: file `%s' does not match pattern, ignoring", c.name,
filePath)
return nil
}
sourceName, metricName = m[0], m[1]
if _, ok := c.metrics[sourceName]; !ok {
c.metrics[sourceName] = make(map[string]*rrdMetric)
}
// Extract metric information from .rrd file
info, err := rrd.Info(filePath)
if err != nil {
logger.Log(logger.LevelWarning, "connector", "rrd[%s]: %s", c.name, err)
return nil
}
// Extract consolidation functions list
cfSet := set.New(set.ThreadSafe)
if cf, ok := info["rra.cf"].([]interface{}); ok {
for _, entry := range cf {
if name, ok := entry.(string); ok {
cfSet.Add(name)
}
}
}
cfList := set.StringSlice(cfSet)
if _, ok := info["ds.index"]; ok {
indexes, ok := info["ds.index"].(map[string]interface{})
if !ok {
return nil
}
for dsName := range indexes {
for _, cfName := range cfList {
metricFullName := metricName + "/" + dsName + "/" + strings.ToLower(cfName)
c.metrics[sourceName][metricFullName] = &rrdMetric{
Dataset: dsName,
FilePath: filePath,
Step: time.Duration(info["step"].(uint)) * time.Second,
Cf: cfName,
}
outputChan <- &catalog.Record{
Origin: originName,
Source: sourceName,
Metric: metricFullName,
Connector: c,
}
}
}
}
return nil
}
if err := utils.WalkDir(c.path, walkFunc); err != nil {
return err
}
return nil
}
// Refresh triggers a full connector data update.
func (connector *RRDConnector) Refresh(originName string, outputChan chan *catalog.CatalogRecord) error {
// Compile pattern
re := regexp.MustCompile(connector.pattern)
// Validate pattern keywords
groups := make(map[string]bool)
for _, key := range re.SubexpNames() {
if key == "" {
continue
} else if key == "source" || key == "metric" {
groups[key] = true
} else {
return fmt.Errorf("invalid pattern keyword `%s'", key)
}
}
if !groups["source"] {
return fmt.Errorf("missing pattern keyword `source'")
} else if !groups["metric"] {
return fmt.Errorf("missing pattern keyword `metric'")
}
// Search for files and parse their path for source/metric pairs
walkFunc := func(filePath string, fileInfo os.FileInfo, err error) error {
var sourceName, metricName string
// Stop if previous error
if err != nil {
return err
}
// Skip non-files
mode := fileInfo.Mode() & os.ModeType
if mode != 0 {
return nil
}
submatch := re.FindStringSubmatch(filePath[len(connector.path)+1:])
if len(submatch) == 0 {
logger.Log(logger.LevelInfo, "connector: rrd", "file `%s' does not match pattern, ignoring", filePath)
return nil
}
if re.SubexpNames()[1] == "source" {
sourceName = submatch[1]
metricName = submatch[2]
} else {
sourceName = submatch[2]
metricName = submatch[1]
}
if _, ok := connector.metrics[sourceName]; !ok {
connector.metrics[sourceName] = make(map[string]*rrdMetric)
}
// Extract metric information from .rrd file
info, err := rrd.Info(filePath)
if err != nil {
logger.Log(logger.LevelWarning, "connector: rrd", "%s", err)
return nil
}
if _, ok := info["ds.index"]; ok {
for dsName := range info["ds.index"].(map[string]interface{}) {
metricFullName := metricName + "/" + dsName
connector.metrics[sourceName][metricFullName] = &rrdMetric{Dataset: dsName, FilePath: filePath}
outputChan <- &catalog.CatalogRecord{
Origin: originName,
Source: sourceName,
Metric: metricFullName,
Connector: connector,
}
}
}
return nil
}
if err := utils.WalkDir(connector.path, walkFunc); err != nil {
return err
}
return nil
}
// Refresh updates the current library by browsing the filesystem for stored data.
func (library *Library) Refresh() error {
var itemType int
// Empty library maps
library.Groups = make(map[string]*Group)
library.Scales = make(map[string]*Scale)
library.Graphs = make(map[string]*Graph)
library.Collections = make(map[string]*Collection)
walkFunc := func(filePath string, fileInfo os.FileInfo, fileError error) error {
mode := fileInfo.Mode() & os.ModeType
if mode != 0 || !strings.HasSuffix(filePath, ".json") {
return nil
}
_, itemID := path.Split(filePath[:len(filePath)-5])
logger.Log(logger.LevelDebug, "library", "loading item `%s' from file `%s'", itemID, filePath)
return library.LoadItem(itemID, itemType)
}
logger.Log(logger.LevelInfo, "library", "refresh started")
for _, itemType = range []int{
LibraryItemSourceGroup,
LibraryItemMetricGroup,
LibraryItemScale,
LibraryItemGraph,
LibraryItemCollection,
} {
dirPath := library.getDirPath(itemType)
if _, err := os.Stat(dirPath); os.IsNotExist(err) {
continue
}
if err := utils.WalkDir(dirPath, walkFunc); err != nil {
logger.Log(logger.LevelError, "library", "%s", err)
}
}
// Update collection items parent-children relations
for _, collection := range library.Collections {
if collection.ParentID == "" {
continue
}
if _, ok := library.Collections[collection.ParentID]; !ok {
logger.Log(logger.LevelError, "library", "unknown parent identifier `%s'", collection.ParentID)
continue
}
collection.Parent = library.Collections[collection.ParentID]
collection.Parent.Children = append(collection.Parent.Children, collection)
}
logger.Log(logger.LevelInfo, "library", "refresh completed")
return nil
}