Exemplo n.º 1
0
func downloadCollections(collections []*CollectionConfig, cache string, stats *report.Recorder, canBypassDisk bool) error {
	if stats != nil {
		t := time.Now()
		defer stats.TimeSince("startup.download", t)
	}
	for _, cfg := range collections {
		if cfg.LocalPath == "" {
			cfg.LocalPath = cfg.SourcePath
		}

		remote := isRemote(cfg.SourcePath)
		if remote {
			cfg.LocalPath = localCache(cfg.SourcePath, cache)
			if _, err := os.Stat(cfg.LocalPath); err == nil {
				if cfg.Debug {
					log.Printf("[FetchRemote] %s already cached: %s.", cfg.Name, cfg.LocalPath)
				}
			} else if !os.IsNotExist(err) {
				return err
			} else {
				err = fetch(cfg, canBypassDisk)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}
Exemplo n.º 2
0
func LoadCollections(collections []*CollectionConfig, cache string, downloadOnly bool, stats *report.Recorder) (*CollectionSet, error) {
	cs := new(CollectionSet)
	cs.Collections = make(map[string]*Reader)

	if len(collections) < 1 {
		return nil, fmt.Errorf("no collections to load!")
	}

	if err := downloadCollections(collections, cache, stats, !downloadOnly); err != nil {
		log.Println("[LoadCollections] Error fetching collections: ", err)
		return nil, err
	}

	if downloadOnly {
		return nil, nil
	}

	t := time.Now()
	for _, cfg := range collections {
		reader, err := NewReaderFromConfig(*cfg)
		if err != nil {
			return nil, err
		}

		cs.Collections[cfg.Name] = reader
	}
	if stats != nil {
		stats.TimeSince("startup.read", t)
	}

	return cs, nil
}