func downloadCollections(collections []*CollectionConfig, cache string, stats *report.Recorder, canBypassDisk bool) error {
if stats != nil {
t := time.Now()
defer stats.TimeSince("startup.download", t)
}
for _, cfg := range collections {
if cfg.LocalPath == "" {
cfg.LocalPath = cfg.SourcePath
}
remote := isRemote(cfg.SourcePath)
if remote {
cfg.LocalPath = localCache(cfg.SourcePath, cache)
if _, err := os.Stat(cfg.LocalPath); err == nil {
if cfg.Debug {
log.Printf("[FetchRemote] %s already cached: %s.", cfg.Name, cfg.LocalPath)
}
} else if !os.IsNotExist(err) {
return err
} else {
err = fetch(cfg, canBypassDisk)
if err != nil {
return err
}
}
}
}
return nil
}
func LoadCollections(collections []*CollectionConfig, cache string, downloadOnly bool, stats *report.Recorder) (*CollectionSet, error) {
cs := new(CollectionSet)
cs.Collections = make(map[string]*Reader)
if len(collections) < 1 {
return nil, fmt.Errorf("no collections to load!")
}
if err := downloadCollections(collections, cache, stats, !downloadOnly); err != nil {
log.Println("[LoadCollections] Error fetching collections: ", err)
return nil, err
}
if downloadOnly {
return nil, nil
}
t := time.Now()
for _, cfg := range collections {
reader, err := NewReaderFromConfig(*cfg)
if err != nil {
return nil, err
}
cs.Collections[cfg.Name] = reader
}
if stats != nil {
stats.TimeSince("startup.read", t)
}
return cs, nil
}