func Crawler(root string) (*core.Node, error) { root = strings.TrimRight(root, "/") name := path.Base(root) node := core.NewNode(name) err := filepath.Walk(root, func(objectPath string, f os.FileInfo, err error) error { if err != nil { return err } if !f.IsDir() && strings.ToLower(filepath.Ext(objectPath)) == CsvExt { rel, err := LoadCsv(objectPath) if err != nil { return fmt.Errorf("LoadCsv file:%s err:%s", objectPath, err) } rPath := path.Dir(strings.TrimPrefix(objectPath, root)) rel.Name = strings.TrimSuffix(path.Base(objectPath), filepath.Ext(objectPath)) node.SetRelation(rPath, rel) } return nil }) return node, err }
func (d *Daemon) Worker(ctx context.Context, ManageCh chan ManageRequest) { node := core.NewNode("root") for { select { case req := <-d.Queue: res := d.work(req, node) req.ResCh <- res if res.Err != nil { log.Printf("work err: %s", res.Err) } case req := <-ManageCh: res := d.manageWork(req, node) req.ResCh <- res if res.Err != nil { log.Printf("manageWork err: %s", res.Err) } case <-ctx.Done(): return } } }