Esempio n. 1
0
func (d *Daemon) Serve(ctx context.Context) error {

	f, err := os.Create(cpuprofile)
	if err != nil {
		log.Fatal(err)
	}
	for i := 0; i < d.WorkerLimit; i++ {
		d.MngQ[i] = make(chan daemon.ManageRequest, 1)
		go d.Worker(ctx, d.MngQ[i])
	}

	names, err := csv.SearchDir(d.LoadDir)
	if err != nil {
		return err
	}
	for _, name := range names {
		dir := path.Join(d.LoadDir, name)
		node, err := csv.Crawler(dir)
		if err != nil {
			return err
		}
		log.Printf("load dir:%s", dir)
		err = d.BroadcastManageReq(daemon.ManageRequest{Cmd: daemon.PutNode, Path: name, Name: name, Node: node})
		if err != nil {
			return err
		}
	}
	startTime := time.Now()

	var streams []core.Stream
	dec := json.NewDecoder(strings.NewReader(jsonStream))
	err = dec.Decode(&streams)
	if err != nil {
		return err
	}
	rs := make([][]*core.Relation, loop)
	elapsendJsonDecode := time.Now().Sub(startTime)
	pprof.StartCPUProfile(f)
	for i := 0; i < loop; i++ {
		rs[i], err = d.QueryStreams(dirname, streams)
		//os.Exit(0)
		if err != nil {
			return err
		}
	}
	pprof.StopCPUProfile()
	elapsendQuery := time.Now().Sub(startTime) - elapsendJsonDecode
	//json.NewEncoder(os.Stdout).Encode(rs)

	elapsedAll := time.Now().Sub(startTime)
	log.Printf("elapsed:%s, json decode:%s, query:%s, json encode:%s,len(rs):%d", elapsedAll, elapsendJsonDecode, elapsendQuery, elapsedAll-elapsendQuery-elapsendJsonDecode, len(rs))
	return nil
}
Esempio n. 2
0
func (d *Daemon) Serve(ctx context.Context) error {

	for i := 0; i < d.WorkerLimit; i++ {
		d.MngQ[i] = make(chan ManageRequest, 1)
		go d.Worker(ctx, d.MngQ[i])
	}

	names, err := csv.SearchDir(d.LoadDir)
	if err != nil {
		return err
	}
	for _, name := range names {
		dir := path.Join(d.LoadDir, name)
		node, err := csv.Crawler(dir)
		if err != nil {
			return err
		}
		log.Printf("load dir:%s", dir)
		err = d.BroadcastManageReq(ManageRequest{Cmd: PutNode, Path: name, Name: name, Node: node})
		if err != nil {
			return err
		}
	}
	mux := http.NewServeMux()
	mux.HandleFunc("/query/", d.Handler)
	s := &http.Server{
		Addr:           d.Listen,
		Handler:        mux,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}
	log.Printf("listen: %s", d.Listen)
	err = s.ListenAndServe()
	if err != nil {
		log.Printf("ListenAndServe err:%s", err)
	}
	return err
}