Ejemplo n.º 1
0
func main() {

	node, err := csv.Crawler("test")
	if err != nil {
		log.Fatalln(err)
	}
	const jsonStream = `{ 
		"union": {
			"input1": {"selection": {
				"input": { "relation": { "name": "dir1/staff2" } },
				"attr": "age",  "selector": ">=", "arg": 31
			}},
			"input2": {"selection": {
				"input": { "relation": { "name": "dir1/staff2" } },
				"attr": "name", "selector": "==", "arg": "山田"
			}}
		}
	}`
	m := core.Stream{}
	if err := json.NewDecoder(strings.NewReader(jsonStream)).Decode(&m); err != nil {
		log.Fatal(err)
	}
	result, err := core.StreamToRelation(m, node)
	if err != nil {
		log.Fatalln(err)
	}
	pp.Print(result)
}
Ejemplo n.º 2
0
func (d *Daemon) Serve(ctx context.Context) error {

	f, err := os.Create(cpuprofile)
	if err != nil {
		log.Fatal(err)
	}
	for i := 0; i < d.WorkerLimit; i++ {
		d.MngQ[i] = make(chan daemon.ManageRequest, 1)
		go d.Worker(ctx, d.MngQ[i])
	}

	names, err := csv.SearchDir(d.LoadDir)
	if err != nil {
		return err
	}
	for _, name := range names {
		dir := path.Join(d.LoadDir, name)
		node, err := csv.Crawler(dir)
		if err != nil {
			return err
		}
		log.Printf("load dir:%s", dir)
		err = d.BroadcastManageReq(daemon.ManageRequest{Cmd: daemon.PutNode, Path: name, Name: name, Node: node})
		if err != nil {
			return err
		}
	}
	startTime := time.Now()

	var streams []core.Stream
	dec := json.NewDecoder(strings.NewReader(jsonStream))
	err = dec.Decode(&streams)
	if err != nil {
		return err
	}
	rs := make([][]*core.Relation, loop)
	elapsendJsonDecode := time.Now().Sub(startTime)
	pprof.StartCPUProfile(f)
	for i := 0; i < loop; i++ {
		rs[i], err = d.QueryStreams(dirname, streams)
		//os.Exit(0)
		if err != nil {
			return err
		}
	}
	pprof.StopCPUProfile()
	elapsendQuery := time.Now().Sub(startTime) - elapsendJsonDecode
	//json.NewEncoder(os.Stdout).Encode(rs)

	elapsedAll := time.Now().Sub(startTime)
	log.Printf("elapsed:%s, json decode:%s, query:%s, json encode:%s,len(rs):%d", elapsedAll, elapsendJsonDecode, elapsendQuery, elapsedAll-elapsendQuery-elapsendJsonDecode, len(rs))
	return nil
}
Ejemplo n.º 3
0
func (d *Daemon) ManageHandler(w http.ResponseWriter, r *http.Request) {
	name := r.PostForm.Get("name")
	if name != "" {
		name = strings.TrimRight(path.Base(r.URL.Path), "/")
	}
	switch r.Method {
	case "PUT":
		node, err := csv.Crawler(r.URL.Path)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			err = fmt.Errorf("csv.Crawler err: %s", err)
			fmt.Fprintln(w, err)
			log.Println(err)
		}
		err = d.BroadcastManageReq(ManageRequest{Cmd: PutNode, Path: r.URL.Path, Name: name, Node: node})
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintln(w, err)
		}
	case "DELETE":
		err := d.BroadcastManageReq(ManageRequest{Cmd: DelNode, Path: r.URL.Path, Name: name})
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintln(w, err)
		}
	case "GET":
		switch r.URL.Path {
		case "/list":
			res := d.sendManageReq(ManageRequest{Cmd: GetNodeList, Path: r.URL.Path, Name: name})
			if res.Err != nil {
				w.WriteHeader(http.StatusBadRequest)
				fmt.Fprintln(w, res.Err)
			}
			json.NewEncoder(w).Encode(res.Body)
		case "/status":
			stat := status.Get()
			json.NewEncoder(w).Encode(stat)
		}
	}
}
Ejemplo n.º 4
0
func (d *Daemon) Serve(ctx context.Context) error {

	for i := 0; i < d.WorkerLimit; i++ {
		d.MngQ[i] = make(chan ManageRequest, 1)
		go d.Worker(ctx, d.MngQ[i])
	}

	names, err := csv.SearchDir(d.LoadDir)
	if err != nil {
		return err
	}
	for _, name := range names {
		dir := path.Join(d.LoadDir, name)
		node, err := csv.Crawler(dir)
		if err != nil {
			return err
		}
		log.Printf("load dir:%s", dir)
		err = d.BroadcastManageReq(ManageRequest{Cmd: PutNode, Path: name, Name: name, Node: node})
		if err != nil {
			return err
		}
	}
	mux := http.NewServeMux()
	mux.HandleFunc("/query/", d.Handler)
	s := &http.Server{
		Addr:           d.Listen,
		Handler:        mux,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}
	log.Printf("listen: %s", d.Listen)
	err = s.ListenAndServe()
	if err != nil {
		log.Printf("ListenAndServe err:%s", err)
	}
	return err
}