func main() {
node, err := csv.Crawler("test")
if err != nil {
log.Fatalln(err)
}
const jsonStream = `{
"union": {
"input1": {"selection": {
"input": { "relation": { "name": "dir1/staff2" } },
"attr": "age", "selector": ">=", "arg": 31
}},
"input2": {"selection": {
"input": { "relation": { "name": "dir1/staff2" } },
"attr": "name", "selector": "==", "arg": "山田"
}}
}
}`
m := core.Stream{}
if err := json.NewDecoder(strings.NewReader(jsonStream)).Decode(&m); err != nil {
log.Fatal(err)
}
result, err := core.StreamToRelation(m, node)
if err != nil {
log.Fatalln(err)
}
pp.Print(result)
}
func (d *Daemon) Serve(ctx context.Context) error {
f, err := os.Create(cpuprofile)
if err != nil {
log.Fatal(err)
}
for i := 0; i < d.WorkerLimit; i++ {
d.MngQ[i] = make(chan daemon.ManageRequest, 1)
go d.Worker(ctx, d.MngQ[i])
}
names, err := csv.SearchDir(d.LoadDir)
if err != nil {
return err
}
for _, name := range names {
dir := path.Join(d.LoadDir, name)
node, err := csv.Crawler(dir)
if err != nil {
return err
}
log.Printf("load dir:%s", dir)
err = d.BroadcastManageReq(daemon.ManageRequest{Cmd: daemon.PutNode, Path: name, Name: name, Node: node})
if err != nil {
return err
}
}
startTime := time.Now()
var streams []core.Stream
dec := json.NewDecoder(strings.NewReader(jsonStream))
err = dec.Decode(&streams)
if err != nil {
return err
}
rs := make([][]*core.Relation, loop)
elapsendJsonDecode := time.Now().Sub(startTime)
pprof.StartCPUProfile(f)
for i := 0; i < loop; i++ {
rs[i], err = d.QueryStreams(dirname, streams)
//os.Exit(0)
if err != nil {
return err
}
}
pprof.StopCPUProfile()
elapsendQuery := time.Now().Sub(startTime) - elapsendJsonDecode
//json.NewEncoder(os.Stdout).Encode(rs)
elapsedAll := time.Now().Sub(startTime)
log.Printf("elapsed:%s, json decode:%s, query:%s, json encode:%s,len(rs):%d", elapsedAll, elapsendJsonDecode, elapsendQuery, elapsedAll-elapsendQuery-elapsendJsonDecode, len(rs))
return nil
}
func (d *Daemon) ManageHandler(w http.ResponseWriter, r *http.Request) {
name := r.PostForm.Get("name")
if name != "" {
name = strings.TrimRight(path.Base(r.URL.Path), "/")
}
switch r.Method {
case "PUT":
node, err := csv.Crawler(r.URL.Path)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
err = fmt.Errorf("csv.Crawler err: %s", err)
fmt.Fprintln(w, err)
log.Println(err)
}
err = d.BroadcastManageReq(ManageRequest{Cmd: PutNode, Path: r.URL.Path, Name: name, Node: node})
if err != nil {
w.WriteHeader(http.StatusBadRequest)
fmt.Fprintln(w, err)
}
case "DELETE":
err := d.BroadcastManageReq(ManageRequest{Cmd: DelNode, Path: r.URL.Path, Name: name})
if err != nil {
w.WriteHeader(http.StatusBadRequest)
fmt.Fprintln(w, err)
}
case "GET":
switch r.URL.Path {
case "/list":
res := d.sendManageReq(ManageRequest{Cmd: GetNodeList, Path: r.URL.Path, Name: name})
if res.Err != nil {
w.WriteHeader(http.StatusBadRequest)
fmt.Fprintln(w, res.Err)
}
json.NewEncoder(w).Encode(res.Body)
case "/status":
stat := status.Get()
json.NewEncoder(w).Encode(stat)
}
}
}
func (d *Daemon) Serve(ctx context.Context) error {
for i := 0; i < d.WorkerLimit; i++ {
d.MngQ[i] = make(chan ManageRequest, 1)
go d.Worker(ctx, d.MngQ[i])
}
names, err := csv.SearchDir(d.LoadDir)
if err != nil {
return err
}
for _, name := range names {
dir := path.Join(d.LoadDir, name)
node, err := csv.Crawler(dir)
if err != nil {
return err
}
log.Printf("load dir:%s", dir)
err = d.BroadcastManageReq(ManageRequest{Cmd: PutNode, Path: name, Name: name, Node: node})
if err != nil {
return err
}
}
mux := http.NewServeMux()
mux.HandleFunc("/query/", d.Handler)
s := &http.Server{
Addr: d.Listen,
Handler: mux,
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
MaxHeaderBytes: 1 << 20,
}
log.Printf("listen: %s", d.Listen)
err = s.ListenAndServe()
if err != nil {
log.Printf("ListenAndServe err:%s", err)
}
return err
}