func main() {
var m runtime.MemStats
/*If that is not working, or it is too much time, you can add a periodic call to
FreeOSMemory (no need to call runtime.GC() before, it is done by debug.FreeOSMemory() )
Something like this: http://play.golang.org/p/mP7_sMpX4F
package main
import (
"runtime/debug"
"time"
)
func main() {
go periodicFree(1 * time.Minute)
// Your program goes here
}
func periodicFree(d time.Duration) {
tick := time.Tick(d)
for _ = range tick {
debug.FreeOSMemory()
}
}
Take into account that every call to FreeOSMemory will take some time (not much)
and it can be partly run in parallel if GOMAXPROCS>1 since Go1.3.*/
debug.FreeOSMemory()
/*Then you can either render it to a dot file with graphical
representation of the heap or convert it to hprof format. To render it to a dot file:
$ go get github.com/randall77/hprof/dumptodot
$ dumptodot heapdump mybinary > heap.dot
and open heap.dot with Graphviz.*/
f, err := os.Create("heapdump")
if err != nil {
panic(err)
}
debug.WriteHeapDump(f.Fd())
fmt.Println(runtime.GOOS)
fmt.Println(runtime.NumCPU())
fmt.Println(runtime.NumGoroutine())
fmt.Println(runtime.GOARCH)
runtime.ReadMemStats(&m)
fmt.Println(m.TotalAlloc)
fmt.Println(m.Alloc)
fmt.Println(m.Sys)
}
// So meta.
func heapdumpHandler(w http.ResponseWriter, r *http.Request) {
f, err := os.Create("metadump")
if err != nil {
panic(err)
}
runtime.GC()
debug.WriteHeapDump(f.Fd())
f.Close()
w.Write([]byte("done"))
}
func TestGetNoteSecretClearedFromMemory(t *testing.T) {
testServer := httptest.NewServer(main.Handlers())
defer testServer.Close()
main.SetupStore()
defer main.TeardownStore()
err := exec.Command("sh", "-c", "head -c 16 /dev/urandom > /tmp/random_secret").Run()
defer os.Remove("/tmp/random_secret")
if err != nil {
t.Error("Error generating random secret:", err)
return
}
out, err := exec.Command("curl", "-d", "@/tmp/random_secret", testServer.URL+"/notes/fc2a4122-e81e-4b10-a31b-d79fbdb33a27").Output()
if err != nil {
t.Error("Error POSTing random secret:", err, out)
return
}
// We already ensure the secret disappears on create, now get it
// to ensure it desappears on access too
err = exec.Command("curl", testServer.URL+"/notes/fc2a4122-e81e-4b10-a31b-d79fbdb33a27").Run()
if err != nil {
t.Error("Error GETing secret:", err)
return
}
// Do a heap dump to see if the secret was cleared from our memory
f, err := os.Create("/tmp/heapdump")
if err != nil {
t.Errorf("Error creating /tmp/heapdump: %s", err)
}
defer os.Remove("/tmp/heapdump")
debug.WriteHeapDump(f.Fd())
f.Close()
dump, err := ioutil.ReadFile("/tmp/heapdump")
if err != nil {
t.Errorf("Error reading /tmp/heapdump: %s", err)
}
randomSecret, err := ioutil.ReadFile("/tmp/random_secret")
if err != nil {
t.Errorf("Error reading random secret: %s", err)
}
if bytes.Contains(dump, randomSecret) {
t.Error("Saved data not cleared from memory", randomSecret)
}
}
func main() {
h1 := Holder{
objects: make(map[string]*Item),
}
h2 := Holder{
objects: make(map[string]*Item),
}
fmt.Println("Setup")
http.Handle("/debug/pprof/heapdump", http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
f, err := ioutil.TempFile("", "dump")
if err != nil {
rw.WriteHeader(http.StatusServiceUnavailable)
io.WriteString(rw, err.Error())
return
}
fmt.Printf("Using %s as heap dump", f.Name())
defer os.Remove(f.Name())
defer f.Close()
debug.WriteHeapDump(f.Fd())
f.Close()
f2, err := os.Open(f.Name())
if err != nil {
rw.WriteHeader(http.StatusServiceUnavailable)
io.WriteString(rw, err.Error())
return
}
defer f2.Close()
io.Copy(rw, f2)
}))
go func() {
log.Println(http.ListenAndServe("localhost:6060", nil))
}()
for i := 0; i < 1000000; i++ {
it := Item(i)
s := strconv.FormatInt(int64(i), 10)
h1.objects[s] = &it
h2.objects[s] = &it
}
fmt.Println("Surviving")
go h1.survive(time.Second)
h1.survive(time.Hour * 24)
}
func main() {
logxml := flag.String("logxml", "../log/log_consumer.xml", "-logxml=../log/log_consumer.xml")
zkhost := flag.String("zkhost", "localhost:2181", "-zkhost=localhost:2181")
flag.Parse()
runtime.GOMAXPROCS(8)
log.LoadConfiguration(*logxml)
go func() {
log.Info(http.ListenAndServe(":38000", nil))
}()
lis := &defualtListener{}
go lis.monitor()
kite := client.NewKiteQClient(*zkhost, "s-mts-test", "123456", lis)
kite.SetBindings([]*binding.Binding{
binding.Bind_Direct("s-mts-test", "relation", "pay-succ", 1000, true),
})
kite.Start()
var s = make(chan os.Signal, 1)
signal.Notify(s, syscall.SIGKILL, syscall.SIGUSR1)
//是否收到kill的命令
for {
cmd := <-s
if cmd == syscall.SIGKILL {
break
} else if cmd == syscall.SIGUSR1 {
//如果为siguser1则进行dump内存
unixtime := time.Now().Unix()
path := "./heapdump-consumer" + fmt.Sprintf("%d", unixtime)
f, err := os.Create(path)
if nil != err {
continue
} else {
debug.WriteHeapDump(f.Fd())
}
}
}
kite.Destory()
}
func main() {
//加载启动参数
so := server.Parse()
runtime.GOMAXPROCS(runtime.NumCPU())
rc := turbo.NewRemotingConfig(
"remoting",
20000, 16*1024,
16*1024, 10000, 10000,
10*time.Second, 160000)
kc := server.NewKiteQConfig(so, rc)
qserver := server.NewKiteQServer(kc)
qserver.Start()
var s = make(chan os.Signal, 1)
signal.Notify(s, syscall.SIGKILL, syscall.SIGUSR1)
//是否收到kill的命令
for {
cmd := <-s
if cmd == syscall.SIGKILL {
break
} else if cmd == syscall.SIGUSR1 {
//如果为siguser1则进行dump内存
unixtime := time.Now().Unix()
path := fmt.Sprintf("./heapdump-kiteq-%d", unixtime)
f, err := os.Create(path)
if nil != err {
continue
} else {
debug.WriteHeapDump(f.Fd())
}
}
}
qserver.Shutdown()
log.InfoLog("kite_server", "KiteQServer IS STOPPED!")
}
func (s *Service) ProcessRequest(ctx *core.Context, m map[string]interface{}, out io.Writer) (map[string]interface{}, error) {
core.Log(core.INFO, ctx, "service.ProcessRequest", "m", m)
timer := core.NewTimer(ctx, "ProcessRequest")
defer func() {
elapsed := timer.Stop()
core.Point(ctx, "rulesservice", "requestTime", elapsed/1000, "Microseconds")
}()
u, given := m["uri"]
if !given {
return nil, fmt.Errorf("No uri.")
}
uri := DWIMURI(ctx, u.(string))
switch uri {
// case "/api/sys/config":
// mutationStore,have,_ := getStringParam(m, "MutationStore", false)
// if have {
// return fmt.Errorf("You can't do that.")
// }
// current s.System.GetConfig()
// logging,have,_ := getStringParam(m, "logging", false)
// if have {
// current.Logging = logging
// }
// err := StoreConfig(current)
// if err != nil {
// return err
// }
// err = s.System.SetConfig(current)
// if err != nil {
// return err
// }
// return `{"status":"happy"}`
case "/api/version":
fmt.Fprintf(out, `{"version":"%s","go":"%s"}`,
APIVersion,
runtime.Version())
case "/api/health":
// Let's have a simple URI.
fmt.Fprintf(out, `{"status":"good"}`)
case "/api/health/shallow":
fmt.Fprintf(out, `{"status":"good"}`)
case "/api/health/deep":
if err := s.HealthDeep(ctx); err != nil {
return m, nil
}
fmt.Fprintf(out, `{"status":"good"}`)
case "/api/health/deeper":
if err := s.HealthDeeper(ctx); err != nil {
return m, nil
}
fmt.Fprintf(out, `{"status":"good"}`)
case "/api/sys/control":
// Temporary implementation so I can test other things.
control, given, _ := GetStringParam(m, "control", false)
target := s.System.Control()
if given {
err := json.Unmarshal([]byte(control), target)
if err != nil {
return nil, err
}
s.System.SetControl(*target)
target = s.System.Control()
js, err := json.Marshal(target)
if err != nil {
return nil, err
}
out.Write([]byte(fmt.Sprintf(`{"result":"okay","control":%s}`, js)))
} else {
js, err := json.Marshal(target)
if err != nil {
return nil, err
}
out.Write(js)
}
case "/api/sys/params":
control, given, _ := GetStringParam(m, "params", false)
target := core.SystemParameters.Copy()
if given {
err := json.Unmarshal([]byte(control), target)
if err != nil {
return nil, err
}
core.SystemParameters = target
target = core.SystemParameters
js, err := json.Marshal(target)
if err != nil {
return nil, err
}
out.Write([]byte(fmt.Sprintf(`{"result":"okay","params":%s}`, js)))
} else {
js, err := json.Marshal(target)
if err != nil {
return nil, err
}
out.Write(js)
}
case "/api/sys/cachedlocations":
js, err := json.Marshal(map[string]interface{}{
"locations": s.System.GetCachedLocations(ctx),
})
if err != nil {
return nil, err
}
out.Write(js)
case "/api/sys/loccontrol":
control, given, _ := GetStringParam(m, "control", false)
ctl := s.System.Control()
target := ctl.DefaultLocControl
if given {
err := json.Unmarshal([]byte(control), target)
if err != nil {
return nil, err
}
js, err := json.Marshal(target)
if err != nil {
return nil, err
}
ctl.DefaultLocControl = target
out.Write([]byte(fmt.Sprintf(`{"result":"okay","control":%s}`, js)))
} else {
js, err := json.Marshal(target)
if err != nil {
return nil, err
}
out.Write(js)
}
case "/api/sys/stats":
stats, err := s.System.GetStats(ctx)
if err != nil {
return nil, err
}
js, err := json.Marshal(&stats)
if err != nil {
return nil, err
}
out.Write(js)
case "/api/sys/runtime":
m, err := GetRuntimes(ctx)
if nil != err {
return nil, err
}
// m["stats"] = GetStats()
js, err := json.Marshal(m)
if err != nil {
return nil, err
}
out.Write(js)
case "/api/sys/util/nowsecs":
fmt.Fprintf(out, `{"secs":%d}`, core.NowSecs())
case "/api/sys/util/js":
code, _, err := GetStringParam(m, "code", true)
bs := make(core.Bindings)
x, err := core.RunJavascript(ctx, &bs, nil, code)
if err != nil {
return m, err
}
js, err := json.Marshal(&x)
if err != nil {
return m, err
}
fmt.Fprintf(out, `{"result":%s}`, js)
case "/api/sys/util/setJavascriptTestValue":
js, _, err := GetStringParam(m, "value", true)
if err != nil {
return nil, err
}
var x interface{}
if err := json.Unmarshal([]byte(js), &x); err != nil {
return nil, err
}
core.JavascriptTestValue = x
fmt.Fprintf(out, `{"result":%s}`, js)
case "/api/sys/admin/panic": // No required params
message, _, _ := GetStringParam(m, "message", false)
panic(message)
case "/api/sys/admin/sleep": // Option d=duration
duration, given, _ := GetStringParam(m, "d", false)
if !given {
duration = "1s"
}
d, err := time.ParseDuration(duration)
if err != nil {
return nil, err
}
time.Sleep(d)
out.Write([]byte(fmt.Sprintf(`{"slept":"%s"}`, d.String())))
case "/api/sys/admin/shutdown":
duration, given, _ := GetStringParam(m, "d", false)
if given && s.Stopper == nil {
return nil, errors.New("no Stopper for given duration")
}
if !given {
duration = "1s"
}
d, err := time.ParseDuration(duration)
if err != nil {
return nil, err
}
go func() {
if s.Stopper != nil {
core.Log(core.INFO, ctx, "/api/admin/shutdown", "Stopper", true)
if err := s.Stopper(ctx, d); err != nil {
core.Log(core.ERROR, ctx, "/api/admin/shutdown", "error", err)
}
}
core.Log(core.INFO, ctx, "/api/admin/shutdown", "Stopper", false)
if err := s.Shutdown(ctx); err != nil {
core.Log(core.ERROR, ctx, "/api/admin/shutdown", "error", err)
}
}()
out.Write([]byte(`{"status":"okay"}`))
case "/api/sys/admin/gcpercent":
percent, _, _ := GetStringParam(m, "percent", true)
n, err := strconv.Atoi(percent)
if err != nil {
return nil, err
}
was := debug.SetGCPercent(n)
fmt.Fprintf(out, `{"status":"okay","was":%d,"now":%d}`, was, n)
case "/api/sys/admin/freemem":
debug.FreeOSMemory()
fmt.Fprintf(out, `{"status":"okay"}`)
case "/api/sys/admin/purgeslurpcache":
core.SlurpCache.Purge()
fmt.Fprintf(out, `{"status":"okay"}`)
case "/api/sys/admin/purgehttppcache":
core.HTTPClientCache.Purge()
fmt.Fprintf(out, `{"status":"okay"}`)
case "/api/sys/admin/purgecaches":
core.SlurpCache.Purge()
core.HTTPClientCache.Purge()
fmt.Fprintf(out, `{"status":"okay"}`)
case "/api/sys/admin/gc":
runtime.GC()
fmt.Fprintf(out, `{"status":"okay"}`)
case "/api/sys/admin/heapdump":
filename, _, _ := GetStringParam(m, "filename", false)
if filename == "" {
filename = "heap.dump"
}
f, err := os.Create(filename)
if err != nil {
return nil, err
}
debug.WriteHeapDump(f.Fd())
if err = f.Close(); err != nil {
return nil, err
}
fmt.Fprintf(out, `{"status":"okay","filename":"%s"}`, filename)
case "/api/sys/util/match": // Params: fact or event,pattern
fact, have, err := getMapParam(m, "fact", false)
if err != nil {
return nil, err
}
if !have {
// Maybe we were given an 'event'. Fine.
fact, _, err = getMapParam(m, "event", true)
}
if err != nil {
return nil, err
}
pattern, _, err := getMapParam(m, "pattern", true)
if err != nil {
return nil, err
}
bss, err := core.Matches(ctx, pattern, fact)
if err != nil {
return nil, err
}
js, err := json.Marshal(&bss)
if err != nil {
return nil, err
}
out.Write(js)
case "/api/sys/admin/timers/names": // No params
names := core.GetTimerNames()
js, err := json.Marshal(names)
if err != nil {
return nil, err
}
out.Write(js)
case "/api/sys/admin/timers/get":
// Param: "name", optional "after" int, optional "limit" int
name, _, err := GetStringParam(m, "name", true)
if err != nil {
return nil, err
}
after, given, _ := GetStringParam(m, "after", false)
if !given {
after = "-1"
}
aft, err := strconv.Atoi(after)
if err != nil {
return nil, err
}
limit, given := m["limit"]
if !given {
limit = float64(-1)
}
history := core.GetTimerHistory(name, aft, int(limit.(float64)))
js, err := json.Marshal(history)
if err != nil {
return m, err
}
out.Write(js)
case "/api/sys/storage/get": // For testing
storage, err := s.System.PeekStorage(ctx)
if err != nil {
return nil, err
}
var acc string
switch impl := storage.(type) {
case *core.MemStorage:
state := impl.State(ctx)
js, err := json.Marshal(&state)
if err != nil {
acc = fmt.Sprintf(`{"type":"%T","error":"%s"}`,
storage,
err.Error())
} else {
acc = fmt.Sprintf(`{"type":"%T","state":%s}`,
storage,
js)
}
default:
acc = fmt.Sprintf(`{"type":"%T"}`, storage)
}
if _, err = out.Write([]byte(acc)); err != nil {
core.Log(core.ERROR, ctx, "/api/sys/storage", "error", err)
}
case "/api/sys/storage/set": // For testing
state, _, err := getMapParam(m, "state", true)
if err != nil {
return nil, err
}
storage, err := s.System.PeekStorage(ctx)
if err != nil {
return nil, err
}
switch impl := storage.(type) {
case *core.MemStorage:
mms := make(map[string]map[string]string)
for loc, pairs := range state {
m, ok := pairs.(map[string]interface{})
if !ok {
return nil, fmt.Errorf("bad pairs %#v (%T)", pairs, pairs)
}
locPairs := make(map[string]string)
for id, val := range m {
s, ok := val.(string)
if !ok {
return nil, fmt.Errorf("bad value %#v (%T)", val, val)
}
locPairs[id] = s
}
mms[loc] = locPairs
}
impl.SetState(ctx, mms)
out.Write([]byte(`{"status":"okay"}`))
default:
return nil, fmt.Errorf(`{"error":"set not supported for %T"}`, storage)
}
case "/api/sys/util/batch": // For testing
// Execute a batch of requests
batch, given := m["requests"]
if !given {
return nil, errors.New("missing 'requests' parameter")
}
var err error
switch xs := batch.(type) {
case []interface{}:
_, err = out.Write([]byte("["))
for i, x := range xs {
if 0 < i {
_, err = out.Write([]byte(","))
}
switch m := x.(type) {
case map[string]interface{}:
_, err = s.ProcessRequest(ctx, m, out)
if err != nil {
problem := fmt.Sprintf(`{"error":"%s"}`, err.Error())
_, err = out.Write([]byte(problem))
}
default:
problem := fmt.Sprintf(`"bad type %T"`, x)
_, err = out.Write([]byte(problem))
}
}
_, err = out.Write([]byte("]"))
default:
return nil, errors.New("'requests' not an array")
}
if err != nil {
core.Log(core.ERROR, ctx, "/api/sys/batch", "error", err)
}
case "/api/loc/admin/size":
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
n, err := s.System.GetSize(ctx, location)
if err != nil {
return nil, err
}
fmt.Fprintf(out, `{"size":%d}`, n)
case "/api/loc/admin/stats":
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
stats, err := s.System.GetLocationStats(ctx, location)
if err != nil {
return nil, err
}
js, err := json.Marshal(&stats)
if err != nil {
return nil, err
}
if _, err = out.Write(js); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/stats", "warning", err)
}
case "/api/loc/util/js":
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
code, _, err := GetStringParam(m, "code", true)
encoding, provided, err := GetStringParam(m, "encoding", false)
if provided {
code, err = core.DecodeString(encoding, code)
if err != nil {
return nil, err
}
}
bs := make(core.Bindings)
var props map[string]interface{}
ctl := s.System.LocControl(ctx, location)
if ctl != nil {
props = ctl.CodeProps
}
libraries := make([]string, 0, 0)
libs, given := m["libraries"]
if given {
switch vv := libs.(type) {
case []interface{}:
for _, lib := range vv {
switch s := lib.(type) {
case string:
libraries = append(libraries, s)
default:
err := fmt.Errorf("Bad library type %T (value= %#v)", lib, lib)
core.Log(core.UERR, ctx, "/api/loc/util/js", "error", err)
return nil, err
}
}
default:
err := fmt.Errorf("Bad 'libraries' type %T (value= %#v)",
libs, libs)
core.Log(core.UERR, ctx, "/api/loc/util/js", "error", err)
return nil, err
}
}
x, err := s.System.RunJavascript(ctx, location, code, libraries, &bs, props)
if err != nil {
return m, err
}
js, err := json.Marshal(&x)
if err != nil {
return m, err
}
fmt.Fprintf(out, `{"result":%s}`, js)
case "/api/loc/admin/create": // Params: location
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
created, err := s.System.CreateLocation(ctx, location)
if err != nil {
return nil, err
}
if !created {
return nil, fmt.Errorf("%s already exists", location)
}
if _, err = out.Write([]byte(`{"status":"okay"}`)); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/admin/create", "warning", err)
}
case "/api/loc/admin/clear": // Params: location
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
err = s.System.ClearLocation(ctx, location)
if err != nil {
return nil, err
}
if _, err = out.Write([]byte(`{"status":"okay"}`)); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/admin/clear", "warning", err)
}
case "/api/loc/admin/updatedmem": // Params: location
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
updated, err := s.System.GetLastUpdatedMem(ctx, location)
if err != nil {
return nil, err
}
resp := fmt.Sprintf(`{"lastUpdated":"%s","source":"memory"}`, updated)
if _, err = out.Write([]byte(resp)); err != nil {
core.Log(core.INFO, ctx, "/api/loc/admin/updatedmem", "warning", err)
}
case "/api/loc/admin/delete": // Params: location
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
err = s.System.DeleteLocation(ctx, location)
if err != nil {
return nil, err
}
if _, err = out.Write([]byte(`{"status":"okay"}`)); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/admin/delete", "warning", err)
}
case "/api/loc/events/ingest": // Params: event
event, _, err := getMapParam(m, "event", true)
if err != nil {
return nil, err
}
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
// ToDo: Not this.
js, err := json.Marshal(event)
if err != nil {
return nil, err
}
ctx.LogAccumulatorLevel = core.EVERYTHING
work, err := s.System.ProcessEvent(ctx, location, string(js))
if err != nil {
return nil, err
}
js, err = json.Marshal(work)
if err != nil {
return nil, err
}
s := fmt.Sprintf(`{"id":"%s","result":%s}`, ctx.Id(), js)
core.Log(core.INFO, ctx, "/api/loc/events/ingest", "got", s)
if _, err = out.Write([]byte(s)); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/events/ingest", "warning", err)
}
case "/api/loc/events/retry": // Params: work
workStr, _, err := GetStringParam(m, "work", true)
if err != nil {
return nil, err
}
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
fr := core.FindRules{}
err = json.Unmarshal([]byte(workStr), &fr)
if err != nil {
return nil, err
}
ctx.LogAccumulatorLevel = core.EVERYTHING
// ToDo: Support number of steps to take.
err = s.System.RetryEventWork(ctx, location, &fr)
js, err := json.Marshal(fr)
if err != nil {
return nil, err
}
s := fmt.Sprintf(`{"id":"%s","result":%s}`, ctx.Id(), js)
core.Log(core.INFO, ctx, "/api/loc/events/retry", "got", s)
if _, err = out.Write([]byte(s)); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/events/retry", "warning", err)
}
case "/api/loc/facts/add": // Params: fact
fact, _, err := getMapParam(m, "fact", true)
if err != nil {
return nil, err
}
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
id, _, err := GetStringParam(m, "id", false)
// ToDo: Not this.
js, err := json.Marshal(fact)
if err != nil {
return nil, err
}
id, err = s.System.AddFact(ctx, location, id, string(js))
if err != nil {
return nil, err
}
m := map[string]interface{}{"id": id}
js, err = json.Marshal(&m)
if err != nil {
return nil, err
}
if _, err = out.Write(js); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/facts/add", "warning", err)
}
case "/api/loc/facts/rem": // Params: id
id, _, err := GetStringParam(m, "id", true)
if err != nil {
return nil, err
}
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
rid, err := s.System.RemFact(ctx, location, id)
if err != nil {
return nil, err
}
m := map[string]interface{}{"removed": rid, "given": id}
js, err := json.Marshal(&m)
if err != nil {
return nil, err
}
if _, err = out.Write(js); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/facts/rem", "warning", err)
}
case "/api/loc/facts/get": // Params: id
id, _, err := GetStringParam(m, "id", true)
if err != nil {
return nil, err
}
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
js, err := s.System.GetFact(ctx, location, id)
if err != nil {
return nil, err
}
bs := []byte(fmt.Sprintf(`{"fact":%s,"id":"%s"}`, js, id))
if _, err = out.Write(bs); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/facts/get", "warning", err)
}
case "/api/loc/facts/search": // Params: pattern, inherited
pattern, _, err := getMapParam(m, "pattern", true)
if err != nil {
return nil, err
}
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
includedInherited, _, err := getBoolParam(m, "inherited", false)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
// ToDo: Not this.
js, err := json.Marshal(pattern)
if err != nil {
return nil, err
}
sr, err := s.System.SearchFacts(ctx, location, string(js), includedInherited)
if err != nil {
return nil, err
}
_, take := m["take"]
if take {
// Warning: Not (yet) atomic!
for _, found := range sr.Found {
_, err := s.System.RemFact(ctx, location, found.Id)
if err != nil {
core.Log(core.ERROR, ctx, "service.ProcessRequest", "app_tag", "/api/loc/facts/search", "error", err, "RemFact", found.Id)
}
// ToDo: Something with error.
}
}
js, err = json.Marshal(sr)
if err != nil {
return nil, err
}
if _, err = out.Write(js); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/facts/take", "warning", err)
}
case "/api/loc/facts/take": // Params: pattern
m["uri"] = "/api/loc/facts/search"
m["take"] = true
s.ProcessRequest(ctx, m, out)
case "/api/loc/facts/replace": // Params: pattern, fact
// Really a 'take' followed by a 'add'.
m["uri"] = "/api/loc/facts/search"
m["take"] = true
core.Log(core.INFO, ctx, "service.ProcessRequest", "app_tag", "/api/loc/facts/replace", "phase", "take")
s.ProcessRequest(ctx, m, ioutil.Discard)
core.Log(core.INFO, ctx, "service.ProcessRequest", "app_tag", "/api/loc/facts/replace", "phase", "add")
m["uri"] = "/api/loc/facts/add"
s.ProcessRequest(ctx, m, out)
case "/api/loc/facts/query": // Params: query
query, _, err := getMapParam(m, "query", true)
if err != nil {
return nil, err
}
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
// ToDo: Not this.
js, err := json.Marshal(query)
if err != nil {
return nil, err
}
qr, err := s.System.Query(ctx, location, string(js))
if err != nil {
return nil, err
}
js, err = json.Marshal(qr)
if err != nil {
return nil, err
}
if _, err = out.Write(js); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/facts/query", "warning", err)
}
case "/api/loc/rules/list": // Params: inherited
location, _, err := GetStringParam(m, "location", true)
if nil != err {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
includedInherited, _, err := getBoolParam(m, "inherited", false)
if err != nil {
return nil, err
}
ss, err := s.System.ListRules(ctx, location, includedInherited)
if nil != err {
return nil, err
}
js, err := json.Marshal(map[string][]string{"ids": ss})
if nil != err {
return nil, err
}
if _, err = out.Write(js); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/rules/list", "warning", err)
}
case "/api/loc/rules/add": // Params: rule
rule, _, err := getMapParam(m, "rule", true)
if err != nil {
return nil, err
}
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
id, _, err := GetStringParam(m, "id", false)
// ToDo: Not this.
js, err := json.Marshal(rule)
if err != nil {
return nil, err
}
id, err = s.System.AddRule(ctx, location, id, string(js))
if err != nil {
return nil, err
}
m := map[string]interface{}{"id": id}
js, err = json.Marshal(&m)
if err != nil {
return nil, err
}
if _, err = out.Write(js); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/rules/add", "warning", err)
}
case "/api/loc/rules/rem": // Params: id
id, _, err := GetStringParam(m, "id", true)
if err != nil {
return nil, err
}
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
rid, err := s.System.RemRule(ctx, location, id)
if err != nil {
return nil, err
}
m := map[string]interface{}{"removed": rid, "given": id}
js, err := json.Marshal(&m)
if err != nil {
return nil, err
}
if _, err = out.Write(js); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/rules/rem", "warning", err)
}
case "/api/loc/rules/disable": // Params: id
id, _, err := GetStringParam(m, "id", true)
if err != nil {
return nil, err
}
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
err = s.System.EnableRule(ctx, location, id, false)
if err != nil {
return nil, err
}
m := map[string]interface{}{"disabled": id}
js, err := json.Marshal(&m)
if err != nil {
return nil, err
}
if _, err = out.Write(js); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/rules/disable", "warning", err)
}
case "/api/loc/rules/enable": // Params: id
id, _, err := GetStringParam(m, "id", true)
if err != nil {
return nil, err
}
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
err = s.System.EnableRule(ctx, location, id, true)
if err != nil {
return nil, err
}
m := map[string]interface{}{"enabled": id}
js, err := json.Marshal(&m)
if err != nil {
return nil, err
}
if _, err = out.Write(js); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/rules/enable", "warning", err)
}
case "/api/loc/rules/enabled": // Params: id
id, _, err := GetStringParam(m, "id", true)
if err != nil {
return nil, err
}
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
enabled, err := s.System.RuleEnabled(ctx, location, id)
if err != nil {
return nil, err
}
m := map[string]interface{}{"ruleId": id, "enabled": enabled}
js, err := json.Marshal(&m)
if err != nil {
return nil, err
}
if _, err = out.Write(js); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/rules/enabled", "warning", err)
}
case "/api/loc/parents": // Params: none means get; "set=[x,y]" means set.
location, _, err := GetStringParam(m, "location", true)
if err != nil {
return nil, err
}
if err := s.checkLocal(ctx, location); err != nil {
return nil, err
}
js, given, err := GetStringParam(m, "set", false)
if given {
var parents []string
if err = json.Unmarshal([]byte(js), &parents); err != nil {
return nil, err
}
_, err := s.System.SetParents(ctx, location, parents)
if err != nil {
return nil, err
}
js := fmt.Sprintf(`{"result": %s}`, js)
if _, err = out.Write([]byte(js)); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/parents/set", "warning", err)
}
} else {
ps, err := s.System.GetParents(ctx, location)
if err != nil {
return nil, err
}
bs, err := json.Marshal(&ps)
if err != nil {
return nil, err
}
bs = []byte(fmt.Sprintf(`{"result":%s}`, bs))
if _, err = out.Write(bs); err != nil {
core.Log(core.ERROR, ctx, "/api/loc/parents/get", "warning", err)
}
}
default:
return nil, fmt.Errorf("Unknown URI '%s'", u)
}
return nil, nil
}
func main() {
runtime.GOMAXPROCS(8)
baseLog := flag.String("logpath", "/home/logs/flume-log", "basic log path ")
instancename := flag.String("instancename", "flume-log", "instance name ")
queuename := flag.String("queuename", "user-log", "config queuename ")
redisHost := flag.String("redis", "redis_node_6008:6008", "redishost")
redisconns := flag.Int("redis-maxconn", 20, "config redis max connetions")
zkhost := flag.String("zkhost", "momo-zk-001.m6:2210,momo-zk-002.m6:2210,momo-zk-003.m6:2210", "zkhost")
business := flag.String("businesses", "location", " businesses")
pprofPort := flag.Int("pport", -1, "pprof port default value is -1 ")
isCompress := flag.Bool("iscompress", false, "is compress")
flag.Parse()
go func() {
if *pprofPort > 0 {
log.Println(http.ListenAndServe(":"+strconv.Itoa(*pprofPort), nil))
}
}()
maxconn := *redisconns
maxIdelTime := 5
log.Printf("queuename:%s,redis:%s,flume:%s\n", *queuename, *redisHost, *zkhost)
queueHosts := make([]config.QueueHostPort, 0)
for _, hp := range parseHostPort(*redisHost) {
qhost := config.QueueHostPort{QueueName: *queuename, Maxconn: maxconn, Timeout: maxIdelTime}
qhost.HostPort = hp
queueHosts = append(queueHosts, qhost)
}
businessArr := strings.Split(*business, ",")
option := config.NewOption(*baseLog, businessArr, *zkhost, queueHosts, *isCompress)
sourcemanager := consumer.NewSourceManager(*instancename, option)
sourcemanager.Start()
log.Println("FLUME_LOG|CMD|[kill -30 $pid dump heap !]")
//接受系统信号量
var s = make(chan os.Signal, 1)
signal.Notify(s, syscall.SIGKILL, syscall.SIGUSR1)
//是否收到kill的命令
for {
cmd := <-s
if cmd == syscall.SIGKILL {
break
} else if cmd == syscall.SIGUSR1 {
//如果为siguser1则进行dump内存
unixtime := time.Now().Unix()
path := *baseLog + "/" + *instancename + "/heapdump-" + *instancename + fmt.Sprintf("%d", unixtime)
f, err := os.Create(path)
if nil != err {
log.Println("FLUME_LOG|ERROR|DUMP HEAP|" + err.Error())
continue
} else {
debug.WriteHeapDump(f.Fd())
log.Println("FLUME_LOG|SUCC|DUMP HEAP|PATH:%s" + path)
}
} else {
log.Println("FLUME_LOG|NO SIGN REG|" + cmd.String())
}
}
sourcemanager.Close()
log.Printf("FLUME_LOG|STOPPED|%s", *instancename)
}
func main() {
//http://127.0.0.1:6060/debug/pprof/
go func() {
runtime.SetBlockProfileRate(1)
log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
}()
flag.Parse()
//CPU追踪
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
} else {
log.Println("start cpu write heap profile....")
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
//内存追踪
if *memprofile != "" {
var err error
memFile, err := os.Create(*memprofile)
if err != nil {
log.Println(err)
} else {
log.Println("start mem write heap profile....")
pprof.WriteHeapProfile(memFile)
defer memFile.Close()
}
}
//协程堵塞追踪
if *blockprofile != "" {
blockFile, err := os.Create(*blockprofile)
if err != nil {
log.Println(err)
} else {
log.Println("start block write heap profile....")
runtime.SetBlockProfileRate(1)
defer pprof.Lookup("block").WriteTo(blockFile, 0)
}
}
//协程运行数
if *goroutineprofile != "" {
goFile, err := os.Create(*goroutineprofile)
if err != nil {
log.Println(err)
} else {
log.Println("start goroutine write heap profile....")
pprof.Lookup("goroutine").WriteTo(goFile, 0)
defer goFile.Close()
}
}
//堆倾卸器
if *heapdumpfile != "" {
heapFile, err := os.Create(*heapdumpfile)
if err != nil {
log.Println(err)
} else {
log.Println("start heapdump write heap profile....")
debug.WriteHeapDump(heapFile.Fd())
defer heapFile.Close()
}
}
mytcp.ServerStart()
}
func main() {
logxml := flag.String("logxml", "./log/log_producer.xml", "-logxml=./log/log_producer.xml")
k := flag.Int("k", 1, "-k=1 //kiteclient num ")
c := flag.Int("c", 1, "-c=100")
tx := flag.Bool("tx", false, "-tx=true send Tx Message")
zkhost := flag.String("zkhost", "localhost:2181", "-zkhost=localhost:2181")
flag.Parse()
runtime.GOMAXPROCS(8)
log.LoadConfiguration(*logxml)
go func() {
log.Info(http.ListenAndServe(":28000", nil))
}()
count := int32(0)
lc := int32(0)
fc := int32(0)
flc := int32(0)
go func() {
for {
tmp := count
ftmp := fc
time.Sleep(1 * time.Second)
fmt.Printf("tps:%d/%d\n", (tmp - lc), (ftmp - flc))
lc = tmp
flc = ftmp
}
}()
wg := &sync.WaitGroup{}
stop := false
clients := make([]*client.KiteQClient, 0, *k)
for j := 0; j < *k; j++ {
kiteClient := client.NewKiteQClient(*zkhost, "pb-mts-test", "123456", &defualtListener{})
kiteClient.SetTopics([]string{"trade"})
kiteClient.Start()
clients = append(clients, kiteClient)
time.Sleep(3 * time.Second)
fmt.Printf("Open Client %d\n", j)
for i := 0; i < *c; i++ {
go func(kite *client.KiteQClient) {
wg.Add(1)
for !stop {
if *tx {
msg := buildBytesMessage(false)
err := kite.SendTxBytesMessage(msg, doTranscation)
if nil != err {
fmt.Printf("SEND TxMESSAGE |FAIL|%s\n", err)
atomic.AddInt32(&fc, 1)
} else {
atomic.AddInt32(&count, 1)
}
} else {
txmsg := buildBytesMessage(true)
err := kite.SendBytesMessage(txmsg)
if nil != err {
// fmt.Printf("SEND MESSAGE |FAIL|%s\n", err)
atomic.AddInt32(&fc, 1)
} else {
atomic.AddInt32(&count, 1)
}
}
}
wg.Done()
}(kiteClient)
}
time.Sleep(10 * time.Second)
}
var s = make(chan os.Signal, 1)
signal.Notify(s, syscall.SIGKILL, syscall.SIGUSR1)
//是否收到kill的命令
for {
cmd := <-s
if cmd == syscall.SIGKILL {
break
} else if cmd == syscall.SIGUSR1 {
//如果为siguser1则进行dump内存
unixtime := time.Now().Unix()
path := "./heapdump-producer" + fmt.Sprintf("%d", unixtime)
f, err := os.Create(path)
if nil != err {
continue
} else {
debug.WriteHeapDump(f.Fd())
}
}
}
wg.Wait()
for _, k := range clients {
k.Destory()
}
}
func main() {
//http://127.0.0.1:6060/debug/pprof/
go func() {
runtime.SetBlockProfileRate(1)
log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
}()
flag.Parse()
//CPU追踪
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
} else {
log.Println("start cpu write heap profile....")
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
//内存追踪
if *memprofile != "" {
var err error
memFile, err := os.Create(*memprofile)
if err != nil {
log.Println(err)
} else {
log.Println("start mem write heap profile....")
pprof.WriteHeapProfile(memFile)
defer memFile.Close()
}
}
//协程堵塞追踪
if *blockprofile != "" {
blockFile, err := os.Create(*blockprofile)
if err != nil {
log.Println(err)
} else {
log.Println("start block write heap profile....")
runtime.SetBlockProfileRate(1)
defer pprof.Lookup("block").WriteTo(blockFile, 0)
}
}
//协程运行数
if *goroutineprofile != "" {
goFile, err := os.Create(*goroutineprofile)
if err != nil {
log.Println(err)
} else {
log.Println("start goroutine write heap profile....")
pprof.Lookup("goroutine").WriteTo(goFile, 0)
defer goFile.Close()
}
}
//堆倾卸器
if *heapdumpfile != "" {
heapFile, err := os.Create(*heapdumpfile)
if err != nil {
log.Println(err)
} else {
log.Println("start heapdump write heap profile....")
debug.WriteHeapDump(heapFile.Fd())
defer heapFile.Close()
}
}
i := 100
for {
if i < 0 {
break
}
i--
file, err := os.Open("test.exe")
if err != nil {
log.Fatalln(err)
}
buf := make([]byte, 1024)
for {
n, err := file.Read(buf)
if err != nil {
if err != io.EOF {
log.Fatalln(err)
}
log.Println("EOF")
}
if n == 0 {
break
}
fmt.Println(n)
}
//file.Close()
}
}
func main() {
fly := flag.Bool("fly", false, "-fly=true //开启服务端优先投递，false为优先存储")
logxml := flag.String("logxml", "./log/log.xml", "-logxml=./log/log.xml")
bindHost := flag.String("bind", ":13800", "-bind=localhost:13800")
zkhost := flag.String("zkhost", "localhost:2181", "-zkhost=localhost:2181")
topics := flag.String("topics", "", "-topics=trade,a,b")
dlqHourPerDay := flag.Int("dlqHourPerDay", 2, "-dlqExecHour=2 过期消息迁移时间点")
db := flag.String("db", "memory://initcap=100000&maxcap=200000",
"-db=mysql://master:3306,slave:3306?db=kite&username=root&password=root&maxConn=500&batchUpdateSize=1000&batchDelSize=1000&flushSeconds=1000")
pprofPort := flag.Int("pport", -1, "pprof port default value is -1 ")
flag.Parse()
//加载log4go的配置
log.LoadConfiguration(*logxml)
flag.VisitAll(func(f *flag.Flag) {
log.InfoLog("kite_server", "KiteQ[%s:%s]", f.Name, f.Value.String())
})
runtime.GOMAXPROCS(runtime.NumCPU())
host, port, _ := net.SplitHostPort(*bindHost)
rc := turbo.NewRemotingConfig(
"remoting-"+*bindHost,
20000, 16*1024,
16*1024, 10000, 10000,
10*time.Second, 160000)
kc := server.NewKiteQConfig("kiteq-"+*bindHost, *bindHost, *zkhost, *fly, 5*time.Second, 8000,
5*time.Second, *dlqHourPerDay /*每天凌晨2点执行*/, strings.Split(*topics, ","), *db, rc)
qserver := server.NewKiteQServer(kc)
qserver.Start()
go func() {
if *pprofPort > 0 {
http.HandleFunc("/stat", qserver.HandleStat)
http.HandleFunc("/binds", qserver.HandleBindings)
log.Error(http.ListenAndServe(host+":"+strconv.Itoa(*pprofPort), nil))
}
}()
var s = make(chan os.Signal, 1)
signal.Notify(s, syscall.SIGKILL, syscall.SIGUSR1)
//是否收到kill的命令
for {
cmd := <-s
if cmd == syscall.SIGKILL {
break
} else if cmd == syscall.SIGUSR1 {
//如果为siguser1则进行dump内存
unixtime := time.Now().Unix()
path := "./heapdump-kiteq-" + host + "_" + port + fmt.Sprintf("%d", unixtime)
f, err := os.Create(path)
if nil != err {
continue
} else {
debug.WriteHeapDump(f.Fd())
}
}
}
qserver.Shutdown()
log.InfoLog("kite_server", "KiteQServer IS STOPPED!")
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
//http://127.0.0.1:6060/debug/pprof/
go func() {
runtime.SetBlockProfileRate(1)
log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
}()
flag.Parse()
//CPU追踪
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
} else {
log.Println("start cpu write heap profile....")
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
//内存追踪
if *memprofile != "" {
var err error
memFile, err := os.Create(*memprofile)
if err != nil {
log.Println(err)
} else {
log.Println("start mem write heap profile....")
pprof.WriteHeapProfile(memFile)
defer memFile.Close()
}
}
//协程堵塞追踪
if *blockprofile != "" {
blockFile, err := os.Create(*blockprofile)
if err != nil {
log.Println(err)
} else {
log.Println("start block write heap profile....")
runtime.SetBlockProfileRate(1)
defer pprof.Lookup("block").WriteTo(blockFile, 0)
}
}
//协程运行数
if *goroutineprofile != "" {
goFile, err := os.Create(*goroutineprofile)
if err != nil {
log.Println(err)
} else {
log.Println("start goroutine write heap profile....")
pprof.Lookup("goroutine").WriteTo(goFile, 0)
defer goFile.Close()
}
}
//堆倾卸器
if *heapdumpfile != "" {
heapFile, err := os.Create(*heapdumpfile)
if err != nil {
log.Println(err)
} else {
log.Println("start heapdump write heap profile....")
debug.WriteHeapDump(heapFile.Fd())
defer heapFile.Close()
}
}
session, err := mgo.Dial("localhost:27017")
if err != nil {
panic(err)
}
defer session.Close()
db = session.DB("test")
c = db.C("t")
insertData()
}