func TestInsert(t *testing.T) {
t.Skip()
var e error
skipIfConnectionIsNil(t)
es := []string{}
qinsert := ctx.NewQuery().From(tableName).Insert()
for i := 1; i <= 3; i++ {
qty := toolkit.RandInt(10)
price := toolkit.RandInt(10) * 50
amount := qty * price
u := &Orders{
toolkit.Sprintf("ord0%d", i+10),
toolkit.Sprintf("item%d", i),
qty,
price,
amount,
toolkit.Sprintf("available"),
}
e = qinsert.Exec(toolkit.M{}.Set("data", u))
if e != nil {
es = append(es, toolkit.Sprintf("Insert fail %d: %s \n", i, e.Error()))
}
}
if len(es) > 0 {
t.Fatal(es)
}
operation = "Test Insert"
sintaks = `
ctx.NewQuery().From(tableName).Insert().
Exec(toolkit.M{}.Set("data", u))`
TestSelect(t)
}
func TestCRUD(t *testing.T) {
skipIfConnectionIsNil(t)
e := ctx.NewQuery().Delete().From(tableName).SetConfig("multiexec", true).Exec(nil)
if e != nil {
t.Fatalf("Delete fail: %s", e.Error())
}
es := []string{}
qinsert := ctx.NewQuery().From(tableName).SetConfig("multiexec", true).Insert()
for i := 1; i <= 50; i++ {
u := &testUser{
toolkit.Sprintf("user%d", i),
toolkit.Sprintf("User %d", i),
toolkit.RandInt(30) + 20, true}
e = qinsert.Exec(toolkit.M{}.Set("data", u))
if e != nil {
es = append(es, toolkit.Sprintf("Insert fail %d: %s \n", i, e.Error()))
}
}
if len(es) > 0 {
t.Fatal(es)
}
e = ctx.NewQuery().Update().From(tableName).Where(dbox.Lte("_id", "user2")).Exec(toolkit.M{}.Set("data", toolkit.M{}.Set("Enable", false)))
if e != nil {
t.Fatalf("Update fail: %s", e.Error())
}
}
func TestFrom(t *testing.T) {
for i := 0; i < sampleCount; i++ {
obj := Obj{}
obj.F = toolkit.ToFloat64(toolkit.RandInt(1000), 2, toolkit.RoundingAuto)
obj.I = toolkit.RandInt(1000) + 5
objs = append(objs, obj)
if i == 0 {
min = obj.I
max = obj.I
sum = obj.I
avg = toolkit.ToFloat64(obj.I, 4, toolkit.RoundingAuto)
} else {
sum += obj.I
avg = toolkit.ToFloat64(sum, 4, toolkit.RoundingAuto) /
toolkit.ToFloat64(i+1, 4, toolkit.RoundingAuto)
if min > obj.I {
min = obj.I
}
if max < obj.I {
max = obj.I
}
}
}
c = crowd.From(&objs)
check(t, c.Error, "")
toolkit.Printf("Data len: %d, max: %d, min: %d, sum: %d, avg: %5.4f\n", c.Len(), max, min, sum, avg)
}
func TestInsert(t *testing.T) {
//t.Skip()
ctx, _ := prepareContext()
defer ctx.Close()
ctx.DeleteMany(new(UserModel), nil)
t0 := time.Now()
count := 20
for i := 1; i <= count; i++ {
fmt.Printf("Insert user no %d ...", i)
u := new(UserModel)
u.ID = "user" + strconv.Itoa(i)
u.FullName = "ORM User " + strconv.Itoa(i)
u.Age = tk.RandInt(20) + 20
u.Email = "*****@*****.**"
u.Password = "******"
u.Enable = 1
u.RandomDate = time.Now().Add(time.Duration(int64(tk.RandInt(1000)) * int64(time.Minute)))
e = ctx.Insert(u)
if e != nil {
t.Errorf("Error Load %d: %s", i, e.Error())
return
} else {
fmt.Println("OK")
}
}
fmt.Printf("Run process for %v \n", time.Since(t0))
}
func TestStorageWrite(t *testing.T) {
skipIfClientNil(t)
es := []string{}
toolkit.Printf("Writing Data:\n")
for i := 0; i < 200; i++ {
dataku := toolkit.RandInt(1000)
totalInt += dataku
//toolkit.Printf("%d ", dataku)
in := toolkit.M{}.Set("key", fmt.Sprintf("public.dataku.%d", i)).Set("data", toolkit.ToBytes(dataku, ""))
writeResult := client.Call("set", in)
if writeResult.Status != toolkit.Status_OK {
es = append(es, toolkit.Sprintf("Fail to write data %d : %d => %s", i, dataku, writeResult.Message))
}
}
if len(es) > 0 {
errorTxt := ""
if len(es) <= 10 {
errorTxt = strings.Join(es, "\n")
} else {
errorTxt = strings.Join(es[:10], "\n") + "\n... And others ..."
}
t.Errorf("Write data fail.\n%s", errorTxt)
}
}
func TestPut(t *testing.T) {
if e := getToken(); e != nil {
//he(t, e)
//return
}
surl := restPrefix + "/put"
for i := 1; i <= 1000; i++ {
randomTxt := createRandomString(toolkit.RandInt(22) + 10)
data := struct {
Token string
Data string
}{
token,
randomTxt,
}
fmt.Printf("Saving %d value %s", i, data.Data)
r, e := toolkit.HttpCall(surl, "XPUT", toolkit.GetEncodeByte(data), nil)
//e = nil
if e != nil {
fmt.Printf("... Fail: %s \n", e.Error())
} else {
if r.StatusCode != 200 {
fmt.Printf("... Fail: %d %s \n", r.StatusCode, r.Status)
} else {
fmt.Println("...Done")
}
fmt.Println("...Done")
}
}
}
func createRandomString(randomLength int) string {
randomTxt := ""
chars := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890!@#$%_-"
for x := 0; x < randomLength; x++ {
ic := toolkit.RandInt(len(chars) - 1)
c := chars[ic]
randomTxt += string(c)
}
return randomTxt
}
func main() {
flag.Parse()
f.Source(func(out chan string) {
for i := 1; i <= 1000; i++ {
txt := createRandomString(toolkit.RandInt(23) + 10)
fmt.Printf("Data %d is %s \n", i, txt)
out <- txt
}
}, 3).Map(func(s string) (int, int) {
return len(s), 1
}).Partition(5).ReduceByKey(func(x, y int) int {
return x + y
}).Sort(nil).Map(func(k int, v int) {
fmt.Printf("Number of data with %d chars are %d \n", k, v)
})
flow.Ready()
f.Run()
}
func TestFind(t *testing.T) {
ms := []toolkit.M{}
for i := 1; i <= 10; i++ {
m := toolkit.M{}
m.Set("_id", i)
m.Set("random", toolkit.RandInt(100))
ms = append(ms, m)
}
toolkit.Printf("Original Value\n%s\n", toolkit.JsonString(ms))
indexes := dbox.Find(ms, []*dbox.Filter{
//dbox.Or(dbox.Lt("random", 20), dbox.And(dbox.Gte("random", 60), dbox.Lte("random", 70)))})
dbox.And(dbox.Gte("random", 30), dbox.Lte("random", 80))})
records := []toolkit.M{}
for _, v := range indexes {
records = append(records, ms[v])
}
for _, r := range records {
toolkit.Printf("Record: %s \n", toolkit.JsonString(r))
}
toolkit.Printf("Find %d records of %d records\n", len(indexes), len(ms))
}
func main() {
// runtime.GOMAXPROCS(runtime.NumCPU())
var err error
flagConfig := flag.String("config", "", "config file")
flagDebugMode := flag.Bool("debug", false, "debug mode")
flagLogPath := flag.String("logpath", "", "log path")
flag.Parse()
tconfig := toolkit.ToString(*flagConfig)
tlogPath := toolkit.ToString(*flagLogPath)
debugMode = *flagDebugMode
configPath = strings.Replace(tconfig, `"`, "", -1)
if tconfig == "" {
sedotan.CheckError(errors.New("-config cannot be empty"))
}
logstdout := false
logfile := true
logPath := strings.Replace(tlogPath, `"`, "", -1)
fmt.Println("Log Path, ", logPath)
if logPath == "" {
logPath, err = os.Getwd()
if err != nil {
logstdout = true
logfile = false
fmt.Println("cannot get log path")
}
}
//Temporary :
var snapshotpath string = filepath.Join(logPath, "daemonsnapshot.csv")
// err = getDataSnapShot(snapshotpath)
// sedotan.CheckError(err)
Log, err = toolkit.NewLog(logstdout, logfile, logPath, "daemonlog-%s", "20060102")
sedotan.CheckError(err)
Log.AddLog(fmt.Sprintf("Start daemon grabbing, config path : %v", configPath), "INFO")
for {
err = nil
daemoninterval := 1 * time.Second
<-time.After(daemoninterval)
thistime = sedotan.TimeNow()
Log.AddLog(fmt.Sprintf("Run daemon"), "INFO")
initiate()
Log.AddLog(fmt.Sprintf("Fetch config grabbing started"), "INFO")
fetchconfig()
if configerr != nil {
Log.AddLog(configerr.Error(), "ERROR")
configerr = nil
}
Log.AddLog(fmt.Sprintf("Get data snapshot"), "INFO")
err = getDataSnapShot(snapshotpath)
if err != nil {
Log.AddLog(fmt.Sprintf("Failed to start grabbing, snapshot error : %v", err.Error()), "ERROR")
continue
}
if len(config) > 0 {
Log.AddLog(fmt.Sprintf("Ready to start grabbing, found %v valid config and %v active config", validconfig, len(config)), "INFO")
} else {
Log.AddLog(fmt.Sprintf("Skip to start grabbing, found %v valid config and 0 active config", validconfig), "ERROR")
}
for _, econfig := range config {
err = nil
eid := econfig.Get("_id", "").(string)
Log.AddLog(fmt.Sprintf("Check config for id : %v", eid), "INFO")
intervalconf, _ := toolkit.ToM(econfig["intervalconf"])
grabconf, _ := toolkit.ToM(econfig["grabconf"])
var isonprocess bool = checkisonprocess(eid, intervalconf, grabconf)
var isconfrun bool = econfig.Get("running", false).(bool) //check config status run/stop (isconfrun)
var istimerun bool = checkistimerun(eid, intervalconf, grabconf)
etype := econfig.Get("sourcetype", "").(string)
//check grab status onprocess/done/na/error -> conf file / snapshot file ? (isonprocess)
//check interval+time start/corn schedulling and check last running for interval(istimerun)
// fmt.Printf("!%v && %v && %v \n", isonprocess, isconfrun, istimerun)
if !isonprocess && isconfrun && istimerun {
Log.AddLog(fmt.Sprintf("Start grabbing for id : %v", eid), "INFO")
// save data snapshot using dbox save
tsnapshot := mapsnapshot[eid]
tsnapshot.Laststartgrab = sedotan.DateToString(thistime)
tsnapshot.Pid = toolkit.RandInt(1000) + 1
tsnapshot.Cgtotal = 0
tsnapshot.Cgprocess = 0
mapsnapshot[eid] = tsnapshot
err = savesnapshot(eid, snapshotpath)
if err != nil {
Log.AddLog(fmt.Sprintf("Save snapshot id : %v, error found : %v", eid, err), "INFO")
continue
}
// run grabbing
go func(id string, etype string, thistime time.Time) {
etype = strings.ToLower(etype)
var cmd *exec.Cmd
aCommand := make([]string, 0, 0)
if runtime.GOOS == "windows" {
aCommand = append(aCommand, "cmd")
aCommand = append(aCommand, "/C")
}
apppath := ""
// Check Type [SourceType_HttpHtml|SourceType_HttpJson|SourceType_DocExcel]
switch {
case strings.Contains(etype, "http"):
if runtime.GOOS == "windows" {
apppath = filepath.Join(EC_APP_PATH, "cli", "sedotanw.exe")
} else {
apppath = filepath.Join(EC_APP_PATH, "cli", "sedotanw")
}
case strings.Contains(etype, "doc"):
if runtime.GOOS == "windows" {
apppath = filepath.Join(EC_APP_PATH, "cli", "sedotans.exe")
} else {
apppath = filepath.Join(EC_APP_PATH, "cli", "sedotans")
}
}
aCommand = append(aCommand, apppath)
aCommand = append(aCommand, `-config="`+configPath+`"`)
aCommand = append(aCommand, `-snapshot="`+snapshotpath+`"`)
aCommand = append(aCommand, `-id="`+eid+`"`)
aCommand = append(aCommand, `-pid=`+toolkit.ToString(mapsnapshot[eid].Pid))
cmd = exec.Command(aCommand[0], aCommand[1:]...)
Log.AddLog(fmt.Sprintf("[%v] run at %v, run : %v", eid, sedotan.DateToString(thistime), cmd.Args), "INFO")
byteoutput, err := cmd.CombinedOutput()
if err != nil {
Log.AddLog(fmt.Sprintf("[%v] run at %v, found error : %v", eid, sedotan.DateToString(thistime), err.Error()), "ERROR")
}
Log.AddLog(fmt.Sprintf("[%v] run at %v, done with message : %v", eid, sedotan.DateToString(thistime), string(byteoutput)), "INFO")
}(eid, etype, thistime)
} else {
Log.AddLog(fmt.Sprintf("Skip grabbing for id : %v", eid), "INFO")
}
}
}
// _, err := sedotan.Process(config)
// sedotan.CheckError(err)
}