Exemple #1
1
func TestInsert(t *testing.T) {
	t.Skip()
	var e error
	skipIfConnectionIsNil(t)

	es := []string{}
	qinsert := ctx.NewQuery().From(tableName).Insert()
	for i := 1; i <= 3; i++ {
		qty := toolkit.RandInt(10)
		price := toolkit.RandInt(10) * 50
		amount := qty * price
		u := &Orders{
			toolkit.Sprintf("ord0%d", i+10),
			toolkit.Sprintf("item%d", i),
			qty,
			price,
			amount,
			toolkit.Sprintf("available"),
		}
		e = qinsert.Exec(toolkit.M{}.Set("data", u))
		if e != nil {
			es = append(es, toolkit.Sprintf("Insert fail %d: %s \n", i, e.Error()))
		}
	}

	if len(es) > 0 {
		t.Fatal(es)
	}

	operation = "Test Insert"
	sintaks = `
		ctx.NewQuery().From(tableName).Insert().
		Exec(toolkit.M{}.Set("data", u))`
	TestSelect(t)
}
Exemple #2
1
func TestCRUD(t *testing.T) {
	skipIfConnectionIsNil(t)
	e := ctx.NewQuery().Delete().From(tableName).SetConfig("multiexec", true).Exec(nil)
	if e != nil {
		t.Fatalf("Delete fail: %s", e.Error())
	}

	es := []string{}
	qinsert := ctx.NewQuery().From(tableName).SetConfig("multiexec", true).Insert()
	for i := 1; i <= 50; i++ {
		u := &testUser{
			toolkit.Sprintf("user%d", i),
			toolkit.Sprintf("User %d", i),
			toolkit.RandInt(30) + 20, true}
		e = qinsert.Exec(toolkit.M{}.Set("data", u))
		if e != nil {
			es = append(es, toolkit.Sprintf("Insert fail %d: %s \n", i, e.Error()))
		}
	}

	if len(es) > 0 {
		t.Fatal(es)
	}

	e = ctx.NewQuery().Update().From(tableName).Where(dbox.Lte("_id", "user2")).Exec(toolkit.M{}.Set("data", toolkit.M{}.Set("Enable", false)))
	if e != nil {
		t.Fatalf("Update fail: %s", e.Error())
	}
}
Exemple #3
0
func TestFrom(t *testing.T) {
	for i := 0; i < sampleCount; i++ {
		obj := Obj{}
		obj.F = toolkit.ToFloat64(toolkit.RandInt(1000), 2, toolkit.RoundingAuto)
		obj.I = toolkit.RandInt(1000) + 5
		objs = append(objs, obj)

		if i == 0 {
			min = obj.I
			max = obj.I
			sum = obj.I
			avg = toolkit.ToFloat64(obj.I, 4, toolkit.RoundingAuto)
		} else {
			sum += obj.I
			avg = toolkit.ToFloat64(sum, 4, toolkit.RoundingAuto) /
				toolkit.ToFloat64(i+1, 4, toolkit.RoundingAuto)
			if min > obj.I {
				min = obj.I
			}
			if max < obj.I {
				max = obj.I
			}
		}
	}

	c = crowd.From(&objs)
	check(t, c.Error, "")
	toolkit.Printf("Data len: %d, max: %d, min: %d, sum: %d, avg: %5.4f\n", c.Len(), max, min, sum, avg)
}
Exemple #4
0
func TestInsert(t *testing.T) {
	//t.Skip()
	ctx, _ := prepareContext()
	defer ctx.Close()

	ctx.DeleteMany(new(UserModel), nil)

	t0 := time.Now()
	count := 20
	for i := 1; i <= count; i++ {
		fmt.Printf("Insert user no %d ...", i)
		u := new(UserModel)
		u.ID = "user" + strconv.Itoa(i)
		u.FullName = "ORM User " + strconv.Itoa(i)
		u.Age = tk.RandInt(20) + 20
		u.Email = "*****@*****.**"
		u.Password = "******"
		u.Enable = 1
		u.RandomDate = time.Now().Add(time.Duration(int64(tk.RandInt(1000)) * int64(time.Minute)))
		e = ctx.Insert(u)
		if e != nil {
			t.Errorf("Error Load %d: %s", i, e.Error())
			return
		} else {
			fmt.Println("OK")
		}
	}
	fmt.Printf("Run process for %v \n", time.Since(t0))
}
Exemple #5
0
func TestStorageWrite(t *testing.T) {
	skipIfClientNil(t)
	es := []string{}

	toolkit.Printf("Writing Data:\n")
	for i := 0; i < 200; i++ {
		dataku := toolkit.RandInt(1000)
		totalInt += dataku
		//toolkit.Printf("%d ", dataku)

		in := toolkit.M{}.Set("key", fmt.Sprintf("public.dataku.%d", i)).Set("data", toolkit.ToBytes(dataku, ""))
		writeResult := client.Call("set", in)
		if writeResult.Status != toolkit.Status_OK {
			es = append(es, toolkit.Sprintf("Fail to write data %d : %d => %s", i, dataku, writeResult.Message))
		}
	}

	if len(es) > 0 {
		errorTxt := ""
		if len(es) <= 10 {
			errorTxt = strings.Join(es, "\n")
		} else {
			errorTxt = strings.Join(es[:10], "\n") + "\n... And others ..."
		}
		t.Errorf("Write data fail.\n%s", errorTxt)
	}
}
Exemple #6
0
func TestPut(t *testing.T) {
	if e := getToken(); e != nil {
		//he(t, e)
		//return
	}

	surl := restPrefix + "/put"
	for i := 1; i <= 1000; i++ {
		randomTxt := createRandomString(toolkit.RandInt(22) + 10)
		data := struct {
			Token string
			Data  string
		}{
			token,
			randomTxt,
		}

		fmt.Printf("Saving %d value %s", i, data.Data)

		r, e := toolkit.HttpCall(surl, "XPUT", toolkit.GetEncodeByte(data), nil)
		//e = nil
		if e != nil {
			fmt.Printf("... Fail: %s \n", e.Error())
		} else {
			if r.StatusCode != 200 {
				fmt.Printf("... Fail: %d %s \n", r.StatusCode, r.Status)
			} else {
				fmt.Println("...Done")
			}
			fmt.Println("...Done")
		}
	}
}
Exemple #7
0
func createRandomString(randomLength int) string {
	randomTxt := ""
	chars := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890!@#$%_-"
	for x := 0; x < randomLength; x++ {
		ic := toolkit.RandInt(len(chars) - 1)
		c := chars[ic]
		randomTxt += string(c)
	}
	return randomTxt
}
Exemple #8
0
func main() {
	flag.Parse()

	f.Source(func(out chan string) {
		for i := 1; i <= 1000; i++ {
			txt := createRandomString(toolkit.RandInt(23) + 10)
			fmt.Printf("Data %d is %s \n", i, txt)
			out <- txt
		}
	}, 3).Map(func(s string) (int, int) {
		return len(s), 1
	}).Partition(5).ReduceByKey(func(x, y int) int {
		return x + y
	}).Sort(nil).Map(func(k int, v int) {
		fmt.Printf("Number of data with %d chars are %d \n", k, v)
	})

	flow.Ready()
	f.Run()
}
func TestFind(t *testing.T) {
	ms := []toolkit.M{}
	for i := 1; i <= 10; i++ {
		m := toolkit.M{}
		m.Set("_id", i)
		m.Set("random", toolkit.RandInt(100))
		ms = append(ms, m)
	}
	toolkit.Printf("Original Value\n%s\n", toolkit.JsonString(ms))

	indexes := dbox.Find(ms, []*dbox.Filter{
		//dbox.Or(dbox.Lt("random", 20), dbox.And(dbox.Gte("random", 60), dbox.Lte("random", 70)))})
		dbox.And(dbox.Gte("random", 30), dbox.Lte("random", 80))})

	records := []toolkit.M{}
	for _, v := range indexes {
		records = append(records, ms[v])
	}
	for _, r := range records {
		toolkit.Printf("Record: %s \n", toolkit.JsonString(r))
	}
	toolkit.Printf("Find %d records of %d records\n", len(indexes), len(ms))
}
Exemple #10
0
func main() {
	// runtime.GOMAXPROCS(runtime.NumCPU())
	var err error

	flagConfig := flag.String("config", "", "config file")
	flagDebugMode := flag.Bool("debug", false, "debug mode")
	flagLogPath := flag.String("logpath", "", "log path")

	flag.Parse()
	tconfig := toolkit.ToString(*flagConfig)
	tlogPath := toolkit.ToString(*flagLogPath)
	debugMode = *flagDebugMode

	configPath = strings.Replace(tconfig, `"`, "", -1)
	if tconfig == "" {
		sedotan.CheckError(errors.New("-config cannot be empty"))
	}

	logstdout := false
	logfile := true

	logPath := strings.Replace(tlogPath, `"`, "", -1)
	fmt.Println("Log Path, ", logPath)
	if logPath == "" {
		logPath, err = os.Getwd()
		if err != nil {
			logstdout = true
			logfile = false
			fmt.Println("cannot get log path")
		}
	}

	//Temporary :
	var snapshotpath string = filepath.Join(logPath, "daemonsnapshot.csv")
	// err = getDataSnapShot(snapshotpath)
	// sedotan.CheckError(err)

	Log, err = toolkit.NewLog(logstdout, logfile, logPath, "daemonlog-%s", "20060102")
	sedotan.CheckError(err)

	Log.AddLog(fmt.Sprintf("Start daemon grabbing, config path : %v", configPath), "INFO")

	for {
		err = nil

		daemoninterval := 1 * time.Second
		<-time.After(daemoninterval)
		thistime = sedotan.TimeNow()

		Log.AddLog(fmt.Sprintf("Run daemon"), "INFO")
		initiate()

		Log.AddLog(fmt.Sprintf("Fetch config grabbing started"), "INFO")
		fetchconfig()
		if configerr != nil {
			Log.AddLog(configerr.Error(), "ERROR")
			configerr = nil
		}

		Log.AddLog(fmt.Sprintf("Get data snapshot"), "INFO")
		err = getDataSnapShot(snapshotpath)
		if err != nil {
			Log.AddLog(fmt.Sprintf("Failed to start grabbing, snapshot error : %v", err.Error()), "ERROR")
			continue
		}

		if len(config) > 0 {
			Log.AddLog(fmt.Sprintf("Ready to start grabbing, found %v valid config and %v active config", validconfig, len(config)), "INFO")
		} else {
			Log.AddLog(fmt.Sprintf("Skip to start grabbing, found %v valid config and 0 active config", validconfig), "ERROR")
		}

		for _, econfig := range config {
			err = nil
			eid := econfig.Get("_id", "").(string)
			Log.AddLog(fmt.Sprintf("Check config for id : %v", eid), "INFO")
			intervalconf, _ := toolkit.ToM(econfig["intervalconf"])
			grabconf, _ := toolkit.ToM(econfig["grabconf"])

			var isonprocess bool = checkisonprocess(eid, intervalconf, grabconf)
			var isconfrun bool = econfig.Get("running", false).(bool) //check config status run/stop (isconfrun)
			var istimerun bool = checkistimerun(eid, intervalconf, grabconf)

			etype := econfig.Get("sourcetype", "").(string)
			//check grab status onprocess/done/na/error -> conf file / snapshot file ? (isonprocess)
			//check interval+time start/corn schedulling and check last running for interval(istimerun)
			// fmt.Printf("!%v && %v && %v \n", isonprocess, isconfrun, istimerun)
			if !isonprocess && isconfrun && istimerun {
				Log.AddLog(fmt.Sprintf("Start grabbing for id : %v", eid), "INFO")

				// save data snapshot using dbox save
				tsnapshot := mapsnapshot[eid]
				tsnapshot.Laststartgrab = sedotan.DateToString(thistime)
				tsnapshot.Pid = toolkit.RandInt(1000) + 1
				tsnapshot.Cgtotal = 0
				tsnapshot.Cgprocess = 0

				mapsnapshot[eid] = tsnapshot

				err = savesnapshot(eid, snapshotpath)
				if err != nil {
					Log.AddLog(fmt.Sprintf("Save snapshot id : %v, error found : %v", eid, err), "INFO")
					continue
				}
				// run grabbing
				go func(id string, etype string, thistime time.Time) {
					etype = strings.ToLower(etype)
					var cmd *exec.Cmd
					aCommand := make([]string, 0, 0)

					if runtime.GOOS == "windows" {
						aCommand = append(aCommand, "cmd")
						aCommand = append(aCommand, "/C")
					}

					apppath := ""
					// Check Type [SourceType_HttpHtml|SourceType_HttpJson|SourceType_DocExcel]
					switch {
					case strings.Contains(etype, "http"):
						if runtime.GOOS == "windows" {
							apppath = filepath.Join(EC_APP_PATH, "cli", "sedotanw.exe")
						} else {
							apppath = filepath.Join(EC_APP_PATH, "cli", "sedotanw")
						}
					case strings.Contains(etype, "doc"):
						if runtime.GOOS == "windows" {
							apppath = filepath.Join(EC_APP_PATH, "cli", "sedotans.exe")
						} else {
							apppath = filepath.Join(EC_APP_PATH, "cli", "sedotans")
						}
					}
					aCommand = append(aCommand, apppath)
					aCommand = append(aCommand, `-config="`+configPath+`"`)
					aCommand = append(aCommand, `-snapshot="`+snapshotpath+`"`)
					aCommand = append(aCommand, `-id="`+eid+`"`)
					aCommand = append(aCommand, `-pid=`+toolkit.ToString(mapsnapshot[eid].Pid))

					cmd = exec.Command(aCommand[0], aCommand[1:]...)

					Log.AddLog(fmt.Sprintf("[%v] run at %v, run : %v", eid, sedotan.DateToString(thistime), cmd.Args), "INFO")
					byteoutput, err := cmd.CombinedOutput()
					if err != nil {
						Log.AddLog(fmt.Sprintf("[%v] run at %v, found error : %v", eid, sedotan.DateToString(thistime), err.Error()), "ERROR")
					}
					Log.AddLog(fmt.Sprintf("[%v] run at %v, done with message : %v", eid, sedotan.DateToString(thistime), string(byteoutput)), "INFO")
				}(eid, etype, thistime)
			} else {
				Log.AddLog(fmt.Sprintf("Skip grabbing for id : %v", eid), "INFO")
			}
		}
	}
	// _, err := sedotan.Process(config)
	// sedotan.CheckError(err)
}