/** * Read records in one batch. */ func batchReads( client *as.Client, keyPrefix string, binName string, size int, ) { // Batch gets into one call. keys := make([]*as.Key, size) for i := 0; i < size; i++ { keys[i], _ = as.NewKey(*shared.Namespace, *shared.Set, keyPrefix+strconv.Itoa(i+1)) } records, err := client.BatchGet(nil, keys, binName) shared.PanicOnError(err) for i := 0; i < len(records); i++ { key := keys[i] record := records[i] level := asl.ERR var value interface{} if record != nil { level = asl.INFO value = record.Bins[binName] } asl.Logger.LogAtLevel(level, "Record: ns=%s set=%s key=%s bin=%s value=%s", key.Namespace(), key.SetName(), key.Value(), binName, value) } if len(records) != size { log.Fatalf("Record size mismatch. Expected %d. Received %d.", size, len(records)) } }
// EnvExists returns true iff the env exists. func EnvExists(as *aerospike.Client, env string) (exists bool, err error) { key, err := envKey(env) if err != nil { return false, err } return as.Exists(nil, key) }
func runExample(client *as.Client) { // Write initial record. key, _ := as.NewKey(*shared.Namespace, *shared.Set, "opkey") bin1 := as.NewBin("optintbin", 7) bin2 := as.NewBin("optstringbin", "string value") log.Printf("Put: namespace=%s set=%s key=%s bin1=%s value1=%s bin2=%s value2=%s", key.Namespace(), key.SetName(), key.Value(), bin1.Name, bin1.Value, bin2.Name, bin2.Value) client.PutBins(shared.WritePolicy, key, bin1, bin2) // Add integer, write new string and read record. bin3 := as.NewBin(bin1.Name, 4) bin4 := as.NewBin(bin2.Name, "new string") log.Println("Add: ", bin3.Value) log.Println("Write: ", bin4.Value) log.Println("Read:") record, err := client.Operate(shared.WritePolicy, key, as.AddOp(bin3), as.PutOp(bin4), as.GetOp()) shared.PanicOnError(err) if record == nil { log.Fatalf( "Failed to get: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value()) } binExpected := as.NewBin(bin3.Name, 11) shared.ValidateBin(key, binExpected, record) shared.ValidateBin(key, bin4, record) }
func runExample(client *as.Client) { log.Printf("Scan parallel: namespace=" + *shared.Namespace + " set=" + *shared.Set) recordCount := 0 begin := time.Now() policy := as.NewScanPolicy() recordset, err := client.ScanAll(policy, *shared.Namespace, *shared.Set) shared.PanicOnError(err) L: for { select { case rec := <-recordset.Records: if rec == nil { break L } recordCount++ if (recordCount % 10000) == 0 { log.Println("Records ", recordCount) } case err := <-recordset.Errors: // if there was an error, stop shared.PanicOnError(err) } } end := time.Now() seconds := float64(end.Sub(begin)) / float64(time.Second) log.Println("Total records returned: ", recordCount) log.Println("Elapsed time: ", seconds, " seconds") performance := shared.Round(float64(recordCount)/float64(seconds), 0.5, 0) log.Println("Records/second: ", performance) }
func syncAppAsDv(ac *as.Client, pool *redis.Pool, app string) error { stm := as.NewStatement(NS_PUSH, SET_DV, "id", "tp", "mt") stm.Addfilter(as.NewEqualFilter("app", app)) p := as.NewQueryPolicy() p.RecordQueueSize = 200000 rs, err := ac.Query(p, stm) if err != nil { fmt.Println("error happen When we sync @app:", app, " @err:", err) return err } latch := utee.NewThrottle(10000) for res := range rs.Results() { if res.Err != nil { return res.Err } bins := res.Record.Bins id, ok := bins["id"].(string) if !ok { // log.Println("warn, id is empty", bins) continue } latch.Acquire() go Deq(pool, latch, id) } return nil }
/** * Read record header data in one batch. */ func batchReadHeaders( client *as.Client, keyPrefix string, size int, ) { // Batch gets into one call. keys := make([]*as.Key, size) for i := 0; i < size; i++ { keys[i], _ = as.NewKey(*shared.Namespace, *shared.Set, keyPrefix+strconv.Itoa(i+1)) } records, err := client.BatchGetHeader(nil, keys) shared.PanicOnError(err) for i := 0; i < len(records); i++ { key := keys[i] record := records[i] level := asl.ERR generation := 0 expiration := 0 if record != nil && (record.Generation > 0 || record.Expiration > 0) { level = asl.INFO generation = record.Generation expiration = record.Expiration } asl.Logger.LogAtLevel(level, "Record: ns=%s set=%s key=%s generation=%d expiration=%d", key.Namespace(), key.SetName(), key.Value(), generation, expiration) } if len(records) != size { log.Fatalf("Record size mismatch. Expected %d. Received %d.", size, len(records)) } }
func runExample(client *as.Client) { log.Println("Scan series: namespace=", *shared.Namespace, " set=", *shared.Set) // Use low scan priority. This will take more time, but it will reduce // the load on the server. policy := as.NewScanPolicy() policy.MaxRetries = 1 policy.Priority = as.LOW nodeList := client.GetNodes() begin := time.Now() for _, node := range nodeList { log.Println("Scan node ", node.GetName()) recordset, err := client.ScanNode(policy, node, *shared.Namespace, *shared.Set) shared.PanicOnError(err) L: for { select { case rec := <-recordset.Records: if rec == nil { break L } metrics, exists := setMap[rec.Key.SetName()] if !exists { metrics = Metrics{} } metrics.count++ metrics.total++ setMap[rec.Key.SetName()] = metrics case err := <-recordset.Errors: // if there was an error, stop shared.PanicOnError(err) } } for k, v := range setMap { log.Println("Node ", node, " set ", k, " count: ", v.count) v.count = 0 } } end := time.Now() seconds := float64(end.Sub(begin)) / float64(time.Second) log.Println("Elapsed time: ", seconds, " seconds") total := 0 for k, v := range setMap { log.Println("Total set ", k, " count: ", v.total) total += v.total } log.Println("Grand total: ", total) performance := shared.Round(float64(total)/seconds, 0.5, 0) log.Println("Records/second: ", performance) }
// DeleteEnv removes an env. func DeleteEnv(as *aerospike.Client, env string) error { key, err := envKey(env) if err != nil { return err } _, err = as.Delete(nil, key) return err }
// ServiceExists returns true iff the service exists. func ServiceExists( as *aerospike.Client, env string, service string) ( exists bool, err error) { key, err := serviceKey(env, service) if err != nil { return false, err } return as.Exists(nil, key) }
// NewEnv stores a new env. func NewEnv(as *aerospike.Client, env string, description string) error { key, err := envKey(env) if err != nil { return err } wp := aerospike.NewWritePolicy(0, 0) wp.RecordExistsAction = aerospike.CREATE_ONLY return as.PutObject(wp, key, &EnvData{Description: description}) }
// SetServiceLiveCodeVersion sets the provided version of the code as the live // one. func SetServiceLiveCodeVersion( as *aerospike.Client, env string, service string, version int64) ( err error) { key, err := serviceKey(env, service) if err != nil { return err } bin := aerospike.NewBin("LiveVer", version) return as.PutBins(nil, key, bin) }
func runExample(client *as.Client) { key, err := as.NewKey(*shared.Namespace, *shared.Set, "addkey") shared.PanicOnError(err) binName := "addbin" // Delete record if it already exists. client.Delete(shared.WritePolicy, key) // Perform some adds and check results. bin := as.NewBin(binName, 10) log.Println("Initial add will create record. Initial value is ", bin.Value, ".") client.AddBins(shared.WritePolicy, key, bin) bin = as.NewBin(binName, 5) log.Println("Add ", bin.Value, " to existing record.") client.AddBins(shared.WritePolicy, key, bin) record, err := client.Get(shared.Policy, key, bin.Name) shared.PanicOnError(err) if record == nil { log.Fatalf( "Failed to get: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value()) } // The value received from the server is an unsigned byte stream. // Convert to an integer before comparing with expected. received := record.Bins[bin.Name] expected := 15 if received == expected { log.Printf("Add successful: ns=%s set=%s key=%s bin=%s value=%s", key.Namespace(), key.SetName(), key.Value(), bin.Name, received) } else { log.Fatalf("Add mismatch: Expected %d. Received %d.", expected, received) } // Demonstrate add and get combined. bin = as.NewBin(binName, 30) log.Println("Add ", bin.Value, " to existing record.") record, err = client.Operate(shared.WritePolicy, key, as.AddOp(bin), as.GetOp()) shared.PanicOnError(err) expected = 45 received = record.Bins[bin.Name] if received == expected { log.Printf("Add successful: ns=%s set=%s key=%s bin=%s value=%s", key.Namespace(), key.SetName(), key.Value(), bin.Name, received) } else { log.Fatalf("Add mismatch: Expected %d. Received %d.", expected, received) } }
func createIndex(c *driver.Client, ns, set string) error { task, err := c.CreateIndex(nil, ns, set, set, urlField, driver.STRING) if err != nil { if err.Error() == "Index already exists" { return nil } return err } return <-task.OnComplete() }
// InitServiceTable initializes the indices for the service table. func InitServiceTable(as *aerospike.Client) error { bin := "Env" task, err := as.CreateIndex( nil, leverOSNamespace, serviceSet, serviceSet+bin, bin, aerospike.STRING) if err != nil { return err } err, isErr := <-task.OnComplete() if isErr && err != nil { return err } return nil }
// UpdateService updates the data associated with a service. func UpdateService( as *aerospike.Client, env string, service string, description string, isPublic bool, liveCodeVersion int64) ( err error) { key, err := serviceKey(env, service) if err != nil { return err } wp := aerospike.NewWritePolicy(0, 0) wp.RecordExistsAction = aerospike.UPDATE_ONLY return as.PutBins( wp, key, aerospike.NewBin("Desc", description), aerospike.NewBin("IsPub", btoi(isPublic)), aerospike.NewBin("LiveVer", liveCodeVersion), ) }
/** * Write records individually. */ func writeRecords( client *as.Client, keyPrefix string, binName string, valuePrefix string, size int, ) { for i := 1; i <= size; i++ { key, _ := as.NewKey(*shared.Namespace, *shared.Set, keyPrefix+strconv.Itoa(i)) bin := as.NewBin(binName, valuePrefix+strconv.Itoa(i)) log.Printf("Put: ns=%s set=%s key=%s bin=%s value=%s", key.Namespace(), key.SetName(), key.Value(), bin.Name, bin.Value) client.PutBins(shared.WritePolicy, key, bin) } }
func writeUsingUdf(client *as.Client) { key, _ := as.NewKey(*shared.Namespace, *shared.Set, "udfkey1") bin := as.NewBin("udfbin1", "string value") client.Execute(shared.WritePolicy, key, "record_example", "writeBin", as.NewValue(bin.Name), bin.Value) record, err := client.Get(shared.Policy, key, bin.Name) shared.PanicOnError(err) expected := bin.Value.String() received := record.Bins[bin.Name].(string) if received == expected { log.Printf("Data matched: namespace=%s set=%s key=%s bin=%s value=%s", key.Namespace(), key.SetName(), key.Value(), bin.Name, received) } else { log.Printf("Data mismatch: Expected %s. Received %s.", expected, received) } }
// NewServiceCodeVersion generates a version number of the code and returns it. func NewServiceCodeVersion( as *aerospike.Client, env string, service string) ( newVersion int64, err error) { key, err := serviceKey(env, service) if err != nil { return 0, err } bin := aerospike.NewBin("NextVer", 1) record, err := as.Operate( nil, key, aerospike.AddOp(bin), aerospike.GetOpForBin("NextVer")) if err != nil { return 0, err } return int64(record.Bins["NextVer"].(int)), nil }
// Read record header data. func runGetHeaderExample(client *as.Client) { key, _ := as.NewKey(*shared.Namespace, *shared.Set, "putgetkey") log.Printf("Get record header: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value()) record, err := client.GetHeader(shared.Policy, key) shared.PanicOnError(err) if record == nil { panic(errors.New(fmt.Sprintf( "Failed to get: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value()))) } // Generation should be greater than zero. Make sure it's populated. if record.Generation == 0 { panic(errors.New(fmt.Sprintf( "Invalid record header: generation=%d expiration=%d", record.Generation, record.Expiration))) } log.Printf("Received: generation=%d expiration=%d (%s)\n", record.Generation, record.Expiration, time.Duration(record.Expiration)*time.Second) }
func writeWithValidation(client *as.Client) { key, _ := as.NewKey(*shared.Namespace, *shared.Set, "udfkey4") binName := "udfbin4" // Lua function writeWithValidation accepts number between 1 and 10. // Write record with valid value. log.Printf("Write with valid value.") client.Execute(shared.WritePolicy, key, "record_example", "writeWithValidation", as.NewValue(binName), as.NewValue(4)) // Write record with invalid value. log.Printf("Write with invalid value.") _, err := client.Execute(shared.WritePolicy, key, "record_example", "writeWithValidation", as.NewValue(binName), as.NewValue(11)) if err == nil { log.Printf("UDF should not have succeeded!") } else { log.Printf("Success. UDF resulted in exception as expected.") } }
// NewService stores a new service. func NewService( as *aerospike.Client, env string, service string, description string, isPublic bool) error { key, err := serviceKey(env, service) if err != nil { return err } wp := aerospike.NewWritePolicy(0, 0) wp.RecordExistsAction = aerospike.CREATE_ONLY return as.PutObject(wp, key, &ServiceData{ Env: env, // TODO: This is not guaranteed to be unique. Should figure out a way // to generate this uniquely. UniqueID: leverutil.RandomHostName(), Description: description, NextCodeVersion: 0, LiveCodeVersion: 0, IsPublic: isPublic, }) }
func runReplaceOnlyExample(client *as.Client) { key, err := as.NewKey(*shared.Namespace, *shared.Set, "replaceonlykey") shared.PanicOnError(err) bin := as.NewBin("bin", "value") // Delete record if it already exists. client.Delete(shared.WritePolicy, key) log.Printf("Replace record requiring that it exists: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value()) wpolicy := as.NewWritePolicy(0, 0) wpolicy.RecordExistsAction = as.REPLACE_ONLY err = client.PutBins(wpolicy, key, bin) if ae, ok := err.(ast.AerospikeError); ok && ae.ResultCode() == ast.KEY_NOT_FOUND_ERROR { log.Printf("Success. `Not found` error returned as expected.") } else { log.Fatalln("Failure. This command should have resulted in an error.") } }
// Execute put and get on a server configured as single-bin. func runSingleBinExample(client *as.Client) { key, _ := as.NewKey(*shared.Namespace, *shared.Set, "putgetkey") bin := as.NewBin("", "value") log.Printf("Single Put: namespace=%s set=%s key=%s value=%s", key.Namespace(), key.SetName(), key.Value(), bin.Value) client.PutBins(shared.WritePolicy, key, bin) log.Printf("Single Get: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value()) record, err := client.Get(shared.Policy, key) shared.PanicOnError(err) if record == nil { panic(errors.New(fmt.Sprintf( "Failed to get: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value()))) } shared.ValidateBin(key, bin, record) }
func runExample(client *as.Client) { // Set LuaPath luaPath, _ := os.Getwd() luaPath += "/udf/" as.SetLuaPath(luaPath) filename := "sum_single_bin" regTask, err := client.RegisterUDFFromFile(nil, luaPath+filename+".lua", filename+".lua", as.LUA) shared.PanicOnError(err) // wait until UDF is created shared.PanicOnError(<-regTask.OnComplete()) sum := 0 for i := 1; i <= keyCount; i++ { sum += i key, err := as.NewKey(*shared.Namespace, *shared.Set, i) shared.PanicOnError(err) bin1 := as.NewBin("bin1", i) client.PutBins(nil, key, bin1) } t := time.Now() stm := as.NewStatement(*shared.Namespace, *shared.Set) res, err := client.QueryAggregate(nil, stm, filename, filename, "bin1") shared.PanicOnError(err) for rec := range res.Results() { log.Printf("Result %f should equal %d\n", rec.Record.Bins["SUCCESS"], sum) } log.Println("Map/Reduce took", time.Now().Sub(t)) }
func runExample(client *as.Client) { key, _ := as.NewKey(*shared.Namespace, *shared.Set, "touchkey") bin := as.NewBin("touchbin", "touchvalue") log.Printf("Create record with 2 second expiration.") writePolicy := as.NewWritePolicy(0, 2) client.PutBins(writePolicy, key, bin) log.Printf("Touch same record with 5 second expiration.") writePolicy.Expiration = 5 record, err := client.Operate(writePolicy, key, as.TouchOp(), as.GetHeaderOp()) if record == nil { log.Fatalf( "Failed to get: namespace=%s set=%s key=%s bin=%s value=%s", key.Namespace(), key.SetName(), key.Value(), bin.Name, nil) } if record.Expiration == 0 { log.Fatalf( "Failed to get record expiration: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value()) } log.Printf("Sleep 3 seconds.") time.Sleep(3 * time.Second) record, err = client.Get(shared.Policy, key, bin.Name) shared.PanicOnError(err) if record == nil { log.Fatalf( "Failed to get: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value()) } log.Printf("Success. Record still exists.") log.Printf("Sleep 4 seconds.") time.Sleep(4 * time.Second) record, err = client.Get(shared.Policy, key, bin.Name) shared.PanicOnError(err) if record == nil { log.Printf("Success. Record expired as expected.") } else { log.Fatalf("Found record when it should have expired.") } }
/** * Check existence of records in one batch. */ func batchExists( client *as.Client, keyPrefix string, size int, ) { // Batch into one call. keys := make([]*as.Key, size) for i := 0; i < size; i++ { keys[i], _ = as.NewKey(*shared.Namespace, *shared.Set, keyPrefix+strconv.Itoa(i+1)) } existsArray, err := client.BatchExists(nil, keys) shared.PanicOnError(err) for i := 0; i < len(existsArray); i++ { key := keys[i] exists := existsArray[i] log.Printf("Record: ns=%s set=%s key=%s exists=%s", key.Namespace(), key.SetName(), key.Value(), exists) } }
func writeIfNotExists(client *as.Client) { key, _ := as.NewKey(*shared.Namespace, *shared.Set, "udfkey3") binName := "udfbin3" // Delete record if it already exists. client.Delete(shared.WritePolicy, key) // Write record only if not already exists. This should succeed. client.Execute(shared.WritePolicy, key, "record_example", "writeUnique", as.NewValue(binName), as.NewValue("first")) // Verify record written. record, err := client.Get(shared.Policy, key, binName) shared.PanicOnError(err) expected := "first" received := record.Bins[binName].(string) if received == expected { log.Printf("Record written: namespace=%s set=%s key=%s bin=%s value=%s", key.Namespace(), key.SetName(), key.Value(), binName, received) } else { log.Printf("Data mismatch: Expected %s. Received %s.", expected, received) } // Write record second time. This should fail. log.Printf("Attempt second write.") client.Execute(shared.WritePolicy, key, "record_example", "writeUnique", as.NewValue(binName), as.NewValue("second")) // Verify record not written. record, err = client.Get(shared.Policy, key, binName) shared.PanicOnError(err) received = record.Bins[binName].(string) if received == expected { log.Printf("Success. Record remained unchanged: namespace=%s set=%s key=%s bin=%s value=%s", key.Namespace(), key.SetName(), key.Value(), binName, received) } else { log.Printf("Data mismatch: Expected %s. Received %s.", expected, received) } }
func writeBlobUsingUdf(client *as.Client) { key, _ := as.NewKey(*shared.Namespace, *shared.Set, "udfkey6") binName := "udfbin6" // Create packed blob using standard java tools. dos := bytes.Buffer{} // dos.Write(9845) dos.WriteString("Hello world.") blob := dos.Bytes() client.Execute(shared.WritePolicy, key, "record_example", "writeBin", as.NewValue(binName), as.NewValue(blob)) received, err := client.Execute(shared.WritePolicy, key, "record_example", "readBin", as.NewValue(binName)) shared.PanicOnError(err) if bytes.Equal(blob, received.([]byte)) { log.Printf("Blob data matched: namespace=%s set=%s key=%s bin=%v value=%v", key.Namespace(), key.SetName(), key.Value(), binName, received) } else { log.Fatalf( "Mismatch: expected=%v received=%v", blob, received) } }
// Execute put and get on a server configured as multi-bin. This is the server default. func runMultiBinExample(client *as.Client) { key, _ := as.NewKey(*shared.Namespace, *shared.Set, "putgetkey") bin1 := as.NewBin("bin1", "value1") bin2 := as.NewBin("bin2", "value2") log.Printf("Put: namespace=%s set=%s key=%s bin1=%s value1=%s bin2=%s value2=%s", key.Namespace(), key.SetName(), key.Value(), bin1.Name, bin1.Value, bin2.Name, bin2.Value) client.PutBins(shared.WritePolicy, key, bin1, bin2) log.Printf("Get: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value()) record, err := client.Get(shared.Policy, key) shared.PanicOnError(err) if record == nil { panic(errors.New(fmt.Sprintf( "Failed to get: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value()))) } shared.ValidateBin(key, bin1, record) shared.ValidateBin(key, bin2, record) }
func writeListMapUsingUdf(client *as.Client) { key, _ := as.NewKey(*shared.Namespace, *shared.Set, "udfkey5") inner := []interface{}{"string2", int64(8)} innerMap := map[interface{}]interface{}{"a": int64(1), int64(2): "b", "list": inner} list := []interface{}{"string1", int64(4), inner, innerMap} binName := "udfbin5" client.Execute(shared.WritePolicy, key, "record_example", "writeBin", as.NewValue(binName), as.NewValue(list)) received, err := client.Execute(shared.WritePolicy, key, "record_example", "readBin", as.NewValue(binName)) shared.PanicOnError(err) if testEq(received.([]interface{}), list) { log.Printf("UDF data matched: namespace=%s set=%s key=%s bin=%s value=%s", key.Namespace(), key.SetName(), key.Value(), binName, received) } else { log.Println("UDF data mismatch") log.Println("Expected ", list) log.Println("Received ", received) } }