/**
* Read records in one batch.
*/
func batchReads(
client *as.Client,
keyPrefix string,
binName string,
size int,
) {
// Batch gets into one call.
keys := make([]*as.Key, size)
for i := 0; i < size; i++ {
keys[i], _ = as.NewKey(*shared.Namespace, *shared.Set, keyPrefix+strconv.Itoa(i+1))
}
records, err := client.BatchGet(nil, keys, binName)
shared.PanicOnError(err)
for i := 0; i < len(records); i++ {
key := keys[i]
record := records[i]
level := asl.ERR
var value interface{}
if record != nil {
level = asl.INFO
value = record.Bins[binName]
}
asl.Logger.LogAtLevel(level, "Record: ns=%s set=%s key=%s bin=%s value=%s",
key.Namespace(), key.SetName(), key.Value(), binName, value)
}
if len(records) != size {
log.Fatalf("Record size mismatch. Expected %d. Received %d.", size, len(records))
}
}
// EnvExists returns true iff the env exists.
func EnvExists(as *aerospike.Client, env string) (exists bool, err error) {
key, err := envKey(env)
if err != nil {
return false, err
}
return as.Exists(nil, key)
}
func runExample(client *as.Client) {
// Write initial record.
key, _ := as.NewKey(*shared.Namespace, *shared.Set, "opkey")
bin1 := as.NewBin("optintbin", 7)
bin2 := as.NewBin("optstringbin", "string value")
log.Printf("Put: namespace=%s set=%s key=%s bin1=%s value1=%s bin2=%s value2=%s",
key.Namespace(), key.SetName(), key.Value(), bin1.Name, bin1.Value, bin2.Name, bin2.Value)
client.PutBins(shared.WritePolicy, key, bin1, bin2)
// Add integer, write new string and read record.
bin3 := as.NewBin(bin1.Name, 4)
bin4 := as.NewBin(bin2.Name, "new string")
log.Println("Add: ", bin3.Value)
log.Println("Write: ", bin4.Value)
log.Println("Read:")
record, err := client.Operate(shared.WritePolicy, key, as.AddOp(bin3), as.PutOp(bin4), as.GetOp())
shared.PanicOnError(err)
if record == nil {
log.Fatalf(
"Failed to get: namespace=%s set=%s key=%s",
key.Namespace(), key.SetName(), key.Value())
}
binExpected := as.NewBin(bin3.Name, 11)
shared.ValidateBin(key, binExpected, record)
shared.ValidateBin(key, bin4, record)
}
func runExample(client *as.Client) {
log.Printf("Scan parallel: namespace=" + *shared.Namespace + " set=" + *shared.Set)
recordCount := 0
begin := time.Now()
policy := as.NewScanPolicy()
recordset, err := client.ScanAll(policy, *shared.Namespace, *shared.Set)
shared.PanicOnError(err)
L:
for {
select {
case rec := <-recordset.Records:
if rec == nil {
break L
}
recordCount++
if (recordCount % 10000) == 0 {
log.Println("Records ", recordCount)
}
case err := <-recordset.Errors:
// if there was an error, stop
shared.PanicOnError(err)
}
}
end := time.Now()
seconds := float64(end.Sub(begin)) / float64(time.Second)
log.Println("Total records returned: ", recordCount)
log.Println("Elapsed time: ", seconds, " seconds")
performance := shared.Round(float64(recordCount)/float64(seconds), 0.5, 0)
log.Println("Records/second: ", performance)
}
func syncAppAsDv(ac *as.Client, pool *redis.Pool, app string) error {
stm := as.NewStatement(NS_PUSH, SET_DV, "id", "tp", "mt")
stm.Addfilter(as.NewEqualFilter("app", app))
p := as.NewQueryPolicy()
p.RecordQueueSize = 200000
rs, err := ac.Query(p, stm)
if err != nil {
fmt.Println("error happen When we sync @app:", app, " @err:", err)
return err
}
latch := utee.NewThrottle(10000)
for res := range rs.Results() {
if res.Err != nil {
return res.Err
}
bins := res.Record.Bins
id, ok := bins["id"].(string)
if !ok {
// log.Println("warn, id is empty", bins)
continue
}
latch.Acquire()
go Deq(pool, latch, id)
}
return nil
}
/**
* Read record header data in one batch.
*/
func batchReadHeaders(
client *as.Client,
keyPrefix string,
size int,
) {
// Batch gets into one call.
keys := make([]*as.Key, size)
for i := 0; i < size; i++ {
keys[i], _ = as.NewKey(*shared.Namespace, *shared.Set, keyPrefix+strconv.Itoa(i+1))
}
records, err := client.BatchGetHeader(nil, keys)
shared.PanicOnError(err)
for i := 0; i < len(records); i++ {
key := keys[i]
record := records[i]
level := asl.ERR
generation := 0
expiration := 0
if record != nil && (record.Generation > 0 || record.Expiration > 0) {
level = asl.INFO
generation = record.Generation
expiration = record.Expiration
}
asl.Logger.LogAtLevel(level, "Record: ns=%s set=%s key=%s generation=%d expiration=%d",
key.Namespace(), key.SetName(), key.Value(), generation, expiration)
}
if len(records) != size {
log.Fatalf("Record size mismatch. Expected %d. Received %d.", size, len(records))
}
}
func runExample(client *as.Client) {
log.Println("Scan series: namespace=", *shared.Namespace, " set=", *shared.Set)
// Use low scan priority. This will take more time, but it will reduce
// the load on the server.
policy := as.NewScanPolicy()
policy.MaxRetries = 1
policy.Priority = as.LOW
nodeList := client.GetNodes()
begin := time.Now()
for _, node := range nodeList {
log.Println("Scan node ", node.GetName())
recordset, err := client.ScanNode(policy, node, *shared.Namespace, *shared.Set)
shared.PanicOnError(err)
L:
for {
select {
case rec := <-recordset.Records:
if rec == nil {
break L
}
metrics, exists := setMap[rec.Key.SetName()]
if !exists {
metrics = Metrics{}
}
metrics.count++
metrics.total++
setMap[rec.Key.SetName()] = metrics
case err := <-recordset.Errors:
// if there was an error, stop
shared.PanicOnError(err)
}
}
for k, v := range setMap {
log.Println("Node ", node, " set ", k, " count: ", v.count)
v.count = 0
}
}
end := time.Now()
seconds := float64(end.Sub(begin)) / float64(time.Second)
log.Println("Elapsed time: ", seconds, " seconds")
total := 0
for k, v := range setMap {
log.Println("Total set ", k, " count: ", v.total)
total += v.total
}
log.Println("Grand total: ", total)
performance := shared.Round(float64(total)/seconds, 0.5, 0)
log.Println("Records/second: ", performance)
}
// DeleteEnv removes an env.
func DeleteEnv(as *aerospike.Client, env string) error {
key, err := envKey(env)
if err != nil {
return err
}
_, err = as.Delete(nil, key)
return err
}
// ServiceExists returns true iff the service exists.
func ServiceExists(
as *aerospike.Client, env string, service string) (
exists bool, err error) {
key, err := serviceKey(env, service)
if err != nil {
return false, err
}
return as.Exists(nil, key)
}
// NewEnv stores a new env.
func NewEnv(as *aerospike.Client, env string, description string) error {
key, err := envKey(env)
if err != nil {
return err
}
wp := aerospike.NewWritePolicy(0, 0)
wp.RecordExistsAction = aerospike.CREATE_ONLY
return as.PutObject(wp, key, &EnvData{Description: description})
}
// SetServiceLiveCodeVersion sets the provided version of the code as the live
// one.
func SetServiceLiveCodeVersion(
as *aerospike.Client, env string, service string, version int64) (
err error) {
key, err := serviceKey(env, service)
if err != nil {
return err
}
bin := aerospike.NewBin("LiveVer", version)
return as.PutBins(nil, key, bin)
}
func runExample(client *as.Client) {
key, err := as.NewKey(*shared.Namespace, *shared.Set, "addkey")
shared.PanicOnError(err)
binName := "addbin"
// Delete record if it already exists.
client.Delete(shared.WritePolicy, key)
// Perform some adds and check results.
bin := as.NewBin(binName, 10)
log.Println("Initial add will create record. Initial value is ", bin.Value, ".")
client.AddBins(shared.WritePolicy, key, bin)
bin = as.NewBin(binName, 5)
log.Println("Add ", bin.Value, " to existing record.")
client.AddBins(shared.WritePolicy, key, bin)
record, err := client.Get(shared.Policy, key, bin.Name)
shared.PanicOnError(err)
if record == nil {
log.Fatalf(
"Failed to get: namespace=%s set=%s key=%s",
key.Namespace(), key.SetName(), key.Value())
}
// The value received from the server is an unsigned byte stream.
// Convert to an integer before comparing with expected.
received := record.Bins[bin.Name]
expected := 15
if received == expected {
log.Printf("Add successful: ns=%s set=%s key=%s bin=%s value=%s",
key.Namespace(), key.SetName(), key.Value(), bin.Name, received)
} else {
log.Fatalf("Add mismatch: Expected %d. Received %d.", expected, received)
}
// Demonstrate add and get combined.
bin = as.NewBin(binName, 30)
log.Println("Add ", bin.Value, " to existing record.")
record, err = client.Operate(shared.WritePolicy, key, as.AddOp(bin), as.GetOp())
shared.PanicOnError(err)
expected = 45
received = record.Bins[bin.Name]
if received == expected {
log.Printf("Add successful: ns=%s set=%s key=%s bin=%s value=%s",
key.Namespace(), key.SetName(), key.Value(), bin.Name, received)
} else {
log.Fatalf("Add mismatch: Expected %d. Received %d.", expected, received)
}
}
func createIndex(c *driver.Client, ns, set string) error {
task, err := c.CreateIndex(nil, ns, set, set, urlField, driver.STRING)
if err != nil {
if err.Error() == "Index already exists" {
return nil
}
return err
}
return <-task.OnComplete()
}
// InitServiceTable initializes the indices for the service table.
func InitServiceTable(as *aerospike.Client) error {
bin := "Env"
task, err := as.CreateIndex(
nil, leverOSNamespace, serviceSet, serviceSet+bin, bin,
aerospike.STRING)
if err != nil {
return err
}
err, isErr := <-task.OnComplete()
if isErr && err != nil {
return err
}
return nil
}
// UpdateService updates the data associated with a service.
func UpdateService(
as *aerospike.Client, env string, service string, description string,
isPublic bool, liveCodeVersion int64) (
err error) {
key, err := serviceKey(env, service)
if err != nil {
return err
}
wp := aerospike.NewWritePolicy(0, 0)
wp.RecordExistsAction = aerospike.UPDATE_ONLY
return as.PutBins(
wp, key,
aerospike.NewBin("Desc", description),
aerospike.NewBin("IsPub", btoi(isPublic)),
aerospike.NewBin("LiveVer", liveCodeVersion),
)
}
/**
* Write records individually.
*/
func writeRecords(
client *as.Client,
keyPrefix string,
binName string,
valuePrefix string,
size int,
) {
for i := 1; i <= size; i++ {
key, _ := as.NewKey(*shared.Namespace, *shared.Set, keyPrefix+strconv.Itoa(i))
bin := as.NewBin(binName, valuePrefix+strconv.Itoa(i))
log.Printf("Put: ns=%s set=%s key=%s bin=%s value=%s",
key.Namespace(), key.SetName(), key.Value(), bin.Name, bin.Value)
client.PutBins(shared.WritePolicy, key, bin)
}
}
func writeUsingUdf(client *as.Client) {
key, _ := as.NewKey(*shared.Namespace, *shared.Set, "udfkey1")
bin := as.NewBin("udfbin1", "string value")
client.Execute(shared.WritePolicy, key, "record_example", "writeBin", as.NewValue(bin.Name), bin.Value)
record, err := client.Get(shared.Policy, key, bin.Name)
shared.PanicOnError(err)
expected := bin.Value.String()
received := record.Bins[bin.Name].(string)
if received == expected {
log.Printf("Data matched: namespace=%s set=%s key=%s bin=%s value=%s",
key.Namespace(), key.SetName(), key.Value(), bin.Name, received)
} else {
log.Printf("Data mismatch: Expected %s. Received %s.", expected, received)
}
}
// NewServiceCodeVersion generates a version number of the code and returns it.
func NewServiceCodeVersion(
as *aerospike.Client, env string, service string) (
newVersion int64, err error) {
key, err := serviceKey(env, service)
if err != nil {
return 0, err
}
bin := aerospike.NewBin("NextVer", 1)
record, err := as.Operate(
nil, key, aerospike.AddOp(bin),
aerospike.GetOpForBin("NextVer"))
if err != nil {
return 0, err
}
return int64(record.Bins["NextVer"].(int)), nil
}
// Read record header data.
func runGetHeaderExample(client *as.Client) {
key, _ := as.NewKey(*shared.Namespace, *shared.Set, "putgetkey")
log.Printf("Get record header: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value())
record, err := client.GetHeader(shared.Policy, key)
shared.PanicOnError(err)
if record == nil {
panic(errors.New(fmt.Sprintf(
"Failed to get: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value())))
}
// Generation should be greater than zero. Make sure it's populated.
if record.Generation == 0 {
panic(errors.New(fmt.Sprintf(
"Invalid record header: generation=%d expiration=%d", record.Generation, record.Expiration)))
}
log.Printf("Received: generation=%d expiration=%d (%s)\n", record.Generation, record.Expiration, time.Duration(record.Expiration)*time.Second)
}
func writeWithValidation(client *as.Client) {
key, _ := as.NewKey(*shared.Namespace, *shared.Set, "udfkey4")
binName := "udfbin4"
// Lua function writeWithValidation accepts number between 1 and 10.
// Write record with valid value.
log.Printf("Write with valid value.")
client.Execute(shared.WritePolicy, key, "record_example", "writeWithValidation", as.NewValue(binName), as.NewValue(4))
// Write record with invalid value.
log.Printf("Write with invalid value.")
_, err := client.Execute(shared.WritePolicy, key, "record_example", "writeWithValidation", as.NewValue(binName), as.NewValue(11))
if err == nil {
log.Printf("UDF should not have succeeded!")
} else {
log.Printf("Success. UDF resulted in exception as expected.")
}
}
// NewService stores a new service.
func NewService(
as *aerospike.Client, env string, service string,
description string, isPublic bool) error {
key, err := serviceKey(env, service)
if err != nil {
return err
}
wp := aerospike.NewWritePolicy(0, 0)
wp.RecordExistsAction = aerospike.CREATE_ONLY
return as.PutObject(wp, key, &ServiceData{
Env: env,
// TODO: This is not guaranteed to be unique. Should figure out a way
// to generate this uniquely.
UniqueID: leverutil.RandomHostName(),
Description: description,
NextCodeVersion: 0,
LiveCodeVersion: 0,
IsPublic: isPublic,
})
}
func runReplaceOnlyExample(client *as.Client) {
key, err := as.NewKey(*shared.Namespace, *shared.Set, "replaceonlykey")
shared.PanicOnError(err)
bin := as.NewBin("bin", "value")
// Delete record if it already exists.
client.Delete(shared.WritePolicy, key)
log.Printf("Replace record requiring that it exists: namespace=%s set=%s key=%s",
key.Namespace(), key.SetName(), key.Value())
wpolicy := as.NewWritePolicy(0, 0)
wpolicy.RecordExistsAction = as.REPLACE_ONLY
err = client.PutBins(wpolicy, key, bin)
if ae, ok := err.(ast.AerospikeError); ok && ae.ResultCode() == ast.KEY_NOT_FOUND_ERROR {
log.Printf("Success. `Not found` error returned as expected.")
} else {
log.Fatalln("Failure. This command should have resulted in an error.")
}
}
// Execute put and get on a server configured as single-bin.
func runSingleBinExample(client *as.Client) {
key, _ := as.NewKey(*shared.Namespace, *shared.Set, "putgetkey")
bin := as.NewBin("", "value")
log.Printf("Single Put: namespace=%s set=%s key=%s value=%s",
key.Namespace(), key.SetName(), key.Value(), bin.Value)
client.PutBins(shared.WritePolicy, key, bin)
log.Printf("Single Get: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value())
record, err := client.Get(shared.Policy, key)
shared.PanicOnError(err)
if record == nil {
panic(errors.New(fmt.Sprintf(
"Failed to get: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value())))
}
shared.ValidateBin(key, bin, record)
}
func runExample(client *as.Client) {
// Set LuaPath
luaPath, _ := os.Getwd()
luaPath += "/udf/"
as.SetLuaPath(luaPath)
filename := "sum_single_bin"
regTask, err := client.RegisterUDFFromFile(nil, luaPath+filename+".lua", filename+".lua", as.LUA)
shared.PanicOnError(err)
// wait until UDF is created
shared.PanicOnError(<-regTask.OnComplete())
sum := 0
for i := 1; i <= keyCount; i++ {
sum += i
key, err := as.NewKey(*shared.Namespace, *shared.Set, i)
shared.PanicOnError(err)
bin1 := as.NewBin("bin1", i)
client.PutBins(nil, key, bin1)
}
t := time.Now()
stm := as.NewStatement(*shared.Namespace, *shared.Set)
res, err := client.QueryAggregate(nil, stm, filename, filename, "bin1")
shared.PanicOnError(err)
for rec := range res.Results() {
log.Printf("Result %f should equal %d\n", rec.Record.Bins["SUCCESS"], sum)
}
log.Println("Map/Reduce took", time.Now().Sub(t))
}
func runExample(client *as.Client) {
key, _ := as.NewKey(*shared.Namespace, *shared.Set, "touchkey")
bin := as.NewBin("touchbin", "touchvalue")
log.Printf("Create record with 2 second expiration.")
writePolicy := as.NewWritePolicy(0, 2)
client.PutBins(writePolicy, key, bin)
log.Printf("Touch same record with 5 second expiration.")
writePolicy.Expiration = 5
record, err := client.Operate(writePolicy, key, as.TouchOp(), as.GetHeaderOp())
if record == nil {
log.Fatalf(
"Failed to get: namespace=%s set=%s key=%s bin=%s value=%s",
key.Namespace(), key.SetName(), key.Value(), bin.Name, nil)
}
if record.Expiration == 0 {
log.Fatalf(
"Failed to get record expiration: namespace=%s set=%s key=%s",
key.Namespace(), key.SetName(), key.Value())
}
log.Printf("Sleep 3 seconds.")
time.Sleep(3 * time.Second)
record, err = client.Get(shared.Policy, key, bin.Name)
shared.PanicOnError(err)
if record == nil {
log.Fatalf(
"Failed to get: namespace=%s set=%s key=%s",
key.Namespace(), key.SetName(), key.Value())
}
log.Printf("Success. Record still exists.")
log.Printf("Sleep 4 seconds.")
time.Sleep(4 * time.Second)
record, err = client.Get(shared.Policy, key, bin.Name)
shared.PanicOnError(err)
if record == nil {
log.Printf("Success. Record expired as expected.")
} else {
log.Fatalf("Found record when it should have expired.")
}
}
/**
* Check existence of records in one batch.
*/
func batchExists(
client *as.Client,
keyPrefix string,
size int,
) {
// Batch into one call.
keys := make([]*as.Key, size)
for i := 0; i < size; i++ {
keys[i], _ = as.NewKey(*shared.Namespace, *shared.Set, keyPrefix+strconv.Itoa(i+1))
}
existsArray, err := client.BatchExists(nil, keys)
shared.PanicOnError(err)
for i := 0; i < len(existsArray); i++ {
key := keys[i]
exists := existsArray[i]
log.Printf("Record: ns=%s set=%s key=%s exists=%s",
key.Namespace(), key.SetName(), key.Value(), exists)
}
}
func writeIfNotExists(client *as.Client) {
key, _ := as.NewKey(*shared.Namespace, *shared.Set, "udfkey3")
binName := "udfbin3"
// Delete record if it already exists.
client.Delete(shared.WritePolicy, key)
// Write record only if not already exists. This should succeed.
client.Execute(shared.WritePolicy, key, "record_example", "writeUnique", as.NewValue(binName), as.NewValue("first"))
// Verify record written.
record, err := client.Get(shared.Policy, key, binName)
shared.PanicOnError(err)
expected := "first"
received := record.Bins[binName].(string)
if received == expected {
log.Printf("Record written: namespace=%s set=%s key=%s bin=%s value=%s",
key.Namespace(), key.SetName(), key.Value(), binName, received)
} else {
log.Printf("Data mismatch: Expected %s. Received %s.", expected, received)
}
// Write record second time. This should fail.
log.Printf("Attempt second write.")
client.Execute(shared.WritePolicy, key, "record_example", "writeUnique", as.NewValue(binName), as.NewValue("second"))
// Verify record not written.
record, err = client.Get(shared.Policy, key, binName)
shared.PanicOnError(err)
received = record.Bins[binName].(string)
if received == expected {
log.Printf("Success. Record remained unchanged: namespace=%s set=%s key=%s bin=%s value=%s",
key.Namespace(), key.SetName(), key.Value(), binName, received)
} else {
log.Printf("Data mismatch: Expected %s. Received %s.", expected, received)
}
}
func writeBlobUsingUdf(client *as.Client) {
key, _ := as.NewKey(*shared.Namespace, *shared.Set, "udfkey6")
binName := "udfbin6"
// Create packed blob using standard java tools.
dos := bytes.Buffer{}
// dos.Write(9845)
dos.WriteString("Hello world.")
blob := dos.Bytes()
client.Execute(shared.WritePolicy, key, "record_example", "writeBin", as.NewValue(binName), as.NewValue(blob))
received, err := client.Execute(shared.WritePolicy, key, "record_example", "readBin", as.NewValue(binName))
shared.PanicOnError(err)
if bytes.Equal(blob, received.([]byte)) {
log.Printf("Blob data matched: namespace=%s set=%s key=%s bin=%v value=%v",
key.Namespace(), key.SetName(), key.Value(), binName, received)
} else {
log.Fatalf(
"Mismatch: expected=%v received=%v", blob, received)
}
}
// Execute put and get on a server configured as multi-bin. This is the server default.
func runMultiBinExample(client *as.Client) {
key, _ := as.NewKey(*shared.Namespace, *shared.Set, "putgetkey")
bin1 := as.NewBin("bin1", "value1")
bin2 := as.NewBin("bin2", "value2")
log.Printf("Put: namespace=%s set=%s key=%s bin1=%s value1=%s bin2=%s value2=%s",
key.Namespace(), key.SetName(), key.Value(), bin1.Name, bin1.Value, bin2.Name, bin2.Value)
client.PutBins(shared.WritePolicy, key, bin1, bin2)
log.Printf("Get: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value())
record, err := client.Get(shared.Policy, key)
shared.PanicOnError(err)
if record == nil {
panic(errors.New(fmt.Sprintf(
"Failed to get: namespace=%s set=%s key=%s", key.Namespace(), key.SetName(), key.Value())))
}
shared.ValidateBin(key, bin1, record)
shared.ValidateBin(key, bin2, record)
}
func writeListMapUsingUdf(client *as.Client) {
key, _ := as.NewKey(*shared.Namespace, *shared.Set, "udfkey5")
inner := []interface{}{"string2", int64(8)}
innerMap := map[interface{}]interface{}{"a": int64(1), int64(2): "b", "list": inner}
list := []interface{}{"string1", int64(4), inner, innerMap}
binName := "udfbin5"
client.Execute(shared.WritePolicy, key, "record_example", "writeBin", as.NewValue(binName), as.NewValue(list))
received, err := client.Execute(shared.WritePolicy, key, "record_example", "readBin", as.NewValue(binName))
shared.PanicOnError(err)
if testEq(received.([]interface{}), list) {
log.Printf("UDF data matched: namespace=%s set=%s key=%s bin=%s value=%s",
key.Namespace(), key.SetName(), key.Value(), binName, received)
} else {
log.Println("UDF data mismatch")
log.Println("Expected ", list)
log.Println("Received ", received)
}
}