// A default client creation. JSON with poolsize = 10
func (this *Connections) Create(entry *RouterEntry) (client.Client, error) {
c := client.NewJson(entry.Address, entry.JsonPort)
c.PoolSize = 5
c.MaxInFlight = 250
err := c.Connect()
return c, err
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
//assumes a running server on port 8009
log.Println("HERE")
// client := NewHttp("localhost:8010")
client := c.NewJson("localhost", 8009)
// with Poolsize = 10 and maxInflight = 500
// 2013/05/16 12:56:28 Pinged 100000 in 38 seconds
client.PoolSize = 10
client.MaxInflight = 500
client.Connect()
defer client.Close()
//warm it up
res, err := client.ApiCallSync(cheshire.NewRequest("/ping", "GET"), 10*time.Second)
if err != nil {
log.Printf("error %s")
}
log.Println(res)
// start the http server for profiling
// go func() {
// log.Println(http.ListenAndServe("localhost:6060", nil))
// }()
resChan := make(chan *cheshire.Response, 20)
errorChan := make(chan error, 200)
total := 100000
start := time.Now().Unix()
go func() {
for i := 0; i < total; i++ {
if i%1000 == 0 {
log.Printf("Sending %d", i)
}
err := client.ApiCall(cheshire.NewRequest("/ping", "GET"), resChan, errorChan)
if err != nil {
log.Printf("apicall error %s", err)
}
}
}()
count := 0
log.Println("Starting select!")
for {
select {
case <-resChan:
count++
if count%1000 == 0 {
log.Printf("Recieved 1000 more, total %d, total time: %d", count, (time.Now().Unix() - start))
}
case <-errorChan:
count++
log.Printf("ERROR FROM CHAN %s", err)
}
if count == total {
log.Println("FINISHED!")
break
}
}
log.Printf("Pinged %d in %d", total, (time.Now().Unix() - start))
}
// Copies the partition data from one server to another.
// This does not lock the partition, that should happen
func CopyData(services *Services, routerTable *shards.RouterTable, partition int, from, to *shards.RouterEntry) (int, error) {
//Move the data!
moved := 0
//create a new json connection
fromClient := client.NewJson(from.Address, from.JsonPort)
err := fromClient.Connect()
if err != nil {
return moved, err
}
defer fromClient.Close()
toClient := client.NewJson(to.Address, to.JsonPort)
err = toClient.Connect()
if err != nil {
return moved, err
}
defer toClient.Close()
request := cheshire.NewRequest(shards.DATA_PULL, "GET")
request.Params().Put("partition", partition)
responseChan := make(chan *cheshire.Response, 10)
errorChan := make(chan error)
fromClient.ApiCall(
request,
responseChan,
errorChan,
)
toResponseChan := make(chan *cheshire.Response, 10)
toErrorChan := make(chan error)
for {
select {
case response := <-responseChan:
//TODO: send the data to the toClient
request := cheshire.NewRequest(shards.DATA_PUSH, "PUT")
d, ok := response.GetDynMap("data")
if !ok {
services.Logger.Printf("ERROR: packet missing data :: %s ", request)
continue
}
request.Params().Put("data", d)
request.Params().Put("partition", partition)
toClient.ApiCall(
request,
toResponseChan,
toErrorChan,
)
// keep a log of items moved
moved++
if moved%100 == 0 {
services.Logger.Printf("Moving partition %d... Moved %d objects so far", partition, moved)
}
//check for completion
if response.TxnStatus() == "complete" {
// FINISHED!
services.Logger.Printf("SUCCESSFULLY Moved partition %d. Moved %d objects!", partition, moved)
break
}
case err := <-errorChan:
services.Logger.Printf("ERROR While Moving data from %s -- %s", from.Address, err)
return moved, err
case response := <-toResponseChan:
if response.StatusCode() != 200 {
services.Logger.Printf("ERROR While Moving data from %s -- %s. \n Continuing...", from.Address, response.StatusMessage())
}
//do nothing,
case err := <-toErrorChan:
services.Logger.Printf("ERROR While Moving data to %s -- %s", to.Address, err)
return moved, err
}
}
// Now make sure we got responses for all the PUT data ops
count := 0
for toClient.CurrentInFlight() > 1 {
services.Logger.Printf("NOW Waiting for success messages to complete")
select {
case response := <-toResponseChan:
//do nothing,
if response.StatusCode() != 200 {
services.Logger.Printf("ERROR While Moving data to %s -- %s. \n Continuing...", to.Address, response.StatusMessage())
}
case err := <-toErrorChan:
services.Logger.Printf("ERROR While Moving data to %s -- %s", to.Address, err)
return moved, err
default:
if count > 30 {
services.Logger.Printf("GAH. Waited 30 seconds for completion. there seems to be a problem,.")
return moved, fmt.Errorf("GAH. Waited 30 seconds for completion. there seems to be a problem.")
}
time.Sleep(1 * time.Second)
}
count++
}
return moved, err
}
func (this *Manager) createConnection(entry *RouterEntry) client.Client {
c := client.NewJson(entry.Address, entry.JsonPort)
return c
}