Example #1
0
// A default client creation.  JSON with poolsize = 10
func (this *Connections) Create(entry *RouterEntry) (client.Client, error) {
	c := client.NewJson(entry.Address, entry.JsonPort)
	c.PoolSize = 5
	c.MaxInFlight = 250
	err := c.Connect()
	return c, err
}
Example #2
0
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	//assumes a running server on port 8009
	log.Println("HERE")
	// client := NewHttp("localhost:8010")
	client := c.NewJson("localhost", 8009)
	// with Poolsize = 10 and maxInflight = 500
	// 2013/05/16 12:56:28 Pinged 100000 in 38 seconds

	client.PoolSize = 10
	client.MaxInflight = 500
	client.Connect()
	defer client.Close()
	//warm it up
	res, err := client.ApiCallSync(cheshire.NewRequest("/ping", "GET"), 10*time.Second)
	if err != nil {
		log.Printf("error %s")
	}
	log.Println(res)

	// start the http server for profiling
	// go func() {
	//     log.Println(http.ListenAndServe("localhost:6060", nil))
	// }()

	resChan := make(chan *cheshire.Response, 20)
	errorChan := make(chan error, 200)
	total := 100000
	start := time.Now().Unix()
	go func() {

		for i := 0; i < total; i++ {
			if i%1000 == 0 {
				log.Printf("Sending %d", i)
			}
			err := client.ApiCall(cheshire.NewRequest("/ping", "GET"), resChan, errorChan)
			if err != nil {
				log.Printf("apicall error %s", err)
			}
		}
	}()
	count := 0

	log.Println("Starting select!")
	for {
		select {
		case <-resChan:
			count++
			if count%1000 == 0 {
				log.Printf("Recieved 1000 more, total %d, total time: %d", count, (time.Now().Unix() - start))
			}

		case <-errorChan:
			count++
			log.Printf("ERROR FROM CHAN %s", err)
		}

		if count == total {
			log.Println("FINISHED!")
			break
		}
	}

	log.Printf("Pinged %d in %d", total, (time.Now().Unix() - start))
}
Example #3
0
// Copies the partition data from one server to another.
// This does not lock the partition, that should happen
func CopyData(services *Services, routerTable *shards.RouterTable, partition int, from, to *shards.RouterEntry) (int, error) {
	//Move the data!
	moved := 0

	//create a new json connection
	fromClient := client.NewJson(from.Address, from.JsonPort)
	err := fromClient.Connect()
	if err != nil {
		return moved, err
	}
	defer fromClient.Close()

	toClient := client.NewJson(to.Address, to.JsonPort)
	err = toClient.Connect()
	if err != nil {
		return moved, err
	}
	defer toClient.Close()

	request := cheshire.NewRequest(shards.DATA_PULL, "GET")
	request.Params().Put("partition", partition)

	responseChan := make(chan *cheshire.Response, 10)
	errorChan := make(chan error)

	fromClient.ApiCall(
		request,
		responseChan,
		errorChan,
	)

	toResponseChan := make(chan *cheshire.Response, 10)
	toErrorChan := make(chan error)

	for {
		select {
		case response := <-responseChan:
			//TODO: send the data to the toClient
			request := cheshire.NewRequest(shards.DATA_PUSH, "PUT")
			d, ok := response.GetDynMap("data")
			if !ok {
				services.Logger.Printf("ERROR: packet missing data :: %s ", request)
				continue
			}
			request.Params().Put("data", d)
			request.Params().Put("partition", partition)
			toClient.ApiCall(
				request,
				toResponseChan,
				toErrorChan,
			)

			// keep a log of items moved
			moved++
			if moved%100 == 0 {
				services.Logger.Printf("Moving partition %d... Moved %d objects so far", partition, moved)
			}

			//check for completion
			if response.TxnStatus() == "complete" {
				// FINISHED!
				services.Logger.Printf("SUCCESSFULLY Moved partition %d. Moved %d objects!", partition, moved)
				break
			}

		case err := <-errorChan:
			services.Logger.Printf("ERROR While Moving data from %s -- %s", from.Address, err)
			return moved, err

		case response := <-toResponseChan:
			if response.StatusCode() != 200 {
				services.Logger.Printf("ERROR While Moving data from %s -- %s.  \n Continuing...", from.Address, response.StatusMessage())
			}

			//do nothing,
		case err := <-toErrorChan:
			services.Logger.Printf("ERROR While Moving data to %s -- %s", to.Address, err)
			return moved, err
		}
	}

	// Now make sure we got responses for all the PUT data ops
	count := 0
	for toClient.CurrentInFlight() > 1 {
		services.Logger.Printf("NOW Waiting for success messages to complete")
		select {
		case response := <-toResponseChan:
			//do nothing,
			if response.StatusCode() != 200 {
				services.Logger.Printf("ERROR While Moving data to %s -- %s.  \n Continuing...", to.Address, response.StatusMessage())
			}

		case err := <-toErrorChan:
			services.Logger.Printf("ERROR While Moving data to %s -- %s", to.Address, err)
			return moved, err
		default:
			if count > 30 {
				services.Logger.Printf("GAH. Waited 30 seconds for completion.  there seems to be a problem,.")
				return moved, fmt.Errorf("GAH. Waited 30 seconds for completion.  there seems to be a problem.")
			}
			time.Sleep(1 * time.Second)
		}
		count++
	}

	return moved, err
}
Example #4
0
func (this *Manager) createConnection(entry *RouterEntry) client.Client {
	c := client.NewJson(entry.Address, entry.JsonPort)
	return c
}