Ejemplo n.º 1
0
//go test -v github.com/trendrr/goshire/cheshire
func TestClient(t *testing.T) {
	//NOT actually a test ;)

	//assumes a running server on port 8009
	log.Println("HERE")
	client := NewHttp("localhost:8010")
	// client := NewJson("localhost", 8009)
	// client.Connect()
	defer client.Close()
	//warm it up
	res, err := client.ApiCallSync(cheshire.NewRequest("/ping", "GET"), 10*time.Second)
	if err != nil {
		t.Errorf("error %s")
	}
	log.Println(res)

	resChan := make(chan *cheshire.Response, 2)
	errorChan := make(chan error)
	total := 10000
	start := time.Now().Unix()
	go func() {

		for i := 0; i < total; i++ {
			client.ApiCall(cheshire.NewRequest("/ping", "GET"), resChan, errorChan)
		}
	}()
	count := 0

	log.Println("Starting select!")
	for {
		select {
		case <-resChan:
			count++
			if count%500 == 0 {
				log.Printf("Pinged 500 more, total time: %d", (time.Now().Unix() - start))
			}

		case <-errorChan:

			log.Println(err)
		}

		if count == total {
			log.Println("FINISHED!")
			break
		}
	}

	log.Printf("Pinged %d in %d", total, (time.Now().Unix() - start))
}
Ejemplo n.º 2
0
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	//assumes a running server on port 8009
	log.Println("HERE")

	// start the http server for profiling
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()

	// client := NewHttp("localhost:8010")
	client := c.NewBin("localhost", 8011)
	// client := c.NewJson("localhost", 8009)

	// with Poolsize = 10 and maxInflight = 500
	// 2013/05/16 12:56:28 Pinged 100000 in 38 seconds

	client.PoolSize = 10
	client.MaxInFlight = 500
	err := client.Connect()
	if err != nil {
		log.Println(err)
	}
	defer client.Close()
	//warm it up
	res, err := client.ApiCallSync(cheshire.NewRequest("/ping", "GET"), 10*time.Second)
	if err != nil {
		log.Printf("error %s")
	}
	log.Println(res)

	asyncTest(client, 1000000)

}
Ejemplo n.º 3
0
// Does a checkin with the requested client.  returns the
// router table revision of the connection.
func (this *Manager) Checkin(client client.Client) (int64, error) {
	response, err := client.ApiCallSync(cheshire.NewRequest("/chs/checkin", "GET"), 10*time.Second)
	if err != nil {
		return int64(0), err
	}
	revision := response.MustInt64("router_table_revision", int64(0))
	return revision, nil
}
Ejemplo n.º 4
0
func TestHttpClient(t *testing.T) {
	client := NewHttp("localhost:8010")
	res, err := client.ApiCallSync(cheshire.NewRequest("/ping", "GET"), 10*time.Second)
	log.Println(res)
	if err != nil {
		t.Errorf("error %s", err)
	}
}
Ejemplo n.º 5
0
func TestJsonClient(t *testing.T) {
	client := NewJson("localhost", 8009)
	client.Connect()
	defer client.Close()
	res, err := client.ApiCallSync(cheshire.NewRequest("/ping", "GET"), 10*time.Second)
	log.Println(res)
	if err != nil {
		t.Errorf("error %s", err)
	}
}
Ejemplo n.º 6
0
func (this *JsonClient) pingLoop() {
	pingTimer := time.Tick(25 * time.Second)
	for !this.Closed() {
		<-pingTimer
		for i := 0; i < this.PoolSize; i++ {
			//Do the ping
			_, err := this.doApiCallSync(cheshire.NewRequest(this.PingUri, "GET"), 10*time.Second)
			if err != nil {
				log.Printf("Error in ping %s", err)
			}
		}
	}
}
Ejemplo n.º 7
0
func asyncTest(client c.Client, total int) {

	resChan := make(chan *cheshire.Response, 20)
	errorChan := make(chan error, 200)

	start := time.Now().Unix()

	sent := total
	go func() {

		for i := 0; i < total; i++ {
			if i%1000 == 0 {
				log.Printf("Sending %d", i)
			}
			err := client.ApiCall(cheshire.NewRequest("/ping", "GET"), resChan, errorChan)
			if err != nil {
				log.Printf("apicall error %s", err)
				sent--
			}
			// if i % 2 == 0 {
			//     time.Sleep(1 * time.Millisecond)
			// }
		}
	}()
	count := 0

	log.Println("Starting select!")
	for {
		select {
		case res := <-resChan:
			count++
			if count%1000 == 0 {
				log.Printf("Recieved 1000 more, total %d, total time: %d", count, (time.Now().Unix() - start))
				log.Printf("RESULT %s", res)
			}

		case err := <-errorChan:
			count++
			log.Printf("ERROR FROM CHAN %s", err)
		}

		if count == sent {
			log.Println("FINISHED!")
			break
		}
	}

	log.Printf("Pinged %d in %d", total, (time.Now().Unix() - start))

}
Ejemplo n.º 8
0
func syncTest(client c.Client, total int) {

	start := time.Now().Unix()

	for i := 0; i < total; i++ {
		if i%1000 == 0 {
			log.Printf("Sending %d", i)
		}
		_, err := client.ApiCallSync(cheshire.NewRequest("/ping", "GET"), 2*time.Second)
		if err != nil {
			log.Printf("apicall error %s", err)
		}
	}

	log.Printf("Pinged %d in %d", total, (time.Now().Unix() - start))

}
Ejemplo n.º 9
0
// internal lock code shared by unlock and lock. (different endpoints)
func locking(endpoint string, services *Services, routerTable *shards.RouterTable, partition int) error {

	// Lock All partitions
	for _, e := range routerTable.Entries {
		response, err := client.HttpApiCallSync(
			fmt.Sprintf("%s:%d", e.Address, e.HttpPort),
			cheshire.NewRequest(endpoint, "GET"),
			5*time.Second)
		if err != nil {
			services.Logger.Printf("ERROR While contacting %s -- %s", e.Address, err)
			//TODO: retry?
			continue
		}
		if response.StatusCode() != 200 {
			//TODO: retry?
			services.Logger.Printf("ERROR While locking partition %s -- %s", e.Address, response.StatusMessage())
		}
	}
	return nil
}
Ejemplo n.º 10
0
// Finds the RouterTable from the given client
//
func RequestRouterTable(c client.Client) (*RouterTable, error) {
	response, err := c.ApiCallSync(cheshire.NewRequest(ROUTERTABLE_GET, "GET"), 10*time.Second)
	if err != nil {
		return nil, err
	}
	if response.StatusCode() != 200 {
		return nil, fmt.Errorf("Error from server %d %s", response.StatusCode(), response.StatusMessage())
	}

	mp, ok := response.GetDynMap("router_table")
	if !ok {
		return nil, fmt.Errorf("No router_table in response : %s", response)
	}

	table, err := ToRouterTable(mp)
	if err != nil {
		return nil, err
	}
	return table, nil
}
Ejemplo n.º 11
0
// Delete the requested partition from the entry.
// this does not lock, and does not update the router table
func DeletePartition(services *Services, entry *shards.RouterEntry, partition int) error {

	services.Logger.Printf("DELETING Partition %d From %s", partition, entry.Address)
	request := cheshire.NewRequest(shards.PARTITION_DELETE, "DELETE")
	request.Params().Put("partition", partition)

	services.Logger.Printf("NOW Deleting the partition from the origin server %s", entry.Address)

	response, err := client.HttpApiCallSync(
		fmt.Sprintf("%s:%d", entry.Address, entry.HttpPort),
		request,
		300*time.Second)

	if err != nil {
		return err
	}

	if response.StatusCode() != 200 {
		return fmt.Errorf("ERROR While deleting partition: %s", response.StatusMessage())
	}
	return nil
}
Ejemplo n.º 12
0
func (this *JsonClient) eventLoop() {
	//client event loop pings, and listens for client disconnects.
	c := time.Tick(5 * time.Second)
	defer log.Println("CLOSED!!!!!!!!")
	for !this.isClosed {
		select {
		case <-this.exitChan:
			log.Println("Exiting Client")
			//close all connections
			for _, conn := range this.conn {
				conn.Close()
			}
			this.isClosed = true
			break
		case <-c:
			//ping all the connections.

			for _, conn := range this.conn {

				_, err := this.doApiCallSync(conn, cheshire.NewRequest(this.PingUri, "GET"), 10*time.Second)
				if err != nil {
					log.Printf("COULDNT PING (%s) Attempting to reconnect", err)
					this.reconnect(conn)
				}
			}

		case conn := <-this.disconnectChan:
			log.Printf("DISCONNECTED %s:%p, attempting reconnect", this.Host, this.Port)
			//reconnect

			//attempt a reconnect immediately.
			// else will attempt to reconnect at next ping
			//
			this.reconnect(conn)
		}
	}

}
Ejemplo n.º 13
0
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	//assumes a running server on port 8009
	log.Println("HERE")
	// client := NewHttp("localhost:8010")
	client := c.NewJson("localhost", 8009)
	// with Poolsize = 10 and maxInflight = 500
	// 2013/05/16 12:56:28 Pinged 100000 in 38 seconds

	client.PoolSize = 10
	client.MaxInflight = 500
	client.Connect()
	defer client.Close()
	//warm it up
	res, err := client.ApiCallSync(cheshire.NewRequest("/ping", "GET"), 10*time.Second)
	if err != nil {
		log.Printf("error %s")
	}
	log.Println(res)

	// start the http server for profiling
	// go func() {
	//     log.Println(http.ListenAndServe("localhost:6060", nil))
	// }()

	resChan := make(chan *cheshire.Response, 20)
	errorChan := make(chan error, 200)
	total := 100000
	start := time.Now().Unix()
	go func() {

		for i := 0; i < total; i++ {
			if i%1000 == 0 {
				log.Printf("Sending %d", i)
			}
			err := client.ApiCall(cheshire.NewRequest("/ping", "GET"), resChan, errorChan)
			if err != nil {
				log.Printf("apicall error %s", err)
			}
		}
	}()
	count := 0

	log.Println("Starting select!")
	for {
		select {
		case <-resChan:
			count++
			if count%1000 == 0 {
				log.Printf("Recieved 1000 more, total %d, total time: %d", count, (time.Now().Unix() - start))
			}

		case <-errorChan:
			count++
			log.Printf("ERROR FROM CHAN %s", err)
		}

		if count == total {
			log.Println("FINISHED!")
			break
		}
	}

	log.Printf("Pinged %d in %d", total, (time.Now().Unix() - start))
}
Ejemplo n.º 14
0
// Checkin to an entry.  will update their router table if it is out of date.  will update our router table if out of date.
func EntryCheckin(routerTable *shards.RouterTable, entry *shards.RouterEntry) (*shards.RouterTable, bool, error) {
	// make sure our routertable is up to date.
	response, err := client.HttpApiCallSync(
		fmt.Sprintf("%s:%d", entry.Address, entry.HttpPort),
		cheshire.NewRequest(shards.CHECKIN, "GET"),
		5*time.Second)
	if err != nil {
		return routerTable, false, fmt.Errorf("ERROR While contacting %s -- %s", entry.Address, err)
	}

	entry.LastSeenAt = time.Now()

	rev := response.MustInt64("rt_revision", 0)
	if rev == routerTable.Revision {
		return routerTable, false, nil
	}

	if rev < routerTable.Revision {
		//updating server.
		//set the new routertable.
		req := cheshire.NewRequest(shards.ROUTERTABLE_SET, "POST")
		req.Params().Put("router_table", req.ToDynMap())

		response, err = client.HttpApiCallSync(
			fmt.Sprintf("%s:%d", entry.Address, entry.HttpPort),
			req,
			5*time.Second)
		if err != nil {
			return routerTable, false, fmt.Errorf("ERROR While contacting for router table update %s -- %s", entry.Address, err)
		}
		if response.StatusCode() != 200 {
			return routerTable, false, fmt.Errorf("Error trying to Set router table %s -- %s", entry.Address, response.StatusMessage())
		}
	} else {
		//updating local

		log.Printf("Found updated router table at: %s", entry.Address)
		//get the new routertable.
		response, err = client.HttpApiCallSync(
			fmt.Sprintf("%s:%d", entry.Address, entry.HttpPort),
			cheshire.NewRequest(shards.ROUTERTABLE_GET, "GET"),
			5*time.Second)
		if err != nil {
			return routerTable, false, fmt.Errorf("ERROR While contacting %s -- %s", entry.Address, err)
		}
		mp, ok := response.GetDynMap("router_table")
		if !ok {
			return routerTable, false, fmt.Errorf("ERROR from %s -- BAD ROUTER TABLE RESPONSE %s", entry.Address, response)
		}

		rt, err := shards.ToRouterTable(mp)
		if err != nil {
			return routerTable, false, fmt.Errorf("ERROR While parsing router table %s -- %s", entry.Address, err)
		}

		log.Printf("SUCCESSFULLY update router table to revision %d", rt.Revision)
		routerTable = rt
		return routerTable, true, nil

	}
	return routerTable, false, nil
}
Ejemplo n.º 15
0
// Copies the partition data from one server to another.
// This does not lock the partition, that should happen
func CopyData(services *Services, routerTable *shards.RouterTable, partition int, from, to *shards.RouterEntry) (int, error) {
	//Move the data!
	moved := 0

	//create a new json connection
	fromClient := client.NewJson(from.Address, from.JsonPort)
	err := fromClient.Connect()
	if err != nil {
		return moved, err
	}
	defer fromClient.Close()

	toClient := client.NewJson(to.Address, to.JsonPort)
	err = toClient.Connect()
	if err != nil {
		return moved, err
	}
	defer toClient.Close()

	request := cheshire.NewRequest(shards.DATA_PULL, "GET")
	request.Params().Put("partition", partition)

	responseChan := make(chan *cheshire.Response, 10)
	errorChan := make(chan error)

	fromClient.ApiCall(
		request,
		responseChan,
		errorChan,
	)

	toResponseChan := make(chan *cheshire.Response, 10)
	toErrorChan := make(chan error)

	for {
		select {
		case response := <-responseChan:
			//TODO: send the data to the toClient
			request := cheshire.NewRequest(shards.DATA_PUSH, "PUT")
			d, ok := response.GetDynMap("data")
			if !ok {
				services.Logger.Printf("ERROR: packet missing data :: %s ", request)
				continue
			}
			request.Params().Put("data", d)
			request.Params().Put("partition", partition)
			toClient.ApiCall(
				request,
				toResponseChan,
				toErrorChan,
			)

			// keep a log of items moved
			moved++
			if moved%100 == 0 {
				services.Logger.Printf("Moving partition %d... Moved %d objects so far", partition, moved)
			}

			//check for completion
			if response.TxnStatus() == "complete" {
				// FINISHED!
				services.Logger.Printf("SUCCESSFULLY Moved partition %d. Moved %d objects!", partition, moved)
				break
			}

		case err := <-errorChan:
			services.Logger.Printf("ERROR While Moving data from %s -- %s", from.Address, err)
			return moved, err

		case response := <-toResponseChan:
			if response.StatusCode() != 200 {
				services.Logger.Printf("ERROR While Moving data from %s -- %s.  \n Continuing...", from.Address, response.StatusMessage())
			}

			//do nothing,
		case err := <-toErrorChan:
			services.Logger.Printf("ERROR While Moving data to %s -- %s", to.Address, err)
			return moved, err
		}
	}

	// Now make sure we got responses for all the PUT data ops
	count := 0
	for toClient.CurrentInFlight() > 1 {
		services.Logger.Printf("NOW Waiting for success messages to complete")
		select {
		case response := <-toResponseChan:
			//do nothing,
			if response.StatusCode() != 200 {
				services.Logger.Printf("ERROR While Moving data to %s -- %s.  \n Continuing...", to.Address, response.StatusMessage())
			}

		case err := <-toErrorChan:
			services.Logger.Printf("ERROR While Moving data to %s -- %s", to.Address, err)
			return moved, err
		default:
			if count > 30 {
				services.Logger.Printf("GAH. Waited 30 seconds for completion.  there seems to be a problem,.")
				return moved, fmt.Errorf("GAH. Waited 30 seconds for completion.  there seems to be a problem.")
			}
			time.Sleep(1 * time.Second)
		}
		count++
	}

	return moved, err
}