Example #1
0
func worker(prot common.Prot, rw *bufio.ReadWriter, keys chan []byte, wg *sync.WaitGroup) {
	r := rand.New(rand.NewSource(time.Now().UnixNano()))

	for key := range keys {
		curOpCount := atomic.AddUint64(opCount, 1)
		if curOpCount%10000 == 0 {
			log.Println(curOpCount)
		}

		valLen := r.Intn(9*1024) + 1024
		value := common.RandData(nil, valLen, true)

		// continue on even if there's errors here
		if err := prot.Set(rw, key, value); err != nil {
			log.Println("Error during set:", err.Error())
			if err == io.EOF {
				log.Println("End of file. Aborting!")
				wg.Done()
				return
			}
		}

		opaque := r.Int()

		// pass the test if the data matches
		ret, err := prot.GetWithOpaque(rw, key, opaque)
		if err != nil {
			log.Println("Error getting data for key", string(key), ":", err.Error())
			if err == io.EOF {
				log.Println("End of file. Aborting!")
				wg.Done()
				return
			}
			continue
		}

		if !bytes.Equal(value, ret) {
			log.Println("Data returned from server does not match!",
				"\nData len sent:", len(value),
				"\nData len recv:", len(ret))
			//log.Println("sent:", string(value))
			//log.Println("got:", string(ret))
		}
	}

	wg.Done()
}
Example #2
0
func communicator(prot common.Prot, conn net.Conn, tasks <-chan *common.Task, metrics chan<- metric, comms *sync.WaitGroup) {
	rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))

	for item := range tasks {
		var err error
		start := timer.Now()

		switch item.Cmd {
		case common.Set:
			err = prot.Set(rw, item.Key, item.Value)
		case common.Add:
			err = prot.Add(rw, item.Key, item.Value)
		case common.Replace:
			err = prot.Replace(rw, item.Key, item.Value)
		case common.Append:
			err = prot.Append(rw, item.Key, item.Value)
		case common.Prepend:
			err = prot.Prepend(rw, item.Key, item.Value)
		}

		if err != nil {
			// don't print not stored
			if !isMiss(err) {
				fmt.Printf("Error performing operation %s on key %s: %s\n", item.Cmd, item.Key, err.Error())
			}
			// if the socket was closed, stop. Otherwise keep on hammering.
			if err == io.EOF {
				break
			}
		}

		m := metricPool.Get().(metric)
		m.d = timer.Since(start)
		m.op = item.Cmd
		m.miss = isMiss(err)
		metrics <- m

		taskPool.Put(item)
	}

	conn.Close()
	comms.Done()
}
Example #3
0
func main() {
	var prot common.Prot
	if f.Binary {
		var b binprot.BinProt
		prot = b
	} else {
		var t textprot.TextProt
		prot = t
	}

	wg := &sync.WaitGroup{}
	wg.Add(f.NumWorkers)

	for i := 0; i < f.NumWorkers; i++ {
		go func(prot common.Prot, wg *sync.WaitGroup) {
			conn, err := common.Connect("localhost", f.Port)
			if err != nil {
				panic("Couldn't connect")
			}

			r := rand.New(rand.NewSource(common.RandSeed()))
			rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))

			// 0 to 100k data
			for i := 0; i < 102400; i++ {
				key := common.RandData(r, f.KeyLength, false)
				value := common.RandData(nil, i, true)

				prot.Set(rw, key, value)
				prot.Get(rw, key)
			}

			fmt.Println("Done.")
			wg.Done()
		}(prot, wg)
	}

	wg.Wait()
}
Example #4
0
File: fill.go Project: Netflix/rend
func worker(numops int, prot common.Prot, wg *sync.WaitGroup) {
	conn, err := common.Connect(f.Host, f.Port)
	if err != nil {
		panic(err.Error())
	}

	rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
	r := rand.New(rand.NewSource(time.Now().UnixNano()))

	for i := 0; i < numops; i++ {
		curOpCount := atomic.AddUint64(opCount, 1)
		if curOpCount%10000 == 0 {
			log.Println(curOpCount)
		}

		// Generate key anew each time
		key := common.RandData(r, f.KeyLength, false)

		// Value size between 5k and 20k
		valLen := r.Intn(15*1024) + 5*1024
		value := common.RandData(nil, valLen, true)

		// continue on even if there's errors here
		if err := prot.Set(rw, key, value); err != nil {
			log.Println("Error during set:", err.Error())
			conn.Close()

			conn, err = common.Connect(f.Host, f.Port)
			if err != nil {
				panic(err.Error())
			}

			rw = bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
		}
	}

	wg.Done()
}
Example #5
0
func communicator(prot common.Prot, conn net.Conn, tasks <-chan *common.Task, metrics chan<- metric, comms *sync.WaitGroup) {
	r := rand.New(rand.NewSource(common.RandSeed()))
	rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))

	for item := range tasks {
		var err error
		start := time.Now()

		switch item.Cmd {
		case common.Set:
			err = prot.Set(rw, item.Key, item.Value)
		case common.Get:
			err = prot.Get(rw, item.Key)
		case common.Gat:
			err = prot.GAT(rw, item.Key)
		case common.Bget:
			err = prot.BatchGet(rw, batchkeys(r, item.Key))
		case common.Delete:
			err = prot.Delete(rw, item.Key)
		case common.Touch:
			err = prot.Touch(rw, item.Key)
		}

		if err != nil {
			if err != binprot.ErrKeyNotFound {
				fmt.Printf("Error performing operation %s on key %s: %s\n", item.Cmd, item.Key, err.Error())
			}
			// if the socket was closed, stop. Otherwise keep on hammering.
			if err == io.EOF {
				break
			}
		}

		metrics <- metric{
			d:  time.Since(start),
			op: item.Cmd,
		}
	}

	conn.Close()
	comms.Done()
}
Example #6
0
func communicator(prot common.Prot, conn net.Conn, tasks <-chan *common.Task, metrics chan<- metric, comms *sync.WaitGroup) {
	r := rand.New(rand.NewSource(common.RandSeed()))
	rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))

	for item := range tasks {
		var err error
		start := time.Now()

		switch item.Cmd {
		case common.Set:
			err = prot.Set(rw, item.Key, item.Value)
		case common.Add:
			err = prot.Add(rw, item.Key, item.Value)
		case common.Replace:
			err = prot.Replace(rw, item.Key, item.Value)
		case common.Get:
			_, err = prot.Get(rw, item.Key)
		case common.Gat:
			_, err = prot.GAT(rw, item.Key)
		case common.Bget:
			bk := batchkeys(r, item.Key)
			_, err = prot.BatchGet(rw, bk)
			bkpool.Put(bk)
		case common.Delete:
			err = prot.Delete(rw, item.Key)
		case common.Touch:
			err = prot.Touch(rw, item.Key)
		}

		if err != nil {
			// don't print get misses, adds not stored, and replaces not stored
			if !(err == binprot.ErrKeyNotFound || err == binprot.ErrKeyExists) {
				fmt.Printf("Error performing operation %s on key %s: %s\n", item.Cmd, item.Key, err.Error())
			}
			// if the socket was closed, stop. Otherwise keep on hammering.
			if err == io.EOF {
				break
			}
		}

		m := metricPool.Get().(metric)
		m.d = time.Since(start)
		m.op = item.Cmd
		metrics <- m

		taskPool.Put(item)
	}

	conn.Close()
	comms.Done()
}