Exemplo n.º 1
0
// Given a set of changes, downloads the associated revisions.
func pullChanges(c *api.SyncGatewayClient, changes []*api.Change, wakeup time.Time) (int, interface{}) {
	docs := []api.BulkDocsEntry{}
	var newLastSeq interface{}
	for _, change := range changes {
		newLastSeq = change.Seq
		for _, changeItem := range change.Changes {
			bulk := api.BulkDocsEntry{ID: change.ID, Rev: changeItem.Rev}
			docs = append(docs, bulk)
		}
	}
	if len(docs) == 1 {
		if !c.GetSingleDoc(docs[0].ID, docs[0].Rev, wakeup) {
			docs = nil
			glExpvars.Add("total_doc_failed_to_pull", 1)
		} else {
			glExpvars.Add("total_doc_pulled", 1)
		}
	} else {
		if !c.GetBulkDocs(docs, wakeup) {
			docs = nil
			glExpvars.Add("total_doc_failed_to_pull", int64(len(docs)))
		} else {
			glExpvars.Add("total_doc_pulled", int64(len(docs)))
		}
	}
	return len(docs), newLastSeq
}
Exemplo n.º 2
0
func createSession(admin *api.SyncGatewayClient, user *workload.User, config workload.Config) {
	userMeta := api.UserAuth{Name: user.Name, Password: "******", AdminChannels: []string{user.Channel}}
	admin.AddUser(user.Name, userMeta)

	session := api.Session{Name: user.Name, TTL: 2592000} // 1 month
	user.Cookie = admin.CreateSession(user.Name, session)
}
Exemplo n.º 3
0
func runUser(user *workload.User, config workload.Config, wg *sync.WaitGroup) {
	c := api.SyncGatewayClient{}
	c.Init(config.Hostname, config.Database, config.Port, config.AdminPort, config.LogRequests)
	c.AddCookie(&user.Cookie)

	log.Printf("Starting new %s (%s)", user.Type, user.Name)
	if user.Type == "pusher" {
		go workload.RunNewPusher(user.Schedule, user.Name, &c, user.Channel, config.DocSize, config.SendAttachment, config.DocSizeDistribution, user.SeqId, config.SleepTimeMs, wg)
	} else {
		go workload.RunNewPuller(user.Schedule, &c, user.Channel, user.Name, config.FeedType, wg)
	}

}
Exemplo n.º 4
0
func RunNewPuller(schedule RunSchedule, c *api.SyncGatewayClient, channel, name, feedType string, wg *sync.WaitGroup) {
	defer wg.Done()

	glExpvars.Add("user_active", 1)
	var wakeupTime = time.Now()

	var lastSeq interface{}
	lastSeqFloat := c.GetLastSeq()
	if lastSeqFloat < 0 {
		Log("Puller, unable to establish last sequence number, exiting")
		return
	}
	lastSeq = lastSeqFloat
	if lastSeqFloat > MaxFirstFetch {
		//FIX: This generates a sequence ID using internal knowledge of the gateway's sequence format.
		lastSeq = lastSeqFloat - MaxFirstFetch // (for use with simple_sequences branch)
	}
	var changesFeed <-chan *api.Change
	var changesResponse *http.Response
	var cancelChangesFeed *bool

	var pendingChanges []*api.Change
	var fetchTimer <-chan time.Time

	var checkpointSeqId int64 = 0
	var checkpointTimer <-chan time.Time

	online := false
	scheduleIndex := 0
	start := time.Now()
	timer := time.NewTimer(schedule[scheduleIndex].start)
	Log("Puller %s first transition at %v", name, schedule[scheduleIndex].start)

outer:
	for {
		select {
		case <-timer.C:
			// timer went off, transition modes
			timeOffset := time.Since(start)
			if online {
				glExpvars.Add("user_awake", -1)
				online = false
				scheduleIndex++
				if scheduleIndex < len(schedule) {
					nextOnIn := schedule[scheduleIndex].start - timeOffset
					timer = time.NewTimer(nextOnIn)
					Log("Puller %s going offline, next on at %v", name, nextOnIn)
					if nextOnIn < 0 {
						log.Printf("WARNING: puller negative timer, exiting")
						return
					}
				} else {
					Log("Puller %s going offline, for good", name)
				}

				// transitioning off, cancel the changes feed, nil our changes feed channel
				*cancelChangesFeed = false
				if changesResponse != nil {
					changesResponse.Body.Close()
				}
				changesFeed = nil
				fetchTimer = nil
				checkpointTimer = nil
				pendingChanges = nil
			} else {
				glExpvars.Add("user_awake", 1)
				online = true
				if schedule[scheduleIndex].end != -1 {
					nextOffIn := schedule[scheduleIndex].end - timeOffset
					timer = time.NewTimer(nextOffIn)
					Log("Puller %s going online, next off at %v", name, nextOffIn)
					if nextOffIn < 0 {
						log.Printf("WARNING: puller negative timer, exiting")
						glExpvars.Add("user_awake", -1)
						return
					}
				} else {
					Log("Puller %s going online, for good", name)
				}

				// reset our wakeupTime to now
				wakeupTime = time.Now()
				Log("new wakeup time %v", wakeupTime)

				// transitioning on, start a changes feed
				changesFeed, cancelChangesFeed, changesResponse = c.GetChangesFeed(feedType, lastSeq)
				Log("** Puller %s watching changes using %s feed...", name, feedType)
			}
		case change, ok := <-changesFeed:
			// Received a change from the feed:
			if !ok {
				break outer
			}
			Log("Puller %s received %+v", name, *change)
			pendingChanges = append(pendingChanges, change)
			if fetchTimer == nil {
				fetchTimer = time.NewTimer(FetchDelay).C
			}
		case <-fetchTimer:
			// Time to get documents from the server:
			fetchTimer = nil
			var nDocs int
			nDocs, lastSeq = pullChanges(c, pendingChanges, wakeupTime)
			pendingChanges = nil
			Log("Puller %s done reading %d docs", name, nDocs)
			if nDocs > 0 && checkpointTimer == nil {
				checkpointTimer = time.NewTimer(CheckpointInterval).C
			}
		case <-checkpointTimer:
			// Time to save a checkpoint:
			checkpointTimer = nil
			checkpoint := api.Checkpoint{LastSequence: lastSeq}
			checkpointHash := fmt.Sprintf("%s-%s", name, Hash(strconv.FormatInt(checkpointSeqId, 10)))
			// save checkpoint asynchronously
			go c.SaveCheckpoint(checkpointHash, checkpoint)
			checkpointSeqId += 1
			Log("Puller %s saved remote checkpoint", name)
		}
	}

}
Exemplo n.º 5
0
func RunNewPusher(schedule RunSchedule, name string, c *api.SyncGatewayClient, channel string, size int, sendAttachment bool, dist DocSizeDistribution, seqId, sleepTime int, wg *sync.WaitGroup) {
	defer wg.Done()

	glExpvars.Add("user_active", 1)
	// if config contains DocSize, always generate this fixed document size
	if size != 0 {
		dist = DocSizeDistribution{
			&DocSizeDistributionElement{
				Prob:    100,
				MinSize: size,
				MaxSize: size,
			},
		}
	}

	docSizeGenerator, err := NewDocSizeGenerator(dist)
	if err != nil {
		Log("Error starting docuemnt pusher: %v", err)
		return
	}

	docIterator := DocIterator(seqId*DocsPerUser, (seqId+1)*DocsPerUser, docSizeGenerator, channel, sendAttachment)

	docsToSend := 0

	online := false
	scheduleIndex := 0
	start := time.Now()
	timer := time.NewTimer(schedule[scheduleIndex].start)
	var lastSend time.Time
	var pushTimer <-chan time.Time

	for {
		select {
		case <-timer.C:
			// timer went off, transition modes
			timeOffset := time.Since(start)
			if online {
				glExpvars.Add("user_awake", -1)
				online = false
				pushTimer = nil
				scheduleIndex++
				if scheduleIndex < len(schedule) {
					nextOnIn := schedule[scheduleIndex].start - timeOffset
					timer = time.NewTimer(nextOnIn)
					Log("Pusher %s going offline, next on at %v", name, nextOnIn)
					if nextOnIn < 0 {
						log.Printf("WARNING: pusher %s negative timer nextOnTime, exiting", name)
						return
					}
				}
			} else {
				glExpvars.Add("user_awake", 1)
				online = true
				pushTimer = time.NewTimer(0).C
				if schedule[scheduleIndex].end != -1 {
					nextOffIn := schedule[scheduleIndex].end - timeOffset
					timer = time.NewTimer(nextOffIn)
					Log("Pusher %s going online, next off at %v", name, nextOffIn)
					if nextOffIn < 0 {
						log.Printf("WARNING: pusher %s negative timer nextOffTime, exiting", name)
						glExpvars.Add("user_awake", -1)
						return
					}
				}
			}
		case <-pushTimer:

			if lastSend.IsZero() {
				docsToSend = 1
			} else {
				//log.Printf("time since last %v", time.Since(lastSend))
				//log.Printf("durration: %v", (time.Duration(sleepTime) * time.Millisecond))
				docsToSend = int(time.Since(lastSend) / (time.Duration(sleepTime) * time.Millisecond))
				//log.Printf("docs to send: %v", docsToSend)
			}
			if docsToSend > 0 {
				Log("Pusher online sending %d docs", docsToSend)
				// generage docs
				docs := make([]api.Doc, docsToSend)
				for i := 0; i < docsToSend; i++ {
					nextDoc := <-docIterator
					docs[i] = nextDoc
				}
				// send revs diff
				revsDiff := map[string][]string{}
				for _, doc := range docs {
					revsDiff[doc.Id] = []string{doc.Rev}
				}

				c.PostRevsDiff(revsDiff)
				// set the creation time in doc id
				nowString := "_" + strconv.Itoa(int(time.Now().UnixNano()/1e6)) // time since epoch in ms as string
				for i, doc := range docs {
					doc.Id = doc.Id + nowString
					docs[i] = doc
				}
				// send bulk docs
				bulkDocs := map[string]interface{}{
					"docs":      docs,
					"new_edits": false,
				}
				Log("Pusher #%d saved %d docs", seqId, docsToSend)
				if c.PostBulkDocs(bulkDocs) {
					glExpvars.Add("total_doc_pushed", int64(docsToSend))
				} else {
					glExpvars.Add("total_doc_failed_to_push", int64(docsToSend))
				}
				docsToSend = 0
				lastSend = time.Now()
				// reset the timer
				pushTimer = time.NewTimer(time.Duration(sleepTime) * time.Millisecond).C
			}
		}
	}

	glExpvars.Add("user_active", -1)

}
Exemplo n.º 6
0
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	// start up an http server, just to serve up expvars
	go http.ListenAndServe(":9876", nil)

	var config workload.Config
	workload.ReadConfig(&config)

	admin := api.SyncGatewayClient{}
	admin.Init(config.Hostname, config.Database, config.Port, config.AdminPort, config.LogRequests)
	if !admin.Valid() {
		log.Fatalf("unable to connect to sync_gateway, check the hostname and database")
	}

	pendingUsers := make(chan *workload.User)
	users := make([]*workload.User, config.NumPullers+config.NumPushers)

	// start a routine to place pending users into array
	go func() {
		for pendingUser := range pendingUsers {

			// users = append(users, pendingUser)
			users[pendingUser.SeqId-config.UserOffset] = pendingUser
		}
	}()

	rampUpDelay := config.RampUpIntervalMs / (config.NumPullers + config.NumPushers)
	// use a fixed number of workers to create the users/sessions
	userIterator := workload.UserIterator(config.NumPullers, config.NumPushers, config.UserOffset, config.ChannelActiveUsers, config.ChannelConcurrentUsers, config.MinUserOffTimeMs, config.MaxUserOffTimeMs, rampUpDelay, config.RunTimeMs)
	adminWg := sync.WaitGroup{}
	worker := func() {
		defer adminWg.Done()
		for user := range userIterator {
			createSession(&admin, user, config)
			pendingUsers <- user
		}
	}

	for i := 0; i < 16; i++ {
		adminWg.Add(1)
		go worker()
	}

	// wait for all the workers to finish
	adminWg.Wait()
	// close the pending users channel to free that routine
	close(pendingUsers)

	numChannels := (config.NumPullers + config.NumPushers) / config.ChannelActiveUsers
	channelRampUpDelayMs := time.Duration(config.RampUpIntervalMs/numChannels) * time.Millisecond
	wg := sync.WaitGroup{}
	channel := ""
	for _, user := range users {
		nextChannel := user.Channel
		if channel != nextChannel {
			if channel != "" {
				time.Sleep(channelRampUpDelayMs)
			}
			channel = nextChannel
		}
		wg := sync.WaitGroup{}
		go runUser(user, config, &wg)
		wg.Add(1)
	}

	if config.RunTimeMs > 0 {
		time.Sleep(time.Duration(config.RunTimeMs-config.RampUpIntervalMs) * time.Millisecond)
		log.Println("Shutting down clients")
	} else {
		wg.Wait()
	}
}