コード例 #1
0
ファイル: support.go プロジェクト: WangZhenfei/arvados
func (this KeepClient) putReplicas(
	hash string,
	tr *streamer.AsyncStream,
	expectedLength int64) (locator string, replicas int, err error) {

	// Take the hash of locator and timestamp in order to identify this
	// specific transaction in log statements.
	requestId := fmt.Sprintf("%x", md5.Sum([]byte(locator+time.Now().String())))[0:8]

	// Calculate the ordering for uploading to servers
	sv := NewRootSorter(this.WritableLocalRoots(), hash).GetSortedRoots()

	// The next server to try contacting
	next_server := 0

	// The number of active writers
	active := 0

	// Used to communicate status from the upload goroutines
	upload_status := make(chan uploadStatus)
	defer close(upload_status)

	// Desired number of replicas
	remaining_replicas := this.Want_replicas

	for remaining_replicas > 0 {
		for active < remaining_replicas {
			// Start some upload requests
			if next_server < len(sv) {
				log.Printf("[%v] Begin upload %s to %s", requestId, hash, sv[next_server])
				go this.uploadToKeepServer(sv[next_server], hash, tr.MakeStreamReader(), upload_status, expectedLength, requestId)
				next_server += 1
				active += 1
			} else {
				if active == 0 {
					return locator, (this.Want_replicas - remaining_replicas), InsufficientReplicasError
				} else {
					break
				}
			}
		}
		log.Printf("[%v] Replicas remaining to write: %v active uploads: %v",
			requestId, remaining_replicas, active)

		// Now wait for something to happen.
		status := <-upload_status
		active -= 1

		if status.statusCode == 200 {
			// good news!
			remaining_replicas -= status.replicas_stored
			locator = status.response
		}
	}

	return locator, this.Want_replicas, nil
}
コード例 #2
0
ファイル: support.go プロジェクト: aflyhorse/arvados
func (this *KeepClient) putReplicas(
	hash string,
	tr *streamer.AsyncStream,
	expectedLength int64) (locator string, replicas int, err error) {

	// Generate an arbitrary ID to identify this specific
	// transaction in debug logs.
	requestID := rand.Int31()

	// Calculate the ordering for uploading to servers
	sv := NewRootSorter(this.WritableLocalRoots(), hash).GetSortedRoots()

	// The next server to try contacting
	next_server := 0

	// The number of active writers
	active := 0

	// Used to communicate status from the upload goroutines
	upload_status := make(chan uploadStatus)
	defer func() {
		// Wait for any abandoned uploads (e.g., we started
		// two uploads and the first replied with replicas=2)
		// to finish before closing the status channel.
		go func() {
			for active > 0 {
				<-upload_status
			}
			close(upload_status)
		}()
	}()

	// Desired number of replicas
	remaining_replicas := this.Want_replicas

	replicasPerThread := this.replicasPerService
	if replicasPerThread < 1 {
		// unlimited or unknown
		replicasPerThread = remaining_replicas
	}

	retriesRemaining := 1 + this.Retries
	var retryServers []string

	for retriesRemaining > 0 {
		retriesRemaining -= 1
		next_server = 0
		retryServers = []string{}
		for remaining_replicas > 0 {
			for active*replicasPerThread < remaining_replicas {
				// Start some upload requests
				if next_server < len(sv) {
					log.Printf("[%08x] Begin upload %s to %s", requestID, hash, sv[next_server])
					go this.uploadToKeepServer(sv[next_server], hash, tr.MakeStreamReader(), upload_status, expectedLength, requestID)
					next_server += 1
					active += 1
				} else {
					if active == 0 && retriesRemaining == 0 {
						return locator, (this.Want_replicas - remaining_replicas), InsufficientReplicasError
					} else {
						break
					}
				}
			}
			log.Printf("[%08x] Replicas remaining to write: %v active uploads: %v",
				requestID, remaining_replicas, active)

			// Now wait for something to happen.
			if active > 0 {
				status := <-upload_status
				active -= 1

				if status.statusCode == 200 {
					// good news!
					remaining_replicas -= status.replicas_stored
					locator = status.response
				} else if status.statusCode == 0 || status.statusCode == 408 || status.statusCode == 429 ||
					(status.statusCode >= 500 && status.statusCode != 503) {
					// Timeout, too many requests, or other server side failure
					// Do not retry when status code is 503, which means the keep server is full
					retryServers = append(retryServers, status.url[0:strings.LastIndex(status.url, "/")])
				}
			} else {
				break
			}
		}

		sv = retryServers
	}

	return locator, this.Want_replicas, nil
}