func (this KeepClient) putReplicas(
hash string,
tr *streamer.AsyncStream,
expectedLength int64) (locator string, replicas int, err error) {
// Take the hash of locator and timestamp in order to identify this
// specific transaction in log statements.
requestId := fmt.Sprintf("%x", md5.Sum([]byte(locator+time.Now().String())))[0:8]
// Calculate the ordering for uploading to servers
sv := NewRootSorter(this.WritableLocalRoots(), hash).GetSortedRoots()
// The next server to try contacting
next_server := 0
// The number of active writers
active := 0
// Used to communicate status from the upload goroutines
upload_status := make(chan uploadStatus)
defer close(upload_status)
// Desired number of replicas
remaining_replicas := this.Want_replicas
for remaining_replicas > 0 {
for active < remaining_replicas {
// Start some upload requests
if next_server < len(sv) {
log.Printf("[%v] Begin upload %s to %s", requestId, hash, sv[next_server])
go this.uploadToKeepServer(sv[next_server], hash, tr.MakeStreamReader(), upload_status, expectedLength, requestId)
next_server += 1
active += 1
} else {
if active == 0 {
return locator, (this.Want_replicas - remaining_replicas), InsufficientReplicasError
} else {
break
}
}
}
log.Printf("[%v] Replicas remaining to write: %v active uploads: %v",
requestId, remaining_replicas, active)
// Now wait for something to happen.
status := <-upload_status
active -= 1
if status.statusCode == 200 {
// good news!
remaining_replicas -= status.replicas_stored
locator = status.response
}
}
return locator, this.Want_replicas, nil
}
func (this *KeepClient) putReplicas(
hash string,
tr *streamer.AsyncStream,
expectedLength int64) (locator string, replicas int, err error) {
// Generate an arbitrary ID to identify this specific
// transaction in debug logs.
requestID := rand.Int31()
// Calculate the ordering for uploading to servers
sv := NewRootSorter(this.WritableLocalRoots(), hash).GetSortedRoots()
// The next server to try contacting
next_server := 0
// The number of active writers
active := 0
// Used to communicate status from the upload goroutines
upload_status := make(chan uploadStatus)
defer func() {
// Wait for any abandoned uploads (e.g., we started
// two uploads and the first replied with replicas=2)
// to finish before closing the status channel.
go func() {
for active > 0 {
<-upload_status
}
close(upload_status)
}()
}()
// Desired number of replicas
remaining_replicas := this.Want_replicas
replicasPerThread := this.replicasPerService
if replicasPerThread < 1 {
// unlimited or unknown
replicasPerThread = remaining_replicas
}
retriesRemaining := 1 + this.Retries
var retryServers []string
for retriesRemaining > 0 {
retriesRemaining -= 1
next_server = 0
retryServers = []string{}
for remaining_replicas > 0 {
for active*replicasPerThread < remaining_replicas {
// Start some upload requests
if next_server < len(sv) {
log.Printf("[%08x] Begin upload %s to %s", requestID, hash, sv[next_server])
go this.uploadToKeepServer(sv[next_server], hash, tr.MakeStreamReader(), upload_status, expectedLength, requestID)
next_server += 1
active += 1
} else {
if active == 0 && retriesRemaining == 0 {
return locator, (this.Want_replicas - remaining_replicas), InsufficientReplicasError
} else {
break
}
}
}
log.Printf("[%08x] Replicas remaining to write: %v active uploads: %v",
requestID, remaining_replicas, active)
// Now wait for something to happen.
if active > 0 {
status := <-upload_status
active -= 1
if status.statusCode == 200 {
// good news!
remaining_replicas -= status.replicas_stored
locator = status.response
} else if status.statusCode == 0 || status.statusCode == 408 || status.statusCode == 429 ||
(status.statusCode >= 500 && status.statusCode != 503) {
// Timeout, too many requests, or other server side failure
// Do not retry when status code is 503, which means the keep server is full
retryServers = append(retryServers, status.url[0:strings.LastIndex(status.url, "/")])
}
} else {
break
}
}
sv = retryServers
}
return locator, this.Want_replicas, nil
}
