// Send segments for the specified file to r.toGet. Send a *cfReader
// to rdrChan if the specified file is found (even if it's empty).
// Then, close rdrChan.
func (kc *KeepClient) queueSegmentsToGet(m manifest.Manifest, filename string, rdrChan chan *cfReader) {
defer close(rdrChan)
// q is a queue of FileSegments that we have received but
// haven't yet been able to send to toGet.
var q []*manifest.FileSegment
var r *cfReader
for seg := range m.FileSegmentIterByName(filename) {
if r == nil {
// We've just discovered that the requested
// filename does appear in the manifest, so we
// can return a real reader (not nil) from
// CollectionFileReader().
r = newCFReader(kc)
rdrChan <- r
}
q = append(q, seg)
r.totalSize += uint64(seg.Len)
// Send toGet as many segments as we can until it
// blocks.
Q:
for len(q) > 0 {
select {
case r.toGet <- q[0]:
q = q[1:]
default:
break Q
}
}
}
if r == nil {
// File not found.
return
}
close(r.countDone)
for _, seg := range q {
r.toGet <- seg
}
close(r.toGet)
}
// ProcessCollections read from api server
func ProcessCollections(arvLogger *logger.Logger,
receivedCollections []SdkCollectionInfo,
defaultReplicationLevel int,
UUIDToCollection map[string]Collection) (latestModificationDate time.Time) {
for _, sdkCollection := range receivedCollections {
collection := Collection{UUID: StrCopy(sdkCollection.UUID),
OwnerUUID: StrCopy(sdkCollection.OwnerUUID),
ReplicationLevel: sdkCollection.Redundancy,
BlockDigestToSize: make(map[blockdigest.BlockDigest]int)}
if sdkCollection.ModifiedAt.IsZero() {
loggerutil.FatalWithMessage(arvLogger,
fmt.Sprintf(
"Arvados SDK collection returned with unexpected zero "+
"modification date. This probably means that either we failed to "+
"parse the modification date or the API server has changed how "+
"it returns modification dates: %+v",
collection))
}
if sdkCollection.ModifiedAt.After(latestModificationDate) {
latestModificationDate = sdkCollection.ModifiedAt
}
if collection.ReplicationLevel == 0 {
collection.ReplicationLevel = defaultReplicationLevel
}
manifest := manifest.Manifest{sdkCollection.ManifestText}
manifestSize := uint64(len(sdkCollection.ManifestText))
if _, alreadySeen := UUIDToCollection[collection.UUID]; !alreadySeen {
totalManifestSize += manifestSize
}
if manifestSize > maxManifestSize {
maxManifestSize = manifestSize
}
blockChannel := manifest.BlockIterWithDuplicates()
for block := range blockChannel {
if storedSize, stored := collection.BlockDigestToSize[block.Digest]; stored && storedSize != block.Size {
message := fmt.Sprintf(
"Collection %s contains multiple sizes (%d and %d) for block %s",
collection.UUID,
storedSize,
block.Size,
block.Digest)
loggerutil.FatalWithMessage(arvLogger, message)
}
collection.BlockDigestToSize[block.Digest] = block.Size
}
collection.TotalSize = 0
for _, size := range collection.BlockDigestToSize {
collection.TotalSize += size
}
UUIDToCollection[collection.UUID] = collection
// Clear out all the manifest strings that we don't need anymore.
// These hopefully form the bulk of our memory usage.
manifest.Text = ""
sdkCollection.ManifestText = ""
}
return
}