// Send segments for the specified file to r.toGet. Send a *cfReader
// to rdrChan if the specified file is found (even if it's empty).
// Then, close rdrChan.
func (kc *KeepClient) queueSegmentsToGet(m manifest.Manifest, filename string, rdrChan chan *cfReader) {
	defer close(rdrChan)

	// q is a queue of FileSegments that we have received but
	// haven't yet been able to send to toGet.
	var q []*manifest.FileSegment
	var r *cfReader
	for seg := range m.FileSegmentIterByName(filename) {
		if r == nil {
			// We've just discovered that the requested
			// filename does appear in the manifest, so we
			// can return a real reader (not nil) from
			// CollectionFileReader().
			r = newCFReader(kc)
			rdrChan <- r
		}
		q = append(q, seg)
		r.totalSize += uint64(seg.Len)
		// Send toGet as many segments as we can until it
		// blocks.
	Q:
		for len(q) > 0 {
			select {
			case r.toGet <- q[0]:
				q = q[1:]
			default:
				break Q
			}
		}
	}
	if r == nil {
		// File not found.
		return
	}
	close(r.countDone)
	for _, seg := range q {
		r.toGet <- seg
	}
	close(r.toGet)
}
Exemple #2
0
// ProcessCollections read from api server
func ProcessCollections(arvLogger *logger.Logger,
	receivedCollections []SdkCollectionInfo,
	defaultReplicationLevel int,
	UUIDToCollection map[string]Collection) (latestModificationDate time.Time) {
	for _, sdkCollection := range receivedCollections {
		collection := Collection{UUID: StrCopy(sdkCollection.UUID),
			OwnerUUID:         StrCopy(sdkCollection.OwnerUUID),
			ReplicationLevel:  sdkCollection.Redundancy,
			BlockDigestToSize: make(map[blockdigest.BlockDigest]int)}

		if sdkCollection.ModifiedAt.IsZero() {
			loggerutil.FatalWithMessage(arvLogger,
				fmt.Sprintf(
					"Arvados SDK collection returned with unexpected zero "+
						"modification date. This probably means that either we failed to "+
						"parse the modification date or the API server has changed how "+
						"it returns modification dates: %+v",
					collection))
		}

		if sdkCollection.ModifiedAt.After(latestModificationDate) {
			latestModificationDate = sdkCollection.ModifiedAt
		}

		if collection.ReplicationLevel == 0 {
			collection.ReplicationLevel = defaultReplicationLevel
		}

		manifest := manifest.Manifest{sdkCollection.ManifestText}
		manifestSize := uint64(len(sdkCollection.ManifestText))

		if _, alreadySeen := UUIDToCollection[collection.UUID]; !alreadySeen {
			totalManifestSize += manifestSize
		}
		if manifestSize > maxManifestSize {
			maxManifestSize = manifestSize
		}

		blockChannel := manifest.BlockIterWithDuplicates()
		for block := range blockChannel {
			if storedSize, stored := collection.BlockDigestToSize[block.Digest]; stored && storedSize != block.Size {
				message := fmt.Sprintf(
					"Collection %s contains multiple sizes (%d and %d) for block %s",
					collection.UUID,
					storedSize,
					block.Size,
					block.Digest)
				loggerutil.FatalWithMessage(arvLogger, message)
			}
			collection.BlockDigestToSize[block.Digest] = block.Size
		}
		collection.TotalSize = 0
		for _, size := range collection.BlockDigestToSize {
			collection.TotalSize += size
		}
		UUIDToCollection[collection.UUID] = collection

		// Clear out all the manifest strings that we don't need anymore.
		// These hopefully form the bulk of our memory usage.
		manifest.Text = ""
		sdkCollection.ManifestText = ""
	}

	return
}