示例#1
0
// ProcessCollections read from api server
func ProcessCollections(arvLogger *logger.Logger,
	receivedCollections []SdkCollectionInfo,
	defaultReplicationLevel int,
	UUIDToCollection map[string]Collection) (latestModificationDate time.Time) {
	for _, sdkCollection := range receivedCollections {
		collection := Collection{UUID: StrCopy(sdkCollection.UUID),
			OwnerUUID:         StrCopy(sdkCollection.OwnerUUID),
			ReplicationLevel:  sdkCollection.Redundancy,
			BlockDigestToSize: make(map[blockdigest.BlockDigest]int)}

		if sdkCollection.ModifiedAt.IsZero() {
			loggerutil.FatalWithMessage(arvLogger,
				fmt.Sprintf(
					"Arvados SDK collection returned with unexpected zero "+
						"modification date. This probably means that either we failed to "+
						"parse the modification date or the API server has changed how "+
						"it returns modification dates: %+v",
					collection))
		}

		if sdkCollection.ModifiedAt.After(latestModificationDate) {
			latestModificationDate = sdkCollection.ModifiedAt
		}

		if collection.ReplicationLevel == 0 {
			collection.ReplicationLevel = defaultReplicationLevel
		}

		manifest := manifest.Manifest{sdkCollection.ManifestText}
		manifestSize := uint64(len(sdkCollection.ManifestText))

		if _, alreadySeen := UUIDToCollection[collection.UUID]; !alreadySeen {
			totalManifestSize += manifestSize
		}
		if manifestSize > maxManifestSize {
			maxManifestSize = manifestSize
		}

		blockChannel := manifest.BlockIterWithDuplicates()
		for block := range blockChannel {
			if storedSize, stored := collection.BlockDigestToSize[block.Digest]; stored && storedSize != block.Size {
				message := fmt.Sprintf(
					"Collection %s contains multiple sizes (%d and %d) for block %s",
					collection.UUID,
					storedSize,
					block.Size,
					block.Digest)
				loggerutil.FatalWithMessage(arvLogger, message)
			}
			collection.BlockDigestToSize[block.Digest] = block.Size
		}
		collection.TotalSize = 0
		for _, size := range collection.BlockDigestToSize {
			collection.TotalSize += size
		}
		UUIDToCollection[collection.UUID] = collection

		// Clear out all the manifest strings that we don't need anymore.
		// These hopefully form the bulk of our memory usage.
		manifest.Text = ""
		sdkCollection.ManifestText = ""
	}

	return
}