Example #1
0
// Returns true if no update needs to be performed.
func (sub *Sub) sendUpdate(srpcClient *srpc.Client) (bool, subStatus) {
	logger := sub.herd.logger
	if !sub.pendingSafetyClear {
		// Perform a cheap safety check.
		if sub.requiredImage.Filter != nil &&
			len(sub.fileSystem.InodeTable)>>1 >
				len(sub.requiredImage.FileSystem.InodeTable) {
			return false, statusUnsafeUpdate
		}
	}
	var request subproto.UpdateRequest
	var reply subproto.UpdateResponse
	if idle, missing := sub.buildUpdateRequest(&request); missing {
		return false, statusMissingComputedFile
	} else if idle {
		return true, statusSynced
	}
	if sub.mdb.DisableUpdates || sub.herd.updatesDisabledReason != "" {
		return false, statusUpdatesDisabled
	}
	sub.status = statusSendingUpdate
	sub.lastUpdateTime = time.Now()
	if err := client.CallUpdate(srpcClient, request, &reply); err != nil {
		srpcClient.Close()
		logger.Printf("Error calling %s:Subd.Update(): %s\n", sub, err)
		if err == srpc.ErrorAccessToMethodDenied {
			return false, statusUpdateDenied
		}
		return false, statusFailedToUpdate
	}
	sub.pendingSafetyClear = false
	return false, statusUpdating
}
Example #2
0
func pollSubcommand(getSubClient getSubClientFunc, args []string) {
	var err error
	var srpcClient *srpc.Client
	for iter := 0; *numPolls < 0 || iter < *numPolls; iter++ {
		if iter > 0 {
			time.Sleep(time.Duration(*interval) * time.Second)
		}
		if srpcClient == nil {
			srpcClient = getSubClient()
		}
		var request sub.PollRequest
		var reply sub.PollResponse
		request.ShortPollOnly = *shortPoll
		pollStartTime := time.Now()
		err = client.CallPoll(srpcClient, request, &reply)
		fmt.Printf("Poll duration: %s\n", time.Since(pollStartTime))
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error calling: %s\n", err)
			os.Exit(1)
		}
		if *newConnection {
			srpcClient.Close()
			srpcClient = nil
		}
		fs := reply.FileSystem
		if fs == nil {
			if !*shortPoll {
				fmt.Println("No FileSystem pointer")
			}
		} else {
			fs.RebuildInodePointers()
			if *debug {
				fs.List(os.Stdout)
			} else {
				fmt.Println(fs)
			}
			fmt.Printf("Num objects: %d\n", len(reply.ObjectCache))
			if *file != "" {
				f, err := os.Create(*file)
				if err != nil {
					fmt.Fprintf(os.Stderr, "Error creating: %s: %s\n",
						*file, err)
					os.Exit(1)
				}
				encoder := gob.NewEncoder(f)
				encoder.Encode(fs)
				f.Close()
			}
		}
		if reply.LastSuccessfulImageName != "" {
			fmt.Printf("Last successful image: \"%s\"\n",
				reply.LastSuccessfulImageName)
		}
	}
	time.Sleep(time.Duration(*wait) * time.Second)
}
Example #3
0
func pollSubcommand(srpcClient *srpc.Client, args []string) {
	var err error
	clientName := fmt.Sprintf("%s:%d", *subHostname, *subPortNum)
	for iter := 0; *numPolls < 0 || iter < *numPolls; iter++ {
		if iter > 0 {
			time.Sleep(time.Duration(*interval) * time.Second)
		}
		if srpcClient == nil {
			srpcClient, err = srpc.DialHTTP("tcp", clientName)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error dialing\t%s\n", err)
				os.Exit(1)
			}
		}
		var request sub.PollRequest
		var reply sub.PollResponse
		pollStartTime := time.Now()
		err = client.CallPoll(srpcClient, request, &reply)
		fmt.Printf("Poll duration: %s\n", time.Since(pollStartTime))
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error calling\t%s\n", err)
			os.Exit(1)
		}
		if *newConnection {
			srpcClient.Close()
			srpcClient = nil
		}
		fs := reply.FileSystem
		if fs == nil {
			fmt.Println("No FileSystem pointer")
		} else {
			fs.RebuildInodePointers()
			if *debug {
				fs.List(os.Stdout)
			} else {
				fmt.Println(fs)
			}
			fmt.Printf("Num objects: %d\n", len(reply.ObjectCache))
			if *file != "" {
				f, err := os.Create(*file)
				if err != nil {
					fmt.Fprintf(os.Stderr, "Error creating: %s\t%s\n",
						*file, err)
					os.Exit(1)
				}
				encoder := gob.NewEncoder(f)
				encoder.Encode(fs)
				f.Close()
			}
		}
	}
	time.Sleep(time.Duration(*wait) * time.Second)
}
Example #4
0
func (sub *Sub) updateConfiguration(srpcClient *srpc.Client,
	pollReply subproto.PollResponse) {
	if pollReply.ScanCount < 1 {
		return
	}
	sub.herd.RLock()
	newConf := sub.herd.configurationForSubs
	sub.herd.RUnlock()
	if compareConfigs(pollReply.CurrentConfiguration, newConf) {
		return
	}
	if err := client.SetConfiguration(srpcClient, newConf); err != nil {
		srpcClient.Close()
		logger := sub.herd.logger
		logger.Printf("Error setting configuration for sub: %s: %s\n",
			sub, err)
		return
	}
}
Example #5
0
func (sub *Sub) cleanup(srpcClient *srpc.Client) {
	logger := sub.herd.logger
	unusedObjects := make(map[hash.Hash]bool)
	for _, hash := range sub.objectCache {
		unusedObjects[hash] = false // Potential cleanup candidate.
	}
	for _, inode := range sub.fileSystem.InodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				if _, ok := unusedObjects[inode.Hash]; ok {
					unusedObjects[inode.Hash] = true // Must clean this one up.
				}
			}
		}
	}
	image := sub.plannedImage
	if image != nil {
		for _, inode := range image.FileSystem.InodeTable {
			if inode, ok := inode.(*filesystem.RegularInode); ok {
				if inode.Size > 0 {
					if clean, ok := unusedObjects[inode.Hash]; !clean && ok {
						delete(unusedObjects, inode.Hash)
					}
				}
			}
		}
	}
	if len(unusedObjects) < 1 {
		return
	}
	hashes := make([]hash.Hash, 0, len(unusedObjects))
	for hash := range unusedObjects {
		hashes = append(hashes, hash)
	}
	if err := client.Cleanup(srpcClient, hashes); err != nil {
		srpcClient.Close()
		logger.Printf("Error calling %s:Subd.Cleanup(): %s\n", sub, err)
	}
}
Example #6
0
func (m *Manager) loadImage(imageClient *srpc.Client, name string) (
	*srpc.Client, *image.Image, error) {
	if imageClient == nil {
		var err error
		imageClient, err = srpc.DialHTTP("tcp", m.imageServerAddress, 0)
		if err != nil {
			if !m.loggedDialFailure {
				m.logger.Printf("Error dialing: %s: %s\n",
					m.imageServerAddress, err)
				m.loggedDialFailure = true
			}
			return nil, nil, err
		}
	}
	img, err := client.GetImage(imageClient, name)
	if err != nil {
		m.logger.Printf("Error calling: %s\n", err)
		imageClient.Close()
		return nil, nil, err
	}
	if img == nil || m.scheduleExpiration(img, name) {
		return imageClient, nil, nil
	}
	if err := img.FileSystem.RebuildInodePointers(); err != nil {
		m.logger.Printf("Error building inode pointers for image: %s %s",
			name, err)
		return imageClient, nil, err
	}
	// Build cache data now to avoid potential concurrent builds later.
	img.FileSystem.InodeToFilenamesTable()
	img.FileSystem.FilenameToInodeTable()
	img.FileSystem.HashToInodesTable()
	img.FileSystem.ComputeTotalDataBytes()
	img.FileSystem.BuildEntryMap()
	m.logger.Printf("Got image: %s\n", name)
	return imageClient, img, nil
}
Example #7
0
// Returns true if all required objects are available.
func (sub *Sub) fetchMissingObjects(srpcClient *srpc.Client, image *image.Image,
	pushComputedFiles bool) (
	bool, subStatus) {
	if image == nil {
		return false, statusImageNotReady
	}
	logger := sub.herd.logger
	subObj := lib.Sub{
		Hostname:       sub.mdb.Hostname,
		Client:         srpcClient,
		FileSystem:     sub.fileSystem,
		ComputedInodes: sub.computedInodes,
		ObjectCache:    sub.objectCache,
		ObjectGetter:   sub.herd.objectServer}
	objectsToFetch, objectsToPush := lib.BuildMissingLists(subObj, image,
		pushComputedFiles, false, logger)
	if objectsToPush == nil {
		return false, statusMissingComputedFile
	}
	var returnAvailable bool = true
	var returnStatus subStatus = statusSynced
	if len(objectsToFetch) > 0 {
		logger.Printf("Calling %s:Subd.Fetch() for: %d objects\n",
			sub, len(objectsToFetch))
		err := client.Fetch(srpcClient, sub.herd.imageManager.String(),
			objectsToFetch)
		if err != nil {
			srpcClient.Close()
			logger.Printf("Error calling %s:Subd.Fetch(): %s\n", sub, err)
			if err == srpc.ErrorAccessToMethodDenied {
				return false, statusFetchDenied
			}
			return false, statusFailedToFetch
		}
		returnAvailable = false
		returnStatus = statusFetching
	}
	if len(objectsToPush) > 0 {
		sub.herd.pushSemaphore <- struct{}{}
		defer func() { <-sub.herd.pushSemaphore }()
		sub.status = statusPushing
		err := lib.PushObjects(subObj, objectsToPush, logger)
		if err != nil {
			if err == srpc.ErrorAccessToMethodDenied {
				return false, statusPushDenied
			}
			if err == lib.ErrorFailedToGetObject {
				return false, statusFailedToGetObject
			}
			return false, statusFailedToPush
		}
		if returnAvailable {
			// Update local copy of objectcache, since there will not be
			// another Poll() before the update computation.
			for hashVal := range objectsToPush {
				sub.objectCache = append(sub.objectCache, hashVal)
			}
		}
	}
	return returnAvailable, returnStatus
}
Example #8
0
func (sub *Sub) poll(srpcClient *srpc.Client, previousStatus subStatus) {
	// If the planned image has just become available, force a full poll.
	if previousStatus == statusSynced &&
		!sub.havePlannedImage &&
		sub.plannedImage != nil {
		sub.havePlannedImage = true
		sub.generationCount = 0 // Force a full poll.
	}
	// If the computed files have changed since the last sync, force a full poll
	if previousStatus == statusSynced &&
		sub.computedFilesChangeTime.After(sub.lastSyncTime) {
		sub.generationCount = 0 // Force a full poll.
	}
	// If the last update was disabled and updates are enabled now, force a full
	// poll.
	if previousStatus == statusUpdatesDisabled &&
		sub.herd.updatesDisabledReason == "" && !sub.mdb.DisableUpdates {
		sub.generationCount = 0 // Force a full poll.
	}
	// If the last update was disabled due to a safety check and there is a
	// pending SafetyClear, force a full poll to re-compute the update.
	if previousStatus == statusUnsafeUpdate && sub.pendingSafetyClear {
		sub.generationCount = 0 // Force a full poll.
	}
	var request subproto.PollRequest
	request.HaveGeneration = sub.generationCount
	var reply subproto.PollResponse
	haveImage := false
	if sub.requiredImage == nil {
		request.ShortPollOnly = true
	} else {
		haveImage = true
	}
	logger := sub.herd.logger
	sub.lastPollStartTime = time.Now()
	if err := client.CallPoll(srpcClient, request, &reply); err != nil {
		srpcClient.Close()
		if err == io.EOF {
			return
		}
		sub.pollTime = time.Time{}
		if err == srpc.ErrorAccessToMethodDenied {
			sub.status = statusPollDenied
		} else {
			sub.status = statusFailedToPoll
		}
		logger.Printf("Error calling %s.Poll(): %s\n", sub, err)
		return
	}
	sub.lastPollSucceededTime = time.Now()
	sub.lastSuccessfulImageName = reply.LastSuccessfulImageName
	if reply.GenerationCount == 0 {
		sub.reclaim()
		sub.generationCount = 0
	}
	sub.lastScanDuration = reply.DurationOfLastScan
	if fs := reply.FileSystem; fs == nil {
		sub.lastPollWasFull = false
		sub.lastShortPollDuration =
			sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
		shortPollDistribution.Add(sub.lastShortPollDuration)
		if !sub.startTime.Equal(reply.StartTime) {
			sub.generationCount = 0 // Sub has restarted: force a full poll.
		}
	} else {
		sub.lastPollWasFull = true
		if err := fs.RebuildInodePointers(); err != nil {
			sub.status = statusFailedToPoll
			logger.Printf("Error building pointers for: %s %s\n", sub, err)
			return
		}
		fs.BuildEntryMap()
		sub.fileSystem = fs
		sub.objectCache = reply.ObjectCache
		sub.generationCount = reply.GenerationCount
		sub.lastFullPollDuration =
			sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
		fullPollDistribution.Add(sub.lastFullPollDuration)
	}
	sub.startTime = reply.StartTime
	sub.pollTime = reply.PollTime
	sub.updateConfiguration(srpcClient, reply)
	if reply.FetchInProgress {
		sub.status = statusFetching
		return
	}
	if reply.UpdateInProgress {
		sub.status = statusUpdating
		return
	}
	if reply.GenerationCount < 1 {
		sub.status = statusSubNotReady
		return
	}
	if previousStatus == statusFetching && reply.LastFetchError != "" {
		logger.Printf("Fetch failure for: %s: %s\n", sub, reply.LastFetchError)
		sub.status = statusFailedToFetch
		if sub.fileSystem == nil {
			sub.generationCount = 0 // Force a full poll next cycle.
			return
		}
	}
	if previousStatus == statusUpdating {
		// Transition from updating to update ended (may be partial/failed).
		if reply.LastUpdateError != "" {
			logger.Printf("Update failure for: %s: %s\n",
				sub, reply.LastUpdateError)
			sub.status = statusFailedToUpdate
		} else {
			sub.status = statusWaitingForNextFullPoll
		}
		sub.scanCountAtLastUpdateEnd = reply.ScanCount
		sub.reclaim()
		return
	}
	if sub.checkCancel() {
		// Configuration change pending: skip further processing. Do not reclaim
		// file-system and objectcache data: it will speed up the next Poll.
		return
	}
	if !haveImage {
		if sub.requiredImageName == "" {
			sub.status = statusImageUndefined
		} else {
			sub.status = statusImageNotReady
		}
		return
	}
	if previousStatus == statusFailedToUpdate ||
		previousStatus == statusWaitingForNextFullPoll {
		if sub.scanCountAtLastUpdateEnd == reply.ScanCount {
			// Need to wait until sub has performed a new scan.
			if sub.fileSystem != nil {
				sub.reclaim()
			}
			sub.status = previousStatus
			return
		}
		if sub.fileSystem == nil {
			// Force a full poll next cycle so that we can see the state of the
			// sub.
			sub.generationCount = 0
			sub.status = previousStatus
			return
		}
	}
	if sub.fileSystem == nil {
		sub.status = previousStatus
		return
	}
	if idle, status := sub.fetchMissingObjects(srpcClient, sub.requiredImage,
		true); !idle {
		sub.status = status
		sub.reclaim()
		return
	}
	sub.status = statusComputingUpdate
	if idle, status := sub.sendUpdate(srpcClient); !idle {
		sub.status = status
		sub.reclaim()
		return
	}
	if idle, status := sub.fetchMissingObjects(srpcClient, sub.plannedImage,
		false); !idle {
		if status != statusImageNotReady {
			sub.status = status
			sub.reclaim()
			return
		}
	}
	if previousStatus == statusWaitingForNextFullPoll &&
		!sub.lastUpdateTime.IsZero() {
		sub.lastSyncTime = time.Now()
	}
	sub.status = statusSynced
	sub.cleanup(srpcClient)
	sub.reclaim()
}