// Returns true if no update needs to be performed.
func (sub *Sub) sendUpdate(srpcClient *srpc.Client) (bool, subStatus) {
logger := sub.herd.logger
if !sub.pendingSafetyClear {
// Perform a cheap safety check.
if sub.requiredImage.Filter != nil &&
len(sub.fileSystem.InodeTable)>>1 >
len(sub.requiredImage.FileSystem.InodeTable) {
return false, statusUnsafeUpdate
}
}
var request subproto.UpdateRequest
var reply subproto.UpdateResponse
if idle, missing := sub.buildUpdateRequest(&request); missing {
return false, statusMissingComputedFile
} else if idle {
return true, statusSynced
}
if sub.mdb.DisableUpdates || sub.herd.updatesDisabledReason != "" {
return false, statusUpdatesDisabled
}
sub.status = statusSendingUpdate
sub.lastUpdateTime = time.Now()
if err := client.CallUpdate(srpcClient, request, &reply); err != nil {
srpcClient.Close()
logger.Printf("Error calling %s:Subd.Update(): %s\n", sub, err)
if err == srpc.ErrorAccessToMethodDenied {
return false, statusUpdateDenied
}
return false, statusFailedToUpdate
}
sub.pendingSafetyClear = false
return false, statusUpdating
}
func pollSubcommand(getSubClient getSubClientFunc, args []string) {
var err error
var srpcClient *srpc.Client
for iter := 0; *numPolls < 0 || iter < *numPolls; iter++ {
if iter > 0 {
time.Sleep(time.Duration(*interval) * time.Second)
}
if srpcClient == nil {
srpcClient = getSubClient()
}
var request sub.PollRequest
var reply sub.PollResponse
request.ShortPollOnly = *shortPoll
pollStartTime := time.Now()
err = client.CallPoll(srpcClient, request, &reply)
fmt.Printf("Poll duration: %s\n", time.Since(pollStartTime))
if err != nil {
fmt.Fprintf(os.Stderr, "Error calling: %s\n", err)
os.Exit(1)
}
if *newConnection {
srpcClient.Close()
srpcClient = nil
}
fs := reply.FileSystem
if fs == nil {
if !*shortPoll {
fmt.Println("No FileSystem pointer")
}
} else {
fs.RebuildInodePointers()
if *debug {
fs.List(os.Stdout)
} else {
fmt.Println(fs)
}
fmt.Printf("Num objects: %d\n", len(reply.ObjectCache))
if *file != "" {
f, err := os.Create(*file)
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating: %s: %s\n",
*file, err)
os.Exit(1)
}
encoder := gob.NewEncoder(f)
encoder.Encode(fs)
f.Close()
}
}
if reply.LastSuccessfulImageName != "" {
fmt.Printf("Last successful image: \"%s\"\n",
reply.LastSuccessfulImageName)
}
}
time.Sleep(time.Duration(*wait) * time.Second)
}
func pollSubcommand(srpcClient *srpc.Client, args []string) {
var err error
clientName := fmt.Sprintf("%s:%d", *subHostname, *subPortNum)
for iter := 0; *numPolls < 0 || iter < *numPolls; iter++ {
if iter > 0 {
time.Sleep(time.Duration(*interval) * time.Second)
}
if srpcClient == nil {
srpcClient, err = srpc.DialHTTP("tcp", clientName)
if err != nil {
fmt.Fprintf(os.Stderr, "Error dialing\t%s\n", err)
os.Exit(1)
}
}
var request sub.PollRequest
var reply sub.PollResponse
pollStartTime := time.Now()
err = client.CallPoll(srpcClient, request, &reply)
fmt.Printf("Poll duration: %s\n", time.Since(pollStartTime))
if err != nil {
fmt.Fprintf(os.Stderr, "Error calling\t%s\n", err)
os.Exit(1)
}
if *newConnection {
srpcClient.Close()
srpcClient = nil
}
fs := reply.FileSystem
if fs == nil {
fmt.Println("No FileSystem pointer")
} else {
fs.RebuildInodePointers()
if *debug {
fs.List(os.Stdout)
} else {
fmt.Println(fs)
}
fmt.Printf("Num objects: %d\n", len(reply.ObjectCache))
if *file != "" {
f, err := os.Create(*file)
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating: %s\t%s\n",
*file, err)
os.Exit(1)
}
encoder := gob.NewEncoder(f)
encoder.Encode(fs)
f.Close()
}
}
}
time.Sleep(time.Duration(*wait) * time.Second)
}
func (sub *Sub) updateConfiguration(srpcClient *srpc.Client,
pollReply subproto.PollResponse) {
if pollReply.ScanCount < 1 {
return
}
sub.herd.RLock()
newConf := sub.herd.configurationForSubs
sub.herd.RUnlock()
if compareConfigs(pollReply.CurrentConfiguration, newConf) {
return
}
if err := client.SetConfiguration(srpcClient, newConf); err != nil {
srpcClient.Close()
logger := sub.herd.logger
logger.Printf("Error setting configuration for sub: %s: %s\n",
sub, err)
return
}
}
func (sub *Sub) cleanup(srpcClient *srpc.Client) {
logger := sub.herd.logger
unusedObjects := make(map[hash.Hash]bool)
for _, hash := range sub.objectCache {
unusedObjects[hash] = false // Potential cleanup candidate.
}
for _, inode := range sub.fileSystem.InodeTable {
if inode, ok := inode.(*filesystem.RegularInode); ok {
if inode.Size > 0 {
if _, ok := unusedObjects[inode.Hash]; ok {
unusedObjects[inode.Hash] = true // Must clean this one up.
}
}
}
}
image := sub.plannedImage
if image != nil {
for _, inode := range image.FileSystem.InodeTable {
if inode, ok := inode.(*filesystem.RegularInode); ok {
if inode.Size > 0 {
if clean, ok := unusedObjects[inode.Hash]; !clean && ok {
delete(unusedObjects, inode.Hash)
}
}
}
}
}
if len(unusedObjects) < 1 {
return
}
hashes := make([]hash.Hash, 0, len(unusedObjects))
for hash := range unusedObjects {
hashes = append(hashes, hash)
}
if err := client.Cleanup(srpcClient, hashes); err != nil {
srpcClient.Close()
logger.Printf("Error calling %s:Subd.Cleanup(): %s\n", sub, err)
}
}
func (m *Manager) loadImage(imageClient *srpc.Client, name string) (
*srpc.Client, *image.Image, error) {
if imageClient == nil {
var err error
imageClient, err = srpc.DialHTTP("tcp", m.imageServerAddress, 0)
if err != nil {
if !m.loggedDialFailure {
m.logger.Printf("Error dialing: %s: %s\n",
m.imageServerAddress, err)
m.loggedDialFailure = true
}
return nil, nil, err
}
}
img, err := client.GetImage(imageClient, name)
if err != nil {
m.logger.Printf("Error calling: %s\n", err)
imageClient.Close()
return nil, nil, err
}
if img == nil || m.scheduleExpiration(img, name) {
return imageClient, nil, nil
}
if err := img.FileSystem.RebuildInodePointers(); err != nil {
m.logger.Printf("Error building inode pointers for image: %s %s",
name, err)
return imageClient, nil, err
}
// Build cache data now to avoid potential concurrent builds later.
img.FileSystem.InodeToFilenamesTable()
img.FileSystem.FilenameToInodeTable()
img.FileSystem.HashToInodesTable()
img.FileSystem.ComputeTotalDataBytes()
img.FileSystem.BuildEntryMap()
m.logger.Printf("Got image: %s\n", name)
return imageClient, img, nil
}
// Returns true if all required objects are available.
func (sub *Sub) fetchMissingObjects(srpcClient *srpc.Client, image *image.Image,
pushComputedFiles bool) (
bool, subStatus) {
if image == nil {
return false, statusImageNotReady
}
logger := sub.herd.logger
subObj := lib.Sub{
Hostname: sub.mdb.Hostname,
Client: srpcClient,
FileSystem: sub.fileSystem,
ComputedInodes: sub.computedInodes,
ObjectCache: sub.objectCache,
ObjectGetter: sub.herd.objectServer}
objectsToFetch, objectsToPush := lib.BuildMissingLists(subObj, image,
pushComputedFiles, false, logger)
if objectsToPush == nil {
return false, statusMissingComputedFile
}
var returnAvailable bool = true
var returnStatus subStatus = statusSynced
if len(objectsToFetch) > 0 {
logger.Printf("Calling %s:Subd.Fetch() for: %d objects\n",
sub, len(objectsToFetch))
err := client.Fetch(srpcClient, sub.herd.imageManager.String(),
objectsToFetch)
if err != nil {
srpcClient.Close()
logger.Printf("Error calling %s:Subd.Fetch(): %s\n", sub, err)
if err == srpc.ErrorAccessToMethodDenied {
return false, statusFetchDenied
}
return false, statusFailedToFetch
}
returnAvailable = false
returnStatus = statusFetching
}
if len(objectsToPush) > 0 {
sub.herd.pushSemaphore <- struct{}{}
defer func() { <-sub.herd.pushSemaphore }()
sub.status = statusPushing
err := lib.PushObjects(subObj, objectsToPush, logger)
if err != nil {
if err == srpc.ErrorAccessToMethodDenied {
return false, statusPushDenied
}
if err == lib.ErrorFailedToGetObject {
return false, statusFailedToGetObject
}
return false, statusFailedToPush
}
if returnAvailable {
// Update local copy of objectcache, since there will not be
// another Poll() before the update computation.
for hashVal := range objectsToPush {
sub.objectCache = append(sub.objectCache, hashVal)
}
}
}
return returnAvailable, returnStatus
}
func (sub *Sub) poll(srpcClient *srpc.Client, previousStatus subStatus) {
// If the planned image has just become available, force a full poll.
if previousStatus == statusSynced &&
!sub.havePlannedImage &&
sub.plannedImage != nil {
sub.havePlannedImage = true
sub.generationCount = 0 // Force a full poll.
}
// If the computed files have changed since the last sync, force a full poll
if previousStatus == statusSynced &&
sub.computedFilesChangeTime.After(sub.lastSyncTime) {
sub.generationCount = 0 // Force a full poll.
}
// If the last update was disabled and updates are enabled now, force a full
// poll.
if previousStatus == statusUpdatesDisabled &&
sub.herd.updatesDisabledReason == "" && !sub.mdb.DisableUpdates {
sub.generationCount = 0 // Force a full poll.
}
// If the last update was disabled due to a safety check and there is a
// pending SafetyClear, force a full poll to re-compute the update.
if previousStatus == statusUnsafeUpdate && sub.pendingSafetyClear {
sub.generationCount = 0 // Force a full poll.
}
var request subproto.PollRequest
request.HaveGeneration = sub.generationCount
var reply subproto.PollResponse
haveImage := false
if sub.requiredImage == nil {
request.ShortPollOnly = true
} else {
haveImage = true
}
logger := sub.herd.logger
sub.lastPollStartTime = time.Now()
if err := client.CallPoll(srpcClient, request, &reply); err != nil {
srpcClient.Close()
if err == io.EOF {
return
}
sub.pollTime = time.Time{}
if err == srpc.ErrorAccessToMethodDenied {
sub.status = statusPollDenied
} else {
sub.status = statusFailedToPoll
}
logger.Printf("Error calling %s.Poll(): %s\n", sub, err)
return
}
sub.lastPollSucceededTime = time.Now()
sub.lastSuccessfulImageName = reply.LastSuccessfulImageName
if reply.GenerationCount == 0 {
sub.reclaim()
sub.generationCount = 0
}
sub.lastScanDuration = reply.DurationOfLastScan
if fs := reply.FileSystem; fs == nil {
sub.lastPollWasFull = false
sub.lastShortPollDuration =
sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
shortPollDistribution.Add(sub.lastShortPollDuration)
if !sub.startTime.Equal(reply.StartTime) {
sub.generationCount = 0 // Sub has restarted: force a full poll.
}
} else {
sub.lastPollWasFull = true
if err := fs.RebuildInodePointers(); err != nil {
sub.status = statusFailedToPoll
logger.Printf("Error building pointers for: %s %s\n", sub, err)
return
}
fs.BuildEntryMap()
sub.fileSystem = fs
sub.objectCache = reply.ObjectCache
sub.generationCount = reply.GenerationCount
sub.lastFullPollDuration =
sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
fullPollDistribution.Add(sub.lastFullPollDuration)
}
sub.startTime = reply.StartTime
sub.pollTime = reply.PollTime
sub.updateConfiguration(srpcClient, reply)
if reply.FetchInProgress {
sub.status = statusFetching
return
}
if reply.UpdateInProgress {
sub.status = statusUpdating
return
}
if reply.GenerationCount < 1 {
sub.status = statusSubNotReady
return
}
if previousStatus == statusFetching && reply.LastFetchError != "" {
logger.Printf("Fetch failure for: %s: %s\n", sub, reply.LastFetchError)
sub.status = statusFailedToFetch
if sub.fileSystem == nil {
sub.generationCount = 0 // Force a full poll next cycle.
return
}
}
if previousStatus == statusUpdating {
// Transition from updating to update ended (may be partial/failed).
if reply.LastUpdateError != "" {
logger.Printf("Update failure for: %s: %s\n",
sub, reply.LastUpdateError)
sub.status = statusFailedToUpdate
} else {
sub.status = statusWaitingForNextFullPoll
}
sub.scanCountAtLastUpdateEnd = reply.ScanCount
sub.reclaim()
return
}
if sub.checkCancel() {
// Configuration change pending: skip further processing. Do not reclaim
// file-system and objectcache data: it will speed up the next Poll.
return
}
if !haveImage {
if sub.requiredImageName == "" {
sub.status = statusImageUndefined
} else {
sub.status = statusImageNotReady
}
return
}
if previousStatus == statusFailedToUpdate ||
previousStatus == statusWaitingForNextFullPoll {
if sub.scanCountAtLastUpdateEnd == reply.ScanCount {
// Need to wait until sub has performed a new scan.
if sub.fileSystem != nil {
sub.reclaim()
}
sub.status = previousStatus
return
}
if sub.fileSystem == nil {
// Force a full poll next cycle so that we can see the state of the
// sub.
sub.generationCount = 0
sub.status = previousStatus
return
}
}
if sub.fileSystem == nil {
sub.status = previousStatus
return
}
if idle, status := sub.fetchMissingObjects(srpcClient, sub.requiredImage,
true); !idle {
sub.status = status
sub.reclaim()
return
}
sub.status = statusComputingUpdate
if idle, status := sub.sendUpdate(srpcClient); !idle {
sub.status = status
sub.reclaim()
return
}
if idle, status := sub.fetchMissingObjects(srpcClient, sub.plannedImage,
false); !idle {
if status != statusImageNotReady {
sub.status = status
sub.reclaim()
return
}
}
if previousStatus == statusWaitingForNextFullPoll &&
!sub.lastUpdateTime.IsZero() {
sub.lastSyncTime = time.Now()
}
sub.status = statusSynced
sub.cleanup(srpcClient)
sub.reclaim()
}
