// Returns true if no update needs to be performed.
func (sub *Sub) sendUpdate(srpcClient *srpc.Client) (bool, subStatus) {
logger := sub.herd.logger
if !sub.pendingSafetyClear {
// Perform a cheap safety check.
if sub.requiredImage.Filter != nil &&
len(sub.fileSystem.InodeTable)>>1 >
len(sub.requiredImage.FileSystem.InodeTable) {
return false, statusUnsafeUpdate
}
}
var request subproto.UpdateRequest
var reply subproto.UpdateResponse
if idle, missing := sub.buildUpdateRequest(&request); missing {
return false, statusMissingComputedFile
} else if idle {
return true, statusSynced
}
if sub.mdb.DisableUpdates || sub.herd.updatesDisabledReason != "" {
return false, statusUpdatesDisabled
}
sub.status = statusSendingUpdate
sub.lastUpdateTime = time.Now()
if err := client.CallUpdate(srpcClient, request, &reply); err != nil {
srpcClient.Close()
logger.Printf("Error calling %s:Subd.Update(): %s\n", sub, err)
if err == srpc.ErrorAccessToMethodDenied {
return false, statusUpdateDenied
}
return false, statusFailedToUpdate
}
sub.pendingSafetyClear = false
return false, statusUpdating
}
func deleteUnreferencedObjects(client *srpc.Client, percentage uint8,
bytes uint64) error {
request := imageserver.DeleteUnreferencedObjectsRequest{percentage, bytes}
var reply imageserver.DeleteUnreferencedObjectsResponse
return client.RequestReply("ImageServer.DeleteUnreferencedObjects",
request, &reply)
}
func callPoll(client *srpc.Client, request sub.PollRequest,
reply *sub.PollResponse) error {
conn, err := client.Call("Subd.Poll")
if err != nil {
return err
}
defer conn.Close()
encoder := gob.NewEncoder(conn)
if err := encoder.Encode(request); err != nil {
return err
}
conn.Flush()
str, err := conn.ReadString('\n')
if err != nil {
return err
}
if str != "\n" {
return errors.New(str)
}
if err := gob.NewDecoder(conn).Decode(reply); err != nil {
return err
}
if reply.FileSystemFollows {
reply.FileSystem, err = filesystem.Decode(conn)
if err != nil {
return err
}
reply.ObjectCache, err = objectcache.Decode(conn)
if err != nil {
return err
}
}
return nil
}
func fetch(srpcClient *srpc.Client, hashesFilename string) error {
hashesFile, err := os.Open(hashesFilename)
if err != nil {
return err
}
defer hashesFile.Close()
scanner := bufio.NewScanner(hashesFile)
serverAddress := fmt.Sprintf("%s:%d",
*objectServerHostname, *objectServerPortNum)
hashes := make([]hash.Hash, 0)
for scanner.Scan() {
hashval, err := objectcache.FilenameToHash(scanner.Text())
if err != nil {
return err
}
hashes = append(hashes, hashval)
}
if err := scanner.Err(); err != nil {
return err
}
return srpcClient.RequestReply("Subd.Fetch", sub.FetchRequest{
ServerAddress: serverAddress,
Wait: true,
Hashes: hashes},
&sub.FetchResponse{})
}
func configureSubs(client *srpc.Client) error {
var request dominator.ConfigureSubsRequest
var reply dominator.ConfigureSubsResponse
request.ScanSpeedPercent = *scanSpeedPercent
request.NetworkSpeedPercent = *networkSpeedPercent
request.ScanExclusionList = scanExcludeList
return client.RequestReply("Dominator.ConfigureSubs", request, &reply)
}
func disableUpdates(client *srpc.Client, reason string) error {
if reason == "" {
return errors.New("cannot disable updates: no reason given")
}
var request dominator.DisableUpdatesRequest
var reply dominator.DisableUpdatesResponse
request.Reason = reason
return client.RequestReply("Dominator.DisableUpdates", request, &reply)
}
func checkImage(client *srpc.Client, name string) (bool, error) {
request := imageserver.CheckImageRequest{name}
var reply imageserver.CheckImageResponse
err := client.RequestReply("ImageServer.CheckImage", request, &reply)
if err != nil {
return false, err
}
return reply.ImageExists, nil
}
func pollSubcommand(getSubClient getSubClientFunc, args []string) {
var err error
var srpcClient *srpc.Client
for iter := 0; *numPolls < 0 || iter < *numPolls; iter++ {
if iter > 0 {
time.Sleep(time.Duration(*interval) * time.Second)
}
if srpcClient == nil {
srpcClient = getSubClient()
}
var request sub.PollRequest
var reply sub.PollResponse
request.ShortPollOnly = *shortPoll
pollStartTime := time.Now()
err = client.CallPoll(srpcClient, request, &reply)
fmt.Printf("Poll duration: %s\n", time.Since(pollStartTime))
if err != nil {
fmt.Fprintf(os.Stderr, "Error calling: %s\n", err)
os.Exit(1)
}
if *newConnection {
srpcClient.Close()
srpcClient = nil
}
fs := reply.FileSystem
if fs == nil {
if !*shortPoll {
fmt.Println("No FileSystem pointer")
}
} else {
fs.RebuildInodePointers()
if *debug {
fs.List(os.Stdout)
} else {
fmt.Println(fs)
}
fmt.Printf("Num objects: %d\n", len(reply.ObjectCache))
if *file != "" {
f, err := os.Create(*file)
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating: %s: %s\n",
*file, err)
os.Exit(1)
}
encoder := gob.NewEncoder(f)
encoder.Encode(fs)
f.Close()
}
}
if reply.LastSuccessfulImageName != "" {
fmt.Printf("Last successful image: \"%s\"\n",
reply.LastSuccessfulImageName)
}
}
time.Sleep(time.Duration(*wait) * time.Second)
}
func setDefaultImage(client *srpc.Client, imageName string) error {
var request dominator.SetDefaultImageRequest
var reply dominator.SetDefaultImageResponse
request.ImageName = imageName
if err := client.RequestReply("Dominator.SetDefaultImage", request,
&reply); err != nil {
return err
}
return nil
}
func getSubsConfiguration(client *srpc.Client) error {
var request dominator.GetSubsConfigurationRequest
var reply dominator.GetSubsConfigurationResponse
if err := client.RequestReply("Dominator.GetSubsConfiguration", request,
&reply); err != nil {
return err
}
fmt.Println(sub.Configuration(reply))
return nil
}
func getDefaultImage(client *srpc.Client) error {
var request dominator.GetDefaultImageRequest
var reply dominator.GetDefaultImageResponse
if err := client.RequestReply("Dominator.GetDefaultImage", request,
&reply); err != nil {
return err
}
if reply.ImageName != "" {
fmt.Println(reply.ImageName)
}
return nil
}
func pollSubcommand(srpcClient *srpc.Client, args []string) {
var err error
clientName := fmt.Sprintf("%s:%d", *subHostname, *subPortNum)
for iter := 0; *numPolls < 0 || iter < *numPolls; iter++ {
if iter > 0 {
time.Sleep(time.Duration(*interval) * time.Second)
}
if srpcClient == nil {
srpcClient, err = srpc.DialHTTP("tcp", clientName)
if err != nil {
fmt.Fprintf(os.Stderr, "Error dialing\t%s\n", err)
os.Exit(1)
}
}
var request sub.PollRequest
var reply sub.PollResponse
pollStartTime := time.Now()
err = client.CallPoll(srpcClient, request, &reply)
fmt.Printf("Poll duration: %s\n", time.Since(pollStartTime))
if err != nil {
fmt.Fprintf(os.Stderr, "Error calling\t%s\n", err)
os.Exit(1)
}
if *newConnection {
srpcClient.Close()
srpcClient = nil
}
fs := reply.FileSystem
if fs == nil {
fmt.Println("No FileSystem pointer")
} else {
fs.RebuildInodePointers()
if *debug {
fs.List(os.Stdout)
} else {
fmt.Println(fs)
}
fmt.Printf("Num objects: %d\n", len(reply.ObjectCache))
if *file != "" {
f, err := os.Create(*file)
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating: %s\t%s\n",
*file, err)
os.Exit(1)
}
encoder := gob.NewEncoder(f)
encoder.Encode(fs)
f.Close()
}
}
}
time.Sleep(time.Duration(*wait) * time.Second)
}
func newObjectAdderQueue(client *srpc.Client) (*ObjectAdderQueue, error) {
var objQ ObjectAdderQueue
var err error
objQ.conn, err = client.Call("ObjectServer.AddObjects")
if err != nil {
return nil, err
}
objQ.encoder = gob.NewEncoder(objQ.conn)
getResponseChan := make(chan bool, 65536)
errorChan := make(chan error, 1024)
objQ.getResponseChan = getResponseChan
objQ.errorChan = errorChan
objQ.sendSemaphore = make(chan bool, 1)
go readResponses(objQ.conn, getResponseChan, errorChan)
return &objQ, nil
}
func (sub *Sub) updateConfiguration(srpcClient *srpc.Client,
pollReply subproto.PollResponse) {
if pollReply.ScanCount < 1 {
return
}
sub.herd.RLock()
newConf := sub.herd.configurationForSubs
sub.herd.RUnlock()
if compareConfigs(pollReply.CurrentConfiguration, newConf) {
return
}
if err := client.SetConfiguration(srpcClient, newConf); err != nil {
srpcClient.Close()
logger := sub.herd.logger
logger.Printf("Error setting configuration for sub: %s: %s\n",
sub, err)
return
}
}
func listDirectories(client *srpc.Client) ([]image.Directory, error) {
conn, err := client.Call("ImageServer.ListDirectories")
if err != nil {
return nil, err
}
defer conn.Close()
directories := make([]image.Directory, 0)
decoder := gob.NewDecoder(conn)
for {
var directory image.Directory
if err := decoder.Decode(&directory); err != nil {
return nil, err
}
if directory.Name == "" {
break
}
directories = append(directories, directory)
}
return directories, nil
}
func listImages(client *srpc.Client) ([]string, error) {
conn, err := client.Call("ImageServer.ListImages")
if err != nil {
return nil, err
}
defer conn.Close()
images := make([]string, 0)
for {
line, err := conn.ReadString('\n')
if err != nil {
return nil, err
}
line = line[:len(line)-1]
if line == "" {
break
}
images = append(images, line)
}
return images, nil
}
func callAddImage(client *srpc.Client, request imageserver.AddImageRequest,
reply *imageserver.AddImageResponse) error {
conn, err := client.Call("ImageServer.AddImage")
if err != nil {
return err
}
defer conn.Close()
encoder := gob.NewEncoder(conn)
if err := encoder.Encode(request); err != nil {
return err
}
conn.Flush()
str, err := conn.ReadString('\n')
if err != nil {
return err
}
if str != "\n" {
return errors.New(str)
}
return gob.NewDecoder(conn).Decode(reply)
}
func listUnreferencedObjects(client *srpc.Client) (
map[hash.Hash]uint64, error) {
conn, err := client.Call("ImageServer.ListUnreferencedObjects")
if err != nil {
return nil, err
}
defer conn.Close()
objects := make(map[hash.Hash]uint64)
decoder := gob.NewDecoder(conn)
for {
var object imageserver.Object
if err := decoder.Decode(&object); err != nil {
return nil, err
}
if object.Size < 1 {
break
}
objects[object.Hash] = object.Size
}
return objects, nil
}
func callFetch(client *srpc.Client, request sub.FetchRequest,
reply *sub.FetchResponse) error {
conn, err := client.Call("Subd.Fetch")
if err != nil {
return err
}
defer conn.Close()
encoder := gob.NewEncoder(conn)
if err := encoder.Encode(request); err != nil {
return err
}
conn.Flush()
str, err := conn.ReadString('\n')
if err != nil {
return err
}
if str != "\n" {
return errors.New(str)
}
return gob.NewDecoder(conn).Decode(reply)
}
func (sub *Sub) cleanup(srpcClient *srpc.Client) {
logger := sub.herd.logger
unusedObjects := make(map[hash.Hash]bool)
for _, hash := range sub.objectCache {
unusedObjects[hash] = false // Potential cleanup candidate.
}
for _, inode := range sub.fileSystem.InodeTable {
if inode, ok := inode.(*filesystem.RegularInode); ok {
if inode.Size > 0 {
if _, ok := unusedObjects[inode.Hash]; ok {
unusedObjects[inode.Hash] = true // Must clean this one up.
}
}
}
}
image := sub.plannedImage
if image != nil {
for _, inode := range image.FileSystem.InodeTable {
if inode, ok := inode.(*filesystem.RegularInode); ok {
if inode.Size > 0 {
if clean, ok := unusedObjects[inode.Hash]; !clean && ok {
delete(unusedObjects, inode.Hash)
}
}
}
}
}
if len(unusedObjects) < 1 {
return
}
hashes := make([]hash.Hash, 0, len(unusedObjects))
for hash := range unusedObjects {
hashes = append(hashes, hash)
}
if err := client.Cleanup(srpcClient, hashes); err != nil {
srpcClient.Close()
logger.Printf("Error calling %s:Subd.Cleanup(): %s\n", sub, err)
}
}
func (m *Manager) loadImage(imageClient *srpc.Client, name string) (
*srpc.Client, *image.Image, error) {
if imageClient == nil {
var err error
imageClient, err = srpc.DialHTTP("tcp", m.imageServerAddress, 0)
if err != nil {
if !m.loggedDialFailure {
m.logger.Printf("Error dialing: %s: %s\n",
m.imageServerAddress, err)
m.loggedDialFailure = true
}
return nil, nil, err
}
}
img, err := client.GetImage(imageClient, name)
if err != nil {
m.logger.Printf("Error calling: %s\n", err)
imageClient.Close()
return nil, nil, err
}
if img == nil || m.scheduleExpiration(img, name) {
return imageClient, nil, nil
}
if err := img.FileSystem.RebuildInodePointers(); err != nil {
m.logger.Printf("Error building inode pointers for image: %s %s",
name, err)
return imageClient, nil, err
}
// Build cache data now to avoid potential concurrent builds later.
img.FileSystem.InodeToFilenamesTable()
img.FileSystem.FilenameToInodeTable()
img.FileSystem.HashToInodesTable()
img.FileSystem.ComputeTotalDataBytes()
img.FileSystem.BuildEntryMap()
m.logger.Printf("Got image: %s\n", name)
return imageClient, img, nil
}
func getFiles(client *srpc.Client, filenames []string,
readerFunc func(reader io.Reader, size uint64) error) error {
conn, err := client.Call("Subd.GetFiles")
if err != nil {
return err
}
defer conn.Close()
go sendRequests(conn, filenames)
decoder := gob.NewDecoder(conn)
for range filenames {
var reply sub.GetFileResponse
if err := decoder.Decode(&reply); err != nil {
return err
}
if reply.Error != nil {
return reply.Error
}
if err := readerFunc(&io.LimitedReader{R: conn, N: int64(reply.Size)},
reply.Size); err != nil {
return err
}
}
return nil
}
func fetch(client *srpc.Client, serverAddress string,
hashes []hash.Hash) error {
request := sub.FetchRequest{ServerAddress: serverAddress, Hashes: hashes}
var reply sub.FetchResponse
return client.RequestReply("Subd.Fetch", request, &reply)
}
func setConfiguration(client *srpc.Client, config sub.Configuration) error {
var request sub.SetConfigurationRequest
request = sub.SetConfigurationRequest(config)
var reply sub.SetConfigurationResponse
return client.RequestReply("Subd.SetConfiguration", request, &reply)
}
func clearSafetyShutoff(client *srpc.Client, subHostname string) error {
var request dominator.ClearSafetyShutoffRequest
var reply dominator.ClearSafetyShutoffResponse
request.Hostname = subHostname
return client.RequestReply("Dominator.ClearSafetyShutoff", request, &reply)
}
func cleanup(client *srpc.Client, hashes []hash.Hash) error {
request := sub.CleanupRequest{hashes}
var reply sub.CleanupResponse
return client.RequestReply("Subd.Cleanup", request, &reply)
}
func (sub *Sub) poll(srpcClient *srpc.Client, previousStatus subStatus) {
// If the planned image has just become available, force a full poll.
if previousStatus == statusSynced &&
!sub.havePlannedImage &&
sub.plannedImage != nil {
sub.havePlannedImage = true
sub.generationCount = 0 // Force a full poll.
}
// If the computed files have changed since the last sync, force a full poll
if previousStatus == statusSynced &&
sub.computedFilesChangeTime.After(sub.lastSyncTime) {
sub.generationCount = 0 // Force a full poll.
}
// If the last update was disabled and updates are enabled now, force a full
// poll.
if previousStatus == statusUpdatesDisabled &&
sub.herd.updatesDisabledReason == "" && !sub.mdb.DisableUpdates {
sub.generationCount = 0 // Force a full poll.
}
// If the last update was disabled due to a safety check and there is a
// pending SafetyClear, force a full poll to re-compute the update.
if previousStatus == statusUnsafeUpdate && sub.pendingSafetyClear {
sub.generationCount = 0 // Force a full poll.
}
var request subproto.PollRequest
request.HaveGeneration = sub.generationCount
var reply subproto.PollResponse
haveImage := false
if sub.requiredImage == nil {
request.ShortPollOnly = true
} else {
haveImage = true
}
logger := sub.herd.logger
sub.lastPollStartTime = time.Now()
if err := client.CallPoll(srpcClient, request, &reply); err != nil {
srpcClient.Close()
if err == io.EOF {
return
}
sub.pollTime = time.Time{}
if err == srpc.ErrorAccessToMethodDenied {
sub.status = statusPollDenied
} else {
sub.status = statusFailedToPoll
}
logger.Printf("Error calling %s.Poll(): %s\n", sub, err)
return
}
sub.lastPollSucceededTime = time.Now()
sub.lastSuccessfulImageName = reply.LastSuccessfulImageName
if reply.GenerationCount == 0 {
sub.reclaim()
sub.generationCount = 0
}
sub.lastScanDuration = reply.DurationOfLastScan
if fs := reply.FileSystem; fs == nil {
sub.lastPollWasFull = false
sub.lastShortPollDuration =
sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
shortPollDistribution.Add(sub.lastShortPollDuration)
if !sub.startTime.Equal(reply.StartTime) {
sub.generationCount = 0 // Sub has restarted: force a full poll.
}
} else {
sub.lastPollWasFull = true
if err := fs.RebuildInodePointers(); err != nil {
sub.status = statusFailedToPoll
logger.Printf("Error building pointers for: %s %s\n", sub, err)
return
}
fs.BuildEntryMap()
sub.fileSystem = fs
sub.objectCache = reply.ObjectCache
sub.generationCount = reply.GenerationCount
sub.lastFullPollDuration =
sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
fullPollDistribution.Add(sub.lastFullPollDuration)
}
sub.startTime = reply.StartTime
sub.pollTime = reply.PollTime
sub.updateConfiguration(srpcClient, reply)
if reply.FetchInProgress {
sub.status = statusFetching
return
}
if reply.UpdateInProgress {
sub.status = statusUpdating
return
}
if reply.GenerationCount < 1 {
sub.status = statusSubNotReady
return
}
if previousStatus == statusFetching && reply.LastFetchError != "" {
logger.Printf("Fetch failure for: %s: %s\n", sub, reply.LastFetchError)
sub.status = statusFailedToFetch
if sub.fileSystem == nil {
sub.generationCount = 0 // Force a full poll next cycle.
return
}
}
if previousStatus == statusUpdating {
// Transition from updating to update ended (may be partial/failed).
if reply.LastUpdateError != "" {
logger.Printf("Update failure for: %s: %s\n",
sub, reply.LastUpdateError)
sub.status = statusFailedToUpdate
} else {
sub.status = statusWaitingForNextFullPoll
}
sub.scanCountAtLastUpdateEnd = reply.ScanCount
sub.reclaim()
return
}
if sub.checkCancel() {
// Configuration change pending: skip further processing. Do not reclaim
// file-system and objectcache data: it will speed up the next Poll.
return
}
if !haveImage {
if sub.requiredImageName == "" {
sub.status = statusImageUndefined
} else {
sub.status = statusImageNotReady
}
return
}
if previousStatus == statusFailedToUpdate ||
previousStatus == statusWaitingForNextFullPoll {
if sub.scanCountAtLastUpdateEnd == reply.ScanCount {
// Need to wait until sub has performed a new scan.
if sub.fileSystem != nil {
sub.reclaim()
}
sub.status = previousStatus
return
}
if sub.fileSystem == nil {
// Force a full poll next cycle so that we can see the state of the
// sub.
sub.generationCount = 0
sub.status = previousStatus
return
}
}
if sub.fileSystem == nil {
sub.status = previousStatus
return
}
if idle, status := sub.fetchMissingObjects(srpcClient, sub.requiredImage,
true); !idle {
sub.status = status
sub.reclaim()
return
}
sub.status = statusComputingUpdate
if idle, status := sub.sendUpdate(srpcClient); !idle {
sub.status = status
sub.reclaim()
return
}
if idle, status := sub.fetchMissingObjects(srpcClient, sub.plannedImage,
false); !idle {
if status != statusImageNotReady {
sub.status = status
sub.reclaim()
return
}
}
if previousStatus == statusWaitingForNextFullPoll &&
!sub.lastUpdateTime.IsZero() {
sub.lastSyncTime = time.Now()
}
sub.status = statusSynced
sub.cleanup(srpcClient)
sub.reclaim()
}
func chownDirectory(client *srpc.Client, dirname, ownerGroup string) error {
request := imageserver.ChangeOwnerRequest{DirectoryName: dirname,
OwnerGroup: ownerGroup}
var reply imageserver.ChangeOwnerResponse
return client.RequestReply("ImageServer.ChownDirectory", request, &reply)
}
func getConfiguration(client *srpc.Client) (sub.Configuration, error) {
var request sub.GetConfigurationRequest
var reply sub.GetConfigurationResponse
err := client.RequestReply("Subd.GetConfiguration", request, &reply)
return sub.Configuration(reply), err
}
// Returns true if all required objects are available.
func (sub *Sub) fetchMissingObjects(srpcClient *srpc.Client, image *image.Image,
pushComputedFiles bool) (
bool, subStatus) {
if image == nil {
return false, statusImageNotReady
}
logger := sub.herd.logger
subObj := lib.Sub{
Hostname: sub.mdb.Hostname,
Client: srpcClient,
FileSystem: sub.fileSystem,
ComputedInodes: sub.computedInodes,
ObjectCache: sub.objectCache,
ObjectGetter: sub.herd.objectServer}
objectsToFetch, objectsToPush := lib.BuildMissingLists(subObj, image,
pushComputedFiles, false, logger)
if objectsToPush == nil {
return false, statusMissingComputedFile
}
var returnAvailable bool = true
var returnStatus subStatus = statusSynced
if len(objectsToFetch) > 0 {
logger.Printf("Calling %s:Subd.Fetch() for: %d objects\n",
sub, len(objectsToFetch))
err := client.Fetch(srpcClient, sub.herd.imageManager.String(),
objectsToFetch)
if err != nil {
srpcClient.Close()
logger.Printf("Error calling %s:Subd.Fetch(): %s\n", sub, err)
if err == srpc.ErrorAccessToMethodDenied {
return false, statusFetchDenied
}
return false, statusFailedToFetch
}
returnAvailable = false
returnStatus = statusFetching
}
if len(objectsToPush) > 0 {
sub.herd.pushSemaphore <- struct{}{}
defer func() { <-sub.herd.pushSemaphore }()
sub.status = statusPushing
err := lib.PushObjects(subObj, objectsToPush, logger)
if err != nil {
if err == srpc.ErrorAccessToMethodDenied {
return false, statusPushDenied
}
if err == lib.ErrorFailedToGetObject {
return false, statusFailedToGetObject
}
return false, statusFailedToPush
}
if returnAvailable {
// Update local copy of objectcache, since there will not be
// another Poll() before the update computation.
for hashVal := range objectsToPush {
sub.objectCache = append(sub.objectCache, hashVal)
}
}
}
return returnAvailable, returnStatus
}