func main() {
flag.Usage = printUsage
flag.Parse()
if flag.NArg() < 1 {
printUsage()
os.Exit(2)
}
setupTls(*certFile, *keyFile)
clientName := fmt.Sprintf("%s:%d",
*imageServerHostname, *imageServerPortNum)
imageClient, err := rpc.DialHTTP("tcp", clientName)
if err != nil {
fmt.Fprintf(os.Stderr, "Error dialing\t%s\n", err)
os.Exit(1)
}
imageSClient, err := srpc.DialHTTP("tcp", clientName)
if err != nil {
fmt.Fprintf(os.Stderr, "Error dialing\t%s\n", err)
os.Exit(1)
}
objectClient := objectclient.NewObjectClient(clientName)
for _, subcommand := range subcommands {
if flag.Arg(0) == subcommand.command {
if flag.NArg()-1 != subcommand.numArgs {
printUsage()
os.Exit(2)
}
subcommand.cmdFunc(imageClient, imageSClient, objectClient,
flag.Args()[1:])
os.Exit(3)
}
}
printUsage()
os.Exit(2)
}
func getImageRetry(clientName, imageName string,
timeoutTime time.Time) (*image.Image, error) {
imageSrpcClient, err := srpc.DialHTTP("tcp", clientName, 0)
if err != nil {
return nil, err
}
defer imageSrpcClient.Close()
for ; time.Now().Before(timeoutTime); time.Sleep(time.Second) {
img, err := imgclient.GetImage(imageSrpcClient, imageName)
if err != nil {
return nil, err
} else if img != nil {
if err := img.FileSystem.RebuildInodePointers(); err != nil {
return nil, err
}
img.FileSystem.InodeToFilenamesTable()
img.FileSystem.FilenameToInodeTable()
img.FileSystem.HashToInodesTable()
img.FileSystem.ComputeTotalDataBytes()
img.FileSystem.BuildEntryMap()
return img, nil
}
}
return nil, errors.New("timed out getting image")
}
func main() {
flag.Usage = printUsage
flag.Parse()
if flag.NArg() < 1 {
printUsage()
os.Exit(2)
}
if err := setupclient.SetupTls(true); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
clientName := fmt.Sprintf("%s:%d", *domHostname, *domPortNum)
client, err := srpc.DialHTTP("tcp", clientName, 0)
if err != nil {
fmt.Fprintf(os.Stderr, "Error dialing\t%s\n", err)
os.Exit(1)
}
for _, subcommand := range subcommands {
if flag.Arg(0) == subcommand.command {
if flag.NArg()-1 != subcommand.numArgs {
printUsage()
os.Exit(2)
}
subcommand.cmdFunc(client, flag.Args()[1:])
os.Exit(3)
}
}
printUsage()
os.Exit(2)
}
func (sub *Sub) connectAndPoll() {
previousStatus := sub.status
sub.status = statusConnecting
hostname := strings.SplitN(sub.hostname, "*", 2)[0]
address := fmt.Sprintf("%s:%d", hostname, constants.SubPortNumber)
sub.lastConnectionStartTime = time.Now()
srpcClient, err := srpc.DialHTTP("tcp", address)
if err != nil {
if err, ok := err.(*net.OpError); ok {
if _, ok := err.Err.(*net.DNSError); ok {
sub.status = statusDNSError
return
}
}
sub.status = statusFailedToConnect
return
}
defer srpcClient.Close()
sub.status = statusWaitingToPoll
sub.lastConnectionSucceededTime = time.Now()
sub.lastConnectDuration =
sub.lastConnectionSucceededTime.Sub(sub.lastConnectionStartTime)
connectDistribution.Add(sub.lastConnectDuration)
if sub.herd.getImage(sub.requiredImage) == nil {
sub.status = statusImageNotReady
return
}
sub.herd.pollSemaphore <- true
sub.status = statusPolling
sub.poll(srpcClient, previousStatus)
<-sub.herd.pollSemaphore
}
func (objClient *ObjectClient) getObjects(hashes []hash.Hash) (
*ObjectsReader, error) {
client, err := srpc.DialHTTP("tcp", objClient.address)
if err != nil {
return nil, errors.New(fmt.Sprintf("Error dialing\t%s\n", err))
}
conn, err := client.Call("ObjectServer.GetObjects")
if err != nil {
return nil, errors.New(fmt.Sprintf("Error calling\t%s\n", err))
}
var request objectserver.GetObjectsRequest
var reply objectserver.GetObjectsResponse
request.Exclusive = objClient.exclusiveGet
request.Hashes = hashes
encoder := gob.NewEncoder(conn)
encoder.Encode(request)
conn.Flush()
var objectsReader ObjectsReader
objectsReader.client = client
objectsReader.reader = conn
decoder := gob.NewDecoder(objectsReader.reader)
err = decoder.Decode(&reply)
if err != nil {
return nil, err
}
if reply.ResponseString != "" {
return nil, errors.New(reply.ResponseString)
}
objectsReader.nextIndex = -1
objectsReader.sizes = reply.ObjectSizes
return &objectsReader, nil
}
func getSubClient() *srpc.Client {
clientName := fmt.Sprintf("%s:%d", *subHostname, *subPortNum)
client, err := srpc.DialHTTP("tcp", clientName, time.Second*5)
if err != nil {
fmt.Fprintf(os.Stderr, "Error dialing %s: %s\n", clientName, err)
os.Exit(3)
}
return client
}
func pollSubcommand(srpcClient *srpc.Client, args []string) {
var err error
clientName := fmt.Sprintf("%s:%d", *subHostname, *subPortNum)
for iter := 0; *numPolls < 0 || iter < *numPolls; iter++ {
if iter > 0 {
time.Sleep(time.Duration(*interval) * time.Second)
}
if srpcClient == nil {
srpcClient, err = srpc.DialHTTP("tcp", clientName)
if err != nil {
fmt.Fprintf(os.Stderr, "Error dialing\t%s\n", err)
os.Exit(1)
}
}
var request sub.PollRequest
var reply sub.PollResponse
pollStartTime := time.Now()
err = client.CallPoll(srpcClient, request, &reply)
fmt.Printf("Poll duration: %s\n", time.Since(pollStartTime))
if err != nil {
fmt.Fprintf(os.Stderr, "Error calling\t%s\n", err)
os.Exit(1)
}
if *newConnection {
srpcClient.Close()
srpcClient = nil
}
fs := reply.FileSystem
if fs == nil {
fmt.Println("No FileSystem pointer")
} else {
fs.RebuildInodePointers()
if *debug {
fs.List(os.Stdout)
} else {
fmt.Println(fs)
}
fmt.Printf("Num objects: %d\n", len(reply.ObjectCache))
if *file != "" {
f, err := os.Create(*file)
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating: %s\t%s\n",
*file, err)
os.Exit(1)
}
encoder := gob.NewEncoder(f)
encoder.Encode(fs)
f.Close()
}
}
}
time.Sleep(time.Duration(*wait) * time.Second)
}
func getClients() (*srpc.Client, *objectclient.ObjectClient) {
if imageSrpcClient == nil {
var err error
clientName := fmt.Sprintf("%s:%d",
*imageServerHostname, *imageServerPortNum)
imageSrpcClient, err = srpc.DialHTTP("tcp", clientName, 0)
if err != nil {
fmt.Fprintf(os.Stderr, "Error dialing\t%s\n", err)
os.Exit(1)
}
theObjectClient = objectclient.NewObjectClient(clientName)
}
return imageSrpcClient, theObjectClient
}
func (objClient *ObjectClient) getClient() (*srpc.Client, error) {
if objClient.client != nil {
return objClient.client, nil
}
if objClient.address == "" {
return nil, errors.New("no client address")
}
srpcClient, err := srpc.DialHTTP("tcp", objClient.address, 0)
if err != nil {
return nil, fmt.Errorf("error dialing: %s: %s", objClient.address, err)
}
objClient.client = srpcClient
return objClient.client, nil
}
func copyMissingObjects(fs *filesystem.FileSystem, imageSClient *srpc.Client,
objectClient *objectclient.ObjectClient, subName string) error {
// Check to see which objects are in the objectserver.
hashes := make([]hash.Hash, 0, fs.NumRegularInodes)
for hash, _ := range fs.HashToInodesTable() {
hashes = append(hashes, hash)
}
objectSizes, err := objectClient.CheckObjects(hashes)
if err != nil {
return err
}
missingHashes := make([]hash.Hash, 0)
for index, size := range objectSizes {
if size < 1 {
missingHashes = append(missingHashes, hashes[index])
}
}
if len(missingHashes) < 1 {
return nil
}
// Get missing objects from sub.
filesForMissingObjects := make([]string, 0, len(missingHashes))
for _, hash := range missingHashes {
if inums, ok := fs.HashToInodesTable()[hash]; !ok {
return fmt.Errorf("no inode for object: %x", hash)
} else if files, ok := fs.InodeToFilenamesTable()[inums[0]]; !ok {
return fmt.Errorf("no file for inode: %d", inums[0])
} else {
filesForMissingObjects = append(filesForMissingObjects, files[0])
}
}
objAdderQueue, err := objectclient.NewObjectAdderQueue(imageSClient)
if err != nil {
return err
}
subClient, err := srpc.DialHTTP("tcp",
fmt.Sprintf("%s:%d", subName, constants.SubPortNumber), 0)
if err != nil {
return fmt.Errorf("error dialing %s", err)
}
defer subClient.Close()
if err := subclient.GetFiles(subClient, filesForMissingObjects,
func(reader io.Reader, size uint64) error {
_, err := objAdderQueue.Add(reader, size)
return err
}); err != nil {
return err
}
return objAdderQueue.Close()
}
func (objClient *ObjectClient) addObject(reader io.Reader, length uint64,
expectedHash *hash.Hash) (hash.Hash, bool, error) {
var request objectserver.AddObjectRequest
var reply objectserver.AddObjectResponse
if length < 1 {
return reply.Hash, false, errors.New(
"zero length object cannot be added")
}
srpcClient, err := srpc.DialHTTP("tcp", objClient.address)
if err != nil {
return reply.Hash, false, errors.New(
fmt.Sprintf("Error dialing\t%s\n", err.Error()))
}
defer srpcClient.Close()
conn, err := srpcClient.Call("ObjectServer.AddObjects")
if err != nil {
return reply.Hash, false, err
}
defer conn.Close()
request.Length = length
request.ExpectedHash = expectedHash
encoder := gob.NewEncoder(conn)
encoder.Encode(request)
nCopied, err := io.Copy(conn, reader)
if err != nil {
return reply.Hash, false, err
}
if uint64(nCopied) != length {
return reply.Hash, false, errors.New(fmt.Sprintf(
"failed to copy, wanted: %d, got: %d bytes", length, nCopied))
}
conn.Flush()
decoder := gob.NewDecoder(conn)
if err := decoder.Decode(&reply); err != nil {
return reply.Hash, false, err
}
if reply.Error != nil {
return reply.Hash, false, err
}
if expectedHash != nil && *expectedHash != reply.Hash {
return reply.Hash, false, errors.New(fmt.Sprintf(
"received hash: %x != expected: %x",
reply.Hash, *expectedHash))
}
return reply.Hash, reply.Added, nil
}
func (herd *Herd) getImageHaveLock(name string) *image.Image {
if name == "" {
return nil
}
if image := herd.imagesByName[name]; image != nil {
return image
}
// Image not yet known. If it was recently found to be missing, report it
// as missing. This avoids hammering the imageserver with "are we there
// yet?", "are we there yet?", "are we there yet?" queries.
if lastCheck, ok := herd.missingImages[name]; ok {
if time.Since(lastCheck).Seconds() < 1 {
return nil
}
}
imageClient, err := srpc.DialHTTP("tcp", herd.imageServerAddress)
if err != nil {
herd.logger.Println(err)
return nil
}
defer imageClient.Close()
var request imageserver.GetImageRequest
request.ImageName = name
var reply imageserver.GetImageResponse
err = client.CallGetImage(imageClient, request, &reply)
if err != nil {
herd.logger.Printf("Error calling\t%s\n", err)
return nil
}
if reply.Image == nil {
herd.missingImages[name] = time.Now()
} else {
if err := reply.Image.FileSystem.RebuildInodePointers(); err != nil {
herd.logger.Printf("Error building inode pointers for image: %s %s",
name, err)
return nil
}
delete(herd.missingImages, name)
reply.Image.FileSystem.BuildEntryMap()
reply.Image.FileSystem.BuildInodeToFilenamesTable()
reply.Image.FileSystem.BuildHashToInodesTable()
herd.imagesByName[name] = reply.Image
herd.logger.Printf("Got image: %s\n", name)
}
return reply.Image
}
func addObjects(address string, filenames []string) error {
client, err := srpc.DialHTTP("tcp", address, 0)
if err != nil {
return err
}
defer client.Close()
objQ, err := objectclient.NewObjectAdderQueue(client)
if err != nil {
return err
}
for _, filename := range filenames {
if err := addObject(objQ, filename); err != nil {
objQ.Close()
return err
}
}
return objQ.Close()
}
func pollImage(name string) (*filesystem.FileSystem, error) {
clientName := fmt.Sprintf("%s:%d", name, constants.SubPortNumber)
srpcClient, err := srpc.DialHTTP("tcp", clientName, 0)
if err != nil {
return nil, errors.New(fmt.Sprintf("Error dialing %s", err))
}
defer srpcClient.Close()
var request sub.PollRequest
var reply sub.PollResponse
if err = subclient.CallPoll(srpcClient, request, &reply); err != nil {
return nil, err
}
if reply.FileSystem == nil {
return nil, errors.New("no poll data")
}
reply.FileSystem.RebuildInodePointers()
return reply.FileSystem, nil
}
func getSubClientRetry() *srpc.Client {
clientName := fmt.Sprintf("%s:%d", *subHostname, *subPortNum)
var client *srpc.Client
var err error
for time.Now().Before(timeoutTime) {
client, err = srpc.DialHTTP("tcp", clientName, time.Second*5)
if err == nil {
return client
}
if err == srpc.ErrorMissingCertificate ||
err == srpc.ErrorBadCertificate ||
err == srpc.ErrorAccessToMethodDenied {
// Never going to happen. Bail out.
fmt.Fprintf(os.Stderr, "Error dialing %s: %s\n", clientName, err)
os.Exit(3)
}
}
fmt.Fprintf(os.Stderr, "Error dialing %s: %s\n", clientName, err)
os.Exit(3)
return nil
}
func newObjectAdderQueue(objClient *ObjectClient) (*ObjectAdderQueue, error) {
var objQ ObjectAdderQueue
var err error
objQ.client, err = srpc.DialHTTP("tcp", objClient.address)
if err != nil {
return nil, errors.New(fmt.Sprintf("Error dialing\t%s\n", err.Error()))
}
objQ.conn, err = objQ.client.Call("ObjectServer.AddObjects")
if err != nil {
objQ.client.Close()
return nil, err
}
objQ.encoder = gob.NewEncoder(objQ.conn)
getResponseChan := make(chan bool, 65536)
errorChan := make(chan error, 1024)
objQ.getResponseChan = getResponseChan
objQ.errorChan = errorChan
objQ.sendSemaphore = make(chan bool, 1)
go readResponses(objQ.conn, getResponseChan, errorChan)
return &objQ, nil
}
func manageSource(sourceName string, sourceReconnectChannel chan<- string,
clientRequestChannel <-chan *proto.ClientRequest,
serverMessageChannel chan<- *serverMessageType, logger *log.Logger) {
closeNotifyChannel := make(chan struct{})
initialRetryTimeout := time.Millisecond * 100
retryTimeout := initialRetryTimeout
reconnect := false
for ; ; time.Sleep(retryTimeout) {
if retryTimeout < time.Minute {
retryTimeout *= 2
}
client, err := srpc.DialHTTP("tcp", sourceName, time.Second*15)
if err != nil {
logger.Printf("error connecting to: %s: %s\n", sourceName, err)
continue
}
conn, err := client.Call("FileGenerator.Connect")
if err != nil {
client.Close()
logger.Println(err)
continue
}
retryTimeout = initialRetryTimeout
// The server keeps the same encoder/decoder pair over the lifetime of
// the connection, so we must do the same.
go handleServerMessages(sourceName, gob.NewDecoder(conn),
serverMessageChannel, closeNotifyChannel, logger)
if reconnect {
sourceReconnectChannel <- sourceName
} else {
reconnect = true
}
sendClientRequests(conn, clientRequestChannel, closeNotifyChannel,
logger)
conn.Close()
client.Close()
}
}
func (m *Manager) loadImage(imageClient *srpc.Client, name string) (
*srpc.Client, *image.Image, error) {
if imageClient == nil {
var err error
imageClient, err = srpc.DialHTTP("tcp", m.imageServerAddress, 0)
if err != nil {
if !m.loggedDialFailure {
m.logger.Printf("Error dialing: %s: %s\n",
m.imageServerAddress, err)
m.loggedDialFailure = true
}
return nil, nil, err
}
}
img, err := client.GetImage(imageClient, name)
if err != nil {
m.logger.Printf("Error calling: %s\n", err)
imageClient.Close()
return nil, nil, err
}
if img == nil || m.scheduleExpiration(img, name) {
return imageClient, nil, nil
}
if err := img.FileSystem.RebuildInodePointers(); err != nil {
m.logger.Printf("Error building inode pointers for image: %s %s",
name, err)
return imageClient, nil, err
}
// Build cache data now to avoid potential concurrent builds later.
img.FileSystem.InodeToFilenamesTable()
img.FileSystem.FilenameToInodeTable()
img.FileSystem.HashToInodesTable()
img.FileSystem.ComputeTotalDataBytes()
img.FileSystem.BuildEntryMap()
m.logger.Printf("Got image: %s\n", name)
return imageClient, img, nil
}
func serverWatchDaemon(mdbServerHostname string, mdbServerPortNum uint,
mdbFileName string, mdbChannel chan<- *mdb.Mdb, logger *log.Logger) {
var lastMdb *mdb.Mdb
if file, err := os.Open(mdbFileName); err == nil {
lastMdb = loadFile(file, mdbFileName, logger)
file.Close()
if lastMdb != nil {
sort.Sort(lastMdb)
mdbChannel <- lastMdb
}
}
address := fmt.Sprintf("%s:%d", mdbServerHostname, mdbServerPortNum)
for ; ; time.Sleep(time.Second) {
client, err := srpc.DialHTTP("tcp", address, time.Second*15)
if err != nil {
logger.Println(err)
continue
}
conn, err := client.Call("MdbServer.GetMdbUpdates")
if err != nil {
logger.Println(err)
client.Close()
continue
}
decoder := gob.NewDecoder(conn)
firstUpdate := true
for {
var mdbUpdate mdbserver.MdbUpdate
if err := decoder.Decode(&mdbUpdate); err != nil {
logger.Println(err)
break
} else {
if firstUpdate {
lastMdb = &mdb.Mdb{}
firstUpdate = false
}
lastMdb = processUpdate(lastMdb, mdbUpdate)
sort.Sort(lastMdb)
mdbChannel <- lastMdb
if file, err := os.Create(mdbFileName + "~"); err != nil {
logger.Println(err)
} else {
writer := bufio.NewWriter(file)
var err error
if isGob(mdbFileName) {
encoder := gob.NewEncoder(writer)
err = encoder.Encode(lastMdb.Machines)
} else {
err = jsonwriter.WriteWithIndent(writer, " ",
lastMdb.Machines)
}
if err != nil {
logger.Println(err)
os.Remove(mdbFileName + "~")
} else {
writer.Flush()
file.Close()
os.Rename(mdbFileName+"~", mdbFileName)
}
}
}
}
conn.Close()
client.Close()
}
}
