func showTimeTaken(startTime time.Time) {
if *showTimes {
stopTime := time.Now()
fmt.Fprintf(os.Stderr, "took %s\n",
format.Duration(stopTime.Sub(startTime)))
}
}
func (sub *Sub) showBusy(writer io.Writer) {
if sub.busy {
if sub.busyStartTime.IsZero() {
fmt.Fprintln(writer, " <td>busy</td>")
} else {
fmt.Fprintf(writer, " <td>%s</td>\n",
format.Duration(time.Since(sub.busyStartTime)))
}
} else {
if sub.busyStartTime.IsZero() {
fmt.Fprintln(writer, " <td></td>")
} else {
fmt.Fprintf(writer, " <td><font color=\"grey\">%s</font></td>\n",
format.Duration(sub.busyStopTime.Sub(sub.busyStartTime)))
}
}
}
func showDuration(writer io.Writer, duration time.Duration, highlight bool) {
if duration < 1 {
fmt.Fprintf(writer, " <td></td>\n")
} else {
str := format.Duration(duration)
if highlight {
str = "<b>" + str + "</b>"
}
fmt.Fprintf(writer, " <td>%s</td>\n", str)
}
}
func writeHeader(writer io.Writer, req *http.Request, noGC bool) {
fmt.Fprintf(writer, "Start time: %s<br>\n", startTime.Format(timeFormat))
uptime := time.Since(startTime) + time.Millisecond*50
uptime = (uptime / time.Millisecond / 100) * time.Millisecond * 100
fmt.Fprintf(writer, "Uptime: %s<br>\n", format.Duration(uptime))
uTime, sTime := getRusage()
userCpuTime := uTime.Sub(startUtime)
sysCpuTime := sTime.Sub(startStime)
cpuTime := userCpuTime + sysCpuTime
fmt.Fprintf(writer, "CPU Time: %.1f%% (User: %s Sys: %s)<br>\n",
float64(cpuTime*100)/float64(uptime), userCpuTime, sysCpuTime)
var memStatsBeforeGC runtime.MemStats
runtime.ReadMemStats(&memStatsBeforeGC)
if noGC {
fmt.Fprintf(writer, "Allocated memory: %s<br>\n",
format.FormatBytes(memStatsBeforeGC.Alloc))
fmt.Fprintf(writer, "System memory: %s<br>\n",
format.FormatBytes(memStatsBeforeGC.Sys))
} else {
var memStatsAfterGC runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&memStatsAfterGC)
fmt.Fprintf(writer, "Allocated memory: %s (%s after GC)<br>\n",
format.FormatBytes(memStatsBeforeGC.Alloc),
format.FormatBytes(memStatsAfterGC.Alloc))
fmt.Fprintf(writer, "System memory: %s (%s after GC)<br>\n",
format.FormatBytes(memStatsBeforeGC.Sys),
format.FormatBytes(memStatsAfterGC.Sys))
}
fmt.Fprintln(writer, "Raw <a href=\"metrics\">metrics</a><br>")
if req != nil {
protocol := "http"
if req.TLS != nil {
protocol = "https"
}
host := strings.Split(req.Host, ":")[0]
fmt.Fprintf(writer,
"Local <a href=\"%s://%s:6910/\">system health agent</a>",
protocol, host)
}
}
func (t *rpcType) doFetch(request sub.FetchRequest) error {
defer t.clearFetchInProgress()
objectServer := objectclient.NewObjectClient(request.ServerAddress)
defer objectServer.Close()
benchmark := false
linkSpeed, haveLinkSpeed := netspeed.GetSpeedToAddress(
request.ServerAddress)
if haveLinkSpeed {
t.logFetch(request, linkSpeed)
} else {
if t.networkReaderContext.MaximumSpeed() < 1 {
benchmark = enoughBytesForBenchmark(objectServer, request)
if benchmark {
objectServer.SetExclusiveGetObjects(true)
t.logger.Printf("Fetch(%s) %d objects and benchmark speed\n",
request.ServerAddress, len(request.Hashes))
} else {
t.logFetch(request, 0)
}
} else {
t.logFetch(request, t.networkReaderContext.MaximumSpeed())
}
}
objectsReader, err := objectServer.GetObjects(request.Hashes)
if err != nil {
t.logger.Printf("Error getting object reader: %s\n", err.Error())
return err
}
defer objectsReader.Close()
var totalLength uint64
defer t.rescanObjectCacheFunction()
timeStart := time.Now()
for _, hash := range request.Hashes {
length, reader, err := objectsReader.NextObject()
if err != nil {
t.logger.Println(err)
return err
}
r := io.Reader(reader)
if haveLinkSpeed {
if linkSpeed > 0 {
r = rateio.NewReaderContext(linkSpeed,
uint64(t.networkReaderContext.SpeedPercent()),
&rateio.ReadMeasurer{}).NewReader(reader)
}
} else if !benchmark {
r = t.networkReaderContext.NewReader(reader)
}
err = readOne(t.objectsDir, hash, length, r)
reader.Close()
if err != nil {
t.logger.Println(err)
return err
}
totalLength += length
}
duration := time.Since(timeStart)
speed := uint64(float64(totalLength) / duration.Seconds())
if benchmark {
file, err := os.Create(t.netbenchFilename)
if err == nil {
fmt.Fprintf(file, "%d\n", speed)
file.Close()
}
t.networkReaderContext.InitialiseMaximumSpeed(speed)
}
t.logger.Printf("Fetch() complete. Read: %s in %s (%s/s)\n",
format.FormatBytes(totalLength), format.Duration(duration),
format.FormatBytes(speed))
return nil
}
func pushImage(getSubClient getSubClientFunc, imageName string) error {
logger := log.New(os.Stderr, "", log.LstdFlags)
computedInodes := make(map[string]*filesystem.RegularInode)
// Start querying the imageserver for the image.
imageServerAddress := fmt.Sprintf("%s:%d",
*imageServerHostname, *imageServerPortNum)
imgChannel := getImageChannel(imageServerAddress, imageName, timeoutTime)
startTime := showStart("getSubClient()")
srpcClient := getSubClient()
showTimeTaken(startTime)
subObj := lib.Sub{
Hostname: *subHostname,
Client: srpcClient,
ComputedInodes: computedInodes}
if *computedFilesRoot == "" {
subObj.ObjectGetter = nullObjectGetterType{}
} else {
fs, err := scanner.ScanFileSystem(*computedFilesRoot, nil, nil, nil,
nil, nil)
if err != nil {
return err
}
subObj.ObjectGetter = fs
for filename, inum := range fs.FilenameToInodeTable() {
if inode, ok := fs.InodeTable[inum].(*filesystem.RegularInode); ok {
computedInodes[filename] = inode
}
}
}
startTime = showStart("<-imgChannel")
imageResult := <-imgChannel
showTimeTaken(startTime)
fmt.Fprintf(os.Stderr, "Background image fetch took %s\n",
format.Duration(imageResult.duration))
img := imageResult.image
var err error
if *filterFile != "" {
img.Filter, err = filter.Load(*filterFile)
if err != nil {
return err
}
}
if *triggersFile != "" {
img.Triggers, err = triggers.Load(*triggersFile)
if err != nil {
return err
}
} else if *triggersString != "" {
img.Triggers, err = triggers.Decode([]byte(*triggersString))
if err != nil {
return err
}
}
if err := pollFetchAndPush(&subObj, img, imageServerAddress, timeoutTime,
logger); err != nil {
return err
}
var updateRequest sub.UpdateRequest
var updateReply sub.UpdateResponse
startTime = showStart("lib.BuildUpdateRequest()")
if lib.BuildUpdateRequest(subObj, img, &updateRequest, true, logger) {
showBlankLine()
return errors.New("missing computed file(s)")
}
showTimeTaken(startTime)
updateRequest.ImageName = imageName
updateRequest.Wait = true
startTime = showStart("Subd.Update()")
err = client.CallUpdate(srpcClient, updateRequest, &updateReply)
if err != nil {
showBlankLine()
return err
}
showTimeTaken(startTime)
return nil
}