func writeHeader(writer io.Writer) { fmt.Fprintf(writer, "Start time: %s<br>\n", startTime.Format(timeFormat)) uptime := time.Since(startTime) + time.Millisecond*50 uptime = (uptime / time.Millisecond / 100) * time.Millisecond * 100 fmt.Fprintf(writer, "Uptime: %s<br>\n", uptime) var rusage syscall.Rusage syscall.Getrusage(syscall.RUSAGE_SELF, &rusage) userCpuTime := time.Duration(rusage.Utime.Sec)*time.Second + time.Duration(rusage.Utime.Usec)*time.Microsecond sysCpuTime := time.Duration(rusage.Stime.Sec)*time.Second + time.Duration(rusage.Stime.Usec)*time.Microsecond cpuTime := rusage.Utime.Sec + rusage.Stime.Sec fmt.Fprintf(writer, "CPU Time: %.1f%% (User: %s Sys: %s)<br>\n", float64(cpuTime*100)/float64(uptime.Seconds()), userCpuTime, sysCpuTime) var memStatsBeforeGC, memStatsAfterGC runtime.MemStats runtime.ReadMemStats(&memStatsBeforeGC) runtime.GC() runtime.ReadMemStats(&memStatsAfterGC) fmt.Fprintf(writer, "Allocated memory: %s (%s after GC)<br>\n", format.FormatBytes(memStatsBeforeGC.Alloc), format.FormatBytes(memStatsAfterGC.Alloc)) fmt.Fprintf(writer, "System memory: %s (%s after GC)<br>\n", format.FormatBytes(memStatsBeforeGC.Sys), format.FormatBytes(memStatsAfterGC.Sys)) fmt.Fprintln(writer, "Raw <a href=\"metrics\">metrics</a>") }
func (t *rpcType) doFetch(request sub.FetchRequest) { defer t.clearFetchInProgress() objectServer := objectclient.NewObjectClient(request.ServerAddress) benchmark := false if t.networkReaderContext.MaximumSpeed() < 1 { benchmark = enoughBytesForBenchmark(objectServer, request) if benchmark { objectServer.SetExclusiveGetObjects(true) t.logger.Println("Benchmarking network speed") } } objectsReader, err := objectServer.GetObjects(request.Hashes) if err != nil { t.logger.Printf("Error getting object reader:\t%s\n", err.Error()) if *exitOnFetchFailure { os.Exit(1) } return } defer objectsReader.Close() var totalLength uint64 timeStart := time.Now() for _, hash := range request.Hashes { length, reader, err := objectsReader.NextObject() if err != nil { t.logger.Println(err) if *exitOnFetchFailure { os.Exit(1) } return } err = readOne(t.objectsDir, hash, length, t.networkReaderContext.NewReader(reader)) reader.Close() if err != nil { t.logger.Println(err) if *exitOnFetchFailure { os.Exit(1) } return } totalLength += length } duration := time.Since(timeStart) speed := uint64(float64(totalLength) / duration.Seconds()) if benchmark { file, err := os.Create(t.netbenchFilename) if err == nil { fmt.Fprintf(file, "%d\n", speed) file.Close() } t.networkReaderContext.InitialiseMaximumSpeed(speed) } t.logger.Printf("Fetch() complete. Read: %s in %s (%s/s)\n", format.FormatBytes(totalLength), duration, format.FormatBytes(speed)) t.rescanObjectCacheChannel <- true }
func (configuration *Configuration) writeHtml(writer io.Writer) { speed := "unknown" ctx := configuration.NetworkReaderContext if ctx.MaximumSpeed() > 0 { speed = fmt.Sprintf("%s/s (%d%% of %s/s)", format.FormatBytes( ctx.MaximumSpeed()*uint64(ctx.SpeedPercent())/100), ctx.SpeedPercent(), format.FormatBytes(ctx.MaximumSpeed())) } fmt.Fprintf(writer, "Network Speed: %s<br>\n", speed) }
func (imageObjectServers *imageObjectServersType) WriteHtml(writer io.Writer) { // TODO(rgooch): These statistics should be cached and the cache invalidated // when images and objects are added/deleted. var rusageStart, rusageStop syscall.Rusage syscall.Getrusage(syscall.RUSAGE_SELF, &rusageStart) objectsMap := imageObjectServers.objSrv.ListObjectSizes() var totalBytes uint64 for _, bytes := range objectsMap { totalBytes += bytes } numObjects := len(objectsMap) fmt.Fprintf(writer, "Number of objects: %d, consumimg %s<br>\n", numObjects, format.FormatBytes(totalBytes)) for _, imageName := range imageObjectServers.imdb.ListImages() { image := imageObjectServers.imdb.GetImage(imageName) if image == nil { continue } for _, inode := range image.FileSystem.InodeTable { if inode, ok := inode.(*filesystem.RegularInode); ok { delete(objectsMap, inode.Hash) } } } var unreferencedBytes uint64 for _, bytes := range objectsMap { unreferencedBytes += bytes } unreferencedObjectsPercent := 0.0 if numObjects > 0 { unreferencedObjectsPercent = 100.0 * float64(len(objectsMap)) / float64(numObjects) } unreferencedBytesPercent := 0.0 if totalBytes > 0 { unreferencedBytesPercent = 100.0 * float64(unreferencedBytes) / float64(totalBytes) } syscall.Getrusage(syscall.RUSAGE_SELF, &rusageStop) statisticsComputeCpuTimeDistribution.Add(time.Duration( rusageStop.Utime.Sec)*time.Second + time.Duration(rusageStop.Utime.Usec)*time.Microsecond - time.Duration(rusageStart.Utime.Sec)*time.Second - time.Duration(rusageStart.Utime.Usec)*time.Microsecond) fmt.Fprintf(writer, "Number of unreferenced objects: %d (%.1f%%), "+ "consuming %s (%.1f%%)<br>\n", len(objectsMap), unreferencedObjectsPercent, format.FormatBytes(unreferencedBytes), unreferencedBytesPercent) }
func (ctx *ReaderContext) format() string { var blocksString string if ctx.maxBlocksPerSecond > 0 { blocksString = fmt.Sprintf("(%d blocks/s)", ctx.maxBlocksPerSecond) } else { blocksString = "" } speedPercent := uint64(ctx.GetContext().SpeedPercent()) return fmt.Sprintf("max speed=%s/s%s limit=%d%% %s/s(%d blocks/s)", format.FormatBytes(ctx.maxBytesPerSecond), blocksString, speedPercent, format.FormatBytes(ctx.maxBytesPerSecond*speedPercent/100), ctx.maxBlocksPerSecond*speedPercent/100) }
func (t *srpcType) DeleteUnreferencedObjects(conn *srpc.Conn, request imageserver.DeleteUnreferencedObjectsRequest, reply *imageserver.DeleteUnreferencedObjectsResponse) error { username := conn.Username() if username == "" { t.logger.Printf("DeleteUnreferencedObjects(%d%%, %s)\n", request.Percentage, format.FormatBytes(request.Bytes)) } else { t.logger.Printf("DeleteUnreferencedObjects(%d%%, %s) by %s\n", request.Percentage, format.FormatBytes(request.Bytes), username) } return t.imageDataBase.DeleteUnreferencedObjects(request.Percentage, request.Bytes) }
func (fs *FileSystem) String() string { return fmt.Sprintf("Tree: %d inodes, total file size: %s, number of regular inodes: %d\nObjectCache: %d objects\n", len(fs.InodeTable), format.FormatBytes(fs.TotalDataBytes), fs.NumRegularInodes, len(fs.ObjectCache)) }
func showImage(writer io.Writer, name string, image *image.Image) { fmt.Fprintf(writer, " <tr>\n") fmt.Fprintf(writer, " <td><a href=\"showImage?%s\">%s</a></td>\n", name, name) fmt.Fprintf(writer, " <td><a href=\"listImage?%s\">%s</a></td>\n", name, format.FormatBytes(image.FileSystem.TotalDataBytes)) fmt.Fprintf(writer, " <td><a href=\"listImage?%s\">%d</a></td>\n", name, image.FileSystem.NumRegularInodes) if numInodes := image.FileSystem.NumComputedRegularInodes(); numInodes < 1 { fmt.Fprintln(writer, " <td>0</td>") } else { fmt.Fprintf(writer, " <td><a href=\"listComputedInodes?%s\">%d</a></td>\n", name, numInodes) } if image.Filter == nil { fmt.Fprintln(writer, " <td>(sparse filter)</td>") } else if len(image.Filter.FilterLines) < 1 { fmt.Fprintln(writer, " <td>0</td>") } else { fmt.Fprintf(writer, " <td><a href=\"listFilter?%s\">%d</a></td>\n", name, len(image.Filter.FilterLines)) } if image.Triggers == nil || len(image.Triggers.Triggers) < 1 { fmt.Fprintln(writer, " <td>0</td>") } else { fmt.Fprintf(writer, " <td><a href=\"listTriggers?%s\">%d</a></td>\n", name, len(image.Triggers.Triggers)) } fmt.Fprintf(writer, " </tr>\n") }
func (t *rpcType) logFetch(request sub.FetchRequest, speed uint64) { speedString := "unlimited speed" if speed > 0 { speedString = format.FormatBytes( speed*uint64(t.networkReaderContext.SpeedPercent())/100) + "/s" } t.logger.Printf("Fetch(%s) %d objects at %s\n", request.ServerAddress, len(request.Hashes), speedString) }
func writeHeader(writer io.Writer) { fmt.Fprintf(writer, "Start time: %s<br>\n", startTime) uptime := time.Since(startTime) fmt.Fprintf(writer, "Uptime: %s<br>\n", uptime) var rusage syscall.Rusage syscall.Getrusage(syscall.RUSAGE_SELF, &rusage) cpuTime := rusage.Utime.Sec + rusage.Stime.Sec fmt.Fprintf(writer, "CPU Time: %d%%<br>\n", cpuTime*100/int64(uptime.Seconds())) var memStatsBeforeGC, memStatsAfterGC runtime.MemStats runtime.ReadMemStats(&memStatsBeforeGC) runtime.GC() runtime.ReadMemStats(&memStatsAfterGC) fmt.Fprintf(writer, "Allocated memory: %s (%s after GC)<br>\n", format.FormatBytes(memStatsBeforeGC.Alloc), format.FormatBytes(memStatsAfterGC.Alloc)) fmt.Fprintf(writer, "System memory: %s (%s after GC)<br>\n", format.FormatBytes(memStatsBeforeGC.Sys), format.FormatBytes(memStatsAfterGC.Sys)) }
func showImage(writer io.Writer, name string, image *image.Image) { fmt.Fprintf(writer, " <tr>\n") fmt.Fprintf(writer, " <td>%s</td>\n", name) fmt.Fprintf(writer, " <td><a href=\"listImage?%s\">%s</a></td>\n", name, format.FormatBytes(image.FileSystem.TotalDataBytes)) fmt.Fprintf(writer, " <td><a href=\"listImage?%s\">%d</a></td>\n", name, image.FileSystem.NumRegularInodes) fmt.Fprintf(writer, " <td><a href=\"listFilter?%s\">%d</a></td>\n", name, len(image.Filter.FilterLines)) fmt.Fprintf(writer, " <td><a href=\"listTriggers?%s\">%d</a></td>\n", name, len(image.Triggers.Triggers)) fmt.Fprintf(writer, " </tr>\n") }
func (fs *fileSystem) writeHtml(writer io.Writer) { usedBytes := fs.size - fs.free usedPercent := float64(usedBytes) * 100 / float64(fs.size) fmt.Fprintf(writer, " <tr>\n") fmt.Fprintf(writer, " <td><center>%s</td>\n", fs.mountPoint) fmt.Fprintf(writer, " <td><center>%s</td>\n", format.FormatBytes(fs.size)) fmt.Fprintf(writer, " <td><center>%.1f%%</td>\n", usedPercent) fmt.Fprint(writer, " <td>") fs.writeHtmlBar(writer) fmt.Fprintln(writer, "</td>") fmt.Fprintln(writer, " </tr>") }
func (fsh *FileSystemHistory) writeHtml(writer io.Writer) { fmt.Fprintf(writer, "Scan count: %d<br>\n", fsh.scanCount) fmt.Fprintf(writer, "Generation count: %d<br>\n", fsh.generationCount) if fsh.scanCount > 0 { fmt.Fprintf(writer, "Last scan completed: %s<br>\n", fsh.timeOfLastScan) fmt.Fprintf(writer, "Duration of last scan: %s<br>\n", fsh.durationOfLastScan) fsh.fileSystem.WriteHtml(writer) tmp := format.FormatBytes(uint64(float64( fsh.fileSystem.TotalDataBytes) / fsh.durationOfLastScan.Seconds())) fmt.Fprintf(writer, "Scan rate: %s/s<br>\n", tmp) } fmt.Fprintf(writer, "Duration of current scan: %s<br>\n", time.Since(fsh.timeOfLastScan)) if fsh.generationCount > 0 { fmt.Fprintf(writer, "Last change: %s<br>\n", fsh.timeOfLastChange) } }
// Benchmark the read speed of the underlying block device for a given file. func main() { pathname := "/" if len(os.Args) == 2 { pathname = os.Args[1] } bytesPerSecond, blocksPerSecond, err := fsbench.GetReadSpeed(pathname) if err != nil { fmt.Printf("Error! %s\n", err) return } ctx := fsrateio.NewReaderContext(bytesPerSecond, blocksPerSecond, 0) fmt.Println(ctx) var file *os.File file, err = os.Open(pathname) if err != nil { fmt.Printf("Error! %s\n", err) return } rd := bufio.NewReader(ctx.NewReader(file)) buffer := make([]byte, 65536) timeStart := time.Now() tread := 0 for { n := 0 n, err = rd.Read(buffer) if n < 1 && err == io.EOF { break } if err != nil { fmt.Printf("Error! %s\n", err) return } tread += n } bytesPerSecond = uint64(float64(tread) / time.Since(timeStart).Seconds()) fmt.Printf("%s/s\n", format.FormatBytes(bytesPerSecond)) }
func writeHeader(writer io.Writer, req *http.Request, noGC bool) { fmt.Fprintf(writer, "Start time: %s<br>\n", startTime.Format(timeFormat)) uptime := time.Since(startTime) + time.Millisecond*50 uptime = (uptime / time.Millisecond / 100) * time.Millisecond * 100 fmt.Fprintf(writer, "Uptime: %s<br>\n", format.Duration(uptime)) uTime, sTime := getRusage() userCpuTime := uTime.Sub(startUtime) sysCpuTime := sTime.Sub(startStime) cpuTime := userCpuTime + sysCpuTime fmt.Fprintf(writer, "CPU Time: %.1f%% (User: %s Sys: %s)<br>\n", float64(cpuTime*100)/float64(uptime), userCpuTime, sysCpuTime) var memStatsBeforeGC runtime.MemStats runtime.ReadMemStats(&memStatsBeforeGC) if noGC { fmt.Fprintf(writer, "Allocated memory: %s<br>\n", format.FormatBytes(memStatsBeforeGC.Alloc)) fmt.Fprintf(writer, "System memory: %s<br>\n", format.FormatBytes(memStatsBeforeGC.Sys)) } else { var memStatsAfterGC runtime.MemStats runtime.GC() runtime.ReadMemStats(&memStatsAfterGC) fmt.Fprintf(writer, "Allocated memory: %s (%s after GC)<br>\n", format.FormatBytes(memStatsBeforeGC.Alloc), format.FormatBytes(memStatsAfterGC.Alloc)) fmt.Fprintf(writer, "System memory: %s (%s after GC)<br>\n", format.FormatBytes(memStatsBeforeGC.Sys), format.FormatBytes(memStatsAfterGC.Sys)) } fmt.Fprintln(writer, "Raw <a href=\"metrics\">metrics</a><br>") if req != nil { protocol := "http" if req.TLS != nil { protocol = "https" } host := strings.Split(req.Host, ":")[0] fmt.Fprintf(writer, "Local <a href=\"%s://%s:6910/\">system health agent</a>", protocol, host) } }
func writeNamedStat(writer io.Writer, name string, value uint64) { fmt.Fprintf(writer, " %s=%s\n", name, format.FormatBytes(value)) }
func (t *rpcType) doFetch(request sub.FetchRequest) error { defer t.clearFetchInProgress() objectServer := objectclient.NewObjectClient(request.ServerAddress) defer objectServer.Close() benchmark := false linkSpeed, haveLinkSpeed := netspeed.GetSpeedToAddress( request.ServerAddress) if haveLinkSpeed { t.logFetch(request, linkSpeed) } else { if t.networkReaderContext.MaximumSpeed() < 1 { benchmark = enoughBytesForBenchmark(objectServer, request) if benchmark { objectServer.SetExclusiveGetObjects(true) t.logger.Printf("Fetch(%s) %d objects and benchmark speed\n", request.ServerAddress, len(request.Hashes)) } else { t.logFetch(request, 0) } } else { t.logFetch(request, t.networkReaderContext.MaximumSpeed()) } } objectsReader, err := objectServer.GetObjects(request.Hashes) if err != nil { t.logger.Printf("Error getting object reader: %s\n", err.Error()) return err } defer objectsReader.Close() var totalLength uint64 defer t.rescanObjectCacheFunction() timeStart := time.Now() for _, hash := range request.Hashes { length, reader, err := objectsReader.NextObject() if err != nil { t.logger.Println(err) return err } r := io.Reader(reader) if haveLinkSpeed { if linkSpeed > 0 { r = rateio.NewReaderContext(linkSpeed, uint64(t.networkReaderContext.SpeedPercent()), &rateio.ReadMeasurer{}).NewReader(reader) } } else if !benchmark { r = t.networkReaderContext.NewReader(reader) } err = readOne(t.objectsDir, hash, length, r) reader.Close() if err != nil { t.logger.Println(err) return err } totalLength += length } duration := time.Since(timeStart) speed := uint64(float64(totalLength) / duration.Seconds()) if benchmark { file, err := os.Create(t.netbenchFilename) if err == nil { fmt.Fprintf(file, "%d\n", speed) file.Close() } t.networkReaderContext.InitialiseMaximumSpeed(speed) } t.logger.Printf("Fetch() complete. Read: %s in %s (%s/s)\n", format.FormatBytes(totalLength), format.Duration(duration), format.FormatBytes(speed)) return nil }
func (fs *FileSystem) writeHtml(writer io.Writer) { fmt.Fprintf(writer, "Scanned: <a href=\"dumpFileSystem\">%s</a><br>\n", format.FormatBytes(fs.TotalDataBytes)) }
func main() { flag.Parse() var err error bytesPerSecond, blocksPerSecond, err := fsbench.GetReadSpeed(*rootDir) if err != nil { fmt.Printf("Error! %s\n", err) return } var configuration scanner.Configuration configuration.ScanFilter, err = filter.NewFilter(nil) if err != nil { fmt.Printf("Unable to create empty filter\t%s\n", err) os.Exit(1) } configuration.FsScanContext = fsrateio.NewReaderContext(bytesPerSecond, blocksPerSecond, 0) if *scanSpeed != 0 { configuration.FsScanContext.GetContext().SetSpeedPercent(*scanSpeed) } fmt.Println(configuration.FsScanContext) syscall.Setpriority(syscall.PRIO_PROCESS, 0, 10) var prev_fs *scanner.FileSystem for iter := 0; *numScans < 0 || iter < *numScans; iter++ { timeStart := time.Now() fs, err := scanner.ScanFileSystem(*rootDir, *objectCache, &configuration) timeStop := time.Now() if iter > 0 { fmt.Println() } if err != nil { fmt.Printf("Error! %s\n", err) return } fmt.Print(fs) fmt.Printf("Total scanned: %s,\t", format.FormatBytes(fs.TotalDataBytes)) bytesPerSecond := uint64(float64(fs.TotalDataBytes) / timeStop.Sub(timeStart).Seconds()) fmt.Printf("%s/s\n", format.FormatBytes(bytesPerSecond)) if prev_fs != nil { if !scanner.CompareFileSystems(prev_fs, fs, os.Stdout) { fmt.Println("Scan results different from last run") } } runtime.GC() // Clean up before showing memory statistics. memstats.WriteMemoryStats(os.Stdout) if *debugFile != "" { file, err := os.Create(*debugFile) if err != nil { fmt.Printf("Error! %s\n", err) return } w := bufio.NewWriter(file) fs.DebugWrite(w, "") w.Flush() file.Close() } if *gobFile != "" { file, err := os.Create(*gobFile) if err != nil { fmt.Printf("Error creating: %s\t%s\n", *gobFile, err) os.Exit(1) } encoder := gob.NewEncoder(file) encoderStartTime := time.Now() encoder.Encode(fs) fmt.Printf("Encoder time: %s\n", time.Since(encoderStartTime)) file.Close() } prev_fs = fs time.Sleep(time.Duration(*interval) * time.Second) } }
func (ctx *ReaderContext) format() string { return fmt.Sprintf("max speed=%s/s limit=%d%% %s/s", format.FormatBytes(ctx.maxIOPerSecond), ctx.speedPercent, format.FormatBytes(ctx.maxIOPerSecond*ctx.speedPercent/100)) }