Пример #1
0
func main() {
	flag.Parse()
	var err error
	bytesPerSecond, blocksPerSecond, err := fsbench.GetReadSpeed(*rootDir)
	if err != nil {
		fmt.Printf("Error! %s\n", err)
		return
	}
	var configuration scanner.Configuration
	configuration.ScanFilter, err = filter.NewFilter(nil)
	if err != nil {
		fmt.Printf("Unable to create empty filter\t%s\n", err)
		os.Exit(1)
	}
	configuration.FsScanContext = fsrateio.NewReaderContext(bytesPerSecond,
		blocksPerSecond, 0)
	if *scanSpeed != 0 {
		configuration.FsScanContext.GetContext().SetSpeedPercent(*scanSpeed)
	}
	fmt.Println(configuration.FsScanContext)
	syscall.Setpriority(syscall.PRIO_PROCESS, 0, 10)
	var prev_fs *scanner.FileSystem
	for iter := 0; *numScans < 0 || iter < *numScans; iter++ {
		timeStart := time.Now()
		fs, err := scanner.ScanFileSystem(*rootDir, *objectCache,
			&configuration)
		timeStop := time.Now()
		if iter > 0 {
			fmt.Println()
		}
		if err != nil {
			fmt.Printf("Error! %s\n", err)
			return
		}
		fmt.Print(fs)
		fmt.Printf("Total scanned: %s,\t",
			format.FormatBytes(fs.TotalDataBytes))
		bytesPerSecond := uint64(float64(fs.TotalDataBytes) /
			timeStop.Sub(timeStart).Seconds())
		fmt.Printf("%s/s\n", format.FormatBytes(bytesPerSecond))
		if prev_fs != nil {
			if !scanner.CompareFileSystems(prev_fs, fs, os.Stdout) {
				fmt.Println("Scan results different from last run")
			}
		}
		runtime.GC() // Clean up before showing memory statistics.
		memstats.WriteMemoryStats(os.Stdout)
		if *debugFile != "" {
			file, err := os.Create(*debugFile)
			if err != nil {
				fmt.Printf("Error! %s\n", err)
				return
			}
			w := bufio.NewWriter(file)
			fs.DebugWrite(w, "")
			w.Flush()
			file.Close()
		}
		if *gobFile != "" {
			file, err := os.Create(*gobFile)
			if err != nil {
				fmt.Printf("Error creating: %s\t%s\n", *gobFile, err)
				os.Exit(1)
			}
			encoder := gob.NewEncoder(file)
			encoderStartTime := time.Now()
			encoder.Encode(fs)
			fmt.Printf("Encoder time: %s\n", time.Since(encoderStartTime))
			file.Close()
		}
		prev_fs = fs
		time.Sleep(time.Duration(*interval) * time.Second)
	}
}
Пример #2
0
func main() {
	flag.Parse()
	workingRootDir := path.Join(*subdDir, "root")
	objectsDir := path.Join(*subdDir, "objects")
	tmpDir := path.Join(*subdDir, "tmp")
	netbenchFilename := path.Join(*subdDir, "netbench")
	oldTriggersFilename := path.Join(*subdDir, "triggers.previous")
	if !createDirectory(workingRootDir) {
		os.Exit(1)
	}
	if !sanityCheck() {
		os.Exit(1)
	}
	if !createDirectory(objectsDir) {
		os.Exit(1)
	}
	if !createDirectory(tmpDir) {
		os.Exit(1)
	}
	if !mountTmpfs(tmpDir) {
		os.Exit(1)
	}
	if !unshareAndBind(workingRootDir) {
		os.Exit(1)
	}
	runtime.GOMAXPROCS(int(*maxThreads))
	bytesPerSecond, blocksPerSecond, firstScan, ok := getCachedFsSpeed(
		workingRootDir, tmpDir)
	if !ok {
		os.Exit(1)
	}
	circularBuffer := logbuf.New(*logbufLines)
	logger := log.New(circularBuffer, "", log.LstdFlags)
	var configuration scanner.Configuration
	var err error
	configuration.ScanFilter, err = filter.NewFilter(constants.ScanExcludeList)
	if err != nil {
		fmt.Printf("Unable to set default scan exclusions\t%s\n", err)
		os.Exit(1)
	}
	configuration.FsScanContext = fsrateio.NewReaderContext(bytesPerSecond,
		blocksPerSecond, 0)
	defaultSpeed := configuration.FsScanContext.GetContext().SpeedPercent()
	if firstScan {
		configuration.FsScanContext.GetContext().SetSpeedPercent(100)
	}
	if *showStats {
		fmt.Println(configuration.FsScanContext)
	}
	var fsh scanner.FileSystemHistory
	fsChannel := scanner.StartScannerDaemon(workingRootDir, objectsDir,
		&configuration, logger)
	networkReaderContext := rateio.NewReaderContext(
		getCachedNetworkSpeed(netbenchFilename),
		constants.DefaultNetworkSpeedPercent, &rateio.ReadMeasurer{})
	configuration.NetworkReaderContext = networkReaderContext
	rescanObjectCacheChannel := rpcd.Setup(&configuration, &fsh, objectsDir,
		networkReaderContext, netbenchFilename, oldTriggersFilename, logger)
	httpd.AddHtmlWriter(&fsh)
	httpd.AddHtmlWriter(&configuration)
	httpd.AddHtmlWriter(circularBuffer)
	html.RegisterHtmlWriterForPattern("/dumpFileSystem", "Scanned File System",
		&DumpableFileSystemHistory{&fsh})
	err = httpd.StartServer(*portNum)
	if err != nil {
		fmt.Printf("Unable to create http server\t%s\n", err)
		os.Exit(1)
	}
	fsh.Update(nil)
	invalidateNextScanObjectCache := false
	sighupChannel := make(chan os.Signal)
	signal.Notify(sighupChannel, syscall.SIGHUP)
	sigtermChannel := make(chan os.Signal)
	signal.Notify(sigtermChannel, syscall.SIGTERM, syscall.SIGINT)
	writePidfile()
	for iter := 0; true; {
		select {
		case <-sighupChannel:
			err = syscall.Exec(os.Args[0], os.Args, os.Environ())
			if err != nil {
				logger.Printf("Unable to Exec:%s\t%s\n", os.Args[0], err)
			}
		case <-sigtermChannel:
			gracefulCleanup()
		case fs := <-fsChannel:
			if *showStats {
				fmt.Printf("Completed cycle: %d\n", iter)
			}
			if invalidateNextScanObjectCache {
				fs.ScanObjectCache()
				invalidateNextScanObjectCache = false
			}
			fsh.Update(fs)
			iter++
			runtime.GC() // An opportune time to take out the garbage.
			if *showStats {
				fmt.Print(fsh)
				fmt.Print(fsh.FileSystem())
				memstats.WriteMemoryStats(os.Stdout)
				fmt.Println()
			}
			if firstScan {
				configuration.FsScanContext.GetContext().SetSpeedPercent(
					defaultSpeed)
				firstScan = false
				if *showStats {
					fmt.Println(configuration.FsScanContext)
				}
			}
		case <-rescanObjectCacheChannel:
			invalidateNextScanObjectCache = true
			fsh.UpdateObjectCacheOnly()
		}
	}
}
Пример #3
0
func main() {
	flag.Parse()
	workingRootDir := path.Join(*subdDir, "root")
	objectsDir := path.Join(*subdDir, "objects")
	tmpDir := path.Join(*subdDir, "tmp")
	if !createDirectory(workingRootDir) {
		os.Exit(1)
	}
	if !sanityCheck() {
		os.Exit(1)
	}
	if !createDirectory(objectsDir) {
		os.Exit(1)
	}
	if !createDirectory(tmpDir) {
		os.Exit(1)
	}
	if !mountTmpfs(tmpDir) {
		os.Exit(1)
	}
	if !unshareAndBind(workingRootDir) {
		os.Exit(1)
	}
	bytesPerSecond, blocksPerSecond, firstScan, ok := getCachedSpeed(
		workingRootDir, tmpDir)
	if !ok {
		os.Exit(1)
	}
	ctx := fsrateio.NewContext(bytesPerSecond, blocksPerSecond)
	defaultSpeed := ctx.SpeedPercent()
	if firstScan {
		ctx.SetSpeedPercent(100)
	}
	if *showStats {
		fmt.Println(ctx)
	}
	var fsh scanner.FileSystemHistory
	fsChannel := scanner.StartScannerDaemon(workingRootDir, objectsDir, ctx)
	err := httpd.StartServer(*portNum, &fsh)
	if err != nil {
		fmt.Printf("Unable to create http server\t%s\n", err)
		os.Exit(1)
	}
	fsh.Update(nil)
	for iter := 0; true; iter++ {
		if *showStats {
			fmt.Printf("Starting cycle: %d\n", iter)
		}
		fsh.Update(<-fsChannel)
		runtime.GC() // An opportune time to take out the garbage.
		if *showStats {
			fmt.Print(fsh)
			fmt.Print(fsh.FileSystem())
			memstats.WriteMemoryStats(os.Stdout)
			fmt.Println()
		}
		if firstScan {
			ctx.SetSpeedPercent(defaultSpeed)
			firstScan = false
			if *showStats {
				fmt.Println(ctx)
			}
		}
	}
}
Пример #4
0
func main() {
	flag.Parse()
	var err error
	bytesPerSecond, blocksPerSecond, err := fsbench.GetReadSpeed(*rootDir)
	if err != nil {
		fmt.Printf("Error! %s\n", err)
		return
	}
	ctx := fsrateio.NewContext(bytesPerSecond, blocksPerSecond)
	if *scanSpeed != 0 {
		ctx.SetSpeedPercent(*scanSpeed)
	}
	fmt.Println(ctx)
	syscall.Setpriority(syscall.PRIO_PROCESS, 0, 10)
	var prev_fs *scanner.FileSystem
	sleepDuration, _ := time.ParseDuration(fmt.Sprintf("%ds", *interval))
	for iter := 0; *numScans < 0 || iter < *numScans; iter++ {
		timeStart := time.Now()
		fs, err := scanner.ScanFileSystem(*rootDir, *objectCache, ctx)
		timeStop := time.Now()
		if iter > 0 {
			fmt.Println()
		}
		if err != nil {
			fmt.Printf("Error! %s\n", err)
			return
		}
		fmt.Print(fs)
		fmt.Printf("Total scanned: %s,\t",
			fsrateio.FormatBytes(fs.TotalDataBytes))
		bytesPerSecond := uint64(float64(fs.TotalDataBytes) /
			timeStop.Sub(timeStart).Seconds())
		fmt.Printf("%s/s\n", fsrateio.FormatBytes(bytesPerSecond))
		if prev_fs != nil {
			if !scanner.Compare(prev_fs, fs, os.Stdout) {
				fmt.Println("Scan results different from last run")
			}
		}
		runtime.GC() // Clean up before showing memory statistics.
		memstats.WriteMemoryStats(os.Stdout)
		if *debugFile != "" {
			file, err := os.Create(*debugFile)
			if err != nil {
				fmt.Printf("Error! %s\n", err)
				return
			}
			w := bufio.NewWriter(file)
			fs.DebugWrite(w, "")
			w.Flush()
			file.Close()
		}
		if *rpcFile != "" {
			file, err := os.Create(*rpcFile)
			if err != nil {
				fmt.Printf("Error creating: %s\t%s\n", *rpcFile, err)
				os.Exit(1)
			}
			encoder := gob.NewEncoder(file)
			encoder.Encode(fs)
			file.Close()
		}
		prev_fs = fs
		time.Sleep(sleepDuration)
	}
}
Пример #5
0
func main() {
	flag.Parse()
	tricorder.RegisterFlags()
	subdDirPathname := path.Join(*rootDir, *subdDir)
	workingRootDir := path.Join(subdDirPathname, "root")
	objectsDir := path.Join(workingRootDir, *subdDir, "objects")
	tmpDir := path.Join(subdDirPathname, "tmp")
	netbenchFilename := path.Join(subdDirPathname, "netbench")
	oldTriggersFilename := path.Join(subdDirPathname, "triggers.previous")
	if !createDirectory(workingRootDir) {
		os.Exit(1)
	}
	if !sanityCheck() {
		os.Exit(1)
	}
	if !createDirectory(tmpDir) {
		os.Exit(1)
	}
	if !mountTmpfs(tmpDir) {
		os.Exit(1)
	}
	if !unshareAndBind(workingRootDir) {
		os.Exit(1)
	}
	if !createDirectory(objectsDir) {
		os.Exit(1)
	}
	runtime.GOMAXPROCS(int(*maxThreads))
	circularBuffer := logbuf.New()
	logger := log.New(circularBuffer, "", log.LstdFlags)
	if err := setupserver.SetupTls(); err != nil {
		logger.Println(err)
		circularBuffer.Flush()
		if !*permitInsecureMode {
			os.Exit(1)
		}
	}
	bytesPerSecond, blocksPerSecond, firstScan, ok := getCachedFsSpeed(
		workingRootDir, tmpDir)
	if !ok {
		os.Exit(1)
	}
	publishFsSpeed(bytesPerSecond, blocksPerSecond)
	var configuration scanner.Configuration
	var err error
	configuration.ScanFilter, err = filter.New(scanExcludeList)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to set default scan exclusions: %s\n",
			err)
		os.Exit(1)
	}
	configuration.FsScanContext = fsrateio.NewReaderContext(bytesPerSecond,
		blocksPerSecond, *defaultScanSpeedPercent)
	defaultSpeed := configuration.FsScanContext.GetContext().SpeedPercent()
	if firstScan {
		configuration.FsScanContext.GetContext().SetSpeedPercent(100)
	}
	if *showStats {
		fmt.Println(configuration.FsScanContext)
	}
	var fsh scanner.FileSystemHistory
	mainFunc := func(fsChannel <-chan *scanner.FileSystem,
		disableScanner func(disableScanner bool)) {
		networkReaderContext := rateio.NewReaderContext(
			getCachedNetworkSpeed(netbenchFilename),
			*defaultNetworkSpeedPercent, &rateio.ReadMeasurer{})
		configuration.NetworkReaderContext = networkReaderContext
		invalidateNextScanObjectCache := false
		rpcdHtmlWriter :=
			rpcd.Setup(&configuration, &fsh, objectsDir,
				workingRootDir, networkReaderContext, netbenchFilename,
				oldTriggersFilename, disableScanner,
				func() {
					invalidateNextScanObjectCache = true
					fsh.UpdateObjectCacheOnly()
				},
				logger)
		configMetricsDir, err := tricorder.RegisterDirectory("/config")
		if err != nil {
			fmt.Fprintf(os.Stderr,
				"Unable to create /config metrics directory: %s\n",
				err)
			os.Exit(1)
		}
		configuration.RegisterMetrics(configMetricsDir)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Unable to create config metrics: %s\n", err)
			os.Exit(1)
		}
		httpd.AddHtmlWriter(rpcdHtmlWriter)
		httpd.AddHtmlWriter(&fsh)
		httpd.AddHtmlWriter(&configuration)
		httpd.AddHtmlWriter(circularBuffer)
		html.RegisterHtmlWriterForPattern("/dumpFileSystem",
			"Scanned File System",
			&DumpableFileSystemHistory{&fsh})
		if err = httpd.StartServer(*portNum); err != nil {
			fmt.Fprintf(os.Stderr, "Unable to create http server: %s\n", err)
			os.Exit(1)
		}
		fsh.Update(nil)
		sighupChannel := make(chan os.Signal)
		signal.Notify(sighupChannel, syscall.SIGHUP)
		sigtermChannel := make(chan os.Signal)
		signal.Notify(sigtermChannel, syscall.SIGTERM, syscall.SIGINT)
		writePidfile()
		for iter := 0; true; {
			select {
			case <-sighupChannel:
				logger.Printf("Caught SIGHUP: re-execing with: %v\n", os.Args)
				circularBuffer.Flush()
				err = syscall.Exec(os.Args[0], os.Args, os.Environ())
				if err != nil {
					logger.Printf("Unable to Exec:%s: %s\n", os.Args[0], err)
				}
			case <-sigtermChannel:
				logger.Printf("Caught SIGTERM: performing graceful cleanup\n")
				circularBuffer.Flush()
				gracefulCleanup()
			case fs := <-fsChannel:
				if *showStats {
					fmt.Printf("Completed cycle: %d\n", iter)
				}
				if invalidateNextScanObjectCache {
					fs.ScanObjectCache()
					invalidateNextScanObjectCache = false
				}
				fsh.Update(fs)
				iter++
				runtime.GC() // An opportune time to take out the garbage.
				if *showStats {
					fmt.Print(fsh)
					fmt.Print(fsh.FileSystem())
					memstats.WriteMemoryStats(os.Stdout)
					fmt.Println()
				}
				if firstScan {
					configuration.FsScanContext.GetContext().SetSpeedPercent(
						defaultSpeed)
					firstScan = false
					if *showStats {
						fmt.Println(configuration.FsScanContext)
					}
				}
			}
		}
	}
	scanner.StartScanning(workingRootDir, objectsDir, &configuration, logger,
		mainFunc)
}