예제 #1
0
파일: main.go 프로젝트: balamurugana/mc
// Main starts mc application
func Main() {
	// Enable profiling supported modes are [cpu, mem, block].
	// ``MC_PROFILER`` supported options are [cpu, mem, block].
	switch os.Getenv("MC_PROFILER") {
	case "cpu":
		defer profile.Start(profile.CPUProfile, profile.ProfilePath(mustGetProfileDir())).Stop()
	case "mem":
		defer profile.Start(profile.MemProfile, profile.ProfilePath(mustGetProfileDir())).Stop()
	case "block":
		defer profile.Start(profile.BlockProfile, profile.ProfilePath(mustGetProfileDir())).Stop()
	}

	probe.Init() // Set project's root source path.
	probe.SetAppInfo("Release-Tag", ReleaseTag)
	probe.SetAppInfo("Commit", ShortCommitID)

	app := registerApp()
	app.Before = registerBefore
	app.ExtraInfo = func() map[string]string {
		if _, e := pb.GetTerminalWidth(); e != nil {
			globalQuiet = true
		}
		if globalDebug {
			return getSystemData()
		}
		return make(map[string]string)
	}
	app.RunAndExitOnError()
}
예제 #2
0
func (p PkgProfile) Start() ProfilerStart {
	if *FLAGS.PROFILE_MEM {
		PROFILE = profile.Start(profile.MemProfile, profile.ProfilePath("."))
	} else {
		PROFILE = profile.Start(profile.CPUProfile, profile.ProfilePath("."))
	}
	return PROFILE
}
예제 #3
0
func Profile(mode string) Stop {
	var stop Stop
	switch mode {
	case "mem":
		stop = profileOnExit(profile.Start(profile.MemProfile, profile.ProfilePath("."), profile.NoShutdownHook))
	case "cpu":
		stop = profileOnExit(profile.Start(profile.CPUProfile, profile.ProfilePath("."), profile.NoShutdownHook))
	case "block":
		stop = profileOnExit(profile.Start(profile.BlockProfile, profile.ProfilePath("."), profile.NoShutdownHook))
	default:
		stop = stopper{}
	}
	return stop
}
예제 #4
0
파일: utils.go 프로젝트: tixu/mmjira
func activateProfiline(profileType string, dir string) {
	// activating
	switch profileType {
	case "cpu":
		defer profile.Start(profile.ProfilePath(dir), profile.CPUProfile).Stop()
	case "mem":
		defer profile.Start(profile.ProfilePath(dir), profile.MemProfile).Stop()
	case "block":
		defer profile.Start(profile.ProfilePath(dir), profile.BlockProfile).Stop()
	default:
		// do nothing
	}

}
예제 #5
0
파일: main.go 프로젝트: Yelp/fullerite
func start(ctx *cli.Context) {
	if ctx.Bool("profile") {
		defer profile.Start(profile.CPUProfile).Stop()
		defer profile.Start(profile.MemProfile).Stop()
		defer profile.Start(profile.BlockProfile).Stop()
		defer profile.Start(profile.ProfilePath("."))
	}
	quit := make(chan bool)
	initLogrus(ctx)
	log.Info("Starting fullerite...")

	c, err := config.ReadConfig(ctx.String("config"))
	if err != nil {
		return
	}
	handlers := createHandlers(c)
	hook := NewLogErrorHook(handlers)
	log.Logger.Hooks.Add(hook)

	startHandlers(handlers)
	collectors := startCollectors(c)

	collectorStatChan := make(chan metric.CollectorEmission)

	internalServer := internalserver.New(c,
		handlerStatFunc(handlers),
		readCollectorStat(collectorStatChan))

	go internalServer.Run()

	readFromCollectors(collectors, handlers, collectorStatChan)

	<-quit
}
예제 #6
0
// Starts a profiler returns nil if profiler is not enabled, caller needs to handle this.
func startProfiler(profiler string) interface {
	Stop()
} {
	// Set ``MINIO_PROFILE_DIR`` to the directory where profiling information should be persisted
	profileDir := os.Getenv("MINIO_PROFILE_DIR")
	// Enable profiler if ``MINIO_PROFILER`` is set. Supported options are [cpu, mem, block].
	switch profiler {
	case "cpu":
		return profile.Start(profile.CPUProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir))
	case "mem":
		return profile.Start(profile.MemProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir))
	case "block":
		return profile.Start(profile.BlockProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir))
	default:
		return nil
	}
}
예제 #7
0
func main() {
	p := profile.Start(profile.MemProfile, profile.ProfilePath("."), profile.NoShutdownHook)
	// initialize representation
	index, _ = iindex.IndexDirectory("/home/rhibbitts/Dropbox/Notes/WorkNotes")
	// run  user interface
	ui()
	p.Stop()
}
예제 #8
0
func main() {
	p := profile.Start(
		profile.MemProfile,
		profile.ProfilePath("."))
	defer p.Stop()

	q := queue.NewDeque()
	fill(q)
	clear(q)
}
예제 #9
0
파일: prof.go 프로젝트: cagnosolutions/bpt
func main() {
	prof := profile.CPUProfile
	//prof := profile.MemProfile
	defer profile.Start(prof, profile.ProfilePath(".")).Stop()

	t := bptx.NewTree()
	for i := 0; i < 10000; i++ {
		x := bptx.UUID()
		t.Set(x, x)
	}
	t.Close()
}
예제 #10
0
파일: prof.go 프로젝트: cagnosolutions/bpt
func main() {
	//prof := profile.CPUProfile
	prof := profile.MemProfile
	defer profile.Start(prof, profile.ProfilePath(".")).Stop()

	t := bpt.NewTree()
	for i := 0; i < 1000000; i++ {
		x := []byte(fmt.Sprintf("%x", i))
		t.Set(x, x)
	}
	t.Close()
}
예제 #11
0
파일: main.go 프로젝트: samlecuyer/ecumene
func main() {
	flag.Parse()

	input := flag.Arg(0)
	output := flag.Arg(1)
	cwd, _ := os.Getwd()
	defer profile.Start(profile.ProfilePath(cwd)).Stop()

	m, _ := mapping.NewMap(input)

	r := rendering.NewRenderer(m, 5000, 5000)

	r.ClipToMap()
	r.DrawToFile(output)
}
예제 #12
0
func main() {
	p := profile.Start(
		profile.MemProfile,
		profile.ProfilePath("."),
		profile.NoShutdownHook)
	defer p.Stop()

	u := &command.Unit{"Test-01", 0, 0}
	s := command.NewCommandStack(10)
	s.Do(command.NewMoveUnitCommand(u, 10, 10))

	for i := 0; i < 10000000; i++ {
		s.Do(command.NewMoveUnitCommand(u, i*10, i*10))
	}
}
예제 #13
0
func main() {
	p := profile.Start(
		profile.MemProfile,
		profile.ProfilePath("."))
	defer p.Stop()

	q := &queue.Queue{}

	for i := 0; i < 20000000; i++ {
		n := string(i)
		q.Put(n)
	}

	for i := 0; i < 20000000; i++ {
		if q.Get() == nil {
			panic("value is nil, expected *node\n")
		}
	}
}
예제 #14
0
func main() {
	kingpin.Version(version)
	kingpin.Parse()
	logging.LogStd(fmt.Sprintf("Starting firehose-to-syslog %s ", version), true)

	logging.SetupLogging(*syslogServer, *debug)

	c := cfclient.Config{
		ApiAddress:        *apiEndpoint,
		Username:          *user,
		Password:          *password,
		SkipSslValidation: *skipSSLValidation,
	}
	cfClient := cfclient.NewClient(&c)

	if len(*dopplerEndpoint) > 0 {
		cfClient.Endpoint.DopplerEndpoint = *dopplerEndpoint
	}
	logging.LogStd(fmt.Sprintf("Using %s as doppler endpoint", cfClient.Endpoint.DopplerEndpoint), true)

	logging.LogStd("Setting up event routing!", true)
	err := events.SetupEventRouting(*wantedEvents)
	if err != nil {
		log.Fatal("Error setting up event routing: ", err)
		os.Exit(1)

	}

	//Use bolt for in-memory  - file caching
	db, err := bolt.Open(*boltDatabasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		log.Fatal("Error opening bolt db: ", err)
		os.Exit(1)

	}
	defer db.Close()

	if *modeProf != "" {
		switch *modeProf {
		case "cpu":
			defer profile.Start(profile.CPUProfile, profile.ProfilePath(*pathProf)).Stop()
		case "mem":
			defer profile.Start(profile.MemProfile, profile.ProfilePath(*pathProf)).Stop()
		case "block":
			defer profile.Start(profile.BlockProfile, profile.ProfilePath(*pathProf)).Stop()
		default:
			// do nothing
		}
	}

	caching.SetCfClient(cfClient)
	caching.SetAppDb(db)
	caching.CreateBucket()

	//Let's Update the database the first time
	logging.LogStd("Start filling app/space/org cache.", true)
	apps := caching.GetAllApp()
	logging.LogStd(fmt.Sprintf("Done filling cache! Found [%d] Apps", len(apps)), true)

	// Ticker Pooling the CC every X sec
	ccPooling := time.NewTicker(*tickerTime)

	go func() {
		for range ccPooling.C {
			apps = caching.GetAllApp()
		}
	}()

	// Parse extra fields from cmd call
	extraFields, err := extrafields.ParseExtraFields(*extraFields)
	if err != nil {
		log.Fatal("Error parsing extra fields: ", err)
		os.Exit(1)
	}

	if *logEventTotals == true {
		events.LogEventTotals(*logEventTotalsTime, *dopplerEndpoint)
	}

	if logging.Connect() || *debug {

		logging.LogStd("Connected to Syslog Server! Connecting to Firehose...", true)

		firehose := firehose.CreateFirehoseChan(cfClient.Endpoint.DopplerEndpoint, cfClient.GetToken(), *subscriptionId, *skipSSLValidation)
		if firehose != nil {
			logging.LogStd("Firehose Subscription Succesfull! Routing events...", true)
			events.RouteEvents(firehose, extraFields)
		} else {
			logging.LogError("Failed connecting to Firehose...Please check settings and try again!", "")
		}

	} else {
		logging.LogError("Failed connecting to the Syslog Server...Please check settings and try again!", "")
	}
}
예제 #15
0
파일: main.go 프로젝트: n054/weave
func main() {
	procs := runtime.NumCPU()
	// packet sniffing can block an OS thread, so we need one thread
	// for that plus at least one more.
	if procs < 2 {
		procs = 2
	}
	runtime.GOMAXPROCS(procs)

	var (
		justVersion        bool
		config             mesh.Config
		networkConfig      weave.NetworkConfig
		protocolMinVersion int
		resume             bool
		ifaceName          string
		routerName         string
		nickName           string
		password           string
		pktdebug           bool
		logLevel           string
		prof               string
		bufSzMB            int
		noDiscovery        bool
		httpAddr           string
		ipamConfig         ipamConfig
		dockerAPI          string
		peers              []string
		noDNS              bool
		dnsConfig          dnsConfig
		datapathName       string
		trustedSubnetStr   string
		dbPrefix           string
		isAWSVPC           bool

		defaultDockerHost = "unix:///var/run/docker.sock"
	)

	if val := os.Getenv("DOCKER_HOST"); val != "" {
		defaultDockerHost = val
	}

	mflag.BoolVar(&justVersion, []string{"#version", "-version"}, false, "print version and exit")
	mflag.StringVar(&config.Host, []string{"-host"}, "", "router host")
	mflag.IntVar(&config.Port, []string{"#port", "-port"}, mesh.Port, "router port")
	mflag.IntVar(&protocolMinVersion, []string{"-min-protocol-version"}, mesh.ProtocolMinVersion, "minimum weave protocol version")
	mflag.BoolVar(&resume, []string{"-resume"}, false, "resume connections to previous peers")
	mflag.StringVar(&ifaceName, []string{"#iface", "-iface"}, "", "name of interface to capture/inject from (disabled if blank)")
	mflag.StringVar(&routerName, []string{"#name", "-name"}, "", "name of router (defaults to MAC of interface)")
	mflag.StringVar(&nickName, []string{"#nickname", "-nickname"}, "", "nickname of peer (defaults to hostname)")
	mflag.StringVar(&password, []string{"#password", "-password"}, "", "network password")
	mflag.StringVar(&logLevel, []string{"-log-level"}, "info", "logging level (debug, info, warning, error)")
	mflag.BoolVar(&pktdebug, []string{"#pktdebug", "#-pktdebug", "-pkt-debug"}, false, "enable per-packet debug logging")
	mflag.StringVar(&prof, []string{"#profile", "-profile"}, "", "enable profiling and write profiles to given path")
	mflag.IntVar(&config.ConnLimit, []string{"#connlimit", "#-connlimit", "-conn-limit"}, 30, "connection limit (0 for unlimited)")
	mflag.BoolVar(&noDiscovery, []string{"#nodiscovery", "#-nodiscovery", "-no-discovery"}, false, "disable peer discovery")
	mflag.IntVar(&bufSzMB, []string{"#bufsz", "-bufsz"}, 8, "capture buffer size in MB")
	mflag.StringVar(&httpAddr, []string{"#httpaddr", "#-httpaddr", "-http-addr"}, "", "address to bind HTTP interface to (disabled if blank, absolute path indicates unix domain socket)")
	mflag.StringVar(&ipamConfig.Mode, []string{"-ipalloc-init"}, "", "allocator initialisation strategy (consensus, seed or observer)")
	mflag.StringVar(&ipamConfig.IPRangeCIDR, []string{"#iprange", "#-iprange", "-ipalloc-range"}, "", "IP address range reserved for automatic allocation, in CIDR notation")
	mflag.StringVar(&ipamConfig.IPSubnetCIDR, []string{"#ipsubnet", "#-ipsubnet", "-ipalloc-default-subnet"}, "", "subnet to allocate within by default, in CIDR notation")
	mflag.IntVar(&ipamConfig.PeerCount, []string{"#initpeercount", "#-initpeercount", "-init-peer-count"}, 0, "number of peers in network (for IP address allocation)")
	mflag.StringVar(&dockerAPI, []string{"#api", "#-api", "-docker-api"}, defaultDockerHost, "Docker API endpoint")
	mflag.BoolVar(&noDNS, []string{"-no-dns"}, false, "disable DNS server")
	mflag.StringVar(&dnsConfig.Domain, []string{"-dns-domain"}, nameserver.DefaultDomain, "local domain to server requests for")
	mflag.StringVar(&dnsConfig.ListenAddress, []string{"-dns-listen-address"}, nameserver.DefaultListenAddress, "address to listen on for DNS requests")
	mflag.IntVar(&dnsConfig.TTL, []string{"-dns-ttl"}, nameserver.DefaultTTL, "TTL for DNS request from our domain")
	mflag.DurationVar(&dnsConfig.ClientTimeout, []string{"-dns-fallback-timeout"}, nameserver.DefaultClientTimeout, "timeout for fallback DNS requests")
	mflag.StringVar(&dnsConfig.EffectiveListenAddress, []string{"-dns-effective-listen-address"}, "", "address DNS will actually be listening, after Docker port mapping")
	mflag.StringVar(&dnsConfig.ResolvConf, []string{"-resolv-conf"}, "", "path to resolver configuration for fallback DNS lookups")
	mflag.StringVar(&datapathName, []string{"-datapath"}, "", "ODP datapath name")
	mflag.StringVar(&trustedSubnetStr, []string{"-trusted-subnets"}, "", "comma-separated list of trusted subnets in CIDR notation")
	mflag.StringVar(&dbPrefix, []string{"-db-prefix"}, "/weavedb/weave", "pathname/prefix of filename to store data")
	mflag.BoolVar(&isAWSVPC, []string{"-awsvpc"}, false, "use AWS VPC for routing")

	// crude way of detecting that we probably have been started in a
	// container, with `weave launch` --> suppress misleading paths in
	// mflags error messages.
	if os.Args[0] == "/home/weave/weaver" { // matches the Dockerfile ENTRYPOINT
		os.Args[0] = "weave"
		mflag.CommandLine.Init("weave", mflag.ExitOnError)
	}

	mflag.Parse()

	peers = mflag.Args()
	if resume && len(peers) > 0 {
		Log.Fatalf("You must not specify an initial peer list in conjunction with --resume")
	}

	common.SetLogLevel(logLevel)

	if justVersion {
		fmt.Printf("weave router %s\n", version)
		os.Exit(0)
	}

	Log.Println("Command line options:", options())

	if prof != "" {
		defer profile.Start(profile.CPUProfile, profile.ProfilePath(prof), profile.NoShutdownHook).Stop()
	}

	if protocolMinVersion < mesh.ProtocolMinVersion || protocolMinVersion > mesh.ProtocolMaxVersion {
		Log.Fatalf("--min-protocol-version must be in range [%d,%d]", mesh.ProtocolMinVersion, mesh.ProtocolMaxVersion)
	}
	config.ProtocolMinVersion = byte(protocolMinVersion)

	if pktdebug {
		networkConfig.PacketLogging = packetLogging{}
	} else {
		networkConfig.PacketLogging = nopPacketLogging{}
	}

	overlay, bridge := createOverlay(datapathName, ifaceName, isAWSVPC, config.Host, config.Port, bufSzMB)
	networkConfig.Bridge = bridge

	name := peerName(routerName, bridge.Interface())

	if nickName == "" {
		var err error
		nickName, err = os.Hostname()
		checkFatal(err)
	}

	config.Password = determinePassword(password)
	config.TrustedSubnets = parseTrustedSubnets(trustedSubnetStr)
	config.PeerDiscovery = !noDiscovery

	if isAWSVPC && len(config.Password) > 0 {
		Log.Fatalf("--awsvpc mode is not compatible with the --password option")
	}

	db, err := db.NewBoltDB(dbPrefix + "data.db")
	checkFatal(err)
	defer db.Close()

	router := weave.NewNetworkRouter(config, networkConfig, name, nickName, overlay, db)
	Log.Println("Our name is", router.Ourself)

	if peers, err = router.InitialPeers(resume, peers); err != nil {
		Log.Fatal("Unable to get initial peer set: ", err)
	}

	var dockerCli *docker.Client
	dockerVersion := "none"
	if dockerAPI != "" {
		dc, err := docker.NewClient(dockerAPI)
		if err != nil {
			Log.Fatal("Unable to start docker client: ", err)
		} else {
			Log.Info(dc.Info())
		}
		dockerCli = dc
		dockerVersion = dockerCli.DockerVersion()
	}

	network := ""
	if isAWSVPC {
		network = "awsvpc"
	}
	checkForUpdates(dockerVersion, network)

	observeContainers := func(o docker.ContainerObserver) {
		if dockerCli != nil {
			if err := dockerCli.AddObserver(o); err != nil {
				Log.Fatal("Unable to start watcher", err)
			}
		}
	}
	isKnownPeer := func(name mesh.PeerName) bool {
		return router.Peers.Fetch(name) != nil
	}

	var (
		allocator     *ipam.Allocator
		defaultSubnet address.CIDR
		trackerName   string
	)
	if ipamConfig.Enabled() {
		var t tracker.LocalRangeTracker
		if isAWSVPC {
			Log.Infoln("Creating AWSVPC LocalRangeTracker")
			t, err = tracker.NewAWSVPCTracker()
			if err != nil {
				Log.Fatalf("Cannot create AWSVPC LocalRangeTracker: %s", err)
			}
			trackerName = "awsvpc"
		}
		allocator, defaultSubnet = createAllocator(router, ipamConfig, db, t, isKnownPeer)
		observeContainers(allocator)
		if dockerCli != nil {
			ids, err := dockerCli.AllContainerIDs()
			checkFatal(err)
			allocator.PruneOwned(ids)
		}
	}

	var (
		ns        *nameserver.Nameserver
		dnsserver *nameserver.DNSServer
	)
	if !noDNS {
		ns, dnsserver = createDNSServer(dnsConfig, router.Router, isKnownPeer)
		observeContainers(ns)
		ns.Start()
		defer ns.Stop()
		dnsserver.ActivateAndServe()
		defer dnsserver.Stop()
	}

	router.Start()
	if errors := router.InitiateConnections(peers, false); len(errors) > 0 {
		Log.Fatal(common.ErrorMessages(errors))
	}

	// The weave script always waits for a status call to succeed,
	// so there is no point in doing "weave launch --http-addr ''".
	// This is here to support stand-alone use of weaver.
	if httpAddr != "" {
		muxRouter := mux.NewRouter()
		if allocator != nil {
			allocator.HandleHTTP(muxRouter, defaultSubnet, trackerName, dockerCli)
		}
		if ns != nil {
			ns.HandleHTTP(muxRouter, dockerCli)
		}
		router.HandleHTTP(muxRouter)
		HandleHTTP(muxRouter, version, router, allocator, defaultSubnet, ns, dnsserver)
		http.Handle("/", common.LoggingHTTPHandler(muxRouter))
		Log.Println("Listening for HTTP control messages on", httpAddr)
		go listenAndServeHTTP(httpAddr)
	}

	common.SignalHandlerLoop(router)
}
예제 #16
0
파일: imagec.go 프로젝트: kjplatz/vic
func main() {
	defer func() {
		if r := recover(); r != nil {
			fmt.Fprintf(os.Stderr, string(sf.FormatError(fmt.Errorf("%s : %s", r, debug.Stack()))))
		}
	}()

	if version.Show() {
		fmt.Fprintf(os.Stdout, "%s\n", version.String())
		return
	}

	// Enable profiling if mode is set
	switch options.profiling {
	case "cpu":
		defer profile.Start(profile.CPUProfile, profile.ProfilePath("."), profile.Quiet).Stop()
	case "mem":
		defer profile.Start(profile.MemProfile, profile.ProfilePath("."), profile.Quiet).Stop()
	case "block":
		defer profile.Start(profile.BlockProfile, profile.ProfilePath("."), profile.Quiet).Stop()
	default:
		// do nothing
	}

	// Register our custom Error hook
	log.AddHook(NewErrorHook(os.Stderr))

	// Enable runtime tracing if tracing is true
	if options.tracing {
		tracing, err := os.Create(time.Now().Format("2006-01-02T150405.pprof"))
		if err != nil {
			log.Fatalf("Failed to create tracing logfile: %s", err)
		}
		defer tracing.Close()

		if err := trace.Start(tracing); err != nil {
			log.Fatalf("Failed to start tracing: %s", err)
		}
		defer trace.Stop()
	}

	// Open the log file
	f, err := os.OpenFile(options.logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatalf("Failed to open the logfile %s: %s", options.logfile, err)
	}
	defer f.Close()

	// Initiliaze logger with default TextFormatter
	log.SetFormatter(&log.TextFormatter{DisableColors: true, FullTimestamp: true})

	// Set the log level
	if options.debug {
		log.SetLevel(log.DebugLevel)
	}

	// SetOutput to log file and/or stdout
	log.SetOutput(f)
	if options.stdout {
		log.SetOutput(io.MultiWriter(os.Stdout, f))
	}

	// Parse the -reference parameter
	if err = ParseReference(); err != nil {
		log.Fatalf(err.Error())
	}

	// Host is either the host's UUID (if run on vsphere) or the hostname of
	// the system (if run standalone)
	host, err := sys.UUID()
	if host != "" {
		log.Infof("Using UUID (%s) for imagestore name", host)
	} else if options.standalone {
		host, err = os.Hostname()
		log.Infof("Using host (%s) for imagestore name", host)
	}

	if err != nil {
		log.Fatalf("Failed to return the UUID or host name: %s", err)
	}

	if !options.standalone {
		log.Debugf("Running with portlayer")

		// Ping the server to ensure it's at least running
		ok, err := PingPortLayer()
		if err != nil || !ok {
			log.Fatalf("Failed to ping portlayer: %s", err)
		}
	} else {
		log.Debugf("Running standalone")
	}

	// Calculate (and overwrite) the registry URL and make sure that it responds to requests
	options.registry, err = LearnRegistryURL(options)
	if err != nil {
		log.Fatalf("Error while pulling image: %s", err)
	}

	// Get the URL of the OAuth endpoint
	url, err := LearnAuthURL(options)
	if err != nil {
		log.Fatalf("Failed to obtain OAuth endpoint: %s", err)
	}

	// Get the OAuth token - if only we have a URL
	if url != nil {
		token, err := FetchToken(url)
		if err != nil {
			log.Fatalf("Failed to fetch OAuth token: %s", err)
		}
		options.token = token
	}

	// HACK: Required to learn the name of the vmdk from given reference
	// Used by docker personality until metadata support lands
	if !options.resolv {
		progress.Message(po, "", "Pulling from "+options.image)
	}

	// Get the manifest
	manifest, err := FetchImageManifest(options)
	if err != nil {
		if strings.Contains(err.Error(), "image not found") {
			log.Fatalf("Error: image %s not found", options.image)
		} else {
			log.Fatalf("Error while pulling image manifest: %s", err)
		}
	}

	// Create the ImageWithMeta slice to hold Image structs
	images, imageLayer, err := ImagesToDownload(manifest, host)
	if err != nil {
		log.Fatalf(err.Error())
	}

	// HACK: Required to learn the name of the vmdk from given reference
	// Used by docker personality until metadata support lands
	if options.resolv {
		if len(images) > 0 {
			fmt.Printf("%s", images[0].meta)
			os.Exit(0)
		}
		os.Exit(1)
	}

	// Fetch the blobs from registry
	if err := DownloadImageBlobs(images); err != nil {
		log.Fatalf(err.Error())
	}

	if err := CreateImageConfig(images, manifest); err != nil {
		log.Fatalf(err.Error())
	}

	// Write blobs to the storage layer
	if err := WriteImageBlobs(images); err != nil {
		log.Fatalf(err.Error())
	}

	if err := updateImageMetadata(imageLayer, manifest); err != nil {
		log.Fatalf(err.Error())
	}

	progress.Message(po, "", "Digest: "+manifest.Digest)
	if len(images) > 0 {
		progress.Message(po, "", "Status: Downloaded newer image for "+options.image+":"+options.tag)
	} else {
		progress.Message(po, "", "Status: Image is up to date for "+options.image+":"+options.tag)
	}
}
예제 #17
0
func main() {
	players, profilingOn, numWorkers := parseCommandLine()
	startTime := time.Now()
	if len(players) == 0 {
		panic("Could not find players")
	}

	// Start profiler
	if profilingOn {
		newLog.Info("Running profiler")
		defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop()
	}

	// Create random Parent solutions to start
	parentSolutions := make([]Solution, numParents)
	for i, _ := range parentSolutions {
		ourPlayers := make([]Player, len(players))
		copy(ourPlayers, players)
		randomizeTeams(ourPlayers)
		solutionScore, _ := ScoreSolution(ourPlayers)
		parentSolutions[i] = Solution{ourPlayers, solutionScore}
	}

	// Use the random starting solutions to determine the worst case for each of
	// our criteria
	PopulateWorstCases(parentSolutions)

	// Start our worker goroutines
	tasks := make(chan workerTask, numSolutionsPerRun)
	results := make(chan Solution, numSolutionsPerRun)
	for i := 0; i < numWorkers; i++ {
		go worker(tasks, results)
	}
	defer close(tasks)

	// Allow user to signal exit
	doneSignal := make(chan os.Signal, 1)
	signal.Notify(doneSignal, syscall.SIGINT)

	topScore := parentSolutions[0].score
	numRunsCompleted := 0
	topScoreRunNumber := 0
	for {
		// If we have a new best score, save and print it!
		if topScore != parentSolutions[0].score {
			topScore = parentSolutions[0].score
			topScoreRunNumber = numRunsCompleted
			if newLog.IsEnabledFor(logging.DEBUG) && numRunsCompleted > 20 {
				newLog.Info("\nNew top score! Run number %d. Score: %.02f",
					numRunsCompleted, topScore)
				PrintTeams(parentSolutions[0])
				PrintSolutionScoring(parentSolutions[0])
			}
		}

		// Create new solutions, and save the best ones
		newSolutions := performRun(parentSolutions, tasks, results)
		sort.Sort(ByScore(newSolutions))
		for i, _ := range parentSolutions {
			parentSolutions[i] = newSolutions[i]
		}

		numRunsCompleted += 1
		if timeToClose(numRunsCompleted, topScoreRunNumber, doneSignal) {
			break
		}
	}

	// Display our solution to the user
	topSolution := parentSolutions[0]
	fmt.Printf("Exiting after %d runs. Top score was found on run #%d\n",
		numRunsCompleted, topScoreRunNumber)
	PrintTeams(topSolution)
	PrintSolutionScoring(topSolution)
	newLog.Debug("Program runtime: %.02fs", time.Since(startTime).Seconds())
}
예제 #18
0
func ExampleProfilePath() {
	// set the location that the profile will be written to
	defer profile.Start(profile.ProfilePath(os.Getenv("HOME")))
}
예제 #19
0
파일: main.go 프로젝트: fanyang01/crawler
func main() {
	defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop()

	logger := log15.Root()
	logger.SetHandler(log15.MultiHandler(
		log15.LvlFilterHandler(
			log15.LvlInfo,
			log15.Must.FileHandler(filepath.Join(dir, "crawler.log"), log15.LogfmtFormat()),
		),
		log15.LvlFilterHandler(log15.LvlError, log15.StdoutHandler),
	))

	csv, err := os.Open(seedfile)
	if err != nil {
		log.Fatal(err)
	}
	var urls []string
	scanner := bufio.NewScanner(csv)
	for scanner.Scan() {
		url := scanner.Text()
		if !strings.HasPrefix(url, "#") {
			urls = append(urls, url)
		}
	}
	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}
	csv.Close()

	pattern := &extract.Pattern{
		File: []string{
			"", "*.?htm?", `/[^\.]*/`,
			`/.*\.(jpg|JPG|png|PNG|jpeg|JPEG|gif|GIF)/`,
			`/.*\.(php|jsp|aspx|asp|cgi|do)/`,
			"*.css", "*.js",
		},
		// ExcludeFile: []string{
		// 	"*.doc?", "*.xls?", "*.ppt?",
		// 	"*.pdf", "*.rar", "*.zip",
		// 	"*.ico", "*.apk", "*.exe",
		// 	"*.mp4", "*.mkv",
		// },
	}
	ctrl = &Controller{
		extractor: &extract.Extractor{
			Matcher:  extract.MustCompile(pattern),
			MaxDepth: 4,
			Pos: []struct{ Tag, Attr string }{
				{"a", "href"},
				{"img", "src"},
				{"link", "href"},
				{"script", "src"},
			},
			SniffFlags: extract.SniffWindowLocation,
			Redirect:   true,
			SpanHosts:  true,
			SubDomain:  true,
			ResolveIP:  true,
		},
		downloader: &download.Downloader{
			Dir: dir,
		},
		trie:        urltrie.NewHosts(threshold),
		count:       count.NewHosts(),
		fingerprint: fingerprint.NewStore(0, 4, 4096),
		limiter:     ratelimit.New(rate),
		logger:      logger.New("worker", "controller"),
	}
	ctrl.complete.hosts = make(map[string]bool)

	store, err := boltstore.New(filepath.Join(dir, "bolt.db"), nil, nil)
	if err != nil {
		log.Fatal(err)
	}
	// queue, err := diskqueue.NewDefault(store.DB)
	queue := ratelimitq.NewWaitQueue(&ratelimitq.Option{
		Limit:     ctrl.Interval,
		Secondary: diskheap.New(store.DB, []byte("HEAP"), 16),
	})

	go func() {
		http.Handle("/count/", http.HandlerFunc(handleCount))
		log.Fatal(http.ListenAndServe("localhost:7869", nil))
	}()

	cw := crawler.New(&crawler.Config{
		Controller: ctrl,
		Logger:     logger,
		Store:      store,
		Queue:      queue,
	})
	if err := cw.Crawl(urls[offset-1 : offset-1+nseed]...); err != nil {
		log.Fatal(err)
	}
	cw.Wait()
}