Exemple #1
0
func loadCsvFile(name, urlpath string, timestamp time.Time, force bool) (*csv.Reader, error) {
	filename := filepath.Join(csvDir, name+timestamp.Format(".2006-01-02")+".csv")
	if !force {
		file, err := os.Open(filename)
		if err != nil {
			if !os.IsNotExist(err) {
				return nil, err
			}
		} else {
			return csv.NewReader(file), nil
		}
	}
	log.Info("Downloading %s.csv from Google Spreadsheets", name)
	resp, err := http.Get(csvUrlPrefix + urlpath + csvUrlSuffix)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	err = ioutil.WriteFile(filename, body, 0644)
	if err != nil {
		return nil, err
	}
	return csv.NewReader(bytes.NewBuffer(body)), nil
}
Exemple #2
0
func renderMarkdown(input []byte) (out []byte, err error) {
	defer func() {
		if e := recover(); e != nil {
			if e, ok := e.(error); ok {
				log.Info("hilite error: restarting, due to: %s", e)
				setupPygments()
				err = e
			}
		}
	}()
	r := &Html{blackfriday.HtmlRenderer(DefaultHtmlFlags, "", "")}
	out = blackfriday.Markdown(input, r, DefaultExtensions)
	return
}
Exemple #3
0
func parseCsv(force bool) {
	now := time.Now()
	file := getCsvFile("devices", devicesUrlKey, now, force)
	if file == nil {
		return
	}
	var (
		line []string
		err  error
	)
	log.Info("Parsing devices.csv")
	for {
		line, err = file.Read()
		if err != nil {
			if err == io.EOF {
				break
			}
			log.Error("Error parsing devices.csv: %s", err)
			return
		}
		_ = line
	}
	// file = getCsvFile("members", membersUrlKey, now, force)
	// if file == nil {
	// 	return
	// }
	// log.Info("Parsing devices.csv")
	// for {
	// 	line, err = file.Read()
	// 	if err != nil {
	// 		if err == io.EOF {
	// 			break
	// 		}
	// 		log.Error("Error parsing devices.csv: %s", err)
	// 		return
	// 	}
	// 	_ = line
	// }
	// file = getCsvFile("opening", openingUrlKey, now, force)
	// if file == nil {
	// 	return
	// }
}
Exemple #4
0
func (trustmap *TrustMap) Compute() (result Computation) {
	log.Info("Computing ...")
	return
}
Exemple #5
0
func parseWifi() {

	if parsedAlready {
		log.Info("Re-parsing logs: %s", wifiLogDir)
	} else {
		log.Info("Parsing logs: %s", wifiLogDir)
		parsedAlready = true
	}

	files, err := ioutil.ReadDir(wifiLogDir)
	if err != nil {
		log.StandardError(err)
		return
	}

	if len(files) == 0 {
		log.Error("No files found to parse in %s", wifiLogDir)
		return
	}

	var (
		c, l     int
		current  []byte
		file     *os.File
		filename string
		isPrefix bool
		key      string
		line     []byte
		reader   *bufio.Reader
		received int64
		sent     int64
		session  string
		split    []string
		status   string
		// pending  map[string]string
	)

	startTime := time.Now()

	var i int

	for _, info := range files {
		filename = filepath.Join(wifiLogDir, info.Name())
		log.Info("Parsing: %s", filename)
		file, err = os.Open(filename)
		if err != nil {
			log.StandardError(err)
			return
		}
		reader = bufio.NewReader(file)
		for {
			line, isPrefix, err = reader.ReadLine()
			if err != nil {
				if err == io.EOF {
					break
				}
				log.StandardError(err)
				return
			}
			current = append(current, line...)
			if isPrefix {
				continue
			}
			split = strings.Split(string(current), ",")
			l = len(split)
			c = 6
			session = ""
			status = ""
			received = 0
			sent = 0
			for c < l {
				key = split[c]
				c += 2
				switch key {
				case "40":
					status = split[c+1]
				case "42":
					received, _ = strconv.ParseInt(split[c+1], 10, 64)
				case "43":
					sent, _ = strconv.ParseInt(split[c+1], 10, 64)
				case "44":
					session = split[c+1]
				}
				// log.Info("key: %s", key)
			}
			_ = session
			_ = status
			_ = received
			_ = sent
			current = current[0:0]
			i += 1
			if i >= 0 {
				break
			}
		}
		break
	}

	log.Info("Finished parsing (%s)", time.Since(startTime))

}
Exemple #6
0
func main() {

	// Define the options for the command line and config file options parser.
	opts := optparse.Parser(
		"Usage: wifistat <config.yaml> [options]\n",
		"wifistat 0.0.1")

	addr := opts.StringConfig("addr", ":9040",
		"the host:port address for the web server [:9040]")

	csv := opts.StringConfig("csv-dir", "csv",
		"the path to the csv files directory [csv]")

	wifi := opts.StringConfig("wifi-logs-dir", "iaslogs",
		"the path to the Wi-Fi logs directory [iaslogs]")

	membership := opts.BoolConfig("member-analytics", false,
		"enable membership-based analytics")

	devices := opts.StringConfig("devices-url", "",
		"the url key for the devices.csv Google Spreadsheet")

	members := opts.StringConfig("members-url", "",
		"the url key for the members.csv Google Spreadsheet")

	opening := opts.StringConfig("opening-url", "",
		"the url key for the opening.csv Google Spreadsheet")

	// Parse the command line options.
	os.Args[0] = "wifistat"
	_, root, _ := runtime.DefaultOpts("wifistat", opts, os.Args)

	// Compute option variables.
	wifiLogDir = runtime.JoinPath(root, *wifi)
	csvDir = runtime.JoinPath(root, *csv)
	err := os.MkdirAll(csvDir, 0755)
	if err != nil {
		runtime.StandardError(err)
	}

	// Handle member analytics options.
	if *membership {
		enableMemberAnalytics = true
		devicesUrlKey = *devices
		if devicesUrlKey == "" {
			runtime.Error("You need to specify the `devices-url` command-line option.")
		}
		membersUrlKey = *members
		if membersUrlKey == "" {
			runtime.Error("You need to specify the `members-url` command-line option.")
		}
		openingUrlKey = *opening
		if openingUrlKey == "" {
			runtime.Error("You need to specify the `opening-url` command-line option.")
		}
	}

	// Parse the logs.
	parseCsv(false)
	parseWifi()

	// Register the various handlers.
	http.HandleFunc("/", handleRequest)
	http.HandleFunc("/reload", handleReload)

	// Start the web server.
	log.Info("Running wifistat on %s", *addr)
	err = http.ListenAndServe(*addr, nil)
	if err != nil {
		runtime.StandardError(err)
	}

	runtime.Exit(0)

}
Exemple #7
0
Fichier : bolt.go Projet : tav/bolt
func main() {

	// Setup temporary console logging.
	log.DisableConsoleTimestamp()
	log.AddConsoleLogger()

	// Set default values for command-line params.
	boltFilename := "Boltfile"
	genExecutablePath := ""

	recompile := false
	skipNext := true
	maxIdx := len(os.Args) - 1
	newArgs := []string{"bolt"}

	// Extract higher-level command-line arguments.
	for idx, arg := range os.Args {
		if skipNext {
			skipNext = false
			continue
		}
		if arg == "--gen" && idx != maxIdx {
			var err error
			genExecutablePath, err = filepath.Abs(os.Args[idx+1])
			if err != nil {
				runtime.StandardError(err)
			}
			skipNext = true
		} else if arg == "--boltfile" && idx != maxIdx {
			boltFilename = os.Args[idx+1]
			skipNext = true
		} else if arg == "--recompile" {
			recompile = true
		} else {
			newArgs = append(newArgs, arg)
		}
	}

	// Try and find the directory containing the Boltfile.
	boltdir, err := findBoltDir(boltFilename)
	if err != nil {
		if _, ok := err.(*fsutil.NotFound); ok {
			log.Error("Couldn't find Boltfile")
			runtime.Exit(1)
		}
		runtime.StandardError(err)
	}

	// Generate the path to the corresponding temp directory.
	boltpath := filepath.Join(boltdir, boltFilename)
	hash := sha1.New()
	hash.Write([]byte(boltpath))
	digest := fmt.Sprintf("%x", hash.Sum(nil))
	tempdir := filepath.Join(os.TempDir(), "bolt-"+digest)

	// See if the temp directory exists and if not create it.
	exists, err := fsutil.Exists(tempdir)
	if !exists {
		if _, ok := err.(*fsutil.NotFound); !ok {
			runtime.Error("Couldn't access the temp directory: %s: %s", tempdir, err)
		}
		err = os.Mkdir(tempdir, 0744)
		if err != nil {
			runtime.Error("Couldn't create the temp directory: %s: %s", tempdir, err)
		}
	}

	// See if an up-to-date generated binary already exists and, if so, run it.
	binpath := filepath.Join(tempdir, "bolt")
	if !recompile {
		boltstat, _ := os.Stat(boltpath)
		if genExecutablePath == "" {
			binstat, err := os.Stat(binpath)
			if err == nil {
				if boltstat.ModTime().Before(binstat.ModTime()) {
					runBoltExecutable(binpath, boltdir, newArgs)
					return
				}
			}
		}
	}

	// Parse the Boltfile.
	spec, err := parseBoltfile(boltpath, boltdir)
	if err != nil {
		exitForParserErrors(boltFilename, err)
	}

	// Exit if no tasks were found.
	if len(spec.Tasks) == 0 {
		runtime.Error("No tasks were found in %s", boltpath)
	}

	// Fudge the path to the executable that needs to be generated depending on
	// whether --gen-executable was specified or not.
	genOnly := true
	if genExecutablePath == "" {
		genExecutablePath = binpath
		genOnly = false
	}

	// Generate the executable.
	err = genExecutable(genExecutablePath, tempdir, spec)
	if err != nil {
		runtime.StandardError(err)
	}

	// Exit early if --gen-executable was specified.
	if genOnly {
		log.Info("%s successfully compiled to %s", boltFilename, genExecutablePath)
		runtime.Exit(0)
	}

	// Otherwise, run the executable.
	runBoltExecutable(binpath, boltdir, newArgs)

}
Exemple #8
0
func ampFrontend(argv []string, usage string) {

	// Define the options for the command line and config file options parser.
	opts := optparse.Parser(
		"Usage: amp frontend <config.yaml> [options]\n\n    " + usage + "\n")

	httpsHost := opts.StringConfig("https-host", "",
		"the host to bind the HTTPS Frontends to")

	httpsPort := opts.IntConfig("https-port", 9040,
		"the base port for the HTTPS Frontends [9040]")

	officialHost := opts.StringConfig("offficial-host", "",
		"the official public host for the HTTPS Frontends")

	primaryHosts := opts.StringConfig("primary-hosts", "",
		"limit the primary HTTPS Frontend to the specified host pattern")

	primaryCert := opts.StringConfig("primary-cert", "cert/primary.cert",
		"the path to the primary host's TLS certificate [cert/primary.cert]")

	primaryKey := opts.StringConfig("primary-key", "cert/primary.key",
		"the path to the primary host's TLS key [cert/primary.key]")

	noSecondary := opts.BoolConfig("no-secondary", false,
		"disable the secondary HTTPS Frontend [false]")

	secondaryHosts := opts.StringConfig("secondary-hosts", "",
		"limit the secondary HTTPS Frontend to the specified host pattern")

	secondaryCert := opts.StringConfig("secondary-cert", "cert/secondary.cert",
		"the path to the secondary host's TLS certificate [cert/secondary.cert]")

	secondaryKey := opts.StringConfig("secondary-key", "cert/secondary.key",
		"the path to the secondary host's TLS key [cert/secondary.key]")

	errorDirectory := opts.StringConfig("error-dir", "error",
		"the path to the HTTP error files directory [error]")

	staticDirectory := opts.StringConfig("static-dir", "www",
		"the path to the static files directory [www]")

	staticMaxAge := opts.IntConfig("static-max-age", 86400,
		"max-age cache header value when serving the static files [86400]")

	hstsMaxAge := opts.IntConfig("hsts-max-age", 50000000,
		"max-age in seconds for HTTP Strict Transport Security [50000000]")

	noRedirect := opts.BoolConfig("no-redirect", false,
		"disable the HTTP Redirector [false]")

	httpHost := opts.StringConfig("http-host", "",
		"the host to bind the HTTP Redirector to")

	httpPort := opts.IntConfig("http-port", 9080,
		"the port to bind the HTTP Redirector to [9080]")

	httpRedirectURL := opts.StringConfig("redirect-url", "",
		"the URL that the HTTP Redirector redirects to")

	singleNode := opts.StringConfig("single-node", "",
		"the upstream single node address if running without a master")

	masterNodes := opts.StringConfig("master-nodes", "localhost:8060",
		"comma-separated addresses of amp master nodes [localhost:8060]")

	masterCert := opts.StringConfig("master-cert", "cert/master.cert",
		"the path to the amp master certificate [cert/master.cert]")

	ironKeyPath := opts.StringConfig("iron-key", "cert/iron.key",
		"the path to the key used for iron strings [cert/iron.key]")

	maintenanceMode := opts.BoolConfig("maintenance", false,
		"start up in maintenance mode [false]")

	_, instanceDirectory, _ := runtime.DefaultOpts("frontend", opts, argv)

	// Ensure that the directory containing static files exists.
	staticPath := runtime.JoinPath(instanceDirectory, *staticDirectory)
	dirInfo, err := os.Stat(staticPath)
	if err == nil {
		if !dirInfo.IsDirectory() {
			runtime.Error("%q is not a directory", staticPath)
		}
	} else {
		runtime.StandardError(err)
	}

	// Ensure that the directory containing error files exists.
	errorPath := runtime.JoinPath(instanceDirectory, *errorDirectory)
	dirInfo, err = os.Stat(errorPath)
	if err == nil {
		if !dirInfo.IsDirectory() {
			runtime.Error("%q is not a directory", errorPath)
		}
	} else {
		runtime.StandardError(err)
	}

	// If ``--official-host`` hasn't been specified, generate it from the given
	// frontend host and base port values -- assuming ``localhost`` for a blank
	// host.
	publicHost := *officialHost
	if publicHost == "" {
		if *httpsHost == "" {
			publicHost = fmt.Sprintf("localhost:%d", *httpsPort)
		} else {
			publicHost = fmt.Sprintf("%s:%d", *httpsHost, *httpsPort)
		}
	}

	// Compute the HSTS max age header value.
	hsts := fmt.Sprintf("max-age=%d", *hstsMaxAge)

	// Pre-format the Cache-Control header for static files.
	staticCache := fmt.Sprintf("public, max-age=%d", *staticMaxAge)
	staticMaxAge64 := int64(*staticMaxAge)

	// Compute the variables related to redirects.
	redirectURL := "https://" + publicHost
	redirectHTML := []byte(fmt.Sprintf(
		`Please <a href="%s">click here if your browser doesn't redirect</a> automatically.`,
		redirectURL))

	// Compute the path to the Iron key.
	ironPath := runtime.JoinPath(instanceDirectory, *ironKeyPath)

	// Instantiate the master client.
	masterClient, err := master.NewClient(
		*masterNodes, runtime.JoinPath(instanceDirectory, *masterCert))

	if err != nil {
		runtime.StandardError(err)
	}

	var noMaster bool
	if *singleNode != "" {
		noMaster = true
	}

	// Let the user know how many CPUs we're currently running on.
	log.Info("Running the Amp Frontend on %d CPUs.", runtime.CPUCount)

	// Initialise the TLS config.
	tlsconf.Init()

	// Initialise a container for the HTTPSFrontends.
	webFrontends := make([]*server.HTTPSFrontend, 1)

	// Compute the variables related to detecting valid hosts.
	primaryWildcard, primaryAddr := getValidAddr(*primaryHosts)
	secondaryWildcard, secondaryAddr := getValidAddr(*secondaryHosts)

	// Instantiate the primary ``HTTPSFrontend`` object.
	frontend := &server.HTTPSFrontend{
		HSTS:            hsts,
		MaintenanceMode: *maintenanceMode,
		MasterClient:    masterClient,
		NoMaster:        noMaster,
		RedirectHTML:    redirectHTML,
		RedirectURL:     redirectURL,
		SingleNode:      *singleNode,
		StaticCache:     staticCache,
		StaticMaxAge:    staticMaxAge64,
		ValidAddress:    primaryAddr,
		ValidWildcard:   primaryWildcard,
	}

	frontend.LoadAssets(errorPath, ironPath, staticPath)
	frontend.Run(*httpsHost, *httpsPort,
		runtime.JoinPath(instanceDirectory, *primaryCert),
		runtime.JoinPath(instanceDirectory, *primaryKey))

	webFrontends[0] = frontend

	// Setup and run the secondary HTTPSFrontend.
	if !*noSecondary {
		frontend = &server.HTTPSFrontend{
			HSTS:            hsts,
			MaintenanceMode: *maintenanceMode,
			MasterClient:    masterClient,
			NoMaster:        noMaster,
			RedirectHTML:    redirectHTML,
			RedirectURL:     redirectURL,
			SingleNode:      *singleNode,
			StaticCache:     staticCache,
			StaticMaxAge:    staticMaxAge64,
			ValidAddress:    secondaryAddr,
			ValidWildcard:   secondaryWildcard,
		}
		frontend.LoadAssets(errorPath, ironPath, staticPath)
		frontend.Run(*httpsHost, *httpsPort+1,
			runtime.JoinPath(instanceDirectory, *secondaryCert),
			runtime.JoinPath(instanceDirectory, *secondaryKey))
		webFrontends = append(webFrontends, frontend)
	}

	// Create a channel which is used to toggle maintenance mode based on
	// process signals.
	maintenanceChannel := make(chan bool, 1)

	// Fork a goroutine which toggles the maintenance mode in a single place and
	// thus ensures thread safety.
	go func() {
		for {
			enabledState := <-maintenanceChannel
			for _, frontend := range webFrontends {
				if enabledState {
					frontend.MaintenanceMode = true
				} else {
					frontend.LoadAssets(errorPath, ironPath, staticPath)
					frontend.MaintenanceMode = false
				}
			}
		}
	}()

	// Register the signal handlers for SIGUSR1 and SIGUSR2.
	runtime.SignalHandlers[os.SIGUSR1] = func() {
		maintenanceChannel <- true
	}

	runtime.SignalHandlers[os.SIGUSR2] = func() {
		maintenanceChannel <- false
	}

	// Enter a wait loop if the HTTP Redirector has been disabled.
	if *noRedirect {
		loopForever := make(chan bool, 1)
		<-loopForever
	}

	// Otherwise, setup and run the HTTP Redirector.
	if *httpHost == "" {
		*httpHost = "localhost"
	}

	if *httpRedirectURL == "" {
		*httpRedirectURL = "https://" + publicHost
	}

	redirector := &server.HTTPRedirector{*httpRedirectURL}
	redirector.Run(*httpHost, *httpPort)

	// Enter the wait loop for the process to be killed.
	loopForever := make(chan bool, 1)
	<-loopForever

}