func loadCsvFile(name, urlpath string, timestamp time.Time, force bool) (*csv.Reader, error) {
filename := filepath.Join(csvDir, name+timestamp.Format(".2006-01-02")+".csv")
if !force {
file, err := os.Open(filename)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
}
} else {
return csv.NewReader(file), nil
}
}
log.Info("Downloading %s.csv from Google Spreadsheets", name)
resp, err := http.Get(csvUrlPrefix + urlpath + csvUrlSuffix)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
err = ioutil.WriteFile(filename, body, 0644)
if err != nil {
return nil, err
}
return csv.NewReader(bytes.NewBuffer(body)), nil
}
func renderMarkdown(input []byte) (out []byte, err error) {
defer func() {
if e := recover(); e != nil {
if e, ok := e.(error); ok {
log.Info("hilite error: restarting, due to: %s", e)
setupPygments()
err = e
}
}
}()
r := &Html{blackfriday.HtmlRenderer(DefaultHtmlFlags, "", "")}
out = blackfriday.Markdown(input, r, DefaultExtensions)
return
}
func parseCsv(force bool) {
now := time.Now()
file := getCsvFile("devices", devicesUrlKey, now, force)
if file == nil {
return
}
var (
line []string
err error
)
log.Info("Parsing devices.csv")
for {
line, err = file.Read()
if err != nil {
if err == io.EOF {
break
}
log.Error("Error parsing devices.csv: %s", err)
return
}
_ = line
}
// file = getCsvFile("members", membersUrlKey, now, force)
// if file == nil {
// return
// }
// log.Info("Parsing devices.csv")
// for {
// line, err = file.Read()
// if err != nil {
// if err == io.EOF {
// break
// }
// log.Error("Error parsing devices.csv: %s", err)
// return
// }
// _ = line
// }
// file = getCsvFile("opening", openingUrlKey, now, force)
// if file == nil {
// return
// }
}
func (trustmap *TrustMap) Compute() (result Computation) {
log.Info("Computing ...")
return
}
func parseWifi() {
if parsedAlready {
log.Info("Re-parsing logs: %s", wifiLogDir)
} else {
log.Info("Parsing logs: %s", wifiLogDir)
parsedAlready = true
}
files, err := ioutil.ReadDir(wifiLogDir)
if err != nil {
log.StandardError(err)
return
}
if len(files) == 0 {
log.Error("No files found to parse in %s", wifiLogDir)
return
}
var (
c, l int
current []byte
file *os.File
filename string
isPrefix bool
key string
line []byte
reader *bufio.Reader
received int64
sent int64
session string
split []string
status string
// pending map[string]string
)
startTime := time.Now()
var i int
for _, info := range files {
filename = filepath.Join(wifiLogDir, info.Name())
log.Info("Parsing: %s", filename)
file, err = os.Open(filename)
if err != nil {
log.StandardError(err)
return
}
reader = bufio.NewReader(file)
for {
line, isPrefix, err = reader.ReadLine()
if err != nil {
if err == io.EOF {
break
}
log.StandardError(err)
return
}
current = append(current, line...)
if isPrefix {
continue
}
split = strings.Split(string(current), ",")
l = len(split)
c = 6
session = ""
status = ""
received = 0
sent = 0
for c < l {
key = split[c]
c += 2
switch key {
case "40":
status = split[c+1]
case "42":
received, _ = strconv.ParseInt(split[c+1], 10, 64)
case "43":
sent, _ = strconv.ParseInt(split[c+1], 10, 64)
case "44":
session = split[c+1]
}
// log.Info("key: %s", key)
}
_ = session
_ = status
_ = received
_ = sent
current = current[0:0]
i += 1
if i >= 0 {
break
}
}
break
}
log.Info("Finished parsing (%s)", time.Since(startTime))
}
func main() {
// Define the options for the command line and config file options parser.
opts := optparse.Parser(
"Usage: wifistat <config.yaml> [options]\n",
"wifistat 0.0.1")
addr := opts.StringConfig("addr", ":9040",
"the host:port address for the web server [:9040]")
csv := opts.StringConfig("csv-dir", "csv",
"the path to the csv files directory [csv]")
wifi := opts.StringConfig("wifi-logs-dir", "iaslogs",
"the path to the Wi-Fi logs directory [iaslogs]")
membership := opts.BoolConfig("member-analytics", false,
"enable membership-based analytics")
devices := opts.StringConfig("devices-url", "",
"the url key for the devices.csv Google Spreadsheet")
members := opts.StringConfig("members-url", "",
"the url key for the members.csv Google Spreadsheet")
opening := opts.StringConfig("opening-url", "",
"the url key for the opening.csv Google Spreadsheet")
// Parse the command line options.
os.Args[0] = "wifistat"
_, root, _ := runtime.DefaultOpts("wifistat", opts, os.Args)
// Compute option variables.
wifiLogDir = runtime.JoinPath(root, *wifi)
csvDir = runtime.JoinPath(root, *csv)
err := os.MkdirAll(csvDir, 0755)
if err != nil {
runtime.StandardError(err)
}
// Handle member analytics options.
if *membership {
enableMemberAnalytics = true
devicesUrlKey = *devices
if devicesUrlKey == "" {
runtime.Error("You need to specify the `devices-url` command-line option.")
}
membersUrlKey = *members
if membersUrlKey == "" {
runtime.Error("You need to specify the `members-url` command-line option.")
}
openingUrlKey = *opening
if openingUrlKey == "" {
runtime.Error("You need to specify the `opening-url` command-line option.")
}
}
// Parse the logs.
parseCsv(false)
parseWifi()
// Register the various handlers.
http.HandleFunc("/", handleRequest)
http.HandleFunc("/reload", handleReload)
// Start the web server.
log.Info("Running wifistat on %s", *addr)
err = http.ListenAndServe(*addr, nil)
if err != nil {
runtime.StandardError(err)
}
runtime.Exit(0)
}
func main() {
// Setup temporary console logging.
log.DisableConsoleTimestamp()
log.AddConsoleLogger()
// Set default values for command-line params.
boltFilename := "Boltfile"
genExecutablePath := ""
recompile := false
skipNext := true
maxIdx := len(os.Args) - 1
newArgs := []string{"bolt"}
// Extract higher-level command-line arguments.
for idx, arg := range os.Args {
if skipNext {
skipNext = false
continue
}
if arg == "--gen" && idx != maxIdx {
var err error
genExecutablePath, err = filepath.Abs(os.Args[idx+1])
if err != nil {
runtime.StandardError(err)
}
skipNext = true
} else if arg == "--boltfile" && idx != maxIdx {
boltFilename = os.Args[idx+1]
skipNext = true
} else if arg == "--recompile" {
recompile = true
} else {
newArgs = append(newArgs, arg)
}
}
// Try and find the directory containing the Boltfile.
boltdir, err := findBoltDir(boltFilename)
if err != nil {
if _, ok := err.(*fsutil.NotFound); ok {
log.Error("Couldn't find Boltfile")
runtime.Exit(1)
}
runtime.StandardError(err)
}
// Generate the path to the corresponding temp directory.
boltpath := filepath.Join(boltdir, boltFilename)
hash := sha1.New()
hash.Write([]byte(boltpath))
digest := fmt.Sprintf("%x", hash.Sum(nil))
tempdir := filepath.Join(os.TempDir(), "bolt-"+digest)
// See if the temp directory exists and if not create it.
exists, err := fsutil.Exists(tempdir)
if !exists {
if _, ok := err.(*fsutil.NotFound); !ok {
runtime.Error("Couldn't access the temp directory: %s: %s", tempdir, err)
}
err = os.Mkdir(tempdir, 0744)
if err != nil {
runtime.Error("Couldn't create the temp directory: %s: %s", tempdir, err)
}
}
// See if an up-to-date generated binary already exists and, if so, run it.
binpath := filepath.Join(tempdir, "bolt")
if !recompile {
boltstat, _ := os.Stat(boltpath)
if genExecutablePath == "" {
binstat, err := os.Stat(binpath)
if err == nil {
if boltstat.ModTime().Before(binstat.ModTime()) {
runBoltExecutable(binpath, boltdir, newArgs)
return
}
}
}
}
// Parse the Boltfile.
spec, err := parseBoltfile(boltpath, boltdir)
if err != nil {
exitForParserErrors(boltFilename, err)
}
// Exit if no tasks were found.
if len(spec.Tasks) == 0 {
runtime.Error("No tasks were found in %s", boltpath)
}
// Fudge the path to the executable that needs to be generated depending on
// whether --gen-executable was specified or not.
genOnly := true
if genExecutablePath == "" {
genExecutablePath = binpath
genOnly = false
}
// Generate the executable.
err = genExecutable(genExecutablePath, tempdir, spec)
if err != nil {
runtime.StandardError(err)
}
// Exit early if --gen-executable was specified.
if genOnly {
log.Info("%s successfully compiled to %s", boltFilename, genExecutablePath)
runtime.Exit(0)
}
// Otherwise, run the executable.
runBoltExecutable(binpath, boltdir, newArgs)
}
func ampFrontend(argv []string, usage string) {
// Define the options for the command line and config file options parser.
opts := optparse.Parser(
"Usage: amp frontend <config.yaml> [options]\n\n " + usage + "\n")
httpsHost := opts.StringConfig("https-host", "",
"the host to bind the HTTPS Frontends to")
httpsPort := opts.IntConfig("https-port", 9040,
"the base port for the HTTPS Frontends [9040]")
officialHost := opts.StringConfig("offficial-host", "",
"the official public host for the HTTPS Frontends")
primaryHosts := opts.StringConfig("primary-hosts", "",
"limit the primary HTTPS Frontend to the specified host pattern")
primaryCert := opts.StringConfig("primary-cert", "cert/primary.cert",
"the path to the primary host's TLS certificate [cert/primary.cert]")
primaryKey := opts.StringConfig("primary-key", "cert/primary.key",
"the path to the primary host's TLS key [cert/primary.key]")
noSecondary := opts.BoolConfig("no-secondary", false,
"disable the secondary HTTPS Frontend [false]")
secondaryHosts := opts.StringConfig("secondary-hosts", "",
"limit the secondary HTTPS Frontend to the specified host pattern")
secondaryCert := opts.StringConfig("secondary-cert", "cert/secondary.cert",
"the path to the secondary host's TLS certificate [cert/secondary.cert]")
secondaryKey := opts.StringConfig("secondary-key", "cert/secondary.key",
"the path to the secondary host's TLS key [cert/secondary.key]")
errorDirectory := opts.StringConfig("error-dir", "error",
"the path to the HTTP error files directory [error]")
staticDirectory := opts.StringConfig("static-dir", "www",
"the path to the static files directory [www]")
staticMaxAge := opts.IntConfig("static-max-age", 86400,
"max-age cache header value when serving the static files [86400]")
hstsMaxAge := opts.IntConfig("hsts-max-age", 50000000,
"max-age in seconds for HTTP Strict Transport Security [50000000]")
noRedirect := opts.BoolConfig("no-redirect", false,
"disable the HTTP Redirector [false]")
httpHost := opts.StringConfig("http-host", "",
"the host to bind the HTTP Redirector to")
httpPort := opts.IntConfig("http-port", 9080,
"the port to bind the HTTP Redirector to [9080]")
httpRedirectURL := opts.StringConfig("redirect-url", "",
"the URL that the HTTP Redirector redirects to")
singleNode := opts.StringConfig("single-node", "",
"the upstream single node address if running without a master")
masterNodes := opts.StringConfig("master-nodes", "localhost:8060",
"comma-separated addresses of amp master nodes [localhost:8060]")
masterCert := opts.StringConfig("master-cert", "cert/master.cert",
"the path to the amp master certificate [cert/master.cert]")
ironKeyPath := opts.StringConfig("iron-key", "cert/iron.key",
"the path to the key used for iron strings [cert/iron.key]")
maintenanceMode := opts.BoolConfig("maintenance", false,
"start up in maintenance mode [false]")
_, instanceDirectory, _ := runtime.DefaultOpts("frontend", opts, argv)
// Ensure that the directory containing static files exists.
staticPath := runtime.JoinPath(instanceDirectory, *staticDirectory)
dirInfo, err := os.Stat(staticPath)
if err == nil {
if !dirInfo.IsDirectory() {
runtime.Error("%q is not a directory", staticPath)
}
} else {
runtime.StandardError(err)
}
// Ensure that the directory containing error files exists.
errorPath := runtime.JoinPath(instanceDirectory, *errorDirectory)
dirInfo, err = os.Stat(errorPath)
if err == nil {
if !dirInfo.IsDirectory() {
runtime.Error("%q is not a directory", errorPath)
}
} else {
runtime.StandardError(err)
}
// If ``--official-host`` hasn't been specified, generate it from the given
// frontend host and base port values -- assuming ``localhost`` for a blank
// host.
publicHost := *officialHost
if publicHost == "" {
if *httpsHost == "" {
publicHost = fmt.Sprintf("localhost:%d", *httpsPort)
} else {
publicHost = fmt.Sprintf("%s:%d", *httpsHost, *httpsPort)
}
}
// Compute the HSTS max age header value.
hsts := fmt.Sprintf("max-age=%d", *hstsMaxAge)
// Pre-format the Cache-Control header for static files.
staticCache := fmt.Sprintf("public, max-age=%d", *staticMaxAge)
staticMaxAge64 := int64(*staticMaxAge)
// Compute the variables related to redirects.
redirectURL := "https://" + publicHost
redirectHTML := []byte(fmt.Sprintf(
`Please <a href="%s">click here if your browser doesn't redirect</a> automatically.`,
redirectURL))
// Compute the path to the Iron key.
ironPath := runtime.JoinPath(instanceDirectory, *ironKeyPath)
// Instantiate the master client.
masterClient, err := master.NewClient(
*masterNodes, runtime.JoinPath(instanceDirectory, *masterCert))
if err != nil {
runtime.StandardError(err)
}
var noMaster bool
if *singleNode != "" {
noMaster = true
}
// Let the user know how many CPUs we're currently running on.
log.Info("Running the Amp Frontend on %d CPUs.", runtime.CPUCount)
// Initialise the TLS config.
tlsconf.Init()
// Initialise a container for the HTTPSFrontends.
webFrontends := make([]*server.HTTPSFrontend, 1)
// Compute the variables related to detecting valid hosts.
primaryWildcard, primaryAddr := getValidAddr(*primaryHosts)
secondaryWildcard, secondaryAddr := getValidAddr(*secondaryHosts)
// Instantiate the primary ``HTTPSFrontend`` object.
frontend := &server.HTTPSFrontend{
HSTS: hsts,
MaintenanceMode: *maintenanceMode,
MasterClient: masterClient,
NoMaster: noMaster,
RedirectHTML: redirectHTML,
RedirectURL: redirectURL,
SingleNode: *singleNode,
StaticCache: staticCache,
StaticMaxAge: staticMaxAge64,
ValidAddress: primaryAddr,
ValidWildcard: primaryWildcard,
}
frontend.LoadAssets(errorPath, ironPath, staticPath)
frontend.Run(*httpsHost, *httpsPort,
runtime.JoinPath(instanceDirectory, *primaryCert),
runtime.JoinPath(instanceDirectory, *primaryKey))
webFrontends[0] = frontend
// Setup and run the secondary HTTPSFrontend.
if !*noSecondary {
frontend = &server.HTTPSFrontend{
HSTS: hsts,
MaintenanceMode: *maintenanceMode,
MasterClient: masterClient,
NoMaster: noMaster,
RedirectHTML: redirectHTML,
RedirectURL: redirectURL,
SingleNode: *singleNode,
StaticCache: staticCache,
StaticMaxAge: staticMaxAge64,
ValidAddress: secondaryAddr,
ValidWildcard: secondaryWildcard,
}
frontend.LoadAssets(errorPath, ironPath, staticPath)
frontend.Run(*httpsHost, *httpsPort+1,
runtime.JoinPath(instanceDirectory, *secondaryCert),
runtime.JoinPath(instanceDirectory, *secondaryKey))
webFrontends = append(webFrontends, frontend)
}
// Create a channel which is used to toggle maintenance mode based on
// process signals.
maintenanceChannel := make(chan bool, 1)
// Fork a goroutine which toggles the maintenance mode in a single place and
// thus ensures thread safety.
go func() {
for {
enabledState := <-maintenanceChannel
for _, frontend := range webFrontends {
if enabledState {
frontend.MaintenanceMode = true
} else {
frontend.LoadAssets(errorPath, ironPath, staticPath)
frontend.MaintenanceMode = false
}
}
}
}()
// Register the signal handlers for SIGUSR1 and SIGUSR2.
runtime.SignalHandlers[os.SIGUSR1] = func() {
maintenanceChannel <- true
}
runtime.SignalHandlers[os.SIGUSR2] = func() {
maintenanceChannel <- false
}
// Enter a wait loop if the HTTP Redirector has been disabled.
if *noRedirect {
loopForever := make(chan bool, 1)
<-loopForever
}
// Otherwise, setup and run the HTTP Redirector.
if *httpHost == "" {
*httpHost = "localhost"
}
if *httpRedirectURL == "" {
*httpRedirectURL = "https://" + publicHost
}
redirector := &server.HTTPRedirector{*httpRedirectURL}
redirector.Run(*httpHost, *httpPort)
// Enter the wait loop for the process to be killed.
loopForever := make(chan bool, 1)
<-loopForever
}