func getNodules(paths []string) (nodules []*nodule.Nodule) {
if paths == nil {
cwd, err := os.Getwd()
if err != nil {
runtime.StandardError(err)
}
paths = []string{cwd}
}
nodules = make([]*nodule.Nodule, 0)
seen := make(map[string]bool)
for _, path := range paths {
data, err := nodule.Find(path)
if err != nil {
runtime.StandardError(err)
}
for _, nodule := range data {
if seen[nodule.Path] {
continue
}
nodules = append(nodules, nodule)
seen[nodule.Path] = true
}
}
return
}
func (app *Application) Init(env map[string]interface{}) {
conf := app.Config
// Create the log directory if it doesn't exist.
logPath := runtime.JoinPath(app.Path, *conf.logDirectory)
err := os.MkdirAll(logPath, 0755)
if err != nil {
runtime.StandardError(err)
}
// Create the run directory if it doesn't exist.
runPath := runtime.JoinPath(app.Path, *conf.runDirectory)
err = os.MkdirAll(runPath, 0755)
if err != nil {
runtime.StandardError(err)
}
// Initialise the process-related resources.
runtime.Init()
runtime.InitProcess(app.Name, runPath)
fmt.Printf("Running %s on %s:%d\n", app.Name, *conf.host, *conf.port)
app.HandleRequests()
}
func ampNode(argv []string, usage string) {
opts := optparse.Parser(
"Usage: amp node <config.yaml> [options]\n\n " + usage + "\n")
nodeHost := opts.StringConfig("node-host", "",
"the host to bind this node to")
nodePort := opts.IntConfig("node-port", 8050,
"the port to bind this node to [8050]")
ctrlHost := opts.StringConfig("control-host", "",
"the host to bind the nodule control socket to")
ctrlPort := opts.IntConfig("control-port", 8051,
"the port to bind the nodule control socket to [8051]")
nodules := opts.StringConfig("nodules", "*",
"comma-separated list of nodules to initialise [*]")
nodulePaths := opts.StringConfig("nodule-paths", ".",
"comma-separated list of nodule container directories [nodule]")
masterNodes := opts.StringConfig("master-nodes", "localhost:8060",
"comma-separated addresses of amp master nodes [localhost:8060]")
masterKeyPath := opts.StringConfig("master-key", "cert/master.key",
"the path to the file containing the amp master public key [cert/master.key]")
debug, _, runPath := runtime.DefaultOpts("node", opts, argv, nil)
masterClient, err := master.NewClient(*masterNodes, *masterKeyPath)
if err != nil {
runtime.StandardError(err)
}
logging.AddConsoleFilter(nodule.FilterConsoleLog)
node, err := nodule.NewHost(
runPath, *nodeHost, *nodePort, *ctrlHost, *ctrlPort, *nodules,
strings.SplitN(*nodulePaths, ",", -1), masterClient)
if err != nil {
runtime.StandardError(err)
}
err = node.Run(debug)
logging.Wait()
if err != nil {
runtime.Exit(1)
}
}
// The ``getFiles`` utility function populates the given ``mapping`` with
// ``StaticFile`` instances for all files found within a given ``directory``.
func getFiles(directory string, mapping map[string]*StaticFile, root string) {
if debugMode {
fmt.Printf("Caching static files in: %s\n", directory)
}
path, err := os.Open(directory)
if err != nil {
runtime.StandardError(err)
}
for {
items, err := path.Readdir(100)
if err != nil || len(items) == 0 {
break
}
for _, item := range items {
name := item.Name()
key := fmt.Sprintf("%s/%s", root, name)
if item.IsDir() {
getFiles(filepath.Join(directory, name), mapping, key)
} else {
content := make([]byte, item.Size())
file, err := os.Open(filepath.Join(directory, name))
if err != nil {
runtime.StandardError(err)
}
_, err = file.Read(content[:])
if err != nil && err != io.EOF {
runtime.StandardError(err)
}
mimetype := mime.TypeByExtension(filepath.Ext(name))
if mimetype == "" {
mimetype = "application/octet-stream"
}
hash := sha1.New()
hash.Write(content)
buffer := &bytes.Buffer{}
encoder := base64.NewEncoder(base64.URLEncoding, buffer)
encoder.Write(hash.Sum(nil))
encoder.Close()
mapping[key] = &StaticFile{
content: content,
etag: fmt.Sprintf(`"%s"`, buffer.String()),
mimetype: mimetype,
size: fmt.Sprintf("%d", len(content)),
}
}
}
}
}
// The ``getErrorInfo`` utility function loads the specified error file from the
// given directory and returns its content and file size.
func getErrorInfo(directory, filename string) ([]byte, string) {
path := filepath.Join(directory, filename)
file, err := os.Open(path)
if err != nil {
runtime.StandardError(err)
}
info, err := os.Stat(path)
if err != nil {
runtime.StandardError(err)
}
buffer := make([]byte, info.Size())
_, err = file.Read(buffer[:])
if err != nil && err != io.EOF {
runtime.StandardError(err)
}
return buffer, fmt.Sprintf("%d", info.Size())
}
func exitForParserErrors(filename string, err error) {
if err, ok := err.(scanner.ErrorList); ok {
for _, e := range err {
log.Error("%s:%s", filename, e)
}
runtime.Exit(1)
} else {
runtime.StandardError(err)
}
}
func ensureDirectory(root, path string) (directory string) {
if path == "" {
directory = root
} else {
directory = runtime.JoinPath(root, path)
}
_, err := os.Open(directory)
if err != nil {
runtime.StandardError(err)
}
return
}
func runProcess(amp, cmd, path, config string, console bool, quit chan bool) {
var files []*os.File
if console {
files = []*os.File{nil, os.Stdout, os.Stderr}
} else {
files = []*os.File{nil, nil, nil}
}
logging.Info("Running: amp %s %s", cmd, config)
process, err := os.StartProcess(
amp,
[]string{"amp", cmd, config},
&os.ProcAttr{
Dir: path,
Env: os.Environ(),
Files: files,
})
if err != nil {
runtime.StandardError(err)
}
waitmsg, err := process.Wait(0)
if err != nil {
runtime.StandardError(err)
}
if waitmsg.ExitStatus() != 0 {
runtime.Error("ERROR: Got %s when running `amp %s %s`\n",
waitmsg, cmd, config)
}
quit <- true
}
func (h *Hilite) Run() {
stdin, w, _ := os.Pipe()
r, stdout, _ := os.Pipe()
h.r = r
h.w = w
proc, err := os.StartProcess("hilite.py", []string{runPath + "/hilite.py"}, &os.ProcAttr{
Files: []*os.File{stdin, stdout, nil},
})
if err != nil {
runtime.StandardError(err)
}
h.p = proc
go proc.Wait()
}
func runBoltExecutable(path, boltdir string, args []string) {
p, err := os.StartProcess(path, args, &os.ProcAttr{
Dir: boltdir,
Files: []*os.File{nil, os.Stdout, os.Stderr},
})
if err != nil {
runtime.StandardError(err)
}
state, _ := p.Wait()
if state.Success() {
runtime.Exit(0)
} else {
runtime.Exit(1)
}
}
func main() {
// Define the options for the command line and config file options parser.
opts := optparse.Parser(
"Usage: live-server <config.yaml> [options]\n",
"live-server 0.0.1")
debug := opts.Bool([]string{"-d", "--debug"}, false,
"enable debug mode")
genConfig := opts.Bool([]string{"-g", "--gen-config"}, false,
"show the default yaml config")
frontendHost := opts.StringConfig("frontend-host", "",
"the host to bind the HTTPS Frontends to")
frontendPort := opts.IntConfig("frontend-port", 9040,
"the base port for the HTTPS Frontends [9040]")
officialHost := opts.StringConfig("offficial-host", "",
"the official public host for the HTTPS Frontends")
primaryHosts := opts.StringConfig("primary-hosts", "",
"limit the primary HTTPS Frontend to the specified host pattern")
primaryCert := opts.StringConfig("primary-cert", "cert/primary.cert",
"the path to the primary host's TLS certificate [cert/primary.cert]")
primaryKey := opts.StringConfig("primary-key", "cert/primary.key",
"the path to the primary host's TLS key [cert/primary.key]")
noSecondary := opts.BoolConfig("no-secondary", false,
"disable the secondary HTTPS Frontend [false]")
secondaryHosts := opts.StringConfig("secondary-hosts", "",
"limit the secondary HTTPS Frontend to the specified host pattern")
secondaryCert := opts.StringConfig("secondary-cert", "cert/secondary.cert",
"the path to the secondary host's TLS certificate [cert/secondary.cert]")
secondaryKey := opts.StringConfig("secondary-key", "cert/secondary.key",
"the path to the secondary host's TLS key [cert/secondary.key]")
errorDirectory := opts.StringConfig("error-dir", "error",
"the path to the HTTP error files directory [error]")
logDirectory := opts.StringConfig("log-dir", "log",
"the path to the log directory [log]")
runDirectory := opts.StringConfig("run-dir", "run",
"the path to the run directory to store locks, pid files, etc. [run]")
staticDirectory := opts.StringConfig("static-dir", "www",
"the path to the static files directory [www]")
staticMaxAge := opts.IntConfig("static-max-age", 86400,
"max-age cache header value when serving the static files [86400]")
noLivequery := opts.BoolConfig("no-livequery", false,
"disable the LiveQuery node and WebSocket/Comet support [false]")
websocketPrefix := opts.StringConfig("websocket-prefix", "/.live/ws",
"URL path prefix for WebSocket requests [/.live/ws]")
cometPrefix := opts.StringConfig("comet-prefix", "/.live/poll",
"URL path prefix for Comet requests [/.live/poll]")
livequeryHost := opts.StringConfig("livequery-host", "",
"the host to bind the LiveQuery node to")
livequeryPort := opts.IntConfig("livequery-port", 9050,
"the port (both UDP and TCP) to bind the LiveQuery node to [9050]")
livequeryExpiry := opts.IntConfig("livequery-expiry", 40,
"maximum number of seconds a LiveQuery subscription is valid [40]")
cookieKeyPath := opts.StringConfig("cookie-key", "cert/cookie.key",
"the path to the file containing the key used to sign cookies [cert/cookie.key]")
cookieName := opts.StringConfig("cookie-name", "user",
"the property name of the cookie containing the user id [user]")
acceptors := opts.StringConfig("acceptor-nodes", "localhost:9060",
"comma-separated addresses of Acceptor nodes [localhost:9060]")
acceptorKeyPath := opts.StringConfig("acceptor-key", "cert/acceptor.key",
"the path to the file containing the Acceptor secret key [cert/acceptor.key]")
runAcceptor := opts.BoolConfig("run-as-acceptor", false,
"run as an Acceptor node [false]")
acceptorIndex := opts.IntConfig("acceptor-index", 0,
"this node's index in the Acceptor nodes address list [0]")
leaseExpiry := opts.IntConfig("lease-expiry", 7,
"maximum number of seconds a lease from an Acceptor node is valid [7]")
noRedirect := opts.BoolConfig("no-redirect", false,
"disable the HTTP Redirector [false]")
httpHost := opts.StringConfig("http-host", "",
"the host to bind the HTTP Redirector to")
httpPort := opts.IntConfig("http-port", 9080,
"the port to bind the HTTP Redirector to [9080]")
redirectURL := opts.StringConfig("redirect-url", "",
"the URL that the HTTP Redirector redirects to")
pingPath := opts.StringConfig("ping-path", "/.ping",
`URL path for a "ping" request [/.ping]`)
enableHSTS := opts.BoolConfig("enable-hsts", false,
"enable HTTP Strict Transport Security (HSTS) on redirects [false]")
hstsMaxAge := opts.IntConfig("hsts-max-age", 50000000,
"max-age value of HSTS in number of seconds [50000000]")
upstreamHost := opts.StringConfig("upstream-host", "localhost",
"the upstream host to connect to [localhost]")
upstreamPort := opts.IntConfig("upstream-port", 8080,
"the upstream port to connect to [8080]")
upstreamTLS := opts.BoolConfig("upstream-tls", false,
"use TLS when connecting to upstream [false]")
logRotate := opts.StringConfig("log-rotate", "never",
"specify one of 'hourly', 'daily' or 'never' [never]")
noConsoleLog := opts.BoolConfig("no-console-log", false,
"disable server requests being logged to the console [false]")
maintenanceMode := opts.BoolConfig("maintenance", false,
"start up in maintenance mode [false]")
extraConfig := opts.StringConfig("extra-config", "",
"path to a YAML config file with additional options")
// Parse the command line options.
os.Args[0] = "live-server"
args := opts.Parse(os.Args)
// Print the default YAML config file if the ``-g`` flag was specified.
if *genConfig {
opts.PrintDefaultConfigFile("live-server")
runtime.Exit(0)
}
// Setup the console logger early.
if !*noConsoleLog {
log.AddConsoleLogger()
log.ConsoleFilters["ls"] = func(items []interface{}) (bool, []interface{}) {
return true, items[2 : len(items)-2]
}
}
// Set the debug mode flag if the ``-d`` flag was specified.
debugMode = *debug
var instanceDirectory string
var configPath string
var err error
// Assume the parent directory of the config as the instance directory.
if len(args) >= 1 {
if args[0] == "help" {
opts.PrintUsage()
runtime.Exit(0)
}
configPath, err = filepath.Abs(filepath.Clean(args[0]))
if err != nil {
runtime.StandardError(err)
}
err = opts.ParseConfig(configPath, os.Args)
if err != nil {
runtime.StandardError(err)
}
instanceDirectory, _ = filepath.Split(configPath)
} else {
opts.PrintUsage()
runtime.Exit(0)
}
// Load the extra config file with additional options if one has been
// specified.
if *extraConfig != "" {
extraConfigPath, err := filepath.Abs(filepath.Clean(*extraConfig))
if err != nil {
runtime.StandardError(err)
}
extraConfigPath = runtime.JoinPath(instanceDirectory, extraConfigPath)
err = opts.ParseConfig(extraConfigPath, os.Args)
if err != nil {
runtime.StandardError(err)
}
}
// Create the log directory if it doesn't exist.
logPath := runtime.JoinPath(instanceDirectory, *logDirectory)
err = os.MkdirAll(logPath, 0755)
if err != nil {
runtime.StandardError(err)
}
// Create the run directory if it doesn't exist.
runPath := runtime.JoinPath(instanceDirectory, *runDirectory)
err = os.MkdirAll(runPath, 0755)
if err != nil {
runtime.StandardError(err)
}
// Initialise the runtime -- which will run ``live-server`` on multiple
// processors if possible.
runtime.Init()
// Handle running as an Acceptor node if ``--run-as-acceptor`` was
// specified.
if *runAcceptor {
// Exit if the `--acceptor-index`` is negative.
if *acceptorIndex < 0 {
runtime.Error("The --acceptor-index cannot be negative.")
}
var index int
var selfAddress string
var acceptorNodes []string
// Generate a list of all the acceptor node addresses and exit if we
// couldn't find the address four ourselves at the given index.
for _, acceptor := range strings.Split(*acceptors, ",") {
acceptor = strings.TrimSpace(acceptor)
if acceptor != "" {
if index == *acceptorIndex {
selfAddress = acceptor
} else {
acceptorNodes = append(acceptorNodes, acceptor)
}
}
index += 1
}
if selfAddress == "" {
runtime.Error("Couldn't determine the address for the acceptor.")
}
// Initialise the process-related resources.
runtime.InitProcess(fmt.Sprintf("acceptor-%d", *acceptorIndex), runPath)
return
}
// Initialise the process-related resources.
runtime.InitProcess("live-server", runPath)
// Ensure that the directory containing static files exists.
staticPath := runtime.JoinPath(instanceDirectory, *staticDirectory)
dirInfo, err := os.Stat(staticPath)
if err == nil {
if !dirInfo.IsDir() {
runtime.Error("Static path %q is not a directory", staticPath)
}
} else {
runtime.StandardError(err)
}
// Load up all static files into a mapping.
staticFiles := make(map[string]*StaticFile)
getFiles(staticPath, staticFiles, "")
// Pre-format the Cache-Control header for static files.
staticCache := fmt.Sprintf("public, max-age=%d", *staticMaxAge)
staticMaxAge64 := time.Duration(*staticMaxAge)
// Exit if the directory containing the 50x.html files isn't present.
errorPath := runtime.JoinPath(instanceDirectory, *errorDirectory)
dirInfo, err = os.Stat(errorPath)
if err == nil {
if !dirInfo.IsDir() {
runtime.Error("Error path %q is not a directory", errorPath)
}
} else {
runtime.StandardError(err)
}
// Load the content for the HTTP ``400``, ``500``, ``502`` and ``503``
// errors.
error400, error400Length = getErrorInfo(errorPath, "400.html")
error500, error500Length = getErrorInfo(errorPath, "500.html")
error502, error502Length = getErrorInfo(errorPath, "502.html")
error503, error503Length = getErrorInfo(errorPath, "503.html")
// Initialise the TLS config.
tlsconf.Init()
// Setup the file loggers.
var rotate int
switch *logRotate {
case "daily":
rotate = log.RotateDaily
case "hourly":
rotate = log.RotateHourly
case "never":
rotate = log.RotateNever
default:
runtime.Error("Unknown log rotation format %q", *logRotate)
}
_, err = log.AddFileLogger("live-server", logPath, rotate, log.InfoLog)
if err != nil {
runtime.Error("Couldn't initialise logfile: %s", err)
}
_, err = log.AddFileLogger("error", logPath, rotate, log.ErrorLog)
if err != nil {
runtime.Error("Couldn't initialise logfile: %s", err)
}
var liveMode bool
// Setup the live support as long as it hasn't been disabled.
if !*noLivequery {
go handleLiveMessages()
acceptorKey, err = ioutil.ReadFile(runtime.JoinPath(instanceDirectory, *acceptorKeyPath))
if err != nil {
runtime.StandardError(err)
}
cookieKey, err = ioutil.ReadFile(runtime.JoinPath(instanceDirectory, *cookieKeyPath))
if err != nil {
runtime.StandardError(err)
}
liveMode = true
_ = *livequeryHost
_ = *livequeryPort
_ = *cookieName
_ = *leaseExpiry
livequeryTimeout = (time.Duration(*livequeryExpiry) / 2) * 1000000000
}
// Create a container for the Frontend instances.
frontends := make([]*Frontend, 0)
// Create a channel which is used to toggle the state of the live-server's
// maintenance mode based on process signals.
maintenanceChannel := make(chan bool, 1)
// Fork a goroutine which toggles the maintenance mode in a single place and
// thus ensures "thread safety".
go func() {
for {
enabledState := <-maintenanceChannel
for _, frontend := range frontends {
if enabledState {
frontend.maintenanceMode = true
} else {
frontend.maintenanceMode = false
}
}
}
}()
// Register the signal handlers for SIGUSR1 and SIGUSR2.
runtime.SignalHandlers[os.SIGUSR1] = func() {
maintenanceChannel <- true
}
runtime.SignalHandlers[os.SIGUSR2] = func() {
maintenanceChannel <- false
}
// Let the user know how many CPUs we're currently running on.
fmt.Printf("Running live-server with %d CPUs:\n", runtime.CPUCount)
// If ``--public-address`` hasn't been specified, generate it from the given
// frontend host and base port values -- assuming ``localhost`` for a blank
// host.
publicHost := *officialHost
if publicHost == "" {
if *frontendHost == "" {
publicHost = fmt.Sprintf("localhost:%d", *frontendPort)
} else {
publicHost = fmt.Sprintf("%s:%d", *frontendHost, *frontendPort)
}
}
// Setup and run the primary HTTPS Frontend.
frontends = append(frontends, initFrontend("primary", *frontendHost,
*frontendPort, publicHost, *primaryHosts, *primaryCert, *primaryKey,
*cometPrefix, *websocketPrefix, instanceDirectory, *upstreamHost,
*upstreamPort, *upstreamTLS, *maintenanceMode, liveMode, staticCache,
staticFiles, staticMaxAge64))
// Setup and run the secondary HTTPS Frontend.
if !*noSecondary {
frontends = append(frontends, initFrontend("secondary", *frontendHost,
*frontendPort+1, publicHost, *secondaryHosts, *secondaryCert,
*secondaryKey, *cometPrefix, *websocketPrefix, instanceDirectory,
*upstreamHost, *upstreamPort, *upstreamTLS, *maintenanceMode,
liveMode, staticCache, staticFiles, staticMaxAge64))
}
// Enter a wait loop if the HTTP Redirector has been disabled.
if *noRedirect {
loopForever := make(chan bool, 1)
<-loopForever
}
// Otherwise, setup the HTTP Redirector.
if *httpHost == "" {
*httpHost = "localhost"
}
if *redirectURL == "" {
*redirectURL = "https://" + publicHost
}
httpAddr := fmt.Sprintf("%s:%d", *httpHost, *httpPort)
httpListener, err := net.Listen("tcp", httpAddr)
if err != nil {
runtime.Error("Cannot listen on %s: %v", httpAddr, err)
}
hsts := ""
if *enableHSTS {
hsts = fmt.Sprintf("max-age=%d", *hstsMaxAge)
}
redirector := &Redirector{
hsts: hsts,
pingPath: *pingPath,
url: *redirectURL,
}
// Start a goroutine which runs the HTTP redirector.
go func() {
err = http.Serve(httpListener, redirector)
if err != nil {
runtime.Error("Couldn't serve HTTP Redirector: %s", err)
}
}()
fmt.Printf("* HTTP Redirector running on http://%s:%d -> %s\n",
*httpHost, *httpPort, *redirectURL)
// Enter the wait loop for the process to be killed.
loopForever := make(chan bool, 1)
<-loopForever
}
func ampRun(argv []string, usage string) {
opts := optparse.Parser(
"Usage: amp run <instance-path> [options]\n\n " + usage + "\n")
profile := opts.String([]string{"--profile"}, "development",
"the config profile to use [development]", "NAME")
masterPath := opts.String([]string{"--master"}, "master",
"the path to the amp master node directory [master]", "PATH")
nodePath := opts.String([]string{"--node"}, "node",
"the path to the amp node directory [node]", "PATH")
frontendPath := opts.String([]string{"--frontend"}, "frontend",
"the path to the amp frontend directory [frontend]", "PATH")
noConsoleLog := opts.Bool([]string{"--no-console-log"}, false,
"disable output to stdout/stderr")
args := opts.Parse(argv)
if len(args) == 0 {
opts.PrintUsage()
runtime.Exit(0)
}
root, err := filepath.Abs(filepath.Clean(args[0]))
if err != nil {
runtime.StandardError(err)
}
amp, err := exec.LookPath("amp")
if err != nil {
runtime.StandardError(err)
}
config := *profile + ".yaml"
quit := make(chan bool, 1)
var console bool
if !*noConsoleLog {
logging.AddConsoleLogger()
console = true
}
runtime.Init()
ensureDirectory(root, "")
for _, spec := range [][]string{
{"master", ensureDirectory(root, *masterPath)},
{"node", ensureDirectory(root, *nodePath)},
{"frontend", ensureDirectory(root, *frontendPath)},
} {
go runProcess(amp, spec[0], spec[1], config, console, quit)
<-time.After(2000000000)
}
// Enter the wait loop for the process to be killed.
<-quit
}
func parseBoltfile(boltpath, boltdir string) (*Spec, error) {
extra := 0
// Open the Boltfile for parsing.
boltfile, err := os.Open(boltpath)
if err != nil {
runtime.StandardError(err)
}
defer boltfile.Close()
// Try and parse the Boltfile.
fileset := token.NewFileSet()
f, err := parser.ParseFile(fileset, "", boltfile, parser.ParseComments)
if err != nil {
// If `package main` has been omitted, auto-insert it.
if strings.Contains(err.Error(), "expected 'package'") {
extra += 1
buf := &bytes.Buffer{}
buf.Write([]byte("package main\n"))
boltfile.Seek(0, 0)
io.Copy(buf, boltfile)
fileset = token.NewFileSet()
f, err = parser.ParseFile(fileset, "", buf, parser.ParseComments)
if err != nil {
return nil, err
}
} else {
return nil, err
}
}
// Check if the `bolt` and `fmt` packages have been imported.
boltImported := false
fmtImported := false
for _, spec := range f.Imports {
path, err := strconv.Unquote(spec.Path.Value)
if err != nil {
continue
}
switch path {
case "bolt":
boltImported = true
case "fmt":
fmtImported = true
}
}
// If not, auto-insert them.
if !boltImported {
addImport(f, `"bolt"`)
extra += 1
}
if !fmtImported {
addImport(f, `"fmt"`)
extra += 1
}
buf := &bytes.Buffer{}
spec := &Spec{FileSet: fileset, Root: f, Extra: extra}
// Find all the tasks.
for _, decl := range f.Decls {
funcDecl, ok := decl.(*ast.FuncDecl)
if !ok {
continue
}
if funcDecl.Name.Name == "onload" {
spec.Init = true
insertBoltContext(funcDecl)
continue
}
if funcDecl.Doc == nil {
continue
}
doc := strings.TrimSpace(funcDecl.Doc.Text())
if doc == "" {
continue
}
task := &Task{FuncName: funcDecl.Name.Name}
id := task.FuncName
split := strings.SplitN(doc, " ", 2)
if first := split[0]; len(first) >= 2 {
idx := len(first) - 1
if first[idx] == ':' {
id = first[:idx]
if len(split) > 1 {
doc = split[1]
} else {
doc = ""
}
}
}
buf.Reset()
yaml.NormaliseID(buf, id)
task.Doc = doc
task.ID = buf.String()
spec.Tasks = append(spec.Tasks, task)
insertBoltContext(funcDecl)
}
// Rewrite the AST.
visitor := &Rewriter{}
ast.Walk(visitor, f)
return spec, nil
}
func (app *Application) ParseOpts() {
opts := app.Opts
conf := app.Config
conf.host = opts.StringConfig("weblite-host", "localhost",
"the host to bind this weblite server to [localhost]")
conf.port = opts.IntConfig("weblite-port", 8080,
"the port to bind this weblite server to [8080]")
conf.frontendHost = opts.StringConfig("frontend-host", "localhost",
"the frontend host to connect to [localhost]")
conf.frontendPort = opts.IntConfig("frontend-port", 9040,
"the frontend port to connect to [9040]")
conf.frontendConnections = opts.IntConfig("frontend-cxns", 5,
"the number of frontend connections to maintain [5]")
conf.frontendTLS = opts.BoolConfig("frontend-tls", false,
"use TLS when connecting to the frontend [false]")
conf.runDirectory = opts.StringConfig("run-dir", "run",
"the path to the run directory to store locks, pid files, etc. [run]")
conf.errorDirectory = opts.StringConfig("error-dir", "error",
"the path to the error templates directory [error]")
conf.logDirectory = opts.StringConfig("log-dir", "log",
"the path to the log directory [log]")
conf.logRotate = opts.StringConfig("log-rotate", "never",
"specify one of 'hourly', 'daily' or 'never' [never]")
conf.noConsoleLog = opts.BoolConfig("no-console-log", false,
"disable server requests being logged to the console [false]")
extraConfig := opts.StringConfig("extra-config", "",
"path to a YAML config file with additional options")
// Parse the command line options.
os.Args[0] = app.Name
args := opts.Parse(os.Args)
// Print the default YAML config file if the ``-g`` flag was specified.
if *conf.genConfig {
opts.PrintDefaultConfigFile()
runtime.Exit(0)
}
var instanceDirectory string
// Assume the parent directory of the config as the instance directory.
if len(args) >= 1 {
if args[0] == "help" {
opts.PrintUsage()
runtime.Exit(0)
}
configPath, err := filepath.Abs(filepath.Clean(args[0]))
if err != nil {
runtime.StandardError(err)
}
err = opts.ParseConfig(configPath, os.Args)
if err != nil {
runtime.StandardError(err)
}
instanceDirectory, _ = filepath.Split(configPath)
} else {
opts.PrintUsage()
runtime.Exit(0)
}
// Load the extra config file with additional options if one has been
// specified.
if *extraConfig != "" {
extraConfigPath, err := filepath.Abs(filepath.Clean(*extraConfig))
if err != nil {
runtime.StandardError(err)
}
extraConfigPath = runtime.JoinPath(instanceDirectory, extraConfigPath)
err = opts.ParseConfig(extraConfigPath, os.Args)
if err != nil {
runtime.StandardError(err)
}
}
// Set the debug mode flag if the ``-d`` flag was specified.
app.Debug = *conf.debug
app.Path = instanceDirectory
for _, hook := range app.Hooks {
hook()
}
}
func main() {
// Setup temporary console logging.
log.DisableConsoleTimestamp()
log.AddConsoleLogger()
// Set default values for command-line params.
boltFilename := "Boltfile"
genExecutablePath := ""
recompile := false
skipNext := true
maxIdx := len(os.Args) - 1
newArgs := []string{"bolt"}
// Extract higher-level command-line arguments.
for idx, arg := range os.Args {
if skipNext {
skipNext = false
continue
}
if arg == "--gen" && idx != maxIdx {
var err error
genExecutablePath, err = filepath.Abs(os.Args[idx+1])
if err != nil {
runtime.StandardError(err)
}
skipNext = true
} else if arg == "--boltfile" && idx != maxIdx {
boltFilename = os.Args[idx+1]
skipNext = true
} else if arg == "--recompile" {
recompile = true
} else {
newArgs = append(newArgs, arg)
}
}
// Try and find the directory containing the Boltfile.
boltdir, err := findBoltDir(boltFilename)
if err != nil {
if _, ok := err.(*fsutil.NotFound); ok {
log.Error("Couldn't find Boltfile")
runtime.Exit(1)
}
runtime.StandardError(err)
}
// Generate the path to the corresponding temp directory.
boltpath := filepath.Join(boltdir, boltFilename)
hash := sha1.New()
hash.Write([]byte(boltpath))
digest := fmt.Sprintf("%x", hash.Sum(nil))
tempdir := filepath.Join(os.TempDir(), "bolt-"+digest)
// See if the temp directory exists and if not create it.
exists, err := fsutil.Exists(tempdir)
if !exists {
if _, ok := err.(*fsutil.NotFound); !ok {
runtime.Error("Couldn't access the temp directory: %s: %s", tempdir, err)
}
err = os.Mkdir(tempdir, 0744)
if err != nil {
runtime.Error("Couldn't create the temp directory: %s: %s", tempdir, err)
}
}
// See if an up-to-date generated binary already exists and, if so, run it.
binpath := filepath.Join(tempdir, "bolt")
if !recompile {
boltstat, _ := os.Stat(boltpath)
if genExecutablePath == "" {
binstat, err := os.Stat(binpath)
if err == nil {
if boltstat.ModTime().Before(binstat.ModTime()) {
runBoltExecutable(binpath, boltdir, newArgs)
return
}
}
}
}
// Parse the Boltfile.
spec, err := parseBoltfile(boltpath, boltdir)
if err != nil {
exitForParserErrors(boltFilename, err)
}
// Exit if no tasks were found.
if len(spec.Tasks) == 0 {
runtime.Error("No tasks were found in %s", boltpath)
}
// Fudge the path to the executable that needs to be generated depending on
// whether --gen-executable was specified or not.
genOnly := true
if genExecutablePath == "" {
genExecutablePath = binpath
genOnly = false
}
// Generate the executable.
err = genExecutable(genExecutablePath, tempdir, spec)
if err != nil {
runtime.StandardError(err)
}
// Exit early if --gen-executable was specified.
if genOnly {
log.Info("%s successfully compiled to %s", boltFilename, genExecutablePath)
runtime.Exit(0)
}
// Otherwise, run the executable.
runBoltExecutable(binpath, boltdir, newArgs)
}
func main() {
// Define the options for the command line and config file options parser.
opts := optparse.Parser(
"Usage: wifistat <config.yaml> [options]\n",
"wifistat 0.0.1")
addr := opts.StringConfig("addr", ":9040",
"the host:port address for the web server [:9040]")
csv := opts.StringConfig("csv-dir", "csv",
"the path to the csv files directory [csv]")
wifi := opts.StringConfig("wifi-logs-dir", "iaslogs",
"the path to the Wi-Fi logs directory [iaslogs]")
membership := opts.BoolConfig("member-analytics", false,
"enable membership-based analytics")
devices := opts.StringConfig("devices-url", "",
"the url key for the devices.csv Google Spreadsheet")
members := opts.StringConfig("members-url", "",
"the url key for the members.csv Google Spreadsheet")
opening := opts.StringConfig("opening-url", "",
"the url key for the opening.csv Google Spreadsheet")
// Parse the command line options.
os.Args[0] = "wifistat"
_, root, _ := runtime.DefaultOpts("wifistat", opts, os.Args)
// Compute option variables.
wifiLogDir = runtime.JoinPath(root, *wifi)
csvDir = runtime.JoinPath(root, *csv)
err := os.MkdirAll(csvDir, 0755)
if err != nil {
runtime.StandardError(err)
}
// Handle member analytics options.
if *membership {
enableMemberAnalytics = true
devicesUrlKey = *devices
if devicesUrlKey == "" {
runtime.Error("You need to specify the `devices-url` command-line option.")
}
membersUrlKey = *members
if membersUrlKey == "" {
runtime.Error("You need to specify the `members-url` command-line option.")
}
openingUrlKey = *opening
if openingUrlKey == "" {
runtime.Error("You need to specify the `opening-url` command-line option.")
}
}
// Parse the logs.
parseCsv(false)
parseWifi()
// Register the various handlers.
http.HandleFunc("/", handleRequest)
http.HandleFunc("/reload", handleReload)
// Start the web server.
log.Info("Running wifistat on %s", *addr)
err = http.ListenAndServe(*addr, nil)
if err != nil {
runtime.StandardError(err)
}
runtime.Exit(0)
}
func ampFrontend(argv []string, usage string) {
// Define the options for the command line and config file options parser.
opts := optparse.Parser(
"Usage: amp frontend <config.yaml> [options]\n\n " + usage + "\n")
httpsHost := opts.StringConfig("https-host", "",
"the host to bind the HTTPS Frontends to")
httpsPort := opts.IntConfig("https-port", 9040,
"the base port for the HTTPS Frontends [9040]")
officialHost := opts.StringConfig("offficial-host", "",
"the official public host for the HTTPS Frontends")
primaryHosts := opts.StringConfig("primary-hosts", "",
"limit the primary HTTPS Frontend to the specified host pattern")
primaryCert := opts.StringConfig("primary-cert", "cert/primary.cert",
"the path to the primary host's TLS certificate [cert/primary.cert]")
primaryKey := opts.StringConfig("primary-key", "cert/primary.key",
"the path to the primary host's TLS key [cert/primary.key]")
noSecondary := opts.BoolConfig("no-secondary", false,
"disable the secondary HTTPS Frontend [false]")
secondaryHosts := opts.StringConfig("secondary-hosts", "",
"limit the secondary HTTPS Frontend to the specified host pattern")
secondaryCert := opts.StringConfig("secondary-cert", "cert/secondary.cert",
"the path to the secondary host's TLS certificate [cert/secondary.cert]")
secondaryKey := opts.StringConfig("secondary-key", "cert/secondary.key",
"the path to the secondary host's TLS key [cert/secondary.key]")
errorDirectory := opts.StringConfig("error-dir", "error",
"the path to the HTTP error files directory [error]")
staticDirectory := opts.StringConfig("static-dir", "www",
"the path to the static files directory [www]")
staticMaxAge := opts.IntConfig("static-max-age", 86400,
"max-age cache header value when serving the static files [86400]")
hstsMaxAge := opts.IntConfig("hsts-max-age", 50000000,
"max-age in seconds for HTTP Strict Transport Security [50000000]")
noRedirect := opts.BoolConfig("no-redirect", false,
"disable the HTTP Redirector [false]")
httpHost := opts.StringConfig("http-host", "",
"the host to bind the HTTP Redirector to")
httpPort := opts.IntConfig("http-port", 9080,
"the port to bind the HTTP Redirector to [9080]")
httpRedirectURL := opts.StringConfig("redirect-url", "",
"the URL that the HTTP Redirector redirects to")
singleNode := opts.StringConfig("single-node", "",
"the upstream single node address if running without a master")
masterNodes := opts.StringConfig("master-nodes", "localhost:8060",
"comma-separated addresses of amp master nodes [localhost:8060]")
masterCert := opts.StringConfig("master-cert", "cert/master.cert",
"the path to the amp master certificate [cert/master.cert]")
ironKeyPath := opts.StringConfig("iron-key", "cert/iron.key",
"the path to the key used for iron strings [cert/iron.key]")
maintenanceMode := opts.BoolConfig("maintenance", false,
"start up in maintenance mode [false]")
_, instanceDirectory, _ := runtime.DefaultOpts("frontend", opts, argv)
// Ensure that the directory containing static files exists.
staticPath := runtime.JoinPath(instanceDirectory, *staticDirectory)
dirInfo, err := os.Stat(staticPath)
if err == nil {
if !dirInfo.IsDirectory() {
runtime.Error("%q is not a directory", staticPath)
}
} else {
runtime.StandardError(err)
}
// Ensure that the directory containing error files exists.
errorPath := runtime.JoinPath(instanceDirectory, *errorDirectory)
dirInfo, err = os.Stat(errorPath)
if err == nil {
if !dirInfo.IsDirectory() {
runtime.Error("%q is not a directory", errorPath)
}
} else {
runtime.StandardError(err)
}
// If ``--official-host`` hasn't been specified, generate it from the given
// frontend host and base port values -- assuming ``localhost`` for a blank
// host.
publicHost := *officialHost
if publicHost == "" {
if *httpsHost == "" {
publicHost = fmt.Sprintf("localhost:%d", *httpsPort)
} else {
publicHost = fmt.Sprintf("%s:%d", *httpsHost, *httpsPort)
}
}
// Compute the HSTS max age header value.
hsts := fmt.Sprintf("max-age=%d", *hstsMaxAge)
// Pre-format the Cache-Control header for static files.
staticCache := fmt.Sprintf("public, max-age=%d", *staticMaxAge)
staticMaxAge64 := int64(*staticMaxAge)
// Compute the variables related to redirects.
redirectURL := "https://" + publicHost
redirectHTML := []byte(fmt.Sprintf(
`Please <a href="%s">click here if your browser doesn't redirect</a> automatically.`,
redirectURL))
// Compute the path to the Iron key.
ironPath := runtime.JoinPath(instanceDirectory, *ironKeyPath)
// Instantiate the master client.
masterClient, err := master.NewClient(
*masterNodes, runtime.JoinPath(instanceDirectory, *masterCert))
if err != nil {
runtime.StandardError(err)
}
var noMaster bool
if *singleNode != "" {
noMaster = true
}
// Let the user know how many CPUs we're currently running on.
log.Info("Running the Amp Frontend on %d CPUs.", runtime.CPUCount)
// Initialise the TLS config.
tlsconf.Init()
// Initialise a container for the HTTPSFrontends.
webFrontends := make([]*server.HTTPSFrontend, 1)
// Compute the variables related to detecting valid hosts.
primaryWildcard, primaryAddr := getValidAddr(*primaryHosts)
secondaryWildcard, secondaryAddr := getValidAddr(*secondaryHosts)
// Instantiate the primary ``HTTPSFrontend`` object.
frontend := &server.HTTPSFrontend{
HSTS: hsts,
MaintenanceMode: *maintenanceMode,
MasterClient: masterClient,
NoMaster: noMaster,
RedirectHTML: redirectHTML,
RedirectURL: redirectURL,
SingleNode: *singleNode,
StaticCache: staticCache,
StaticMaxAge: staticMaxAge64,
ValidAddress: primaryAddr,
ValidWildcard: primaryWildcard,
}
frontend.LoadAssets(errorPath, ironPath, staticPath)
frontend.Run(*httpsHost, *httpsPort,
runtime.JoinPath(instanceDirectory, *primaryCert),
runtime.JoinPath(instanceDirectory, *primaryKey))
webFrontends[0] = frontend
// Setup and run the secondary HTTPSFrontend.
if !*noSecondary {
frontend = &server.HTTPSFrontend{
HSTS: hsts,
MaintenanceMode: *maintenanceMode,
MasterClient: masterClient,
NoMaster: noMaster,
RedirectHTML: redirectHTML,
RedirectURL: redirectURL,
SingleNode: *singleNode,
StaticCache: staticCache,
StaticMaxAge: staticMaxAge64,
ValidAddress: secondaryAddr,
ValidWildcard: secondaryWildcard,
}
frontend.LoadAssets(errorPath, ironPath, staticPath)
frontend.Run(*httpsHost, *httpsPort+1,
runtime.JoinPath(instanceDirectory, *secondaryCert),
runtime.JoinPath(instanceDirectory, *secondaryKey))
webFrontends = append(webFrontends, frontend)
}
// Create a channel which is used to toggle maintenance mode based on
// process signals.
maintenanceChannel := make(chan bool, 1)
// Fork a goroutine which toggles the maintenance mode in a single place and
// thus ensures thread safety.
go func() {
for {
enabledState := <-maintenanceChannel
for _, frontend := range webFrontends {
if enabledState {
frontend.MaintenanceMode = true
} else {
frontend.LoadAssets(errorPath, ironPath, staticPath)
frontend.MaintenanceMode = false
}
}
}
}()
// Register the signal handlers for SIGUSR1 and SIGUSR2.
runtime.SignalHandlers[os.SIGUSR1] = func() {
maintenanceChannel <- true
}
runtime.SignalHandlers[os.SIGUSR2] = func() {
maintenanceChannel <- false
}
// Enter a wait loop if the HTTP Redirector has been disabled.
if *noRedirect {
loopForever := make(chan bool, 1)
<-loopForever
}
// Otherwise, setup and run the HTTP Redirector.
if *httpHost == "" {
*httpHost = "localhost"
}
if *httpRedirectURL == "" {
*httpRedirectURL = "https://" + publicHost
}
redirector := &server.HTTPRedirector{*httpRedirectURL}
redirector.Run(*httpHost, *httpPort)
// Enter the wait loop for the process to be killed.
loopForever := make(chan bool, 1)
<-loopForever
}