// The subscribe message is of the format::
//
// <sqid> [<key>, ...] [<key>, ...]
//
// The two sets of keys are assumed to be disjoint sets, i.e. they have no
// elements in common.
func subscribe(message []byte) {
buf := bytes.NewBuffer(message)
dec := &ArgoDecoder{buf, make([]byte, 20)}
sqid, err := dec.ReadString()
if err != nil {
log.Error("Error decoding X-Live Subscribe SQID %s %s", message, err)
return
}
if sqid == "" {
return
}
keys1, err := dec.ReadStringSlice()
if err != nil {
log.Error("Error decoding X-Live Subscribe Keys-1 %s %s", message, err)
return
}
var keys2 []string
if buf.Len() > 0 {
keys2, err = dec.ReadStringSlice()
if err != nil {
log.Error("Error decoding X-Live Subscribe Keys-2 %s %s", message, err)
return
}
}
pubsub.Subscribe(sqid, keys1, keys2)
}
func handleWebSocket(conn *websocket.Conn) {
respStatus := http.StatusOK
defer func() {
conn.Close()
logRequest(HTTPS_WEBSOCKET, respStatus, conn.Request().Host, conn.Request())
}()
request := make([]byte, 1024)
for {
n, err := conn.Read(request)
if err != nil {
if debugMode {
log.Error("Error reading on WebSocket: %s", err)
}
break
}
response, status := getLiveItems(string(request[:n]))
if status != http.StatusOK {
respStatus = status
break
}
if len(response) != 0 {
if _, err = conn.Write(response); err != nil {
break
}
}
}
}
// The publish message is of the format::
//
// <item-id> [<key>, ...]
//
func publish(message []byte) {
buf := bytes.NewBuffer(message)
dec := &ArgoDecoder{buf, make([]byte, 20)}
itemID, err := dec.ReadString()
if err != nil {
log.Error("Error decoding X-Live Publish ItemID %s %s", message, err)
return
}
if itemID == "" {
return
}
keys, err := dec.ReadStringSlice()
if err != nil {
log.Error("Error decoding X-Live Publish Keys %s %s", message, err)
return
}
pubsub.Publish(itemID, keys)
}
func getCsvFile(name, urlpath string, timestamp time.Time, force bool) *csv.Reader {
file, err := loadCsvFile(name, urlpath, timestamp, force)
if err != nil {
log.Error("Couldn't load %s.csv: %s", name, err)
if !force {
runtime.Exit(1)
}
return nil
}
_, err = file.Read()
if err != nil {
log.Error("Error parsing header line from %s.csv: %s", name, err)
if !force {
runtime.Exit(1)
}
return nil
}
return file
}
func exitForParserErrors(filename string, err error) {
if err, ok := err.(scanner.ErrorList); ok {
for _, e := range err {
log.Error("%s:%s", filename, e)
}
runtime.Exit(1)
} else {
runtime.StandardError(err)
}
}
func handleLiveMessages() {
for message := range liveChannel {
cmd := message[0]
switch cmd {
case 0:
go publish(message[1:])
case 1:
go subscribe(message[1:])
default:
log.Error("Got unexpected X-Live payload: %s", message)
}
}
}
func getLiveItems(request string) ([]byte, int) {
reqs := strings.Split(request, ",")
if len(reqs) == 0 {
return nil, http.StatusBadRequest
}
sid := reqs[0] /* XXX validate sid */
result, refresh, ok := pubsub.Listen(sid, reqs[1:], livequeryTimeout)
if ok {
data := make(map[string]map[string][]string)
data["items"] = result
response, err := json.Marshal(data)
if err != nil {
log.Error("ERROR encoding JSON: %s for: %v", err, data)
return nil, http.StatusInternalServerError
}
return response, http.StatusOK
}
refreshCount := len(refresh)
if refreshCount != 0 {
data := make(map[string][]string)
qids := make([]string, refreshCount)
i := 0
for qid, _ := range refresh {
qids[i] = qid
i++
}
data["refresh"] = qids
response, err := json.Marshal(data)
if err != nil {
log.Error("ERROR encoding JSON: %s for: %v", err, data)
return nil, http.StatusInternalServerError
}
return response, http.StatusOK
}
return nil, http.StatusNotFound
}
func parseCsv(force bool) {
now := time.Now()
file := getCsvFile("devices", devicesUrlKey, now, force)
if file == nil {
return
}
var (
line []string
err error
)
log.Info("Parsing devices.csv")
for {
line, err = file.Read()
if err != nil {
if err == io.EOF {
break
}
log.Error("Error parsing devices.csv: %s", err)
return
}
_ = line
}
// file = getCsvFile("members", membersUrlKey, now, force)
// if file == nil {
// return
// }
// log.Info("Parsing devices.csv")
// for {
// line, err = file.Read()
// if err != nil {
// if err == io.EOF {
// break
// }
// log.Error("Error parsing devices.csv: %s", err)
// return
// }
// _ = line
// }
// file = getCsvFile("opening", openingUrlKey, now, force)
// if file == nil {
// return
// }
}
func (frontend *Frontend) ServeHTTP(conn http.ResponseWriter, req *http.Request) {
originalHost := req.Host
// Set default headers.
headers := conn.Header()
headers.Set("X-Content-Type-Options", "nosniff")
// Redirect all requests to the "official" public host if the Host header
// doesn't match.
if !frontend.isValidHost(originalHost) {
headers.Set("Location", frontend.redirectURL)
conn.WriteHeader(http.StatusMovedPermanently)
conn.Write(frontend.redirectHTML)
logRequest(HTTPS_REDIRECT, http.StatusMovedPermanently, originalHost, req)
return
}
// Return the HTTP 503 error page if we're in maintenance mode.
if frontend.maintenanceMode {
headers.Set(contentType, textHTML)
headers.Set(contentLength, error503Length)
conn.WriteHeader(http.StatusServiceUnavailable)
conn.Write(error503)
logRequest(HTTPS_MAINTENANCE, http.StatusServiceUnavailable, originalHost, req)
return
}
reqPath := req.URL.Path
// Handle requests for any files exposed within the static directory.
if staticFile, ok := frontend.staticFiles[reqPath]; ok {
expires := time.Now().Add(frontend.staticMaxAge)
headers.Set("Expires", expires.Format(http.TimeFormat))
headers.Set("Cache-Control", frontend.staticCache)
headers.Set("Etag", staticFile.etag)
if req.Header.Get("If-None-Match") == staticFile.etag {
conn.WriteHeader(http.StatusNotModified)
logRequest(HTTPS_STATIC, http.StatusNotModified, originalHost, req)
return
}
// Special case /.well-known/oauth.json?callback= requests.
if reqPath == "/.well-known/oauth.json" && req.URL.RawQuery != "" {
query, err := url.ParseQuery(req.URL.RawQuery)
if err != nil {
log.Error("Error parsing oauth.json query string %q: %s",
req.URL.RawQuery, err)
serveError400(conn, originalHost, req)
return
}
if callbackList, found := query["callback"]; found {
callback := callbackList[0]
if callback != "" {
respLen := len(callback) + len(staticFile.content) + 2
headers.Set(contentType, "text/javascript")
headers.Set(contentLength, fmt.Sprintf("%d", respLen))
conn.WriteHeader(http.StatusOK)
conn.Write([]byte(callback))
conn.Write([]byte{'('})
conn.Write(staticFile.content)
conn.Write([]byte{')'})
logRequest(HTTPS_STATIC, http.StatusOK, originalHost, req)
return
}
}
}
headers.Set(contentType, staticFile.mimetype)
headers.Set(contentLength, staticFile.size)
conn.WriteHeader(http.StatusOK)
conn.Write(staticFile.content)
logRequest(HTTPS_STATIC, http.StatusOK, originalHost, req)
return
}
if frontend.liveMode {
// Handle WebSocket requests.
if strings.HasPrefix(reqPath, frontend.websocketPrefix) {
websocket.Handler(handleWebSocket).ServeHTTP(conn, req)
return
}
// Handle long-polling Comet requests.
if strings.HasPrefix(reqPath, frontend.cometPrefix) {
query, err := url.ParseQuery(req.URL.RawQuery)
if err != nil {
log.Error("Error parsing Comet query string %q: %s",
req.URL.RawQuery, err)
serveError400(conn, originalHost, req)
return
}
queryReq, found := query["q"]
if !found {
serveError400(conn, originalHost, req)
return
}
response, status := getLiveItems(queryReq[0])
headers.Set(contentType, applicationJSON)
headers.Set(contentLength, fmt.Sprintf("%d", len(response)))
conn.WriteHeader(status)
conn.Write(response)
logRequest(HTTPS_COMET, status, originalHost, req)
return
}
}
// Open a connection to the upstream server.
upstreamConn, err := net.Dial("tcp", frontend.upstreamAddr)
if err != nil {
log.Error("Couldn't connect to upstream: %s", err)
serveError502(conn, originalHost, req)
return
}
var clientIP string
var upstream net.Conn
splitPoint := strings.LastIndex(req.RemoteAddr, ":")
if splitPoint == -1 {
clientIP = req.RemoteAddr
} else {
clientIP = req.RemoteAddr[0:splitPoint]
}
if frontend.upstreamTLS {
upstream = tls.Client(upstreamConn, tlsconf.Config)
defer upstream.Close()
} else {
upstream = upstreamConn
}
// Modify the request Host: and User-Agent: headers.
req.Host = frontend.upstreamHost
req.Header.Set(
"User-Agent",
fmt.Sprintf("%s, %s, %s", req.UserAgent(), clientIP, originalHost))
// Send the request to the upstream server.
err = req.Write(upstream)
if err != nil {
log.Error("Error writing to the upstream server: %s", err)
serveError502(conn, originalHost, req)
return
}
// Parse the response from upstream.
resp, err := http.ReadResponse(bufio.NewReader(upstream), req)
if err != nil {
log.Error("Error parsing response from upstream: %s", err)
serveError502(conn, originalHost, req)
return
}
defer resp.Body.Close()
// Set a variable to hold the X-Live header value if present.
var liveLength int
if frontend.liveMode {
xLive := resp.Header.Get("X-Live")
if xLive != "" {
// If the X-Live header was set, parse it into an int.
liveLength, err = strconv.Atoi(xLive)
if err != nil {
log.Error("Error converting X-Live header value %q: %s", xLive, err)
serveError500(conn, originalHost, req)
return
}
resp.Header.Del("X-Live")
}
}
var body []byte
if liveLength > 0 {
var gzipSet bool
var respBody io.ReadCloser
// Check Content-Encoding to see if upstream sent gzipped content.
if resp.Header.Get("Content-Encoding") == "gzip" {
gzipSet = true
respBody, err = gzip.NewReader(resp.Body)
if err != nil {
log.Error("Error reading gzipped response from upstream: %s", err)
serveError500(conn, originalHost, req)
return
}
defer respBody.Close()
} else {
respBody = resp.Body
}
// Read the X-Live content from the response body.
liveMessage := make([]byte, liveLength)
n, err := respBody.Read(liveMessage)
if n != liveLength || err != nil {
log.Error("Error reading X-Live response from upstream: %s", err)
serveError500(conn, originalHost, req)
return
}
// Read the response to send back to the original request.
body, err = ioutil.ReadAll(respBody)
if err != nil {
log.Error("Error reading non X-Live response from upstream: %s", err)
serveError500(conn, originalHost, req)
return
}
// Re-encode the response if it had been gzipped by upstream.
if gzipSet {
buffer := &bytes.Buffer{}
encoder, err := gzip.NewWriter(buffer)
if err != nil {
log.Error("Error creating a new gzip Writer: %s", err)
serveError500(conn, originalHost, req)
return
}
n, err = encoder.Write(body)
if n != len(body) || err != nil {
log.Error("Error writing to the gzip Writer: %s", err)
serveError500(conn, originalHost, req)
return
}
err = encoder.Close()
if err != nil {
log.Error("Error finalising the write to the gzip Writer: %s", err)
serveError500(conn, originalHost, req)
return
}
body = buffer.Bytes()
}
resp.Header.Set(contentLength, fmt.Sprintf("%d", len(body)))
liveChannel <- liveMessage
} else {
// Read the full response body.
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
log.Error("Error reading response from upstream: %s", err)
serveError502(conn, originalHost, req)
return
}
}
// Set the received headers back to the initial connection.
for k, values := range resp.Header {
for _, v := range values {
headers.Add(k, v)
}
}
// Write the response body back to the initial connection.
conn.WriteHeader(resp.StatusCode)
conn.Write(body)
logRequest(HTTPS_UPSTREAM, resp.StatusCode, originalHost, req)
}
func parseWifi() {
if parsedAlready {
log.Info("Re-parsing logs: %s", wifiLogDir)
} else {
log.Info("Parsing logs: %s", wifiLogDir)
parsedAlready = true
}
files, err := ioutil.ReadDir(wifiLogDir)
if err != nil {
log.StandardError(err)
return
}
if len(files) == 0 {
log.Error("No files found to parse in %s", wifiLogDir)
return
}
var (
c, l int
current []byte
file *os.File
filename string
isPrefix bool
key string
line []byte
reader *bufio.Reader
received int64
sent int64
session string
split []string
status string
// pending map[string]string
)
startTime := time.Now()
var i int
for _, info := range files {
filename = filepath.Join(wifiLogDir, info.Name())
log.Info("Parsing: %s", filename)
file, err = os.Open(filename)
if err != nil {
log.StandardError(err)
return
}
reader = bufio.NewReader(file)
for {
line, isPrefix, err = reader.ReadLine()
if err != nil {
if err == io.EOF {
break
}
log.StandardError(err)
return
}
current = append(current, line...)
if isPrefix {
continue
}
split = strings.Split(string(current), ",")
l = len(split)
c = 6
session = ""
status = ""
received = 0
sent = 0
for c < l {
key = split[c]
c += 2
switch key {
case "40":
status = split[c+1]
case "42":
received, _ = strconv.ParseInt(split[c+1], 10, 64)
case "43":
sent, _ = strconv.ParseInt(split[c+1], 10, 64)
case "44":
session = split[c+1]
}
// log.Info("key: %s", key)
}
_ = session
_ = status
_ = received
_ = sent
current = current[0:0]
i += 1
if i >= 0 {
break
}
}
break
}
log.Info("Finished parsing (%s)", time.Since(startTime))
}
func main() {
// Setup temporary console logging.
log.DisableConsoleTimestamp()
log.AddConsoleLogger()
// Set default values for command-line params.
boltFilename := "Boltfile"
genExecutablePath := ""
recompile := false
skipNext := true
maxIdx := len(os.Args) - 1
newArgs := []string{"bolt"}
// Extract higher-level command-line arguments.
for idx, arg := range os.Args {
if skipNext {
skipNext = false
continue
}
if arg == "--gen" && idx != maxIdx {
var err error
genExecutablePath, err = filepath.Abs(os.Args[idx+1])
if err != nil {
runtime.StandardError(err)
}
skipNext = true
} else if arg == "--boltfile" && idx != maxIdx {
boltFilename = os.Args[idx+1]
skipNext = true
} else if arg == "--recompile" {
recompile = true
} else {
newArgs = append(newArgs, arg)
}
}
// Try and find the directory containing the Boltfile.
boltdir, err := findBoltDir(boltFilename)
if err != nil {
if _, ok := err.(*fsutil.NotFound); ok {
log.Error("Couldn't find Boltfile")
runtime.Exit(1)
}
runtime.StandardError(err)
}
// Generate the path to the corresponding temp directory.
boltpath := filepath.Join(boltdir, boltFilename)
hash := sha1.New()
hash.Write([]byte(boltpath))
digest := fmt.Sprintf("%x", hash.Sum(nil))
tempdir := filepath.Join(os.TempDir(), "bolt-"+digest)
// See if the temp directory exists and if not create it.
exists, err := fsutil.Exists(tempdir)
if !exists {
if _, ok := err.(*fsutil.NotFound); !ok {
runtime.Error("Couldn't access the temp directory: %s: %s", tempdir, err)
}
err = os.Mkdir(tempdir, 0744)
if err != nil {
runtime.Error("Couldn't create the temp directory: %s: %s", tempdir, err)
}
}
// See if an up-to-date generated binary already exists and, if so, run it.
binpath := filepath.Join(tempdir, "bolt")
if !recompile {
boltstat, _ := os.Stat(boltpath)
if genExecutablePath == "" {
binstat, err := os.Stat(binpath)
if err == nil {
if boltstat.ModTime().Before(binstat.ModTime()) {
runBoltExecutable(binpath, boltdir, newArgs)
return
}
}
}
}
// Parse the Boltfile.
spec, err := parseBoltfile(boltpath, boltdir)
if err != nil {
exitForParserErrors(boltFilename, err)
}
// Exit if no tasks were found.
if len(spec.Tasks) == 0 {
runtime.Error("No tasks were found in %s", boltpath)
}
// Fudge the path to the executable that needs to be generated depending on
// whether --gen-executable was specified or not.
genOnly := true
if genExecutablePath == "" {
genExecutablePath = binpath
genOnly = false
}
// Generate the executable.
err = genExecutable(genExecutablePath, tempdir, spec)
if err != nil {
runtime.StandardError(err)
}
// Exit early if --gen-executable was specified.
if genOnly {
log.Info("%s successfully compiled to %s", boltFilename, genExecutablePath)
runtime.Exit(0)
}
// Otherwise, run the executable.
runBoltExecutable(binpath, boltdir, newArgs)
}
func Error(format string, v ...interface{}) {
log.Error(format, v...)
Exit(1)
}