Ejemplo n.º 1
0
// The subscribe message is of the format::
//
//     <sqid> [<key>, ...] [<key>, ...]
//
// The two sets of keys are assumed to be disjoint sets, i.e. they have no
// elements in common.
func subscribe(message []byte) {
	buf := bytes.NewBuffer(message)
	dec := &ArgoDecoder{buf, make([]byte, 20)}
	sqid, err := dec.ReadString()
	if err != nil {
		log.Error("Error decoding X-Live Subscribe SQID %s %s", message, err)
		return
	}
	if sqid == "" {
		return
	}
	keys1, err := dec.ReadStringSlice()
	if err != nil {
		log.Error("Error decoding X-Live Subscribe Keys-1 %s %s", message, err)
		return
	}
	var keys2 []string
	if buf.Len() > 0 {
		keys2, err = dec.ReadStringSlice()
		if err != nil {
			log.Error("Error decoding X-Live Subscribe Keys-2 %s %s", message, err)
			return
		}
	}
	pubsub.Subscribe(sqid, keys1, keys2)
}
Ejemplo n.º 2
0
func handleWebSocket(conn *websocket.Conn) {
	respStatus := http.StatusOK
	defer func() {
		conn.Close()
		logRequest(HTTPS_WEBSOCKET, respStatus, conn.Request().Host, conn.Request())
	}()
	request := make([]byte, 1024)
	for {
		n, err := conn.Read(request)
		if err != nil {
			if debugMode {
				log.Error("Error reading on WebSocket: %s", err)
			}
			break
		}
		response, status := getLiveItems(string(request[:n]))
		if status != http.StatusOK {
			respStatus = status
			break
		}
		if len(response) != 0 {
			if _, err = conn.Write(response); err != nil {
				break
			}
		}
	}
}
Ejemplo n.º 3
0
// The publish message is of the format::
//
//     <item-id> [<key>, ...]
//
func publish(message []byte) {
	buf := bytes.NewBuffer(message)
	dec := &ArgoDecoder{buf, make([]byte, 20)}
	itemID, err := dec.ReadString()
	if err != nil {
		log.Error("Error decoding X-Live Publish ItemID %s %s", message, err)
		return
	}
	if itemID == "" {
		return
	}
	keys, err := dec.ReadStringSlice()
	if err != nil {
		log.Error("Error decoding X-Live Publish Keys %s %s", message, err)
		return
	}
	pubsub.Publish(itemID, keys)
}
Ejemplo n.º 4
0
func getCsvFile(name, urlpath string, timestamp time.Time, force bool) *csv.Reader {
	file, err := loadCsvFile(name, urlpath, timestamp, force)
	if err != nil {
		log.Error("Couldn't load %s.csv: %s", name, err)
		if !force {
			runtime.Exit(1)
		}
		return nil
	}
	_, err = file.Read()
	if err != nil {
		log.Error("Error parsing header line from %s.csv: %s", name, err)
		if !force {
			runtime.Exit(1)
		}
		return nil
	}
	return file
}
Ejemplo n.º 5
0
Archivo: bolt.go Proyecto: tav/bolt
func exitForParserErrors(filename string, err error) {
	if err, ok := err.(scanner.ErrorList); ok {
		for _, e := range err {
			log.Error("%s:%s", filename, e)
		}
		runtime.Exit(1)
	} else {
		runtime.StandardError(err)
	}
}
Ejemplo n.º 6
0
func handleLiveMessages() {
	for message := range liveChannel {
		cmd := message[0]
		switch cmd {
		case 0:
			go publish(message[1:])
		case 1:
			go subscribe(message[1:])
		default:
			log.Error("Got unexpected X-Live payload: %s", message)
		}
	}
}
Ejemplo n.º 7
0
func getLiveItems(request string) ([]byte, int) {
	reqs := strings.Split(request, ",")
	if len(reqs) == 0 {
		return nil, http.StatusBadRequest
	}
	sid := reqs[0] /* XXX validate sid */
	result, refresh, ok := pubsub.Listen(sid, reqs[1:], livequeryTimeout)
	if ok {
		data := make(map[string]map[string][]string)
		data["items"] = result
		response, err := json.Marshal(data)
		if err != nil {
			log.Error("ERROR encoding JSON: %s for: %v", err, data)
			return nil, http.StatusInternalServerError
		}
		return response, http.StatusOK
	}
	refreshCount := len(refresh)
	if refreshCount != 0 {
		data := make(map[string][]string)
		qids := make([]string, refreshCount)
		i := 0
		for qid, _ := range refresh {
			qids[i] = qid
			i++
		}
		data["refresh"] = qids
		response, err := json.Marshal(data)
		if err != nil {
			log.Error("ERROR encoding JSON: %s for: %v", err, data)
			return nil, http.StatusInternalServerError
		}
		return response, http.StatusOK
	}
	return nil, http.StatusNotFound
}
Ejemplo n.º 8
0
func parseCsv(force bool) {
	now := time.Now()
	file := getCsvFile("devices", devicesUrlKey, now, force)
	if file == nil {
		return
	}
	var (
		line []string
		err  error
	)
	log.Info("Parsing devices.csv")
	for {
		line, err = file.Read()
		if err != nil {
			if err == io.EOF {
				break
			}
			log.Error("Error parsing devices.csv: %s", err)
			return
		}
		_ = line
	}
	// file = getCsvFile("members", membersUrlKey, now, force)
	// if file == nil {
	// 	return
	// }
	// log.Info("Parsing devices.csv")
	// for {
	// 	line, err = file.Read()
	// 	if err != nil {
	// 		if err == io.EOF {
	// 			break
	// 		}
	// 		log.Error("Error parsing devices.csv: %s", err)
	// 		return
	// 	}
	// 	_ = line
	// }
	// file = getCsvFile("opening", openingUrlKey, now, force)
	// if file == nil {
	// 	return
	// }
}
Ejemplo n.º 9
0
func (frontend *Frontend) ServeHTTP(conn http.ResponseWriter, req *http.Request) {

	originalHost := req.Host

	// Set default headers.
	headers := conn.Header()
	headers.Set("X-Content-Type-Options", "nosniff")

	// Redirect all requests to the "official" public host if the Host header
	// doesn't match.
	if !frontend.isValidHost(originalHost) {
		headers.Set("Location", frontend.redirectURL)
		conn.WriteHeader(http.StatusMovedPermanently)
		conn.Write(frontend.redirectHTML)
		logRequest(HTTPS_REDIRECT, http.StatusMovedPermanently, originalHost, req)
		return
	}

	// Return the HTTP 503 error page if we're in maintenance mode.
	if frontend.maintenanceMode {
		headers.Set(contentType, textHTML)
		headers.Set(contentLength, error503Length)
		conn.WriteHeader(http.StatusServiceUnavailable)
		conn.Write(error503)
		logRequest(HTTPS_MAINTENANCE, http.StatusServiceUnavailable, originalHost, req)
		return
	}

	reqPath := req.URL.Path

	// Handle requests for any files exposed within the static directory.
	if staticFile, ok := frontend.staticFiles[reqPath]; ok {
		expires := time.Now().Add(frontend.staticMaxAge)
		headers.Set("Expires", expires.Format(http.TimeFormat))
		headers.Set("Cache-Control", frontend.staticCache)
		headers.Set("Etag", staticFile.etag)
		if req.Header.Get("If-None-Match") == staticFile.etag {
			conn.WriteHeader(http.StatusNotModified)
			logRequest(HTTPS_STATIC, http.StatusNotModified, originalHost, req)
			return
		}
		// Special case /.well-known/oauth.json?callback= requests.
		if reqPath == "/.well-known/oauth.json" && req.URL.RawQuery != "" {
			query, err := url.ParseQuery(req.URL.RawQuery)
			if err != nil {
				log.Error("Error parsing oauth.json query string %q: %s",
					req.URL.RawQuery, err)
				serveError400(conn, originalHost, req)
				return
			}
			if callbackList, found := query["callback"]; found {
				callback := callbackList[0]
				if callback != "" {
					respLen := len(callback) + len(staticFile.content) + 2
					headers.Set(contentType, "text/javascript")
					headers.Set(contentLength, fmt.Sprintf("%d", respLen))
					conn.WriteHeader(http.StatusOK)
					conn.Write([]byte(callback))
					conn.Write([]byte{'('})
					conn.Write(staticFile.content)
					conn.Write([]byte{')'})
					logRequest(HTTPS_STATIC, http.StatusOK, originalHost, req)
					return
				}
			}
		}
		headers.Set(contentType, staticFile.mimetype)
		headers.Set(contentLength, staticFile.size)
		conn.WriteHeader(http.StatusOK)
		conn.Write(staticFile.content)
		logRequest(HTTPS_STATIC, http.StatusOK, originalHost, req)
		return
	}

	if frontend.liveMode {

		// Handle WebSocket requests.
		if strings.HasPrefix(reqPath, frontend.websocketPrefix) {
			websocket.Handler(handleWebSocket).ServeHTTP(conn, req)
			return
		}

		// Handle long-polling Comet requests.
		if strings.HasPrefix(reqPath, frontend.cometPrefix) {
			query, err := url.ParseQuery(req.URL.RawQuery)
			if err != nil {
				log.Error("Error parsing Comet query string %q: %s",
					req.URL.RawQuery, err)
				serveError400(conn, originalHost, req)
				return
			}
			queryReq, found := query["q"]
			if !found {
				serveError400(conn, originalHost, req)
				return
			}
			response, status := getLiveItems(queryReq[0])
			headers.Set(contentType, applicationJSON)
			headers.Set(contentLength, fmt.Sprintf("%d", len(response)))
			conn.WriteHeader(status)
			conn.Write(response)
			logRequest(HTTPS_COMET, status, originalHost, req)
			return
		}

	}

	// Open a connection to the upstream server.
	upstreamConn, err := net.Dial("tcp", frontend.upstreamAddr)
	if err != nil {
		log.Error("Couldn't connect to upstream: %s", err)
		serveError502(conn, originalHost, req)
		return
	}

	var clientIP string
	var upstream net.Conn

	splitPoint := strings.LastIndex(req.RemoteAddr, ":")
	if splitPoint == -1 {
		clientIP = req.RemoteAddr
	} else {
		clientIP = req.RemoteAddr[0:splitPoint]
	}

	if frontend.upstreamTLS {
		upstream = tls.Client(upstreamConn, tlsconf.Config)
		defer upstream.Close()
	} else {
		upstream = upstreamConn
	}

	// Modify the request Host: and User-Agent: headers.
	req.Host = frontend.upstreamHost
	req.Header.Set(
		"User-Agent",
		fmt.Sprintf("%s, %s, %s", req.UserAgent(), clientIP, originalHost))

	// Send the request to the upstream server.
	err = req.Write(upstream)
	if err != nil {
		log.Error("Error writing to the upstream server: %s", err)
		serveError502(conn, originalHost, req)
		return
	}

	// Parse the response from upstream.
	resp, err := http.ReadResponse(bufio.NewReader(upstream), req)
	if err != nil {
		log.Error("Error parsing response from upstream: %s", err)
		serveError502(conn, originalHost, req)
		return
	}

	defer resp.Body.Close()

	// Set a variable to hold the X-Live header value if present.
	var liveLength int

	if frontend.liveMode {
		xLive := resp.Header.Get("X-Live")
		if xLive != "" {
			// If the X-Live header was set, parse it into an int.
			liveLength, err = strconv.Atoi(xLive)
			if err != nil {
				log.Error("Error converting X-Live header value %q: %s", xLive, err)
				serveError500(conn, originalHost, req)
				return
			}
			resp.Header.Del("X-Live")
		}
	}

	var body []byte

	if liveLength > 0 {

		var gzipSet bool
		var respBody io.ReadCloser

		// Check Content-Encoding to see if upstream sent gzipped content.
		if resp.Header.Get("Content-Encoding") == "gzip" {
			gzipSet = true
			respBody, err = gzip.NewReader(resp.Body)
			if err != nil {
				log.Error("Error reading gzipped response from upstream: %s", err)
				serveError500(conn, originalHost, req)
				return
			}
			defer respBody.Close()
		} else {
			respBody = resp.Body
		}

		// Read the X-Live content from the response body.
		liveMessage := make([]byte, liveLength)
		n, err := respBody.Read(liveMessage)
		if n != liveLength || err != nil {
			log.Error("Error reading X-Live response from upstream: %s", err)
			serveError500(conn, originalHost, req)
			return
		}

		// Read the response to send back to the original request.
		body, err = ioutil.ReadAll(respBody)
		if err != nil {
			log.Error("Error reading non X-Live response from upstream: %s", err)
			serveError500(conn, originalHost, req)
			return
		}

		// Re-encode the response if it had been gzipped by upstream.
		if gzipSet {
			buffer := &bytes.Buffer{}
			encoder, err := gzip.NewWriter(buffer)
			if err != nil {
				log.Error("Error creating a new gzip Writer: %s", err)
				serveError500(conn, originalHost, req)
				return
			}
			n, err = encoder.Write(body)
			if n != len(body) || err != nil {
				log.Error("Error writing to the gzip Writer: %s", err)
				serveError500(conn, originalHost, req)
				return
			}
			err = encoder.Close()
			if err != nil {
				log.Error("Error finalising the write to the gzip Writer: %s", err)
				serveError500(conn, originalHost, req)
				return
			}
			body = buffer.Bytes()
		}

		resp.Header.Set(contentLength, fmt.Sprintf("%d", len(body)))
		liveChannel <- liveMessage

	} else {
		// Read the full response body.
		body, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Error("Error reading response from upstream: %s", err)
			serveError502(conn, originalHost, req)
			return
		}
	}

	// Set the received headers back to the initial connection.
	for k, values := range resp.Header {
		for _, v := range values {
			headers.Add(k, v)
		}
	}

	// Write the response body back to the initial connection.
	conn.WriteHeader(resp.StatusCode)
	conn.Write(body)
	logRequest(HTTPS_UPSTREAM, resp.StatusCode, originalHost, req)

}
Ejemplo n.º 10
0
func parseWifi() {

	if parsedAlready {
		log.Info("Re-parsing logs: %s", wifiLogDir)
	} else {
		log.Info("Parsing logs: %s", wifiLogDir)
		parsedAlready = true
	}

	files, err := ioutil.ReadDir(wifiLogDir)
	if err != nil {
		log.StandardError(err)
		return
	}

	if len(files) == 0 {
		log.Error("No files found to parse in %s", wifiLogDir)
		return
	}

	var (
		c, l     int
		current  []byte
		file     *os.File
		filename string
		isPrefix bool
		key      string
		line     []byte
		reader   *bufio.Reader
		received int64
		sent     int64
		session  string
		split    []string
		status   string
		// pending  map[string]string
	)

	startTime := time.Now()

	var i int

	for _, info := range files {
		filename = filepath.Join(wifiLogDir, info.Name())
		log.Info("Parsing: %s", filename)
		file, err = os.Open(filename)
		if err != nil {
			log.StandardError(err)
			return
		}
		reader = bufio.NewReader(file)
		for {
			line, isPrefix, err = reader.ReadLine()
			if err != nil {
				if err == io.EOF {
					break
				}
				log.StandardError(err)
				return
			}
			current = append(current, line...)
			if isPrefix {
				continue
			}
			split = strings.Split(string(current), ",")
			l = len(split)
			c = 6
			session = ""
			status = ""
			received = 0
			sent = 0
			for c < l {
				key = split[c]
				c += 2
				switch key {
				case "40":
					status = split[c+1]
				case "42":
					received, _ = strconv.ParseInt(split[c+1], 10, 64)
				case "43":
					sent, _ = strconv.ParseInt(split[c+1], 10, 64)
				case "44":
					session = split[c+1]
				}
				// log.Info("key: %s", key)
			}
			_ = session
			_ = status
			_ = received
			_ = sent
			current = current[0:0]
			i += 1
			if i >= 0 {
				break
			}
		}
		break
	}

	log.Info("Finished parsing (%s)", time.Since(startTime))

}
Ejemplo n.º 11
0
Archivo: bolt.go Proyecto: tav/bolt
func main() {

	// Setup temporary console logging.
	log.DisableConsoleTimestamp()
	log.AddConsoleLogger()

	// Set default values for command-line params.
	boltFilename := "Boltfile"
	genExecutablePath := ""

	recompile := false
	skipNext := true
	maxIdx := len(os.Args) - 1
	newArgs := []string{"bolt"}

	// Extract higher-level command-line arguments.
	for idx, arg := range os.Args {
		if skipNext {
			skipNext = false
			continue
		}
		if arg == "--gen" && idx != maxIdx {
			var err error
			genExecutablePath, err = filepath.Abs(os.Args[idx+1])
			if err != nil {
				runtime.StandardError(err)
			}
			skipNext = true
		} else if arg == "--boltfile" && idx != maxIdx {
			boltFilename = os.Args[idx+1]
			skipNext = true
		} else if arg == "--recompile" {
			recompile = true
		} else {
			newArgs = append(newArgs, arg)
		}
	}

	// Try and find the directory containing the Boltfile.
	boltdir, err := findBoltDir(boltFilename)
	if err != nil {
		if _, ok := err.(*fsutil.NotFound); ok {
			log.Error("Couldn't find Boltfile")
			runtime.Exit(1)
		}
		runtime.StandardError(err)
	}

	// Generate the path to the corresponding temp directory.
	boltpath := filepath.Join(boltdir, boltFilename)
	hash := sha1.New()
	hash.Write([]byte(boltpath))
	digest := fmt.Sprintf("%x", hash.Sum(nil))
	tempdir := filepath.Join(os.TempDir(), "bolt-"+digest)

	// See if the temp directory exists and if not create it.
	exists, err := fsutil.Exists(tempdir)
	if !exists {
		if _, ok := err.(*fsutil.NotFound); !ok {
			runtime.Error("Couldn't access the temp directory: %s: %s", tempdir, err)
		}
		err = os.Mkdir(tempdir, 0744)
		if err != nil {
			runtime.Error("Couldn't create the temp directory: %s: %s", tempdir, err)
		}
	}

	// See if an up-to-date generated binary already exists and, if so, run it.
	binpath := filepath.Join(tempdir, "bolt")
	if !recompile {
		boltstat, _ := os.Stat(boltpath)
		if genExecutablePath == "" {
			binstat, err := os.Stat(binpath)
			if err == nil {
				if boltstat.ModTime().Before(binstat.ModTime()) {
					runBoltExecutable(binpath, boltdir, newArgs)
					return
				}
			}
		}
	}

	// Parse the Boltfile.
	spec, err := parseBoltfile(boltpath, boltdir)
	if err != nil {
		exitForParserErrors(boltFilename, err)
	}

	// Exit if no tasks were found.
	if len(spec.Tasks) == 0 {
		runtime.Error("No tasks were found in %s", boltpath)
	}

	// Fudge the path to the executable that needs to be generated depending on
	// whether --gen-executable was specified or not.
	genOnly := true
	if genExecutablePath == "" {
		genExecutablePath = binpath
		genOnly = false
	}

	// Generate the executable.
	err = genExecutable(genExecutablePath, tempdir, spec)
	if err != nil {
		runtime.StandardError(err)
	}

	// Exit early if --gen-executable was specified.
	if genOnly {
		log.Info("%s successfully compiled to %s", boltFilename, genExecutablePath)
		runtime.Exit(0)
	}

	// Otherwise, run the executable.
	runBoltExecutable(binpath, boltdir, newArgs)

}
Ejemplo n.º 12
0
func Error(format string, v ...interface{}) {
	log.Error(format, v...)
	Exit(1)
}