コード例 #1
0
ファイル: pvproxy.go プロジェクト: eliza411/pipeviz-1
// TODO return msgid sent back from pipeviz backend as uint64
func (c client) send(m *ingest.Message) error {
	j, err := json.Marshal(m)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("POST", c.target, bytes.NewReader(j))
	// TODO is it safe to reuse the header map like this?
	req.Header = c.h
	if err != nil {
		return err
	}

	resp, err := c.c.Post(c.target, "application/json", bytes.NewReader(j))
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"system": "pvproxy",
			"err":    err,
		}).Warn("Error returned from backend")
		return err
	}

	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		return fmt.Errorf("Pipeviz backend rejected message with code %d", resp.StatusCode)
	}

	return nil
}
コード例 #2
0
ファイル: broker.go プロジェクト: sdboyer/pipeviz
// New creates a pointer to a new, fully initialized GraphBroker.
func New() *GraphBroker {
	gb := &GraphBroker{lock: sync.RWMutex{}, subs: make(map[GraphReceiver]GraphSender), id: atomic.AddUint64(&brokerCount, 1)}

	log.WithFields(log.Fields{
		"system":    "broker",
		"broker-id": gb.id,
	}).Debug("New graph broker created")
	return gb
}
コード例 #3
0
ファイル: pvproxy.go プロジェクト: eliza411/pipeviz-1
// Run sets up and runs the proxying HTTP server, then blocks.
func (s *srv) Run(cmd *cobra.Command, args []string) {
	if s.vflag {
		fmt.Println("pvproxy version", version.Version())
		return
	}

	setUpLogging(s)

	mux := web.New()
	cl := newClient(s.target, 5*time.Second)

	mux.Use(log.NewHTTPLogger("pvproxy"))

	if s.key != "" && s.cert == "" {
		s.cert = s.key + ".crt"
	}
	useTLS := s.key != "" && s.cert != ""
	if useTLS {
		sec := secure.New(secure.Options{
			AllowedHosts:         nil,                                             // TODO allow a way to declare these
			SSLRedirect:          false,                                           // we have just one port to work with, so an internal redirect can't work
			SSLTemporaryRedirect: false,                                           // Use 301, not 302
			SSLProxyHeaders:      map[string]string{"X-Forwarded-Proto": "https"}, // list of headers that indicate we're using TLS (which would have been set by TLS-terminating proxy)
			STSSeconds:           315360000,                                       // 1yr HSTS time, as is generally recommended
			STSIncludeSubdomains: false,                                           // don't include subdomains; it may not be correct in general case TODO allow config
			STSPreload:           false,                                           // can't know if this is correct for general case TODO allow config
			FrameDeny:            true,                                            // proxy is write-only, no reason this should ever happen
			ContentTypeNosniff:   true,                                            // again, write-only
			BrowserXssFilter:     true,                                            // again, write-only
		})

		mux.Use(sec.Handler)
	}

	mux.Post("/github/push", githubIngestor(cl, cmd))

	var addr string
	if s.bindAll {
		addr = ":" + strconv.Itoa(s.port)
	} else {
		addr = s.bind + ":" + strconv.Itoa(s.port)
	}

	var err error
	if useTLS {
		err = graceful.ListenAndServeTLS(addr, s.cert, s.key, mux)
	} else {
		err = graceful.ListenAndServe(addr, mux)
	}

	if err != nil {
		logrus.WithFields(logrus.Fields{
			"system": "pvproxy",
			"err":    err,
		}).Fatal("pvproxy httpd terminated")
	}
}
コード例 #4
0
ファイル: websocket.go プロジェクト: sdboyer/pipeviz
func graphToSock(ws *websocket.Conn, g system.CoreGraph) {
	j, err := graphToJSON(g)
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"system": "webapp",
			"err":    err,
		}).Error("Error while marshaling graph into JSON for transmission over websocket")
	}

	if j != nil {
		ws.SetWriteDeadline(time.Now().Add(writeWait))
		if err := ws.WriteMessage(websocket.TextMessage, j); err != nil {
			logrus.WithFields(logrus.Fields{
				"system": "webapp",
				"err":    err,
			}).Error("Error while writing graph data to websocket")
			return
		}
	}
}
コード例 #5
0
ファイル: schema.go プロジェクト: eliza411/pipeviz-1
func init() {
	raw, err := Asset("schema.json")
	if err != nil {
		// It is correct to fatal out here because there is no use case for importing
		// the schema package that does not require the master schema to be present
		logrus.WithFields(logrus.Fields{
			"system": "schema",
			"err":    err,
		}).Fatal("Failed to locate raw bytes/file for master schema")
	}

	schema, err = gojsonschema.NewSchema(gojsonschema.NewStringLoader(string(raw)))
	if err != nil {
		// Correct to fatal, again for the same reasons
		logrus.WithFields(logrus.Fields{
			"system": "schema",
			"err":    err,
		}).Fatal("Failed to create master schema object")
	}
}
コード例 #6
0
ファイル: broker.go プロジェクト: sdboyer/pipeviz
// Unsubscribe immediately removes the provided channel from the subscribers
// list, and closes the channel.
//
// If the provided channel is not in the subscribers list, it will have no
// effect on the brokering behavior, and the channel will not be closed.
func (gb *GraphBroker) Unsubscribe(recv GraphReceiver) {
	gb.lock.Lock()

	log.WithFields(log.Fields{
		"system":    "broker",
		"broker-id": gb.id,
	}).Debug("Receiver unsubscribed from graph broker")

	if c, exists := gb.subs[recv]; exists {
		delete(gb.subs, recv)
		close(c)
	}
	gb.lock.Unlock()
}
コード例 #7
0
ファイル: broker.go プロジェクト: sdboyer/pipeviz
// Fanout initiates a goroutine that fans out each graph passed through the
// provided channel to all of its subscribers. New subscribers added after
// the fanout goroutine starts are automatically incorporated.
func (gb *GraphBroker) Fanout(input GraphReceiver) {
	go func() {
		log.WithFields(log.Fields{
			"system":    "broker",
			"broker-id": gb.id,
		}).Debug("New fanout initiated from graph broker")

		for in := range input {
			// for now we just iterate straight through and send in one goroutine
			log.WithFields(log.Fields{
				"system":    "broker",
				"broker-id": gb.id,
				"msgid":     in.MsgID(),
			}).Debug("Received new graph, sending to all subscribers")

			i := 1
			for k, c := range gb.subs {
				log.WithFields(log.Fields{
					"system":    "broker",
					"broker-id": gb.id,
					"subnum":    i,
					"msgid":     in.MsgID(),
				}).Debug("Sending graph to subscriber")
				// take a read lock at each stage of the loop. this guarantees that a
				// sub/unsub can interrupt at each point; mostly this is crucial because
				// otherwise we run the risk of sending on a closed channel.
				gb.lock.RLock()
				if _, exists := gb.subs[k]; exists {
					c <- in
				}
				gb.lock.RUnlock()
				i++
			}
		}
	}()
}
コード例 #8
0
ファイル: broker.go プロジェクト: sdboyer/pipeviz
// Subscribe creates a channel that receives all graphs passed into the
// broker. In general, subscribers should avoid doing a lot of work in the
// receiving goroutine, as it could block other subscribers.
// TODO nonblocking impl
func (gb *GraphBroker) Subscribe() GraphReceiver {
	gb.lock.Lock()

	log.WithFields(log.Fields{
		"system":    "broker",
		"broker-id": gb.id,
	}).Debug("New subscriber to graph broker")

	// Unbuffered. Listeners must be careful not to do too much in their receiving
	// goroutine, lest they create a pileup!
	c := make(chan system.CoreGraph, 0)
	gb.subs[c] = c

	gb.lock.Unlock()
	return c
}
コード例 #9
0
ファイル: pipeviz_nodbg.go プロジェクト: sdboyer/pipeviz
func setUpLogging() {
	// For now, either log to syslog OR stdout
	if *useSyslog {
		hook, err := logrus_syslog.NewSyslogHook("", "", syslog.LOG_INFO, "")
		if err == nil {
			logrus.AddHook(hook)
		} else {
			logrus.WithFields(logrus.Fields{
				"system": "main",
				"err":    err,
			}).Fatal("Could not connect to syslog, exiting")
		}
	} else {
		logrus.SetFormatter(&logrus.TextFormatter{
			FullTimestamp:  true,
			DisableSorting: true,
		})
	}
}
コード例 #10
0
ファイル: message.go プロジェクト: sdboyer/pipeviz
// UnificationForm translates all data in the message into the standard
// UnifyInstructionForm, suitable for merging into the dataset.
func (m Message) UnificationForm() []system.UnifyInstructionForm {
	logEntry := log.WithFields(log.Fields{
		"system": "interpet",
	})

	var ret []system.UnifyInstructionForm

	for _, e := range m.Env {
		logEntry.WithField("vtype", "environment").Debug("Preparing to translate into UnifyInstructionForm")
		ret = append(ret, e.UnificationForm()...)
	}
	for _, e := range m.Ls {
		logEntry.WithField("vtype", "logic state").Debug("Preparing to translate into UnifyInstructionForm")
		ret = append(ret, e.UnificationForm()...)
	}
	for _, e := range m.Pds {
		logEntry.WithField("vtype", "parent dataset").Debug("Preparing to translate into UnifyInstructionForm")
		ret = append(ret, e.UnificationForm()...)
	}
	for _, e := range m.Ds {
		logEntry.WithField("vtype", "dataset").Debug("Preparing to translate into UnifyInstructionForm")
		ret = append(ret, e.UnificationForm()...)
	}
	for _, e := range m.P {
		logEntry.WithField("vtype", "process").Debug("Preparing to translate into UnifyInstructionForm")
		ret = append(ret, e.UnificationForm()...)
	}
	for _, e := range m.C {
		logEntry.WithField("vtype", "git commit").Debug("Preparing to translate into UnifyInstructionForm")
		ret = append(ret, e.UnificationForm()...)
	}
	for _, e := range m.Cm {
		logEntry.WithField("vtype", "commit meta").Debug("Preparing to translate into UnifyInstructionForm")
		ret = append(ret, e.UnificationForm()...)
	}

	for _, e := range m.Yp {
		logEntry.WithField("vtype", "yum-pkg").Debug("Preparing to translate into UnifyInstructionForm")
		ret = append(ret, e.UnificationForm()...)
	}

	return ret
}
コード例 #11
0
ファイル: logstash_test.go プロジェクト: sdboyer/pipeviz
func TestLogstashFormatter(t *testing.T) {
	assert := assert.New(t)

	lf := LogstashFormatter{Type: "abc"}

	fields := logrus.Fields{
		"message": "def",
		"level":   "ijk",
		"type":    "lmn",
		"one":     1,
		"pi":      3.14,
		"bool":    true,
	}

	entry := logrus.WithFields(fields)
	entry.Message = "msg"
	entry.Level = logrus.InfoLevel

	b, _ := lf.Format(entry)

	var data map[string]interface{}
	dec := json.NewDecoder(bytes.NewReader(b))
	dec.UseNumber()
	dec.Decode(&data)

	// base fields
	assert.Equal(json.Number("1"), data["@version"])
	assert.NotEmpty(data["@timestamp"])
	assert.Equal("abc", data["type"])
	assert.Equal("msg", data["message"])
	assert.Equal("info", data["level"])

	// substituted fields
	assert.Equal("def", data["fields.message"])
	assert.Equal("ijk", data["fields.level"])
	assert.Equal("lmn", data["fields.type"])

	// formats
	assert.Equal(json.Number("1"), data["one"])
	assert.Equal(json.Number("3.14"), data["pi"])
	assert.Equal(true, data["bool"])
}
コード例 #12
0
ファイル: websocket.go プロジェクト: sdboyer/pipeviz
func openSocket(w http.ResponseWriter, r *http.Request) {
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		entry := logrus.WithFields(logrus.Fields{
			"system": "webapp",
			"err":    err,
		})

		if _, ok := err.(websocket.HandshakeError); !ok {
			entry.Error("Error on attempting upgrade to websocket")
		} else {
			entry.Warn("Handshake error on websocket upgrade")
		}
		return
	}

	clientCount++
	go wsWriter(ws)
	wsReader(ws)
	clientCount--
}
コード例 #13
0
ファイル: http.go プロジェクト: sdboyer/pipeviz
// NewHTTPLogger returns an HTTPLogger, suitable for use as http middleware.
func NewHTTPLogger(system string) func(h http.Handler) http.Handler {
	middleware := func(h http.Handler) http.Handler {
		entry := logrus.WithFields(logrus.Fields{
			"system": system,
		})

		fn := func(w http.ResponseWriter, r *http.Request) {
			lw := mutil.WrapWriter(w)

			entry.WithFields(logrus.Fields{
				"uri":    r.URL.String(),
				"method": r.Method,
				"remote": r.RemoteAddr,
			}).Info("Beginning request processing")

			t1 := time.Now()
			h.ServeHTTP(lw, r)

			if lw.Status() == 0 {
				lw.WriteHeader(http.StatusOK)
			}

			entry.WithFields(logrus.Fields{
				"status": lw.Status(),
				"uri":    r.URL.String(),
				"method": r.Method,
				"remote": r.RemoteAddr,
				"wall":   time.Now().Sub(t1).String(),
			}).Info("Request processing complete")
		}

		return http.HandlerFunc(fn)
	}

	return middleware
}
コード例 #14
0
ファイル: server.go プロジェクト: eliza411/pipeviz-1
// ListenAndServe initiates the webapp http listener.
//
// This blocks on the http listening loop, so it should typically be called in its own goroutine.
func (s *WebAppServer) ListenAndServe(addr, pubdir, key, cert string, showVersion bool) {
	mf := web.New()
	useTLS := key != "" && cert != ""

	mf.Use(log.NewHTTPLogger("webapp"))

	if useTLS {
		sec := secure.New(secure.Options{
			AllowedHosts:         nil,                                             // TODO allow a way to declare these
			SSLRedirect:          false,                                           // we have just one port to work with, so an internal redirect can't work
			SSLTemporaryRedirect: false,                                           // Use 301, not 302
			SSLProxyHeaders:      map[string]string{"X-Forwarded-Proto": "https"}, // list of headers that indicate we're using TLS (which would have been set by TLS-terminating proxy)
			STSSeconds:           315360000,                                       // 1yr HSTS time, as is generally recommended
			STSIncludeSubdomains: false,                                           // don't include subdomains; it may not be correct in general case TODO allow config
			STSPreload:           false,                                           // can't know if this is correct for general case TODO allow config
			FrameDeny:            false,                                           // pipeviz is exactly the kind of thing where embedding is appropriate
			ContentTypeNosniff:   true,                                            // shouldn't be an issue for pipeviz, but doesn't hurt...probably?
			BrowserXssFilter:     false,                                           // really shouldn't be necessary for pipeviz
		})

		mf.Use(func(h http.Handler) http.Handler {
			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				err := sec.Process(w, r)

				// If there was an error, do not continue.
				if err != nil {
					logrus.WithFields(logrus.Fields{
						"system": "webapp",
						"err":    err,
					}).Warn("Error from security middleware, dropping request")
					return
				}

				h.ServeHTTP(w, r)
			})
		})
	}

	// If showing version, add a middleware to do it automatically for everything
	if showVersion {
		mf.Use(func(h http.Handler) http.Handler {
			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("Server", version.Version())
				h.ServeHTTP(w, r)
			})
		})
	}

	mf.Get("/sock", s.openSocket)
	mf.Get("/message/:mid", s.getMessage)
	mf.Get("/*", http.StripPrefix("/", http.FileServer(http.Dir(pubdir))))

	mf.Compile()

	// kick off a goroutine to grab the latest graph and listen for cancel
	go func() {
		var g system.CoreGraph
		for {
			select {
			case <-s.cancel:
				s.unsub(s.receiver)
				return

			case g = <-s.receiver:
				s.latest = g
			}
		}
	}()

	var err error
	if useTLS {
		err = graceful.ListenAndServeTLS(addr, cert, key, mf)
	} else {
		err = graceful.ListenAndServe(addr, mf)
	}

	if err != nil {
		logrus.WithFields(logrus.Fields{
			"system": "webapp",
			"err":    err,
		}).Fatal("ListenAndServe returned with an error")
	}

	// TODO allow returning err
}
コード例 #15
0
ファイル: github.go プロジェクト: sdboyer/pipeviz
func (gpe githubPushEvent) ToMessage(token string) *ingest.Message {
	msg := new(ingest.Message)
	client := http.Client{Timeout: 2 * time.Second}

	for _, c := range gpe.Commits {
		// don't include commits we know not to be new - make that someone else's job
		if !c.Distinct {
			continue
		}

		// take up to 50 bytes for subject
		subjlen := len(c.Message)
		if subjlen > 50 {
			subjlen = 50
		}

		// github doesn't include parent commit list in push payload (UGHHHH). so, call out for it.
		// TODO spawn a goroutine per commit to do these in parallel
		url := strings.Replace(gpe.Repository.GitCommitsURL, "{/sha}", "/"+c.Sha, 1)
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			logrus.WithFields(logrus.Fields{
				"system": "pvproxy",
				"err":    err,
				"sha1":   c.Sha,
			}).Warn("Error while creating request for additional information from github; skipping commit.")
			continue
		}

		if token != "" {
			req.Header.Set("Authentication", "token "+token)
		}

		resp, err := client.Do(req)
		if err != nil {
			// just drop the problematic commit
			logrus.WithFields(logrus.Fields{
				"system": "pvproxy",
				"err":    err,
				"sha1":   c.Sha,
			}).Warn("Request to github to retrieve commit parent info failed; commit dropped.")
			continue
		}

		if !statusIsOK(resp) {
			logrus.WithFields(logrus.Fields{
				"system": "pvproxy",
				"status": resp.StatusCode,
				"sha1":   c.Sha,
			}).Warn("Github responded with non-2xx response when requesting parent commit data.")
			continue
		}

		// skip err here, it's effectively caught by the json unmarshaler
		bod, _ := ioutil.ReadAll(resp.Body)

		var jmap interface{}
		err = json.Unmarshal(bod, &jmap)
		if err != nil {
			logrus.WithFields(logrus.Fields{
				"system": "pvproxy",
				"err":    err,
				"sha1":   c.Sha,
			}).Warn("Bad JSON response from github when requesting commit parent info; commit dropped.")
			continue
		}

		var parents []string
		for _, iparent := range jmap.(map[string]interface{})["parents"].([]interface{}) {
			parent := iparent.(map[string]interface{})
			parents = append(parents, parent["sha"].(string))
		}

		t, err := time.Parse(time.RFC3339, c.Timestamp)
		if err != nil {
			logrus.WithFields(logrus.Fields{
				"system":     "pvproxy",
				"err":        err,
				"datestring": c.Timestamp,
				"sha1":       c.Sha,
			}).Warn("Error on parsing date field in github payload; commit dropped.")
		}

		msg.Add(semantic.Commit{
			Sha1Str:    c.Sha,
			Subject:    c.Message[:subjlen],
			Author:     fmt.Sprintf("%q <%s>", c.Author.Name, c.Author.Email),
			Date:       t.Format(gitDateFormat),
			Repository: gpe.Repository.Ident,
			ParentsStr: parents,
		})
	}

	if gpe.Ref[:11] == "refs/heads/" {
		msg.Add(semantic.CommitMeta{
			Sha1Str:  gpe.HeadCommit.Sha,
			Branches: []string{gpe.Ref[11:]},
		})
	} else if gpe.Ref[:10] == "refs/tags/" {
		msg.Add(semantic.CommitMeta{
			Sha1Str: gpe.HeadCommit.Sha,
			Tags:    []string{gpe.Ref[10:]},
		})
	}

	return msg
}
コード例 #16
0
ファイル: pipeviz.go プロジェクト: sdboyer/pipeviz
// RunWebapp runs the pipeviz http frontend webapp on the specified address.
//
// This blocks on the http listening loop, so it should typically be called in its own goroutine.
func RunWebapp(addr, key, cert string, f mlog.RecordGetter) {
	mf := web.New()
	useTLS := key != "" && cert != ""

	if useTLS {
		sec := secure.New(secure.Options{
			AllowedHosts:         nil,                                             // TODO allow a way to declare these
			SSLRedirect:          false,                                           // we have just one port to work with, so an internal redirect can't work
			SSLTemporaryRedirect: false,                                           // Use 301, not 302
			SSLProxyHeaders:      map[string]string{"X-Forwarded-Proto": "https"}, // list of headers that indicate we're using TLS (which would have been set by TLS-terminating proxy)
			STSSeconds:           315360000,                                       // 1yr HSTS time, as is generally recommended
			STSIncludeSubdomains: false,                                           // don't include subdomains; it may not be correct in general case TODO allow config
			STSPreload:           false,                                           // can't know if this is correct for general case TODO allow config
			FrameDeny:            false,                                           // pipeviz is exactly the kind of thing where embedding is appropriate
			ContentTypeNosniff:   true,                                            // shouldn't be an issue for pipeviz, but doesn't hurt...probably?
			BrowserXssFilter:     false,                                           // really shouldn't be necessary for pipeviz
		})

		mf.Use(func(h http.Handler) http.Handler {
			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				err := sec.Process(w, r)

				// If there was an error, do not continue.
				if err != nil {
					log.WithFields(log.Fields{
						"system": "webapp",
						"err":    err,
					}).Warn("Error from security middleware, dropping request")
					return
				}

				h.ServeHTTP(w, r)
			})
		})
	}

	// A middleware to attach the mlog-getting func to the env for later use.
	mf.Use(func(c *web.C, h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			if c.Env == nil {
				c.Env = make(map[interface{}]interface{})
			}
			c.Env["mlogGet"] = f
			h.ServeHTTP(w, r)
		})
	})

	webapp.RegisterToMux(mf)

	mf.Compile()

	var err error
	if useTLS {
		err = graceful.ListenAndServeTLS(addr, cert, key, mf)
	} else {
		err = graceful.ListenAndServe(addr, mf)
	}

	if err != nil {
		log.WithFields(log.Fields{
			"system": "webapp",
			"err":    err,
		}).Fatal("ListenAndServe returned with an error")
	}
}
コード例 #17
0
ファイル: pipeviz.go プロジェクト: sdboyer/pipeviz
func main() {
	pflag.Parse()
	setUpLogging()

	src, err := schema.Master()
	if err != nil {
		log.WithFields(log.Fields{
			"system": "main",
			"err":    err,
		}).Fatal("Could not locate master schema file, exiting")
	}

	// The master JSON schema used for validating all incoming messages
	masterSchema, err := gjs.NewSchema(gjs.NewStringLoader(string(src)))
	if err != nil {
		log.WithFields(log.Fields{
			"system": "main",
			"err":    err,
		}).Fatal("Error while creating a schema object from the master schema file, exiting")
	}

	// Channel to receive persisted messages from HTTP workers. 1000 cap to allow
	// some wiggle room if there's a sudden burst of messages and the interpreter
	// gets behind.
	interpretChan := make(chan *mlog.Record, 1000)

	var listenAt string
	if *bindAll == false {
		listenAt = "127.0.0.1:"
	} else {
		listenAt = ":"
	}

	var j mlog.Store
	switch *mlstore {
	case "bolt":
		j, err = boltdb.NewBoltStore(*dbPath + "/mlog.bolt")
		if err != nil {
			log.WithFields(log.Fields{
				"system": "main",
				"err":    err,
			}).Fatal("Error while setting up bolt mlog storage, exiting")
		}
	case "memory":
		j = mem.NewMemStore()
	default:
		log.WithFields(log.Fields{
			"system":  "main",
			"storage": *mlstore,
		}).Fatal("Invalid storage type requested for mlog, exiting")
	}

	// Restore the graph from the mlog (or start from nothing if mlog is empty)
	// TODO move this down to after ingestor is started
	g, err := restoreGraph(j)
	if err != nil {
		log.WithFields(log.Fields{
			"system": "main",
			"err":    err,
		}).Fatal("Error while rebuilding the graph from the mlog")
	}

	// Kick off fanout on the master/singleton graph broker. This will bridge between
	// the state machine and the listeners interested in the machine's state.
	brokerChan := make(chan system.CoreGraph, 0)
	broker.Get().Fanout(brokerChan)
	brokerChan <- g

	srv := ingest.New(j, masterSchema, interpretChan, brokerChan, MaxMessageSize)

	// Kick off the http message ingestor.
	// TODO let config/params control address
	go func() {
		if *ingestKey != "" && *ingestCert == "" {
			*ingestCert = *ingestKey + ".crt"
		}
		err := srv.RunHTTPIngestor(listenAt+strconv.Itoa(DefaultIngestionPort), *ingestKey, *ingestCert)
		if err != nil {
			log.WithFields(log.Fields{
				"system": "main",
				"err":    err,
			}).Fatal("Error while starting the ingestion http server")
		}
	}()

	// Kick off the intermediary interpretation goroutine that receives persisted
	// messages from the ingestor, merges them into the state graph, then passes
	// them along to the graph broker.
	go srv.Interpret(g)

	// And finally, kick off the webapp.
	// TODO let config/params control address
	if *webappKey != "" && *webappCert == "" {
		*webappCert = *webappKey + ".crt"
	}
	go RunWebapp(listenAt+strconv.Itoa(DefaultAppPort), *webappKey, *webappCert, j.Get)

	// Block on goji's graceful waiter, allowing the http connections to shut down nicely.
	// FIXME using this should be unnecessary if we're crash-only
	graceful.Wait()
}
コード例 #18
0
ファイル: traverse.go プロジェクト: sdboyer/pipeviz
func (g *coreGraph) adjacentWith(egoID uint64, vef system.VEFilter, in bool) (vts system.VertexTupleVector) {
	etype, eprops := vef.EType(), vef.EProps()
	vtype, vprops := vef.VType(), vef.VProps()
	vt, err := g.Get(egoID)
	if err != nil {
		// vertex doesn't exist
		return
	}

	// Allow some quick parallelism by continuing to search edges while loading vertices
	// TODO performance test this to see if it's at all worth it
	vidchan := make(chan uint64, 10)

	var feef func(k string, v ps.Any)
	// TODO specialize the func for zero-cases
	feef = func(k string, v ps.Any) {
		edge := v.(system.StdEdge)
		if etype != system.ETypeNone && etype != edge.EType {
			// etype doesn't match
			return
		}

		for _, p := range eprops {
			eprop, exists := edge.Props.Lookup(p.K)
			if !exists {
				return
			}

			deprop := eprop.(system.Property)
			switch tv := deprop.Value.(type) {
			default:
				if tv != p.V {
					return
				}
			case []byte:
				cmptv, ok := p.V.([]byte)
				if !ok || !bytes.Equal(tv, cmptv) {
					return
				}
			}
		}

		if in {
			vidchan <- edge.Source
		} else {
			vidchan <- edge.Target
		}
	}

	go func() {
		if in {
			vt.InEdges.ForEach(feef)
		} else {
			vt.OutEdges.ForEach(feef)
		}
		close(vidchan)
	}()

	// Keep track of the vertices we've collected, for deduping
	visited := make(map[uint64]struct{})
VertexInspector:
	for vid := range vidchan {
		if _, seenit := visited[vid]; seenit {
			// already visited this vertex; go back to waiting on the chan
			continue
		}
		// mark this vertex as black
		visited[vid] = struct{}{}

		adjvt, err := g.Get(vid)
		if err != nil {
			log.WithFields(log.Fields{
				"system": "engine",
				"err":    err,
			}).Error("Attempted to get nonexistent vid during traversal - should be impossible")
			continue
		}

		// FIXME can't rely on Typ() method here, need to store it
		if vtype != system.VTypeNone && vtype != adjvt.Vertex.Typ() {
			continue
		}

		for _, p := range vprops {
			vprop, exists := adjvt.Vertex.Props().Lookup(p.K)
			if !exists {
				continue VertexInspector
			}

			dvprop := vprop.(system.Property)
			switch tv := dvprop.Value.(type) {
			default:
				if tv != p.V {
					continue VertexInspector
				}
			case []byte:
				cmptv, ok := p.V.([]byte)
				if !ok || !bytes.Equal(tv, cmptv) {
					continue VertexInspector
				}
			}
		}

		vts = append(vts, adjvt)
	}

	return
}