// Rebuilds the graph from the extant entries in a mlog.
func restoreGraph(j mlog.Store) (system.CoreGraph, error) {
g := represent.NewGraph()
var item *mlog.Record
tot, err := j.Count()
if err != nil {
// mlog failed to report a count for some reason, bail out
return g, err
} else if tot > 0 {
// we manually iterate to the count because we assume that any messages
// that come in while we do this processing will be queued elsewhere.
for i := uint64(1); i-1 < tot; i++ {
item, err = j.Get(i)
if err != nil {
// TODO returning out here could end us up somwehere weird
return g, err
}
msg := &ingest.Message{}
json.Unmarshal(item.Message, msg)
g = g.Merge(item.Index, msg.UnificationForm())
}
}
return g, nil
}
func runDotDumper(cmd *cobra.Command, args []string) {
g := represent.NewGraph()
raw, err := schema.Master()
if err != nil {
panic(fmt.Sprint("Failed to open master schema file, test must abort. message:", err.Error()))
}
schemaMaster, err := gjs.NewSchema(gjs.NewStringLoader(string(raw)))
if err != nil {
panic("bad schema...?")
}
if len(args) < 1 {
log.Fatalf("Must provide at least one directory argument to dotdumper.")
}
var k uint64 = 0
for _, dir := range args {
fl, err := ioutil.ReadDir(dir)
if err != nil {
erro.Printf("Failed to read directory '%v' with error %v\n", dir, err)
}
for _, f := range fl {
if match, _ := regexp.MatchString("\\.json$", f.Name()); match && !f.IsDir() {
src, err := ioutil.ReadFile(dir + "/" + f.Name())
if err != nil {
erro.Printf("Failed to read fixture file %v/%v\n", dir, f.Name())
continue
}
result, err := schemaMaster.Validate(gjs.NewStringLoader(string(src)))
if err != nil {
erro.Printf("Validation process terminated with errors for %v/%v. Error: \n%v\n", dir, f.Name(), err.Error())
continue
}
if !result.Valid() {
for _, desc := range result.Errors() {
erro.Printf("\t%s\n", desc)
}
} else {
k++
m := ingest.Message{}
json.Unmarshal(src, &m)
g = g.Merge(k, m.UnificationForm())
fmt.Printf("Merged message %v/%v into graph\n", dir, f.Name())
}
}
}
}
pathflag := cmd.Flags().Lookup("output")
if pathflag.Changed {
ioutil.WriteFile(pathflag.Value.String(), GenerateDot(g), 0644)
} else {
fmt.Println(string(GenerateDot(g)))
}
}
// New creates a new webapp server, ready to be kicked off.
func New(receiver broker.GraphReceiver, unsub func(broker.GraphReceiver), cancel <-chan struct{}, f mlog.RecordGetter) *WebAppServer {
return &WebAppServer{
receiver: receiver,
unsub: unsub,
latest: represent.NewGraph(),
cancel: cancel,
mlogGetter: f,
}
}
func init() {
startTime := time.Now().UTC()
expvar.Publish("Uptime", expvar.Func(func() interface{} { return int64(time.Since(startTime)) }))
expvar.Publish("Goroutines", expvar.Func(func() interface{} { return runtime.NumGoroutine() }))
// subscribe to the broker in order to report data about current graph
c := broker.Get().Subscribe()
// Instantiate a real, empty graph to ensure the interface type is never nil when it might be called
var g system.CoreGraph = represent.NewGraph()
go func() {
for latest := range c {
g = latest
}
}()
expvar.Publish("MsgId", expvar.Func(func() interface{} { return g.MsgId() }))
go func() {
log.Println(http.ListenAndServe("localhost:6060", nil))
}()
}
// Ensures that all broker behavior works as expected
func TestBroker(t *testing.T) {
g := represent.NewGraph()
br := New()
input := make(chan system.CoreGraph, 0)
tallyChan := make(chan system.CoreGraph, 0)
var tally int
lstn := func(c <-chan system.CoreGraph) {
for g := range c {
tallyChan <- g
}
}
// kick off the tallying channel, the final target
go func() {
for _ = range tallyChan {
tally++
}
}()
// Kick off fanout goroutine with no listeners registered
br.Fanout(input)
input <- g
// give a bit of time for the channels to pass it through
time.Sleep(time.Millisecond)
if tally != 0 {
t.Fatalf("Broker has no listeners, fanout should not have increased the tally, but somehow %v message made it through", tally)
}
// Now add one listener
go lstn(br.Subscribe())
input <- g
time.Sleep(time.Millisecond)
if tally != 1 {
t.Fatalf("Fanout to single listener failed; should have gotten one message, but got %v", tally)
}
tally = 0
// Add a second listener, which should be incorporated by the already-running fanout goroutine
l2 := br.Subscribe()
go lstn(l2)
if len(br.subs) != 2 {
t.Fatalf("Should be two channels in the subs list after second subscription, but got %v", len(br.subs))
}
// Kick off second fanout goroutine
input2 := make(chan system.CoreGraph, 0)
br.Fanout(input2)
input <- g
input2 <- g
time.Sleep(time.Millisecond)
if tally != 4 {
t.Fatalf("Two fanouts to two listeners failed; should have gotten four messages, but got %v", tally)
}
tally = 0
// Now unsubscribe the second listener
br.Unsubscribe(l2)
if len(br.subs) != 1 {
t.Fatalf("Should be just one channel in the subs list after unsubscription, but got %v", len(br.subs))
}
input <- g
input2 <- g
time.Sleep(time.Millisecond)
if tally != 2 {
t.Fatalf("Two fanouts to what should be one listener after unsubscribing failed; should have gotten one message, but got %v", tally)
}
close(input)
close(input2)
close(tallyChan)
}
"github.com/pipeviz/pipeviz/log"
"github.com/pipeviz/pipeviz/mlog"
"github.com/pipeviz/pipeviz/represent"
"github.com/pipeviz/pipeviz/represent/q"
"github.com/pipeviz/pipeviz/types/system"
)
var (
publicDir = filepath.Join(defaultBase("github.com/pipeviz/pipeviz/webapp/"), "public")
)
var (
// Subscribe to the master broker and store latest locally as it comes
brokerListen = broker.Get().Subscribe()
// Initially set the latestGraph to a new, empty one to avoid nil pointer
latestGraph = represent.NewGraph()
// Count of active websocket clients (for expvars)
clientCount int64
)
const (
// Time allowed to write data to the client.
writeWait = 10 * time.Second
// Time allowed to read the next pong message from the client.
pongWait = 60 * time.Second
// Send pings to client with this period; less than pongWait.
pingPeriod = (pongWait * 9) / 10
)
func init() {
// Kick off goroutine to listen on the graph broker and keep our local pointer up to date