func runDotDumper(cmd *cobra.Command, args []string) {
g := represent.NewGraph()
raw, err := schema.Master()
if err != nil {
panic(fmt.Sprint("Failed to open master schema file, test must abort. message:", err.Error()))
}
schemaMaster, err := gjs.NewSchema(gjs.NewStringLoader(string(raw)))
if err != nil {
panic("bad schema...?")
}
if len(args) < 1 {
log.Fatalf("Must provide at least one directory argument to dotdumper.")
}
var k uint64 = 0
for _, dir := range args {
fl, err := ioutil.ReadDir(dir)
if err != nil {
erro.Printf("Failed to read directory '%v' with error %v\n", dir, err)
}
for _, f := range fl {
if match, _ := regexp.MatchString("\\.json$", f.Name()); match && !f.IsDir() {
src, err := ioutil.ReadFile(dir + "/" + f.Name())
if err != nil {
erro.Printf("Failed to read fixture file %v/%v\n", dir, f.Name())
continue
}
result, err := schemaMaster.Validate(gjs.NewStringLoader(string(src)))
if err != nil {
erro.Printf("Validation process terminated with errors for %v/%v. Error: \n%v\n", dir, f.Name(), err.Error())
continue
}
if !result.Valid() {
for _, desc := range result.Errors() {
erro.Printf("\t%s\n", desc)
}
} else {
k++
m := ingest.Message{}
json.Unmarshal(src, &m)
g = g.Merge(k, m.UnificationForm())
fmt.Printf("Merged message %v/%v into graph\n", dir, f.Name())
}
}
}
}
pathflag := cmd.Flags().Lookup("output")
if pathflag.Changed {
ioutil.WriteFile(pathflag.Value.String(), GenerateDot(g), 0644)
} else {
fmt.Println(string(GenerateDot(g)))
}
}
func BenchmarkValidateMessageOneAndTwo(b *testing.B) {
d1 := MsgJSON[0]
d2 := MsgJSON[1]
msg1 := gjs.NewStringLoader(string(d1))
msg2 := gjs.NewStringLoader(string(d2))
for i := 0; i < b.N; i++ {
masterSchema.Validate(msg1)
masterSchema.Validate(msg2)
}
}
func BenchmarkValidateMessageOneAndTwo(b *testing.B) {
d1 := msgJSON[0]
d2 := msgJSON[1]
msg1 := gojsonschema.NewStringLoader(string(d1))
msg2 := gojsonschema.NewStringLoader(string(d2))
for i := 0; i < b.N; i++ {
schema.Master().Validate(msg1)
schema.Master().Validate(msg2)
}
}
// Reads all message fixtures from fixtures/ein and validates them
// against the master message schema (schema.json).
func TestMessageValidity(t *testing.T) {
files, err := ioutil.ReadDir("../fixtures/ein/")
if err != nil {
t.Error("Failed to scan message fixtures dir:", err.Error())
t.FailNow()
}
for _, f := range files {
if testing.Verbose() {
t.Log("Beginning validation on", f.Name())
}
src, _ := ioutil.ReadFile("../fixtures/ein/" + f.Name())
msg := gjs.NewStringLoader(string(src))
result, err := masterSchema.Validate(msg)
if err != nil {
panic(err.Error())
}
if result.Valid() {
if testing.Verbose() {
t.Log(f.Name(), "passed validation")
}
} else {
for _, desc := range result.Errors() {
t.Errorf("%s\n", strings.Replace(desc.String(), "root", f.Name(), 1))
}
}
}
}
func init() {
for i := range make([]struct{}, 8) {
m := &ingest.Message{}
path := fmt.Sprintf("../fixtures/ein/%v.json", i+1)
f, err := ioutil.ReadFile(path)
if err != nil {
panic("json fnf: " + path)
}
MsgJSON = append(MsgJSON, f)
_ = json.Unmarshal(f, m)
Msgs = append(Msgs, m)
}
src, err := schema.Master()
if err != nil {
panic(fmt.Sprint("Failed to open master schema file, test must abort. Note: this test must be run from the pipeviz repo root. message:", err.Error()))
}
masterSchema, err = gjs.NewSchema(gjs.NewStringLoader(string(src)))
if err != nil {
panic(fmt.Sprint("Failed to create a schema object from the master schema.json:", err.Error()))
}
}
func BenchmarkValidateMessageTwo(b *testing.B) {
d := MsgJSON[0]
msg := gjs.NewStringLoader(string(d))
for i := 0; i < b.N; i++ {
masterSchema.Validate(msg)
}
}
func BenchmarkValidateMessageTwo(b *testing.B) {
d := msgJSON[0]
msg := gojsonschema.NewStringLoader(string(d))
for i := 0; i < b.N; i++ {
schema.Master().Validate(msg)
}
}
func runValidate(cmd *cobra.Command, args []string) {
var errors int
for _, dir := range args {
finfo, err := os.Stat(dir)
if err != nil {
fmt.Printf("Could not stat '%s' with error: %s\n", dir, err)
continue
}
if !finfo.IsDir() {
continue
}
fl, err := ioutil.ReadDir(dir)
if err != nil {
fmt.Printf("Failed to read directory '%v' with error %v\n", dir, err)
}
for _, f := range fl {
if match, _ := regexp.MatchString("\\.json$", f.Name()); match && !f.IsDir() {
src, err := ioutil.ReadFile(dir + "/" + f.Name())
if err != nil {
errors |= FileReadFail
fmt.Printf("Failed to read fixture file %v/%v\n", dir, f.Name())
continue
}
result, err := schema.Master().Validate(gjs.NewStringLoader(string(src)))
if err != nil {
errors |= ValidationError
fmt.Printf("Validation process terminated with errors for %v/%v. Error: \n%v\n", dir, f.Name(), err.Error())
continue
}
if !result.Valid() {
errors |= ValidationFail
fmt.Printf("Errors in %v/%v:\n", dir, f.Name())
for _, desc := range result.Errors() {
fmt.Printf("\t%s\n", desc)
}
} else {
fmt.Printf("%v/%v passed validation\n", dir, f.Name())
}
}
}
}
os.Exit(errors)
}
func validateAndPrint(w io.Writer, v interface{}) {
msg, err := toJSONBytes(v)
if err != nil {
fmt.Fprintf(w, err.Error())
return
}
// Validate the current state of the message
result, err := schemaMaster.Validate(gojsonschema.NewStringLoader(string(msg)))
if err != nil {
fmt.Fprintf(w, "\nError while attempting to validate data: %s\n", err.Error())
return
}
if !result.Valid() {
fmt.Fprintln(w, "\nAs it stands now, the data will fail validation if sent to a pipeviz server. Errors:")
for _, desc := range result.Errors() {
fmt.Fprintf(w, "\t%s\n", desc)
}
}
}
func init() {
raw, err := Asset("schema.json")
if err != nil {
// It is correct to fatal out here because there is no use case for importing
// the schema package that does not require the master schema to be present
logrus.WithFields(logrus.Fields{
"system": "schema",
"err": err,
}).Fatal("Failed to locate raw bytes/file for master schema")
}
schema, err = gojsonschema.NewSchema(gojsonschema.NewStringLoader(string(raw)))
if err != nil {
// Correct to fatal, again for the same reasons
logrus.WithFields(logrus.Fields{
"system": "schema",
"err": err,
}).Fatal("Failed to create master schema object")
}
}
// Tests that all message fixtures pass schema validation against the master
// message schema (../schema/schema.json).
func TestFixtureValidity(t *testing.T) {
// Ensure stable order, for user's sanity
names := fixtures.AssetNames()
sort.Strings(names)
// If this test is run without having generated assets via go-bindata, it could
// pass. This should be precluded because the test won't compile without the
// functions created by go-bindata, but this is an easy guard to put in place, so
// might as well.
if len(names) == 0 {
t.Error("No asset files found for validation - has code generation been run correctly? (should be `make gen`)")
}
for _, name := range names {
if testing.Verbose() {
t.Log("Beginning validation on", name)
}
b, err := fixtures.Asset(name)
result, err := schema.Master().Validate(gojsonschema.NewStringLoader(string(b)))
if err != nil {
t.Errorf("Error while validating fixture %v: %s", name, err)
panic(err.Error())
}
if result.Valid() {
if testing.Verbose() {
t.Log(name, "passed validation")
}
} else {
buf := bytes.NewBufferString(fmt.Sprintf("%s failed validation with the following errors:\n", name))
for _, desc := range result.Errors() {
buf.WriteString(fmt.Sprintf("%s\n", strings.Replace(desc.String(), "root", name, 1)))
}
t.Errorf(buf.String())
}
}
}
func main() {
pflag.Parse()
setUpLogging()
src, err := schema.Master()
if err != nil {
log.WithFields(log.Fields{
"system": "main",
"err": err,
}).Fatal("Could not locate master schema file, exiting")
}
// The master JSON schema used for validating all incoming messages
masterSchema, err := gjs.NewSchema(gjs.NewStringLoader(string(src)))
if err != nil {
log.WithFields(log.Fields{
"system": "main",
"err": err,
}).Fatal("Error while creating a schema object from the master schema file, exiting")
}
// Channel to receive persisted messages from HTTP workers. 1000 cap to allow
// some wiggle room if there's a sudden burst of messages and the interpreter
// gets behind.
interpretChan := make(chan *mlog.Record, 1000)
var listenAt string
if *bindAll == false {
listenAt = "127.0.0.1:"
} else {
listenAt = ":"
}
var j mlog.Store
switch *mlstore {
case "bolt":
j, err = boltdb.NewBoltStore(*dbPath + "/mlog.bolt")
if err != nil {
log.WithFields(log.Fields{
"system": "main",
"err": err,
}).Fatal("Error while setting up bolt mlog storage, exiting")
}
case "memory":
j = mem.NewMemStore()
default:
log.WithFields(log.Fields{
"system": "main",
"storage": *mlstore,
}).Fatal("Invalid storage type requested for mlog, exiting")
}
// Restore the graph from the mlog (or start from nothing if mlog is empty)
// TODO move this down to after ingestor is started
g, err := restoreGraph(j)
if err != nil {
log.WithFields(log.Fields{
"system": "main",
"err": err,
}).Fatal("Error while rebuilding the graph from the mlog")
}
// Kick off fanout on the master/singleton graph broker. This will bridge between
// the state machine and the listeners interested in the machine's state.
brokerChan := make(chan system.CoreGraph, 0)
broker.Get().Fanout(brokerChan)
brokerChan <- g
srv := ingest.New(j, masterSchema, interpretChan, brokerChan, MaxMessageSize)
// Kick off the http message ingestor.
// TODO let config/params control address
go func() {
if *ingestKey != "" && *ingestCert == "" {
*ingestCert = *ingestKey + ".crt"
}
err := srv.RunHTTPIngestor(listenAt+strconv.Itoa(DefaultIngestionPort), *ingestKey, *ingestCert)
if err != nil {
log.WithFields(log.Fields{
"system": "main",
"err": err,
}).Fatal("Error while starting the ingestion http server")
}
}()
// Kick off the intermediary interpretation goroutine that receives persisted
// messages from the ingestor, merges them into the state graph, then passes
// them along to the graph broker.
go srv.Interpret(g)
// And finally, kick off the webapp.
// TODO let config/params control address
if *webappKey != "" && *webappCert == "" {
*webappCert = *webappKey + ".crt"
}
go RunWebapp(listenAt+strconv.Itoa(DefaultAppPort), *webappKey, *webappCert, j.Get)
// Block on goji's graceful waiter, allowing the http connections to shut down nicely.
// FIXME using this should be unnecessary if we're crash-only
graceful.Wait()
}
// runGenLS is the main entry point for running the logic state-generating
// ls subcommand.
func (lsc lsCmd) runGenLS(cmd *cobra.Command, args []string) {
ls := &semantic.LogicState{}
if !cmd.Flags().Lookup("no-detect").Changed {
*ls = lsc.detectDefaults()
}
// Write directly to stdout, at least for now
w := os.Stdout
// Prep schema to validate the messages as we go
raw, err := schema.Master()
if err != nil {
fmt.Fprintln(w, "WARNING: Failed to open master schema file; pvc cannot validate outgoing messages.")
}
schemaMaster, err = gjs.NewSchema(gjs.NewStringLoader(string(raw)))
if err != nil {
panic("bad schema...?")
}
client := http.Client{Timeout: 5 * time.Second}
fmt.Fprintln(w, "Generating a logic state message...")
reader := bufio.NewReader(os.Stdin)
MenuLoop:
for {
fmt.Fprintf(w, "\n")
lsc.printCurrentState(w, *ls)
var input string
for {
fmt.Fprintf(w, "\nSelect a value to edit by number, (p)rint current JSON message, (s)end, or (q)uit: ")
l, err := fmt.Fscanln(reader, &input)
if l > 1 || err != nil {
continue
}
switch input {
case "q", "quit":
fmt.Fprintf(w, "\nQuitting; message was not sent\n")
os.Exit(1)
case "s", "send":
msg, err := toJSONBytes(*ls)
if err != nil {
log.Fatalf("\nFailed to marshal JSON of logic state object, no message sent\n")
}
resp, err := client.Post(cmd.Flags().Lookup("target").Value.String(), "application/json", bytes.NewReader(msg))
if err != nil {
log.Fatalf(err.Error())
}
bod, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
log.Fatalf(err.Error())
}
if resp.StatusCode >= 200 && resp.StatusCode <= 300 {
fmt.Printf("Message accepted (HTTP code %v), msgid %v\n", resp.StatusCode, string(bod))
} else {
fmt.Printf("Message was rejected with HTTP code %v and message %v\n", resp.StatusCode, string(bod))
}
break MenuLoop
case "p", "print":
byts, err := toJSONBytes(*ls)
if err != nil {
fmt.Fprintf(w, "Error while generating JSON for printing: %q", err.Error())
continue MenuLoop
}
var prettied bytes.Buffer
err = json.Indent(&prettied, byts, "", " ")
if err != nil {
fmt.Fprintf(w, "Error while generating JSON for printing: %q", err.Error())
continue MenuLoop
}
fmt.Fprintf(w, "\nMessage that will be sent to %s:\n", cmd.Flags().Lookup("target").Value)
prettied.WriteTo(w)
w.WriteString("\n")
default:
num, interr := strconv.Atoi(input)
if interr != nil {
continue
} else if 0 < num && num < 10 {
switch num {
case 1:
lsc.collectPath(w, reader, ls)
case 2:
lsc.collectHostFQDN(w, reader, ls)
case 3:
lsc.collectHostNick(w, reader, ls)
case 4:
lsc.collectCommit(w, reader, ls)
case 5:
lsc.collectVersion(w, reader, ls)
case 6:
lsc.collectSemver(w, reader, ls)
case 7:
lsc.collectLgroup(w, reader, ls)
case 8:
lsc.collectNick(w, reader, ls)
case 9:
lsc.collectType(w, reader, ls)
}
continue MenuLoop
} else {
continue
}
}
}
}
}