Beispiel #1
0
func runDotDumper(cmd *cobra.Command, args []string) {
	g := represent.NewGraph()
	raw, err := schema.Master()
	if err != nil {
		panic(fmt.Sprint("Failed to open master schema file, test must abort. message:", err.Error()))
	}

	schemaMaster, err := gjs.NewSchema(gjs.NewStringLoader(string(raw)))
	if err != nil {
		panic("bad schema...?")
	}

	if len(args) < 1 {
		log.Fatalf("Must provide at least one directory argument to dotdumper.")
	}

	var k uint64 = 0
	for _, dir := range args {
		fl, err := ioutil.ReadDir(dir)
		if err != nil {
			erro.Printf("Failed to read directory '%v' with error %v\n", dir, err)
		}

		for _, f := range fl {
			if match, _ := regexp.MatchString("\\.json$", f.Name()); match && !f.IsDir() {
				src, err := ioutil.ReadFile(dir + "/" + f.Name())
				if err != nil {
					erro.Printf("Failed to read fixture file %v/%v\n", dir, f.Name())
					continue
				}

				result, err := schemaMaster.Validate(gjs.NewStringLoader(string(src)))
				if err != nil {
					erro.Printf("Validation process terminated with errors for %v/%v. Error: \n%v\n", dir, f.Name(), err.Error())
					continue
				}

				if !result.Valid() {
					for _, desc := range result.Errors() {
						erro.Printf("\t%s\n", desc)
					}
				} else {
					k++
					m := ingest.Message{}
					json.Unmarshal(src, &m)

					g = g.Merge(k, m.UnificationForm())
					fmt.Printf("Merged message %v/%v into graph\n", dir, f.Name())
				}
			}
		}
	}

	pathflag := cmd.Flags().Lookup("output")
	if pathflag.Changed {
		ioutil.WriteFile(pathflag.Value.String(), GenerateDot(g), 0644)
	} else {
		fmt.Println(string(GenerateDot(g)))
	}
}
Beispiel #2
0
func BenchmarkValidateMessageOneAndTwo(b *testing.B) {
	d1 := MsgJSON[0]
	d2 := MsgJSON[1]
	msg1 := gjs.NewStringLoader(string(d1))
	msg2 := gjs.NewStringLoader(string(d2))

	for i := 0; i < b.N; i++ {
		masterSchema.Validate(msg1)
		masterSchema.Validate(msg2)
	}
}
Beispiel #3
0
func BenchmarkValidateMessageOneAndTwo(b *testing.B) {
	d1 := msgJSON[0]
	d2 := msgJSON[1]
	msg1 := gojsonschema.NewStringLoader(string(d1))
	msg2 := gojsonschema.NewStringLoader(string(d2))

	for i := 0; i < b.N; i++ {
		schema.Master().Validate(msg1)
		schema.Master().Validate(msg2)
	}
}
Beispiel #4
0
// Reads all message fixtures from fixtures/ein and validates them
// against the master message schema (schema.json).
func TestMessageValidity(t *testing.T) {
	files, err := ioutil.ReadDir("../fixtures/ein/")
	if err != nil {
		t.Error("Failed to scan message fixtures dir:", err.Error())
		t.FailNow()
	}

	for _, f := range files {
		if testing.Verbose() {
			t.Log("Beginning validation on", f.Name())
		}

		src, _ := ioutil.ReadFile("../fixtures/ein/" + f.Name())
		msg := gjs.NewStringLoader(string(src))
		result, err := masterSchema.Validate(msg)

		if err != nil {
			panic(err.Error())
		}

		if result.Valid() {
			if testing.Verbose() {
				t.Log(f.Name(), "passed validation")
			}
		} else {
			for _, desc := range result.Errors() {
				t.Errorf("%s\n", strings.Replace(desc.String(), "root", f.Name(), 1))
			}
		}
	}
}
Beispiel #5
0
func init() {
	for i := range make([]struct{}, 8) {
		m := &ingest.Message{}

		path := fmt.Sprintf("../fixtures/ein/%v.json", i+1)
		f, err := ioutil.ReadFile(path)
		if err != nil {
			panic("json fnf: " + path)
		}

		MsgJSON = append(MsgJSON, f)
		_ = json.Unmarshal(f, m)
		Msgs = append(Msgs, m)
	}

	src, err := schema.Master()
	if err != nil {
		panic(fmt.Sprint("Failed to open master schema file, test must abort. Note: this test must be run from the pipeviz repo root. message:", err.Error()))
	}

	masterSchema, err = gjs.NewSchema(gjs.NewStringLoader(string(src)))
	if err != nil {
		panic(fmt.Sprint("Failed to create a schema object from the master schema.json:", err.Error()))
	}
}
Beispiel #6
0
func BenchmarkValidateMessageTwo(b *testing.B) {
	d := MsgJSON[0]
	msg := gjs.NewStringLoader(string(d))

	for i := 0; i < b.N; i++ {
		masterSchema.Validate(msg)
	}
}
Beispiel #7
0
func BenchmarkValidateMessageTwo(b *testing.B) {
	d := msgJSON[0]
	msg := gojsonschema.NewStringLoader(string(d))

	for i := 0; i < b.N; i++ {
		schema.Master().Validate(msg)
	}
}
Beispiel #8
0
func runValidate(cmd *cobra.Command, args []string) {
	var errors int
	for _, dir := range args {
		finfo, err := os.Stat(dir)
		if err != nil {
			fmt.Printf("Could not stat '%s' with error: %s\n", dir, err)
			continue
		}
		if !finfo.IsDir() {
			continue
		}

		fl, err := ioutil.ReadDir(dir)
		if err != nil {
			fmt.Printf("Failed to read directory '%v' with error %v\n", dir, err)
		}

		for _, f := range fl {
			if match, _ := regexp.MatchString("\\.json$", f.Name()); match && !f.IsDir() {
				src, err := ioutil.ReadFile(dir + "/" + f.Name())
				if err != nil {
					errors |= FileReadFail
					fmt.Printf("Failed to read fixture file %v/%v\n", dir, f.Name())
					continue
				}

				result, err := schema.Master().Validate(gjs.NewStringLoader(string(src)))
				if err != nil {
					errors |= ValidationError
					fmt.Printf("Validation process terminated with errors for %v/%v. Error: \n%v\n", dir, f.Name(), err.Error())
					continue
				}

				if !result.Valid() {
					errors |= ValidationFail
					fmt.Printf("Errors in %v/%v:\n", dir, f.Name())
					for _, desc := range result.Errors() {
						fmt.Printf("\t%s\n", desc)
					}
				} else {
					fmt.Printf("%v/%v passed validation\n", dir, f.Name())
				}
			}
		}
	}
	os.Exit(errors)
}
Beispiel #9
0
func validateAndPrint(w io.Writer, v interface{}) {
	msg, err := toJSONBytes(v)
	if err != nil {
		fmt.Fprintf(w, err.Error())
		return
	}

	// Validate the current state of the message
	result, err := schemaMaster.Validate(gojsonschema.NewStringLoader(string(msg)))
	if err != nil {
		fmt.Fprintf(w, "\nError while attempting to validate data: %s\n", err.Error())
		return
	}
	if !result.Valid() {
		fmt.Fprintln(w, "\nAs it stands now, the data will fail validation if sent to a pipeviz server. Errors:")
		for _, desc := range result.Errors() {
			fmt.Fprintf(w, "\t%s\n", desc)
		}
	}
}
Beispiel #10
0
func init() {
	raw, err := Asset("schema.json")
	if err != nil {
		// It is correct to fatal out here because there is no use case for importing
		// the schema package that does not require the master schema to be present
		logrus.WithFields(logrus.Fields{
			"system": "schema",
			"err":    err,
		}).Fatal("Failed to locate raw bytes/file for master schema")
	}

	schema, err = gojsonschema.NewSchema(gojsonschema.NewStringLoader(string(raw)))
	if err != nil {
		// Correct to fatal, again for the same reasons
		logrus.WithFields(logrus.Fields{
			"system": "schema",
			"err":    err,
		}).Fatal("Failed to create master schema object")
	}
}
Beispiel #11
0
// Tests that all message fixtures pass schema validation against the master
// message schema (../schema/schema.json).
func TestFixtureValidity(t *testing.T) {
	// Ensure stable order, for user's sanity
	names := fixtures.AssetNames()
	sort.Strings(names)

	// If this test is run without having generated assets via go-bindata, it could
	// pass. This should be precluded because the test won't compile without the
	// functions created by go-bindata, but this is an easy guard to put in place, so
	// might as well.
	if len(names) == 0 {
		t.Error("No asset files found for validation - has code generation been run correctly? (should be `make gen`)")
	}

	for _, name := range names {
		if testing.Verbose() {
			t.Log("Beginning validation on", name)
		}

		b, err := fixtures.Asset(name)
		result, err := schema.Master().Validate(gojsonschema.NewStringLoader(string(b)))

		if err != nil {
			t.Errorf("Error while validating fixture %v: %s", name, err)
			panic(err.Error())
		}

		if result.Valid() {
			if testing.Verbose() {
				t.Log(name, "passed validation")
			}
		} else {
			buf := bytes.NewBufferString(fmt.Sprintf("%s failed validation with the following errors:\n", name))
			for _, desc := range result.Errors() {
				buf.WriteString(fmt.Sprintf("%s\n", strings.Replace(desc.String(), "root", name, 1)))
			}

			t.Errorf(buf.String())
		}
	}
}
Beispiel #12
0
func main() {
	pflag.Parse()
	setUpLogging()

	src, err := schema.Master()
	if err != nil {
		log.WithFields(log.Fields{
			"system": "main",
			"err":    err,
		}).Fatal("Could not locate master schema file, exiting")
	}

	// The master JSON schema used for validating all incoming messages
	masterSchema, err := gjs.NewSchema(gjs.NewStringLoader(string(src)))
	if err != nil {
		log.WithFields(log.Fields{
			"system": "main",
			"err":    err,
		}).Fatal("Error while creating a schema object from the master schema file, exiting")
	}

	// Channel to receive persisted messages from HTTP workers. 1000 cap to allow
	// some wiggle room if there's a sudden burst of messages and the interpreter
	// gets behind.
	interpretChan := make(chan *mlog.Record, 1000)

	var listenAt string
	if *bindAll == false {
		listenAt = "127.0.0.1:"
	} else {
		listenAt = ":"
	}

	var j mlog.Store
	switch *mlstore {
	case "bolt":
		j, err = boltdb.NewBoltStore(*dbPath + "/mlog.bolt")
		if err != nil {
			log.WithFields(log.Fields{
				"system": "main",
				"err":    err,
			}).Fatal("Error while setting up bolt mlog storage, exiting")
		}
	case "memory":
		j = mem.NewMemStore()
	default:
		log.WithFields(log.Fields{
			"system":  "main",
			"storage": *mlstore,
		}).Fatal("Invalid storage type requested for mlog, exiting")
	}

	// Restore the graph from the mlog (or start from nothing if mlog is empty)
	// TODO move this down to after ingestor is started
	g, err := restoreGraph(j)
	if err != nil {
		log.WithFields(log.Fields{
			"system": "main",
			"err":    err,
		}).Fatal("Error while rebuilding the graph from the mlog")
	}

	// Kick off fanout on the master/singleton graph broker. This will bridge between
	// the state machine and the listeners interested in the machine's state.
	brokerChan := make(chan system.CoreGraph, 0)
	broker.Get().Fanout(brokerChan)
	brokerChan <- g

	srv := ingest.New(j, masterSchema, interpretChan, brokerChan, MaxMessageSize)

	// Kick off the http message ingestor.
	// TODO let config/params control address
	go func() {
		if *ingestKey != "" && *ingestCert == "" {
			*ingestCert = *ingestKey + ".crt"
		}
		err := srv.RunHTTPIngestor(listenAt+strconv.Itoa(DefaultIngestionPort), *ingestKey, *ingestCert)
		if err != nil {
			log.WithFields(log.Fields{
				"system": "main",
				"err":    err,
			}).Fatal("Error while starting the ingestion http server")
		}
	}()

	// Kick off the intermediary interpretation goroutine that receives persisted
	// messages from the ingestor, merges them into the state graph, then passes
	// them along to the graph broker.
	go srv.Interpret(g)

	// And finally, kick off the webapp.
	// TODO let config/params control address
	if *webappKey != "" && *webappCert == "" {
		*webappCert = *webappKey + ".crt"
	}
	go RunWebapp(listenAt+strconv.Itoa(DefaultAppPort), *webappKey, *webappCert, j.Get)

	// Block on goji's graceful waiter, allowing the http connections to shut down nicely.
	// FIXME using this should be unnecessary if we're crash-only
	graceful.Wait()
}
Beispiel #13
0
// runGenLS is the main entry point for running the logic state-generating
// ls subcommand.
func (lsc lsCmd) runGenLS(cmd *cobra.Command, args []string) {
	ls := &semantic.LogicState{}

	if !cmd.Flags().Lookup("no-detect").Changed {
		*ls = lsc.detectDefaults()
	}

	// Write directly to stdout, at least for now
	w := os.Stdout

	// Prep schema to validate the messages as we go
	raw, err := schema.Master()
	if err != nil {
		fmt.Fprintln(w, "WARNING: Failed to open master schema file; pvc cannot validate outgoing messages.")
	}

	schemaMaster, err = gjs.NewSchema(gjs.NewStringLoader(string(raw)))
	if err != nil {
		panic("bad schema...?")
	}

	client := http.Client{Timeout: 5 * time.Second}

	fmt.Fprintln(w, "Generating a logic state message...")
	reader := bufio.NewReader(os.Stdin)
MenuLoop:
	for {
		fmt.Fprintf(w, "\n")
		lsc.printCurrentState(w, *ls)

		var input string
		for {
			fmt.Fprintf(w, "\nSelect a value to edit by number, (p)rint current JSON message, (s)end, or (q)uit: ")
			l, err := fmt.Fscanln(reader, &input)
			if l > 1 || err != nil {
				continue
			}

			switch input {
			case "q", "quit":
				fmt.Fprintf(w, "\nQuitting; message was not sent\n")
				os.Exit(1)
			case "s", "send":
				msg, err := toJSONBytes(*ls)
				if err != nil {
					log.Fatalf("\nFailed to marshal JSON of logic state object, no message sent\n")
				}

				resp, err := client.Post(cmd.Flags().Lookup("target").Value.String(), "application/json", bytes.NewReader(msg))
				if err != nil {
					log.Fatalf(err.Error())
				}

				bod, err := ioutil.ReadAll(resp.Body)
				resp.Body.Close()
				if err != nil {
					log.Fatalf(err.Error())
				}

				if resp.StatusCode >= 200 && resp.StatusCode <= 300 {
					fmt.Printf("Message accepted (HTTP code %v), msgid %v\n", resp.StatusCode, string(bod))
				} else {
					fmt.Printf("Message was rejected with HTTP code %v and message %v\n", resp.StatusCode, string(bod))
				}
				break MenuLoop

			case "p", "print":
				byts, err := toJSONBytes(*ls)
				if err != nil {
					fmt.Fprintf(w, "Error while generating JSON for printing: %q", err.Error())
					continue MenuLoop
				}

				var prettied bytes.Buffer
				err = json.Indent(&prettied, byts, "", "    ")
				if err != nil {
					fmt.Fprintf(w, "Error while generating JSON for printing: %q", err.Error())
					continue MenuLoop
				}

				fmt.Fprintf(w, "\nMessage that will be sent to %s:\n", cmd.Flags().Lookup("target").Value)
				prettied.WriteTo(w)
				w.WriteString("\n")

			default:
				num, interr := strconv.Atoi(input)
				if interr != nil {
					continue
				} else if 0 < num && num < 10 {
					switch num {
					case 1:
						lsc.collectPath(w, reader, ls)
					case 2:
						lsc.collectHostFQDN(w, reader, ls)
					case 3:
						lsc.collectHostNick(w, reader, ls)
					case 4:
						lsc.collectCommit(w, reader, ls)
					case 5:
						lsc.collectVersion(w, reader, ls)
					case 6:
						lsc.collectSemver(w, reader, ls)
					case 7:
						lsc.collectLgroup(w, reader, ls)
					case 8:
						lsc.collectNick(w, reader, ls)
					case 9:
						lsc.collectType(w, reader, ls)
					}
					continue MenuLoop
				} else {
					continue
				}
			}
		}
	}
}