func ProcessFile(file string) ([]byte, error) {
ext := filepath.Ext(file)
isYaml := ext == ".yaml" || ext == ".yml"
isJson := ext == ".json"
if !isYaml && !isJson {
return nil, nil
}
bytes, err := ioutil.ReadFile(file)
if err != nil {
return nil, kerr.Wrap("NMWROTKPLJ", err)
}
if isYaml {
j, err := yaml.YAMLToJSON(bytes)
if err != nil {
return nil, kerr.Wrap("FAFJCYESRH", err)
}
bytes = j
}
return bytes, nil
}
// ScanFiles takes a chanel of files
func ScanFilesToBytes(ctx context.Context, in chan File) chan Content {
out := make(chan Content)
go func() {
defer close(out)
for {
select {
case value, open := <-in:
if !open {
return
}
if value.Err != nil {
out <- Content{"", nil, kerr.Wrap("PQUCOUYLJE", value.Err)}
return
}
bytes, err := ProcessFile(value.File)
// process returns Bytes == nil for non json files, so we should skip them
if bytes != nil || err != nil {
out <- Content{value.File, bytes, err}
}
case <-ctx.Done():
out <- Content{"", nil, kerr.Wrap("AFBJCTFOKX", ctx.Err())}
return
}
}
}()
return out
}
// Generate generates the source code for type structs, and writes the
// generated.go to the filesystem.
func Generate(ctx context.Context, env *envctx.Env) error {
wgctx.Add(ctx, "Generate")
defer wgctx.Done(ctx, "Generate")
cmd := cmdctx.FromContext(ctx)
cmd.Printf("Generating types for %s... ", env.Path)
outputDir := env.Dir
filename := "generated.go"
source, err := generate.Structs(ctx, env)
if err != nil {
return kerr.Wrap("XFNESBLBTQ", err)
}
// We only backup in the system structs and types files because they are
// the only generated files we ever need to roll back
backup := env.Path == "kego.io/system"
if err = save(outputDir, source, filename, backup); err != nil {
return kerr.Wrap("UONJTTSTWW", err)
} else {
cmd.Println("OK.")
}
return nil
}
func walkFile(path string) error {
b, err := ioutil.ReadFile(path)
if err != nil {
return kerr.Wrap("FGAHVRNUPV", err)
}
fset := token.NewFileSet()
file, err := parser.ParseFile(fset, path, nil, parser.ParseComments)
if err != nil {
return kerr.Wrap("ALRIEYJBJE", err)
}
// visitor implements ast.Visitor
v := &visitor{b, false}
ast.Walk(v, file)
// if we made a replacement, re-write the modified file
if v.Found {
fmt.Println(path)
f, err := os.Create(path)
if err != nil {
return kerr.Wrap("QFPDQRTIRS", err)
}
defer f.Close()
if err := printer.Fprint(f, fset, file); err != nil {
return kerr.Wrap("VEPVDWFWEF", err)
}
}
return nil
}
func mutateRestoreNode(ctx context.Context, n *node.Node, p *node.Node, b *node.Node) error {
n.Restore(ctx, b)
if p == nil {
return nil
}
switch p.Type.NativeJsonType(ctx) {
case system.J_MAP:
// don't have to call n.InitialiseMapItem because the node is already
// initialized
if err := n.AddToMap(ctx, p, n.Key, true); err != nil {
return kerr.Wrap("TOPLOONYCL", err)
}
case system.J_ARRAY:
// don't have to call n.InitialiseArrayItem because the node is already
// initialized
if err := n.AddToArray(ctx, p, n.Index, true); err != nil {
return kerr.Wrap("WFXSQYOEAY", err)
}
case system.J_OBJECT:
// don't have to call n.InitialiseObjectField because the node is
// already initialized
if err := n.AddToObject(ctx, p, n.Rule, n.Key, true); err != nil {
return kerr.Wrap("QMBJQMLOCY", err)
}
}
return nil
}
func getInfo(ctx context.Context, dir string) (info *generate.InfoStruct, found bool, err error) {
f, err := os.Open(filepath.Join(dir, "generated.go"))
if err != nil {
if os.IsNotExist(err) {
return nil, false, nil
}
return nil, false, kerr.Wrap("TLFTCRNBKK", err)
}
defer f.Close()
scanner := bufio.NewScanner(f)
scanner.Scan()
if err := scanner.Err(); err != nil {
return nil, false, kerr.Wrap("HLDYEAPLEQ", err)
}
if !strings.HasPrefix(scanner.Text(), "// info:{") {
return nil, false, nil
}
data := []byte(scanner.Text()[8:])
var i generate.InfoStruct
if err := json.Unmarshal(data, &i); err != nil {
return nil, false, kerr.Wrap("UJXKJVLXHG", err)
}
return &i, true, nil
}
func extractFields(ctx context.Context, fields map[string]*system.Field, t *system.Type) error {
for t.Alias != nil {
t = system.WrapRule(ctx, t.Alias).Parent
}
if !t.Basic && !t.Interface {
// All types apart from Basic types embed system:object
ob, ok := system.GetTypeFromCache(ctx, "kego.io/system", "object")
if !ok {
return kerr.New("YRFWOTIGFT", "Type system:object not found in sys ctx")
}
if err := extractFields(ctx, fields, ob); err != nil {
return kerr.Wrap("DTQEFALIMM", err)
}
}
for _, embedRef := range t.Embed {
embed, ok := system.GetTypeFromCache(ctx, embedRef.Package, embedRef.Name)
if !ok {
return kerr.New("SLIRILCARQ", "Type %s not found in sys ctx", embedRef)
}
if err := extractFields(ctx, fields, embed); err != nil {
return kerr.Wrap("JWAPCVIYBJ", err)
}
}
for name, rule := range t.Fields {
if _, ok := fields[name]; ok {
return kerr.New("BARXPFXQNB", "Duplicate field %s", name)
}
fields[name] = &system.Field{Name: name, Rule: rule, Origin: t.Id}
}
return nil
}
func (n *Node) SetValue(ctx context.Context, value interface{}) error {
switch n.JsonType {
case system.J_STRING:
val := value.(string)
if n.ValueString == val {
// ignore the change if there's no change to the value
return nil
}
if err := n.SetValueString(ctx, val); err != nil {
return kerr.Wrap("NCIMXDORED", err)
}
case system.J_BOOL:
val := value.(bool)
if n.ValueBool == val {
// ignore the change if there's no change to the value
return nil
}
if err := n.SetValueBool(ctx, val); err != nil {
return kerr.Wrap("HKFEEMFRHR", err)
}
case system.J_NUMBER:
val := value.(float64)
if n.ValueNumber == val {
// ignore the change if there's no change to the value
return nil
}
if err := n.SetValueNumber(ctx, val); err != nil {
return kerr.Wrap("LBEBBNFJVG", err)
}
}
return nil
}
func buildRulesCollectionChildren(ctx context.Context, n *node.Node, cache map[*node.Node][]system.RuleInterface) error {
rules := n.Rule.Interface.(system.ObjectInterface).GetObject(nil).Rules
// If we have additional rules on the main items rule, we should add them to rules
items, err := n.Rule.ItemsRule()
if err != nil {
return kerr.Wrap("YFNERJIKWF", err)
}
itemsRuleOb := items.Interface.(system.ObjectInterface).GetObject(nil)
if len(itemsRuleOb.Rules) > 0 {
rules = append(rules, itemsRuleOb.Rules...)
}
for _, child := range n.Array {
err := buildRulesObject(ctx, child, cache, rules)
if err != nil {
return kerr.Wrap("FWKXQRMPQJ", err)
}
}
for _, child := range n.Map {
err := buildRulesObject(ctx, child, cache, rules)
if err != nil {
return kerr.Wrap("MKYHIDMORV", err)
}
}
return nil
}
func (n *Node) setZero(ctx context.Context, null bool, missing bool) error {
if missing && !null {
return kerr.New("NYQULBBBHO", "If missing, must also be null")
}
if missing && (n.Parent == nil || n.Parent.JsonType != system.J_OBJECT) {
return kerr.New("XRYLQWRNPH", "Parent must be J_OBJECT")
}
if n.Type == nil {
return kerr.New("ABXFQOYCBA", "Can't set value without a type")
}
if n.Type.IsNativeCollection() && n.Rule == nil {
return kerr.New("VGKTIRMDTJ", "Can't create collection zero value without a rule")
}
n.Missing = missing
n.Null = null
n.ValueString = ""
n.ValueNumber = 0.0
n.ValueBool = false
n.Array = []*Node{}
n.Map = map[string]*Node{}
if null {
n.JsonType = system.J_NULL
} else {
// if this node was previously null, we must reset the json type
n.JsonType = n.Type.NativeJsonType(ctx)
}
var rv reflect.Value
if n.Type.IsNativeCollection() {
var err error
if rv, err = n.Rule.ZeroValue(null); err != nil {
return kerr.Wrap("WNQLTRJRBD", err)
}
} else {
// this is for both objects and native values
var err error
if rv, err = n.Type.ZeroValue(ctx, null); err != nil {
return kerr.Wrap("UDBVTIDRIK", err)
}
}
n.Value = rv.Interface()
n.setVal(rv)
if !null && n.Type.IsNativeObject() {
if err := n.initialiseFields(ctx, nil, true); err != nil {
return kerr.Wrap("VSAXCHGCOG", err)
}
if err := n.setCorrectTypeField(ctx); err != nil {
return kerr.Wrap("CUCJDNBBSU", err)
}
}
return nil
}
func (n *Node) initialiseCollectionItem(ctx context.Context, parent *Node, key string, index int) error {
n.resetAllValues()
n.Parent = parent
n.Index = index
n.Key = key
n.Missing = false
var collectionRule *system.RuleWrapper
if n.Parent.Type.Alias != nil {
collectionRule = system.WrapRule(ctx, n.Parent.Type.Alias)
} else {
collectionRule = parent.Rule
}
rule, err := collectionRule.ItemsRule()
if err != nil {
return kerr.Wrap("SBJVMGUOOA", err)
}
n.Rule = rule
t, err := extractType(ctx, system.Pack(nil), rule)
if err != nil {
return kerr.Wrap("EQNRHQWXFJ", err)
}
if err := n.setType(ctx, t); err != nil {
return kerr.Wrap("UPAQMUGDNH", err)
}
return nil
}
func MarshalIndent(ctx context.Context, v interface{}, prefix, indent string) ([]byte, error) {
i, err := system.Repack(ctx, v)
if err != nil {
return nil, kerr.Wrap("QNHVVOBKFO", err)
}
b, err := json.MarshalIndent(i, prefix, indent)
if err != nil {
return nil, kerr.Wrap("QXXEBOMOFG", err)
}
return b, nil
}
func Marshal(ctx context.Context, v interface{}) ([]byte, error) {
i, err := system.Repack(ctx, v)
if err != nil {
return nil, kerr.Wrap("JVGOWMVMXN", err)
}
b, err := json.Marshal(i)
if err != nil {
return nil, kerr.Wrap("LXDTUOBQPD", err)
}
return b, nil
}
func Open(ctx context.Context, filename string) (value interface{}, err error) {
bytes, err := scanner.ProcessFile(filename)
if err != nil {
return nil, kerr.Wrap("HPWXWFTKWA", err)
}
err = system.Unmarshal(ctx, bytes, &value)
if err != nil {
return nil, kerr.Wrap("CXIULJCEBE", err)
}
return
}
func (n *Node) SetValueZero(ctx context.Context, null bool, t *system.Type) error {
if t != nil {
if err := n.setType(ctx, t); err != nil {
return kerr.Wrap("TGPRRSUSMQ", err)
}
}
if err := n.setZero(ctx, null, false); err != nil {
return kerr.Wrap("JWCKCAJXUB", err)
}
return nil
}
func validateBytes(ctx context.Context, bytes []byte) (errors []ValidationError, err error) {
n, err := node.Unmarshal(ctx, bytes)
if err != nil {
return nil, kerr.Wrap("QIVNOQKCQF", err)
}
errors, err = ValidateNode(ctx, n)
if err != nil {
return nil, kerr.Wrap("RVKNMWKQHD", err)
}
return errors, nil
}
func getInfo(infoBase64 string) (info *shared.Info, err error) {
info = &shared.Info{}
infoBytes, err := base64.StdEncoding.DecodeString(infoBase64)
if err != nil {
return nil, kerr.Wrap("UTKDDLYKKH", err)
}
buf := bytes.NewBuffer(infoBytes)
if err := gob.NewDecoder(buf).Decode(info); err != nil {
return nil, kerr.Wrap("AAFXLQRUEW", err)
}
return info, nil
}
func Repack(ctx context.Context, data interface{}) (interface{}, error) {
if re, ok := data.(Repacker); ok {
p, pkg, name, jt, err := re.Repack(ctx)
if err != nil {
return nil, kerr.Wrap("ISAOPMMEYI", err)
}
if jt == J_OBJECT {
return p, nil
}
typRef := NewReference(pkg, name)
typeVal, err := typRef.ValueContext(ctx)
if err != nil {
return nil, kerr.Wrap("IWSCKOWIMW", err)
}
out := map[string]interface{}{}
out["type"] = typeVal
out["value"] = p
return out, nil
}
switch data.(type) {
case string, float64, bool:
return data, nil
}
val := reflect.ValueOf(data)
switch val.Kind() {
case reflect.Slice:
out := []interface{}{}
for i := 0; i < val.Len(); i++ {
c, err := Repack(ctx, val.Index(i).Interface())
if err != nil {
return nil, kerr.Wrap("KSHOKVIBRS", err)
}
out = append(out, c)
}
return out, nil
case reflect.Map:
out := map[string]interface{}{}
for _, key := range val.MapKeys() {
c, err := Repack(ctx, val.MapIndex(key).Interface())
if err != nil {
return nil, kerr.Wrap("SOLGDVLWSK", err)
}
out[key.Interface().(string)] = c
}
return out, nil
}
return nil, kerr.New("POKFWVHEDY", "%T does can't be repacked", data)
}
func (n *Node) setCorrectTypeField(ctx context.Context) error {
typeField, ok := n.Map["type"]
if !ok {
return kerr.New("DQKGYKFQKJ", "type field not found")
}
typeString, err := n.Type.Id.ValueContext(ctx)
if err != nil {
return kerr.Wrap("MRHEBUUXBB", err)
}
if err := typeField.SetValueString(ctx, typeString); err != nil {
return kerr.Wrap("CURDKCQLGS", err)
}
return nil
}
func Marshal(ctx context.Context, data interface{}) ([]byte, error) {
p, err := Repack(ctx, data)
if err != nil {
return nil, kerr.Wrap("OMFSDKIBCY", err)
}
b, err := json.Marshal(p)
if err != nil {
return nil, kerr.Wrap("YCEBRGOCGO", err)
}
return b, nil
}
func scanForTypesAndExports(ctx context.Context, env *envctx.Env, cache *sysctx.SysPackageInfo, hash *PackageHasher) error {
// While we're scanning for types, we should use a custom unpacking env,
// because the env from the context is the one of the local package.
files := scanner.ScanDirToFiles(ctx, env.Dir, env.Recursive)
bytes := scanner.ScanFilesToBytes(ctx, files)
localContext := envctx.NewContext(ctx, env)
for b := range bytes {
if b.Err != nil {
return kerr.Wrap("JACKALTIGG", b.Err)
}
o := &system.ObjectStub{}
if err := system.Unmarshal(localContext, b.Bytes, o); err != nil {
return kerr.Wrap("HCYGNBDFFA", err)
}
if o.Type == nil {
return kerr.New("NUKWIHYFMQ", "%s has no type", b.File)
}
if o.Id == nil && *o.Type != *system.NewReference("kego.io/system", "package") {
// we tolerate missing ID only for system:package
return kerr.New("DLLMKTDYFW", "%s has no id", b.File)
}
relativeFile, err := filepath.Rel(env.Dir, b.File)
if err != nil {
return kerr.Wrap("AWYRJSCYQS", err)
}
switch *o.Type {
case *system.NewReference("kego.io/system", "type"):
if err := ProcessTypeFileBytes(ctx, env, relativeFile, b.Bytes, cache, hash); err != nil {
return kerr.Wrap("IVEFDDSKHE", err)
}
case *system.NewReference("kego.io/system", "package"):
cache.PackageBytes = b.Bytes
cache.PackageFilename = relativeFile
default:
cache.Globals.Set(o.Id.Name, relativeFile)
if o.Export {
cache.Exports.Set(o.Id.Name, o.Type.Name, o.Type.Package, b.Bytes)
if hash != nil {
hash.Exports[o.Id.Name+" "+o.Type.Name+" "+o.Type.Package] = cityhash.CityHash64(b.Bytes, uint32(len(b.Bytes)))
}
}
}
}
return nil
}
func Unmarshal(ctx context.Context, in []byte, out interface{}) error {
var i interface{}
if err := json.Unmarshal(in, &i); err != nil {
return kerr.Wrap("PDTPGAYXRX", err)
}
p := Pack(i)
if err := Unpack(ctx, p, out); err != nil {
return kerr.Wrap("BCTGSARAXH", err)
}
return nil
}
func UnpackRefelctValue(ctx context.Context, p Packed, val reflect.Value) error {
if up, ok := val.Interface().(Unpacker); ok {
if err := up.Unpack(ctx, p, false); err != nil {
return kerr.Wrap("FGSFQHHMNE", err)
}
return nil
}
if val.Type() == reflect.TypeOf("") {
unpacked, err := UnpackString(ctx, p)
if err != nil {
return kerr.Wrap("XDGRKOVCHD", err)
}
val.Set(reflect.ValueOf(unpacked))
return nil
}
if val.Type() == reflect.TypeOf(0.0) {
unpacked, err := UnpackNumber(ctx, p)
if err != nil {
return kerr.Wrap("IPYJYKLNBE", err)
}
val.Set(reflect.ValueOf(unpacked))
return nil
}
if val.Type() == reflect.TypeOf(false) {
unpacked, err := UnpackBool(ctx, p)
if err != nil {
return kerr.Wrap("RBILDURHWV", err)
}
val.Set(reflect.ValueOf(unpacked))
return nil
}
if val.Type().Kind() == reflect.Ptr && val.Type().Elem().Kind() == reflect.Interface {
i, err := UnpackUnknownType(ctx, p, true, "", "")
if err != nil {
return kerr.Wrap("TDEFTWSBFD", err)
}
val.Set(reflect.ValueOf(i))
} else {
return kerr.New("IFCTNULWUA", "If unmarshaling into an unknown type, must pass in a *interface{}. Found: %v", val.Type())
}
return nil
}
func Editor(ctx context.Context, env *envctx.Env) (source []byte, err error) {
g := builder.New("main")
g.Imports.Anonymous("kego.io/system")
g.Imports.Anonymous(env.Path)
for _, p := range env.Aliases {
g.Imports.Anonymous(p)
}
/*
func main() {
if err := client.Start(); err != nil {
fmt.Println(err.Error())
}
}
*/
g.Println("func main() {")
{
clientStart := builder.Reference("kego.io/editor/client", "Start", env.Path, g.Imports.Add)
g.Println("if err := ", clientStart, "(); err != nil {")
{
fmtPrintln := builder.Reference("fmt", "Println", env.Path, g.Imports.Add)
g.Println(fmtPrintln, "(err.Error())")
}
g.Println("}")
}
g.Println("}")
b, err := g.Build()
if err != nil {
return nil, kerr.Wrap("CBTOLUQOGL", err)
}
return b, nil
}
func (p *Parser) kegoProduction(value interface{}) func(*node.Node) (bool, error) {
logger.Print("Creating kegoProduction validator ", value)
return func(n *node.Node) (bool, error) {
if n.Null || n.Missing {
return false, nil
}
tokenString := value.(string)
kegoType := tokenString[1 : len(tokenString)-1]
r, err := system.NewReferenceFromString(p.ctx, kegoType)
if err != nil {
return false, kerr.Wrap("RWDOYBBDVK", err)
}
logger.Print("kegoProduction ? ", n.Type.Id.Value(), " == ", r.Value())
if n.Type.Id.Value() == r.Value() {
return true, nil
}
// If the specified type is an interface, we should check to see if the
// node implements the interface.
if i, ok := r.GetType(p.ctx); ok && i.Interface {
rt, ok := i.GetReflectType(p.ctx)
if ok {
if n.Type.Implements(p.ctx, rt) {
return true, nil
}
}
}
return false, nil
}
}
// TypeDefinition returns the Go source for the definition of the type
// [optional pointer][collection prefix][optional pointer][type name]
func TypeDefinition(ctx context.Context, field system.RuleInterface, path string, getAlias func(string) string) (string, error) {
outer := system.WrapRule(ctx, field)
outerPointer := outer.PassedAsPointerString(ctx)
// if the rule is a complex collection, with possibly several maps and
// arrays, this iterates over the rule and returns the go collection prefix
// - e.g. []map[string] for an array of maps. It also returns the inner rule.
prefix, inner, err := collectionPrefixInnerRule(ctx, "", outer)
if err != nil {
return "", kerr.Wrap("SOGEFOPJHB", err)
}
innerPointer := ""
// if we have a prefix we should also work out the innerPointer
if prefix != "" && inner.PassedAsPointer(ctx) {
innerPointer = "*"
}
var n string
if inner.Struct.Interface {
n = system.GoInterfaceName(inner.Parent.Id.Name)
} else {
n = system.GoName(inner.Parent.Id.Name)
}
name := Reference(inner.Parent.Id.Package, n, path, getAlias)
return fmt.Sprint(outerPointer, prefix, innerPointer, name), nil
}
func (p *Parser) GetNodes(selector string) ([]*node.Node, error) {
nodes, err := p.evaluateSelector(selector)
if err != nil {
return nil, kerr.Wrap("QFXXVGGHSS", err)
}
return nodes, nil
}
func (r *ReferenceRule) Enforce(ctx context.Context, data interface{}) (fail bool, messages []string, err error) {
if i, ok := data.(ReferenceInterface); ok && i != nil {
data = i.GetReference(ctx)
}
v, ok := data.(*Reference)
if !ok && data != nil {
return true, nil, kerr.New("BYDVGGETWW", "Reference rule: value %T should be *system.Reference", data)
}
// Pattern restriction should be the same as StringRule
var s *String
if v != nil {
s = NewString(v.Name)
}
sr := StringRule{
Rule: &Rule{Optional: r.Optional},
Pattern: r.Pattern,
PatternNot: r.PatternNot,
}
if fail, messages, err = sr.Enforce(ctx, s); err != nil {
return true, nil, kerr.Wrap("KYYJLYOSHT", err)
}
return
}
func comparePackageHash(ctx context.Context, path string) (changes bool, err error) {
scache := sysctx.FromContext(ctx)
spi, ok := scache.Get(path)
if !ok {
return false, kerr.New("NHXWLPHCHL", "%s not found in sys ctx", path)
}
for _, aliasPath := range spi.Aliases {
changes, err := comparePackageHash(ctx, aliasPath)
if err != nil {
return false, kerr.Wrap("DGJTLHQOCQ", err)
}
if changes {
return true, nil
}
}
jcache := jsonctx.FromContext(ctx)
jpi, ok := jcache.Packages.Get(path)
if !ok {
return true, nil
}
// pcache.Environment.Hash is computed after parsing all the data files.
// h.Hash is in generated.go (jsonctx.InitPackage), and correct when the types were generated.
if jpi.Hash != spi.Hash {
return true, nil
}
return false, nil
}
func (*Repacker) Process(ctx context.Context, data []byte, i *interface{}) error {
n, err := node.Unmarshal(ctx, data)
if err != nil {
return kerr.Wrap("RXAARYNHLP", err)
}
return system.Unpack(ctx, node.Pack(n), i)
}
