func TestNew(t *testing.T) { if _, err := New("fakefile"); err == nil { t.Errorf("wanted to return error for nonexistent file") } testInput := &redditproto.UserAgent{} if err := proto.UnmarshalText(` user_agent: "test" client_id: "id" client_secret: "secret" username: "******" password: "******" `, testInput); err != nil { t.Errorf("failed to build test expectation proto: %v", err) } testFile, err := ioutil.TempFile("", "user_agent") if err != nil { t.Errorf("failed to make test input file: %v", err) } if err := proto.MarshalText(testFile, testInput); err != nil { t.Errorf("failed to write test input file: %v", err) } if _, err := New(testFile.Name()); err != nil { t.Errorf("error: %v", err) } }
func TestLoad(t *testing.T) { expected := &redditproto.UserAgent{} if err := proto.UnmarshalText(` user_agent: "test" client_id: "id" client_secret: "secret" username: "******" password: "******" `, expected); err != nil { t.Errorf("failed to build test expectation proto: %v", err) } testFile, err := ioutil.TempFile("", "user_agent") if err != nil { t.Errorf("failed to make test input file: %v", err) } if err := proto.MarshalText(testFile, expected); err != nil { t.Errorf("failed to write test input file: %v", err) } if _, err := loadAgentFile("notarealfile"); err == nil { t.Errorf("wanted error returned with nonexistent file as input") } actual, err := loadAgentFile(testFile.Name()) if err != nil { t.Errorf("failed: %v", err) } if !proto.Equal(expected, actual) { t.Errorf("got %v; wanted %v", actual, expected) } }
func BenchmarkMarshalTextUnbuffered(b *testing.B) { w := ioutil.Discard m := newTestMessage() for i := 0; i < b.N; i++ { proto.MarshalText(w, m) } }
func MarshalText(w io.Writer, m *Manifest) error { var bm pb.Manifest for _, resource := range m.Resources { bm.Resource = append(bm.Resource, toProto(resource)) } return proto.MarshalText(w, &bm) }
func BenchmarkMarshalTextBuffered(b *testing.B) { buf := new(bytes.Buffer) m := newTestMessage() for i := 0; i < b.N; i++ { buf.Reset() proto.MarshalText(buf, m) } }
func dumpWeights(w http.ResponseWriter, r *http.Request) { def := &caffe2.TensorProtos{} data, err := ioutil.ReadFile("inception_tensors.pb") if err != nil { log.Fatal(err) } proto.Unmarshal(data, def) proto.MarshalText(w, def) }
func dumpNet(w http.ResponseWriter, r *http.Request) { def := &caffe2.NetDef{} data, err := ioutil.ReadFile("inception_net.pb") if err != nil { log.Fatal(err) } proto.Unmarshal(data, def) proto.MarshalText(w, def) }
func TestMarshalTextCustomMessage(t *testing.T) { buf := new(bytes.Buffer) if err := proto.MarshalText(buf, &textMessage{}); err != nil { t.Fatalf("proto.MarshalText: %v", err) } s := buf.String() if s != "custom" { t.Errorf("Got %q, expected %q", s, "custom") } }
func TestMarshalText(t *testing.T) { buf := new(bytes.Buffer) if err := proto.MarshalText(buf, newTestMessage()); err != nil { t.Fatalf("proto.MarshalText: %v", err) } s := buf.String() if s != text { t.Errorf("Got:\n===\n%v===\nExpected:\n===\n%v===\n", s, text) } }
func TestInitAcls(t *testing.T) { if _, err := os.Stat("./tmpdir"); os.IsNotExist(err) { err = os.Mkdir("./tmpdir", 0777) if err != nil { t.Fatal(err) } } trustedEntities := TrustedEntities{ TrustedProgramTaoNames: []string{fmt.Sprintf("%v", programName)}, TrustedHostTaoNames: []string{fmt.Sprintf("%v", hostName)}, TrustedMachineInfos: []string{machineName}} f, err := os.Create("./tmpdir/TrustedEntities") if err != nil { t.Fatal(err) } err = proto.MarshalText(f, &trustedEntities) if err != nil { t.Fatal(err) } err = f.Close() if err != nil { t.Fatal(err) } aclGuardType := "ACLs" aclGuardPath := "acls" cfg := tao.DomainConfig{ DomainInfo: &tao.DomainDetails{ GuardType: &aclGuardType}, AclGuardInfo: &tao.ACLGuardDetails{ SignedAclsPath: &aclGuardPath}} domain, err := tao.CreateDomain(cfg, "./tmpdir/domain", []byte("xxx")) if err != nil { t.Fatal(err) } err = InitAcls(domain, "./tmpdir/TrustedEntities") if err != nil { t.Fatal(err) } machinePrin := auth.Prin{ Type: "MachineInfo", KeyHash: auth.Str(machineName), } if !domain.Guard.IsAuthorized(*programName, "Execute", []string{}) || !domain.Guard.IsAuthorized(*hostName, "Host", []string{}) || !domain.Guard.IsAuthorized(machinePrin, "Root", []string{}) { t.Fatal("Authorization checks fail") } err = os.RemoveAll("./tmpdir") if err != nil { t.Fatal(err) } }
func TestMarshalTextNil(t *testing.T) { want := "<nil>" tests := []proto.Message{nil, (*pb.MyMessage)(nil)} for i, test := range tests { buf := new(bytes.Buffer) if err := proto.MarshalText(buf, test); err != nil { t.Fatal(err) } if got := buf.String(); got != want { t.Errorf("%d: got %q want %q", i, got, want) } } }
func TestStringEscaping(t *testing.T) { testCases := []struct { in *pb.Strings out string }{ { // Test data from C++ test (TextFormatTest.StringEscape). // Single divergence: we don't escape apostrophes. &pb.Strings{StringField: proto.String("\"A string with ' characters \n and \r newlines and \t tabs and \001 slashes \\ and multiple spaces")}, "string_field: \"\\\"A string with ' characters \\n and \\r newlines and \\t tabs and \\001 slashes \\\\ and multiple spaces\"\n", }, { // Test data from the same C++ test. &pb.Strings{StringField: proto.String("\350\260\267\346\255\214")}, "string_field: \"\\350\\260\\267\\346\\255\\214\"\n", }, { // Some UTF-8. &pb.Strings{StringField: proto.String("\x00\x01\xff\x81")}, `string_field: "\000\001\377\201"` + "\n", }, } for i, tc := range testCases { var buf bytes.Buffer if err := proto.MarshalText(&buf, tc.in); err != nil { t.Errorf("proto.MarsalText: %v", err) continue } s := buf.String() if s != tc.out { t.Errorf("#%d: Got:\n%s\nExpected:\n%s\n", i, s, tc.out) continue } // Check round-trip. pb := new(pb.Strings) if err := proto.UnmarshalText(s, pb); err != nil { t.Errorf("#%d: UnmarshalText: %v", i, err) continue } if !proto.Equal(pb, tc.in) { t.Errorf("#%d: Round-trip failed:\nstart: %v\n end: %v", i, tc.in, pb) } } }
func TestMarshalTextFailing(t *testing.T) { // Try lots of different sizes to exercise more error code-paths. for lim := 0; lim < len(text); lim++ { buf := new(limitedWriter) buf.limit = lim err := proto.MarshalText(buf, newTestMessage()) // We expect a certain error, but also some partial results in the buffer. if err != outOfSpace { t.Errorf("Got:\n===\n%v===\nExpected:\n===\n%v===\n", err, outOfSpace) } s := buf.b.String() x := text[:buf.limit] if s != x { t.Errorf("Got:\n===\n%v===\nExpected:\n===\n%v===\n", s, x) } } }
func modifySolverNetSpec(sourceBytes []byte) ([]byte, error) { // read into object with protobuf (must have already generated go protobuf code) netParam := &caffe.NetParameter{} if err := proto.UnmarshalText(string(sourceBytes), netParam); err != nil { return nil, err } // modify object fields for _, layerParam := range netParam.Layers { switch *layerParam.Type { case caffe.V1LayerParameter_IMAGE_DATA: if isTrainingPhase(layerParam) { layerParam.ImageDataParam.Source = proto.String(TRAINING_INDEX) } if isTestingPhase(layerParam) { layerParam.ImageDataParam.Source = proto.String(TESTING_INDEX) } case caffe.V1LayerParameter_DATA: if isTrainingPhase(layerParam) { layerParam.DataParam.Source = proto.String(TRAINING_DIR) } if isTestingPhase(layerParam) { layerParam.DataParam.Source = proto.String(TESTING_DIR) } } } buf := new(bytes.Buffer) if err := proto.MarshalText(buf, netParam); err != nil { return nil, err } return buf.Bytes(), nil }
func modifySolverSpec(source []byte) ([]byte, error) { // read into object with protobuf (must have already generated go protobuf code) solverParam := &caffe.SolverParameter{} if err := proto.UnmarshalText(string(source), solverParam); err != nil { return nil, err } // modify object fields solverParam.Net = proto.String("solver-net.prototxt") solverParam.SnapshotPrefix = proto.String("snapshot") buf := new(bytes.Buffer) if err := proto.MarshalText(buf, solverParam); err != nil { return nil, err } return buf.Bytes(), nil }
func SafeWriteFile(filename string, msg proto.Message) error { f, err := ioutil.TempFile(filepath.Dir(filename), "tmpfile") if err != nil { return fmt.Errorf("can't create temporary file: %s", err) } w := bufio.NewWriter(f) if err := proto.MarshalText(w, msg); err != nil { f.Close() os.Remove(f.Name()) return fmt.Errorf("marshal failed: %s", f.Name(), err) } w.Flush() f.Sync() f.Close() f.Sync() if err := os.Rename(f.Name(), filename); err != nil { os.Remove(f.Name()) return fmt.Errorf("rename failed: %s", err) } f.Sync() return nil }
Short: "Dump the contents of the manifest in protobuf text format", Run: func(cmd *cobra.Command, args []string) { var p []byte var err error if len(args) < 1 { p, err = ioutil.ReadAll(os.Stdin) if err != nil { log.Fatalf("error reading manifest: %v", err) } } else { p, err = ioutil.ReadFile(args[0]) if err != nil { log.Fatalf("error reading manifest: %v", err) } } var bm pb.Manifest if err := proto.Unmarshal(p, &bm); err != nil { log.Fatalf("error unmarshaling manifest: %v", err) } // TODO(stevvooe): For now, just dump the text format. Turn this into // nice text output later. if err := proto.MarshalText(os.Stdout, &bm); err != nil { log.Fatalf("error dumping manifest: %v", err) } }, }
func outputText(w http.ResponseWriter, r *resumepb.Resume) error { w.Header().Add("Content-Type", "text/plain") return proto.MarshalText(w, r) }
func TestWalkFS(t *testing.T) { rand.Seed(1) // Testing: // 1. Setup different files: // - links // - sibling directory - relative // - sibling directory - absolute // - parent directory - absolute // - parent directory - relative // - illegal links // - parent directory - relative, out of root // - parent directory - absolute, out of root // - regular files // - character devices // - what about sticky bits? // 2. Build the manifest. // 3. Verify expected result. testResources := []dresource{ { path: "a", mode: 0644, }, { kind: rhardlink, path: "a-hardlink", target: "a", }, { kind: rdirectory, path: "b", mode: 0755, }, { kind: rhardlink, path: "b/a-hardlink", target: "a", }, { path: "b/a", mode: 0600 | os.ModeSticky, }, { kind: rdirectory, path: "c", mode: 0755, }, { path: "c/a", mode: 0644, }, { kind: rrelsymlink, path: "c/ca-relsymlink", mode: 0600, target: "a", }, { kind: rrelsymlink, path: "c/a-relsymlink", mode: 0600, target: "../a", }, { kind: rabssymlink, path: "c/a-abssymlink", mode: 0600, target: "a", }, // TODO(stevvooe): Make sure we can test this case and get proper // errors when it is encountered. // { // // create a bad symlink and make sure we don't include it. // kind: relsymlink, // path: "c/a-badsymlink", // mode: 0600, // target: "../../..", // }, // TODO(stevvooe): Must add tests for xattrs, with symlinks, // directorys and regular files. { kind: rnamedpipe, path: "fifo", mode: 0666 | os.ModeNamedPipe, }, { kind: rdirectory, path: "/dev", mode: 0755, }, // NOTE(stevvooe): Below here, we add a few simple character devices. // Block devices are untested but should be nearly the same as // character devices. // devNullResource, // devZeroResource, } root, err := ioutil.TempDir("", "continuity-test-") if err != nil { t.Fatalf("error creating temporary directory: %v", err) } defer os.RemoveAll(root) generateTestFiles(t, root, testResources) ctx, err := NewContext(root) if err != nil { t.Fatalf("error getting context: %v", err) } bm, err := BuildManifest(ctx) if err != nil { t.Fatalf("error building manifest: %v, %#T", err, err) } proto.MarshalText(os.Stdout, bm) t.Fail() // TODO(stevvooe): Actually test input/output matches }
func fuzz(data []byte, text bool) int { ctors := []func() proto.Message{ func() proto.Message { return new(pb.M0) }, func() proto.Message { return new(pb.M1) }, func() proto.Message { return new(pb.M2) }, func() proto.Message { return new(pb.M3) }, func() proto.Message { return new(pb.M4) }, func() proto.Message { return new(pb.M5) }, func() proto.Message { return new(pb.M6) }, func() proto.Message { return new(pb.M7) }, func() proto.Message { return new(pb.M8) }, func() proto.Message { return new(pb.M9) }, func() proto.Message { return new(pb.M10) }, func() proto.Message { return new(pb.M11) }, func() proto.Message { return new(pb.M12) }, func() proto.Message { return new(pb.M13) }, func() proto.Message { return new(pb.M14) }, func() proto.Message { return new(pb.M15) }, func() proto.Message { return new(pb.M16) }, func() proto.Message { return new(pb.M17) }, func() proto.Message { return new(pb.M18) }, func() proto.Message { return new(pb.M19) }, func() proto.Message { return new(pb.M20) }, func() proto.Message { return new(pb.M21) }, func() proto.Message { return new(pb.M22) }, func() proto.Message { return new(pb.M23) }, func() proto.Message { return new(pb.M24) }, func() proto.Message { return new(pb.M25) }, } datas := "" if text { datas = string(data) } score := 0 for _, ctor := range ctors { v := ctor() var err error if text { err = proto.UnmarshalText(datas, v) } else { err = proto.Unmarshal(data, v) } if err != nil { continue } score = 1 var data1 []byte if text { var buf bytes.Buffer err = proto.MarshalText(&buf, v) data1 = buf.Bytes() } else { data1, err = proto.Marshal(v) } if err != nil { panic(err) } v1 := ctor() if text { err = proto.UnmarshalText(string(data1), v1) } else { err = proto.Unmarshal(data1, v1) } if err != nil { panic(err) } if !DeepEqual(v, v1) { fmt.Printf("v0: %#v\n", v) fmt.Printf("v1: %#v\n", v1) panic(fmt.Sprintf("non idempotent marshal of %T", v)) } if text { // TODO: Marshal/Unmarshal to binary. } else { var buf bytes.Buffer err := proto.MarshalText(&buf, v) if err != nil { fmt.Printf("failed to MarshalText: %#v\n", v) panic(err) } v2 := ctor() err = proto.UnmarshalText(buf.String(), v2) if err != nil { if strings.Contains(err.Error(), "unexpected byte 0x2f") { continue // known bug } fmt.Printf("failed to UnmarshalText: %q\n", buf.Bytes()) panic(err) } if !DeepEqual(v, v2) { fmt.Printf("v0: %#v\n", v) fmt.Printf("v2: %#v\n", v2) panic(fmt.Sprintf("non idempotent text marshal of %T", v)) } } } return score }
func main() { flag.Usage = usage flag.Parse() if *helpShort || *helpLong || flag.NArg() == 0 { flag.Usage() os.Exit(1) } fs, err := parser.ParseFiles(flag.Args(), strings.Split(*importPath, ",")) if err != nil { fatalf("%v", err) } fds, err := gendesc.Generate(fs) if err != nil { fatalf("Failed generating descriptors: %v", err) } if *descriptorOnly { proto.MarshalText(os.Stdout, fds) os.Exit(0) } //fmt.Println("-----") //proto.MarshalText(os.Stdout, fds) //fmt.Println("-----") // Prepare request. cgRequest := &plugin.CodeGeneratorRequest{ FileToGenerate: flag.Args(), ProtoFile: fds.File, } if *params != "" { cgRequest.Parameter = params } buf, err := proto.Marshal(cgRequest) if err != nil { fatalf("Failed marshaling CG request: %v", err) } // Find plugin. pluginPath := fullPath(*pluginBinary, strings.Split(os.Getenv("PATH"), ":")) if pluginPath == "" { fatalf("Failed finding plugin binary %q", *pluginBinary) } // Run the plugin subprocess. cmd := &exec.Cmd{ Path: pluginPath, Stdin: bytes.NewBuffer(buf), Stderr: os.Stderr, } buf, err = cmd.Output() if err != nil { fatalf("Failed running plugin: %v", err) } // Parse the response. cgResponse := new(plugin.CodeGeneratorResponse) if err = proto.Unmarshal(buf, cgResponse); err != nil { fatalf("Failed unmarshaling CG response: %v", err) } // TODO: check cgResponse.Error for _, f := range cgResponse.File { // TODO: If f.Name is nil, the content should be appended to the previous file. if f.Name == nil || f.Content == nil { fatalf("Malformed CG response") } if err := ioutil.WriteFile(*f.Name, []byte(*f.Content), 0644); err != nil { fatalf("Failed writing output file: %v", err) } } }
func main() { flag.Parse() if len(*makeAnnounce) > 0 { msgBytes, err := ioutil.ReadFile(*makeAnnounce) if err != nil { panic(err) } announce := &pond.Message{ Id: proto.Uint64(0), Time: proto.Int64(time.Now().Unix()), Body: msgBytes, MyNextDh: []byte{}, BodyEncoding: pond.Message_RAW.Enum(), } announceBytes, err := proto.Marshal(announce) if err != nil { panic(err) } os.Stdout.Write(announceBytes) return } if len(*baseDirectory) == 0 { log.Fatalf("Must give --base-directory") return } configPath := filepath.Join(*baseDirectory, configFilename) var identity [32]byte if *initFlag { if err := os.MkdirAll(*baseDirectory, 0700); err != nil { log.Fatalf("Failed to create base directory: %s", err) return } if _, err := io.ReadFull(rand.Reader, identity[:]); err != nil { log.Fatalf("Failed to read random bytes: %s", err) return } if err := ioutil.WriteFile(filepath.Join(*baseDirectory, identityFilename), identity[:], 0600); err != nil { log.Fatalf("Failed to write identity file: %s", err) return } defaultConfig := &protos.Config{ Port: proto.Uint32(uint32(*port)), } configFile, err := os.OpenFile(configPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { log.Fatalf("Failed to create config file: %s", err) } proto.MarshalText(configFile, defaultConfig) configFile.Close() } identityBytes, err := ioutil.ReadFile(filepath.Join(*baseDirectory, identityFilename)) if err != nil { log.Print("Use --init to setup a new base directory") log.Fatalf("Failed to read identity file: %s", err) return } if len(identityBytes) != 32 { log.Fatalf("Identity file is not 32 bytes long") return } copy(identity[:], identityBytes) config := new(protos.Config) configBytes, err := ioutil.ReadFile(configPath) if err != nil { log.Fatalf("No config file found") } if err := proto.UnmarshalText(string(configBytes), config); err != nil { log.Fatalf("Failed to parse config: %s", err) } if err := maybeConvertMessagesToNewFormat(*baseDirectory); err != nil { log.Fatalf("Failed to convert messages to new naming scheme: %s", err) } ip := net.IPv4(127, 0, 0, 1) // IPv4 loopback interface if config.Address != nil { if ip = net.ParseIP(*config.Address); ip == nil { log.Fatalf("Failed to parse address from config: %s", ip) } } listenAddr := net.TCPAddr{ IP: ip, Port: int(*config.Port), } listener, err := net.ListenTCP("tcp", &listenAddr) if err != nil { log.Fatalf("Failed to listen on port: %s", err) } var identityPublic [32]byte curve25519.ScalarBaseMult(&identityPublic, &identity) identityString := strings.Replace(base32.StdEncoding.EncodeToString(identityPublic[:]), "=", "", -1) log.Printf("Started. Listening on port %d with identity %s", listener.Addr().(*net.TCPAddr).Port, identityString) server := NewServer(*baseDirectory, config.GetAllowRegistration()) if *lifelineFd > -1 { lifeline := os.NewFile(uintptr(*lifelineFd), "lifeline") go func() { var buf [1]byte lifeline.Read(buf[:]) os.Exit(255) }() } for { conn, err := listener.Accept() if err != nil { log.Printf("Error accepting connection: %s", err) continue } go handleConnection(server, conn, &identity) } }
func main() { vars := []proto.Message{ &pb.M0{F: proto.Int32(0)}, &pb.M0{F: proto.Int32(-1000000), XXX_unrecognized: []byte{0, 1, 2}}, &pb.M1{F: proto.Int64(0)}, &pb.M1{F: proto.Int64(100)}, &pb.M1{F: proto.Int64(123123123123123123)}, &pb.M1{F: proto.Int64(-100)}, &pb.M1{F: proto.Int64(-123123123123123123)}, &pb.M2{}, &pb.M2{F: proto.Uint32(123123)}, &pb.M3{}, &pb.M3{F: proto.Uint64(123123123123123123)}, &pb.M4{F: proto.Int32(123123)}, &pb.M5{F: proto.Int64(123123)}, &pb.M5{F: proto.Int64(-123123)}, &pb.M6{XXX_unrecognized: []byte{0, 1, 2}}, &pb.M6{F: proto.Uint32(123123), XXX_unrecognized: []byte{0, 1, 2}}, &pb.M7{F: proto.Uint64(123123123123)}, &pb.M8{F: proto.Int32(-123123)}, &pb.M9{F: proto.Int64(-123123123123)}, &pb.M10{F: proto.Float64(123123.123123)}, &pb.M11{F: proto.Float32(123123.123123)}, &pb.M12{F: proto.Bool(true)}, &pb.M13{}, &pb.M13{F: proto.String("")}, &pb.M13{F: proto.String("foo")}, &pb.M13{F: proto.String("&pb.M6{F: proto.Uint32(123123), XXX_unrecognized: []byte{0,1,2}},")}, &pb.M13{F: proto.String("\x00\x01\x02")}, &pb.M14{}, &pb.M14{F: []byte{0, 1, 2}}, &pb.M14{F: []byte("&pb.M6{F: proto.Uint32(123123), XXX_unrecognized: []byte{0,1,2}},")}, &pb.M15{F0: proto.Int32(123)}, &pb.M15{F0: proto.Int32(123), F1: proto.String("foo"), F2: []byte{1, 2, 3}, F4: proto.Bool(false)}, &pb.M16{}, &pb.M16{F: pb.Corpus_UNIVERSAL.Enum()}, &pb.M16{F: pb.Corpus_PRODUCTS.Enum()}, &pb.M17{F: &pb.M15{F0: proto.Int32(123)}}, &pb.M17{F: &pb.M15{F0: proto.Int32(123), F1: proto.String("foo"), F2: []byte{1, 2, 3}, F4: proto.Bool(false)}}, func() proto.Message { v := &pb.M18{F0: proto.String("foo")} proto.SetExtension(v, pb.E_F1, 42) return v }(), &pb.M19{}, &pb.M19{F: []int32{0, -123, 500, 123123123}}, &pb.M20{F: []string{"", "foo", "\x00\x01\x02"}}, &pb.M21{F: []*pb.M15{&pb.M15{F0: proto.Int32(123)}, &pb.M15{F0: proto.Int32(123), F1: proto.String("foo"), F2: []byte{1, 2, 3}, F4: proto.Bool(false)}}}, &pb.M22{F: []*pb.M2{&pb.M2{}}}, &pb.M22{F: []*pb.M2{&pb.M2{}, &pb.M2{F: proto.Uint32(123123)}}}, &pb.M23{}, &pb.M23{F: map[int32]string{42: "", 11: "foo", 123123123: "\x00\x01\x02"}}, &pb.M24{F: map[string]*pb.M2{"": &pb.M2{}, "foo": &pb.M2{}, "\x00\x01\x02": &pb.M2{F: proto.Uint32(123123)}}}, &pb.M25{}, &pb.M25{F0: proto.String("")}, &pb.M25{F0: proto.String("foo")}, &pb.M25{F1: &pb.M2{}}, &pb.M25{F1: &pb.M2{F: proto.Uint32(123123)}}, &pb.M25{F2: pb.Corpus_UNIVERSAL.Enum()}, } for i, v := range vars { if false { data, err := proto.Marshal(v) if err != nil { panic(err) } f, err := os.Create(fmt.Sprintf("/tmp/proto/%v", i)) if err != nil { panic(err) } f.Write(data) f.Close() } else { f, err := os.Create(fmt.Sprintf("/tmp/proto/%v", i)) if err != nil { panic(err) } fmt.Printf("%v: %+v\n", i, v) err = proto.MarshalText(f, v) if err != nil { panic(err) } f.Close() } } }
func printResp(v proto.Message, err error) { if err != nil { log.Fatal(err) } proto.MarshalText(os.Stdout, v) }
func main() { flag.Parse() if flag.NArg() != 1 { flag.Usage() os.Exit(1) } config := common.ReadConfigOrExit(flag.Arg(0)) transitionSystem, ok := common.TransitionSystems[config.Parser.System] if !ok { log.Fatalf("Unknown transition system: %s", config.Parser.System) } labelNumberer := common.ReadTransitionsOrExit(config.Parser.Transitions, transitionSystem) var layers []*caffe.LayerParameter layers = []*caffe.LayerParameter{ createDataLayer("test/train-lmdb", caffe.Phase_TRAIN, caffe.DataParameter_LMDB), createDataLayer("test/validation-lmdb", caffe.Phase_TEST, caffe.DataParameter_LMDB), createDropoutLayer("data_dropout", "data", "data", 0.1), } ilas := common.ReadIlasOrExit(config.Parser.Inputs) vecs := common.ReadEmbeddingsOrExit(config.Embeddings.Word) tvecs := common.ReadEmbeddingsOrExit(config.Embeddings.Tag) rvecs := common.ReadEmbeddingsOrExit(config.Embeddings.DepRel) fvecs := common.ReadEmbeddingsOrExit(config.Embeddings.Feature) cvecs := common.ReadEmbeddingsOrExit(config.Embeddings.Char) layerVecs := map[addr.Layer]*go2vec.Embeddings{ addr.TOKEN: vecs, addr.TAG: tvecs, addr.DEPREL: rvecs, addr.FEATURE: fvecs, addr.CHAR: cvecs, } wordSplits, wordNames := splitLayers(layerVecs, ilas, addr.TOKEN, "words", *wordEmbeddings) tagSplits, tagNames := splitLayers(layerVecs, ilas, addr.TAG, "tags", *tagEmbeddings) relSplits, relNames := splitLayers(layerVecs, ilas, addr.DEPREL, "deprels", *deprelEmbeddings) featureSplits, featureNames := splitLayers(layerVecs, ilas, addr.FEATURE, "features", *featureEmbeddings) charSplits, charNames := splitCharLayers(layerVecs, ilas, "chars") splits := make([]uint, 0, len(wordSplits)+len(tagSplits)+len(relSplits)) splits = append(splits, wordSplits...) splits = append(splits, tagSplits...) splits = append(splits, relSplits...) splits = append(splits, featureSplits...) splits = append(splits, charSplits...) splitNames := make([]string, 0, len(wordNames)+len(tagNames)+len(relNames)+len(featureNames)) splitNames = append(splitNames, wordNames...) splitNames = append(splitNames, tagNames...) splitNames = append(splitNames, relNames...) splitNames = append(splitNames, featureNames...) splitNames = append(splitNames, charNames...) layers = append(layers, createSliceLayer("slice", "data", splits, splitNames)) concatLayers := make([]string, 0) if *wordEmbeddings { layers = append(layers, createEmbeddingLayers("words", "wordembed", wordSplits, 50, false)...) concatLayers = append(concatLayers, names("wordembed", wordSplits)...) } else { layers = append(layers, createActivationLayer("flat_words", "words", "flat_words", "Flatten")) concatLayers = append(concatLayers, "flat_words") } if *tagEmbeddings { layers = append(layers, createEmbeddingLayers("tags", "tagembed", tagSplits, 50, false)...) concatLayers = append(concatLayers, names("tagembed", tagSplits)...) } else { layers = append(layers, createActivationLayer("flat_tags", "tags", "flat_tags", "Flatten")) concatLayers = append(concatLayers, "flat_tags") } if *deprelEmbeddings { layers = append(layers, createEmbeddingLayers("deprels", "relembed", relSplits, 50, false)...) concatLayers = append(concatLayers, names("relembed", relSplits)...) } else { layers = append(layers, createActivationLayer("flat_deprels", "deprels", "flat_deprels", "Flatten")) concatLayers = append(concatLayers, "flat_deprels") } if *featureEmbeddings { layers = append(layers, createEmbeddingLayers("features", "featureembed", featureSplits, 50, false)...) concatLayers = append(concatLayers, names("featureembed", relSplits)...) } else if layerVecs[addr.FEATURE] != nil { layers = append(layers, createActivationLayer("flat_features", "features", "flat_features", "Flatten")) concatLayers = append(concatLayers, "flat_features") } layers = append(layers, createEmbeddingLayers("chars", "morph", charSplits, 210, true)...) for _, name := range names("morph", charSplits) { layers = append(layers, createActivationLayer(fmt.Sprintf("%s_activation", name), name, name, "Sigmoid")) } concatLayers = append(concatLayers, names("morph", charSplits)...) layers = append(layers, createConcatLayer("concat", "concat", concatLayers), createInnerProductLayer("ip", "concat", "ip", 200), createActivationLayer("tanh", "ip", "ip", "TanH"), createDropoutLayer("dropout", "ip", "ip", 0.05), createInnerProductLayer("ip2", "ip", "ip2", uint32(labelNumberer.Size())), createLossLayer("softmax", "ip2", "label", "softmax", "SoftmaxWithLoss"), createAccuracyLayer("accuracy", "ip2", "label", "accuracy", "Accuracy"), ) net := &caffe.NetParameter{ Name: proto.String("dparnn-net"), Layer: layers, } proto.MarshalText(os.Stdout, net) }