func TestNew(t *testing.T) {
if _, err := New("fakefile"); err == nil {
t.Errorf("wanted to return error for nonexistent file")
}
testInput := &redditproto.UserAgent{}
if err := proto.UnmarshalText(`
user_agent: "test"
client_id: "id"
client_secret: "secret"
username: "******"
password: "******"
`, testInput); err != nil {
t.Errorf("failed to build test expectation proto: %v", err)
}
testFile, err := ioutil.TempFile("", "user_agent")
if err != nil {
t.Errorf("failed to make test input file: %v", err)
}
if err := proto.MarshalText(testFile, testInput); err != nil {
t.Errorf("failed to write test input file: %v", err)
}
if _, err := New(testFile.Name()); err != nil {
t.Errorf("error: %v", err)
}
}
func TestLoad(t *testing.T) {
expected := &redditproto.UserAgent{}
if err := proto.UnmarshalText(`
user_agent: "test"
client_id: "id"
client_secret: "secret"
username: "******"
password: "******"
`, expected); err != nil {
t.Errorf("failed to build test expectation proto: %v", err)
}
testFile, err := ioutil.TempFile("", "user_agent")
if err != nil {
t.Errorf("failed to make test input file: %v", err)
}
if err := proto.MarshalText(testFile, expected); err != nil {
t.Errorf("failed to write test input file: %v", err)
}
if _, err := loadAgentFile("notarealfile"); err == nil {
t.Errorf("wanted error returned with nonexistent file as input")
}
actual, err := loadAgentFile(testFile.Name())
if err != nil {
t.Errorf("failed: %v", err)
}
if !proto.Equal(expected, actual) {
t.Errorf("got %v; wanted %v", actual, expected)
}
}
func BenchmarkMarshalTextUnbuffered(b *testing.B) {
w := ioutil.Discard
m := newTestMessage()
for i := 0; i < b.N; i++ {
proto.MarshalText(w, m)
}
}
func MarshalText(w io.Writer, m *Manifest) error {
var bm pb.Manifest
for _, resource := range m.Resources {
bm.Resource = append(bm.Resource, toProto(resource))
}
return proto.MarshalText(w, &bm)
}
func BenchmarkMarshalTextBuffered(b *testing.B) {
buf := new(bytes.Buffer)
m := newTestMessage()
for i := 0; i < b.N; i++ {
buf.Reset()
proto.MarshalText(buf, m)
}
}
func dumpWeights(w http.ResponseWriter, r *http.Request) {
def := &caffe2.TensorProtos{}
data, err := ioutil.ReadFile("inception_tensors.pb")
if err != nil {
log.Fatal(err)
}
proto.Unmarshal(data, def)
proto.MarshalText(w, def)
}
func dumpNet(w http.ResponseWriter, r *http.Request) {
def := &caffe2.NetDef{}
data, err := ioutil.ReadFile("inception_net.pb")
if err != nil {
log.Fatal(err)
}
proto.Unmarshal(data, def)
proto.MarshalText(w, def)
}
func TestMarshalTextCustomMessage(t *testing.T) {
buf := new(bytes.Buffer)
if err := proto.MarshalText(buf, &textMessage{}); err != nil {
t.Fatalf("proto.MarshalText: %v", err)
}
s := buf.String()
if s != "custom" {
t.Errorf("Got %q, expected %q", s, "custom")
}
}
func TestMarshalText(t *testing.T) {
buf := new(bytes.Buffer)
if err := proto.MarshalText(buf, newTestMessage()); err != nil {
t.Fatalf("proto.MarshalText: %v", err)
}
s := buf.String()
if s != text {
t.Errorf("Got:\n===\n%v===\nExpected:\n===\n%v===\n", s, text)
}
}
func TestInitAcls(t *testing.T) {
if _, err := os.Stat("./tmpdir"); os.IsNotExist(err) {
err = os.Mkdir("./tmpdir", 0777)
if err != nil {
t.Fatal(err)
}
}
trustedEntities := TrustedEntities{
TrustedProgramTaoNames: []string{fmt.Sprintf("%v", programName)},
TrustedHostTaoNames: []string{fmt.Sprintf("%v", hostName)},
TrustedMachineInfos: []string{machineName}}
f, err := os.Create("./tmpdir/TrustedEntities")
if err != nil {
t.Fatal(err)
}
err = proto.MarshalText(f, &trustedEntities)
if err != nil {
t.Fatal(err)
}
err = f.Close()
if err != nil {
t.Fatal(err)
}
aclGuardType := "ACLs"
aclGuardPath := "acls"
cfg := tao.DomainConfig{
DomainInfo: &tao.DomainDetails{
GuardType: &aclGuardType},
AclGuardInfo: &tao.ACLGuardDetails{
SignedAclsPath: &aclGuardPath}}
domain, err := tao.CreateDomain(cfg, "./tmpdir/domain", []byte("xxx"))
if err != nil {
t.Fatal(err)
}
err = InitAcls(domain, "./tmpdir/TrustedEntities")
if err != nil {
t.Fatal(err)
}
machinePrin := auth.Prin{
Type: "MachineInfo",
KeyHash: auth.Str(machineName),
}
if !domain.Guard.IsAuthorized(*programName, "Execute", []string{}) ||
!domain.Guard.IsAuthorized(*hostName, "Host", []string{}) ||
!domain.Guard.IsAuthorized(machinePrin, "Root", []string{}) {
t.Fatal("Authorization checks fail")
}
err = os.RemoveAll("./tmpdir")
if err != nil {
t.Fatal(err)
}
}
func TestMarshalTextNil(t *testing.T) {
want := "<nil>"
tests := []proto.Message{nil, (*pb.MyMessage)(nil)}
for i, test := range tests {
buf := new(bytes.Buffer)
if err := proto.MarshalText(buf, test); err != nil {
t.Fatal(err)
}
if got := buf.String(); got != want {
t.Errorf("%d: got %q want %q", i, got, want)
}
}
}
func TestStringEscaping(t *testing.T) {
testCases := []struct {
in *pb.Strings
out string
}{
{
// Test data from C++ test (TextFormatTest.StringEscape).
// Single divergence: we don't escape apostrophes.
&pb.Strings{StringField: proto.String("\"A string with ' characters \n and \r newlines and \t tabs and \001 slashes \\ and multiple spaces")},
"string_field: \"\\\"A string with ' characters \\n and \\r newlines and \\t tabs and \\001 slashes \\\\ and multiple spaces\"\n",
},
{
// Test data from the same C++ test.
&pb.Strings{StringField: proto.String("\350\260\267\346\255\214")},
"string_field: \"\\350\\260\\267\\346\\255\\214\"\n",
},
{
// Some UTF-8.
&pb.Strings{StringField: proto.String("\x00\x01\xff\x81")},
`string_field: "\000\001\377\201"` + "\n",
},
}
for i, tc := range testCases {
var buf bytes.Buffer
if err := proto.MarshalText(&buf, tc.in); err != nil {
t.Errorf("proto.MarsalText: %v", err)
continue
}
s := buf.String()
if s != tc.out {
t.Errorf("#%d: Got:\n%s\nExpected:\n%s\n", i, s, tc.out)
continue
}
// Check round-trip.
pb := new(pb.Strings)
if err := proto.UnmarshalText(s, pb); err != nil {
t.Errorf("#%d: UnmarshalText: %v", i, err)
continue
}
if !proto.Equal(pb, tc.in) {
t.Errorf("#%d: Round-trip failed:\nstart: %v\n end: %v", i, tc.in, pb)
}
}
}
func TestMarshalTextFailing(t *testing.T) {
// Try lots of different sizes to exercise more error code-paths.
for lim := 0; lim < len(text); lim++ {
buf := new(limitedWriter)
buf.limit = lim
err := proto.MarshalText(buf, newTestMessage())
// We expect a certain error, but also some partial results in the buffer.
if err != outOfSpace {
t.Errorf("Got:\n===\n%v===\nExpected:\n===\n%v===\n", err, outOfSpace)
}
s := buf.b.String()
x := text[:buf.limit]
if s != x {
t.Errorf("Got:\n===\n%v===\nExpected:\n===\n%v===\n", s, x)
}
}
}
func modifySolverNetSpec(sourceBytes []byte) ([]byte, error) {
// read into object with protobuf (must have already generated go protobuf code)
netParam := &caffe.NetParameter{}
if err := proto.UnmarshalText(string(sourceBytes), netParam); err != nil {
return nil, err
}
// modify object fields
for _, layerParam := range netParam.Layers {
switch *layerParam.Type {
case caffe.V1LayerParameter_IMAGE_DATA:
if isTrainingPhase(layerParam) {
layerParam.ImageDataParam.Source = proto.String(TRAINING_INDEX)
}
if isTestingPhase(layerParam) {
layerParam.ImageDataParam.Source = proto.String(TESTING_INDEX)
}
case caffe.V1LayerParameter_DATA:
if isTrainingPhase(layerParam) {
layerParam.DataParam.Source = proto.String(TRAINING_DIR)
}
if isTestingPhase(layerParam) {
layerParam.DataParam.Source = proto.String(TESTING_DIR)
}
}
}
buf := new(bytes.Buffer)
if err := proto.MarshalText(buf, netParam); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func modifySolverSpec(source []byte) ([]byte, error) {
// read into object with protobuf (must have already generated go protobuf code)
solverParam := &caffe.SolverParameter{}
if err := proto.UnmarshalText(string(source), solverParam); err != nil {
return nil, err
}
// modify object fields
solverParam.Net = proto.String("solver-net.prototxt")
solverParam.SnapshotPrefix = proto.String("snapshot")
buf := new(bytes.Buffer)
if err := proto.MarshalText(buf, solverParam); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func SafeWriteFile(filename string, msg proto.Message) error {
f, err := ioutil.TempFile(filepath.Dir(filename), "tmpfile")
if err != nil {
return fmt.Errorf("can't create temporary file: %s", err)
}
w := bufio.NewWriter(f)
if err := proto.MarshalText(w, msg); err != nil {
f.Close()
os.Remove(f.Name())
return fmt.Errorf("marshal failed: %s", f.Name(), err)
}
w.Flush()
f.Sync()
f.Close()
f.Sync()
if err := os.Rename(f.Name(), filename); err != nil {
os.Remove(f.Name())
return fmt.Errorf("rename failed: %s", err)
}
f.Sync()
return nil
}
Short: "Dump the contents of the manifest in protobuf text format",
Run: func(cmd *cobra.Command, args []string) {
var p []byte
var err error
if len(args) < 1 {
p, err = ioutil.ReadAll(os.Stdin)
if err != nil {
log.Fatalf("error reading manifest: %v", err)
}
} else {
p, err = ioutil.ReadFile(args[0])
if err != nil {
log.Fatalf("error reading manifest: %v", err)
}
}
var bm pb.Manifest
if err := proto.Unmarshal(p, &bm); err != nil {
log.Fatalf("error unmarshaling manifest: %v", err)
}
// TODO(stevvooe): For now, just dump the text format. Turn this into
// nice text output later.
if err := proto.MarshalText(os.Stdout, &bm); err != nil {
log.Fatalf("error dumping manifest: %v", err)
}
},
}
func outputText(w http.ResponseWriter, r *resumepb.Resume) error {
w.Header().Add("Content-Type", "text/plain")
return proto.MarshalText(w, r)
}
func TestWalkFS(t *testing.T) {
rand.Seed(1)
// Testing:
// 1. Setup different files:
// - links
// - sibling directory - relative
// - sibling directory - absolute
// - parent directory - absolute
// - parent directory - relative
// - illegal links
// - parent directory - relative, out of root
// - parent directory - absolute, out of root
// - regular files
// - character devices
// - what about sticky bits?
// 2. Build the manifest.
// 3. Verify expected result.
testResources := []dresource{
{
path: "a",
mode: 0644,
},
{
kind: rhardlink,
path: "a-hardlink",
target: "a",
},
{
kind: rdirectory,
path: "b",
mode: 0755,
},
{
kind: rhardlink,
path: "b/a-hardlink",
target: "a",
},
{
path: "b/a",
mode: 0600 | os.ModeSticky,
},
{
kind: rdirectory,
path: "c",
mode: 0755,
},
{
path: "c/a",
mode: 0644,
},
{
kind: rrelsymlink,
path: "c/ca-relsymlink",
mode: 0600,
target: "a",
},
{
kind: rrelsymlink,
path: "c/a-relsymlink",
mode: 0600,
target: "../a",
},
{
kind: rabssymlink,
path: "c/a-abssymlink",
mode: 0600,
target: "a",
},
// TODO(stevvooe): Make sure we can test this case and get proper
// errors when it is encountered.
// {
// // create a bad symlink and make sure we don't include it.
// kind: relsymlink,
// path: "c/a-badsymlink",
// mode: 0600,
// target: "../../..",
// },
// TODO(stevvooe): Must add tests for xattrs, with symlinks,
// directorys and regular files.
{
kind: rnamedpipe,
path: "fifo",
mode: 0666 | os.ModeNamedPipe,
},
{
kind: rdirectory,
path: "/dev",
mode: 0755,
},
// NOTE(stevvooe): Below here, we add a few simple character devices.
// Block devices are untested but should be nearly the same as
// character devices.
// devNullResource,
// devZeroResource,
}
root, err := ioutil.TempDir("", "continuity-test-")
if err != nil {
t.Fatalf("error creating temporary directory: %v", err)
}
defer os.RemoveAll(root)
generateTestFiles(t, root, testResources)
ctx, err := NewContext(root)
if err != nil {
t.Fatalf("error getting context: %v", err)
}
bm, err := BuildManifest(ctx)
if err != nil {
t.Fatalf("error building manifest: %v, %#T", err, err)
}
proto.MarshalText(os.Stdout, bm)
t.Fail() // TODO(stevvooe): Actually test input/output matches
}
func fuzz(data []byte, text bool) int {
ctors := []func() proto.Message{
func() proto.Message { return new(pb.M0) },
func() proto.Message { return new(pb.M1) },
func() proto.Message { return new(pb.M2) },
func() proto.Message { return new(pb.M3) },
func() proto.Message { return new(pb.M4) },
func() proto.Message { return new(pb.M5) },
func() proto.Message { return new(pb.M6) },
func() proto.Message { return new(pb.M7) },
func() proto.Message { return new(pb.M8) },
func() proto.Message { return new(pb.M9) },
func() proto.Message { return new(pb.M10) },
func() proto.Message { return new(pb.M11) },
func() proto.Message { return new(pb.M12) },
func() proto.Message { return new(pb.M13) },
func() proto.Message { return new(pb.M14) },
func() proto.Message { return new(pb.M15) },
func() proto.Message { return new(pb.M16) },
func() proto.Message { return new(pb.M17) },
func() proto.Message { return new(pb.M18) },
func() proto.Message { return new(pb.M19) },
func() proto.Message { return new(pb.M20) },
func() proto.Message { return new(pb.M21) },
func() proto.Message { return new(pb.M22) },
func() proto.Message { return new(pb.M23) },
func() proto.Message { return new(pb.M24) },
func() proto.Message { return new(pb.M25) },
}
datas := ""
if text {
datas = string(data)
}
score := 0
for _, ctor := range ctors {
v := ctor()
var err error
if text {
err = proto.UnmarshalText(datas, v)
} else {
err = proto.Unmarshal(data, v)
}
if err != nil {
continue
}
score = 1
var data1 []byte
if text {
var buf bytes.Buffer
err = proto.MarshalText(&buf, v)
data1 = buf.Bytes()
} else {
data1, err = proto.Marshal(v)
}
if err != nil {
panic(err)
}
v1 := ctor()
if text {
err = proto.UnmarshalText(string(data1), v1)
} else {
err = proto.Unmarshal(data1, v1)
}
if err != nil {
panic(err)
}
if !DeepEqual(v, v1) {
fmt.Printf("v0: %#v\n", v)
fmt.Printf("v1: %#v\n", v1)
panic(fmt.Sprintf("non idempotent marshal of %T", v))
}
if text {
// TODO: Marshal/Unmarshal to binary.
} else {
var buf bytes.Buffer
err := proto.MarshalText(&buf, v)
if err != nil {
fmt.Printf("failed to MarshalText: %#v\n", v)
panic(err)
}
v2 := ctor()
err = proto.UnmarshalText(buf.String(), v2)
if err != nil {
if strings.Contains(err.Error(), "unexpected byte 0x2f") {
continue // known bug
}
fmt.Printf("failed to UnmarshalText: %q\n", buf.Bytes())
panic(err)
}
if !DeepEqual(v, v2) {
fmt.Printf("v0: %#v\n", v)
fmt.Printf("v2: %#v\n", v2)
panic(fmt.Sprintf("non idempotent text marshal of %T", v))
}
}
}
return score
}
func main() {
flag.Usage = usage
flag.Parse()
if *helpShort || *helpLong || flag.NArg() == 0 {
flag.Usage()
os.Exit(1)
}
fs, err := parser.ParseFiles(flag.Args(), strings.Split(*importPath, ","))
if err != nil {
fatalf("%v", err)
}
fds, err := gendesc.Generate(fs)
if err != nil {
fatalf("Failed generating descriptors: %v", err)
}
if *descriptorOnly {
proto.MarshalText(os.Stdout, fds)
os.Exit(0)
}
//fmt.Println("-----")
//proto.MarshalText(os.Stdout, fds)
//fmt.Println("-----")
// Prepare request.
cgRequest := &plugin.CodeGeneratorRequest{
FileToGenerate: flag.Args(),
ProtoFile: fds.File,
}
if *params != "" {
cgRequest.Parameter = params
}
buf, err := proto.Marshal(cgRequest)
if err != nil {
fatalf("Failed marshaling CG request: %v", err)
}
// Find plugin.
pluginPath := fullPath(*pluginBinary, strings.Split(os.Getenv("PATH"), ":"))
if pluginPath == "" {
fatalf("Failed finding plugin binary %q", *pluginBinary)
}
// Run the plugin subprocess.
cmd := &exec.Cmd{
Path: pluginPath,
Stdin: bytes.NewBuffer(buf),
Stderr: os.Stderr,
}
buf, err = cmd.Output()
if err != nil {
fatalf("Failed running plugin: %v", err)
}
// Parse the response.
cgResponse := new(plugin.CodeGeneratorResponse)
if err = proto.Unmarshal(buf, cgResponse); err != nil {
fatalf("Failed unmarshaling CG response: %v", err)
}
// TODO: check cgResponse.Error
for _, f := range cgResponse.File {
// TODO: If f.Name is nil, the content should be appended to the previous file.
if f.Name == nil || f.Content == nil {
fatalf("Malformed CG response")
}
if err := ioutil.WriteFile(*f.Name, []byte(*f.Content), 0644); err != nil {
fatalf("Failed writing output file: %v", err)
}
}
}
func main() {
flag.Parse()
if len(*makeAnnounce) > 0 {
msgBytes, err := ioutil.ReadFile(*makeAnnounce)
if err != nil {
panic(err)
}
announce := &pond.Message{
Id: proto.Uint64(0),
Time: proto.Int64(time.Now().Unix()),
Body: msgBytes,
MyNextDh: []byte{},
BodyEncoding: pond.Message_RAW.Enum(),
}
announceBytes, err := proto.Marshal(announce)
if err != nil {
panic(err)
}
os.Stdout.Write(announceBytes)
return
}
if len(*baseDirectory) == 0 {
log.Fatalf("Must give --base-directory")
return
}
configPath := filepath.Join(*baseDirectory, configFilename)
var identity [32]byte
if *initFlag {
if err := os.MkdirAll(*baseDirectory, 0700); err != nil {
log.Fatalf("Failed to create base directory: %s", err)
return
}
if _, err := io.ReadFull(rand.Reader, identity[:]); err != nil {
log.Fatalf("Failed to read random bytes: %s", err)
return
}
if err := ioutil.WriteFile(filepath.Join(*baseDirectory, identityFilename), identity[:], 0600); err != nil {
log.Fatalf("Failed to write identity file: %s", err)
return
}
defaultConfig := &protos.Config{
Port: proto.Uint32(uint32(*port)),
}
configFile, err := os.OpenFile(configPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
log.Fatalf("Failed to create config file: %s", err)
}
proto.MarshalText(configFile, defaultConfig)
configFile.Close()
}
identityBytes, err := ioutil.ReadFile(filepath.Join(*baseDirectory, identityFilename))
if err != nil {
log.Print("Use --init to setup a new base directory")
log.Fatalf("Failed to read identity file: %s", err)
return
}
if len(identityBytes) != 32 {
log.Fatalf("Identity file is not 32 bytes long")
return
}
copy(identity[:], identityBytes)
config := new(protos.Config)
configBytes, err := ioutil.ReadFile(configPath)
if err != nil {
log.Fatalf("No config file found")
}
if err := proto.UnmarshalText(string(configBytes), config); err != nil {
log.Fatalf("Failed to parse config: %s", err)
}
if err := maybeConvertMessagesToNewFormat(*baseDirectory); err != nil {
log.Fatalf("Failed to convert messages to new naming scheme: %s", err)
}
ip := net.IPv4(127, 0, 0, 1) // IPv4 loopback interface
if config.Address != nil {
if ip = net.ParseIP(*config.Address); ip == nil {
log.Fatalf("Failed to parse address from config: %s", ip)
}
}
listenAddr := net.TCPAddr{
IP: ip,
Port: int(*config.Port),
}
listener, err := net.ListenTCP("tcp", &listenAddr)
if err != nil {
log.Fatalf("Failed to listen on port: %s", err)
}
var identityPublic [32]byte
curve25519.ScalarBaseMult(&identityPublic, &identity)
identityString := strings.Replace(base32.StdEncoding.EncodeToString(identityPublic[:]), "=", "", -1)
log.Printf("Started. Listening on port %d with identity %s", listener.Addr().(*net.TCPAddr).Port, identityString)
server := NewServer(*baseDirectory, config.GetAllowRegistration())
if *lifelineFd > -1 {
lifeline := os.NewFile(uintptr(*lifelineFd), "lifeline")
go func() {
var buf [1]byte
lifeline.Read(buf[:])
os.Exit(255)
}()
}
for {
conn, err := listener.Accept()
if err != nil {
log.Printf("Error accepting connection: %s", err)
continue
}
go handleConnection(server, conn, &identity)
}
}
func main() {
vars := []proto.Message{
&pb.M0{F: proto.Int32(0)},
&pb.M0{F: proto.Int32(-1000000), XXX_unrecognized: []byte{0, 1, 2}},
&pb.M1{F: proto.Int64(0)},
&pb.M1{F: proto.Int64(100)},
&pb.M1{F: proto.Int64(123123123123123123)},
&pb.M1{F: proto.Int64(-100)},
&pb.M1{F: proto.Int64(-123123123123123123)},
&pb.M2{},
&pb.M2{F: proto.Uint32(123123)},
&pb.M3{},
&pb.M3{F: proto.Uint64(123123123123123123)},
&pb.M4{F: proto.Int32(123123)},
&pb.M5{F: proto.Int64(123123)},
&pb.M5{F: proto.Int64(-123123)},
&pb.M6{XXX_unrecognized: []byte{0, 1, 2}},
&pb.M6{F: proto.Uint32(123123), XXX_unrecognized: []byte{0, 1, 2}},
&pb.M7{F: proto.Uint64(123123123123)},
&pb.M8{F: proto.Int32(-123123)},
&pb.M9{F: proto.Int64(-123123123123)},
&pb.M10{F: proto.Float64(123123.123123)},
&pb.M11{F: proto.Float32(123123.123123)},
&pb.M12{F: proto.Bool(true)},
&pb.M13{},
&pb.M13{F: proto.String("")},
&pb.M13{F: proto.String("foo")},
&pb.M13{F: proto.String("&pb.M6{F: proto.Uint32(123123), XXX_unrecognized: []byte{0,1,2}},")},
&pb.M13{F: proto.String("\x00\x01\x02")},
&pb.M14{},
&pb.M14{F: []byte{0, 1, 2}},
&pb.M14{F: []byte("&pb.M6{F: proto.Uint32(123123), XXX_unrecognized: []byte{0,1,2}},")},
&pb.M15{F0: proto.Int32(123)},
&pb.M15{F0: proto.Int32(123), F1: proto.String("foo"), F2: []byte{1, 2, 3}, F4: proto.Bool(false)},
&pb.M16{},
&pb.M16{F: pb.Corpus_UNIVERSAL.Enum()},
&pb.M16{F: pb.Corpus_PRODUCTS.Enum()},
&pb.M17{F: &pb.M15{F0: proto.Int32(123)}},
&pb.M17{F: &pb.M15{F0: proto.Int32(123), F1: proto.String("foo"), F2: []byte{1, 2, 3}, F4: proto.Bool(false)}},
func() proto.Message {
v := &pb.M18{F0: proto.String("foo")}
proto.SetExtension(v, pb.E_F1, 42)
return v
}(),
&pb.M19{},
&pb.M19{F: []int32{0, -123, 500, 123123123}},
&pb.M20{F: []string{"", "foo", "\x00\x01\x02"}},
&pb.M21{F: []*pb.M15{&pb.M15{F0: proto.Int32(123)}, &pb.M15{F0: proto.Int32(123), F1: proto.String("foo"), F2: []byte{1, 2, 3}, F4: proto.Bool(false)}}},
&pb.M22{F: []*pb.M2{&pb.M2{}}},
&pb.M22{F: []*pb.M2{&pb.M2{}, &pb.M2{F: proto.Uint32(123123)}}},
&pb.M23{},
&pb.M23{F: map[int32]string{42: "", 11: "foo", 123123123: "\x00\x01\x02"}},
&pb.M24{F: map[string]*pb.M2{"": &pb.M2{}, "foo": &pb.M2{}, "\x00\x01\x02": &pb.M2{F: proto.Uint32(123123)}}},
&pb.M25{},
&pb.M25{F0: proto.String("")},
&pb.M25{F0: proto.String("foo")},
&pb.M25{F1: &pb.M2{}},
&pb.M25{F1: &pb.M2{F: proto.Uint32(123123)}},
&pb.M25{F2: pb.Corpus_UNIVERSAL.Enum()},
}
for i, v := range vars {
if false {
data, err := proto.Marshal(v)
if err != nil {
panic(err)
}
f, err := os.Create(fmt.Sprintf("/tmp/proto/%v", i))
if err != nil {
panic(err)
}
f.Write(data)
f.Close()
} else {
f, err := os.Create(fmt.Sprintf("/tmp/proto/%v", i))
if err != nil {
panic(err)
}
fmt.Printf("%v: %+v\n", i, v)
err = proto.MarshalText(f, v)
if err != nil {
panic(err)
}
f.Close()
}
}
}
func printResp(v proto.Message, err error) {
if err != nil {
log.Fatal(err)
}
proto.MarshalText(os.Stdout, v)
}
func main() {
flag.Parse()
if flag.NArg() != 1 {
flag.Usage()
os.Exit(1)
}
config := common.ReadConfigOrExit(flag.Arg(0))
transitionSystem, ok := common.TransitionSystems[config.Parser.System]
if !ok {
log.Fatalf("Unknown transition system: %s", config.Parser.System)
}
labelNumberer := common.ReadTransitionsOrExit(config.Parser.Transitions, transitionSystem)
var layers []*caffe.LayerParameter
layers = []*caffe.LayerParameter{
createDataLayer("test/train-lmdb", caffe.Phase_TRAIN, caffe.DataParameter_LMDB),
createDataLayer("test/validation-lmdb", caffe.Phase_TEST, caffe.DataParameter_LMDB),
createDropoutLayer("data_dropout", "data", "data", 0.1),
}
ilas := common.ReadIlasOrExit(config.Parser.Inputs)
vecs := common.ReadEmbeddingsOrExit(config.Embeddings.Word)
tvecs := common.ReadEmbeddingsOrExit(config.Embeddings.Tag)
rvecs := common.ReadEmbeddingsOrExit(config.Embeddings.DepRel)
fvecs := common.ReadEmbeddingsOrExit(config.Embeddings.Feature)
cvecs := common.ReadEmbeddingsOrExit(config.Embeddings.Char)
layerVecs := map[addr.Layer]*go2vec.Embeddings{
addr.TOKEN: vecs,
addr.TAG: tvecs,
addr.DEPREL: rvecs,
addr.FEATURE: fvecs,
addr.CHAR: cvecs,
}
wordSplits, wordNames := splitLayers(layerVecs, ilas, addr.TOKEN, "words", *wordEmbeddings)
tagSplits, tagNames := splitLayers(layerVecs, ilas, addr.TAG, "tags", *tagEmbeddings)
relSplits, relNames := splitLayers(layerVecs, ilas, addr.DEPREL, "deprels", *deprelEmbeddings)
featureSplits, featureNames := splitLayers(layerVecs, ilas, addr.FEATURE, "features", *featureEmbeddings)
charSplits, charNames := splitCharLayers(layerVecs, ilas, "chars")
splits := make([]uint, 0, len(wordSplits)+len(tagSplits)+len(relSplits))
splits = append(splits, wordSplits...)
splits = append(splits, tagSplits...)
splits = append(splits, relSplits...)
splits = append(splits, featureSplits...)
splits = append(splits, charSplits...)
splitNames := make([]string, 0, len(wordNames)+len(tagNames)+len(relNames)+len(featureNames))
splitNames = append(splitNames, wordNames...)
splitNames = append(splitNames, tagNames...)
splitNames = append(splitNames, relNames...)
splitNames = append(splitNames, featureNames...)
splitNames = append(splitNames, charNames...)
layers = append(layers, createSliceLayer("slice", "data", splits, splitNames))
concatLayers := make([]string, 0)
if *wordEmbeddings {
layers = append(layers, createEmbeddingLayers("words", "wordembed", wordSplits, 50, false)...)
concatLayers = append(concatLayers, names("wordembed", wordSplits)...)
} else {
layers = append(layers, createActivationLayer("flat_words", "words", "flat_words", "Flatten"))
concatLayers = append(concatLayers, "flat_words")
}
if *tagEmbeddings {
layers = append(layers, createEmbeddingLayers("tags", "tagembed", tagSplits, 50, false)...)
concatLayers = append(concatLayers, names("tagembed", tagSplits)...)
} else {
layers = append(layers, createActivationLayer("flat_tags", "tags", "flat_tags", "Flatten"))
concatLayers = append(concatLayers, "flat_tags")
}
if *deprelEmbeddings {
layers = append(layers, createEmbeddingLayers("deprels", "relembed", relSplits, 50, false)...)
concatLayers = append(concatLayers, names("relembed", relSplits)...)
} else {
layers = append(layers, createActivationLayer("flat_deprels", "deprels", "flat_deprels", "Flatten"))
concatLayers = append(concatLayers, "flat_deprels")
}
if *featureEmbeddings {
layers = append(layers, createEmbeddingLayers("features", "featureembed", featureSplits, 50, false)...)
concatLayers = append(concatLayers, names("featureembed", relSplits)...)
} else if layerVecs[addr.FEATURE] != nil {
layers = append(layers, createActivationLayer("flat_features", "features", "flat_features", "Flatten"))
concatLayers = append(concatLayers, "flat_features")
}
layers = append(layers, createEmbeddingLayers("chars", "morph", charSplits, 210, true)...)
for _, name := range names("morph", charSplits) {
layers = append(layers, createActivationLayer(fmt.Sprintf("%s_activation", name), name, name, "Sigmoid"))
}
concatLayers = append(concatLayers, names("morph", charSplits)...)
layers = append(layers,
createConcatLayer("concat", "concat", concatLayers),
createInnerProductLayer("ip", "concat", "ip", 200),
createActivationLayer("tanh", "ip", "ip", "TanH"),
createDropoutLayer("dropout", "ip", "ip", 0.05),
createInnerProductLayer("ip2", "ip", "ip2", uint32(labelNumberer.Size())),
createLossLayer("softmax", "ip2", "label", "softmax", "SoftmaxWithLoss"),
createAccuracyLayer("accuracy", "ip2", "label", "accuracy", "Accuracy"),
)
net := &caffe.NetParameter{
Name: proto.String("dparnn-net"),
Layer: layers,
}
proto.MarshalText(os.Stdout, net)
}
