Exemplo n.º 1
0
func main() {
	flag.Parse()

	if flag.NArg() != 1 && flag.NArg() != 2 {
		flag.Usage()
		os.Exit(1)
	}

	config := common.ReadConfigOrExit(flag.Arg(0))

	transitionSystem, ok := common.TransitionSystems[config.Parser.System]
	if !ok {
		log.Fatalf("Unknown transition system: %s", config.Parser.System)
	}

	blasImpl, ok := blasImpls[*blas]
	if !ok {
		log.Fatalf("Unknown blas implementation: %s", *blas)
	}

	labelNumberer := common.ReadTransitionsOrExit(config.Parser.Transitions, transitionSystem)
	parseNetwork := common.ReadModelOrExit(config.Parser.Model, blasImpl)

	var morphNetwork *network.Network
	if len(config.Parser.MorphModel) != 0 {
		morphNetwork = common.ReadModelOrExit(config.Parser.MorphModel, blasImpl)
	}

	ilas := common.ReadIlasOrExit(config.Parser.Inputs)
	normalizer := common.ReadNormalizerOrExit(config.Parser.Normalisation)

	layerEmbeddings := common.MustReadAllEmbeddings(config.Embeddings)

	realizer := input.NewInputVectorRealizer(ilas, layerEmbeddings, normalizer)

	var guide system.Guide
	if morphNetwork != nil {
		log.Print("Using morphology")
		guide = network.NewMorphGuide(parseNetwork, morphNetwork, *labelNumberer, realizer)
	} else {
		guide = network.NewGuide(parseNetwork, *labelNumberer, realizer)
	}
	parser := system.NewGreedyParser(transitionSystem, guide)

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	start := time.Now()
	run(parser)
	elapsed := time.Since(start)
	log.Printf("Parsing took %s", elapsed)
}
Exemplo n.º 2
0
func main() {
	flag.Parse()
	if flag.NArg() != 3 {
		flag.Usage()
		os.Exit(1)
	}

	vecs := common.ReadEmbeddingsOrExit(common.Embedding{flag.Arg(0), false, false})
	network := common.ReadModelOrExit(flag.Arg(1), cblas.Implementation{})

	out, err := os.Create(flag.Arg(2))
	common.ExitIfError("Cannot open output vectors for writing: ", err)
	defer out.Close()

	if network.Layers() != 1 {
		fmt.Fprintf(os.Stderr, "Weight file contains %d layers, expected 1", network.Layers())
		os.Exit(1)
	}

	layer := network.Layer(0)
	weights := layer.W()

	if layer.Inputs() != uint(vecs.Size()) {
		fmt.Fprintf(os.Stderr, "Embedding layer and one-hot size mismatch: %d - %d", layer.Inputs(), vecs.Size())
		os.Exit(1)
	}

	mergedVecs := go2vec.NewEmbeddings(int(layer.Outputs()))

	wordIdx := 0
	vec := make([]float32, layer.Outputs())
	vecs.Iterate(func(word string, vector []float32) bool {
		for idx := range vec {
			vec[idx] = weights[uint(idx)*layer.Inputs()+uint(wordIdx)]
		}

		mergedVecs.Put(word, vec)
		wordIdx++
		return true
	})

	writer := bufio.NewWriter(out)
	mergedVecs.Write(writer)
	writer.Flush()
}