Exemple #1
0
func sgdOnSequences(f *rnn.Bidirectional, s []seqtoseq.Sample) {
	gradient := autofunc.NewGradient(f.Parameters())
	for _, x := range s {
		inRes := seqfunc.ConstResult([][]linalg.Vector{x.Inputs})
		output := f.ApplySeqs(inRes)
		upstreamGrad := make([]linalg.Vector, len(x.Outputs))
		for i, o := range x.Outputs {
			upstreamGrad[i] = o.Copy().Scale(-1)
		}
		output.PropagateGradient([][]linalg.Vector{upstreamGrad}, gradient)
	}
	for _, vec := range gradient {
		for i, x := range vec {
			if x > 0 {
				vec[i] = 1
			} else {
				vec[i] = -1
			}
		}
	}
	gradient.AddToVars(-StepSize)
}
Exemple #2
0
func Train(rnnFile, sampleDir string, stepSize float64) {
	log.Println("Loading samples...")
	samples, err := ReadSamples(sampleDir)
	if err != nil {
		fmt.Fprintln(os.Stderr, "Failed to read samples:", err)
		os.Exit(1)
	}

	var seqFunc *rnn.Bidirectional
	rnnData, err := ioutil.ReadFile(rnnFile)
	if err == nil {
		log.Println("Loaded network from file.")
		seqFunc, err = rnn.DeserializeBidirectional(rnnData)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Failed to deserialize network:", err)
			os.Exit(1)
		}
	} else {
		log.Println("Created network.")
		seqFunc = createNetwork(samples)
	}

	crossLen := int(CrossRatio * float64(samples.Len()))
	log.Println("Using", samples.Len()-crossLen, "training and",
		crossLen, "validation samples...")

	// Always shuffle the samples in the same way.
	rand.Seed(123)
	sgd.ShuffleSampleSet(samples)
	validation := samples.Subset(0, crossLen)
	training := samples.Subset(crossLen, samples.Len())

	gradienter := &sgd.Adam{
		Gradienter: &ctc.RGradienter{
			Learner:        seqFunc,
			SeqFunc:        seqFunc,
			MaxConcurrency: MaxConcurrency,
			MaxSubBatch:    MaxSubBatch,
		},
	}

	var epoch int
	toggleRegularization(seqFunc, true)
	sgd.SGDInteractive(gradienter, training, stepSize, BatchSize, func() bool {
		toggleRegularization(seqFunc, false)
		cost := ctc.TotalCost(seqFunc, training, CostBatchSize, MaxConcurrency)
		crossCost := ctc.TotalCost(seqFunc, validation, CostBatchSize, MaxConcurrency)
		toggleRegularization(seqFunc, true)
		log.Printf("Epoch %d: cost=%e cross=%e", epoch, cost, crossCost)
		epoch++
		return true
	})
	toggleRegularization(seqFunc, false)

	data, err := seqFunc.Serialize()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Failed to serialize:", err)
		os.Exit(1)
	}

	if err := ioutil.WriteFile(rnnFile, data, 0755); err != nil {
		fmt.Fprintln(os.Stderr, "Failed to save:", err)
		os.Exit(1)
	}
}