Ejemplo n.º 1
0
func main() {
	rand.Seed(time.Now().UnixNano())

	outNet := neuralnet.Network{
		&neuralnet.DenseLayer{
			InputCount:  StateSize * 2,
			OutputCount: 10,
		},
		&neuralnet.Sigmoid{},
		&neuralnet.DenseLayer{
			InputCount:  10,
			OutputCount: 2,
		},
		&neuralnet.LogSoftmaxLayer{},
	}
	outNet.Randomize()
	bd := &rnn.Bidirectional{
		Forward:  &rnn.BlockSeqFunc{B: rnn.NewGRU(2, StateSize)},
		Backward: &rnn.BlockSeqFunc{B: rnn.NewGRU(2, StateSize)},
		Output:   &rnn.NetworkSeqFunc{Network: outNet},
	}
	var samples []seqtoseq.Sample
	var sampleSet sgd.SliceSampleSet
	for i := 0; i < TrainingSize; i++ {
		samples = append(samples, generateSequence())
		sampleSet = append(sampleSet, samples[i])
	}

	g := &sgd.RMSProp{
		Gradienter: &seqtoseq.Gradienter{
			SeqFunc:  bd,
			Learner:  bd,
			CostFunc: neuralnet.DotCost{},
		},
	}

	var i int
	sgd.SGDInteractive(g, sampleSet, StepSize, BatchSize, func() bool {
		fmt.Printf("%d epochs: cost=%f\n", i, totalCost(bd, sampleSet))
		i++
		return true
	})

	var testingCorrect, testingTotal int
	for j := 0; j < TestingSize; j++ {
		sample := generateSequence()
		inRes := seqfunc.ConstResult([][]linalg.Vector{sample.Inputs})
		output := bd.ApplySeqs(inRes).OutputSeqs()[0]
		for i, expected := range sample.Outputs {
			actual := output[i]
			if math.Abs(expected[0]-math.Exp(actual[0])) < 0.1 {
				testingCorrect++
			}
			testingTotal++
		}
	}

	fmt.Printf("Got %d/%d (%.2f%%)\n", testingCorrect, testingTotal,
		100*float64(testingCorrect)/float64(testingTotal))
}
Ejemplo n.º 2
0
func createNetwork(samples sgd.SampleSet) *rnn.Bidirectional {
	means := make(linalg.Vector, FeatureCount)
	var count float64

	for i := 0; i < samples.Len(); i++ {
		inputSeq := samples.GetSample(i).(ctc.Sample).Input
		for _, vec := range inputSeq {
			means.Add(vec)
			count++
		}
	}
	means.Scale(-1 / count)

	stddevs := make(linalg.Vector, FeatureCount)
	for i := 0; i < samples.Len(); i++ {
		inputSeq := samples.GetSample(i).(ctc.Sample).Input
		for _, vec := range inputSeq {
			for j, v := range vec {
				stddevs[j] += math.Pow(v+means[j], 2)
			}
		}
	}
	stddevs.Scale(1 / count)
	for i, x := range stddevs {
		stddevs[i] = 1 / math.Sqrt(x)
	}

	outputNet := neuralnet.Network{
		&neuralnet.DropoutLayer{
			KeepProbability: HiddenDropout,
			Training:        false,
		},
		&neuralnet.DenseLayer{
			InputCount:  HiddenSize * 2,
			OutputCount: OutHiddenSize,
		},
		&neuralnet.HyperbolicTangent{},
		&neuralnet.DenseLayer{
			InputCount:  OutHiddenSize,
			OutputCount: len(cubewhisper.Labels) + 1,
		},
		&neuralnet.LogSoftmaxLayer{},
	}
	outputNet.Randomize()

	inputNet := neuralnet.Network{
		&neuralnet.VecRescaleLayer{
			Biases: means,
			Scales: stddevs,
		},
		&neuralnet.GaussNoiseLayer{
			Stddev:   InputNoise,
			Training: false,
		},
	}
	netBlock := rnn.NewNetworkBlock(inputNet, 0)
	forwardBlock := rnn.StackedBlock{
		netBlock,
		rnn.NewGRU(FeatureCount, HiddenSize),
	}
	backwardBlock := rnn.StackedBlock{
		netBlock,
		rnn.NewGRU(FeatureCount, HiddenSize),
	}
	for _, block := range []rnn.StackedBlock{forwardBlock, backwardBlock} {
		for i, param := range block.Parameters() {
			if i%2 == 0 {
				for i := range param.Vector {
					param.Vector[i] = rand.NormFloat64() * WeightStddev
				}
			}
		}
	}
	return &rnn.Bidirectional{
		Forward:  &rnn.BlockSeqFunc{Block: forwardBlock},
		Backward: &rnn.BlockSeqFunc{Block: backwardBlock},
		Output:   &rnn.NetworkSeqFunc{Network: outputNet},
	}
}
Ejemplo n.º 3
0
func TestGRU(t *testing.T) {
	b := rnn.NewGRU(4, 2)
	NewChecker4In(b, b).FullCheck(t)
}