func main() {
rand.Seed(time.Now().UnixNano())
outNet := neuralnet.Network{
&neuralnet.DenseLayer{
InputCount: StateSize * 2,
OutputCount: 10,
},
&neuralnet.Sigmoid{},
&neuralnet.DenseLayer{
InputCount: 10,
OutputCount: 2,
},
&neuralnet.LogSoftmaxLayer{},
}
outNet.Randomize()
bd := &rnn.Bidirectional{
Forward: &rnn.BlockSeqFunc{B: rnn.NewGRU(2, StateSize)},
Backward: &rnn.BlockSeqFunc{B: rnn.NewGRU(2, StateSize)},
Output: &rnn.NetworkSeqFunc{Network: outNet},
}
var samples []seqtoseq.Sample
var sampleSet sgd.SliceSampleSet
for i := 0; i < TrainingSize; i++ {
samples = append(samples, generateSequence())
sampleSet = append(sampleSet, samples[i])
}
g := &sgd.RMSProp{
Gradienter: &seqtoseq.Gradienter{
SeqFunc: bd,
Learner: bd,
CostFunc: neuralnet.DotCost{},
},
}
var i int
sgd.SGDInteractive(g, sampleSet, StepSize, BatchSize, func() bool {
fmt.Printf("%d epochs: cost=%f\n", i, totalCost(bd, sampleSet))
i++
return true
})
var testingCorrect, testingTotal int
for j := 0; j < TestingSize; j++ {
sample := generateSequence()
inRes := seqfunc.ConstResult([][]linalg.Vector{sample.Inputs})
output := bd.ApplySeqs(inRes).OutputSeqs()[0]
for i, expected := range sample.Outputs {
actual := output[i]
if math.Abs(expected[0]-math.Exp(actual[0])) < 0.1 {
testingCorrect++
}
testingTotal++
}
}
fmt.Printf("Got %d/%d (%.2f%%)\n", testingCorrect, testingTotal,
100*float64(testingCorrect)/float64(testingTotal))
}
func createNetwork(samples sgd.SampleSet) *rnn.Bidirectional {
means := make(linalg.Vector, FeatureCount)
var count float64
for i := 0; i < samples.Len(); i++ {
inputSeq := samples.GetSample(i).(ctc.Sample).Input
for _, vec := range inputSeq {
means.Add(vec)
count++
}
}
means.Scale(-1 / count)
stddevs := make(linalg.Vector, FeatureCount)
for i := 0; i < samples.Len(); i++ {
inputSeq := samples.GetSample(i).(ctc.Sample).Input
for _, vec := range inputSeq {
for j, v := range vec {
stddevs[j] += math.Pow(v+means[j], 2)
}
}
}
stddevs.Scale(1 / count)
for i, x := range stddevs {
stddevs[i] = 1 / math.Sqrt(x)
}
outputNet := neuralnet.Network{
&neuralnet.DropoutLayer{
KeepProbability: HiddenDropout,
Training: false,
},
&neuralnet.DenseLayer{
InputCount: HiddenSize * 2,
OutputCount: OutHiddenSize,
},
&neuralnet.HyperbolicTangent{},
&neuralnet.DenseLayer{
InputCount: OutHiddenSize,
OutputCount: len(cubewhisper.Labels) + 1,
},
&neuralnet.LogSoftmaxLayer{},
}
outputNet.Randomize()
inputNet := neuralnet.Network{
&neuralnet.VecRescaleLayer{
Biases: means,
Scales: stddevs,
},
&neuralnet.GaussNoiseLayer{
Stddev: InputNoise,
Training: false,
},
}
netBlock := rnn.NewNetworkBlock(inputNet, 0)
forwardBlock := rnn.StackedBlock{
netBlock,
rnn.NewGRU(FeatureCount, HiddenSize),
}
backwardBlock := rnn.StackedBlock{
netBlock,
rnn.NewGRU(FeatureCount, HiddenSize),
}
for _, block := range []rnn.StackedBlock{forwardBlock, backwardBlock} {
for i, param := range block.Parameters() {
if i%2 == 0 {
for i := range param.Vector {
param.Vector[i] = rand.NormFloat64() * WeightStddev
}
}
}
}
return &rnn.Bidirectional{
Forward: &rnn.BlockSeqFunc{Block: forwardBlock},
Backward: &rnn.BlockSeqFunc{Block: backwardBlock},
Output: &rnn.NetworkSeqFunc{Network: outputNet},
}
}
func TestGRU(t *testing.T) {
b := rnn.NewGRU(4, 2)
NewChecker4In(b, b).FullCheck(t)
}