func main() {
rand.Seed(time.Now().UnixNano())
sampleSet := sgd.SliceSampleSet{}
for i := 0; i < TrainingCount; i++ {
inSeq, outSeq := genEvenOddSeq(rand.Intn(MaxSeqLen-MinSeqLen) + MinSeqLen)
sampleSet = append(sampleSet, seqtoseq.Sample{
Inputs: inSeq,
Outputs: outSeq,
})
}
outNet := neuralnet.Network{
&neuralnet.DenseLayer{
InputCount: HiddenSize,
OutputCount: 2,
},
}
outNet.Randomize()
outBlock := rnn.NewNetworkBlock(outNet, 0)
lstm := rnn.NewLSTM(2, HiddenSize)
net := rnn.StackedBlock{lstm, outBlock}
gradienter := &sgd.RMSProp{
Gradienter: &seqtoseq.Gradienter{
SeqFunc: &rnn.BlockSeqFunc{B: net},
Learner: net,
CostFunc: neuralnet.SigmoidCECost{},
MaxLanes: 1,
},
}
sgd.SGD(gradienter, sampleSet, StepSize, Epochs, BatchSize)
outNet = append(outNet, neuralnet.Sigmoid{})
var scoreSum float64
var scoreTotal float64
for i := 0; i < TestingCount; i++ {
size := rand.Intn(MaxSeqLen-MinSeqLen) + MinSeqLen
ins, outs := genEvenOddSeq(size)
score := runTestSample(ins, outs, net)
scoreSum += score
scoreTotal += 1
}
fmt.Println("Testing success rate:", scoreSum/scoreTotal)
}
func createNetwork(samples sgd.SampleSet) *rnn.Bidirectional {
means := make(linalg.Vector, FeatureCount)
var count float64
for i := 0; i < samples.Len(); i++ {
inputSeq := samples.GetSample(i).(ctc.Sample).Input
for _, vec := range inputSeq {
means.Add(vec)
count++
}
}
means.Scale(-1 / count)
stddevs := make(linalg.Vector, FeatureCount)
for i := 0; i < samples.Len(); i++ {
inputSeq := samples.GetSample(i).(ctc.Sample).Input
for _, vec := range inputSeq {
for j, v := range vec {
stddevs[j] += math.Pow(v+means[j], 2)
}
}
}
stddevs.Scale(1 / count)
for i, x := range stddevs {
stddevs[i] = 1 / math.Sqrt(x)
}
outputNet := neuralnet.Network{
&neuralnet.DropoutLayer{
KeepProbability: HiddenDropout,
Training: false,
},
&neuralnet.DenseLayer{
InputCount: HiddenSize * 2,
OutputCount: OutHiddenSize,
},
&neuralnet.HyperbolicTangent{},
&neuralnet.DenseLayer{
InputCount: OutHiddenSize,
OutputCount: len(cubewhisper.Labels) + 1,
},
&neuralnet.LogSoftmaxLayer{},
}
outputNet.Randomize()
inputNet := neuralnet.Network{
&neuralnet.VecRescaleLayer{
Biases: means,
Scales: stddevs,
},
&neuralnet.GaussNoiseLayer{
Stddev: InputNoise,
Training: false,
},
}
netBlock := rnn.NewNetworkBlock(inputNet, 0)
forwardBlock := rnn.StackedBlock{
netBlock,
rnn.NewGRU(FeatureCount, HiddenSize),
}
backwardBlock := rnn.StackedBlock{
netBlock,
rnn.NewGRU(FeatureCount, HiddenSize),
}
for _, block := range []rnn.StackedBlock{forwardBlock, backwardBlock} {
for i, param := range block.Parameters() {
if i%2 == 0 {
for i := range param.Vector {
param.Vector[i] = rand.NormFloat64() * WeightStddev
}
}
}
}
return &rnn.Bidirectional{
Forward: &rnn.BlockSeqFunc{Block: forwardBlock},
Backward: &rnn.BlockSeqFunc{Block: backwardBlock},
Output: &rnn.NetworkSeqFunc{Network: outputNet},
}
}
