Esempio n. 1
0
func main() {
	rand.Seed(time.Now().UnixNano())

	sampleSet := sgd.SliceSampleSet{}
	for i := 0; i < TrainingCount; i++ {
		inSeq, outSeq := genEvenOddSeq(rand.Intn(MaxSeqLen-MinSeqLen) + MinSeqLen)
		sampleSet = append(sampleSet, seqtoseq.Sample{
			Inputs:  inSeq,
			Outputs: outSeq,
		})
	}

	outNet := neuralnet.Network{
		&neuralnet.DenseLayer{
			InputCount:  HiddenSize,
			OutputCount: 2,
		},
	}
	outNet.Randomize()
	outBlock := rnn.NewNetworkBlock(outNet, 0)
	lstm := rnn.NewLSTM(2, HiddenSize)
	net := rnn.StackedBlock{lstm, outBlock}

	gradienter := &sgd.RMSProp{
		Gradienter: &seqtoseq.Gradienter{
			SeqFunc:  &rnn.BlockSeqFunc{B: net},
			Learner:  net,
			CostFunc: neuralnet.SigmoidCECost{},
			MaxLanes: 1,
		},
	}

	sgd.SGD(gradienter, sampleSet, StepSize, Epochs, BatchSize)

	outNet = append(outNet, neuralnet.Sigmoid{})

	var scoreSum float64
	var scoreTotal float64
	for i := 0; i < TestingCount; i++ {
		size := rand.Intn(MaxSeqLen-MinSeqLen) + MinSeqLen
		ins, outs := genEvenOddSeq(size)
		score := runTestSample(ins, outs, net)
		scoreSum += score
		scoreTotal += 1
	}

	fmt.Println("Testing success rate:", scoreSum/scoreTotal)
}
Esempio n. 2
0
func createNetwork(samples sgd.SampleSet) *rnn.Bidirectional {
	means := make(linalg.Vector, FeatureCount)
	var count float64

	for i := 0; i < samples.Len(); i++ {
		inputSeq := samples.GetSample(i).(ctc.Sample).Input
		for _, vec := range inputSeq {
			means.Add(vec)
			count++
		}
	}
	means.Scale(-1 / count)

	stddevs := make(linalg.Vector, FeatureCount)
	for i := 0; i < samples.Len(); i++ {
		inputSeq := samples.GetSample(i).(ctc.Sample).Input
		for _, vec := range inputSeq {
			for j, v := range vec {
				stddevs[j] += math.Pow(v+means[j], 2)
			}
		}
	}
	stddevs.Scale(1 / count)
	for i, x := range stddevs {
		stddevs[i] = 1 / math.Sqrt(x)
	}

	outputNet := neuralnet.Network{
		&neuralnet.DropoutLayer{
			KeepProbability: HiddenDropout,
			Training:        false,
		},
		&neuralnet.DenseLayer{
			InputCount:  HiddenSize * 2,
			OutputCount: OutHiddenSize,
		},
		&neuralnet.HyperbolicTangent{},
		&neuralnet.DenseLayer{
			InputCount:  OutHiddenSize,
			OutputCount: len(cubewhisper.Labels) + 1,
		},
		&neuralnet.LogSoftmaxLayer{},
	}
	outputNet.Randomize()

	inputNet := neuralnet.Network{
		&neuralnet.VecRescaleLayer{
			Biases: means,
			Scales: stddevs,
		},
		&neuralnet.GaussNoiseLayer{
			Stddev:   InputNoise,
			Training: false,
		},
	}
	netBlock := rnn.NewNetworkBlock(inputNet, 0)
	forwardBlock := rnn.StackedBlock{
		netBlock,
		rnn.NewGRU(FeatureCount, HiddenSize),
	}
	backwardBlock := rnn.StackedBlock{
		netBlock,
		rnn.NewGRU(FeatureCount, HiddenSize),
	}
	for _, block := range []rnn.StackedBlock{forwardBlock, backwardBlock} {
		for i, param := range block.Parameters() {
			if i%2 == 0 {
				for i := range param.Vector {
					param.Vector[i] = rand.NormFloat64() * WeightStddev
				}
			}
		}
	}
	return &rnn.Bidirectional{
		Forward:  &rnn.BlockSeqFunc{Block: forwardBlock},
		Backward: &rnn.BlockSeqFunc{Block: backwardBlock},
		Output:   &rnn.NetworkSeqFunc{Network: outputNet},
	}
}