func main() { rand.Seed(time.Now().UnixNano()) outNet := neuralnet.Network{ &neuralnet.DenseLayer{ InputCount: StateSize * 2, OutputCount: 10, }, &neuralnet.Sigmoid{}, &neuralnet.DenseLayer{ InputCount: 10, OutputCount: 2, }, &neuralnet.LogSoftmaxLayer{}, } outNet.Randomize() bd := &rnn.Bidirectional{ Forward: &rnn.BlockSeqFunc{B: rnn.NewGRU(2, StateSize)}, Backward: &rnn.BlockSeqFunc{B: rnn.NewGRU(2, StateSize)}, Output: &rnn.NetworkSeqFunc{Network: outNet}, } var samples []seqtoseq.Sample var sampleSet sgd.SliceSampleSet for i := 0; i < TrainingSize; i++ { samples = append(samples, generateSequence()) sampleSet = append(sampleSet, samples[i]) } g := &sgd.RMSProp{ Gradienter: &seqtoseq.Gradienter{ SeqFunc: bd, Learner: bd, CostFunc: neuralnet.DotCost{}, }, } var i int sgd.SGDInteractive(g, sampleSet, StepSize, BatchSize, func() bool { fmt.Printf("%d epochs: cost=%f\n", i, totalCost(bd, sampleSet)) i++ return true }) var testingCorrect, testingTotal int for j := 0; j < TestingSize; j++ { sample := generateSequence() inRes := seqfunc.ConstResult([][]linalg.Vector{sample.Inputs}) output := bd.ApplySeqs(inRes).OutputSeqs()[0] for i, expected := range sample.Outputs { actual := output[i] if math.Abs(expected[0]-math.Exp(actual[0])) < 0.1 { testingCorrect++ } testingTotal++ } } fmt.Printf("Got %d/%d (%.2f%%)\n", testingCorrect, testingTotal, 100*float64(testingCorrect)/float64(testingTotal)) }
func printScore(prefix string, n neuralnet.Network, d mnist.DataSet) { classifier := func(v []float64) int { r := n.Apply(&autofunc.Variable{v}) return networkOutput(r) } correctCount := d.NumCorrect(classifier) histogram := d.CorrectnessHistogram(classifier) log.Printf("%s: %d/%d - %s", prefix, correctCount, len(d.Samples), histogram) }
// NewNetworkBlock creates a NetworkBlock. func NewNetworkBlock(n neuralnet.Network, stateSize int) *NetworkBlock { return &NetworkBlock{ batcherBlock: &BatcherBlock{ B: n.BatchLearner(), StateSize: stateSize, Start: &autofunc.Variable{Vector: make(linalg.Vector, stateSize)}, }, network: n, } }
func TestStackedBlock(t *testing.T) { testVars := []*autofunc.Variable{ {Vector: []float64{0.098591, -0.595453, -0.751214, 0.266051}}, {Vector: []float64{0.988517, 0.107284, -0.331529, 0.028565}}, {Vector: []float64{-0.150604, 0.889039, 0.120916, 0.240999}}, {Vector: []float64{0.961058, 0.878608, 0.052284, -0.635746}}, {Vector: []float64{0.31415, -0.2718}}, {Vector: []float64{-0.6}}, } testSeqs := [][]*autofunc.Variable{ {testVars[0], testVars[2]}, {testVars[1]}, {testVars[2], testVars[1], testVars[3]}, } testRV := autofunc.RVector{ testVars[0]: []float64{0.62524, 0.52979, 0.33020, 0.54462}, testVars[1]: []float64{0.13498, 0.12607, 0.35989, 0.23255}, testVars[2]: []float64{0.85996, 0.68435, 0.68506, 0.96907}, testVars[3]: []float64{0.79095, 0.33867, 0.86759, 0.16159}, testVars[4]: []float64{-0.79095, 0.33867}, testVars[5]: []float64{0.33867}, } net1 := neuralnet.Network{ &neuralnet.DenseLayer{ InputCount: 6, OutputCount: 6, }, &neuralnet.HyperbolicTangent{}, } net1.Randomize() net2 := neuralnet.Network{ &neuralnet.DenseLayer{ InputCount: 5, OutputCount: 5, }, &neuralnet.HyperbolicTangent{}, } net2.Randomize() block := &rnn.StackedBlock{ &rnn.BatcherBlock{B: net1.BatchLearner(), StateSize: 2, Start: testVars[4]}, &rnn.BatcherBlock{B: net2.BatchLearner(), StateSize: 1, Start: testVars[5]}, } checker := &BlockChecker{ B: block, Input: testSeqs, Vars: testVars, RV: testRV, } checker.FullCheck(t) }
// firstBitTest builds a neural network to: // - output 0 for inputs starting with a 1 // - output 1 for inputs starting with a 0. func firstBitTest() { trainingSamples := make([]linalg.Vector, FirstBitTrainingSize) trainingOutputs := make([]linalg.Vector, FirstBitTrainingSize) for i := range trainingSamples { trainingSamples[i] = make(linalg.Vector, FirstBitInputSize) for j := range trainingSamples[i] { trainingSamples[i][j] = float64(rand.Intn(2)) } trainingOutputs[i] = []float64{1 - trainingSamples[i][0]} } samples := neuralnet.VectorSampleSet(trainingSamples, trainingOutputs) network := neuralnet.Network{ &neuralnet.DenseLayer{ InputCount: FirstBitInputSize, OutputCount: FirstBitHiddenSize, }, &neuralnet.Sigmoid{}, &neuralnet.DenseLayer{ InputCount: FirstBitHiddenSize, OutputCount: 1, }, &neuralnet.Sigmoid{}, } network.Randomize() batcher := &neuralnet.SingleRGradienter{ Learner: network, CostFunc: neuralnet.MeanSquaredCost{}, } sgd.SGD(batcher, samples, 0.2, 100000, 1) var totalError float64 var maxPossibleError float64 for i := 0; i < 50; i++ { sample := make([]float64, FirstBitInputSize) for j := range sample { sample[j] = float64(rand.Intn(2)) } result := network.Apply(&autofunc.Variable{sample}) output := result.Output()[0] amountError := math.Abs(output - (1 - sample[0])) totalError += amountError maxPossibleError += 1.0 } fmt.Printf("firstBitTest() error rate: %f\n", totalError/maxPossibleError) }
func main() { rand.Seed(time.Now().UnixNano()) sampleSet := sgd.SliceSampleSet{} for i := 0; i < TrainingCount; i++ { inSeq, outSeq := genEvenOddSeq(rand.Intn(MaxSeqLen-MinSeqLen) + MinSeqLen) sampleSet = append(sampleSet, seqtoseq.Sample{ Inputs: inSeq, Outputs: outSeq, }) } outNet := neuralnet.Network{ &neuralnet.DenseLayer{ InputCount: HiddenSize, OutputCount: 2, }, } outNet.Randomize() outBlock := rnn.NewNetworkBlock(outNet, 0) lstm := rnn.NewLSTM(2, HiddenSize) net := rnn.StackedBlock{lstm, outBlock} gradienter := &sgd.RMSProp{ Gradienter: &seqtoseq.Gradienter{ SeqFunc: &rnn.BlockSeqFunc{B: net}, Learner: net, CostFunc: neuralnet.SigmoidCECost{}, MaxLanes: 1, }, } sgd.SGD(gradienter, sampleSet, StepSize, Epochs, BatchSize) outNet = append(outNet, neuralnet.Sigmoid{}) var scoreSum float64 var scoreTotal float64 for i := 0; i < TestingCount; i++ { size := rand.Intn(MaxSeqLen-MinSeqLen) + MinSeqLen ins, outs := genEvenOddSeq(size) score := runTestSample(ins, outs, net) scoreSum += score scoreTotal += 1 } fmt.Println("Testing success rate:", scoreSum/scoreTotal) }
func createNet(d mnist.DataSet) neuralnet.Network { convOutWidth := (d.Width-FilterSize)/FilterStride + 1 convOutHeight := (d.Height-FilterSize)/FilterStride + 1 poolOutWidth := convOutWidth / MaxPoolingSpan if convOutWidth%MaxPoolingSpan != 0 { poolOutWidth++ } poolOutHeight := convOutWidth / MaxPoolingSpan if convOutHeight%MaxPoolingSpan != 0 { poolOutHeight++ } net := neuralnet.Network{ &neuralnet.ConvLayer{ FilterCount: FilterCount, FilterWidth: FilterSize, FilterHeight: FilterSize, Stride: FilterStride, InputWidth: d.Width, InputHeight: d.Height, InputDepth: 1, }, &neuralnet.Sigmoid{}, &neuralnet.MaxPoolingLayer{ XSpan: MaxPoolingSpan, YSpan: MaxPoolingSpan, InputWidth: convOutWidth, InputHeight: convOutHeight, InputDepth: FilterCount, }, &neuralnet.DenseLayer{ InputCount: poolOutWidth * poolOutHeight * FilterCount, OutputCount: HiddenSize, }, &neuralnet.Sigmoid{}, &neuralnet.DenseLayer{ InputCount: HiddenSize, OutputCount: LabelCount, }, &neuralnet.SoftmaxLayer{}, } net.Randomize() return net }
func countCorrect(n neuralnet.Network, s sgd.SampleSet) int { var count int for i := 0; i < s.Len(); i++ { sample := s.GetSample(i).(neuralnet.VectorSample) output := n.Apply(&autofunc.Variable{Vector: sample.Input}).Output() var maxIdx int var maxVal float64 for j, x := range output { if x > maxVal || j == 0 { maxIdx = j maxVal = x } } if sample.Output[maxIdx] == 1 { count++ } } return count }
func TestBaselineChecks(t *testing.T) { network := neuralnet.Network{ &neuralnet.DenseLayer{ InputCount: 4, OutputCount: 6, }, neuralnet.HyperbolicTangent{}, } network.Randomize() for stateSize := 0; stateSize < 4; stateSize++ { start := &autofunc.Variable{Vector: make(linalg.Vector, stateSize)} for i := range start.Vector { start.Vector[i] = rand.NormFloat64() } toTest := &rnn.BlockSeqFunc{ B: &rnn.BatcherBlock{ B: network.BatchLearner(), StateSize: stateSize, Start: start, }, } seqs, rv := randBaselineTestSeqs(network, 4-stateSize) rv[start] = make(linalg.Vector, len(start.Vector)) for i := range rv[start] { rv[start][i] = rand.NormFloat64() } vars := make([]*autofunc.Variable, 0, len(rv)) for v := range rv { vars = append(vars, v) } checker := &functest.SeqRFuncChecker{ F: toTest, Vars: vars, Input: seqs, RV: rv, } checker.FullCheck(t) } }
func trainClassifier(n neuralnet.Network, d mnist.DataSet) { log.Println("Training classifier (ctrl+C to finish)...") killChan := make(chan struct{}) go func() { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) <-c signal.Stop(c) fmt.Println("\nCaught interrupt. Ctrl+C again to terminate.") close(killChan) }() inputs := make([]linalg.Vector, len(d.Samples)) outputs := make([]linalg.Vector, len(d.Samples)) for i, x := range d.IntensityVectors() { inputs[i] = x } for i, x := range d.LabelVectors() { outputs[i] = x } samples := neuralnet.VectorSampleSet(inputs, outputs) batcher := &neuralnet.BatchRGradienter{ Learner: n.BatchLearner(), CostFunc: neuralnet.MeanSquaredCost{}, } crossValidation := mnist.LoadTestingDataSet() sgd.SGDInteractive(batcher, samples, ClassifierStepSize, ClassifierBatchSize, func() bool { printScore("Training", n, d) printScore("Cross", n, crossValidation) return true }) }
func runHorizontalLineTest(name string, network neuralnet.Network) { trainingSamples := make([]linalg.Vector, GridTrainingSize) trainingOutputs := make([]linalg.Vector, GridTrainingSize) for i := range trainingSamples { trainingSamples[i] = randomBitmap() if bitmapHasHorizontal(trainingSamples[i]) { trainingOutputs[i] = []float64{1} } else { trainingOutputs[i] = []float64{0} } } samples := neuralnet.VectorSampleSet(trainingSamples, trainingOutputs) network.Randomize() batcher := &neuralnet.SingleRGradienter{ Learner: network, CostFunc: neuralnet.MeanSquaredCost{}, } sgd.SGD(batcher, samples, 0.1, 1000, 100) var trainingError float64 var maxTrainingError float64 for i, sample := range trainingSamples { result := network.Apply(&autofunc.Variable{sample}) output := result.Output()[0] amountError := math.Abs(output - trainingOutputs[i][0]) trainingError += amountError maxTrainingError += 1.0 } var totalError float64 var maxPossibleError float64 for i := 0; i < 50; i++ { sample := randomBitmap() var expected float64 if bitmapHasHorizontal(sample) { expected = 1 } result := network.Apply(&autofunc.Variable{sample}) output := result.Output()[0] amountError := math.Abs(output - expected) totalError += amountError maxPossibleError += 1.0 } fmt.Printf("%s() training error: %f; cross error: %f\n", name, trainingError/maxTrainingError, totalError/maxPossibleError) }
func TestStateOutBlock(t *testing.T) { net := neuralnet.Network{ &neuralnet.DenseLayer{ InputCount: 8, OutputCount: 4, }, &neuralnet.HyperbolicTangent{}, } net.Randomize() startVar := &autofunc.Variable{Vector: []float64{0.3, -0.3, 0.2, 0.5}} block := &rnn.StateOutBlock{ Block: &rnn.BatcherBlock{ B: net.BatchLearner(), StateSize: 4, Start: startVar, }, } learner := append(stateOutBlockLearner{startVar}, net.Parameters()...) NewChecker4In(block, learner).FullCheck(t) }
func TrainCmd(netPath, dirPath string) { log.Println("Loading samples...") images, width, height, err := LoadTrainingImages(dirPath) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } log.Println("Creating network...") var network neuralnet.Network networkData, err := ioutil.ReadFile(netPath) if err == nil { network, err = neuralnet.DeserializeNetwork(networkData) if err != nil { fmt.Fprintln(os.Stderr, "Failed to load network:", err) os.Exit(1) } log.Println("Loaded network from file.") } else { mean, stddev := sampleStatistics(images) convLayer := &neuralnet.ConvLayer{ FilterCount: FilterCount, FilterWidth: 4, FilterHeight: 4, Stride: 2, InputWidth: width, InputHeight: height, InputDepth: ImageDepth, } maxLayer := &neuralnet.MaxPoolingLayer{ XSpan: 3, YSpan: 3, InputWidth: convLayer.OutputWidth(), InputHeight: convLayer.OutputHeight(), InputDepth: convLayer.OutputDepth(), } convLayer1 := &neuralnet.ConvLayer{ FilterCount: FilterCount1, FilterWidth: 3, FilterHeight: 3, Stride: 2, InputWidth: maxLayer.OutputWidth(), InputHeight: maxLayer.OutputHeight(), InputDepth: maxLayer.InputDepth, } network = neuralnet.Network{ &neuralnet.RescaleLayer{ Bias: -mean, Scale: 1 / stddev, }, convLayer, neuralnet.HyperbolicTangent{}, maxLayer, neuralnet.HyperbolicTangent{}, convLayer1, neuralnet.HyperbolicTangent{}, &neuralnet.DenseLayer{ InputCount: convLayer1.OutputWidth() * convLayer1.OutputHeight() * convLayer1.OutputDepth(), OutputCount: HiddenSize, }, neuralnet.HyperbolicTangent{}, &neuralnet.DenseLayer{ InputCount: HiddenSize, OutputCount: len(images), }, &neuralnet.LogSoftmaxLayer{}, } network.Randomize() log.Println("Created new network.") } samples := neuralSamples(images) sgd.ShuffleSampleSet(samples) validationCount := int(ValidationFraction * float64(samples.Len())) validationSamples := samples.Subset(0, validationCount) trainingSamples := samples.Subset(validationCount, samples.Len()) costFunc := neuralnet.DotCost{} gradienter := &sgd.Adam{ Gradienter: &neuralnet.BatchRGradienter{ Learner: network.BatchLearner(), CostFunc: &neuralnet.RegularizingCost{ Variables: network.Parameters(), Penalty: Regularization, CostFunc: costFunc, }, }, } sgd.SGDInteractive(gradienter, trainingSamples, StepSize, BatchSize, func() bool { log.Printf("Costs: validation=%d/%d cost=%f", countCorrect(network, validationSamples), validationSamples.Len(), neuralnet.TotalCost(costFunc, network, trainingSamples)) return true }) data, _ := network.Serialize() if err := ioutil.WriteFile(netPath, data, 0755); err != nil { fmt.Fprintln(os.Stderr, "Failed to save:", err) os.Exit(1) } }
func createNetwork(samples sgd.SampleSet) *rnn.Bidirectional { means := make(linalg.Vector, FeatureCount) var count float64 for i := 0; i < samples.Len(); i++ { inputSeq := samples.GetSample(i).(ctc.Sample).Input for _, vec := range inputSeq { means.Add(vec) count++ } } means.Scale(-1 / count) stddevs := make(linalg.Vector, FeatureCount) for i := 0; i < samples.Len(); i++ { inputSeq := samples.GetSample(i).(ctc.Sample).Input for _, vec := range inputSeq { for j, v := range vec { stddevs[j] += math.Pow(v+means[j], 2) } } } stddevs.Scale(1 / count) for i, x := range stddevs { stddevs[i] = 1 / math.Sqrt(x) } outputNet := neuralnet.Network{ &neuralnet.DropoutLayer{ KeepProbability: HiddenDropout, Training: false, }, &neuralnet.DenseLayer{ InputCount: HiddenSize * 2, OutputCount: OutHiddenSize, }, &neuralnet.HyperbolicTangent{}, &neuralnet.DenseLayer{ InputCount: OutHiddenSize, OutputCount: len(cubewhisper.Labels) + 1, }, &neuralnet.LogSoftmaxLayer{}, } outputNet.Randomize() inputNet := neuralnet.Network{ &neuralnet.VecRescaleLayer{ Biases: means, Scales: stddevs, }, &neuralnet.GaussNoiseLayer{ Stddev: InputNoise, Training: false, }, } netBlock := rnn.NewNetworkBlock(inputNet, 0) forwardBlock := rnn.StackedBlock{ netBlock, rnn.NewGRU(FeatureCount, HiddenSize), } backwardBlock := rnn.StackedBlock{ netBlock, rnn.NewGRU(FeatureCount, HiddenSize), } for _, block := range []rnn.StackedBlock{forwardBlock, backwardBlock} { for i, param := range block.Parameters() { if i%2 == 0 { for i := range param.Vector { param.Vector[i] = rand.NormFloat64() * WeightStddev } } } } return &rnn.Bidirectional{ Forward: &rnn.BlockSeqFunc{Block: forwardBlock}, Backward: &rnn.BlockSeqFunc{Block: backwardBlock}, Output: &rnn.NetworkSeqFunc{Network: outputNet}, } }
// TestBaselineOutput makes sure that the BatcherBlock + // BlockSeqFunc combo produces the right output, since // that combo will be used for the rest of the tests. func TestBaselineOutput(t *testing.T) { network := neuralnet.Network{ &neuralnet.DenseLayer{ InputCount: 4, OutputCount: 6, }, neuralnet.HyperbolicTangent{}, } network.Randomize() for stateSize := 0; stateSize < 4; stateSize++ { start := &autofunc.Variable{Vector: make(linalg.Vector, stateSize)} for i := range start.Vector { start.Vector[i] = rand.NormFloat64() } toTest := rnn.BlockSeqFunc{ B: &rnn.BatcherBlock{ B: network.BatchLearner(), StateSize: stateSize, Start: start, }, } seqs, rv := randBaselineTestSeqs(network, 4-stateSize) rv[start] = make(linalg.Vector, len(start.Vector)) for i := range rv[start] { rv[start][i] = rand.NormFloat64() } res := toTest.ApplySeqsR(rv, seqfunc.VarRResult(rv, seqs)) actual := res.OutputSeqs() actualR := res.ROutputSeqs() expected, expectedR := manualNetworkSeq(rv, network, start, seqs, stateSize) if len(expected) != len(actual) { t.Errorf("stateSize %d: len(expected) [%d] != len(actual) [%d]", stateSize, len(expected), len(actual)) continue } for i, act := range actual { actR := actualR[i] exp := expected[i] expR := expectedR[i] if len(act) != len(exp) { t.Errorf("stateSize %d seq %d: len(act) [%d] != len(exp) [%d]", stateSize, i, len(act), len(act)) continue } for j, a := range act { x := exp[j] if len(a) != len(x) || x.Copy().Scale(-1).Add(a).MaxAbs() > 1e-5 { t.Errorf("stateSize %d seq %d entry %d: expected %v got %v", stateSize, i, j, x, a) } } for j, a := range actR { x := expR[j] if len(a) != len(x) || x.Copy().Scale(-1).Add(a).MaxAbs() > 1e-5 { t.Errorf("stateSize %d seq %d entry %d (R): expected %v got %v", stateSize, i, j, x, a) } } } } }
func Autoencode(images <-chan image.Image) (neuralnet.Network, error) { firstImage := <-images if firstImage == nil { return nil, errors.New("no readable images") } width := firstImage.Bounds().Dx() height := firstImage.Bounds().Dy() log.Print("Reading images...") tensors := []*neuralnet.Tensor3{ImageTensor(firstImage)} for img := range images { if img.Bounds().Dx() != width || img.Bounds().Dy() != height { log.Printf("Image size %d,%d does not match %d,%d", img.Bounds().Dx(), img.Bounds().Dy(), width, height) } else { tensors = append(tensors, ImageTensor(img)) } } log.Print("Training network (ctrl+c to finish)...") tensorSlices := make([]linalg.Vector, len(tensors)) for i, tensor := range tensors { tensorSlices[i] = tensor.Data } samples := neuralnet.VectorSampleSet(tensorSlices, tensorSlices) average, stddev := statisticalInfo(tensorSlices) network := neuralnet.Network{ &neuralnet.RescaleLayer{ Bias: -average, Scale: 1 / stddev, }, &neuralnet.DenseLayer{ InputCount: width * height * 3, OutputCount: HiddenSize1, }, neuralnet.Sigmoid{}, &neuralnet.DenseLayer{ InputCount: HiddenSize1, OutputCount: HiddenSize2, }, neuralnet.Sigmoid{}, &neuralnet.DenseLayer{ InputCount: HiddenSize2, OutputCount: HiddenSize1, }, neuralnet.Sigmoid{}, &neuralnet.DenseLayer{ InputCount: HiddenSize1, OutputCount: width * height * 3, }, } network.Randomize() ui := hessfree.NewConsoleUI() learner := &hessfree.DampingLearner{ WrappedLearner: &hessfree.NeuralNetLearner{ Layers: network, Output: nil, Cost: neuralnet.SigmoidCECost{}, MaxSubBatch: MaxSubBatch, MaxConcurrency: 2, }, DampingCoeff: 2, UI: ui, } trainer := hessfree.Trainer{ Learner: learner, Samples: samples, BatchSize: samples.Len(), UI: ui, Convergence: hessfree.ConvergenceCriteria{ MinK: 5, }, } trainer.Train() network = append(network, neuralnet.Sigmoid{}) return network, nil }