Example #1
0
func main() {
	rand.Seed(time.Now().UnixNano())

	outNet := neuralnet.Network{
		&neuralnet.DenseLayer{
			InputCount:  StateSize * 2,
			OutputCount: 10,
		},
		&neuralnet.Sigmoid{},
		&neuralnet.DenseLayer{
			InputCount:  10,
			OutputCount: 2,
		},
		&neuralnet.LogSoftmaxLayer{},
	}
	outNet.Randomize()
	bd := &rnn.Bidirectional{
		Forward:  &rnn.BlockSeqFunc{B: rnn.NewGRU(2, StateSize)},
		Backward: &rnn.BlockSeqFunc{B: rnn.NewGRU(2, StateSize)},
		Output:   &rnn.NetworkSeqFunc{Network: outNet},
	}
	var samples []seqtoseq.Sample
	var sampleSet sgd.SliceSampleSet
	for i := 0; i < TrainingSize; i++ {
		samples = append(samples, generateSequence())
		sampleSet = append(sampleSet, samples[i])
	}

	g := &sgd.RMSProp{
		Gradienter: &seqtoseq.Gradienter{
			SeqFunc:  bd,
			Learner:  bd,
			CostFunc: neuralnet.DotCost{},
		},
	}

	var i int
	sgd.SGDInteractive(g, sampleSet, StepSize, BatchSize, func() bool {
		fmt.Printf("%d epochs: cost=%f\n", i, totalCost(bd, sampleSet))
		i++
		return true
	})

	var testingCorrect, testingTotal int
	for j := 0; j < TestingSize; j++ {
		sample := generateSequence()
		inRes := seqfunc.ConstResult([][]linalg.Vector{sample.Inputs})
		output := bd.ApplySeqs(inRes).OutputSeqs()[0]
		for i, expected := range sample.Outputs {
			actual := output[i]
			if math.Abs(expected[0]-math.Exp(actual[0])) < 0.1 {
				testingCorrect++
			}
			testingTotal++
		}
	}

	fmt.Printf("Got %d/%d (%.2f%%)\n", testingCorrect, testingTotal,
		100*float64(testingCorrect)/float64(testingTotal))
}
Example #2
0
func main() {
	training := mnist.LoadTrainingDataSet()
	crossValidation := mnist.LoadTestingDataSet()

	net := createNet(training)

	trainingSamples := dataSetSamples(training)
	gradienter := &neuralnet.BatchRGradienter{
		Learner:  net.BatchLearner(),
		CostFunc: neuralnet.MeanSquaredCost{},
	}
	rmsGrad := &sgd.RMSProp{Gradienter: gradienter}

	sgd.SGDInteractive(rmsGrad, trainingSamples, StepSize, BatchSize, func() bool {
		log.Println("Printing score...")
		printScore("Cross", net, crossValidation)
		log.Println("Running training round...")
		return true
	})
}
Example #3
0
func trainClassifier(n neuralnet.Network, d mnist.DataSet) {
	log.Println("Training classifier (ctrl+C to finish)...")

	killChan := make(chan struct{})

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		<-c
		signal.Stop(c)
		fmt.Println("\nCaught interrupt. Ctrl+C again to terminate.")
		close(killChan)
	}()

	inputs := make([]linalg.Vector, len(d.Samples))
	outputs := make([]linalg.Vector, len(d.Samples))
	for i, x := range d.IntensityVectors() {
		inputs[i] = x
	}
	for i, x := range d.LabelVectors() {
		outputs[i] = x
	}
	samples := neuralnet.VectorSampleSet(inputs, outputs)
	batcher := &neuralnet.BatchRGradienter{
		Learner:  n.BatchLearner(),
		CostFunc: neuralnet.MeanSquaredCost{},
	}

	crossValidation := mnist.LoadTestingDataSet()

	sgd.SGDInteractive(batcher, samples, ClassifierStepSize,
		ClassifierBatchSize, func() bool {
			printScore("Training", n, d)
			printScore("Cross", n, crossValidation)
			return true
		})
}
Example #4
0
func TrainCmd(netPath, dirPath string) {
	log.Println("Loading samples...")
	images, width, height, err := LoadTrainingImages(dirPath)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	log.Println("Creating network...")

	var network neuralnet.Network
	networkData, err := ioutil.ReadFile(netPath)
	if err == nil {
		network, err = neuralnet.DeserializeNetwork(networkData)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Failed to load network:", err)
			os.Exit(1)
		}
		log.Println("Loaded network from file.")
	} else {
		mean, stddev := sampleStatistics(images)
		convLayer := &neuralnet.ConvLayer{
			FilterCount:  FilterCount,
			FilterWidth:  4,
			FilterHeight: 4,
			Stride:       2,

			InputWidth:  width,
			InputHeight: height,
			InputDepth:  ImageDepth,
		}
		maxLayer := &neuralnet.MaxPoolingLayer{
			XSpan:       3,
			YSpan:       3,
			InputWidth:  convLayer.OutputWidth(),
			InputHeight: convLayer.OutputHeight(),
			InputDepth:  convLayer.OutputDepth(),
		}
		convLayer1 := &neuralnet.ConvLayer{
			FilterCount:  FilterCount1,
			FilterWidth:  3,
			FilterHeight: 3,
			Stride:       2,

			InputWidth:  maxLayer.OutputWidth(),
			InputHeight: maxLayer.OutputHeight(),
			InputDepth:  maxLayer.InputDepth,
		}
		network = neuralnet.Network{
			&neuralnet.RescaleLayer{
				Bias:  -mean,
				Scale: 1 / stddev,
			},
			convLayer,
			neuralnet.HyperbolicTangent{},
			maxLayer,
			neuralnet.HyperbolicTangent{},
			convLayer1,
			neuralnet.HyperbolicTangent{},
			&neuralnet.DenseLayer{
				InputCount: convLayer1.OutputWidth() * convLayer1.OutputHeight() *
					convLayer1.OutputDepth(),
				OutputCount: HiddenSize,
			},
			neuralnet.HyperbolicTangent{},
			&neuralnet.DenseLayer{
				InputCount:  HiddenSize,
				OutputCount: len(images),
			},
			&neuralnet.LogSoftmaxLayer{},
		}
		network.Randomize()
		log.Println("Created new network.")
	}

	samples := neuralSamples(images)
	sgd.ShuffleSampleSet(samples)

	validationCount := int(ValidationFraction * float64(samples.Len()))
	validationSamples := samples.Subset(0, validationCount)
	trainingSamples := samples.Subset(validationCount, samples.Len())

	costFunc := neuralnet.DotCost{}
	gradienter := &sgd.Adam{
		Gradienter: &neuralnet.BatchRGradienter{
			Learner: network.BatchLearner(),
			CostFunc: &neuralnet.RegularizingCost{
				Variables: network.Parameters(),
				Penalty:   Regularization,
				CostFunc:  costFunc,
			},
		},
	}
	sgd.SGDInteractive(gradienter, trainingSamples, StepSize, BatchSize, func() bool {
		log.Printf("Costs: validation=%d/%d cost=%f",
			countCorrect(network, validationSamples), validationSamples.Len(),
			neuralnet.TotalCost(costFunc, network, trainingSamples))
		return true
	})

	data, _ := network.Serialize()
	if err := ioutil.WriteFile(netPath, data, 0755); err != nil {
		fmt.Fprintln(os.Stderr, "Failed to save:", err)
		os.Exit(1)
	}
}
Example #5
0
func Train(rnnFile, sampleDir string, stepSize float64) {
	log.Println("Loading samples...")
	samples, err := ReadSamples(sampleDir)
	if err != nil {
		fmt.Fprintln(os.Stderr, "Failed to read samples:", err)
		os.Exit(1)
	}

	var seqFunc *rnn.Bidirectional
	rnnData, err := ioutil.ReadFile(rnnFile)
	if err == nil {
		log.Println("Loaded network from file.")
		seqFunc, err = rnn.DeserializeBidirectional(rnnData)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Failed to deserialize network:", err)
			os.Exit(1)
		}
	} else {
		log.Println("Created network.")
		seqFunc = createNetwork(samples)
	}

	crossLen := int(CrossRatio * float64(samples.Len()))
	log.Println("Using", samples.Len()-crossLen, "training and",
		crossLen, "validation samples...")

	// Always shuffle the samples in the same way.
	rand.Seed(123)
	sgd.ShuffleSampleSet(samples)
	validation := samples.Subset(0, crossLen)
	training := samples.Subset(crossLen, samples.Len())

	gradienter := &sgd.Adam{
		Gradienter: &ctc.RGradienter{
			Learner:        seqFunc,
			SeqFunc:        seqFunc,
			MaxConcurrency: MaxConcurrency,
			MaxSubBatch:    MaxSubBatch,
		},
	}

	var epoch int
	toggleRegularization(seqFunc, true)
	sgd.SGDInteractive(gradienter, training, stepSize, BatchSize, func() bool {
		toggleRegularization(seqFunc, false)
		cost := ctc.TotalCost(seqFunc, training, CostBatchSize, MaxConcurrency)
		crossCost := ctc.TotalCost(seqFunc, validation, CostBatchSize, MaxConcurrency)
		toggleRegularization(seqFunc, true)
		log.Printf("Epoch %d: cost=%e cross=%e", epoch, cost, crossCost)
		epoch++
		return true
	})
	toggleRegularization(seqFunc, false)

	data, err := seqFunc.Serialize()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Failed to serialize:", err)
		os.Exit(1)
	}

	if err := ioutil.WriteFile(rnnFile, data, 0755); err != nil {
		fmt.Fprintln(os.Stderr, "Failed to save:", err)
		os.Exit(1)
	}
}