func main() {
rand.Seed(time.Now().UnixNano())
outNet := neuralnet.Network{
&neuralnet.DenseLayer{
InputCount: StateSize * 2,
OutputCount: 10,
},
&neuralnet.Sigmoid{},
&neuralnet.DenseLayer{
InputCount: 10,
OutputCount: 2,
},
&neuralnet.LogSoftmaxLayer{},
}
outNet.Randomize()
bd := &rnn.Bidirectional{
Forward: &rnn.BlockSeqFunc{B: rnn.NewGRU(2, StateSize)},
Backward: &rnn.BlockSeqFunc{B: rnn.NewGRU(2, StateSize)},
Output: &rnn.NetworkSeqFunc{Network: outNet},
}
var samples []seqtoseq.Sample
var sampleSet sgd.SliceSampleSet
for i := 0; i < TrainingSize; i++ {
samples = append(samples, generateSequence())
sampleSet = append(sampleSet, samples[i])
}
g := &sgd.RMSProp{
Gradienter: &seqtoseq.Gradienter{
SeqFunc: bd,
Learner: bd,
CostFunc: neuralnet.DotCost{},
},
}
var i int
sgd.SGDInteractive(g, sampleSet, StepSize, BatchSize, func() bool {
fmt.Printf("%d epochs: cost=%f\n", i, totalCost(bd, sampleSet))
i++
return true
})
var testingCorrect, testingTotal int
for j := 0; j < TestingSize; j++ {
sample := generateSequence()
inRes := seqfunc.ConstResult([][]linalg.Vector{sample.Inputs})
output := bd.ApplySeqs(inRes).OutputSeqs()[0]
for i, expected := range sample.Outputs {
actual := output[i]
if math.Abs(expected[0]-math.Exp(actual[0])) < 0.1 {
testingCorrect++
}
testingTotal++
}
}
fmt.Printf("Got %d/%d (%.2f%%)\n", testingCorrect, testingTotal,
100*float64(testingCorrect)/float64(testingTotal))
}
func printScore(prefix string, n neuralnet.Network, d mnist.DataSet) {
classifier := func(v []float64) int {
r := n.Apply(&autofunc.Variable{v})
return networkOutput(r)
}
correctCount := d.NumCorrect(classifier)
histogram := d.CorrectnessHistogram(classifier)
log.Printf("%s: %d/%d - %s", prefix, correctCount, len(d.Samples), histogram)
}
// NewNetworkBlock creates a NetworkBlock.
func NewNetworkBlock(n neuralnet.Network, stateSize int) *NetworkBlock {
return &NetworkBlock{
batcherBlock: &BatcherBlock{
B: n.BatchLearner(),
StateSize: stateSize,
Start: &autofunc.Variable{Vector: make(linalg.Vector, stateSize)},
},
network: n,
}
}
func TestStackedBlock(t *testing.T) {
testVars := []*autofunc.Variable{
{Vector: []float64{0.098591, -0.595453, -0.751214, 0.266051}},
{Vector: []float64{0.988517, 0.107284, -0.331529, 0.028565}},
{Vector: []float64{-0.150604, 0.889039, 0.120916, 0.240999}},
{Vector: []float64{0.961058, 0.878608, 0.052284, -0.635746}},
{Vector: []float64{0.31415, -0.2718}},
{Vector: []float64{-0.6}},
}
testSeqs := [][]*autofunc.Variable{
{testVars[0], testVars[2]},
{testVars[1]},
{testVars[2], testVars[1], testVars[3]},
}
testRV := autofunc.RVector{
testVars[0]: []float64{0.62524, 0.52979, 0.33020, 0.54462},
testVars[1]: []float64{0.13498, 0.12607, 0.35989, 0.23255},
testVars[2]: []float64{0.85996, 0.68435, 0.68506, 0.96907},
testVars[3]: []float64{0.79095, 0.33867, 0.86759, 0.16159},
testVars[4]: []float64{-0.79095, 0.33867},
testVars[5]: []float64{0.33867},
}
net1 := neuralnet.Network{
&neuralnet.DenseLayer{
InputCount: 6,
OutputCount: 6,
},
&neuralnet.HyperbolicTangent{},
}
net1.Randomize()
net2 := neuralnet.Network{
&neuralnet.DenseLayer{
InputCount: 5,
OutputCount: 5,
},
&neuralnet.HyperbolicTangent{},
}
net2.Randomize()
block := &rnn.StackedBlock{
&rnn.BatcherBlock{B: net1.BatchLearner(), StateSize: 2, Start: testVars[4]},
&rnn.BatcherBlock{B: net2.BatchLearner(), StateSize: 1, Start: testVars[5]},
}
checker := &BlockChecker{
B: block,
Input: testSeqs,
Vars: testVars,
RV: testRV,
}
checker.FullCheck(t)
}
// firstBitTest builds a neural network to:
// - output 0 for inputs starting with a 1
// - output 1 for inputs starting with a 0.
func firstBitTest() {
trainingSamples := make([]linalg.Vector, FirstBitTrainingSize)
trainingOutputs := make([]linalg.Vector, FirstBitTrainingSize)
for i := range trainingSamples {
trainingSamples[i] = make(linalg.Vector, FirstBitInputSize)
for j := range trainingSamples[i] {
trainingSamples[i][j] = float64(rand.Intn(2))
}
trainingOutputs[i] = []float64{1 - trainingSamples[i][0]}
}
samples := neuralnet.VectorSampleSet(trainingSamples, trainingOutputs)
network := neuralnet.Network{
&neuralnet.DenseLayer{
InputCount: FirstBitInputSize,
OutputCount: FirstBitHiddenSize,
},
&neuralnet.Sigmoid{},
&neuralnet.DenseLayer{
InputCount: FirstBitHiddenSize,
OutputCount: 1,
},
&neuralnet.Sigmoid{},
}
network.Randomize()
batcher := &neuralnet.SingleRGradienter{
Learner: network,
CostFunc: neuralnet.MeanSquaredCost{},
}
sgd.SGD(batcher, samples, 0.2, 100000, 1)
var totalError float64
var maxPossibleError float64
for i := 0; i < 50; i++ {
sample := make([]float64, FirstBitInputSize)
for j := range sample {
sample[j] = float64(rand.Intn(2))
}
result := network.Apply(&autofunc.Variable{sample})
output := result.Output()[0]
amountError := math.Abs(output - (1 - sample[0]))
totalError += amountError
maxPossibleError += 1.0
}
fmt.Printf("firstBitTest() error rate: %f\n", totalError/maxPossibleError)
}
func main() {
rand.Seed(time.Now().UnixNano())
sampleSet := sgd.SliceSampleSet{}
for i := 0; i < TrainingCount; i++ {
inSeq, outSeq := genEvenOddSeq(rand.Intn(MaxSeqLen-MinSeqLen) + MinSeqLen)
sampleSet = append(sampleSet, seqtoseq.Sample{
Inputs: inSeq,
Outputs: outSeq,
})
}
outNet := neuralnet.Network{
&neuralnet.DenseLayer{
InputCount: HiddenSize,
OutputCount: 2,
},
}
outNet.Randomize()
outBlock := rnn.NewNetworkBlock(outNet, 0)
lstm := rnn.NewLSTM(2, HiddenSize)
net := rnn.StackedBlock{lstm, outBlock}
gradienter := &sgd.RMSProp{
Gradienter: &seqtoseq.Gradienter{
SeqFunc: &rnn.BlockSeqFunc{B: net},
Learner: net,
CostFunc: neuralnet.SigmoidCECost{},
MaxLanes: 1,
},
}
sgd.SGD(gradienter, sampleSet, StepSize, Epochs, BatchSize)
outNet = append(outNet, neuralnet.Sigmoid{})
var scoreSum float64
var scoreTotal float64
for i := 0; i < TestingCount; i++ {
size := rand.Intn(MaxSeqLen-MinSeqLen) + MinSeqLen
ins, outs := genEvenOddSeq(size)
score := runTestSample(ins, outs, net)
scoreSum += score
scoreTotal += 1
}
fmt.Println("Testing success rate:", scoreSum/scoreTotal)
}
func createNet(d mnist.DataSet) neuralnet.Network {
convOutWidth := (d.Width-FilterSize)/FilterStride + 1
convOutHeight := (d.Height-FilterSize)/FilterStride + 1
poolOutWidth := convOutWidth / MaxPoolingSpan
if convOutWidth%MaxPoolingSpan != 0 {
poolOutWidth++
}
poolOutHeight := convOutWidth / MaxPoolingSpan
if convOutHeight%MaxPoolingSpan != 0 {
poolOutHeight++
}
net := neuralnet.Network{
&neuralnet.ConvLayer{
FilterCount: FilterCount,
FilterWidth: FilterSize,
FilterHeight: FilterSize,
Stride: FilterStride,
InputWidth: d.Width,
InputHeight: d.Height,
InputDepth: 1,
},
&neuralnet.Sigmoid{},
&neuralnet.MaxPoolingLayer{
XSpan: MaxPoolingSpan,
YSpan: MaxPoolingSpan,
InputWidth: convOutWidth,
InputHeight: convOutHeight,
InputDepth: FilterCount,
},
&neuralnet.DenseLayer{
InputCount: poolOutWidth * poolOutHeight * FilterCount,
OutputCount: HiddenSize,
},
&neuralnet.Sigmoid{},
&neuralnet.DenseLayer{
InputCount: HiddenSize,
OutputCount: LabelCount,
},
&neuralnet.SoftmaxLayer{},
}
net.Randomize()
return net
}
func countCorrect(n neuralnet.Network, s sgd.SampleSet) int {
var count int
for i := 0; i < s.Len(); i++ {
sample := s.GetSample(i).(neuralnet.VectorSample)
output := n.Apply(&autofunc.Variable{Vector: sample.Input}).Output()
var maxIdx int
var maxVal float64
for j, x := range output {
if x > maxVal || j == 0 {
maxIdx = j
maxVal = x
}
}
if sample.Output[maxIdx] == 1 {
count++
}
}
return count
}
func TestBaselineChecks(t *testing.T) {
network := neuralnet.Network{
&neuralnet.DenseLayer{
InputCount: 4,
OutputCount: 6,
},
neuralnet.HyperbolicTangent{},
}
network.Randomize()
for stateSize := 0; stateSize < 4; stateSize++ {
start := &autofunc.Variable{Vector: make(linalg.Vector, stateSize)}
for i := range start.Vector {
start.Vector[i] = rand.NormFloat64()
}
toTest := &rnn.BlockSeqFunc{
B: &rnn.BatcherBlock{
B: network.BatchLearner(),
StateSize: stateSize,
Start: start,
},
}
seqs, rv := randBaselineTestSeqs(network, 4-stateSize)
rv[start] = make(linalg.Vector, len(start.Vector))
for i := range rv[start] {
rv[start][i] = rand.NormFloat64()
}
vars := make([]*autofunc.Variable, 0, len(rv))
for v := range rv {
vars = append(vars, v)
}
checker := &functest.SeqRFuncChecker{
F: toTest,
Vars: vars,
Input: seqs,
RV: rv,
}
checker.FullCheck(t)
}
}
func trainClassifier(n neuralnet.Network, d mnist.DataSet) {
log.Println("Training classifier (ctrl+C to finish)...")
killChan := make(chan struct{})
go func() {
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
<-c
signal.Stop(c)
fmt.Println("\nCaught interrupt. Ctrl+C again to terminate.")
close(killChan)
}()
inputs := make([]linalg.Vector, len(d.Samples))
outputs := make([]linalg.Vector, len(d.Samples))
for i, x := range d.IntensityVectors() {
inputs[i] = x
}
for i, x := range d.LabelVectors() {
outputs[i] = x
}
samples := neuralnet.VectorSampleSet(inputs, outputs)
batcher := &neuralnet.BatchRGradienter{
Learner: n.BatchLearner(),
CostFunc: neuralnet.MeanSquaredCost{},
}
crossValidation := mnist.LoadTestingDataSet()
sgd.SGDInteractive(batcher, samples, ClassifierStepSize,
ClassifierBatchSize, func() bool {
printScore("Training", n, d)
printScore("Cross", n, crossValidation)
return true
})
}
func runHorizontalLineTest(name string, network neuralnet.Network) {
trainingSamples := make([]linalg.Vector, GridTrainingSize)
trainingOutputs := make([]linalg.Vector, GridTrainingSize)
for i := range trainingSamples {
trainingSamples[i] = randomBitmap()
if bitmapHasHorizontal(trainingSamples[i]) {
trainingOutputs[i] = []float64{1}
} else {
trainingOutputs[i] = []float64{0}
}
}
samples := neuralnet.VectorSampleSet(trainingSamples, trainingOutputs)
network.Randomize()
batcher := &neuralnet.SingleRGradienter{
Learner: network,
CostFunc: neuralnet.MeanSquaredCost{},
}
sgd.SGD(batcher, samples, 0.1, 1000, 100)
var trainingError float64
var maxTrainingError float64
for i, sample := range trainingSamples {
result := network.Apply(&autofunc.Variable{sample})
output := result.Output()[0]
amountError := math.Abs(output - trainingOutputs[i][0])
trainingError += amountError
maxTrainingError += 1.0
}
var totalError float64
var maxPossibleError float64
for i := 0; i < 50; i++ {
sample := randomBitmap()
var expected float64
if bitmapHasHorizontal(sample) {
expected = 1
}
result := network.Apply(&autofunc.Variable{sample})
output := result.Output()[0]
amountError := math.Abs(output - expected)
totalError += amountError
maxPossibleError += 1.0
}
fmt.Printf("%s() training error: %f; cross error: %f\n", name,
trainingError/maxTrainingError, totalError/maxPossibleError)
}
func TestStateOutBlock(t *testing.T) {
net := neuralnet.Network{
&neuralnet.DenseLayer{
InputCount: 8,
OutputCount: 4,
},
&neuralnet.HyperbolicTangent{},
}
net.Randomize()
startVar := &autofunc.Variable{Vector: []float64{0.3, -0.3, 0.2, 0.5}}
block := &rnn.StateOutBlock{
Block: &rnn.BatcherBlock{
B: net.BatchLearner(),
StateSize: 4,
Start: startVar,
},
}
learner := append(stateOutBlockLearner{startVar}, net.Parameters()...)
NewChecker4In(block, learner).FullCheck(t)
}
func TrainCmd(netPath, dirPath string) {
log.Println("Loading samples...")
images, width, height, err := LoadTrainingImages(dirPath)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
log.Println("Creating network...")
var network neuralnet.Network
networkData, err := ioutil.ReadFile(netPath)
if err == nil {
network, err = neuralnet.DeserializeNetwork(networkData)
if err != nil {
fmt.Fprintln(os.Stderr, "Failed to load network:", err)
os.Exit(1)
}
log.Println("Loaded network from file.")
} else {
mean, stddev := sampleStatistics(images)
convLayer := &neuralnet.ConvLayer{
FilterCount: FilterCount,
FilterWidth: 4,
FilterHeight: 4,
Stride: 2,
InputWidth: width,
InputHeight: height,
InputDepth: ImageDepth,
}
maxLayer := &neuralnet.MaxPoolingLayer{
XSpan: 3,
YSpan: 3,
InputWidth: convLayer.OutputWidth(),
InputHeight: convLayer.OutputHeight(),
InputDepth: convLayer.OutputDepth(),
}
convLayer1 := &neuralnet.ConvLayer{
FilterCount: FilterCount1,
FilterWidth: 3,
FilterHeight: 3,
Stride: 2,
InputWidth: maxLayer.OutputWidth(),
InputHeight: maxLayer.OutputHeight(),
InputDepth: maxLayer.InputDepth,
}
network = neuralnet.Network{
&neuralnet.RescaleLayer{
Bias: -mean,
Scale: 1 / stddev,
},
convLayer,
neuralnet.HyperbolicTangent{},
maxLayer,
neuralnet.HyperbolicTangent{},
convLayer1,
neuralnet.HyperbolicTangent{},
&neuralnet.DenseLayer{
InputCount: convLayer1.OutputWidth() * convLayer1.OutputHeight() *
convLayer1.OutputDepth(),
OutputCount: HiddenSize,
},
neuralnet.HyperbolicTangent{},
&neuralnet.DenseLayer{
InputCount: HiddenSize,
OutputCount: len(images),
},
&neuralnet.LogSoftmaxLayer{},
}
network.Randomize()
log.Println("Created new network.")
}
samples := neuralSamples(images)
sgd.ShuffleSampleSet(samples)
validationCount := int(ValidationFraction * float64(samples.Len()))
validationSamples := samples.Subset(0, validationCount)
trainingSamples := samples.Subset(validationCount, samples.Len())
costFunc := neuralnet.DotCost{}
gradienter := &sgd.Adam{
Gradienter: &neuralnet.BatchRGradienter{
Learner: network.BatchLearner(),
CostFunc: &neuralnet.RegularizingCost{
Variables: network.Parameters(),
Penalty: Regularization,
CostFunc: costFunc,
},
},
}
sgd.SGDInteractive(gradienter, trainingSamples, StepSize, BatchSize, func() bool {
log.Printf("Costs: validation=%d/%d cost=%f",
countCorrect(network, validationSamples), validationSamples.Len(),
neuralnet.TotalCost(costFunc, network, trainingSamples))
return true
})
data, _ := network.Serialize()
if err := ioutil.WriteFile(netPath, data, 0755); err != nil {
fmt.Fprintln(os.Stderr, "Failed to save:", err)
os.Exit(1)
}
}
func createNetwork(samples sgd.SampleSet) *rnn.Bidirectional {
means := make(linalg.Vector, FeatureCount)
var count float64
for i := 0; i < samples.Len(); i++ {
inputSeq := samples.GetSample(i).(ctc.Sample).Input
for _, vec := range inputSeq {
means.Add(vec)
count++
}
}
means.Scale(-1 / count)
stddevs := make(linalg.Vector, FeatureCount)
for i := 0; i < samples.Len(); i++ {
inputSeq := samples.GetSample(i).(ctc.Sample).Input
for _, vec := range inputSeq {
for j, v := range vec {
stddevs[j] += math.Pow(v+means[j], 2)
}
}
}
stddevs.Scale(1 / count)
for i, x := range stddevs {
stddevs[i] = 1 / math.Sqrt(x)
}
outputNet := neuralnet.Network{
&neuralnet.DropoutLayer{
KeepProbability: HiddenDropout,
Training: false,
},
&neuralnet.DenseLayer{
InputCount: HiddenSize * 2,
OutputCount: OutHiddenSize,
},
&neuralnet.HyperbolicTangent{},
&neuralnet.DenseLayer{
InputCount: OutHiddenSize,
OutputCount: len(cubewhisper.Labels) + 1,
},
&neuralnet.LogSoftmaxLayer{},
}
outputNet.Randomize()
inputNet := neuralnet.Network{
&neuralnet.VecRescaleLayer{
Biases: means,
Scales: stddevs,
},
&neuralnet.GaussNoiseLayer{
Stddev: InputNoise,
Training: false,
},
}
netBlock := rnn.NewNetworkBlock(inputNet, 0)
forwardBlock := rnn.StackedBlock{
netBlock,
rnn.NewGRU(FeatureCount, HiddenSize),
}
backwardBlock := rnn.StackedBlock{
netBlock,
rnn.NewGRU(FeatureCount, HiddenSize),
}
for _, block := range []rnn.StackedBlock{forwardBlock, backwardBlock} {
for i, param := range block.Parameters() {
if i%2 == 0 {
for i := range param.Vector {
param.Vector[i] = rand.NormFloat64() * WeightStddev
}
}
}
}
return &rnn.Bidirectional{
Forward: &rnn.BlockSeqFunc{Block: forwardBlock},
Backward: &rnn.BlockSeqFunc{Block: backwardBlock},
Output: &rnn.NetworkSeqFunc{Network: outputNet},
}
}
// TestBaselineOutput makes sure that the BatcherBlock +
// BlockSeqFunc combo produces the right output, since
// that combo will be used for the rest of the tests.
func TestBaselineOutput(t *testing.T) {
network := neuralnet.Network{
&neuralnet.DenseLayer{
InputCount: 4,
OutputCount: 6,
},
neuralnet.HyperbolicTangent{},
}
network.Randomize()
for stateSize := 0; stateSize < 4; stateSize++ {
start := &autofunc.Variable{Vector: make(linalg.Vector, stateSize)}
for i := range start.Vector {
start.Vector[i] = rand.NormFloat64()
}
toTest := rnn.BlockSeqFunc{
B: &rnn.BatcherBlock{
B: network.BatchLearner(),
StateSize: stateSize,
Start: start,
},
}
seqs, rv := randBaselineTestSeqs(network, 4-stateSize)
rv[start] = make(linalg.Vector, len(start.Vector))
for i := range rv[start] {
rv[start][i] = rand.NormFloat64()
}
res := toTest.ApplySeqsR(rv, seqfunc.VarRResult(rv, seqs))
actual := res.OutputSeqs()
actualR := res.ROutputSeqs()
expected, expectedR := manualNetworkSeq(rv, network, start, seqs, stateSize)
if len(expected) != len(actual) {
t.Errorf("stateSize %d: len(expected) [%d] != len(actual) [%d]", stateSize,
len(expected), len(actual))
continue
}
for i, act := range actual {
actR := actualR[i]
exp := expected[i]
expR := expectedR[i]
if len(act) != len(exp) {
t.Errorf("stateSize %d seq %d: len(act) [%d] != len(exp) [%d]",
stateSize, i, len(act), len(act))
continue
}
for j, a := range act {
x := exp[j]
if len(a) != len(x) || x.Copy().Scale(-1).Add(a).MaxAbs() > 1e-5 {
t.Errorf("stateSize %d seq %d entry %d: expected %v got %v",
stateSize, i, j, x, a)
}
}
for j, a := range actR {
x := expR[j]
if len(a) != len(x) || x.Copy().Scale(-1).Add(a).MaxAbs() > 1e-5 {
t.Errorf("stateSize %d seq %d entry %d (R): expected %v got %v",
stateSize, i, j, x, a)
}
}
}
}
}
func Autoencode(images <-chan image.Image) (neuralnet.Network, error) {
firstImage := <-images
if firstImage == nil {
return nil, errors.New("no readable images")
}
width := firstImage.Bounds().Dx()
height := firstImage.Bounds().Dy()
log.Print("Reading images...")
tensors := []*neuralnet.Tensor3{ImageTensor(firstImage)}
for img := range images {
if img.Bounds().Dx() != width || img.Bounds().Dy() != height {
log.Printf("Image size %d,%d does not match %d,%d",
img.Bounds().Dx(), img.Bounds().Dy(),
width, height)
} else {
tensors = append(tensors, ImageTensor(img))
}
}
log.Print("Training network (ctrl+c to finish)...")
tensorSlices := make([]linalg.Vector, len(tensors))
for i, tensor := range tensors {
tensorSlices[i] = tensor.Data
}
samples := neuralnet.VectorSampleSet(tensorSlices, tensorSlices)
average, stddev := statisticalInfo(tensorSlices)
network := neuralnet.Network{
&neuralnet.RescaleLayer{
Bias: -average,
Scale: 1 / stddev,
},
&neuralnet.DenseLayer{
InputCount: width * height * 3,
OutputCount: HiddenSize1,
},
neuralnet.Sigmoid{},
&neuralnet.DenseLayer{
InputCount: HiddenSize1,
OutputCount: HiddenSize2,
},
neuralnet.Sigmoid{},
&neuralnet.DenseLayer{
InputCount: HiddenSize2,
OutputCount: HiddenSize1,
},
neuralnet.Sigmoid{},
&neuralnet.DenseLayer{
InputCount: HiddenSize1,
OutputCount: width * height * 3,
},
}
network.Randomize()
ui := hessfree.NewConsoleUI()
learner := &hessfree.DampingLearner{
WrappedLearner: &hessfree.NeuralNetLearner{
Layers: network,
Output: nil,
Cost: neuralnet.SigmoidCECost{},
MaxSubBatch: MaxSubBatch,
MaxConcurrency: 2,
},
DampingCoeff: 2,
UI: ui,
}
trainer := hessfree.Trainer{
Learner: learner,
Samples: samples,
BatchSize: samples.Len(),
UI: ui,
Convergence: hessfree.ConvergenceCriteria{
MinK: 5,
},
}
trainer.Train()
network = append(network, neuralnet.Sigmoid{})
return network, nil
}