func testTrainingXOR(t *testing.T, maxBatch, maxGos, batchSize int, single bool) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
net := Network{
&DenseLayer{
InputCount: 2,
OutputCount: 4,
},
&Sigmoid{},
&DenseLayer{
InputCount: 4,
OutputCount: 1,
},
&Sigmoid{},
}
rand.Seed(123123)
net.Randomize()
samples := VectorSampleSet([]linalg.Vector{
{0, 0},
{0, 1},
{1, 0},
{1, 1},
}, []linalg.Vector{{0}, {1}, {1}, {0}})
var gradienter sgd.Gradienter
if single {
gradienter = &SingleRGradienter{
Learner: net,
CostFunc: MeanSquaredCost{},
}
} else {
gradienter = &BatchRGradienter{
Learner: net.BatchLearner(),
CostFunc: MeanSquaredCost{},
MaxGoroutines: maxGos,
MaxBatchSize: maxBatch,
}
}
sgd.SGD(gradienter, samples, 0.9, 1000, batchSize)
for i := 0; i < samples.Len(); i++ {
sample := samples.GetSample(i)
vs := sample.(VectorSample)
output := net.Apply(&autofunc.Variable{vs.Input}).Output()
expected := vs.Output[0]
actual := output[0]
if math.Abs(expected-actual) > 0.08 {
t.Errorf("expected %f for input %v but got %f", expected, sample, actual)
}
}
}
// firstBitTest builds a neural network to:
// - output 0 for inputs starting with a 1
// - output 1 for inputs starting with a 0.
func firstBitTest() {
trainingSamples := make([]linalg.Vector, FirstBitTrainingSize)
trainingOutputs := make([]linalg.Vector, FirstBitTrainingSize)
for i := range trainingSamples {
trainingSamples[i] = make(linalg.Vector, FirstBitInputSize)
for j := range trainingSamples[i] {
trainingSamples[i][j] = float64(rand.Intn(2))
}
trainingOutputs[i] = []float64{1 - trainingSamples[i][0]}
}
samples := neuralnet.VectorSampleSet(trainingSamples, trainingOutputs)
network := neuralnet.Network{
&neuralnet.DenseLayer{
InputCount: FirstBitInputSize,
OutputCount: FirstBitHiddenSize,
},
&neuralnet.Sigmoid{},
&neuralnet.DenseLayer{
InputCount: FirstBitHiddenSize,
OutputCount: 1,
},
&neuralnet.Sigmoid{},
}
network.Randomize()
batcher := &neuralnet.SingleRGradienter{
Learner: network,
CostFunc: neuralnet.MeanSquaredCost{},
}
sgd.SGD(batcher, samples, 0.2, 100000, 1)
var totalError float64
var maxPossibleError float64
for i := 0; i < 50; i++ {
sample := make([]float64, FirstBitInputSize)
for j := range sample {
sample[j] = float64(rand.Intn(2))
}
result := network.Apply(&autofunc.Variable{sample})
output := result.Output()[0]
amountError := math.Abs(output - (1 - sample[0]))
totalError += amountError
maxPossibleError += 1.0
}
fmt.Printf("firstBitTest() error rate: %f\n", totalError/maxPossibleError)
}
func runHorizontalLineTest(name string, network neuralnet.Network) {
trainingSamples := make([]linalg.Vector, GridTrainingSize)
trainingOutputs := make([]linalg.Vector, GridTrainingSize)
for i := range trainingSamples {
trainingSamples[i] = randomBitmap()
if bitmapHasHorizontal(trainingSamples[i]) {
trainingOutputs[i] = []float64{1}
} else {
trainingOutputs[i] = []float64{0}
}
}
samples := neuralnet.VectorSampleSet(trainingSamples, trainingOutputs)
network.Randomize()
batcher := &neuralnet.SingleRGradienter{
Learner: network,
CostFunc: neuralnet.MeanSquaredCost{},
}
sgd.SGD(batcher, samples, 0.1, 1000, 100)
var trainingError float64
var maxTrainingError float64
for i, sample := range trainingSamples {
result := network.Apply(&autofunc.Variable{sample})
output := result.Output()[0]
amountError := math.Abs(output - trainingOutputs[i][0])
trainingError += amountError
maxTrainingError += 1.0
}
var totalError float64
var maxPossibleError float64
for i := 0; i < 50; i++ {
sample := randomBitmap()
var expected float64
if bitmapHasHorizontal(sample) {
expected = 1
}
result := network.Apply(&autofunc.Variable{sample})
output := result.Output()[0]
amountError := math.Abs(output - expected)
totalError += amountError
maxPossibleError += 1.0
}
fmt.Printf("%s() training error: %f; cross error: %f\n", name,
trainingError/maxTrainingError, totalError/maxPossibleError)
}
func main() {
rand.Seed(time.Now().UnixNano())
sampleSet := sgd.SliceSampleSet{}
for i := 0; i < TrainingCount; i++ {
inSeq, outSeq := genEvenOddSeq(rand.Intn(MaxSeqLen-MinSeqLen) + MinSeqLen)
sampleSet = append(sampleSet, seqtoseq.Sample{
Inputs: inSeq,
Outputs: outSeq,
})
}
outNet := neuralnet.Network{
&neuralnet.DenseLayer{
InputCount: HiddenSize,
OutputCount: 2,
},
}
outNet.Randomize()
outBlock := rnn.NewNetworkBlock(outNet, 0)
lstm := rnn.NewLSTM(2, HiddenSize)
net := rnn.StackedBlock{lstm, outBlock}
gradienter := &sgd.RMSProp{
Gradienter: &seqtoseq.Gradienter{
SeqFunc: &rnn.BlockSeqFunc{B: net},
Learner: net,
CostFunc: neuralnet.SigmoidCECost{},
MaxLanes: 1,
},
}
sgd.SGD(gradienter, sampleSet, StepSize, Epochs, BatchSize)
outNet = append(outNet, neuralnet.Sigmoid{})
var scoreSum float64
var scoreTotal float64
for i := 0; i < TestingCount; i++ {
size := rand.Intn(MaxSeqLen-MinSeqLen) + MinSeqLen
ins, outs := genEvenOddSeq(size)
score := runTestSample(ins, outs, net)
scoreSum += score
scoreTotal += 1
}
fmt.Println("Testing success rate:", scoreSum/scoreTotal)
}
func benchmarkTrainingBig(b *testing.B, hiddenSize, batchSize int) {
runtime.GC()
inputs := make([]linalg.Vector, 100)
outputs := make([]linalg.Vector, len(inputs))
for i := range inputs {
inputs[i] = make(linalg.Vector, 1000)
outputs[i] = make(linalg.Vector, len(inputs[i]))
for j := range inputs[i] {
inputs[i][j] = rand.Float64()
outputs[i][j] = rand.Float64()
}
}
samples := VectorSampleSet(inputs, outputs)
network := Network{
&DenseLayer{
InputCount: len(inputs[0]),
OutputCount: hiddenSize,
},
&Sigmoid{},
&DenseLayer{
InputCount: hiddenSize,
OutputCount: 10,
},
&Sigmoid{},
}
network.Randomize()
batcher := &BatchRGradienter{
Learner: network.BatchLearner(),
CostFunc: MeanSquaredCost{},
}
b.ResetTimer()
sgd.SGD(batcher, samples, 0.01, b.N, batchSize)
}