func main() { runtime.GOMAXPROCS(runtime.NumCPU() - 2) gopath := os.Getenv("GOPATH") path := filepath.Join(gopath, "prof", "github.com", "reggo", "reggo", "nnet") nInputs := 10 nOutputs := 3 nLayers := 2 nNeurons := 50 nSamples := 1000000 nRuns := 50 config := &profile.Config{ CPUProfile: true, ProfilePath: path, } defer profile.Start(config).Stop() net, err := nnet.NewSimpleTrainer(nInputs, nOutputs, nLayers, nNeurons, nnet.Linear{}) if err != nil { log.Fatal(err) } // Generate some random data inputs := mat64.NewDense(nSamples, nInputs, nil) outputs := mat64.NewDense(nSamples, nOutputs, nil) for i := 0; i < nSamples; i++ { for j := 0; j < nInputs; j++ { inputs.Set(i, j, rand.Float64()) } for j := 0; j < nOutputs; j++ { outputs.Set(i, j, rand.Float64()) } } // Create trainer prob := train.NewBatchGradBased(net, true, inputs, outputs, nil, nil, nil) nParameters := net.NumParameters() parameters := make([]float64, nParameters) derivative := make([]float64, nParameters) for i := 0; i < nRuns; i++ { net.RandomizeParameters() net.Parameters(parameters) prob.ObjGrad(parameters, derivative) fmt.Println(floats.Sum(derivative)) } }
// TestDeriv uses finite difference to test that the prediction from Deriv // is correct, and tests that computing the loss in parallel works properly // Only does finite difference for the first nTest to save time func TestDeriv(t *testing.T, trainable DerivTester, inputs, trueOutputs common.RowMatrix, name string) { // Set the parameters to something random trainable.RandomizeParameters() // Compute the loss and derivative losser := loss.SquaredDistance{} regularizer := regularize.TwoNorm{} batchGrad := train.NewBatchGradBased(trainable, true, inputs, trueOutputs, nil, losser, regularizer) derivative := make([]float64, trainable.NumParameters()) parameters := trainable.Parameters(nil) // Don't need to check loss, because if predict is right and losser is right then loss must be correct _ = batchGrad.ObjGrad(parameters, derivative) fdDerivative := make([]float64, trainable.NumParameters()) wg := &sync.WaitGroup{} wg.Add(trainable.NumParameters()) for i := 0; i < trainable.NumParameters(); i++ { go func(i int) { newParameters := make([]float64, trainable.NumParameters()) tmpDerivative := make([]float64, trainable.NumParameters()) copy(newParameters, parameters) newParameters[i] += fdStep loss1 := batchGrad.ObjGrad(newParameters, tmpDerivative) newParameters[i] -= 2 * fdStep loss2 := batchGrad.ObjGrad(newParameters, tmpDerivative) newParameters[i] += fdStep fdDerivative[i] = (loss1 - loss2) / (2 * fdStep) wg.Done() }(i) } wg.Wait() if !floats.EqualApprox(derivative, fdDerivative, 1e-6) { t.Errorf("%v: deriv doesn't match: Finite Difference: %v, Analytic: %v", name, fdDerivative, derivative) } }
// TestLinearsolveAndDeriv compares the optimal weights found from gradient-based optimization with those found // from computing a linear solve func TestLinearsolveAndDeriv(t *testing.T, linear train.LinearTrainable, inputs, trueOutputs common.RowMatrix, name string) { // Compare with no weights rows, cols := trueOutputs.Dims() predOutLinear := mat64.NewDense(rows, cols, nil) parametersLinearSolve := train.LinearSolve(linear, nil, inputs, trueOutputs, nil, nil) linear.SetParameters(parametersLinearSolve) linear.Predictor().PredictBatch(inputs, predOutLinear) //fmt.Println("Pred out linear", predOutLinear) linear.RandomizeParameters() parameters := linear.Parameters(nil) batch := train.NewBatchGradBased(linear, true, inputs, trueOutputs, nil, loss.SquaredDistance{}, regularize.None{}) problem := batch settings := multivariate.DefaultSettings() settings.GradAbsTol = 1e-11 //settings. = 0 result, err := multivariate.OptimizeGrad(problem, parameters, settings, nil) if err != nil { t.Errorf("Error training: %v", err) } parametersDeriv := result.Loc deriv := make([]float64, linear.NumParameters()) loss1 := batch.ObjGrad(parametersDeriv, deriv) linear.SetParameters(parametersDeriv) predOutDeriv := mat64.NewDense(rows, cols, nil) linear.Predictor().PredictBatch(inputs, predOutDeriv) linear.RandomizeParameters() init2 := linear.Parameters(nil) batch2 := train.NewBatchGradBased(linear, true, inputs, trueOutputs, nil, loss.SquaredDistance{}, regularize.None{}) problem2 := batch2 result2, err := multivariate.OptimizeGrad(problem2, init2, settings, nil) parametersDeriv2 := result2.Loc //fmt.Println("starting deriv2 loss") deriv2 := make([]float64, linear.NumParameters()) loss2 := batch2.ObjGrad(parametersDeriv2, deriv2) //fmt.Println("starting derivlin loss") derivlinear := make([]float64, linear.NumParameters()) lossLin := batch2.ObjGrad(parametersLinearSolve, derivlinear) _ = loss1 _ = loss2 _ = lossLin /* fmt.Println("param deriv 1 =", parametersDeriv) fmt.Println("param deriv2 =", parametersDeriv2) fmt.Println("linear params =", parametersLinearSolve) fmt.Println("deriv1 loss =", loss1) fmt.Println("deriv2 loss =", loss2) fmt.Println("lin loss =", lossLin) fmt.Println("deriv =", deriv) fmt.Println("deriv2 =", deriv2) fmt.Println("linderiv =", derivlinear) //fmt.Println("Pred out deriv", predOutDeriv) */ /* for i := 0; i < rows; i++ { fmt.Println(predOutLinear.RowView(i), predOutBatch.RowView(i)) } */ if !floats.EqualApprox(parametersLinearSolve, parametersDeriv, 1e-8) { t.Errorf("Parameters don't match for gradient based and linear solve.") //for i := range parametersDeriv { // fmt.Printf("index %v: Deriv = %v, linsolve = %v, diff = %v\n", i, parametersDeriv[i], parametersLinearSolve[i], parametersDeriv[i]-parametersLinearSolve[i]) //} } }