// LinearSolve trains a Linear algorithm. // Assumes inputs and outputs are already scaled // If features is nil will call featurize // Will return nil if regularizer is not a linear regularizer // Is destructive if any of the weights are zero // Losser is always the two-norm // Does not set the value of the parameters (in case this is called in parallel with a different routine) func LinearSolve(linearTrainable LinearTrainable, features *mat64.Dense, inputs, trueOutputs common.RowMatrix, weights []float64, regularizer regularize.Regularizer) (parameters []float64) { // TODO: Allow tikhonov regularization // TODO: Add test for weights // TODO: Need to do something about returning a []float64 if !IsLinearSolveRegularizer(regularizer) { return nil } if features == nil { features = FeaturizeTrainable(linearTrainable, inputs, features) } _, nFeatures := features.Dims() var weightedFeatures, weightedOutput *mat64.Dense if weights != nil { scaledWeight := make([]float64, len(weights)) for i, weight := range weights { scaledWeight[i] = math.Sqrt(weight) } diagWeight := diagonal.NewDiagonal(nFeatures, weights) nSamples, outputDim := trueOutputs.Dims() weightedOutput = mat64.NewDense(nSamples, outputDim, nil) weightedFeatures = mat64.NewDense(nSamples, nFeatures, nil) weightedOutput.Mul(diagWeight, trueOutputs) weightedFeatures.Mul(diagWeight, features) } switch regularizer.(type) { case nil: case regularize.None: default: panic("Shouldn't be here. Must be error in IsLinearRegularizer") } if weights == nil { parameterMat := mat64.Solve(features, trueOutputs) return parameterMat.RawMatrix().Data } parameterMat := mat64.Solve(weightedFeatures, weightedOutput) return parameterMat.RawMatrix().Data }
// Iterative Re-weighting Least Squares Estimation for Generalized Linear Models func (l *glm) Train(A *mat64.Dense, b []float64) ([]float64, error) { if l.config == nil { return nil, fmt.Errorf("config not set") } nrow, ncol := A.Dims() x := mat64.NewDense(ncol, 1, rep(0.0, ncol)) var i int64 var err error for ; i < l.config.MaxIt; i++ { eta := matrixMult(A, x) etaCol := eta.Col(nil, 0) var ( g = make([]float64, nrow) // g = invLink(eta) gprime = make([]float64, nrow) // g = derivativeFn(eta) w = make([]float64, nrow) // w = gprime^2 / variance(g) ) for i, val := range etaCol { g[i] = l.config.F.LinkFn(val) gprime[i] = l.config.F.DerivativeFn(val) w[i] = math.Pow(gprime[i], 2.0) / l.config.F.VarianceFn(g[i]) } // z = eta + (b - g) / gprime z := mat64.NewDense(nrow, 1, nil) eta.Clone(z) z.Apply(func(i, j int, eta float64) float64 { return eta + (b[i]-g[i])/gprime[i] }, z) // convert w = w * I wMat := mat64.NewDense(nrow, nrow, rep(0.0, nrow*nrow)) for i := 0; i < nrow; i++ { wMat.Set(i, i, w[i]) } var ( wa = matrixMult(wMat, A) cprod1 = matrixMult(wa.T(), A) wz = matrixMult(wMat, z) cprod2 = matrixMult(wz.T(), A) ) // save xold for evaluating convergence xold := mat64.NewDense(ncol, 1, nil) x.Clone(xold) // xnew = solve(crossprod(A,W*A), crossprod(A,W*z)) x, err = mat64.Solve(cprod1, cprod2.T()) if err != nil { return nil, err } // convergence = sqrt(crossprod(x - xold)) <= tolerance diff := &mat64.Dense{} diff.Sub(x, xold) conv := matrixMult(diff.T(), diff) if math.Sqrt(conv.At(0, 0)) <= l.config.Tolerance { break } } coef := x.Col(nil, 0) return coef, nil }
// LinearSolve trains a Linear algorithm. // Assumes inputs and outputs are already scaled // If features is nil will call featurize // Will return nil if regularizer is not a linear regularizer // Is destructive if any of the weights are zero // Losser is always the two-norm // Does not set the value of the parameters (in case this is called in parallel with a different routine) func LinearSolve(linearTrainable LinearTrainable, features *mat64.Dense, inputs, trueOutputs common.RowMatrix, weights []float64, regularizer regularize.Regularizer) (parameters []float64) { // TODO: Allow tikhonov regularization // TODO: Add test for weights // TODO: Need to do something about returning a []float64 if !IsLinearSolveRegularizer(regularizer) { return nil } if features == nil { features = FeaturizeTrainable(linearTrainable, inputs, features) } _, nFeatures := features.Dims() var weightedFeatures, weightedOutput *mat64.Dense fmt.Println("In linear solve") if weights != nil { panic("Need functionality to be better. Either banded special case in matrix or do the mulitplication by hand") scaledWeight := make([]float64, len(weights)) for i, weight := range weights { scaledWeight[i] = math.Sqrt(weight) } diagWeight := diagonal.NewDiagonal(len(scaledWeight), scaledWeight) nSamples, outputDim := trueOutputs.Dims() weightedOutput = mat64.NewDense(nSamples, outputDim, nil) weightedFeatures = mat64.NewDense(nSamples, nFeatures, nil) weightedOutput.Copy(trueOutputs) weightedFeatures.Copy(features) // TODO: Replace this with better than mat multiply weightedOutput.Mul(diagWeight, weightedOutput) weightedFeatures.Mul(diagWeight, weightedFeatures) } switch regularizer.(type) { case nil: case regularize.None: default: panic("Shouldn't be here. Must be error in IsLinearRegularizer") } if weights == nil { parameterMat, err := mat64.Solve(features, trueOutputs) if err != nil { panic(err) } return parameterMat.RawMatrix().Data } parameterMat, err := mat64.Solve(weightedFeatures, weightedOutput) if err != nil { panic(err) } return parameterMat.RawMatrix().Data }