func (m *LBFGSMinimizer) Minimize(costfun DiffFunction, init *core.Vector) *core.Vector {
m.costFun = costfun
var cost float64 = costfun.Value(init)
var grad *core.Vector = costfun.Gradient(init).Copy()
var pos *core.Vector = init.Copy()
var terminalCriterion *relativeMeanImprCriterion = NewRelativeMeanImprCriterion(m.tolerance)
terminalCriterion.addCost(cost)
var helper *QuasiNewtonHelper = NewQuasiNewtonHelper(m.numHist, m, pos, grad)
if lbfgs_output_switch {
fmt.Println("Iter\tcost\timprovement")
fmt.Printf("%d\t%e\tUndefined", 0, cost)
}
for iter := 1; iter <= m.maxIteration; iter++ {
dir := grad.Copy()
dir.ApplyScale(-1.0)
helper.ApplyQuasiInverseHession(dir)
newCost, newPos := helper.BackTrackingLineSearch(cost, pos, grad, dir, iter == 1)
if lbfgs_output_switch {
fmt.Println("")
}
if cost == newCost {
break
}
cost = newCost
pos = newPos
grad = costfun.Gradient(pos).Copy()
terminalCriterion.addCost(cost)
if lbfgs_output_switch {
fmt.Printf("%d\t%e\t%e", iter, newCost, terminalCriterion.improvement)
}
if terminalCriterion.isTerminable() || helper.UpdateState(pos, grad) {
if lbfgs_output_switch {
fmt.Println("")
}
break
}
}
return pos
}
func (lr *LROWLQN) updateValueGrad(pos *core.Vector, dataset *core.DataSet) {
var totalLoss float64 = 0.0
var grad *core.Vector = core.NewVector()
for _, sample := range dataset.Samples {
var score float64 = lr.getScore(pos, sample)
var signScore float64 = score
if sample.Label == 0 {
signScore = -score
}
var prob float64
var lnProb float64
if signScore < -30 {
prob = 0
lnProb = signScore
} else if signScore > 30 {
prob = 1
lnProb = 0
} else {
prob = 1.0 / (1.0 + math.Exp(-signScore))
lnProb = math.Log(prob)
}
var scale float64
if sample.Label == 0 {
scale = (1 - prob)
} else {
scale = -(1 - prob)
}
totalLoss += -lnProb
for _, fea := range sample.Features {
grad.AddValue(fea.Id, scale*fea.Value)
}
}
lr.lastPos = pos.Copy()
lr.lastCost = totalLoss
lr.lastGrad = grad
}
func (m *OWLQNMinimizer) Minimize(costfun DiffFunction, init *core.Vector) *core.Vector {
m.costFun = costfun
var cost float64 = m.Evaluate(init)
var grad *core.Vector = costfun.Gradient(init).Copy()
var pos *core.Vector = init.Copy()
var terminalCriterion *relativeMeanImprCriterion = NewRelativeMeanImprCriterion(m.tolerance)
terminalCriterion.addCost(cost)
var helper *QuasiNewtonHelper = NewQuasiNewtonHelper(m.numHist, m, pos, grad)
if owlqn_output_switch {
fmt.Println("Iter\tcost\timprovement")
fmt.Printf("%d\t%e\tUndefined", 0, cost)
}
for iter := 1; iter <= m.maxIteration; iter++ {
// customed steepest descending dir
steepestDescDir := grad.Copy()
m.updateGrad(pos, steepestDescDir)
steepestDescDir.ApplyScale(-1.0)
dir := steepestDescDir.Copy()
// quasi-newton dir
helper.ApplyQuasiInverseHession(dir)
m.fixDirSign(dir, steepestDescDir)
// customed grad for the new position
potentialGrad := grad.Copy()
m.updateGradForNewPos(pos, potentialGrad, dir)
newCost, newPos := helper.BackTrackingLineSearch(cost, pos, potentialGrad, dir, iter == 1)
if owlqn_output_switch {
fmt.Println("")
}
if cost == newCost {
break
}
cost = newCost
pos = newPos
grad = costfun.Gradient(pos).Copy()
terminalCriterion.addCost(cost)
if owlqn_output_switch {
fmt.Printf("%d\t%e\t%e", iter, newCost, terminalCriterion.improvement)
}
if terminalCriterion.isTerminable() || helper.UpdateState(pos, grad) {
if owlqn_output_switch {
fmt.Println("")
}
break
}
}
return pos
}
func Distance(x, y *core.Vector) float64 {
z := x.Copy()
z.AddVector(y, -1)
d := z.NormL2()
return d
}
func (c *KNN) Kernel(x, y *core.Vector) float64 {
z := x.Copy()
z.AddVector(y, -1.0)
ret := math.Exp(-1.0 * z.NormL2() / 20.0)
return ret
}
/*
Given matrix m and vector v, compute inv(m)*v.
Based on Gibbs and MacKay 1997, and Mark N. Gibbs's PhD dissertation
Details:
A - positive seminidefinite matrix
u - a vector
theta - positive number
C = A + I*theta
Returns inv(C)*u - So you need the diagonal noise term for covariance matrix in a sense.
However, this algorithm is numerically stable, the noise term can be very small and the inversion can still be calculated...
*/
func (algo *GaussianProcess) ApproximateInversion(A *core.Matrix, u *core.Vector, theta float64, dim int64) *core.Vector {
max_itr := 500
tol := 0.01
C := core.NewMatrix()
for key, val := range A.Data {
C.Data[key] = val.Copy()
}
// Add theta to diagonal elements
for i := int64(0); i < dim; i++ {
_, ok := C.Data[i]
if !ok {
C.Data[i] = core.NewVector()
}
C.Data[i].Data[i] = C.Data[i].Data[i] + theta
}
var Q_l float64
var Q_u float64
var dQ float64
u_norm := u.Dot(u) / 2
// Lower bound
y_l := core.NewVector()
g_l := u.Copy()
h_l := u.Copy()
lambda_l := float64(0)
gamma_l := float64(0)
var tmp_f1 float64
var tmp_f2 float64
var tmp_v1 *core.Vector
tmp_f1 = g_l.Dot(g_l)
tmp_v1 = C.MultiplyVector(h_l)
// Upper bound
y_u := core.NewVector()
g_u := u.Copy()
h_u := u.Copy()
lambda_u := float64(0)
gamma_u := float64(0)
var tmp_f3 float64
var tmp_f4 float64
var tmp_v3 *core.Vector
var tmp_v4 *core.Vector
tmp_v3 = g_u.MultiplyMatrix(A)
tmp_v4 = C.MultiplyVector(h_u)
tmp_f3 = tmp_v1.Dot(g_u)
for i := 0; i < max_itr; i++ {
// Lower bound
lambda_l = tmp_f1 / h_l.Dot(tmp_v1)
y_l.AddVector(h_l, lambda_l) //y_l next
Q_l = y_l.Dot(u) - 0.5*(y_l.MultiplyMatrix(C)).Dot(y_l)
// Upper bound
lambda_u = tmp_f3 / tmp_v3.Dot(tmp_v4)
y_u.AddVector(h_u, lambda_u) //y_u next
Q_u = (y_u.MultiplyMatrix(A)).Dot(u) - 0.5*((y_u.MultiplyMatrix(C)).MultiplyMatrix(A)).Dot(y_u)
dQ = (u_norm-Q_u)/theta - Q_l
if dQ < tol {
break
}
// Lower bound var updates
g_l.AddVector(tmp_v1, -lambda_l) //g_l next
tmp_f2 = g_l.Dot(g_l)
gamma_l = tmp_f2 / tmp_f1
for key, val := range h_l.Data {
h_l.SetValue(key, val*gamma_l)
}
h_l.AddVector(g_l, 1) //h_l next
tmp_f1 = tmp_f2 //tmp_f1 next
tmp_v1 = C.MultiplyVector(h_l) //tmp_v1 next
// Upper bound var updates
g_u.AddVector(tmp_v4, -lambda_u) //g_u next
tmp_v3 = g_u.MultiplyMatrix(A) //tmp_v3 next
tmp_f4 = tmp_v3.Dot(g_u)
gamma_u = tmp_f4 / tmp_f3
for key, val := range h_u.Data {
h_u.SetValue(key, val*gamma_u)
}
h_u.AddVector(g_u, 1) //h_u next
tmp_v4 = C.MultiplyVector(h_u) //tmp_v4 next
tmp_f3 = tmp_f4 // tmp_f3 next
}
return y_l
}