func (lbfgs *Lbfgs) Iterate(loc *multi.Location, obj *uni.Objective, grad *multi.Gradient, fun common.MultiObjGrad) (common.Status, error) {
counter := lbfgs.counter
q := lbfgs.q
a := lbfgs.a
b := lbfgs.b
rhoHist := lbfgs.rhoHist
sHist := lbfgs.sHist
yHist := lbfgs.yHist
gamma_k := lbfgs.gamma_k
tmp := lbfgs.tmp
p_k := lbfgs.p_k
s_k := lbfgs.s_k
y_k := lbfgs.y_k
z := lbfgs.z
grad.Current(lbfgs.currGrad)
loc.Current(lbfgs.currLoc)
// Calculate search direction
for i, val := range lbfgs.currGrad {
q[i] = val
}
for i := counter - 1; i >= 0; i-- {
a[i] = rhoHist[i] * floats.Dot(sHist[i], q)
copy(tmp, yHist[i])
floats.Scale(a[i], tmp)
floats.Sub(q, tmp)
}
for i := lbfgs.NumStore - 1; i >= counter; i-- {
a[i] = rhoHist[i] * floats.Dot(sHist[i], q)
copy(tmp, yHist[i])
floats.Scale(a[i], tmp)
//fmt.Println(q)
//fmt.Println(tmp)
floats.Sub(q, tmp)
}
// Assume H_0 is the identity times gamma_k
copy(z, q)
floats.Scale(gamma_k, z)
// Second loop for update, going oldest to newest
for i := counter; i < lbfgs.NumStore; i++ {
b[i] = rhoHist[i] * floats.Dot(yHist[i], z)
copy(tmp, sHist[i])
floats.Scale(a[i]-b[i], tmp)
floats.Add(z, tmp)
}
for i := 0; i < counter; i++ {
b[i] = rhoHist[i] * floats.Dot(yHist[i], z)
copy(tmp, sHist[i])
floats.Scale(a[i]-b[i], tmp)
floats.Add(z, tmp)
}
lbfgs.a = a
lbfgs.b = b
copy(p_k, z)
floats.Scale(-1, p_k)
normP_k := floats.Norm(p_k, 2)
// Perform line search -- need to find some way to implement this, especially bookkeeping function values
linesearchResult, err := linesearch.Linesearch(fun, lbfgs.LinesearchMethod, lbfgs.LinesearchSettings, lbfgs.Wolfe, p_k, lbfgs.currLoc, obj.Current(), lbfgs.currGrad)
// In the future add a check to switch to a different linesearcher?
if err != nil {
return common.LinesearchFailure, err
}
x_kp1 := linesearchResult.Loc
f_kp1 := linesearchResult.Obj
g_kp1 := linesearchResult.Grad
alpha_k := linesearchResult.Step
// Update hessian estimate
copy(s_k, p_k)
floats.Scale(alpha_k, s_k)
copy(y_k, g_kp1)
floats.Sub(y_k, lbfgs.currGrad)
skDotYk := floats.Dot(s_k, y_k)
// Bookkeep the results
stepSize := alpha_k * normP_k
lbfgs.step = stepSize
loc.SetCurrent(x_kp1)
//lbfgs.loc.AddToHist(x_kp1)
//fmt.Println(lbfgs.loc.GetHist())
obj.SetCurrent(f_kp1)
grad.SetCurrent(g_kp1)
copy(sHist[counter], s_k)
copy(yHist[counter], y_k)
rhoHist[counter] = 1 / skDotYk
lbfgs.gamma_k = skDotYk / floats.Dot(y_k, y_k)
lbfgs.counter += 1
if lbfgs.counter == lbfgs.NumStore {
lbfgs.counter = 0
}
return common.Continue, nil
}
func (cubic *Cubic) Iterate(loc *uni.Location, obj *uni.Objective, grad *uni.Gradient, fun common.UniObjGrad) (common.Status, error) {
// Initialize
var stepMultiplier float64
updateCurrPoint := false
reverseDirection := false
currG := grad.Current()
currF := obj.Current()
// Evaluate trial point
// Step Size is from the original point
var trialX float64
if cubic.initialGradNegative {
trialX = cubic.step.Current + loc.Initial
} else {
trialX = -cubic.step.Current + loc.Initial
}
/*
fmt.Println(trialX, "trialX")
fmt.Println("cubic current step", cubic.step.Current)
*/
trialF, trialG, err := fun.ObjGrad(trialX)
if err != nil {
return common.UserFunctionError, errors.New("gofunopter: cubic: user defined function error: " + err.Error())
}
var newStepSize float64
/*
fmt.Println()
fmt.Println("curr step size", cubic.step.Current)
fmt.Println("LB", cubic.step.Lb())
fmt.Println("UB", cubic.step.Ub())
fmt.Println("initX", loc.Initial)
fmt.Println("currX", loc.Current())
fmt.Println("trialX ", trialX)
fmt.Println("InitF \t", obj.Init())
fmt.Println("currF \t", currF)
fmt.Println("trialF \t", trialF)
fmt.Println("InitG", grad.Init())
fmt.Println("currG", currG)
fmt.Println("trialG", trialG)
*/
absTrialG := math.Abs(trialG)
// Find guess for next point
deltaF := trialF - currF
decreaseInValue := (deltaF <= 0)
// See if we can trust the deltaF measurement
var canTrustDeltaF bool
divisor := math.Max(math.Abs(trialF), math.Abs(currF))
/*
fmt.Println("Divisor is ", divisor)
fmt.Println("Delta F", deltaF)
*/
if divisor == 0 {
canTrustDeltaF = true // Both are zero, so is >= 0
} else {
if math.Abs(deltaF) > divisor*cubic.FloatingpointRelTol {
// Change large enough to trust
canTrustDeltaF = true
}
// otherwise can't trust
}
changeInDerivSign := (currG > 0 && trialG < 0) || (currG < 0 && trialG > 0)
decreaseInDerivMagnitude := (absTrialG < math.Abs(currG))
/*
fmt.Println("Decrease in value ", decreaseInValue)
fmt.Println("Change in deriv sign ", changeInDerivSign)
fmt.Println("Decrease in deriv mag ", decreaseInDerivMagnitude)
*/
// Find coefficients of the cubic polynomial fit between the current point and the new point
// Derived from fitting a cubic between (0, CurrF) and (1,TrialF).
// Apply transformations later to reshift the coordinate axis
// Need to play games with derivatives
trialFitG := trialG
currFitG := currG
if cubic.initialGradNegative {
trialFitG *= -1
currFitG *= -1
}
if cubic.currStepDirectionPositive {
trialFitG *= -1
currFitG *= -1
}
var a, b, c float64
a = trialG + currG - 2*deltaF
b = 3*deltaF - 2*currG - trialG
c = currG
det := (math.Pow(b, 2) - 3*a*c)
//fmt.Println("det", det)
if a == 0 {
//Perfect quadratic fit
stepMultiplier = -c / (2 * b)
} else if det < 0 {
if decreaseInValue && !changeInDerivSign {
// The trial point has lower function value
// and steeper gradient. Set this location as a
// lower bound for the minimum, and set the
// next point farther in that direction.
cubic.setBound("Lower")
// We know we need to increase in step, but unsure how much, so make a guess
stepMultiplier = cubic.unclearStepIncrease(loc.Current())
if decreaseInDerivMagnitude {
updateCurrPoint = true
}
} else {
// All other conditions we want to decrease the step size, but the
// cubic doesn't give an estimate of how much. Just do a binary search
cubic.setBound("Upper")
//for i := 0; i < 10; i++ {
// fmt.Println("Blah")
//}
//fmt.Println(cubic.step.Curr)
stepMultiplier = 0.5
}
} else {
// Use the cubic projection to guess the minimum location for the line search
minCubic := (-b + math.Sqrt(det)) / (3 * a)
/*
fmt.Println("sht")
fmt.Println("minCubic", minCubic)
fmt.Println("SizeLB",cubic.SizeLB)
fmt.Println("SizeUB",cubic.SizeUB)
*/
switch {
case changeInDerivSign:
// There is a change in derivative sign between the current
// point and the trial point. Make the trial point an upper
// bound for the minimum, and set it as the current point
// if the derivative is smaller in magnitude.
cubic.setBound("Upper")
stepMultiplier = cubic.sizeDecrease(minCubic)
if decreaseInDerivMagnitude && decreaseInValue {
updateCurrPoint = true
reverseDirection = true
}
case (!changeInDerivSign && decreaseInValue) || !canTrustDeltaF:
// No change in derivative sign, but a decrease in
// function value. Want to move more in this direction
// and the trial point is a new lower bound and a new
// base for the cubic approximation
cubic.setBound("Lower")
updateCurrPoint = true
if decreaseInDerivMagnitude {
if minCubic < cubic.StepIncreaseMin {
// Cubic gave a bad approximation (minimum is more
// in this direction). Assume linear decrease in
// derivative
// Check this line
stepMultiplier = math.Abs(currG) / (math.Abs(trialG) - math.Abs(currG))
}
stepMultiplier = cubic.sizeIncrease(stepMultiplier)
} else {
// Found a better point, but the derivative increased
// Use cubic approximation if it gives a reasonable guess
// otherwise just project forward
if minCubic < cubic.StepIncreaseMin {
stepMultiplier = cubic.unclearStepIncrease(loc.Current())
} else {
stepMultiplier = cubic.sizeIncrease(minCubic)
}
}
case (!changeInDerivSign && !decreaseInValue) || canTrustDeltaF:
// Neither a decrease in value nor a change in derivative sign.
// This means there must be a local minimum between the starting location and
// this one. Don't update the cubic point
cubic.setBound("Upper")
stepMultiplier = math.Min(minCubic, 0.75)
stepMultiplier = math.Max(stepMultiplier, 0.25)
default:
panic("Bad logic in cases")
}
}
var deltaXTrialCurrent float64
deltaXTrialCurrent = cubic.step.Current - cubic.deltaCurrent
newDeltaXFromCurrent := deltaXTrialCurrent * stepMultiplier
newStepSize = newDeltaXFromCurrent + cubic.deltaCurrent
// Want to make sure that the new search location isn't pushing beyond
// previously established bounds. If it is, just do a binary search between
// the bounds
if !cubic.step.WithinBounds(newStepSize) {
newStepSize = cubic.step.Midpoint()
}
/*
}
else {
fmt.Println("Can't trust deltaF")
// Can't trust delta F, so just do a binary search
updateCurrPoint = true
switch {
case changeInDerivSign && decreaseInDerivMagnitude:
cubic.setBound("Upper") // this trial is the new upper bound
updateCurrPoint = true
case changeInDerivSign && !decreaseInDerivMagnitude:
cubic.setBound("Upper") // this trial is the new upper bound
updateCurrPoint = false
case !changeInDerivSign && decreaseInDerivMagnitude:
cubic.setBound("Lower")
updateCurrPoint = true
case !changeInDerivSign && !decreaseInDerivMagnitude:
return nFunEvals, errors.New("Gradient tolerance too high for the precision of the gradient")
}
newStepSize = cubic.step.Midpoint()
}
*/
if updateCurrPoint {
loc.SetCurrent(trialX)
cubic.prevF = obj.Current()
obj.SetCurrent(trialF)
grad.SetCurrent(trialG)
cubic.deltaCurrent = trialX - loc.Initial
if cubic.initialGradNegative {
cubic.deltaCurrent *= -1
}
if reverseDirection {
cubic.currStepDirectionPositive = !cubic.currStepDirectionPositive
}
}
cubic.step.Current = newStepSize
return common.Continue, nil
}