func TestLinprog(t *testing.T) {
m := 500
n := 1000
tol := 1e-8
A := mat.RandN(m, n)
c := mat.RandVec(n)
b := mat.NewVec(m)
xt := mat.RandVec(n)
b.Apply(A, xt)
At := A.TrView()
rd := mat.NewVec(n)
rp := mat.NewVec(m)
rs := mat.NewVec(n)
prob := NewStandard(c, A, b)
//Example for printing duality gap and infeasibilities
result := Solve(prob, nil, NewDisplay(2))
rd.Sub(c, result.S)
rd.AddMul(At, result.Y, -1)
rp.Apply(A, result.X)
rp.Sub(b, rp)
rs.Mul(result.X, result.S)
rs.Neg(rs)
dev := (rd.Asum() + rp.Asum() + rs.Asum()) / float64(n)
if dev > tol {
t.Log(dev)
t.Fail()
}
}
func (sol ProjGrad) Solve(o Grad, proj Projection, in *Solution, p *Params, u ...Updater) *Result {
r := NewResult(in)
obj := ObjGradWrapper{r: r, o: o}
r.initGrad(obj)
h := newHelper(r.Solution, u)
n := len(r.X)
s := 1.0 //initial step size
d := mat.NewVec(n)
d.Copy(r.GradX)
d.Scal(-1)
xTemp := mat.NewVec(n)
xTemp.Copy(r.X)
xTemp.Axpy(s/2, d)
proj.Project(xTemp)
xTemp.Sub(xTemp, r.X)
xTemp.Scal(2 / s)
gLin := -xTemp.Nrm2Sq()
lineFunc := NewLineFuncProj(obj, proj, r.X, d)
lsInit := uni.NewSolution()
lsParams := uni.NewParams()
for ; r.Status == NotTerminated; h.update(r, p) {
lsInit.SetX(s)
lsInit.SetLB(0, r.ObjX, gLin)
lsRes := sol.LineSearch.Solve(lineFunc, lsInit, lsParams)
if lsRes.Status < 0 {
r.Status = Status(lsRes.Status)
break
}
s, r.ObjX = lsRes.X, lsRes.ObjX
r.X.Axpy(s, d)
proj.Project(r.X)
obj.ValGrad(r.X, r.GradX)
d.Copy(r.GradX)
d.Scal(-1)
xTemp.Copy(r.X)
xTemp.Axpy(s/2, d)
proj.Project(xTemp)
xTemp.Sub(xTemp, r.X)
xTemp.Scal(2 / s)
gLin = -xTemp.Nrm2Sq()
}
return r
}
func NewLineFuncDeriv(f Grad, x, d mat.Vec) *LineFuncDeriv {
n := len(x)
if len(d) != n {
panic("dimension mismatch")
}
return &LineFuncDeriv{
f: f,
x: x,
d: d,
g: mat.NewVec(n),
xTemp: mat.NewVec(n),
}
}
func TestExampleModel(t *testing.T) {
//badly conditioned Hessian leads to zig-zagging of the steepest descent
//algorithm
condNo := 100.0
optSol := mat.Vec{1, 2}
A := mat.NewFromArray([]float64{condNo, 0, 0, 1}, true, 2, 2)
b := mat.Vec{-2 * optSol[0] * condNo, -2 * optSol[1]}
c := -0.5 * mat.Dot(b, optSol)
//define objective function
fun := opt.NewQuadratic(A, b, c)
//set inital solution estimate
sol := NewSolution(mat.NewVec(2))
//set termination parameters
p := NewParams()
p.IterMax = 5
//Use steepest descent solver to solve the model
result := NewSteepestDescent().Solve(fun, sol, p, NewDisplay(1))
fmt.Println("x =", result.X)
//should be [1,2], but because of the bad conditioning we made little
//progress in the second dimension
//Use a BFGS solver to refine the result:
result = NewLBFGS().Solve(fun, result.Solution, p, NewDisplay(1))
fmt.Println("x =", result.X)
}
func newHelper(in *Solution, u []Updater) *helper {
h := &helper{}
h.initialTime = time.Now()
h.updates = u
if in.GradX != nil {
h.initialGradNorm = in.GradX.Nrm2()
} else {
h.initialGradNorm = nan
}
h.oldX = mat.NewVec(len(in.X)).Scal(nan)
h.temp = mat.NewVec(len(in.X)).Scal(nan)
h.oldObjX = nan
h.gradNorm = nan
return h
}
func NewLineFuncProj(f Function, p Projection, x, d mat.Vec) *LineFuncProj {
n := len(x)
if len(d) != n {
panic("dimension mismatch")
}
return &LineFuncProj{
f: f,
p: p,
x: x,
d: d,
xTemp: mat.NewVec(n),
}
}
func NewLineFunc(f Function, x, d mat.Vec) *LineFunc {
n := len(x)
if len(d) != n {
panic("dimension mismatch")
}
return &LineFunc{
f: f,
x: x,
d: d,
Dir: 1,
xTemp: mat.NewVec(n),
}
}
func NewResult(prob *Problem) *Result {
r := &Result{}
r.Solution = &Solution{}
m, n := prob.A.Dims()
r.X = mat.NewVec(n)
r.S = mat.NewVec(n)
r.Y = mat.NewVec(m)
return r
/*
if in.X != nil {
r.X.Copy(in.X)
}
if in.S != nil {
r.S.Copy(in.S)
}
if in.Y != nil {
r.Y.Copy(in.Y)
}
*/
}
func BenchmarkLinprog(bench *testing.B) {
bench.StopTimer()
m := 50
n := 100
tol := 1e-3
rd := mat.NewVec(n)
rp := mat.NewVec(m)
rs := mat.NewVec(n)
for i := 0; i < bench.N; i++ {
A := mat.RandN(m, n)
c := mat.RandVec(n)
b := mat.NewVec(m)
xt := mat.RandVec(n)
b.Apply(A, xt)
At := A.TrView()
prob := NewStandard(c, A, b)
bench.StartTimer()
result := Solve(prob, nil)
bench.StopTimer()
rd.Sub(c, result.S)
rd.AddMul(At, result.Y, -1)
rp.Apply(A, result.X)
rp.Sub(b, rp)
rs.Mul(result.X, result.S)
rs.Neg(rs)
dev := (rd.Asum() + rp.Asum() + rs.Asum()) / float64(n)
if dev > tol {
bench.Log(dev)
}
}
}
func NewQuadratic(A *mat.Dense, b mat.Vec, c float64) *Quadratic {
m, n := A.Dims()
if m != n {
panic("matrix has to be quadratic")
}
if n != len(b) {
panic("dimension mismatch between A and b")
}
return &Quadratic{
A: A,
B: b,
C: c,
temp: mat.NewVec(n),
}
}
func TestLinprog2(t *testing.T) {
m := 1
n := 5
tol := 1e-8
a := mat.NewVec(n).AddSc(1)
xStar := mat.NewVec(n)
xStar[0] = 1
A := mat.NewFromArray(a, true, m, n)
At := A.TrView()
b := mat.NewVec(m).AddSc(1)
c := mat.NewVec(n)
c[0] = -1
prob := NewStandard(c, A, b)
result := Solve(prob, nil)
rd := mat.NewVec(n)
rp := mat.NewVec(m)
rs := mat.NewVec(n)
rd.Sub(c, result.S)
rd.AddMul(At, result.Y, -1)
rp.Apply(A, result.X)
rp.Sub(b, rp)
rs.Mul(result.X, result.S)
rs.Neg(rs)
dev := (rd.Asum() + rp.Asum() + rs.Asum()) / float64(n)
if dev > tol {
t.Fail()
}
temp := mat.NewVec(n)
temp.Sub(result.X, xStar)
if temp.Nrm2() > tol {
t.Log(result.X)
t.Fail()
}
}
func (sol SteepestDescent) Solve(o Grad, in *Solution, p *Params, u ...Updater) *Result {
r := NewResult(in)
obj := ObjGradWrapper{r: r, o: o}
r.initGrad(obj)
h := newHelper(r.Solution, u)
n := len(r.X)
s := 1.0 //initial step size
gLin := -r.GradX.Nrm2Sq()
d := mat.NewVec(n)
d.Copy(r.GradX)
d.Scal(-1)
lineFunc := NewLineFuncDeriv(obj, r.X, d)
lsInit := uni.NewSolution()
lsParams := uni.NewParams()
for ; r.Status == NotTerminated; h.update(r, p) {
lsInit.SetX(s)
lsInit.SetLB(0, r.ObjX, gLin)
lsRes := sol.LineSearch.Solve(lineFunc, lsInit, lsParams)
if lsRes.Status < 0 {
r.Status = Status(lsRes.Status)
break
}
s, r.ObjX = lsRes.X, lsRes.ObjX
r.X.Axpy(s, d)
obj.ValGrad(r.X, r.GradX)
d.Copy(r.GradX)
d.Scal(-1)
gLin = -d.Nrm2Sq()
}
return r
}
func TestQuadratic(t *testing.T) {
mat.Register(cops)
n := 10
xStar := mat.NewVec(n)
xStar.AddSc(1)
A := mat.RandN(n)
At := A.TrView()
AtA := mat.New(n)
AtA.Mul(At, A)
bTmp := mat.NewVec(n)
bTmp.Apply(A, xStar)
b := mat.NewVec(n)
b.Apply(At, bTmp)
b.Scal(-2)
c := bTmp.Nrm2Sq()
//Define input arguments
obj := opt.NewQuadratic(AtA, b, c)
p := NewParams()
sol := NewSolution(mat.NewVec(n))
//Steepest descent with armijo
stDesc := NewSteepestDescent()
res1 := stDesc.Solve(obj, sol, p, NewDisplay(100))
t.Log(res1.ObjX, res1.FunEvals, res1.GradEvals, res1.Status)
//Steepest descent with Quadratic
stDesc.LineSearch = uni.DerivWrapper{uni.NewQuadratic()}
res2 := stDesc.Solve(obj, sol, p, NewDisplay(100))
t.Log(res2.ObjX, res2.FunEvals, res2.GradEvals, res2.Status)
//LBFGS with armijo
lbfgs := NewLBFGS()
res3 := lbfgs.Solve(obj, sol, p, NewDisplay(10))
t.Log(res3.ObjX, res3.FunEvals, res3.GradEvals, res3.Status)
//constrained problems (constraints described as projection)
projGrad := NewProjGrad()
res4 := projGrad.Solve(obj, opt.RealPlus{}, sol, p, NewDisplay(100))
t.Log(res4.ObjX, res4.FunEvals, res4.GradEvals, res4.Status)
if math.Abs(res1.ObjX) > 0.01 {
t.Fail()
}
if math.Abs(res2.ObjX) > 0.01 {
t.Fail()
}
if math.Abs(res3.ObjX) > 0.01 {
t.Fail()
}
if math.Abs(res4.ObjX) > 0.01 {
t.Fail()
}
}
func (sol LBFGS) Solve(o Grad, in *Solution, p *Params, u ...Updater) *Result {
r := NewResult(in)
obj := ObjGradWrapper{r: r, o: o}
r.initGrad(obj)
h := newHelper(r.Solution, u)
stepSize := 1.0
gLin := 0.0
n := len(r.X)
S := make([]mat.Vec, sol.Mem)
Y := make([]mat.Vec, sol.Mem)
for i := 0; i < sol.Mem; i++ {
S[i] = mat.NewVec(n)
Y[i] = mat.NewVec(n)
}
d := mat.NewVec(n)
xOld := mat.NewVec(n)
gOld := mat.NewVec(n)
sNew := mat.NewVec(n)
yNew := mat.NewVec(n)
alphas := mat.NewVec(sol.Mem)
betas := mat.NewVec(sol.Mem)
rhos := mat.NewVec(sol.Mem)
lineFunc := NewLineFuncDeriv(obj, r.X, d)
lsInit := uni.NewSolution()
lsParams := uni.NewParams()
for ; r.Status == NotTerminated; h.update(r, p) {
d.Copy(r.GradX)
if r.Iter > 0 {
yNew.Sub(r.GradX, gOld)
sNew.Sub(r.X, xOld)
temp := S[len(S)-1]
copy(S[1:], S)
S[0] = temp
S[0].Copy(sNew)
temp = Y[len(S)-1]
copy(Y[1:], Y)
Y[0] = temp
Y[0].Copy(yNew)
copy(rhos[1:], rhos)
rhos[0] = 1 / mat.Dot(sNew, yNew)
for i := 0; i < sol.Mem; i++ {
alphas[i] = rhos[i] * mat.Dot(S[i], d)
d.Axpy(-alphas[i], Y[i])
}
for i := sol.Mem - 1; i >= 0; i-- {
betas[i] = rhos[i] * mat.Dot(Y[i], d)
d.Axpy(alphas[i]-betas[i], S[i])
}
}
d.Scal(-1)
gLin = mat.Dot(d, r.GradX)
lsInit.SetX(stepSize)
lsInit.SetLB(0, r.ObjX, gLin)
lsRes := sol.LineSearch.Solve(lineFunc, lsInit, lsParams)
if lsRes.Status < 0 {
fmt.Println("Linesearch:", lsRes.Status)
d.Copy(r.GradX)
d.Scal(-1)
lsInit.SetLB(0, r.ObjX, -r.GradX.Nrm2Sq())
lsRes = sol.LineSearch.Solve(lineFunc, lsInit, lsParams)
if lsRes.Status < 0 {
fmt.Println("Linesearch:", lsRes.Status)
r.Status = Status(lsRes.Status)
break
}
}
stepSize, r.ObjX = lsRes.X, lsRes.ObjX
xOld.Copy(r.X)
gOld.Copy(r.GradX)
r.X.Axpy(stepSize, d)
obj.ValGrad(r.X, r.GradX)
}
return r
}
func (sol *Rosenbrock) Solve(o Function, in *Solution, p *Params, u ...Updater) *Result {
r := NewResult(in)
obj := ObjWrapper{r: r, o: o}
r.init(obj)
h := newHelper(r.Solution, u)
eps := 1.0
n := len(r.X)
d := make([]mat.Vec, n)
for i := range d {
d[i] = mat.NewVec(n)
d[i][i] = 1
}
lambda := make([]float64, n)
lf := make([]*LineFunc, n)
for i := range lf {
lf[i] = NewLineFunc(obj, r.X, d[i])
}
lsInit := uni.NewSolution()
lsParams := uni.NewParams()
lsParams.XTolAbs = p.XTolAbs
lsParams.XTolRel = p.XTolRel
lsParams.FunTolAbs = 0
lsParams.FunTolRel = 0
for ; r.Status == NotTerminated; h.update(r, p) {
//Search in all directions
for i := range d {
lf[i].Dir = 1
valNeg := 0.0
valPos := lf[i].Val(eps)
if valPos >= r.ObjX {
lf[i].Dir = -1
valNeg = lf[i].Val(eps)
if valNeg >= r.ObjX {
eps *= 0.5
lf[i].Dir = 1
lsInit.SetLB(-eps)
lsInit.SetUB(eps)
lsInit.SetX(0)
} else {
lsInit.SetUB()
lsInit.SetLB()
lsInit.SetX(eps)
}
} else {
lsInit.SetUB()
lsInit.SetLB()
lsInit.SetX(eps)
}
lsRes := sol.LineSearch.Solve(lf[i], lsInit, lsParams)
lambda[i] = lf[i].Dir * lsRes.X
r.X.Axpy(lambda[i], d[i])
r.ObjX = lsRes.ObjX
}
//Find new directions
for i := range d {
if math.Abs(lambda[i]) > p.XTolAbs {
d[i].Scal(lambda[i])
for j := i + 1; j < n; j++ {
d[i].Axpy(lambda[j], d[j])
}
}
}
//Gram-Schmidt, TODO:use QR factorization
for i := range d {
d[i].Scal(1 / d[i].Nrm2())
for j := i + 1; j < n; j++ {
d[j].Axpy(-mat.Dot(d[i], d[j]), d[i])
}
}
}
return r
}
//Predictor-Corrector Interior Point implementation
func (sol *PredCorr) Solve(prob *Problem, p *Params, u ...Updater) *Result {
res := NewResult(prob)
h := newHelper(u)
A := prob.A
m, n := A.Dims()
At := A.TrView()
var mu, sigma float64
res.X.AddSc(1)
res.S.AddSc(1)
res.Rd = mat.NewVec(n)
res.Rp = mat.NewVec(m)
res.Rs = mat.NewVec(n)
x := res.X
s := res.S
y := res.Y
dx := mat.NewVec(n)
ds := mat.NewVec(n)
dy := mat.NewVec(m)
dxAff := mat.NewVec(n)
dsAff := mat.NewVec(n)
dyAff := mat.NewVec(m)
dxCC := mat.NewVec(n)
dsCC := mat.NewVec(n)
dyCC := mat.NewVec(m)
xdivs := mat.NewVec(n)
temp := mat.New(m, n)
lhs := mat.New(m, m)
rhs := mat.NewVec(m)
soli := mat.NewVec(m)
triU := mat.New(m, m)
triUt := triU.TrView()
nTemp1 := mat.NewVec(n)
nTemp2 := mat.NewVec(n)
alpha := 0.0
for {
res.Rd.Sub(prob.C, res.S)
res.Rd.AddMul(At, res.Y, -1)
res.Rp.Apply(A, res.X)
res.Rp.Sub(prob.B, res.Rp)
res.Rs.Mul(res.X, res.S)
res.Rs.Neg(res.Rs)
if h.update(res, p); res.Status != 0 {
break
}
if checkKKT(res, p); res.Status != 0 {
break
}
mu = res.Rs.Asum() / float64(n)
//determining left hand side
temp.ScalCols(A, xdivs.Div(x, s))
lhs.Mul(temp, At)
//factorization
lhs.Chol(triU)
//right hand side
nTemp1.Add(res.Rd, s)
nTemp1.Mul(nTemp1, xdivs)
rhs.Apply(A, nTemp1)
rhs.Add(rhs, res.Rp)
//solving for dyAff
soli.Trsv(triUt, rhs)
dyAff.Trsv(triU, soli)
//calculating other steps (dxAff, dsAff)
nTemp1.Apply(At, dyAff)
dxAff.Sub(nTemp1, res.Rd)
dxAff.Sub(dxAff, s)
dxAff.Mul(dxAff, xdivs)
dsAff.Div(dxAff, xdivs)
dsAff.Add(dsAff, s)
dsAff.Neg(dsAff)
//determining step size
alpha = 1.0
for i := range dxAff {
if dxAff[i] < 0 {
alph := -x[i] / dxAff[i]
if alph < alpha {
alpha = alph
}
}
}
for i := range dsAff {
if dsAff[i] < 0 {
alph := -s[i] / dsAff[i]
if alph < alpha {
alpha = alph
}
}
}
alpha *= 0.99995
//calculating duality gap measure for affine case
nTemp1.Copy(x)
nTemp1.Axpy(alpha, dxAff)
nTemp2.Copy(s)
nTemp2.Axpy(alpha, dsAff)
mu_aff := mat.Dot(nTemp1, nTemp2) / float64(n)
//centering parameter
sigma = math.Pow(mu_aff/mu, 3)
//right hand side for predictor corrector step
res.Rs.Mul(dxAff, dsAff)
res.Rs.Neg(res.Rs)
res.Rs.AddSc(sigma * mu_aff)
nTemp1.Div(res.Rs, s)
nTemp1.Neg(nTemp1)
rhs.Apply(A, nTemp1)
//solving for dyCC
soli.Trsv(triUt, rhs)
dyCC.Trsv(triU, soli)
//calculating other steps (dxAff, dsAff)
nTemp1.Apply(At, dyCC)
dxCC.Mul(nTemp1, x)
dxCC.Add(res.Rs, dxCC)
dxCC.Div(dxCC, s)
dsCC.Mul(dxCC, s)
dsCC.Sub(res.Rs, dsCC)
dsCC.Div(dsCC, x)
dx.Add(dxAff, dxCC)
dy.Add(dyAff, dyCC)
ds.Add(dsAff, dsCC)
alpha = 1
for i := range dx {
if dx[i] < 0 {
alph := -x[i] / dx[i]
if alph < alpha {
alpha = alph
}
}
}
for i := range ds {
if ds[i] < 0 {
alph := -s[i] / ds[i]
if alph < alpha {
alpha = alph
}
}
}
alpha *= 0.99995
x.Axpy(alpha, dx)
y.Axpy(alpha, dy)
s.Axpy(alpha, ds)
}
return res
}
