// Solution of KKT equations by reduction to a 2 x 2 system, a QR
// factorization to eliminate the equality constraints, and a dense
// Cholesky factorization of order n-p.
//
// Computes the QR factorization
//
// A' = [Q1, Q2] * [R; 0]
//
// and returns a function that (1) computes the Cholesky factorization
//
// Q_2^T * (H + GG^T * W^{-1} * W^{-T} * GG) * Q2 = L * L^T,
//
// given H, Df, W, where GG = [Df; G], and (2) returns a function for
// solving
//
// [ H A' GG' ] [ ux ] [ bx ]
// [ A 0 0 ] * [ uy ] = [ by ].
// [ GG 0 -W'*W ] [ uz ] [ bz ]
//
// H is n x n, A is p x n, Df is mnl x n, G is N x n where
// N = dims['l'] + sum(dims['q']) + sum( k**2 for k in dims['s'] ).
//
func kktChol(G *matrix.FloatMatrix, dims *sets.DimensionSet, A *matrix.FloatMatrix, mnl int) (kktFactor, error) {
p, n := A.Size()
cdim := mnl + dims.Sum("l", "q") + dims.SumSquared("s")
cdim_pckd := mnl + dims.Sum("l", "q") + dims.SumPacked("s")
QA := A.Transpose()
tauA := matrix.FloatZeros(p, 1)
lapack.Geqrf(QA, tauA)
Gs := matrix.FloatZeros(cdim, n)
K := matrix.FloatZeros(n, n)
bzp := matrix.FloatZeros(cdim_pckd, 1)
yy := matrix.FloatZeros(p, 1)
checkpnt.AddMatrixVar("tauA", tauA)
checkpnt.AddMatrixVar("Gs", Gs)
checkpnt.AddMatrixVar("K", K)
factor := func(W *sets.FloatMatrixSet, H, Df *matrix.FloatMatrix) (KKTFunc, error) {
// Compute
//
// K = [Q1, Q2]' * (H + GG' * W^{-1} * W^{-T} * GG) * [Q1, Q2]
//
// and take the Cholesky factorization of the 2,2 block
//
// Q_2' * (H + GG^T * W^{-1} * W^{-T} * GG) * Q2.
var err error = nil
minor := 0
if !checkpnt.MinorEmpty() {
minor = checkpnt.MinorTop()
}
// Gs = W^{-T} * GG in packed storage.
if mnl > 0 {
Gs.SetSubMatrix(0, 0, Df)
}
Gs.SetSubMatrix(mnl, 0, G)
checkpnt.Check("00factor_chol", minor)
scale(Gs, W, true, true)
pack2(Gs, dims, mnl)
//checkpnt.Check("10factor_chol", minor)
// K = [Q1, Q2]' * (H + Gs' * Gs) * [Q1, Q2].
blas.SyrkFloat(Gs, K, 1.0, 0.0, la.OptTrans, &la.IOpt{"k", cdim_pckd})
if H != nil {
K.SetSubMatrix(0, 0, matrix.Plus(H, K.GetSubMatrix(0, 0, H.Rows(), H.Cols())))
}
//checkpnt.Check("20factor_chol", minor)
symm(K, n, 0)
lapack.Ormqr(QA, tauA, K, la.OptLeft, la.OptTrans)
lapack.Ormqr(QA, tauA, K, la.OptRight)
//checkpnt.Check("30factor_chol", minor)
// Cholesky factorization of 2,2 block of K.
lapack.Potrf(K, &la.IOpt{"n", n - p}, &la.IOpt{"offseta", p * (n + 1)})
checkpnt.Check("40factor_chol", minor)
solve := func(x, y, z *matrix.FloatMatrix) (err error) {
// Solve
//
// [ 0 A' GG'*W^{-1} ] [ ux ] [ bx ]
// [ A 0 0 ] * [ uy ] = [ by ]
// [ W^{-T}*GG 0 -I ] [ W*uz ] [ W^{-T}*bz ]
//
// and return ux, uy, W*uz.
//
// On entry, x, y, z contain bx, by, bz. On exit, they contain
// the solution ux, uy, W*uz.
//
// If we change variables ux = Q1*v + Q2*w, the system becomes
//
// [ K11 K12 R ] [ v ] [Q1'*(bx+GG'*W^{-1}*W^{-T}*bz)]
// [ K21 K22 0 ] * [ w ] = [Q2'*(bx+GG'*W^{-1}*W^{-T}*bz)]
// [ R^T 0 0 ] [ uy ] [by ]
//
// W*uz = W^{-T} * ( GG*ux - bz ).
minor := 0
if !checkpnt.MinorEmpty() {
minor = checkpnt.MinorTop()
}
// bzp := W^{-T} * bz in packed storage
scale(z, W, true, true)
pack(z, bzp, dims, &la.IOpt{"mnl", mnl})
// x := [Q1, Q2]' * (x + Gs' * bzp)
// = [Q1, Q2]' * (bx + Gs' * W^{-T} * bz)
blas.GemvFloat(Gs, bzp, x, 1.0, 1.0, la.OptTrans, &la.IOpt{"m", cdim_pckd})
lapack.Ormqr(QA, tauA, x, la.OptLeft, la.OptTrans)
// y := x[:p]
// = Q1' * (bx + Gs' * W^{-T} * bz)
blas.Copy(y, yy)
blas.Copy(x, y, &la.IOpt{"n", p})
// x[:p] := v = R^{-T} * by
blas.Copy(yy, x)
lapack.Trtrs(QA, x, la.OptUpper, la.OptTrans, &la.IOpt{"n", p})
// x[p:] := K22^{-1} * (x[p:] - K21*x[:p])
// = K22^{-1} * (Q2' * (bx + Gs' * W^{-T} * bz) - K21*v)
blas.GemvFloat(K, x, x, -1.0, 1.0, &la.IOpt{"m", n - p}, &la.IOpt{"n", p},
&la.IOpt{"offseta", p}, &la.IOpt{"offsety", p})
lapack.Potrs(K, x, &la.IOpt{"n", n - p}, &la.IOpt{"offseta", p * (n + 1)},
&la.IOpt{"offsetb", p})
// y := y - [K11, K12] * x
// = Q1' * (bx + Gs' * W^{-T} * bz) - K11*v - K12*w
blas.GemvFloat(K, x, y, -1.0, 1.0, &la.IOpt{"m", p}, &la.IOpt{"n", n})
// y := R^{-1}*y
// = R^{-1} * (Q1' * (bx + Gs' * W^{-T} * bz) - K11*v
// - K12*w)
lapack.Trtrs(QA, y, la.OptUpper, &la.IOpt{"n", p})
// x := [Q1, Q2] * x
lapack.Ormqr(QA, tauA, x, la.OptLeft)
// bzp := Gs * x - bzp.
// = W^{-T} * ( GG*ux - bz ) in packed storage.
// Unpack and copy to z.
blas.GemvFloat(Gs, x, bzp, 1.0, -1.0, &la.IOpt{"m", cdim_pckd})
unpack(bzp, z, dims, &la.IOpt{"mnl", mnl})
checkpnt.Check("90solve_chol", minor)
return nil
}
return solve, err
}
return factor, nil
}
// Computes analytic center of A*x <= b with A m by n of rank n.
// We assume that b > 0 and the feasible set is bounded.
func acent(A, b *matrix.FloatMatrix, niters int) (x *matrix.FloatMatrix, ntdecrs []float64, err error) {
err = nil
if niters <= 0 {
niters = MAXITERS
}
ntdecrs = make([]float64, 0, niters)
if A.Rows() != b.Rows() {
return nil, nil, errors.New("A.Rows() != b.Rows()")
}
m, n := A.Size()
x = matrix.FloatZeros(n, 1)
H := matrix.FloatZeros(n, n)
// Helper m*n matrix
Dmn := matrix.FloatZeros(m, n)
for i := 0; i < niters; i++ {
// Gradient is g = A^T * (1.0/(b - A*x)). d = 1.0/(b - A*x)
// d is m*1 matrix, g is n*1 matrix
d := matrix.Minus(b, matrix.Times(A, x)).Inv()
g := matrix.Times(A.Transpose(), d)
// Hessian is H = A^T * diag(1./(b-A*x))^2 * A.
// in the original python code expression d[:,n*[0]] creates
// a m*n matrix where each column is copy of column 0.
// We do it here manually.
for i := 0; i < n; i++ {
Dmn.SetColumn(i, d)
}
// Function mul creates element wise product of matrices.
Asc := matrix.Mul(Dmn, A)
blas.SyrkFloat(Asc, H, 1.0, 0.0, linalg.OptTrans)
// Newton step is v = H^-1 * g.
v := matrix.Scale(g, -1.0)
PosvFloat(H, v)
// Directional derivative and Newton decrement.
lam := blas.DotFloat(g, v)
ntdecrs = append(ntdecrs, math.Sqrt(-lam))
if ntdecrs[len(ntdecrs)-1] < TOL {
return x, ntdecrs, err
}
// Backtracking line search.
// y = d .* A*v
y := matrix.Mul(d, matrix.Times(A, v))
step := 1.0
for 1-step*y.Max() < 0 {
step *= BETA
}
search:
for {
// t = -step*y + 1 [e.g. t = 1 - step*y]
t := matrix.Scale(y, -step).Add(1.0)
// ts = sum(log(1-step*y))
ts := t.Log().Sum()
if -ts < ALPHA*step*lam {
break search
}
step *= BETA
}
v.Scale(step)
x.Plus(v)
}
// no solution !!
err = errors.New(fmt.Sprintf("Iteration %d exhausted\n", niters))
return x, ntdecrs, err
}
// Solution of KKT equations with zero 1,1 block, by eliminating the
// equality constraints via a QR factorization, and solving the
// reduced KKT system by another QR factorization.
//
// Computes the QR factorization
//
// A' = [Q1, Q2] * [R1; 0]
//
// and returns a function that (1) computes the QR factorization
//
// W^{-T} * G * Q2 = Q3 * R3
//
// (with columns of W^{-T}*G in packed storage), and (2) returns a function for solving
//
// [ 0 A' G' ] [ ux ] [ bx ]
// [ A 0 0 ] * [ uy ] = [ by ].
// [ G 0 -W'*W ] [ uz ] [ bz ]
//
// A is p x n and G is N x n where N = dims['l'] + sum(dims['q']) +
// sum( k**2 for k in dims['s'] ).
//
func kktQr(G *matrix.FloatMatrix, dims *sets.DimensionSet, A *matrix.FloatMatrix, mnl int) (kktFactor, error) {
p, n := A.Size()
cdim := dims.Sum("l", "q") + dims.SumSquared("s")
cdim_pckd := dims.Sum("l", "q") + dims.SumPacked("s")
QA := A.Transpose()
tauA := matrix.FloatZeros(p, 1)
lapack.Geqrf(QA, tauA)
Gs := matrix.FloatZeros(cdim, n)
tauG := matrix.FloatZeros(n-p, 1)
u := matrix.FloatZeros(cdim_pckd, 1)
vv := matrix.FloatZeros(n, 1)
w := matrix.FloatZeros(cdim_pckd, 1)
checkpnt.AddMatrixVar("tauA", tauA)
checkpnt.AddMatrixVar("tauG", tauG)
checkpnt.AddMatrixVar("Gs", Gs)
checkpnt.AddMatrixVar("qr_u", u)
checkpnt.AddMatrixVar("qr_vv", vv)
factor := func(W *sets.FloatMatrixSet, H, Df *matrix.FloatMatrix) (KKTFunc, error) {
var err error = nil
minor := 0
if !checkpnt.MinorEmpty() {
minor = checkpnt.MinorTop()
}
// Gs = W^{-T}*G, in packed storage.
blas.Copy(G, Gs)
//checkpnt.Check("00factor_qr", minor)
scale(Gs, W, true, true)
//checkpnt.Check("01factor_qr", minor)
pack2(Gs, dims, 0)
//checkpnt.Check("02factor_qr", minor)
// Gs := [ Gs1, Gs2 ]
// = Gs * [ Q1, Q2 ]
lapack.Ormqr(QA, tauA, Gs, la.OptRight, &la.IOpt{"m", cdim_pckd})
//checkpnt.Check("03factor_qr", minor)
// QR factorization Gs2 := [ Q3, Q4 ] * [ R3; 0 ]
lapack.Geqrf(Gs, tauG, &la.IOpt{"n", n - p}, &la.IOpt{"m", cdim_pckd},
&la.IOpt{"offseta", Gs.Rows() * p})
checkpnt.Check("10factor_qr", minor)
solve := func(x, y, z *matrix.FloatMatrix) (err error) {
// On entry, x, y, z contain bx, by, bz. On exit, they
// contain the solution x, y, W*z of
//
// [ 0 A' G'*W^{-1} ] [ x ] [bx ]
// [ A 0 0 ] * [ y ] = [by ].
// [ W^{-T}*G 0 -I ] [ W*z ] [W^{-T}*bz]
//
// The system is solved in five steps:
//
// w := W^{-T}*bz - Gs1*R1^{-T}*by
// u := R3^{-T}*Q2'*bx + Q3'*w
// W*z := Q3*u - w
// y := R1^{-1} * (Q1'*bx - Gs1'*(W*z))
// x := [ Q1, Q2 ] * [ R1^{-T}*by; R3^{-1}*u ]
minor := 0
if !checkpnt.MinorEmpty() {
minor = checkpnt.MinorTop()
}
// w := W^{-T} * bz in packed storage
scale(z, W, true, true)
pack(z, w, dims)
//checkpnt.Check("00solve_qr", minor)
// vv := [ Q1'*bx; R3^{-T}*Q2'*bx ]
blas.Copy(x, vv)
lapack.Ormqr(QA, tauA, vv, la.OptTrans)
lapack.Trtrs(Gs, vv, la.OptUpper, la.OptTrans, &la.IOpt{"n", n - p},
&la.IOpt{"offseta", Gs.Rows() * p}, &la.IOpt{"offsetb", p})
//checkpnt.Check("10solve_qr", minor)
// x[:p] := R1^{-T} * by
blas.Copy(y, x)
lapack.Trtrs(QA, x, la.OptUpper, la.OptTrans, &la.IOpt{"n", p})
//checkpnt.Check("20solve_qr", minor)
// w := w - Gs1 * x[:p]
// = W^{-T}*bz - Gs1*by
blas.GemvFloat(Gs, x, w, -1.0, 1.0, &la.IOpt{"n", p}, &la.IOpt{"m", cdim_pckd})
//checkpnt.Check("30solve_qr", minor)
// u := [ Q3'*w + v[p:]; 0 ]
// = [ Q3'*w + R3^{-T}*Q2'*bx; 0 ]
blas.Copy(w, u)
lapack.Ormqr(Gs, tauG, u, la.OptTrans, &la.IOpt{"k", n - p},
&la.IOpt{"offseta", Gs.Rows() * p}, &la.IOpt{"m", cdim_pckd})
blas.AxpyFloat(vv, u, 1.0, &la.IOpt{"offsetx", p}, &la.IOpt{"n", n - p})
blas.ScalFloat(u, 0.0, &la.IOpt{"offset", n - p})
//checkpnt.Check("40solve_qr", minor)
// x[p:] := R3^{-1} * u[:n-p]
blas.Copy(u, x, &la.IOpt{"offsety", p}, &la.IOpt{"n", n - p})
lapack.Trtrs(Gs, x, la.OptUpper, &la.IOpt{"n", n - p},
&la.IOpt{"offset", Gs.Rows() * p}, &la.IOpt{"offsetb", p})
//checkpnt.Check("50solve_qr", minor)
// x is now [ R1^{-T}*by; R3^{-1}*u[:n-p] ]
// x := [Q1 Q2]*x
lapack.Ormqr(QA, tauA, x)
//checkpnt.Check("60solve_qr", minor)
// u := [Q3, Q4] * u - w
// = Q3 * u[:n-p] - w
lapack.Ormqr(Gs, tauG, u, &la.IOpt{"k", n - p}, &la.IOpt{"m", cdim_pckd},
&la.IOpt{"offseta", Gs.Rows() * p})
blas.AxpyFloat(w, u, -1.0)
//checkpnt.Check("70solve_qr", minor)
// y := R1^{-1} * ( v[:p] - Gs1'*u )
// = R1^{-1} * ( Q1'*bx - Gs1'*u )
blas.Copy(vv, y, &la.IOpt{"n", p})
blas.GemvFloat(Gs, u, y, -1.0, 1.0, &la.IOpt{"m", cdim_pckd},
&la.IOpt{"n", p}, la.OptTrans)
lapack.Trtrs(QA, y, la.OptUpper, &la.IOpt{"n", p})
//checkpnt.Check("80solve_qr", minor)
unpack(u, z, dims)
checkpnt.Check("90solve_qr", minor)
return nil
}
return solve, err
}
return factor, nil
}