//ref LIBLINEAR -- A Library for Large Linear Classification
func get_upper_bound(p *readData.Problem, dr float32, j int) float32 {
var upper_1 float32
var upper_2 float32
var s1 float32
var s2 float32
var s_y_1 float32
var s_y_0 float32 //for label -1
nnz := len(p.A_cols[j].Idxs)
for i := 0; i < len(p.A_cols[j].Idxs); i++ {
sample_index := p.A_cols[j].Idxs[i]
s1 = s1 + p.A_cols[j].Values[i]/(1.0+(mathOp.Exp(p.Ax[sample_index])))
s2 = s2 + p.A_cols[j].Values[i]/(1.0+(mathOp.Exp(-p.Ax[sample_index])))
if p.Labels[sample_index] == 1 {
s_y_1 = s_y_1 + p.A_cols[j].Values[i]
} else {
s_y_0 = s_y_0 + p.A_cols[j].Values[i]
}
}
s1 = s1 / float32(p.L)
s1 = s1 * (mathOp.Exp(-dr*p.Xj_max[j]) - 1.0)
s1 = s1 / p.Xj_max[j] * float32(nnz) / float32(p.L)
upper_1 = float32(nnz) / float32(p.L) * (mathOp.Log(1.0 + s1))
s_y_0 = dr*s_y_0/float32(p.L) + p.Lambda*(mathOp.Abs(p.X[j]+dr)-mathOp.Abs(p.X[j]))
upper_1 = upper_1 + s_y_0
s2 = s2 / float32(p.L)
s2 = s2 * (mathOp.Exp(dr*p.Xj_max[j]) - 1.0)
s2 = s2 / p.Xj_max[j] * float32(nnz) / float32(p.L)
upper_2 = float32(nnz) / float32(p.L) * (mathOp.Log((1.0 + s2)))
s_y_1 = -dr*s_y_1/float32(p.L) + p.Lambda*(mathOp.Abs(p.X[j]+dr)-mathOp.Abs(p.X[j]))
upper_2 = upper_2 + s_y_1
if upper_1 < upper_2 {
return upper_1
} else {
return upper_2
}
}
//0<sigma<1, 0<r<1
func Solve_lr_new_glmnet_cdn(p *readData.Problem, sigma float32, lambda float32) {
obj_old := get_obj_lr(p)
fmt.Printf("iter: 0, obj: %f\n", obj_old)
for i := 1; i <= p.Max_iter; i++ {
// shuffle should be added here
for j := 0; j < p.N; j++ {
if j%10000 == 0 {
// fmt.Printf("j: %d g_j = %f, H_jj = %f, d=%f\n", j, g_j, H_jj, lambda*d)
obj_new := get_obj_lr(p)
fmt.Printf(" iter: %d, obj: %f\n", j, obj_new)
}
if len(p.A_cols[i].Idxs) == 0 {
continue
}
g_j, H_jj := get_g_j_and_H_jj(p, j)
if H_jj < 1e-8 || mathOp.Abs(g_j) < 1e-8 {
continue
}
d := update_d(g_j, H_jj, p.X[j], p.Lambda)
if mathOp.Abs(d) < 1e-8 {
continue
}
// fmt.Printf("j: %d g_j = %f, H_jj = %f, d=%f\n", j, g_j, H_jj, d)
// loss_old := get_obj_lr(p)
var r float32
r = 1.0
right_hand_size := g_j*d + p.Lambda*(mathOp.Abs(p.X[j]+d)-mathOp.Abs(p.X[j]))
//compute upper bound of left hand side of (45)
var upper_1 float32
var upper_2 float32
var upper float32
var s1 float32
var s2 float32
var ss1 float32
var ss2 float32
var s_y_1 float32
var s_y_0 float32 //for label -1
var ss_y_1 float32
var ss_y_0 float32
nnz := len(p.A_cols[j].Idxs)
for i := 0; i < len(p.A_cols[j].Idxs); i++ {
sample_index := p.A_cols[j].Idxs[i]
s1 = s1 + p.A_cols[j].Values[i]/(1.0+(mathOp.Exp(p.Ax[sample_index])))
s2 = s2 + p.A_cols[j].Values[i]/(1.0+(mathOp.Exp(-p.Ax[sample_index])))
if p.Labels[sample_index] == 1 {
s_y_1 = s_y_1 + p.A_cols[j].Values[i]
} else {
s_y_0 = s_y_0 + p.A_cols[j].Values[i]
}
}
s1 = s1 / float32(p.L)
s2 = s2 / float32(p.L)
for {
// loss_new := get_obj_with_d(p, d, j, r) //expensive
ss1 = s1 * (mathOp.Exp(-d*r*p.Xj_max[j]) - 1.0)
ss1 = ss1 / p.Xj_max[j] * float32(nnz) / float32(p.L)
upper_1 = float32(nnz) / float32(p.L) * mathOp.Log(1.0+ss1)
ss_y_0 = d*r*s_y_0/float32(p.L) + p.Lambda*(mathOp.Abs(p.X[j]+d*r)-mathOp.Abs(p.X[j]))
upper_1 = upper_1 + ss_y_0
ss2 = s2 * (mathOp.Exp(d*r*p.Xj_max[j]) - 1.0)
ss2 = ss2 / p.Xj_max[j] * float32(nnz) / float32(p.L)
upper_2 = float32(nnz) / float32(p.L) * mathOp.Log(1.0+ss2)
ss_y_1 = -d*r*s_y_1/float32(p.L) + p.Lambda*(mathOp.Abs(p.X[j]+d*r)-mathOp.Abs(p.X[j]))
upper_2 = upper_2 + ss_y_1
if upper_1 < upper_2 {
upper = upper_1
} else {
upper = upper_2
}
// upper := get_upper_bound(p, d*r, j)
if upper < sigma*r*right_hand_size {
break
}
r = r * lambda
// fmt.Printf("r=%f\n", r)
}
update_Ax(p, p.X[j], p.X[j]+r*d, j)
p.X[j] = p.X[j] + r*d
}
obj_new := get_obj_lr(p)
// if i%10 == 0 {
fmt.Printf("iter: %d, obj: %f\n", i, obj_new)
// }
if mathOp.Abs(obj_new-obj_old) < p.Epsilon*obj_old {
break
}
obj_old = obj_new
}
}