コード例 #1
0
//compute x_i while fixing the other variable
func solve_lasso_CD(p *readData.Problem) {

	//initial pred is zeros, since x is zero vector, hence the residual is just the label vector
	residual := make([]float32, p.L)
	z := make([]float32, p.L)
	for i := 0; i < p.L; i++ {
		residual[i] = float32(p.Labels[i])
	}
	copy(z, residual)
	//	pred := make([]float32, p.L)
	fea_square := make([]float32, p.N)
	for i := 0; i < p.N; i++ {
		fea_square[i] = p.A_cols[i].Multiply_sparse_vector(&(p.A_cols[i]))
	}
	//	fmt.Printf("%v ", fea_square)
	obj_old := get_obj(p)
	fmt.Printf("obj: %f\n", obj_old)
	var iter int
	for iter = 1; iter < p.Max_iter; iter++ {
		for n := 0; n < p.N; n++ {
			update_z(z, residual, p, n)
			temp := p.A_cols[n].Multiply_dense_array(z) / fea_square[n]
			p.X[n] = soft_threshold(temp, float32(p.L)*p.Lambda/fea_square[n])
			update_residual(residual, z, p, n)
		}
		obj_new := get_obj(p)
		fmt.Printf("obj: %f\n", obj_new)
		if mathOp.Abs(obj_new-obj_old) < p.Epsilon {
			break
		}
		obj_old = obj_new
	}
	fmt.Printf("converged in %d iterations\n", iter)
}
//ref LIBLINEAR -- A Library for Large Linear Classification
func get_upper_bound(p *readData.Problem, dr float32, j int) float32 {
	var upper_1 float32
	var upper_2 float32
	var s1 float32
	var s2 float32
	var s_y_1 float32
	var s_y_0 float32 //for label -1
	nnz := len(p.A_cols[j].Idxs)
	for i := 0; i < len(p.A_cols[j].Idxs); i++ {
		sample_index := p.A_cols[j].Idxs[i]
		s1 = s1 + p.A_cols[j].Values[i]/(1.0+(mathOp.Exp(p.Ax[sample_index])))
		s2 = s2 + p.A_cols[j].Values[i]/(1.0+(mathOp.Exp(-p.Ax[sample_index])))
		if p.Labels[sample_index] == 1 {
			s_y_1 = s_y_1 + p.A_cols[j].Values[i]
		} else {
			s_y_0 = s_y_0 + p.A_cols[j].Values[i]
		}
	}
	s1 = s1 / float32(p.L)
	s1 = s1 * (mathOp.Exp(-dr*p.Xj_max[j]) - 1.0)
	s1 = s1 / p.Xj_max[j] * float32(nnz) / float32(p.L)
	upper_1 = float32(nnz) / float32(p.L) * (mathOp.Log(1.0 + s1))
	s_y_0 = dr*s_y_0/float32(p.L) + p.Lambda*(mathOp.Abs(p.X[j]+dr)-mathOp.Abs(p.X[j]))
	upper_1 = upper_1 + s_y_0

	s2 = s2 / float32(p.L)
	s2 = s2 * (mathOp.Exp(dr*p.Xj_max[j]) - 1.0)
	s2 = s2 / p.Xj_max[j] * float32(nnz) / float32(p.L)
	upper_2 = float32(nnz) / float32(p.L) * (mathOp.Log((1.0 + s2)))
	s_y_1 = -dr*s_y_1/float32(p.L) + p.Lambda*(mathOp.Abs(p.X[j]+dr)-mathOp.Abs(p.X[j]))
	upper_2 = upper_2 + s_y_1

	if upper_1 < upper_2 {
		return upper_1
	} else {
		return upper_2
	}
}
コード例 #3
0
func get_obj_lr(p *readData.Problem) float32 {
	var loss float32
	for i := 0; i < p.L; i++ {
		//		ti := (1.0 + float32(math.Exp(float64(p.A_rows[i].Multiply_dense_array(p.X)*float32(-p.Labels[i])))))
		ti := (1.0 + float32(math.Exp(float64(p.Ax[i]*float32(-p.Labels[i])))))
		ti = float32(math.Log(float64(ti)))
		loss = loss + ti
	}
	loss = loss / float32(p.L)
	var l1 float32
	for i := 0; i < p.N; i++ {
		l1 = l1 + mathOp.Abs(p.X[i])
	}
	return loss + l1*p.Lambda
}
コード例 #4
0
//solve lasso with coordinate descent
func get_obj(p *readData.Problem) float32 {
	var loss float32
	var l1 float32
	var t float32
	loss = 0
	l1 = 0
	for i := 0; i < p.L; i++ {
		t = float32(p.Labels[i]) - p.A_rows[i].Multiply_dense_array(p.X)
		loss = loss + t*t
	}
	loss = loss / float32(p.L) / 2.0
	for i := 0; i < p.N; i++ {
		l1 = l1 + mathOp.Abs(p.X[i])
	}
	return loss + l1*p.Lambda
}
コード例 #5
0
func get_obj(p *readData.Problem, u []float32, w []float32) float32 {
	var loss float32
	var l1 float32
	var t float32
	loss = 0
	l1 = 0
	for i := 0; i < p.L; i++ {
		//		t = u[i] - p.A_rows[i].Multiply_dense_array(p.X)
		t = u[i] - p.Ax[i]
		loss = loss + t*t*w[i]
	}
	for i := 0; i < p.N; i++ {
		l1 = l1 + mathOp.Abs(p.X[i])
	}
	return loss + l1*p.Lambda
}
コード例 #6
0
//TODO  speed it up (liblinear)
func Solve_lr_CD(p *readData.Problem) {
	obj_old := get_obj_lr(p)
	fmt.Printf("iter: 0, obj: %f\n", obj_old)
	for i := 1; i <= p.Max_iter; i++ {
		u, w := get_u_and_w(p)
		solve_weighted_lasso_CD(p, u, w)
		obj_new := get_obj_lr(p)
		if i%10 == 0 {
			fmt.Printf("iter: %d, obj: %f\n", i, obj_new)
		}
		if mathOp.Abs(obj_new-obj_old) < p.Epsilon*obj_old {
			break
		}
		obj_old = obj_new
	}
}
コード例 #7
0
// using CD to solve weigthed lasso
//compute x_i while fixing the other variable
func solve_weighted_lasso_CD(p *readData.Problem, u []float32, w []float32) {
	//initial pred is zeros, since x is zero vector, hence the residual is just the label vector
	residual := make([]float32, p.L)
	z := make([]float32, p.L)
	for i := 0; i < p.L; i++ {
		//		residual[i] = u[i] - p.A_rows[i].Multiply_dense_array(p.X)
		residual[i] = u[i] - p.Ax[i]
	}
	copy(z, residual) // can  also be deleted
	//	pred := make([]float32, p.L)
	fea_weithed_norm := make([]float32, p.N) // time consuming
	for i := 0; i < p.N; i++ {
		temp := p.A_cols[i].Dot_product(w)
		fea_weithed_norm[i] = p.A_cols[i].Multiply_sparse_vector(temp)
	}
	//	fmt.Printf("%v ", fea_square)
	obj_old := get_obj(p, u, w)
	//	fmt.Printf("    inner obj: %f\n", obj_old)
	var iter int
	for iter = 1; iter < 100; iter++ {
		for n := 0; n < p.N; n++ {
			weighted_lasso_update_z(z, residual, p, n)
			//			weighted_lasso_update_z_0(z, p, n)
			//only nonzeros entries of z are used
			temp := p.A_cols[n].Multiply_dense_array_weithted(z, w) / fea_weithed_norm[n]
			x_new := soft_threshold(temp, p.Lambda/fea_weithed_norm[n]/2.0)
			update_Ax(p, p.X[n], x_new, n)
			p.X[n] = x_new
			weighted_lasso_update_residual(residual, z, p, n)
			//			weighted_lasso_update_z_1(z, p, n)
		}
		obj_new := get_obj(p, u, w)
		if obj_new > obj_old {
			fmt.Printf("    wrong\n")
		}
		//		fmt.Printf("    inner obj: %f\n", obj_new)
		if mathOp.Abs(obj_new-obj_old) < 0.1*obj_old { //!!
			break
		}
		obj_old = obj_new
	}
	//	fmt.Printf("   inner converged in %d iterations\n", iter)
}
//0<sigma<1, 0<r<1
func Solve_lr_new_glmnet_cdn(p *readData.Problem, sigma float32, lambda float32) {
	obj_old := get_obj_lr(p)
	fmt.Printf("iter: 0, obj: %f\n", obj_old)

	for i := 1; i <= p.Max_iter; i++ {
		// shuffle should be added here
		for j := 0; j < p.N; j++ {
			if j%10000 == 0 {
				//				fmt.Printf("j: %d    g_j = %f, H_jj = %f, d=%f\n", j, g_j, H_jj, lambda*d)
				obj_new := get_obj_lr(p)
				fmt.Printf("    iter: %d, obj: %f\n", j, obj_new)
			}
			if len(p.A_cols[i].Idxs) == 0 {
				continue
			}
			g_j, H_jj := get_g_j_and_H_jj(p, j)
			if H_jj < 1e-8 || mathOp.Abs(g_j) < 1e-8 {
				continue
			}

			d := update_d(g_j, H_jj, p.X[j], p.Lambda)
			if mathOp.Abs(d) < 1e-8 {
				continue
			}
			//			fmt.Printf("j: %d    g_j = %f, H_jj = %f, d=%f\n", j, g_j, H_jj, d)
			//			loss_old := get_obj_lr(p)
			var r float32
			r = 1.0
			right_hand_size := g_j*d + p.Lambda*(mathOp.Abs(p.X[j]+d)-mathOp.Abs(p.X[j]))

			//compute upper bound of left hand side of (45)
			var upper_1 float32
			var upper_2 float32
			var upper float32
			var s1 float32
			var s2 float32
			var ss1 float32
			var ss2 float32
			var s_y_1 float32
			var s_y_0 float32 //for label -1
			var ss_y_1 float32
			var ss_y_0 float32
			nnz := len(p.A_cols[j].Idxs)

			for i := 0; i < len(p.A_cols[j].Idxs); i++ {
				sample_index := p.A_cols[j].Idxs[i]
				s1 = s1 + p.A_cols[j].Values[i]/(1.0+(mathOp.Exp(p.Ax[sample_index])))
				s2 = s2 + p.A_cols[j].Values[i]/(1.0+(mathOp.Exp(-p.Ax[sample_index])))
				if p.Labels[sample_index] == 1 {
					s_y_1 = s_y_1 + p.A_cols[j].Values[i]
				} else {
					s_y_0 = s_y_0 + p.A_cols[j].Values[i]
				}
			}
			s1 = s1 / float32(p.L)
			s2 = s2 / float32(p.L)
			for {
				//				loss_new := get_obj_with_d(p, d, j, r) //expensive
				ss1 = s1 * (mathOp.Exp(-d*r*p.Xj_max[j]) - 1.0)
				ss1 = ss1 / p.Xj_max[j] * float32(nnz) / float32(p.L)
				upper_1 = float32(nnz) / float32(p.L) * mathOp.Log(1.0+ss1)
				ss_y_0 = d*r*s_y_0/float32(p.L) + p.Lambda*(mathOp.Abs(p.X[j]+d*r)-mathOp.Abs(p.X[j]))
				upper_1 = upper_1 + ss_y_0

				ss2 = s2 * (mathOp.Exp(d*r*p.Xj_max[j]) - 1.0)
				ss2 = ss2 / p.Xj_max[j] * float32(nnz) / float32(p.L)
				upper_2 = float32(nnz) / float32(p.L) * mathOp.Log(1.0+ss2)
				ss_y_1 = -d*r*s_y_1/float32(p.L) + p.Lambda*(mathOp.Abs(p.X[j]+d*r)-mathOp.Abs(p.X[j]))
				upper_2 = upper_2 + ss_y_1

				if upper_1 < upper_2 {
					upper = upper_1
				} else {
					upper = upper_2
				}
				//				upper := get_upper_bound(p, d*r, j)
				if upper < sigma*r*right_hand_size {
					break
				}
				r = r * lambda
				//				fmt.Printf("r=%f\n", r)
			}

			update_Ax(p, p.X[j], p.X[j]+r*d, j)
			p.X[j] = p.X[j] + r*d
		}

		obj_new := get_obj_lr(p)
		//		if i%10 == 0 {
		fmt.Printf("iter: %d, obj: %f\n", i, obj_new)
		//		}
		if mathOp.Abs(obj_new-obj_old) < p.Epsilon*obj_old {
			break
		}
		obj_old = obj_new
	}
}