OWLQN

Logistic Regression

OWLQN

• BFGS, LBFGS: Anders Skajaa.Limited Memory BFGS for Nonsmooth Optimization.2010
• LBFGS:J. Nocedal and S. Wright. Numerical Optimization. Springer, 2nd
  edition, 2006.
• OWLQN:Galen Andrew and Jianfeng Gao.Scalable Training of L1-Regularized Log-Linear Models.2007

OWLQN

Galen Andrew and Jianfeng Gao. 2007

//OWLQN
//dir
void OptimizerState::MakeSteepestDescDir() {

	if (l1weight == 0) {
		//l10dir
		scaleInto(dir, grad, -1);
	} else {
		//l10l1
		for (size_t i=0; i<dim; i++) {
			if (x[i] < 0) {
				//xi<0|xi| = - xil1-l1weight
				dir[i] = -grad[i] + l1weight;
			} else if (x[i] > 0) {
				//xi>0|xi| = xil1l1weight
				dir[i] = -grad[i] - l1weight;
			} else {//xi == 0
				if (grad[i] < -l1weight) {
					//xi == 0grad[i] + l1weight < 0dir[i] > 0
					dir[i] = -grad[i] - l1weight;
				} else if (grad[i] > l1weight) {
					//xi == 0grad[i] - l1weight > 0dir[i] < 0
					dir[i] = -grad[i] + l1weight;
				} else {
					//xi == 000
					dir[i] = 0;
				}
			}
		}
	}

	//
	steepestDescDir = dir;
}

• Galen Andrew and Jianfeng Gao. 2007