Example #1
0
/*
 * 离线模型请求串格式
 * http://127.0.0.1:8080/offline?biz=[model name]&src=[hdfs/local]&dst=[redis&local&json]
                &alpha=[0.1]&beta=[0.1]&l1=[10]&l2=[10]&dropout=[0.1]&epoch=[2]
				&push=[push step]&fetch=[fetch step]&threads=[threads number]
				&train=[train file name]&test=[test file name]&debug=[off]&thd=[threshold]
   src:训练、测试数据源为hdfs/local
   dst:模型输出到redis、local和json
   train:训练数据完整路径
   test:测试数据完整路径
*/
func (lan *Lands) offlineServeHttp(w http.ResponseWriter, par *util.ModelParam) error {
	lan.log4goline.Info("[Lands-offlineServeHttp] Begin offline learning...")
	conn := lan.pool.Get()
	defer conn.Close()
	var base_path_off, model_path, train_path, test_path string
	var predict_path string

	//建立模型训练本地路径
	base_path_off = lan.conf.DataPathBase + par.Biz + "/off/"

	timestamp := time.Now().Format(TimeFormatString)
	err := util.Mkdir(base_path_off + "/" + timestamp)
	if err != nil {
		lan.log4goline.Error("[Lands-offlineServeHttp] Offline model make local directory error." + err.Error())
		return errors.New("[Lands-offlineServeHttp] Offline model make local directory error." + err.Error())
	}

	base_path_on := lan.conf.DataPathBase + par.Biz + "/on/workspace"

	err = util.Mkdir(base_path_on)
	if err != nil {
		lan.log4goline.Error("[Lands-offlineServeHttp] Make online local workspace directory error." + err.Error())
		return errors.New("[Lands-offlineServeHttp] Make online local workspace directory error." + err.Error())
	}

	//挂载数据
	lan.log4goline.Info("[Lands-offlineServeHttp] Mount training data.")
	if par.Src == "hdfs" {
		client, err := lan.createHdfsClient()
		if err != nil {
			lan.log4goline.Error("[Lands-offlineServeHttp] Create hdfs client error." + err.Error())
			return errors.New("[Lands-offlineServeHttp] Create hdfs client error." + err.Error())
		}

		model_path = base_path_off + "/" + timestamp + "/model.dat"
		train_path = base_path_off + "/" + timestamp + "/train.dat"
		test_path = base_path_off + "/" + timestamp + "/test.dat"
		predict_path = base_path_off + "/" + timestamp + "/predict.dat"

		client.GetMerge(par.Train, train_path, false)
		if err != nil {
			lan.log4goline.Error("[Lands-offlineServeHttp] Getmerge train data from hdfs to local error." + err.Error())
			return errors.New("[Lands-offlineServeHttp] Getmerge train data from hdfs to local error." + err.Error())
		}

		client.GetMerge(par.Test, test_path, false)
		if err != nil {
			lan.log4goline.Error("[Lands-offlineServeHttp] Getmerge test data from hdfs to local error." + err.Error())
			return errors.New("[Lands-offlineServeHttp] Getmerge test data from hdfs to local error." + err.Error())
		}

	} else if par.Src == "local" {
		model_path = base_path_off + "/" + timestamp + "/model.dat"
		train_path = base_path_off + "/" + timestamp + "/train.dat"
		test_path = base_path_off + "/" + timestamp + "/test.dat"
		predict_path = base_path_off + "/" + timestamp + "/predict.dat"
		err := util.CopyFile(train_path, par.Train)
		if err != nil {
			lan.log4goline.Error("[Lands-offlineServeHttp] Copy train data from local to local error." + err.Error())
			return errors.New("[Lands-offlineServeHttp] Copy train data from local to local error." + err.Error())
		}
		err = util.CopyFile(test_path, par.Test)
		if err != nil {
			lan.log4goline.Error("[Lands-offlineServeHttp] Copy test data from local to local error." + err.Error())
			return errors.New("[Lands-offlineServeHttp] Copy test data from local to local error." + err.Error())
		} else {
			lan.log4goline.Error("[Lands-offlineServeHttp] Training data source path error.")
			return errors.New("[Lands-offlineServeHttp] Training data source path error.")
		}
	}

	lan.log4goline.Info(fmt.Sprintf("[Lands-offlineServeHttp] Model path=%s,train path=%s, test path=%s\n",
		model_path,
		train_path,
		test_path))

	//训练数据格式检查及转换
	lan.log4goline.Info("[Lands-offlineServeHttp] Check training data.")
	_, err = lan.checkData(train_path)
	if err != nil {
		lan.log4goline.Error("[Lands-offlineServeHttp] Check train data from local to local error." + err.Error())
		return errors.New("[Lands-offlineServeHttp] Check train data from local to local error." + err.Error())
	}
	lan.log4goline.Info("[Lands-offlineServeHttp] Check testing data.")
	_, err = lan.checkData(test_path)
	if err != nil {
		lan.log4goline.Error("[Lands-offlineServeHttp] Check test data from local to local error." + err.Error())
		return errors.New("[Lands-offlineServeHttp] Check test data from local to local error." + err.Error())
	}

	//数据抽样
	if !util.UtilFloat64Equal(par.Sample, 1.0) && !util.UtilFloat64Equal(par.Sample, -1.0) {
		lan.log4goline.Info("[Lands-offlineServeHttp] Training data sampling.")
		err = util.FileSampleWithRatio(train_path, par.Sample)
		if err != nil {
			lan.log4goline.Error("[Lands-offlineServeHttp] Train data sampling error." + err.Error())
			return errors.New("[Lands-offlineServeHttp] Train data sampling error." + err.Error())
		}
	}

	//模型训练
	lan.log4goline.Info("[Lands-offlineServeHttp] Offline model training.")
	var fft trainer.FastFtrlTrainer
	fft.SetJobName(par.Biz + " offline " + timestamp)
	if !fft.Initialize(par.Epoch, par.Threads, true, 0, par.Push, par.Fetch) {
		lan.log4goline.Error("[Lands-offlineServeHttp] Initialize ftrl trainer error")
		return errors.New("[Lands-offlineServeHttp] Initialize ftrl trainer error.")
	}

	err = fft.Train(par.Alpha, par.Beta, par.L1, par.L2, par.Dropout, model_path,
		train_path, test_path)
	if err != nil {
		lan.log4goline.Error("[Lands-offlineServeHttp] Training model error." + err.Error())
		return errors.New("[Lands-offlineServeHttp] Training model error." + err.Error())
	}

	lan.log4goline.Info("[Lands-offlineServeHttp] Predict testing data.")
	_, err = predictor.Run(3, []string{par.Biz + " offline " + timestamp, test_path,
		model_path,
		predict_path,
		par.Threshold})

	if err != nil {
		lan.log4goline.Error("[Lands-offlineServeHttp] Predicting model error." + err.Error())
		return errors.New("[Lands-offlineServeHttp] Predicting model error." + err.Error())
	}

	err = util.CopyFile(base_path_on+"/model.dat", base_path_off+"/"+timestamp+"/model.dat")
	if err != nil {
		lan.log4goline.Error("[Lands-offlineServeHttp] Copy model error." + err.Error())
		return errors.New("[Lands-offlineServeHttp] Copy model error." + err.Error())
	}

	//清理目录
	lan.log4goline.Info("[Lands-offlineServeHttp] Clear local directory." + base_path_off)
	err = util.KeepLatestN(base_path_off, 5)
	if err != nil {
		lan.log4goline.Warn("[Lands-offlineServeHttp] Clear local file error." + err.Error())
		errors.New("[Lands-offlineServeHttp] Clear local file error." + err.Error())
	}

	m, err := fft.ParamServer.SaveEncodeModel()
	if err != nil {
		lan.log4goline.Error("[Lands-offlineServeHttp] Save model error." + err.Error())
		return errors.New("[Lands-offlineServeHttp] Save model error." + err.Error())
	}

	fmt.Fprintf(w, "%s", m)

	//模型存入redis
	lan.log4goline.Info("[Lands-offlineServeHttp] Write model to redis.")
	if par.Dst == "redis" {
		key := ModelPrefix + par.Biz
		conn.Send("SET", key, m)
		conn.Flush()
		_, err := conn.Receive()
		if err != nil {
			lan.log4goline.Error(fmt.Sprintln("[Lands-offlineServeHttp] Save model to redis error." + err.Error()))
			return errors.New("[Lands-offlineServeHttp] Save model to redis error." + err.Error())
		}
	}

	lan.log4goline.Info("[Lands-offlineServeHttp] End offline learning.")
	return nil
}
Example #2
0
/*
 * 离线模型请求串格式
 * http://127.0.0.1:8080/predict?biz=[model name]&src=[hdfs/redis/stream]&dst=[local&json]
				&pred=[hdfs file/redis key/instance strings]&debug=[off]&thd=[threshold]
   src:待预测数据源为hdfs/local
   dst:待预测数据输出到local和json
   pred:待预测数据完整路径
*/
func (lan *Lands) predictServeHttp(w http.ResponseWriter, par *util.ModelParam) error {
	lan.log4goline.Info("[Lands-predictServeHttp] Begin predicting...")
	conn := lan.pool.Get()
	defer conn.Close()
	var predict_path string

	base_path_prd := lan.conf.DataPathBase + par.Biz + "/prd/"

	timestamp := time.Now().Format(TimeFormatString)
	base_path_prdtime := base_path_prd + "/" + timestamp

	err := util.Mkdir(base_path_prdtime)
	if err != nil {
		fmt.Fprintf(w, fmt.Sprintf(JsonError, "[Lands-predictServeHttp] make local directory error."+err.Error()))
		return errors.New("[Lands-predictServeHttp] make local directory error." + err.Error())
	}

	base_path_ws := lan.conf.DataPathBase + par.Biz + "/on/workspace/"

	//清理目录
	lan.log4goline.Info("[Lands-predictServeHttp] Clear local directory.")
	err = util.KeepLatestN(base_path_prd, 5)
	if err != nil {
		lan.log4goline.Warn("[Lands-onlineServeHttp] Clear local file error." + err.Error())
		errors.New("[Lands-onlineServeHttp] Clear local file error." + err.Error())
	}

	//模型本地备份
	err = util.CopyFile(base_path_ws+"model_bak.dat", base_path_ws+"model.dat")
	if err != nil {
		lan.log4goline.Error("[Lands-predictServeHttp] Online model write local error." + err.Error())
		return errors.New("[Lands-predictServeHttp] Online model write local error." + err.Error())
	}

	//读取离线模型
	//优先读取redis模型
	lan.log4goline.Info("[Lands-predictServeHttp] Read model from redis.")
	key := ModelPrefix + par.Biz
	var encodemodel string
	reply, err := redis.Values(conn.Do("MGET", key))
	if err != nil {
		lan.log4goline.Error(fmt.Sprintf("[Lands-predictServeHttp] Get model from redis error." + err.Error()))
	} else {
		if _, err := redis.Scan(reply, &encodemodel); err != nil {
			lan.log4goline.Error(fmt.Sprintf("[Lands-onlineServeHttp] Open offline model from redis error." + err.Error()))
		}
	}

	//从redis取到了模型
	if len(encodemodel) != 0 {
		fout, err := os.OpenFile(base_path_ws+"model.dat", os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644)
		defer fout.Close()
		if err != nil {
			lan.log4goline.Error("[Lands-predictServeHttp] Online model from redis write to local error." + err.Error())
		}

		fout.WriteString(encodemodel)
	}

	//挂载数据
	lan.log4goline.Info("[Lands-predictServeHttp] Mount predicting data from local/hdfs/redis.")
	if par.Src == "local" {
		predict_path = base_path_prdtime + "/predict.dat"
		err := util.CopyFile(predict_path, par.Predict)
		if err != nil {
			lan.log4goline.Error("[Lands-predictServeHttp] copy train data from local to local error." + err.Error())
			return errors.New("[Lands-predictServeHttp] copy train data from local to local error." + err.Error())
		}
	} else if par.Src == "hdfs" {
		client, err := lan.createHdfsClient()
		if err != nil {
			lan.log4goline.Error("[Lands-predictServeHttp] Create hdfs client error." + err.Error())
			return errors.New("[Lands-predictServeHttp] Create hdfs client error." + err.Error())
		}

		predict_path = base_path_prdtime + "/predict.dat"

		client.GetMerge(par.Predict, predict_path, false)
		if err != nil {
			lan.log4goline.Error("[Lands-predictServeHttp] Getmerge train data from hdfs to local error." + err.Error())
			return errors.New("[Lands-predictServeHttp] Getmerge train data from hdfs to local error." + err.Error())
		}

	} else if par.Src == "redis" {
		predict_path = base_path_prdtime + "/predict.dat"
		//redis get with key
	} else {
		// run stream
		instances := strings.Split(strings.TrimSpace(par.Predict), lan.conf.LineSpliter)
		if len(instances) == 0 {
			lan.log4goline.Error("[Lands-predictServeHttp] Streaming instances number error.")
			return errors.New("[Lands-predictServeHttp] Streaming instances number error.")
		}

		json, err := predictor.StreamRun(base_path_ws+"model.dat", instances)
		if err != nil {
			lan.log4goline.Error("[Lands-predictServeHttp] Streaming predicting running error." + err.Error())
			return errors.New("[Lands-predictServeHttp] Streaming predicting running error." + err.Error())
		}

		fmt.Fprintf(w, json)
		lan.log4goline.Info("[Lands-predictServeHttp] End predicting.")
		return nil
	}

	json, err := predictor.Run(3, []string{par.Biz + " predict " + timestamp, predict_path,
		base_path_ws + "model.dat",
		base_path_prdtime + "/predict_result.dat",
		par.Threshold})
	if err != nil {
		lan.log4goline.Error("[Lands-predictServeHttp] Predicting running error." + err.Error())
		return errors.New("[Lands-predictServeHttp] Predicting running error." + err.Error())
	}

	fmt.Fprintf(w, json)
	lan.log4goline.Info("[Lands-predictServeHttp] End predicting.")

	return nil
}
Example #3
0
/*
 * 在线模型请求串格式
 * http://127.0.0.1:8080/online?biz=[model name]&src=[redis&stream]&dst=[redis&local&json]
                &epoch=[2]&threads=[threads number]&train=[redis key/instance strings]
				&debug=[off]&thd=[threshold]
   src:训练数据为redis还是即时stream (初始化模型如果redis不存在则读取local,模型key为biz值)
   dst:模型存储到redis、local和json
   train:训练数据来源于redis或stream
*/
func (lan *Lands) onlineServeHttp(w http.ResponseWriter, par *util.ModelParam) error {
	lan.log4goline.Info("[Lands-onlineServeHttp] Begin online learning...")
	conn := lan.pool.Get()
	defer conn.Close()
	var base_path_on string

	base_path_on = lan.conf.DataPathBase + par.Biz + "/on/"

	timestamp := time.Now().Format(TimeFormatString)
	err := util.Mkdir(base_path_on + "/" + timestamp)
	if err != nil {
		lan.log4goline.Error(fmt.Sprintf(fmt.Sprintf(JsonError, "[Lands-onlineServeHttp] Online make local directory error."+err.Error())))
		return errors.New("[Lands-onlineServeHttp] Online make local directory error." + err.Error())
	}

	base_path_ws := base_path_on + "workspace/"

	err = util.Mkdir(base_path_ws)
	if err != nil {
		lan.log4goline.Error(fmt.Sprintf(fmt.Sprintf(JsonError, "[Lands-onlineServeHttp] Online make local workspace directory error."+err.Error())))
		return errors.New("[Lands-onlineServeHttp] Online make local workspace directory error." + err.Error())
	}

	//读取离线模型
	//优先读取redis
	lan.log4goline.Info("[Lands-onlineServeHttp] Read online model.")
	var encodemodel string
	key := ModelPrefix + par.Biz
	reply, err := redis.Values(conn.Do("MGET", key))
	if err != nil {
		//redis读取失败则读本地模型
		fs, err := os.Open(base_path_ws + "model.dat")
		if err != nil {
			lan.log4goline.Error(fmt.Sprintf(fmt.Sprintf(JsonError, "[Lands-onlineServeHttp] Offline model must be trained."+err.Error())))
			return errors.New("[Lands-onlineServeHttp] Offline model must be trained." + err.Error())
		}

		defer fs.Close()

		buf := bufio.NewReader(fs)

		line, err := buf.ReadString('\n')
		encodemodel = s.TrimSpace(line)
		if err != nil && err != io.EOF {
			lan.log4goline.Error(fmt.Sprintf(fmt.Sprintf(JsonError, "[Lands-onlineServeHttp] Open offline model error."+err.Error())))
			return errors.New("[Lands-onlineServeHttp] Open offline model error." + err.Error())
		}
	} else {
		if _, err := redis.Scan(reply, &encodemodel); err != nil {
			lan.log4goline.Error(fmt.Sprintln("[Lands-onlineServeHttp] Open offline model from redis error." + err.Error()))
			return errors.New("[Lands-onlineServeHttp] Open offline model from redis error." + err.Error())
		}
	}

	//模型训练
	lan.log4goline.Info("[Lands-onlineServeHttp] Online model training with lock free ftrl.")
	var lff trainer.LockFreeFtrlTrainer
	lff.SetJobName(par.Biz + " online " + timestamp)
	if !lff.Initialize(par.Epoch, par.Threads, false) {
		lan.log4goline.Error("[Lands-onlineServeHttp] Initialize offline model error.")
		return errors.New("[Lands-onlineServeHttp] Initialize offline model error.")
	}
	instances := strings.Split(par.Train, lan.conf.LineSpliter)
	if len(instances) == 0 {
		lan.log4goline.Error("[Lands-onlineServeHttp] Instances number error.")
		return errors.New("[Lands-onlineServeHttp] Instances number error.")
	}
	model, err := lff.TrainOnline(encodemodel, instances)
	if err != nil {
		lan.log4goline.Error("[Lands-onlineServeHttp] Online model training error." + err.Error())
		return errors.New("[Lands-onlineServeHttp] Online model training error." + err.Error())
	}

	//模型写入本地
	lan.log4goline.Info("[Lands-onlineServeHttp] Write model to local directory.")
	util.CopyFile(base_path_ws+"model_bak.dat", base_path_ws+"model.dat")
	fout, err := os.OpenFile(base_path_on+"/"+timestamp+"/model.dat", os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644)
	defer fout.Close()
	if err != nil {
		lan.log4goline.Error("[Lands-onlineServeHttp] Online model write local error." + err.Error())
		return errors.New("[Lands-onlineServeHttp] Online model write local error." + err.Error())
	}

	fout.WriteString(model)
	err = util.CopyFile(base_path_ws+"model.dat", base_path_on+"/"+timestamp+"/model.dat")
	if err != nil {
		lan.log4goline.Error("[Lands-onlineServeHttp] Copy model error." + err.Error())
		return errors.New("[Lands-onlineServeHttp] Copy model error." + err.Error())
	}

	//清理目录
	lan.log4goline.Info("[Lands-onlineServeHttp] Clear local directory.")
	err = util.KeepLatestN(base_path_on, 5)
	if err != nil {
		lan.log4goline.Warn("[Lands-onlineServeHttp] Clear local file error." + err.Error())
		errors.New("[Lands-onlineServeHttp] Clear local file error." + err.Error())
	}

	//接口输出
	fmt.Fprintf(w, model)

	//模型存入redis
	lan.log4goline.Info("[Lands-onlineServeHttp] Write model to redis.")
	if par.Dst == "redis" {
		key := ModelPrefix + par.Biz
		conn.Send("SET", key, model)
		conn.Flush()
		_, err := conn.Receive()
		if err != nil {
			lan.log4goline.Warn(fmt.Sprintln("[Lands-onlineServeHttp] Save model to redis error." + err.Error()))
			return errors.New("[Lands-onlineServeHttp] Save model to redis error." + err.Error())
		}
	}

	lan.log4goline.Info("[Lands-onlineServeHttp] End online learning.")

	return nil
}