/* * 离线模型请求串格式 * http://127.0.0.1:8080/predict?biz=[model name]&src=[hdfs/redis/stream]&dst=[local&json] &pred=[hdfs file/redis key/instance strings]&debug=[off]&thd=[threshold] src:待预测数据源为hdfs/local dst:待预测数据输出到local和json pred:待预测数据完整路径 */ func (lan *Lands) predictServeHttp(w http.ResponseWriter, par *util.ModelParam) error { lan.log4goline.Info("[Lands-predictServeHttp] Begin predicting...") conn := lan.pool.Get() defer conn.Close() var predict_path string base_path_prd := lan.conf.DataPathBase + par.Biz + "/prd/" timestamp := time.Now().Format(TimeFormatString) base_path_prdtime := base_path_prd + "/" + timestamp err := util.Mkdir(base_path_prdtime) if err != nil { fmt.Fprintf(w, fmt.Sprintf(JsonError, "[Lands-predictServeHttp] make local directory error."+err.Error())) return errors.New("[Lands-predictServeHttp] make local directory error." + err.Error()) } base_path_ws := lan.conf.DataPathBase + par.Biz + "/on/workspace/" //清理目录 lan.log4goline.Info("[Lands-predictServeHttp] Clear local directory.") err = util.KeepLatestN(base_path_prd, 5) if err != nil { lan.log4goline.Warn("[Lands-onlineServeHttp] Clear local file error." + err.Error()) errors.New("[Lands-onlineServeHttp] Clear local file error." + err.Error()) } //模型本地备份 err = util.CopyFile(base_path_ws+"model_bak.dat", base_path_ws+"model.dat") if err != nil { lan.log4goline.Error("[Lands-predictServeHttp] Online model write local error." + err.Error()) return errors.New("[Lands-predictServeHttp] Online model write local error." + err.Error()) } //读取离线模型 //优先读取redis模型 lan.log4goline.Info("[Lands-predictServeHttp] Read model from redis.") key := ModelPrefix + par.Biz var encodemodel string reply, err := redis.Values(conn.Do("MGET", key)) if err != nil { lan.log4goline.Error(fmt.Sprintf("[Lands-predictServeHttp] Get model from redis error." + err.Error())) } else { if _, err := redis.Scan(reply, &encodemodel); err != nil { lan.log4goline.Error(fmt.Sprintf("[Lands-onlineServeHttp] Open offline model from redis error." + err.Error())) } } //从redis取到了模型 if len(encodemodel) != 0 { fout, err := os.OpenFile(base_path_ws+"model.dat", os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644) defer fout.Close() if err != nil { lan.log4goline.Error("[Lands-predictServeHttp] Online model from redis write to local error." + err.Error()) } fout.WriteString(encodemodel) } //挂载数据 lan.log4goline.Info("[Lands-predictServeHttp] Mount predicting data from local/hdfs/redis.") if par.Src == "local" { predict_path = base_path_prdtime + "/predict.dat" err := util.CopyFile(predict_path, par.Predict) if err != nil { lan.log4goline.Error("[Lands-predictServeHttp] copy train data from local to local error." + err.Error()) return errors.New("[Lands-predictServeHttp] copy train data from local to local error." + err.Error()) } } else if par.Src == "hdfs" { client, err := lan.createHdfsClient() if err != nil { lan.log4goline.Error("[Lands-predictServeHttp] Create hdfs client error." + err.Error()) return errors.New("[Lands-predictServeHttp] Create hdfs client error." + err.Error()) } predict_path = base_path_prdtime + "/predict.dat" client.GetMerge(par.Predict, predict_path, false) if err != nil { lan.log4goline.Error("[Lands-predictServeHttp] Getmerge train data from hdfs to local error." + err.Error()) return errors.New("[Lands-predictServeHttp] Getmerge train data from hdfs to local error." + err.Error()) } } else if par.Src == "redis" { predict_path = base_path_prdtime + "/predict.dat" //redis get with key } else { // run stream instances := strings.Split(strings.TrimSpace(par.Predict), lan.conf.LineSpliter) if len(instances) == 0 { lan.log4goline.Error("[Lands-predictServeHttp] Streaming instances number error.") return errors.New("[Lands-predictServeHttp] Streaming instances number error.") } json, err := predictor.StreamRun(base_path_ws+"model.dat", instances) if err != nil { lan.log4goline.Error("[Lands-predictServeHttp] Streaming predicting running error." + err.Error()) return errors.New("[Lands-predictServeHttp] Streaming predicting running error." + err.Error()) } fmt.Fprintf(w, json) lan.log4goline.Info("[Lands-predictServeHttp] End predicting.") return nil } json, err := predictor.Run(3, []string{par.Biz + " predict " + timestamp, predict_path, base_path_ws + "model.dat", base_path_prdtime + "/predict_result.dat", par.Threshold}) if err != nil { lan.log4goline.Error("[Lands-predictServeHttp] Predicting running error." + err.Error()) return errors.New("[Lands-predictServeHttp] Predicting running error." + err.Error()) } fmt.Fprintf(w, json) lan.log4goline.Info("[Lands-predictServeHttp] End predicting.") return nil }
/* * 离线模型请求串格式 * http://127.0.0.1:8080/offline?biz=[model name]&src=[hdfs/local]&dst=[redis&local&json] &alpha=[0.1]&beta=[0.1]&l1=[10]&l2=[10]&dropout=[0.1]&epoch=[2] &push=[push step]&fetch=[fetch step]&threads=[threads number] &train=[train file name]&test=[test file name]&debug=[off]&thd=[threshold] src:训练、测试数据源为hdfs/local dst:模型输出到redis、local和json train:训练数据完整路径 test:测试数据完整路径 */ func (lan *Lands) offlineServeHttp(w http.ResponseWriter, par *util.ModelParam) error { lan.log4goline.Info("[Lands-offlineServeHttp] Begin offline learning...") conn := lan.pool.Get() defer conn.Close() var base_path_off, model_path, train_path, test_path string var predict_path string //建立模型训练本地路径 base_path_off = lan.conf.DataPathBase + par.Biz + "/off/" timestamp := time.Now().Format(TimeFormatString) err := util.Mkdir(base_path_off + "/" + timestamp) if err != nil { lan.log4goline.Error("[Lands-offlineServeHttp] Offline model make local directory error." + err.Error()) return errors.New("[Lands-offlineServeHttp] Offline model make local directory error." + err.Error()) } base_path_on := lan.conf.DataPathBase + par.Biz + "/on/workspace" err = util.Mkdir(base_path_on) if err != nil { lan.log4goline.Error("[Lands-offlineServeHttp] Make online local workspace directory error." + err.Error()) return errors.New("[Lands-offlineServeHttp] Make online local workspace directory error." + err.Error()) } //挂载数据 lan.log4goline.Info("[Lands-offlineServeHttp] Mount training data.") if par.Src == "hdfs" { client, err := lan.createHdfsClient() if err != nil { lan.log4goline.Error("[Lands-offlineServeHttp] Create hdfs client error." + err.Error()) return errors.New("[Lands-offlineServeHttp] Create hdfs client error." + err.Error()) } model_path = base_path_off + "/" + timestamp + "/model.dat" train_path = base_path_off + "/" + timestamp + "/train.dat" test_path = base_path_off + "/" + timestamp + "/test.dat" predict_path = base_path_off + "/" + timestamp + "/predict.dat" client.GetMerge(par.Train, train_path, false) if err != nil { lan.log4goline.Error("[Lands-offlineServeHttp] Getmerge train data from hdfs to local error." + err.Error()) return errors.New("[Lands-offlineServeHttp] Getmerge train data from hdfs to local error." + err.Error()) } client.GetMerge(par.Test, test_path, false) if err != nil { lan.log4goline.Error("[Lands-offlineServeHttp] Getmerge test data from hdfs to local error." + err.Error()) return errors.New("[Lands-offlineServeHttp] Getmerge test data from hdfs to local error." + err.Error()) } } else if par.Src == "local" { model_path = base_path_off + "/" + timestamp + "/model.dat" train_path = base_path_off + "/" + timestamp + "/train.dat" test_path = base_path_off + "/" + timestamp + "/test.dat" predict_path = base_path_off + "/" + timestamp + "/predict.dat" err := util.CopyFile(train_path, par.Train) if err != nil { lan.log4goline.Error("[Lands-offlineServeHttp] Copy train data from local to local error." + err.Error()) return errors.New("[Lands-offlineServeHttp] Copy train data from local to local error." + err.Error()) } err = util.CopyFile(test_path, par.Test) if err != nil { lan.log4goline.Error("[Lands-offlineServeHttp] Copy test data from local to local error." + err.Error()) return errors.New("[Lands-offlineServeHttp] Copy test data from local to local error." + err.Error()) } else { lan.log4goline.Error("[Lands-offlineServeHttp] Training data source path error.") return errors.New("[Lands-offlineServeHttp] Training data source path error.") } } lan.log4goline.Info(fmt.Sprintf("[Lands-offlineServeHttp] Model path=%s,train path=%s, test path=%s\n", model_path, train_path, test_path)) //训练数据格式检查及转换 lan.log4goline.Info("[Lands-offlineServeHttp] Check training data.") _, err = lan.checkData(train_path) if err != nil { lan.log4goline.Error("[Lands-offlineServeHttp] Check train data from local to local error." + err.Error()) return errors.New("[Lands-offlineServeHttp] Check train data from local to local error." + err.Error()) } lan.log4goline.Info("[Lands-offlineServeHttp] Check testing data.") _, err = lan.checkData(test_path) if err != nil { lan.log4goline.Error("[Lands-offlineServeHttp] Check test data from local to local error." + err.Error()) return errors.New("[Lands-offlineServeHttp] Check test data from local to local error." + err.Error()) } //数据抽样 if !util.UtilFloat64Equal(par.Sample, 1.0) && !util.UtilFloat64Equal(par.Sample, -1.0) { lan.log4goline.Info("[Lands-offlineServeHttp] Training data sampling.") err = util.FileSampleWithRatio(train_path, par.Sample) if err != nil { lan.log4goline.Error("[Lands-offlineServeHttp] Train data sampling error." + err.Error()) return errors.New("[Lands-offlineServeHttp] Train data sampling error." + err.Error()) } } //模型训练 lan.log4goline.Info("[Lands-offlineServeHttp] Offline model training.") var fft trainer.FastFtrlTrainer fft.SetJobName(par.Biz + " offline " + timestamp) if !fft.Initialize(par.Epoch, par.Threads, true, 0, par.Push, par.Fetch) { lan.log4goline.Error("[Lands-offlineServeHttp] Initialize ftrl trainer error") return errors.New("[Lands-offlineServeHttp] Initialize ftrl trainer error.") } err = fft.Train(par.Alpha, par.Beta, par.L1, par.L2, par.Dropout, model_path, train_path, test_path) if err != nil { lan.log4goline.Error("[Lands-offlineServeHttp] Training model error." + err.Error()) return errors.New("[Lands-offlineServeHttp] Training model error." + err.Error()) } lan.log4goline.Info("[Lands-offlineServeHttp] Predict testing data.") _, err = predictor.Run(3, []string{par.Biz + " offline " + timestamp, test_path, model_path, predict_path, par.Threshold}) if err != nil { lan.log4goline.Error("[Lands-offlineServeHttp] Predicting model error." + err.Error()) return errors.New("[Lands-offlineServeHttp] Predicting model error." + err.Error()) } err = util.CopyFile(base_path_on+"/model.dat", base_path_off+"/"+timestamp+"/model.dat") if err != nil { lan.log4goline.Error("[Lands-offlineServeHttp] Copy model error." + err.Error()) return errors.New("[Lands-offlineServeHttp] Copy model error." + err.Error()) } //清理目录 lan.log4goline.Info("[Lands-offlineServeHttp] Clear local directory." + base_path_off) err = util.KeepLatestN(base_path_off, 5) if err != nil { lan.log4goline.Warn("[Lands-offlineServeHttp] Clear local file error." + err.Error()) errors.New("[Lands-offlineServeHttp] Clear local file error." + err.Error()) } m, err := fft.ParamServer.SaveEncodeModel() if err != nil { lan.log4goline.Error("[Lands-offlineServeHttp] Save model error." + err.Error()) return errors.New("[Lands-offlineServeHttp] Save model error." + err.Error()) } fmt.Fprintf(w, "%s", m) //模型存入redis lan.log4goline.Info("[Lands-offlineServeHttp] Write model to redis.") if par.Dst == "redis" { key := ModelPrefix + par.Biz conn.Send("SET", key, m) conn.Flush() _, err := conn.Receive() if err != nil { lan.log4goline.Error(fmt.Sprintln("[Lands-offlineServeHttp] Save model to redis error." + err.Error())) return errors.New("[Lands-offlineServeHttp] Save model to redis error." + err.Error()) } } lan.log4goline.Info("[Lands-offlineServeHttp] End offline learning.") return nil }