func (n *GitLabClient) RegisterRunner(runner RunnerCredentials, description, tags string) *RegisterRunnerResponse {
// TODO: pass executor
request := RegisterRunnerRequest{
Info: n.getRunnerVersion(RunnerConfig{}),
Token: runner.Token,
Description: description,
Tags: tags,
}
var response RegisterRunnerResponse
result, statusText := n.do(runner, "POST", "runners/register.json", 201, &request, &response)
shortToken := helpers.ShortenToken(runner.Token)
switch result {
case 201:
logrus.Println(shortToken, "Registering runner...", "succeeded")
return &response
case 403:
logrus.Errorln(shortToken, "Registering runner...", "forbidden (check registration token)")
return nil
case clientError:
logrus.Errorln(shortToken, "Registering runner...", "error", statusText)
return nil
default:
logrus.Errorln(shortToken, "Registering runner...", "failed", statusText)
return nil
}
}
func (n *GitLabClient) GetBuild(config RunnerConfig) (*GetBuildResponse, bool) {
request := GetBuildRequest{
Info: n.getRunnerVersion(config),
Token: config.Token,
}
var response GetBuildResponse
result, statusText := n.do(config.RunnerCredentials, "POST", "builds/register.json", 201, &request, &response)
switch result {
case 201:
logrus.Println(config.ShortDescription(), "Checking for builds...", "received")
return &response, true
case 403:
logrus.Errorln(config.ShortDescription(), "Checking for builds...", "forbidden")
return nil, false
case 404:
logrus.Debugln(config.ShortDescription(), "Checking for builds...", "nothing")
return nil, true
case clientError:
logrus.Errorln(config.ShortDescription(), "Checking for builds...", "error:", statusText)
return nil, false
default:
logrus.Warningln(config.ShortDescription(), "Checking for builds...", "failed:", statusText)
return nil, true
}
}
func (p *TenantService) TenantDetail(token string, tenantId string) (ret interface{}, errorCode string, err error) {
if !bson.IsObjectIdHex(tenantId) {
logrus.Errorln("invalid object id for getTenantDetail: ", tenantId)
err = errors.New("invalid object id for getTenantDetail")
return nil, TENANT_ERROR_CREATE, err
}
code, err := GetTokenService().TokenValidate(token)
if err != nil {
return nil, code, err
}
if authorized := GetAuthService().Authorize("get_tenant", token, tenantId, p.collectionName); !authorized {
logrus.Errorln("required opertion is not allowed!")
return nil, COMMON_ERROR_UNAUTHORIZED, errors.New("Required opertion is not authorized!")
}
selector := bson.M{}
selector["_id"] = bson.ObjectIdHex(tenantId)
ret = entity.Tenant{}
queryStruct := dao.QueryStruct{
CollectionName: p.collectionName,
Selector: selector,
Skip: 0,
Limit: 0,
Sort: ""}
err = dao.HandleQueryOne(&ret, queryStruct)
return
}
func RegisterRunner(url, token, description, tags string) *RegisterRunnerResponse {
// TODO: pass executor
request := RegisterRunnerRequest{
Info: GetRunnerVersion(RunnerConfig{}),
Token: token,
Description: description,
Tags: tags,
}
var response RegisterRunnerResponse
result, statusText := postJSON(getURL(url, "runners/register.json"), 201, &request, &response)
shortToken := helpers.ShortenToken(token)
switch result {
case 201:
log.Println(shortToken, "Registering runner...", "succeeded")
return &response
case 403:
log.Errorln(shortToken, "Registering runner...", "forbidden (check registration token)")
return nil
default:
log.Errorln(shortToken, "Registering runner...", "failed", statusText)
return nil
}
}
func compressResponse(resp *http.Response, compressionLevel int) error {
// Establish a new pipe.
pipeReader, pipeWriter := io.Pipe()
// In a seperate Go routine, compress the request body and copy it to the
// pipe.
go func(r io.ReadCloser) {
// Defer the closing of both the reader and writer.
defer closeLogError(r)
defer closeLogError(pipeWriter)
// Create a new gzip writer, wrapping the original writer,
// and defer its closing.
gzipWriter, err := gzip.NewWriterLevel(pipeWriter, compressionLevel)
if err != nil {
log.Errorln(err)
return
}
defer closeLogError(gzipWriter)
// Copy the response body to the gzip writer.
if _, err := io.Copy(gzipWriter, r); err != nil {
log.Errorln(err)
}
}(resp.Body)
resp.Header.Set("content-encoding", "gzip")
resp.Header.Del("content-length")
resp.Body = pipeReader
return nil
}
//return a query language by authenticate
//this will be used for listAll, list and deleteAll interface
func (p *AuthService) BuildQueryByAuth(operation string, tokenId string) (query bson.M, err error) {
logrus.Infoln("build query object by auth")
token, err := GetTokenService().GetTokenById(tokenId)
if err != nil {
logrus.Errorln("get token error:", err)
return nil, errors.New("get token by id error!")
}
authParam := &AuthParam{auth_token: token}
policyValue, exist := p.authdata[operation]
if !exist {
logrus.Infoln("no auth policy for specific operation, operation:", operation)
query = bson.M{}
return
}
switch checkType := policyValue.(type) {
case entity.OrCheck:
orresult, orsuccess := p.orCheck_query(checkType.Basechecks, checkType.Andchecks, checkType.Orchecks, authParam)
if !orsuccess {
logrus.Warnln("build auth query error")
return nil, errors.New("build auth query error")
}
query, err = format(orresult)
if err != nil {
logrus.Warnln("format query result to bson error %v", err)
return nil, err
}
return
case entity.AndCheck:
andresult, andsuccess := p.andCheck_query(checkType.Basechecks, checkType.Andchecks, checkType.Orchecks, authParam)
if !andsuccess {
logrus.Warnln("build auth query error")
return nil, errors.New("build auth query error")
}
query, err = format(andresult)
if err != nil {
logrus.Warnln("format query result to bson error %v", err)
return nil, err
}
return
case entity.BaseCheck:
baseresult, basesuccess := p.baseCheck_query(checkType.Checktype, checkType.Value, authParam)
if !basesuccess {
logrus.Warnln("build auth query error")
return nil, errors.New("build auth query error")
}
query, err = format(baseresult)
if err != nil {
logrus.Warnln("format query result to bson error %v", err)
return nil, err
}
return
default:
logrus.Errorln("unkonwn check type:", checkType)
return nil, errors.New("unknown check type")
}
}
func (sn *Node) StartGossip() {
go func() {
t := time.Tick(GOSSIP_TIME)
for {
select {
case <-t:
sn.viewmu.Lock()
c := sn.HostListOn(sn.ViewNo)
sn.viewmu.Unlock()
if len(c) == 0 {
log.Errorln(sn.Name(), "StartGossip: none in hostlist for view: ", sn.ViewNo, len(c))
continue
}
sn.randmu.Lock()
from := c[sn.Rand.Int()%len(c)]
sn.randmu.Unlock()
log.Errorln("Gossiping with: ", from)
sn.CatchUp(int(atomic.LoadInt64(&sn.LastAppliedVote)+1), from)
case <-sn.closed:
log.Warnln("stopping gossip: closed")
return
}
}
}()
}
func exportEnvironmentsList(envsList []models.EnvironmentItemModel) error {
log.Info("[BITRISE_CLI] - Exporting workflow environments")
for _, env := range envsList {
envKey, envValue, err := env.GetKeyValue()
if err != nil {
log.Errorln("[BITRISE_CLI] - Failed to get environment key-value pair from env:", env)
return err
}
if envValue != "" {
expand := true
if env["is_expand"] != "" {
boolValue, err := goinp.ParseBool(env["is_expand"])
if err != nil {
log.Error("Failed to parse bool:", err)
return err
}
expand = boolValue
}
if err := bitrise.RunEnvmanAdd(envKey, envValue, expand); err != nil {
log.Errorln("[BITRISE_CLI] - Failed to run envman add")
return err
}
}
}
return nil
}
//terminate specified hosts of a cluster
// Request
// URL:
// PUT /v1/cluster/<CLUSTER_ID>/hosts
// Header:
// X-Auth-Token
// Except Body:
//{
// "host_ids":["568e23655d5c3d173019f1ba","568e2be45d5c3d173019f1bb","568e2bfd5d5c3d173019f1bc","568e2c335d5c3d173019f1bd"]
//}
//
// Response:
//{
// "success": true
//}
//
func (p *Resource) HostsDeleteHandler(req *restful.Request, resp *restful.Response) {
logrus.Infof("HostsDeleteHandler is called!")
x_auth_token := req.HeaderParameter("X-Auth-Token")
code, err := services.TokenValidation(x_auth_token)
if err != nil {
logrus.Errorln("token validation error is %v", err)
response.WriteStatusError(code, err, resp)
return
}
clusterId := req.PathParameter(ParamID)
body := TerminateHostsRequestBody{}
err = json.NewDecoder(req.Request.Body).Decode(&body)
errorCode, err := services.GetClusterService().TerminateHosts(clusterId, body.HostIds, x_auth_token)
if err != nil {
logrus.Errorln("terminate hosts error is %v", err)
response.WriteStatusError(errorCode, err, resp)
return
}
// Write success response
response.WriteSuccess(resp)
return
}
func (s *SlaveSpider) Base() {
method := "SlaveSpider Base"
logrus.Println("SlaveSpider", method)
for {
if len(s.Urls) == 0 {
// logrus.Infoln(method, "Send to Master New urls:", len(s.NewUrls) /*s.NewUrls*/)
send := len(s.NewUrls)
masterToSlave, err := s.Request()
if err != nil {
logrus.Errorln(method, err)
continue
}
s.NewUrls = []string{}
s.Match = []string{}
logrus.Infoln(method, "slaveToMaster url:", send, "get Task:", len(masterToSlave.Task) /*, masterToSlave.Task*/)
s.Urls = masterToSlave.Task
time.Sleep(time.Second * 2)
} else {
err := s.MatchOnce()
if err != nil {
logrus.Errorln(method, err)
continue
}
}
}
return
}
func (p *AuthService) baseCheck_query(checktype string, value string, authParam *AuthParam) (map[string]interface{}, bool) {
basequery := make(map[string]interface{})
if strings.EqualFold(checktype, "role") {
if strings.EqualFold(value, authParam.auth_token.Role.Rolename) {
return basequery, true
} else {
return nil, false
}
} else if strings.EqualFold(checktype, "field") {
valueArrays := strings.Split(value, "=")
if len(valueArrays) != 2 {
logrus.Errorln("a field policy format error! value is :", value)
return nil, false
}
//handle first value
value1 := valueArrays[0]
// handle second value(can be a value or from token)
value2 := valueArrays[1]
if strings.HasPrefix(value2, "%") {
key2 := strings.TrimPrefix(value2, "%")
value2 = p.getValueFromToken(authParam.auth_token, key2)
}
basequery[value1] = value2
return basequery, true
} else {
logrus.Errorln("unsupported checktype:", checktype)
return nil, false
}
}
func (sr *Runner) checkCleared(check stalker.Check) {
log.Debugln("check cleared")
log.Infof("%s %s detected as cleared", check.Hostname, check.Check)
query := map[string]string{"hostname": check.Hostname, "check": check.Check}
cursor, err := r.Db(STALKERDB).Table("notifications").Filter(query).Run(sr.rsess)
if err != nil {
log.Errorln("Error checking for existing notification:", err.Error())
return
}
defer cursor.Close()
result := stalker.Notification{}
cursor.One(&result)
if result.Active == false {
log.Infoln("No notification to clear")
return
}
_, err = r.Db(STALKERDB).Table("notifications").Filter(query).Delete().RunWrite(sr.rsess)
if err != nil {
log.Errorln("Error deleting notification entry:", err.Error())
return
}
sr.emitClear(check)
return
}
func (n *GitLabClient) VerifyRunner(runner RunnerCredentials) bool {
request := VerifyRunnerRequest{
Token: runner.Token,
}
// HACK: we use non-existing build id to check if receive forbidden or not found
result, statusText := n.do(runner, "PUT", fmt.Sprintf("builds/%d", -1), 200, &request, nil)
shortToken := helpers.ShortenToken(runner.Token)
switch result {
case 404:
// this is expected due to fact that we ask for non-existing job
logrus.Println(shortToken, "Veryfing runner...", "is alive")
return true
case 403:
logrus.Errorln(shortToken, "Veryfing runner...", "is removed")
return false
case clientError:
logrus.Errorln(shortToken, "Veryfing runner...", "error", statusText)
return false
default:
logrus.Errorln(shortToken, "Veryfing runner...", "failed", statusText)
return true
}
}
func userLoginParamCheck(doc interface{}) (username string, password string, paraErr error) {
var document interface{}
document, paraErr = mejson.Marshal(doc)
if paraErr != nil {
logrus.Errorf("marshal user err is %v", paraErr)
return
}
docJson := document.(map[string]interface{})
usernameDoc := docJson["username"]
if usernameDoc == nil {
logrus.Errorln("invalid parameter ! username can not be null")
paraErr = errors.New("Invalid parameter!")
return
} else {
username = usernameDoc.(string)
}
passwordDoc := docJson["password"]
if passwordDoc == nil {
logrus.Errorln("invalid parameter ! password can not be null")
paraErr = errors.New("Invalid parameter!")
return
} else {
password = passwordDoc.(string)
}
return
}
// step steps through the pipeline to head.next
func (p *Pipeline) step() {
if p.head == p.tail {
go func() {
defer func() {
if r := recover(); r != nil {
logrus.Errorln("recover executing step function", r)
}
}()
// stop all containers
for _, id := range p.containers {
p.engine.ContainerStop(id)
}
// wait for all logs to terminate
// p.wait.Done() // this is for the ambassador
p.wait.Wait()
// signal completion
p.done <- nil
}()
} else {
go func() {
defer func() {
if r := recover(); r != nil {
logrus.Errorln("recover executing step to head function", r)
}
}()
p.head = p.head.next
p.next <- nil
}()
}
}
func (d NetworkDriver) DeleteEndpoint(request *network.DeleteEndpointRequest) error {
logutils.JSONMessage("DeleteEndpoint", request)
log.Debugf("Removing endpoint %v\n", request.EndpointID)
hostname, err := osutils.GetHostname()
if err != nil {
err = errors.Wrap(err, "Hostname fetching error")
log.Errorln(err)
return err
}
if err = d.client.WorkloadEndpoints().Delete(
api.WorkloadEndpointMetadata{
Name: request.EndpointID,
Node: hostname,
Orchestrator: d.orchestratorID,
Workload: d.containerName}); err != nil {
err = errors.Wrapf(err, "Endpoint %v removal error", request.EndpointID)
log.Errorln(err)
return err
}
logutils.JSONMessage("DeleteEndpoint response JSON=%v", map[string]string{})
return err
}
func (p *TokenService) TokenReGenerate(token string, userId string, tenantId string) (ret TokenId, errorCode string, err error) {
if len(userId) == 0 || len(tenantId) == 0 {
logrus.Errorf("user and tenant id can not be null!")
return ret, "E12002", errors.New("invalid parameter!user and tenant id can not be null")
}
code, err := p.TokenValidate(token)
if err != nil {
return ret, code, err
}
if authorized := GetAuthService().Authorize("regenerate_token", token, "", p.collectionName); !authorized {
logrus.Errorln("required opertion is not allowed!")
return ret, "E12004", errors.New("required opertion is not authorized!")
}
user_name, tenant_name, err := getNamesById(userId, tenantId)
if err != nil {
logrus.Errorln("failed to get user and tenant by id, error is %v", err)
return ret, TOKEN_ERROR_CREATE, err
}
newtoken, err := p.checkAndGenerateToken(user_name, "", tenant_name, false)
if err != nil {
logrus.Errorf("failed to generate token, error is %s", err)
return ret, TOKEN_ERROR_CREATE, err
}
tokenId := TokenId{Id: newtoken}
return tokenId, "", nil
}
func (h *hook) updateKeys() error {
resp, err := http.Get(h.cfg.JWKSetURL)
if err != nil {
log.Errorln("failed to fetch JWK Set: " + err.Error())
return err
}
var parsedJWKs gojwk.Key
err = json.NewDecoder(resp.Body).Decode(&parsedJWKs)
if err != nil {
resp.Body.Close()
log.Errorln("failed to decode JWK JSON: " + err.Error())
return err
}
resp.Body.Close()
keys := map[string]crypto.PublicKey{}
for _, parsedJWK := range parsedJWKs.Keys {
publicKey, err := parsedJWK.DecodePublicKey()
if err != nil {
log.Errorln("failed to decode JWK into public key: " + err.Error())
return err
}
keys[parsedJWK.Kid] = publicKey
}
h.publicKeys = keys
log.Debug("successfully fetched JWK Set")
return nil
}
func (p *UserService) UserDetail(token string, userId string) (ret interface{}, errorCode string, err error) {
code, err := GetTokenService().TokenValidate(token)
if err != nil {
return nil, code, err
}
if authorized := GetAuthService().Authorize("get_user", token, userId, p.userCollectionName); !authorized {
logrus.Errorln("required opertion is not allowed!")
return nil, COMMON_ERROR_UNAUTHORIZED, errors.New("Required opertion is not authorized!")
}
selector := bson.M{}
selector["_id"] = bson.ObjectIdHex(userId)
ret = new(entity.User)
queryStruct := dao.QueryStruct{
CollectionName: p.userCollectionName,
Selector: selector,
Skip: 0,
Limit: 0,
Sort: ""}
err = dao.HandleQueryOne(ret, queryStruct)
logrus.Errorln(ret)
return
}
func configureLogging(v *viper.Viper) {
level, err := log.ParseLevel(v.GetString("log_level"))
if err != nil {
log.Fatalln(err)
}
log.SetLevel(level)
if v.GetString("log_format") == "text" {
log.SetFormatter(&log.TextFormatter{DisableColors: true, FullTimestamp: true})
} else if v.GetString("log_format") == "json" {
log.SetFormatter(&log.JSONFormatter{})
} else {
log.Errorln("Error: log_type invalid, defaulting to text")
log.SetFormatter(&log.TextFormatter{})
}
switch v.GetString("log_target") {
case "stdout":
log.SetOutput(os.Stdout)
case "stderr":
log.SetOutput(os.Stderr)
default:
log.Errorln("Error: log_target invalid, defaulting to Stdout")
log.SetOutput(os.Stdout)
}
}
func (command *RestoreCommand) Run(args []string) int {
flagSet := flag.NewFlagSet("restore", flag.ExitOnError)
flagSet.Usage = func() { fmt.Println(command.Help()) }
sudo := flagSet.Bool("sudo", false, "Use sudo while invoking tar")
if err := flagSet.Parse(args); err != nil {
log.Errorln(err)
return -1
}
args = flagSet.Args()
if len(args) != 2 {
log.Errorln(errors.New("Insufficient argument. Please pass container name and image file name"))
return -1
}
if err := specification.DecompressImage(args[0], args[1], *sudo); err != nil {
log.Errorf("Failed to restore container. Error: %s\n", err)
return -1
}
if err := specification.UpdateUTS(args[0]); err != nil {
log.Errorf("Failed to restore container. Error: %s\n", err)
return -1
}
return 0
}
func (sr *Runner) getChecks(maxChecks int, timeout int) []stalker.Check {
log.Debugln("Getting checks off queue")
checks := make([]stalker.Check, 0)
expireTime := time.Now().Add(3 * time.Second).Unix()
for len(checks) <= maxChecks {
//we've exceeded our try time
if time.Now().Unix() > expireTime {
break
}
rconn := sr.rpool.Get()
defer rconn.Close()
res, err := redis.Values(rconn.Do("BLPOP", sr.conf.workerQueue, timeout))
if err != nil {
if err != redis.ErrNil {
log.Errorln("Error grabbing check from queue:", err.Error())
break
} else {
log.Debugln("redis result:", err)
continue
}
}
var rb []byte
res, err = redis.Scan(res, nil, &rb)
var check stalker.Check
if err := json.Unmarshal(rb, &check); err != nil {
log.Errorln("Error decoding check from queue to json:", err.Error())
break
}
checks = append(checks, check)
}
return checks
}
func (n *GitLabClient) UpdateBuild(config RunnerConfig, id int, state BuildState, trace string) UpdateState {
request := UpdateBuildRequest{
Info: n.getRunnerVersion(config),
Token: config.Token,
State: state,
Trace: trace,
}
result, statusText := n.do(config.RunnerCredentials, "PUT", fmt.Sprintf("builds/%d.json", id), 200, &request, nil)
switch result {
case 200:
logrus.Println(config.ShortDescription(), id, "Submitting build to coordinator...", "ok")
return UpdateSucceeded
case 404:
logrus.Warningln(config.ShortDescription(), id, "Submitting build to coordinator...", "aborted")
return UpdateAbort
case 403:
logrus.Errorln(config.ShortDescription(), id, "Submitting build to coordinator...", "forbidden")
return UpdateAbort
case clientError:
logrus.Errorln(config.ShortDescription(), id, "Submitting build to coordinator...", "error", statusText)
return UpdateAbort
default:
logrus.Warningln(config.ShortDescription(), id, "Submitting build to coordinator...", "failed", statusText)
return UpdateFailed
}
}
// open opens a new database connection with the specified
// driver and connection string and returns a store.
func open(driver, config string) *sql.DB {
db, err := sql.Open(driver, config)
if err != nil {
logrus.Errorln(err)
logrus.Fatalln("database connection failed")
}
if driver == "mysql" {
// per issue https://github.com/go-sql-driver/mysql/issues/257
db.SetMaxIdleConns(0)
}
setupMeddler(driver)
if err := pingDatabase(db); err != nil {
logrus.Errorln(err)
logrus.Fatalln("database ping attempts failed")
}
if err := setupDatabase(driver, db); err != nil {
logrus.Errorln(err)
logrus.Fatalln("migration failed")
}
cleanupDatabase(db)
return db
}
func (s *Server) mapRoutes() {
r := s.router
cwd, _ := os.Getwd()
staticPath := path.Join(cwd, s.config.static)
var staticURL string
var staticBundleURL string
if s.config.hot {
// create the prefix necessary to load bundles from hmr server
staticBundleURL = s.config.hmr
staticURL = s.config.address
} else {
// ensure bundles exist if not hot reloading
ensureBundles(s.config.js, s.config.style, staticPath)
if s.config.dev {
staticBundleURL = s.config.address
staticURL = s.config.address
} else {
staticBundleURL = s.config.serve
staticURL = s.config.serve
}
}
staticBundleURL = path.Join(staticBundleURL, s.config.static)
staticURL = path.Join(staticURL, s.config.static)
log.Warnln(s.config.js)
log.Warnln(s.config.static)
// create the default app (the route used to serve the client app)
app := defaultApp{}
// load template
f, err := os.Open(s.config.template)
if err != nil {
log.Errorln("Tpl err", err)
os.Exit(1)
}
defer f.Close()
b, err := ioutil.ReadAll(f)
if err != nil {
log.Errorln("Tpl read err", err)
os.Exit(1)
}
tpl, err := template.New("app").Parse(string(b))
if err != nil {
log.Errorln("Tpl parse err", err)
os.Exit(1)
}
app.data = map[string]interface{}{
"Js": path.Join(staticBundleURL, s.config.js),
"Style": path.Join(staticBundleURL, s.config.style),
"Static": staticURL,
"Hot": s.config.hot,
}
log.Warnln(app.data)
app.template = tpl
// httprouter fileserver
r.ServeFiles(path.Join(base(s.config.static), "*filepath"), http.Dir(staticPath))
// if it's not an api call then we use the app, after first checking
// if there's a file matching the route
r.NotFound = app
}
func (command *FetchCommand) Run(args []string) int {
flagSet := flag.NewFlagSet("fetch", flag.ExitOnError)
flagSet.Usage = func() { fmt.Println(command.Help()) }
name := flagSet.String("name", "", "Name of the container (Default: random UUID)")
bucket := flagSet.String("bucket", "", "S3 bucket")
key := flagSet.String("key", "", "S3 key")
region := flagSet.String("region", "us-west-1", "S3 region")
sudo := flagSet.Bool("sudo", false, "Use sudo during decompression of image")
flagSet.Parse(args)
if *bucket == "" {
log.Errorf("Must provide the s3 bucket name")
return -1
}
if *key == "" {
log.Errorf("Must provide the s3 key name")
return -1
}
svc := s3.New(session.New(), aws.NewConfig().WithRegion(*region))
fo, err := ioutil.TempFile(os.TempDir(), "nut")
if err != nil {
log.Error(err)
return -1
}
params := &s3.GetObjectInput{
Bucket: aws.String(*bucket),
Key: aws.String(*key),
}
resp, downloadErr := svc.GetObject(params)
if downloadErr != nil {
log.Error(downloadErr)
return -1
}
defer resp.Body.Close()
if _, copyError := io.Copy(fo, resp.Body); copyError != nil {
log.Errorln(copyError)
return -1
}
log.Infof("Image written to: %s\n", fo.Name())
fo.Close()
if *name == "" {
uuid, err := specification.UUID()
if err != nil {
log.Errorln(err)
return -1
}
name = &uuid
}
if err := specification.DecompressImage(*name, fo.Name(), *sudo); err != nil {
log.Errorln(err)
return -1
}
if err := specification.UpdateUTS(*name); err != nil {
log.Errorln(err)
return -1
}
return 0
}
// When client asks for val to be timestamped
// It blocks until it get a stamp reply back
func (c *Client) TimeStamp(val []byte, TSServerName string) error {
c.Mux.Lock()
if c.Error != nil {
c.Mux.Unlock()
return c.Error
}
c.reqno++
myReqno := c.reqno
c.doneChan[c.reqno] = make(chan error, 1) // new done channel for new req
c.Mux.Unlock()
// send request to TSServer
// log.Println("SENDING TIME STAMP REQUEST TO: ", TSServerName)
err := c.PutToServer(TSServerName,
&TimeStampMessage{
Type: StampRequestType,
ReqNo: myReqno,
Sreq: &StampRequest{Val: val}})
if err != nil {
if err != coconet.ErrNotEstablished {
if sign.DEBUG {
log.Warn("error timestamping: ", err)
}
}
// pass back up all errors from putting to server
return err
}
// get channel associated with request
c.Mux.Lock()
myChan := c.doneChan[myReqno]
c.Mux.Unlock()
// wait until ProcessStampReply signals that reply was received
select {
case err = <-myChan:
// log.Println("-------------client received response from" + TSServerName)
break
case <-time.After(10 * ROUND_TIME):
if sign.DEBUG == true {
log.Errorln(errors.New("client timeouted on waiting for response from" + TSServerName))
}
break
// err = ErrClientToTSTimeout
}
if err != nil {
if sign.DEBUG {
log.Errorln("error received from DoneChan:", err)
}
return err
}
// delete channel as it is of no longer meaningful
c.Mux.Lock()
delete(c.doneChan, myReqno)
c.Mux.Unlock()
return err
}
/* Export a file from the stash somewhere... if the somewhere is a
directory, we tack on file's name, if it's a file we export to
the new file name */
func (self *Meta) ExportFile(node Node, file FilePointer, loc string) {
log.Debugln("Opening stash: ", node.Id)
fl, err := os.Open(config.Stash_loc + "/" + node.Id)
st, err := fl.Stat()
if err != nil || st.IsDir() {
log.Errorln("Invalid stash, failed to open:", node.Id)
return
}
defer fl.Close()
log.Debugln("Testing for directory?")
if st, err := os.Stat(loc); err == nil { //adjusts if loc is a dir
if st.IsDir() {
loc += "/" + file.Name
}
}
log.Debugln("Opening output file: ", loc)
of, err := os.OpenFile(loc, os.O_CREATE|os.O_RDWR, 0666)
if err != nil {
log.Errorln("Failed to open output location:", loc)
return
}
defer of.Close()
sp := file.Size
wrote := 0
buff := make([]byte, 4096)
log.Debugln("This is the file object:\n", file, "\n")
log.Debugln("Scanning input file for ", file.Size, " bytes")
for {
sz, err := fl.Read(buff)
if err != nil && err != io.EOF {
panic(err)
}
if err == io.EOF || sp <= 0 {
log.Debugln("EOF at ", sz, " bytes")
break
}
if int64(sz) > file.Size {
sz = int(file.Size)
}
wr, err := of.Write(buff[:sz])
//log.Debugln("Read ", sz, " bytes, Wrote ", wr, " bytes to output file")
if err != nil {
log.Errorln("Failure during file export from stash:", loc)
}
sp -= int64(wr)
wrote += wr
log.Debugln(sp, " bytes left to write")
}
log.Debugln("Wrote: ", wrote, " bytes total")
}
func (p *UserService) Create(userParam UserParam, token string) (errorCode string, userId string, err error) {
if len(userParam.UserName) == 0 || len(userParam.Email) == 0 || len(userParam.Password) == 0 {
logrus.Error("invalid parameter for user create!")
return "", COMMON_ERROR_INVALIDATE, errors.New("invalid parameter! parameter should not be null")
}
code, err := GetTokenService().TokenValidate(token)
if err != nil {
return code, "", err
}
if authorized := GetAuthService().Authorize("create_user", token, "", p.userCollectionName); !authorized {
logrus.Errorln("required opertion is not allowed!")
return "", COMMON_ERROR_UNAUTHORIZED, errors.New("Required opertion is not authorized!")
}
username := userParam.UserName
email := userParam.Email
password := userParam.Password
company := userParam.Company
if !IsUserNameValid(username) {
logrus.Errorf("username invalid! username is: %v", username)
return COMMON_ERROR_INVALIDATE, "", errors.New("invalid username")
}
_, err = GetTenantService().getTenantByName(username)
if err == nil {
logrus.Errorln("user already exist!")
return USER_ERROR_EXIST, "", errors.New("The username has already been registered, please specified another one!")
}
encryPassword := HashString(password)
tenantId, errte := GetTenantService().createAndInsertTenant(username, username)
if errte != nil {
logrus.Errorf("create and insert new tenant error, err is %v", errte)
return USER_ERROR_REG, "", errte
}
role, errrole := GetRoleService().getRoleByName(admin_role)
if errrole != nil {
logrus.Errorf("get role error is %v", errrole)
return ROLE_ERROR_GET, "", errrole
}
userId, err = p.createAndInsertUser(username, encryPassword, email, tenantId, role.ObjectId.Hex(), company)
if err != nil {
logrus.Errorf("create and insert new user error, err is %v", err)
return USER_ERROR_REG, "", err
}
return "", userId, nil
}
func (s *MasterSpider) Base(w http.ResponseWriter, r *http.Request) {
method := "MasterSpider Base"
logrus.Infof("%-15s|%-15s|%-15s|", Url, r.Method, r.RemoteAddr)
//read request
body := r.Body
defer body.Close()
bodyBytes, err := ioutil.ReadAll(body)
if err != nil {
logrus.Errorln(method, err)
w.Write([]byte(""))
return
}
var slaveToMaster *SlaveToMaster
err = json.Unmarshal(bodyBytes, &slaveToMaster)
if err != nil {
logrus.Errorln(method, err)
w.Write([]byte(""))
return
}
newUrls := s.FilterUrls(slaveToMaster.NewUrls)
idxBefore := len(s.Urls)
//add the urls which slave given to the all list.
s.Urls.addList(newUrls)
idxAfter := len(s.Urls)
//record all urls and match to file.
err = s.SaveToFile(slaveToMaster, idxBefore, idxAfter)
//master send task to slave
var masterToSlave MasterToSlave
if s.Index >= len(s.Urls) {
masterToSlave.Task = []string{}
} else {
s.idxLock.Lock()
s.Index, masterToSlave.Task = s.Urls.GiveUrls(s.Index)
s.idxLock.Unlock()
}
logrus.Debugln("all urls length:", len(s.Urls), "index:", s.Index, "send task:", len(masterToSlave.Task), "slaveToMaster match:", len(slaveToMaster.Match), "slaveToMaster NewUrls:", len(slaveToMaster.NewUrls))
// logrus.Debugln("index", s.Index, "send urls:", masterToSlave.Task)
// logrus.Debugln("slave give urls:", slaveToMaster.NewUrls)
tmparr, err := json.Marshal(masterToSlave)
if err != nil {
logrus.Errorln(method, err)
}
w.Write(tmparr)
}
