Beispiel #1
0
func (a *ABF) handleResponse(resp *http.Response, testing bool) error {

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	var result map[string]interface{}

	err = json.Unmarshal(body, &result)
	if err != nil {
		return err
	}

	var lists []interface{}

	err = dig.Get(&result, &lists, "build_lists")
	if err != nil {
		return err
	}

	for _, v := range lists {
		asserted := v.(map[string]interface{})
		id := dig.Uint64(&asserted, "id")
		strID := "[" + to.String(id) + "]"

		var num int
		err = models.DB.Model(&models.List{}).Where("integration_name = ? AND integration_one LIKE ?", "abf", "%"+strID+"%").Count(&num).Error
		if err != nil && err != gorm.ErrRecordNotFound {
			klog.Criticalf("abf: couldn't check db for existing (list %v): %v", id, err)
			continue
		}

		if num != 0 {
			klog.Infof("abf: ignoring list, already processed (list %v)", id)
			continue
		}

		json, err := a.getJSONList(id)
		if err != nil {
			klog.Criticalf("abf: couldn't retrieve the build list JSON (list %v): %v", id, err)
			continue
		}

		// check if arch is on whitelist
		if abfArchWhitelistSet != nil && !abfArchWhitelistSet.Contains(dig.String(&json, "arch", "name")) {
			klog.Infof("abf: ignoring list, arch not on whitelist (list %v)", id)
			continue
		}

		// check if platform is on whitelist
		if !abfPlatformsSet.Contains(dig.String(&json, "save_to_repository", "platform", "name")) {
			klog.Infof("abf: ignoring list, platform not on whitelist (list %v)", id)
			continue
		}

		// *check for duplicates before continuing
		// *we only check for duplicates in the same platform; different platforms have different conditions

		// check for duplicates
		var duplicate models.List
		err = models.DB.Where("platform = ? AND integration_two = ? AND stage_current <> ?",
			dig.String(&json, "save_to_repository", "platform", "name"),
			dig.String(&json, "commit_hash"),
			models.ListStageFinished).First(&duplicate).Error

		if err != nil && err != gorm.ErrRecordNotFound {
			klog.Criticalf("abf: couldn't check db for duplicates (list %v): %v", id, err)
			continue
		}

		if err == nil { // we had no problem finding a duplicate, so handle and continue
			duplicate.IntegrationOne += ";" + strID
			duplicate.Variants += ";" + dig.String(&json, "arch", "name")

			if testing { // we got this from the build completed (not in testing) list
				// send id to testing
				go a.sendToTesting(id)
			}

			err = models.DB.Save(&duplicate).Error
			if err != nil {
				klog.Criticalf("abf: couldn't save duplicate modification to %v (list %v): %v", duplicate.ID, id, err)
			}

			pkgs := a.makePkgList(json)
			for _, listpkg := range pkgs {
				listpkg.ListID = duplicate.ID

				err = models.DB.Create(&listpkg).Error
				if err != nil {
					klog.Criticalf("abf: couldn't save new list package to %v (list %v): %v", duplicate.ID, id, err)
				}
			}

			// add a link to it
			newLink := models.ListLink{
				ListID: duplicate.ID,
				Name:   fmt.Sprintf("Build List for %v", dig.String(&json, "arch", "name")),
				URL:    fmt.Sprintf("%s/build_lists/%v", abfURL, id),
			}
			if err := models.DB.Create(&newLink).Error; err != nil {
				klog.Criticalf("abf: couldn't save new list link to %v (list %v): %v", duplicate.ID, id, err)
			}

			// ok, we're done here
			continue
		}

		list, err := a.makeBuildList(json)
		if err != nil {
			klog.Criticalf("abf: couldn't make new list (list %v): %v\n", id, err)
			continue
		}

		if testing {
			// Now send it to testing
			go a.sendToTesting(id)
		}

		if err := models.DB.Create(list).Error; err != nil {
			klog.Criticalf("abf: couldn't create new list in db (list %v): %v", id, err)
			continue
		}
	}

	return nil
}
Beispiel #2
0
func main() {
	klog.Info("starting kahinah v4")

	// -- mux -----------------------------------------------------------------
	mux := goji.NewMux()

	// -- middleware ----------------------------------------------------------

	// logging middleware (base middleware)
	mux.UseC(func(inner goji.Handler) goji.Handler {
		return goji.HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
			klog.Debugf("req  (%v): path (%v), user-agent (%v), referrer (%v)", r.RemoteAddr, r.RequestURI, r.UserAgent(), r.Referer())
			wp := mutil.WrapWriter(rw) // proxy the rw for info later
			inner.ServeHTTPC(ctx, wp, r)
			klog.Debugf("resp (%v): status (%v), bytes written (%v)", r.RemoteAddr, wp.Status(), wp.BytesWritten())
		})
	})

	// rendering middleware (required by panic)
	renderer := urender.New(urender.Options{
		Directory:  "views",
		Layout:     "layout",
		Extensions: []string{".tmpl", ".tpl"},
		Funcs: []template.FuncMap{
			template.FuncMap{
				"rfc3339": func(t time.Time) string {
					return t.Format(time.RFC3339)
				},
				"since": func(t time.Time) string {
					hrs := time.Since(t).Hours()
					return fmt.Sprintf("%dd %02dhrs", int(hrs)/24, int(hrs)%24)
				},
				"emailat": func(s string) string {
					return strings.Replace(s, "@", " [@T] ", -1)
				},
				"mapaccess": func(s interface{}, m map[string]string) string {
					return m[to.String(s)]
				},
				"url":     render.ConvertURL,
				"urldata": render.ConvertURLWithData,
			},
		},
		IndentJSON:    true,
		IndentXML:     true,
		IsDevelopment: conf.GetDefault("runMode", "dev").(string) == "dev",
	})

	mux.UseC(func(inner goji.Handler) goji.Handler {
		return goji.HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
			newCtx := render.NewContext(ctx, renderer)
			inner.ServeHTTPC(newCtx, rw, r)
		})
	})

	// panic middleware
	mux.UseC(controllers.PanicMiddleware)

	// not found middleware
	mux.UseC(func(inner goji.Handler) goji.Handler {
		return goji.HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
			routeFound := middleware.Pattern(ctx)

			if routeFound != nil {
				inner.ServeHTTPC(ctx, rw, r)
				return
			}

			panic(controllers.ErrNotFound)
		})
	})

	// authentication (cas) middleware
	if enable, ok := conf.GetDefault("authentication.cas.enable", false).(bool); ok && enable {
		url, _ := url.Parse(conf.Get("authentication.cas.url").(string))

		casClient := cas.NewClient(&cas.Options{
			URL: url,
		})

		mux.Use(casClient.Handle)
	}

	// sessions middleware
	sessionConfig := &sessionmw.Config{
		Secret:      []byte(securecookie.GenerateRandomKey(64)),
		BlockSecret: []byte(securecookie.GenerateRandomKey(32)),
		Store:       kv.NewMemStore(),
		Name:        "kahinah",
	}
	mux.UseC(sessionConfig.Handler)

	// csrf middleware
	mux.UseC(csrf.Protect(securecookie.GenerateRandomKey(64), csrf.Secure(false)))

	// data rendering middleware
	mux.UseC(func(inner goji.Handler) goji.Handler {
		return goji.HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
			newCtx := data.RenderMiddleware(ctx, rw, r)
			inner.ServeHTTPC(newCtx, rw, r)
			data.RenderAfterware(newCtx, rw, r)
		})
	})

	// --------------------------------------------------------------------
	// HANDLERS
	// --------------------------------------------------------------------

	getHandlers := map[string]goji.HandlerFunc{

		// static paths
		"/static/*": controllers.StaticHandler,

		// main page
		"/": controllers.MainHandler,

		// builds - json
		"/i/list/json/": controllers.ListsAPIHandler,

		// builds
		"/i/list/": controllers.ListsHandler,

		// build - specific
		"/b/:id/": controllers.BuildGetHandler,

		// build - specific - json
		"/b/:id/json/": controllers.BuildGetJSONHandler,

		// activity - json
		"/i/activity/json/": controllers.ActivityJSONHandler,

		// activity - html
		"/i/activity/": controllers.ActivityHandler,

		// admin
		"/admin/": controllers.AdminGetHandler,

		// authentication - login
		"/u/login/": controllers.UserLoginHandler,

		// authentication - logout
		"/u/logout/": controllers.UserLogoutHandler,
	}

	postHandlers := map[string]goji.HandlerFunc{

		// webhooks
		"/hook/*": controllers.IntegrationHandler,

		// build - specific
		"/b/:id/": controllers.BuildPostHandler,

		// admin
		"/admin/": controllers.AdminPostHandler,
	}

	for k, v := range getHandlers {
		if len(k) > 1 && strings.HasSuffix(k, "/") {
			getHandlerRedirectName := render.ConvertURLRelative(k[:len(k)-1])
			klog.Debugf("get handler setup: redirecting %v", getHandlerRedirectName)
			mux.HandleFunc(pat.Get(getHandlerRedirectName), controllers.RedirectHandler)
		}
		getHandlerUseName := render.ConvertURLRelative(k)
		klog.Debugf("get handler setup: using %v", getHandlerUseName)
		mux.HandleC(pat.Get(getHandlerUseName), v)
	}

	for k, v := range postHandlers {
		if len(k) > 1 && strings.HasSuffix(k, "/") {
			postHandlerRedirectName := render.ConvertURLRelative(k[:len(k)-1])
			klog.Debugf("post handler setup: redirecting %v", postHandlerRedirectName)
			mux.HandleFunc(pat.Post(postHandlerRedirectName), controllers.RedirectHandler)
		}
		postHandlerUseName := render.ConvertURLRelative(k)
		klog.Debugf("post handler setup: using %v", postHandlerUseName)
		mux.HandleC(pat.Post(postHandlerUseName), v)
	}

	// -- cronjobs ----------------------------------------------------------

	cronRunner := cron.New()

	// integration polling
	if pollRate, ok := conf.Get("integration.poll").(string); ok && pollRate != "" {
		pollFunc := func() {
			pollAllErr := integration.PollAll()
			for name, err := range pollAllErr {
				klog.Warningf("integration polling failed for %v: %v", name, err)
			}
		}
		cronRunner.AddFunc(pollRate, pollFunc)

		// and do an initial poll
		pollFunc()
	}

	// process new stages/check processes every 10 seconds
	cronRunner.AddFunc("@every 10s", func() {
		models.CheckAllListStages()
	})

	// run the job scheduler every minute
	cronRunner.AddFunc("@every 1m", func() {
		job.ProcessQueue()
	})

	// start cron
	cronRunner.Start()

	// -- server setup --------------------------------------------------------

	// bind and listen
	listenAddr := conf.GetDefault("listenAddr", "0.0.0.0").(string)
	listenPort := conf.GetDefault("listenPort", 3000).(int64)

	klog.Infof("listening to %v:%v", listenAddr, listenPort)
	if err := http.ListenAndServe(fmt.Sprintf("%v:%v", listenAddr, listenPort), mux); err != nil {
		klog.Fatalf("unable to serve: %v", err)
	}

	cronRunner.Stop()

	klog.Info("processing leftover jobs...")
	close(job.Queue)
	for len(job.Queue) > 0 {
		job.ProcessQueue()
	}
}