/*
Judge run and judge other answers
*/
func Judge(dir, path string, answer []Answer, tests []os.FileInfo, timelimit float64, isConcurrent bool, logger *logrus.Entry) {
logger.WithFields(logrus.Fields{
"state": "check",
"type": "info",
"info": "check answers start"}).Info("Check answers start")
chk, err := chkSetup(dir, path, logger)
if err != nil {
return
}
//file setup
fileSetup(dir, tests, answer, isConcurrent, logger)
//testing
for k, s := range answer { //tests
id := k + 1
name := filepath.Base(s.Name)
outdir := filepath.Join(dir, "temp", fmt.Sprintf("ans_%d", id))
info := ProgInfo{Src: s.Name, Out: filepath.Join(outdir, fmt.Sprintf("ans_%d.out", id))}
test(outdir, name, id, info, tests, timelimit, chk, isConcurrent, logger)
}
logger.WithFields(logrus.Fields{
"state": "check",
"type": "info",
"info": "check answers end"}).Info("Check answers finished")
}
func (k *Kickass) search(s Searcher, log *logrus.Entry) ([]polochon.Torrent, error) {
users := s.users()
result := []polochon.Torrent{}
log = log.WithFields(logrus.Fields{
"search_category": s.category(),
"search_string": s.searchStr(),
})
if err := s.validate(); err != nil {
log.Error(err)
return nil, err
}
for _, u := range users {
torrents, err := k.searchUser(s, log, u)
if err != nil {
return nil, err
}
result = append(result, torrents...)
}
return polochon.FilterTorrents(result), nil
}
// Fire fires the event to the ELK beat
func (elk *UDPHook) Fire(e *log.Entry) error {
// Make a copy to safely modify
entry := e.WithFields(nil)
if frameNo := findFrame(); frameNo != -1 {
t := newTrace(frameNo-1, nil)
entry.Data[FileField] = t.String()
entry.Data[FunctionField] = t.Func()
}
data, err := json.Marshal(Frame{
Time: elk.Clock.Now().UTC(),
Type: "trace",
Entry: entry.Data,
Message: entry.Message,
Level: entry.Level.String(),
})
if err != nil {
return Wrap(err)
}
c, err := net.ListenPacket("udp", ":0")
if err != nil {
return Wrap(err)
}
ra, err := net.ResolveUDPAddr("udp", "127.0.0.1:5000")
if err != nil {
return Wrap(err)
}
_, err = (c.(*net.UDPConn)).WriteToUDP(data, ra)
return Wrap(err)
}
// GetLogrusLogger returns the logrus logger for the context. If one more keys
// are provided, they will be resolved on the context and included in the
// logger. Only use this function if specific logrus functionality is
// required.
func getLogrusLogger(ctx Context, keys ...interface{}) *logrus.Entry {
var logger *logrus.Entry
// Get a logger, if it is present.
loggerInterface := ctx.Value("logger")
if loggerInterface != nil {
if lgr, ok := loggerInterface.(*logrus.Entry); ok {
logger = lgr
}
}
if logger == nil {
// If no logger is found, just return the standard logger.
logger = logrus.NewEntry(logrus.StandardLogger())
}
fields := logrus.Fields{}
for _, key := range keys {
v := ctx.Value(key)
if v != nil {
fields[fmt.Sprint(key)] = v
}
}
return logger.WithFields(fields)
}
// GetLogrusLogger returns the logrus logger for the context. If one more keys
// are provided, they will be resolved on the context and included in the
// logger. Only use this function if specific logrus functionality is
// required.
func getLogrusLogger(ctx Context, keys ...interface{}) *logrus.Entry {
var logger *logrus.Entry
// Get a logger, if it is present.
loggerInterface := ctx.Value("logger")
if loggerInterface != nil {
if lgr, ok := loggerInterface.(*logrus.Entry); ok {
logger = lgr
}
}
if logger == nil {
fields := logrus.Fields{}
// Fill in the instance id, if we have it.
instanceID := ctx.Value("instance.id")
if instanceID != nil {
fields["instance.id"] = instanceID
}
fields["go.version"] = runtime.Version()
// If no logger is found, just return the standard logger.
logger = logrus.StandardLogger().WithFields(fields)
}
fields := logrus.Fields{}
for _, key := range keys {
v := ctx.Value(key)
if v != nil {
fields[fmt.Sprint(key)] = v
}
}
return logger.WithFields(fields)
}
// DefaultBefore is the default func assigned to *Middleware.Before
func DefaultBefore(entry *logrus.Entry, req *http.Request, remoteAddr string) *logrus.Entry {
return entry.WithFields(logrus.Fields{
"request": req.RequestURI,
"method": req.Method,
"remote": remoteAddr,
})
}
// Format formats a message from container
func (f *formatter) Format(entry *log.Entry) ([]byte, error) {
e := entry.WithFields(log.Fields{
"container": fmt.Sprintf("%.12s", f.containerID),
})
e.Message = entry.Message
e.Level = f.level
return f.delegate.Format(e)
}
// DefaultAfter is the default func assigned to *Middleware.After
func DefaultAfter(entry *logrus.Entry, res negroni.ResponseWriter, latency time.Duration, name string) *logrus.Entry {
return entry.WithFields(logrus.Fields{
"status": res.Status(),
"text_status": http.StatusText(res.Status()),
"took": latency,
fmt.Sprintf("measure#%s.latency", name): latency.Nanoseconds(),
})
}
// Format formats a message from container
func (f *formatter) Format(entry *log.Entry) ([]byte, error) {
e := entry.WithFields(log.Fields{
"container": f.container.Name.String(),
})
e.Message = entry.Message
e.Level = f.level
return f.delegate.Format(e)
}
func (k *Kickass) listMoviesByUser(movies map[string]*polochon.Movie, user string, log *logrus.Entry) error {
query := &kickass.Query{
User: user,
OrderBy: "seeders",
Order: "desc",
Category: string(MoviesCategory),
}
log = log.WithField("explore_user", user)
torrents, err := k.client.ListByUser(query)
if err != nil {
return err
}
for _, t := range torrents {
torrentStr := torrentGuessitStr(t)
guess, err := guessit.Guess(torrentStr)
if err != nil {
continue
}
// Get the torrent quality
torrentQuality := polochon.Quality(guess.Quality)
if !torrentQuality.IsAllowed() {
log.Infof("kickass: unhandled quality: %q", torrentQuality)
continue
}
// Get the movie if its already in the map
m, ok := movies[guess.Title]
if !ok {
// Create a new movie
m = polochon.NewMovie(polochon.MovieConfig{})
m.Title = guess.Title
if guess.Year != 0 {
m.Year = guess.Year
}
}
log.WithFields(logrus.Fields{
"torrent_quality": guess.Quality,
"movie_title": guess.Title,
}).Debug("Adding torrent to the list")
m.Torrents = append(m.Torrents, polochon.Torrent{
Quality: torrentQuality,
URL: t.MagnetURL,
Seeders: t.Seed,
Leechers: t.Leech,
Source: moduleName,
UploadUser: user,
})
movies[m.Title] = m
}
return nil
}
/*
Open opens file from path
*/
func Open(path string, logger *logrus.Entry) *os.File {
ret, err := os.Open(path)
if err != nil {
logger.WithFields(logrus.Fields{
"type": "error",
"error": err.Error()}).Panic("File Open Failed")
}
return ret
}
/*
CreateFile creates file from path
*/
func CreateFile(name string, logger *logrus.Entry) *os.File {
ret, err := os.Create(name)
if err != nil {
logger.WithFields(logrus.Fields{
"type": "error",
"error": err.Error()}).Panic("File Creation Failed")
}
return ret
}
func (k *Kickass) searchUser(s Searcher, log *logrus.Entry, user string) ([]polochon.Torrent, error) {
query := &kickass.Query{
User: user,
OrderBy: "seeders",
Order: "desc",
Category: string(s.category()),
Search: s.searchStr(),
}
log = log.WithField("search_user", user)
torrents, err := k.client.Search(query)
if err != nil {
return nil, err
}
result := []polochon.Torrent{}
for _, t := range torrents {
torrentStr := torrentGuessitStr(t)
guess, err := guessit.Guess(torrentStr)
if err != nil {
continue
}
if !s.isValidGuess(guess, log) {
continue
}
// Default quality
if s.category() == ShowsCategory && guess.Quality == "" {
guess.Quality = string(polochon.Quality480p)
}
// Get the torrent quality
torrentQuality := polochon.Quality(guess.Quality)
if !torrentQuality.IsAllowed() {
log.Infof("kickass: unhandled quality: %q", torrentQuality)
continue
}
log.WithFields(logrus.Fields{
"torrent_quality": guess.Quality,
"torrent_name": torrentStr,
}).Debug("Adding torrent to the list")
// Add the torrent
result = append(result, polochon.Torrent{
Quality: torrentQuality,
URL: t.MagnetURL,
Seeders: t.Seed,
Leechers: t.Leech,
Source: moduleName,
UploadUser: user,
})
}
return result, nil
}
func handleConnection(conn net.Conn, jobd *jobserver.JobServer, cbor *codec.CborHandle, reqLogger *logrus.Logger) {
defer conn.Close()
var reqLog, errLog *logrus.Entry
fields := logrus.Fields{
"remote": conn.RemoteAddr(),
}
errLog = logrus.WithFields(fields)
if reqLogger != nil {
reqLog = reqLogger.WithFields(fields)
}
jobdv := reflect.ValueOf(jobd)
reader := bufio.NewReader(conn)
decoder := codec.NewDecoder(reader, cbor)
writer := bufio.NewWriter(conn)
encoder := codec.NewEncoder(writer, cbor)
for {
var request cborrpc.Request
err := decoder.Decode(&request)
if err == io.EOF {
if reqLog != nil {
reqLog.Debug("Connection closed")
}
return
} else if err != nil {
errLog.WithError(err).Error("Error reading message")
return
}
if reqLog != nil {
reqLog.WithFields(logrus.Fields{
"id": request.ID,
"method": request.Method,
}).Debug("Request")
}
response := doRequest(jobdv, request)
if reqLog != nil {
entry := reqLog.WithField("id", response.ID)
if response.Error != "" {
entry = entry.WithField("error", response.Error)
}
entry.Debug("Response")
}
err = encoder.Encode(response)
if err != nil {
errLog.WithError(err).Error("Error encoding response")
return
}
err = writer.Flush()
if err != nil {
errLog.WithError(err).Error("Error writing response")
return
}
}
}
/*
ToInfo converts *os.File to os.FileInfo
*/
func ToInfo(file *os.File, logger *logrus.Entry) os.FileInfo {
ret, err := file.Stat()
if err != nil {
logger.WithFields(logrus.Fields{
"type": "error",
"error": IE.String(),
"info": err.Error()}).Panic("Input files not found")
}
return ret
}
/*
ReadDir reads directory from path
*/
func ReadDir(dir string, logger *logrus.Entry) []os.FileInfo {
files, err := ioutil.ReadDir(dir)
if err != nil {
logger.WithFields(logrus.Fields{
"type": "error",
"error": IE.String(),
"info": err.Error()}).Panic("Input files not found")
}
return files
}
func (s *Sentry) checkLocation(mirror *bouncer.MirrorsActiveResult, location *bouncer.LocationsActiveResult, mirrorLog *logrus.Entry) *checkLocationResult {
locationLog := mirrorLog.WithFields(logrus.Fields{
"location": location.Path,
})
lang := "en-US"
if strings.Contains(location.Path, "/firefox/") &&
!strings.Contains(location.Path, "/namoroka/") &&
!strings.Contains(location.Path, "/devpreview/") &&
!strings.Contains(location.Path, "3.6b1") &&
!strings.Contains(location.Path, "wince-arm") &&
!strings.Contains(strings.ToLower(location.Path), "euballot") {
lang = "zh-TW"
} else if strings.Contains(location.Path, "/thunderbird/") {
if strings.Contains(location.Path, "3.1a1") {
lang = "tr"
} else {
lang = "zh-TW"
}
} else if strings.Contains(location.Path, "/seamonkey/") {
if strings.Contains(location.Path, "2.0.5") || strings.Contains(location.Path, "2.0.6") {
lang = "zh-CN"
} else {
lang = "tr"
}
} else if strings.Contains(strings.ToLower(location.Path), "-euballot") {
lang = "sv-SE"
}
path := strings.Replace(location.Path, ":lang", lang, -1)
url := mirror.BaseURL + path
start := time.Now()
active, healthy := true, false
resp, err := s.HeadLocation(url)
elapsed := time.Now().Sub(start)
if err != nil {
locationLog.WithError(err).Errorf("%s TOOK=%v", url, elapsed)
return &checkLocationResult{Active: true, Healthy: false}
}
defer resp.Body.Close()
if resp.StatusCode == 200 && !strings.Contains(resp.Header.Get("Content-Type"), "text/html") {
active, healthy = true, true
} else if resp.StatusCode == 404 || resp.StatusCode == 403 {
active, healthy = false, false
}
locationLog.Infof("%s TOOK=%v RC=%d", url, elapsed, resp.StatusCode)
return &checkLocationResult{Active: active, Healthy: healthy}
}
// Format implements logrus.Formatter interface and adds file and line
func (tf *TextFormatter) Format(e *log.Entry) ([]byte, error) {
if frameNo := findFrame(); frameNo != -1 {
t := newTrace(frameNo, nil)
new := e.WithFields(log.Fields{FileField: t.Loc(), FunctionField: t.FuncName()})
new.Time = e.Time
new.Level = e.Level
new.Message = e.Message
e = new
}
return (&tf.TextFormatter).Format(e)
}
/*
CreateDir creates directory.
*/
func CreateDir(dir string, logger *logrus.Entry) {
_, err := os.Stat(dir)
if err != nil {
err := os.MkdirAll(dir, 0755)
if err != nil {
logger.WithFields(logrus.Fields{
"type": "error",
"error": err.Error()}).Panic("Create Directory Failed")
}
}
}
func (c *Cleaner) deleteDirectory(directoryPath string, log *logrus.Entry) error {
// Delete the directory
deletePath := filepath.Join(c.config.Downloader.Cleaner.TrashDir, path.Base(directoryPath))
log.WithFields(logrus.Fields{
"old_path": directoryPath,
"new_path": deletePath,
}).Debug("deleting folder")
return os.Rename(directoryPath, deletePath)
}
// joinFields on a entry.
func joinFields(e *logrus.Entry, fs []logrus.Fields) *logrus.Entry {
if len(fs) == 0 {
return e
}
for i := range fs {
e = e.WithFields(fs[i])
}
return e
}
func (exes *executions) write(logger *logrus.Entry, ws *websocket.Conn) {
exes.writerWg.Add(1)
go func() {
defer exes.writerWg.Done()
for msg := range exes.messages {
logger.WithFields(logrus.Fields{"response": msg}).Info("response")
if ws.WriteJSON(msg) != nil {
break
}
}
}()
}
// addFileLine is a helper that adds the file & line of the original
// caller of the Logging function if possible.
func addFileLine(entry *log.Entry) *log.Entry {
if _, file, line, ok := runtime.Caller(2); ok {
// cut all of the filepath before the "src" folder
if idx := strings.Index(file, "/src/"); idx > -1 {
file = file[idx+5:]
}
entry = entry.WithFields(log.Fields{
"file": file,
"line": line,
})
}
return entry
}
func (h TcpHealthCheck) VerifyResponse(answer string, contextLogger *log.Entry) bool {
ansLogger := contextLogger.WithFields(log.Fields{
"answer": answer,
"expect": h.Expect,
})
if strings.Contains(answer, h.Expect) {
ansLogger.Debug("Healthy response")
return true
}
ansLogger.Debug("Unhealthy response")
return false
}
func addSourceLocations(e *logrus.Entry) *logrus.Entry {
_, file, line, _ := runtime.Caller(2)
// attempt to strip the project prefix
parts := strings.SplitN(file, "horizon/", 2)
if len(parts) == 2 {
file = parts[1]
}
return e.WithFields(logrus.Fields{
"file": file,
"line": line,
})
}
//implementation of generating input testcases
func test(dir, name string, id int, info ProgInfo, tests []os.FileInfo, timelimit float64, chk checker, isConcurrent bool, logger *logrus.Entry) {
logger = logger.WithFields(logrus.Fields{
"state": "test",
"id": id,
"name": name})
logger.WithFields(logrus.Fields{
"type": "info",
"info": fmt.Sprintf("test start")}).Infof("answer#%d(%s): start", id, name)
//compilation
_, err := Compile(info)
if err != nil {
logger.WithFields(logrus.Fields{
"type": "error",
"error": CE.String(),
"info": err.Error()}).Errorf("answer#%d(%s): %s", id, name, CE.String())
return
}
cnt := 0
//generation
if isConcurrent {
//counter
inc := func() func() {
m := new(sync.Mutex)
return func() {
m.Lock()
cnt++
m.Unlock()
}
}()
wg := new(sync.WaitGroup)
w := make(chan bool, GetWorkerCount())
for _, file := range tests {
wg.Add(1)
w <- true
go func(x os.FileInfo) {
defer func() {
wg.Done()
<-w
}()
if judge(dir, name, id, info, x, timelimit, chk, logger) {
inc()
}
}(file)
}
wg.Wait()
close(w)
} else {
for _, x := range tests {
if judge(dir, name, id, info, x, timelimit, chk, logger) {
cnt++
}
}
}
logger.WithFields(logrus.Fields{
"type": "info",
"info": fmt.Sprintf("test end")}).Infof("answer#%d(%s): end %d/%d files passed", id, name, cnt, len(tests))
}
func customAfter(entry *logrus.Entry, res negroni.ResponseWriter, latency time.Duration, name string) *logrus.Entry {
fields := logrus.Fields{
"ALL_DONE": true,
"RESPONSE_STATUS": res.Status(),
fmt.Sprintf("%s_LATENCY", strings.ToUpper(name)): latency,
}
// one way to replace an existing entry key
if requestId, ok := entry.Data["request_id"]; ok {
fields["REQUEST_ID"] = requestId
delete(entry.Data, "request_id")
}
return entry.WithFields(fields)
}
func claimTask(client client.Queue, taskID string, runID int, workerID string, workerGroup string, log *logrus.Entry) (*taskClaim, error) {
log.Info("Claiming task")
payload := queue.TaskClaimRequest{
WorkerGroup: workerGroup,
WorkerID: workerID,
}
tcrsp, err := client.ClaimTask(taskID, strconv.Itoa(runID), &payload)
// check if an error occurred...
if err != nil {
switch err := err.(type) {
case httpbackoff.BadHttpResponseCode:
e := &updateError{
err: err.Error(),
statusCode: err.HttpResponseCode,
}
var errorMessage string
switch {
case err.HttpResponseCode == 401 || err.HttpResponseCode == 403:
errorMessage = fmt.Sprintf("Not authorized to claim task %s (status code %v).", taskID, err.HttpResponseCode)
case err.HttpResponseCode >= 500:
errorMessage = fmt.Sprintf("Server error (status code %v) when attempting to claim task %v.", err.HttpResponseCode, taskID)
case err.HttpResponseCode != 200:
errorMessage = fmt.Sprintf("Received http status code %v when claiming task %v.", err.HttpResponseCode, taskID)
}
log.WithFields(logrus.Fields{
"error": err,
"statusCode": err.HttpResponseCode,
}).Error(errorMessage)
return nil, e
default:
log.WithFields(logrus.Fields{
"error": err,
}).Error(fmt.Sprintf("Unexpected error occurred when claiming task %s", taskID))
return nil, err
}
}
return &taskClaim{
taskID: taskID,
runID: runID,
taskClaim: tcrsp,
definition: &tcrsp.Task,
}, nil
}
// RemoveContainer removes a container
func RemoveContainer(logger *log.Entry, client client.DockerClient, containerID string, expectContainer bool) {
logger.Debug("Removing container")
err := client.RemoveContainer(docker.RemoveContainerOptions{
ID: containerID,
RemoveVolumes: true,
})
switch err.(type) {
case *docker.NoSuchContainer:
if !expectContainer {
return
}
case nil:
return
}
logger.WithFields(log.Fields{"container": containerID}).Warnf(
"Failed to remove container: %s", err)
}
/*
Load problem from directory
*/
func Load(path string, logger *logrus.Entry) (Problem, error) {
logger = logger.WithField("state", "load_problem")
path = filepath.Join(path, "problem.toml")
var problem Problem
_, err := toml.DecodeFile(path, &problem)
if err != nil {
logger.WithFields(logrus.Fields{
"type": "error",
"error": IE.String(),
"info": err.Error()}).Error("Loading problem: Failed")
return problem, err
}
logger.WithFields(logrus.Fields{
"type": "info",
"info": problem}).Infof("Loaded Problem: %s", problem.Name)
return problem, nil
}
