func fetch(url string, ch chan<- string) { start := time.Now() resp, err := http.Get(url) if err != nil { ch <- fmt.Sprint(err) // send to channel ch return } file, err := os.Create(sanitize.BaseName(url) + " " + time.Now().String()) if err != nil { ch <- fmt.Sprint(err) } defer file.Close() nbytes, err := io.Copy(file, resp.Body) resp.Body.Close() // don't leak resources if err != nil { ch <- fmt.Sprintf("while reading %s: %v", url, err) return } secs := time.Since(start).Seconds() ch <- fmt.Sprintf("%.2fs %7d %s", secs, nbytes, url) }
func generateFileName(fileName string) string { ext := filepath.Ext(fileName) randomExts := []string{".jpg", ".webm", ".gif", ".jpeg", ".png"} for _, extension := range randomExts { if extension == ext { return randomFileName(5, ext) } } return sanitize.BaseName(fileName) + ext }
//Makes sure that a string is a valid PostgreSQL identifier func postgresify(identifier string) string { str := sanitize.BaseName(identifier) str = strings.ToLower(identifier) str = strings.TrimSpace(str) replacements := map[string]string{ " ": "_", "/": "_", ".": "_", ":": "_", ";": "_", "|": "_", "-": "_", ",": "_", "#": "_", "[": "", "]": "", "{": "", "}": "", "(": "", ")": "", "?": "", "!": "", "$": "", "%": "", "*": "", "\"": "", } for oldString, newString := range replacements { str = strings.Replace(str, oldString, newString, -1) } if len(str) == 0 { str = fmt.Sprintf("_col%d", rand.Intn(10000)) } else { firstLetter := string(str[0]) if _, err := strconv.Atoi(firstLetter); err == nil { str = "_" + str } } return str }
// Handle events func handleEvents(w http.ResponseWriter, r *http.Request) { // Since multiple requests could come in at once, ensure we have a lock // around all file operations eventMutex.Lock() defer eventMutex.Unlock() vars := mux.Vars(r) apiKey := sanitize.BaseName(vars["apikey"]) if apiKey == "" { handleError(w, "An API Key must be provided", http.StatusBadRequest) return } dataFileSession := fmt.Sprintf(dataFileBaseName, apiKey) // Stat the file, so we can find its current permissions var fi os.FileInfo var errStat error fi, errStat = os.Stat(dataFileSession) if errStat != nil { f, err := os.Create(dataFileSession) if err != nil { handleError(w, "Error when creating session file", http.StatusInternalServerError) } fi, _ = f.Stat() f.WriteString("[]") f.Close() } // Read the events from the file. eventData, err := ioutil.ReadFile(dataFileSession) if err != nil { handleError(w, fmt.Sprintf("Unable to read the data file (%s): %s", dataFileSession, err), http.StatusInternalServerError) return } switch r.Method { case "POST": // Decode the JSON data events := make([]eventItem, 0) if err := json.Unmarshal(eventData, &events); err != nil { handleError(w, fmt.Sprintf("Unable to Unmarshal events from data file (%s): %s", dataFileSession, err), http.StatusInternalServerError) return } response, _ := ioutil.ReadAll(r.Body) TraceLogger.Println("New event payload received", string(response)) // Add a new event to the in memory slice of events var mjEvent mailjetAPIEvent err1 := json.Unmarshal(response, &mjEvent) if err1 != nil { handleError(w, err1.Error(), http.StatusBadRequest) return } var mjEventPayload eventPayload json.Unmarshal(response, &mjEventPayload) newEventItem := eventItem{ EventType: mjEvent.Event, Payload: mjEventPayload, } events = append([]eventItem{newEventItem}, events...) if config.MaxEventsCount > 0 && len(events) > config.MaxEventsCount { events = events[:config.MaxEventsCount] } // Marshal the events to indented json. var err3 error eventData, err3 = json.Marshal(events) if err3 != nil { handleError(w, fmt.Sprintf("Unable to marshal events to json: %s", err3), http.StatusInternalServerError) return } // Write out the events to the file, preserving permissions err2 := ioutil.WriteFile(dataFileSession, eventData, fi.Mode()) if err2 != nil { handleError(w, fmt.Sprintf("Unable to write events to data file (%s): %s", dataFileSession, err3), http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") w.Header().Set("Cache-Control", "no-cache") io.Copy(w, bytes.NewReader(eventData)) case "GET": w.Header().Set("Content-Type", "application/json") w.Header().Set("Cache-Control", "no-cache") // stream the contents of the file to the response io.Copy(w, bytes.NewReader(eventData)) default: // Don't know the method, so error handleError(w, fmt.Sprintf("Unsupported method: %s", r.Method), http.StatusMethodNotAllowed) } }
// cacheEntryFilename creates a filename-safe name in a subdirectory // of the configured cache dir, with any access token stripped out. func cacheEntryFilename(c *Context, url string) string { newUrl := strings.Replace(url, fmt.Sprintf("access_token=%s", c.Token), "", 1) return filepath.Join(c.CacheDir, c.Repo, sanitize.BaseName(newUrl)) }