func DeleteImage(ctx context.Context) error {
var currentUser, ok = ctx.Data()["user"].(m.User)
if ok {
imageId := ctx.FormValue("imageId")
if imageId == "" {
return goweb.API.Respond(ctx, 200, nil, []string{"You have to specify an image to be deleted."})
}
var image m.Image
err := m.GetDB("Image").FindId(bson.ObjectIdHex(imageId)).One(&image)
if err != nil {
log.Error("DeleteImage, remove query " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"You have to specify an image to be deleted."})
}
if image.User.Hex() != currentUser.Id.Hex() {
return goweb.API.Respond(ctx, 200, nil, []string{"You cannot delete this image."})
} else {
m.GetDB("Image").RemoveId(image.Id)
err = os.Remove(image.Url)
}
return goweb.API.RespondWithData(ctx, "Image deleted.")
} else {
return goweb.API.Respond(ctx, 200, nil, []string{"Please log in to delete an images."})
}
}
func LinkGpsToImages(track m.Track) {
var possibleImages []m.Image
m.GetDB("Image").Find(bson.M{"date": bson.M{"$lte": track.MaxDate, "$gte": track.MinDate}, "user": track.User}).All(&possibleImages)
if len(possibleImages) == 0 {
return
}
var best m.Coordinate
var before time.Time
var after time.Time
for i := 0; i < len(possibleImages); i++ {
before = possibleImages[i].Date.Add(time.Duration(-10 * time.Second))
after = possibleImages[i].Date.Add(time.Duration(10 * time.Second))
for j := 0; j < len(track.Coordinates); j++ {
if before.Before(track.Coordinates[j].Date) && after.After(track.Coordinates[j].Date) {
best = track.Coordinates[j]
break
}
}
if best.Lat != "" && best.Lon != "" {
possibleImages[i].Lat = best.Lat
possibleImages[i].Lon = best.Lon
err := m.GetDB("Image").UpdateId(possibleImages[i].Id, possibleImages[i])
if err != nil {
log.Error("LinkGpdToImages, update, " + err.Error())
}
}
}
}
func ProcessZip(currentUser m.User, outputZipPath string) {
var imagesFound = 0
var tablesFound = 0
var WalkImageCallback = func(path string, fi os.FileInfo, err error) error {
if err != nil {
return nil
}
if fi.IsDir() {
return nil
}
if !strings.Contains(fi.Name(), ".txt") {
imagesFound++
log.Info(strconv.Itoa(imagesFound))
file, err := os.Open(path)
if err != nil {
// log.Error("Failed to read image file from zip: " + err.Error())
return nil
}
ProcessImage(currentUser, file, nil, true)
}
return nil
}
var WalkITableCallback = func(path string, fi os.FileInfo, err error) error {
if err != nil {
return nil
}
if fi.IsDir() {
return nil
}
if strings.Contains(fi.Name(), ".txt") {
tablesFound++
// log.Info(strconv.Itoa(tablesFound))
file, err := os.Open(path)
if err != nil {
log.Error("Failed to read txt file from zip: " + err.Error())
return nil
}
defer file.Close()
ParseImageTable(currentUser, file)
}
return nil
}
log.Info("Received zip file for processing")
filepath.Walk(outputZipPath, WalkImageCallback)
log.Info("Images found in zip: " + strconv.Itoa(imagesFound))
log.Info("Starting processing of image tables.")
filepath.Walk(outputZipPath, WalkITableCallback)
log.Info("Tables found in zip: " + strconv.Itoa(tablesFound))
os.Remove(outputZipPath)
}
func Connect() {
// Database
// mongolab.com
var dbServer = "mongolab"
_db, err = mgo.Dial("mongodb://127.0.0.1:27017/gms")
if err != nil {
log.Error("Error on database connection: " + err.Error())
_db, err = mgo.Dial("mongodb://127.0.0.1:27017/gms")
dbServer = "local db"
if err != nil {
panic(err)
}
}
// This makes the db monotonic
_db.SetMode(mgo.Monotonic, true)
// Database name (test) and the collection names ("User") are set
collections := _db.DB(db_name)
db_map["User"] = collections.C("User")
db_map["Passport"] = collections.C("Passport")
db_map["Comment"] = collections.C("Comment")
db_map["Track"] = collections.C("Track")
db_map["Image"] = collections.C("Image")
db_map["Video"] = collections.C("Video")
db_map["Cluster"] = collections.C("Cluster")
db_map["RecommendUser"] = collections.C("RecommendUser")
db_map["TrendingAllUsers"] = collections.C("TrendingAllUsers")
db_map["RecommendAllUsers"] = collections.C("RecommendAllUsers")
log.Info("Database connection established with " + dbServer)
}
func Root(ctx context.Context) error {
file, err := ioutil.ReadFile("../index.html")
if err != nil {
log.Error("Error reading index.html: " + err.Error())
return goweb.Respond.With(ctx, http.StatusOK, []byte(`<html><body><h2>Log in with...</h2>
<ul><li><a href="/api/auth/facebook/login/">Facebook</a></li></ul></body></html>`))
}
return goweb.Respond.With(ctx, http.StatusOK, file)
}
func GetVideos(ctx context.Context) error {
var result []m.Video
// Get the info from the context
fromDate, fromDateExists := utils.ParseDateFromContext("from_", ctx)
toDate, toDateExists := utils.ParseDateFromContext("to_", ctx)
limit, err := strconv.Atoi(ctx.FormValue("limit"))
if err != nil {
limit = 5
}
skip, err := strconv.Atoi(ctx.FormValue("skip"))
if err != nil {
skip = 0
}
var currentUser, ok = ctx.Data()["user"].(m.User)
isPersonal, err := strconv.ParseBool(ctx.FormValue("personal"))
if err != nil {
isPersonal = false
}
query := bson.M{}
if ok && isPersonal {
if fromDateExists || toDateExists {
query["date"] = bson.M{"$gte": fromDate, "$lte": toDate}
query["user"] = currentUser.Id
} else {
query["user"] = currentUser.Id
}
} else {
isPersonal = false
if fromDateExists || toDateExists {
query["date"] = bson.M{"$gte": fromDate, "$lte": toDate}
}
}
if !fromDateExists && !toDateExists {
var lastVideo m.Image
var oneQuery = bson.M{}
if isPersonal {
oneQuery["user"] = currentUser.Id
}
if err = m.GetDB("Video").Find(oneQuery).Sort("-date").One(&lastVideo); err == nil {
query["date"] = bson.M{"$gte": lastVideo.Date.AddDate(0, 0, -1), "$lte": lastVideo.Date.AddDate(0, 0, 1)}
}
}
if err := m.GetDB("Video").Find(query).Skip(skip).Limit(limit).All(&result); err != nil {
log.Error("Failed to search for tracks: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to find the tracks."})
}
return goweb.API.RespondWithData(ctx, result)
}
func LoadCypherKey() error {
// Load the encryption key
var err error
key, err = ioutil.ReadFile("../key.pub")
if err != nil {
log.Error("Error reading key: " + err.Error())
return err
}
return nil
}
func UploadVideo(ctx context.Context) error {
var currentUser, ok = ctx.Data()["user"].(m.User)
if ok {
file, _, err := ctx.HttpRequest().FormFile("file")
if err != nil {
log.Error("Error on image upload FormFile " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to upload the image."})
}
defer file.Close()
outputPath := "../uploads/" + currentUser.Username + "/" + base64.StdEncoding.EncodeToString([]byte(currentUser.Username+time.Now().String()))
if err := os.MkdirAll("../uploads/"+currentUser.Username+"/", 0777); err != nil {
log.Error("Error in creating output file " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to upload the image."})
}
outputFile, err := os.Create(outputPath)
if err != nil {
log.Error("Error in creating output file " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to upload the image."})
}
defer outputFile.Close()
if _, err = io.Copy(outputFile, file); err != nil {
log.Error("Error in copying file " + err.Error())
os.Remove(outputPath)
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to upload the image."})
}
if err = m.GetDB("Video").Insert(&m.Video{currentUser.Id, outputPath, time.Now()}); err != nil {
log.Error("Error in saving video file to db " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to upload the image."})
}
return goweb.API.Respond(ctx, 200, nil, nil)
} else {
return goweb.API.Respond(ctx, 200, nil, []string{"Please log in to upload your videos."})
}
}
func ProcessImage(currentUser m.User, file io.ReadCloser, imageWaitGroup *sync.WaitGroup, closeReader bool) error {
if closeReader {
defer file.Close()
}
outputPath := "../uploads/" + currentUser.Username + "/" + base64.StdEncoding.EncodeToString([]byte(currentUser.Username+time.Now().String()))
if err := os.MkdirAll("../uploads/"+currentUser.Username+"/", 0777); err != nil {
log.Error("Error in creating output dir " + err.Error())
return errors.New("Failed to upload the image.")
}
outputFile, err := os.Create(outputPath)
if err != nil {
log.Error("Error in creating output file, process image " + err.Error())
return errors.New("Failed to upload the image.")
}
defer outputFile.Close()
if _, err = io.Copy(outputFile, file); err != nil {
log.Error("Error in copying file " + err.Error())
os.Remove(outputPath)
return errors.New("Failed to upload the image.")
}
var lat, long string
var date time.Time
openedFile, err := os.Open(outputPath)
if err != nil {
log.Error("Error opening file in image upload " + err.Error())
}
defer openedFile.Close()
if exifParser, _ := exif.Decode(openedFile); exifParser == nil {
lat = ""
long = ""
date = time.Now()
} else {
lat = ""
long = ""
date, err = exifParser.DateTime()
if err != nil {
log.Error("Error in exif parser date: " + err.Error())
date = time.Now()
}
}
image := m.Image{bson.NewObjectId(), currentUser.Id, outputPath, lat, long, date, time.Now(), 1, "", true, bson.NewObjectId(),
"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", false}
if err = m.GetDB("Image").Insert(&image); err != nil {
log.Error("Error in saving image file to db " + err.Error())
return errors.New("Failed to upload the image.")
}
LinkImageToGps(image, imageWaitGroup)
return nil
}
func Connect(ctx context.Context) error {
provider_type := ctx.PathValue("provider")
action := ctx.PathValue("action")
if provider_type == "facebook" {
provider, err := gomniauth.Provider(provider_type)
if err != nil {
log.Error("Error on getting provider: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"An error has occured."})
}
state := gomniauth.NewState("after", "success")
// if you want to request additional scopes from the provider,
// pass them as login?scope=scope1,scope2
//options := objx.MSI("scope", ctx.QueryValue("scope"))
authUrl, err := provider.GetBeginAuthURL(state, nil)
if err != nil {
log.Error("Error on getting url: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"An error has occured."})
}
// redirect
return goweb.Respond.WithRedirect(ctx, authUrl)
} else if provider_type == "local" && ctx.MethodString() == "POST" {
// This is taken care of in separate functions.
// Local login only with POST
if action == "login" {
return nil
} else if action == "register" {
return nil
} else if action == "connect" {
return nil
} else {
return goweb.API.Respond(ctx, 200, nil, []string{"Invalid action."})
}
} else {
return goweb.API.Respond(ctx, 200, nil, []string{"Invalid provider type."})
}
}
func UploadImgData(ctx context.Context) error {
var currentUser, ok = ctx.Data()["user"].(m.User)
if ok {
file, _, err := ctx.HttpRequest().FormFile("file")
if err != nil {
log.Error("Error on image upload FormFile " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to upload the image."})
}
defer file.Close()
utils.ParseImageTable(currentUser, file)
return goweb.API.Respond(ctx, 200, nil, nil)
} else {
return goweb.API.Respond(ctx, 200, nil, []string{"Please log in to upload your images."})
}
}
func UploadZip(ctx context.Context) error {
var currentUser, ok = ctx.Data()["user"].(m.User)
if ok {
file, _, err := ctx.HttpRequest().FormFile("file")
if err != nil {
log.Error("Error on zip upload FormFile " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to upload the zip."})
}
defer file.Close()
outputZipPath := "../uploads/temp/" + base64.StdEncoding.EncodeToString([]byte(currentUser.Username+time.Now().String()))
if err := os.MkdirAll("../uploads/temp/", 0777); err != nil {
log.Error("Error in creating output dir " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to upload the zip."})
}
outputZipFile, err := os.Create(outputZipPath)
if err != nil {
log.Error("Error in creating output file in zip " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to upload the zip."})
}
defer outputZipFile.Close()
if _, err = io.Copy(outputZipFile, file); err != nil {
log.Error("Error in copying file " + err.Error())
os.Remove(outputZipPath)
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to upload the zip."})
}
var extention = ""
if runtime.GOOS == "windows" {
extention = ".exe"
}
extractPath := "../uploads/temp/" + base64.StdEncoding.EncodeToString([]byte(currentUser.Username+time.Now().String()))
commandString := fmt.Sprintf(`7za%s e %s -o%s -p%s -aoa`, extention, outputZipPath, extractPath, "SuperSecurePassword")
commandSlice := strings.Fields(commandString)
commandCall := exec.Command(commandSlice[0], commandSlice[1:]...)
// err = commandCall.Run()
value, err := commandCall.Output()
if err != nil {
log.Error("in unarchiving zip file: " + err.Error())
log.Error("Full info about the error: " + string(value))
}
go utils.ProcessZip(currentUser, extractPath)
return goweb.API.Respond(ctx, 200, nil, nil)
} else {
return goweb.API.Respond(ctx, 200, nil, []string{"Please log in to upload a zip."})
}
}
// This should be the work dispatching thread, only dispatching work between 0 and 7
func RunProcessing() {
// create a channel for the pool
imageChannel := make(chan m.Image, 50)
// create a work done channel
done := make(chan int, threads)
counter := 0
for true {
var notProcessedImages []m.Image
if counter < 1 {
counter = 4
count, err := m.GetDB("Image").Find(bson.M{"processed": false}).Count()
if err != nil {
log.Error("Could not count the unprocessed images")
} else {
log.Info("Total unprocessed images left: " + strconv.Itoa(count))
}
} else {
counter--
}
m.GetDB("Image").Find(bson.M{"processed": false}).Limit(50).All(¬ProcessedImages)
if len(notProcessedImages) == 0 {
log.Info("No images found, sleeping for 1 hour")
counter = 0
time.Sleep(1 * time.Hour)
continue
}
for _, image := range notProcessedImages {
imageChannel <- image
}
for i := threads; i > 0; i-- {
go ProcessImage(imageChannel, done)
}
for i := threads; i > 0; i-- {
<-done
}
}
}
func LinkImageToGps(image m.Image, imageWaitGroup *sync.WaitGroup) {
var possibleTracks []m.Track
m.GetDB("Track").Find(bson.M{"minDate": bson.M{"$lte": image.Date}, "maxDate": bson.M{"$gte": image.Date}, "user": image.User}).All(&possibleTracks)
var best m.Coordinate
before := image.Date.Add(time.Duration(-10 * time.Second))
after := image.Date.Add(time.Duration(10 * time.Second))
for i := 0; i < len(possibleTracks); i++ {
for j := 0; j < len(possibleTracks[i].Coordinates); j++ {
if before.Before(possibleTracks[i].Coordinates[j].Date) && after.After(possibleTracks[i].Coordinates[j].Date) {
best = possibleTracks[i].Coordinates[j]
break
}
}
if best.Lat != "" && best.Lon != "" {
break
}
}
// attach location
if best.Lat != "" && best.Lon != "" {
image.Lat = best.Lat
image.Lon = best.Lon
}
// save the image
err := m.GetDB("Image").UpdateId(image.Id, image)
if err != nil {
log.Error("LinkImageToGps, update, " + err.Error())
}
if imageWaitGroup != nil {
imageWaitGroup.Done()
}
}
func Register(ctx context.Context) error {
var currentUser, ok = ctx.Data()["user"].(m.User)
if ok {
return goweb.API.RespondWithData(ctx, currentUser.Token)
} else {
username := ctx.FormValue("username")
email := ctx.FormValue("email")
password := ctx.FormValue("password")
// Check or basic auth header
authHeader := ctx.HttpRequest().Header.Get("Authorization")
if len(authHeader) > 0 {
info, err := base64.StdEncoding.DecodeString(strings.Split(authHeader, " ")[1])
if err == nil {
decryptedInfo := strings.Split(string(info), ":")
username = decryptedInfo[0]
password = decryptedInfo[1]
email = decryptedInfo[2]
}
}
if len(username) < 3 {
return goweb.API.Respond(ctx, 200, nil, []string{"The username has to be at least 4 characters long."})
}
if len(email) == 0 {
return goweb.API.Respond(ctx, 200, nil, []string{"Please supply an email address."})
}
err := utils.CheckValidPassword(password)
if err != nil {
return goweb.API.Respond(ctx, 200, nil, []string{err.Error()})
}
var oldUser m.User
err = m.GetDB("User").Find(bson.M{"username": username}).One(&oldUser)
if err == nil {
log.Debug("Username already taken.")
return goweb.API.Respond(ctx, 200, nil, []string{"Username already taken."})
}
var token = nonce.NewToken()
nonce.MarkToken(token)
newUser := m.User{bson.NewObjectId(), username, email, true, token, time.Now()}
err = m.GetDB("User").Insert(&newUser)
if err != nil {
log.Error("Error on local registration, new user: "******"Failed to register."})
}
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), 10)
err = m.GetDB("Passport").Insert(&m.Passport{bson.NewObjectId(), newUser.Id, string(hashedPassword), "local", "", "", ""})
if err != nil {
log.Error("Error on local registration, new user passport: " + err.Error())
m.GetDB("User").RemoveId(newUser.Id)
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to create your new passport."})
}
log.Info("New user registered local")
return goweb.API.RespondWithData(ctx, newUser.Token)
}
}
func ImagePostprocessing(image m.Image, imageWaitGroup *sync.WaitGroup) {
// get the blur score
var ext = ".sh"
if runtime.GOOS == "windows" {
ext = ".bat"
}
blurArr, err := exec.Command("./processing/blurScript"+ext, image.Url).Output()
if err != nil {
log.Error("ImagePostprocessing, get blur score: " + err.Error())
}
image.Blur, err = strconv.ParseFloat(strings.Trim(string(blurArr), "\n "), 64)
if err != nil {
log.Error("Find blur " + err.Error())
image.Blur = 1
}
image.Show = image.Blur < BlurThreshold
// get the phash
// Old call
phash, err := exec.Command("./processing/phashScript"+ext, image.Url).Output()
if err != nil {
log.Error("ImagePostprocessing, get phash score: " + err.Error())
}
image.Phash = string(phash)
// hash, err := phash.ImageHashDCT("../" + image.Url)
// if err != nil {
// log.Error("LinkImageToGps, phash: " + err.Error())
// image.Phash = ""
// } else {
// image.Phash = strconv.FormatUint(hash, 2)
// log.Error(image.Phash)
// }
// cluster (by date + phash(humming distance) + user)
var clusters []m.Cluster
mutex.Lock()
err = m.GetDB("Cluster").Find(bson.M{"user": image.User, "date": bson.M{"$gt": time.Now().AddDate(0, 0, -1), "$lt": time.Now().AddDate(0, 0, 1)}}).All(&clusters)
if err != nil || len(clusters) == 0 {
// create a new cluster
CreateNewCluster(image)
} else {
found := false
for index := range clusters {
if Distance(clusters[index].Phash, image.Phash) < ClusterThreshold {
// add it to the cluster
image.Cluster = clusters[index].Id
if !clusters[index].HasSelected && image.Show {
clusters[index].HasSelected = true
err = m.GetDB("Cluster").UpdateId(clusters[index].Id, clusters[index])
if err != nil {
log.Error("ImagePostprocessing, update cluster: " + err.Error())
}
} else {
image.Show = false
}
found = true
}
}
if !found {
CreateNewCluster(image)
}
}
mutex.Unlock()
image.Processed = true
// save the image
err = m.GetDB("Image").UpdateId(image.Id, image)
if err != nil {
log.Error("ImagePostprocessing, update, " + err.Error())
}
if imageWaitGroup != nil {
imageWaitGroup.Done()
}
}
func Login(ctx context.Context) error {
var currentUser, ok = ctx.Data()["user"].(m.User)
if ok {
return goweb.API.RespondWithData(ctx, currentUser.Token)
} else {
username := ctx.FormValue("username")
password := ctx.FormValue("password")
// Check or basic auth header
authHeader := ctx.HttpRequest().Header.Get("Authorization")
if len(authHeader) > 0 {
info, err := base64.StdEncoding.DecodeString(strings.Split(authHeader, " ")[1])
if err == nil {
decryptedInfo := strings.Split(string(info), ":")
username = decryptedInfo[0]
password = decryptedInfo[1]
}
}
if len(username) < 3 || len(password) < 6 {
return goweb.API.Respond(ctx, 200, nil, []string{"Invalid data supplied, please fill all fields."})
}
user := m.User{}
query := bson.M{}
if strings.Contains(username, "@") {
query["username"] = username
} else {
query["email"] = username
}
err := m.GetDB("User").Find(query).One(&user)
if err != nil {
log.Error("Error in find user on login " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"No such username, please register."})
}
passport := m.Passport{}
err = m.GetDB("Passport").Find(bson.M{"provider": "local", "user": user.Id}).One(&passport)
if err != nil {
log.Error("Error in find user passport on login " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"You have registered through Facebook."})
}
err = bcrypt.CompareHashAndPassword([]byte(passport.Password), []byte(password))
if err != nil {
return goweb.API.Respond(ctx, 200, nil, []string{"Incorrect password."})
}
log.Info("User found on local login")
if user.Token == "" || user.TokenUpdate.Before(time.Now().AddDate(0, 0, -1)) {
var token = nonce.NewToken()
nonce.MarkToken(token)
user.Token = token
user.TokenUpdate = time.Now()
err := m.GetDB("User").UpdateId(user.Id, user)
if err != nil {
log.Error("Failed to update token: " + err.Error())
}
}
ctx.HttpResponseWriter().Header().Add("cookie", "token="+user.Token)
return goweb.API.RespondWithData(ctx, user.Token)
}
}
func UploadTrail(ctx context.Context) error {
var currentUser, ok = ctx.Data()["user"].(m.User)
if ok {
// read the file from the request
file, _, err := ctx.HttpRequest().FormFile("file")
if err != nil {
log.Error("Error on gps upload FormFile " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to upload the file."})
}
defer file.Close()
outputPath := "../uploads/" + currentUser.Username + "/gps/" + base64.StdEncoding.EncodeToString([]byte(currentUser.Username+time.Now().String()))
if err := os.MkdirAll("../uploads/"+currentUser.Username+"/gps/", 0777); err != nil {
log.Error("Error in creating output dir, gps " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to upload the gps."})
}
outputFile, err := os.Create(outputPath)
if err != nil {
log.Error("Error in creating output file, gps " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to upload the gps."})
}
defer outputFile.Close()
if _, err = io.Copy(outputFile, file); err != nil {
log.Error("Error in copying file, gps " + err.Error())
os.Remove(outputPath)
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to upload the gps."})
}
var latIndex int = 4
var lonIndex int = 6
var TimeDate bool = true
//get rid of heading
scanner := bufio.NewScanner(file)
if scanner.Scan() {
line := scanner.Text()
lineRecords := strings.Split(line, ",")
if strings.EqualFold(lineRecords[2], "DATE") {
latIndex++
lonIndex++
TimeDate = false
}
}
//initialise vars
var lastLon string = ""
var lastLat string = ""
var track m.Track
var lastTrackID string = ""
var LastDateArray []string
var currDateArray []string
var date string
var date2 string
//first record
if scanner.Scan() {
record := scanner.Text()
recordSlice := strings.Split(record, ",")
if TimeDate {
LastDateArray = strings.Split(recordSlice[2], " ")
} else {
date = recordSlice[2] + " " + recordSlice[3]
LastDateArray = strings.Split(date, " ")
}
lastTrackID = strconv.FormatInt(utils.GetTrackID(LastDateArray, currDateArray), 10)
track = m.Track{bson.NewObjectId(), currentUser.Id, lastTrackID, utils.ParseDateFromStrings(LastDateArray[0], LastDateArray[1]), []m.Coordinate{}, time.Now(), time.Now(), time.Now()}
if strings.EqualFold(recordSlice[latIndex+1], "S") {
recordSlice[latIndex] = "-" + recordSlice[latIndex]
}
if strings.EqualFold(recordSlice[lonIndex+1], "W") {
recordSlice[lonIndex] = "-" + recordSlice[lonIndex]
}
coor := m.Coordinate{recordSlice[latIndex], recordSlice[lonIndex], utils.ParseDateFromStrings(LastDateArray[0], LastDateArray[1])}
track.Coordinates = append(track.Coordinates, coor)
lastLat = recordSlice[latIndex]
lastLon = recordSlice[lonIndex]
}
//read records
for scanner.Scan() { //if there is something left
line := scanner.Text() //read one line
s := strings.Split(line, ",") //split to slice
if TimeDate {
currDateArray = strings.Split(s[2], " ")
} else {
date2 = s[2] + " " + s[3]
currDateArray = strings.Split(date2, " ")
}
if (strings.EqualFold(lastLat, s[latIndex])) || (strings.EqualFold(lastLon, s[lonIndex])) {
continue //if same lat or lon as previous row: skip it
} else {
lastLat = s[latIndex] //update last record
lastLon = s[lonIndex]
trackID := strconv.FormatInt(utils.GetTrackID(LastDateArray, currDateArray), 10)
if strings.EqualFold(s[latIndex+1], "S") {
s[latIndex] = "-" + s[latIndex]
}
if strings.EqualFold(s[lonIndex+1], "W") {
s[lonIndex] = "-" + s[lonIndex]
}
if strings.EqualFold(lastTrackID, trackID) { //add new coordinate to current track
track.Coordinates = append(track.Coordinates, m.Coordinate{s[latIndex], s[lonIndex], utils.ParseDateFromStrings(currDateArray[0], currDateArray[1])})
} else {
//store track and start a new one!
min, max := utils.GetMinMaxDateFromCoordinates(track.Coordinates)
doc := m.Track{Id: bson.NewObjectId(), User: currentUser.Id, TrackID: track.TrackID, Date: track.Date, Coordinates: track.Coordinates, MinDate: min, MaxDate: max, Uploaded: time.Now()}
err = m.GetDB("Track").Insert(doc)
if err != nil {
log.Error("Can't insert track: " + err.Error())
} else {
utils.LinkGpsToImages(doc)
}
track = m.Track{bson.NewObjectId(), currentUser.Id, trackID, utils.ParseDateFromStrings(currDateArray[0], currDateArray[1]), []m.Coordinate{}, time.Now(), time.Now(), time.Now()}
track.Coordinates = append(track.Coordinates, m.Coordinate{s[latIndex], s[lonIndex], utils.ParseDateFromStrings(currDateArray[0], currDateArray[1])})
lastTrackID = trackID
}
LastDateArray = currDateArray
}
}
//store last track
min, max := utils.GetMinMaxDateFromCoordinates(track.Coordinates)
doc := m.Track{Id: bson.NewObjectId(), User: currentUser.Id, TrackID: track.TrackID, Date: track.Date, Coordinates: track.Coordinates, MinDate: min, MaxDate: max}
err = m.GetDB("Track").Insert(doc)
if err != nil {
log.Error("Can't insert track: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to upload the track."})
}
utils.LinkGpsToImages(doc)
return goweb.API.Respond(ctx, 200, nil, nil)
} else {
return goweb.API.Respond(ctx, 200, nil, []string{"Please log in to upload your GPS data."})
}
}
func ParseImageTable(currentUser m.User, file io.Reader) error {
outputPath := "../uploads/" + currentUser.Username + "/tables/" + base64.StdEncoding.EncodeToString([]byte(currentUser.Username+time.Now().String()))
if err := os.MkdirAll("../uploads/"+currentUser.Username+"/tables/", 0777); err != nil {
log.Error("Error in creating output dir " + err.Error())
return errors.New("Failed to upload the image.")
}
outputFile, err := os.Create(outputPath)
if err != nil {
log.Error("Error in creating output file, process image " + err.Error())
return errors.New("Failed to upload the image.")
}
defer outputFile.Close()
if _, err = io.Copy(outputFile, file); err != nil {
log.Error("Error in copying file " + err.Error())
os.Remove(outputPath)
return errors.New("Failed to upload the image.")
}
parser := csv.NewReader(file)
parser.Read() // dump the header
timeLayout := "2006-01-02T15:04:05-0700"
for true {
items, err := parser.Read()
if err != nil {
return errors.New("Failed to parse the image table")
break
}
imageDate, err := time.Parse(timeLayout, items[0])
if err != nil {
log.Error("parse date from image table " + err.Error())
continue
}
var image m.Image
if err = m.GetDB("Image").Find(bson.M{"user": currentUser.Id, "date": imageDate}).One(&image); err != nil {
if err.Error() != "not found" {
log.Error("find image from image table " + err.Error())
}
continue
} else {
image.CamId = items[1]
image.Size = items[2]
image.Type = items[3]
image.PropInfo = items[4]
image.Accx = items[5]
image.Accy = items[6]
image.Accz = items[7]
image.Magx = items[8]
image.Magy = items[9]
image.Magz = items[10]
image.Red = items[11]
image.Green = items[12]
image.Blue = items[13]
image.Lum = items[14]
image.Tem = items[15]
image.GpsStatus = items[16]
// 17, 18 lat and long
image.Alt = items[19]
image.GS = items[20]
image.Herr = items[21]
image.Verr = items[22]
image.Exp = items[23]
image.Gain = items[24]
image.RBal = items[25]
image.GBal = items[26]
image.BBal = items[27]
image.Xor = items[28]
image.Yor = items[29]
image.Zor = items[30]
image.Stags = items[31]
image.Tags = items[32]
// save the image
err = m.GetDB("Image").UpdateId(image.Id, image)
if err != nil {
log.Error("LinkImageToGps, update, " + err.Error())
}
}
}
return nil
}
func GetStats(ctx context.Context) error {
var err error
var pageType = ctx.FormValue("page")
query := bson.M{}
//barchartQuery := bson.M{}
var totalFileSize int64 = 0
var imageCount int = 0
if pageType == "pstats" {
var currentUser, ok = ctx.Data()["user"].(m.User)
if ok {
fmt.Println("USER OK")
query["user"] = currentUser.Id
//barchartQuery["user"] = currentUser.Id
var identifier = currentUser.Username
if identifier == "" {
identifier = currentUser.Email
}
var dirs []os.FileInfo
dirs, err = ioutil.ReadDir("../uploads/" + identifier)
if err != nil {
log.Error("Failed to read dir:" + err.Error())
}
fmt.Println("ID:" + identifier)
for i := 0; i < len(dirs); i++ {
totalFileSize += dirs[i].Size()
}
imageCount = len(dirs)
} else {
return goweb.API.Respond(ctx, 200, nil, []string{"Please log in to view your stats."})
}
} else {
var subdirInfo []os.FileInfo
var dirInfo []os.FileInfo
dirInfo, err = ioutil.ReadDir("../uploads/")
if err != nil {
log.Error("Failed to read dir:" + err.Error())
}
for i := 0; i < len(dirInfo); i++ {
if dirInfo[i].IsDir() {
subdirInfo, err = ioutil.ReadDir("../uploads/" + dirInfo[i].Name())
if err != nil {
log.Error("Failed to read dir:" + err.Error())
}
for j := 0; j < len(subdirInfo); j++ {
totalFileSize += subdirInfo[j].Size()
}
imageCount += len(subdirInfo)
}
}
}
// convert to MB
var fileSizeInMB = strconv.FormatFloat((float64(totalFileSize) / 1000000), 'f', 2, 32)
fmt.Println(fileSizeInMB)
// Get the info from the context
fromDate, fromDateExists := utils.ParseDateFromContext("from_", ctx)
toDate, toDateExists := utils.ParseDateFromContext("to_", ctx)
//Total number of users
userCount, err := m.GetDB("User").Count()
if err != nil {
log.Error("Failed to find user count: " + err.Error())
}
//Total number of tracks
trailCount, err := m.GetDB("Track").Find(query).Count()
if err != nil {
log.Error("Failed to find track count: " + err.Error())
}
// Query add date/time
if fromDateExists || toDateExists {
query["date"] = bson.M{"$gte": fromDate, "$lte": toDate}
}
//Filtered images and tracks by date/time and user
filteredImageCount, err := m.GetDB("Image").Find(query).Count()
if err != nil {
log.Error("Failed to find image count: " + err.Error())
}
filteredTrackCount, err := m.GetDB("Track").Find(query).Count()
if err != nil {
log.Error("Failed to find track count: " + err.Error())
}
fmt.Print("FILTERED IMAGE AND TRACK COUNT: ")
fmt.Print(filteredImageCount)
fmt.Print(" ")
fmt.Print(filteredTrackCount)
fmt.Println("")
fmt.Println("")
// Barchart number of images
var dDay, dMonth, dYear = getDifference(fromDate, toDate)
var imageCurrentCount int
var imageCountBarchartValues []int
var gpsCurrentCount int
var gpsCountBarchartValues []int
var currentLabel string
var barchartLabels []string
var nextDate, endNextDate time.Time
var February = leapYearDays(fromDate.Year())
var months = [13]int{0, 31, February, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
if dYear < 1 {
if dMonth < 1 { // days view
for i := 0; i <= dDay; i++ {
if i == 0 {
nextDate, endNextDate = getDates(fromDate, 0, 0)
}
query["date"] = bson.M{"$gte": nextDate, "$lte": endNextDate}
imageCurrentCount, err = m.GetDB("Image").Find(query).Count()
if err != nil {
log.Error("Failed to find image count: " + err.Error())
}
imageCountBarchartValues = append(imageCountBarchartValues, imageCurrentCount)
gpsCurrentCount, err = m.GetDB("Track").Find(query).Count()
if err != nil {
log.Error("Failed to find gps count: " + err.Error())
}
gpsCountBarchartValues = append(gpsCountBarchartValues, gpsCurrentCount)
currentLabel = strconv.Itoa(nextDate.Day()) + " " + nextDate.Month().String() + " " + strconv.Itoa(nextDate.Year())
barchartLabels = append(barchartLabels, currentLabel)
nextDate, endNextDate = getDates(nextDate.AddDate(0, 0, 1), 0, 0)
}
} else { // months view
for i := 0; i <= dMonth; i++ {
if i == 0 { //first third
nextDate, endNextDate = getDates(fromDate, 0, months[int(fromDate.Month())])
} else if i == dMonth { //last third
nextDate = nextDate.AddDate(0, 1, 1-nextDate.Day())
nextDate, endNextDate = getDates(nextDate, 0, toDate.Day())
} else { // everything else from the beginning to the end of month
nextDate = nextDate.AddDate(0, 1, 1-nextDate.Day())
mEnd := months[int(nextDate.Month())]
nextDate, endNextDate = getDates(nextDate, 0, mEnd)
}
query["date"] = bson.M{"$gte": nextDate, "$lte": endNextDate}
imageCurrentCount, err = m.GetDB("Image").Find(query).Count()
if err != nil {
log.Error("Failed to find image count: " + err.Error())
}
imageCountBarchartValues = append(imageCountBarchartValues, imageCurrentCount)
gpsCurrentCount, err = m.GetDB("Track").Find(query).Count()
if err != nil {
log.Error("Failed to find gps count: " + err.Error())
}
gpsCountBarchartValues = append(gpsCountBarchartValues, gpsCurrentCount)
currentLabel = nextDate.Month().String() + " " + strconv.Itoa(nextDate.Year())
barchartLabels = append(barchartLabels, currentLabel)
}
}
} else { // years view
for i := 0; i <= dYear; i++ {
if i == 0 { //first third
nextDate, endNextDate = getDates(fromDate, 12, months[int(fromDate.Month())]) //31?
} else if i == dMonth { //last third
nextDate = nextDate.AddDate(1, 1-int(nextDate.Month()), 1-nextDate.Day())
nextDate, endNextDate = getDates(nextDate, int(toDate.Month()), toDate.Day())
} else { // everything else from the beginning to the end of year
nextDate = nextDate.AddDate(1, 1-int(nextDate.Month()), 1-nextDate.Day())
nextDate, endNextDate = getDates(nextDate, 12, months[int(nextDate.Month())]) //31?
}
query["date"] = bson.M{"$gte": nextDate, "$lte": endNextDate}
imageCurrentCount, err = m.GetDB("Image").Find(query).Count()
if err != nil {
log.Error("Failed to find image count: " + err.Error())
}
imageCountBarchartValues = append(imageCountBarchartValues, imageCurrentCount)
gpsCurrentCount, err = m.GetDB("Track").Find(query).Count()
if err != nil {
log.Error("Failed to find gps count: " + err.Error())
}
gpsCountBarchartValues = append(gpsCountBarchartValues, gpsCurrentCount)
currentLabel = nextDate.Month().String() + " " + strconv.Itoa(nextDate.Year())
barchartLabels = append(barchartLabels, currentLabel)
}
}
var stats = []int{userCount, imageCount, trailCount, filteredImageCount, filteredTrackCount}
var result []interface{} = []interface{}{imageCountBarchartValues, barchartLabels, gpsCountBarchartValues, stats, fileSizeInMB}
return goweb.API.RespondWithData(ctx, result)
}
func Callback(ctx context.Context) error {
provider_type := ctx.PathValue("provider")
provider, err := gomniauth.Provider(provider_type)
if err != nil {
log.Error("Error on getting provider: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"An error has occured."})
}
creds, err := provider.CompleteAuth(ctx.QueryParams())
if err != nil {
log.Error("Error on completing auth: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"An error has occured."})
}
// load the user
// https://github.com/stretchr/gomniauth/blob/master/common/user.go
user, userErr := provider.GetUser(creds)
if userErr != nil {
log.Error("Error on getting user: "******"An error has occured."})
}
passport := m.Passport{}
err = m.GetDB("Passport").
Find(bson.M{"provider": provider_type,
"identifier": user.IDForProvider(provider_type)}).
One(&passport)
if err != nil {
if err.Error() == "not found" {
var currentUser, ok = ctx.Data()["user"].(m.User)
if ok {
err = m.GetDB("Passport").
Insert(&m.Passport{bson.NewObjectId(), currentUser.Id, "", provider_type,
user.IDForProvider(provider_type), fmt.Sprintf("%v", creds.Map.Get("access_token").Data()),
fmt.Sprintf("%v", creds.Map.Get("refresh_token").Data())})
if err != nil {
log.Error("Error on registration with provider " + provider_type + ", new passport: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Could not create your new authorization."})
}
log.Info("Connecting user")
url, _ := url.Parse(utils.EnvUrl() + "/#/fblogin/?token=" + currentUser.Token)
return goweb.Respond.WithRedirect(ctx, url)
} else {
// No user, create user, create passport
var token = nonce.NewToken()
nonce.MarkToken(token)
newUser := m.User{bson.NewObjectId(), user.Nickname(), user.Email(), true, token, time.Now()}
err = m.GetDB("User").Insert(&newUser)
if err != nil {
log.Error("Error on registration with provider " + provider_type + ", new user: "******"Failed to register."})
}
err = m.GetDB("Passport").
Insert(&m.Passport{bson.NewObjectId(), newUser.Id, "", provider_type,
user.IDForProvider(provider_type), fmt.Sprintf("%v", creds.Map.Get("access_token").Data()),
fmt.Sprintf("%v", creds.Map.Get("refresh_token").Data())})
if err != nil {
log.Error("Error on registration with provider " + provider_type + ", new user passport: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to create your new passport."})
}
log.Info("New user registered")
url, _ := url.Parse(utils.EnvUrl() + "/#/fblogin/?token=" + newUser.Token)
return goweb.Respond.WithRedirect(ctx, url)
}
} else {
log.Error("Error on registration with provider " + provider_type + ", new passport: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Could not find your authorization."})
}
} else {
// login the user
var user = m.User{}
fmt.Println(passport)
err = m.GetDB("User").Find(bson.M{"_id": passport.User}).One(&user)
if err != nil {
log.Error("Error on login with provider " + provider_type + ", user query: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Could not find you on the database."})
}
log.Info("Found user returning id")
url, _ := url.Parse(utils.EnvUrl() + "/#/fblogin/?token=" + user.Token)
return goweb.Respond.WithRedirect(ctx, url)
}
}
func GetImages(ctx context.Context) error {
var result []m.Image
// Get the info from the context
fromDate, fromDateExists := utils.ParseDateFromContext("from_", ctx)
toDate, toDateExists := utils.ParseDateFromContext("to_", ctx)
searchType := ctx.FormValue("upload")
if searchType != "" {
searchType = "uploaded"
} else {
searchType = "date"
}
limit, err := strconv.Atoi(ctx.FormValue("limit"))
if err != nil {
limit = 20
}
skip, err := strconv.Atoi(ctx.FormValue("skip"))
if err != nil {
skip = 0
}
var currentUser, ok = ctx.Data()["user"].(m.User)
isPersonal, err := strconv.ParseBool(ctx.FormValue("personal"))
if err != nil {
isPersonal = false
}
query := bson.M{}
if ok && isPersonal {
if fromDateExists || toDateExists {
query[searchType] = bson.M{"$gte": fromDate, "$lte": toDate}
query["user"] = currentUser.Id
} else {
query["user"] = currentUser.Id
}
} else {
isPersonal = false
if fromDateExists || toDateExists {
query[searchType] = bson.M{"$gte": fromDate, "$lte": toDate}
}
}
addLocation := true
var lat, long, radius float64
rad, err := strconv.Atoi(ctx.FormValue("radius"))
if err != nil {
addLocation = false
}
if addLocation {
radius = float64(rad)
lat, err = strconv.ParseFloat(ctx.FormValue("lat"), 64)
if err != nil {
addLocation = false
}
long, err = strconv.ParseFloat(ctx.FormValue("log"), 64)
if err != nil {
addLocation = false
}
}
if addLocation {
query["lat"] = bson.M{"$gt": lat - radius, "$lt": lat + radius}
query["log"] = bson.M{"$gt": long - radius, "$lt": long + radius}
}
showAll := ctx.FormValue("allImages")
if showAll == "" {
query["show"] = true
}
keyMoments := ctx.FormValue("keyMoments")
if keyMoments != "" {
query["processed"] = true
}
if err = m.GetDB("Image").Find(query).Sort("-" + searchType).Skip(skip).Limit(limit).All(&result); err != nil {
log.Error("Failed to search for images: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to find the images."})
}
return goweb.API.RespondWithData(ctx, result)
}
func main() {
log.Info("Glasgow Memories Server")
log.Info("=======================")
utils.InitEnv()
var Address = ":" + utils.EnvPort()
var baseURL = utils.EnvUrl()
m.Connect()
defer m.Close()
// prepare the decryption key
if utils.LoadCypherKey() != nil {
log.Error("Failed to load the decryption key.")
return
}
// GOMNIAUTH
gomniauth.SetSecurityKey(signature.RandomKey(64))
gomniauth.WithProviders(
facebook.New("1497244403859030", "fbbb08c47e0441bcf23ea82b5f340fe5",
baseURL+"/api/auth/facebook/callback/"),
)
// Attach the DB collection references to the context in order to pass it around
goweb.MapBefore(func(ctx context.Context) error {
var user = m.User{}
cookieC, err := ctx.HttpRequest().Cookie("token")
var cookie string
if err != nil {
cookie = ctx.FormValue("token")
if cookie == "" {
return nil
}
} else {
cookie = cookieC.Value
}
err = m.GetDB("User").Find(bson.M{"token": cookie}).One(&user)
if err != nil {
// log.Info("MapBefore 2 " + err.Error())
return nil
}
ctx.Data()["user"] = user
return nil
})
goweb.MapStatic("/static", "../static") // This is the directory with all static UI files
goweb.MapStatic("/uploads", "../uploads") // This is the directory where we should store uploaded files
// ENDPOINTS
goweb.Map("GET", "/", endpoints.Root)
goweb.Map("POST", "api/auth/local/register", endpoints.Register)
goweb.Map("POST", "api/auth/local/login", endpoints.Login)
goweb.Map("GET", "api/auth/{provider}/callback", endpoints.Callback)
goweb.Map([]string{"GET", "POST"}, "api/auth/{provider}/{action}", endpoints.Connect)
goweb.Map("POST", "api/upload/image", endpoints.UploadImage)
goweb.Map("GET", "api/images/get", endpoints.GetImages)
goweb.Map("POST", "api/upload/csv", endpoints.UploadTrail)
goweb.Map("GET", "api/trails/get", endpoints.GetTrails)
goweb.Map("POST", "api/upload/video", endpoints.UploadVideo)
goweb.Map("GET", "api/videos/get", endpoints.GetVideos)
goweb.Map("GET", "api/user", endpoints.GetUserInfo)
goweb.Map("GET", "api/stats/get", endpoints.GetStats)
goweb.Map("GET", "api/popLocations", endpoints.GetPopularLocations)
goweb.Map("POST", "api/upload/imagetable", endpoints.UploadImageTable)
goweb.Map("POST", "api/upload/zip", endpoints.UploadZip)
// TODO: Add new endpoints here
goweb.Map(endpoints.NotFound)
// Remove the information from the data just in case the call is intercepted
goweb.MapAfter(func(ctx context.Context) error {
ctx.Data()["user"] = ""
return nil
})
// setup the API responder
codecService := services.NewWebCodecService()
codecService.RemoveCodec("text/xml")
apiResponder := responders.NewGowebAPIResponder(codecService, goweb.Respond)
apiResponder.StandardFieldDataKey = "data"
apiResponder.StandardFieldStatusKey = "status"
apiResponder.StandardFieldErrorsKey = "errors"
goweb.API = apiResponder
// SERVER
s := &http.Server{
Addr: Address,
Handler: goweb.DefaultHttpHandler(),
ReadTimeout: 5 * time.Minute,
WriteTimeout: 5 * time.Minute,
MaxHeaderBytes: 1 << 20,
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
listener, listenErr := net.Listen("tcp", Address)
log.Info("Server port: " + Address)
log.Info("Server running at: " + baseURL + "\n")
if listenErr != nil {
log.Error("Could not listen: " + listenErr.Error())
}
go func() {
for _ = range c {
// sig is a ^C, handle it
// stop the HTTP server
log.Info("Stopping the server...\n")
listener.Close()
log.Info("Server stopped.\n")
}
}()
// begin the server
log.Error("Error in Serve: " + s.Serve(listener).Error())
}
