func ProcessZip(currentUser m.User, outputZipPath string) {
var imagesFound = 0
var tablesFound = 0
var WalkImageCallback = func(path string, fi os.FileInfo, err error) error {
if err != nil {
return nil
}
if fi.IsDir() {
return nil
}
if !strings.Contains(fi.Name(), ".txt") {
imagesFound++
log.Info(strconv.Itoa(imagesFound))
file, err := os.Open(path)
if err != nil {
// log.Error("Failed to read image file from zip: " + err.Error())
return nil
}
ProcessImage(currentUser, file, nil, true)
}
return nil
}
var WalkITableCallback = func(path string, fi os.FileInfo, err error) error {
if err != nil {
return nil
}
if fi.IsDir() {
return nil
}
if strings.Contains(fi.Name(), ".txt") {
tablesFound++
// log.Info(strconv.Itoa(tablesFound))
file, err := os.Open(path)
if err != nil {
log.Error("Failed to read txt file from zip: " + err.Error())
return nil
}
defer file.Close()
ParseImageTable(currentUser, file)
}
return nil
}
log.Info("Received zip file for processing")
filepath.Walk(outputZipPath, WalkImageCallback)
log.Info("Images found in zip: " + strconv.Itoa(imagesFound))
log.Info("Starting processing of image tables.")
filepath.Walk(outputZipPath, WalkITableCallback)
log.Info("Tables found in zip: " + strconv.Itoa(tablesFound))
os.Remove(outputZipPath)
}
func main() {
log.Info("Glasgow Memories Server Processing Module")
m.Connect()
defer m.Close()
threads = runtime.NumCPU() - 2
if threads < 2 {
threads = 2
}
log.Info("Starting processing with " + strconv.Itoa(threads) + " threads.")
// schedule the execution
RunProcessing()
}
func Connect() {
// Database
// mongolab.com
var dbServer = "mongolab"
_db, err = mgo.Dial("mongodb://127.0.0.1:27017/gms")
if err != nil {
log.Error("Error on database connection: " + err.Error())
_db, err = mgo.Dial("mongodb://127.0.0.1:27017/gms")
dbServer = "local db"
if err != nil {
panic(err)
}
}
// This makes the db monotonic
_db.SetMode(mgo.Monotonic, true)
// Database name (test) and the collection names ("User") are set
collections := _db.DB(db_name)
db_map["User"] = collections.C("User")
db_map["Passport"] = collections.C("Passport")
db_map["Comment"] = collections.C("Comment")
db_map["Track"] = collections.C("Track")
db_map["Image"] = collections.C("Image")
db_map["Video"] = collections.C("Video")
db_map["Cluster"] = collections.C("Cluster")
db_map["RecommendUser"] = collections.C("RecommendUser")
db_map["TrendingAllUsers"] = collections.C("TrendingAllUsers")
db_map["RecommendAllUsers"] = collections.C("RecommendAllUsers")
log.Info("Database connection established with " + dbServer)
}
// This should be the work dispatching thread, only dispatching work between 0 and 7
func RunProcessing() {
// create a channel for the pool
imageChannel := make(chan m.Image, 50)
// create a work done channel
done := make(chan int, threads)
counter := 0
for true {
var notProcessedImages []m.Image
if counter < 1 {
counter = 4
count, err := m.GetDB("Image").Find(bson.M{"processed": false}).Count()
if err != nil {
log.Error("Could not count the unprocessed images")
} else {
log.Info("Total unprocessed images left: " + strconv.Itoa(count))
}
} else {
counter--
}
m.GetDB("Image").Find(bson.M{"processed": false}).Limit(50).All(¬ProcessedImages)
if len(notProcessedImages) == 0 {
log.Info("No images found, sleeping for 1 hour")
counter = 0
time.Sleep(1 * time.Hour)
continue
}
for _, image := range notProcessedImages {
imageChannel <- image
}
for i := threads; i > 0; i-- {
go ProcessImage(imageChannel, done)
}
for i := threads; i > 0; i-- {
<-done
}
}
}
func Callback(ctx context.Context) error {
provider_type := ctx.PathValue("provider")
provider, err := gomniauth.Provider(provider_type)
if err != nil {
log.Error("Error on getting provider: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"An error has occured."})
}
creds, err := provider.CompleteAuth(ctx.QueryParams())
if err != nil {
log.Error("Error on completing auth: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"An error has occured."})
}
// load the user
// https://github.com/stretchr/gomniauth/blob/master/common/user.go
user, userErr := provider.GetUser(creds)
if userErr != nil {
log.Error("Error on getting user: "******"An error has occured."})
}
passport := m.Passport{}
err = m.GetDB("Passport").
Find(bson.M{"provider": provider_type,
"identifier": user.IDForProvider(provider_type)}).
One(&passport)
if err != nil {
if err.Error() == "not found" {
var currentUser, ok = ctx.Data()["user"].(m.User)
if ok {
err = m.GetDB("Passport").
Insert(&m.Passport{bson.NewObjectId(), currentUser.Id, "", provider_type,
user.IDForProvider(provider_type), fmt.Sprintf("%v", creds.Map.Get("access_token").Data()),
fmt.Sprintf("%v", creds.Map.Get("refresh_token").Data())})
if err != nil {
log.Error("Error on registration with provider " + provider_type + ", new passport: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Could not create your new authorization."})
}
log.Info("Connecting user")
url, _ := url.Parse(utils.EnvUrl() + "/#/fblogin/?token=" + currentUser.Token)
return goweb.Respond.WithRedirect(ctx, url)
} else {
// No user, create user, create passport
var token = nonce.NewToken()
nonce.MarkToken(token)
newUser := m.User{bson.NewObjectId(), user.Nickname(), user.Email(), true, token, time.Now()}
err = m.GetDB("User").Insert(&newUser)
if err != nil {
log.Error("Error on registration with provider " + provider_type + ", new user: "******"Failed to register."})
}
err = m.GetDB("Passport").
Insert(&m.Passport{bson.NewObjectId(), newUser.Id, "", provider_type,
user.IDForProvider(provider_type), fmt.Sprintf("%v", creds.Map.Get("access_token").Data()),
fmt.Sprintf("%v", creds.Map.Get("refresh_token").Data())})
if err != nil {
log.Error("Error on registration with provider " + provider_type + ", new user passport: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to create your new passport."})
}
log.Info("New user registered")
url, _ := url.Parse(utils.EnvUrl() + "/#/fblogin/?token=" + newUser.Token)
return goweb.Respond.WithRedirect(ctx, url)
}
} else {
log.Error("Error on registration with provider " + provider_type + ", new passport: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Could not find your authorization."})
}
} else {
// login the user
var user = m.User{}
fmt.Println(passport)
err = m.GetDB("User").Find(bson.M{"_id": passport.User}).One(&user)
if err != nil {
log.Error("Error on login with provider " + provider_type + ", user query: " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"Could not find you on the database."})
}
log.Info("Found user returning id")
url, _ := url.Parse(utils.EnvUrl() + "/#/fblogin/?token=" + user.Token)
return goweb.Respond.WithRedirect(ctx, url)
}
}
func Login(ctx context.Context) error {
var currentUser, ok = ctx.Data()["user"].(m.User)
if ok {
return goweb.API.RespondWithData(ctx, currentUser.Token)
} else {
username := ctx.FormValue("username")
password := ctx.FormValue("password")
// Check or basic auth header
authHeader := ctx.HttpRequest().Header.Get("Authorization")
if len(authHeader) > 0 {
info, err := base64.StdEncoding.DecodeString(strings.Split(authHeader, " ")[1])
if err == nil {
decryptedInfo := strings.Split(string(info), ":")
username = decryptedInfo[0]
password = decryptedInfo[1]
}
}
if len(username) < 3 || len(password) < 6 {
return goweb.API.Respond(ctx, 200, nil, []string{"Invalid data supplied, please fill all fields."})
}
user := m.User{}
query := bson.M{}
if strings.Contains(username, "@") {
query["username"] = username
} else {
query["email"] = username
}
err := m.GetDB("User").Find(query).One(&user)
if err != nil {
log.Error("Error in find user on login " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"No such username, please register."})
}
passport := m.Passport{}
err = m.GetDB("Passport").Find(bson.M{"provider": "local", "user": user.Id}).One(&passport)
if err != nil {
log.Error("Error in find user passport on login " + err.Error())
return goweb.API.Respond(ctx, 200, nil, []string{"You have registered through Facebook."})
}
err = bcrypt.CompareHashAndPassword([]byte(passport.Password), []byte(password))
if err != nil {
return goweb.API.Respond(ctx, 200, nil, []string{"Incorrect password."})
}
log.Info("User found on local login")
if user.Token == "" || user.TokenUpdate.Before(time.Now().AddDate(0, 0, -1)) {
var token = nonce.NewToken()
nonce.MarkToken(token)
user.Token = token
user.TokenUpdate = time.Now()
err := m.GetDB("User").UpdateId(user.Id, user)
if err != nil {
log.Error("Failed to update token: " + err.Error())
}
}
ctx.HttpResponseWriter().Header().Add("cookie", "token="+user.Token)
return goweb.API.RespondWithData(ctx, user.Token)
}
}
func Register(ctx context.Context) error {
var currentUser, ok = ctx.Data()["user"].(m.User)
if ok {
return goweb.API.RespondWithData(ctx, currentUser.Token)
} else {
username := ctx.FormValue("username")
email := ctx.FormValue("email")
password := ctx.FormValue("password")
// Check or basic auth header
authHeader := ctx.HttpRequest().Header.Get("Authorization")
if len(authHeader) > 0 {
info, err := base64.StdEncoding.DecodeString(strings.Split(authHeader, " ")[1])
if err == nil {
decryptedInfo := strings.Split(string(info), ":")
username = decryptedInfo[0]
password = decryptedInfo[1]
email = decryptedInfo[2]
}
}
if len(username) < 3 {
return goweb.API.Respond(ctx, 200, nil, []string{"The username has to be at least 4 characters long."})
}
if len(email) == 0 {
return goweb.API.Respond(ctx, 200, nil, []string{"Please supply an email address."})
}
err := utils.CheckValidPassword(password)
if err != nil {
return goweb.API.Respond(ctx, 200, nil, []string{err.Error()})
}
var oldUser m.User
err = m.GetDB("User").Find(bson.M{"username": username}).One(&oldUser)
if err == nil {
log.Debug("Username already taken.")
return goweb.API.Respond(ctx, 200, nil, []string{"Username already taken."})
}
var token = nonce.NewToken()
nonce.MarkToken(token)
newUser := m.User{bson.NewObjectId(), username, email, true, token, time.Now()}
err = m.GetDB("User").Insert(&newUser)
if err != nil {
log.Error("Error on local registration, new user: "******"Failed to register."})
}
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), 10)
err = m.GetDB("Passport").Insert(&m.Passport{bson.NewObjectId(), newUser.Id, string(hashedPassword), "local", "", "", ""})
if err != nil {
log.Error("Error on local registration, new user passport: " + err.Error())
m.GetDB("User").RemoveId(newUser.Id)
return goweb.API.Respond(ctx, 200, nil, []string{"Failed to create your new passport."})
}
log.Info("New user registered local")
return goweb.API.RespondWithData(ctx, newUser.Token)
}
}
func main() {
log.Info("Glasgow Memories Server")
log.Info("=======================")
utils.InitEnv()
var Address = ":" + utils.EnvPort()
var baseURL = utils.EnvUrl()
m.Connect()
defer m.Close()
// prepare the decryption key
if utils.LoadCypherKey() != nil {
log.Error("Failed to load the decryption key.")
return
}
// GOMNIAUTH
gomniauth.SetSecurityKey(signature.RandomKey(64))
gomniauth.WithProviders(
facebook.New("1497244403859030", "fbbb08c47e0441bcf23ea82b5f340fe5",
baseURL+"/api/auth/facebook/callback/"),
)
// Attach the DB collection references to the context in order to pass it around
goweb.MapBefore(func(ctx context.Context) error {
var user = m.User{}
cookieC, err := ctx.HttpRequest().Cookie("token")
var cookie string
if err != nil {
cookie = ctx.FormValue("token")
if cookie == "" {
return nil
}
} else {
cookie = cookieC.Value
}
err = m.GetDB("User").Find(bson.M{"token": cookie}).One(&user)
if err != nil {
// log.Info("MapBefore 2 " + err.Error())
return nil
}
ctx.Data()["user"] = user
return nil
})
goweb.MapStatic("/static", "../static") // This is the directory with all static UI files
goweb.MapStatic("/uploads", "../uploads") // This is the directory where we should store uploaded files
// ENDPOINTS
goweb.Map("GET", "/", endpoints.Root)
goweb.Map("POST", "api/auth/local/register", endpoints.Register)
goweb.Map("POST", "api/auth/local/login", endpoints.Login)
goweb.Map("GET", "api/auth/{provider}/callback", endpoints.Callback)
goweb.Map([]string{"GET", "POST"}, "api/auth/{provider}/{action}", endpoints.Connect)
goweb.Map("POST", "api/upload/image", endpoints.UploadImage)
goweb.Map("GET", "api/images/get", endpoints.GetImages)
goweb.Map("POST", "api/upload/csv", endpoints.UploadTrail)
goweb.Map("GET", "api/trails/get", endpoints.GetTrails)
goweb.Map("POST", "api/upload/video", endpoints.UploadVideo)
goweb.Map("GET", "api/videos/get", endpoints.GetVideos)
goweb.Map("GET", "api/user", endpoints.GetUserInfo)
goweb.Map("GET", "api/stats/get", endpoints.GetStats)
goweb.Map("GET", "api/popLocations", endpoints.GetPopularLocations)
goweb.Map("POST", "api/upload/imagetable", endpoints.UploadImageTable)
goweb.Map("POST", "api/upload/zip", endpoints.UploadZip)
// TODO: Add new endpoints here
goweb.Map(endpoints.NotFound)
// Remove the information from the data just in case the call is intercepted
goweb.MapAfter(func(ctx context.Context) error {
ctx.Data()["user"] = ""
return nil
})
// setup the API responder
codecService := services.NewWebCodecService()
codecService.RemoveCodec("text/xml")
apiResponder := responders.NewGowebAPIResponder(codecService, goweb.Respond)
apiResponder.StandardFieldDataKey = "data"
apiResponder.StandardFieldStatusKey = "status"
apiResponder.StandardFieldErrorsKey = "errors"
goweb.API = apiResponder
// SERVER
s := &http.Server{
Addr: Address,
Handler: goweb.DefaultHttpHandler(),
ReadTimeout: 5 * time.Minute,
WriteTimeout: 5 * time.Minute,
MaxHeaderBytes: 1 << 20,
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
listener, listenErr := net.Listen("tcp", Address)
log.Info("Server port: " + Address)
log.Info("Server running at: " + baseURL + "\n")
if listenErr != nil {
log.Error("Could not listen: " + listenErr.Error())
}
go func() {
for _ = range c {
// sig is a ^C, handle it
// stop the HTTP server
log.Info("Stopping the server...\n")
listener.Close()
log.Info("Server stopped.\n")
}
}()
// begin the server
log.Error("Error in Serve: " + s.Serve(listener).Error())
}