// Update updates the form gallery in the MongoDB database
// collection.
func Update(context interface{}, db *db.DB, id string, gallery *Gallery) error {
log.Dev(context, "Update", "Started : Gallery[%s]", id)
if !bson.IsObjectIdHex(id) {
log.Error(context, "Update", ErrInvalidID, "Completed")
return ErrInvalidID
}
if err := gallery.Validate(); err != nil {
log.Error(context, "Update", err, "Completed")
return err
}
objectID := bson.ObjectIdHex(id)
gallery.DateUpdated = time.Now()
f := func(c *mgo.Collection) error {
log.Dev(context, "Update", "MGO : db.%s.update(%s, %s)", c.Name, mongo.Query(objectID), mongo.Query(gallery))
return c.UpdateId(objectID, gallery)
}
if err := db.ExecuteMGO(context, Collection, f); err != nil {
log.Error(context, "Update", err, "Completed")
return err
}
log.Dev(context, "Update", "Completed")
return nil
}
// List retrives the form galleries for a given form from the MongoDB database
// collection.
func List(context interface{}, db *db.DB, formID string) ([]Gallery, error) {
log.Dev(context, "List", "Started")
if !bson.IsObjectIdHex(formID) {
log.Error(context, "List", ErrInvalidID, "Completed")
return nil, ErrInvalidID
}
formObjectID := bson.ObjectIdHex(formID)
var galleries = make([]Gallery, 0)
f := func(c *mgo.Collection) error {
q := bson.M{
"form_id": formObjectID,
}
log.Dev(context, "List", "MGO : db.%s.find(%s)", c.Name, mongo.Query(q))
return c.Find(q).All(&galleries)
}
if err := db.ExecuteMGO(context, Collection, f); err != nil {
log.Error(context, "List", err, "Completed")
return nil, err
}
if err := hydrateMany(context, db, galleries); err != nil {
log.Error(context, "List", err, "Completed")
return nil, err
}
log.Dev(context, "List", "Completed")
return galleries, nil
}
// GetAll retrieves a list of regexs.
func GetAll(context interface{}, db *db.DB, tags []string) ([]Regex, error) {
log.Dev(context, "GetAll", "Started : Tags[%v]", tags)
key := "grs" + strings.Join(tags, "-")
if v, found := cache.Get(key); found {
rgxs := v.([]Regex)
log.Dev(context, "GetAll", "Completed : CACHE : Rgxs[%d]", len(rgxs))
return rgxs, nil
}
var rgxs []Regex
f := func(c *mgo.Collection) error {
log.Dev(context, "GetAll", "MGO : db.%s.find({}).sort([\"name\"])", c.Name)
return c.Find(nil).All(&rgxs)
}
if err := db.ExecuteMGO(context, Collection, f); err != nil {
if err == mgo.ErrNotFound {
err = ErrNotFound
}
log.Error(context, "GetAll", err, "Completed")
return nil, err
}
if rgxs == nil {
log.Error(context, "GetAll", ErrNotFound, "Completed")
return nil, ErrNotFound
}
cache.Set(key, rgxs, gc.DefaultExpiration)
log.Dev(context, "GetAll", "Completed : Rgxs[%d]", len(rgxs))
return rgxs, nil
}
// Remove removes an item into the items collection and remove any
// corresponding quads from the graph database.
func Remove(context interface{}, db *db.DB, graph *cayley.Handle, itemID string) error {
log.Dev(context, "Remove", "Started : ID[%s]", itemID)
// Get the item from the items collection.
items, err := item.GetByIDs(context, db, []string{itemID})
if err != nil {
log.Error(context, "Remove", err, "Completed")
return err
}
// Prepare the item map data.
itmMap := map[string]interface{}{
"item_id": items[0].ID,
"type": items[0].Type,
"version": items[0].Version,
"data": items[0].Data,
}
// Remove the corresponding relationships from the graph.
if err := wire.RemoveFromGraph(context, db, graph, itmMap); err != nil {
log.Error(context, "Remove", err, "Completed")
return err
}
// Delete the item.
if err := item.Delete(context, db, itemID); err != nil {
log.Error(context, "Remove", err, "Completed")
return err
}
log.Dev(context, "Remove", "Completed")
return nil
}
// Auth handles token authentication.
func Auth(h app.Handler) app.Handler {
// Check if authentication is turned off.
on, err := cfg.Bool(cfgAuth)
if err == nil && !on {
return func(c *app.Context) error {
log.Dev(c.SessionID, "Auth", "******> Authentication Off")
return h(c)
}
}
// Turn authentication on.
return func(c *app.Context) error {
token := c.Request.Header.Get("Authorization")
log.Dev(c.SessionID, "Auth", "Started : Token[%s]", token)
if len(token) < 5 || token[0:5] != "Basic" {
log.Error(c.SessionID, "Auth", app.ErrNotAuthorized, "Validating token")
return app.ErrNotAuthorized
}
var err error
if c.User, err = auth.ValidateWebToken(c.SessionID, c.DB, token[6:]); err != nil {
log.Error(c.SessionID, "Auth", err, "Validating token")
return app.ErrNotAuthorized
}
log.Dev(c.SessionID, "Auth", "Completed")
return h(c)
}
}
// Create adds a new Submission based on a given Form into
// the MongoDB database collection.
func Create(context interface{}, db *db.DB, formID string, submission *Submission) error {
log.Dev(context, "Create", "Started : Form[%s]", formID)
if !bson.IsObjectIdHex(formID) {
log.Error(context, "Create", ErrInvalidID, "Completed")
return ErrInvalidID
}
if err := submission.Validate(); err != nil {
return err
}
// FIXME: handle Number field maybe with https://docs.mongodb.com/v3.0/tutorial/create-an-auto-incrementing-field/ to resolve race condition
count, err := Count(context, db, formID)
if err != nil {
log.Error(context, "Create", err, "Completed")
return err
}
submission.Number = count + 1
f := func(c *mgo.Collection) error {
log.Dev(context, "Create", "MGO : db.%s.insert(%s)", c.Name, mongo.Query(submission))
return c.Insert(submission)
}
if err := db.ExecuteMGO(context, Collection, f); err != nil {
log.Error(context, "Create", err, "Completed")
return err
}
log.Dev(context, "Create", "Completed")
return nil
}
// CreateWebToken return a token and session that can be used to authenticate a user.
func CreateWebToken(context interface{}, db *db.DB, u *User, expires time.Duration) (string, error) {
log.Dev(context, "CreateWebToken", "Started : PublicID[%s]", u.PublicID)
// Do we have a valid session right now?
s, err := session.GetByLatest(context, db, u.PublicID)
if err != nil && err != mgo.ErrNotFound {
log.Error(context, "CreateUser", err, "Completed")
return "", err
}
// If we don't have one or it has been expired create
// a new one.
if err == mgo.ErrNotFound || s.IsExpired(context) {
if s, err = session.Create(context, db, u.PublicID, expires); err != nil {
log.Error(context, "CreateUser", err, "Completed")
return "", err
}
}
// Set the return arguments though we will explicitly
// return them. Don't want any confusion.
token, err := u.WebToken(s.SessionID)
if err != nil {
log.Error(context, "CreateUser", err, "Completed")
return "", err
}
log.Dev(context, "CreateWebToken", "Completed : WebToken[%s]", token)
return token, nil
}
// GetUserWebToken return a token if one exists and is valid.
func GetUserWebToken(context interface{}, db *db.DB, publicID string) (string, error) {
log.Dev(context, "GetUserWebToken", "Started : PublicID[%s]", publicID)
// Do we have a valid session right now?
s, err := session.GetByLatest(context, db, publicID)
if err != nil {
log.Error(context, "GetUserWebToken", err, "Completed")
return "", err
}
// If it is expired return failure.
if s.IsExpired(context) {
err := errors.New("Session expired.")
log.Error(context, "GetUserWebToken", err, "Completed")
return "", err
}
// Pull the user information.
u, err := GetUserByPublicID(context, db, publicID, true)
if err != nil {
log.Error(context, "GetUserWebToken", err, "Completed")
return "", err
}
// Generate a token that works right now.
token, err := u.WebToken(s.SessionID)
if err != nil {
log.Error(context, "GetUserWebToken", err, "Completed")
return "", err
}
log.Dev(context, "GetUserWebToken", "Completed : WebToken[%s]", token)
return token, nil
}
// UpdateUserPassword updates an existing user's password and token in the database.
func UpdateUserPassword(context interface{}, db *db.DB, u *User, password string) error {
log.Dev(context, "UpdateUserPassword", "Started : PublicID[%s]", u.PublicID)
if err := u.Validate(); err != nil {
log.Error(context, "UpdateUserPassword", err, "Completed")
return err
}
if len(password) < 8 {
err := errors.New("Invalid password length")
log.Error(context, "UpdateUserPassword", err, "Completed")
return err
}
newPassHash, err := crypto.BcryptPassword(u.PrivateID + password)
if err != nil {
log.Error(context, "UpdateUserPassword", err, "Completed")
return err
}
f := func(c *mgo.Collection) error {
q := bson.M{"public_id": u.PublicID}
upd := bson.M{"$set": bson.M{"password": newPassHash, "modified_at": time.Now().UTC()}}
log.Dev(context, "UpdateUserPassword", "MGO : db.%s.Update(%s, CAN'T SHOW)", c.Name, mongo.Query(q))
return c.Update(q, upd)
}
if err := db.ExecuteMGO(context, Collection, f); err != nil {
log.Error(context, "UpdateUserPassword", err, "Completed")
return err
}
log.Dev(context, "UpdateUserPassword", "Completed")
return nil
}
// TransformRow transform a row of data into the coral schema
func TransformRow(row map[string]interface{}, strategyTableName string) (interface{}, []map[string]interface{}, error) { // id row, transformation, error
var newRows []map[string]interface{}
var err error
table := strategy.GetEntities()[strategyTableName]
idField := GetID(strategyTableName)
id := row[idField]
if table.Local == "" {
return "", nil, fmt.Errorf("No table %v found in the strategy file.", table)
}
// if has an array field type array
if strategy.HasArrayField(table) {
newRows, err = transformRowWithArrayField(strategyTableName, row, table.Fields)
if err != nil {
log.Error(uuid, "fiddler.transformRow", err, "Transform the row into several coral documents.")
}
} else {
newRow, err := transformRow(strategyTableName, row, table.Fields)
if err != nil {
log.Error(uuid, "fiddler.transformRow", err, "Transform the row into coral.")
return id, nil, err
}
newRows = append(newRows, newRow)
}
return id, newRows, err
}
// saveResult processes the $save command for this result.
func saveResult(context interface{}, save map[string]interface{}, results []bson.M, data map[string]interface{}) error {
// {"$map": "list"}
// Capture the key and value and process the save.
for cmd, value := range save {
name, ok := value.(string)
if !ok {
err := fmt.Errorf("Save key \"%v\" is a %T but must be a string", value, value)
log.Error(context, "saveResult", err, "Extracting save key")
return err
}
switch cmd {
// Save the results into the map under the specified key.
case "$map":
log.Dev(context, "saveResult", "Saving result to map[%s]", name)
data[name] = results
return nil
default:
err := fmt.Errorf("Invalid save location %q", cmd)
log.Error(context, "saveResult", err, "Nothing saved")
return err
}
}
err := errors.New("Missing save document")
log.Error(context, "saveResult", err, "Nothing saved")
return err
}
func main() {
const context = "main"
// Create a task value for execution.
t := Task{
Name: "test task",
}
// Start the job running with a specified duration.
if err := runner.Run(context, time.Second, &t); err != nil {
switch err {
case runner.ErrTimeout:
// The task did not finish within the specified duration.
log.Error(context, "main", err, "Task timeout")
case runner.ErrSignaled:
// The user hit <control> c and we shutdown early.
log.Error(context, "main", err, "Shutdown early")
default:
// An error occurred in the processing of the task.
log.Error(context, "main", err, "Processing error")
}
os.Exit(1)
}
log.User(context, "main", "Completed")
}
// UpdateStatus updates a form submissions status inside the MongoDB database
// collection.
func UpdateStatus(context interface{}, db *db.DB, id, status string) (*Submission, error) {
log.Dev(context, "UpdateStatus", "Started : Submission[%s]", id)
if !bson.IsObjectIdHex(id) {
log.Error(context, "UpdateStatus", ErrInvalidID, "Completed")
return nil, ErrInvalidID
}
objectID := bson.ObjectIdHex(id)
f := func(c *mgo.Collection) error {
u := bson.M{
"$set": bson.M{
"status": status,
"date_updated": time.Now(),
},
}
log.Dev(context, "UpdateStatus", "MGO : db.%s.update(%s, %s)", c.Name, mongo.Query(objectID), mongo.Query(u))
return c.UpdateId(objectID, u)
}
if err := db.ExecuteMGO(context, Collection, f); err != nil {
log.Error(context, "UpdateStatus", err, "Completed")
return nil, err
}
submission, err := Retrieve(context, db, id)
if err != nil {
log.Error(context, "UpdateStatus", err, "Completed")
return nil, err
}
log.Dev(context, "UpdateStatus", "Completed")
return submission, nil
}
// Count returns the count of current submissions for a given
// form id in the Form Submissions MongoDB database collection.
func Count(context interface{}, db *db.DB, formID string) (int, error) {
log.Dev(context, "Count", "Completed : Form[%s]", formID)
if !bson.IsObjectIdHex(formID) {
log.Error(context, "Count", ErrInvalidID, "Completed")
return 0, ErrInvalidID
}
formObjectID := bson.ObjectIdHex(formID)
var count int
f := func(c *mgo.Collection) error {
var err error
q := bson.M{
"form_id": formObjectID,
}
log.Dev(context, "Count", "MGO : db.%s.find(%s).count()", c.Name, mongo.Query(q))
count, err = c.Find(q).Count()
return err
}
if err := db.ExecuteMGO(context, Collection, f); err != nil {
log.Error(context, "Count", err, "Completed")
return 0, err
}
log.Dev(context, "Count", "Completed")
return count, nil
}
// RetrieveMany retrieves a list of Submission's from the MongoDB database collection.
func RetrieveMany(context interface{}, db *db.DB, ids []string) ([]Submission, error) {
log.Dev(context, "RetrieveMany", "Started")
var objectIDs = make([]bson.ObjectId, len(ids))
for i, id := range ids {
if !bson.IsObjectIdHex(id) {
log.Error(context, "RetrieveMany", ErrInvalidID, "Completed")
return nil, ErrInvalidID
}
objectIDs[i] = bson.ObjectIdHex(id)
}
var submissions []Submission
f := func(c *mgo.Collection) error {
q := bson.M{
"_id": bson.M{
"$in": objectIDs,
},
}
log.Dev(context, "RetrieveMany", "MGO : db.%s.find(%s)", c.Name, mongo.Query(q))
return c.Find(q).All(&submissions)
}
if err := db.ExecuteMGO(context, Collection, f); err != nil {
log.Error(context, "RetrieveMany", err, "Completed")
return nil, err
}
log.Dev(context, "RetrieveMany", "Started")
return submissions, nil
}
// RemoveFlag removes a flag from a given Submission in
// the MongoDB database collection.
func RemoveFlag(context interface{}, db *db.DB, id, flag string) (*Submission, error) {
log.Dev(context, "RemoveFlag", "Started : Submission[%s]", id)
if !bson.IsObjectIdHex(id) {
log.Error(context, "RemoveFlag", ErrInvalidID, "Completed")
return nil, ErrInvalidID
}
objectID := bson.ObjectIdHex(id)
f := func(c *mgo.Collection) error {
u := bson.M{
"$pull": bson.M{
"flags": flag,
},
}
log.Dev(context, "RemoveFlag", "MGO : db.%s.update(%s, %s)", c.Name, mongo.Query(objectID), mongo.Query(u))
return c.UpdateId(objectID, u)
}
if err := db.ExecuteMGO(context, Collection, f); err != nil {
log.Error(context, "RemoveFlag", err, "Completed")
return nil, err
}
submission, err := Retrieve(context, db, id)
if err != nil {
log.Error(context, "RemoveFlag", err, "Completed")
return nil, err
}
log.Dev(context, "RemoveFlag", "Completed")
return submission, nil
}
func main() {
// Initialize logging
logLevel := func() int {
ll, err := cfg.Int(cfgLoggingLevel)
if err != nil {
return log.USER
}
return ll
}
log.Init(os.Stderr, logLevel)
// Generate UUID to use with the logs
uid := uuid.New()
if err := sponge.Init(uid); err != nil {
log.Error(uid, "main", err, "Unable to initialize configuration.")
os.Exit(-1)
}
if err := cmd.RootCmd.Execute(); err != nil {
log.Error(uid, "main", err, "Unable to execute the command.")
os.Exit(-1)
}
}
// UpsertForm upserts the provided form into the MongoDB database collection and
// creates a gallery based on it.
func UpsertForm(context interface{}, db *db.DB, f *form.Form) error {
log.Dev(context, "UpsertForm", "Started : Form[%s]", f.ID.Hex())
var isNewForm bool
// If there was no ID provided, we should set one. UpsertForm might optionally add
// a form ID to ensure that we don't duplicate the FormGallery.
if f.ID.Hex() == "" {
isNewForm = true
}
if err := form.Upsert(context, db, f); err != nil {
log.Error(context, "UpsertForm", err, "Completed")
return err
}
if isNewForm {
// Create the new gallery that we will create that is based on the current
// form ID.
g := gallery.Gallery{
FormID: f.ID,
}
if err := gallery.Create(context, db, &g); err != nil {
log.Error(context, "UpsertForm", err, "Completed")
return err
}
}
log.Dev(context, "UpsertForm", "Completed")
return nil
}
// DeleteSubmission deletes a submission as well as updating a form's stats.
func DeleteSubmission(context interface{}, db *db.DB, id, formID string) error {
log.Dev(context, "DeleteSubmission", "Started : Submission[%s]", id)
if !bson.IsObjectIdHex(id) {
log.Error(context, "DeleteSubmission", ErrInvalidID, "Completed")
return ErrInvalidID
}
if !bson.IsObjectIdHex(formID) {
log.Error(context, "Delete", ErrInvalidID, "Completed")
return ErrInvalidID
}
if err := submission.Delete(context, db, id); err != nil {
log.Error(context, "DeleteSubmission", err, "Completed")
return err
}
if _, err := form.UpdateStats(context, db, formID); err != nil {
log.Error(context, "DeleteSubmission", err, "Completed")
return err
}
log.Dev(context, "DeleteSubmission", "Started")
return nil
}
// ConnectionMySQL returns the connection string
func connectionAPI(pageAfter string) *url.URL {
credA, ok := credential.(str.CredentialService)
if !ok {
err := fmt.Errorf("Error when asserting type CredentialService on credential.")
log.Error(uuid, "api.connectionAPI", err, "Asserting type.")
}
data := QueryData{}
// Get all the data from credentials we need
data.Basicurl = credA.GetEndpoint() //"https://comments-api.ext.nile.works/v1/search"
data.Appkey = credA.GetAppKey()
data.Next = pageAfter //credA.GetPageAfterField() // field that we are going to get the next value from
data.Attributes = credA.GetAttributes() // Attributes for the query. Eg, for WaPo we have scope and sortOrder
urltemplate, urltemplatepagination := credA.GetQueryFormat() //format for the query
regexToEscape := credA.GetRegexToEscape()
//url.QueryEscape(
surl, err := formatURL(data, urltemplate, urltemplatepagination, regexToEscape)
if err != nil {
log.Error(uuid, "api.connectionAPI", err, "Parsing url %s", surl)
}
u, err := url.Parse(surl)
if err != nil {
log.Error(uuid, "api.connectionAPI", err, "Parsing url %s", surl)
}
return u
}
// DecodeWebToken breaks a web token into its parts.
func DecodeWebToken(context interface{}, webToken string) (sessionID string, token string, err error) {
log.Dev(context, "DecodeWebToken", "Started : WebToken[%s]", webToken)
// Decode the web token to break it into its parts.
data, err := base64.StdEncoding.DecodeString(webToken)
if err != nil {
log.Error(context, "DecodeWebToken", err, "Completed")
return "", "", err
}
// Split the web token.
str := strings.Split(string(data), ":")
if len(str) != 2 {
err := errors.New("Invalid token")
log.Error(context, "DecodeWebToken", err, "Completed")
return "", "", err
}
// Pull out the session and token.
sessionID = str[0]
token = str[1]
log.Dev(context, "DecodeWebToken", "Completed : SessionID[%s] Token[%s]", sessionID, token)
return sessionID, token, nil
}
// Retrieve retrieves a form gallery from the MongoDB database
// collection as well as hydrating the form gallery with form submissions.
func Retrieve(context interface{}, db *db.DB, id string) (*Gallery, error) {
log.Dev(context, "Retrieve", "Started : Gallery[%s]", id)
if !bson.IsObjectIdHex(id) {
log.Error(context, "Retrieve", ErrInvalidID, "Completed")
return nil, ErrInvalidID
}
objectID := bson.ObjectIdHex(id)
var gallery Gallery
f := func(c *mgo.Collection) error {
log.Dev(context, "Retrieve", "MGO : db.%s.find(%s)", c.Name, mongo.Query(objectID))
return c.FindId(objectID).One(&gallery)
}
if err := db.ExecuteMGO(context, Collection, f); err != nil {
log.Error(context, "Retrieve", err, "Completed")
return nil, err
}
if err := hydrate(context, db, &gallery); err != nil {
log.Error(context, "Retrieve", err, "Completed")
return nil, err
}
log.Dev(context, "Retrieve", "Completed")
return &gallery, nil
}
// New creates a new strategy struct variable from the json file
func New() (Strategy, error) {
var strategy Strategy
var err error
//read STRATEGY_CONF env variable
strategyFile := os.Getenv("STRATEGY_CONF")
if strategyFile == "" {
log.Fatal(uuid, "strategy.new", "Enviromental variable STRATEGY_CONF not setup.")
return strategy, err
}
strategy, err = read(strategyFile)
if err != nil {
log.Error(uuid, "strategy.new", err, "Reading strategy file %s.", strategyFile)
return Strategy{}, err
}
err = strategy.setCredential()
if err != nil {
log.Error(uuid, "strategy.new", err, "Setting credentials.")
return Strategy{}, err
}
return strategy, err
}
// varLookup looks up variables and returns their values as the specified type.
func varLookup(context interface{}, cmd, variable string, vars map[string]string, results map[string]interface{}) (interface{}, error) {
// {"field": "#cmd:variable"}
// Before: {"field": "#number:variable_name"} After: {"field": 1234}
// Before: {"field": "#string:variable_name"} After: {"field": "value"}
// Before: {"field": "#date:variable_name"} After: {"field": time.Time}
// Before: {"field": "#objid:variable_name"} After: {"field": mgo.ObjectId}
// Before: {"field": "#regex:/pattern/<options>"} After: {"field": bson.RegEx}
// Before: {"field": "#since:3600"} After: {"field": time.Time}
// Before: {"field": "#data.0:doc.station_id"} After: {"field": "23453"}
// If the variable does not exist, use the variable straight up.
param, exists := vars[variable]
if !exists {
param = variable
}
// Do we have a command that is not known.
if len(cmd) < 4 {
err := fmt.Errorf("Unknown command %q", cmd)
log.Error(context, "varLookup", err, "Checking cmd is proper length")
return nil, err
}
// Let's perform the right action per command.
switch cmd[0:4] {
case "numb":
return number(context, param)
case "stri":
return param, nil
case "date":
return isoDate(context, param)
case "obji":
return objID(context, param)
case "rege":
return regExp(context, param)
case "time":
return adjTime(context, param)
case "data":
if len(cmd) == 6 {
return dataLookup(context, cmd[5:6], param, results)
}
err := errors.New("Data command is missing the operator")
log.Error(context, "varLookup", err, "Checking cmd is data")
return nil, err
default:
err := fmt.Errorf("Unknown command %q", cmd)
log.Error(context, "varLookup", err, "Checking cmd in default case")
return nil, err
}
}
func process(coralName string, data []map[string]interface{}) {
// Transform the data row by row
log.User(uuid, "sponge.process", "# Transforming data to the coral schema.\n")
log.User(uuid, "sponge.process", "# And importing %v documents.", len(data))
// Initialize benchmarking for current table
start := time.Now()
blockStart := time.Now()
blockSize := int64(1000) // number of documents between each report
documents := int64(0)
totalDocuments := int64(len(data))
for _, row := range data {
// output benchmarking for each block of documents
if documents%blockSize == 0 && documents > 0 {
// calculate stats
percentComplete := float64(documents) / float64(totalDocuments) * float64(100)
msSinceStart := time.Since(start).Nanoseconds() / int64(1000000)
msSinceBlock := time.Since(blockStart).Nanoseconds() / int64(1000000)
timeRemaining := int64(float64(time.Since(start).Seconds()) / float64(percentComplete) * float64(100))
//log stats
log.User(uuid, "sponge.process", "%v%% (%v/%v imported) %vms, %vms avg - last %v in %vms, %vms avg -- est time remaining %vs\n", int64(percentComplete), documents, totalDocuments, msSinceStart, msSinceStart/documents, blockSize, msSinceBlock, msSinceBlock/blockSize, int64(timeRemaining))
blockStart = time.Now()
}
documents = documents + 1
// transform the row
id, newRows, err := fiddler.TransformRow(row, coralName)
if err != nil {
log.Error(uuid, "sponge.process", err, "Error when transforming the row %s.", row)
//RECORD to report about failing transformation
if options.ReportOnFailedRecords {
report.Record(coralName, id, "Failing transform data", err)
}
}
// Usually newRows only will have a document but in the case that we have subcollections
// we may get more than one document from a transformation
for _, newRow := range newRows {
log.Dev(uuid, "sponge.process", "Transforming: %v into %v.", row, newRow)
// send the row to pillar
err = coral.AddRow(newRow, coralName)
if err != nil {
log.Error(uuid, "sponge.process", err, "Error when adding a row") // thae row %v to %s.", string(newRow), modelName)
//RECORD to report about failing adding row to coral db
if options.ReportOnFailedRecords {
report.Record(coralName, id, "Failing add row to coral", err)
}
}
}
}
}
// GetQueryData returns the raw data from the table based on the ids
func (m MySQL) GetQueryData(entityname string, options *Options, ids []string) ([]map[string]interface{}, error) { //offset int, limit int, orderby string
// Get the corresponding entity to the entityname
tableName := strategy.GetEntityForeignName(entityname)
tableFields := strategy.GetEntityForeignFields(entityname) // []map[string]string
// open a connection
db, err := m.open()
if err != nil {
log.Error(uuid, "mysql.getquerydata", err, "Error connecting to mysql database.")
return nil, err
}
defer m.close(db)
// Fields for that external source table
f := make([]string, 0, len(tableFields))
for _, field := range tableFields {
if field != nil {
f = append(f, field["foreign"].(string))
}
}
// all the fields
fields := strings.Join(f, ", ")
// if we are ordering by
if len(options.Orderby) == 0 {
options.Orderby = strategy.GetOrderBy(entityname)
}
var queryWhere string
// if we are quering specifics recrords
if len(ids) > 0 {
idField := strategy.GetIDField(entityname)
queryWhere = fmt.Sprintf("where %s in (%s)", idField, strings.Join(ids, ", "))
}
// Get only the fields that we are going to use
// the query string . To Do. Select only the stuff you are going to use
query := strings.Join([]string{"SELECT", fields, "from", tableName, queryWhere, "order by", options.Orderby, "limit", fmt.Sprintf("%v", options.Offset), ", ", fmt.Sprintf("%v", options.Limit)}, " ")
data, err := gosqljson.QueryDbToMapJSON(db, "lower", query)
if err != nil {
log.Error(uuid, "mysql.getquerydata", err, "Running SQL query.")
return nil, err
}
byt := []byte(data)
var dat []map[string]interface{}
err = json.Unmarshal(byt, &dat)
if err != nil {
log.Error(uuid, "mysql.getquerydata", err, "Unmarshalling the query.")
return nil, err
}
return dat, nil
}
// GetData returns the raw data from that entity
func (m MySQL) GetData(entityname string, options *Options) ([]map[string]interface{}, error) { //offset int, limit int, orderby string, q string
// Get the corresponding table to the modelName
tableName := strategy.GetEntityForeignName(entityname)
tableFields := strategy.GetEntityForeignFields(entityname) // []map[string]string
// open a connection
db, err := m.open()
if err != nil {
log.Error(uuid, "mysql.getdata", err, "Connecting to mysql database.")
return nil, err
}
defer m.close(db)
// Fields for that external source table
f := make([]string, 0, len(tableFields))
for _, field := range tableFields {
if field != nil {
f = append(f, field["foreign"].(string))
}
}
fields := strings.Join(f, ", ")
if options.Orderby == "" {
options.Orderby = strategy.GetOrderBy(entityname)
}
// Get only the fields that we are going to use
// the query string . To Do. Select only the stuff you are going to use
//query := strings.Join([]string{"SELECT", fields, "from", tableName, "order by", orderby, "limit", fmt.Sprintf("%v", offset), ", ", fmt.Sprintf("%v", limit)}, " ")
var where string
if options.Query != "" {
where = fmt.Sprintf("where %s ", options.Query)
}
query := fmt.Sprintf("SELECT %s from %s %s order by %s limit %v, %v", fields, tableName, where, options.Orderby, options.Offset, options.Limit)
data, err := gosqljson.QueryDbToMapJSON(db, "lower", query)
if err != nil {
log.Error(uuid, "mysql.getdata", err, "Running SQL query.")
return nil, err
}
byt := []byte(data)
var dat []map[string]interface{}
err = json.Unmarshal(byt, &dat)
if err != nil {
log.Error(uuid, "mysql.getdata", err, "Unmarshalling the result of the query.")
return nil, err
}
return dat, nil
}
// Create creates a new FormSubmission based on the payload of replies and the
// formID that is being submitted.
// 200 Success, 400 Bad Request, 404 Not Found, 500 Internal
func (formSubmissionHandle) Create(c *web.Context) error {
var payload struct {
Recaptcha string `json:"recaptcha"`
Answers []submission.AnswerInput `json:"replies"`
}
if err := json.NewDecoder(c.Request.Body).Decode(&payload); err != nil {
return err
}
formID := c.Params["form_id"]
// We should check to see if the form has a recaptcha property.
f, err := form.Retrieve(c.SessionID, c.Ctx["DB"].(*db.DB), formID)
if err != nil {
return err
}
// If the recaptcha is enabled on the form, then we should check that the
// response contains the data we need and if it's valid.
if enabled, ok := f.Settings["recaptcha"].(bool); ok && enabled {
if len(payload.Recaptcha) <= 0 {
log.Error(c.SessionID, "FormSubmission : Create", ErrInvalidCaptcha, "Payload empty")
return ErrInvalidCaptcha
}
// Check to see if Recaptcha has been enabled on the server.
if recaptchaSecret, ok := c.Web.Ctx["recaptcha"].(string); ok {
// Communicate with the Google Recaptcha Web Service to validate the
// request.
if err := ValidateReacaptchaResponse(c, recaptchaSecret, payload.Recaptcha); err != nil {
log.Error(c.SessionID, "FormSubmission : Create", err, "Recaptcha validation failed")
return ErrInvalidCaptcha
}
} else {
log.Dev(c.SessionID, "FormSubmission : Create", "Recaptcha disabled, will not check")
}
}
s, err := ask.CreateSubmission(c.SessionID, c.Ctx["DB"].(*db.DB), formID, payload.Answers)
if err != nil {
return err
}
c.Respond(s, http.StatusOK)
return nil
}
// API returns a handler for a set of routes.
func API() http.Handler {
mongoURI := cfg.MustURL(cfgMongoURI)
// The web framework middleware for Mongo is using the name of the
// database as the name of the master session by convention. So use
// cfg.DB as the second argument when creating the master session.
if err := db.RegMasterSession("startup", mongoURI.Path, mongoURI.String(), 25*time.Second); err != nil {
log.Error("startup", "Init", err, "Initializing MongoDB")
os.Exit(1)
}
w := web.New(logm.Midware, errorm.Midware)
publicKey, err := cfg.String(cfgAuthPublicKey)
if err != nil || publicKey == "" {
log.User("startup", "Init", "%s is missing, internal authentication is disabled", cfgAuthPublicKey)
}
// If the public key is provided then add the auth middleware or fail using
// the provided public key.
if publicKey != "" {
log.Dev("startup", "Init", "Initializing Auth")
authm, err := authm.Midware(publicKey, authm.MidwareOpts{})
if err != nil {
log.Error("startup", "Init", err, "Initializing Auth")
os.Exit(1)
}
// Apply the authentication middleware on top of the application as the
// first middleware.
w.Use(authm)
}
// Add the Mongo and Cayley middlewares possibly after the auth middleware.
w.Use(mongo.Midware(mongoURI), cayley.Midware(mongoURI))
if cors, err := cfg.Bool(cfgEnableCORS); err == nil && cors {
log.Dev("startup", "Init", "Initializing CORS : CORS Enabled")
w.Use(w.CORS())
} else {
log.Dev("startup", "Init", "CORS Disabled")
}
log.Dev("startup", "Init", "Initalizing routes")
routes(w)
return w
}
// Midware for catching errors.
func Midware(next web.Handler) web.Handler {
// Create the handler that will be attached in the middleware chain.
h := func(c *web.Context) error {
log.Dev(c.SessionID, "error : Midware", "Started")
// In the event of a panic, we want to capture it here so we can send an
// error down the stack.
defer func() {
if err := recover(); err != nil {
// Respond with the error.
c.RespondError("internal server error", http.StatusInternalServerError)
// Log out that we caught the error.
switch err := err.(type) {
case error:
log.Error(c.SessionID, "error : Midware", err, "Panic Caught")
default:
log.Error(c.SessionID, "error : Midware", fmt.Errorf("%v", err), "Panic Caught")
}
// Print out the stack.
log.Dev(c.SessionID, "error : Midware", "Panic Stacktrace:\n%s", debug.Stack())
_, filePath, line, _ := runtime.Caller(4)
log.Dev(c.SessionID, "error : Midware", "Panic Traced to %s:%d", filePath, line)
}
}()
if err := next(c); err != nil {
// Respond with the error.
c.Error(err)
// Log out that we caught the error.
log.Dev(c.SessionID, "error : Midware", "Completed : Error Caught : %s", err.Error())
return nil
}
log.Dev(c.SessionID, "error : Midware", "Completed")
return nil
}
return h
}