// Use for sending a file to be managed by the auto manager. The auto manager will decide if and when the file should
// be sent to the bundle manager for upload.
func (self *autoManager) addAutoFile(fs fileState, newFileName string, groups [][2]string) {
common.Dprintln("Entering addAutoFile")
defer common.Dprintln("Leaving addAutoFile")
self.mutex.Lock()
defer self.mutex.Unlock()
self.refreshWorkingBundlesNoLock(false)
common.Dprintf("addAutoFile working with %+v", fs)
//Skip this file if it has been seen before.
if fs.passOff != notSeenBefore {
common.Dprintf("file %s, %s, %s, has been seen before...skipping.", fs.userName, fs.ruleName, fs.fullPath)
return
}
af := &autoFile{fs: &fs, newFileName: newFileName, groups: groups}
//Get the corresponding WatchRule for this fileState.
wr := getWatchRule(fs.userName, fs.ruleName)
if wr == nil {
log.Printf("getWatchRule returned no watchRule for user %s and rule %s, addAutoFile will be re-attempted later.",
fs.userName, fs.ruleName)
return
}
//Bundle the file and log any error.
err := self.bundleAutoFileNoLock(af, fs.userName, fs.ruleName, wr)
if err != nil {
log.Printf("bundleAutoFileNoLock failed with error %v, addAutoFile will be re-attempted later.", err)
return
}
}
func (self *autoManager) scheduleSubmit() {
common.Dprintln("Entering scheduleSubmit")
defer common.Dprintln("Leaving scheduleSubmit")
self.mutex.Lock()
defer self.mutex.Unlock()
self.refreshWorkingBundlesNoLock(false)
for _, v := range self.workingBundles {
if v.autoSubmitBid == nil {
continue
}
common.Dprintf("scheduleSubmit workingBundle %+v", *v)
//Get the BundleMD for autoSubmitBid
b, err := autoBm.BundleGet(v.user, *v.autoSubmitBid)
if err != nil {
log.Printf("BundleGet(%s, %d) failed %v", v.user, *v.autoSubmitBid, err)
continue
}
//Get the amount of time since the last modification to the BundleMD
modTime := time.Now().UnixNano() - v.autoSubmitLastTouched
//Get the number of files in this BundleMD
count, err := b.BundleFileCountGet()
if err != nil {
log.Printf("Failed to get file count for bundle id %d, error %v", v.autoSubmitBid, err)
continue
}
common.Dprintf("modTime == %d, bundleModThreshold == %d", modTime, bundleModThreshold)
//Submit only the autoSubmitBid if its activity has slowed or it has met the maximum number
//of files threshold.
if time.Duration(modTime) >= bundleModThreshold ||
count >= maxNumFilesBundle {
//Submit...
err := b.Submit()
if err != nil {
log.Printf("Failed to submit bundle, error %v", err)
continue
}
common.Dprintf("Bundle %d submitted!!!", v.autoSubmitBid)
//Set the userBundles auto bundle to nil so a new one will get created on the next file add.
self.workingBundlesNeedsRefresh = true
v.autoSubmitBid = nil
err = self.stateManager.db.setUserBundles(v)
if err != nil {
log.Printf("Failed to set userBundles %+v for, error %v", v, err)
continue
}
}
}
}
// Get a map of current working (i.e. in progress or yet to be submitted) bundle id's for each user.
func (self *stateDatabase) getWorkingBundles() map[string]*userBundles {
common.Dprintln("Entering getWorkingBundles")
defer common.Dprintln("Leaving getWorkingBundles")
self.mutex.Lock()
defer self.mutex.Unlock()
sql := fmt.Sprintf("SELECT user, auto_submit_bid, no_auto_submit_bid, auto_submit_last_touched FROM user_bundles")
s, err := self.conn.Prepare(sql)
if err != nil {
log.Fatalf("getWorkingBundles query preparation %s, failed with error %v", sql, err)
}
defer s.Finalize()
err = s.Exec()
if err != nil {
log.Fatalf("getWorkingBundles query execution %s, failed with error %v", sql, err)
}
ret := make(map[string]*userBundles)
var user string
var auto_submit_bid string
var no_auto_submit_bid string
var auto_submit_last_touched int64
for {
if !s.Next() {
break
}
err = s.Scan(&user, &auto_submit_bid, &no_auto_submit_bid, &auto_submit_last_touched)
if err != nil {
log.Fatalf("%v", err)
}
ub := new(userBundles)
ub.user = user
if auto_submit_bid != "" {
i, err := strconv.Atoi(auto_submit_bid)
if err != nil {
log.Fatalf("SQL query %s returned unexpected auto_submit_bid %s, error %v", sql, auto_submit_bid, err)
}
ub.autoSubmitBid = &i
}
if no_auto_submit_bid != "" {
i, err := strconv.Atoi(no_auto_submit_bid)
if err != nil {
log.Fatalf("SQL query %s returned unexpected no_bundle_file_id %s, error %v", sql, no_auto_submit_bid, err)
}
ub.noAutoSubmitBid = &i
}
ub.autoSubmitLastTouched = auto_submit_last_touched
ret[user] = ub
}
return ret
}
func (self *autoManager) recover() {
common.Dprintln("Entering recover")
defer common.Dprintln("Leaving recover")
self.mutex.Lock()
defer self.mutex.Unlock()
//Get list of fileStates in bundleManager that have autoPassOffState addingToBundle.
states, err := self.stateManager.getFileStatesInLimbo()
if err != nil {
log.Fatalf("Could not get file states with state addingToBundle. Error, %v", err)
}
for _, v := range states {
log.Printf("Recovering file %s", v.fullPath)
var isInBundle bool
if v.bundleId != nil && v.bundleFileId != nil {
b, err := autoBm.BundleGet(v.userName, *v.bundleId)
if err != nil {
isInBundle = false
} else {
_, err := autoBm.BundleFileGet(b, v.userName, *v.bundleId, *v.bundleFileId)
if err != nil {
isInBundle = false
} else {
isInBundle = true
}
}
} else {
isInBundle = false
}
if isInBundle {
v.passOff = inBundle
} else {
v.bundleId = nil
v.bundleFileId = nil
v.passOff = notSeenBefore
}
err = self.stateManager.setFileState(v)
if err != nil {
log.Printf("Failed to set state on fileState %+v, error %v", v, err)
continue
}
}
}
func (self *stateDatabase) setUserBundles(ub *userBundles) error {
common.Dprintln("Entering setUserBundles")
defer common.Dprintln("Leaving setUserBundles")
if ub == nil {
return errors.New("ub must not be nil")
}
self.mutex.Lock()
defer self.mutex.Unlock()
var autoSql string
if ub.autoSubmitBid == nil {
autoSql = "NULL"
ub.autoSubmitLastTouched = 0
} else {
autoSql = fmt.Sprintf("%d", *ub.autoSubmitBid)
ub.autoSubmitLastTouched = time.Now().UnixNano()
}
var noAutoSql string
if ub.noAutoSubmitBid == nil {
noAutoSql = "NULL"
} else {
noAutoSql = fmt.Sprintf("%d", *ub.noAutoSubmitBid)
}
sql := fmt.Sprintf("INSERT OR REPLACE INTO user_bundles "+
"(user, auto_submit_bid, no_auto_submit_bid, auto_submit_last_touched) "+
"VALUES "+
"(\"%s\", %s, %s, %d);",
ub.user, autoSql, noAutoSql, ub.autoSubmitLastTouched)
common.Dprintf("%s", sql)
err := self.conn.Exec(sql)
if err != nil {
return err
}
return nil
}
func (self *autoManager) monitorSubmitted() {
common.Dprintln("Entering cleanupSubmitted")
defer common.Dprintln("Leaving cleanupSubmitted")
self.mutex.Lock()
defer self.mutex.Unlock()
users, bundle_ids, err := self.stateManager.db.getFileStatesProgressing()
if err != nil {
log.Printf("Failed to getFileStatesProgressing. error %v", err)
return
}
var i int
for i = 0; i < len(users); i++ {
b, err := autoBm.BundleGet(users[i], bundle_ids[i])
if err != nil {
log.Printf("BundleGet(%s, %d) failed %v", users[i], bundle_ids[i], err)
continue
}
bState, err := b.StateGet()
if err != nil {
log.Printf("Could not get state for BundleMD %+v, error %v", b, err)
continue
}
if bState == upload.BundleState_Unsubmitted || bState == upload.BundleState_ToBundle || bState == upload.BundleState_Submitted {
//FIXME This cuts out extra expensive checks when the state should not change. But, if state gets to submitted quickly and some files DOE'ed, then this can hold up files needlessly until state changes to ERROR or SUCCESS. Really, the check should be to check only once per bundle when state of bundle's DOE files aren't going to change. This means maintaining per bundle state in the db.
continue
}
bfs, err := b.FilesGet()
if err != nil {
log.Printf("FileIdsGet() returned error %v", err)
continue
}
flagged, err := b.FlaggedToDelete()
if err != nil {
log.Printf("Failed to get flagged to delete returned error %v", err)
continue
}
for _, bf := range bfs {
bfStateMsg, err := bf.DisableOnErrorMsgGet()
if err != nil {
log.Printf("Could not get state for BundleFileMD %+v, error %v", bf, err)
continue
}
//Get fileState(s) for this BundleFileMD
bfid := bf.IdGet()
states, err := self.stateManager.db.getFileStatesByBundleFileId(bfid)
if err != nil || len(states) < 1 {
log.Printf("Could not get any fileStates for bundle file id %d, error %v", bfid, err)
continue
}
if bfStateMsg != "" ||
bState == upload.BundleState_Error || (flagged && bState == upload.BundleState_Unsubmitted) {
for _, v := range states {
v.bundleId = nil
v.bundleFileId = nil
v.passOff = notSeenBefore
self.stateManager.setFileState(v)
}
continue
}
if bState == upload.BundleState_Safe {
for _, v := range states {
v.passOff = done
t, err := bf.MtimeGet()
if err != nil {
log.Printf("Could not get time for %+v, error %v", bf, err)
} else {
v.lastModified = t.UnixNano()
}
self.stateManager.setFileState(v)
}
continue
}
}
if flagged ||
bState == upload.BundleState_Safe ||
bState == upload.BundleState_Error {
//FIXME consider this information with the above error handling. It probably should not delete state until all above is cleared.
err := b.Delete("auto")
if err != nil {
log.Printf("Failed to delete BundleMD, error %v", err)
}
}
}
}
func (self *stateDatabase) setupDatabase() {
self.mutex.Lock()
defer self.mutex.Unlock()
common.Dprintln("Entered stateDatabase.setupDatabase")
//Open or create the database
c, err := sqlite.Open(self.path)
if err != nil {
log.Fatalf("Could not open or create %s, error: %v\n", self.path, err)
}
self.conn = c
//Turn on foreign key constraints
err = self.conn.Exec("PRAGMA foreign_keys = ON;")
if err != nil {
log.Fatalf("%v\n", err)
}
//Prepare to get the current database version
s, err := self.conn.Prepare("select value from system where name = \"version\";")
//The system table does not exist
if err != nil && err.Error() == "SQL error or missing database: no such table: system" {
err = self.conn.Exec("create table system(name string primary key, value string);")
if err != nil {
log.Fatalf("%v\n", err)
}
fileStatesSchema := "CREATE TABLE file_states(" +
"id INTEGER PRIMARY KEY AUTOINCREMENT, " +
"last_modified INTEGER NOT NULL, " +
"digest STRING NULLABLE, " +
"full_path STRING NOT NULL, " +
"last_seen INTEGER NOT NULL, " +
"bundle_id INTEGER NULLABLE, " +
"bundle_file_id INTEGER NULLABLE, " +
"pass_off INTEGER NOT NULL, " +
"user_name STRING NOT NULL, " +
"rule_name STRING NOT NULL, " +
"auto_added_to_bundle_time INTEGER NOT NULL);"
//Create the file_states table
err = self.conn.Exec(fileStatesSchema)
if err != nil {
log.Fatalf("%v\nSQL: %s\n", err, fileStatesSchema)
}
userBundlesSchema := "CREATE TABLE user_bundles(" +
"user STRING PRIMARY KEY, " +
"auto_submit_bid INTEGER NULLABLE, " +
"no_auto_submit_bid INTEGER NULLABLE, " +
"auto_submit_last_touched INTEGER NOT NULL);"
//Create the user_bundles table
err = self.conn.Exec(userBundlesSchema)
if err != nil {
log.Fatalf("%v\nSQL: %s\n", err, userBundlesSchema)
}
schema := strings.Split("create index file_states_full_path ON file_states(full_path);"+
"create index file_states_id ON file_states(id);", ";")
for _, sql := range schema {
if strings.TrimSpace(sql) == "" {
continue
}
err = self.conn.Exec(sql + ";")
if err != nil {
log.Printf("%v\nSQL: %s\n", err, sql)
os.Exit(1)
}
}
//Set the database version
err = self.conn.Exec("INSERT INTO system(name, value) VALUES(\"version\", \"" +
strconv.Itoa(int(_DATABASE_CODE_VER)) + "\");")
if err != nil {
log.Fatalf("%v\n", err)
}
//Prepare statement to get the database version
s, err = self.conn.Prepare("select value from system where name = \"version\";")
if err != nil {
log.Fatalf("%v\n", err)
}
}
//Get the database version
err = s.Exec()
if err != nil {
log.Fatalf("%v\n", err)
}
version := ""
for {
if !s.Next() {
break
}
var value string
err = s.Scan(&value)
if err != nil {
log.Fatalf("%v", err)
}
version = value
}
if version == "" {
log.Fatalf("Failed to get a version from database.\n")
}
ver, err := strconv.ParseInt(version, 10, 32)
if err != nil {
log.Fatalf("Unable to read version %s %v\n", version, err)
}
log.Printf("Got file manager database version %v\n", ver)
if ver > _DATABASE_CODE_VER {
log.Fatalf("File manager database is too new(%v). Version is %v\n", ver, _DATABASE_CODE_VER)
}
if ver < _DATABASE_CODE_VER {
if ver == 1 {
schema := strings.Split("begin transaction;"+
"create index file_states_full_path ON file_states(full_path);"+
"create index file_states_id ON file_states(id);"+
"update system set value = 2 where name=\"version\";"+
"commit;", ";")
for _, sql := range schema {
if strings.TrimSpace(sql) == "" {
continue
}
err = self.conn.Exec(sql + ";")
if err != nil {
log.Printf("Failed to upgrade schema! %v\nSQL: %s\n", err, sql)
os.Exit(1)
}
}
ver = 2
}
}
if ver != _DATABASE_CODE_VER {
log.Fatalf("File state manager database needs to be upgraded(%v). I'm %v\n", ver, _DATABASE_CODE_VER)
}
}