예제 #1
0
// Use for sending a file to be managed by the auto manager.  The auto manager will decide if and when the file should
// be sent to the bundle manager for upload.
func (self *autoManager) addAutoFile(fs fileState, newFileName string, groups [][2]string) {
	common.Dprintln("Entering addAutoFile")
	defer common.Dprintln("Leaving addAutoFile")

	self.mutex.Lock()
	defer self.mutex.Unlock()

	self.refreshWorkingBundlesNoLock(false)

	common.Dprintf("addAutoFile working with %+v", fs)

	//Skip this file if it has been seen before.
	if fs.passOff != notSeenBefore {
		common.Dprintf("file %s, %s, %s, has been seen before...skipping.", fs.userName, fs.ruleName, fs.fullPath)
		return
	}

	af := &autoFile{fs: &fs, newFileName: newFileName, groups: groups}

	//Get the corresponding WatchRule for this fileState.
	wr := getWatchRule(fs.userName, fs.ruleName)
	if wr == nil {
		log.Printf("getWatchRule returned no watchRule for user %s and rule %s, addAutoFile will be re-attempted later.",
			fs.userName, fs.ruleName)
		return
	}

	//Bundle the file and log any error.
	err := self.bundleAutoFileNoLock(af, fs.userName, fs.ruleName, wr)
	if err != nil {
		log.Printf("bundleAutoFileNoLock failed with error %v, addAutoFile will be re-attempted later.", err)
		return
	}
}
예제 #2
0
func (self *autoManager) scheduleSubmit() {
	common.Dprintln("Entering scheduleSubmit")
	defer common.Dprintln("Leaving scheduleSubmit")

	self.mutex.Lock()
	defer self.mutex.Unlock()

	self.refreshWorkingBundlesNoLock(false)

	for _, v := range self.workingBundles {
		if v.autoSubmitBid == nil {
			continue
		}

		common.Dprintf("scheduleSubmit workingBundle %+v", *v)

		//Get the BundleMD for autoSubmitBid
		b, err := autoBm.BundleGet(v.user, *v.autoSubmitBid)
		if err != nil {
			log.Printf("BundleGet(%s, %d) failed %v", v.user, *v.autoSubmitBid, err)
			continue
		}

		//Get the amount of time since the last modification to the BundleMD
		modTime := time.Now().UnixNano() - v.autoSubmitLastTouched

		//Get the number of files in this BundleMD
		count, err := b.BundleFileCountGet()
		if err != nil {
			log.Printf("Failed to get file count for bundle id %d, error %v", v.autoSubmitBid, err)
			continue
		}

		common.Dprintf("modTime == %d, bundleModThreshold == %d", modTime, bundleModThreshold)

		//Submit only the autoSubmitBid if its activity has slowed or it has met the maximum number
		//of files threshold.
		if time.Duration(modTime) >= bundleModThreshold ||
			count >= maxNumFilesBundle {
			//Submit...
			err := b.Submit()
			if err != nil {
				log.Printf("Failed to submit bundle, error %v", err)
				continue
			}

			common.Dprintf("Bundle %d submitted!!!", v.autoSubmitBid)

			//Set the userBundles auto bundle to nil so a new one will get created on the next file add.
			self.workingBundlesNeedsRefresh = true
			v.autoSubmitBid = nil
			err = self.stateManager.db.setUserBundles(v)
			if err != nil {
				log.Printf("Failed to set userBundles %+v for, error %v", v, err)
				continue
			}
		}
	}
}
예제 #3
0
func (self *autoManager) refreshWorkingBundlesNoLock(force bool) {
	if force || self.workingBundlesNeedsRefresh {
		common.Dprintf("refreshing workingBundles")
		self.workingBundles = self.stateManager.db.getWorkingBundles()
		self.workingBundlesNeedsRefresh = false
	}
}
예제 #4
0
func (self *autoManager) runAuto() {
	go func() {
		for {
			common.Dprintf("runAuto sleeping for %v", autoTimerFreq)
			t := time.NewTimer(autoTimerFreq)
			<-t.C
			self.scheduleSubmit()
			self.monitorSubmitted()
		}
	}()
}
예제 #5
0
func (self *watcher) processPaths() {
	self.pathMutex.Lock()
	tmpPaths := self.paths
	self.pathMutex.Unlock()
	for _, path := range tmpPaths {
		pathFi, err := os.Stat(path)
		if err != nil {
			common.Dprintf("Could not stat %s, %v", path, err)
			continue
		}

		if !pathFi.IsDir() {
			self.callFound(path, path)
		} else {
			w := func(newPath string, info os.FileInfo, err error) error {
				//FIXME work around broken Go!
				//See issue http://code.google.com/p/go/issues/detail?id=3486
				if info.IsDir() && err == nil {
					f, err := os.Open(newPath)
					if err != nil {
						return filepath.SkipDir
					}
					f.Close()
				}
				if info.IsDir() && err != nil {
					common.Dprintf("%s skipped, error %v\n", newPath, err)
					return filepath.SkipDir
				} else if !info.IsDir() {
					self.callFound(path, newPath)
				}
				return nil
			}
			err = filepath.Walk(path, w)
			if err != nil {
				common.Dprintf("Walk failed %v with %v\n", path, err)
			}
		}
	}
}
func (self *stateDatabase) setFileState(fs *fileState) error {
	if fs == nil {
		return errors.New("fs is nil")
	}

	tmp := self.getFileState(fs.userName, fs.ruleName, fs.fullPath)

	self.mutex.Lock()
	defer self.mutex.Unlock()

	var sql string

	if tmp == nil { //fs is a new fileState in the database.
		//The insert statement relies on AUTOINCREMENT
		if fs.bundleId == nil || fs.bundleFileId == nil {
			sql = fmt.Sprintf(
				"INSERT INTO file_states"+
					"(id, last_modified, digest, full_path, last_seen, bundle_id, bundle_file_id, pass_off, user_name, rule_name, auto_added_to_bundle_time) "+
					"VALUES "+
					"(NULL, %d, \"%s\", \"%s\", %d, NULL, NULL, %d, \"%s\", \"%s\", %d);",
				fs.lastModified, fs.digest, fs.fullPath, fs.lastSeen, fs.passOff, fs.userName, fs.ruleName, fs.autoAddedToBundleTime)
		} else {
			sql = fmt.Sprintf(
				"INSERT INTO file_states"+
					"(id, last_modified, digest, full_path, last_seen, bundle_id, bundle_file_id, pass_off, user_name, rule_name, auto_added_to_bundle_time) "+
					"VALUES "+
					"(NULL, %d, \"%s\", \"%s\", %d, %d, %d, %d, \"%s\", \"%s\", %d);",
				fs.lastModified, fs.digest, fs.fullPath, fs.lastSeen, fs.bundleId, fs.bundleFileId, fs.passOff, fs.userName, fs.ruleName, fs.autoAddedToBundleTime)
		}
	} else { //fs is in the database, updating.
		if fs.bundleId == nil || fs.bundleFileId == nil {
			sql = fmt.Sprintf("UPDATE file_states "+
				"SET last_modified=%d, digest=\"%s\", full_path=\"%s\", last_seen=%d, pass_off=%d, user_name=\"%s\", rule_name=\"%s\", auto_added_to_bundle_time=%d "+
				"WHERE id=%d;",
				fs.lastModified, fs.digest, fs.fullPath, fs.lastSeen, fs.passOff, fs.userName, fs.ruleName, fs.autoAddedToBundleTime, fs.id)
		} else {
			sql = fmt.Sprintf("UPDATE file_states "+
				"SET last_modified=%d, digest=\"%s\", full_path=\"%s\", last_seen=%d, bundle_id=%d, bundle_file_id=%d, pass_off=%d, user_name=\"%s\", rule_name=\"%s\", auto_added_to_bundle_time=%d "+
				"WHERE id=%d;",
				fs.lastModified, fs.digest, fs.fullPath, fs.lastSeen, *fs.bundleId, *fs.bundleFileId, fs.passOff, fs.userName, fs.ruleName, fs.autoAddedToBundleTime, fs.id)
		}
	}

	common.Dprintf("%s", sql)

	return self.conn.Exec(sql)
}
func (self *stateDatabase) setUserBundles(ub *userBundles) error {
	common.Dprintln("Entering setUserBundles")
	defer common.Dprintln("Leaving setUserBundles")

	if ub == nil {
		return errors.New("ub must not be nil")
	}

	self.mutex.Lock()
	defer self.mutex.Unlock()

	var autoSql string
	if ub.autoSubmitBid == nil {
		autoSql = "NULL"
		ub.autoSubmitLastTouched = 0
	} else {
		autoSql = fmt.Sprintf("%d", *ub.autoSubmitBid)
		ub.autoSubmitLastTouched = time.Now().UnixNano()
	}

	var noAutoSql string
	if ub.noAutoSubmitBid == nil {
		noAutoSql = "NULL"
	} else {
		noAutoSql = fmt.Sprintf("%d", *ub.noAutoSubmitBid)
	}

	sql := fmt.Sprintf("INSERT OR REPLACE INTO user_bundles "+
		"(user, auto_submit_bid, no_auto_submit_bid, auto_submit_last_touched) "+
		"VALUES "+
		"(\"%s\", %s, %s, %d);",
		ub.user, autoSql, noAutoSql, ub.autoSubmitLastTouched)

	common.Dprintf("%s", sql)

	err := self.conn.Exec(sql)
	if err != nil {
		return err
	}

	return nil
}
예제 #8
0
//TODO - break this function up into more digestable bits, it's deeply nested and painful to read.
func (self *matcher) found(path string, fullpath string) {
	//TODO - remove...
	//common.Dprintf("Found path: %s fullpath: %s\n", path, fullpath)
	fileinfo, err := os.Lstat(fullpath)
	if err != nil {
		log.Printf("Failed to stat %s\n", fullpath)
		return
	}
	reversePaths := *self.reversePaths
	rlea := reversePaths[path]
	if rlea == nil {
		return
	}
	if len(rlea) < 1 {
		common.Dprintf("No reverse lookup entries found for %s", path)
	}

	//Process each ReverseLookupEntry set for path
	for _, rle := range rlea {
		if rle.Rule != nil && (rle.Prefix == "" || strings.HasPrefix(fullpath, rle.Prefix+string(os.PathSeparator)) || fullpath == rle.Prefix) {
			//Determine if the fullpath matches the exclude pattern for the corresponding WatchRule.
			excluded := false
			for i := 0; i < len(rle.Rule.ExcludePatterns); i++ {
				//FIXME This can perform better by precompiling patterns and putting them in rle. Consider doing this later.
				matched, err := regexp.Match(rle.Rule.ExcludePatterns[i], []byte(fullpath))
				if err != nil || matched == false {
					continue
				}
				excluded = true
			}
			if excluded {
				common.Dprintf("Excluding %s %s %s", rle.User, rle.Rule.Name, fullpath)
				continue
			}

			//Get the fileState for this path
			filestate, err := fsm.getFileState(rle.User, rle.Rule.Name, fullpath)
			if err != nil {
				log.Printf("Failed to get a fileState for %s, %s, %s, error %v",
					rle.User, rle.Rule.Name, fullpath, err)
				continue
			}

			//The file has changed, it should be processed (a.k.a matched)
			if checkFileChanged(fullpath, fileinfo, filestate) {
				ok, err := common.UserAccess(rle.User, fullpath)
				if err != nil {
					log.Printf("User access to %v could not be granted", err)
				}
				if ok {
					groups := make([][2]string, 0)
					for _, sg := range rle.Rule.StaticMetadata {
						groups = append(groups, [2]string{sg.Type, sg.Name})
					}

					//TODO - NGT 4/25/12 This is difficult to read.  It looks like it is both determining if the file matches
					//and generating groups.  A great candidate for refactoring into one or two functions...
					//Aha, after reading more closely it appears the groups list is being "appended" (or potentially replaced) with additional
					//type/name pairs based on groups that were passed through regular expressions in the WatchRules.
					//If so, this needs to be clearly explained in comments.  Also, we need to document in the UI somewhere the
					//regular expressions format allowed by the regex engine (even if it's just a link) or a couple canned examples...
					//This code is nothing short of magic :)
					for _, rule := range rle.Rule.MetadataPattern {
						common.Dprintf("Got pattern %v\n", rule.Pattern)
						//FIXME This can perform better by precompiling patterns and putting them in rle. Consider doing this later.
						re, err := regexp.Compile(rule.Pattern)
						if err != nil {
							break
						}
						groupMap := make(map[string]string)
						for _, x := range rule.Group {
							groupMap[x.Pattern] = x.Value
						}
						subexpnames := re.SubexpNames()
						patternMapList := []patternMapEntry{}
						for id, x := range subexpnames {
							if x != "" && groupMap[x] != "" {
								common.Dprintf("Subexp: %v %v\n", x, groupMap[x])
								patternMapList = append(patternMapList, patternMapEntry{id, groupMap[x]})
							}
						}
						match := re.FindStringSubmatch(fullpath)
						if match != nil {
							common.Dprintf("%s matched.\n", fullpath)
							for _, x := range patternMapList {
								common.Dprintf("Map: %v %v %v\n", x.value, match[x.id], fullpath)
								groups = append(groups, [2]string{x.value, match[x.id]})
							}
						}
					}

					//TODO - another block that is a good candidate for refactor into a function
					//Generate the new file name based on the rename patterns
					var newfilename string = fullpath
					for _, rule := range rle.Rule.RenamePatterns {
						re, err := regexp.Compile(rule.Pattern)
						if err != nil {
							break
						}
						match := re.FindStringSubmatchIndex(fullpath)
						if match != nil {
							n := re.ExpandString([]byte{}, rule.Value, newfilename, match)
							if n != nil {
								common.Dprintf("Renaming %s %s\n", newfilename, string(n))
								newfilename = string(n)
							}
						}
					}
					var tmpprefix string
					if rle.Prefix == "" {
						tmpprefix = path
					} else {
						tmpprefix = rle.Prefix
					}
					tmpprefix += string(os.PathSeparator)
					if strings.HasPrefix(newfilename, tmpprefix) {
						newfilename = newfilename[len(tmpprefix):]
					}
					newfilename = strings.Replace(newfilename, string(os.PathSeparator), "/", -1)

					am.addAutoFile(*filestate, newfilename, groups)
				}
			} else {
				common.Dprintf("checkFileChanged(%s, %+v, %+v) returned false skipping...", fullpath, fileinfo, filestate)
			}
		} else {
			common.Dprintf("reverse lookup entry %+v was skipped for processing", rle)
		}
	}
}
예제 #9
0
//TODO - break this function up into more digestable bits, it's deeply nested and painful to read.
func (self *matcher) updatePaths(allConfigs *map[string]*UserConfig) {
	reversePaths := map[string][]ReverseLookupEntry{}
	paths := map[string]bool{}
	for user, uc := range *allConfigs {
		for _, rule := range uc.Rules {
			for _, path := range rule.Paths {
				paths[path] = true
				if reversePaths[path] == nil {
					reversePaths[path] = make([]ReverseLookupEntry, 0)
				}
				reversePaths[path] = append(reversePaths[path], ReverseLookupEntry{User: user, Rule: rule})
			}
		}
	}
	var newpaths filePathSlice
	for key, _ := range paths {
		newpaths = append(newpaths, key)
	}
	newpaths.Sort()
	var valid []bool = make([]bool, len(newpaths))
	for i := 0; i < len(newpaths); i++ {
		valid[i] = true
	}
	for i := 0; i < len(newpaths); i++ {
		path := newpaths[i]
		common.Dprintf("newpath - %d %s\n", i, path)
		for j := i + 1; j < len(newpaths); j++ {
			if valid[j] && strings.HasPrefix(newpaths[j], path+string(os.PathSeparator)) {
				common.Dprintf("Collapsable - %s\n", newpaths[j])
				rlea := reversePaths[newpaths[j]]
				if rlea != nil {
					for _, rle := range rlea {
						rle.Prefix = newpaths[j]
						reversePaths[path] = append(reversePaths[path], rle)
					}
				}
				valid[j] = false
			}
			if valid[j] {
				common.Dprintf("cmpnewpath - %s %s %t\n", path, newpaths[j], strings.HasPrefix(newpaths[j], path+string(os.PathSeparator)))
			}
		}
	}
	var tmpnewpaths filePathSlice
	tmpReversePaths := map[string][]ReverseLookupEntry{}
	for i := 0; i < len(newpaths); i++ {
		if valid[i] {
			tmpnewpaths = append(tmpnewpaths, newpaths[i])
			tmpReversePaths[newpaths[i]] = reversePaths[newpaths[i]]
		}
	}
	newpaths = tmpnewpaths
	reversePaths = tmpReversePaths
	for path, rlea := range reversePaths {
		if rlea != nil {
			for _, rle := range rlea {
				if rle.Rule != nil {
					common.Dprintf("rle - %s %s %s %s\n", path, rle.User, rle.Rule.Name, rle.Prefix)
				}
			}
		}
	}
	if w != nil {
		w.updatePaths([]string(newpaths))
	}
	common.Dprintf("Unique paths: %v\n", newpaths)
	self.reversePaths = &reversePaths
}