Beispiel #1
0
// ScanMulti runs multiple scanner tools in parallel. It passes command-line
// options from opt to each one, and it sends the JSON representation of cfg
// (the repo/tree's Config) to each tool's stdin.
func ScanMulti(scanners [][]string, opt Options, treeConfig map[string]interface{}) ([]*unit.SourceUnit, error) {
	if treeConfig == nil {
		treeConfig = map[string]interface{}{}
	}

	var (
		units []*unit.SourceUnit
		mu    sync.Mutex
	)

	run := parallel.NewRun(runtime.GOMAXPROCS(0))
	for _, scanner_ := range scanners {
		scanner := scanner_
		run.Acquire()
		go func() {
			defer run.Release()

			units2, err := Scan(scanner, opt, treeConfig)
			if err != nil {
				run.Error(fmt.Errorf("scanner %v: %s", scanner, err))
				return
			}

			mu.Lock()
			defer mu.Unlock()
			units = append(units, units2...)
		}()
	}
	err := run.Wait()
	// Return error only if none of the commands succeeded.
	if len(units) == 0 {
		return nil, err
	}
	return units, nil
}
Beispiel #2
0
// refsAtOffsets reads the refs at the given serialized byte offsets
// from the ref data file and returns them in arbitrary order.
func (s *fsUnitStore) refsAtOffsets(ofs byteOffsets, fs []RefFilter) (refs []*graph.Ref, err error) {
	vlog.Printf("%s: reading refs at %d offsets with filters %v...", s, len(ofs), fs)
	f, err := openFetcherOrOpen(s.fs, unitRefsFilename)
	if err != nil {
		return nil, err
	}
	defer func() {
		err2 := f.Close()
		if err == nil {
			err = err2
		}
	}()

	ffs := refFilters(fs)

	p := parFetches(s.fs, fs)
	if p == 0 {
		return nil, nil
	}

	var refsLock sync.Mutex
	par := parallel.NewRun(p)
	for _, ofs_ := range ofs {
		ofs := ofs_
		par.Acquire()
		go func() {
			defer par.Release()

			if _, moreOK := LimitRemaining(fs); !moreOK {
				return
			}

			// Guess how many bytes this ref is. The s3vfs (if that's the
			// VFS impl in use) will autofetch beyond that if needed.
			const byteEstimate = decodeBufSize
			r, err := rangeReader(s.fs, unitRefsFilename, f, ofs, byteEstimate)
			if err != nil {
				par.Error(err)
				return
			}
			dec := Codec.NewDecoder(r)
			var ref graph.Ref
			if _, err := dec.Decode(&ref); err != nil {
				par.Error(err)
				return
			}
			if ffs.SelectRef(&ref) {
				refsLock.Lock()
				refs = append(refs, &ref)
				refsLock.Unlock()
			}
		}()
	}
	if err := par.Wait(); err != nil {
		return refs, err
	}
	sort.Sort(refsByFileStartEnd(refs))
	vlog.Printf("%s: read %v refs at %d offsets with filters %v.", s, len(refs), len(ofs), fs)
	return refs, nil
}
Beispiel #3
0
// BuildIndexes builds all indexes on store and its lower-level stores
// that match the specified criteria. It returns the status of each
// index that was built (or rebuilt).
func BuildIndexes(store interface{}, c IndexCriteria, indexChan chan<- IndexStatus) ([]IndexStatus, error) {
	var built []IndexStatus
	var builtMu sync.Mutex
	indexChan2 := make(chan IndexStatus)
	done := make(chan struct{})
	go func() {
		var par *parallel.Run
		lastDependsOnChildren := false
		for sx := range indexChan2 {
			doBuild := func(sx IndexStatus) {
				start := time.Now()
				err := sx.store.BuildIndex(sx.Name, sx.index)
				sx.BuildDuration = time.Since(start)
				if err == nil {
					sx.Stale = false
				} else {
					sx.BuildError = err.Error()
				}
				builtMu.Lock()
				built = append(built, sx)
				builtMu.Unlock()
				if indexChan != nil {
					indexChan <- sx
				}
			}

			// Run indexes in parallel, but if we
			// encounter an index that depends on children, wait for
			// all previously seen indexes to finish before building
			// those indexes.
			if sx.DependsOnChildren != lastDependsOnChildren && par != nil {
				par.Wait()
				par = nil
			}
			if par == nil {
				par = parallel.NewRun(MaxIndexParallel)
			}
			sx_ := sx
			par.Acquire()
			go func() {
				defer par.Release()
				doBuild(sx_)
			}()

			lastDependsOnChildren = sx.DependsOnChildren
		}
		if par != nil {
			par.Wait()
		}
		done <- struct{}{}
	}()
	err := listIndexes(store, c, indexChan2, nil)
	close(indexChan2)
	<-done
	return built, err
}
Beispiel #4
0
func (s unitStores) Refs(f ...RefFilter) ([]*graph.Ref, error) {
	uss, err := openUnitStores(s.opener, f)
	if err != nil {
		return nil, err
	}

	c_unitStores_Refs_last_numUnitsQueried.set(0)
	var (
		allRefsMu sync.Mutex
		allRefs   []*graph.Ref
	)
	par := parallel.NewRun(storeFetchPar)
	for u, us := range uss {
		if us == nil {
			continue
		}
		u, us := u, us

		c_unitStores_Refs_last_numUnitsQueried.increment()

		par.Acquire()
		go func() {
			defer par.Release()
			if _, moreOK := LimitRemaining(f); !moreOK {
				return
			}
			fCopy := filtersForUnit(u, f).([]RefFilter)
			fCopy = withImpliedUnit(fCopy, u)

			refs, err := us.Refs(fCopy...)
			if err != nil && !isStoreNotExist(err) {
				par.Error(err)
				return
			}
			for _, ref := range refs {
				ref.UnitType = u.Type
				ref.Unit = u.Name
				if ref.DefUnitType == "" {
					ref.DefUnitType = u.Type
				}
				if ref.DefUnit == "" {
					ref.DefUnit = u.Name
				}
			}

			allRefsMu.Lock()
			allRefs = append(allRefs, refs...)
			allRefsMu.Unlock()
		}()
	}
	err = par.Wait()
	return allRefs, err
}
Beispiel #5
0
// ReadCached reads a Tree's configuration from all of its source unit
// definition files (which may either be in a local VFS rooted at a
// .srclib-cache/<COMMITID> dir, or a remote VFS). It does not read
// the Srcfile; the Srcfile's directives are already accounted for in
// the cached source unit definition files.
//
// bdfs should be a VFS obtained from a call to
// (buildstore.RepoBuildStore).Commit.
func ReadCached(bdfs vfs.FileSystem) (*Tree, error) {
	if _, err := bdfs.Lstat("."); os.IsNotExist(err) {
		return nil, fmt.Errorf("build cache dir does not exist (did you run `srclib config` to create it)?")
	} else if err != nil {
		return nil, err
	}

	// Collect all **/*.unit.json files.
	var unitFiles []string
	unitSuffix := buildstore.DataTypeSuffix(unit.SourceUnit{})
	w := fs.WalkFS(".", rwvfs.Walkable(rwvfs.ReadOnly(bdfs)))
	for w.Step() {
		if err := w.Err(); err != nil {
			return nil, err
		}
		if path := w.Path(); strings.HasSuffix(path, unitSuffix) {
			unitFiles = append(unitFiles, path)
		}
	}

	// Parse units
	sort.Strings(unitFiles)
	units := make([]*unit.SourceUnit, len(unitFiles))
	par := parallel.NewRun(runtime.GOMAXPROCS(0))
	for i_, unitFile_ := range unitFiles {
		i, unitFile := i_, unitFile_
		par.Acquire()
		go func() {
			defer par.Release()
			f, err := bdfs.Open(unitFile)
			if err != nil {
				par.Error(err)
				return
			}
			if err := json.NewDecoder(f).Decode(&units[i]); err != nil {
				f.Close()
				par.Error(err)
				return
			}
			if err := f.Close(); err != nil {
				par.Error(err)
				return
			}
		}()
	}
	if err := par.Wait(); err != nil {
		return nil, err
	}
	return &Tree{SourceUnits: units}, nil
}
Beispiel #6
0
// RemoveAll removes a tree recursively.
func RemoveAll(path string, vfs rwvfs.WalkableFileSystem) error {
	w := fs.WalkFS(path, vfs)

	remove := func(par *parallel.Run, path string) {
		par.Acquire()
		go func() {
			defer par.Release()
			if err := vfs.Remove(path); err != nil {
				par.Error(err)
			}
		}()
	}

	var dirs []string // remove dirs after removing all files
	filesPar := parallel.NewRun(20)
	for w.Step() {
		if err := w.Err(); err != nil {
			return err
		}
		if w.Stat().IsDir() {
			dirs = append(dirs, w.Path())
		} else {
			remove(filesPar, w.Path())
		}
	}

	if err := filesPar.Wait(); err != nil {
		return err
	}

	dirsPar := parallel.NewRun(20)
	sort.Sort(sort.Reverse(sort.StringSlice(dirs))) // reverse so we delete leaf dirs first
	for _, dir := range dirs {
		remove(dirsPar, dir)
	}
	return dirsPar.Wait()
}
Beispiel #7
0
func brokenRefsOnly(refs []*graph.Ref, s interface{}) ([]*graph.Ref, error) {
	uniqRefDefs := map[graph.DefKey][]*graph.Ref{}
	loggedDefRepos := map[string]struct{}{}
	for _, ref := range refs {
		if ref.Repo != ref.DefRepo {
			if _, logged := loggedDefRepos[ref.DefRepo]; !logged {
				// TODO(sqs): need to skip these because we don't know the
				// "DefCommitID" in the def's repo, and ByDefKey requires
				// the key to have a CommitID.
				log.Printf("WARNING: Can't check resolution of cross-repo ref (ref.Repo=%q != ref.DefRepo=%q) - cross-repo ref checking is not yet implemented. (This log message will not be repeated.)", ref.Repo, ref.DefRepo)
				loggedDefRepos[ref.DefRepo] = struct{}{}
			}
			continue
		}
		def := ref.DefKey()
		def.CommitID = ref.CommitID
		uniqRefDefs[def] = append(uniqRefDefs[def], ref)
	}

	var (
		brokenRefs  []*graph.Ref
		brokenRefMu sync.Mutex
		par         = parallel.NewRun(runtime.GOMAXPROCS(0))
	)
	for def_, refs_ := range uniqRefDefs {
		def, refs := def_, refs_
		par.Acquire()
		go func() {
			defer par.Release()
			defs, err := s.(store.RepoStore).Defs(store.ByDefKey(def))
			if err != nil {
				par.Error(err)
				return
			}
			if len(defs) == 0 {
				brokenRefMu.Lock()
				brokenRefs = append(brokenRefs, refs...)
				brokenRefMu.Unlock()
			}
		}()
	}
	err := par.Wait()
	sort.Sort(graph.Refs(brokenRefs))
	return brokenRefs, err
}
Beispiel #8
0
func (s unitStores) Defs(fs ...DefFilter) ([]*graph.Def, error) {
	uss, err := openUnitStores(s.opener, fs)
	if err != nil {
		return nil, err
	}

	var (
		allDefs   []*graph.Def
		allDefsMu sync.Mutex
	)
	par := parallel.NewRun(storeFetchPar)
	for u_, us_ := range uss {
		u, us := u_, us_
		if us == nil {
			continue
		}

		par.Acquire()
		go func() {
			defer par.Release()
			defs, err := us.Defs(filtersForUnit(u, fs).([]DefFilter)...)
			if err != nil && !isStoreNotExist(err) {
				par.Error(err)
				return
			}
			for _, def := range defs {
				def.UnitType = u.Type
				def.Unit = u.Name
			}
			allDefsMu.Lock()
			allDefs = append(allDefs, defs...)
			allDefsMu.Unlock()
		}()
	}
	err = par.Wait()
	return allDefs, err
}
Beispiel #9
0
func (s repoStores) Defs(f ...DefFilter) ([]*graph.Def, error) {
	rss, err := openRepoStores(s.opener, f)
	if err != nil {
		return nil, err
	}

	var (
		allDefs   []*graph.Def
		allDefsMu sync.Mutex
	)
	par := parallel.NewRun(storeFetchPar)
	for repo_, rs_ := range rss {
		repo, rs := repo_, rs_
		if rs == nil {
			continue
		}

		par.Acquire()
		go func() {
			defer par.Release()
			defs, err := rs.Defs(filtersForRepo(repo, f).([]DefFilter)...)
			if err != nil && !isStoreNotExist(err) {
				par.Error(err)
				return
			}
			for _, def := range defs {
				def.Repo = repo
			}
			allDefsMu.Lock()
			allDefs = append(allDefs, defs...)
			allDefsMu.Unlock()
		}()
	}
	err = par.Wait()
	return allDefs, err
}
Beispiel #10
0
func (s repoStores) Units(f ...UnitFilter) ([]*unit.SourceUnit, error) {
	rss, err := openRepoStores(s.opener, f)
	if err != nil {
		return nil, err
	}

	var (
		allUnits   []*unit.SourceUnit
		allUnitsMu sync.Mutex
	)
	par := parallel.NewRun(storeFetchPar)
	for repo_, rs_ := range rss {
		repo, rs := repo_, rs_
		if rs == nil {
			continue
		}

		par.Acquire()
		go func() {
			defer par.Release()
			units, err := rs.Units(filtersForRepo(repo, f).([]UnitFilter)...)
			if err != nil && !isStoreNotExist(err) {
				par.Error(err)
				return
			}
			for _, unit := range units {
				unit.Repo = repo
			}
			allUnitsMu.Lock()
			allUnits = append(allUnits, units...)
			allUnitsMu.Unlock()
		}()
	}
	err = par.Wait()
	return allUnits, err
}
Beispiel #11
0
// refsAtByteRanges reads the refs at the given serialized byte ranges
// from the ref data file and returns them in arbitrary order.
func (s *fsUnitStore) refsAtByteRanges(brs []byteRanges, fs []RefFilter) (refs []*graph.Ref, err error) {
	vlog.Printf("%s: reading refs at %d byte ranges with filters %v...", s, len(brs), fs)
	f, err := openFetcherOrOpen(s.fs, unitRefsFilename)
	if err != nil {
		return nil, err
	}
	defer func() {
		err2 := f.Close()
		if err == nil {
			err = err2
		}
	}()

	ffs := refFilters(fs)

	p := parFetches(s.fs, fs)
	if p == 0 {
		return nil, nil
	}

	// See how many bytes we need to read to get the refs in all
	// byteRanges.
	readLengths := make([]int64, len(brs))
	totalRefs := 0
	for i, br := range brs {
		var n int64
		for _, b := range br[1:] {
			n += b
			totalRefs++
		}
		readLengths[i] = n
	}

	var refsLock sync.Mutex
	par := parallel.NewRun(p)
	for i_, br_ := range brs {
		i, br := i_, br_
		par.Acquire()
		go func() {
			defer par.Release()

			if _, moreOK := LimitRemaining(fs); !moreOK {
				return
			}

			r, err := rangeReader(s.fs, unitRefsFilename, f, br.start(), readLengths[i])
			if err != nil {
				par.Error(err)
				return
			}
			dec := Codec.NewDecoder(r)
			for range br[1:] {
				var ref graph.Ref
				if _, err := dec.Decode(&ref); err != nil {
					par.Error(err)
					return
				}
				if ffs.SelectRef(&ref) {
					refsLock.Lock()
					refs = append(refs, &ref)
					refsLock.Unlock()
				}
			}
		}()
	}
	if err := par.Wait(); err != nil {
		return refs, err
	}
	sort.Sort(refsByFileStartEnd(refs))
	vlog.Printf("%s: read %d refs at %d byte ranges with filters %v.", s, len(refs), len(brs), fs)
	return refs, nil
}
Beispiel #12
0
// listIndexes lists indexes in s (a store) asynchronously, sending
// status objects to ch. If f != nil, it is called to set/modify
// fields on each status object before the IndexStatus object is sent to
// the channel.
func listIndexes(s interface{}, c IndexCriteria, ch chan<- IndexStatus, f func(*IndexStatus)) error {
	switch s := s.(type) {
	case indexedStore:
		xx := s.Indexes()
		var waitingOnChildren []IndexStatus
		for name, x := range xx {
			st := IndexStatus{
				Name:  name,
				Type:  strings.TrimPrefix(reflect.TypeOf(x).String(), "*store."),
				index: x,
				store: s,
			}

			if !strings.Contains(st.Name, c.Name) {
				continue
			}
			if !strings.Contains(st.Type, c.Type) {
				continue
			}

			fi, err := s.statIndex(name)
			if os.IsNotExist(err) {
				st.Stale = true
			} else if err != nil {
				st.Error = err.Error()
			} else {
				st.Size = fi.Size()
			}

			switch x.(type) {
			case unitRefIndexBuilder, defQueryTreeIndexBuilder:
				st.DependsOnChildren = true
			}

			if c.Stale != nil && st.Stale != *c.Stale {
				continue
			}

			if f != nil {
				f(&st)
			}

			if c.Unit != nil && c.Unit != NoSourceUnit {
				if st.Unit == nil || *c.Unit != *st.Unit {
					continue
				}
			}

			if st.DependsOnChildren {
				waitingOnChildren = append(waitingOnChildren, st)
			} else {
				ch <- st
			}
		}

		switch s := s.(type) {
		case *indexedTreeStore:
			if err := listIndexes(s.fsTreeStore, c, ch, f); err != nil {
				return err
			}
		case *indexedUnitStore:
			if err := listIndexes(s.fsUnitStore, c, ch, f); err != nil {
				return err
			}
		}

		for _, si := range waitingOnChildren {
			ch <- si
		}

	case repoStoreOpener:
		var rss map[string]RepoStore
		if c.Repo == "" {
			var err error
			rss, err = s.openAllRepoStores()
			if err != nil && !isStoreNotExist(err) {
				return err
			}
		} else {
			rss = map[string]RepoStore{c.Repo: s.openRepoStore(c.Repo)}
		}

		// Sort repos for determinism.
		repos := make([]string, 0, len(rss))
		for repo := range rss {
			repos = append(repos, repo)
		}
		sort.Strings(repos)

		if c.ReposOffset != 0 {
			if c.ReposOffset < len(repos) {
				repos = repos[c.ReposOffset:]
			} else {
				log.Printf("Warning: A ReposOffset (%d) was specified that equals or exceeds the total number of repos (%d).", c.ReposOffset, len(repos))
			}
		}
		if c.ReposLimit != 0 && c.ReposLimit < len(repos) {
			repos = repos[:c.ReposLimit]
		}

		for _, repo := range repos {
			rs := rss[repo]
			err := listIndexes(rs, c, ch, func(x *IndexStatus) {
				x.Repo = repo
				if f != nil {
					f(x)
				}
			})
			if err != nil {
				return err
			}
		}

	case treeStoreOpener:
		var tss map[string]TreeStore
		if c.CommitID == "" {
			var err error
			tss, err = s.openAllTreeStores()
			if err != nil && !isStoreNotExist(err) {
				return err
			}
		} else {
			tss = map[string]TreeStore{c.CommitID: s.openTreeStore(c.CommitID)}
		}
		for commitID, ts := range tss {
			err := listIndexes(ts, c, ch, func(x *IndexStatus) {
				x.CommitID = commitID
				if f != nil {
					f(x)
				}
			})
			if err != nil {
				return err
			}
		}

	case unitStoreOpener:
		if c.Unit == NoSourceUnit {
			return nil
		}
		var uss map[unit.ID2]UnitStore
		if c.Unit == nil {
			var err error
			uss, err = s.openAllUnitStores()
			if err != nil && !isStoreNotExist(err) {
				return err
			}
		} else {
			uss = map[unit.ID2]UnitStore{*c.Unit: s.openUnitStore(*c.Unit)}
		}
		if len(uss) > 0 {
			par := parallel.NewRun(MaxIndexParallel)
			for unit_, us_ := range uss {
				unit, us := unit_, us_
				par.Acquire()
				go func() {
					defer par.Release()
					unitCopy := unit
					if err := listIndexes(us, c, ch, func(x *IndexStatus) {
						x.Unit = &unitCopy
						if f != nil {
							f(x)
						}
					}); err != nil {
						par.Error(err)
					}
				}()
			}
			if err := par.Wait(); err != nil {
				return err
			}
		}

	}
	return nil
}
Beispiel #13
0
func (c *SimpleRepoCmd) Execute(args []string) error {
	if err := c.validate(); err != nil {
		return err
	}

	if err := removeGlob(".srclib-*"); err != nil {
		return err
	}

	units := make([]*unit.SourceUnit, 0)
	unitNames := hierarchicalNames("u", "unit", "", c.NUnits)
	for _, unitName := range unitNames {
		units = append(units, &unit.SourceUnit{
			Key: unit.Key{
				Name:     unitName,
				Type:     "GoPackage",
				Repo:     c.Repo,
				CommitID: c.CommitID,
			},
			Info: unit.Info{
				Files: []string{},
				Dir:   unitName,
			},
		})
	}

	if c.GenSource {
		if err := resetSource(); err != nil {
			return err
		}

		// generate source files
		par := parallel.NewRun(runtime.GOMAXPROCS(0))
		for _, ut_ := range units {
			ut := ut_
			par.Acquire()
			go func() {
				defer par.Release()
				if err := c.genUnit(ut); err != nil {
					par.Error(err)
				}
			}()
		}
		if err := par.Wait(); err != nil {
			return err
		}

		// get commit ID
		commitID, err := getGitCommitID()
		if err != nil {
			return err
		}

		// update command to generate graph data
		c.CommitID = commitID
		c.GenSource = false
	}

	// generate graph data
	par := parallel.NewRun(runtime.GOMAXPROCS(0))
	for _, ut_ := range units {
		ut := ut_
		ut.CommitID = c.CommitID
		par.Acquire()
		go func() {
			defer par.Release()
			if err := c.genUnit(ut); err != nil {
				par.Error(err)
			}
		}()
	}
	if err := par.Wait(); err != nil {
		return err
	}

	return nil
}
Beispiel #14
0
func (s *indexedTreeStore) buildIndexes(xs map[string]Index, units []*unit.SourceUnit, unitRefIndexes map[unit.ID2]*defRefsIndex, unitDefQueryIndexes map[unit.ID2]*defQueryIndex) error {
	// TODO(sqs): there's a race condition here if multiple imports
	// are running concurrently, they could clobber each other's
	// indexes. (S3 is eventually consistent.)

	var getUnitsErr error
	var getUnitsOnce sync.Once
	getUnits := func() ([]*unit.SourceUnit, error) {
		getUnitsOnce.Do(func() {
			if getUnitsErr == nil && units == nil {
				units, getUnitsErr = s.fsTreeStore.Units()
			}
			if units == nil {
				units = []*unit.SourceUnit{}
			}
		})
		return units, getUnitsErr
	}

	var getUnitRefIndexesErr error
	var getUnitRefIndexesOnce sync.Once
	var unitRefIndexesLock sync.Mutex
	getUnitRefIndexes := func() (map[unit.ID2]*defRefsIndex, error) {
		getUnitRefIndexesOnce.Do(func() {
			if getUnitRefIndexesErr == nil && unitRefIndexes == nil {
				// Read in the defRefsIndex for all source units.
				units, err := getUnits()
				if err != nil {
					getUnitRefIndexesErr = err
					return
				}

				// Use openUnitStore on the list from getUnits so we
				// don't need to traverse the FS tree to enumerate all
				// the source units again (which is slow).
				uss := make(map[unit.ID2]UnitStore, len(units))
				for _, u := range units {
					uss[u.ID2()] = s.fsTreeStore.openUnitStore(u.ID2())
				}

				unitRefIndexes = make(map[unit.ID2]*defRefsIndex, len(units))
				par := parallel.NewRun(runtime.GOMAXPROCS(0))
				for u_, us_ := range uss {
					u := u_
					us, ok := us_.(*indexedUnitStore)
					if !ok {
						continue
					}

					par.Acquire()
					go func() {
						defer par.Release()
						x := us.indexes[defToRefsIndexName]
						if err := prepareIndex(us.fs, defToRefsIndexName, x); err != nil {
							par.Error(err)
							return
						}
						unitRefIndexesLock.Lock()
						defer unitRefIndexesLock.Unlock()
						unitRefIndexes[u] = x.(*defRefsIndex)
					}()
				}
				getUnitRefIndexesErr = par.Wait()
			}
			if unitRefIndexes == nil {
				unitRefIndexes = map[unit.ID2]*defRefsIndex{}
			}
		})
		return unitRefIndexes, getUnitRefIndexesErr
	}

	var getUnitDefQueryIndexesErr error
	var getUnitDefQueryIndexesOnce sync.Once
	var unitDefQueryIndexesLock sync.Mutex
	getUnitDefQueryIndexes := func() (map[unit.ID2]*defQueryIndex, error) {
		getUnitDefQueryIndexesOnce.Do(func() {
			if getUnitDefQueryIndexesErr == nil && unitDefQueryIndexes == nil {
				// Read in the defQueryIndex for all source units.
				units, err := getUnits()
				if err != nil {
					getUnitDefQueryIndexesErr = err
					return
				}

				// Use openUnitStore on the list from getUnits so we
				// don't need to traverse the FS tree to enumerate all
				// the source units again (which is slow).
				uss := make(map[unit.ID2]UnitStore, len(units))
				for _, u := range units {
					uss[u.ID2()] = s.fsTreeStore.openUnitStore(u.ID2())
				}

				unitDefQueryIndexes = make(map[unit.ID2]*defQueryIndex, len(units))
				par := parallel.NewRun(runtime.GOMAXPROCS(0))
				for u_, us_ := range uss {
					u := u_
					us, ok := us_.(*indexedUnitStore)
					if !ok {
						continue
					}

					par.Acquire()
					go func() {
						defer par.Release()
						x := us.indexes[defQueryIndexName]
						if err := prepareIndex(us.fs, defQueryIndexName, x); err != nil {
							par.Error(err)
							return
						}
						unitDefQueryIndexesLock.Lock()
						defer unitDefQueryIndexesLock.Unlock()
						unitDefQueryIndexes[u] = x.(*defQueryIndex)
					}()
				}
				getUnitDefQueryIndexesErr = par.Wait()
			}
			if unitDefQueryIndexes == nil {
				unitDefQueryIndexes = map[unit.ID2]*defQueryIndex{}
			}
		})
		return unitDefQueryIndexes, getUnitDefQueryIndexesErr
	}

	par := parallel.NewRun(runtime.GOMAXPROCS(0))
	for name_, x_ := range xs {
		name, x := name_, x_
		par.Acquire()
		go func() {
			defer par.Release()
			switch x := x.(type) {
			case unitIndexBuilder:
				units, err := getUnits()
				if err != nil {
					par.Error(err)
					return
				}
				if err := x.Build(units); err != nil {
					par.Error(err)
					return
				}
			case unitRefIndexBuilder:
				unitRefIndexes, err := getUnitRefIndexes()
				if err != nil {
					par.Error(err)
					return
				}
				if err := x.Build(unitRefIndexes); err != nil {
					par.Error(err)
					return
				}
			case defQueryTreeIndexBuilder:
				unitDefQueryIndexes, err := getUnitDefQueryIndexes()
				if err != nil {
					par.Error(err)
					return
				}
				if err := x.Build(unitDefQueryIndexes); err != nil {
					par.Error(err)
					return
				}
			default:
				par.Error(fmt.Errorf("don't know how to build index %q of type %T", name, x))
				return
			}
			if x, ok := x.(persistedIndex); ok {
				if err := writeIndex(s.fs, name, x); err != nil {
					par.Error(err)
					return
				}
			}
		}()
	}
	return par.Wait()
}
Beispiel #15
0
// Import imports build data into a RepoStore or MultiRepoStore.
func Import(buildDataFS vfs.FileSystem, stor interface{}, opt ImportOpt) error {
	// Traverse the build data directory for this repo and commit to
	// create the makefile that lists the targets (which are the data
	// files we will import).
	treeConfig, err := config.ReadCached(buildDataFS)
	if err != nil {
		return fmt.Errorf("error calling config.ReadCached: %s", err)
	}
	mf, err := plan.CreateMakefile(".", nil, "", treeConfig)
	if err != nil {
		return fmt.Errorf("error calling plan.Makefile: %s", err)
	}

	// hasIndexableData is set if at least one source unit's graph data is
	// successfully imported to the graph store.
	//
	// This flag is set concurrently in calls to importGraphData, but it doesn't
	// need to be protected by a mutex since its value is only modified in one
	// direction (false to true), and it is only read in the sequential section
	// after parallel.NewRun completes.
	//
	// However, we still protect it with a mutex to avoid data race errors from the
	// Go race detector.
	var (
		mu               sync.Mutex
		hasIndexableData bool
	)

	importGraphData := func(graphFile string, sourceUnit *unit.SourceUnit) error {
		var data graph.Output
		if err := readJSONFileFS(buildDataFS, graphFile, &data); err != nil {
			if err == errEmptyJSONFile {
				log.Printf("Warning: the JSON file is empty for unit %s %s.", sourceUnit.Type, sourceUnit.Name)
				return nil
			}
			if os.IsNotExist(err) {
				log.Printf("Warning: no build data for unit %s %s.", sourceUnit.Type, sourceUnit.Name)
				return nil
			}
			return fmt.Errorf("error reading JSON file %s for unit %s %s: %s", graphFile, sourceUnit.Type, sourceUnit.Name, err)
		}
		if opt.DryRun || GlobalOpt.Verbose {
			log.Printf("# Importing graph data (%d defs, %d refs, %d docs, %d anns) for unit %s %s", len(data.Defs), len(data.Refs), len(data.Docs), len(data.Anns), sourceUnit.Type, sourceUnit.Name)
			if opt.DryRun {
				return nil
			}
		}

		// HACK: Transfer docs to [def].Docs.
		docsByPath := make(map[string]*graph.Doc, len(data.Docs))
		for _, doc := range data.Docs {
			docsByPath[doc.Path] = doc
		}
		for _, def := range data.Defs {
			if doc, present := docsByPath[def.Path]; present {
				def.Docs = append(def.Docs, &graph.DefDoc{Format: doc.Format, Data: doc.Data})
			}
		}

		switch imp := stor.(type) {
		case store.RepoImporter:
			if err := imp.Import(opt.CommitID, sourceUnit, data); err != nil {
				return fmt.Errorf("error running store.RepoImporter.Import: %s", err)
			}
		case store.MultiRepoImporter:
			if err := imp.Import(opt.Repo, opt.CommitID, sourceUnit, data); err != nil {
				return fmt.Errorf("error running store.MultiRepoImporter.Import: %s", err)
			}
		default:
			return fmt.Errorf("store (type %T) does not implement importing", stor)
		}

		mu.Lock()
		hasIndexableData = true
		mu.Unlock()

		return nil
	}

	par := parallel.NewRun(10)
	for _, rule_ := range mf.Rules {
		rule := rule_
		switch rule := rule.(type) {
		case *grapher.GraphUnitRule:
			if (opt.Unit != "" && rule.Unit.Name != opt.Unit) || (opt.UnitType != "" && rule.Unit.Type != opt.UnitType) {
				continue
			}
			par.Acquire()
			go func() {
				defer par.Release()
				if err := importGraphData(rule.Target(), rule.Unit); err != nil {
					par.Error(err)
				}
			}()
		case *grapher.GraphMultiUnitsRule:
			for target_, sourceUnit_ := range rule.Targets() {
				target, sourceUnit := target_, sourceUnit_
				if (opt.Unit != "" && sourceUnit.Name != opt.Unit) || (opt.UnitType != "" && sourceUnit.Type != opt.UnitType) {
					continue
				}
				par.Acquire()
				go func() {
					defer par.Release()
					if err := importGraphData(target, sourceUnit); err != nil {
						par.Error(err)
					}
				}()
			}
		}
	}
	if err := par.Wait(); err != nil {
		return err
	}

	if hasIndexableData && !opt.NoIndex {
		if GlobalOpt.Verbose {
			log.Printf("# Building indexes")
		}
		switch s := stor.(type) {
		case store.RepoIndexer:
			if err := s.Index(opt.CommitID); err != nil {
				return fmt.Errorf("Error indexing commit %s: %s", opt.CommitID, err)
			}
		case store.MultiRepoIndexer:
			if err := s.Index(opt.Repo, opt.CommitID); err != nil {
				return fmt.Errorf("error indexing %s@%s: %s", opt.Repo, opt.CommitID, err)
			}
		}
	}

	switch imp := stor.(type) {
	case store.RepoImporter:
		if err := imp.CreateVersion(opt.CommitID); err != nil {
			return fmt.Errorf("error running store.RepoImporter.CreateVersion: %s", err)
		}
	case store.MultiRepoImporter:
		if err := imp.CreateVersion(opt.Repo, opt.CommitID); err != nil {
			return fmt.Errorf("error running store.MultiRepoImporter.CreateVersion: %s", err)
		}
	}

	return nil
}
Beispiel #16
0
func (s *indexedUnitStore) buildIndexes(xs map[string]Index, data *graph.Output, defOfs byteOffsets, refFBRs fileByteRanges, refOfs byteOffsets) error {
	var defs []*graph.Def
	var refs []*graph.Ref
	if data != nil {
		// Allow us to distinguish between empty (empty slice) and not-yet-fetched (nil).
		defs = data.Defs
		if defs == nil {
			defs = []*graph.Def{}
		}
		refs = data.Refs
		if refs == nil {
			refs = []*graph.Ref{}
		}
	}

	var getDefsErr error
	var getDefsOnce sync.Once
	getDefs := func() ([]*graph.Def, byteOffsets, error) {
		getDefsOnce.Do(func() {
			// Don't refetch if passed in as arg or if getData was
			// already called.
			if defs == nil {
				defs, defOfs, getDefsErr = s.fsUnitStore.readDefs()
			}
			if defs == nil {
				defs = []*graph.Def{}
			}
		})
		return defs, defOfs, getDefsErr
	}

	var getRefsErr error
	var getRefsOnce sync.Once
	getRefs := func() ([]*graph.Ref, fileByteRanges, byteOffsets, error) {
		getRefsOnce.Do(func() {
			// Don't refetch if passed in as arg or if getData was
			// already called.
			if refs == nil {
				refs, refFBRs, refOfs, getRefsErr = s.fsUnitStore.readRefs()
			}
			if refs == nil {
				refs = []*graph.Ref{}
			}
		})
		return refs, refFBRs, refOfs, getRefsErr
	}

	par := parallel.NewRun(runtime.GOMAXPROCS(0))
	for name_, x_ := range xs {
		name, x := name_, x_
		par.Acquire()
		go func() {
			defer par.Release()
			switch x := x.(type) {
			case defIndexBuilder:
				defs, defOfs, err := getDefs()
				if err != nil {
					par.Error(err)
					return
				}
				if err := x.Build(defs, defOfs); err != nil {
					par.Error(err)
					return
				}
			case refIndexBuilder:
				refs, refFBRs, refOfs, err := getRefs()
				if err != nil {
					par.Error(err)
					return
				}
				if err := x.Build(refs, refFBRs, refOfs); err != nil {
					par.Error(err)
					return
				}
			default:
				par.Error(fmt.Errorf("don't know how to build index %q of type %T", name, x))
				return
			}
			if x, ok := x.(persistedIndex); ok {
				if err := writeIndex(s.fs, name, x); err != nil {
					par.Error(err)
					return
				}
			}
		}()
	}
	return par.Wait()
}
Beispiel #17
0
func (c *URefsRepoCmd) Execute(args []string) error {
	if err := c.validate(); err != nil {
		return err
	}

	if err := removeGlob(".srclib-*"); err != nil {
		return err
	}

	units := make([]*unit.SourceUnit, 0)
	unitNames := hierarchicalNames("u", "unit", "", c.NUnits)
	for _, unitName := range unitNames {
		ut := &unit.SourceUnit{
			Key: unit.Key{
				Name:     unitName,
				Type:     "GoPackage",
				Repo:     c.Repo,
				CommitID: c.CommitID,
			},
			Info: unit.Info{
				Files: []string{},
				Dir:   unitName,
			},
		}
		units = append(units, ut)
	}

	if c.GenSource {
		if err := resetSource(); err != nil {
			return err
		}

		// generate source files
		par := parallel.NewRun(runtime.GOMAXPROCS(0))
		for _, ut := range units {
			ut := ut
			par.Acquire()
			go func() {
				defer par.Release()
				if _, _, _, err := c.genUnit(ut, units); err != nil {
					par.Error(err)
				}
			}()
		}
		if err := par.Wait(); err != nil {
			return err
		}

		// get commit ID
		commitID, err := getGitCommitID()
		if err != nil {
			return err
		}

		// update command to generate graph data
		c.CommitID = commitID
		c.GenSource = false
	}

	// generate graph data
	gr := make(map[string]*graph.Output)
	for _, ut := range units {
		gr[ut.Name] = &graph.Output{}
	}
	var grmu sync.Mutex
	par := parallel.NewRun(runtime.GOMAXPROCS(0))
	for _, ut := range units {
		ut := ut
		ut.CommitID = c.CommitID
		par.Acquire()
		go func() {
			defer par.Release()

			defs, refs, reffiles, err := c.genUnit(ut, units)
			if err != nil {
				par.Error(err)
				return
			}

			grmu.Lock()
			defer grmu.Unlock()

			gr[ut.Name].Defs = append(gr[ut.Name].Defs, defs...)
			for utName, utRefs := range refs {
				gr[utName].Refs = append(gr[utName].Refs, utRefs...)
			}
			for _, ut2 := range units {
				ut2.Files = append(ut2.Files, reffiles[ut2.Name]...)
			}
		}()
	}
	if err := par.Wait(); err != nil {
		return err
	}
	for _, ut := range units {
		utgraph := gr[ut.Name]
		utgraph.Docs = make([]*graph.Doc, 0)
		if err := writeSrclibCache(ut, utgraph, make([]*dep.Resolution, 0)); err != nil {
			return err
		}
	}

	return nil
}