// ScanMulti runs multiple scanner tools in parallel. It passes command-line
// options from opt to each one, and it sends the JSON representation of cfg
// (the repo/tree's Config) to each tool's stdin.
func ScanMulti(scanners [][]string, opt Options, treeConfig map[string]interface{}) ([]*unit.SourceUnit, error) {
if treeConfig == nil {
treeConfig = map[string]interface{}{}
}
var (
units []*unit.SourceUnit
mu sync.Mutex
)
run := parallel.NewRun(runtime.GOMAXPROCS(0))
for _, scanner_ := range scanners {
scanner := scanner_
run.Acquire()
go func() {
defer run.Release()
units2, err := Scan(scanner, opt, treeConfig)
if err != nil {
run.Error(fmt.Errorf("scanner %v: %s", scanner, err))
return
}
mu.Lock()
defer mu.Unlock()
units = append(units, units2...)
}()
}
err := run.Wait()
// Return error only if none of the commands succeeded.
if len(units) == 0 {
return nil, err
}
return units, nil
}
// refsAtOffsets reads the refs at the given serialized byte offsets
// from the ref data file and returns them in arbitrary order.
func (s *fsUnitStore) refsAtOffsets(ofs byteOffsets, fs []RefFilter) (refs []*graph.Ref, err error) {
vlog.Printf("%s: reading refs at %d offsets with filters %v...", s, len(ofs), fs)
f, err := openFetcherOrOpen(s.fs, unitRefsFilename)
if err != nil {
return nil, err
}
defer func() {
err2 := f.Close()
if err == nil {
err = err2
}
}()
ffs := refFilters(fs)
p := parFetches(s.fs, fs)
if p == 0 {
return nil, nil
}
var refsLock sync.Mutex
par := parallel.NewRun(p)
for _, ofs_ := range ofs {
ofs := ofs_
par.Acquire()
go func() {
defer par.Release()
if _, moreOK := LimitRemaining(fs); !moreOK {
return
}
// Guess how many bytes this ref is. The s3vfs (if that's the
// VFS impl in use) will autofetch beyond that if needed.
const byteEstimate = decodeBufSize
r, err := rangeReader(s.fs, unitRefsFilename, f, ofs, byteEstimate)
if err != nil {
par.Error(err)
return
}
dec := Codec.NewDecoder(r)
var ref graph.Ref
if _, err := dec.Decode(&ref); err != nil {
par.Error(err)
return
}
if ffs.SelectRef(&ref) {
refsLock.Lock()
refs = append(refs, &ref)
refsLock.Unlock()
}
}()
}
if err := par.Wait(); err != nil {
return refs, err
}
sort.Sort(refsByFileStartEnd(refs))
vlog.Printf("%s: read %v refs at %d offsets with filters %v.", s, len(refs), len(ofs), fs)
return refs, nil
}
// BuildIndexes builds all indexes on store and its lower-level stores
// that match the specified criteria. It returns the status of each
// index that was built (or rebuilt).
func BuildIndexes(store interface{}, c IndexCriteria, indexChan chan<- IndexStatus) ([]IndexStatus, error) {
var built []IndexStatus
var builtMu sync.Mutex
indexChan2 := make(chan IndexStatus)
done := make(chan struct{})
go func() {
var par *parallel.Run
lastDependsOnChildren := false
for sx := range indexChan2 {
doBuild := func(sx IndexStatus) {
start := time.Now()
err := sx.store.BuildIndex(sx.Name, sx.index)
sx.BuildDuration = time.Since(start)
if err == nil {
sx.Stale = false
} else {
sx.BuildError = err.Error()
}
builtMu.Lock()
built = append(built, sx)
builtMu.Unlock()
if indexChan != nil {
indexChan <- sx
}
}
// Run indexes in parallel, but if we
// encounter an index that depends on children, wait for
// all previously seen indexes to finish before building
// those indexes.
if sx.DependsOnChildren != lastDependsOnChildren && par != nil {
par.Wait()
par = nil
}
if par == nil {
par = parallel.NewRun(MaxIndexParallel)
}
sx_ := sx
par.Acquire()
go func() {
defer par.Release()
doBuild(sx_)
}()
lastDependsOnChildren = sx.DependsOnChildren
}
if par != nil {
par.Wait()
}
done <- struct{}{}
}()
err := listIndexes(store, c, indexChan2, nil)
close(indexChan2)
<-done
return built, err
}
func (s unitStores) Refs(f ...RefFilter) ([]*graph.Ref, error) {
uss, err := openUnitStores(s.opener, f)
if err != nil {
return nil, err
}
c_unitStores_Refs_last_numUnitsQueried.set(0)
var (
allRefsMu sync.Mutex
allRefs []*graph.Ref
)
par := parallel.NewRun(storeFetchPar)
for u, us := range uss {
if us == nil {
continue
}
u, us := u, us
c_unitStores_Refs_last_numUnitsQueried.increment()
par.Acquire()
go func() {
defer par.Release()
if _, moreOK := LimitRemaining(f); !moreOK {
return
}
fCopy := filtersForUnit(u, f).([]RefFilter)
fCopy = withImpliedUnit(fCopy, u)
refs, err := us.Refs(fCopy...)
if err != nil && !isStoreNotExist(err) {
par.Error(err)
return
}
for _, ref := range refs {
ref.UnitType = u.Type
ref.Unit = u.Name
if ref.DefUnitType == "" {
ref.DefUnitType = u.Type
}
if ref.DefUnit == "" {
ref.DefUnit = u.Name
}
}
allRefsMu.Lock()
allRefs = append(allRefs, refs...)
allRefsMu.Unlock()
}()
}
err = par.Wait()
return allRefs, err
}
// ReadCached reads a Tree's configuration from all of its source unit
// definition files (which may either be in a local VFS rooted at a
// .srclib-cache/<COMMITID> dir, or a remote VFS). It does not read
// the Srcfile; the Srcfile's directives are already accounted for in
// the cached source unit definition files.
//
// bdfs should be a VFS obtained from a call to
// (buildstore.RepoBuildStore).Commit.
func ReadCached(bdfs vfs.FileSystem) (*Tree, error) {
if _, err := bdfs.Lstat("."); os.IsNotExist(err) {
return nil, fmt.Errorf("build cache dir does not exist (did you run `srclib config` to create it)?")
} else if err != nil {
return nil, err
}
// Collect all **/*.unit.json files.
var unitFiles []string
unitSuffix := buildstore.DataTypeSuffix(unit.SourceUnit{})
w := fs.WalkFS(".", rwvfs.Walkable(rwvfs.ReadOnly(bdfs)))
for w.Step() {
if err := w.Err(); err != nil {
return nil, err
}
if path := w.Path(); strings.HasSuffix(path, unitSuffix) {
unitFiles = append(unitFiles, path)
}
}
// Parse units
sort.Strings(unitFiles)
units := make([]*unit.SourceUnit, len(unitFiles))
par := parallel.NewRun(runtime.GOMAXPROCS(0))
for i_, unitFile_ := range unitFiles {
i, unitFile := i_, unitFile_
par.Acquire()
go func() {
defer par.Release()
f, err := bdfs.Open(unitFile)
if err != nil {
par.Error(err)
return
}
if err := json.NewDecoder(f).Decode(&units[i]); err != nil {
f.Close()
par.Error(err)
return
}
if err := f.Close(); err != nil {
par.Error(err)
return
}
}()
}
if err := par.Wait(); err != nil {
return nil, err
}
return &Tree{SourceUnits: units}, nil
}
// RemoveAll removes a tree recursively.
func RemoveAll(path string, vfs rwvfs.WalkableFileSystem) error {
w := fs.WalkFS(path, vfs)
remove := func(par *parallel.Run, path string) {
par.Acquire()
go func() {
defer par.Release()
if err := vfs.Remove(path); err != nil {
par.Error(err)
}
}()
}
var dirs []string // remove dirs after removing all files
filesPar := parallel.NewRun(20)
for w.Step() {
if err := w.Err(); err != nil {
return err
}
if w.Stat().IsDir() {
dirs = append(dirs, w.Path())
} else {
remove(filesPar, w.Path())
}
}
if err := filesPar.Wait(); err != nil {
return err
}
dirsPar := parallel.NewRun(20)
sort.Sort(sort.Reverse(sort.StringSlice(dirs))) // reverse so we delete leaf dirs first
for _, dir := range dirs {
remove(dirsPar, dir)
}
return dirsPar.Wait()
}
func brokenRefsOnly(refs []*graph.Ref, s interface{}) ([]*graph.Ref, error) {
uniqRefDefs := map[graph.DefKey][]*graph.Ref{}
loggedDefRepos := map[string]struct{}{}
for _, ref := range refs {
if ref.Repo != ref.DefRepo {
if _, logged := loggedDefRepos[ref.DefRepo]; !logged {
// TODO(sqs): need to skip these because we don't know the
// "DefCommitID" in the def's repo, and ByDefKey requires
// the key to have a CommitID.
log.Printf("WARNING: Can't check resolution of cross-repo ref (ref.Repo=%q != ref.DefRepo=%q) - cross-repo ref checking is not yet implemented. (This log message will not be repeated.)", ref.Repo, ref.DefRepo)
loggedDefRepos[ref.DefRepo] = struct{}{}
}
continue
}
def := ref.DefKey()
def.CommitID = ref.CommitID
uniqRefDefs[def] = append(uniqRefDefs[def], ref)
}
var (
brokenRefs []*graph.Ref
brokenRefMu sync.Mutex
par = parallel.NewRun(runtime.GOMAXPROCS(0))
)
for def_, refs_ := range uniqRefDefs {
def, refs := def_, refs_
par.Acquire()
go func() {
defer par.Release()
defs, err := s.(store.RepoStore).Defs(store.ByDefKey(def))
if err != nil {
par.Error(err)
return
}
if len(defs) == 0 {
brokenRefMu.Lock()
brokenRefs = append(brokenRefs, refs...)
brokenRefMu.Unlock()
}
}()
}
err := par.Wait()
sort.Sort(graph.Refs(brokenRefs))
return brokenRefs, err
}
func (s unitStores) Defs(fs ...DefFilter) ([]*graph.Def, error) {
uss, err := openUnitStores(s.opener, fs)
if err != nil {
return nil, err
}
var (
allDefs []*graph.Def
allDefsMu sync.Mutex
)
par := parallel.NewRun(storeFetchPar)
for u_, us_ := range uss {
u, us := u_, us_
if us == nil {
continue
}
par.Acquire()
go func() {
defer par.Release()
defs, err := us.Defs(filtersForUnit(u, fs).([]DefFilter)...)
if err != nil && !isStoreNotExist(err) {
par.Error(err)
return
}
for _, def := range defs {
def.UnitType = u.Type
def.Unit = u.Name
}
allDefsMu.Lock()
allDefs = append(allDefs, defs...)
allDefsMu.Unlock()
}()
}
err = par.Wait()
return allDefs, err
}
func (s repoStores) Defs(f ...DefFilter) ([]*graph.Def, error) {
rss, err := openRepoStores(s.opener, f)
if err != nil {
return nil, err
}
var (
allDefs []*graph.Def
allDefsMu sync.Mutex
)
par := parallel.NewRun(storeFetchPar)
for repo_, rs_ := range rss {
repo, rs := repo_, rs_
if rs == nil {
continue
}
par.Acquire()
go func() {
defer par.Release()
defs, err := rs.Defs(filtersForRepo(repo, f).([]DefFilter)...)
if err != nil && !isStoreNotExist(err) {
par.Error(err)
return
}
for _, def := range defs {
def.Repo = repo
}
allDefsMu.Lock()
allDefs = append(allDefs, defs...)
allDefsMu.Unlock()
}()
}
err = par.Wait()
return allDefs, err
}
func (s repoStores) Units(f ...UnitFilter) ([]*unit.SourceUnit, error) {
rss, err := openRepoStores(s.opener, f)
if err != nil {
return nil, err
}
var (
allUnits []*unit.SourceUnit
allUnitsMu sync.Mutex
)
par := parallel.NewRun(storeFetchPar)
for repo_, rs_ := range rss {
repo, rs := repo_, rs_
if rs == nil {
continue
}
par.Acquire()
go func() {
defer par.Release()
units, err := rs.Units(filtersForRepo(repo, f).([]UnitFilter)...)
if err != nil && !isStoreNotExist(err) {
par.Error(err)
return
}
for _, unit := range units {
unit.Repo = repo
}
allUnitsMu.Lock()
allUnits = append(allUnits, units...)
allUnitsMu.Unlock()
}()
}
err = par.Wait()
return allUnits, err
}
// refsAtByteRanges reads the refs at the given serialized byte ranges
// from the ref data file and returns them in arbitrary order.
func (s *fsUnitStore) refsAtByteRanges(brs []byteRanges, fs []RefFilter) (refs []*graph.Ref, err error) {
vlog.Printf("%s: reading refs at %d byte ranges with filters %v...", s, len(brs), fs)
f, err := openFetcherOrOpen(s.fs, unitRefsFilename)
if err != nil {
return nil, err
}
defer func() {
err2 := f.Close()
if err == nil {
err = err2
}
}()
ffs := refFilters(fs)
p := parFetches(s.fs, fs)
if p == 0 {
return nil, nil
}
// See how many bytes we need to read to get the refs in all
// byteRanges.
readLengths := make([]int64, len(brs))
totalRefs := 0
for i, br := range brs {
var n int64
for _, b := range br[1:] {
n += b
totalRefs++
}
readLengths[i] = n
}
var refsLock sync.Mutex
par := parallel.NewRun(p)
for i_, br_ := range brs {
i, br := i_, br_
par.Acquire()
go func() {
defer par.Release()
if _, moreOK := LimitRemaining(fs); !moreOK {
return
}
r, err := rangeReader(s.fs, unitRefsFilename, f, br.start(), readLengths[i])
if err != nil {
par.Error(err)
return
}
dec := Codec.NewDecoder(r)
for range br[1:] {
var ref graph.Ref
if _, err := dec.Decode(&ref); err != nil {
par.Error(err)
return
}
if ffs.SelectRef(&ref) {
refsLock.Lock()
refs = append(refs, &ref)
refsLock.Unlock()
}
}
}()
}
if err := par.Wait(); err != nil {
return refs, err
}
sort.Sort(refsByFileStartEnd(refs))
vlog.Printf("%s: read %d refs at %d byte ranges with filters %v.", s, len(refs), len(brs), fs)
return refs, nil
}
// listIndexes lists indexes in s (a store) asynchronously, sending
// status objects to ch. If f != nil, it is called to set/modify
// fields on each status object before the IndexStatus object is sent to
// the channel.
func listIndexes(s interface{}, c IndexCriteria, ch chan<- IndexStatus, f func(*IndexStatus)) error {
switch s := s.(type) {
case indexedStore:
xx := s.Indexes()
var waitingOnChildren []IndexStatus
for name, x := range xx {
st := IndexStatus{
Name: name,
Type: strings.TrimPrefix(reflect.TypeOf(x).String(), "*store."),
index: x,
store: s,
}
if !strings.Contains(st.Name, c.Name) {
continue
}
if !strings.Contains(st.Type, c.Type) {
continue
}
fi, err := s.statIndex(name)
if os.IsNotExist(err) {
st.Stale = true
} else if err != nil {
st.Error = err.Error()
} else {
st.Size = fi.Size()
}
switch x.(type) {
case unitRefIndexBuilder, defQueryTreeIndexBuilder:
st.DependsOnChildren = true
}
if c.Stale != nil && st.Stale != *c.Stale {
continue
}
if f != nil {
f(&st)
}
if c.Unit != nil && c.Unit != NoSourceUnit {
if st.Unit == nil || *c.Unit != *st.Unit {
continue
}
}
if st.DependsOnChildren {
waitingOnChildren = append(waitingOnChildren, st)
} else {
ch <- st
}
}
switch s := s.(type) {
case *indexedTreeStore:
if err := listIndexes(s.fsTreeStore, c, ch, f); err != nil {
return err
}
case *indexedUnitStore:
if err := listIndexes(s.fsUnitStore, c, ch, f); err != nil {
return err
}
}
for _, si := range waitingOnChildren {
ch <- si
}
case repoStoreOpener:
var rss map[string]RepoStore
if c.Repo == "" {
var err error
rss, err = s.openAllRepoStores()
if err != nil && !isStoreNotExist(err) {
return err
}
} else {
rss = map[string]RepoStore{c.Repo: s.openRepoStore(c.Repo)}
}
// Sort repos for determinism.
repos := make([]string, 0, len(rss))
for repo := range rss {
repos = append(repos, repo)
}
sort.Strings(repos)
if c.ReposOffset != 0 {
if c.ReposOffset < len(repos) {
repos = repos[c.ReposOffset:]
} else {
log.Printf("Warning: A ReposOffset (%d) was specified that equals or exceeds the total number of repos (%d).", c.ReposOffset, len(repos))
}
}
if c.ReposLimit != 0 && c.ReposLimit < len(repos) {
repos = repos[:c.ReposLimit]
}
for _, repo := range repos {
rs := rss[repo]
err := listIndexes(rs, c, ch, func(x *IndexStatus) {
x.Repo = repo
if f != nil {
f(x)
}
})
if err != nil {
return err
}
}
case treeStoreOpener:
var tss map[string]TreeStore
if c.CommitID == "" {
var err error
tss, err = s.openAllTreeStores()
if err != nil && !isStoreNotExist(err) {
return err
}
} else {
tss = map[string]TreeStore{c.CommitID: s.openTreeStore(c.CommitID)}
}
for commitID, ts := range tss {
err := listIndexes(ts, c, ch, func(x *IndexStatus) {
x.CommitID = commitID
if f != nil {
f(x)
}
})
if err != nil {
return err
}
}
case unitStoreOpener:
if c.Unit == NoSourceUnit {
return nil
}
var uss map[unit.ID2]UnitStore
if c.Unit == nil {
var err error
uss, err = s.openAllUnitStores()
if err != nil && !isStoreNotExist(err) {
return err
}
} else {
uss = map[unit.ID2]UnitStore{*c.Unit: s.openUnitStore(*c.Unit)}
}
if len(uss) > 0 {
par := parallel.NewRun(MaxIndexParallel)
for unit_, us_ := range uss {
unit, us := unit_, us_
par.Acquire()
go func() {
defer par.Release()
unitCopy := unit
if err := listIndexes(us, c, ch, func(x *IndexStatus) {
x.Unit = &unitCopy
if f != nil {
f(x)
}
}); err != nil {
par.Error(err)
}
}()
}
if err := par.Wait(); err != nil {
return err
}
}
}
return nil
}
func (c *SimpleRepoCmd) Execute(args []string) error {
if err := c.validate(); err != nil {
return err
}
if err := removeGlob(".srclib-*"); err != nil {
return err
}
units := make([]*unit.SourceUnit, 0)
unitNames := hierarchicalNames("u", "unit", "", c.NUnits)
for _, unitName := range unitNames {
units = append(units, &unit.SourceUnit{
Key: unit.Key{
Name: unitName,
Type: "GoPackage",
Repo: c.Repo,
CommitID: c.CommitID,
},
Info: unit.Info{
Files: []string{},
Dir: unitName,
},
})
}
if c.GenSource {
if err := resetSource(); err != nil {
return err
}
// generate source files
par := parallel.NewRun(runtime.GOMAXPROCS(0))
for _, ut_ := range units {
ut := ut_
par.Acquire()
go func() {
defer par.Release()
if err := c.genUnit(ut); err != nil {
par.Error(err)
}
}()
}
if err := par.Wait(); err != nil {
return err
}
// get commit ID
commitID, err := getGitCommitID()
if err != nil {
return err
}
// update command to generate graph data
c.CommitID = commitID
c.GenSource = false
}
// generate graph data
par := parallel.NewRun(runtime.GOMAXPROCS(0))
for _, ut_ := range units {
ut := ut_
ut.CommitID = c.CommitID
par.Acquire()
go func() {
defer par.Release()
if err := c.genUnit(ut); err != nil {
par.Error(err)
}
}()
}
if err := par.Wait(); err != nil {
return err
}
return nil
}
func (s *indexedTreeStore) buildIndexes(xs map[string]Index, units []*unit.SourceUnit, unitRefIndexes map[unit.ID2]*defRefsIndex, unitDefQueryIndexes map[unit.ID2]*defQueryIndex) error {
// TODO(sqs): there's a race condition here if multiple imports
// are running concurrently, they could clobber each other's
// indexes. (S3 is eventually consistent.)
var getUnitsErr error
var getUnitsOnce sync.Once
getUnits := func() ([]*unit.SourceUnit, error) {
getUnitsOnce.Do(func() {
if getUnitsErr == nil && units == nil {
units, getUnitsErr = s.fsTreeStore.Units()
}
if units == nil {
units = []*unit.SourceUnit{}
}
})
return units, getUnitsErr
}
var getUnitRefIndexesErr error
var getUnitRefIndexesOnce sync.Once
var unitRefIndexesLock sync.Mutex
getUnitRefIndexes := func() (map[unit.ID2]*defRefsIndex, error) {
getUnitRefIndexesOnce.Do(func() {
if getUnitRefIndexesErr == nil && unitRefIndexes == nil {
// Read in the defRefsIndex for all source units.
units, err := getUnits()
if err != nil {
getUnitRefIndexesErr = err
return
}
// Use openUnitStore on the list from getUnits so we
// don't need to traverse the FS tree to enumerate all
// the source units again (which is slow).
uss := make(map[unit.ID2]UnitStore, len(units))
for _, u := range units {
uss[u.ID2()] = s.fsTreeStore.openUnitStore(u.ID2())
}
unitRefIndexes = make(map[unit.ID2]*defRefsIndex, len(units))
par := parallel.NewRun(runtime.GOMAXPROCS(0))
for u_, us_ := range uss {
u := u_
us, ok := us_.(*indexedUnitStore)
if !ok {
continue
}
par.Acquire()
go func() {
defer par.Release()
x := us.indexes[defToRefsIndexName]
if err := prepareIndex(us.fs, defToRefsIndexName, x); err != nil {
par.Error(err)
return
}
unitRefIndexesLock.Lock()
defer unitRefIndexesLock.Unlock()
unitRefIndexes[u] = x.(*defRefsIndex)
}()
}
getUnitRefIndexesErr = par.Wait()
}
if unitRefIndexes == nil {
unitRefIndexes = map[unit.ID2]*defRefsIndex{}
}
})
return unitRefIndexes, getUnitRefIndexesErr
}
var getUnitDefQueryIndexesErr error
var getUnitDefQueryIndexesOnce sync.Once
var unitDefQueryIndexesLock sync.Mutex
getUnitDefQueryIndexes := func() (map[unit.ID2]*defQueryIndex, error) {
getUnitDefQueryIndexesOnce.Do(func() {
if getUnitDefQueryIndexesErr == nil && unitDefQueryIndexes == nil {
// Read in the defQueryIndex for all source units.
units, err := getUnits()
if err != nil {
getUnitDefQueryIndexesErr = err
return
}
// Use openUnitStore on the list from getUnits so we
// don't need to traverse the FS tree to enumerate all
// the source units again (which is slow).
uss := make(map[unit.ID2]UnitStore, len(units))
for _, u := range units {
uss[u.ID2()] = s.fsTreeStore.openUnitStore(u.ID2())
}
unitDefQueryIndexes = make(map[unit.ID2]*defQueryIndex, len(units))
par := parallel.NewRun(runtime.GOMAXPROCS(0))
for u_, us_ := range uss {
u := u_
us, ok := us_.(*indexedUnitStore)
if !ok {
continue
}
par.Acquire()
go func() {
defer par.Release()
x := us.indexes[defQueryIndexName]
if err := prepareIndex(us.fs, defQueryIndexName, x); err != nil {
par.Error(err)
return
}
unitDefQueryIndexesLock.Lock()
defer unitDefQueryIndexesLock.Unlock()
unitDefQueryIndexes[u] = x.(*defQueryIndex)
}()
}
getUnitDefQueryIndexesErr = par.Wait()
}
if unitDefQueryIndexes == nil {
unitDefQueryIndexes = map[unit.ID2]*defQueryIndex{}
}
})
return unitDefQueryIndexes, getUnitDefQueryIndexesErr
}
par := parallel.NewRun(runtime.GOMAXPROCS(0))
for name_, x_ := range xs {
name, x := name_, x_
par.Acquire()
go func() {
defer par.Release()
switch x := x.(type) {
case unitIndexBuilder:
units, err := getUnits()
if err != nil {
par.Error(err)
return
}
if err := x.Build(units); err != nil {
par.Error(err)
return
}
case unitRefIndexBuilder:
unitRefIndexes, err := getUnitRefIndexes()
if err != nil {
par.Error(err)
return
}
if err := x.Build(unitRefIndexes); err != nil {
par.Error(err)
return
}
case defQueryTreeIndexBuilder:
unitDefQueryIndexes, err := getUnitDefQueryIndexes()
if err != nil {
par.Error(err)
return
}
if err := x.Build(unitDefQueryIndexes); err != nil {
par.Error(err)
return
}
default:
par.Error(fmt.Errorf("don't know how to build index %q of type %T", name, x))
return
}
if x, ok := x.(persistedIndex); ok {
if err := writeIndex(s.fs, name, x); err != nil {
par.Error(err)
return
}
}
}()
}
return par.Wait()
}
// Import imports build data into a RepoStore or MultiRepoStore.
func Import(buildDataFS vfs.FileSystem, stor interface{}, opt ImportOpt) error {
// Traverse the build data directory for this repo and commit to
// create the makefile that lists the targets (which are the data
// files we will import).
treeConfig, err := config.ReadCached(buildDataFS)
if err != nil {
return fmt.Errorf("error calling config.ReadCached: %s", err)
}
mf, err := plan.CreateMakefile(".", nil, "", treeConfig)
if err != nil {
return fmt.Errorf("error calling plan.Makefile: %s", err)
}
// hasIndexableData is set if at least one source unit's graph data is
// successfully imported to the graph store.
//
// This flag is set concurrently in calls to importGraphData, but it doesn't
// need to be protected by a mutex since its value is only modified in one
// direction (false to true), and it is only read in the sequential section
// after parallel.NewRun completes.
//
// However, we still protect it with a mutex to avoid data race errors from the
// Go race detector.
var (
mu sync.Mutex
hasIndexableData bool
)
importGraphData := func(graphFile string, sourceUnit *unit.SourceUnit) error {
var data graph.Output
if err := readJSONFileFS(buildDataFS, graphFile, &data); err != nil {
if err == errEmptyJSONFile {
log.Printf("Warning: the JSON file is empty for unit %s %s.", sourceUnit.Type, sourceUnit.Name)
return nil
}
if os.IsNotExist(err) {
log.Printf("Warning: no build data for unit %s %s.", sourceUnit.Type, sourceUnit.Name)
return nil
}
return fmt.Errorf("error reading JSON file %s for unit %s %s: %s", graphFile, sourceUnit.Type, sourceUnit.Name, err)
}
if opt.DryRun || GlobalOpt.Verbose {
log.Printf("# Importing graph data (%d defs, %d refs, %d docs, %d anns) for unit %s %s", len(data.Defs), len(data.Refs), len(data.Docs), len(data.Anns), sourceUnit.Type, sourceUnit.Name)
if opt.DryRun {
return nil
}
}
// HACK: Transfer docs to [def].Docs.
docsByPath := make(map[string]*graph.Doc, len(data.Docs))
for _, doc := range data.Docs {
docsByPath[doc.Path] = doc
}
for _, def := range data.Defs {
if doc, present := docsByPath[def.Path]; present {
def.Docs = append(def.Docs, &graph.DefDoc{Format: doc.Format, Data: doc.Data})
}
}
switch imp := stor.(type) {
case store.RepoImporter:
if err := imp.Import(opt.CommitID, sourceUnit, data); err != nil {
return fmt.Errorf("error running store.RepoImporter.Import: %s", err)
}
case store.MultiRepoImporter:
if err := imp.Import(opt.Repo, opt.CommitID, sourceUnit, data); err != nil {
return fmt.Errorf("error running store.MultiRepoImporter.Import: %s", err)
}
default:
return fmt.Errorf("store (type %T) does not implement importing", stor)
}
mu.Lock()
hasIndexableData = true
mu.Unlock()
return nil
}
par := parallel.NewRun(10)
for _, rule_ := range mf.Rules {
rule := rule_
switch rule := rule.(type) {
case *grapher.GraphUnitRule:
if (opt.Unit != "" && rule.Unit.Name != opt.Unit) || (opt.UnitType != "" && rule.Unit.Type != opt.UnitType) {
continue
}
par.Acquire()
go func() {
defer par.Release()
if err := importGraphData(rule.Target(), rule.Unit); err != nil {
par.Error(err)
}
}()
case *grapher.GraphMultiUnitsRule:
for target_, sourceUnit_ := range rule.Targets() {
target, sourceUnit := target_, sourceUnit_
if (opt.Unit != "" && sourceUnit.Name != opt.Unit) || (opt.UnitType != "" && sourceUnit.Type != opt.UnitType) {
continue
}
par.Acquire()
go func() {
defer par.Release()
if err := importGraphData(target, sourceUnit); err != nil {
par.Error(err)
}
}()
}
}
}
if err := par.Wait(); err != nil {
return err
}
if hasIndexableData && !opt.NoIndex {
if GlobalOpt.Verbose {
log.Printf("# Building indexes")
}
switch s := stor.(type) {
case store.RepoIndexer:
if err := s.Index(opt.CommitID); err != nil {
return fmt.Errorf("Error indexing commit %s: %s", opt.CommitID, err)
}
case store.MultiRepoIndexer:
if err := s.Index(opt.Repo, opt.CommitID); err != nil {
return fmt.Errorf("error indexing %s@%s: %s", opt.Repo, opt.CommitID, err)
}
}
}
switch imp := stor.(type) {
case store.RepoImporter:
if err := imp.CreateVersion(opt.CommitID); err != nil {
return fmt.Errorf("error running store.RepoImporter.CreateVersion: %s", err)
}
case store.MultiRepoImporter:
if err := imp.CreateVersion(opt.Repo, opt.CommitID); err != nil {
return fmt.Errorf("error running store.MultiRepoImporter.CreateVersion: %s", err)
}
}
return nil
}
func (s *indexedUnitStore) buildIndexes(xs map[string]Index, data *graph.Output, defOfs byteOffsets, refFBRs fileByteRanges, refOfs byteOffsets) error {
var defs []*graph.Def
var refs []*graph.Ref
if data != nil {
// Allow us to distinguish between empty (empty slice) and not-yet-fetched (nil).
defs = data.Defs
if defs == nil {
defs = []*graph.Def{}
}
refs = data.Refs
if refs == nil {
refs = []*graph.Ref{}
}
}
var getDefsErr error
var getDefsOnce sync.Once
getDefs := func() ([]*graph.Def, byteOffsets, error) {
getDefsOnce.Do(func() {
// Don't refetch if passed in as arg or if getData was
// already called.
if defs == nil {
defs, defOfs, getDefsErr = s.fsUnitStore.readDefs()
}
if defs == nil {
defs = []*graph.Def{}
}
})
return defs, defOfs, getDefsErr
}
var getRefsErr error
var getRefsOnce sync.Once
getRefs := func() ([]*graph.Ref, fileByteRanges, byteOffsets, error) {
getRefsOnce.Do(func() {
// Don't refetch if passed in as arg or if getData was
// already called.
if refs == nil {
refs, refFBRs, refOfs, getRefsErr = s.fsUnitStore.readRefs()
}
if refs == nil {
refs = []*graph.Ref{}
}
})
return refs, refFBRs, refOfs, getRefsErr
}
par := parallel.NewRun(runtime.GOMAXPROCS(0))
for name_, x_ := range xs {
name, x := name_, x_
par.Acquire()
go func() {
defer par.Release()
switch x := x.(type) {
case defIndexBuilder:
defs, defOfs, err := getDefs()
if err != nil {
par.Error(err)
return
}
if err := x.Build(defs, defOfs); err != nil {
par.Error(err)
return
}
case refIndexBuilder:
refs, refFBRs, refOfs, err := getRefs()
if err != nil {
par.Error(err)
return
}
if err := x.Build(refs, refFBRs, refOfs); err != nil {
par.Error(err)
return
}
default:
par.Error(fmt.Errorf("don't know how to build index %q of type %T", name, x))
return
}
if x, ok := x.(persistedIndex); ok {
if err := writeIndex(s.fs, name, x); err != nil {
par.Error(err)
return
}
}
}()
}
return par.Wait()
}
func (c *URefsRepoCmd) Execute(args []string) error {
if err := c.validate(); err != nil {
return err
}
if err := removeGlob(".srclib-*"); err != nil {
return err
}
units := make([]*unit.SourceUnit, 0)
unitNames := hierarchicalNames("u", "unit", "", c.NUnits)
for _, unitName := range unitNames {
ut := &unit.SourceUnit{
Key: unit.Key{
Name: unitName,
Type: "GoPackage",
Repo: c.Repo,
CommitID: c.CommitID,
},
Info: unit.Info{
Files: []string{},
Dir: unitName,
},
}
units = append(units, ut)
}
if c.GenSource {
if err := resetSource(); err != nil {
return err
}
// generate source files
par := parallel.NewRun(runtime.GOMAXPROCS(0))
for _, ut := range units {
ut := ut
par.Acquire()
go func() {
defer par.Release()
if _, _, _, err := c.genUnit(ut, units); err != nil {
par.Error(err)
}
}()
}
if err := par.Wait(); err != nil {
return err
}
// get commit ID
commitID, err := getGitCommitID()
if err != nil {
return err
}
// update command to generate graph data
c.CommitID = commitID
c.GenSource = false
}
// generate graph data
gr := make(map[string]*graph.Output)
for _, ut := range units {
gr[ut.Name] = &graph.Output{}
}
var grmu sync.Mutex
par := parallel.NewRun(runtime.GOMAXPROCS(0))
for _, ut := range units {
ut := ut
ut.CommitID = c.CommitID
par.Acquire()
go func() {
defer par.Release()
defs, refs, reffiles, err := c.genUnit(ut, units)
if err != nil {
par.Error(err)
return
}
grmu.Lock()
defer grmu.Unlock()
gr[ut.Name].Defs = append(gr[ut.Name].Defs, defs...)
for utName, utRefs := range refs {
gr[utName].Refs = append(gr[utName].Refs, utRefs...)
}
for _, ut2 := range units {
ut2.Files = append(ut2.Files, reffiles[ut2.Name]...)
}
}()
}
if err := par.Wait(); err != nil {
return err
}
for _, ut := range units {
utgraph := gr[ut.Name]
utgraph.Docs = make([]*graph.Doc, 0)
if err := writeSrclibCache(ut, utgraph, make([]*dep.Resolution, 0)); err != nil {
return err
}
}
return nil
}