// CreateMakefile creates a Makefile to build a tree. The cwd should
// be the root of the tree you want to make (due to some probably
// unnecessary assumptions that CreateMaker makes).
func CreateMakefile() (*makex.Makefile, error) {
localRepo, err := OpenRepo(".")
if err != nil {
return nil, err
}
buildStore, err := buildstore.LocalRepo(localRepo.RootDir)
if err != nil {
return nil, err
}
treeConfig, err := config.ReadCached(buildStore.Commit(localRepo.CommitID))
if err != nil {
return nil, err
}
if len(treeConfig.SourceUnits) == 0 {
log.Printf("No source unit files found. Did you mean to run `%s config`? (This is not an error; it just means that srclib didn't find anything to build or analyze here.)", srclib.CommandName)
}
// TODO(sqs): buildDataDir is hardcoded.
buildDataDir := filepath.Join(buildstore.BuildDataDirName, localRepo.CommitID)
mf, err := plan.CreateMakefile(buildDataDir, buildStore, localRepo.VCSType, treeConfig)
if err != nil {
return nil, err
}
return mf, nil
}
func TestCreateMakefile(t *testing.T) {
oldChooseTool := toolchain.ChooseTool
defer func() { toolchain.ChooseTool = oldChooseTool }()
toolchain.ChooseTool = func(op, unitType string) (*srclib.ToolRef, error) {
return &srclib.ToolRef{
Toolchain: "tc",
Subcmd: "t",
}, nil
}
buildDataDir := "testdata"
c := &config.Tree{
SourceUnits: []*unit.SourceUnit{
{
Key: unit.Key{
Name: "n",
Type: "t",
},
Info: unit.Info{
Files: []string{"f"},
Ops: map[string][]byte{
"graph": nil,
"depresolve": nil,
},
},
},
},
}
mf, err := plan.CreateMakefile(buildDataDir, nil, "", c)
if err != nil {
t.Fatal(err)
}
want := `
.PHONY: all
all: testdata/n/t.depresolve.json testdata/n/t.graph.json
testdata/n/t.depresolve.json: testdata/n/t.unit.json
srclib tool "tc" "t" < $^ 1> $@
testdata/n/t.graph.json: testdata/n/t.unit.json
srclib tool "tc" "t" < $< | srclib internal normalize-graph-data --unit-type "t" --dir . 1> $@
.DELETE_ON_ERROR:
`
gotBytes, err := makex.Marshal(mf)
if err != nil {
t.Fatal(err)
}
want = strings.TrimSpace(want)
got := string(bytes.TrimSpace(gotBytes))
if got != want {
t.Errorf("got makefile:\n==========\n%s\n==========\n\nwant makefile:\n==========\n%s\n==========", got, want)
}
}
// CreateMakefile creates a Makefile to build a tree. The cwd should
// be the root of the tree you want to make (due to some probably
// unnecessary assumptions that CreateMaker makes).
func CreateMakefile(execOpt ToolchainExecOpt, verbose bool) (*makex.Makefile, error) {
localRepo, err := OpenRepo(".")
if err != nil {
return nil, err
}
buildStore, err := buildstore.LocalRepo(localRepo.RootDir)
if err != nil {
return nil, err
}
treeConfig, err := config.ReadCached(buildStore.Commit(localRepo.CommitID))
if err != nil {
return nil, err
}
if len(treeConfig.SourceUnits) == 0 {
log.Printf("No source unit files found. Did you mean to run `%s config`? (This is not an error; it just means that srclib didn't find anything to build or analyze here.)", srclib.CommandName)
}
toolchainExecOptArgs, err := flagutil.MarshalArgs(&execOpt)
if err != nil {
return nil, err
}
// TODO(sqs): buildDataDir is hardcoded.
buildDataDir := filepath.Join(buildstore.BuildDataDirName, localRepo.CommitID)
mf, err := plan.CreateMakefile(buildDataDir, buildStore, localRepo.VCSType, treeConfig, plan.Options{
ToolchainExecOpt: strings.Join(toolchainExecOptArgs, " "),
Verbose: verbose,
})
if err != nil {
return nil, err
}
return mf, nil
}
func TestCreateMakefile(t *testing.T) {
buildDataDir := "testdata"
c := &config.Tree{
SourceUnits: []*unit.SourceUnit{
{
Name: "n",
Type: "t",
Files: []string{"f"},
Ops: map[string]*srclib.ToolRef{
"graph": {Toolchain: "tc", Subcmd: "t"},
"depresolve": {Toolchain: "tc", Subcmd: "t"},
},
},
},
}
mf, err := plan.CreateMakefile(buildDataDir, nil, "", c, plan.Options{NoCache: true})
if err != nil {
t.Fatal(err)
}
sep := string(filepath.Separator)
want := `
all: testdata` + sep + `n` + sep + `t.graph.json testdata` + sep + `n` + sep + `t.depresolve.json
testdata` + sep + `n` + sep + `t.graph.json: testdata` + sep + `n` + sep + `t.unit.json f
src tool "tc" "t" < $< | src internal normalize-graph-data --unit-type "t" --dir . 1> $@
testdata` + sep + `n` + sep + `t.depresolve.json: testdata` + sep + `n` + sep + `t.unit.json
src tool "tc" "t" < $^ 1> $@
.DELETE_ON_ERROR:
`
gotBytes, err := makex.Marshal(mf)
if err != nil {
t.Fatal(err)
}
want = strings.TrimSpace(want)
got := string(bytes.TrimSpace(gotBytes))
if got != want {
t.Errorf("got makefile:\n==========\n%s\n==========\n\nwant makefile:\n==========\n%s\n==========", got, want)
}
}
// Import imports build data into a RepoStore or MultiRepoStore.
func Import(buildDataFS vfs.FileSystem, stor interface{}, opt ImportOpt) error {
// Traverse the build data directory for this repo and commit to
// create the makefile that lists the targets (which are the data
// files we will import).
treeConfig, err := config.ReadCached(buildDataFS)
if err != nil {
return fmt.Errorf("error calling config.ReadCached: %s", err)
}
mf, err := plan.CreateMakefile(".", nil, "", treeConfig, plan.Options{NoCache: true})
if err != nil {
return fmt.Errorf("error calling plan.Makefile: %s", err)
}
var (
mu sync.Mutex
hasIndexableData bool
)
par := parallel.NewRun(10)
for _, rule_ := range mf.Rules {
rule := rule_
if opt.Unit != "" || opt.UnitType != "" {
type ruleForSourceUnit interface {
SourceUnit() *unit.SourceUnit
}
if rule, ok := rule.(ruleForSourceUnit); ok {
u := rule.SourceUnit()
if (opt.Unit != "" && u.Name != opt.Unit) || (opt.UnitType != "" && u.Type != opt.UnitType) {
continue
}
} else {
// Skip all non-source-unit rules if --unit or
// --unit-type are specified.
continue
}
}
par.Do(func() error {
switch rule := rule.(type) {
case *grapher.GraphUnitRule:
var data graph.Output
if err := readJSONFileFS(buildDataFS, rule.Target(), &data); err != nil {
if os.IsNotExist(err) {
log.Printf("Warning: no build data for unit %s %s.", rule.Unit.Type, rule.Unit.Name)
return nil
}
return fmt.Errorf("error reading JSON file %s for unit %s %s: %s", rule.Target(), rule.Unit.Type, rule.Unit.Name, err)
}
if opt.DryRun || GlobalOpt.Verbose {
log.Printf("# Importing graph data (%d defs, %d refs, %d docs, %d anns) for unit %s %s", len(data.Defs), len(data.Refs), len(data.Docs), len(data.Anns), rule.Unit.Type, rule.Unit.Name)
if opt.DryRun {
return nil
}
}
// HACK: Transfer docs to [def].Docs.
docsByPath := make(map[string]*graph.Doc, len(data.Docs))
for _, doc := range data.Docs {
docsByPath[doc.Path] = doc
}
for _, def := range data.Defs {
if doc, present := docsByPath[def.Path]; present {
def.Docs = append(def.Docs, &graph.DefDoc{Format: doc.Format, Data: doc.Data})
}
}
switch imp := stor.(type) {
case store.RepoImporter:
if err := imp.Import(opt.CommitID, rule.Unit, data); err != nil {
return fmt.Errorf("error running store.RepoImporter.Import: %s", err)
}
case store.MultiRepoImporter:
if err := imp.Import(opt.Repo, opt.CommitID, rule.Unit, data); err != nil {
return fmt.Errorf("error running store.MultiRepoImporter.Import: %s", err)
}
default:
return fmt.Errorf("store (type %T) does not implement importing", stor)
}
mu.Lock()
hasIndexableData = true
mu.Unlock()
}
return nil
})
}
if err := par.Wait(); err != nil {
return err
}
if hasIndexableData && !opt.NoIndex {
if GlobalOpt.Verbose {
log.Printf("# Building indexes")
}
switch s := stor.(type) {
case store.RepoIndexer:
if err := s.Index(opt.CommitID); err != nil {
return fmt.Errorf("Error indexing commit %s: %s", opt.CommitID, err)
}
case store.MultiRepoIndexer:
if err := s.Index(opt.Repo, opt.CommitID); err != nil {
return fmt.Errorf("error indexing %s@%s: %s", opt.Repo, opt.CommitID, err)
}
}
}
return nil
}
// Import imports build data into a RepoStore or MultiRepoStore.
func Import(buildDataFS vfs.FileSystem, stor interface{}, opt ImportOpt) error {
// Traverse the build data directory for this repo and commit to
// create the makefile that lists the targets (which are the data
// files we will import).
treeConfig, err := config.ReadCached(buildDataFS)
if err != nil {
return fmt.Errorf("error calling config.ReadCached: %s", err)
}
mf, err := plan.CreateMakefile(".", nil, "", treeConfig)
if err != nil {
return fmt.Errorf("error calling plan.Makefile: %s", err)
}
// hasIndexableData is set if at least one source unit's graph data is
// successfully imported to the graph store.
//
// This flag is set concurrently in calls to importGraphData, but it doesn't
// need to be protected by a mutex since its value is only modified in one
// direction (false to true), and it is only read in the sequential section
// after parallel.NewRun completes.
//
// However, we still protect it with a mutex to avoid data race errors from the
// Go race detector.
var (
mu sync.Mutex
hasIndexableData bool
)
importGraphData := func(graphFile string, sourceUnit *unit.SourceUnit) error {
var data graph.Output
if err := readJSONFileFS(buildDataFS, graphFile, &data); err != nil {
if err == errEmptyJSONFile {
log.Printf("Warning: the JSON file is empty for unit %s %s.", sourceUnit.Type, sourceUnit.Name)
return nil
}
if os.IsNotExist(err) {
log.Printf("Warning: no build data for unit %s %s.", sourceUnit.Type, sourceUnit.Name)
return nil
}
return fmt.Errorf("error reading JSON file %s for unit %s %s: %s", graphFile, sourceUnit.Type, sourceUnit.Name, err)
}
if opt.DryRun || GlobalOpt.Verbose {
log.Printf("# Importing graph data (%d defs, %d refs, %d docs, %d anns) for unit %s %s", len(data.Defs), len(data.Refs), len(data.Docs), len(data.Anns), sourceUnit.Type, sourceUnit.Name)
if opt.DryRun {
return nil
}
}
// HACK: Transfer docs to [def].Docs.
docsByPath := make(map[string]*graph.Doc, len(data.Docs))
for _, doc := range data.Docs {
docsByPath[doc.Path] = doc
}
for _, def := range data.Defs {
if doc, present := docsByPath[def.Path]; present {
def.Docs = append(def.Docs, &graph.DefDoc{Format: doc.Format, Data: doc.Data})
}
}
switch imp := stor.(type) {
case store.RepoImporter:
if err := imp.Import(opt.CommitID, sourceUnit, data); err != nil {
return fmt.Errorf("error running store.RepoImporter.Import: %s", err)
}
case store.MultiRepoImporter:
if err := imp.Import(opt.Repo, opt.CommitID, sourceUnit, data); err != nil {
return fmt.Errorf("error running store.MultiRepoImporter.Import: %s", err)
}
default:
return fmt.Errorf("store (type %T) does not implement importing", stor)
}
mu.Lock()
hasIndexableData = true
mu.Unlock()
return nil
}
par := parallel.NewRun(10)
for _, rule_ := range mf.Rules {
rule := rule_
switch rule := rule.(type) {
case *grapher.GraphUnitRule:
if (opt.Unit != "" && rule.Unit.Name != opt.Unit) || (opt.UnitType != "" && rule.Unit.Type != opt.UnitType) {
continue
}
par.Acquire()
go func() {
defer par.Release()
if err := importGraphData(rule.Target(), rule.Unit); err != nil {
par.Error(err)
}
}()
case *grapher.GraphMultiUnitsRule:
for target_, sourceUnit_ := range rule.Targets() {
target, sourceUnit := target_, sourceUnit_
if (opt.Unit != "" && sourceUnit.Name != opt.Unit) || (opt.UnitType != "" && sourceUnit.Type != opt.UnitType) {
continue
}
par.Acquire()
go func() {
defer par.Release()
if err := importGraphData(target, sourceUnit); err != nil {
par.Error(err)
}
}()
}
}
}
if err := par.Wait(); err != nil {
return err
}
if hasIndexableData && !opt.NoIndex {
if GlobalOpt.Verbose {
log.Printf("# Building indexes")
}
switch s := stor.(type) {
case store.RepoIndexer:
if err := s.Index(opt.CommitID); err != nil {
return fmt.Errorf("Error indexing commit %s: %s", opt.CommitID, err)
}
case store.MultiRepoIndexer:
if err := s.Index(opt.Repo, opt.CommitID); err != nil {
return fmt.Errorf("error indexing %s@%s: %s", opt.Repo, opt.CommitID, err)
}
}
}
switch imp := stor.(type) {
case store.RepoImporter:
if err := imp.CreateVersion(opt.CommitID); err != nil {
return fmt.Errorf("error running store.RepoImporter.CreateVersion: %s", err)
}
case store.MultiRepoImporter:
if err := imp.CreateVersion(opt.Repo, opt.CommitID); err != nil {
return fmt.Errorf("error running store.MultiRepoImporter.CreateVersion: %s", err)
}
}
return nil
}
func coverage(repo *Repo) (map[string]*cvg.Coverage, error) {
// Gather file data
codeFileData := make(map[string]*codeFileDatum) // data for each file needed to compute coverage
filepath.Walk(repo.RootDir, func(path string, info os.FileInfo, err error) error {
if filepath.IsAbs(path) {
var err error
path, err = filepath.Rel(repo.RootDir, path)
if err != nil {
return err
}
}
if info.IsDir() {
if strings.HasPrefix(info.Name(), ".") {
return filepath.SkipDir // don't search hidden directories
}
return nil
}
path = filepath.ToSlash(path)
ext := strings.ToLower(filepath.Ext(path))
if lang, isCodeFile := extToLang[ext]; isCodeFile {
// omitting special files (auto-generated, temporary, ...)
if shouldIgnoreFile(path, lang) {
return nil
}
b, err := ioutil.ReadFile(path)
if err != nil {
return err
}
loc := numLines(b)
codeFileData[path] = &codeFileDatum{LoC: loc, Language: lang}
}
return nil
})
// Gather ref/def data for each file
bdfs, err := GetBuildDataFS(repo.CommitID)
if err != nil {
return nil, err
}
treeConfig, err := config.ReadCached(bdfs)
if err != nil {
return nil, fmt.Errorf("error calling config.ReadCached: %s", err)
}
mf, err := plan.CreateMakefile(".", nil, "", treeConfig)
if err != nil {
return nil, fmt.Errorf("error calling plan.Makefile: %s", err)
}
defKeys := make(map[graph.DefKey]struct{})
data := make([]graph.Output, 0, len(mf.Rules))
parseGraphData := func(graphFile string, sourceUnit *unit.SourceUnit) error {
var item graph.Output
if err := readJSONFileFS(bdfs, graphFile, &item); err != nil {
if err == errEmptyJSONFile {
log.Printf("Warning: the JSON file is empty for unit %s %s.", sourceUnit.Type, sourceUnit.Name)
return nil
}
if os.IsNotExist(err) {
log.Printf("Warning: no build data for unit %s %s.", sourceUnit.Type, sourceUnit.Name)
return nil
}
return fmt.Errorf("error reading JSON file %s for unit %s %s: %s", graphFile, sourceUnit.Type, sourceUnit.Name, err)
}
data = append(data, item)
for _, file := range sourceUnit.Files {
if datum, exists := codeFileData[file]; exists {
datum.Seen = true
}
}
for _, def := range item.Defs {
defKeys[def.DefKey] = struct{}{}
}
return nil
}
for _, rule_ := range mf.Rules {
switch rule := rule_.(type) {
case *grapher.GraphUnitRule:
if err := parseGraphData(rule.Target(), rule.Unit); err != nil {
return nil, err
}
case *grapher.GraphMultiUnitsRule:
for target, sourceUnit := range rule.Targets() {
if err := parseGraphData(target, sourceUnit); err != nil {
return nil, err
}
}
}
}
missingKeys := make(map[graph.DefKey]struct{})
for _, item := range data {
var validRefs []*graph.Ref
for _, ref := range item.Refs {
if datum, exists := codeFileData[ref.File]; exists {
datum.NumRefs++
if ref.DefUnitType == "URL" || ref.DefRepo != "" {
validRefs = append(validRefs, ref)
datum.NumRefsValid++
} else if _, defExists := defKeys[ref.DefKey()]; defExists {
validRefs = append(validRefs, ref)
datum.NumRefsValid++
} else if GlobalOpt.Verbose {
if _, reported := missingKeys[ref.DefKey()]; !reported {
missingKeys[ref.DefKey()] = struct{}{}
sample := ref.DefKey().Path
candidates := make([]graph.DefKey, 0, 1)
for key := range defKeys {
if key.Path == sample {
candidates = append(candidates, key)
}
}
log.Printf("No matching def for %s, candidates are %v", ref.String(), candidates)
}
}
}
}
for _, def := range item.Defs {
if datum, exists := codeFileData[def.File]; exists {
datum.NumDefs++
}
}
}
// Compute coverage from per-file data
type langStats struct {
numFiles int
numIndexedFiles int
numDefs int
numRefs int
numRefsValid int
uncoveredFiles []string
undiscoveredFiles []string
loc int
}
stats := make(map[string]*langStats)
for file, datum := range codeFileData {
if _, exist := stats[datum.Language]; !exist {
stats[datum.Language] = &langStats{}
}
s := stats[datum.Language]
s.loc += datum.LoC
s.numDefs += datum.NumDefs
s.numRefs += datum.NumRefs
s.numRefsValid += datum.NumRefsValid
if datum.Seen {
// this file is listed in the source unit and found by the scanner
s.numFiles++
density := float64(datum.NumDefs+datum.NumRefsValid) / float64(datum.LoC)
if density > fileTokThresh {
s.numIndexedFiles++
} else {
if GlobalOpt.Verbose {
log.Printf("Uncovered file %s - density: %f, defs: %d, refs: %d, lines of code: %d",
file, density, datum.NumDefs, datum.NumRefsValid, datum.LoC)
}
s.uncoveredFiles = append(s.uncoveredFiles, file)
}
} else {
// this file is not listed in the source unit but found by the scanner
if GlobalOpt.Verbose {
log.Printf("Undiscovered file %s", file)
}
s.undiscoveredFiles = append(s.undiscoveredFiles, file)
}
}
cov := make(map[string]*cvg.Coverage)
for lang, s := range stats {
cov[lang] = &cvg.Coverage{
FileScore: divideSentinel(float64(s.numIndexedFiles), float64(s.numFiles), -1),
RefScore: divideSentinel(float64(s.numRefsValid), float64(s.numRefs), -1),
TokDensity: divideSentinel(float64(s.numDefs+s.numRefs), float64(s.loc), -1),
UncoveredFiles: s.uncoveredFiles,
UndiscoveredFiles: s.undiscoveredFiles,
}
}
return cov, nil
}