// StartService starts up the program as a micro-service server.
func (l *service) Start() error {
// set a fixed socket and manually start process to help with debugging.
// l.socketAddr = "@01c67"
// return nil
// Start up the mirco-service
log.Println("starting process", l.program)
log.Println(l.args)
cmd := exec.Command(l.program, l.args...)
p, err := cmd.StdoutPipe()
if err != nil {
return errors.Trace(err)
}
if err := cmd.Start(); err != nil {
return errors.Trace(err)
}
l.process = cmd.Process
// Get the socket address from the server.
b := bufio.NewReader(p)
line, _, err := b.ReadLine()
if err != nil {
log.Println("unable to get socket address from server, stopping tenet")
l.Stop()
return errors.Trace(err)
}
l.socketAddr = strings.TrimSuffix(string(line), "\n")
return nil
}
func (b *Base) SendError(err error) {
log.Println("sending error", err)
log.Println(b.errorsc == nil)
select {
case b.errorsc <- err:
default:
}
}
// Close closes the connection and, if local, stops the backing service.
func (t *tenetService) Close() error {
log.Println("closing conn")
if t.Service == nil {
return errors.New("attempted to close a nil service. Has it been started?")
}
err := t.conn.Close()
log.Println("stopping service")
if err1 := t.Service.Stop(); err1 != nil {
err = err1
}
return err
}
func (r *review) raiseIssue(issueName string, iRange *issueRange, opts []RaiseIssueOption) Review {
if r.areAllContextsMatched() {
return r
}
b := r.baseTenet()
// TODO(waigani) error handle this.
i := b.registeredIssues[issueName]
if i == nil {
// Yes panic, this is a developer error.
msg := fmt.Sprintf("issue %q cannot be raised before it is registered", issueName)
panic(msg)
}
issue := &Issue{}
x := *i
x.copyTo(issue)
// TODO(waigani) This a Go blemish. Is there a nicer way to copy a struct with a map?
issue.CommVars = map[string]interface{}{}
for _, opt := range opts {
opt(issue)
}
// TODO(waigani) this is a quick hack. We need to pull File out of *Issue.
issue.file = r.File()
issue.setSource(iRange)
if err := r.setContextualComment(issue); err != nil {
// If no comment has been set for the context in which this issue was
// found, don't raise it.
if err == errNoCommentForContext {
return r
}
issue.Err = err
}
log.Println("sending issue")
r.sendIssue(issue)
log.Println("not blocked")
if r.areAllContextsMatched() {
// This is our last issue raised, close the issue chan. Note: if a
// tenet reviews async, this will become a race condition.
r.Close()
}
return r
}
// NewService allows you to run a program on the localhost as a micro-service.
func NewService(program string, args ...string) *service {
log.Println("NewLocal service")
return &service{
program: program,
args: args,
}
}
func (t *tenet) OpenService() (TenetService, error) {
ds, err := t.Driver.Service()
if err != nil {
log.Print(err.Error()) // TODO(waigani) this logs are quick hacks. Work out the error paths and log them all at the root.
return nil, errors.Trace(err)
}
cfg := &api.Config{}
for k, v := range t.options {
cfg.Options = append(cfg.Options,
&api.Option{
Name: k,
Value: fmt.Sprintf("%v", v),
})
}
s := &tenetService{
Service: ds,
cfg: cfg,
editFilename: t.Driver.EditFilename,
editIssue: t.Driver.EditIssue,
mutex: &sync.Mutex{},
}
if err := s.start(); err != nil {
log.Println("got err opening service")
// TODO(waigani) add retry logic here. 1. Keep retrying until service
// is up. 2. Keep retrying until service is connected.
log.Printf("err: %#v", errors.ErrorStack(err))
return nil, errors.Trace(err)
}
log.Print("opened service, no issue")
return s, nil
}
func (l *service) DialGRPC() (*grpc.ClientConn, error) {
if l.socketAddr == "" {
return nil, errors.New("socket address is empty. Is the service started?")
}
log.Println("dialing server")
return grpc.Dial(l.socketAddr, grpc.WithDialer(dialer()), grpc.WithInsecure())
}
// Closes a review. Can be called more than once.
func (r *review) Close() {
if r.waitc != nil {
log.Println("closing review and waitc")
close(r.waitc)
os.RemoveAll(r.tmpdir)
r.waitc = nil
}
}
// StopService stops the backing tenet server. If Common as a non-nil
// connection to the server, that will be closed.
func (l *service) Stop() (err error) {
if l.process != nil {
log.Println("killing process")
if err = l.process.Kill(); err != nil {
log.Fatalf("did not stop %s: %v", l.program, err)
}
}
return
}
func (s *TenetSuite) CheckFiles(c *gc.C, files []string, expectedIssues ...ExpectedIssue) {
log.Println("CheckFiles")
br := s.baseReview()
br.StartReview()
s.sendFiles(files...)
s.AssertExpectedIssues(c, ReadAllIssues(c, br), expectedIssues...)
}
func main() {
err := app.New().Run(os.Args)
if err != nil {
if errors.Cause(err).Error() == "ui" {
if e, ok := err.(*errors.Err); ok {
log.Println(e.Underlying())
fmt.Println(e.Underlying())
os.Exit(1)
}
}
panic(errors.ErrorStack(err))
}
}
func ReadAllIssues(c *gc.C, r tenet.BaseReview) []*tenet.Issue {
log.Println("reading all issues")
var issues []*tenet.Issue
l:
for {
select {
case issue, ok := <-r.Issues():
if !ok {
break l
}
issues = append(issues, issue)
case <-time.After(10 * time.Second):
c.Fatal("timed out waiting for issues")
break l
}
}
return issues
}
func (b *Base) NewReview() *review {
r := &review{
tenet: b,
issuesc: make(chan *Issue),
filesc: make(chan *api.File),
waitc: make(chan struct{}),
fileDoneMap: map[string]bool{},
}
go func() {
<-r.waitc
if r.issuesc != nil {
log.Println("closing issuesc")
close(r.issuesc)
}
r.waitc = nil
}()
return r
}
// TODO(waigani) call this handleTenetError and make a TenetError type - only
// those can be passed in.
// posOfErr is the position of the node/line that was being parsed when the
// error occoured.
func (b *Base) addErrOnErr(err error, f File, posOfErr interface{}) bool {
if err != nil {
// TODO(waigani) this log is a quick hack. We should read all the errs off errorsc.
log.Println(err.Error())
errCtx := &errWithContext{err: err}
switch p := posOfErr.(type) {
case token.Pos:
fset := f.Fset()
pos := fset.Position(p)
errCtx.errLine = &pos
case int:
line := f.(BaseFile).linePosition(p)
errCtx.errLine = &line
default:
panic(fmt.Sprintf("unknown posOfErr type: %T", posOfErr))
}
b.errorsc <- errCtx
return true
}
return false
}
// StartReview listens for files sent to r.SendFile(filename) and reviews them.
func (r *review) StartReview() {
go func() {
defer r.Close()
log.Println("started review")
b := base(r.tenet)
// check files synchronously to ensure correct ordering and that we stop
// after the context is full.
fset := token.NewFileSet()
log.Println("reading off filesc")
for {
select {
case file, ok := <-r.filesc:
if !ok && file == nil {
log.Println("all files reviewed.")
return
}
f, err := buildFile(file.Name, "", fset, file.Lines)
if err != nil {
log.Println("could not build file")
b.SendError(errors.Annotatef(err, "could not find file: %q", file))
continue
}
log.Println("checking file", file)
err = r.check(f)
b.addErrOnErr(err, f, 0)
log.Println("finished checking file", file)
case <-time.After(3 * time.Second):
b.errorsc <- errors.New("timed out waiting for file")
return
}
}
}()
}
// Build up a config object by following directories up or down.
func buildConfigRecursive(cfgPath string, cascadeDir CascadeDirection, cfg *Config) error {
cfgPath, err := filepath.Abs(cfgPath)
if err != nil {
return nil
}
currentCfg, err := ReadConfigFile(cfgPath)
if err == nil {
// Add the non-tenet properties - always when cascading down, otherwise
// only if not already specified
// TODO: Use reflection here to avoid forgotten values?
if cascadeDir == CascadeDown || cfg.Include == "" {
cfg.Include = currentCfg.Include
}
if cascadeDir == CascadeDown || cfg.Template == "" {
cfg.Template = currentCfg.Template
}
// DEMOWARE: Need a better way to assign tenets to groups without duplication
for _, g := range currentCfg.TenetGroups {
DupeCheck:
for _, t := range g.Tenets {
for _, e := range cfg.allTenets {
if e.Name == t.Name {
continue DupeCheck
}
}
cfg.AddTenet(t, g.Name)
cfg.allTenets = append(cfg.allTenets, t)
}
}
// Asign group properties
for _, g := range currentCfg.TenetGroups {
for i, cg := range cfg.TenetGroups {
if cg.Name == g.Name {
if g.Template != "" {
cfg.TenetGroups[i].Template = g.Template
}
}
}
}
} else if !os.IsNotExist(err) {
// Just leave the current state of cfg on encountering an error
log.Println("error reading file: %s", cfgPath)
return nil
}
currentDir, filename := path.Split(cfgPath)
switch cascadeDir {
case CascadeUp:
if currentDir == "/" || (currentCfg != nil && !currentCfg.Cascade) {
return nil
}
parent := path.Dir(path.Dir(currentDir))
if err := buildConfigRecursive(path.Join(parent, filename), cascadeDir, cfg); err != nil {
return err
}
case CascadeDown:
files, err := filepath.Glob(path.Join(currentDir, "*"))
if err != nil {
return nil
}
for _, f := range files {
file, err := os.Open(f)
if err != nil {
log.Println("error reading file: %s", file)
return nil
}
defer file.Close()
if fi, err := file.Stat(); err == nil && fi.IsDir() {
if err := buildConfigRecursive(path.Join(f, filename), cascadeDir, cfg); err != nil {
return err
}
}
}
default:
return errors.New("invalid cascade direction")
}
return nil
}
// Review sets up two goroutines. One to send files to the service from filesc,
// the other to recieve issues from the service on issuesc.
func (t *tenetService) Review(filesc <-chan *api.File, issuesc chan<- *api.Issue, filesTM *tomb.Tomb) error {
stream, err := t.client.Review(context.Background())
if err != nil {
return err
}
// first setup our issues chan to read from the service.
go func(issuesc chan<- *api.Issue) {
for {
log.Println("waiting for issues")
issue, err := stream.Recv()
if err != nil {
if err == io.EOF ||
grpc.ErrorDesc(err) == "transport is closing" ||
err.Error() == "timed out waiting for issues" { // TODO(waigani) error type
log.Println("closing issuesc")
// Close our local issues channel.
close(issuesc)
return
}
// TODO(waigani) in what error cases should we close issuesc?
// Any other err we keep calm and carry on.
log.Println("ERROR receiving an issue : %s", err.Error())
continue
}
issuesc <- t.editIssue(issue)
}
}(issuesc)
// next, setup a goroutine to send our files to the service to review.
go func(filesc <-chan *api.File) {
for {
select {
case file, ok := <-filesc:
if !ok && file == nil {
log.Println("client filesc closed. Closing send stream.")
// Close the file send stream.
err := stream.CloseSend()
if err != nil {
log.Println(err.Error())
}
return
}
file.Name = t.editFilename(file.Name)
if err := stream.Send(file); err != nil {
log.Println("failed to send a file %q: %v", file.Name, err)
}
log.Printf("sent file %q\n", file.Name)
// Each tenet has a 5 second idle time. If we don't find any
// files to send it in that time, we close this tenet down.
case <-time.After(5 * time.Second):
// this will close this instance of the tenet.
filesTM.Kill(errors.New("timed out waiting for a filename"))
return
}
}
}(filesc)
return nil
}
// returns a chan of tenet reviews and a cancel chan that blocks until the user cancels.
func reviewQueue(ctx *cli.Context, mappings <-chan cfgMap, changed *map[string][]int, errc chan error) (<-chan *tenetReview, chan struct{}) {
reviews := make(map[string]*tenetReview)
reviewChannel := make(chan *tenetReview)
cleanupWG := &sync.WaitGroup{}
// setup a cancel exit path.
cancelc := make(chan os.Signal, 1)
signal.Notify(cancelc, os.Interrupt)
signal.Notify(cancelc, syscall.SIGTERM)
cancelledc := make(chan struct{})
cancelled := func() bool {
select {
case _, ok := <-cancelledc:
if ok {
close(cancelc)
}
return true
default:
return false
}
}
// Kill all open tenets on cancel.
go func() {
var i int
for {
<-cancelc
if i > 0 {
// on the second exit, just do it.
fmt.Print("failed.\nSome docker containers may still be running.")
os.Exit(1)
}
i++
go func() {
// TODO(waigani) add progress bar here
fmt.Print("\ncleaning up tenets ... ")
// Anything waiting on the cancelled chan will now fire.
close(cancelledc)
// Wait for all tenets to be cleaned up.
cleanupWG.Wait()
// say bye.
fmt.Println("done.")
os.Exit(1)
}()
}
}()
// TODO(waigani) reenable buffering to:
// 1. Allow found tenets to keep running.
// 2. Stop building new tenets until there is room in the buffer.
// TODO(waigani) make cfg vars
// buffLimit := 3
// if ctx.Bool("keep-all") {
// buffLimit = 100
// }
// buff := util.NewBuffer(buffLimit, cancelledc)
go func() {
for m := range mappings {
// Glob all the files in the associated directories for this config, and assign to each tenet by hash
for _, tc := range m.cfg.AllTenets() {
if cancelled() {
// empty files and dirs to stop feeding tenet reviews in progress.
m.files = []string{}
m.dirs = []string{}
return
}
// Open the tenet service if we haven't seen this config before.
configHash := tc.hash()
r, found := reviews[configHash]
if !found {
// Don't build a new tenet until there is room in the buffer.
// Found tenets will keep running until they are not fed files for 5 seconds.
// WaitRoom will not block if we get a cancel signal.
// buff.WaitRoom()
tn, err := newTenet(ctx, tc)
if err != nil {
errc <- err
continue
}
// Note: service should not be called outside this if block.
service, err := tn.OpenService()
if err != nil {
errc <- err
continue
}
info, err := service.Info()
if err != nil {
errc <- err
continue
}
r = &tenetReview{
configHash: configHash,
filesc: make(chan *api.File),
issuesc: make(chan *api.Issue),
info: info,
issuesWG: &sync.WaitGroup{},
filesTM: &tomb.Tomb{},
}
reviews[configHash] = r
// Setup the takedown of this review.
r.issuesWG.Add(1)
cleanupWG.Add(1)
// buff.Add(1)
go func(r *tenetReview) {
// The following fires when:
select {
// 1. all files have been sent or timed out
// 2. the tenet buffer is full
case <-r.filesTM.Dying():
// 3. lingo has been stopped
case <-cancelledc:
}
// make room for another tenet to start and ensure
// that any configHash's matching this one will have
// to start a new tenet instance.
delete(reviews, configHash)
// buff.Add(-1)
// signal to the tenet that no more files are coming.
close(r.filesc)
// wait for the tenet to signal to us that it's finished it's review.
r.issuesWG.Wait()
// we can now safely close the backing service.
if err := service.Close(); err != nil {
log.Println("ERROR closing sevice:", err)
}
log.Println("cleanup done")
cleanupWG.Done()
}(r)
// Make sure we're ready to handle results before we start
// the review.
reviewChannel <- r
// Start this tenet's review.
service.Review(r.filesc, r.issuesc, r.filesTM)
}
regexPattern, globPattern := fileExtFilterForLang(r.info.Language)
for _, d := range m.dirs {
files, err := filepath.Glob(path.Join(d, globPattern))
if err != nil {
// Non-fatal
log.Printf("Error reading files in %s: %v\n", d, err)
}
l:
for i, f := range files {
select {
case <-cancelledc:
log.Println("user cancelled, dropping files.")
case <-r.filesTM.Dying():
dropped := len(files) - i
log.Print("WARNING a tenet review timed out waiting for files to be sent. %d files dropped", dropped)
break l
case r.filesc <- &api.File{Name: f}:
}
}
}
z:
for i, f := range m.files {
if matches, err := regexp.MatchString(regexPattern, f); !matches {
if err != nil {
log.Println("error in regex: ", regexPattern)
}
continue
}
fileinfo := &api.File{Name: f}
if changed != nil {
if diffLines, ok := (*changed)[f]; ok { // This can be false if --diff and fileargs are specified
for _, l := range diffLines {
fileinfo.Lines = append(fileinfo.Lines, int64(l))
}
}
}
// TODO: Refactor so as not to have copy/pasted code with above dir handler
select {
case <-cancelledc:
log.Println("user cancelled, dropping files.")
case <-r.filesTM.Dying():
dropped := len(m.files) - i
log.Print("WARNING a tenet review timed out waiting for files to be sent. %d files dropped", dropped)
break z
case r.filesc <- fileinfo:
}
}
}
}
for _, r := range reviews {
// this says all files have been sent. For this review.
r.filesTM.Done()
}
// wait for all tenets to be cleaned up.
cleanupWG.Wait()
// Closing this chan will start the wind down to end the lingo
// process.
close(reviewChannel)
}()
return reviewChannel, cancelledc
}
// confirm returns true if the issue should be kept or false if it should be
// dropped.
func (c IssueConfirmer) Confirm(attempt int, issue *tenet.Issue) bool {
if c.keepAll {
return true
}
if attempt == 0 {
fmt.Println(c.FormatPlainText(issue))
}
attempt++
var options string
fmt.Print("\n[o]pen")
if c.output {
fmt.Print(" [d]iscard [K]eep")
}
fmt.Print(": ")
fmt.Scanln(&options)
// if err != nil || n != 1 {
// // TODO(waigani) handle invalid input
// fmt.Println("invalid input", n, err)
// }
switch options {
case "o":
var app string
defaultEditor := "vi" // TODO(waigani) is vi an okay default?
if editor != "" {
defaultEditor = editor
}
fmt.Printf("application (%s):", defaultEditor)
fmt.Scanln(&app)
filename := issue.Position.Start.Filename
if app == "" {
app = defaultEditor
}
// c := issue.Position.Start.Column // TODO(waigani) use column
l := issue.Position.Start.Line
cmd, err := util.OpenFileCmd(app, filename, l)
if err != nil {
fmt.Println(err)
return c.Confirm(attempt, issue)
}
if err = cmd.Start(); err != nil {
log.Println(err)
}
if err = cmd.Wait(); err != nil {
log.Println(err)
}
editor = app
c.Confirm(attempt, issue)
case "d":
issue.Discard = true
// TODO(waigani) only prompt for reason if we're sending to a service.
fmt.Print("reason: ")
in := bufio.NewReader(os.Stdin)
issue.DiscardReason, _ = in.ReadString('\n')
// TODO(waigani) we are now always returning true. Returning a bool at
// all doesn't make sense and KeptIssues in commands/common.go should
// be renamed to "AllIssues" or the like.
return true
case "", "k", "K", "\n":
return true
default:
fmt.Printf("invalid input: %s\n", options)
fmt.Println(options)
c.Confirm(attempt, issue)
}
return true
}