// doTest checks that Sample picks items evenly. "n" samples are
// drawn from a list of numbers of length "space". This is repeated
// "iters" times. The number of times a particular number is drawn
// should be within "tolerance" of the expected number.
func doTest(t *testing.T, n, space, iters int, tolerance float64) {
count := make([]int, space)
for i := 0; i < iters; i++ {
s := stream.Sequence(
stream.Numbers(0, space-1),
stream.SampleWithSeed(n, int64(i)),
)
stream.ForEach(s, func(s string) {
num := -1 // Will cause panic below if Scan fails
fmt.Sscan(s, &num)
count[num]++
})
}
// Check that all counts are approximately equal.
expected := (float64(iters) * float64(n)) / float64(space)
minExpected := expected * (1.0 - tolerance)
maxExpected := expected * (1.0 + tolerance)
for i, n := range count {
if float64(n) < minExpected || float64(n) > maxExpected {
t.Errorf("%d has %d samples; expected range [%f,%f]\n",
i, n, minExpected, maxExpected)
}
}
}
func ExampleForEach() {
err := stream.ForEach(stream.Numbers(1, 5), func(s string) {
fmt.Print(s)
})
if err != nil {
panic(err)
}
// Output:
// 12345
}
func ExampleSequence() {
stream.ForEach(stream.Sequence(
stream.Numbers(1, 25),
stream.Grep("3"),
), func(s string) { fmt.Println(s) })
// Output:
// 3
// 13
// 23
}
func TestStyle(t *testing.T) {
pkg, err := build.Import(cockroachDB, "", build.FindOnly)
if err != nil {
t.Skip(err)
}
t.Run("TestCopyrightHeaders", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-LE", `^// (Copyright|Code generated by)`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Errorf(`%s <- missing license header`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestMissingLeakTest", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "util/leaktest/check-leaktest.sh")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestTabsInShellScripts", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nF", "\t", "--", "*.sh")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Errorf(`%s <- tab detected, use spaces instead`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestEnvutil", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `os\.(Getenv|LookupEnv)`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`^cmd/`),
stream.GrepNot(`^(build/style_test\.go|((util/(log|envutil|sdnotify))|acceptance(/.*)?)/\w+\.go)\b`),
), func(s string) {
t.Errorf(`%s <- forbidden; use "envutil" instead`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestSyncutil", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `sync\.(RW)?Mutex`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`^util/syncutil/mutex_sync\.go\b`),
), func(s string) {
t.Errorf(`%s <- forbidden; use "syncutil.{,RW}Mutex" instead`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestTimeutil", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `time\.(Now|Since)`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`^util/(log|syncutil|timeutil)/\w+\.go\b`),
), func(s string) {
t.Errorf(`%s <- forbidden; use "timeutil" instead`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestGrpc", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `grpc.NewServer\([^)]*\)`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`^rpc/context(_test)?\.go\b`),
), func(s string) {
t.Errorf(`%s <- forbidden; use "rpc.NewServer" instead`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestProtoClone", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `\.Clone\([^)]+\)`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`protoutil\.Clone\([^)]+\)`),
stream.GrepNot(`^util/protoutil/clone(_test)?\.go\b`),
), func(s string) {
t.Errorf(`%s <- forbidden; use "protoutil.Clone" instead`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestProtoMarshal", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `\.Marshal\([^)]+\)`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`(json|yaml|protoutil|Field)\.Marshal`),
stream.GrepNot(`^util/protoutil/marshal(_test)?\.go\b`),
), func(s string) {
t.Errorf(`%s <- forbidden; use "protoutil.Marshal" instead`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestImportNames", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `^(import|\s+)(\w+ )?"database/sql"$`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`gosql "database/sql"`),
), func(s string) {
t.Errorf(`%s <- forbidden; import "database/sql" as "gosql" to avoid confusion with "cockroach/sql"`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestMisspell", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "ls-files")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.Map(func(s string) string {
return filepath.Join(pkg.Dir, s)
}),
stream.Xargs("misspell"),
), func(s string) {
t.Errorf(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestGofmtSimplify", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "gofmt", "-s", "-d", "-l", ".")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestCrlfmt", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "crlfmt", "-ignore", `\.pb(\.gw)?\.go`, "-tab", "2", ".")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
if t.Failed() {
args := append([]string(nil), cmd.Args[1:len(cmd.Args)-1]...)
args = append(args, "-w", ".")
for i := range args {
args[i] = strconv.Quote(args[i])
}
t.Logf("run the following to fix your formatting:\n"+
"\n%s %s\n\n"+
"Don't forget to add amend the result to the correct commits.",
cmd.Args[0], strings.Join(args, " "),
)
}
})
t.Run("TestVet", func(t *testing.T) {
t.Parallel()
// `go tool vet` is a special snowflake that emits all its output on
// `stderr.
cmd := exec.Command("go", "tool", "vet", "-all", "-shadow", "-printfuncs",
strings.Join([]string{
"Info:1",
"Infof:1",
"InfofDepth:2",
"Warning:1",
"Warningf:1",
"WarningfDepth:2",
"Error:1",
"Errorf:1",
"ErrorfDepth:2",
"Fatal:1",
"Fatalf:1",
"FatalfDepth:2",
"Event:1",
"Eventf:1",
"ErrEvent:1",
"ErrEventf:1",
"VEvent:2",
"VEventf:2",
"UnimplementedWithIssueErrorf:1",
}, ","),
".",
)
cmd.Dir = pkg.Dir
var b bytes.Buffer
cmd.Stdout = &b
cmd.Stderr = &b
switch err := cmd.Run(); err.(type) {
case nil:
case *exec.ExitError:
// Non-zero exit is expected.
default:
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
stream.FilterFunc(func(arg stream.Arg) error {
scanner := bufio.NewScanner(&b)
for scanner.Scan() {
arg.Out <- scanner.Text()
}
return scanner.Err()
}),
stream.GrepNot(`declaration of "?(pE|e)rr"? shadows`),
stream.GrepNot(`\.pb\.gw\.go:[0-9]+: declaration of "?ctx"? shadows`),
), func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
})
t.Run("TestUnused", func(t *testing.T) {
t.Parallel()
// NB: this doesn't use `pkgScope` because `unused` produces many false
// positives unless it inspects all our packages.
cmd, stderr, filter, err := dirCmd(pkg.Dir, "unused", "-reflect=false", "-exported", "./...")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`pkg/sql/(pgwire/pgerror/codes.go|parser/yacc(par|tab))|(field no|type No)Copy `),
), func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestForbiddenImports", func(t *testing.T) {
t.Parallel()
filter := stream.FilterFunc(func(arg stream.Arg) error {
for _, useAllFiles := range []bool{false, true} {
buildContext := build.Default
buildContext.CgoEnabled = true
buildContext.UseAllFiles = useAllFiles
outer:
for path := range buildutil.ExpandPatterns(&buildContext, []string{cockroachDB + "/..."}) {
importPkg, err := buildContext.Import(path, pkg.Dir, 0)
switch err.(type) {
case nil:
for _, s := range importPkg.Imports {
arg.Out <- importPkg.ImportPath + ": " + s
}
for _, s := range importPkg.TestImports {
arg.Out <- importPkg.ImportPath + ": " + s
}
for _, s := range importPkg.XTestImports {
arg.Out <- importPkg.ImportPath + ": " + s
}
case *build.NoGoError:
case *build.MultiplePackageError:
if useAllFiles {
continue outer
}
default:
return errors.Wrapf(err, "error loading package %s", path)
}
}
}
return nil
})
if err := stream.ForEach(stream.Sequence(
filter,
stream.Sort(),
stream.Uniq(),
stream.GrepNot(`cockroach/pkg/cmd/`),
stream.Grep(` (github\.com/golang/protobuf/proto|github\.com/satori/go\.uuid|log|path|context)$`),
stream.GrepNot(`cockroach/pkg/(base|security|util/(log|randutil|stop)): log$`),
stream.GrepNot(`cockroach/pkg/(server/serverpb|ts/tspb): github\.com/golang/protobuf/proto$`),
stream.GrepNot(`cockroach/pkg/util/uuid: github\.com/satori/go\.uuid$`),
), func(s string) {
if strings.HasSuffix(s, " path") {
t.Errorf(`%s <- please use "path/filepath" instead of "path"`, s)
}
if strings.HasSuffix(s, " log") {
t.Errorf(`%s <- please use "util/log" instead of "log"`, s)
}
if strings.HasSuffix(s, " github.com/golang/protobuf/proto") {
t.Errorf(`%s <- please use "github.com/gogo/protobuf/proto" instead of "github.com/golang/protobuf/proto"`, s)
}
if strings.HasSuffix(s, " github.com/satori/go.uuid") {
t.Errorf(`%s <- please use "util/uuid" instead of "github.com/satori/go.uuid"`, s)
}
if strings.HasSuffix(s, " context") {
t.Errorf(`%s <- please use "golang.org/x/net/context" instead of "context"`, s)
}
}); err != nil {
t.Error(err)
}
})
// Things that are packaged scoped are below here.
pkgScope, ok := os.LookupEnv("PKG")
if !ok {
pkgScope = "./..."
}
t.Run("TestErrCheck", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "errcheck", "-ignore", "bytes:Write.*,io:Close,net:Close,net/http:Close,net/rpc:Close,os:Close,database/sql:Close", pkgScope)
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Errorf(`%s <- unchecked error`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestReturnCheck", func(t *testing.T) {
t.Skip("TODO(dt): need to update this upstream or pull it into repo")
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "returncheck", pkgScope)
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Errorf(`%s <- unchecked error`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestGolint", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "golint", pkgScope)
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`((\.pb|\.pb\.gw|embedded|_string)\.go|sql/parser/(yaccpar|sql\.y):)`),
), func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestUnconvert", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "unconvert", pkgScope)
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`\.pb\.go:`),
), func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestMetacheck", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(
pkg.Dir,
"metacheck",
"-ignore",
strings.Join([]string{
"github.com/cockroachdb/cockroach/pkg/security/securitytest/embedded.go:S1013",
"github.com/cockroachdb/cockroach/pkg/ui/embedded.go:S1013",
// Intentionally compare an unsigned integer <= 0 to avoid knowledge
// of the type at the caller and for consistency with convention.
"github.com/cockroachdb/cockroach/pkg/storage/replica.go:SA4003",
// Allow a comment to refer to an "unused" argument.
//
// TODO(bdarnell): remove when/if #8360 is fixed.
"github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:SA4009",
// Loop intentionally exits unconditionally; it's cleaner than
// explicitly checking length and extracting the first element.
"github.com/cockroachdb/cockroach/pkg/sql/errors.go:SA4004",
// A value assigned to a variable is never read; this might be worth
// investigating, but it's cumbersome because the reported file
// differs from the source.
//
// Reported as pkg/sql/parser/yaccpar, but the real file is sql.y.
"github.com/cockroachdb/cockroach/pkg/sql/parser/sql.y:SA4006",
}, " "),
pkgScope,
)
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
}