func ExampleSampleWithSeed() {
stream.Run(
stream.Numbers(1, 100),
stream.SampleWithSeed(2, 100),
stream.Sort().Num(1),
stream.WriteLines(os.Stdout),
)
// Output:
// 11
// 46
}
func ExampleSorter_By() {
stream.Run(
stream.Items("bananas", "apples", "pears"),
stream.Sort().By(func(a, b string) bool { return len(a) < len(b) }),
stream.WriteLines(os.Stdout),
)
// Output:
// pears
// apples
// bananas
}
func ExampleCommand_outputOnly() {
stream.Run(
stream.Command("find", ".", "-type", "f", "-print"),
stream.Grep(`^\./stream.*\.go$`),
stream.Sort(),
stream.WriteLines(os.Stdout),
)
// Output:
// ./stream.go
// ./stream_test.go
}
func ExampleReadLines() {
stream.Run(
stream.ReadLines(bytes.NewBufferString("the\nquick\nbrown\nfox\n")),
stream.Sort(),
stream.WriteLines(os.Stdout),
)
// Output:
// brown
// fox
// quick
// the
}
func ExampleSort() {
stream.Run(
stream.Items("banana", "apple", "cheese", "apple"),
stream.Sort(),
stream.WriteLines(os.Stdout),
)
// Output:
// apple
// apple
// banana
// cheese
}
func ExampleSorter_TextDecreasing() {
stream.Run(
stream.Items(
"10 bananas",
"20 apples",
"30", // Will sort first since column 2 is missing
),
stream.Sort().TextDecreasing(2),
stream.WriteLines(os.Stdout),
)
// Output:
// 10 bananas
// 20 apples
// 30
}
func ExampleSorter_NumDecreasing() {
stream.Run(
stream.Items(
"a 100",
"b 20",
"c notanumber",
"d",
),
stream.Sort().NumDecreasing(2),
stream.WriteLines(os.Stdout),
)
// Output:
// c notanumber
// a 100
// b 20
// d
}
func ExampleSorter_Num() {
stream.Run(
stream.Items(
"a 100",
"b 20.3",
"c notanumber", // Will sort last since column 2 is not a number
"d", // Will sort earliest since column 2 is missing
),
stream.Sort().Num(2),
stream.WriteLines(os.Stdout),
)
// Output:
// d
// b 20.3
// a 100
// c notanumber
}
func ExampleParallel() {
stream.Run(
stream.Items("hello", "there", "how", "are", "you?"),
stream.Parallel(4,
stream.Map(func(s string) string {
return fmt.Sprintf("%d %s", len(s), s)
}),
),
stream.Sort(),
stream.WriteLines(os.Stdout),
)
// Output:
// 3 are
// 3 how
// 4 you?
// 5 hello
// 5 there
}
func ExampleSort_multipleColumns() {
// Sort numerically by column 1. Break ties by sorting
// lexicographically by column 2.
stream.Run(
stream.Items(
"1970 march",
"1970 feb",
"1950 june",
"1980 sep",
),
stream.Sort().Num(1).Text(2),
stream.WriteLines(os.Stdout),
)
// Output:
// 1950 june
// 1970 feb
// 1970 march
// 1980 sep
}
func TestStyle(t *testing.T) {
pkg, err := build.Import(cockroachDB, "", build.FindOnly)
if err != nil {
t.Skip(err)
}
t.Run("TestCopyrightHeaders", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-LE", `^// (Copyright|Code generated by)`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Errorf(`%s <- missing license header`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestMissingLeakTest", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "util/leaktest/check-leaktest.sh")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestTabsInShellScripts", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nF", "\t", "--", "*.sh")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Errorf(`%s <- tab detected, use spaces instead`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestEnvutil", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `os\.(Getenv|LookupEnv)`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`^cmd/`),
stream.GrepNot(`^(build/style_test\.go|((util/(log|envutil|sdnotify))|acceptance(/.*)?)/\w+\.go)\b`),
), func(s string) {
t.Errorf(`%s <- forbidden; use "envutil" instead`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestSyncutil", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `sync\.(RW)?Mutex`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`^util/syncutil/mutex_sync\.go\b`),
), func(s string) {
t.Errorf(`%s <- forbidden; use "syncutil.{,RW}Mutex" instead`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestTimeutil", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `time\.(Now|Since)`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`^util/(log|syncutil|timeutil)/\w+\.go\b`),
), func(s string) {
t.Errorf(`%s <- forbidden; use "timeutil" instead`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestGrpc", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `grpc.NewServer\([^)]*\)`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`^rpc/context(_test)?\.go\b`),
), func(s string) {
t.Errorf(`%s <- forbidden; use "rpc.NewServer" instead`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestProtoClone", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `\.Clone\([^)]+\)`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`protoutil\.Clone\([^)]+\)`),
stream.GrepNot(`^util/protoutil/clone(_test)?\.go\b`),
), func(s string) {
t.Errorf(`%s <- forbidden; use "protoutil.Clone" instead`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestProtoMarshal", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `\.Marshal\([^)]+\)`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`(json|yaml|protoutil|Field)\.Marshal`),
stream.GrepNot(`^util/protoutil/marshal(_test)?\.go\b`),
), func(s string) {
t.Errorf(`%s <- forbidden; use "protoutil.Marshal" instead`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestImportNames", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `^(import|\s+)(\w+ )?"database/sql"$`, "--", "*.go")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`gosql "database/sql"`),
), func(s string) {
t.Errorf(`%s <- forbidden; import "database/sql" as "gosql" to avoid confusion with "cockroach/sql"`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestMisspell", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "ls-files")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.Map(func(s string) string {
return filepath.Join(pkg.Dir, s)
}),
stream.Xargs("misspell"),
), func(s string) {
t.Errorf(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestGofmtSimplify", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "gofmt", "-s", "-d", "-l", ".")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestCrlfmt", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "crlfmt", "-ignore", `\.pb(\.gw)?\.go`, "-tab", "2", ".")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
if t.Failed() {
args := append([]string(nil), cmd.Args[1:len(cmd.Args)-1]...)
args = append(args, "-w", ".")
for i := range args {
args[i] = strconv.Quote(args[i])
}
t.Logf("run the following to fix your formatting:\n"+
"\n%s %s\n\n"+
"Don't forget to add amend the result to the correct commits.",
cmd.Args[0], strings.Join(args, " "),
)
}
})
t.Run("TestVet", func(t *testing.T) {
t.Parallel()
// `go tool vet` is a special snowflake that emits all its output on
// `stderr.
cmd := exec.Command("go", "tool", "vet", "-all", "-shadow", "-printfuncs",
strings.Join([]string{
"Info:1",
"Infof:1",
"InfofDepth:2",
"Warning:1",
"Warningf:1",
"WarningfDepth:2",
"Error:1",
"Errorf:1",
"ErrorfDepth:2",
"Fatal:1",
"Fatalf:1",
"FatalfDepth:2",
"Event:1",
"Eventf:1",
"ErrEvent:1",
"ErrEventf:1",
"VEvent:2",
"VEventf:2",
"UnimplementedWithIssueErrorf:1",
}, ","),
".",
)
cmd.Dir = pkg.Dir
var b bytes.Buffer
cmd.Stdout = &b
cmd.Stderr = &b
switch err := cmd.Run(); err.(type) {
case nil:
case *exec.ExitError:
// Non-zero exit is expected.
default:
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
stream.FilterFunc(func(arg stream.Arg) error {
scanner := bufio.NewScanner(&b)
for scanner.Scan() {
arg.Out <- scanner.Text()
}
return scanner.Err()
}),
stream.GrepNot(`declaration of "?(pE|e)rr"? shadows`),
stream.GrepNot(`\.pb\.gw\.go:[0-9]+: declaration of "?ctx"? shadows`),
), func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
})
t.Run("TestUnused", func(t *testing.T) {
t.Parallel()
// NB: this doesn't use `pkgScope` because `unused` produces many false
// positives unless it inspects all our packages.
cmd, stderr, filter, err := dirCmd(pkg.Dir, "unused", "-reflect=false", "-exported", "./...")
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`pkg/sql/(pgwire/pgerror/codes.go|parser/yacc(par|tab))|(field no|type No)Copy `),
), func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestForbiddenImports", func(t *testing.T) {
t.Parallel()
filter := stream.FilterFunc(func(arg stream.Arg) error {
for _, useAllFiles := range []bool{false, true} {
buildContext := build.Default
buildContext.CgoEnabled = true
buildContext.UseAllFiles = useAllFiles
outer:
for path := range buildutil.ExpandPatterns(&buildContext, []string{cockroachDB + "/..."}) {
importPkg, err := buildContext.Import(path, pkg.Dir, 0)
switch err.(type) {
case nil:
for _, s := range importPkg.Imports {
arg.Out <- importPkg.ImportPath + ": " + s
}
for _, s := range importPkg.TestImports {
arg.Out <- importPkg.ImportPath + ": " + s
}
for _, s := range importPkg.XTestImports {
arg.Out <- importPkg.ImportPath + ": " + s
}
case *build.NoGoError:
case *build.MultiplePackageError:
if useAllFiles {
continue outer
}
default:
return errors.Wrapf(err, "error loading package %s", path)
}
}
}
return nil
})
if err := stream.ForEach(stream.Sequence(
filter,
stream.Sort(),
stream.Uniq(),
stream.GrepNot(`cockroach/pkg/cmd/`),
stream.Grep(` (github\.com/golang/protobuf/proto|github\.com/satori/go\.uuid|log|path|context)$`),
stream.GrepNot(`cockroach/pkg/(base|security|util/(log|randutil|stop)): log$`),
stream.GrepNot(`cockroach/pkg/(server/serverpb|ts/tspb): github\.com/golang/protobuf/proto$`),
stream.GrepNot(`cockroach/pkg/util/uuid: github\.com/satori/go\.uuid$`),
), func(s string) {
if strings.HasSuffix(s, " path") {
t.Errorf(`%s <- please use "path/filepath" instead of "path"`, s)
}
if strings.HasSuffix(s, " log") {
t.Errorf(`%s <- please use "util/log" instead of "log"`, s)
}
if strings.HasSuffix(s, " github.com/golang/protobuf/proto") {
t.Errorf(`%s <- please use "github.com/gogo/protobuf/proto" instead of "github.com/golang/protobuf/proto"`, s)
}
if strings.HasSuffix(s, " github.com/satori/go.uuid") {
t.Errorf(`%s <- please use "util/uuid" instead of "github.com/satori/go.uuid"`, s)
}
if strings.HasSuffix(s, " context") {
t.Errorf(`%s <- please use "golang.org/x/net/context" instead of "context"`, s)
}
}); err != nil {
t.Error(err)
}
})
// Things that are packaged scoped are below here.
pkgScope, ok := os.LookupEnv("PKG")
if !ok {
pkgScope = "./..."
}
t.Run("TestErrCheck", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "errcheck", "-ignore", "bytes:Write.*,io:Close,net:Close,net/http:Close,net/rpc:Close,os:Close,database/sql:Close", pkgScope)
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Errorf(`%s <- unchecked error`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestReturnCheck", func(t *testing.T) {
t.Skip("TODO(dt): need to update this upstream or pull it into repo")
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "returncheck", pkgScope)
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Errorf(`%s <- unchecked error`, s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestGolint", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "golint", pkgScope)
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`((\.pb|\.pb\.gw|embedded|_string)\.go|sql/parser/(yaccpar|sql\.y):)`),
), func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestUnconvert", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(pkg.Dir, "unconvert", pkgScope)
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(stream.Sequence(
filter,
stream.GrepNot(`\.pb\.go:`),
), func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
t.Run("TestMetacheck", func(t *testing.T) {
t.Parallel()
cmd, stderr, filter, err := dirCmd(
pkg.Dir,
"metacheck",
"-ignore",
strings.Join([]string{
"github.com/cockroachdb/cockroach/pkg/security/securitytest/embedded.go:S1013",
"github.com/cockroachdb/cockroach/pkg/ui/embedded.go:S1013",
// Intentionally compare an unsigned integer <= 0 to avoid knowledge
// of the type at the caller and for consistency with convention.
"github.com/cockroachdb/cockroach/pkg/storage/replica.go:SA4003",
// Allow a comment to refer to an "unused" argument.
//
// TODO(bdarnell): remove when/if #8360 is fixed.
"github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:SA4009",
// Loop intentionally exits unconditionally; it's cleaner than
// explicitly checking length and extracting the first element.
"github.com/cockroachdb/cockroach/pkg/sql/errors.go:SA4004",
// A value assigned to a variable is never read; this might be worth
// investigating, but it's cumbersome because the reported file
// differs from the source.
//
// Reported as pkg/sql/parser/yaccpar, but the real file is sql.y.
"github.com/cockroachdb/cockroach/pkg/sql/parser/sql.y:SA4006",
}, " "),
pkgScope,
)
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
if err := stream.ForEach(filter, func(s string) {
t.Error(s)
}); err != nil {
t.Error(err)
}
if err := cmd.Wait(); err != nil {
if out := stderr.String(); len(out) > 0 {
t.Fatalf("err=%s, stderr=%s", err, out)
}
}
})
}
func BenchmarkSort3(b *testing.B) {
stream.Run(
stream.Repeat("the 3 musketeers", b.N),
stream.Sort().Num(2).Text(1).Text(3),
)
}
func BenchmarkSort(b *testing.B) {
stream.Run(
stream.Repeat("hello", b.N),
stream.Sort(),
)
}