// Samples 'k' unique ints from the range [0, n)
func SampleInts(n int, k int) (res []int, err error) {
if k < 0 {
err = errors.Newf("invalid sample size k")
return
}
if n < k {
err = errors.Newf("sample size k larger than n")
return
}
picked := set.NewSet()
for picked.Len() < k {
i := Intn(n)
picked.Add(i)
}
res = make([]int, k)
e := 0
for i := range picked.Iter() {
res[e] = i.(int)
e++
}
return
}
func serializeClauses(
clauses []Clause,
separator []byte,
out *bytes.Buffer) (err error) {
if clauses == nil || len(clauses) == 0 {
return errors.Newf("Empty clauses. Generated sql: %s", out.String())
}
if clauses[0] == nil {
return errors.Newf("nil clause. Generated sql: %s", out.String())
}
if err = clauses[0].SerializeSql(out); err != nil {
return
}
for _, c := range clauses[1:] {
out.Write(separator)
if c == nil {
return errors.Newf("nil clause. Generated sql: %s", out.String())
}
if err = c.SerializeSql(out); err != nil {
return
}
}
return nil
}
func (c *aliasColumn) SerializeSqlForColumnList(includeTableName bool, database Database, out *bytes.Buffer) error {
if !validIdentifierName(c.name) {
return errors.Newf(
"Invalid alias name `%s`. Generated sql: %s",
c.name,
out.String())
}
if c.expression == nil {
return errors.Newf(
"Cannot alias a nil expression. Generated sql: %s",
out.String())
}
out.WriteByte('(')
if c.expression == nil {
return errors.Newf("nil alias clause. Generate sql: %s", out.String())
}
if err := c.expression.SerializeSql(database, out); err != nil {
return err
}
out.WriteString(") AS ")
out.WriteRune(database.EscapeCharacter())
out.WriteString(c.name)
out.WriteRune(database.EscapeCharacter())
return nil
}
func (c *aliasColumn) SerializeSqlForColumnList(out *bytes.Buffer) error {
if !validIdentifierName(c.name) {
return errors.Newf(
"Invalid alias name `%s`. Generated sql: %s",
c.name,
out.String())
}
if c.expression == nil {
return errors.Newf(
"Cannot alias a nil expression. Generated sql: %s",
out.String())
}
out.WriteByte('(')
if c.expression == nil {
return errors.Newf("nil alias clause. Generate sql: %s", out.String())
}
if err := c.expression.SerializeSql(out); err != nil {
return err
}
out.WriteString(") AS `")
out.WriteString(c.name)
out.WriteByte('`')
return nil
}
func (p *baseRowsEventParser) parseRowsHeader(raw *RawV4Event) (
id uint64,
flags uint16,
extraInfo []byte,
width int,
remaining []byte,
err error) {
if p.context == nil {
return 0, 0, nil, 0, nil, &TableContextNotSetError{
errors.New("Table context not set"),
}
}
data := raw.FixedLengthData()
id = LittleEndian.Uint48(data)
if id != p.context.TableId() {
return 0, 0, nil, 0, nil, errors.Newf(
"mismatch table id (event: %d; context: %d name: %s)",
id,
p.context.TableId(),
p.context.TableName())
}
flags = LittleEndian.Uint16(data[6:])
remaining = raw.VariableLengthData()
if p.version == mysql_proto.RowsEventVersion_V1 {
extraInfo = nil
} else {
extraInfoBlobLen := LittleEndian.Uint16(data[8:])
extraInfo, remaining, err = p.parseExtraInfoBlob(
extraInfoBlobLen,
remaining)
if err != nil {
return 0, 0, nil, 0, nil, err
}
}
w, remaining, err := readFieldLength(remaining)
if err != nil {
return 0, 0, nil, 0, nil, err
}
if w > uint64(p.context.NumColumns()) {
return 0, 0, nil, 0, nil, errors.Newf(
"row width (%d / %d) greater than # of columns (%d) in table %s",
w,
width,
p.context.NumColumns(),
string(p.context.TableName()))
}
return id, flags, extraInfo, int(w), remaining, nil
}
func (s *Source) validate() (err error) {
if strings.ToLower(s.Hash) == "skip" {
return
}
file, err := os.Open(s.Path)
if err != nil {
err = &HashError{
errors.Wrap(err, "source: Failed to open file for hash"),
}
return
}
defer file.Close()
var hash hash.Hash
switch len(s.Hash) {
case 32:
hash = md5.New()
case 40:
hash = sha1.New()
case 64:
hash = sha256.New()
case 128:
hash = sha512.New()
default:
err = &HashError{
errors.Newf("source: Unknown hash type for hash '%s'", s.Hash),
}
return
}
_, err = io.Copy(hash, file)
if err != nil {
return
}
sum := hash.Sum([]byte{})
hexSum := fmt.Sprintf("%x", sum)
if hexSum != s.Hash {
err = &HashError{
errors.Newf("source: Hash verification failed for '%s'", s.Source),
}
return
}
return
}
// See ResourcePool for documentation.
func (p *MultiResourcePool) Register(resourceLocation string) error {
if resourceLocation == "" {
return errors.New("Registering invalid resource location")
}
p.rwMutex.Lock()
defer p.rwMutex.Unlock()
if p.isLameDuck {
return errors.Newf(
"Cannot register %s to lame duck resource pool",
resourceLocation)
}
if _, inMap := p.locationPools[resourceLocation]; inMap {
return nil
}
pool := p.createPool(p.options)
if err := pool.Register(resourceLocation); err != nil {
return err
}
p.locationPools[resourceLocation] = pool
return nil
}
// Writes an image to a file. The extension of the file is used to determine the
// encoding method.
func CreateImage(i image.Image, file string) error {
ext := filepath.Ext(file)
m := mime.TypeByExtension(ext)
switch m {
case "image/jpeg", "image/png", "image/tiff":
default:
return errors.Newf("unsupported extension/mime type: %s %s", ext, m)
}
f, err := os.Create(file)
if err != nil {
return errors.Wrap(err, "could not create file")
}
defer f.Close()
switch m {
case "image/jpeg":
err = jpeg.Encode(f, i, &jpeg.Options{Quality: 98})
case "image/png":
err = png.Encode(f, i)
case "image/tiff":
err = tiff.Encode(f, i, &tiff.Options{Compression: tiff.Deflate, Predictor: true})
default:
panic("unreachable")
}
if err != nil {
return errors.Wrap(err, "could not encode image")
}
return nil
}
// Peek returns a slice of the look ahead buffer which holds numBytes
// number of bytes. If the look ahead buffer does not already hold enough
// bytes, it will read from the underlying reader to populate the rest.
// NOTE: the returned slice is not a copy of the raw buffer.
func (b *LookAheadBuffer) Peek(numBytes int) ([]byte, error) {
if numBytes < 0 {
return nil, errors.New("Cannot fill negative numBytes")
}
if numBytes > len(b.buffer) {
return nil, errors.Newf(
"Buffer full (buffer size: %d n: %d)",
len(b.buffer),
numBytes)
}
var err error
var numRead int
if b.bytesBuffered < numBytes {
numRead, err = io.ReadAtLeast(
b.src,
b.buffer[b.bytesBuffered:],
numBytes-b.bytesBuffered)
if err != nil {
if err == io.ErrUnexpectedEOF {
// ErrUnexpectedEOF is a terrible error only returned by
// ReadAtLeast. Return EOF (i.e., the original error) instead
// ErrUnexpectedEOF since no one ever checks for this.
err = io.EOF
}
}
}
b.bytesBuffered += numRead
if numBytes > b.bytesBuffered {
numBytes = b.bytesBuffered
}
return b.buffer[:numBytes], err
}
// See ConnectionPool for documentation.
func (p *MultiConnectionPool) Register(network string, address string) error {
if network == "" && address == "" {
return errors.New("Registering invalid (network, address)")
}
key := NetworkAddress{
Network: network,
Address: address,
}
p.rwMutex.Lock()
defer p.rwMutex.Unlock()
if p.isLameDuck {
return errors.Newf(
"Cannot register (%s, %s) to lame duck connection pool",
network,
address)
}
if _, inMap := p.addressPools[key]; inMap {
return nil
}
pool := p.createPool(p.options)
if err := pool.Register(network, address); err != nil {
return err
}
p.addressPools[key] = pool
return nil
}
func GetPacker(pac *pack.Pack, distro, release string) (
pcker Packer, err error) {
switch constants.DistroPack[distro] {
case "pacman":
pcker = &pacman.Pacman{
Pack: pac,
}
case "debian":
pcker = &debian.Debian{
Pack: pac,
}
case "redhat":
pcker = &redhat.Redhat{
Pack: pac,
}
default:
system := distro
if release != "" {
system += "-" + release
}
err = &UnknownSystem{
errors.Newf("packer: Unkown system %s", system),
}
return
}
return
}
func (conj *conjunctExpression) SerializeSql(out *bytes.Buffer) (err error) {
if len(conj.expressions) == 0 {
return errors.Newf(
"Empty conjunction. Generated sql: %s",
out.String())
}
clauses := make([]Clause, len(conj.expressions), len(conj.expressions))
for i, expr := range conj.expressions {
clauses[i] = expr
}
use_parentheses := len(clauses) > 1
if use_parentheses {
out.WriteByte('(')
}
if err = serializeClauses(clauses, conj.conjunction, out); err != nil {
return
}
if use_parentheses {
out.WriteByte(')')
}
return nil
}
func (arith *arithmeticExpression) SerializeSql(out *bytes.Buffer) (err error) {
if len(arith.expressions) == 0 {
return errors.Newf(
"Empty arithmetic expression. Generated sql: %s",
out.String())
}
clauses := make([]Clause, len(arith.expressions), len(arith.expressions))
for i, expr := range arith.expressions {
clauses[i] = expr
}
useParentheses := len(clauses) > 1
if useParentheses {
out.WriteByte('(')
}
if err = serializeClauses(clauses, arith.operator, out); err != nil {
return
}
if useParentheses {
out.WriteByte(')')
}
return nil
}
func (*singletonSuite) TestGet(c *C) {
callCount := 0
init := func() (interface{}, error) {
callCount++
if callCount < 2 {
return nil, errors.Newf("fail")
}
v := 5
return &v, nil
}
s := NewSingleton(init)
// Verify init hasn't been called yet
c.Assert(callCount, Equals, 0)
// Verify init gets called with Get()
_, err := s.Get()
c.Assert(err, NotNil)
c.Assert(callCount, Equals, 1)
// Verify failed init gets retried with Get()
data1, err := s.Get()
c.Assert(err, IsNil)
c.Assert(callCount, Equals, 2)
c.Assert(*(data1.(*int)), Equals, 5)
// Verify that subsequent Get() after a successful one return the same object and don't call
// init
data2, err := s.Get()
c.Assert(err, IsNil)
c.Assert(callCount, Equals, 2)
c.Assert(data1, Equals, data2)
}
func Parse() (err error) {
flag.Parse()
cmd := flag.Arg(0)
switch cmd {
case "build":
err = Build()
case "create":
err = Create()
case "project":
err = Project()
case "pull":
err = utils.PullContainers()
case "genkey":
err = GenKey()
case "list-targets":
err = ListTargets()
default:
err = &UnknownCommand{
errors.Newf("cmd: Unknown command '%s'", cmd),
}
}
return
}
// Note: this is equivalent to net_field_length in sql-common/pack.c
func readFieldLength(valBytes []byte) (
length uint64,
remaining []byte,
err error) {
if len(valBytes) == 0 {
return 0, nil, errors.New("Empty field length input")
}
val := uint64(valBytes[0])
if val < 251 {
return val, valBytes[1:], nil
} else if val == 251 {
return NullLength, valBytes[1:], nil
}
size := 9
if val == 252 {
size = 3
} else if val == 253 {
size = 4
}
if len(valBytes) < size {
return 0, nil, errors.Newf(
"Invalid field length input (expected at least %d bytes)",
size)
}
// NOTE: mysql's net_field_length implementation is somewhat broken.
// In particular, when net_store_length encode a ulong using 8 bytes,
// net_field_length will only read the first 4 bytes, and ignore the
// rest ....
return bytesToLEUint(valBytes[1:size]), valBytes[size:], nil
}
func (c *inExpression) SerializeSql(out *bytes.Buffer) error {
if c.err != nil {
return errors.Wrap(c.err, "Invalid IN expression")
}
if c.lhs == nil {
return errors.Newf(
"lhs of in expression is nil. Generated sql: %s",
out.String())
}
// We'll serialize the lhs even if we don't need it to ensure no error
buf := &bytes.Buffer{}
err := c.lhs.SerializeSql(buf)
if err != nil {
return err
}
if c.rhs == nil {
out.WriteString("FALSE")
return nil
}
out.WriteString(buf.String())
out.WriteString(" IN ")
err = c.rhs.SerializeSql(out)
if err != nil {
return err
}
return nil
}
func (m *Mirror) Create() (err error) {
keyPath := filepath.Join(m.Root, "sign.key")
m.Signing, err = utils.Exists(keyPath)
if err != nil {
return
}
if m.Signing {
err = signing.ImportKey(keyPath)
if err != nil {
return
}
err = signing.CreateRedhatConf()
if err != nil {
return
}
}
switch constants.DistroPack[m.Distro] {
case "pacman":
err = m.createPacman()
case "debian":
err = m.createDebian()
case "redhat":
err = m.createRedhat()
default:
err = &UnknownType{
errors.Newf("mirror: Unknown type '%s'", m.Distro),
}
}
return
}
// ConverAssignRowNullable is the same as ConvertAssignRow except that it allows
// nil as a value for the row or any of the row values. In thoses cases, the
// corresponding values are ignored.
func ConvertAssignRowNullable(row []Value, dest ...interface{}) error {
if len(row) != len(dest) {
return errors.Newf(
"# of row entries %d does not match # of destinations %d",
len(row),
len(dest))
}
if row == nil {
return nil
}
for i := 0; i < len(row); i++ {
if row[i].IsNull() {
continue
}
err := ConvertAssign(row[i], dest[i])
if err != nil {
return err
}
}
return nil
}
func (s *lockStatementImpl) String(database string) (sql string, err error) {
if !validIdentifierName(database) {
return "", errors.New("Invalid database name specified")
}
if len(s.locks) == 0 {
return "", errors.New("No locks added")
}
buf := new(bytes.Buffer)
buf.WriteString("LOCK TABLES ")
for idx, lock := range s.locks {
if lock.t == nil {
return "", errors.Newf("nil table. Generated sql: %s", buf.String())
}
if err = lock.t.SerializeSql(database, buf); err != nil {
return
}
if lock.w {
buf.WriteString(" WRITE")
} else {
buf.WriteString(" READ")
}
if idx != len(s.locks)-1 {
buf.WriteString(", ")
}
}
return buf.String(), nil
}
// See ResourcePool for documentation.
func (p *RoundRobinResourcePool) Register(resourceLocation string) error {
if resourceLocation == "" {
return errors.New("Registering invalid resource location")
}
p.rwMutex.Lock()
defer p.rwMutex.Unlock()
if p.isLameDuck {
return errors.Newf(
"Cannot register %s to lame duck resource pool",
resourceLocation)
}
for _, locPool := range p.pools {
if locPool.ResourceLocation == resourceLocation {
return nil
}
}
pool := p.createPool(p.options)
if err := pool.Register(resourceLocation); err != nil {
return err
}
p.pools = append(
p.pools,
&ResourceLocationPool{
ResourceLocation: resourceLocation,
Pool: pool,
})
shuffle(p.pools)
return nil
}
// Returns instance that isn't marked down, if all instances are
// marked as down it will just choose a next one.
func (pool *LoadBalancedPool) getInstance() (
idx int,
instance *instancePool,
isDown bool,
err error) {
pool.lock.RLock()
defer pool.lock.RUnlock()
if len(pool.instanceList) == 0 {
return 0, nil, false, errors.Newf("no available instances")
}
now := time.Now().Unix()
for i := 0; i < len(pool.instanceList); i++ {
switch pool.strategy {
case LBRoundRobin:
// In RoundRobin strategy instanceIdx keeps changing, to
// achieve round robin load balancing.
instanceIdx := atomic.AddUint64(&pool.instanceIdx, 1)
idx = int(instanceIdx % uint64(len(pool.instanceList)))
case LBSortedFixed:
// In SortedFixed strategy instances are always traversed in same
// exact order.
idx = i
case LBShuffledFixed:
// In ShuffledFixed strategy instances are also always traversed in
// same exact order.
idx = i
}
if pool.markDownUntil[idx] < now {
break
}
}
return idx, pool.instanceList[idx], (pool.markDownUntil[idx] >= now), nil
}
func (s *unionStatementImpl) String(database string) (sql string, err error) {
// XXX(teisenbe): Once sqlproxy gets fixed, re-parenthesis the UNION selects
// We don't have any use cases where they matter, but if one were to use limits/order by with
// selects and unions, it could be problematic
if len(s.selects) == 0 {
return "", errors.Newf("Union statement must have at least one SELECT")
}
if len(s.selects) == 1 {
return s.selects[0].String(database)
}
buf := new(bytes.Buffer)
for i, statement := range s.selects {
if i != 0 {
buf.WriteString(" UNION ")
}
//buf.WriteString("(")
selectSql, err := statement.String(database)
if err != nil {
return "", err
}
buf.WriteString(selectSql)
//buf.WriteString(")")
}
return buf.String(), nil
}
func (f *FileLock) performLock(flockHow int) (performErr error) {
if f.fileH != nil {
return errors.Newf("FileLock is already acquired!")
}
filePath := path.Join(f.path, f.prefix+f.name)
// GoLang os.OpenFile creates file with FD_CLOEXEC flag already set on it.
// This means file will get closed automatically once the process exits,
// thus we dont need to manually set that flag.
fileH, err := os.Create(filePath)
if err != nil {
return err
}
defer func() {
if performErr != nil {
_ = fileH.Close()
}
}()
if err := syscall.Flock(int(fileH.Fd()), flockHow); err != nil {
return err
}
f.fileH = fileH
return nil
}
func (r *logFileV4EventReader) maybeCheckLogFormatVersion() error {
if r.passedLogFormatVersionCheck {
return nil
}
headerBytes, err := r.peekHeaderBytes(sizeOfFormatVersionHeader)
if err != nil {
return err
}
header := formatVersionHeader{}
_, err = readLittleEndian(headerBytes, &header)
if err != nil {
return err
}
if version := header.version(); version != 4 {
return errors.Newf(
"Binary log reader does not support V%d binlog format",
version)
}
r.passedLogFormatVersionCheck = true
return nil
}
func Project() (err error) {
path, err := os.Getwd()
if err != nil {
err = &FileError{
errors.Wrapf(err, "cmd: Failed to get working directory"),
}
return
}
proj := &project.Project{
Root: path,
}
err = proj.Init()
if err != nil {
return
}
cmd := flag.Arg(1)
switch cmd {
case "init":
err = proj.InitProject()
case "build":
err = proj.Build(flag.Arg(2))
case "repo":
err = proj.Repo(flag.Arg(2))
default:
err = &UnknownCommand{
errors.Newf("cmd: Unknown cmd '%s'", cmd),
}
}
return
}
func (this *watcher) doDel(path string) error {
info, found := this.watchedByPath[path]
if !found {
return errors.Newf("can't remove non-existent kevent watch for: %s", path)
}
var kbuf [1]syscall.Kevent_t
watchEntry := &kbuf[0]
syscall.SetKevent(watchEntry, info.fd, syscall.EVFILT_VNODE, syscall.EV_DELETE)
entryFlags := watchEntry.Flags
success, errno := syscall.Kevent(this.kq, kbuf[:], nil, nil)
if success == sysCallFailed {
return os.NewSyscallError("kevent_rm_watch", errno)
} else if entryFlags&syscall.EV_ERROR == syscall.EV_ERROR {
return errors.New("kevent rm error")
}
syscall.Close(info.fd)
//Remove childs if it's directory
for _, child := range info.childes {
this.doDel(child)
}
delete(this.watchedByPath, path)
delete(this.watchedByFD, info.fd)
return nil
}
// This decrements the key's counter by delta. If the counter does not
// exist, one of two things may happen:
// 1. If the expiration value is all one-bits (0xffffffff), the operation
// will fail with StatusNotFound.
// 2. For all other expiration values, the operation will succeed by
// seeding the value for this key with the provided initValue to expire
// with the provided expiration time. The flags will be set to zero.
//
// NOTE:
// 1. If you want to set the value of the counter with add/set/replace,
// the objects data must be the ascii representation of the value and
// not the byte values of a 64 bit integer.
// 2. Decrementing a counter will never result in a "negative value" (or
// cause the counter to "wrap"). instead the counter is set to 0.
func (c *MockClient) Decrement(
key string,
delta uint64,
initValue uint64,
expiration uint32) CountResponse {
return NewCountErrorResponse(key, errors.Newf("Decrement not implemented"))
}
func (r *Redhat) Build() (err error) {
err = r.makeDirs()
if err != nil {
return
}
defer r.remDirs()
err = r.createSpec([]string{})
if err != nil {
return
}
err = r.rpmBuild()
if err != nil {
return
}
files, err := r.getFiles()
if err != nil {
return
}
if len(files) == 0 {
err = &BuildError{
errors.Newf("redhat: Failed to find rpms files '%s'",
r.rpmsDir),
}
return
}
r.remDirs()
err = r.makeDirs()
if err != nil {
return
}
err = r.createSpec(files)
if err != nil {
return
}
err = r.rpmBuild()
if err != nil {
return
}
err = r.clean()
if err != nil {
return
}
err = r.copy()
if err != nil {
return
}
return
}
func (c *ShardedClient) connectionError(shard int, err error) error {
if err == nil {
return errors.Newf(
"Connection unavailable for memcache shard %d", shard)
}
return errors.Wrapf(
err,
"Connection unavailable for memcache shard %d", shard)
}