// Capacity queries the underlying file system for disk capacity information. func (r *RocksDB) Capacity() (roachpb.StoreCapacity, error) { fileSystemUsage := gosigar.FileSystemUsage{} dir := r.dir if dir == "" { dir = "/tmp" } if err := fileSystemUsage.Get(dir); err != nil { return roachpb.StoreCapacity{}, err } if fileSystemUsage.Total > math.MaxInt64 { return roachpb.StoreCapacity{}, fmt.Errorf("unsupported disk size %s, max supported size is %s", humanize.IBytes(fileSystemUsage.Total), util.IBytes(math.MaxInt64)) } if fileSystemUsage.Avail > math.MaxInt64 { return roachpb.StoreCapacity{}, fmt.Errorf("unsupported disk size %s, max supported size is %s", humanize.IBytes(fileSystemUsage.Avail), util.IBytes(math.MaxInt64)) } fsuTotal := int64(fileSystemUsage.Total) fsuAvail := int64(fileSystemUsage.Avail) // If no size limitation have been placed on the store size or if the // limitation is greater than what's available, just return the actual // totals. if r.maxSize == 0 || r.maxSize >= fsuTotal || r.dir == "" { return roachpb.StoreCapacity{ Capacity: fsuTotal, Available: fsuAvail, }, nil } // Find the total size of all the files in the r.dir and all its // subdirectories. var totalUsedBytes int64 if errOuter := filepath.Walk(r.dir, func(path string, info os.FileInfo, err error) error { if err != nil { return nil } if info.Mode().IsRegular() { totalUsedBytes += info.Size() } return nil }); errOuter != nil { return roachpb.StoreCapacity{}, errOuter } available := r.maxSize - totalUsedBytes if available > fsuAvail { available = fsuAvail } if available < 0 { available = 0 } return roachpb.StoreCapacity{ Capacity: r.maxSize, Available: available, }, nil }
// Open creates options and opens the database. If the database // doesn't yet exist at the specified directory, one is initialized // from scratch. The RocksDB Open and Close methods are reference // counted such that subsequent Open calls to an already opened // RocksDB instance only bump the reference count. The RocksDB is only // closed when a sufficient number of Close calls are performed to // bring the reference count down to 0. func (r *RocksDB) Open() error { if r.rdb != nil { return nil } if r.memtableBudget < minMemtableBudget { return util.Errorf("memtable budget must be at least %s: %s", humanize.IBytes(minMemtableBudget), util.IBytes(r.memtableBudget)) } if len(r.dir) != 0 { log.Infof("opening rocksdb instance at %q", r.dir) } status := C.DBOpen(&r.rdb, goToCSlice([]byte(r.dir)), C.DBOptions{ cache_size: C.uint64_t(r.cacheSize), memtable_budget: C.uint64_t(r.memtableBudget), allow_os_buffer: C.bool(true), logging_enabled: C.bool(log.V(3)), }) err := statusToError(status) if err != nil { return util.Errorf("could not open rocksdb instance: %s", err) } // Start a goroutine that will finish when the underlying handle // is deallocated. This is used to check a leak in tests. go func() { <-r.deallocated }() r.stopper.AddCloser(r) return nil }
// String returns a fully parsable version of the store spec. func (ss StoreSpec) String() string { var buffer bytes.Buffer if len(ss.Path) != 0 { fmt.Fprintf(&buffer, "path=%s,", ss.Path) } if ss.InMemory { fmt.Fprint(&buffer, "type=mem,") } if ss.SizeInBytes > 0 { fmt.Fprintf(&buffer, "size=%s,", util.IBytes(ss.SizeInBytes)) } if ss.SizePercent > 0 { fmt.Fprintf(&buffer, "size=%s%%,", humanize.Ftoa(ss.SizePercent)) } if len(ss.Attributes.Attrs) > 0 { fmt.Fprint(&buffer, "attrs=") for i, attr := range ss.Attributes.Attrs { if i != 0 { fmt.Fprint(&buffer, ":") } fmt.Fprintf(&buffer, attr) } fmt.Fprintf(&buffer, ",") } // Trim the extra comma from the end if it exists. if l := buffer.Len(); l > 0 { buffer.Truncate(l - 1) } return buffer.String() }
// InitStores initializes ctx.Engines based on ctx.Stores. func (ctx *Context) InitStores(stopper *stop.Stopper) error { // TODO(peter): The comments and docs say that CacheSize and MemtableBudget // are split evenly if there are multiple stores, but we aren't doing that // currently. See #4979 and #4980. for _, spec := range ctx.Stores.Specs { var sizeInBytes = spec.SizeInBytes if spec.InMemory { if spec.SizePercent > 0 { sysMem, err := GetTotalMemory() if err != nil { return fmt.Errorf("could not retrieve system memory") } sizeInBytes = int64(float64(sysMem) * spec.SizePercent / 100) } if sizeInBytes != 0 && sizeInBytes < minimumStoreSize { return fmt.Errorf("%f%% of memory is only %s bytes, which is below the minimum requirement of %s", spec.SizePercent, util.IBytes(sizeInBytes), util.IBytes(minimumStoreSize)) } ctx.Engines = append(ctx.Engines, engine.NewInMem(spec.Attributes, sizeInBytes, stopper)) } else { if spec.SizePercent > 0 { fileSystemUsage := gosigar.FileSystemUsage{} if err := fileSystemUsage.Get(spec.Path); err != nil { return err } sizeInBytes = int64(float64(fileSystemUsage.Total) * spec.SizePercent / 100) } if sizeInBytes != 0 && sizeInBytes < minimumStoreSize { return fmt.Errorf("%f%% of %s's total free space is only %s bytes, which is below the minimum requirement of %s", spec.SizePercent, spec.Path, util.IBytes(sizeInBytes), util.IBytes(minimumStoreSize)) } ctx.Engines = append(ctx.Engines, engine.NewRocksDB(spec.Attributes, spec.Path, ctx.CacheSize/int64(len(ctx.Stores.Specs)), ctx.MemtableBudget, sizeInBytes, stopper)) } } if len(ctx.Engines) == 1 { log.Infof("1 storage engine initialized") } else { log.Infof("%d storage engines initialized", len(ctx.Engines)) } return nil }
// GetTotalMemory returns either the total system memory or if possible the // cgroups available memory. func GetTotalMemory() (int64, error) { mem := gosigar.Mem{} if err := mem.Get(); err != nil { return 0, err } if mem.Total > math.MaxInt64 { return 0, fmt.Errorf("inferred memory size %s exceeds maximum supported memory size %s", humanize.IBytes(mem.Total), humanize.Bytes(math.MaxInt64)) } totalMem := int64(mem.Total) if runtime.GOOS == "linux" { var err error var buf []byte if buf, err = ioutil.ReadFile(defaultCGroupMemPath); err != nil { if log.V(1) { log.Infof("can't read available memory from cgroups (%s), using system memory %s instead", err, util.IBytes(totalMem)) } return totalMem, nil } var cgAvlMem uint64 if cgAvlMem, err = strconv.ParseUint(strings.TrimSpace(string(buf)), 10, 64); err != nil { if log.V(1) { log.Infof("can't parse available memory from cgroups (%s), using system memory %s instead", err, util.IBytes(totalMem)) } return totalMem, nil } if cgAvlMem > math.MaxInt64 { if log.V(1) { log.Infof("available memory from cgroups is too large and unsupported %s using system memory %s instead", humanize.IBytes(cgAvlMem), util.IBytes(totalMem)) } return totalMem, nil } return int64(cgAvlMem), nil } return totalMem, nil }
func (b *bytesValue) String() string { // This uses the MiB, GiB, etc suffixes. If we use humanize.Bytes() we get // the MB, GB, etc suffixes, but the conversion is done in multiples of 1000 // vs 1024. return util.IBytes(*b.val) }
// newStoreSpec parses the string passed into a --store flag and returns a // StoreSpec if it is correctly parsed. // There are four possible fields that can be passed in, comma separated: // - path=xxx The directory in which to the rocks db instance should be // located, required unless using a in memory storage. // - type=mem This specifies that the store is an in memory storage instead of // an on disk one. mem is currently the only other type available. // - size=xxx The optional maximum size of the storage. This can be in one of a // few different formats. // - 10000000000 -> 10000000000 bytes // - 20GB -> 20000000000 bytes // - 20GiB -> 21474836480 bytes // - 0.02TiB -> 21474836480 bytes // - 20% -> 20% of the available space // - 0.2 -> 20% of the available space // - attrs=xxx:yyy:zzz A colon separated list of optional attributes. // Note that commas are forbidden within any field name or value. func newStoreSpec(value string) (StoreSpec, error) { if len(value) == 0 { return StoreSpec{}, fmt.Errorf("no value specified") } var ss StoreSpec used := make(map[string]struct{}) for _, split := range strings.Split(value, ",") { if len(split) == 0 { continue } subSplits := strings.SplitN(split, "=", 2) var field string var value string if len(subSplits) == 1 { field = "path" value = subSplits[0] } else { field = strings.ToLower(subSplits[0]) value = subSplits[1] } if _, ok := used[field]; ok { return StoreSpec{}, fmt.Errorf("%s field was used twice in store definition", field) } used[field] = struct{}{} if len(field) == 0 { continue } if len(value) == 0 { return StoreSpec{}, fmt.Errorf("no value specified for %s", field) } switch field { case "path": if len(value) == 0 { } ss.Path = value case "size": if len(value) == 0 { return StoreSpec{}, fmt.Errorf("no size specified") } if unicode.IsDigit(rune(value[len(value)-1])) && (strings.HasPrefix(value, "0.") || strings.HasPrefix(value, ".")) { // Value is a percentage without % sign. var err error ss.SizePercent, err = strconv.ParseFloat(value, 64) ss.SizePercent *= 100 if err != nil { return StoreSpec{}, fmt.Errorf("could not parse store size (%s) %s", value, err) } if ss.SizePercent > 100 || ss.SizePercent < 1 { return StoreSpec{}, fmt.Errorf("store size (%s) must be between 1%% and 100%%", value) } } else if strings.HasSuffix(value, "%") { // Value is a percentage. var err error ss.SizePercent, err = strconv.ParseFloat(value[:len(value)-1], 64) if err != nil { return StoreSpec{}, fmt.Errorf("could not parse store size (%s) %s", value, err) } if ss.SizePercent > 100 || ss.SizePercent < 1 { return StoreSpec{}, fmt.Errorf("store size (%s) must be between 1%% and 100%%", value) } } else { var err error ss.SizeInBytes, err = util.ParseBytes(value) if err != nil { return StoreSpec{}, fmt.Errorf("could not parse store size (%s) %s", value, err) } if ss.SizeInBytes < minimumStoreSize { return StoreSpec{}, fmt.Errorf("store size (%s) must be larger than %s", value, util.IBytes(minimumStoreSize)) } } case "attrs": if len(value) == 0 { return StoreSpec{}, fmt.Errorf("no attributes specified") } // Check to make sure there are no duplicate attributes. attrMap := make(map[string]struct{}) for _, attribute := range strings.Split(value, ":") { if _, ok := attrMap[attribute]; ok { return StoreSpec{}, fmt.Errorf("duplicate attribute given for store: %s", attribute) } attrMap[attribute] = struct{}{} } for attribute := range attrMap { ss.Attributes.Attrs = append(ss.Attributes.Attrs, attribute) } sort.Strings(ss.Attributes.Attrs) case "type": if value == "mem" { ss.InMemory = true } else { return StoreSpec{}, fmt.Errorf("%s is not a valid store type", value) } default: return StoreSpec{}, fmt.Errorf("%s is not a valid store field", field) } } if ss.InMemory { // Only in memory stores don't need a path and require a size. if ss.Path != "" { return StoreSpec{}, fmt.Errorf("path specified for in memory store") } if ss.SizePercent == 0 && ss.SizeInBytes == 0 { return StoreSpec{}, fmt.Errorf("size must be specified for an in memory store") } } else if ss.Path == "" { return StoreSpec{}, fmt.Errorf("no path specified") } return ss, nil }