func (self *ConstantOccupancyWorkload) Init(p Properties) (err error) {
catch(&err)
propStr := p.GetDefault(PropertyDiskSize, PropertyDiskSizeDefault)
diskSize, err := strconv.ParseInt(propStr, 0, 64)
try(err)
propStr = p.GetDefault(PropertyStorageAge, PropertyStorageAgeDefault)
storageAges, err := strconv.ParseInt(propStr, 0, 64)
try(err)
propStr = p.GetDefault(PropertyOccupancy, PropertyOccupancyDefault)
occupancy, err := strconv.ParseFloat(propStr, 64)
try(err)
_, ok1 := p[PropertyRecordCount]
_, ok2 := p[PropertyInsertCount]
_, ok3 := p[PropertyOperationCount]
if ok1 || ok2 || ok3 {
Warnf("Warning: record, insert or operation count was set prior to initting ConstantOccupancyWorkload. Overriding old values.")
}
gen, err := self.CoreWorkload.getFieldLengthGenerator(p)
try(err)
fieldSize := gen.Mean()
propStr = p.GetDefault(PropertyFieldCount, PropertyFieldCountDefault)
fieldCount, err := strconv.ParseInt(propStr, 0, 64)
try(err)
objectCount := int64(occupancy * (float64(diskSize) / (fieldSize * float64(fieldCount))))
if objectCount == 0 {
try(g.NewErrorf("Object count was zero. Perhaps diskSize is too low?"))
}
p.Add(PropertyRecordCount, fmt.Sprintf("%d", objectCount))
p.Add(PropertyOperationCount, fmt.Sprintf("%d", storageAges*objectCount))
p.Add(PropertyInsertCount, fmt.Sprintf("%d", objectCount))
try(self.CoreWorkload.Init(p))
return
}
func (self *CoreWorkload) getFieldLengthGenerator(p Properties) (g.IntegerGenerator, error) {
var fieldLengthGenerator g.IntegerGenerator
fieldLengthDistribution := p.GetDefault(PropertyFieldLengthDistribution, PropertyFieldLengthDistributionDefault)
propStr := p.GetDefault(PropertyFieldLength, PropertyFieldCountDefault)
fieldLength, err := strconv.ParseInt(propStr, 0, 64)
if err != nil {
return nil, err
}
fieldLengthHistogramFile := p.GetDefault(PropertyFieldLengthHistogramFile, PropertyFieldLengthHistogramFileDefault)
switch fieldLengthDistribution {
case "constant":
fieldLengthGenerator = g.NewConstantIntegerGenerator(fieldLength)
case "uniform":
fieldLengthGenerator = g.NewUniformIntegerGenerator(1, fieldLength)
case "zipfian":
fieldLengthGenerator = g.NewZipfianGeneratorByInterval(1, fieldLength-1)
case "histogram":
fieldLengthGenerator, err = g.NewHistogramGeneratorFromFile(fieldLengthHistogramFile)
if err != nil {
return nil, err
}
default:
return nil, g.NewErrorf("unknown field length distribution %s", fieldLengthDistribution)
}
return fieldLengthGenerator, nil
}
func NewWorkload(className string) (Workload, error) {
f, ok := Workloads[className]
if !ok {
return nil, g.NewErrorf("unsupported workload: %s", className)
}
w := f()
return w, nil
}
func NewMeasurementExporter(className string, w io.WriteCloser) (MeasurementExporter, error) {
f, ok := MeasurementExporters[className]
if !ok {
return nil, g.NewErrorf("unsupported measurement exporter: %s", className)
}
e := f(w)
return e, nil
}
func NewDB(database string, props Properties) (DB, error) {
f, ok := Databases[database]
if !ok {
return nil, g.NewErrorf("unsupported database: %s", database)
}
db := f()
db.SetProperties(props)
return NewDBWrapper(db), nil
}
func LoadProperties(fileName string) (Properties, error) {
ret := NewProperties()
f, err := os.Open(fileName)
if err != nil {
return ret, err
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
if regexIgnorable.MatchString(line) {
continue
}
parts := regexIgnorable.FindAllString(line, -1)
if parts == nil {
return ret, g.NewErrorf("invalid workload file: %s, line: %s", fileName, line)
}
ret.Add(parts[0], parts[1])
}
return ret, scanner.Err()
}
// Initialize the scenario.
// Called once, in the main routine, before any operations are started.
func (self *CoreWorkload) Init(p Properties) error {
table := p.GetDefault(PropertyTableName, PropertyTableNameDefault)
propStr := p.GetDefault(PropertyFieldCount, PropertyFieldCountDefault)
fieldCount, err := strconv.ParseInt(propStr, 0, 64)
if err != nil {
return err
}
fieldPrefix := p.GetDefault(PropertyFieldPrefix, PropertyFieldPrefixDefault)
fieldNames := make([]string, 0, fieldCount)
for i := int64(0); i < fieldCount; i++ {
fieldNames = append(fieldNames, fmt.Sprintf("%s%d", fieldPrefix, i))
}
fieldLengthGenerator, err := self.getFieldLengthGenerator(p)
if err != nil {
return err
}
propStr = p.GetDefault(PropertyReadProportion, PropertyReadProportionDefault)
readProportion, err := strconv.ParseFloat(propStr, 64)
if err != nil {
return err
}
propStr = p.GetDefault(PropertyUpdateProportion, PropertyUpdateProportionDefault)
updateProportion, err := strconv.ParseFloat(propStr, 64)
if err != nil {
return err
}
propStr = p.GetDefault(PropertyInsertProportion, PropertyInsertProportionDefault)
insertProportion, err := strconv.ParseFloat(propStr, 64)
if err != nil {
return err
}
propStr = p.GetDefault(PropertyScanProportion, PropertyScanProportionDefault)
scanProportion, err := strconv.ParseFloat(propStr, 64)
if err != nil {
return err
}
propStr = p.GetDefault(PropertyReadModifyWriteProportion, PropertyReadModifyWriteProportionDefault)
readModifyWriteProportion, err := strconv.ParseFloat(propStr, 64)
if err != nil {
return err
}
propStr = p.GetDefault(PropertyRecordCount, PropertyRecordCountDefault)
recordCount, err := strconv.ParseInt(propStr, 0, 64)
if err != nil {
return err
}
if recordCount == 0 {
recordCount = math.MaxInt32
}
requestDistrib := p.GetDefault(PropertyRequestDistribution, PropertyRequestDistributionDefault)
propStr = p.GetDefault(PropertyMaxScanLength, PropertyMaxScanLengthDefault)
maxScanLength, err := strconv.ParseInt(propStr, 0, 64)
if err != nil {
return err
}
scanLengthDistrib := p.GetDefault(PropertyScanLengthDistribution, PropertyScanLengthDistributionDefault)
propStr = p.GetDefault(PropertyInsertStart, PropertyInsertStartDefault)
insertStart, err := strconv.ParseInt(propStr, 0, 64)
if err != nil {
return err
}
propStr = p.GetDefault(PropertyReadAllFields, PropertyReadAllFieldsDefault)
readAllFields, err := strconv.ParseBool(propStr)
if err != nil {
return err
}
propStr = p.GetDefault(PropertyWriteAllFields, PropertyWriteAllFieldsDefault)
writeAllFields, err := strconv.ParseBool(propStr)
if err != nil {
return err
}
propStr = p.GetDefault(PropertyDataIntegrity, PropertyDataIntegrityDefault)
dataIntegrity, err := strconv.ParseBool(propStr)
if err != nil {
return err
}
propStr = p.GetDefault(PropertyFieldLengthDistribution, PropertyFieldLengthDistributionDefault)
isConstant := (propStr == "constant")
// Confirm that fieldLengthGenerator returns a constant if data integrity check requested.
if dataIntegrity && isConstant {
return g.NewErrorf("must have constant field size to check data integrity")
}
propStr = p.GetDefault(PropertyInsertOrder, PropertyInsertOrderDefault)
var orderedInserts bool
var keyChooser g.IntegerGenerator
if propStr == "hashed" {
orderedInserts = false
} else if requestDistrib == "exponential" {
propStr = p.GetDefault(PropertyExponentialPercentile, PropertyExponentialPercentileDefault)
percentile, err := strconv.ParseFloat(propStr, 64)
if err != nil {
return err
}
propStr = p.GetDefault(PropertyExponentialFraction, PropertyExponentialFractionDefault)
fraction, err := strconv.ParseFloat(propStr, 64)
if err != nil {
return err
}
keyChooser = g.NewExponentialGenerator(percentile, float64(recordCount)*fraction)
} else {
orderedInserts = true
}
keyPrefix := p.GetDefault(PropertyKeyPrefix, PropertyKeyPrefixDefault)
keySequence := g.NewCounterGenerator(insertStart)
operationChooser := g.NewDiscreteGenerator()
if readProportion > 0 {
operationChooser.AddValue(readProportion, "READ")
}
if updateProportion > 0 {
operationChooser.AddValue(updateProportion, "UPDATE")
}
if insertProportion > 0 {
operationChooser.AddValue(insertProportion, "INSERT")
}
if scanProportion > 0 {
operationChooser.AddValue(scanProportion, "SCAN")
}
if readModifyWriteProportion > 0 {
operationChooser.AddValue(readModifyWriteProportion, "READMODIFYWRITE")
}
transactionInsertKeySequence := g.NewAcknowledgedCounterGenerator(recordCount)
switch requestDistrib {
case "uniform":
keyChooser = g.NewUniformIntegerGenerator(0, recordCount-1)
case "zipfian":
// It does this by generating a random "next key" in part by
// taking the modulus over the number of keys.
// If the number of keys changes, this would shift the modulus,
// and we don't want that to change which keys are popular
// so we'll actually construct the scrambled zipfian generator
// with a keyspace that is larger than exists at the beginning
// of the test. That is, we'll predict the number of inserts, and
// tell the scrambled zipfian generator the number of existing keys
// plus the number of predicted keys as the total keyspace.
// Then, if the generator picks a key that hasn't been inserted yet,
// will just ignore it and pick another key. This way, the size of
// the keyspace doesn't change from the prespective of the scrambled
// zipfian generator.
propStr = p.GetDefault(PropertyOperationCount, "")
opCount, err := strconv.ParseInt(propStr, 0, 64)
if err != nil {
return err
}
// 2.0 is fudge factor
expectedNewKeys := int64(float64(opCount) * insertProportion * 2.0)
keyChooser = g.NewScrambledZipfianGeneratorByItems(recordCount + expectedNewKeys)
case "latest":
keyChooser = g.NewSkewedLatestGenerator(transactionInsertKeySequence.CounterGenerator)
case "hotspot":
propStr = p.GetDefault(HotspotDataFraction, HotspotDataFractionDefault)
hotSetFraction, err := strconv.ParseFloat(propStr, 64)
if err != nil {
return err
}
propStr = p.GetDefault(HotspotOpnFraction, HotspotOpnFractionDefault)
hotOpnFraction, err := strconv.ParseFloat(propStr, 64)
if err != nil {
return err
}
keyChooser = g.NewHotspotIntegerGenerator(0, recordCount-1, hotSetFraction, hotOpnFraction)
default:
return g.NewErrorf("unknown request distribution %s", requestDistrib)
}
fieldChooser := g.NewUniformIntegerGenerator(0, fieldCount-1)
var scanLengthChooser g.IntegerGenerator
switch scanLengthDistrib {
case "uniform":
scanLengthChooser = g.NewUniformIntegerGenerator(1, maxScanLength)
case "zipfian":
scanLengthChooser = g.NewZipfianGeneratorByInterval(1, maxScanLength)
default:
return g.NewErrorf("distribution %s not allowed for scan length", scanLengthDistrib)
}
propStr = p.GetDefault(InsertionRetryLimit, InsertionRetryLimitDefault)
insertionRetryLimit, err := strconv.ParseInt(propStr, 0, 64)
if err != nil {
return err
}
propStr = p.GetDefault(InsertionRetryInterval, InsertionRetryIntervalDefault)
insertionRetryInterval, err := strconv.ParseInt(propStr, 0, 64)
if err != nil {
return err
}
// set all fields
self.table = table
self.fieldCount = fieldCount
self.fieldNames = fieldNames
self.fieldLengthGenerator = fieldLengthGenerator
self.readAllFields = readAllFields
self.writeAllFields = writeAllFields
self.dataIntegrity = dataIntegrity
self.keyPrefix = keyPrefix
self.keySequence = keySequence
self.operationChooser = operationChooser
self.keyChooser = keyChooser
self.fieldChooser = fieldChooser
self.transactionInsertKeySequence = transactionInsertKeySequence
self.scanLengthChooser = scanLengthChooser
self.orderedInserts = orderedInserts
self.recordCount = recordCount
self.insertionRetryLimit = insertionRetryLimit
self.insertionRetryInterval = insertionRetryInterval
return nil
}
