Ejemplo n.º 1
0
func (self *ConstantOccupancyWorkload) Init(p Properties) (err error) {
	catch(&err)
	propStr := p.GetDefault(PropertyDiskSize, PropertyDiskSizeDefault)
	diskSize, err := strconv.ParseInt(propStr, 0, 64)
	try(err)
	propStr = p.GetDefault(PropertyStorageAge, PropertyStorageAgeDefault)
	storageAges, err := strconv.ParseInt(propStr, 0, 64)
	try(err)
	propStr = p.GetDefault(PropertyOccupancy, PropertyOccupancyDefault)
	occupancy, err := strconv.ParseFloat(propStr, 64)
	try(err)
	_, ok1 := p[PropertyRecordCount]
	_, ok2 := p[PropertyInsertCount]
	_, ok3 := p[PropertyOperationCount]
	if ok1 || ok2 || ok3 {
		Warnf("Warning: record, insert or operation count was set prior to initting ConstantOccupancyWorkload. Overriding old values.")
	}
	gen, err := self.CoreWorkload.getFieldLengthGenerator(p)
	try(err)
	fieldSize := gen.Mean()
	propStr = p.GetDefault(PropertyFieldCount, PropertyFieldCountDefault)
	fieldCount, err := strconv.ParseInt(propStr, 0, 64)
	try(err)
	objectCount := int64(occupancy * (float64(diskSize) / (fieldSize * float64(fieldCount))))
	if objectCount == 0 {
		try(g.NewErrorf("Object count was zero. Perhaps diskSize is too low?"))
	}
	p.Add(PropertyRecordCount, fmt.Sprintf("%d", objectCount))
	p.Add(PropertyOperationCount, fmt.Sprintf("%d", storageAges*objectCount))
	p.Add(PropertyInsertCount, fmt.Sprintf("%d", objectCount))
	try(self.CoreWorkload.Init(p))
	return
}
Ejemplo n.º 2
0
func (self *CoreWorkload) getFieldLengthGenerator(p Properties) (g.IntegerGenerator, error) {
	var fieldLengthGenerator g.IntegerGenerator
	fieldLengthDistribution := p.GetDefault(PropertyFieldLengthDistribution, PropertyFieldLengthDistributionDefault)
	propStr := p.GetDefault(PropertyFieldLength, PropertyFieldCountDefault)
	fieldLength, err := strconv.ParseInt(propStr, 0, 64)
	if err != nil {
		return nil, err
	}
	fieldLengthHistogramFile := p.GetDefault(PropertyFieldLengthHistogramFile, PropertyFieldLengthHistogramFileDefault)
	switch fieldLengthDistribution {
	case "constant":
		fieldLengthGenerator = g.NewConstantIntegerGenerator(fieldLength)
	case "uniform":
		fieldLengthGenerator = g.NewUniformIntegerGenerator(1, fieldLength)
	case "zipfian":
		fieldLengthGenerator = g.NewZipfianGeneratorByInterval(1, fieldLength-1)
	case "histogram":
		fieldLengthGenerator, err = g.NewHistogramGeneratorFromFile(fieldLengthHistogramFile)
		if err != nil {
			return nil, err
		}
	default:
		return nil, g.NewErrorf("unknown field length distribution %s", fieldLengthDistribution)
	}
	return fieldLengthGenerator, nil
}
Ejemplo n.º 3
0
func NewWorkload(className string) (Workload, error) {
	f, ok := Workloads[className]
	if !ok {
		return nil, g.NewErrorf("unsupported workload: %s", className)
	}
	w := f()
	return w, nil
}
Ejemplo n.º 4
0
func NewMeasurementExporter(className string, w io.WriteCloser) (MeasurementExporter, error) {
	f, ok := MeasurementExporters[className]
	if !ok {
		return nil, g.NewErrorf("unsupported measurement exporter: %s", className)
	}
	e := f(w)
	return e, nil
}
Ejemplo n.º 5
0
Archivo: db.go Proyecto: hhkbp2/yabf
func NewDB(database string, props Properties) (DB, error) {
	f, ok := Databases[database]
	if !ok {
		return nil, g.NewErrorf("unsupported database: %s", database)
	}
	db := f()
	db.SetProperties(props)
	return NewDBWrapper(db), nil
}
Ejemplo n.º 6
0
Archivo: utils.go Proyecto: hhkbp2/yabf
func LoadProperties(fileName string) (Properties, error) {
	ret := NewProperties()
	f, err := os.Open(fileName)
	if err != nil {
		return ret, err
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		if regexIgnorable.MatchString(line) {
			continue
		}
		parts := regexIgnorable.FindAllString(line, -1)
		if parts == nil {
			return ret, g.NewErrorf("invalid workload file: %s, line: %s", fileName, line)
		}
		ret.Add(parts[0], parts[1])
	}
	return ret, scanner.Err()
}
Ejemplo n.º 7
0
// Initialize the scenario.
// Called once, in the main routine, before any operations are started.
func (self *CoreWorkload) Init(p Properties) error {
	table := p.GetDefault(PropertyTableName, PropertyTableNameDefault)

	propStr := p.GetDefault(PropertyFieldCount, PropertyFieldCountDefault)
	fieldCount, err := strconv.ParseInt(propStr, 0, 64)
	if err != nil {
		return err
	}
	fieldPrefix := p.GetDefault(PropertyFieldPrefix, PropertyFieldPrefixDefault)
	fieldNames := make([]string, 0, fieldCount)
	for i := int64(0); i < fieldCount; i++ {
		fieldNames = append(fieldNames, fmt.Sprintf("%s%d", fieldPrefix, i))
	}
	fieldLengthGenerator, err := self.getFieldLengthGenerator(p)
	if err != nil {
		return err
	}

	propStr = p.GetDefault(PropertyReadProportion, PropertyReadProportionDefault)
	readProportion, err := strconv.ParseFloat(propStr, 64)
	if err != nil {
		return err
	}
	propStr = p.GetDefault(PropertyUpdateProportion, PropertyUpdateProportionDefault)
	updateProportion, err := strconv.ParseFloat(propStr, 64)
	if err != nil {
		return err
	}
	propStr = p.GetDefault(PropertyInsertProportion, PropertyInsertProportionDefault)
	insertProportion, err := strconv.ParseFloat(propStr, 64)
	if err != nil {
		return err
	}
	propStr = p.GetDefault(PropertyScanProportion, PropertyScanProportionDefault)
	scanProportion, err := strconv.ParseFloat(propStr, 64)
	if err != nil {
		return err
	}
	propStr = p.GetDefault(PropertyReadModifyWriteProportion, PropertyReadModifyWriteProportionDefault)
	readModifyWriteProportion, err := strconv.ParseFloat(propStr, 64)
	if err != nil {
		return err
	}
	propStr = p.GetDefault(PropertyRecordCount, PropertyRecordCountDefault)
	recordCount, err := strconv.ParseInt(propStr, 0, 64)
	if err != nil {
		return err
	}
	if recordCount == 0 {
		recordCount = math.MaxInt32
	}
	requestDistrib := p.GetDefault(PropertyRequestDistribution, PropertyRequestDistributionDefault)
	propStr = p.GetDefault(PropertyMaxScanLength, PropertyMaxScanLengthDefault)
	maxScanLength, err := strconv.ParseInt(propStr, 0, 64)
	if err != nil {
		return err
	}
	scanLengthDistrib := p.GetDefault(PropertyScanLengthDistribution, PropertyScanLengthDistributionDefault)
	propStr = p.GetDefault(PropertyInsertStart, PropertyInsertStartDefault)
	insertStart, err := strconv.ParseInt(propStr, 0, 64)
	if err != nil {
		return err
	}
	propStr = p.GetDefault(PropertyReadAllFields, PropertyReadAllFieldsDefault)
	readAllFields, err := strconv.ParseBool(propStr)
	if err != nil {
		return err
	}
	propStr = p.GetDefault(PropertyWriteAllFields, PropertyWriteAllFieldsDefault)
	writeAllFields, err := strconv.ParseBool(propStr)
	if err != nil {
		return err
	}
	propStr = p.GetDefault(PropertyDataIntegrity, PropertyDataIntegrityDefault)
	dataIntegrity, err := strconv.ParseBool(propStr)
	if err != nil {
		return err
	}
	propStr = p.GetDefault(PropertyFieldLengthDistribution, PropertyFieldLengthDistributionDefault)
	isConstant := (propStr == "constant")
	// Confirm that fieldLengthGenerator returns a constant if data integrity check requested.
	if dataIntegrity && isConstant {
		return g.NewErrorf("must have constant field size to check data integrity")
	}
	propStr = p.GetDefault(PropertyInsertOrder, PropertyInsertOrderDefault)
	var orderedInserts bool
	var keyChooser g.IntegerGenerator
	if propStr == "hashed" {
		orderedInserts = false
	} else if requestDistrib == "exponential" {
		propStr = p.GetDefault(PropertyExponentialPercentile, PropertyExponentialPercentileDefault)
		percentile, err := strconv.ParseFloat(propStr, 64)
		if err != nil {
			return err
		}
		propStr = p.GetDefault(PropertyExponentialFraction, PropertyExponentialFractionDefault)
		fraction, err := strconv.ParseFloat(propStr, 64)
		if err != nil {
			return err
		}
		keyChooser = g.NewExponentialGenerator(percentile, float64(recordCount)*fraction)
	} else {
		orderedInserts = true
	}

	keyPrefix := p.GetDefault(PropertyKeyPrefix, PropertyKeyPrefixDefault)
	keySequence := g.NewCounterGenerator(insertStart)
	operationChooser := g.NewDiscreteGenerator()

	if readProportion > 0 {
		operationChooser.AddValue(readProportion, "READ")
	}
	if updateProportion > 0 {
		operationChooser.AddValue(updateProportion, "UPDATE")
	}
	if insertProportion > 0 {
		operationChooser.AddValue(insertProportion, "INSERT")
	}
	if scanProportion > 0 {
		operationChooser.AddValue(scanProportion, "SCAN")
	}
	if readModifyWriteProportion > 0 {
		operationChooser.AddValue(readModifyWriteProportion, "READMODIFYWRITE")
	}

	transactionInsertKeySequence := g.NewAcknowledgedCounterGenerator(recordCount)
	switch requestDistrib {
	case "uniform":
		keyChooser = g.NewUniformIntegerGenerator(0, recordCount-1)
	case "zipfian":
		// It does this by generating a random "next key" in part by
		// taking the modulus over the number of keys.
		// If the number of keys changes, this would shift the modulus,
		// and we don't want that to change which keys are popular
		// so we'll actually construct the scrambled zipfian generator
		// with a keyspace that is larger than exists at the beginning
		// of the test. That is, we'll predict the number of inserts, and
		// tell the scrambled zipfian generator the number of existing keys
		// plus the number of predicted keys as the total keyspace.
		// Then, if the generator picks a key that hasn't been inserted yet,
		// will just ignore it and pick another key. This way, the size of
		// the keyspace doesn't change from the prespective of the scrambled
		// zipfian generator.
		propStr = p.GetDefault(PropertyOperationCount, "")
		opCount, err := strconv.ParseInt(propStr, 0, 64)
		if err != nil {
			return err
		}
		// 2.0 is fudge factor
		expectedNewKeys := int64(float64(opCount) * insertProportion * 2.0)
		keyChooser = g.NewScrambledZipfianGeneratorByItems(recordCount + expectedNewKeys)
	case "latest":
		keyChooser = g.NewSkewedLatestGenerator(transactionInsertKeySequence.CounterGenerator)
	case "hotspot":
		propStr = p.GetDefault(HotspotDataFraction, HotspotDataFractionDefault)
		hotSetFraction, err := strconv.ParseFloat(propStr, 64)
		if err != nil {
			return err
		}
		propStr = p.GetDefault(HotspotOpnFraction, HotspotOpnFractionDefault)
		hotOpnFraction, err := strconv.ParseFloat(propStr, 64)
		if err != nil {
			return err
		}
		keyChooser = g.NewHotspotIntegerGenerator(0, recordCount-1, hotSetFraction, hotOpnFraction)
	default:
		return g.NewErrorf("unknown request distribution %s", requestDistrib)
	}

	fieldChooser := g.NewUniformIntegerGenerator(0, fieldCount-1)
	var scanLengthChooser g.IntegerGenerator
	switch scanLengthDistrib {
	case "uniform":
		scanLengthChooser = g.NewUniformIntegerGenerator(1, maxScanLength)
	case "zipfian":
		scanLengthChooser = g.NewZipfianGeneratorByInterval(1, maxScanLength)
	default:
		return g.NewErrorf("distribution %s not allowed for scan length", scanLengthDistrib)
	}

	propStr = p.GetDefault(InsertionRetryLimit, InsertionRetryLimitDefault)
	insertionRetryLimit, err := strconv.ParseInt(propStr, 0, 64)
	if err != nil {
		return err
	}
	propStr = p.GetDefault(InsertionRetryInterval, InsertionRetryIntervalDefault)
	insertionRetryInterval, err := strconv.ParseInt(propStr, 0, 64)
	if err != nil {
		return err
	}

	// set all fields
	self.table = table
	self.fieldCount = fieldCount
	self.fieldNames = fieldNames
	self.fieldLengthGenerator = fieldLengthGenerator
	self.readAllFields = readAllFields
	self.writeAllFields = writeAllFields
	self.dataIntegrity = dataIntegrity
	self.keyPrefix = keyPrefix
	self.keySequence = keySequence
	self.operationChooser = operationChooser
	self.keyChooser = keyChooser
	self.fieldChooser = fieldChooser
	self.transactionInsertKeySequence = transactionInsertKeySequence
	self.scanLengthChooser = scanLengthChooser
	self.orderedInserts = orderedInserts
	self.recordCount = recordCount
	self.insertionRetryLimit = insertionRetryLimit
	self.insertionRetryInterval = insertionRetryInterval
	return nil
}