// SimulateFromFile runs a simulation for a single specified batch of jobs and appends the results to a
// specified file in CSV format
func SimulateFromFile(
clusterSize int, sysLoad int, inputPath string, outputPath string,
outputLock *sync.Mutex, threadCapper *models.Semaphore,
waitGroup *sync.WaitGroup, schedulerType int) {
threadCapper.Lock()
jobs := utils.ReadJobs(inputPath)
eventQueue := make(models.EventQueue, len(jobs))
for i := 0; i < len(jobs); i++ {
job := jobs[i]
event := models.Event{
Time: job.Arrival,
Job: &job,
Type: models.Arrival,
}
eventQueue[i] = &event
}
heap.Init(&eventQueue)
nodeQueue := make(models.NodeQueue, clusterSize)
for i := 0; i < clusterSize; i++ {
node := models.Node{
CurJob: nil,
EstCompletion: -1,
Index: i,
}
nodeQueue[i] = &node
}
jobQueue := make(models.JobQueue, 0)
var scheduler scheduling.Scheduler
switch schedulerType {
case Markov:
scheduler = &scheduling.MarkovScheduler{
QueuedJobs: &jobQueue,
Nodes: &nodeQueue,
ScheduleSize: clusterSize * 384,
}
case StandardEDF:
scheduler = &scheduling.StandardEDFScheduler{
QueuedJobs: &jobQueue,
Nodes: &nodeQueue,
}
case OmniscientEDF:
scheduler = &scheduling.OmniscientEDFScheduler{
QueuedJobs: &jobQueue,
Nodes: &nodeQueue,
}
}
scheduler.Init()
jobsSeen, jobsRejected, jobsCompleted, jobsKilled, jobsDlMissed := 0, 0, 0, 0, 0
for eventQueue.Len() > 0 {
event := heap.Pop(&eventQueue).(*models.Event)
switch event.Type {
case models.Arrival:
jobsSeen++
if scheduler.IsJobAdmitted(event.Job) {
node := heap.Pop(&nodeQueue).(*models.Node)
if node.CurJob == nil { // Start job right away if there's a Node waiting
startNextJob(event, node, heap.Pop(&jobQueue).(*models.Job), scheduler, &eventQueue)
}
// Completion events from other Jobs will automatically start the next Job for us if
// no nodes are ready now
heap.Push(&nodeQueue, node)
} else { // !IsJobAdmitted
jobsRejected++
}
case models.Stretch:
event.Node.EstCompletion += scheduler.GetAllowance()
heap.Fix(&nodeQueue, event.Node.Index)
if scheduler.GetAllocation(event.Job)+scheduler.GetAllowance() >= event.Job.RealExec {
heap.Push(&eventQueue, &models.Event{
Job: event.Job,
Time: event.Time + event.Job.RealExec - scheduler.GetAllocation(event.Job),
Node: event.Node,
Type: models.Complete,
})
} else {
heap.Push(&eventQueue, &models.Event{
Job: event.Job,
Time: event.Node.EstCompletion,
Node: event.Node,
Type: models.Kill,
})
}
scheduler.DeductAllowance()
case models.Complete:
jobsCompleted++
scheduler.SuccessCallback(event.Job)
if jobQueue.Len() > 0 {
startNextJob(event, event.Node, heap.Pop(&jobQueue).(*models.Job), scheduler, &eventQueue)
} else {
event.Node.CurJob = nil
event.Node.EstCompletion = -1
}
heap.Fix(&nodeQueue, event.Node.Index)
case models.Kill:
jobsKilled++
scheduler.FailCallback(event.Job)
if jobQueue.Len() > 0 {
startNextJob(event, event.Node, heap.Pop(&jobQueue).(*models.Job), scheduler, &eventQueue)
} else {
event.Node.CurJob = nil
event.Node.EstCompletion = -1
}
heap.Fix(&nodeQueue, event.Node.Index)
case models.Miss:
jobsDlMissed++
scheduler.FailCallback(event.Job)
if jobQueue.Len() > 0 {
startNextJob(event, event.Node, heap.Pop(&jobQueue).(*models.Job), scheduler, &eventQueue)
} else {
event.Node.CurJob = nil
event.Node.EstCompletion = -1
}
heap.Fix(&nodeQueue, event.Node.Index)
}
}
outputLock.Lock()
var record = []string{
fmt.Sprintf("%d", clusterSize),
fmt.Sprintf("%d", sysLoad),
fmt.Sprintf("%d", jobsSeen),
fmt.Sprintf("%d", jobsCompleted),
fmt.Sprintf("%d", jobsRejected),
fmt.Sprintf("%d", jobsKilled),
fmt.Sprintf("%d", jobsDlMissed),
}
file, err := os.OpenFile(outputPath, os.O_APPEND|os.O_WRONLY, 0666)
utils.Check(err)
writer := csv.NewWriter(file)
err = writer.Write(record)
utils.Check(err)
writer.Flush()
if writer.Error() != nil {
fmt.Println(writer.Error())
} else {
file.Sync()
}
file.Close()
outputLock.Unlock()
threadCapper.Free()
waitGroup.Done()
}
func readWrapper(filePath string, waitGroup *sync.WaitGroup, threadCapper *models.Semaphore) {
threadCapper.Lock()
utils.ReadJobs(filePath)
threadCapper.Free()
waitGroup.Done()
}
// GenJobs generates and returns a slice of Jobs with timings
// derived from exponentially distributed random variables.
func GenJobs(sysLoad float64, simTime int, clusterSize int, outPath string, stats chan string,
precision chan float64, threadCap *models.Semaphore, jobPool *sync.Pool) {
threadCap.Lock()
var totalComputation = 0
var numJobs = round(
(sysLoad * float64(simTime) * float64(clusterSize)) / realExecAvg)
var numSubmitters = round(math.Sqrt(float64(numJobs)))
submitterEstAvgs := make([]float64, numSubmitters)
for i := 0; i < numSubmitters; i++ {
for {
submitterEstAvgs[i] = rand.ExpFloat64() * estExecAvg
if submitterEstAvgs[i] > estExecAvg/15.0 && submitterEstAvgs[i] < estExecAvg*15.0 {
break
}
}
}
var burstSize int
var burstAvg = sysLoad * float64(clusterSize) * burstMult
var interval int
var intervalLambda = float64(simTime) / (float64(numJobs) / burstAvg)
var jobs []models.Job
var arrival = 0
var estExec, realExec, deadline, submitter int
for i := 0; i < numJobs; {
submitter = rand.Intn(numSubmitters)
if i != 0 {
interval = round(rand.ExpFloat64() * float64(intervalLambda))
arrival += interval
}
burstSize = round(rand.ExpFloat64() * burstAvg)
for j := 0; j < burstSize; j++ {
realExec = round(rand.ExpFloat64() * realExecAvg)
for { // estExec must always be > realExec
estExec = round(rand.ExpFloat64() * submitterEstAvgs[submitter])
if estExec > realExec {
break
}
}
for { // deadline must always be > estExec (n.b. this implicitly makes it > realExec)
deadline = round(rand.ExpFloat64() * deadlineAvg)
if deadline > estExec {
break
}
}
var newJob *models.Job
poolFetch := jobPool.Get()
if poolFetch == nil {
newJob = &models.Job{}
} else {
newJob = poolFetch.(*models.Job)
}
newJob.ID = i
newJob.SubmitterID = submitter
newJob.Arrival = arrival
newJob.EstExec = estExec
newJob.RealExec = realExec
newJob.Deadline = deadline
jobs = append(jobs, *newJob)
jobPool.Put(newJob)
totalComputation += realExec
i++
}
}
WriteJobsToCSV(jobs, outPath)
var trueLoad = (float64(totalComputation) / (float64(simTime) * float64(clusterSize)))
precision <- (trueLoad / sysLoad)
var statsMsg = "Generated jobs for:\n" +
" Cluster Size: %v\n" +
" System Load: %v\n" +
" Time Span: %v\n" +
" Actual computation load: %v\n"
statsMsg = fmt.Sprintf(statsMsg, clusterSize, sysLoad, simTime, trueLoad)
stats <- statsMsg
threadCap.Free()
}