func init() {
// Parse the flags
flag.Parse()
cnf = config.Config{
Broker: *broker,
ResultBackend: *resultBackend,
Exchange: *exchange,
ExchangeType: *exchangeType,
DefaultQueue: *defaultQueue,
BindingKey: *bindingKey,
}
// Parse the config
// NOTE: If a config file is present, it has priority over flags
data, err := config.ReadFromFile(*configPath)
if err == nil {
err = config.ParseYAMLConfig(&data, &cnf)
errors.Fail(err, "Could not parse config file")
}
server, err := machinery.NewServer(&cnf)
errors.Fail(err, "Could not initialize server")
// Register tasks
tasks := map[string]interface{}{
"add": exampletasks.Add,
"multiply": exampletasks.Multiply,
}
server.RegisterTasks(tasks)
// The second argument is a consumer tag
// Ideally, each worker should have a unique tag (worker1, worker2 etc)
worker = server.NewWorker("machinery_worker")
}
func setup(brokerURL, backend string) *Server {
cnf := config.Config{
Broker: brokerURL,
ResultBackend: backend,
Exchange: "test_exchange",
ExchangeType: "direct",
DefaultQueue: "test_queue",
BindingKey: "test_task",
}
server, err := NewServer(&cnf)
errors.Fail(err, "Could not initialize server")
tasks := map[string]interface{}{
"add": func(args ...int64) (int64, error) {
sum := int64(0)
for _, arg := range args {
sum += arg
}
return sum, nil
},
"multiply": func(args ...int64) (int64, error) {
sum := int64(1)
for _, arg := range args {
sum *= arg
}
return sum, nil
},
}
server.RegisterTasks(tasks)
return server
}
func init() {
// Parse the flags
flag.Parse()
cnf = config.Config{
Broker: *broker,
ResultBackend: *resultBackend,
Exchange: *exchange,
ExchangeType: *exchangeType,
DefaultQueue: *defaultQueue,
BindingKey: *bindingKey,
}
// Parse the config
// NOTE: If a config file is present, it has priority over flags
data, err := config.ReadFromFile(*configPath)
if err == nil {
err = config.ParseYAMLConfig(&data, &cnf)
errors.Fail(err, "Could not parse config file")
}
server, err = machinery.NewServer(&cnf)
errors.Fail(err, "Could not initialize server")
}
func main() {
/*
* First, let's try sending a single task
*/
asyncResult, err := server.SendTask(&task0)
errors.Fail(err, "Could not send task")
result, err := asyncResult.Get()
errors.Fail(err, "Getting task state failed with error")
fmt.Printf("%v\n", result.Interface())
/*
* Now let's explore ways of sending multiple tasks
*/
// Now let's try a parallel execution
group := machinery.NewGroup(&task1, &task2, &task3)
asyncResults, err := server.SendGroup(group)
errors.Fail(err, "Could not send group")
for _, asyncResult := range asyncResults {
result, err = asyncResult.Get()
errors.Fail(err, "Getting task state failed with error")
fmt.Printf("%v\n", result.Interface())
}
// Now let's try chaining task results
chain := machinery.NewChain(&task1, &task2, &task3)
chainAsyncResult, err := server.SendChain(chain)
errors.Fail(err, "Could not send chain")
result, err = chainAsyncResult.Get()
errors.Fail(err, "Getting chain result failed with error")
fmt.Printf("%v\n", result.Interface())
}
func main() {
err := worker.Launch()
errors.Fail(err, "Could not launch worker")
}
func init() {
// Parse the flags
flag.Parse()
cnf = config.Config{
Broker: *broker,
ResultBackend: *resultBackend,
Exchange: *exchange,
ExchangeType: *exchangeType,
DefaultQueue: *defaultQueue,
BindingKey: *bindingKey,
}
// Parse the config
// NOTE: If a config file is present, it has priority over flags
data, err := config.ReadFromFile(*configPath)
if err == nil {
err = config.ParseYAMLConfig(&data, &cnf)
errors.Fail(err, "Could not parse config file")
}
server, err = machinery.NewServer(&cnf)
errors.Fail(err, "Could not initialize server")
// Let's define some example tasks
task0 = signatures.TaskSignature{
Name: "add",
Args: []signatures.TaskArg{
signatures.TaskArg{
Type: "int64",
Value: 1,
},
signatures.TaskArg{
Type: "int64",
Value: 1,
},
},
}
task1 = signatures.TaskSignature{
Name: "add",
Args: []signatures.TaskArg{
signatures.TaskArg{
Type: "int64",
Value: 1,
},
signatures.TaskArg{
Type: "int64",
Value: 1,
},
},
}
task2 = signatures.TaskSignature{
Name: "add",
Args: []signatures.TaskArg{
signatures.TaskArg{
Type: "int64",
Value: 5,
},
signatures.TaskArg{
Type: "int64",
Value: 6,
},
},
}
task3 = signatures.TaskSignature{
Name: "multiply",
Args: []signatures.TaskArg{
signatures.TaskArg{
Type: "int64",
Value: 4,
},
},
}
}
func main() {
/*
* First, let's try sending a single task
*/
initTasks()
fmt.Println("Single task:")
asyncResult, err := server.SendTask(&task0)
errors.Fail(err, "Could not send task")
result, err := asyncResult.Get()
errors.Fail(err, "Getting task state failed with error")
fmt.Printf("1 + 1 = %v\n", result.Interface())
/*
* Now let's explore ways of sending multiple tasks
*/
// Now let's try a parallel execution
initTasks()
fmt.Println("Group of tasks (parallel execution):")
group := machinery.NewGroup(&task0, &task1, &task2)
asyncResults, err := server.SendGroup(group)
errors.Fail(err, "Could not send group")
for _, asyncResult := range asyncResults {
result, err = asyncResult.Get()
errors.Fail(err, "Getting task state failed with error")
fmt.Printf(
"%v + %v = %v\n",
asyncResult.Signature.Args[0].Value,
asyncResult.Signature.Args[1].Value,
result.Interface(),
)
}
// Now let's try a group with a chord
initTasks()
fmt.Println("Group of tasks with a callback (chord):")
group = machinery.NewGroup(&task0, &task1, &task2)
chord := machinery.NewChord(group, &task4)
chordAsyncResult, err := server.SendChord(chord)
errors.Fail(err, "Could not send chord")
result, err = chordAsyncResult.Get()
errors.Fail(err, "Getting task state failed with error")
fmt.Printf("(1 + 1) * (2 + 2) * (5 + 6) = %v\n", result.Interface())
// Now let's try chaining task results
initTasks()
fmt.Println("Chain of tasks:")
chain := machinery.NewChain(&task0, &task1, &task2, &task3)
chainAsyncResult, err := server.SendChain(chain)
errors.Fail(err, "Could not send chain")
result, err = chainAsyncResult.Get()
errors.Fail(err, "Getting chain result failed with error")
fmt.Printf("(((1 + 1) + (2 + 2)) + (5 + 6)) * 4 = %v\n", result.Interface())
}
func main() {
// First, let's try sending a single task
task0 := signatures.TaskSignature{
Name: "add",
Args: []signatures.TaskArg{
signatures.TaskArg{
Type: "int64",
Value: 1,
},
signatures.TaskArg{
Type: "int64",
Value: 1,
},
},
}
asyncResult, err := server.SendTask(&task0)
errors.Fail(err, "Could not send task")
result, err := asyncResult.Get()
errors.Fail(err, "Task failed with error")
fmt.Printf("%v\n", result.Interface())
// Now, let's try a chain of tasks
task1 := signatures.TaskSignature{
Name: "add",
Args: []signatures.TaskArg{
signatures.TaskArg{
Type: "int64",
Value: 1,
},
signatures.TaskArg{
Type: "int64",
Value: 1,
},
},
}
task2 := signatures.TaskSignature{
Name: "add",
Args: []signatures.TaskArg{
signatures.TaskArg{
Type: "int64",
Value: 5,
},
signatures.TaskArg{
Type: "int64",
Value: 6,
},
},
}
task3 := signatures.TaskSignature{
Name: "multiply",
Args: []signatures.TaskArg{
signatures.TaskArg{
Type: "int64",
Value: 4,
},
},
}
chain := machinery.NewChain(&task1, &task2, &task3)
chainAsyncResult, err := server.SendChain(chain)
errors.Fail(err, "Could not send task")
result, err = chainAsyncResult.Get()
errors.Fail(err, "Chain failed with error")
fmt.Printf("%v\n", result.Interface())
}
func TestWorkerOnlyConsumesRegisteredTaskAMQP(t *testing.T) {
amqpURL := os.Getenv("AMQP_URL")
if amqpURL != "" {
cnf := config.Config{
Broker: amqpURL,
ResultBackend: amqpURL,
Exchange: "test_exchange",
ExchangeType: "direct",
DefaultQueue: "test_queue",
BindingKey: "test_task",
}
server1, err := machinery.NewServer(&cnf)
errors.Fail(err, "Could not initialize server")
server1.RegisterTask("add", func(args ...int64) (int64, error) {
sum := int64(0)
for _, arg := range args {
sum += arg
}
return sum, nil
})
server2, err := machinery.NewServer(&cnf)
errors.Fail(err, "Could not initialize server")
server2.RegisterTask("multiply", func(args ...int64) (int64, error) {
sum := int64(1)
for _, arg := range args {
sum *= arg
}
return sum, nil
})
task1 := signatures.TaskSignature{
Name: "add",
Args: []signatures.TaskArg{
signatures.TaskArg{
Type: "int64",
Value: 2,
},
signatures.TaskArg{
Type: "int64",
Value: 3,
},
},
}
task2 := signatures.TaskSignature{
Name: "multiply",
Args: []signatures.TaskArg{
signatures.TaskArg{
Type: "int64",
Value: 4,
},
signatures.TaskArg{
Type: "int64",
Value: 5,
},
},
}
worker1 := server1.NewWorker("test_worker")
worker2 := server2.NewWorker("test_worker2")
go worker1.Launch()
go worker2.Launch()
group := machinery.NewGroup(&task2, &task1)
asyncResults, err := server1.SendGroup(group)
if err != nil {
t.Error(err)
}
expectedResults := []int64{5, 20}
actualResults := make([]int64, 2)
for i, asyncResult := range asyncResults {
result, err := asyncResult.Get()
if err != nil {
t.Error(err)
}
intResult, ok := result.Interface().(int64)
if !ok {
t.Errorf("Could not convert %v to int64", result.Interface())
}
actualResults[i] = intResult
}
worker1.Quit()
worker2.Quit()
sort.Sort(ascendingInt64s(actualResults))
if !reflect.DeepEqual(expectedResults, actualResults) {
t.Errorf(
"expected results = %v, actual results = %v",
expectedResults,
actualResults,
)
}
}
}