Exemple #1
0
func init() {
	// Parse the flags
	flag.Parse()

	cnf = config.Config{
		Broker:        *broker,
		ResultBackend: *resultBackend,
		Exchange:      *exchange,
		ExchangeType:  *exchangeType,
		DefaultQueue:  *defaultQueue,
		BindingKey:    *bindingKey,
	}

	// Parse the config
	// NOTE: If a config file is present, it has priority over flags
	data, err := config.ReadFromFile(*configPath)
	if err == nil {
		err = config.ParseYAMLConfig(&data, &cnf)
		errors.Fail(err, "Could not parse config file")
	}

	server, err := machinery.NewServer(&cnf)
	errors.Fail(err, "Could not initialize server")

	// Register tasks
	tasks := map[string]interface{}{
		"add":      exampletasks.Add,
		"multiply": exampletasks.Multiply,
	}
	server.RegisterTasks(tasks)

	// The second argument is a consumer tag
	// Ideally, each worker should have a unique tag (worker1, worker2 etc)
	worker = server.NewWorker("machinery_worker")
}
func setup(brokerURL, backend string) *Server {
	cnf := config.Config{
		Broker:        brokerURL,
		ResultBackend: backend,
		Exchange:      "test_exchange",
		ExchangeType:  "direct",
		DefaultQueue:  "test_queue",
		BindingKey:    "test_task",
	}

	server, err := NewServer(&cnf)
	errors.Fail(err, "Could not initialize server")

	tasks := map[string]interface{}{
		"add": func(args ...int64) (int64, error) {
			sum := int64(0)
			for _, arg := range args {
				sum += arg
			}
			return sum, nil
		},
		"multiply": func(args ...int64) (int64, error) {
			sum := int64(1)
			for _, arg := range args {
				sum *= arg
			}
			return sum, nil
		},
	}
	server.RegisterTasks(tasks)

	return server
}
Exemple #3
0
func init() {
	// Parse the flags
	flag.Parse()

	cnf = config.Config{
		Broker:        *broker,
		ResultBackend: *resultBackend,
		Exchange:      *exchange,
		ExchangeType:  *exchangeType,
		DefaultQueue:  *defaultQueue,
		BindingKey:    *bindingKey,
	}

	// Parse the config
	// NOTE: If a config file is present, it has priority over flags
	data, err := config.ReadFromFile(*configPath)
	if err == nil {
		err = config.ParseYAMLConfig(&data, &cnf)
		errors.Fail(err, "Could not parse config file")
	}

	server, err = machinery.NewServer(&cnf)
	errors.Fail(err, "Could not initialize server")
}
Exemple #4
0
func main() {
	/*
	 * First, let's try sending a single task
	 */

	asyncResult, err := server.SendTask(&task0)
	errors.Fail(err, "Could not send task")

	result, err := asyncResult.Get()
	errors.Fail(err, "Getting task state failed with error")
	fmt.Printf("%v\n", result.Interface())

	/*
	 * Now let's explore ways of sending multiple tasks
	 */

	// Now let's try a parallel execution
	group := machinery.NewGroup(&task1, &task2, &task3)
	asyncResults, err := server.SendGroup(group)
	errors.Fail(err, "Could not send group")

	for _, asyncResult := range asyncResults {
		result, err = asyncResult.Get()
		errors.Fail(err, "Getting task state failed with error")
		fmt.Printf("%v\n", result.Interface())
	}

	// Now let's try chaining task results
	chain := machinery.NewChain(&task1, &task2, &task3)
	chainAsyncResult, err := server.SendChain(chain)
	errors.Fail(err, "Could not send chain")

	result, err = chainAsyncResult.Get()
	errors.Fail(err, "Getting chain result failed with error")
	fmt.Printf("%v\n", result.Interface())
}
Exemple #5
0
func main() {
	err := worker.Launch()
	errors.Fail(err, "Could not launch worker")
}
Exemple #6
0
func init() {
	// Parse the flags
	flag.Parse()

	cnf = config.Config{
		Broker:        *broker,
		ResultBackend: *resultBackend,
		Exchange:      *exchange,
		ExchangeType:  *exchangeType,
		DefaultQueue:  *defaultQueue,
		BindingKey:    *bindingKey,
	}

	// Parse the config
	// NOTE: If a config file is present, it has priority over flags
	data, err := config.ReadFromFile(*configPath)
	if err == nil {
		err = config.ParseYAMLConfig(&data, &cnf)
		errors.Fail(err, "Could not parse config file")
	}

	server, err = machinery.NewServer(&cnf)
	errors.Fail(err, "Could not initialize server")

	// Let's define some example tasks

	task0 = signatures.TaskSignature{
		Name: "add",
		Args: []signatures.TaskArg{
			signatures.TaskArg{
				Type:  "int64",
				Value: 1,
			},
			signatures.TaskArg{
				Type:  "int64",
				Value: 1,
			},
		},
	}

	task1 = signatures.TaskSignature{
		Name: "add",
		Args: []signatures.TaskArg{
			signatures.TaskArg{
				Type:  "int64",
				Value: 1,
			},
			signatures.TaskArg{
				Type:  "int64",
				Value: 1,
			},
		},
	}

	task2 = signatures.TaskSignature{
		Name: "add",
		Args: []signatures.TaskArg{
			signatures.TaskArg{
				Type:  "int64",
				Value: 5,
			},
			signatures.TaskArg{
				Type:  "int64",
				Value: 6,
			},
		},
	}

	task3 = signatures.TaskSignature{
		Name: "multiply",
		Args: []signatures.TaskArg{
			signatures.TaskArg{
				Type:  "int64",
				Value: 4,
			},
		},
	}
}
Exemple #7
0
func main() {
	/*
	 * First, let's try sending a single task
	 */
	initTasks()
	fmt.Println("Single task:")

	asyncResult, err := server.SendTask(&task0)
	errors.Fail(err, "Could not send task")

	result, err := asyncResult.Get()
	errors.Fail(err, "Getting task state failed with error")
	fmt.Printf("1 + 1 = %v\n", result.Interface())

	/*
	 * Now let's explore ways of sending multiple tasks
	 */

	// Now let's try a parallel execution
	initTasks()
	fmt.Println("Group of tasks (parallel execution):")

	group := machinery.NewGroup(&task0, &task1, &task2)
	asyncResults, err := server.SendGroup(group)
	errors.Fail(err, "Could not send group")

	for _, asyncResult := range asyncResults {
		result, err = asyncResult.Get()
		errors.Fail(err, "Getting task state failed with error")
		fmt.Printf(
			"%v + %v = %v\n",
			asyncResult.Signature.Args[0].Value,
			asyncResult.Signature.Args[1].Value,
			result.Interface(),
		)
	}

	// Now let's try a group with a chord
	initTasks()
	fmt.Println("Group of tasks with a callback (chord):")

	group = machinery.NewGroup(&task0, &task1, &task2)
	chord := machinery.NewChord(group, &task4)
	chordAsyncResult, err := server.SendChord(chord)
	errors.Fail(err, "Could not send chord")

	result, err = chordAsyncResult.Get()
	errors.Fail(err, "Getting task state failed with error")
	fmt.Printf("(1 + 1) * (2 + 2) * (5 + 6) = %v\n", result.Interface())

	// Now let's try chaining task results
	initTasks()
	fmt.Println("Chain of tasks:")

	chain := machinery.NewChain(&task0, &task1, &task2, &task3)
	chainAsyncResult, err := server.SendChain(chain)
	errors.Fail(err, "Could not send chain")

	result, err = chainAsyncResult.Get()
	errors.Fail(err, "Getting chain result failed with error")
	fmt.Printf("(((1 + 1) + (2 + 2)) + (5 + 6)) * 4 = %v\n", result.Interface())
}
Exemple #8
0
func main() {
	// First, let's try sending a single task
	task0 := signatures.TaskSignature{
		Name: "add",
		Args: []signatures.TaskArg{
			signatures.TaskArg{
				Type:  "int64",
				Value: 1,
			},
			signatures.TaskArg{
				Type:  "int64",
				Value: 1,
			},
		},
	}

	asyncResult, err := server.SendTask(&task0)
	errors.Fail(err, "Could not send task")

	result, err := asyncResult.Get()
	errors.Fail(err, "Task failed with error")
	fmt.Printf("%v\n", result.Interface())

	// Now, let's try a chain of tasks
	task1 := signatures.TaskSignature{
		Name: "add",
		Args: []signatures.TaskArg{
			signatures.TaskArg{
				Type:  "int64",
				Value: 1,
			},
			signatures.TaskArg{
				Type:  "int64",
				Value: 1,
			},
		},
	}

	task2 := signatures.TaskSignature{
		Name: "add",
		Args: []signatures.TaskArg{
			signatures.TaskArg{
				Type:  "int64",
				Value: 5,
			},
			signatures.TaskArg{
				Type:  "int64",
				Value: 6,
			},
		},
	}

	task3 := signatures.TaskSignature{
		Name: "multiply",
		Args: []signatures.TaskArg{
			signatures.TaskArg{
				Type:  "int64",
				Value: 4,
			},
		},
	}

	chain := machinery.NewChain(&task1, &task2, &task3)
	chainAsyncResult, err := server.SendChain(chain)
	errors.Fail(err, "Could not send task")

	result, err = chainAsyncResult.Get()
	errors.Fail(err, "Chain failed with error")
	fmt.Printf("%v\n", result.Interface())
}
func TestWorkerOnlyConsumesRegisteredTaskAMQP(t *testing.T) {
	amqpURL := os.Getenv("AMQP_URL")

	if amqpURL != "" {
		cnf := config.Config{
			Broker:        amqpURL,
			ResultBackend: amqpURL,
			Exchange:      "test_exchange",
			ExchangeType:  "direct",
			DefaultQueue:  "test_queue",
			BindingKey:    "test_task",
		}

		server1, err := machinery.NewServer(&cnf)
		errors.Fail(err, "Could not initialize server")

		server1.RegisterTask("add", func(args ...int64) (int64, error) {
			sum := int64(0)
			for _, arg := range args {
				sum += arg
			}
			return sum, nil
		})

		server2, err := machinery.NewServer(&cnf)
		errors.Fail(err, "Could not initialize server")

		server2.RegisterTask("multiply", func(args ...int64) (int64, error) {
			sum := int64(1)
			for _, arg := range args {
				sum *= arg
			}
			return sum, nil
		})

		task1 := signatures.TaskSignature{
			Name: "add",
			Args: []signatures.TaskArg{
				signatures.TaskArg{
					Type:  "int64",
					Value: 2,
				},
				signatures.TaskArg{
					Type:  "int64",
					Value: 3,
				},
			},
		}

		task2 := signatures.TaskSignature{
			Name: "multiply",
			Args: []signatures.TaskArg{
				signatures.TaskArg{
					Type:  "int64",
					Value: 4,
				},
				signatures.TaskArg{
					Type:  "int64",
					Value: 5,
				},
			},
		}

		worker1 := server1.NewWorker("test_worker")
		worker2 := server2.NewWorker("test_worker2")
		go worker1.Launch()
		go worker2.Launch()

		group := machinery.NewGroup(&task2, &task1)
		asyncResults, err := server1.SendGroup(group)
		if err != nil {
			t.Error(err)
		}

		expectedResults := []int64{5, 20}
		actualResults := make([]int64, 2)

		for i, asyncResult := range asyncResults {
			result, err := asyncResult.Get()
			if err != nil {
				t.Error(err)
			}
			intResult, ok := result.Interface().(int64)
			if !ok {
				t.Errorf("Could not convert %v to int64", result.Interface())
			}
			actualResults[i] = intResult
		}

		worker1.Quit()
		worker2.Quit()

		sort.Sort(ascendingInt64s(actualResults))

		if !reflect.DeepEqual(expectedResults, actualResults) {
			t.Errorf(
				"expected results = %v, actual results = %v",
				expectedResults,
				actualResults,
			)
		}
	}
}