// NewUnit takes a component type and returns a Fleet unit
// that includes the relevant systemd service template
func NewUnit(component string, templatePaths []string, decorate bool) (uf *unit.UnitFile, err error) {
template, err := readTemplate(component, templatePaths)
if err != nil {
return
}
if decorate {
decorator, err := readDecorator(component)
if err != nil {
return nil, err
}
uf, err = unit.NewUnitFile(string(template) + "\n" + string(decorator))
} else {
uf, err = unit.NewUnitFile(string(template))
}
return
}
func newUnitFile(t *testing.T, contents string) *unit.UnitFile {
uf, err := unit.NewUnitFile(contents)
if err != nil {
t.Fatalf("error creating NewUnitFile from %s: %v", contents, err)
}
return uf
}
// getUnitByHash retrieves from the Registry the Unit associated with the given Hash
func (r *EtcdRegistry) getUnitByHash(hash unit.Hash) *unit.UnitFile {
key := r.hashedUnitPath(hash)
opts := &etcd.GetOptions{
Recursive: true,
}
resp, err := r.kAPI.Get(r.ctx(), key, opts)
if err != nil {
if isEtcdError(err, etcd.ErrorCodeKeyNotFound) {
err = nil
}
return nil
}
var um unitModel
if err := unmarshal(resp.Node.Value, &um); err != nil {
log.Errorf("error unmarshaling Unit(%s): %v", hash, err)
return nil
}
u, err := unit.NewUnitFile(um.Raw)
if err != nil {
log.Errorf("error parsing Unit(%s): %v", hash, err)
return nil
}
return u
}
func newUF(t *testing.T, contents string) unit.UnitFile {
uf, err := unit.NewUnitFile(contents)
if err != nil {
t.Fatalf("error creating new unit file from %v: %v", contents, err)
}
return *uf
}
func newUnit(t *testing.T, str string) *unit.UnitFile {
u, err := unit.NewUnitFile(str)
if err != nil {
t.Fatalf("Unexpected error creating unit from %q: %v", str, err)
}
return u
}
// getUnitByHash retrieves from the Registry the Unit associated with the given Hash
func (r *EtcdRegistry) getUnitByHash(hash unit.Hash) *unit.UnitFile {
req := etcd.Get{
Key: r.hashedUnitPath(hash),
Recursive: true,
}
resp, err := r.etcd.Do(&req)
if err != nil {
if isKeyNotFound(err) {
err = nil
}
return nil
}
var um unitModel
if err := unmarshal(resp.Node.Value, &um); err != nil {
log.Errorf("error unmarshaling Unit(%s): %v", hash, err)
return nil
}
u, err := unit.NewUnitFile(um.Raw)
if err != nil {
log.Errorf("error parsing Unit(%s): %v", hash, err)
return nil
}
return u
}
func StartUnitsInDir(path string) {
files, _ := ioutil.ReadDir(path)
for _, f := range files {
unitpath := fmt.Sprintf("v1-alpha/units/%s", f.Name())
url := getFullAPIURL("10001", unitpath)
filepath := fmt.Sprintf("%s/%s", path, f.Name())
readfile, err := ioutil.ReadFile(filepath)
checkForErrors(err)
content := string(readfile)
u, _ := unit.NewUnitFile(content)
options_bytes, _ := json.Marshal(u.Options)
options_str := lowerCasingOfUnitOptionsStr(string(options_bytes))
json_str := fmt.Sprintf(
`{"name": "%s", "desiredState":"launched", "options": %s}`,
f.Name(),
options_str)
resp := httpPutRequest(url, []byte(json_str))
if resp.StatusCode != 204 {
body, err := ioutil.ReadAll(resp.Body)
log.Printf("[Error] in HTTP Body: %s", body)
checkForErrors(err)
}
}
}
func TestRpcUnitToJobUnit(t *testing.T) {
contents := `
[Unit]
Description = Foo
`
unitFile, err := unit.NewUnitFile(contents)
if err != nil {
t.Fatalf("unexpected error parsing unit %q: %v", contents, err)
}
want := &pb.Unit{
Name: "foo",
Unit: unitFile.ToPB(),
DesiredState: pb.TargetState_LOADED,
}
expect := &job.Unit{
Name: "foo",
Unit: *unitFile,
TargetState: job.JobStateLoaded,
}
got := rpcUnitToJobUnit(want)
if !reflect.DeepEqual(got, expect) {
t.Fatalf("got %#v, expected %#v", got, expect)
}
}
func newUnitWithMetadata(t *testing.T, metadata string) unit.UnitFile {
contents := fmt.Sprintf("[X-Fleet]\nMachineMetadata=%s", metadata)
u, err := unit.NewUnitFile(contents)
if err != nil {
t.Fatalf("error creating unit from %q: %v", contents, err)
}
return *u
}
// getUnitFromFile attempts to load a Unit from a given filename
// It returns the Unit or nil, and any error encountered
func getUnitFromFile(file string) (*unit.UnitFile, error) {
out, err := ioutil.ReadFile(file)
if err != nil {
panic(err)
}
return unit.NewUnitFile(string(out))
}
// getUnitFromStdin attempts to load a Unit from stdin
func getUnitFromStdin() (*unit.UnitFile, error) {
bytes, err := ioutil.ReadAll(os.Stdin)
if err != nil {
panic(err)
}
return unit.NewUnitFile(string(bytes))
}
func TestFakeRegistryUnitLifecycle(t *testing.T) {
reg := NewFakeRegistry()
units, err := reg.Units()
if err != nil {
t.Fatalf("Received error while calling Jobs: %v", err)
}
if !reflect.DeepEqual([]job.Unit{}, units) {
t.Fatalf("Expected no units, got %v", units)
}
uf, _ := unit.NewUnitFile("")
u1 := job.Unit{Name: "u1.service", Unit: *uf, TargetState: job.JobStateLoaded}
err = reg.CreateUnit(&u1)
if err != nil {
t.Fatalf("Received error while calling CreateUnit: %v", err)
}
units, err = reg.Units()
if err != nil {
t.Fatalf("Received error while calling Units: %v", err)
}
if len(units) != 1 {
t.Fatalf("Expected 1 Unit, got %v", units)
}
if !reflect.DeepEqual(u1, units[0]) {
t.Fatalf("Expected unit %v, got %v", u1, units[0])
}
err = reg.ScheduleUnit("u1.service", "XXX")
if err != nil {
t.Fatalf("Received error while calling ScheduleUnit: %v", err)
}
su, err := reg.ScheduledUnit("u1.service")
if err != nil {
t.Fatalf("Received error while calling ScheduledUnit: %v", err)
}
if su.TargetMachineID != "XXX" {
t.Fatalf("Unit should be scheduled to XXX, got %v", su.TargetMachineID)
}
err = reg.DestroyUnit("u1.service")
if err != nil {
t.Fatalf("Received error while calling DestroyUnit: %v", err)
}
units, err = reg.Units()
if err != nil {
t.Fatalf("Received error while calling Units: %v", err)
}
if !reflect.DeepEqual([]job.Unit{}, units) {
t.Fatalf("Expected no units, got %v", units)
}
}
// NewUnit takes a component type and returns a Fleet unit
// that includes the relevant systemd service template
func NewUnit(component string) (uf *unit.UnitFile, err error) {
template, err := readTemplate(component)
if err != nil {
return
}
uf, err = unit.NewUnitFile(string(template))
if err != nil {
return
}
return
}
// getUnitFromFile attempts to load a Unit from a given filename
// It returns the Unit or nil, and any error encountered
func getUnitFromFile(file string) (*unit.UnitFile, error) {
out, err := ioutil.ReadFile(file)
if err != nil {
return nil, err
}
unitName := path.Base(file)
log.Debugf("Unit(%s) found in local filesystem", unitName)
return unit.NewUnitFile(string(out))
}
func (d *deployer) buildWantedUnits() (map[string]*schema.Unit, error) {
units := make(map[string]*schema.Unit)
servicesDefinition, err := d.serviceDefinitionClient.servicesDefinition()
if err != nil {
return nil, err
}
for _, srv := range servicesDefinition.Services {
vars := make(map[string]interface{})
serviceTemplate, err := d.serviceDefinitionClient.serviceFile(srv)
if err != nil {
log.Printf("%v", err)
continue
}
vars["version"] = srv.Version
serviceFile, err := renderedServiceFile(serviceTemplate, vars)
if err != nil {
log.Printf("%v", err)
return nil, err
}
// fleet deploy
uf, err := unit.NewUnitFile(serviceFile)
if err != nil {
//Broken service file, skip it and continue
log.Printf("WARNING service file %s is incorrect: %v [SKIPPING]", srv.Name, err)
continue
}
if srv.Count == 0 && !strings.Contains(srv.Name, "@") {
u := &schema.Unit{
Name: srv.Name,
Options: schema.MapUnitFileToSchemaUnitOptions(uf),
DesiredState: srv.DesiredState,
}
units[srv.Name] = u
} else if srv.Count > 0 && strings.Contains(srv.Name, "@") {
for i := 0; i < srv.Count; i++ {
xName := strings.Replace(srv.Name, "@", fmt.Sprintf("@%d", i+1), -1)
u := &schema.Unit{
Name: xName,
Options: schema.MapUnitFileToSchemaUnitOptions(uf),
DesiredState: srv.DesiredState,
}
units[u.Name] = u
}
} else {
log.Printf("WARNING skipping service: %s, incorrect service definition", srv.Name)
}
}
return units, nil
}
func newTestJobFromUnitContents(t *testing.T, name, contents string) *job.Job {
u, err := unit.NewUnitFile(contents)
if err != nil {
t.Fatalf("error creating Unit from %q: %v", contents, err)
}
j := job.NewJob(name, *u)
if j == nil {
t.Fatalf("error creating Job %q from %q", name, u)
}
return j
}
func (u *Unit) GetUnitContentAsFleeted() (string, error) {
unitFileContent, err := ioutil.ReadFile(u.unitPath)
if err != nil {
return "", err
}
fleetunit, err := unit.NewUnitFile(string(unitFileContent))
if err != nil {
return "", err
}
return convertMultilineUnitToString([]byte(fleetunit.String())), nil
}
// NewDataUnit takes a component type and returns a Fleet unit
// that is hard-scheduled to a machine ID
func NewDataUnit(component string, machineID string) (uf *unit.UnitFile, err error) {
template, err := readTemplate(component)
if err != nil {
return
}
// replace CHANGEME with random machineID
replaced := strings.Replace(string(template), "CHANGEME", machineID, 1)
uf, err = unit.NewUnitFile(replaced)
if err != nil {
return
}
return
}
func hashUnitFile(loc string) (unit.Hash, error) {
b, err := ioutil.ReadFile(loc)
if err != nil {
return unit.Hash{}, err
}
uf, err := unit.NewUnitFile(string(b))
if err != nil {
return unit.Hash{}, err
}
return uf.Hash(), nil
}
func (srv *Service) unitToOptions(desiredUnit Unit) ([]*schema.UnitOption, error) {
uf, err := unit.NewUnitFile(srv.UnitTemplate)
if err != nil {
return nil, mask(err)
}
options := schema.MapUnitFileToSchemaUnitOptions(uf)
options = append(options, &schema.UnitOption{
Section: "X-Fleet",
Name: "MachineID",
Value: desiredUnit.MachineID,
})
return options, nil
}
func fleetUnit(t *testing.T, opts ...string) unit.UnitFile {
contents := "[X-Fleet]"
for _, v := range opts {
contents = fmt.Sprintf("%s\n%s", contents, v)
}
u, err := unit.NewUnitFile(contents)
if u == nil || err != nil {
t.Fatalf("Failed creating test unit: unit=%v, err=%v", u, err)
}
return *u
}
func startUnitFile(unitFile string) {
filename := filepath.Base(unitFile)
unitFilepath := fmt.Sprintf(
"fleet/%s/units/%s", Conf.FleetAPIVersion, filename)
url := getFullAPIURL(Conf.FleetAPIPort, unitFilepath)
log.Printf("Starting unit file: %s", filename)
statusCode := 0
for statusCode != 204 {
readfile, err := ioutil.ReadFile(unitFile)
goutils.PrintErrors(
goutils.ErrorParams{Err: err, CallerNum: 2, Fatal: false})
content := string(readfile)
u, _ := unit.NewUnitFile(content)
options_bytes, _ := json.Marshal(u.Options)
options_str := lowerCasingOfUnitOptionsStr(string(options_bytes))
json_str := fmt.Sprintf(
`{"name": "%s", "desiredState":"launched", "options": %s}`,
filename,
options_str)
headers := map[string]string{
"Content-Type": "application/json",
}
p := goutils.HttpRequestParams{
HttpRequestType: "PUT",
Url: url,
Data: json_str,
Headers: headers,
}
statusCode, _, _ = goutils.HttpCreateRequest(p)
time.Sleep(1 * time.Second)
/*
log.Printf(
"curl -H \"Content-Type: application/json\" -X PUT "+
"-d %q localhost:10001/v1-alpha/units/%s",
json_str, filename)
body, err := ioutil.ReadAll(resp.Body)
log.Printf("Status Code: %s", statusCode)
log.Printf("[Error] in HTTP Body: %s - %v", body, err)
goutils.PrintErrors(
goutils.ErrorParams{Err: err, CallerNum: 2, Fatal: false})
*/
}
}
func TestInMemoryUnitState(t *testing.T) {
inmemoryRegistry := newInmemoryRegistry()
scheduleUnit := &pb.ScheduledUnit{
Name: "foo",
CurrentState: pb.TargetState_INACTIVE,
MachineID: "machine1",
}
inmemoryRegistry.scheduledUnits[scheduleUnit.Name] = *scheduleUnit
contents := `
[Unit]
Description = Foo
`
unitFile, err := unit.NewUnitFile(contents)
if err != nil {
t.Fatalf("unexpected error parsing unit %q: %v", contents, err)
}
unit := &pb.Unit{
Name: "foo",
Unit: unitFile.ToPB(),
DesiredState: pb.TargetState_LOADED,
}
machineID := "testMachine"
ttl := 2 * time.Second
inmemoryRegistry.CreateUnit(unit)
inmemoryRegistry.ScheduleUnit(unit.Name, machineID)
stateLoaded := &pb.UnitState{
Name: unit.Name,
Hash: "heh",
LoadState: "active",
ActiveState: "loaded",
SubState: "active",
MachineID: machineID,
}
inmemoryRegistry.SaveUnitState(unit.Name, stateLoaded, ttl)
if !inmemoryRegistry.isUnitLoaded(unit.Name, machineID) {
u, ok := inmemoryRegistry.Unit(unit.Name)
if !ok {
t.Fatalf("unexpected error unit not found %s", unit.Name)
}
t.Fatalf("unexpected error unit expected to be loaded %v", u)
}
resUs := inmemoryRegistry.UnitState(unit.Name)
if resUs == nil || len(resUs.Name) == 0 {
t.Fatal("Invalid unit state in the in-memory registry")
}
}
func TestInMemoryScheduleUnit(t *testing.T) {
inmemoryRegistry := newInmemoryRegistry()
scheduleUnit := &pb.ScheduledUnit{
Name: "foo",
CurrentState: pb.TargetState_INACTIVE,
MachineID: "machine1",
}
inmemoryRegistry.scheduledUnits[scheduleUnit.Name] = *scheduleUnit
contents := `
[Unit]
Description = Foo
`
unitFile, err := unit.NewUnitFile(contents)
if err != nil {
t.Fatalf("unexpected error parsing unit %q: %v", contents, err)
}
unit := &pb.Unit{
Name: "foo",
Unit: unitFile.ToPB(),
DesiredState: pb.TargetState_LOADED,
}
machineID := "testMachine"
inmemoryRegistry.CreateUnit(unit)
inmemoryRegistry.ScheduleUnit(unit.Name, machineID)
unitsLen := len(inmemoryRegistry.Units())
if unitsLen == 0 {
t.Fatalf("unexpected amount of units in the in-memory registry got %d expected 1", unitsLen)
}
if !inmemoryRegistry.isScheduled(unit.Name, machineID) {
t.Fatalf("unexpected error unit should be scheduled %s %s", unit.Name, machineID)
}
inmemoryRegistry.UnscheduleUnit(unit.Name, machineID)
if inmemoryRegistry.isScheduled("foo", "testMachine") {
t.Fatalf("unexpected error unit should NOT be scheduled %s %s", unit.Name, machineID)
}
if !inmemoryRegistry.DestroyUnit(unit.Name) {
t.Fatalf("unexpected error unit have to be destroy %s", unit.Name)
}
unitsLen = len(inmemoryRegistry.Units())
if unitsLen > 0 {
t.Fatalf("unexpected amount of units in the in-memory registry got %d expected 0", unitsLen)
}
}
func (r *EtcdRegistry) unitFromEtcdNode(hash unit.Hash, etcdNode *etcd.Node) *unit.UnitFile {
var um unitModel
if err := unmarshal(etcdNode.Value, &um); err != nil {
log.Errorf("error unmarshaling Unit(%s): %v", hash, err)
return nil
}
u, err := unit.NewUnitFile(um.Raw)
if err != nil {
log.Errorf("error parsing Unit(%s): %v", hash, err)
return nil
}
return u
}
// lazyCreateUnits iterates over a set of unit names and, for each, attempts to
// ensure that a unit by that name exists in the Registry, by checking a number
// of conditions and acting on the first one that succeeds, in order of:
// 1. a unit by that name already existing in the Registry
// 2. a unit file by that name existing on disk
// 3. a corresponding unit template (if applicable) existing in the Registry
// 4. a corresponding unit template (if applicable) existing on disk
// Any error encountered during these steps is returned immediately (i.e.
// subsequent Jobs are not acted on). An error is also returned if none of the
// above conditions match a given Job.
func (f *FleetTunnel) lazyCreateUnits(units UnitDataList, events chan Event) error {
errchan := make(chan error)
blockAttempts := f.BlockAttempts
var wg sync.WaitGroup
for i := 0; i < units.Len(); i++ {
u := units.Get(i)
name := u.Name()
create, err := f.checkUnitCreation(name)
if err != nil {
return err
} else if !create {
continue
}
// Assume that the name references a local unit file on
// disk or if it is an instance unit and if so get its
// corresponding unit
uf, err := unit.NewUnitFile(u.Content())
if err != nil {
return err
}
events <- newEvent(name, "creating unit")
_, err = f.createUnit(name, uf)
if err != nil {
return err
}
wg.Add(1)
go f.checkUnitState(name, job.JobStateInactive, blockAttempts, events, &wg, errchan)
}
go func() {
wg.Wait()
close(errchan)
}()
var ae aerr.AggregateError
for msg := range errchan {
ae.Add(maskAny(fmt.Errorf("Error waiting on unit creation: %v\n", msg)))
}
if !ae.IsEmpty() {
return maskAny(&ae)
}
return nil
}
func (b *Builder) UseCustomUnitFileService(filePath string) error {
filename, _ := filepath.Abs(filePath)
unitFile, err := ioutil.ReadFile(filename)
if err != nil {
log.Logger().Errorf("unable to read yaml file %v", err)
return err
}
contents := string(unitFile)
if strings.Contains(contents, "Global=true") {
log.Logger().Warningf("Global fleet scheduling option 'Global=true' can cause undesired results")
}
b.unitFile, err = unit.NewUnitFile(contents)
if err != nil {
log.Logger().Errorf("error creating Unit from %q: %v", contents, err)
return err
}
return nil
}
func TestLegacyPayload(t *testing.T) {
contents := `
[Service]
ExecStart=/bin/sleep 30000
`[1:]
legacyPayloadContents := `{"Name":"sleep.service","Unit":{"Contents":{"Service":{"ExecStart":"/bin/sleep 30000"}},"Raw":"[Service]\nExecStart=/bin/sleep 30000\n"}}`
want, err := unit.NewUnitFile(contents)
if err != nil {
t.Fatalf("unexpected error creating unit from %q: %v", contents, err)
}
var ljp LegacyJobPayload
err = unmarshal(legacyPayloadContents, &ljp)
if err != nil {
t.Error("Error unmarshaling legacy payload:", err)
}
got := ljp.Unit
if !reflect.DeepEqual(*want, got) {
t.Errorf("Unit from legacy payload does not match expected!\nwant:\n%s\ngot:\n%s", *want, got)
}
}
func (ljp *LegacyJobPayload) UnmarshalJSON(data []byte) error {
var ljpm legacyJobPayloadModel
err := json.Unmarshal(data, &ljpm)
if err != nil {
return fmt.Errorf("unable to JSON-deserialize object: %s", err)
}
var u *unit.UnitFile
if len(ljpm.Unit.Raw) > 0 {
u, err = unit.NewUnitFile(ljpm.Unit.Raw)
} else {
u, err = unit.NewUnitFromLegacyContents(ljpm.Unit.Contents)
}
if err != nil {
return err
}
ljp.Unit = *u
ljp.Name = ljpm.Name
return nil
}
func TestSystemdUnitFlow(t *testing.T) {
uDir, err := ioutil.TempDir("", "fleet-")
if err != nil {
t.Fatalf("Failed creating tempdir: %v", err)
}
defer os.RemoveAll(uDir)
mgr, err := systemd.NewSystemdUnitManager(uDir, false)
if err != nil {
t.Fatalf("Failed initializing SystemdUnitManager: %v", err)
}
units, err := mgr.Units()
if err != nil {
t.Fatalf("Failed calling Units(): %v", err)
}
if len(units) > 0 {
t.Fatalf("Expected no units to be returned, got %v", units)
}
contents := `[Service]
ExecStart=/usr/bin/sleep 3000
`
name := fmt.Sprintf("fleet-unit-%d.service", rand.Int63())
uf, err := unit.NewUnitFile(contents)
if err != nil {
t.Fatalf("Invalid unit file: %v", err)
}
hash := uf.Hash().String()
j := job.NewJob(name, *uf)
if err := mgr.Load(j.Name, j.Unit); err != nil {
t.Fatalf("Failed loading job: %v", err)
}
units, err = mgr.Units()
if err != nil {
t.Fatalf("Failed calling Units(): %v", err)
}
if !reflect.DeepEqual([]string{name}, units) {
t.Fatalf("Expected [hello.service], got %v", units)
}
err = waitForUnitState(mgr, name, unit.UnitState{"loaded", "inactive", "dead", "", hash, ""})
if err != nil {
t.Error(err.Error())
}
mgr.TriggerStart(name)
err = waitForUnitState(mgr, name, unit.UnitState{"loaded", "active", "running", "", hash, ""})
if err != nil {
t.Error(err.Error())
}
mgr.TriggerStop(name)
mgr.Unload(name)
units, err = mgr.Units()
if err != nil {
t.Fatalf("Failed calling Units(): %v", err)
}
if len(units) > 0 {
t.Fatalf("Expected no units to be returned, got %v", units)
}
}