Exemplo n.º 1
0
// Initializes JobDB with data from the file config.JobDBPath if it exists and
// starts the goroutine that runs handleJobDBRequests.
// Not an init() because main() needs to set up some things first.
func JobsInit() {
	if jobDB != nil {
		panic("JobsInit() called twice")
	}
	jobdb_storer := &LoggingFileStorer{xml.FileStorer{config.JobDBPath}}
	var delay time.Duration = config.DBPersistDelay
	jobDB = xml.NewDB("jobdb", jobdb_storer, delay)
	if !config.FreshDatabase {
		xml, err := xml.FileToHash(config.JobDBPath)
		if err != nil {
			if os.IsNotExist(err) {
				/* File does not exist is not an error that needs to be reported */
			} else {
				util.Log(0, "ERROR! JobsInit() reading '%v': %v", config.JobDBPath, err)
			}
		} else {
			jobDB.Init(xml)
		}
	}

	// Remove all non-local jobs (they may be stale and new_server
	// causes full sync anyway)
	jobDB.Remove(xml.FilterNot(xml.FilterSimple("siserver", config.ServerSourceAddress)))

	// The following loop goes through all jobs and does the following
	// * find the the greatest id number used in the db
	// * schedule processing of pending actions for all timestamps
	var count uint64 = 0
	for job := jobDB.Query(xml.FilterAll).First("job"); job != nil; job = job.Next() {
		id, err := strconv.ParseUint(job.Text("id"), 10, 64)
		if err != nil {
			panic(err)
		}
		if id > count {
			count = id
		}

		scheduleProcessPendingActions(job.Text("timestamp"))
	}
	nextID = util.Counter(count + 1)

	go handleJobDBRequests()
}
Exemplo n.º 2
0
// Unit tests for the package github.com/mbenkmann/golib/util.
func Util_test() {
	fmt.Printf("\n==== util ===\n\n")

	addr, err := util.Resolve("1.2.3.4", "")
	check(err, nil)
	check(addr, "1.2.3.4")

	addr, err = util.Resolve("1.2.3.4:5", "")
	check(err, nil)
	check(addr, "1.2.3.4:5")

	addr, err = util.Resolve("::1:5", "")
	check(err, nil)
	check(addr, "[::1:5]")

	addr, err = util.Resolve("localhost:65535", "")
	check(err, nil)
	check(addr, "127.0.0.1:65535")

	addr, err = util.Resolve("localhost", "")
	check(err, nil)
	check(addr, "127.0.0.1")

	addr, err = util.Resolve("::1", "")
	check(err, nil)
	check(addr, "127.0.0.1")

	addr, err = util.Resolve("[::1]", "")
	check(err, nil)
	check(addr, "127.0.0.1")

	addr, err = util.Resolve("[::1]:12345", "")
	check(err, nil)
	check(addr, "127.0.0.1:12345")

	addr, err = util.Resolve("localhost:65535", "foo")
	check(err, nil)
	check(addr, "foo:65535")

	addr, err = util.Resolve("localhost", "foo")
	check(err, nil)
	check(addr, "foo")

	addr, err = util.Resolve("::1", "foo")
	check(err, nil)
	check(addr, "foo")

	addr, err = util.Resolve("[::1]", "foo")
	check(err, nil)
	check(addr, "foo")

	addr, err = util.Resolve("[::1]:12345", "foo")
	check(err, nil)
	check(addr, "foo:12345")

	addr, err = util.Resolve("", "")
	check(hasWords(err, "no", "such", "host"), "")
	check(addr, "")

	addr, err = util.Resolve(":10", "")
	check(hasWords(err, "no", "such", "host"), "")
	check(addr, ":10")

	check(util.WaitForDNS(3*time.Second), true)

	h, _ := exec.Command("hostname").CombinedOutput()
	hostname := strings.TrimSpace(string(h))

	ipp, _ := exec.Command("hostname", "-I").CombinedOutput()
	ips := strings.Fields(strings.TrimSpace(string(ipp)))
	addr, err = util.Resolve(hostname+":234", config.IP)
	check(err, nil)
	ip := ""
	for _, ip2 := range ips {
		if addr == ip2+":234" {
			ip = ip2
		}
	}
	check(addr, ip+":234")

	testLogging()

	buf := make([]byte, 80)
	for i := range buf {
		buf[i] = byte(util_test_rng.Intn(26) + 'a')
	}

	crap1 := &crappyConnection1{}
	n, err := util.WriteAll(crap1, buf)
	check(string(*crap1), string(buf))
	check(n, len(buf))
	check(err, nil)

	crap2 := &crappyConnection2{}
	n, err = util.WriteAll(crap2, buf)
	check(string(*crap2), string(buf))
	check(n, len(buf))
	check(err, nil)

	stalled1 := &stalledConnection1{}
	n, err = util.WriteAll(stalled1, buf)
	check(string(*stalled1), string(buf[0:16]))
	check(n, 16)
	check(err, io.ErrShortWrite)

	stalled2 := &stalledConnection2{}
	n, err = util.WriteAll(stalled2, buf)
	check(string(*stalled2), string(buf[0:16]))
	check(n, 16)
	check(err, io.ErrShortWrite)

	broken := &brokenConnection{}
	n, err = util.WriteAll(broken, buf)
	check(string(*broken), string(buf[0:16]))
	check(n, 16)
	check(err, io.ErrClosedPipe)

	panicker := func() {
		foobar = "bar"
		panic("foo")
	}

	var buffy bytes.Buffer
	util.LoggersSuspend()
	util.LoggerAdd(&buffy)
	defer util.LoggersRestore()

	util.WithPanicHandler(panicker)
	time.Sleep(200 * time.Millisecond) // make sure log message is written out
	check(foobar, "bar")
	check(len(buffy.String()) > 10, true)

	listener, err := net.Listen("tcp", "127.0.0.1:39390")
	if err != nil {
		panic(err)
	}

	go func() {
		r, err := listener.Accept()
		if err != nil {
			panic(err)
		}
		buf := make([]byte, 1)
		r.Read(buf)
		time.Sleep(10 * time.Second)
		r.Read(buf)
	}()
	long := make([]byte, 10000000)
	longstr := string(long)
	buffy.Reset()
	t0 := time.Now()
	util.SendLnTo("127.0.0.1:39390", longstr, 5*time.Second)
	duration := time.Since(t0)
	check(duration > 4*time.Second && duration < 6*time.Second, true)
	time.Sleep(200 * time.Millisecond) // make sure log message is written out
	check(strings.Contains(buffy.String(), "ERROR"), true)

	go func() {
		conn, err := listener.Accept()
		if err != nil {
			panic(err)
		}
		ioutil.ReadAll(conn)
	}()
	long = make([]byte, 10000000)
	longstr = string(long)
	buffy.Reset()
	t0 = time.Now()
	util.SendLnTo("127.0.0.1:39390", longstr, 5*time.Second)
	duration = time.Since(t0)
	check(duration < 2*time.Second, true)
	time.Sleep(200 * time.Millisecond) // make sure log message is written out
	check(buffy.String(), "")

	// Test that ReadLn() times out properly
	go func() {
		_, err := net.Dial("tcp", "127.0.0.1:39390")
		if err != nil {
			panic(err)
		}
	}()
	conn, err := listener.Accept()
	if err != nil {
		panic(err)
	}
	t0 = time.Now()
	st, err := util.ReadLn(conn, 5*time.Second)
	duration = time.Since(t0)
	check(duration > 4*time.Second && duration < 6*time.Second, true)
	check(st, "")
	check(hasWords(err, "timeout"), "")

	// Test that ReadLn() returns io.EOF if last line not terminated by \n
	go func() {
		conn, err := net.Dial("tcp", "127.0.0.1:39390")
		if err != nil {
			panic(err)
		}
		conn.Write([]byte("foo\r"))
		conn.Close()
	}()
	conn, err = listener.Accept()
	if err != nil {
		panic(err)
	}
	st, err = util.ReadLn(conn, 5*time.Second)
	check(err, io.EOF)
	check(st, "foo")

	go func() {
		conn, err := net.Dial("tcp", "127.0.0.1:39390")
		if err != nil {
			panic(err)
		}
		conn.Write([]byte("\r\r\n\rfo\ro\nbar\r\nfoxtrott"))
		conn.Close()
	}()
	conn, err = listener.Accept()
	if err != nil {
		panic(err)
	}
	// Test proper trimming of multiple \r
	st, err = util.ReadLn(conn, 0)
	check(err, nil)
	check(st, "")
	// Test that the empty first line has actually been read
	// and that the next ReadLn() reads the 2nd line
	// Also test that negative timeouts work the same as timeout==0
	// Also test that \r is not trimmed at start and within line.
	st, err = util.ReadLn(conn, -1*time.Second)
	check(err, nil)
	check(st, "\rfo\ro")
	// Check 3rd line
	st, err = util.ReadLn(conn, 0)
	check(err, nil)
	check(st, "bar")
	// Check 4th line and io.EOF error
	st, err = util.ReadLn(conn, 0)
	check(err, io.EOF)
	check(st, "foxtrott")

	// Test that delayed reads work with timeout==0
	go func() {
		conn, err := net.Dial("tcp", "127.0.0.1:39390")
		if err != nil {
			panic(err)
		}
		time.Sleep(1 * time.Second)
		_, err = conn.Write([]byte("foo\r\n"))
		if err != nil {
			panic(err)
		}
		time.Sleep(2 * time.Second)
	}()
	conn, err = listener.Accept()
	if err != nil {
		panic(err)
	}
	t0 = time.Now()
	st, err = util.ReadLn(conn, time.Duration(0))
	duration = time.Since(t0)
	check(duration < 2*time.Second, true)
	check(duration > 800*time.Millisecond, true)
	check(err, nil)
	check(st, "foo")

	counter := util.Counter(13)
	var b1 UintArray = make([]uint64, 100)
	var b2 UintArray = make([]uint64, 100)
	done := make(chan bool)
	fill := func(b UintArray) {
		for i := 0; i < 100; i++ {
			b[i] = <-counter
			time.Sleep(1 * time.Millisecond)
		}
		done <- true
	}
	go fill(b1)
	go fill(b2)
	<-done
	<-done
	check(sort.IsSorted(&b1), true)
	check(sort.IsSorted(&b2), true)
	var b3 UintArray = make([]uint64, 200)
	i := 0
	j := 0
	k := 0
	for i < 100 || j < 100 {
		if i == 100 {
			b3[k] = b2[j]
			j++
			k++
			continue
		}
		if j == 100 {
			b3[k] = b1[i]
			i++
			k++
			continue
		}
		if b1[i] == b2[j] {
			check(b1[i] != b2[j], true)
			break
		}
		if b1[i] < b2[j] {
			b3[k] = b1[i]
			i++
		} else {
			b3[k] = b2[j]
			j++
		}
		k++
	}

	one_streak := true
	b5 := make([]uint64, 200)
	for i := 0; i < 200; i++ {
		if i < 100 && b1[i] != uint64(13+i) && b2[i] != uint64(13+i) {
			one_streak = false
		}
		b5[i] = uint64(13 + i)
	}

	check(b3, b5)
	check(one_streak, false) // Check whether goroutines were actually executed concurrently rather than in sequence

	tempdir, err := ioutil.TempDir("", "util-test-")
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(tempdir)
	fpath := tempdir + "/foo.log"
	logfile := util.LogFile(fpath)
	check(logfile.Close(), nil)
	n, err = util.WriteAll(logfile, []byte("Test"))
	check(err, nil)
	check(n, 4)
	check(logfile.Close(), nil)
	n, err = util.WriteAll(logfile, []byte("12"))
	check(err, nil)
	check(n, 2)
	n, err = util.WriteAll(logfile, []byte("3"))
	check(err, nil)
	check(n, 1)
	check(os.Rename(fpath, fpath+".old"), nil)
	n, err = util.WriteAll(logfile, []byte("Fo"))
	check(err, nil)
	check(n, 2)
	f2, _ := os.OpenFile(fpath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
	f2.Write([]byte("o"))
	f2.Close()
	n, err = util.WriteAll(logfile, []byte("bar"))
	check(err, nil)
	check(n, 3)
	check(logfile.Close(), nil)
	data, err := ioutil.ReadFile(fpath)
	check(err, nil)
	if err == nil {
		check(string(data), "Foobar")
	}
	data, err = ioutil.ReadFile(fpath + ".old")
	check(err, nil)
	if err == nil {
		check(string(data), "Test123")
	}

	test_time := time.Date(2013, time.January, 20, 14, 7, 21, 0, time.Local)
	check(util.MakeTimestamp(test_time), "20130120140721")
	test_time = time.Date(2013, time.January, 20, 14, 7, 21, 0, time.UTC)
	check(util.MakeTimestamp(test_time), "20130120140721")
	test_time = time.Date(2013, time.January, 20, 14, 7, 21, 0, time.FixedZone("Fooistan", 45678))
	check(util.MakeTimestamp(test_time), "20130120140721")
	illegal := time.Unix(0, 0)
	buffy.Reset()
	check(util.ParseTimestamp(""), illegal)
	time.Sleep(200 * time.Millisecond) // make sure log message is written out
	check(strings.Contains(buffy.String(), "ERROR"), true)
	buffy.Reset()
	check(util.ParseTimestamp("20139910101010"), illegal)
	time.Sleep(200 * time.Millisecond) // make sure log message is written out
	check(strings.Contains(buffy.String(), "ERROR"), true)
	check(util.ParseTimestamp("20131110121314"), time.Date(2013, time.November, 10, 12, 13, 14, 0, time.Local))
	check(util.MakeTimestamp(util.ParseTimestamp(util.MakeTimestamp(test_time))), util.MakeTimestamp(test_time))
	test_time = test_time.Add(2400 * time.Hour)
	check(util.MakeTimestamp(util.ParseTimestamp(util.MakeTimestamp(test_time))), util.MakeTimestamp(test_time))
	test_time = test_time.Add(2400 * time.Hour)
	check(util.MakeTimestamp(util.ParseTimestamp(util.MakeTimestamp(test_time))), util.MakeTimestamp(test_time))
	test_time = test_time.Add(2400 * time.Hour)
	check(util.MakeTimestamp(util.ParseTimestamp(util.MakeTimestamp(test_time))), util.MakeTimestamp(test_time))
	test_time = test_time.Add(2400 * time.Hour)
	check(util.MakeTimestamp(util.ParseTimestamp(util.MakeTimestamp(test_time))), util.MakeTimestamp(test_time))

	diff := time.Since(util.ParseTimestamp(util.MakeTimestamp(time.Now())))
	if diff < time.Second {
		diff = 0
	}
	check(diff, time.Duration(0))

	t0 = time.Now()
	util.WaitUntil(t0.Add(-10 * time.Second))
	util.WaitUntil(t0.Add(-100 * time.Minute))
	dur := time.Now().Sub(t0)
	if dur < 1*time.Second {
		dur = 0
	}
	check(dur, 0)
	t0 = time.Now()
	util.WaitUntil(t0.Add(1200 * time.Millisecond))
	dur = time.Now().Sub(t0)
	if dur >= 1200*time.Millisecond && dur <= 1300*time.Millisecond {
		dur = 1200 * time.Millisecond
	}
	check(dur, 1200*time.Millisecond)

	mess := "WaitUntil(Jesus first birthday) takes forever"
	go func() {
		util.WaitUntil(time.Date(1, time.December, 25, 0, 0, 0, 0, time.UTC))
		mess = ""
	}()
	time.Sleep(100 * time.Millisecond)
	check(mess, "")

	mess = "WaitUntil(1000-11-10 00:00:00) takes forever"
	go func() {
		util.WaitUntil(time.Date(1000, time.October, 11, 0, 0, 0, 0, time.UTC))
		mess = ""
	}()
	time.Sleep(100 * time.Millisecond)
	check(mess, "")

	testBase64()
}