예제 #1
0
// This is mostly a horrible test harness for the moment.
func main() {
	var (
		server      = kingpin.Flag("server", "Usenet server hostname").Short('s').Required().String()
		username    = kingpin.Flag("username", "Usenet server username").Short('u').Required().String()
		passfile    = kingpin.Flag("passfile", "Usenet server password file").Short('p').Required().ExistingFile()
		connections = kingpin.Flag("connections", "Usenet max connections").Short('c').Default("1").Int()
		retention   = kingpin.Flag("retention", "Usenet retention in days. 0 for unlimited.").Short('r').Default("0").Int()
		needsgroup  = kingpin.Flag("switchgroup", "Send GROUP before getting article").Short('g').Default("false").Bool()
		nzbfile     = kingpin.Arg("nzb", "NZB file to download").Required().ExistingFile()
		outdir      = kingpin.Arg("outdir", "File to download to").Required().ExistingDir()
	)
	kingpin.Parse()

	n, err := nzb.NewFromFile(*nzbfile)
	if err != nil {
		log.Fatalf("Unable to open NZB file: %v", err)
	}

	o := make([]grabber.ServerOption, 0)
	if *retention > 0 {
		o = append(o, grabber.Retention(time.Duration(*retention)*Day))
	}
	if *needsgroup {
		o = append(o, grabber.MustBeInGroup())
	}

	password, err := util.PasswordFromFile(*passfile)
	if err != nil {
		log.Fatalf("Couldn't read password from %v", *passfile)
	}

	nntpServer, err := nntp.NewServer(*server, 119, *connections, nntp.Credentials(*username, password))
	if err != nil {
		log.Fatalf("Error setting up strategy: %v", err)
	}

	grabServer, err := grabber.NewServer(nntpServer, *server, o...)
	if err != nil {
		log.Fatalf("Error setting up strategy: %v", err)
	}

	grabStrategy, err := grabber.NewStrategy([]grabber.Serverer{grabServer})
	if err != nil {
		log.Fatalf("Error setting up strategy: %v", err)
	}

	g, err := grabber.New(*outdir, grabStrategy, grabber.FromNZB(n))
	if err != nil {
		log.Fatalf("Unable to setup grabber: %v", err)
	}

	g.HandleGrabs()

	if err := g.GrabAll(); err != nil {
		log.Fatalf("Unable to grab all the things: %v", err)
	}

	watchGrabber(g)

	pp := postprocess.New(g.WorkDir())
	if err := pp.Assemble(g.Files()); err != nil {
		log.Println(err)
	}

	if err := g.Shutdown(nil); err != nil {
		log.Fatalf("Unable to shutdown grabber: %v", err)
	}

}
예제 #2
0
func TestGrabber(t *testing.T) {
	t.Parallel()
	for _, tt := range grabberTests {
		servers := make([]Serverer, 0, len(tt.servers))
		for i, ns := range tt.servers {
			s, err := NewServer(ns, ns.Address(), tt.options[i]...)
			if err != nil {
				t.Fatalf("NewServer(%#v, %#v, %#v): %v", ns, ns.Address(), tt.options[i], err)
			}
			servers = append(servers, s)
		}
		ss, err := NewStrategy(servers)
		if err != nil {
			t.Fatalf("NewStrategy(%+v): %v", servers, err)
		}

		n, err := nzb.NewFromFile(tt.f)
		if err != nil {
			t.Fatalf("nzb.NewFromFile(%v): %v", tt.f, err)
			continue
		}

		g, err := New(
			"/tmp",
			ss,
			FromNZB(n, tt.filters...),
			Decoder(createFakeErroreyDecoder),
			SegmentFileCreator(createFakeOut),
		)

		if g.Name() != tt.name {
			t.Errorf("g.Name == %v, wanted %v", g.Name(), tt.name)
		}

		if len(g.Metadata()) != tt.metadata {
			t.Errorf("%v len(g.Metadata) == %v, wanted %v", g.Name(), len(g.Metadata()), tt.metadata)
		}
		if len(g.Files()) != tt.files {
			t.Errorf("%v len(g.Files) == %v, wanted %v", g.Name(), len(g.Files()), tt.files)
		}

		par2Files, filteredFiles, pausedFiles := 0, 0, 0
		for _, f := range g.Files() {
			if f.IsPar2() {
				par2Files++
			}
			if f.IsFiltered() {
				filteredFiles++
			}
			if f.State() == Paused {
				pausedFiles++
			}
		}

		if par2Files != tt.par2Files {
			t.Errorf("%v par2Files == %v, wanted %v", g.Name(), par2Files, tt.par2Files)
		}
		if filteredFiles != tt.filteredFiles {
			t.Errorf("%v filteredFiles == %v, wanted %v", g.Name(), filteredFiles, tt.filteredFiles)
		}
		if pausedFiles != tt.pausedFiles {
			t.Errorf("%v pausedFiles == %v, wanted %v", g.Name(), pausedFiles, tt.pausedFiles)
		}

		g.HandleGrabs()

		if err := g.GrabAll(); err != nil {
			t.Errorf("%v g.GrabAll(): %v", g.Name(), err)
		}

		if err := g.Pause(); err != nil {
			t.Errorf("%v g.Pause(): %v", g.Name(), err)
		}

		//TODO(negz): Debug sporadic resume bug. Smells like fighting mutexes.
		if err := g.Resume(); err != nil {
			t.Errorf("%v g.Resume(): %v", g.Name(), err)
		}

		if !GrabberDone(g) {
			t.Errorf("%v timed out waiting to become postprocessable", g.Name())
			g.Shutdown(nil)
			for _, f := range g.Files() {
				t.Logf("File state: %v", f.State())
			}
			continue
		}

		for _, f := range g.Files() {
			if f.IsPar2() {
				if err := g.GrabFile(f); err != nil {
					t.Errorf("%v g.GrabFile(%v): %v", g.Name(), f, err)
				}
			}
		}

		if !GrabberDone(g) {
			t.Errorf("%v timed out waiting to become postprocessable after requesting additional par2 files", g.Name())
		}

		g.Shutdown(nil)
	}
}