Esempio n. 1
0
func (s *bucketstore) compactSwapFile(bsf *bucketstorefile, compactPath string) error {
	idx, ver, err := parseStoreFileName(filepath.Base(bsf.path), STORE_FILE_SUFFIX)
	if err != nil {
		return err
	}

	nextName := makeStoreFileName(idx, ver+1, STORE_FILE_SUFFIX)
	nextPath := path.Join(filepath.Dir(bsf.path), nextName)

	if err = os.Rename(compactPath, nextPath); err != nil {
		return err
	}

	nextFile, err := fileService.OpenFile(nextPath, os.O_RDWR|os.O_CREATE)
	if err != nil {
		return err
	}

	nextBSF := NewBucketStoreFile(nextPath, nextFile, bsf.stats)
	nextStore, err := gkvlite.NewStore(nextBSF)
	if err != nil {
		// TODO: Rollback the previous *.orig rename.
		return err
	}
	nextBSF.store = nextStore

	atomic.StorePointer(&s.bsf, unsafe.Pointer(nextBSF))

	bsf.apply(func() {
		bsf.purge = true // Mark the old file as purgable.
	})

	return nil
}
Esempio n. 2
0
func (m *mainMenuUI) loadGame(name string) {
	var w World

	f, err := os.OpenFile(filepath.Join(SaveDirName, name+".sav"), os.O_RDWR, 0666)
	if err != nil {
		m.err = err.Error()
		m.state = menuStateError
		return
	}
	w.store, err = gkvlite.NewStore(f)
	if err != nil {
		m.err = err.Error()
		m.state = menuStateError
		f.Close()
		return
	}
	w.storeFile = f

	err = w.init()
	if err != nil {
		m.err = err.Error()
		m.state = menuStateError
		f.Close()
		return
	}

	worldLock.Lock()
	world = &w
	worldLock.Unlock()
}
Esempio n. 3
0
func initkv() {
	dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
	kvFile, _ = os.OpenFile(fmt.Sprintf("%s/database.gkvlite", dir), os.O_RDWR|os.O_CREATE, 0666)
	KV, _ = gkvlite.NewStore(kvFile)
	Users = KV.SetCollection("users", nil)
	Channels = KV.SetCollection("channels", nil)
}
Esempio n. 4
0
func TestDumpColls(t *testing.T) {
	x := 0
	printf := func(format string, a ...interface{}) (n int, err error) {
		x++
		return 0, nil
	}

	store, _ := gkvlite.NewStore(nil)
	c := store.SetCollection("test", nil)
	n, err := dumpColl(printf, c, "")
	if err != nil || n != 0 || x != 0 {
		t.Errorf("expected dumpColl on empty coll to work, got: %v, %v", n, err)
	}
	n, err = dumpCollAsItems(printf, c, "")
	if err != nil || n != 0 || x != 0 {
		t.Errorf("expected dumpCollAsItems on empty coll to work, got: %v, %v", n, err)
	}
	c.Set([]byte("test-key"), (&item{}).toValueBytes())
	x = 0
	n, err = dumpColl(printf, c, "")
	if err != nil || n != 1 || x != 1 {
		t.Errorf("expected dumpColl on 1-item coll to work, got: %v, %v", n, err)
	}
	x = 0
	n, err = dumpCollAsItems(printf, c, "")
	if err != nil || n != 1 || x != 1 {
		t.Errorf("expected dumpCollAsItems on 1-item coll to work, got: %v, %v", n, err)
	}
}
Esempio n. 5
0
// Returns a gkvlite collection
func getCacheStore() *gkvlite.Store {
	f, err := os.OpenFile(cachePath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
	if err != nil {
		log.Fatal(err)
	}
	s, err := gkvlite.NewStore(f)
	if err != nil {
		log.Fatal(err)
	}
	return s
}
Esempio n. 6
0
func newBucketStore(path string, settings BucketSettings) (res *bucketstore, err error) {
	var file FileLike
	if settings.MemoryOnly < MemoryOnly_LEVEL_PERSIST_NOTHING {
		file, err = fileService.OpenFile(path, os.O_RDWR|os.O_CREATE)
		if err != nil {
			fmt.Printf("!!!! %v\n", err)
			return nil, err
		}
	}

	bsf := NewBucketStoreFile(path, file, &BucketStoreStats{})
	bsfForGKVLite := bsf
	if settings.MemoryOnly >= MemoryOnly_LEVEL_PERSIST_NOTHING {
		bsfForGKVLite = nil
	}

	store, err := gkvlite.NewStore(bsfForGKVLite)
	if err != nil {
		return nil, err
	}
	bsf.store = store

	var bsfMemoryOnly *bucketstorefile
	if settings.MemoryOnly > MemoryOnly_LEVEL_PERSIST_EVERYTHING {
		bsfMemoryOnly = NewBucketStoreFile(path, file, bsf.stats)
		bsfMemoryOnly.store, err = gkvlite.NewStore(nil)
		if err != nil {
			return nil, err
		}
	}

	return &bucketstore{
		bsf:           unsafe.Pointer(bsf),
		bsfMemoryOnly: bsfMemoryOnly,
		endch:         make(chan bool),
		partitions:    make(map[uint16]*partitionstore),
		stats:         bsf.stats,
	}, nil
}
Esempio n. 7
0
func NewPster(dbpath string, errlog *log.Logger) (*SimplePster, error) { // {{{1
	var store *gkvlite.Store
	file, err := os.OpenFile(dbpath, os.O_RDWR|os.O_CREATE|os.O_SYNC, 0660)
	if err != nil {
		return nil, err
	}
	store, err = gkvlite.NewStore(file)
	if err != nil {
		return nil, err
	}
	return &SimplePster{
		file:    file,
		store:   store,
		rlog:    store.SetCollection("rlog", nil),
		rfields: store.SetCollection("rfields", nil),
		err:     errlog,
	}, nil
}
Esempio n. 8
0
func mainDo(args []string) error {
	if len(args) < 1 {
		return fmt.Errorf("missing gkvlite file arg")
	}
	fname, args := args[0], args[1:]

	f, err := os.Open(fname)
	if err != nil || f == nil {
		return fmt.Errorf("could not open file: %v", fname)
	}
	defer f.Close()

	s, err := gkvlite.NewStore(f)
	if err != nil || s == nil {
		return fmt.Errorf("could not create store from file: %v", fname)
	}

	cmd := "names"
	if len(args) > 0 {
		cmd, args = args[0], args[1:]
	}
	switch cmd {
	case "names":
		collNames := s.GetCollectionNames()
		for _, collName := range collNames {
			fmt.Printf("%s\n", collName)
		}
	case "items":
		if len(args) < 1 {
			return fmt.Errorf("missing 'items <collection-name>' param")
		}
		collName := args[0]
		coll := s.GetCollection(collName)
		if coll == nil {
			return fmt.Errorf("could not find collection: %v", collName)
		}
		return coll.VisitItemsAscendEx(nil, true, emitItem)
	default:
		return fmt.Errorf("unknown command: %v", cmd)
	}

	return nil
}
Esempio n. 9
0
func compress(fn string) (err error) {
	f, err := os.OpenFile(fn, os.O_RDWR, 0666)
	if err != nil {
		return err
	}
	defer func() {
		err_ := f.Close()
		if err == nil {
			err = err_
		}
	}()

	tmpf, err := ioutil.TempFile("", "gkvlite")
	if err != nil {
		return err
	}
	defer os.Remove(tmpf.Name())
	defer tmpf.Close()

	store, err := gkvlite.NewStore(f)
	if err != nil {
		return err
	}

	store2, err := store.CopyTo(tmpf, 1000000)
	if err != nil {
		return err
	}

	err = f.Truncate(0)
	if err != nil {
		return err
	}

	_, err = store2.CopyTo(f, 1000000)
	if err != nil {
		return err
	}

	return nil
}
Esempio n. 10
0
// NewBasicStore initializes BasicStore and creates Jobs collection
//
// It initializes data storage to a path passed in as an argument
// It returns an error if the store could not be initialized or creation of jobs collection failed
func NewBasicStore(fileName string) (*BasicStore, error) {
	file, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0600)
	if err != nil {
		return nil, err
	}

	s, err := gkvlite.NewStore(file)
	if err != nil {
		return nil, err
	}

	if s.GetCollection("jobs") == nil {
		s.SetCollection("jobs", nil)
		if err := s.Flush(); err != nil {
			return nil, err
		}
	}

	return &BasicStore{
		store: s,
	}, nil
}
Esempio n. 11
0
func (s *bucketstore) compactGo(bsf *bucketstorefile, compactPath string) error {
	// TODO: Should cleanup all old, previous attempts to rescue disk space.
	os.Remove(compactPath) // Clean up any previous attempts.

	compactFile, err := fileService.OpenFile(compactPath, os.O_RDWR|os.O_CREATE|os.O_EXCL)
	if err != nil {
		return err
	}
	defer func() {
		if compactFile != nil {
			compactFile.Close()
			os.Remove(compactPath)
		}
	}()
	compactStore, err := gkvlite.NewStore(compactFile)
	if err != nil {
		return err
	}

	// TODO: Parametrize writeEvery.
	writeEvery := 1000

	lastChanges := make(map[uint16]*gkvlite.Item) // Last items in changes colls.
	collNames := bsf.store.GetCollectionNames()   // Names of collections to process.
	collRest := make([]string, 0, len(collNames)) // Names of unprocessed collections.
	vbids := make([]uint16, 0, len(collNames))    // VBucket id's that we processed.

	// Process compaction in a few steps: first, unlocked,
	// snapshot-based collection copying meant to handle most of each
	// vbucket's data; and then, locked copying of any vbucket
	// mutations (deltas) that happened in the meantime.  Then, while
	// still holding all vbucket collection locks, we copy any
	// remaining non-vbucket collections and atomically swap the files.
	for _, collName := range collNames {
		if !strings.HasSuffix(collName, COLL_SUFFIX_CHANGES) {
			if !strings.HasSuffix(collName, COLL_SUFFIX_KEYS) {
				// It's neither a changes nor a keys collection.
				collRest = append(collRest, collName)
			}
			continue
		}
		vbid, lastChange, err :=
			s.copyVBucketColls(bsf, collName, compactStore, writeEvery)
		if err != nil {
			return err
		}
		lastChanges[uint16(vbid)] = lastChange
		vbids = append(vbids, uint16(vbid))
	}

	return s.copyBucketStoreDeltas(bsf, compactStore,
		vbids, 0, lastChanges, writeEvery, func() (err error) {
			// Copy any remaining (simple) collections (lifke COLL_VBMETA).
			err = s.copyRemainingColls(bsf, collRest, compactStore, writeEvery)
			if err != nil {
				return err
			}
			err = compactStore.Flush()
			if err != nil {
				return err
			}
			compactFile.Close()

			return s.compactSwapFile(bsf, compactPath) // The last step.
		})
}
Esempio n. 12
0
func TestCollRangeCopy(t *testing.T) {
	s, _ := gkvlite.NewStore(nil)
	x := s.SetCollection("x", nil)
	testFillColl(x, []string{"b", "c", "d"})

	d, _ := gkvlite.NewStore(nil)
	dx := d.SetCollection("dx", nil)
	err := collRangeCopy(x, dx, []byte("a"), []byte("a"), []byte("z"))
	testCheckColl(t, dx, "a", []string{"b", "c", "d"}, nil)

	d, _ = gkvlite.NewStore(nil)
	dx = d.SetCollection("dx", nil)
	err = collRangeCopy(x, dx, []byte("a"), []byte("a"), []byte("a"))
	if err != nil {
		t.Errorf("expected collRangeCopy to not have error")
	}
	testCheckColl(t, dx, "a", []string{}, nil)

	d, _ = gkvlite.NewStore(nil)
	dx = d.SetCollection("dx", nil)
	err = collRangeCopy(x, dx, []byte("z"), []byte("z"), []byte("a"))
	if err != nil {
		t.Errorf("expected collRangeCopy to not have error")
	}
	testCheckColl(t, dx, "a", []string{}, nil)

	d, _ = gkvlite.NewStore(nil)
	dx = d.SetCollection("dx", nil)
	err = collRangeCopy(x, dx, []byte("a"), []byte("a"), []byte("b"))
	if err != nil {
		t.Errorf("expected collRangeCopy to not have error")
	}
	testCheckColl(t, dx, "a", []string{}, nil)

	d, _ = gkvlite.NewStore(nil)
	dx = d.SetCollection("dx", nil)
	err = collRangeCopy(x, dx, []byte("d"), []byte("z"), []byte("z"))
	if err != nil {
		t.Errorf("expected collRangeCopy to not have error")
	}
	testCheckColl(t, dx, "a", []string{}, nil)

	d, _ = gkvlite.NewStore(nil)
	dx = d.SetCollection("dx", nil)
	err = collRangeCopy(x, dx, []byte("d"), []byte("d"), []byte("z"))
	if err != nil {
		t.Errorf("expected collRangeCopy to not have error")
	}
	testCheckColl(t, dx, "a", []string{"d"}, nil)

	d, _ = gkvlite.NewStore(nil)
	dx = d.SetCollection("dx", nil)
	err = collRangeCopy(x, dx, []byte("a"), []byte("c"), []byte("d"))
	if err != nil {
		t.Errorf("expected collRangeCopy to not have error")
	}
	testCheckColl(t, dx, "a", []string{"c"}, nil)

	d, _ = gkvlite.NewStore(nil)
	dx = d.SetCollection("dx", nil)
	err = collRangeCopy(x, dx, []byte("a"), []byte("b"), []byte("c"))
	if err != nil {
		t.Errorf("expected collRangeCopy to not have error")
	}
	testCheckColl(t, dx, "a", []string{"b"}, nil)
}
Esempio n. 13
0
func (m *mainMenuUI) inputKey(key termbox.Key, ch rune, mod termbox.Modifier) bool {
	switch m.state {
	case menuStateMain:
		switch {
		case key == termbox.KeyArrowDown:
			m.choiceIndex = (m.choiceIndex + 1) % (len(m.saveNames) + 1)
		case key == termbox.KeyArrowUp:
			m.choiceIndex = (m.choiceIndex + (len(m.saveNames) + 1 - 1)) % (len(m.saveNames) + 1)
		case key == termbox.KeyArrowLeft || key == termbox.KeyArrowRight:
			// silently ignore
		case key == termbox.KeyEnter:
			if m.choiceIndex < len(m.saveNames) {
				m.loadGame(m.saveNames[m.choiceIndex])
			} else {
				switch m.choiceIndex - len(m.saveNames) {
				case 0:
					m.newGame()
				}
			}
		case key == termbox.KeyEsc:
			return false
		default:
			panic(fmt.Sprintf("%v, %v, %v", key, ch, mod))
		}

	case menuStateError:
		switch {
		case key == termbox.KeyEsc:
			m.err = ""
			m.state = menuStateMain
		default:
			panic(fmt.Sprintf("%v, %v, %v", key, ch, mod))
		}

	case menuStateNew:
		const fieldCount = 2

		switch {
		case key == termbox.KeyEsc:
			m.state = menuStateMain
			m.choiceIndex = len(m.saveNames)
		case key == termbox.KeyEnter:
			m.choiceIndex++
			if m.choiceIndex >= fieldCount {
				m.choiceIndex = fieldCount - 1
				if len(m.saveName) == 0 {
					m.choiceIndex = 0
					fmt.Print("\a")
					return true
				}
				_ = os.MkdirAll(SaveDirName, 0777)
				f, err := os.OpenFile(filepath.Join(SaveDirName, string(m.saveName)+".sav"), os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
				var w *World
				if err == nil {
					var store *gkvlite.Store
					store, err = gkvlite.NewStore(f)
					w = &World{store: store, storeFile: f}
				}
				if err == nil {
					err = w.setSeed(NewSeed(string(m.seed)))
				}

				if err == nil {
					err = w.init()
				}

				if err == nil {
					m.saveNames = append(m.saveNames, string(m.saveName))
					worldLock.Lock()
					world = w
					worldLock.Unlock()
				} else {
					if f != nil {
						f.Close()
					}
					m.err = err.Error()
					m.state = menuStateError
				}
			}
		case key == termbox.KeyArrowDown:
			m.choiceIndex = (m.choiceIndex + 1) % fieldCount
		case key == termbox.KeyArrowUp:
			m.choiceIndex = (m.choiceIndex + (fieldCount - 1)) % fieldCount
		case key == termbox.KeyArrowLeft || key == termbox.KeyArrowRight:
			fmt.Print("\a")
		case m.choiceIndex == 0 && (key == termbox.KeyBackspace || key == termbox.KeyBackspace2):
			if len(m.saveName) == 0 {
				fmt.Print("\a")
			} else {
				m.saveName = m.saveName[:len(m.saveName)-1]
			}
		case m.choiceIndex == 0 && key == termbox.KeySpace:
			if len(m.saveName) == 0 {
				fmt.Print("\a")
			} else {
				m.saveName = append(m.saveName, ' ')
			}
		case m.choiceIndex == 0 && ch != 0:
			// unicode.Punctuation is not included due to characters like /
			if ch != '_' && ch != '-' && !unicode.In(ch, unicode.Letter, unicode.Number, unicode.Symbol) {
				fmt.Print("\a")
			} else {
				m.saveName = append(m.saveName, ch)
			}
		case m.choiceIndex == 1 && (key == termbox.KeyBackspace || key == termbox.KeyBackspace2):
			if len(m.seed) == 0 {
				fmt.Print("\a")
			} else {
				m.seed = m.seed[:len(m.seed)-1]
			}
		case m.choiceIndex == 1 && key == termbox.KeySpace:
			m.seed = append(m.seed, ' ')
		case m.choiceIndex == 1 && ch != 0:
			m.seed = append(m.seed, ch)
		default:
			panic(fmt.Sprintf("%v, %v, %v", key, ch, mod))
		}

	default:
		panic(fmt.Sprintf("%v, %v, %v", key, ch, mod))
	}

	return true
}
Esempio n. 14
0
func main() {
	//var start string
	//flag.StringVar(&start, "start", "", "starting url")
	//flag.Parse()

	flag.Parse()
	if len(flag.Args()) < 2 {
		log.Fatal("Usage: mirror start-url credentials")
	}
	start := flag.Args()[0]
	username, password := Split2(flag.Args()[1], ":")

	// http://stackoverflow.com/questions/18414212/golang-how-to-follow-location-with-cookie
	options := cookiejar.Options{}
	cookie_jar, err := cookiejar.New(&options)
	client := &http.Client{
		Jar: cookie_jar,
	}
	v := url.Values{"user": {username}, "passwrd": {password}}
	req, err := http.NewRequest("POST", start+"index.php?action=login2",
		bytes.NewReader([]byte(v.Encode())))
	// ugh
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Add("Accept-Encoding", "identity")
	pres, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	contents, err := ioutil.ReadAll(pres.Body)
	if !strings.Contains(string(contents), "action=unread") {
		log.Fatal("Login did not work")
	}

	f, err := os.OpenFile("smfmirror.gkvlite", os.O_RDWR, 0666)
	if err != nil {
		f, err = os.Create("smfmirror.gkvlite")
	}
	if err != nil {
		panic(err)
	}
	s, err := gkvlite.NewStore(f)
	if err != nil {
		panic(err)
	}
	c := s.GetCollection("smfmirror")
	if c == nil {
		c = s.SetCollection("smfmirror", nil)
	}
	pboard_links, err := c.Get([]byte("board_links"))
	var board_links []string
	if pboard_links != nil {
		err := json.Unmarshal(pboard_links, &board_links)
		if err != nil {
			panic(err)
		}
	} else {
		board_links = loadBoardLinks(client, start)
		b, err := json.Marshal(board_links)
		if err != nil {
			panic(err)
		}
		//c.Set([]byte("hello"), []byte("world"))
		//a, x := c.Get([]byte("hello"))
		//fmt.Printf("%v %v\n", a, x)
		c.Set([]byte("board_links"), b)
		s.Flush()
		s.Close()
		f.Sync()
		f.Close()
	}

	for _, url := range board_links {
		fmt.Printf("%v\n", url)

	}
	//fmt.Printf("%v\n", board_links)
}