func genNewArticle(title string) {
fmt.Printf("genNewArticle: %q\n", title)
store, err := NewStore()
if err != nil {
log.Fatalf("NewStore() failed with %s", err)
}
newId := findUniqueArticleId(store.articles)
name := sanitizeForFile(title) + ".md"
fmt.Printf("new id: %d, name: %s\n", newId, name)
t := time.Now()
dir := "blog_posts"
d := t.Format("2006-01")
path := filepath.Join(dir, d, name)
s := fmt.Sprintf(`Id: %d
Title: %s
Date: %s
Format: Markdown
--------------`, newId, title, t.Format(time.RFC3339))
for i := 1; i < 10; i++ {
if !u.PathExists(path) {
break
}
name := sanitizeForFile(title) + "-" + strconv.Itoa(i) + ".md"
path = filepath.Join(dir, d, name)
}
u.PanicIf(u.PathExists(path))
fmt.Printf("path: %s\n", path)
u.CreateDirForFileMust(path)
ioutil.WriteFile(path, []byte(s), 0644)
}
func getAppEngineTmplDir() string {
// when running locally
d := filepath.Join("..", "tmpl")
if u.PathExists(d) {
return d
}
// when running on a server
d = "appengtmpl"
if u.PathExists(d) {
return d
}
logger.Errorf("getAppEngineTmplDir(): %q dir doesn't exist", d)
return ""
}
func getWwwDir() string {
// when running locally
d := filepath.Join("..", "www")
if u.PathExists(d) {
return d
}
// when running on a server
d = "www"
if u.PathExists(d) {
return d
}
logger.Errorf("getWwwDir(): %q dir doesn't exist", d)
return ""
}
func serveFileFromDir(w http.ResponseWriter, r *http.Request, dir, fileName string) {
filePath := filepath.Join(dir, fileName)
if !u.PathExists(filePath) {
logger.Noticef("serveFileFromDir() file %q doesn't exist, referer: %q", fileName, getReferer(r))
}
http.ServeFile(w, r, filePath)
}
func NewStore(dataDir, forumName string) (*Store, error) {
dataFilePath := filepath.Join(dataDir, "forum", forumName+".txt")
store := &Store{
dataDir: dataDir,
forumName: forumName,
posts: make([]*Post, 0),
topics: make([]Topic, 0),
}
var err error
if u.PathExists(dataFilePath) {
if err = store.readExistingData(dataFilePath); err != nil {
fmt.Printf("readExistingData() failed with %s", err)
return nil, err
}
} else {
f, err := os.Create(dataFilePath)
if err != nil {
fmt.Printf("NewStore(): os.Create(%s) failed with %s", dataFilePath, err)
return nil, err
}
f.Close()
}
verifyTopics(store.topics)
store.dataFile, err = os.OpenFile(dataFilePath, os.O_APPEND|os.O_RDWR, 0666)
if err != nil {
fmt.Printf("NewStore(): os.OpenFile(%s) failed with %s", dataFilePath, err)
return nil, err
}
return store, nil
}
func serveFileFromDir(w http.ResponseWriter, r *http.Request, dir, fileName string) {
filePath := filepath.Join(dir, fileName)
if !u.PathExists(filePath) {
fmt.Printf("serveFileFromDir() file=%s doesn't exist\n", filePath)
}
http.ServeFile(w, r, filePath)
}
func NewStoreCrashes(dataDir string) (*StoreCrashes, error) {
dataFilePath := filepath.Join(dataDir, "data", "crashesdata.txt")
store := &StoreCrashes{
dataDir: dataDir,
crashes: make([]*Crash, 0),
apps: make([]*App, 0),
versions: make([]*string, 0),
ips: make(map[string]*string),
crashingLines: make(map[string]*string),
}
var err error
if u.PathExists(dataFilePath) {
err = store.readExistingCrashesData(dataFilePath)
if err != nil {
logger.Errorf("NewStoreCrashes(): readExistingCrashesData() failed with %s\n", err)
return nil, err
}
} else {
f, err := os.Create(dataFilePath)
if err != nil {
logger.Errorf("NewStoreCrashes(): os.Create(%s) failed with %s", dataFilePath, err)
return nil, err
}
f.Close()
}
store.dataFile, err = os.OpenFile(dataFilePath, os.O_APPEND|os.O_RDWR, 0666)
if err != nil {
logger.Errorf("NewStoreCrashes(): os.OpenFile(%s) failed with %s", dataFilePath, err)
return nil, err
}
logger.Noticef("crashes: %d, versions: %d, ips: %d, crashing lines: %d", len(store.crashes), len(store.versions), len(store.ips), len(store.crashingLines))
return store, nil
}
// data dir is ../../data on the server or ~/data/apptranslator locally
// the important part is that it's outside of directory with the code
func getDataDir() string {
if dataDir != "" {
return dataDir
}
// on the server, must be done first because ExpandTildeInPath()
// doesn't work when cross-compiled on mac for linux
dataDir = filepath.Join("..", "..", "data")
if u.PathExists(dataDir) {
return dataDir
}
dataDir = u.ExpandTildeInPath("~/data/apptranslator")
if u.PathExists(dataDir) {
return dataDir
}
log.Fatal("data directory (../../data or ../apptranslatordata) doesn't exist")
return ""
}
// /static/$rest
func handleStatic(w http.ResponseWriter, r *http.Request) {
fileName := r.URL.Path[len("/static/"):]
path := filepath.Join("www", fileName)
if u.PathExists(path) {
LogInfof("%s\n", path)
http.ServeFile(w, r, path)
} else {
LogInfof("file %q doesn't exist, referer: %q\n", path, getReferer(r))
http.NotFound(w, r)
}
}
func readAppData(app *App) error {
var path string
path = app.storeCsvFilePath()
if u.PathExists(path) {
if l, err := store.NewStoreCsv(path); err == nil {
app.store = l
return nil
}
}
return fmt.Errorf("readAppData: %q data file doesn't exist", path)
}
func serveFileFromDir(w http.ResponseWriter, r *http.Request, dir, fileName string) {
if redirectIfFoundMatching(w, r, dir, fileName) {
return
}
filePath := filepath.Join(dir, fileName)
if u.PathExists(filePath) {
//logger.Noticef("serveFileFromDir(): %q", filePath)
http.ServeFile(w, r, filePath)
} else {
logger.Noticef("serveFileFromDir() file %q doesn't exist, referer: %q", fileName, getReferer(r))
http.NotFound(w, r)
}
}
func removeStoreFiles(basePath string) {
path := idxFilePath(basePath)
os.Remove(path)
nSegment := 0
for {
path = segmentFilePath(basePath, nSegment)
if !u.PathExists(path) {
break
}
os.Remove(path)
nSegment += 1
}
}
// tests if s3 credentials are valid and aborts if aren't
func ensureValidConfig(config *BackupConfig) {
if !u.PathExists(config.LocalDir) {
log.Fatalf("Invalid s3 backup: directory to backup %q doesn't exist\n", config.LocalDir)
}
if !strings.HasSuffix(config.S3Dir, bucketDelim) {
config.S3Dir += bucketDelim
}
_, err := listBackupFiles(config, 10)
if err != nil {
log.Fatalf("Invalid s3 backup: bucket.List failed %s\n", err)
}
}
func serveFile(w http.ResponseWriter, r *http.Request, fileName string) {
//LogVerbosef("serverFile: fileName='%s'\n", fileName)
path := filepath.Join("www", fileName)
if hasZipResources() {
serveResourceFromZip(w, r, path)
return
}
if u.PathExists(path) {
http.ServeFile(w, r, path)
} else {
LogVerbosef("file '%s' doesn't exist\n", path)
http.NotFound(w, r)
}
}
func getDataDir() string {
if dataDir != "" {
return dataDir
}
// on the server, must be done first because ExpandTildeInPath()
// doesn't work when cross-compiled on mac for linux
serverDir := filepath.Join("..", "..", "data")
dataDir = serverDir
if u.PathExists(dataDir) {
return dataDir
}
// locally
localDir := u.ExpandTildeInPath("~/data/blog")
dataDir = localDir
if u.PathExists(dataDir) {
return dataDir
}
log.Fatalf("data directory (%q or %q) doesn't exist", serverDir, localDir)
return ""
}
func NewWithLimit(basePath string, maxSegmentSize int) (store *Store, err error) {
store = &Store{
basePath: basePath,
blobs: make([]blob, 0),
sha1HexToBlobNo: make(map[string]int),
maxSegmentSize: maxSegmentSize,
cachedSegmentNo: -1,
}
idxPath := idxFilePath(basePath)
idxDidExist := u.PathExists(idxPath)
if idxDidExist {
if err = store.readIndex(); err != nil {
return nil, err
}
}
if store.idxFile, err = os.OpenFile(idxPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644); err != nil {
return nil, err
}
store.idxCsvWriter = csv.NewWriter(store.idxFile)
if !idxDidExist {
rec := [][]string{{idxHdr}}
err = store.idxCsvWriter.WriteAll(rec)
if err != nil {
return nil, err
}
}
segmentPath := segmentFilePath(store.basePath, store.currSegmentNo)
stat, err := os.Stat(segmentPath)
if err != nil {
// TODO: fail if error is different than "file doesn't exist"
if store.currSegmentNo != 0 {
store.Close()
return nil, errSegmentFileMissing
}
store.currSegmentFile, err = os.Create(segmentPath)
if err != nil {
store.Close()
return nil, err
}
} else {
store.currSegmentSize = int(stat.Size())
store.currSegmentFile, err = os.OpenFile(segmentPath, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
store.Close()
return nil, err
}
}
return store, nil
}
func NewForum(config *ForumConfig) *Forum {
forum := &Forum{ForumConfig: *config}
sidebarTmplPath := filepath.Join("forums", fmt.Sprintf("%s_sidebar.html", forum.ForumUrl))
if !u.PathExists(sidebarTmplPath) {
panic(fmt.Sprintf("sidebar template %s for forum %s doesn't exist", sidebarTmplPath, forum.ForumUrl))
}
forum.SidebarTmpl = template.Must(template.ParseFiles(sidebarTmplPath))
store, err := NewStore(getDataDir(), config.DataDir)
if err != nil {
panic("failed to create store for a forum")
}
logger.Noticef("%d topics, %d posts in forum %q", store.TopicsCount(), store.PostsCount(), config.ForumUrl)
forum.Store = store
return forum
}
func (store *Store) readIndex() error {
// at this point idx file must exist
file, err := os.Open(idxFilePath(store.basePath))
if err != nil {
return err
}
defer file.Close()
// TODO: would be faster (and easier?) to use a bitset since we know
// segment numbers are consequitive integers
segments := make([]int, 0)
csvReader := csv.NewReader(file)
csvReader.Comma = ','
csvReader.FieldsPerRecord = -1
rec, err := csvReader.Read()
if err != nil || len(rec) != 1 || rec[0] != idxHdr {
return errInvalidIndexHdr
}
var blob blob
for {
if rec, err = csvReader.Read(); err != nil {
break
}
if blob, err = decodeIndexLine(rec); err != nil {
break
}
appendIntIfNotExists(&segments, blob.nSegment)
store.appendBlob(blob)
}
if err == io.EOF {
err = nil
}
// verify segment files exist
// TODO: also verify offset + size is <= size of segment file
for _, nSegment := range segments {
path := segmentFilePath(store.basePath, nSegment)
if !u.PathExists(path) {
return errSegmentFileMissing
}
store.currSegmentNo = nSegment
}
return nil
}
func NewStoreCsv(path string) (*StoreCsv, error) {
//fmt.Printf("NewStoreCsv: %q\n", path)
var err error
s := &StoreCsv{
filePath: path,
strings: NewStringInterner(),
users: NewStringInterner(),
edits: make([]TranslationRec, 0),
}
if u.PathExists(path) {
if err = s.readExistingRecords(path); err != nil {
return nil, err
}
}
s.setActiveStrings(s.activeStrings)
if s.file, s.w, err = openCsv(path); err != nil {
return nil, err
}
return s, nil
}
func (s *StoreCrashes) MessageFileExists(sha1 []byte) bool {
p := s.MessageFilePath(sha1)
return u.PathExists(p)
}