func main() {
flag.Parse()
if *servingTable == "" {
flagutil.UsageError("missing --serving_table")
} else if *outPath == "" {
flagutil.UsageError("missing --out")
}
ctx := context.Background()
db, err := leveldb.Open(*servingTable, nil)
if err != nil {
log.Fatalf("Error opening db at %q: %v", *servingTable, err)
}
defer db.Close()
tbl := &table.KVProto{db}
esr := &esrchsrv.Table{&table.KVInverted{db}}
ix := index.Create(*outPath)
for _, ticket := range filesInTable(ctx, esr) {
var fd srvpb.FileDecorations
if err := tbl.Lookup(ctx, xsrv.DecorationsKey(ticket), &fd); err != nil {
log.Fatalf("Error looking up decoration for %q: %v", ticket, err)
}
ix.Add(ticket, bytes.NewReader(fd.SourceText))
}
ix.Flush()
}
func main() {
flag.Usage = usage
flag.Parse()
args := flag.Args()
if *listFlag {
ix := index.Open(index.File())
for _, arg := range ix.Paths() {
fmt.Printf("%s\n", arg)
}
return
}
if *cpuProfile != "" {
f, err := os.Create(*cpuProfile)
if err != nil {
log.Fatal(err)
}
defer f.Close()
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
if *resetFlag && len(args) == 0 {
os.Remove(index.File())
return
}
if len(args) == 0 {
ix := index.Open(index.File())
for _, arg := range ix.Paths() {
args = append(args, arg)
}
}
// Translate paths to absolute paths so that we can
// generate the file list in sorted order.
for i, arg := range args {
a, err := filepath.Abs(arg)
if err != nil {
log.Printf("%s: %s", arg, err)
args[i] = ""
continue
}
args[i] = a
}
sort.Strings(args)
for len(args) > 0 && args[0] == "" {
args = args[1:]
}
master := index.File()
if _, err := os.Stat(master); err != nil {
// Does not exist.
*resetFlag = true
}
file := master
if !*resetFlag {
file += "~"
}
ix := index.Create(file)
ix.Verbose = *verboseFlag
ix.AddPaths(args)
for _, arg := range args {
log.Printf("index %s", arg)
filepath.Walk(arg, func(path string, info os.FileInfo, err error) error {
if _, elem := filepath.Split(path); elem != "" {
// Skip various temporary or "hidden" files or directories.
if elem[0] == '.' || elem[0] == '#' || elem[0] == '~' || elem[len(elem)-1] == '~' {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
}
if err != nil {
log.Printf("%s: %s", path, err)
return nil
}
if info != nil && info.Mode()&os.ModeType == 0 {
ix.AddFile(path)
}
return nil
})
}
log.Printf("flush index")
ix.Flush()
if !*resetFlag {
log.Printf("merge %s %s", master, file)
index.Merge(file+"~", master, file)
os.Remove(file)
os.Rename(file+"~", master)
}
log.Printf("done")
return
}
func main() {
flag.Usage = usage
flag.Parse()
args := flag.Args()
if *listFlag {
ix := index.Open(index.File())
for _, arg := range ix.Paths() {
fmt.Printf("%s\n", arg)
}
return
}
if *cpuProfile != "" {
f, err := os.Create(*cpuProfile)
if err != nil {
log.Fatal(err)
}
defer f.Close()
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
if *resetFlag && len(args) == 0 {
os.Remove(index.File())
return
}
if len(args) == 0 {
ix := index.Open(index.File())
for _, arg := range ix.Paths() {
args = append(args, arg)
}
}
// Translate paths to absolute paths so that we can
// generate the file list in sorted order.
for i, arg := range args {
a, err := filepath.Abs(arg)
if err != nil {
log.Printf("%s: %s", arg, err)
args[i] = ""
continue
}
args[i] = a
}
sort.Strings(args)
for len(args) > 0 && args[0] == "" {
args = args[1:]
}
if *indexFlag != "" {
index.SetFile(*indexFlag)
}
master := index.File()
if _, err := os.Stat(master); err != nil {
// Does not exist.
*resetFlag = true
}
file := master
if !*resetFlag {
file += "~"
}
ix := index.Create(file)
ix.Verbose = *verboseFlag
ix.LogSkip = *skipFlag || ix.Verbose
var skipExtsList []string
if *skipExts != "" {
skipExtsList = strings.Split(strings.ToLower(*skipExts), ",")
}
if len(skipExtsList) > 0 {
log.Printf("Skipping files with extensions: %#v", skipExtsList)
}
ix.AddPaths(args)
for _, arg := range args {
log.Printf("index %s", arg)
filepath.Walk(arg, func(path string, info os.FileInfo, err error) error {
if _, elem := filepath.Split(path); elem != "" {
// Skip various temporary or "hidden" files or directories.
if elem[0] == '.' || elem[0] == '#' || elem[0] == '~' || elem[len(elem)-1] == '~' ||
strings.Contains(elem, ".backup") || strings.Contains(elem, ".old") {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
}
if err != nil {
log.Printf("%s: %s", path, err)
return nil
}
if info != nil && info.Mode()&os.ModeType == 0 {
if len(skipExtsList) > 0 {
ext := strings.ToLower(pth.Ext(path))
if len(ext) > 0 {
ext = ext[1:]
}
for _, e := range skipExtsList {
if e == ext {
if ix.LogSkip {
log.Printf("Skipping %s due to forbidden extension.\n", path)
}
return nil
}
}
}
ix.AddFile(path)
}
return nil
})
}
log.Printf("flush index")
ix.Flush()
if !*resetFlag {
log.Printf("merge %s %s", master, file)
index.Merge(file+"~", master, file)
os.Remove(file)
os.Rename(file+"~", master)
}
log.Printf("done")
return
}