func (m Memory) Generate() ([]i3.Output, error) {
mem, err := mem.VirtualMemory()
if err != nil {
return nil, err
}
total := mem.Total
used := total - mem.Buffers - mem.Cached - mem.Free
percUsed := (100 * used) / total
var color string
switch {
case m.IsCrit(int(percUsed)):
color = i3.DefaultColors.Crit
case m.IsWarn(int(percUsed)):
color = i3.DefaultColors.Warn
default:
color = i3.DefaultColors.OK
}
out := make([]i3.Output, 1)
out[0] = i3.Output{
Name: m.Name,
FullText: fmt.Sprintf(FormatString, percUsed, bytefmt.ByteSize(used), bytefmt.ByteSize(total)),
Color: color,
Separator: true,
}
return out, nil
}
func (t *TerminalUI) AdjustMemory(stats Statistics) {
// memory gauges
mem := make([]*ui.Gauge, len(stats.Instances))
for i, idx := range stats.Instances {
// show max 8 instances
if i > 7 {
break
}
memory := uint64(stats.Data[idx].Stats.Usage.Memory)
quota := uint64(stats.Data[idx].Stats.MemoryQuota)
percent := int(math.Ceil((float64(memory) / float64(quota)) * 100.0))
mem[i] = ui.NewGauge()
mem[i].Percent = percent
mem[i].Height = 13 - min(len(stats.Instances), 8)
mem[i].Border.Label = fmt.Sprintf("Memory - Instance %d: %d%% (%s / %s)",
i, percent, bytefmt.ByteSize(memory), bytefmt.ByteSize(quota))
mem[i].Border.FgColor = ui.ColorWhite
mem[i].Border.LabelFgColor = ui.ColorWhite
mem[i].BarColor = colors[i%6]
mem[i].PercentColor = ui.ColorWhite
}
t.Memory = mem
// update layout
ui.Body.Rows = []*ui.Row{
ui.NewRow(
ui.NewCol(3, 0, t.Usage),
ui.NewCol(3, 0, t.Summary),
ui.NewCol(6, 0, t.Disk)),
ui.NewRow(
ui.NewCol(6, 0, t.CPU),
t.newMemCol(6, 0, t.Memory)),
}
}
func DiskUsage(path string) (disk DiskStatus) {
fs := syscall.Statfs_t{}
err := syscall.Statfs(path, &fs)
if err != nil {
return
}
fmt.Print("",
"Type:", fs.Type, "\n", /* Type of filesystem (see below) */
"Bsize:", fs.Bsize, "\n", /* Optimal transfer block size */
"Blocks:", fs.Blocks, "\n", /* Total data blocks in filesystem */
"Bfree:", fs.Bfree, "\n", /* Free blocks in filesystem */
"Bavail:", fs.Bavail, "\n", /* Free blocks available to unprivileged user*/
"Files:", fs.Files, "\n", /* Total file nodes in filesystem */
"Ffree:", fs.Ffree, "\n", /* Free file nodes in filesystem */
//"Fsid:", fs.Bsize, "\n",/* Filesystem ID */
"Namelen:", fs.Namelen, "\n", /* Maximum length of filenames */
"Frsize:", fs.Frsize, "\n", /* Fragment size (since Linux 2.6) */
"Flags:", fs.Flags, "\n", /* Mount flags of filesystem (since Linux 2.6.36) */
//"Spare:", fs.Bsize, "\n", /* Padding bytes reserved for future use */
)
disk.All = fs.Blocks * uint64(fs.Bsize)
disk.Free = fs.Bfree * uint64(fs.Bsize)
disk.Used = disk.All - disk.Free
fmt.Println("",
"Total:", disk.All, bytefmt.ByteSize(disk.All), "\n",
"Used:", disk.Used, bytefmt.ByteSize(disk.Used), "\n",
"Free:", disk.Free, bytefmt.ByteSize(disk.Free))
return
}
func (t *TerminalUI) AdjustSummary(stats Statistics) {
var uptime int64
var cpuUsage float64
var memUsage, memQuota, diskUsage, diskQuota int64
for _, idx := range stats.Instances {
uptime = max(uptime, stats.Data[idx].Stats.Uptime)
cpuUsage += stats.Data[idx].Stats.Usage.CPU
memUsage += stats.Data[idx].Stats.Usage.Memory
diskUsage += stats.Data[idx].Stats.Usage.Disk
memQuota += stats.Data[idx].Stats.MemoryQuota
diskQuota += stats.Data[idx].Stats.DiskQuota
}
up, _ := time.ParseDuration(fmt.Sprintf("%ds", uptime))
summaryText := fmt.Sprintf(
`Instances running: %d
Uptime: %s
Up Since: %s
Total CPU Usage: %.4f%%
Total Memory Usage: %s / %s
Total Disk Usage: %s / %s`,
len(stats.Instances),
up.String(),
time.Now().Add(-1*up).Format("Mon, 02 Jan 2006 15:04 MST"),
cpuUsage*100.0,
bytefmt.ByteSize(uint64(memUsage)), bytefmt.ByteSize(uint64(memQuota)),
bytefmt.ByteSize(uint64(diskUsage)), bytefmt.ByteSize(uint64(diskQuota)))
t.Summary.Text = summaryText
}
//Initalisiert Dynamischen Pc
func (y *PCDynamic) PCInitDynamic(s *SSH) error {
/*
Id int `id`
MemFree string `memfree`
MemAvailable string `memavail`
MemUsed uint64 `memused`
MemUsedProzent float64 `memusedprozent`
LoadAvg float64 `loadavg`
NumProcess uint64 `numprocess`
*/
loadavg := &linuxproc.LoadAvg{}
meminfo := &linuxproc.MemInfo{}
err := errors.New("")
if s.Client == nil {
err := errors.New("Could not find a ssh Client ")
revel.ERROR.Print(err)
return err
} else {
s.Download("/proc/loadavg", tempDir+"/loadavg"+strconv.Itoa(y.Id))
loadavg, err = linuxproc.ReadLoadAvg("/proc/loadavg")
if err != nil {
revel.ERROR.Print(err)
return err
}
y.LoadAvg = loadavg.Last1Min
y.NumProcess = loadavg.ProcessRunning
s.Download("/proc/meminfo", tempDir+"/meminfo"+strconv.Itoa(y.Id))
meminfo, err = linuxproc.ReadMemInfo("/home/pr0gramming/Dokumente/meminfo" + strconv.Itoa(y.Id))
if err != nil {
revel.ERROR.Print(err)
return err
}
}
y.MemUsed = bytefmt.ByteSize((meminfo.MemTotal - meminfo.MemFree) * 1024)
y.MemFree = bytefmt.ByteSize(meminfo.MemFree * 1024)
y.MemAvailable = bytefmt.ByteSize(meminfo.MemAvailable * 1024)
a := float64(meminfo.MemTotal)
b := float64(meminfo.MemFree)
y.MemUsedProzent = strconv.FormatFloat(float64(((a-b)/a)*100), 'f', -1, 64)
//println("test: " + (y.MemUsedProzent))
return nil
}
func (me *CoreService) Status() *Sac {
out := NewSac()
m := runtime.MemStats{}
runtime.ReadMemStats(&m)
mem := NewSac()
mem_gen := NewSac().
Set("alloc", bytefmt.ByteSize(m.Alloc)).
Set("total_alloc", bytefmt.ByteSize(m.TotalAlloc))
mem.Set("general", mem_gen)
mem_hp := NewSac().
Set("alloc", bytefmt.ByteSize(m.HeapAlloc)).
Set("sys", bytefmt.ByteSize(m.HeapAlloc)).
Set("idle", bytefmt.ByteSize(m.HeapIdle)).
Set("inuse", bytefmt.ByteSize(m.HeapInuse)).
Set("released", bytefmt.ByteSize(m.HeapReleased)).
Set("objects", bytefmt.ByteSize(m.HeapObjects))
mem.Set("heap", mem_hp)
out.Set("mem", mem).
Set("server-time", time.Now().Format("2006-01-02 15:04:05 MST")).
Set("go-version", runtime.Version()[2:]).
Set("aqua-version", release)
return out
}
func (g DiskUsageGenerator) Generate() ([]i3.Output, error) {
items := make([]i3.Output, len(g.Items))
for i, item := range g.Items {
usage, err := getUsage(item.Path)
if err != nil {
return nil, err
}
free := bytefmt.ByteSize(usage.Free * bytefmt.BYTE)
items[i].FullText = fmt.Sprintf(freeFormat, item.Name, free)
freePercent := 100 - int(usage.UsedPerc)
var color string
switch {
case freePercent < g.CritThreshold:
color = i3.DefaultColors.Crit
case freePercent < g.WarnThreshold:
color = i3.DefaultColors.Warn
default:
color = i3.DefaultColors.OK
}
items[i].Color = color
items[i].Name = g.Name
items[i].Instance = item.Path
}
items[len(items)-1].Separator = true
return items, nil
}
func (factory *DropletRunnerCommandFactory) listDroplets(context *cli.Context) {
if !factory.ensureBlobStoreVerified() {
return
}
droplets, err := factory.dropletRunner.ListDroplets()
if err != nil {
factory.UI.SayLine(fmt.Sprintf("Error listing droplets: %s", err))
factory.ExitHandler.Exit(exit_codes.CommandFailed)
return
}
sort.Sort(dropletSliceSortedByCreated(droplets))
w := &tabwriter.Writer{}
w.Init(factory.UI, 12, 8, 1, '\t', 0)
fmt.Fprintln(w, "Droplet\tCreated At\tSize")
for _, droplet := range droplets {
size := bytefmt.ByteSize(uint64(droplet.Size))
if !droplet.Created.IsZero() {
fmt.Fprintf(w, "%s\t%s\t%s\n", droplet.Name, droplet.Created.Format("01/02 15:04:05.00"), size)
} else {
fmt.Fprintf(w, "%s\t\t%s\n", droplet.Name, size)
}
}
w.Flush()
}
func (runner *Runner) downloadBuildpacks() error {
// Do we have a custom buildpack?
for _, buildpackName := range runner.config.BuildpackOrder() {
buildpackUrl, err := url.Parse(buildpackName)
if err != nil {
return fmt.Errorf("Invalid buildpack url (%s): %s", buildpackName, err.Error())
}
if !buildpackUrl.IsAbs() {
continue
}
destination := runner.config.BuildpackPath(buildpackName)
if IsZipFile(buildpackUrl.Path) {
var size uint64
zipDownloader := NewZipDownloader(runner.config.SkipCertVerify())
size, err = zipDownloader.DownloadAndExtract(buildpackUrl, destination)
if err == nil {
fmt.Printf("Downloaded buildpack `%s` (%s)", buildpackUrl.String(), bytefmt.ByteSize(size))
}
} else {
err = GitClone(*buildpackUrl, destination)
}
if err != nil {
return err
}
}
return nil
}
func main() {
flag.Parse()
log.Print("scanning")
start := time.Now()
frontend := hath.NewFrontend(dir)
files := make(chan hath.File)
progress := make(chan hath.Progress)
wg := new(sync.WaitGroup)
wg.Add(1)
go func() {
for p := range progress {
log.Println("scan progress", p)
}
}()
go func() {
defer close(files)
defer wg.Done()
if err := frontend.Scan(files, progress); err != nil {
log.Println("error while scanning:", err)
} else {
log.Println("scanned")
}
}()
w := new(walker)
for f := range files {
// fmt.Print(".")
w.count++
w.sum += f.Size
}
wg.Wait()
log.Println("end")
end := time.Now()
duration := end.Sub(start)
log.Println("count", w.count, "size", bytefmt.ByteSize(uint64(w.sum)), "duration", duration)
}
func Fetch(url string) (*ServerInfo, error) {
if url == "" {
return nil, fmt.Errorf("Error fetching server info: url can not be empty.")
}
resp, err := http.Get(url)
if err != nil {
return nil, err
}
v := &struct {
MemStats runtime.MemStats `"json":"memstats"`
}{}
if err := json.NewDecoder(resp.Body).Decode(v); err != nil {
log.Fatalln(err.Error())
}
p := time.Duration(v.MemStats.PauseTotalNs) * time.Nanosecond
return &ServerInfo{
GC: GC{
CPUFraction: v.MemStats.GCCPUFraction,
Num: v.MemStats.NumGC,
PauseTotal: p,
PauseTotalStr: p.String(),
Enabled: v.MemStats.EnableGC,
Debug: v.MemStats.DebugGC,
},
Mem: Mem{
TotalAlloc: v.MemStats.TotalAlloc,
TotalAllocStr: bytefmt.ByteSize(v.MemStats.TotalAlloc),
},
}, nil
}
func main() {
argsWithoutProg := os.Args[1:]
if len(argsWithoutProg) == 0 {
fmt.Println("Usage: go run sum.go filter")
return
}
filter := strings.Join(argsWithoutProg, " ")
file, err := os.Open("diff.txt")
check(err)
defer file.Close()
var added, removed uint64
re, err := regexp.Compile(`(.+)\:`)
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
if len(line) > 3 && strings.Contains(line[3:], filter) {
if line[0:3] == "Inc" || line[0:3] == "New" {
res := re.FindStringSubmatch(line[4:])
if len(res) > 0 {
added += SizeInBytes(res[1])
}
} else if line[0:3] == "Dec" || line[0:3] == "Rem" {
res := re.FindStringSubmatch(line[4:])
if len(res) > 0 {
removed += SizeInBytes(res[1])
}
}
}
}
color.Green("Added: " + bytefmt.ByteSize(added))
color.Yellow("Removed: " + bytefmt.ByteSize(removed))
diff := int64(added) - int64(removed)
if diff < 0 {
color.Yellow("= Net total: -" + bytefmt.ByteSize(uint64(-diff)))
} else {
color.Green("= Net total: " + bytefmt.ByteSize(uint64(diff)))
}
}
func main() {
var delete = false
flag.BoolVar(&delete, "delete", false, "Delete all JPG sidecar files.")
flag.Parse()
if len(flag.Args()) < 1 {
log.Fatalf("Must supply a path to a Lightroom catalog file.")
}
path := flag.Args()[0]
log.Printf("Processing %s", path)
catalog, err := NewCatalog(path)
if err != nil {
log.Fatal(err)
}
if !delete {
stats := catalog.GetSidecarStats()
log.Printf("There are %d sidecar entries totalling %s bytes on disk.",
stats.Count, bytefmt.ByteSize(uint64(stats.TotalSizeBytes)))
return
}
var processed_count uint
var error_count uint
var skip_count uint
var missing_count uint
var count uint
catalog.ProcessSidecars(func(sidecarPath string, originalPath string) error {
if _, err := os.Stat(sidecarPath); err == nil {
if _, err := os.Stat(originalPath); os.IsNotExist(err) {
log.Printf("Missing original path for sidecar; Skipping %s", sidecarPath)
skip_count++
return nil
}
log.Printf("Deleting %s", sidecarPath)
err = os.Remove(sidecarPath)
if err != nil {
log.Printf("Error deleting %s; %v.", sidecarPath, err)
error_count++
return err
} else {
processed_count++
}
} else {
log.Printf("Missing %s", sidecarPath)
missing_count++
}
count++
return nil
})
log.Printf("Done.")
log.Printf(" Total: %d", count)
log.Printf(" Deleted: %d", processed_count)
log.Printf(" Skipped: %d", skip_count)
log.Printf(" Missing: %d", missing_count)
}
func main() {
flag.Parse()
runtime.GOMAXPROCS(*cpus)
w := new(writer)
r := new(reader)
worker := func(wr *writer) {
hasher := sha1.New()
dst := io.MultiWriter(wr, hasher)
defer log.Printf("%x", hasher.Sum(nil))
io.Copy(dst, r)
}
for index := 0; index < *workers; index++ {
go worker(w)
}
time.Sleep(*duration)
bytes := bytefmt.ByteSize(w.sum)
speed := uint64(float64(w.sum) / duration.Seconds())
log.Println(duration, bytes, "total;", bytefmt.ByteSize(speed), "per second")
log.Println(processed, "is rought", bytefmt.ByteSize(hath.GetRoughCacheSize(processed)))
}
func convertSize(s string) string {
// aus 314572800000 wird 962.1G
ret, err := strconv.ParseUint(s, 10, 64)
if err != nil {
log.Error(err.Error())
return ""
}
str := bytefmt.ByteSize(ret)
log.Info("Aus " + s + " wurde " + str)
return str
}
func main() {
flag.Parse()
g := hath.FileGenerator{
SizeMax: sizeMax,
SizeMin: sizeMin,
ResolutionMax: resMax,
ResolutionMin: resMin,
Dir: dir,
}
files := make(chan hath.File)
worker := func(work chan hath.File) {
log.Println("starting worker")
for {
f, err := g.New()
if err != nil {
log.Fatal(err)
}
work <- f
}
}
for i := 0; i < workers; i++ {
go worker(files)
}
fmt.Printf("%+v\n", g)
start := time.Now()
var i int64
var total int64
for i = 0; i < count; i++ {
f := <-files
total += f.Size
fmt.Println(f)
}
end := time.Now()
duration := end.Sub(start)
totalWrote := bytefmt.ByteSize(uint64(total))
perSecond := float64(total) / duration.Seconds()
rate := bytefmt.ByteSize(uint64(perSecond))
fmt.Printf("OK for %v\n", duration)
fmt.Printf("%s at rate %s/s\n", totalWrote, rate)
log.Println(count, "is rought", bytefmt.ByteSize(hath.GetRoughCacheSize(count)))
}
func watchGrabber(g grabber.Grabberer) {
tick := time.NewTicker(3 * time.Second)
defer tick.Stop()
for {
select {
case <-tick.C:
log.Printf("Downloading at %v", bytefmt.ByteSize(uint64(g.Strategy().DownloadRate())))
case <-g.PostProcessable():
return
}
}
}
func MemStat() MemStatus {
// self mem stat
memStat := new(runtime.MemStats)
runtime.ReadMemStats(memStat)
mem := MemStatus{}
mem.Self = memStat.Alloc
// system mem stat
sysInfo := new(syscall.Sysinfo_t)
err := syscall.Sysinfo(sysInfo)
if err == nil {
mem.All = uint64(sysInfo.Totalram)
mem.Free = uint64(sysInfo.Freeram)
mem.Used = mem.All - mem.Free
}
fmt.Println("",
"Self Alloc:", mem.Self, bytefmt.ByteSize(mem.Self), "\n",
"All:", mem.All, bytefmt.ByteSize(mem.All), "\n",
"Used:", mem.Used, bytefmt.ByteSize(mem.Used), "\n",
"Free:", mem.Free, bytefmt.ByteSize(mem.Free),
)
return mem
}
func (factory *AppExaminerCommandFactory) printInstanceSummary(writer io.Writer, actualInstances []app_examiner.InstanceInfo) {
w := tabwriter.NewWriter(writer, minColumnWidth, 8, 1, '\t', 0)
printHorizontalRule(w, "=")
fmt.Fprintf(w, fmt.Sprintf("%s\t%s\t%s\t%s\t%s\t%s\n",
"Instance",
colors.NoColor("State")+" ",
"Crashes",
"CPU",
"Memory",
"Uptime"),
)
printHorizontalRule(w, "-")
for _, instance := range actualInstances {
metricsSlice := []string{"N/A", "N/A"}
if instance.HasMetrics {
metricsSlice = []string{
fmt.Sprintf("%.2f%%", instance.Metrics.CpuPercentage),
fmt.Sprintf("%s", bytefmt.ByteSize(instance.Metrics.MemoryBytes)),
}
}
if instance.PlacementError == "" && instance.State != "CRASHED" {
uptime := factory.clock.Now().Sub(time.Unix(0, instance.Since))
roundedUptime := uptime - (uptime % time.Second)
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
strconv.Itoa(instance.Index),
presentation.PadAndColorInstanceState(instance),
strconv.Itoa(instance.CrashCount),
strings.Join(metricsSlice, "\t"),
fmt.Sprint(roundedUptime),
)
} else {
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
strconv.Itoa(instance.Index),
presentation.PadAndColorInstanceState(instance),
strconv.Itoa(instance.CrashCount),
strings.Join(metricsSlice, "\t"),
"N/A",
)
}
}
w.Flush()
}
func download(uri string, folder string, httpClient *http.Client, fileName string) (path string, newEpisode bool, err error) {
fileName = filepath.Join(folder, fileName)
fileName = sanitize.Path(fileName)
logger.Debug.Println("Local resource path : " + fileName)
tmpFilename := fileName + ".part"
resourceName := filepath.Base(folder) + " - " + filepath.Base(fileName)
defer removeTempFile(tmpFilename)
if !pathExists(fileName) {
logger.Debug.Println("New resource available : " + resourceName)
// TODO: check file existence first with io.IsExist
output, err := os.Create(tmpFilename)
if err != nil {
return fileName, newEpisode, err
}
defer output.Close()
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
return fileName, newEpisode, err
}
req.Close = true
response, err := httpClient.Do(req)
if err != nil {
return fileName, newEpisode, err
}
defer response.Body.Close()
n, err := io.Copy(output, response.Body)
if err != nil {
return fileName, newEpisode, err
}
logger.Debug.Println("Resource downloaded : " + resourceName + " (" + bytefmt.ByteSize(uint64(n)) + ")")
os.Rename(tmpFilename, fileName)
newEpisode = true
} else {
logger.Debug.Println("No download since the file exists", fileName)
newEpisode = false
}
return fileName, newEpisode, err
}
func (t *TerminalUI) AdjustDisk(stats Statistics) {
var quota int64
var data []int
for i, idx := range stats.Instances {
// show max 8 instances
if i > 7 {
break
}
mb := int((stats.Data[idx].Stats.Usage.Disk / 1024) / 1024)
data = append(data, mb)
quota = stats.Data[idx].Stats.DiskQuota
}
t.Disk.Data = data
t.Disk.BarWidth = 20 - min(len(stats.Instances), 8)
t.Disk.Border.Label = fmt.Sprintf("Disk Usage (in MB) - Quota: %s", bytefmt.ByteSize(uint64(quota)))
}
func (factory *AppExaminerCommandFactory) printInstanceInfo(writer io.Writer, actualInstances []app_examiner.InstanceInfo) {
w := tabwriter.NewWriter(writer, minColumnWidth, 8, 1, '\t', 0)
instanceBar := func(index, state string) {
fmt.Fprintf(w, "%sInstance %s [%s]\n", indentHeading, index, state)
printHorizontalRule(w, "-")
}
printHorizontalRule(w, "=")
for _, instance := range actualInstances {
instanceBar(fmt.Sprint(instance.Index), presentation.ColorInstanceState(instance))
if instance.PlacementError == "" && instance.State != "CRASHED" {
fmt.Fprintf(w, "%s\t%s\n", "InstanceGuid", instance.InstanceGuid)
fmt.Fprintf(w, "%s\t%s\n", "Cell ID", instance.CellID)
fmt.Fprintf(w, "%s\t%s\n", "Ip", instance.Ip)
portMappingStrings := make([]string, 0)
for _, portMapping := range instance.Ports {
portMappingStrings = append(portMappingStrings, fmt.Sprintf("%d:%d", portMapping.HostPort, portMapping.ContainerPort))
}
fmt.Fprintf(w, "%s\t%s\n", "Port Mapping", strings.Join(portMappingStrings, ";"))
uptime := factory.clock.Now().Sub(time.Unix(0, instance.Since))
roundedUptime := uptime - (uptime % time.Second)
fmt.Fprintf(w, "%s\t%s\n", "Uptime", fmt.Sprint(roundedUptime))
} else if instance.State != "CRASHED" {
fmt.Fprintf(w, "%s\t%s\n", "Placement Error", instance.PlacementError)
}
fmt.Fprintf(w, "%s \t%d \n", "Crash Count", instance.CrashCount)
if instance.HasMetrics {
fmt.Fprintf(w, "%s \t%.2f%% \n", "CPU", instance.Metrics.CpuPercentage)
fmt.Fprintf(w, "%s \t%s \n", "Memory", bytefmt.ByteSize(instance.Metrics.MemoryBytes))
}
printHorizontalRule(w, "-")
}
w.Flush()
}
func listEntries(baseurl string, serverhash string, verbose bool) {
resp, err := http.Get(fmt.Sprintf("%s/ls?target=%s", baseurl, serverhash))
if err != nil {
panic(err)
}
entrylist := &Index{}
rawbytes, err := ioutil.ReadAll(resp.Body)
//fmt.Println("got raw bytes", string(rawbytes))
if err != nil {
fmt.Println("Error reading response from localhost\n")
panic(err)
}
err = json.Unmarshal(rawbytes, entrylist)
if err != nil {
fmt.Println("Failed to unmarshal:", err)
}
seen := make(map[string]bool)
hidelist := getHideList(serverhash, baseurl, entrylist)
// reverse the list
for i := len(entrylist.Entries) - 1; i >= 0; i-- {
if verbose {
//ts := entrylist.Entries[i].Timestamp.Format(time.RFC3339)
ts := entrylist.Entries[i].Timestamp.Format("2006-01-02T15:04")
fmt.Println(entrylist.Entries[i].Hash, ts, bytefmt.ByteSize(uint64(entrylist.Entries[i].Size)), entrylist.Entries[i].Name)
continue
}
_, exists := seen[entrylist.Entries[i].Name]
_, existsh := hidelist[entrylist.Entries[i].Name]
if !exists && !existsh {
fmt.Println(entrylist.Entries[i].Hash, entrylist.Entries[i].Name)
}
seen[entrylist.Entries[i].Name] = true
}
}
func (step *downloadStep) perform() error {
step.emit("Downloading %s...\n", step.model.Artifact)
downloadedFile, downloadedSize, err := step.fetch()
if err != nil {
return NewEmittableError(err, "Downloading failed")
}
err = step.streamIn(step.model.To, downloadedFile)
if err != nil {
return NewEmittableError(err, "Copying into the container failed")
}
if downloadedSize != 0 {
step.emit("Downloaded %s (%s)\n", step.model.Artifact, bytefmt.ByteSize(uint64(downloadedSize)))
} else {
step.emit("Downloaded %s\n", step.model.Artifact)
}
return nil
}
func (o *onedrive) SyncWith(up *onedrive, downDir, upDir string, jobCount int) {
items := o.Children(downDir)
if len(items) == 0 {
return
}
if jobCount > 0 {
o.startJobs(jobCount)
}
err := up.Mkdir(upDir)
if err != nil {
log.Fatal("Can't create directory")
}
upItems := up.Children(upDir)
MAIN:
for _, item := range items {
if item.folder {
fmt.Println("Directory:", item.name)
o.SyncWith(up, downDir+"/"+item.name, upDir+"/"+item.name, 0)
} else {
for _, upItem := range upItems {
if upItem.name == item.name && upItem.size == item.size {
if upItem.hash == item.hash {
size := bytefmt.ByteSize(uint64(item.size))
fmt.Println("Online:", item.name, size)
continue MAIN
}
}
}
o.jobs <- jobItem{up: up, upDir: upDir, item: item}
}
}
if jobCount > 0 {
close(o.jobs)
o.wait.Wait()
}
}
func main() {
jar, _ := cookiejar.New(nil)
LOGIN_URL := "https://sso.teachable.com/secure/24/users/sign_in?reset_purchase_session=1"
SCHOOL_ID := "24" // id of bitfountain on usefedora.com
// commandline inputs for email, password, and course url
emailPtr := flag.String("email", "", "Email of the user")
courseUrlPtr := flag.String("course", "", "URL of the course")
passwordPtr := flag.String("pass", "", "Password of the user")
optionalStartSectionNoPtr := flag.Int("section", 0, "Section number to start the downloads at, starts at 0")
// Optional start lecture for the download process
flag.Parse()
if *emailPtr == "" {
log.Fatal("Please provide your email address")
}
if *passwordPtr == "" {
log.Fatal("Please provide your password")
}
if *courseUrlPtr == "" {
log.Fatal("Please provide the bitfountain course url")
}
fmt.Printf("\nStarting download at section %d ...\n", *optionalStartSectionNoPtr)
// Display the starting course for the users reference
// Unnecessary, I just like it like that :)
// pass, err := terminal.ReadPassword()
// if err != nil {
// log.Fatal(err)
// }
// fmt.Printf("\n\npass:: %s", pass)
fmt.Println("Logging in ...")
client := http.Client{Jar: jar}
respLogin, err := client.Get(LOGIN_URL)
if err != nil {
log.Fatal(err)
}
loginPage, err := goquery.NewDocumentFromResponse(respLogin)
if err != nil {
log.Fatal(err)
}
defer respLogin.Body.Close()
token, _ := loginPage.Find("input[name='authenticity_token']").Attr("value")
resp, err := client.PostForm(LOGIN_URL,
url.Values{
"utf8": {"✓"},
"authenticity_token": {token},
"user[school_id]": {SCHOOL_ID},
"user[email]": {*emailPtr},
"user[password]": {*passwordPtr},
"commit": {"Log in"},
})
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
fmt.Println("Logged in. Fetching course sections ...")
currentDir, err := osext.ExecutableFolder()
if err != nil {
log.Fatal(err)
}
courseDir := filepath.Join(currentDir, path.Base(*courseUrlPtr))
// Every bitfountain course is split into multiple sections with each
// section having multiple lectures (videos)
type Lecture struct {
name string
id string
filePath string
url string // bitfountain lecture page url
}
type Section struct {
name string
sectionDir string
lectures []Lecture
}
// Get the Course page (contains the list of sections and lectures)
respCourseDetails, err := client.Get(*courseUrlPtr)
if err != nil {
log.Fatal(err)
}
defer respCourseDetails.Body.Close()
doc, err := goquery.NewDocumentFromResponse(respCourseDetails)
if err != nil {
log.Fatal(err)
}
sections := []Section{}
// Find all the sections in the course, and loop over them
doc.Find(".course-section").Each(func(i int, s *goquery.Selection) {
if i >= *optionalStartSectionNoPtr {
// Only add the section if it is above the requested section index
// Get the name of the current section
name := s.Find(".section-title").Text()
name = getDashedName(name, i)
sectionDir := filepath.Join(courseDir, name)
newLectures := []Lecture{}
// Find all the lectures in the current section, and loop over them
s.Find(".section-item").Each(func(i int, l *goquery.Selection) {
// Get the name of the lecture (video)
lectureName := l.Find(".lecture-name").Text()
// Get the lecture id from the attribute. This will be used to
// construct the url of each lecture's page
item, _ := l.Find(".item").Attr("href")
parts := strings.Split(item, "/")
lectureId := strings.TrimSpace(parts[len(parts)-1])
lectureFileName := fmt.Sprint(getDashedName(lectureName, i), ".mp4")
// The video will be stored locally at the lectureFilePath
filePath := filepath.Join(sectionDir, lectureFileName)
// We will need to visit the lecturePageUrl, to get the Wistia
// video download link
url := *courseUrlPtr + "/lectures/" + lectureId
newLectures = append(newLectures, Lecture{
name: lectureName,
id: lectureId,
filePath: filePath,
url: url,
})
})
newSection := Section{
name: name,
sectionDir: sectionDir,
lectures: newLectures,
}
sections = append(sections, newSection)
}
// This should provide rudamentary functionality to the optionalStartSectionNoPtr param
})
fmt.Println("Done! Starting course download...")
// Just providing the user with more feedback
// create course directory
os.Mkdir(courseDir, 0777)
for _, section := range sections {
// check if the section dir exists
_, err := os.Stat(section.sectionDir)
if err != nil {
// section dir does not exist
// create section directory
os.Mkdir(section.sectionDir, 0777)
}
fmt.Printf("\n\n\n%s", section.name)
for _, lecture := range section.lectures {
fmt.Printf("\n\n\t%s", lecture.name)
// Visit the lecture's url
respLecture, err := client.Get(lecture.url)
if err != nil {
log.Fatal(err)
}
lecturePage, err := goquery.NewDocumentFromResponse(respLecture)
if err != nil {
log.Fatal(err)
}
defer respLecture.Body.Close()
// Get the Wistia download link on the lecture's page
videoUrl, _ := lecturePage.Find("a.download").Attr("href")
// Parse the URL and ensure there are no errors.
parsedVideoUrl, err := url.Parse(videoUrl)
if err != nil {
log.Fatal(err)
}
// Sometimes the videoUrl does not have http / https as Scheme,
// We'll add http as the Scheme, because if its empty Go will
// throw an error
if parsedVideoUrl.Scheme != "http" || parsedVideoUrl.Scheme != "https" {
parsedVideoUrl.Scheme = "http"
videoUrl = parsedVideoUrl.String()
parsedVideoUrl, _ = url.Parse(videoUrl)
}
// Some of the lectures only have assignments, no video
// We need to skip them
if parsedVideoUrl.Host == "" {
fmt.Printf("\n\t\tNo video found on this lecture's page. Moving on to the next lecture ...")
continue
}
var wistiaVideoSize int64
var wistiaVideoSizeString string
var existingFileSizeOnDiskString string
// Create some new variables for the converted byte strings
// It's all for readability, instead of placing the converted string straight into the printf
resp, err := client.Get(videoUrl)
if err != nil {
log.Fatal(err)
} else {
contentLength := resp.Header.Get("Content-Length")
wistiaVideoSize, _ = strconv.ParseInt(contentLength, 10, 64)
// contentType := resp.Header.Get("Content-Type")
wistiaVideoSizeString = bytefmt.ByteSize(uint64(wistiaVideoSize))
// Convert the video size to a human readable format and place it in a new var
fmt.Printf("\n\t\twistiaVideoSize: %s", wistiaVideoSizeString)
}
// check if video file already exists
if fileStat, err := os.Stat(lecture.filePath); err == nil {
existingFileSizeOnDisk := fileStat.Size()
existingFileSizeOnDiskString = bytefmt.ByteSize(uint64(existingFileSizeOnDisk))
// Convert the video size to a human readable format and place it in a new var
fmt.Printf("\n\t\texistingFileSizeOnDisk: %s", existingFileSizeOnDiskString)
if existingFileSizeOnDisk == wistiaVideoSize {
fmt.Println("\n\t\tFull video file exists; moving to next lecture...")
continue
} else {
fmt.Printf("\n\t\tVideo file exists but not fully, will download again")
}
}
defer resp.Body.Close()
fmt.Printf("\n\t\tCreating file ...")
out, err := os.Create(lecture.filePath)
if err != nil {
log.Fatal(err)
}
defer out.Close()
fmt.Printf(" Done!\n\t\tDownloading Video ... \n")
// Create the progress bar
bar := pb.New(int(wistiaVideoSize)).SetUnits(pb.U_BYTES).SetRefreshRate(time.Second / 2)
bar.ShowSpeed = true
bar.SetMaxWidth(80)
bar.Format("[=> ]")
// Custom init options
bar.Callback = func(progressBarString string) {
fmt.Printf("\r\t\t%v", progressBarString)
}
// Custom formatting callback, makes the progress bar stay on the same line, but have tabbing to keep inline with the other elements
bar.Start()
// Start the progress bar
writer := io.MultiWriter(out, bar)
// Create multi writer
_, ioErr := io.Copy(writer, resp.Body)
if ioErr != nil {
log.Fatal(err)
}
// Copy (download) the file
bar.Finish()
// Stop the progress bar
fmt.Printf("\t\tDone!")
}
}
}
func (o *onedrive) syncFile(up *onedrive, upDir string, item onedriveItem) bool {
var b bytes.Buffer
var size int64 = 0
buffer := make([]byte, chunk)
resp := o.get(api + item.link + ":/content")
defer resp.Body.Close()
uploadUrl, err := up.createSession(item.name, upDir)
if err != nil || len(uploadUrl) == 0 {
fmt.Println("Session:", err, uploadUrl)
return false
}
fmt.Println("Upload:", item.name, "-", bytefmt.ByteSize(uint64(item.size)))
tries := 0
for {
if tries > 3 {
fmt.Println("Too many tries")
return false
}
num, err := io.ReadAtLeast(resp.Body, buffer, chunk)
b.Reset()
b.Write(buffer)
b.Truncate(num)
req, err2 := http.NewRequest("PUT", uploadUrl, &b)
if err2 != nil {
return false
}
r := fmt.Sprintf("bytes %d-%d/%d", size, size+int64(num)-1, item.size)
req.Header.Add("Content-Length", fmt.Sprintf("%d", num))
req.Header.Add("Content-Range", r)
res, err2 := up.client().Do(req)
if err2 != nil {
return false
}
res.Body.Close()
size += int64(num)
from := bytefmt.ByteSize(uint64(size))
to := bytefmt.ByteSize(uint64(item.size))
fmt.Println(from, "/", to, "Status:", res.StatusCode, "Name:", item.name)
if err != nil || res.StatusCode >= 400 {
if item.size > size || res.StatusCode >= 400 {
fmt.Println("\nError:", res.StatusCode, err, r, "\n")
resp.Body.Close()
resp, size = o.resume(up, uploadUrl, item)
if size == 0 {
return false
}
tries++
continue
}
break
}
tries = 0
}
fmt.Println("Uploaded:", item.name)
return true
}
func list(w http.ResponseWriter, r *http.Request) {
startTime := time.Now()
var fileList webList
var totalSize uint64
fileList.Version = strings.Title(strings.TrimPrefix(runtime.Version(), "go"))
fileList.TotalCount = 0
totalSize = 0
fileList.FilterYear = append(fileList.FilterYear, 0)
filterYear := startTime.Year()
if getYear, err := strconv.Atoi(r.URL.Query().Get("year")); err == nil {
filterYear = getYear
}
if err = filepath.Walk(*osPath, func(path string, f os.FileInfo, err error) error {
finfo, err := os.Stat(path)
if err != nil {
return err
}
if !finfo.IsDir() {
fileYear := finfo.ModTime().Year()
if !intInSlice(fileYear, fileList.FilterYear) {
fileList.FilterYear = append(fileList.FilterYear, fileYear)
}
if filterYear != fileYear && filterYear != 0 {
return nil
}
fileList.TotalCount++
totalSize += uint64(finfo.Size())
path = strings.TrimPrefix(path, *osPath)
smallPath := strings.TrimPrefix(*osPath, "./")
if strings.HasPrefix(path, smallPath) {
path = strings.TrimPrefix(path, smallPath)
}
if strings.HasPrefix(path, "/") {
path = strings.TrimPrefix(path, "/")
}
timearr := strings.Split(fmt.Sprintf("%q", finfo.ModTime()), " ")
thisfile := webFile{
Name: fmt.Sprintf("%s%s", *webPath, path),
Size: bytefmt.ByteSize(uint64(finfo.Size())),
Time: fmt.Sprintf("%s %s", timearr[0][1:], strings.Split(timearr[1], ".")[0]),
}
if startTime.Format("2006-01-02") == timearr[0][1:] {
thisfile.Today = true
} else {
thisfile.Today = false
}
fileList.List = append(fileList.List, thisfile)
}
return nil
}); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
const tpl = `<html>
<head>
<title>List</title>
<style>
table {border-collapse:collapse;}
table td {border: 1px solid #afafaf;padding: 1px 3px;}
table tr:hover {background-color:#f0f0ef}
abbr {border-bottom:1px dotted black;cursor:help;}
a {text-decoration:none;}
a:hover {text-decoration:underline;}
.now {background-color:#daffda;}
</style>
</head>
<body>
<small>
Year: {{range $year := .FilterYear}}[<a href="/list?year={{$year}}">{{if eq $year 0}}ALL{{else}}{{$year}}{{end}}</a>] {{end}}
</small>
<hr>
<table>
{{$domain := .MainDomain}}
{{range .List}}
<tr>
<td><a href="{{.Name}}">{{.Name}}</a></td>
<td{{if .Today}} class="now"{{end}}>{{.Time}}</td>
<td>{{.Size}}</td>
<td>[<a href="https://darklog.apps.merolabs.com/search/vkey={{$domain}}{{.Name}}" target="_blank">L</a>]</td>
</tr>
{{end}}
</table>
<hr>
<small><abbr title="GoLang Version">Go</abbr>: {{.Version}}</small> |
<small><abbr title="Generation Time">GT</abbr>: {{.LoadTime}}</small> |
<small><abbr title="Total Count">TC</abbr>: {{.TotalCount}}</small> |
<small><abbr title="Total Size">TS</abbr>: {{.TotalSize}}</small>
</body>
</html>`
t, err := template.New("webpage").Parse(tpl)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
sort.Sort(sort.IntSlice(fileList.FilterYear))
fileList.MainDomain = *mainDomain
fileList.TotalSize = bytefmt.ByteSize(totalSize)
fileList.LoadTime = fmt.Sprintf("%q", time.Since(startTime))
if err = t.Execute(w, fileList); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
func deployCmd(c *CLI, ctx *cli.Context) {
// 0. prepare API
var source string
api := c.GetAPIClient(ctx)
site := c.GetSite(ctx)
instance := c.GetInstance(ctx, nil)
buildLabel := fmt.Sprintf("%s-%%s", filepath.Base(filepath.Dir(siteCfg.filename)))
if ctx.Args().First() != "" {
source = ctx.Args().First()
var sourceLabel string
if len(source) > 8 {
sourceLabel = source[:8]
} else {
sourceLabel = source
}
buildLabel = fmt.Sprintf(buildLabel, sourceLabel)
} else {
var ok bool
source, ok = siteCfg.instances[*instance.Label]
if !ok {
fatal(fmt.Sprintf("no branch could be found for %s", *instance.Label))
} else {
buildLabel = fmt.Sprintf(buildLabel, fmt.Sprintf("%s-%s", siteCfg.vcs.Branch, siteCfg.vcs.Commit[:8]))
}
}
if siteCfg.Deploy == nil {
fatal("gondor.yml is missing the deploy configuration.")
}
fmt.Printf("-----> Preparing build of %s (%s)\n", source, buildLabel)
cleanup := func(err error) {
if err != nil {
fatal(err.Error())
}
}
// 1. create a build
build := &gondor.Build{
Site: site.URL,
Instance: instance.URL,
Label: &buildLabel,
BuildpackURL: &siteCfg.BuildpackURL,
}
if err := api.Builds.Create(build); err != nil {
cleanup(err)
}
// 2. perform build from source blob
fmt.Printf(" Running git archive --format=tar %s... ", source)
f, err := ioutil.TempFile("", fmt.Sprintf("%s-", c.Name))
if err != nil {
fmt.Println("error")
fmt.Printf(" %s\n", err)
cleanup(nil)
}
cmd := exec.Command("git", "archive", "--format=tar", source)
stdout, err := cmd.StdoutPipe()
if err != nil {
fmt.Println("error")
fmt.Printf(" %s\n", err)
os.Remove(f.Name())
cleanup(nil)
}
w := bufio.NewWriter(f)
if err := cmd.Start(); err != nil {
fmt.Println("error")
fmt.Printf(" %s\n", err)
os.Remove(f.Name())
cleanup(nil)
}
go io.Copy(w, stdout)
if err := cmd.Wait(); err != nil {
fmt.Println("failed")
fmt.Printf(" %s\n", err)
os.Remove(f.Name())
cleanup(nil)
}
fmt.Println("done")
w.Flush()
var size uint64
fi, err := os.Stat(f.Name())
if err == nil {
size = uint64(fi.Size())
}
msg := " Uploading tarball%s... "
if size > 0 {
msg = fmt.Sprintf(msg, fmt.Sprintf(" (%s)", bytefmt.ByteSize(size)))
} else {
msg = fmt.Sprintf(msg, "")
}
fmt.Print(msg)
f.Seek(0, 0)
endpoint, err := build.Perform(f)
if err != nil {
fmt.Println("error")
fmt.Printf(" %s\n", err)
os.Remove(f.Name())
cleanup(nil)
}
fmt.Println("done")
os.Remove(f.Name())
re := remoteExec{
endpoint: endpoint,
enableTty: false,
httpClient: c.GetHttpClient(ctx),
tlsConfig: c.GetTLSConfig(ctx),
callback: func(ok bool, err error) {
if err != nil {
fmt.Println("error")
fmt.Printf(" %s\n", err)
cleanup(nil)
}
if !ok {
fmt.Println("failed")
cleanup(nil)
}
fmt.Println("ok")
},
}
fmt.Printf("-----> Attaching to build process... ")
exitCode, err := re.execute()
if err != nil {
fatal(err.Error())
}
if exitCode > 0 {
os.Exit(exitCode)
}
// 3. create a deployment for the instance pointed at the release
fmt.Printf("\n-----> Deploying to %s... ", *instance.Label)
errc := make(chan error)
for _, serviceName := range siteCfg.Deploy.Services {
service, err := api.Services.Get(*instance.URL, serviceName)
if err != nil {
fmt.Println("error")
fmt.Printf(" %s\n", err)
os.Exit(1)
}
go func() {
deployment := &gondor.Deployment{
Service: service.URL,
Build: build.URL,
}
if err := api.Deployments.Create(deployment); err != nil {
errc <- err
return
}
if err := deployment.Wait(); err != nil {
errc <- err
return
}
errc <- nil
}()
}
if err := <-errc; err != nil {
fmt.Println("error")
fmt.Printf(" %s\n", err)
os.Exit(1)
}
fmt.Println("done")
}
if err != nil {
log.Debug(err.Error())
log.Fatal("Could not retrieve statistics")
}
log.Alert("\nSystem Statistics\n")
log.Stats("Host", client.Host)
log.Stats("Version", stats.Version)
if cmd.Flag("verbose").Value.String() == "true" {
log.Stats("Total messages", stats.MessageCount)
}
log.Stats("Pending messages", stats.PendingMessages)
log.Stats("Connected clients",
fmt.Sprintf("%d / %d", stats.ConnectedClients, stats.TotalMailboxes))
if cmd.Flag("verbose").Value.String() == "true" {
log.Stats("Database version", stats.DBVersion)
log.Stats("CPU's in use", stats.CPUCount)
log.Stats("Active threads", stats.Threads)
log.Stats("Memory in use", bytefmt.ByteSize(stats.MemoryAllocated))
log.Stats("Lookups", fmt.Sprintf("%d", stats.Lookups))
log.Stats("Next GC at", bytefmt.ByteSize(stats.NextGC))
log.Stats("File store count", fmt.Sprintf("%d", stats.FileStoreCount))
log.Stats("File store size", bytefmt.ByteSize(uint64(stats.FileStoreSize)))
}
},
}
func init() {
RootCmd.AddCommand(infoCmd)
infoCmd.Flags().BoolP("verbose", "v", false, "Show all details.")
}