// Seek implements http.File. func (f *HTTPFile) Seek(offset int64, whence int) (int64, error) { if f.closed { return 0, os.ErrInvalid } if offset < 0 { return int64(f.off), fmt.Errorf("cannot seek before start of file") } switch whence { case 0: noff := int64(f.off) + offset if noff > mathutil.MaxInt { return int64(f.off), fmt.Errorf("seek target overflows int: %d", noff) } f.off = mathutil.Min(int(offset), len(f.content)) if f.off == int(offset) { return offset, nil } return int64(f.off), io.EOF case 1: noff := int64(f.off) + offset if noff > mathutil.MaxInt { return int64(f.off), fmt.Errorf("seek target overflows int: %d", noff) } off := mathutil.Min(f.off+int(offset), len(f.content)) if off == f.off+int(offset) { f.off = off return int64(off), nil } f.off = off return int64(off), io.EOF case 2: noff := int64(f.off) - offset if noff < 0 { return int64(f.off), fmt.Errorf("cannot seek before start of file") } f.off = len(f.content) - int(offset) return int64(f.off), nil default: return int64(f.off), fmt.Errorf("seek: invalid whence %d", whence) } }
//Pick cells to form distal connections to. func (tm *TemporalMemory) pickCellsToLearnOn(n int, segment int, winnerCells []int, connections *TemporalMemoryConnections) []int { candidates := make([]int, len(winnerCells)) copy(candidates, winnerCells) for _, val := range connections.SynapsesForSegment(segment) { syn := connections.DataForSynapse(val) for idx, val := range candidates { if val == syn.SourceCell { candidates = append(candidates[:idx], candidates[idx+1:]...) break } } } //Shuffle candidates for i := range candidates { j := rand.Intn(i + 1) candidates[i], candidates[j] = candidates[j], candidates[i] } n = mathutil.Min(n, len(candidates)) return candidates[:n] }
func (f *mem) ReadAt(b []byte, off int64) (n int, err error) { avail := f.size - off pi := off >> f.pgBits po := int(off) & f.pgMask rem := len(b) if int64(rem) >= avail { rem = int(avail) err = io.EOF } var zeroPage *[]byte for rem != 0 && avail > 0 { pg := f.m[pi] if pg == nil { if zeroPage == nil { zeroPage = buffer.CGet(f.pgSize) defer buffer.Put(zeroPage) } pg = zeroPage } nc := copy(b[:mathutil.Min(rem, f.pgSize)], (*pg)[po:]) pi++ po = 0 rem -= nc n += nc b = b[nc:] } return n, err }
func (f *file) ReadAt(b []byte, off int64) (n int, err error) { avail := f.size - off pi := off >> f.pgBits po := int(off) & f.pgMask rem := len(b) if int64(rem) >= avail { rem = int(avail) err = io.EOF } for rem != 0 && avail > 0 { pg := f.m[pi] if pg == nil { if pg, err = f.page(pi); err != nil { return n, err } } nc := copy(b[:mathutil.Min(rem, f.pgSize)], pg[po:]) pi++ po = 0 rem -= nc n += nc b = b[nc:] } return n, err }
func (f *bitFiler) ReadAt(b []byte, off int64) (n int, err error) { avail := f.size - off pgI := off >> bfBits pgO := int(off & bfMask) rem := len(b) if int64(rem) >= avail { rem = int(avail) err = io.EOF } for rem != 0 && avail > 0 { pg := f.m[pgI] if pg == nil { pg = &bitPage{} pg.pdata = buffer.CGet(bfSize) pg.data = *pg.pdata if f.parent != nil { _, err = f.parent.ReadAt(pg.data, off&^bfMask) if err != nil && !fileutil.IsEOF(err) { return } err = nil } f.m[pgI] = pg } nc := copy(b[:mathutil.Min(rem, bfSize)], pg.data[pgO:]) pgI++ pgO = 0 rem -= nc n += nc b = b[nc:] off += int64(nc) } return }
func (t *treeCache) getTree(db *DB, prefix int, name string, canCreate bool, cacheSize int) (r *lldb.BTree, err error) { m := t.get() r, ok := m[name] if ok { return } root, err := db.root() if err != nil { return } val, err := root.get(prefix, name) if err != nil { return } switch x := val.(type) { case nil: if !canCreate { return } var h int64 r, h, err = lldb.CreateBTree(db.alloc, collate) if err != nil { return nil, err } if err = root.set(h, prefix, name); err != nil { return nil, err } case int64: if r, err = lldb.OpenBTree(db.alloc, collate, x); err != nil { return nil, err } default: return nil, &lldb.ErrINVAL{Src: "corrupted root directory value for", Val: fmt.Sprintf("%q, %q", prefix, name)} } if len(m) > cacheSize { i, j, n := 0, cacheSize/2, mathutil.Min(cacheSize/20, 10) loop: for k := range m { if i++; i >= j { delete(m, k) if n == 0 { break loop } n-- } } } m[name] = r return }
func (c *cache) get2(n int) (r *node, isZeroed bool) { s := *c lens := len(s) if lens == 0 { return &node{b: make([]byte, n, mathutil.Min(2*n, maxBuf))}, true } i := sort.Search(lens, func(x int) bool { return len(s[x].b) >= n }) if i == lens { i-- s[i].b, isZeroed = make([]byte, n, mathutil.Min(2*n, maxBuf)), true } r = s[i] r.b = r.b[:n] copy(s[i:], s[i+1:]) s = s[:lens-1] *c = s return }
// Execute implements Instruction. // // Stack frame before Execute: // // +----------------+ // | Return address | // +----------------+ // SP -> | | // // // Stack frame after Execute: // // +----------------+ // | Return address | // +----------------+ // | Previous FP | // +----------------+ // | [LNL]int | Parent Frame Pointers // +----------------+ // | LNL | Lexical Nesting Level // +----------------+ // FP -> | [NVars]int | Local variables // +----------------+ // SP -> | ! Evaluation stack func (n *Enter) Execute(m *VM) { m.push(m.FP) cur := 0 if n.LNL != 0 { cur = m.read(m.FP - 1) fp0x := m.FP - 1 - cur for i := 0; i < mathutil.Min(cur, n.LNL); i++ { m.push(m.read(fp0x)) fp0x++ } } if n.LNL > cur { m.push(m.FP) } m.push(n.LNL) m.FP = m.SP() m.Stack = append(m.Stack, make([]int, n.NVars)...) }
// PutBytes stores b in the DB and returns its id. Zero length byte slices are // guaranteed to return zero ID. PutBytes Locks the DB before updating it. func (d *MemDB) PutBytes(b []byte) int { if len(b) == 0 { return 0 } if len(b) == 1 { return int(b[0]) + 1 } d.mu.Lock() // W+ id := d.nextID pi := id >> dbPageShift if pi < len(d.pages) { p := d.pages[pi] off := id & dbPageMask if n := cap(p) - off - maxUvarint; n >= len(b) { p = p[:cap(p)] l := binary.PutUvarint(p[off:], uint64(len(b))) copy(p[off+l:], b) n = l + len(b) d.pages[pi] = p[:off+n] d.nextID += n d.mu.Unlock() // W- return id + 257 } pi++ } p := make([]byte, mathutil.Max(dbPageSize, maxUvarint+len(b))) p = p[:binary.PutUvarint(p, uint64(len(b)))] p = append(p, b...) d.pages = append(d.pages, p) id = pi << dbPageShift d.nextID = id + mathutil.Min(dbPageSize, len(p)) d.mu.Unlock() // W- return id + 257 }
// Readdir implements http.File. func (f *HTTPFile) Readdir(count int) ([]os.FileInfo, error) { if f.isFile { return nil, fmt.Errorf("not a directory: %s", f.name) } if count <= 0 { r := f.dirEntries f.dirEntries = f.dirEntries[:0] return r, nil } rq := mathutil.Min(count, len(f.dirEntries)) r := f.dirEntries[:rq] f.dirEntries = f.dirEntries[rq:] if len(r) != 0 { return r, nil } return nil, io.EOF }
// ReadAt implements Filer. func (f *MemFiler) ReadAt(b []byte, off int64) (n int, err error) { avail := f.size - off pgI := off >> pgBits pgO := int(off & pgMask) rem := len(b) if int64(rem) >= avail { rem = int(avail) err = io.EOF } for rem != 0 && avail > 0 { pg := f.m[pgI] if pg == nil { pg = &zeroPage } nc := copy(b[:mathutil.Min(rem, pgSize)], pg[pgO:]) pgI++ pgO = 0 rem -= nc n += nc b = b[nc:] } return }
func (f *File) readAt(b []byte, off int64, bits bool) (n int, err error) { var fsize int64 if !bits { fsize, err = f.Size() if err != nil { return } } avail := fsize - off pgI := off >> pgBits pgO := int(off & pgMask) rem := len(b) if !bits && int64(rem) >= avail { rem = int(avail) err = io.EOF } for rem != 0 && (bits || avail > 0) { v, err := (*Array)(f).Get(pgI) if err != nil { return n, err } pg, _ := v.([]byte) if len(pg) == 0 { pg = zeroPage[:] } nc := copy(b[:mathutil.Min(rem, pgSize)], pg[pgO:]) pgI++ pgO = 0 rem -= nc n += nc b = b[nc:] } return }
func BenchmarkRollbackFiler(b *testing.B) { rng := rand.New(rand.NewSource(42)) type t struct { off int64 b []byte } a := []t{} for rem := b.N; rem > 0; { off := rng.Int63() n := mathutil.Min(rng.Intn(1e3)+1, rem) a = append(a, t{off, rndBytes(rng, n)}) rem -= n } var r *RollbackFiler f := NewMemFiler() checkpoint := func(sz int64) (err error) { return f.Truncate(sz) } r, err := NewRollbackFiler(f, checkpoint, f) if err != nil { b.Fatal(err) } if err := r.BeginUpdate(); err != nil { b.Fatal(err) } b.ResetTimer() for _, v := range a { if _, err := r.WriteAt(v.b, v.off); err != nil { b.Fatal(err) } } }
func main1(in string) (err error) { var out io.Writer if nm := *oOut; nm != "" { var f *os.File var e error if f, err = os.Create(nm); err != nil { return err } defer func() { if e1 := f.Close(); e1 != nil && err == nil { err = e1 } }() w := bufio.NewWriter(f) defer func() { if e1 := w.Flush(); e1 != nil && err == nil { err = e1 } }() buf := bytes.NewBuffer(nil) out = buf defer func() { var dest []byte if dest, e = format.Source(buf.Bytes()); e != nil { dest = buf.Bytes() } if _, e = w.Write(dest); e != nil && err == nil { err = e } }() } var rep io.Writer if nm := *oReport; nm != "" { f, err1 := os.Create(nm) if err1 != nil { return err1 } defer func() { if e := f.Close(); e != nil && err == nil { err = e } }() w := bufio.NewWriter(f) defer func() { if e := w.Flush(); e != nil && err == nil { err = e } }() rep = w } var xerrors []byte if nm := *oXErrors; nm != "" { b, err1 := ioutil.ReadFile(nm) if err1 != nil { return err1 } xerrors = b } p, err := y.ProcessFile(token.NewFileSet(), in, &y.Options{ //NoDefault: *oNoDefault, AllowConflicts: true, Closures: *oClosures, LA: *oLA, Reducible: *oReducible, Report: rep, Resolved: *oResolved, XErrorsName: *oXErrors, XErrorsSrc: xerrors, }) if err != nil { return err } if fn := *oXErrorsGen; fn != "" { f, err := os.OpenFile(fn, os.O_RDWR|os.O_CREATE, 0666) if err != nil { return err } b := bufio.NewWriter(f) if err := p.SkeletonXErrors(b); err != nil { return err } if err := b.Flush(); err != nil { return err } if err := f.Close(); err != nil { return err } } msu := make(map[*y.Symbol]int, len(p.Syms)) // sym -> usage for nm, sym := range p.Syms { if nm == "" || nm == "ε" || nm == "$accept" || nm == "#" { continue } msu[sym] = 0 } var minArg, maxArg int for _, state := range p.Table { for _, act := range state { msu[act.Sym]++ k, arg := act.Kind() if k == 'a' { continue } if k == 'r' { arg = -arg } minArg, maxArg = mathutil.Min(minArg, arg), mathutil.Max(maxArg, arg) } } su := make(symsUsed, 0, len(msu)) for sym, used := range msu { su = append(su, symUsed{sym, used}) } sort.Sort(su) // ----------------------------------------------------------- Prologue f := strutil.IndentFormatter(out, "\t") f.Format("// CAUTION: Generated file - DO NOT EDIT.\n\n") f.Format("%s", injectImport(p.Prologue)) f.Format(` type %[1]sSymType %i%s%u type %[1]sXError struct { state, xsym int } `, *oPref, p.UnionSrc) // ---------------------------------------------------------- Constants nsyms := map[string]*y.Symbol{} a := make([]string, 0, len(msu)) maxTokName := 0 for sym := range msu { nm := sym.Name if nm == "$default" || nm == "$end" || sym.IsTerminal && nm[0] != '\'' && sym.Value > 0 { maxTokName = mathutil.Max(maxTokName, len(nm)) a = append(a, nm) } nsyms[nm] = sym } sort.Strings(a) f.Format("\nconst (%i\n") for _, v := range a { nm := v switch nm { case "error": nm = *oPref + "ErrCode" case "$default": nm = *oPref + "Default" case "$end": nm = *oPref + "EofCode" } f.Format("%s%s = %d\n", nm, strings.Repeat(" ", maxTokName-len(nm)+1), nsyms[v].Value) } minArg-- // eg: [-13, 42], minArg -14 maps -13 to 1 so zero cell values -> empty. f.Format("\n%sMaxDepth = 200\n", *oPref) f.Format("%sTabOfs = %d\n", *oPref, minArg) f.Format("%u)") // ---------------------------------------------------------- Variables f.Format("\n\nvar (%i\n") // Lex translation table f.Format("%sXLAT = map[int]int{%i\n", *oPref) xlat := make(map[int]int, len(su)) var errSym int for i, v := range su { if v.sym.Name == "error" { errSym = i } xlat[v.sym.Value] = i f.Format("%6d: %3d, // %s (%dx)\n", v.sym.Value, i, v.sym.Name, msu[v.sym]) } f.Format("%u}\n") // Symbol names f.Format("\n%sSymNames = []string{%i\n", *oPref) for _, v := range su { f.Format("%q,\n", v.sym.Name) } f.Format("%u}\n") // Reduction table f.Format("\n%sReductions = []struct{xsym, components int}{%i\n", *oPref) for _, rule := range p.Rules { f.Format("{%d, %d},\n", xlat[rule.Sym.Value], len(rule.Components)) } f.Format("%u}\n") // XError table f.Format("\n%[1]sXErrors = map[%[1]sXError]string{%i\n", *oPref) for _, xerr := range p.XErrors { state := xerr.Stack[len(xerr.Stack)-1] xsym := -1 if xerr.Lookahead != nil { xsym = xlat[xerr.Lookahead.Value] } f.Format("%[1]sXError{%d, %d}: \"%s\",\n", *oPref, state, xsym, xerr.Msg) } f.Format("%u}\n\n") // Parse table tbits := 32 switch n := mathutil.BitLen(maxArg - minArg + 1); { case n < 8: tbits = 8 case n < 16: tbits = 16 } f.Format("%sParseTab = [%d][]uint%d{%i\n", *oPref, len(p.Table), tbits) nCells := 0 var tabRow sortutil.Uint64Slice for si, state := range p.Table { tabRow = tabRow[:0] max := 0 for _, act := range state { sym := act.Sym xsym, ok := xlat[sym.Value] if !ok { panic("internal error 001") } max = mathutil.Max(max, xsym) kind, arg := act.Kind() switch kind { case 'a': arg = 0 case 'r': arg *= -1 } tabRow = append(tabRow, uint64(xsym)<<32|uint64(arg-minArg)) } nCells += max tabRow.Sort() col := -1 if si%5 == 0 { f.Format("// %d\n", si) } f.Format("{") for i, v := range tabRow { xsym := int(uint32(v >> 32)) arg := int(uint32(v)) if col+1 != xsym { f.Format("%d: ", xsym) } switch { case i == len(tabRow)-1: f.Format("%d", arg) default: f.Format("%d, ", arg) } col = xsym } f.Format("},\n") } f.Format("%u}\n") fmt.Fprintf(os.Stderr, "Parse table entries: %d of %d, x %d bits == %d bytes\n", nCells, len(p.Table)*len(msu), tbits, nCells*tbits/8) if n := p.ConflictsSR; n != 0 { fmt.Fprintf(os.Stderr, "conflicts: %d shift/reduce\n", n) } if n := p.ConflictsRR; n != 0 { fmt.Fprintf(os.Stderr, "conflicts: %d reduce/reduce\n", n) } f.Format(`%u) var %[1]sDebug = 0 type %[1]sLexer interface { Lex(lval *%[1]sSymType) int Errorf(format string, a ...interface{}) Errors() []error } type %[1]sLexerEx interface { %[1]sLexer Reduced(rule, state int, lval *%[1]sSymType) bool } func %[1]sSymName(c int) (s string) { x, ok := %[1]sXLAT[c] if ok { return %[1]sSymNames[x] } return __yyfmt__.Sprintf("%%d", c) } func %[1]slex1(yylex %[1]sLexer, lval *%[1]sSymType) (n int) { n = yylex.Lex(lval) if n <= 0 { n = %[1]sEofCode } if %[1]sDebug >= 3 { __yyfmt__.Printf("\nlex %%s(%%#x %%d), %[4]s: %[3]s\n", %[1]sSymName(n), n, n, %[4]s) } return n } func %[1]sParse(yylex %[1]sLexer, parser *Parser) int { const yyError = %[2]d yyEx, _ := yylex.(%[1]sLexerEx) var yyn int parser.yylval = %[1]sSymType{} parser.yyVAL = %[1]sSymType{} yyS := parser.cache Nerrs := 0 /* number of errors */ Errflag := 0 /* error recovery flag */ yyerrok := func() { if %[1]sDebug >= 2 { __yyfmt__.Printf("yyerrok()\n") } Errflag = 0 } _ = yyerrok yystate := 0 yychar := -1 var yyxchar int var yyshift int yyp := -1 goto yystack ret0: return 0 ret1: return 1 yystack: /* put a state and value onto the stack */ yyp++ if yyp >= len(yyS) { nyys := make([]%[1]sSymType, len(yyS)*2) copy(nyys, yyS) yyS = nyys parser.cache = yyS } yyS[yyp] = parser.yyVAL yyS[yyp].yys = yystate yynewstate: if yychar < 0 { yychar = %[1]slex1(yylex, &parser.yylval) var ok bool if yyxchar, ok = %[1]sXLAT[yychar]; !ok { yyxchar = len(%[1]sSymNames) // > tab width } } if %[1]sDebug >= 4 { var a []int for _, v := range yyS[:yyp+1] { a = append(a, v.yys) } __yyfmt__.Printf("state stack %%v\n", a) } row := %[1]sParseTab[yystate] yyn = 0 if yyxchar < len(row) { if yyn = int(row[yyxchar]); yyn != 0 { yyn += %[1]sTabOfs } } switch { case yyn > 0: // shift yychar = -1 parser.yyVAL = parser.yylval yystate = yyn yyshift = yyn if %[1]sDebug >= 2 { __yyfmt__.Printf("shift, and goto state %%d\n", yystate) } if Errflag > 0 { Errflag-- } goto yystack case yyn < 0: // reduce case yystate == 1: // accept if %[1]sDebug >= 2 { __yyfmt__.Println("accept") } goto ret0 } if yyn == 0 { /* error ... attempt to resume parsing */ switch Errflag { case 0: /* brand new error */ if %[1]sDebug >= 1 { __yyfmt__.Printf("no action for %%s in state %%d\n", %[1]sSymName(yychar), yystate) } msg, ok := %[1]sXErrors[%[1]sXError{yystate, yyxchar}] if !ok { msg, ok = %[1]sXErrors[%[1]sXError{yystate, -1}] } if !ok && yyshift != 0 { msg, ok = %[1]sXErrors[%[1]sXError{yyshift, yyxchar}] } if !ok { msg, ok = %[1]sXErrors[%[1]sXError{yyshift, -1}] } if !ok || msg == "" { msg = "syntax error" } // ignore goyacc error message yylex.Errorf("") Nerrs++ fallthrough case 1, 2: /* incompletely recovered error ... try again */ Errflag = 3 /* find a state where "error" is a legal shift action */ for yyp >= 0 { row := %[1]sParseTab[yyS[yyp].yys] if yyError < len(row) { yyn = int(row[yyError])+%[1]sTabOfs if yyn > 0 { // hit if %[1]sDebug >= 2 { __yyfmt__.Printf("error recovery found error shift in state %%d\n", yyS[yyp].yys) } yystate = yyn /* simulate a shift of "error" */ goto yystack } } /* the current p has no shift on "error", pop stack */ if %[1]sDebug >= 2 { __yyfmt__.Printf("error recovery pops state %%d\n", yyS[yyp].yys) } yyp-- } /* there is no state on the stack with an error shift ... abort */ if %[1]sDebug >= 2 { __yyfmt__.Printf("error recovery failed\n") } goto ret1 case 3: /* no shift yet; clobber input char */ if %[1]sDebug >= 2 { __yyfmt__.Printf("error recovery discards %%s\n", %[1]sSymName(yychar)) } if yychar == %[1]sEofCode { goto ret1 } yychar = -1 goto yynewstate /* try again in the same state */ } } r := -yyn x0 := %[1]sReductions[r] x, n := x0.xsym, x0.components yypt := yyp _ = yypt // guard against "declared and not used" yyp -= n if yyp+1 >= len(yyS) { nyys := make([]%[1]sSymType, len(yyS)*2) copy(nyys, yyS) yyS = nyys parser.cache = yyS } parser.yyVAL = yyS[yyp+1] /* consult goto table to find next state */ exState := yystate yystate = int(%[1]sParseTab[yyS[yyp].yys][x])+%[1]sTabOfs /* reduction by production r */ if %[1]sDebug >= 2 { __yyfmt__.Printf("reduce using rule %%v (%%s), and goto state %%d\n", r, %[1]sSymNames[x], yystate) } switch r {%i `, *oPref, errSym, *oDlvalf, *oDlval) for r, rule := range p.Rules { if rule.Action == nil { continue } action := rule.Action.Values if len(action) == 0 { continue } if len(action) == 1 { part := action[0] if part.Type == parser.ActionValueGo { src := part.Src src = src[1 : len(src)-1] // Remove lead '{' and trail '}' if strings.TrimSpace(src) == "" { continue } } } components := rule.Components typ := rule.Sym.Type max := len(components) if p1 := rule.Parent; p1 != nil { max = rule.MaxParentDlr components = p1.Components } f.Format("case %d: ", r) for _, part := range action { num := part.Num switch part.Type { case parser.ActionValueGo: f.Format("%s", part.Src) case parser.ActionValueDlrDlr: f.Format("parser.yyVAL.%s", typ) if typ == "" { panic("internal error 002") } case parser.ActionValueDlrNum: typ := p.Syms[components[num-1]].Type if typ == "" { panic("internal error 003") } f.Format("yyS[yypt-%d].%s", max-num, typ) case parser.ActionValueDlrTagDlr: f.Format("parser.yyVAL.%s", part.Tag) case parser.ActionValueDlrTagNum: f.Format("yyS[yypt-%d].%s", max-num, part.Tag) } } f.Format("\n") } f.Format(`%u } if yyEx != nil && yyEx.Reduced(r, exState, &parser.yyVAL) { return -1 } goto yystack /* stack new state and value */ } %[2]s `, *oPref, p.Tail) _ = oNoLines //TODO Ignored for now return nil }
// []interface{}{qltype, ...}->[]interface{}{lldb scalar type, ...} // + long blobs are (pre)written to a chain of chunks. func (s *file) flatten(data []interface{}) (err error) { for i, v := range data { tag := 0 var b []byte switch x := v.(type) { case []byte: tag = qBlob b = x case *big.Int: tag = qBigInt b, err = s.codec.encode(x) case *big.Rat: tag = qBigRat b, err = s.codec.encode(x) case time.Time: tag = qTime b, err = s.codec.encode(x) case time.Duration: tag = qDuration b, err = s.codec.encode(x) default: continue } if err != nil { return } const chunk = 1 << 16 chunks := 0 var next int64 var buf []byte for rem := len(b); rem > shortBlob; { n := mathutil.Min(rem, chunk) part := b[rem-n:] b = b[:rem-n] rem -= n switch next { case 0: // last chunk buf, err = lldb.EncodeScalars([]interface{}{part}...) default: // middle chunk buf, err = lldb.EncodeScalars([]interface{}{next, part}...) } if err != nil { return } h, err := s.a.Alloc(buf) if err != nil { return err } next = h chunks++ } switch next { case 0: // single chunk buf, err = lldb.EncodeScalars([]interface{}{tag, b}...) default: // multi chunks buf, err = lldb.EncodeScalars([]interface{}{tag, next, b}...) } if err != nil { return } data[i] = buf } return }
/* Free up some synapses in this segment. We always free up inactive synapses (lowest permanence freed up first) before we start to free up active ones. param numToFree number of synapses to free up param inactiveSynapseIndices list of the inactive synapse indices. */ func (s *Segment) freeNSynapses(numToFree int, inactiveSynapseIndices []int) { //Make sure numToFree isn't larger than the total number of syns we have if numToFree > len(s.syns) { panic("Number to free cannot be larger than existing synapses.") } if s.tp.params.Verbosity >= 5 { fmt.Println("freeNSynapses with numToFree=", numToFree) fmt.Println("inactiveSynapseIndices= ", inactiveSynapseIndices) } var candidates []int // Remove the lowest perm inactive synapses first if len(inactiveSynapseIndices) > 0 { perms := make([]float64, len(inactiveSynapseIndices)) for idx, _ := range perms { perms[idx] = s.syns[idx].Permanence } var indexes []int floats.Argsort(perms, indexes) //sort perms cSize := mathutil.Min(numToFree, len(perms)) candidates = make([]int, cSize) //indexes[0:cSize] for i := 0; i < cSize; i++ { candidates[i] = inactiveSynapseIndices[indexes[i]] } } // Do we need more? if so, remove the lowest perm active synapses too var activeSynIndices []int if len(candidates) < numToFree { for i := 0; i < len(s.syns); i++ { if !utils.ContainsInt(i, inactiveSynapseIndices) { activeSynIndices = append(activeSynIndices, i) } } perms := make([]float64, len(activeSynIndices)) for i := range perms { perms[i] = s.syns[i].Permanence } var indexes []int floats.Argsort(perms, indexes) moreToFree := numToFree - len(candidates) //moreCandidates := make([]int, moreToFree) for i := 0; i < moreToFree; i++ { candidates = append(candidates, activeSynIndices[indexes[i]]) } } if s.tp.params.Verbosity >= 4 { fmt.Printf("Deleting %v synapses from segment to make room for new ones: %v \n", len(candidates), candidates) fmt.Println("Before:", s.ToString()) } // Delete candidate syns by copying undeleted to new slice var newSyns []Synapse for idx, val := range s.syns { if !utils.ContainsInt(idx, candidates) { newSyns = append(newSyns, val) } } s.syns = newSyns if s.tp.params.Verbosity >= 4 { fmt.Println("After:", s.ToString()) } }
func minmax(a, b int) { fmt.Println("Min:", mathutil.Min(a, b)) fmt.Println("Max:", mathutil.Max(a, b)) }
func (tp *TrivialPredictor) infer(activeColumns []int) { numColsToPredict := int(0.5 + tp.AverageDensity*float64(tp.NumOfCols)) //for method in self.methods: for _, method := range tp.Methods { // Copy t-1 into t copy(tp.State[method].ActiveStateLast, tp.State[method].ActiveState) copy(tp.State[method].PredictedStateLast, tp.State[method].PredictedState) copy(tp.State[method].ConfidenceLast, tp.State[method].Confidence) utils.FillSliceBool(tp.State[method].ActiveState, false) utils.FillSliceBool(tp.State[method].PredictedState, false) utils.FillSliceFloat64(tp.State[method].Confidence, 0.0) for _, val := range activeColumns { tp.State[method].ActiveState[val] = true } var predictedCols []int switch method { case Random: // Randomly predict N columns //predictedCols = RandomInts(numColsToPredict, tp.NumOfCols) break case Zeroth: // Always predict the top N most frequent columns var inds []int ints.Argsort(tp.ColumnCount, inds) predictedCols = inds[len(inds)-numColsToPredict:] break case Last: // Always predict the last input for idx, val := range tp.State[method].ActiveState { if val { predictedCols = append(predictedCols, idx) } } break case All: // Always predict all columns for i := 0; i < tp.NumOfCols; i++ { predictedCols = append(predictedCols, i) } break case Lots: // Always predict 2 * the top N most frequent columns numColsToPredict := mathutil.Min(2*numColsToPredict, tp.NumOfCols) var inds []int ints.Argsort(tp.ColumnCount, inds) predictedCols = inds[len(inds)-numColsToPredict:] break default: panic("prediction method not implemented") } for _, val := range predictedCols { tp.State[method].PredictedState[val] = true tp.State[method].Confidence[val] = 1.0 } if tp.Verbosity > 1 { fmt.Println("Random prediction:", method) fmt.Println(" numColsToPredict:", numColsToPredict) fmt.Println(predictedCols) } } }
//Creates a new spatial pooler func NewSpatialPooler(spParams SpParams) *SpatialPooler { sp := SpatialPooler{} //Validate inputs sp.numColumns = utils.ProdInt(spParams.ColumnDimensions) sp.numInputs = utils.ProdInt(spParams.InputDimensions) if sp.numColumns < 1 { panic("Must have at least 1 column") } if sp.numInputs < 1 { panic("must have at least 1 input") } if spParams.NumActiveColumnsPerInhArea < 1 && (spParams.LocalAreaDensity < 1) && (spParams.LocalAreaDensity >= 0.5) { panic("Num active colums invalid") } sp.InputDimensions = spParams.InputDimensions sp.ColumnDimensions = spParams.ColumnDimensions sp.PotentialRadius = int(mathutil.Min(spParams.PotentialRadius, sp.numInputs)) sp.PotentialPct = spParams.PotentialPct sp.GlobalInhibition = spParams.GlobalInhibition sp.LocalAreaDensity = spParams.LocalAreaDensity sp.NumActiveColumnsPerInhArea = spParams.NumActiveColumnsPerInhArea sp.StimulusThreshold = spParams.StimulusThreshold sp.SynPermInactiveDec = spParams.SynPermInactiveDec sp.SynPermActiveInc = spParams.SynPermActiveInc sp.SynPermBelowStimulusInc = spParams.SynPermConnected / 10.0 sp.SynPermConnected = spParams.SynPermConnected sp.MinPctOverlapDutyCycles = spParams.MinPctOverlapDutyCycle sp.MinPctActiveDutyCycles = spParams.MinPctActiveDutyCycle sp.DutyCyclePeriod = spParams.DutyCyclePeriod sp.MaxBoost = spParams.MaxBoost sp.Seed = spParams.Seed sp.SpVerbosity = spParams.SpVerbosity // Extra parameter settings sp.SynPermMin = 0 sp.SynPermMax = 1 sp.SynPermTrimThreshold = sp.SynPermActiveInc / 2.0 if sp.SynPermTrimThreshold >= sp.SynPermConnected { panic("Syn perm threshold >= syn connected.") } sp.UpdatePeriod = 50 sp.InitConnectedPct = 0.5 /* # Internal state version = 1.0 iterationNum = 0 iterationLearnNum = 0 */ /* Store the set of all inputs that are within each column's potential pool. 'potentialPools' is a matrix, whose rows represent cortical columns, and whose columns represent the input bits. if potentialPools[i][j] == 1, then input bit 'j' is in column 'i's potential pool. A column can only be connected to inputs in its potential pool. The indices refer to a falttenned version of both the inputs and columns. Namely, irrespective of the topology of the inputs and columns, they are treated as being a one dimensional array. Since a column is typically connected to only a subset of the inputs, many of the entries in the matrix are 0. Therefore the the potentialPool matrix is stored using the SparseBinaryMatrix class, to reduce memory footprint and compuation time of algorithms that require iterating over the data strcuture. */ sp.potentialPools = NewDenseBinaryMatrix(sp.numColumns, sp.numInputs) /* Initialize the permanences for each column. Similar to the 'potentialPools', the permances are stored in a matrix whose rows represent the cortial columns, and whose columns represent the input bits. if permanences[i][j] = 0.2, then the synapse connecting cortical column 'i' to input bit 'j' has a permanence of 0.2. Here we also use the SparseMatrix class to reduce the memory footprint and computation time of algorithms that require iterating over the data structure. This permanence matrix is only allowed to have non-zero elements where the potential pool is non-zero. */ //Assumes 70% sparsity elms := make(map[int]float64, int(float64(sp.numColumns*sp.numInputs)*0.3)) sp.permanences = matrix.MakeSparseMatrix(elms, sp.numColumns, sp.numInputs) /* Initialize a tiny random tie breaker. This is used to determine winning columns where the overlaps are identical. */ sp.tieBreaker = make([]float64, sp.numColumns) for i := 0; i < len(sp.tieBreaker); i++ { sp.tieBreaker[i] = 0.01 * rand.Float64() } /* 'connectedSynapses' is a similar matrix to 'permanences' (rows represent cortial columns, columns represent input bits) whose entries represent whether the cortial column is connected to the input bit, i.e. its permanence value is greater than 'synPermConnected'. While this information is readily available from the 'permanence' matrix, it is stored separately for efficiency purposes. */ sp.connectedSynapses = NewDenseBinaryMatrix(sp.numColumns, sp.numInputs) /* Stores the number of connected synapses for each column. This is simply a sum of each row of 'ConnectedSynapses'. again, while this information is readily available from 'ConnectedSynapses', it is stored separately for efficiency purposes. */ sp.connectedCounts = make([]int, sp.numColumns) /* Initialize the set of permanence values for each columns. Ensure that each column is connected to enough input bits to allow it to be activated */ for i := 0; i < sp.numColumns; i++ { potential := sp.mapPotential(i, true) sp.potentialPools.ReplaceRow(i, potential) perm := sp.initPermanence(potential, sp.InitConnectedPct) sp.updatePermanencesForColumn(perm, i, true) } sp.overlapDutyCycles = make([]float64, sp.numColumns) sp.activeDutyCycles = make([]float64, sp.numColumns) sp.minOverlapDutyCycles = make([]float64, sp.numColumns) sp.minActiveDutyCycles = make([]float64, sp.numColumns) sp.boostFactors = make([]float64, sp.numColumns) for i := 0; i < len(sp.boostFactors); i++ { sp.boostFactors[i] = 1.0 } /* The inhibition radius determines the size of a column's local neighborhood. of a column. A cortical column must overcome the overlap score of columns in his neighborhood in order to become actives. This radius is updated every learning round. It grows and shrinks with the average number of connected synapses per column. */ sp.inhibitionRadius = 0 sp.updateInhibitionRadius(sp.avgConnectedSpanForColumnND, sp.avgColumnsPerInput) if sp.spVerbosity > 0 { sp.printParameters() } return &sp }
func TestMarshal(t *testing.T) { now := time.Now() dur := time.Millisecond schema8 := testSchema8{ A: true, //B: 1, C: 2, D: 3, E: 4, F: 5, //G: 6, H: 7, I: 8, J: 9, K: 10, L: 11, M: 12, N: -1, O: -2, P: []byte("abc"), Q: *big.NewInt(1), R: *big.NewRat(3, 2), S: "string", T: now, U: dur, } schema8.PA = &schema8.A //schema8.PB = &schema8.B schema8.PC = &schema8.C schema8.PD = &schema8.D schema8.PE = &schema8.E schema8.PF = &schema8.F //schema8.PG = &schema8.G schema8.PH = &schema8.H schema8.PI = &schema8.I schema8.PJ = &schema8.J schema8.PK = &schema8.K schema8.PL = &schema8.L schema8.PM = &schema8.M schema8.PN = &schema8.N schema8.PO = &schema8.O schema8.PP = &schema8.P schema8.PQ = &schema8.Q schema8.PR = &schema8.R schema8.PS = &schema8.S schema8.PT = &schema8.T schema8.PU = &schema8.U type u int tab := []struct { inst interface{} err bool r []interface{} }{ {42, true, nil}, {new(u), true, nil}, {testSchema8{}, false, []interface{}{ false, //int64(0), int8(0), int16(0), int32(0), int64(0), //uint64(0), uint8(0), uint16(0), uint32(0), uint64(0), float32(0), float64(0), complex64(0), complex128(0), []byte(nil), big.Int{}, big.Rat{}, "", time.Time{}, time.Duration(0), nil, //nil, nil, nil, nil, nil, //nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, }}, {&testSchema8{}, false, []interface{}{ false, //int64(0), int8(0), int16(0), int32(0), int64(0), //uint64(0), uint8(0), uint16(0), uint32(0), uint64(0), float32(0), float64(0), complex64(0), complex128(0), []byte(nil), big.Int{}, big.Rat{}, "", time.Time{}, time.Duration(0), nil, //nil, nil, nil, nil, nil, //nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, }}, {schema8, false, []interface{}{ true, //int64(1), int8(2), int16(3), int32(4), int64(5), //uint64(6), uint8(7), uint16(8), uint32(9), uint64(10), float32(11), float64(12), complex64(-1), complex128(-2), []byte("abc"), *big.NewInt(1), *big.NewRat(3, 2), "string", now, dur, true, //int64(1), int8(2), int16(3), int32(4), int64(5), //uint64(6), uint8(7), uint16(8), uint32(9), uint64(10), float32(11), float64(12), complex64(-1), complex128(-2), []byte("abc"), *big.NewInt(1), *big.NewRat(3, 2), "string", now, dur, }}, {&schema8, false, []interface{}{ true, //int64(1), int8(2), int16(3), int32(4), int64(5), //uint64(6), uint8(7), uint16(8), uint32(9), uint64(10), float32(11), float64(12), complex64(-1), complex128(-2), []byte("abc"), *big.NewInt(1), *big.NewRat(3, 2), "string", now, dur, true, //int64(1), int8(2), int16(3), int32(4), int64(5), //uint64(6), uint8(7), uint16(8), uint32(9), uint64(10), float32(11), float64(12), complex64(-1), complex128(-2), []byte("abc"), *big.NewInt(1), *big.NewRat(3, 2), "string", now, dur, }}, } for iTest, test := range tab { r, err := Marshal(test.inst) if g, e := err != nil, test.err; g != e { t.Fatal(iTest, g, e) } if err != nil { t.Log(err) continue } for i := 0; i < mathutil.Min(len(r), len(test.r)); i++ { g, e := r[i], test.r[i] use(e) switch x := g.(type) { case bool: switch y := e.(type) { case bool: if x != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case int: switch y := e.(type) { case int64: if int64(x) != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case int8: switch y := e.(type) { case int8: if x != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case int16: switch y := e.(type) { case int16: if x != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case int32: switch y := e.(type) { case int32: if x != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case int64: switch y := e.(type) { case int64: if x != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case uint: switch y := e.(type) { case uint64: if uint64(x) != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case uint8: switch y := e.(type) { case uint8: if x != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case uint16: switch y := e.(type) { case uint16: if x != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case uint32: switch y := e.(type) { case uint32: if x != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case uint64: switch y := e.(type) { case uint64: if x != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case float32: switch y := e.(type) { case float32: if x != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case float64: switch y := e.(type) { case float64: if x != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case complex64: switch y := e.(type) { case complex64: if x != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case complex128: switch y := e.(type) { case complex128: if x != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case []byte: switch y := e.(type) { case []byte: if bytes.Compare(x, y) != 0 { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case big.Int: switch y := e.(type) { case big.Int: if x.Cmp(&y) != 0 { t.Fatal(iTest, &x, &y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case big.Rat: switch y := e.(type) { case big.Rat: if x.Cmp(&y) != 0 { t.Fatal(iTest, &x, &y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case string: switch y := e.(type) { case string: if x != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case time.Time: switch y := e.(type) { case time.Time: if !x.Equal(y) { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case time.Duration: switch y := e.(type) { case time.Duration: if x != y { t.Fatal(iTest, x, y) } default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } case nil: switch y := e.(type) { case nil: // ok default: t.Fatalf("%d: %T <-> %T", iTest, x, y) } default: panic(fmt.Errorf("%T", x)) } } if g, e := len(r), len(test.r); g != e { t.Fatal(iTest, g, e) } } }