/
util.go
224 lines (200 loc) · 5.83 KB
/
util.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
package bucketstore
import (
"bytes"
"encoding/binary"
"fmt"
"math"
)
func Uint64ToBytes(v uint64) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, v)
return b
}
func BytesToUint64(b []byte) uint64 {
return binary.BigEndian.Uint64(b)
}
func BytesToFloat64(b []byte) float64 {
bits := binary.BigEndian.Uint64(b)
return math.Float64frombits(bits)
}
func StringToBytes(v string) []byte {
return []byte(v)
}
const (
ValueTypeBool = 0x01
ValueTypeString = 0x02
ValueTypeFloat64 = 0x03
ValueTypeNil = 0x04
valueTypeNoIndex = 0x00
)
//
// # Index specification.
//
// ## index key structure
//
// The following byte slice is index key structure.
//
// <valueType> + <value> + "0x00 0xFF" + <key>
//
// "0x00 0xFF" is a separator this spec reason is the below.
// 0x00 is least byte pattern to match prefix string by seek.
// 0xFF is not in the utf8 code.
//
// Example: true
// 0x01 0x01 0x00 0xFF <key>
//
// Example: "abc"
// 0x02 0x61 0x62 0x63 0x00 0xFF <key>
//
// Example: 1
// 0x03 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x01 0x00 0xFF <key>
//
// ## Description
//
// To search value using index, you can user IndexCursor that is the low level API.
// This cursor can move to the fist and last point matching bytes pattern in the specific property.
//
// Example1)
// If you want to search items with "ab" prefix. IndexCoursor seek by the following bytes pattern.
//
// 0x02 0x61 0x62 0x00 0xFF
//
// This bytes the below structure.
//
// <valueType> + <value> + "0x00 0xFF"
//
// If you have the "abc" value in your database like the above example this pattern is below.
//
// 0x02 0x61 0x62 0x63 0x00 0xFF
//
// IndexCursor bytes pattern is "0x02 0x61 0x62 0x00 0xFF", so It moves to before "abc" "abd" "abb" etc...
// And It walks using "Next" method until not to match the prefix .
//
// Example2)
// If you want to search items with "ab" prefix descending order. IndexCoursor seek by the following bytes pattern.
//
// 0x02 0x61 0x62 0xFF 0xFF
//
// This bytes the below structure.
//
// <valueType> + <value> + "0xFF 0xFF"
//
// If you have the "abc" value in your database like the above example this pattern is below.
//
// 0x02 0x61 0x62 0x63 0x00 0xFF
//
// IndexCursor bytes pattern is "0x02 0x61 0x62 0xFF 0xFF", so It moves to after "abc" "abd" "abb" etc...
// For instance, cursor moves to "aca" that is the fist point after all matching pattern space.
// You should move one previous position that is the last matching point.
// It walks using "Prev" method until not to match the prefix.
//
const (
sep1 = 0x00
sep2 = 0xFF
maxValueSize = 255
)
func genIndexKey(value interface{}, key []byte) []byte {
valueBytes, valueTypeByte := toIndexedBytes(value)
if valueTypeByte == valueTypeNoIndex {
// does not index.
return nil
}
// only support indexing in max 255 bytes.
size := len(valueBytes)
if valueTypeByte == ValueTypeString && size > maxValueSize {
valueBytes = valueBytes[:maxValueSize]
size = len(valueBytes)
}
return append(append(append([]byte{valueTypeByte}, valueBytes[:]...), sep1, sep2), key[:]...)
}
func genIndexPrefixForSeekFirst(valueType byte, value []byte) []byte {
// only support indexing in max 255 bytes.
if valueType == ValueTypeString && len(value) > maxValueSize {
value = value[:maxValueSize]
}
return append(append([]byte{valueType}, value[:]...), sep1, sep2)
}
// genIndexPrefixForSeekLast generates prefix bytes pattern to find item
// that is one more biggger than specified one.
func genIndexPrefixForSeekLast(valueType byte, value []byte) []byte {
// only support indexing in max 255 bytes.
if valueType == ValueTypeString && len(value) > maxValueSize {
value = value[:maxValueSize]
}
return append(append([]byte{valueType}, value[:]...), 0xFF, sep2)
}
func genIndexFilter(value interface{}) ([]byte, byte) {
valueBytes, valueTypeByte := toIndexedBytes(value)
if valueTypeByte == valueTypeNoIndex {
// does not index.
return nil, valueTypeNoIndex
}
return genIndexPrefixForSeekFirst(valueTypeByte, valueBytes), valueTypeByte
}
func getValueFromIndexKey(indexKey []byte) ([]byte, error) {
// get value type
switch indexKey[0] {
case ValueTypeBool:
return indexKey[1:2], nil
case ValueTypeString:
i := bytes.Index(indexKey[1:], []byte{sep1, sep2})
if i == -1 {
return nil, fmt.Errorf("got a illegal formatted key %v", indexKey)
}
return indexKey[1:i+1], nil
case ValueTypeFloat64:
return indexKey[1:9], nil
case ValueTypeNil:
return nil, nil
}
return nil, fmt.Errorf("got a illegal formatted key %v", indexKey)
}
func toIndexedBytes(value interface{}) ([]byte, byte) {
// https://golang.org/pkg/encoding/json/#Unmarshal
switch converted := value.(type) {
case bool:
// JSON boolean
if converted {
return []byte{1}, ValueTypeBool
} else {
return []byte{0}, ValueTypeBool
}
case string:
// JSON string
return []byte(converted), ValueTypeString
case float64:
// JSON number
// http://stackoverflow.com/questions/22491876/convert-byte-array-uint8-to-float64-in-golang
bits := math.Float64bits(converted)
bytes := make([]byte, 8)
binary.BigEndian.PutUint64(bytes, bits)
return bytes, ValueTypeFloat64
case int:
// JSON number
c := float64(converted)
bits := math.Float64bits(c)
bytes := make([]byte, 8)
binary.BigEndian.PutUint64(bytes, bits)
return bytes, ValueTypeFloat64
case int64:
// JSON number
c := float64(converted)
bits := math.Float64bits(c)
bytes := make([]byte, 8)
binary.BigEndian.PutUint64(bytes, bits)
return bytes, ValueTypeFloat64
case nil:
// JSON null
return nil, ValueTypeNil
case []interface{}:
// JSON array
// not support indexing...
return nil, valueTypeNoIndex
case map[string]interface{}:
// JSON object
// not support indexing...
return nil, valueTypeNoIndex
default:
return nil, valueTypeNoIndex
}
}