/
common_test.go
336 lines (311 loc) · 15.3 KB
/
common_test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
package marctools
import (
"bytes"
"database/sql"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
_ "github.com/mattn/go-sqlite3"
"github.com/miku/marc22"
)
func TestCount(t *testing.T) {
var tests = []struct {
in string
out int64
}{
{"./fixtures/authoritybibs.mrc", 9},
{"./fixtures/deweybrowse.mrc", 1},
{"./fixtures/heb.mrc", 1},
{"./fixtures/journals.mrc", 10},
{"./fixtures/testbug2.mrc", 1},
{"./fixtures/weird_ids.mrc", 8},
}
for _, tt := range tests {
count := RecordCount(tt.in)
if count != tt.out {
t.Errorf("RecordCount(%s) => %d, want: %d", tt.in, count, tt.out)
}
}
}
func TestIDList(t *testing.T) {
var tests = []struct {
in string
out []string
}{
{"./fixtures/deweybrowse.mrc", []string{"testdeweybrowse"}},
{"./fixtures/testbug2.mrc", []string{"testbug2"}},
{"./fixtures/heb.mrc", []string{"testbug1"}},
// {"./fixtures/weird_ids.mrc", []string{`|||pipe|||"quote`,
// `dot.dash-underscore__3.colon:18`,
// `dot.dash-underscore__3.space`,
// `dollar$ign/slashcombo`,
// `all'kinds"of'quotes"`,
// `<angle>brackets&ersands`,
// `hashes#coming@ya`,
// `wehave?sand%stospare`}},
{"./fixtures/journals.mrc", []string{"testsample1",
"testsample2",
"testsample3",
"testsample4",
"testsample5",
"testsample6",
"testsample7",
"testsample8",
"testsample9",
"testsample10"}},
}
for _, tt := range tests {
ids := IdentifierList(tt.in, false)
if len(ids) != len(tt.out) {
t.Errorf("IDList(%s) => %+v, want: %+v", tt.in, ids, tt.out)
}
for i := 0; i < len(ids); i++ {
if ids[i] != tt.out[i] {
t.Errorf("List element mismatch in IDList(%s)[%d] => %+v, want: %+v",
tt.in, i, ids[i], tt.out[i])
}
}
}
}
func TestMarcMap(t *testing.T) {
var tests = []struct {
in string
out string
}{
{"./fixtures/deweybrowse.mrc", "testdeweybrowse\t0\t613\n"},
{"./fixtures/journals.mrc",
"testsample1\t0\t1571\n" +
"testsample2\t1571\t1195\n" +
"testsample3\t2766\t1057\n" +
"testsample4\t3823\t1361\n" +
"testsample5\t5184\t1707\n" +
"testsample6\t6891\t1532\n" +
"testsample7\t8423\t1426\n" +
"testsample8\t9849\t1251\n" +
"testsample9\t11100\t2173\n" +
"testsample10\t13273\t1195\n"},
}
var b bytes.Buffer
for _, tt := range tests {
b.Reset()
MarcMap(tt.in, &b, false)
output := b.String()
if output != tt.out {
t.Errorf("MarcMap(%s) => %s, want: %s", tt.in, output, tt.out)
}
}
}
func TestMarcMapSqlite(t *testing.T) {
file, err := ioutil.TempFile("", "marctools-TestMarcMapSqlite-")
if err != nil {
t.Errorf("%s\n", err)
}
MarcMapSqlite("./fixtures/journals.mrc", file.Name(), false)
db, err := sql.Open("sqlite3", file.Name())
if err != nil {
t.Errorf("%s\n", err)
}
defer db.Close()
var offset, length string
err = db.QueryRow("SELECT offset, length FROM seekmap WHERE id=?", "testsample3").Scan(&offset, &length)
switch {
case err == sql.ErrNoRows:
t.Errorf("%s\n", err)
case err != nil:
t.Errorf("%s\n", err)
default:
if offset != "2766" || length != "1057" {
t.Errorf("(%s, %s), want: (2766, 1057)\n", offset, length)
}
}
}
func TestMarcSplitDirectoryPrefix(t *testing.T) {
dir, err := ioutil.TempDir("", "marctools-TestMarcSplitDirectoryPrefix-")
if err != nil {
t.Errorf("%s\n", err)
}
prefix := "prefix-"
MarcSplitDirectoryPrefix("./fixtures/authoritybibs.mrc", 3, dir, prefix)
expectedFiles := []string{filepath.Join(dir, fmt.Sprintf("%s%08d", prefix, 0)),
filepath.Join(dir, fmt.Sprintf("%s%08d", prefix, 1)),
filepath.Join(dir, fmt.Sprintf("%s%08d", prefix, 2))}
for _, filename := range expectedFiles {
if _, err := os.Stat(filename); os.IsNotExist(err) {
t.Errorf("%s expected: %s\n", filename, err)
}
recordCount := RecordCount(filename)
if recordCount != 3 {
t.Errorf("unexpected record count in %s: %d, want: %d", filename, recordCount, 3)
}
}
}
func TestKeyValueStringToMap(t *testing.T) {
var tests = []struct {
in string
out map[string]string
}{
{"key=value", map[string]string{"key": "value"}},
{"k1=v1, k2=v2", map[string]string{"k1": "v1", "k2": "v2"}},
{"k1 = v1,k2= v2", map[string]string{"k1": "v1", "k2": "v2"}},
{"k1=v1, k1=v2", map[string]string{"k1": "v2"}},
}
for _, tt := range tests {
out, err := KeyValueStringToMap(tt.in)
if err != nil {
t.Errorf("KeyValueStringToMap(%s) err'd: %s", tt.in, err)
}
eq := reflect.DeepEqual(out, tt.out)
if !eq {
t.Errorf("KeyValueStringToMap(%s) => %+v, want: %+v", tt.in, out, tt.out)
}
}
failures := []string{"key=",
"keyvalue",
"key=value=value",
"key=value=value,=",
"k1=v1,,, k1=v2"}
for _, value := range failures {
out, err := KeyValueStringToMap(value)
if err == nil {
t.Errorf("KeyValueStringToMap(%s) => %s, want: err!", value, out)
}
}
}
func TestStringToMapSet(t *testing.T) {
var tests = []struct {
in string
out map[string]bool
}{
{"key", map[string]bool{"key": true}},
{"k1,k2", map[string]bool{"k1": true, "k2": true}},
{"k1, k2", map[string]bool{"k1": true, "k2": true}},
}
for _, tt := range tests {
out := StringToMapSet(tt.in)
eq := reflect.DeepEqual(out, tt.out)
if !eq {
t.Errorf("StringToMapSet(%s) => %+v, want: %+v", tt.in, out, tt.out)
}
}
}
var recordMapTests = []struct {
record string
filterMap map[string]bool
includeLeader bool
out string
}{
// test with includeLeader=false
{`00613cam a2200229Ma 4500001001600000005001700016008004100033020001500074035002300089040002500112041001800137043001200155050002400167049000900191082001600200082001600216100003000232245002200262250002300284260004700307300002900354 testdeweybrowse 20110419140028.0 110214s1992 it a b 001 0 ita d a8820737493 a(OCoLC)ocm30585539 aRBN cRBN dOCLCG dPVU 1 aita alat hlat ae-it--- 14 aDG848.15 b.V53 1992 aPVUM a123.45 .I39 a123.46 .Q39 1 aPerson, Fake, d1668-1744. 10 aDewey browse test aFictional edition. aMorano : bCentro di Studi Vichiani, c1992. a296 p. : bill. ; c24 cm. `,
map[string]bool{},
false,
`{"001":"testdeweybrowse","005":"20110419140028.0","008":"110214s1992 it a b 001 0 ita d","020":[{"a":["8820737493"],"ind1":" ","ind2":" "}],"035":[{"a":["(OCoLC)ocm30585539"],"ind1":" ","ind2":" "}],"040":[{"a":["RBN"],"c":["RBN"],"d":["OCLCG","PVU"],"ind1":" ","ind2":" "}],"041":[{"a":["ita","lat"],"h":["lat"],"ind1":"1","ind2":" "}],"043":[{"a":["e-it---"],"ind1":" ","ind2":" "}],"049":[{"a":["PVUM"],"ind1":" ","ind2":" "}],"050":[{"a":["DG848.15"],"b":[".V53 1992"],"ind1":"1","ind2":"4"}],"082":[{"a":["123.45 .I39"],"ind1":" ","ind2":" "},{"a":["123.46 .Q39"],"ind1":" ","ind2":" "}],"100":[{"a":["Person, Fake,"],"d":["1668-1744."],"ind1":"1","ind2":" "}],"245":[{"a":["Dewey browse test"],"ind1":"1","ind2":"0"}],"250":[{"a":["Fictional edition."],"ind1":" ","ind2":" "}],"260":[{"a":["Morano :"],"b":["Centro di Studi Vichiani,"],"c":["1992."],"ind1":" ","ind2":" "}],"300":[{"a":["296 p. :"],"b":["ill. ;"],"c":["24 cm."],"ind1":" ","ind2":" "}]}`},
// test with includeLeader=true
{`00613cam a2200229Ma 4500001001600000005001700016008004100033020001500074035002300089040002500112041001800137043001200155050002400167049000900191082001600200082001600216100003000232245002200262250002300284260004700307300002900354 testdeweybrowse 20110419140028.0 110214s1992 it a b 001 0 ita d a8820737493 a(OCoLC)ocm30585539 aRBN cRBN dOCLCG dPVU 1 aita alat hlat ae-it--- 14 aDG848.15 b.V53 1992 aPVUM a123.45 .I39 a123.46 .Q39 1 aPerson, Fake, d1668-1744. 10 aDewey browse test aFictional edition. aMorano : bCentro di Studi Vichiani, c1992. a296 p. : bill. ; c24 cm. `,
map[string]bool{},
true,
`{"001":"testdeweybrowse","005":"20110419140028.0","008":"110214s1992 it a b 001 0 ita d","020":[{"a":["8820737493"],"ind1":" ","ind2":" "}],"035":[{"a":["(OCoLC)ocm30585539"],"ind1":" ","ind2":" "}],"040":[{"a":["RBN"],"c":["RBN"],"d":["OCLCG","PVU"],"ind1":" ","ind2":" "}],"041":[{"a":["ita","lat"],"h":["lat"],"ind1":"1","ind2":" "}],"043":[{"a":["e-it---"],"ind1":" ","ind2":" "}],"049":[{"a":["PVUM"],"ind1":" ","ind2":" "}],"050":[{"a":["DG848.15"],"b":[".V53 1992"],"ind1":"1","ind2":"4"}],"082":[{"a":["123.45 .I39"],"ind1":" ","ind2":" "},{"a":["123.46 .Q39"],"ind1":" ","ind2":" "}],"100":[{"a":["Person, Fake,"],"d":["1668-1744."],"ind1":"1","ind2":" "}],"245":[{"a":["Dewey browse test"],"ind1":"1","ind2":"0"}],"250":[{"a":["Fictional edition."],"ind1":" ","ind2":" "}],"260":[{"a":["Morano :"],"b":["Centro di Studi Vichiani,"],"c":["1992."],"ind1":" ","ind2":" "}],"300":[{"a":["296 p. :"],"b":["ill. ;"],"c":["24 cm."],"ind1":" ","ind2":" "}],"leader":{"ba":"229","cs":"a","ic":"2","impldef":"m Ma ","length":"613","lol":"4","losp":"5","raw":"00613cam a2200229Ma 4500","sfcl":"2","status":"c","type":"a"}}`},
// test with includeLeader=false and a simple filter
{`00613cam a2200229Ma 4500001001600000005001700016008004100033020001500074035002300089040002500112041001800137043001200155050002400167049000900191082001600200082001600216100003000232245002200262250002300284260004700307300002900354 testdeweybrowse 20110419140028.0 110214s1992 it a b 001 0 ita d a8820737493 a(OCoLC)ocm30585539 aRBN cRBN dOCLCG dPVU 1 aita alat hlat ae-it--- 14 aDG848.15 b.V53 1992 aPVUM a123.45 .I39 a123.46 .Q39 1 aPerson, Fake, d1668-1744. 10 aDewey browse test aFictional edition. aMorano : bCentro di Studi Vichiani, c1992. a296 p. : bill. ; c24 cm. `,
map[string]bool{"001": true, "005": true},
false,
`{"001":"testdeweybrowse","005":"20110419140028.0"}`},
{`00092 2200061 4500001000600000040001200006040001200018 12345 aValue 1 aValue 2 `,
map[string]bool{},
false,
`{"001":"12345","040":[{"a":["Value 1"],"ind1":" ","ind2":" "},{"a":["Value 2"],"ind1":" ","ind2":" "}]}`,
},
{`00077 2200049 4500001000600000040002100006 23456 aValue 1 aValue 2 `,
map[string]bool{},
false,
`{"001":"23456","040":[{"a":["Value 1","Value 2"],"ind1":" ","ind2":" "}]}`,
},
}
func TestRecordToMap(t *testing.T) {
for _, tt := range recordMapTests {
reader := strings.NewReader(tt.record)
record, err := marc22.ReadRecord(reader)
if err != nil {
t.Error(err)
}
result := RecordMap(record, tt.filterMap, tt.includeLeader)
if result == nil {
t.Error("RecordToMap should not return nil")
}
b, err := json.Marshal(result)
if err != nil {
t.Error("RecordToMap should return something JSON-serializable")
}
if string(b) != tt.out {
t.Errorf("RecordToMap(%s, %+v, %v) => %+v, want: %+v", tt.record, tt.filterMap, tt.includeLeader, string(b), tt.out)
}
}
}
var recordToTSVTests = []struct {
record string
tags []string
fillNA string
separator string
skipIncompleteLines bool
out string
}{
{`00613cam a2200229Ma 4500001001600000005001700016008004100033020001500074035002300089040002500112041001800137043001200155050002400167049000900191082001600200082001600216100003000232245002200262250002300284260004700307300002900354 testdeweybrowse 20110419140028.0 110214s1992 it a b 001 0 ita d a8820737493 a(OCoLC)ocm30585539 aRBN cRBN dOCLCG dPVU 1 aita alat hlat ae-it--- 14 aDG848.15 b.V53 1992 aPVUM a123.45 .I39 a123.46 .Q39 1 aPerson, Fake, d1668-1744. 10 aDewey browse test aFictional edition. aMorano : bCentro di Studi Vichiani, c1992. a296 p. : bill. ; c24 cm. `,
[]string{"001"},
"<NULL>",
"",
true,
"testdeweybrowse\n",
},
{`00613cam a2200229Ma 4500001001600000005001700016008004100033020001500074035002300089040002500112041001800137043001200155050002400167049000900191082001600200082001600216100003000232245002200262250002300284260004700307300002900354 testdeweybrowse 20110419140028.0 110214s1992 it a b 001 0 ita d a8820737493 a(OCoLC)ocm30585539 aRBN cRBN dOCLCG dPVU 1 aita alat hlat ae-it--- 14 aDG848.15 b.V53 1992 aPVUM a123.45 .I39 a123.46 .Q39 1 aPerson, Fake, d1668-1744. 10 aDewey browse test aFictional edition. aMorano : bCentro di Studi Vichiani, c1992. a296 p. : bill. ; c24 cm. `,
[]string{"001", "020.a", "082.a"},
"<NULL>",
"",
true,
"testdeweybrowse\t8820737493\t123.45 .I39\n",
},
{`00613cam a2200229Ma 4500001001600000005001700016008004100033020001500074035002300089040002500112041001800137043001200155050002400167049000900191082001600200082001600216100003000232245002200262250002300284260004700307300002900354 testdeweybrowse 20110419140028.0 110214s1992 it a b 001 0 ita d a8820737493 a(OCoLC)ocm30585539 aRBN cRBN dOCLCG dPVU 1 aita alat hlat ae-it--- 14 aDG848.15 b.V53 1992 aPVUM a123.45 .I39 a123.46 .Q39 1 aPerson, Fake, d1668-1744. 10 aDewey browse test aFictional edition. aMorano : bCentro di Studi Vichiani, c1992. a296 p. : bill. ; c24 cm. `,
[]string{"001", "020.a", "082.a"},
"<NULL>",
"|",
true,
"testdeweybrowse\t8820737493\t123.45 .I39|123.46 .Q39\n",
},
{`00613cam a2200229Ma 4500001001600000005001700016008004100033020001500074035002300089040002500112041001800137043001200155050002400167049000900191082001600200082001600216100003000232245002200262250002300284260004700307300002900354 testdeweybrowse 20110419140028.0 110214s1992 it a b 001 0 ita d a8820737493 a(OCoLC)ocm30585539 aRBN cRBN dOCLCG dPVU 1 aita alat hlat ae-it--- 14 aDG848.15 b.V53 1992 aPVUM a123.45 .I39 a123.46 .Q39 1 aPerson, Fake, d1668-1744. 10 aDewey browse test aFictional edition. aMorano : bCentro di Studi Vichiani, c1992. a296 p. : bill. ; c24 cm. `,
[]string{"999"},
"<NULL>",
"|",
false,
"<NULL>\n",
},
{`00613cam a2200229Ma 4500001001600000005001700016008004100033020001500074035002300089040002500112041001800137043001200155050002400167049000900191082001600200082001600216100003000232245002200262250002300284260004700307300002900354 testdeweybrowse 20110419140028.0 110214s1992 it a b 001 0 ita d a8820737493 a(OCoLC)ocm30585539 aRBN cRBN dOCLCG dPVU 1 aita alat hlat ae-it--- 14 aDG848.15 b.V53 1992 aPVUM a123.45 .I39 a123.46 .Q39 1 aPerson, Fake, d1668-1744. 10 aDewey browse test aFictional edition. aMorano : bCentro di Studi Vichiani, c1992. a296 p. : bill. ; c24 cm. `,
[]string{"999"},
"<NULL>",
"|",
true,
"",
},
{`00613cam a2200229Ma 4500001001600000005001700016008004100033020001500074035002300089040002500112041001800137043001200155050002400167049000900191082001600200082001600216100003000232245002200262250002300284260004700307300002900354 testdeweybrowse 20110419140028.0 110214s1992 it a b 001 0 ita d a8820737493 a(OCoLC)ocm30585539 aRBN cRBN dOCLCG dPVU 1 aita alat hlat ae-it--- 14 aDG848.15 b.V53 1992 aPVUM a123.45 .I39 a123.46 .Q39 1 aPerson, Fake, d1668-1744. 10 aDewey browse test aFictional edition. aMorano : bCentro di Studi Vichiani, c1992. a296 p. : bill. ; c24 cm. `,
[]string{"001", "@Length"},
"<NULL>",
"",
false,
"testdeweybrowse\t613\n",
},
}
func TestRecordToTSV(t *testing.T) {
for _, tt := range recordToTSVTests {
reader := strings.NewReader(tt.record)
record, err := marc22.ReadRecord(reader)
if err != nil {
t.Error(err)
}
result := RecordToTSV(record, tt.tags, tt.fillNA, tt.separator, tt.skipIncompleteLines)
if result != tt.out {
t.Errorf("RecordToTSV(%s, %v, %s, %s, %t) => %+v, want: %+v", record, tt.tags, tt.fillNA, tt.separator, tt.skipIncompleteLines, result, tt.out)
}
}
}