forked from stellar-deprecated/horizon
/
main.go
210 lines (176 loc) · 5.65 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
// Package ingest contains the ingestion system for horizon. This system takes
// data produced by the connected stellar-core database, transforms it and
// inserts it into the horizon database.
package ingest
import (
"time"
sq "github.com/lann/squirrel"
"github.com/rcrowley/go-metrics"
"github.com/stellar/horizon/cache"
"github.com/stellar/horizon/db2"
"github.com/stellar/horizon/db2/core"
)
const (
// CurrentVersion reflects the latest version of the ingestion
// algorithm. As rows are ingested into the horizon database, this version is
// used to tag them. In the future, any breaking changes introduced by a
// developer should be accompanied by an increase in this value.
//
// Scripts, that have yet to be ported to this codebase can then be leveraged
// to re-ingest old data with the new algorithm, providing a seamless
// transition when the ingested data's structure changes.
CurrentVersion = 8
)
// Cursor iterates through a stellar core database's ledgers
type Cursor struct {
// FirstLedger is the beginning of the range of ledgers (inclusive) that will
// attempt to be ingested in this session.
FirstLedger int32
// LastLedger is the end of the range of ledgers (inclusive) that will
// attempt to be ingested in this session.
LastLedger int32
// DB is the stellar-core db that data is ingested from.
DB *db2.Repo
Metrics *IngesterMetrics
// Err is the error that caused this iteration to fail, if any.
Err error
lg int32
tx int
op int
data *LedgerBundle
}
// EffectIngestion is a helper struct to smooth the ingestion of effects. this
// struct will track what the correct operation to use and order to use when
// adding effects into an ingestion.
type EffectIngestion struct {
Dest *Ingestion
OperationID int64
Accounts *cache.HistoryAccount
err error
added int
}
// LedgerBundle represents a single ledger's worth of novelty created by one
// ledger close
type LedgerBundle struct {
Sequence int32
Header core.LedgerHeader
TransactionFees []core.TransactionFee
Transactions []core.Transaction
}
// System represents the data ingestion subsystem of horizon.
type System struct {
// HorizonDB is the connection to the horizon database that ingested data will
// be written to.
HorizonDB *db2.Repo
// CoreDB is the stellar-core db that data is ingested from.
CoreDB *db2.Repo
Metrics IngesterMetrics
// Network is the passphrase for the network being imported
Network string
tick *time.Ticker
historySequence int32
coreSequence int32
}
// IngesterMetrics tracks all the metrics for the ingestion subsystem
type IngesterMetrics struct {
ClearLedgerTimer metrics.Timer
IngestLedgerTimer metrics.Timer
LoadLedgerTimer metrics.Timer
}
// Ingestion receives write requests from a Session
type Ingestion struct {
// DB is the sql repo to be used for writing any rows into the horizon
// database.
DB *db2.Repo
ledgers sq.InsertBuilder
transactions sq.InsertBuilder
transaction_participants sq.InsertBuilder
operations sq.InsertBuilder
operation_participants sq.InsertBuilder
effects sq.InsertBuilder
accounts sq.InsertBuilder
}
// Session represents a single attempt at ingesting data into the history
// database.
type Session struct {
Cursor *Cursor
Ingestion *Ingestion
// Network is the passphrase for the network being imported
Network string
// ClearExisting causes the session to clear existing data from the horizon db
// when the session is run.
ClearExisting bool
// Metrics is a reference to where the session should record its metric information
Metrics *IngesterMetrics
//
// Results fields
//
// Err is the error that caused this session to fail, if any.
Err error
// Ingested is the number of ledgers that were successfully ingested during
// this session.
Ingested int
accountCache *cache.HistoryAccount
}
// New initializes the ingester, causing it to begin polling the stellar-core
// database for now ledgers and ingesting data into the horizon database.
func New(network string, core, horizon *db2.Repo) *System {
i := &System{
Network: network,
HorizonDB: horizon,
CoreDB: core,
}
i.Metrics.ClearLedgerTimer = metrics.NewTimer()
i.Metrics.IngestLedgerTimer = metrics.NewTimer()
i.Metrics.LoadLedgerTimer = metrics.NewTimer()
i.tick = time.NewTicker(1 * time.Second)
return i
}
// NewSession initialize a new ingestion session, from `first` to `last` using
// `i`.
func NewSession(first, last int32, i *System) *Session {
hdb := i.HorizonDB.Clone()
return &Session{
Ingestion: &Ingestion{
DB: hdb,
},
Cursor: &Cursor{
FirstLedger: first,
LastLedger: last,
DB: i.CoreDB,
Metrics: &i.Metrics,
},
Network: i.Network,
Metrics: &i.Metrics,
accountCache: cache.NewHistoryAccount(hdb),
}
}
// ReingestAll re-ingests all data
func ReingestAll(network string, core, horizon *db2.Repo) (int, error) {
i := New(network, core, horizon)
return i.ReingestAll()
}
func ReingestOutdated(network string, core, horizon *db2.Repo) (int, error) {
i := New(network, core, horizon)
return i.ReingestOutdated()
}
// ReingestSingle re-ingests a single ledger
func ReingestSingle(network string, core, horizon *db2.Repo, sequence int32) error {
i := New(network, core, horizon)
return i.ReingestSingle(sequence)
}
// RunOnce runs a single ingestion session
func RunOnce(network string, core, horizon *db2.Repo) (*Session, error) {
i := New(network, core, horizon)
err := i.updateLedgerState()
if err != nil {
return nil, err
}
is := NewSession(
i.historySequence+1,
i.coreSequence,
i,
)
is.Run()
return is, is.Err
}