aboutsummaryrefslogtreecommitdiff
path: root/six/info_file_parser.go
diff options
context:
space:
mode:
Diffstat (limited to 'six/info_file_parser.go')
-rw-r--r--six/info_file_parser.go279
1 files changed, 279 insertions, 0 deletions
diff --git a/six/info_file_parser.go b/six/info_file_parser.go
new file mode 100644
index 0000000..cf98280
--- /dev/null
+++ b/six/info_file_parser.go
@@ -0,0 +1,279 @@
1// SPDX-License-Identifier: GPL-2.0-only
2// Copyright (C) 2020 Michael Crute <mike@crute.us>. All rights reserved.
3//
4// Use of this source code is governed by a license that can be found in the
5// LICENSE file.
6
7package six
8
9import (
10 "fmt"
11 "io/ioutil"
12 "net"
13 "net/http"
14 "regexp"
15 "strings"
16)
17
18const (
19 roaUrl = "https://www.seattleix.net/rs/rpki_roas/%d.txt" // ASN
20 pfxUrl = "https://www.seattleix.net/rs/irr_prefixes/%d.v%d.txt" // ASN, IP Version
21 asnUrl = "https://www.seattleix.net/rs/irr_asns/%d.v%d.txt" // ASN, IP Version
22 asSetUrl = "https://www.seattleix.net/rs/irr_as-set_prefixes/%d.v%d.txt" // ASN, IP Version
23 rsPrefixUrl = "https://www.seattleix.net/rs/rs%d.%d.v%d/%d:1:%d:v%d.txt" // RS#, MTU, IP Version, ASN, MTU, IP Version
24 rsErrUrl = "https://www.seattleix.net/rs/rs%d.%d.v%d/%d:1:%d:v%d.%serr.txt" // RS#, MTU, IP Version, ASN, MTU, IP Version, x for xerr
25)
26
27// Date, ASN, MTU, IP Version, Net Name, Network, Source, AS Path (may have {}), Error Message
28// See tests for line examples
29var errorLineRegex = regexp.MustCompile(`(?P<datetime>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) <[^>]+> (?P<asn>\d+):1:(?P<mtu>\d+):v(?P<ip_version>[46]):(?P<net_name>[^:]+): (?P<target>(?:[0-9\.\/]+|[0-9a-f\/:]+)) from (?P<source>(?:[0-9\.]+|[0-9a-f:]+?)):? (?:(?:bgp_path)?\(path (?P<path>[\d {}]+)\):?)? ?(?P<error>.*)`)
30
31func fetchParseRSErrors(rsNum, asn int, forTransit bool) *Errors {
32 transit := ""
33 if forTransit {
34 transit = "x"
35 }
36
37 // Not everyone will have every file and parseErrorFiles ignores nil so
38 // just ignore errors here
39 v4, _ := fetchFile(rsErrUrl, rsNum, 1500, IPv4, asn, 1500, IPv4, transit)
40 v6, _ := fetchFile(rsErrUrl, rsNum, 1500, IPv6, asn, 1500, IPv6, transit)
41 v4j, _ := fetchFile(rsErrUrl, rsNum, 9000, IPv4, asn, 9000, IPv4, transit)
42 v6j, _ := fetchFile(rsErrUrl, rsNum, 9000, IPv6, asn, 9000, IPv6, transit)
43
44 return parseErrorFiles(v4, v6, v4j, v6j)
45}
46
47func fetchParseRoutes(rsNum, asn int, ipv IPVersion, forJumbo bool) (*RouteFile, error) {
48 mtu := 1500
49 if forJumbo {
50 mtu = 9000
51 }
52
53 f, err := fetchFile(rsPrefixUrl, rsNum, mtu, ipv, asn, mtu, ipv)
54 if err != nil {
55 return nil, err
56 }
57
58 return parseRoutesFile(f), nil
59}
60
61func fetchParseASSet(asn int, ipv IPVersion) (*ASSet, error) {
62 f, err := fetchFile(asnUrl, asn, ipv)
63 if err != nil {
64 return nil, err
65 }
66 return parseASSetFile(f), nil
67}
68
69func fetchParsePrefixList(asn int, ipv IPVersion, forAsSet bool) (*PrefixSet, error) {
70 url := pfxUrl
71 if forAsSet {
72 url = asSetUrl
73 }
74
75 f, err := fetchFile(url, asn, ipv)
76 if err != nil {
77 return nil, err
78 }
79 return parsePrefixSetFile(f), nil
80}
81
82func fetchParseROAFile(asn int) (*ROASet, error) {
83 f, err := fetchFile(roaUrl, asn)
84 if err != nil {
85 return nil, err
86 }
87 return parseROAFile(f), nil
88}
89
90func fetchFile(urlTpl string, args ...interface{}) ([]string, error) {
91 r, err := http.Get(fmt.Sprintf(urlTpl, args...))
92 if err != nil {
93 return nil, err
94 }
95 defer r.Body.Close()
96
97 b, err := ioutil.ReadAll(r.Body)
98 if err != nil {
99 return nil, err
100 }
101
102 return strings.Split(string(b), "\n"), nil
103}
104
105func parseROAFile(lines []string) *ROASet {
106 rs := &ROASet{
107 ROAS: []ROA{},
108 }
109
110 for _, line := range lines {
111 if strings.HasPrefix(line, "Timestamp:") {
112 rs.Timestamp = *mustParseLongTime(line[11:])
113 continue
114 } else if strings.HasPrefix(line, "Trust Anchor") {
115 continue
116 } else if strings.TrimSpace(line) == "" {
117 continue
118 } else {
119 lp := strings.Split(line, "|")
120 rs.ROAS = append(rs.ROAS, ROA{
121 TrustAnchor: strings.TrimSpace(lp[0]),
122 ASN: mustParseInt(lp[1]),
123 MaxLength: mustParseInt(lp[2]),
124 Prefix: *parseIPNetFromCIDR(lp[3]),
125 })
126 }
127 }
128
129 return rs
130}
131
132func parsePrefixSetFile(lines []string) *PrefixSet {
133 ps := &PrefixSet{
134 Prefixes: []net.IPNet{},
135 }
136
137 for _, line := range lines {
138 if strings.HasPrefix(line, "Timestamp:") {
139 ps.Timestamp = *mustParseLongTime(line[11:])
140 continue
141 } else if strings.HasPrefix(line, "as-set:") {
142 ps.ASSet = &strings.Split(line, " ")[1]
143 continue
144 } else if strings.TrimSpace(line) == "" {
145 continue
146 } else {
147 // Data formatting error on SIX side
148 if strings.HasSuffix(line, ",") {
149 line = line[:len(line)-1]
150 }
151 ps.Prefixes = append(ps.Prefixes, *parseIPNetFromCIDR(line))
152 }
153 }
154
155 return ps
156}
157
158func parseASSetFile(lines []string) *ASSet {
159 as := &ASSet{
160 ASNumbers: []int{},
161 }
162
163 for _, line := range lines {
164 if strings.HasPrefix(line, "Timestamp:") {
165 as.Timestamp = *mustParseLongTime(line[11:])
166 continue
167 } else if strings.HasPrefix(line, "as-set:") {
168 as.Name = strings.Split(line, " ")[1]
169 continue
170 } else if strings.TrimSpace(line) == "" {
171 continue
172 } else {
173 as.ASNumbers = append(as.ASNumbers, mustParseInt(line))
174 }
175 }
176
177 return as
178}
179
180func parseErrorFiles(v4, v6, v4j, v6j []string) *Errors {
181 e := &Errors{}
182
183 if v4 != nil {
184 parseErrorFile(v4, e, IPv4, false)
185 }
186 if v6 != nil {
187 parseErrorFile(v6, e, IPv6, false)
188 }
189 if v4j != nil {
190 parseErrorFile(v4j, e, IPv4, true)
191 }
192 if v6j != nil {
193 parseErrorFile(v6j, e, IPv6, true)
194 }
195
196 return e
197}
198
199func getTarget(e *Errors, ipv IPVersion, jumbo bool) *ErrorRecords {
200 if ipv == IPv4 && jumbo {
201 if e.IPv4Jumbo == nil {
202 e.IPv4Jumbo = NewErrorRecords()
203 }
204 return e.IPv4Jumbo
205 } else if ipv == IPv6 && jumbo {
206 if e.IPv6Jumbo == nil {
207 e.IPv6Jumbo = NewErrorRecords()
208 }
209 return e.IPv6Jumbo
210 } else if ipv == IPv4 {
211 if e.IPv4 == nil {
212 e.IPv4 = NewErrorRecords()
213 }
214 return e.IPv4
215 } else if ipv == IPv6 {
216 if e.IPv6 == nil {
217 e.IPv6 = NewErrorRecords()
218 }
219 return e.IPv6
220 } else {
221 // This should not be possible
222 panic("Could not find target")
223 }
224}
225
226func parseErrorFile(lines []string, e *Errors, ipv IPVersion, jumbo bool) {
227 t := getTarget(e, ipv, jumbo)
228
229 for _, line := range lines {
230 l := parseErrorLine(line)
231 if l == nil {
232 // Skip blank lines, which sometimes happen
233 if strings.TrimSpace(line) == "" {
234 continue
235 } else {
236 t.UnparsableLines = append(t.UnparsableLines, line)
237 }
238 } else {
239 t.Records = append(t.Records, *l)
240 }
241 }
242}
243
244func parseErrorLine(line string) *ErrorRecord {
245 res := errorLineRegex.FindAllStringSubmatch(line, -1)
246 if len(res) == 0 {
247 return nil
248 } else {
249 row := res[0][1:]
250 return &ErrorRecord{
251 Timestamp: *mustParseLongTimeNoZone(row[0]),
252 ASN: mustParseInt(row[1]),
253 MTU: mustParseInt(row[2]),
254 Version: IPVersion(mustParseInt(row[3])),
255 NetworkName: row[4],
256 Network: *parseIPNetFromCIDR(row[5]),
257 Router: *parseIP(row[6]),
258 Path: parseASPath(row[7]),
259 Message: row[8],
260 }
261 }
262}
263
264func parseRoutesFile(lines []string) *RouteFile {
265 ps := &RouteFile{
266 Lines: []string{},
267 }
268 for _, line := range lines {
269 if strings.HasPrefix(line, "Timestamp:") {
270 ps.Timestamp = *mustParseLongTime(line[11:])
271 continue
272 } else if strings.TrimSpace(line) == "" {
273 continue
274 } else {
275 ps.Lines = append(ps.Lines, line)
276 }
277 }
278 return ps
279}
© 2015 Mike Crute