From 651151bce6be01a5cef0e8eb400e0a028b93d887 Mon Sep 17 00:00:00 2001 From: Mike Crute Date: Thu, 13 Feb 2020 06:02:52 +0000 Subject: Initial import --- LICENSE | 278 ++++++++++++++++++++++++++++++++++++++++++ Makefile | 8 ++ README.rst | 57 +++++++++ cmd/six_monitor/main.go | 250 ++++++++++++++++++++++++++++++++++++++ go.mod | 10 ++ go.sum | 108 +++++++++++++++++ six/feed_parser.go | 159 ++++++++++++++++++++++++ six/helpers.go | 112 +++++++++++++++++ six/info_file_parser.go | 279 +++++++++++++++++++++++++++++++++++++++++++ six/info_file_parser_test.go | 256 +++++++++++++++++++++++++++++++++++++++ six/participant.go | 224 ++++++++++++++++++++++++++++++++++ six/participant_builder.go | 113 ++++++++++++++++++ 12 files changed, 1854 insertions(+) create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.rst create mode 100644 cmd/six_monitor/main.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 six/feed_parser.go create mode 100644 six/helpers.go create mode 100644 six/info_file_parser.go create mode 100644 six/info_file_parser_test.go create mode 100644 six/participant.go create mode 100644 six/participant_builder.go diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7d5393a --- /dev/null +++ b/LICENSE @@ -0,0 +1,278 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5bd7d42 --- /dev/null +++ b/Makefile @@ -0,0 +1,8 @@ +SRC_FILES = $(shell find . -name '*.go') + +six_status_exporter: test $(SRC_FILES) + go build -o $@ cmd/six_monitor/main.go + +.PHONY: test +test: + go test ./... diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..ff27969 --- /dev/null +++ b/README.rst @@ -0,0 +1,57 @@ +========================================= +Seattle Internet Exchange Status Reporter +========================================= + +This code is licensed under the GPL version 2.0 only. The code is maintained at +`code.crute.me `_ and +mirrored to `GitHub `_. + +This is a server and library for parsing and reporting on the status of a +participant at the `Seattle Internet Exchange `_. The +server provides a Prometheus endpoint reporting the stats of a participant's +ASN on the exchange. The stats are gathered from the `participant data`_ +exported by the exchange operators. + +The library can be used to process the SIX participants feed as well as to +fetch and parse additional data that is exported by the SIX route servers. It's +mainly useful for generating reports in the case of errors or building +monitoring infrastructure around SIX data. + +Installing +========== +The default ``make`` target will create the correct binary. You can also build +using regular ``go build``, look at the ``Makefile`` for more details. + +:: + + make + ./six_status_exporter YOUR-ASN + +By default the server will run on port ``9119`` and export a Prometheus metrics +endpoint at ``/metrics``. + +Library +======= +The main models are located in `six/participant.go`_ and contain the important +details about the library. To generate the models from the SIX CSV consume the +``ParseSIXCSV`` and ``FetchParseSIXCSV`` which take a ``io.Reader`` and no +arguments, respectively. Those functions are defined in `six/feed_parser.go`_. + +Contributing +============ +The authors welcome and appreciate contributions. To contribute please open a +pull request on the GitHub page linked above, or email a patch in ``git am`` +format to the author. To have the best contribution experience, please: + +* Don't break backwards compatibility of public interfaces +* Write tests for your new feature/bug fix (run ``make test``) +* Ensure that existing tests pass +* Update the readme/documentation, if necessary +* Ensure your code follows ``go fmt`` standards + +All code is reviewed before acceptance and changes may be requested to better +follow the conventions of the existing API. + +.. _six/participant.go: https://code.crute.me/pomonaconsulting/six_monitoring/tree/six/participant.go +.. _six/feed_parser.go: https://code.crute.me/pomonaconsulting/six_monitoring/tree/six/feed_parser.go +.. _participant data: https://www.seattleix.net/participants/> diff --git a/cmd/six_monitor/main.go b/cmd/six_monitor/main.go new file mode 100644 index 0000000..cd41d8f --- /dev/null +++ b/cmd/six_monitor/main.go @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2020 Michael Crute . All rights reserved. +// +// Use of this source code is governed by a license that can be found in the +// LICENSE file. + +package main + +import ( + "flag" + "fmt" + "log" + "net/http" + _ "net/http/pprof" + "os" + "strconv" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/common/version" + + "code.crute.me/pomonaconsulting/six_monitoring/six" +) + +const ( + namespace = "sixstatus" + exporter = "six_status_exporter" +) + +var ( + up = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "up"), + "Was the SIX query successful?", + nil, nil, + ) + rsPrefixCount = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "route_server", "prefix_count"), + "Number of prefixes advertised to a route server.", + []string{"ipversion", "mtu", "server"}, nil, + ) + rsErrorCount = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "route_server", "error_count"), + "Number of errors reported by a route server.", + []string{"ipversion", "mtu", "server"}, nil, + ) + rsTransitErrorCount = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "route_server", "transit_error_count"), + "Number of transit errors reported by a route server.", + []string{"ipversion", "mtu", "server"}, nil, + ) + irrPrefixCount = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "irr", "prefix_count"), + "Number of prefixes resolved from IRR records.", + []string{"ipversion"}, nil, + ) + irrASNCount = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "irr", "asn_count"), + "Number of ASNs resolved from IRR records.", + []string{"ipversion"}, nil, + ) + irrAsSetPrefixCount = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "irr", "as_set_prefix_count"), + "Number of prefixes in the AS-SET resolved from IRR records.", + []string{"ipversion"}, nil, + ) + pdbPrefixCount = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "peeringdb", "prefix_count"), + "Number of prefixes configured in PeeringDB.", + []string{"ipversion"}, nil, + ) + roaCount = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "roa_count"), + "Number of resolved ROA records.", + nil, nil, + ) +) + +type SIXCollector struct { + ASN int +} + +var _ prometheus.Collector = (*SIXCollector)(nil) + +func (c *SIXCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- up + ch <- rsPrefixCount + ch <- rsErrorCount + ch <- rsTransitErrorCount + ch <- irrPrefixCount + ch <- irrASNCount + ch <- irrAsSetPrefixCount + ch <- pdbPrefixCount + ch <- roaCount +} + +func routeServerMetrics(rs *six.RouteServer, ch chan<- prometheus.Metric) { + ch <- prometheus.MustNewConstMetric( + rsPrefixCount, prometheus.GaugeValue, float64(rs.IPv4.Prefixes), + "4", "1500", string(strconv.Itoa(rs.Number)), + ) + ch <- prometheus.MustNewConstMetric( + rsErrorCount, prometheus.GaugeValue, float64(rs.IPv4.Errors), + "4", "1500", string(strconv.Itoa(rs.Number)), + ) + ch <- prometheus.MustNewConstMetric( + rsTransitErrorCount, prometheus.GaugeValue, float64(rs.IPv4.TransitErrors), + "4", "1500", string(strconv.Itoa(rs.Number)), + ) + ch <- prometheus.MustNewConstMetric( + rsPrefixCount, prometheus.GaugeValue, float64(rs.IPv6.Prefixes), + "6", "1500", string(strconv.Itoa(rs.Number)), + ) + ch <- prometheus.MustNewConstMetric( + rsErrorCount, prometheus.GaugeValue, float64(rs.IPv6.Errors), + "6", "1500", string(strconv.Itoa(rs.Number)), + ) + ch <- prometheus.MustNewConstMetric( + rsTransitErrorCount, prometheus.GaugeValue, float64(rs.IPv6.TransitErrors), + "6", "1500", string(strconv.Itoa(rs.Number)), + ) + ch <- prometheus.MustNewConstMetric( + rsPrefixCount, prometheus.GaugeValue, float64(rs.IPv4Jumbo.Prefixes), + "4", "9000", string(strconv.Itoa(rs.Number)), + ) + ch <- prometheus.MustNewConstMetric( + rsErrorCount, prometheus.GaugeValue, float64(rs.IPv4Jumbo.Errors), + "4", "9000", string(strconv.Itoa(rs.Number)), + ) + ch <- prometheus.MustNewConstMetric( + rsTransitErrorCount, prometheus.GaugeValue, float64(rs.IPv4Jumbo.TransitErrors), + "4", "9000", string(strconv.Itoa(rs.Number)), + ) + ch <- prometheus.MustNewConstMetric( + rsPrefixCount, prometheus.GaugeValue, float64(rs.IPv6Jumbo.Prefixes), + "6", "9000", string(strconv.Itoa(rs.Number)), + ) + ch <- prometheus.MustNewConstMetric( + rsErrorCount, prometheus.GaugeValue, float64(rs.IPv6Jumbo.Errors), + "6", "9000", string(strconv.Itoa(rs.Number)), + ) + ch <- prometheus.MustNewConstMetric( + rsTransitErrorCount, prometheus.GaugeValue, float64(rs.IPv6Jumbo.TransitErrors), + "6", "9000", string(strconv.Itoa(rs.Number)), + ) +} + +func (c *SIXCollector) Collect(ch chan<- prometheus.Metric) { + sp, err := six.FetchParseSIXCSV() + if err != nil { + log.Printf("error: Fetching and parsing CSV: %s", err) + ch <- prometheus.MustNewConstMetric(up, prometheus.GaugeValue, 0) + return + } + + ps, ok := sp.GetParticipantByASN(c.ASN) + if !ok { + log.Printf("error: No participant for ASN: %d", c.ASN) + ch <- prometheus.MustNewConstMetric(up, prometheus.GaugeValue, 0) + return + } + + p := ps[0] + + ch <- prometheus.MustNewConstMetric(up, prometheus.GaugeValue, 1) + + // ipversion, mtu, server + + if p.RouteServer2 != nil { + routeServerMetrics(p.RouteServer2, ch) + } + + if p.RouteServer3 != nil { + routeServerMetrics(p.RouteServer3, ch) + } + + ch <- prometheus.MustNewConstMetric( + irrPrefixCount, prometheus.GaugeValue, float64(p.IRRv4.PrefixCount), + "4", + ) + ch <- prometheus.MustNewConstMetric( + irrASNCount, prometheus.GaugeValue, float64(p.IRRv4.ASNCount), + "4", + ) + ch <- prometheus.MustNewConstMetric( + irrAsSetPrefixCount, prometheus.GaugeValue, float64(p.IRRv4.ASSetCount), + "4", + ) + ch <- prometheus.MustNewConstMetric( + pdbPrefixCount, prometheus.GaugeValue, float64(p.PeeringDBPrefixCountv4), + "4", + ) + ch <- prometheus.MustNewConstMetric( + irrPrefixCount, prometheus.GaugeValue, float64(p.IRRv6.PrefixCount), + "6", + ) + ch <- prometheus.MustNewConstMetric( + irrASNCount, prometheus.GaugeValue, float64(p.IRRv6.ASNCount), + "6", + ) + ch <- prometheus.MustNewConstMetric( + irrAsSetPrefixCount, prometheus.GaugeValue, float64(p.IRRv6.ASSetCount), + "6", + ) + ch <- prometheus.MustNewConstMetric( + pdbPrefixCount, prometheus.GaugeValue, float64(p.PeeringDBPrefixCountv6), + "6", + ) + + ch <- prometheus.MustNewConstMetric(roaCount, prometheus.GaugeValue, float64(p.ROACount)) +} + +func main() { + var ( + showVersion = flag.Bool("version", false, "Print version information.") + listenAddress = flag.String("web.listen-address", ":9119", "Address to listen on for web interface and telemetry.") + ) + flag.Parse() + + if len(os.Args) != 2 { + fmt.Fprintf(os.Stdout, "usage: %s \n", os.Args[0]) + os.Exit(1) + } + + asn, err := strconv.Atoi(os.Args[1]) + if err != nil { + fmt.Fprintln(os.Stdout, "invalid ASN, must be numeric") + fmt.Fprintf(os.Stdout, "usage: %s \n", os.Args[0]) + os.Exit(1) + } + + if *showVersion { + fmt.Fprintln(os.Stdout, version.Print(exporter)) + os.Exit(0) + } + log.Println("Starting", exporter, version.Info()) + log.Println("Build context", version.BuildContext()) + + prometheus.MustRegister(version.NewCollector(exporter), &SIXCollector{asn}) + + log.Println("Starting Server: ", *listenAddress) + http.Handle("/metrics", promhttp.Handler()) + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(` + SIX Status Exporter +

SIX Status Exporter

+

Metrics

+ `)) + }) + log.Fatal(http.ListenAndServe(*listenAddress, nil)) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2614df2 --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module code.crute.me/pomonaconsulting/six_monitoring + +go 1.13 + +require ( + github.com/prometheus-community/bind_exporter v0.3.0 + github.com/prometheus/client_golang v1.4.1 + github.com/prometheus/common v0.9.1 + github.com/stretchr/testify v1.4.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..5e4f5a3 --- /dev/null +++ b/go.sum @@ -0,0 +1,108 @@ +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus-community/bind_exporter v0.3.0 h1:DsyFrvT607dHUBbHOh+h8jqNeMGP6+WKZRlZNWNMmpg= +github.com/prometheus-community/bind_exporter v0.3.0/go.mod h1:VNrkjpy+wDFpgW+A/8+CQgMM0fISAfpSdhnat/K6+Ic= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= +github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200103143344-a1369afcdac7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/six/feed_parser.go b/six/feed_parser.go new file mode 100644 index 0000000..fb839f7 --- /dev/null +++ b/six/feed_parser.go @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2020 Michael Crute . All rights reserved. +// +// Use of this source code is governed by a license that can be found in the +// LICENSE file. + +package six + +import ( + "encoding/csv" + "fmt" + "io" + "net/http" + "os" +) + +const sixFeedUrl = "https://www.seattleix.net/autogen/participants.csv" + +// Error indicating that a line in the SIX CSV feed could not be parsed. +// Contains the actual error as well as the line number on which the error +// occurred. +type SIXParticipantParseError struct { + Line int + Err error +} + +// Return the error string for the error +func (e *SIXParticipantParseError) Error() string { + return fmt.Sprintf("record on line %d: %s", e.Line, e.Err) +} + +// Returns the wrapped error +func (e *SIXParticipantParseError) Unwrap() error { + return e.Err +} + +// Parser for the SIX participant CSV feed. +type SIXParser struct { + Records []*SIXParticipant + Errors []error + asnIndex map[int][]*SIXParticipant + headers []string +} + +// Builds a new SIXParser +func NewSIXParser() *SIXParser { + return &SIXParser{ + Records: []*SIXParticipant{}, + Errors: []error{}, + asnIndex: map[int][]*SIXParticipant{}, + headers: nil, + } +} + +// After parsing the feed will return a list of participant structs for the +// passed ASN as well as a boolean indicating if the participant exists. +// Participants may have several connections to the exchange and each +// connection is a different participant. +func (p *SIXParser) GetParticipantByASN(asn int) ([]*SIXParticipant, bool) { + r, ok := p.asnIndex[asn] + return r, ok +} + +// Adds a participant to the SIXParticipant struct +func (p *SIXParser) AddParticipant(sp *SIXParticipant) { + p.Records = append(p.Records, sp) + + if _, ok := p.asnIndex[sp.ASN]; !ok { + p.asnIndex[sp.ASN] = []*SIXParticipant{sp} + } else { + p.asnIndex[sp.ASN] = append(p.asnIndex[sp.ASN], sp) + } +} + +func (p *SIXParser) sliceToHeaderMap(s []string) map[string]string { + r := map[string]string{} + for i, k := range p.headers { + r[k] = s[i] + } + return r +} + +func (p *SIXParser) addRow(rn int, r []string) { + defer func() { + if e := recover(); e != nil { + p.Errors = append(p.Errors, &SIXParticipantParseError{ + Line: rn, + Err: e.(error), + }) + } + }() + p.AddParticipant(NewSIXParticipantFromData(p.sliceToHeaderMap(r))) +} + +// ASSUMPTION: The correct row size will not change +// +// Participants that aren't using the route servers tend to have inconsistent +// row data. The head and tail of the row are fine and contain the organization +// and connection information but the middle area that contains route server +// stats generally has 7-9 too few columns. This function will graft the head +// and tail onto an appropriately sized row. +func (p *SIXParser) fixupRow(r []string) []string { + k := make([]string, 49) + copy(k[:18], r[:18]) + copy(k[42:], r[len(r)-7:]) + return k +} + +// Parse an io.Reader containing SIX participant data in CSV format and collect +// the results into the parser for later querying. +func (p *SIXParser) Parse(fr io.Reader) { + rn := 0 + cr := csv.NewReader(fr) + for { + rn++ + row, err := cr.Read() + if err == io.EOF { + return + } + if err != nil { + row = p.fixupRow(row) + p.Errors = append(p.Errors, err) + } + if p.headers == nil { + p.headers = row + continue + } + p.addRow(rn, row) + } +} + +// Create a new SIXParser and parse the CSV file pointed to by the filename. +func ParseSIXCSV(filename string) (*SIXParser, error) { + fp, err := os.Open(filename) + if err != nil { + return nil, err + } + defer fp.Close() + + sp := NewSIXParser() + sp.Parse(fp) + + return sp, nil +} + +// Create a new SIXParser and parse the contents of the SIX participant file +// locate on the SIX http server. +func FetchParseSIXCSV() (*SIXParser, error) { + res, err := http.Get(sixFeedUrl) + if err != nil { + return nil, err + } + defer res.Body.Close() + + sp := NewSIXParser() + sp.Parse(res.Body) + + return sp, nil +} diff --git a/six/helpers.go b/six/helpers.go new file mode 100644 index 0000000..694a79e --- /dev/null +++ b/six/helpers.go @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2020 Michael Crute . All rights reserved. +// +// Use of this source code is governed by a license that can be found in the +// LICENSE file. + +package six + +import ( + "net" + "strconv" + "strings" + "time" +) + +func mustParseInt(a string) int { + a = strings.TrimSpace(a) + + if a == "" { + return 0 + } + + i, err := strconv.Atoi(a) + if err != nil { + panic(err) + } + + return i +} + +func parseYesNo(b string) bool { + return strings.ToLower(strings.TrimSpace(b)) == "yes" +} + +func parseIP(i string) *net.IP { + o := net.ParseIP(i) + return &o +} + +func parseIPNetFromCIDR(i string) *net.IPNet { + _, ipnet, _ := net.ParseCIDR(strings.TrimSpace(i)) + return ipnet +} + +func mustParseTime(t string) *time.Time { + t = strings.TrimSpace(t) + + if t == "" { + return nil + } + + i, err := time.Parse("2006-01-02", t) + if err != nil { + panic(err) + } + + return &i +} + +func mustParseLongTime(t string) *time.Time { + t = strings.TrimSpace(t) + + if t == "" { + return nil + } + + i, err := time.Parse("2006-01-02 15:04:05 MST", t) + if err != nil { + panic(err) + } + + return &i +} + +func mustParseLongTimeNoZone(t string) *time.Time { + t = strings.TrimSpace(t) + + if t == "" { + return nil + } + + i, err := time.Parse("2006-01-02 15:04:05", t) + if err != nil { + panic(err) + } + + return &i +} + +func allEmpty(d []string) bool { + for _, v := range d { + if strings.TrimSpace(v) != "" { + return false + } + } + return true +} + +func parseASPath(p string) []int { + out := []int{} + + for _, i := range strings.Split(p, " ") { + ii, err := strconv.Atoi(strings.TrimSpace(i)) + // Some AS paths contain { and } which are not valid and need to be discard + if err != nil { + continue + } + out = append(out, ii) + } + + return out +} diff --git a/six/info_file_parser.go b/six/info_file_parser.go new file mode 100644 index 0000000..cf98280 --- /dev/null +++ b/six/info_file_parser.go @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2020 Michael Crute . All rights reserved. +// +// Use of this source code is governed by a license that can be found in the +// LICENSE file. + +package six + +import ( + "fmt" + "io/ioutil" + "net" + "net/http" + "regexp" + "strings" +) + +const ( + roaUrl = "https://www.seattleix.net/rs/rpki_roas/%d.txt" // ASN + pfxUrl = "https://www.seattleix.net/rs/irr_prefixes/%d.v%d.txt" // ASN, IP Version + asnUrl = "https://www.seattleix.net/rs/irr_asns/%d.v%d.txt" // ASN, IP Version + asSetUrl = "https://www.seattleix.net/rs/irr_as-set_prefixes/%d.v%d.txt" // ASN, IP Version + rsPrefixUrl = "https://www.seattleix.net/rs/rs%d.%d.v%d/%d:1:%d:v%d.txt" // RS#, MTU, IP Version, ASN, MTU, IP Version + rsErrUrl = "https://www.seattleix.net/rs/rs%d.%d.v%d/%d:1:%d:v%d.%serr.txt" // RS#, MTU, IP Version, ASN, MTU, IP Version, x for xerr +) + +// Date, ASN, MTU, IP Version, Net Name, Network, Source, AS Path (may have {}), Error Message +// See tests for line examples +var errorLineRegex = regexp.MustCompile(`(?P\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) <[^>]+> (?P\d+):1:(?P\d+):v(?P[46]):(?P[^:]+): (?P(?:[0-9\.\/]+|[0-9a-f\/:]+)) from (?P(?:[0-9\.]+|[0-9a-f:]+?)):? (?:(?:bgp_path)?\(path (?P[\d {}]+)\):?)? ?(?P.*)`) + +func fetchParseRSErrors(rsNum, asn int, forTransit bool) *Errors { + transit := "" + if forTransit { + transit = "x" + } + + // Not everyone will have every file and parseErrorFiles ignores nil so + // just ignore errors here + v4, _ := fetchFile(rsErrUrl, rsNum, 1500, IPv4, asn, 1500, IPv4, transit) + v6, _ := fetchFile(rsErrUrl, rsNum, 1500, IPv6, asn, 1500, IPv6, transit) + v4j, _ := fetchFile(rsErrUrl, rsNum, 9000, IPv4, asn, 9000, IPv4, transit) + v6j, _ := fetchFile(rsErrUrl, rsNum, 9000, IPv6, asn, 9000, IPv6, transit) + + return parseErrorFiles(v4, v6, v4j, v6j) +} + +func fetchParseRoutes(rsNum, asn int, ipv IPVersion, forJumbo bool) (*RouteFile, error) { + mtu := 1500 + if forJumbo { + mtu = 9000 + } + + f, err := fetchFile(rsPrefixUrl, rsNum, mtu, ipv, asn, mtu, ipv) + if err != nil { + return nil, err + } + + return parseRoutesFile(f), nil +} + +func fetchParseASSet(asn int, ipv IPVersion) (*ASSet, error) { + f, err := fetchFile(asnUrl, asn, ipv) + if err != nil { + return nil, err + } + return parseASSetFile(f), nil +} + +func fetchParsePrefixList(asn int, ipv IPVersion, forAsSet bool) (*PrefixSet, error) { + url := pfxUrl + if forAsSet { + url = asSetUrl + } + + f, err := fetchFile(url, asn, ipv) + if err != nil { + return nil, err + } + return parsePrefixSetFile(f), nil +} + +func fetchParseROAFile(asn int) (*ROASet, error) { + f, err := fetchFile(roaUrl, asn) + if err != nil { + return nil, err + } + return parseROAFile(f), nil +} + +func fetchFile(urlTpl string, args ...interface{}) ([]string, error) { + r, err := http.Get(fmt.Sprintf(urlTpl, args...)) + if err != nil { + return nil, err + } + defer r.Body.Close() + + b, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, err + } + + return strings.Split(string(b), "\n"), nil +} + +func parseROAFile(lines []string) *ROASet { + rs := &ROASet{ + ROAS: []ROA{}, + } + + for _, line := range lines { + if strings.HasPrefix(line, "Timestamp:") { + rs.Timestamp = *mustParseLongTime(line[11:]) + continue + } else if strings.HasPrefix(line, "Trust Anchor") { + continue + } else if strings.TrimSpace(line) == "" { + continue + } else { + lp := strings.Split(line, "|") + rs.ROAS = append(rs.ROAS, ROA{ + TrustAnchor: strings.TrimSpace(lp[0]), + ASN: mustParseInt(lp[1]), + MaxLength: mustParseInt(lp[2]), + Prefix: *parseIPNetFromCIDR(lp[3]), + }) + } + } + + return rs +} + +func parsePrefixSetFile(lines []string) *PrefixSet { + ps := &PrefixSet{ + Prefixes: []net.IPNet{}, + } + + for _, line := range lines { + if strings.HasPrefix(line, "Timestamp:") { + ps.Timestamp = *mustParseLongTime(line[11:]) + continue + } else if strings.HasPrefix(line, "as-set:") { + ps.ASSet = &strings.Split(line, " ")[1] + continue + } else if strings.TrimSpace(line) == "" { + continue + } else { + // Data formatting error on SIX side + if strings.HasSuffix(line, ",") { + line = line[:len(line)-1] + } + ps.Prefixes = append(ps.Prefixes, *parseIPNetFromCIDR(line)) + } + } + + return ps +} + +func parseASSetFile(lines []string) *ASSet { + as := &ASSet{ + ASNumbers: []int{}, + } + + for _, line := range lines { + if strings.HasPrefix(line, "Timestamp:") { + as.Timestamp = *mustParseLongTime(line[11:]) + continue + } else if strings.HasPrefix(line, "as-set:") { + as.Name = strings.Split(line, " ")[1] + continue + } else if strings.TrimSpace(line) == "" { + continue + } else { + as.ASNumbers = append(as.ASNumbers, mustParseInt(line)) + } + } + + return as +} + +func parseErrorFiles(v4, v6, v4j, v6j []string) *Errors { + e := &Errors{} + + if v4 != nil { + parseErrorFile(v4, e, IPv4, false) + } + if v6 != nil { + parseErrorFile(v6, e, IPv6, false) + } + if v4j != nil { + parseErrorFile(v4j, e, IPv4, true) + } + if v6j != nil { + parseErrorFile(v6j, e, IPv6, true) + } + + return e +} + +func getTarget(e *Errors, ipv IPVersion, jumbo bool) *ErrorRecords { + if ipv == IPv4 && jumbo { + if e.IPv4Jumbo == nil { + e.IPv4Jumbo = NewErrorRecords() + } + return e.IPv4Jumbo + } else if ipv == IPv6 && jumbo { + if e.IPv6Jumbo == nil { + e.IPv6Jumbo = NewErrorRecords() + } + return e.IPv6Jumbo + } else if ipv == IPv4 { + if e.IPv4 == nil { + e.IPv4 = NewErrorRecords() + } + return e.IPv4 + } else if ipv == IPv6 { + if e.IPv6 == nil { + e.IPv6 = NewErrorRecords() + } + return e.IPv6 + } else { + // This should not be possible + panic("Could not find target") + } +} + +func parseErrorFile(lines []string, e *Errors, ipv IPVersion, jumbo bool) { + t := getTarget(e, ipv, jumbo) + + for _, line := range lines { + l := parseErrorLine(line) + if l == nil { + // Skip blank lines, which sometimes happen + if strings.TrimSpace(line) == "" { + continue + } else { + t.UnparsableLines = append(t.UnparsableLines, line) + } + } else { + t.Records = append(t.Records, *l) + } + } +} + +func parseErrorLine(line string) *ErrorRecord { + res := errorLineRegex.FindAllStringSubmatch(line, -1) + if len(res) == 0 { + return nil + } else { + row := res[0][1:] + return &ErrorRecord{ + Timestamp: *mustParseLongTimeNoZone(row[0]), + ASN: mustParseInt(row[1]), + MTU: mustParseInt(row[2]), + Version: IPVersion(mustParseInt(row[3])), + NetworkName: row[4], + Network: *parseIPNetFromCIDR(row[5]), + Router: *parseIP(row[6]), + Path: parseASPath(row[7]), + Message: row[8], + } + } +} + +func parseRoutesFile(lines []string) *RouteFile { + ps := &RouteFile{ + Lines: []string{}, + } + for _, line := range lines { + if strings.HasPrefix(line, "Timestamp:") { + ps.Timestamp = *mustParseLongTime(line[11:]) + continue + } else if strings.TrimSpace(line) == "" { + continue + } else { + ps.Lines = append(ps.Lines, line) + } + } + return ps +} diff --git a/six/info_file_parser_test.go b/six/info_file_parser_test.go new file mode 100644 index 0000000..9994e6f --- /dev/null +++ b/six/info_file_parser_test.go @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2020 Michael Crute . All rights reserved. +// +// Use of this source code is governed by a license that can be found in the +// LICENSE file. + +package six + +import ( + "net" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +var testDate = time.Date(2020, 02, 12, 19, 43, 23, 0, time.UTC) + +const roaTest = ` +Timestamp: 2020-02-12 19:43:23 UTC +Trust Anchor | ASN | Max Length | Prefix + APNIC | 64496 | 24 | 192.0.2.0/24 + RIPE | 64496 | 24 | 198.51.100.0/24 + RIPE | 64496 | 48 | 2001:db8:3a80::/48 + APNIC | 64496 | 48 | 2001:db8:3a81::/48 +` + +func parseCidr(c string) net.IPNet { + _, n, _ := net.ParseCIDR(c) + return *n +} + +func TestROAParsing(t *testing.T) { + rs := parseROAFile(strings.Split(roaTest, "\n")) + + matrix := []struct { + TrustAnchor string + ASN int + MaxLength int + Prefix net.IPNet + }{ + {"APNIC", 64496, 24, parseCidr("192.0.2.0/24")}, + {"RIPE", 64496, 24, parseCidr("198.51.100.0/24")}, + {"RIPE", 64496, 48, parseCidr("2001:db8:3a80::/48")}, + {"APNIC", 64496, 48, parseCidr("2001:db8:3a81::/48")}, + } + + for i, tv := range matrix { + assert.EqualValues(t, tv, rs.ROAS[i]) + } + + assert.Equal(t, rs.Timestamp, testDate) +} + +const prefixListv4 = ` +Timestamp: 2020-02-12 19:43:23 UTC +192.0.2.0/24 +198.51.100.0/24 +` + +func TestPrefixSetv4Parsing(t *testing.T) { + r := parsePrefixSetFile(strings.Split(prefixListv4, "\n")) + + assert.Equal(t, r.Timestamp, testDate) + assert.Equal(t, r.Prefixes[0], parseCidr("192.0.2.0/24")) + assert.Equal(t, r.Prefixes[1], parseCidr("198.51.100.0/24")) +} + +const prefixListv6 = ` +Timestamp: 2020-02-12 19:43:23 UTC +2001:db8:3a80::/48 +2001:db8:3a81::/48 +` + +func TestPrefixSetv6Parsing(t *testing.T) { + r := parsePrefixSetFile(strings.Split(prefixListv6, "\n")) + + assert.Equal(t, r.Timestamp, testDate) + assert.Equal(t, r.Prefixes[0], parseCidr("2001:db8:3a80::/48")) + assert.Equal(t, r.Prefixes[1], parseCidr("2001:db8:3a81::/48")) +} + +const asPrefixListv4 = ` +as-set: AS-TEST +Timestamp: 2020-02-12 19:43:23 UTC +192.0.2.0/24 +198.51.100.0/24 +198.51.100.0/24, +` + +func TestPrefixSetASNv4Parsing(t *testing.T) { + r := parsePrefixSetFile(strings.Split(asPrefixListv4, "\n")) + + assert.Equal(t, r.Timestamp, testDate) + assert.Equal(t, *r.ASSet, "AS-TEST") + assert.Equal(t, r.Prefixes[0], parseCidr("192.0.2.0/24")) + assert.Equal(t, r.Prefixes[1], parseCidr("198.51.100.0/24")) + assert.Equal(t, r.Prefixes[2], parseCidr("198.51.100.0/24")) +} + +const asPrefixListv6 = ` +as-set: AS-TEST +Timestamp: 2020-02-12 19:43:23 UTC +2001:db8:3a80::/48 +2001:db8:3a81::/48 +2001:db8:3a81::/48, +` + +func TestPrefixSetASNv6Parsing(t *testing.T) { + r := parsePrefixSetFile(strings.Split(asPrefixListv6, "\n")) + + assert.Equal(t, r.Timestamp, testDate) + assert.Equal(t, *r.ASSet, "AS-TEST") + assert.Equal(t, r.Prefixes[0], parseCidr("2001:db8:3a80::/48")) + assert.Equal(t, r.Prefixes[1], parseCidr("2001:db8:3a81::/48")) + assert.Equal(t, r.Prefixes[2], parseCidr("2001:db8:3a81::/48")) +} + +const asSetFile = ` +as-set: AS-TEST +Timestamp: 2020-02-12 19:43:23 UTC +64496 +64497 +64498 +` + +func TestASSetParsing(t *testing.T) { + r := parseASSetFile(strings.Split(asSetFile, "\n")) + + assert.Equal(t, r.Timestamp, testDate) + assert.Equal(t, r.Name, "AS-TEST") + assert.Equal(t, r.ASNumbers[0], 64496) + assert.Equal(t, r.ASNumbers[1], 64497) + assert.Equal(t, r.ASNumbers[2], 64498) +} + +const errorFilev4 = ` +2020-02-12 19:43:23 64496:1:1500:v4:Testnet: 192.0.2.0/24 from 192.0.2.1 (path 64496 64497 64497 64497 64499): AS64499 not member of IRR as-set object! Dropping. +2020-02-12 19:43:23 64496:1:1500:v4:Testnet: 198.51.100.0/24 from 192.0.2.1: bgp_path(path 64496 64497 64498 64498 64499) matches pdb_never_via_route_servers_ASNs! Dropping. +2020-02-12 19:43:23 64496:1:1500:v4:Testnet: 192.0.2.0/26 from 192.0.2.1 (path 64496): RPKI ROA_INVALID! Dropping. +` + +const errorFilev6 = ` +2020-02-12 19:43:23 64496:1:1500:v6:Testnet: 2001:db8::/16 from 2001:db8::1: matches bogon_prefixes! Dropping. +2020-02-12 19:43:23 64496:1:1500:v6:Testnet: 2001:db8::/24 from 2001:db8::1: bgp_path(path 64496 64497 64497 64497 64497 64497 64497 64497 64497 64498) matches pdb_never_via_route_servers_ASNs! Dropping. +2020-02-12 19:43:23 64496:1:1500:v6:Testnet: 2001:db8::/32 from 2001:db8::1 (path 64496 64497 64498 64498 64499): AS64499 not member of IRR as-set object! Dropping. +` + +// Last line intentionally not parsable +const errorFilev4j = ` +2020-02-12 19:43:23 64496:1:9000:v4:Testnet: 192.0.2.0/24 from 192.0.2.1: bgp_path(path 64496 64498 { 75757 }) matches bogon_ASNs! Dropping. +2020-02-12 19:43:23 64496:1:9000:v4:Testnet: 198.51.100.0/24 from 192.0.2.1: matches bogon_prefixes! Dropping. +2 20-02-12 19:43:23 64496:1:9000:v4:Testnet: 192.0.2.0/26 from 192.0.2.1: matches bogon_prefixes! Dropping. +` + +// Last line intentionally not parsable +const errorFilev6j = ` +2020-02-12 19:43:23 64496:1:9000:v6:Testnet: 2001:db8::/32 from 2001:db8::1 (path 64496 64497 64499): no IRR route object found! Transit_dropping. +2020-02-12 19:43:23 64496:1:9000:v6:Testnet: 2001:db8::/36 from 2001:db8::1 (path 64496 64498 64499): no IRR route object found! Transit_dropping. +2020-02-12 19:43:23 64496:1:9000:v6:Testnet: 2001:db8::/48 from 2001:db8::1 (path 64496 64498 64498 64498 64499): RPKI ROA_INVALID! Transit_dropping. +2 20-02-12 19:43:23 64496:1:9000:v6:Testnet: 2001:db8::/48 from 2001:db8::1 (path 64496 64498 64498 64498 64499): RPKI ROA_INVALID! Transit_dropping. +` + +func TestParsingErrorFile(t *testing.T) { + e := parseErrorFiles( + strings.Split(errorFilev4, "\n"), + strings.Split(errorFilev6, "\n"), + strings.Split(errorFilev4j, "\n"), + strings.Split(errorFilev6j, "\n")) + + v4t := []struct { + Timestamp time.Time + ASN int + MTU int + Version IPVersion + NetworkName string + Network net.IPNet + Router net.IP + Path []int + Message string + }{ + {testDate, 64496, 1500, IPv4, "Testnet", parseCidr("192.0.2.0/24"), net.ParseIP("192.0.2.1"), []int{64496, 64497, 64497, 64497, 64499}, "AS64499 not member of IRR as-set object! Dropping."}, + {testDate, 64496, 1500, IPv4, "Testnet", parseCidr("198.51.100.0/24"), net.ParseIP("192.0.2.1"), []int{64496, 64497, 64498, 64498, 64499}, "matches pdb_never_via_route_servers_ASNs! Dropping."}, + {testDate, 64496, 1500, IPv4, "Testnet", parseCidr("192.0.2.0/26"), net.ParseIP("192.0.2.1"), []int{64496}, "RPKI ROA_INVALID! Dropping."}, + } + + for i, tv := range v4t { + assert.EqualValues(t, tv, e.IPv4.Records[i]) + } + + v6t := []struct { + Timestamp time.Time + ASN int + MTU int + Version IPVersion + NetworkName string + Network net.IPNet + Router net.IP + Path []int + Message string + }{ + {testDate, 64496, 1500, IPv6, "Testnet", parseCidr("2001:db8::/16"), net.ParseIP("2001:db8::1"), []int{}, "matches bogon_prefixes! Dropping."}, + {testDate, 64496, 1500, IPv6, "Testnet", parseCidr("2001:db8::/24"), net.ParseIP("2001:db8::1"), []int{64496, 64497, 64497, 64497, 64497, 64497, 64497, 64497, 64497, 64498}, "matches pdb_never_via_route_servers_ASNs! Dropping."}, + {testDate, 64496, 1500, IPv6, "Testnet", parseCidr("2001:db8::/32"), net.ParseIP("2001:db8::1"), []int{64496, 64497, 64498, 64498, 64499}, "AS64499 not member of IRR as-set object! Dropping."}, + } + + for i, tv := range v6t { + assert.EqualValues(t, tv, e.IPv6.Records[i]) + } + + v4jt := []struct { + Timestamp time.Time + ASN int + MTU int + Version IPVersion + NetworkName string + Network net.IPNet + Router net.IP + Path []int + Message string + }{ + {testDate, 64496, 9000, IPv4, "Testnet", parseCidr("192.0.2.0/24"), net.ParseIP("192.0.2.1"), []int{64496, 64498, 75757}, "matches bogon_ASNs! Dropping."}, + {testDate, 64496, 9000, IPv4, "Testnet", parseCidr("198.51.100.0/24"), net.ParseIP("192.0.2.1"), []int{}, "matches bogon_prefixes! Dropping."}, + } + + for i, tv := range v4jt { + assert.EqualValues(t, tv, e.IPv4Jumbo.Records[i]) + } + + badLinev4j := "2 20-02-12 19:43:23 64496:1:9000:v4:Testnet: 192.0.2.0/26 from 192.0.2.1: matches bogon_prefixes! Dropping." + assert.Equal(t, badLinev4j, e.IPv4Jumbo.UnparsableLines[0]) + + v6jt := []struct { + Timestamp time.Time + ASN int + MTU int + Version IPVersion + NetworkName string + Network net.IPNet + Router net.IP + Path []int + Message string + }{ + {testDate, 64496, 9000, IPv6, "Testnet", parseCidr("2001:db8::/32"), net.ParseIP("2001:db8::1"), []int{64496, 64497, 64499}, "no IRR route object found! Transit_dropping."}, + {testDate, 64496, 9000, IPv6, "Testnet", parseCidr("2001:db8::/36"), net.ParseIP("2001:db8::1"), []int{64496, 64498, 64499}, "no IRR route object found! Transit_dropping."}, + {testDate, 64496, 9000, IPv6, "Testnet", parseCidr("2001:db8::/48"), net.ParseIP("2001:db8::1"), []int{64496, 64498, 64498, 64498, 64499}, "RPKI ROA_INVALID! Transit_dropping."}, + } + + for i, tv := range v6jt { + assert.EqualValues(t, tv, e.IPv6Jumbo.Records[i]) + } + + badLinev6j := "2 20-02-12 19:43:23 64496:1:9000:v6:Testnet: 2001:db8::/48 from 2001:db8::1 (path 64496 64498 64498 64498 64499): RPKI ROA_INVALID! Transit_dropping." + assert.Equal(t, badLinev6j, e.IPv6Jumbo.UnparsableLines[0]) +} diff --git a/six/participant.go b/six/participant.go new file mode 100644 index 0000000..e8150bb --- /dev/null +++ b/six/participant.go @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2020 Michael Crute . All rights reserved. +// +// Use of this source code is governed by a license that can be found in the +// LICENSE file. + +package six + +import ( + "net" + "time" +) + +// IP protocol version number +type IPVersion int + +const ( + IPv6 IPVersion = 6 + IPv4 = 4 +) + +// IPv4 and IPv6 union type, contains a full network for IPv4 and IPv6 +// addresses +type Addresses struct { + IPv4 *net.IPNet + IPv6 *net.IPNet +} + +// IRR statistical data. Contains the counts of various IRR data for a +// participant. +type IRRData struct { + PrefixCount int + ASNCount int + ASSetCount int +} + +// Set of prefixes, optionally with an AS-SET name that a participant has +// registered with the exchange. +type PrefixSet struct { + Timestamp time.Time + ASSet *string + Prefixes []net.IPNet +} + +// Set of AS numbers with an AS-SET that the participant has registered with +// the exchange. +type ASSet struct { + Timestamp time.Time + Name string + ASNumbers []int +} + +// Set of ROA record summaries that have been parsed from the raw ROA data for +// a participant. +type ROASet struct { + Timestamp time.Time + ROAS []ROA +} + +// Individual ROA summary +type ROA struct { + TrustAnchor string + ASN int + MaxLength int + Prefix net.IPNet +} + +// A SIX participant is a record of an organization's connection to the Seattle +// Internet Exchange. It contains information about their connect, the prefixes +// and ASes that they have registered with the exchange, and their usage of the +// route server. +// +// Not all data exists for all participants, and the data that does exist isn't +// always in a nice clean format. The parsers attempt to make sense of whatever +// is available but some data may be missing or inconsistent. +// +// The default contains summary data for the participant and more detailed data +// can be fetched with the methods attached to this struct as well as to the +// RouteServer structs, if the participant is using the route server. +type SIXParticipant struct { + Organization string + URL string + ASN int + Speed int // Connection speed in Mbit/s + Switch string // Connected Switch or Location + Contact string // Contact Email + Comment string + IsConnected bool + IsVoter bool + Update *time.Time + Options []string // Currently: IPv6 and MTU9k + PeeringPolicy string + ROACount int + PeeringDBPrefixCountv4 int + PeeringDBPrefixCountv6 int + Addresses Addresses + JumboAddresses *Addresses + IRRv4 IRRData + IRRv6 IRRData + RouteServer2 *RouteServer + RouteServer3 *RouteServer +} + +// Get the set of prefixes that the participant has registered. +// +// This does an HTTP fetch to get the detailed file. +func (p *SIXParticipant) GetPrefixes(ip IPVersion) (*PrefixSet, error) { + return fetchParsePrefixList(p.ASN, ip, false) +} + +// Get the prefix to AS associations that the participant has registered. +// +// This does an HTTP fetch to get the detailed file. +func (p *SIXParticipant) GetASPrefixes(ip IPVersion) (*PrefixSet, error) { + return fetchParsePrefixList(p.ASN, ip, true) +} + +// Get the AS sets that the participant has registered +// +// This does an HTTP fetch to get the detailed file. +func (p *SIXParticipant) GetASSet(ip IPVersion) (*ASSet, error) { + return fetchParseASSet(p.ASN, ip) +} + +// Get the ROA set and more detailed records for the ROAs that the participant +// has registered. This does not fetch the raw ROAs, just the summary that the +// exchange has computed. +// +// This does an HTTP fetch to get the detailed file. +func (p *SIXParticipant) GetROASet() (*ROASet, error) { + return fetchParseROAFile(p.ASN) +} + +// List of lines from the file containing route data dumped from the route +// server +type RouteFile struct { + Timestamp time.Time + Lines []string +} + +// Lists of errors that the route server has exported. These are parsed for +// more details if possible, but if that fails the attached ErrorRecords object +// contains a list of un-parsable lines for consumption. +type Errors struct { + IPv4 *ErrorRecords + IPv6 *ErrorRecords + IPv4Jumbo *ErrorRecords + IPv6Jumbo *ErrorRecords +} + +// List of error records for route server errors. Also contains a list of lines +// that could not be parsed by the parser. +type ErrorRecords struct { + Records []ErrorRecord + UnparsableLines []string +} + +// Create a new ErrorRecords struct +func NewErrorRecords() *ErrorRecords { + return &ErrorRecords{ + Records: []ErrorRecord{}, + UnparsableLines: []string{}, + } +} + +// Contains the parsed data for a route server error. +type ErrorRecord struct { + Timestamp time.Time + ASN int + MTU int + Version IPVersion + NetworkName string + Network net.IPNet + Router net.IP + Path []int + Message string +} + +// Data about the participant's connection to a route server. If the +// participant is making use of the route servers this will always contain some +// basic statistics about their connection. More detailed information can be +// obtained by calling the methods attached to this struct. +type RouteServer struct { + Number int + IPv4 RouteServerStats + IPv6 RouteServerStats + IPv4Jumbo RouteServerStats + IPv6Jumbo RouteServerStats + asn int +} + +// Route server statistics. Contains the counts of various types of route +// server entries. +type RouteServerStats struct { + Prefixes int + Errors int + TransitErrors int +} + +// Gets the list of routes being advertised by the participant. Note that this +// file is the raw lines from the route server and no attempt has been made to +// parse these lines. The lines are in BIRD format. +// +// This does an HTTP fetch to get the detailed file. +func (s *RouteServer) GetRoutes(ip IPVersion, jumbo bool) (*RouteFile, error) { + return fetchParseRoutes(s.Number, s.asn, ip, jumbo) +} + +// Gets the errors returned by the route server for all IP protocols and all +// VLANs to which the participant is connected. If the participant is using +// multiple route servers, this data is scoped to the current route server. +// +// This does an HTTP fetch to get the detailed file. +func (s *RouteServer) GetErrors() (*Errors, error) { + return fetchParseRSErrors(s.Number, s.asn, false), nil +} + +// Get a list of transit errors from the route server. Otherwise return value +// and behavior is identical to GetErrors. +// +// This does an HTTP fetch to get the detailed file. +func (s *RouteServer) GetTransitErrors() (*Errors, error) { + return fetchParseRSErrors(s.Number, s.asn, true), nil +} diff --git a/six/participant_builder.go b/six/participant_builder.go new file mode 100644 index 0000000..f18a6d1 --- /dev/null +++ b/six/participant_builder.go @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2020 Michael Crute . All rights reserved. +// +// Use of this source code is governed by a license that can be found in the +// LICENSE file. + +package six + +import ( + "fmt" + "strings" +) + +// Create a new SIXParticipant struct from a map of data that was parsed from +// the participant CSV file. This assumes the column headers from that CSV file +// so it will not work with the other data formats avaiable. +// +// This uses the CSV file because it's both the most rich source of data and +// the easiest to parse. +func NewSIXParticipantFromData(d map[string]string) *SIXParticipant { + r := &SIXParticipant{ + Organization: d["Organization"], + URL: d["URL"], + ASN: mustParseInt(d["ASN"]), + Speed: mustParseInt(d["Speed"]), + Switch: d["Switch"], + Contact: d["Contact"], + Comment: d["Comment"], + IsConnected: parseYesNo(d["Conn?"]), + IsVoter: parseYesNo(d["Voter?"]), + Update: mustParseTime(d["Update"]), + Options: strings.Split(d["Options"], " "), + PeeringPolicy: d["Policy"], + ROACount: mustParseInt(d["rpki:roa"]), + PeeringDBPrefixCountv4: mustParseInt(d["pdb:v4"]), + PeeringDBPrefixCountv6: mustParseInt(d["pdb:v6"]), + Addresses: Addresses{ + IPv4: parseIPNetFromCIDR(d["IPv4"]), + IPv6: parseIPNetFromCIDR(d["IPv6"]), + }, + IRRv4: IRRData{ + PrefixCount: mustParseInt(d["irr:p4"]), + ASNCount: mustParseInt(d["irr:a4"]), + ASSetCount: mustParseInt(d["irr:ap4"]), + }, + IRRv6: IRRData{ + PrefixCount: mustParseInt(d["irr:p6"]), + ASNCount: mustParseInt(d["irr:a6"]), + ASSetCount: mustParseInt(d["irr:ap6"]), + }, + RouteServer2: getRSData(2, d), + RouteServer3: getRSData(3, d), + } + + // Not all participants use the MTU9k VLAN + ja4 := parseIPNetFromCIDR(d["Jumbo IPv4"]) + ja6 := parseIPNetFromCIDR(d["Jumbo IPv6"]) + if ja4 != nil && ja6 != nil { + r.JumboAddresses = &Addresses{IPv4: ja4, IPv6: ja6} + } + + return r +} + +func getRSData(server int, d map[string]string) *RouteServer { + // Extract all the data and determine if it's all empty strings, if so then + // the participant isn't using the route server. If any data is not empty + // then they are. Do integer conversion afterward to avoid ambiguity about + // zero vs empty string. + pd := []string{ + d[fmt.Sprintf("rs%d:v4", server)], + d[fmt.Sprintf("err%d:v4", server)], + d[fmt.Sprintf("xerr%d:v4", server)], + d[fmt.Sprintf("rs%d:v6", server)], + d[fmt.Sprintf("err%d:v6", server)], + d[fmt.Sprintf("xerr%d:v6", server)], + d[fmt.Sprintf("rs%d:v4j", server)], + d[fmt.Sprintf("err%d:v4j", server)], + d[fmt.Sprintf("xerr%d:v4j", server)], + d[fmt.Sprintf("rs%d:v6j", server)], + d[fmt.Sprintf("err%d:v6j", server)], + d[fmt.Sprintf("xerr%d:v6j", server)], + } + + if allEmpty(pd) { + return nil + } + + return &RouteServer{ + Number: server, + asn: mustParseInt(d["ASN"]), + IPv4: RouteServerStats{ + Prefixes: mustParseInt(pd[0]), + Errors: mustParseInt(pd[1]), + TransitErrors: mustParseInt(pd[2]), + }, + IPv6: RouteServerStats{ + Prefixes: mustParseInt(pd[3]), + Errors: mustParseInt(pd[4]), + TransitErrors: mustParseInt(pd[5]), + }, + IPv4Jumbo: RouteServerStats{ + Prefixes: mustParseInt(pd[6]), + Errors: mustParseInt(pd[7]), + TransitErrors: mustParseInt(pd[8]), + }, + IPv6Jumbo: RouteServerStats{ + Prefixes: mustParseInt(pd[9]), + Errors: mustParseInt(pd[10]), + TransitErrors: mustParseInt(pd[11]), + }, + } +} -- cgit v1.2.3