#!/usr/bin/env python
# Copyright 2012-2020 Cumulus Networks, Inc
#

"""
Compare kernel neighbor/route tables with HW neighbor/route tables.
Also compare HW tables for consistency.  Print any comparison differences
to stdout.  Exit(1) if any differences detected.

Both IPv4 and IPv6 entries are considered.

Kernel tables:

    ip -o neigh                       // IPv4/6 neighbor table
    ip -o route show table all        // IPv4/6 route table

Broadcom HW tables:

    XGS family:

    IPv4              IPv6
    ----              ----
    l3 l3table        l3 ip6host      // host tables
    l3 defip          l3 ip6route     // route tables
    l3 multipath      l3 multipath    // multipath table
    l3 egress         l3 egress       // egress table
    l3 intf           l3 intf         // INTF table

    DNX family:

    diag dbal te 0          // host tables
    diag dbal te 3          // host tables
    diag dbal te 8          // route tables
    diag alloc ecmp         // multipath table
    diag alloc fec          // egress table
    diag pp outlif_info_ll  // INTF table

Broadcom HW tables are extracted with /usr/lib/cumulus/bcmcmd.  There is no sync between switchd
and /usr/lib/cumulus/bcmcmd cmds, nor the kernel, so script should be run on a quiet system.

Mellanox HW tables
    IPv4              IPv6
    ----              ----
    neighbor          neighbor        // host tables
    uc_route          uc_route        // route tables
    multipath         multipath       // multipath table
                                      // egress table
    interface         interface       // INTF table

Mellanox HW tables are extracted using the mlx.py module.

"""

from cStringIO import StringIO
from ipaddr import IPv6Address, IPNetwork, IPv4Network, IPv6Network, AddressValueError
from pprint import pformat, pprint
from subprocess import Popen, PIPE, check_output, CalledProcessError
import argparse
import json
import os
import re
import sys
import cumulus.platforms
import socket
import struct
import yaml
wfi_enabled = False

def rfc5952(dest):
    """
    per RFC 5952, convert full IPv6 address to text representation

    e.g.
        2001:0db8::0001 should be rendered as 2001:db8::1
        2001:db8:0:0:0:0:2:1 should be rendered to 2001:db8::2:1
        2001:db8:0000:1:1:1:1:1 should be rendered as 2001:db8:0:1:1:1:1:1
        2001:db8:0:0:1:0:0:1 should be rendered as 2001:db8::1:0:0:1
    """
    if '/' in dest:
        prfx, prfxlen = dest.split('/')
        ipv6 = IPv6Address(prfx)
        prefix = str(ipv6)+"/"+prfxlen
        return prefix
    else:
       ipv6 = IPv6Address(dest)
       return str(ipv6)


def sort_for_humans(list_foo):
    """
    Sort list_foo in the way that humans expect
    http://nedbatchelder.com/blog/200712/human_sorting.html
    """
    convert = lambda text: int(text) if text.isdigit() else text
    alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
    list_foo.sort(key=alphanum_key)
    return list_foo

def format_kernel_json_output(dict_foo):
    dict_foo = json.dumps(dict_foo)
    dict_foo = dict_foo.replace("null", "None")
    dict_foo = eval(dict_foo)
    return dict_foo

def has_recursive_nexthop(nexthops):
    """
    Return True if any of the nexthops are flagged as recursive
    """
    for nexthop in nexthops:
        if nexthop.get('recursive'):
            return True
    return False

def is_service_running(service):
    """
    Return True if service is running fine
    """
    status=False
    cmd = ["systemctl", "status", service]
    try:
        output = check_output(cmd)
        if "running" in output:
            status=True
    except CalledProcessError:
        status=False

    return status

class Kernel(object):

    def __init__(self):
        self.ifaces = {}             # iface       -> (ifindex, mac)
        self.ipv6_local_routes = {}  # v6 prefix/ln-> [(iface, mac), ...]
        self.neigh_macs = {}         # mac         -> [iface, ...]
        self.neighs = {}             # prefix,iface-> (iface, mac)
        self.onlinks = {}
        self.vxlan_onlinks = {}
        self.routes = {}             # prefix/len  -> [(iface, mac), ...]
        self.vrfs = {}               # key is vrf name, value is table id
        self.neigh_exceptions = set()
        self.route_exceptions = set()
        self.l3nexthops = {}
        self.l2nexthops = {}
        self.l3nhgs = {}
        self.l2nhgs = {}
        self.vxlan_to_vlan_map = {}
        self.remote_macs = {}
        self.remotemac_exceptions = set()
        self.l3_nhg_supported_protos = ["zebra", "bgp"]
        self.blackhole_ids = []
        self.offload_flags = {}
        self.offload_flag_mode = 0
    def iface_is_up(self, iface):
        # if an interface is link down or we are ignoring link status for
        # hardware route population (switchd config option)
        if self.ifaces.get(iface):
            return self.ifaces[iface][2] or self.ifaces[iface][3]
        return False

    def keep_down_nexthops(self):
        filename = "/cumulus/switchd/config/route/delete_dead_routes"

        if os.path.exists(filename):
            with open(filename, "r") as sfs:
                value = sfs.read().strip().lower()
                return value != "true"

        return False

    def ignore_neigh_for_forwaring(self, iface, prefix):
        if ':' in prefix:
            return not self.ifaces[iface][5]
        return not self.ifaces[iface][4]
    #
    # This function converts a v6 prefix
    # with '.' format to ipv6 format
    # ::ffff:10.10.3.12 is converted to
    # :ffff:a0a:30c
    def convert_ipv6_vxlan_nexthop(self, prefix):
        octets = prefix.split(':')
        octets = list(filter(None, octets))
        numbers = list(map(int,octets[-1].split('.')))
        octets[-1] = '{:x}{:02x}:{:x}{:02x}'.format(*numbers)
        new_prefix = ":"+":".join(octets)
        return new_prefix

    def collect_data(self):

        def extract_table(line):
            table = 0

            re_table = re.search('table (\S+)', line)
            if re_table:
                table = re_table.group(1)

                if table == 'local':
                    table = 0
                # If this table is a vrf then we need to
                # get the kernel's table ID for this vrf
                elif table in self.vrfs:
                    table = self.vrfs.get(table)
                elif table.isdigit():
                    table = int(table)
                else:
                    table = re.findall(r'\d+', table)[0]
                    table = int(table)

            # Always ignore IPv6 multicast for non-default vrf too
            self.route_exceptions.update(set([(table, 'ff00::/8')]))

            return table

        #
        # build interface to ether mac addr dict
        #
        ipaddr = re.compile("""
         (\d+):\s+                                # ifindex
         ([^ ]+):                                 # iface
         \s([^ ]+)                                # flags
         .*link/(loopback|ether)                  # cruft
         \s([^ ]+)                                # mac addr
        """, re.VERBOSE)

        keep_down_nexthops = self.keep_down_nexthops()

        for line in Popen(["/sbin/ip", "-d", "-o", "link"], shell=False, stdout=PIPE).stdout:
            parsed = ipaddr.findall(line)

            if parsed:
                ifindex, iface, flags, cruft, mac = parsed[0]
                iface = iface.split('@')[0]
                up = 'LOWER_UP' in flags

                v4_on = True
                v6_on = True
                v4_fwd_path = "/proc/sys/net/ipv4/conf/{0}/forwarding".format(iface)
                v6_fwd_path = "/proc/sys/net/ipv6/conf/{0}/forwarding".format(iface)
                if os.path.isfile(v4_fwd_path):
                    for l in Popen(["/bin/cat", v4_fwd_path], shell=False, stdout=PIPE).stdout:
                        v4_on = int(l)
                if os.path.isfile(v6_fwd_path):
                    for l in Popen(["/bin/cat", v6_fwd_path], shell=False, stdout=PIPE).stdout:
                        v6_on = int(l)

                self.ifaces[iface] = (int(ifindex), mac, up, keep_down_nexthops, v4_on, v6_on)

        # Examples of what we are parsing:
        # 103: vrf102: <NOARP,MASTER,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 1000\    link/ether fe:ed:e6:cb:a4:c9 brd ff:ff:ff:ff:ff:ff promiscuity 0 \    vrf table 102 addrgenmode eui64
        # 104: blue: <NOARP,MASTER,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 1000\    link/ether fe:a5:4a:44:b8:fe brd ff:ff:ff:ff:ff:ff promiscuity 0 \    vrf table 103 addrgenmode eui64
        #
        # The first ()s stores the table name...vrf102, blue, etc
        # The second ()s stores the table id...102, 103, etc
        vrftable = re.compile("^\d+:\s+(\S+):.*vrf table (\d+)")

        for line in Popen(["/sbin/ip", "-d", "-o", "link", "show", "type", "vrf"], shell=False, stdout=PIPE).stdout:
            parsed = vrftable.findall(line)

            if parsed:
                (vrf_name, vrf_id) = parsed[0]
                vrf_id = int(vrf_id)
                self.vrfs[vrf_name] = vrf_id

        # In vxlan evpn, the peer vteps are programmed in kernel
        # in both v4 and v6 formath. Keep these nexthops to uniquely
        # identify vxlan routes in kernel. Also the v6 nexthops are
        # programmed with '.' format which causes the script
        # to fail. Add Such prefixes to neigh exceptions.
        set_vxlan_nexthop = set()    # keep all nexthops to match kernel routes
        set_vxlan_v6_nexthop = set() # keep v6 only routes to add in exceptions
        vni_nexthops = {}
        if is_service_running("frr"):
            cmd = ["/usr/bin/vtysh", "-c", "show evpn next-hops vni all json"]
            try:
                output = check_output(cmd).strip()
            except CalledProcessError:
                output = None
            if output is not None:
                try:
                    vni_nexthops = json.loads(output)
                except ValueError:
                    print "ERROR: The json from '%s' is invalid\n%s" %\
                          (' '.join(cmd), output)
            for l3vni, vni_data in vni_nexthops.iteritems():
                for nexthop, nexthop_data in vni_data.iteritems():
                    if '.' in nexthop:
                        set_vxlan_nexthop.add(nexthop)
                    if ':' in nexthop:
                        set_vxlan_v6_nexthop.add(rfc5952(nexthop))

        #
        # build kernel neighbor and mac dict
        #
        # fe80::2e0:ecff:fe36:2e86 dev eth0 lladdr 00:e0:ec:36:2e:86 router REACHABLE
        neigh_with_mac = re.compile("""
         ([^ ]+)                                     # prefix
         \sdev                                       # 'dev'
         \s([^ ]+)                                   # iface
         \slladdr                                    # 'lladdr'
         \s([^ ]+)                                   # mac addr
        """, re.VERBOSE)

        # fe80::202:ff:fe00:1b dev swp1.4  router INCOMPLETE, or
        # fe80::202:ff:fe00:1b dev swp1.4  FAILED
        neigh_without_mac = re.compile("""
         ([^ ]+)                                     # prefix
         \sdev                                       # 'dev'
         \s([^ ]+)                                   # iface
         \s+([^ ]+)                                  # router or nud
        """, re.VERBOSE)

        for line in Popen(["/sbin/ip", "-o", "neigh"], shell=False, stdout=PIPE).stdout:
            parsed_with_mac = neigh_with_mac.findall(line)
            parsed_without_mac = neigh_without_mac.findall(line)

            if parsed_with_mac:
                prefix, iface, mac = parsed_with_mac[0]
                if ignore_iface(iface):
                    self.neigh_exceptions.update(set([prefix]))
                elif self.ignore_neigh_for_forwaring(iface, prefix):
                    self.neigh_exceptions.update(set([prefix]))
                self.neighs[(prefix, iface)] = (iface, mac)
                self.neigh_macs.setdefault(mac, []).append(iface)
                if '.' in prefix and ':' in prefix:
                    self.neigh_exceptions.update(set([prefix]))
                    new_prefix = rfc5952(prefix)
                    self.neighs[(new_prefix, iface)] = (iface, mac)

            elif parsed_without_mac:
                prefix, iface, nud = parsed_without_mac[0]
                if 'FAILED' in nud:
                    self.neigh_exceptions.update(set([prefix]))
                    # let it fall through or we'll have other problems
                if ignore_iface(iface):
                    self.neigh_exceptions.update(set([prefix]))
                elif self.ignore_neigh_for_forwaring(iface, prefix):
                    self.neigh_exceptions.update(set([prefix]))
                self.neighs[(prefix, iface)] = (iface, None)
                if '.' in prefix and ':' in prefix:
                    self.neigh_exceptions.update(set([prefix]))
                    new_prefix = rfc5952(prefix)
                    self.neighs[(new_prefix, iface)] = (iface, mac)

        #Exclude the VRR/anycast addresses
        setipv4 = set()
        setipv6 = set()
        myipv4 = os.popen('ip addr show | grep -A 2 "\-v0" | grep "\<inet\>" | awk \'{ print $2 }\' | awk -F "/" \'{ print $1 }\'').read().strip()
        myipv6 = os.popen('ip addr show | grep -A 4 "\-v0" | grep "\<inet6\>" | awk \'{ print $2 }\' | awk -F "/" \'{ print $1 }\'').read().strip()
        setipv4 = myipv4.split('\n')
        setipv6 = myipv6.split('\n')
        self.neigh_exceptions.update(setipv4)
        self.neigh_exceptions.update(setipv6)

        # build nexthops , l2/l3nhg dict
        cmd = ["ip", "-j", "nexthop", "show"]
        output = check_output(cmd).strip()
        output = json.loads(output)
        nhrawoutput = format_kernel_json_output(output) if output else []
        for entry in nhrawoutput:
            ids = str(entry['id'])
            if "gateway" in entry.keys():
                gateway = entry.get("gateway", None)
                dev = entry.get("dev", None)
                if entry.get("protocol", None) in self.l3_nhg_supported_protos:
                    self.l3nexthops[ids] = (gateway, dev)
                else:
                    self.l2nexthops[ids]= gateway
            elif "group" in entry.keys():
                groups = entry.get("group", None)
                grouplist = []
                for grpdict in groups:
                    grouplist.append(str(grpdict['id']))
                if entry.get("protocol", None) in self.l3_nhg_supported_protos:
                    self.l3nhgs[ids] = grouplist
                else:
                    self.l2nhgs[ids] = grouplist
            elif "blackhole" in entry.keys():
                self.blackhole_ids.append(ids)

        #build vlan-vni mapping dict 
        cmd = ["net", "show", "evpn", "access-vlan", "json"]
        output = check_output(cmd).strip()
        output = json.loads(output)
        vxlandict = format_kernel_json_output(output) if output else []
        for entry in vxlandict:
            vxlanIf = entry.get("vxlanIf", None)
            if vxlanIf:
                self.vxlan_to_vlan_map[vxlanIf] = {}
                self.vxlan_to_vlan_map[vxlanIf]["vni"] = entry.get("vni", None)
                self.vxlan_to_vlan_map[vxlanIf]["vlan"] = entry.get("vlan", None)

        # build fdb remote mac dict
        cmd = ["bridge", "-j", "fdb", "show"]
        output = check_output(cmd).strip()
        output = json.loads(output)
        fdbdict = format_kernel_json_output(output) if output else []
        for entry in fdbdict:
            if entry["state"] == "permanent":
                continue
            mac = entry.get("mac", None)
            ifname = entry.get("ifname", None)
            vlan = entry.get("vlan", None)
            if entry["state"] == "static" and not self.iface_is_up(entry["ifname"]):
                self.remotemac_exceptions.update(set([(mac, vlan)]))
            if "dst" in entry.keys():
                if self.vxlan_to_vlan_map.get(ifname, None):
                    vlan = self.vxlan_to_vlan_map[ifname]["vlan"]
                dst_ip = entry["dst"]
                self.remote_macs[(mac, vlan)] = [dst_ip]
            elif "nhid" in entry.keys():
                if self.vxlan_to_vlan_map.get(ifname, None):
                    vlan = self.vxlan_to_vlan_map[ifname]["vlan"]
                nhid = str(entry["nhid"])
                grouplist = self.l2nhgs.get(nhid, None)
                nexthoplist = []
                if grouplist:
                    for ids in grouplist:
                        nexthop = self.l2nexthops[ids]
                        nexthoplist.append(nexthop)
                self.remote_macs[(mac, vlan)] = nexthoplist

        #
        # build kernel route dict
        #
        route = re.compile("(^default |^local |^broadcast |^anycast |^blackhole |^) *([^ ]+)")
        self.missing_neighbors = []
        # Always ignore IPv6 multicast
        self.route_exceptions.update(set([(0, 'ff00::/8')]))

        for line in Popen(["/sbin/ip", "-o", "route", "show", "table", "all"],
                          shell=False, stdout=PIPE).stdout:

            begin, prefix = route.findall(line)[0]
            flags = ""
            offload_flag = re.findall("rt_offload",line)
            offload_failed_flag = re.findall("rt_offload_failed",line)
            trap_flag = re.findall("rt_trap",line)
            if offload_flag:
                flags = "offload"
            if trap_flag:
                flags = "trap"
            if offload_failed_flag:
                flags = "offload_failed"
            mgmt = re.findall("mgmt", line)
            if begin == "default ":
                prefix = "0.0.0.0/0"
                iface = re.findall('dev ([^ ]+)', line)[0]
                table = extract_table(line)
                vias = re.findall("via ([^ ]+) ", line)
                devs = re.findall("dev ([^ ]+)", line)
                add_exception = False

                for iface, via in zip(devs, vias):

                    if (via, iface) in self.neighs:
                        neigh_iface, neigh_mac = self.neighs[(via, iface)]

                        if iface == neigh_iface:
                            self.routes.setdefault((table, prefix), []).append((iface, neigh_mac, via))

                            if ignore_iface(iface):
                                add_exception = True
                    else:
                        self.missing_neighbors.append((table, iface, via))
                        self.routes.setdefault((table, prefix), []).append((iface, 'no neighbor found', via))

                #table 10000 and more belongs to PBR. Ignore these
                if table >= 10000:
                    add_exception = True

                if add_exception:
                    self.route_exceptions.update(set([(table, prefix)]))
                continue

            # table 10 us used by rdnbrd, ignore these
            elif ' table 10 ' in line:
                continue

            elif begin == "broadcast ":
                table = extract_table(line)
                self.route_exceptions.update(set([(table, prefix)]))

            elif begin == "local " or begin == "anycast ":
                iface = re.findall('dev ([^ ]+)', line)[0]
                if begin == "local " and not self.iface_is_up(iface):
                    # skip local routes that point to down interface
                    continue

                if IPNetwork(prefix).is_loopback:
                    continue

            elif begin == "blackhole ":
                table = extract_table(line)
                self.route_exceptions.update(set([(table, prefix)]))
                continue
    
            # Most lines will fall into this category
            elif begin is "":
                if prefix == "unreachable" or prefix == "none" or prefix == "multicast":
                    continue

                # MPLS label, cl-route-check does not yet support validating
                # MPLS entries so we are going to ignore this for now.
                if prefix.isdigit():
                    continue

            else:
                continue

            is_ipv6_local_route = False

            if prefix.count("/") == 0:       # missing prefix length
                if prefix.count(".") > 0:    # IPv4
                    prefix += "/32"
                else:                           # IPv6
                    is_ipv6_local_route = True  # IPv6 local route
                    prefix += "/128"

            table = extract_table(line)
            vias = re.findall("via ([^ ]+) ", line)
            devs = re.findall("dev ([^ ]+)", line)
            protos = re.findall("proto ([^ ]+)", line)
            nhid = re.findall("nhid ([^ ]+)", line)

            is_onlink = False
            if 'onlink' in line:
                is_onlink = True

            if nhid:
                nhid = nhid[0]
                is_onlink = True
                grouplist = self.l3nhgs.get(nhid, [])
                if grouplist:
                    for grp in grouplist:
                        (gateway, iface) = self.l3nexthops.get(grp, (None, None))
                        vias.append(gateway)
                        devs.append(iface)
                else:
                    (gateway, iface) = self.l3nexthops.get(nhid, (None,None))
                    vias.append(gateway)
                    devs.append(iface)

            # add nexthop paths to route
            for iface, via in zip(devs, vias):
                if iface == "lo":
                    if is_ipv6_local_route:
                        self.ipv6_local_routes.setdefault((table, prefix), []).append((iface, None, None))
                    else:
                        self.routes.setdefault((table, prefix), []).append((iface, None, None))
                    self.route_exceptions.update(set([(table, prefix)]))
                    self.routes.setdefault((table, prefix), []).append((iface, None, None))

                elif (via, iface) in self.neighs:
                    neigh_iface, neigh_mac = self.neighs[(via, iface)]

                    if iface == neigh_iface:
                        self.routes.setdefault((table, prefix), []).append((iface, neigh_mac, via))
                        if ignore_iface(iface) or not self.iface_is_up(iface):
                            self.route_exceptions.update(set([(table, prefix)]))
                else:
                    self.missing_neighbors.append((table, iface, via))
                    self.routes.setdefault((table, prefix), []).append((iface, 'no neighbor found', via))

                if via in prefix and iface != "lo":
                    self.onlinks.setdefault(prefix, []).append((iface, None))
                # to uniquely identify vxlan evpn symmetric routes
                # check if the route has onlink keyword and via is
                # one of vxlan nexthops and proto is bgp
                if is_onlink and 'bgp' in protos:
                    if via in set_vxlan_nexthop:
                        self.vxlan_onlinks.setdefault(prefix, []).append((iface, None))
                    elif ':' in via and rfc5952(via) in set_vxlan_v6_nexthop:
                        self.vxlan_onlinks.setdefault(prefix, []).append((iface, None))

            # add directly connected path
            if devs and not vias:
                iface = devs[0]
                if ignore_iface(iface) or not self.iface_is_up(iface):
                    self.route_exceptions.update(set([(table, prefix)]))
                self.routes.setdefault((table, prefix), []).append((iface, None, None))
            if wfi_enabled and not mgmt:
                if flags and not (prefix.startswith("fe80") or prefix.startswith("ff00") or prefix.startswith("127") or  prefix.startswith("0")):
                    self.offload_flags.setdefault((str(table) + " " + str(prefix) + " " + flags), [])
        if self.missing_neighbors:
            self.missing_neighbors = sorted(list(set(self.missing_neighbors)))


class Asic(object):

    def __init__(self):

        #
        # The various tables from kernel/HW are collected into dictionaries
        # with these mappings:
        #
        self.ip4_routes   = {}  # ip4 prefix/len  -> egress_ndx | mp_ndx
        self.ip6_routes   = {}  # ip6 prefix/len  -> egress_ndx | mp_ndx
        self.route_2_rif  = {}  # ip4/ip6 prefix/len -> egress_ndx
        self.route_2_ecmp = {}  # ip4/ip6 prefix/len -> mp_ndx
        self.egresses     = {}  # egress_ndx  -> (intf, mac)
        self.hosts        = {}  # prefix      -> egress_ndx
        self.hostsipv6    = {}  # v6 prefix   -> egress_ndx (split_ipv6_local only)
        self.hroutes      = {}  # prefix/len  -> [(intf, mac), ...]
        self.intfs        = {}  # intf        -> mac
        self.multipaths   = {}  # mp_ndx      -> [egress_ndx, egress_ndx, ...]
        self.remote_mac_table = {}  # (mac, vlan) -> [nexthops]
        self.l2_ecmp      = {}

        # Asic-specific exceptions
        self.egress_exceptions  = set()
        self.mp_exceptions      = set()
        self.hroutes_exceptions = set()

        # chip ID to chip name map
        self.chip_id_map = {}

        self.chip = self.get_chipname()

        # Certain switch ASICs (e.g. BCM56150 aka Hurricane2) will maintain
        # the kernel IPv6 /128 local routes in the hardware IPv6 host table
        # instead of the defip/ip6route table.  In such cases we need to diff
        # kernel IPv6 local routes against the hardware IPv6 host table instead
        # of defip/ip6route table.
        self.split_ipv6_local = False
        self.offload_flags = {}
    def get_id_2_name(self, chip_id):
        try:
            return self.chip_id_map[chip_id]
        except KeyError:
            return 'Unknown(%s)' % chip_id

    def build_route_dict(self):
        for dest, rif in self.route_2_rif.items():
            if rif in self.egresses:
                intf, mac = self.egresses[rif]
                self.hroutes[dest] = [(intf, mac)]
        for dest, ecmp_id in self.route_2_ecmp.items():
            self.hroutes[dest] = []
            if ecmp_id in self.multipaths:
                for n in self.multipaths[ecmp_id]:
                    if n in self.egresses:
                        intf, mac = self.egresses[n]
                        self.hroutes.setdefault(dest, []).append((intf, mac))


class Broadcom(Asic):

    def __init__(self):

        import bcmshell
        self.bs = bcmshell.bcmshell(keepopen=True, timeout=180)
        Asic.__init__(self)

        # the blackhole and CPU-bound interfaces will not
        # always be represented in the route or neighbor tables
        self.egress_exceptions = set(["100000", "100002"])

        kernel_neighbor_ipv6_in_host_ipv6 = ('Hurricane2',)
        if self.chip in kernel_neighbor_ipv6_in_host_ipv6:
            self.split_ipv6_local = True

        self.chip_id_map = {'56150': 'Hurricane2'}

    def get_chipname(self):
        unit_str = self.bs.run('show unit')
        unit_re = re.compile('Unit \d+ chip BCM(?P<chip_id>\d+)')
        m = unit_re.match(unit_str)

        if not m:
            raise RuntimeError('could not determine the platform from unit string:\n\n%s\n' % unit_str)

        chip_id = m.group('chip_id')
        return self.get_id_2_name(chip_id)

    def collect_data(self):
        #
        # build HW intf table from "l3 intf show"
        #
        l3intf = re.compile("""
         \d+\s+                                   # Unit
         (\d+)\s+                                 # Intf
         \d+\s+                                   # VRF
         \d+\s+                                   # Group
         \d+\s+                                   # VLAN
         ([^ ]+)\s+                               # Source Mac
         \d+\s+                                   # MTU
        """, re.VERBOSE)

        for line in self.bs.run("l3 intf show").splitlines():
            parsed = l3intf.findall(line)

            if parsed:
                intf, mac = parsed[0]
                self.intfs[int(intf)] = mac

        #
        # build HW host table from "l3 l3table show" and "l3 ip6host show"
        #
        host = re.compile("""
         \d+\s+                                   # Entry
         (\d+)\s+                                 # VRF
         ([^ ]+)\s+                               # prefix
         [^ ]+\s+                                 # Mac Address
         (\d+)                                    # INTF
        """, re.VERBOSE)

        for line in self.bs.run("l3 l3table show").splitlines():
            parsed = host.findall(line)

            if parsed:
                vrf, prefix, ndx = parsed[0]
                vrf = int(vrf)
                assert (vrf, prefix) not in self.hosts,\
                    "'l3 l3table show' (%s, %s) is already in %s" % (vrf, prefix, pformat(self.hosts))
                self.hosts[(vrf, prefix)] = ndx

        for line in self.bs.run("l3 ip6host show").splitlines():
            parsed = host.findall(line)

            if parsed:
                vrf, prefix, ndx = parsed[0]
                vrf = int(vrf)
                prefix = rfc5952(prefix)

                if self.split_ipv6_local:
                    assert (vrf, prefix) not in self.hostsipv6,\
                        "'l3 ip6host show' (%s, %s) is already in %s" % (vrf, prefix, pformat(self.hostsipv6))
                    self.hostsipv6[(vrf, prefix)] = ndx
                else:
                    assert (vrf, prefix) not in self.hosts,\
                        "'l3 ip6host show' (%s, %s) is already in %s" % (vrf, prefix, pformat(self.hosts))
                    self.hosts[(vrf, prefix)] = ndx

        #
        # build HW egress dict from "l3 egress show"
        #
        egress = re.compile("""
         (\d+)\s+                                 # Entry
         ([^ ]+)                                  # Mac addr
         \s+\d+\s+                                # Vlan
         (\d+)                                    # INTF
        """, re.VERBOSE)

        for line in self.bs.run("l3 egress show").splitlines():
            parsed = egress.findall(line)

            if parsed:
                egress_ndx, mac, intf = parsed[0]
                assert egress_ndx not in self.egresses,\
                    "'l3 egress show' %s is already in %s" % (egress_ndx, pformat(self.egresses))
                self.egresses[egress_ndx] = (int(intf), mac)

        #
        # build HW multipath dict from "l3 multipath show"
        #
        multipath = re.compile("Multipath Egress Object (\d+)")
        ref_count = re.compile("Reference count: (\d+)")
        interfaces = re.compile("(\d+)")
        mp_ndx = None

        for line in self.bs.run("l3 multipath show").splitlines():
            mparsed = multipath.findall(line)
            iparsed = interfaces.findall(line)
            rparsed = ref_count.findall(line)

            if mparsed:
                mp_ndx = mparsed[0]
            elif rparsed:
                pass
            elif iparsed:
                self.multipaths.setdefault(mp_ndx, []).extend(iparsed)

        #
        # build HW IPv4 routes dict from "l3 defip show"
        #
        route = re.compile("""
         \d+\s+                                    # '#'
         (\d+)\s+                                  # VRF
         ([^ ]+)\s+                                # Net addr prefix/len
         [^ ]+\s+                                  # Next Hop Mac addr
         (\d+)                                     # INTF
        """, re.VERBOSE)

        for line in self.bs.run("l3 defip show").splitlines():
            parsed = route.findall(line)

            if parsed:
                vrf, dest, ndx = parsed[0]
                vrf = int(vrf)

                # If not table 0 add 1000 so the HW vrf table ID matches the kernel vrf table ID
                if vrf:
                    vrf += 1000

                assert (vrf, dest) not in self.ip4_routes,\
                    "'l3 defip show' (%s, %s) is already in %s" % (vrf, dest, pformat(self.ip4_routes))
                self.ip4_routes[(vrf, dest)] = ndx
                if ndx in self.egresses:
                    self.route_2_rif[(vrf, dest)] = ndx
                elif ndx in self.multipaths:
                    self.route_2_ecmp[(vrf, dest)] = ndx

        #
        # build HW IPv6 route dict from "l3 ip6route show"
        #
        for line in self.bs.run("l3 ip6route show").splitlines():
            parsed = route.findall(line)

            if parsed:
                vrf, dest, ndx = parsed[0]
                vrf = int(vrf)

                # If not table 0 add 1000 so the HW vrf table ID matches the kernel vrf table ID
                if vrf:
                    vrf += 1000

                dest = rfc5952(dest)
                assert (vrf, dest) not in self.ip6_routes,\
                    "'l3 ip6route show' (%s, %s) is already in %s" % (vrf, dest, pformat(self.ip6_routes))
                self.ip6_routes[(vrf, dest)] = ndx
                if ndx in self.egresses:
                    self.route_2_rif[(vrf, dest)] = ndx
                elif ndx in self.multipaths:
                    self.route_2_ecmp[(vrf, dest)] = ndx

        self.bs.close()

        #
        # Combine IPv4 and IPv6 routes
        #
        self.routes = dict(self.ip4_routes.items() + self.ip6_routes.items())

        self.build_route_dict()


class BroadcomDNX(Asic):

    def __init__(self):
        import bcmshell

        self.bs = bcmshell.bcmshell(timeout=180, prompt='BCM\.[0-9]+>\s+$')
        Asic.__init__(self)

        # CPU FEC
        self.egress_exceptions = set(["4096"])

        # Supports Qumran-MX only
        self.chip_id_map = {'88370': 'Qumran-MX', '88375': 'Qumran-MX'}

    def get_chipname(self):
        unit_str = self.bs.run('show unit')

        unit_re = re.compile('Unit \d+ chip BCM(?P<chip_id>\d+)')
        m = unit_re.match(unit_str)
        if not m:
            raise RuntimeError('could not determine the platform from unit string:\n\n%s\n' % unit_str)

        chip_id = m.group('chip_id')

        return self.get_id_2_name(chip_id)

    def get_mac_from_encap(self, encap_id):
        # First 4K are reserved for L2 LIFs
        encap_id = int(encap_id[-4:], 16) + 4096

        # Sample output
        #   Local Out_LIF:0x00002006 -> Type:(null) Bank:1 Offset:3
        #   LL Encapsulation:
        #        dest_mac:00:02:00:00:00:11
        #        out_vid_valid: 1
        #        out_vid: 3002
        #        pcp_dei_valid: 0
        #        pcp_dei: 0
        #        tpid_index: 0
        #        ll_remark_profile: 0
        #        out_ac_valid: 0
        #        out_ac_lsb: 0
        #        oam_lif_set: 0
        #        outlif_profile: 0x00
        #        Next_eep: 0xffffffff
        command = "diag pp lif_show id=" + str(encap_id) + " type=out"
        for line in self.bs.run(command).splitlines():
            if "dest_mac:" in line:
                mac = line[-17:]
                return mac

        return "00:00:00:00:00:00"

    def get_mac_from_outlif(self, fec):
        # First 4K are reserved for L2 LIFs
        fec = int(fec) + 4096

        # Sample output
        #   Local Out_LIF:0x00002006 -> Type:(null) Bank:1 Offset:3
        #   LL Encapsulation:
        #        dest_mac:00:02:00:00:00:11
        #        out_vid_valid: 1
        #        out_vid: 3002
        #        pcp_dei_valid: 0
        #        pcp_dei: 0
        #        tpid_index: 0
        #        ll_remark_profile: 0
        #        out_ac_valid: 0
        #        out_ac_lsb: 0
        #        oam_lif_set: 0
        #        outlif_profile: 0x00
        #        Next_eep: 0xffffffff
        command = "diag pp lif_show id=" + str(fec) + " type=out"
        for line in self.bs.run(command).splitlines():
            if "dest_mac:" in line:
                mac = line[-17:]
                return mac

        return "00:00:00:00:00:00"

    def get_ecmp_members(self, ecmp_fec):
        fecs = []

        # Sample output
        #   ECMP get successfull
        #   fec[0] = 0x20001004
        #   fec[1] = 0x20001005
        command = "l3 ecmp get ecmp_id=%s" % ecmp_fec
        fec_entry_pat = re.compile('\]\s+=\s+')
        for line in self.bs.run(command).splitlines():
            if "fec[" in line:
                junk, fec = line.split("] = ");
                fec = fec[-4:]
                fecs.append(str(int(fec, 16)))

        return fecs

    ############################
    # Egresses = FECs
    #
    # Sample output: (diag alloc fec info=1)
    #     FEC-id: 0x1007     encap id: 0x40001020 port: 0x8000002  intf 0x2        access: X
    ############################
    def collect_egresses(self):
        try:
            disp_str = self.bs.run('diag alloc fec info=1')
        except IOError:
            raise RuntimeError('diag alloc fec info=1 timed out')

        egress_pat = re.compile('FEC-id:\s+0x(?P<egress_id>.*)\s+encap\s+id: (?P<encap_id>.*)\sport: (?P<port>.*)\s+intf\s0x(?P<intf>.*)\s+access: (?P<access>.*)', re.VERBOSE)
        for line in disp_str.splitlines():
            # Skip unwanted lines
            if "FEC-id: " not in line:
                continue

            parsed = egress_pat.findall(line)
            if parsed:
                egress_ndx, encap_id, port, intf, access = parsed[0]

                egress_ndx = str(int(egress_ndx, 16))

                intf = int(intf, 16)

                port = int(port[-2:], 16)

                mac = self.get_mac_from_outlif(egress_ndx)
                if mac != 0:
                    assert egress_ndx not in self.egresses, "'FEC' %s is already in %s" % (egress_ndx, pformat(self.egresses))
                    self.egresses[egress_ndx] = (intf, mac)

    ############################
    # ECMP Egresses = FECs
    #
    # Sample output: (diag alloc ecmp)
    #     Pool ECMP id(49) Total number of entries: 4095   Used entries: 2   Low entry ID is: 1(0x1)
    #     List of used entries in this pool:
    #           2         3
    ############################
    def collect_ecmp_egresses(self):
        try:
            disp_str = self.bs.run('diag alloc ecmp')
        except IOError:
            raise RuntimeError('diag alloc ecmp timed out')

        # Get ECMP ids allocated
        ecmp_fec_pat = re.compile("(\d+)")
        for line in disp_str.splitlines():
            if "Used entries: " in line:
                junk, num_ecmp_fecs = line.split('Used entries: ')

                # No ECMP egresses allocated
                if num_ecmp_fecs[0] == 0:
                    return

                continue

            ecmp_fecs_parsed = ecmp_fec_pat.findall(line)
            if ecmp_fecs_parsed != []:
                for fec in ecmp_fecs_parsed:
                    fec_list = self.get_ecmp_members(fec)
                    self.multipaths[fec] = fec_list

    ############################
    # Host Routes - LEM routes - FLP IPv4 UC LEM, ID (3)
    #
    # Sample output : (diag dbal te 3)
    #     Entry 19: Prefix= 2 fwd_ipv4_dip=0x14010107  Full buffer=0x1000 00000000 14010107   Payload=0x00000000 00021005
    ############################
    def collect_ipv4_host_routes(self):
        try:
            disp_str = self.bs.run('diag dbal te 3')
        except IOError:
            raise RuntimeError('diag dbal te 3 timed out')

        lem_route_pat = re.compile('Entry\s+\d+: Prefix= \d+\s+fwd_ipv4_dip=(?P<dip>.*)\s+vrf=(?P<vrf>.*)\s+Full buffer=0x\d+\s+\d+\s+\d+\s+Payload=0x\d+\s+(?P<fec>.*)')
        for line in disp_str.splitlines():
            if "Entry " not in line:
                continue

            parsed = lem_route_pat.findall(line)
            if parsed:
                dest, vrf, ndx = parsed[0]

                vrf = int(vrf, 16)
                if vrf:
                    vrf += 1000

                dest = socket.inet_ntoa(struct.pack('!I', int(dest, 16)))

                ndx = str(int(ndx[len(ndx) - 5:], 16))

                if (vrf, dest) not in self.hosts:
                    self.hosts[(vrf, dest)] = ndx

    ############################
    # Transit routes - LEM routes - FLP IPv4 UC LEM, ID (0)
    #
    # Sample output : (diag dbal te 0)
    #     Entry  5: Prefix= 2 fwd_ipv4_dip=0x9090203  vrf=0x2  Full buffer=0x1000 00000002 09090203   Payload=0x00000000 00021136
    ############################
    def collect_lem_ipv4_routes(self):
        try:
            disp_str = self.bs.run('diag dbal te 0')
        except IOError:
            raise RuntimeError('diag dbal te 0 timed out')

        lem_route_pat = re.compile('Entry\s+\d+: Prefix= \d+\s+fwd_ipv4_dip=(?P<dip>.*)\s+vrf=(?P<vrf>.*)\s+Full buffer=0x\d+\s+\d+\s+\d+\s+Payload=0x\d+\s+(?P<fec>.*)')
        for line in disp_str.splitlines():
            if "Entry " not in line:
                continue

            parsed = lem_route_pat.findall(line)
            if parsed:
                dest, vrf, ndx = parsed[0]

                vrf = int(vrf, 16)
                if vrf:
                    vrf += 1000

                dest = socket.inet_ntoa(struct.pack('!I', int(dest, 16)))

                ndx = str(int(ndx[len(ndx) - 5:], 16))

                if ndx == '4096':
                    # Handle local routes
                    assert (vrf, dest) not in self.hosts, "'Local route' (%s, %s) is already in %s" % (vrf, dest, pformat(self.hosts))
                    self.hosts[(vrf, dest)] = ndx
                    continue

                dest = dest + '/32'
                assert (vrf, dest) not in self.ip4_routes, "'Route' (%s, %s) is already in %s" % (vrf, dest, pformat(self.ip4_routes))
                self.ip4_routes[(vrf, dest)] = ndx

                if ndx in self.egresses:
                    self.route_2_rif[(vrf, dest)] = ndx
                elif ndx in self.multipaths:
                    self.route_2_ecmp[(vrf, dest)] = ndx

    ############################
    # Transit routes - KAPS routes - FLP IPv4 UC KAPS, ID (8)
    #
    # Sample output :(diag dbal te 8)
    #     |Raw: 0xc002140702000000000000000000000000000000/040 |TBL:0x3/2/2 |VRF: 0x0002/14/14 |DIPv4: 0x14070200/24/32 |Result: 0x21001/20|
    #     |     0xc003140703000000000000000000000000000000/040 |    0x3/2/2 |     0x0003/14/14 |       0x14070300/24/32 |        0x21001/20|
    ############################
    def collect_kaps_ipv4_routes(self):
        try:
            disp_str = self.bs.run('diag dbal te 8')
        except IOError:
            raise RuntimeError('diag dbal te 8 timed out')

        lines = disp_str.splitlines()
        for line in lines[2:]:
            # Filter out unwanted lines
            if line and "|" != line[0]:
                continue

            # Fetch vrf, dip and nexthop fields
            fields = line.split("|")
            if not fields[3:6]:
                continue
            fec = fields.pop(5).lstrip("Result:")
            dest = fields.pop(4).lstrip("DIPv4:")
            vrf = fields.pop(3).lstrip("VRF:")

            # vrf
            vrf, mask, size = vrf.split('/')
            vrf = int(vrf, 16)
            if vrf:
                vrf += 1000

            # fec
            fec, size = fec.split('/')
            fec = int(fec[len(fec) - 4:], 16)
            fec = str(fec)

            # destination prefix
            dest, mask, size = dest.split('/')
            prefix = socket.inet_ntoa(struct.pack('!I', int(dest, 16)))

            if mask == '32':
                # Handle local routes
                assert (vrf, prefix) not in self.hosts, "'Local route' (%s, %s) is already in %s" % (vrf, prefix, pformat(self.hosts))
                self.hosts[(vrf, prefix)] = fec
                continue

            prefix = prefix + '/' + mask
            assert (vrf, prefix) not in self.ip4_routes, "'Route' (%s, %s) is already in %s" % (vrf, dest, pformat(self.ip4_routes))
            self.ip4_routes[(vrf, prefix)] = fec

            if fec in self.egresses:
                self.route_2_rif[(vrf, prefix)] = fec
            elif fec in self.multipaths:
                self.route_2_ecmp[(vrf, prefix)] = fec

    def collect_data(self):
        ###############
        #   Nexthops  #
        ###############
        # Unipath egresses = FECs
        self.collect_egresses()

        # Multipath egresses = ECMP-FECs
        self.collect_ecmp_egresses()

        ##################
        #   Host routes  #
        ##################
        # Routes - KAPS routes - FLP IPv4 UC KAPS, ID (8)
        self.collect_ipv4_host_routes()

        # TODO: Add IPv6 host routes

        #####################
        #   Transit routes  #
        #####################
        # Routes - LEM routes - FLP IPv4 UC LEM, ID (0)
        self.collect_lem_ipv4_routes()

        # Routes - KAPS routes - FLP IPv4 UC KAPS, ID (8)
        self.collect_kaps_ipv4_routes()

        # TODO: Add IPv6 routes

        self.bs.close()

        # Combine IPv4 and IPv6 routes
        self.routes = dict(self.ip4_routes.items() + self.ip6_routes.items())

        self.build_route_dict()

class Mellanox(Asic):

    def __init__(self):
        Asic.__init__(self)
        # ecmp ID == 0 is an invalid flag, not a true ID value
        self.mp_exceptions = {0: None}
        self.neighbor_table = {}
        self.vfid_vlan_map = {}

    def get_chipname(self):
        pass

    def _read_ip_route_entries(self, source_table, ip_version):
        for p, entry in source_table.items():
            flags = ""
            prefix = p.split(' ')[1]
            ndx = 0
            target_table = self.ip4_routes

            if ip_version == 'ipv6':
                prefix = rfc5952(prefix)
                target_table = self.ip6_routes

            vrid = int(entry['vrid'])

            # If not table 0 add 1000 so the HW vrf table ID matches the kernel vrf table ID
            if vrid:
                vrid += 1000

            entry_type = entry['type']
            action = entry['action']
            rif = int(entry['erif'])

            invalid_actions = ['TRAP', 'DROP']
            if entry_type == 'NEXT_HOP' and action not in invalid_actions:
                ecmp_id = int(entry['ecmp_id'])
                eid = ecmp_id + 100000
                if eid not in self.multipaths:
                    self.multipaths[eid] = []
                    ecmp_table = mlx_get_ecmp(ecmp_id=ecmp_id)
                    if ecmp_table:
                        for ecmp_out in ecmp_table[ecmp_id]:
                            for nh in entry['next_hops']:
                                key = '{0} {1}'.format(nh, ecmp_out[0])
                                if key in self.neighbor_table:
                                    self.multipaths[eid].append(self.neighbor_table.keys().index(key))

                if entry['next_hop_cnt'] == 1:
                    if self.multipaths[eid]:
                        ndx = self.multipaths[eid][0]
                        self.route_2_rif[(vrid, prefix)] = ndx
                else:
                    ndx = ecmp_id + 100000
                self.route_2_ecmp[(vrid, prefix)] = ecmp_id + 100000
                flags = "offload"
            elif entry_type == 'LOCAL':
                ndx = rif + 200000
                self.route_2_rif[(vrid, prefix)] = ndx
                self.egresses[ndx] = (rif, "00:00:00:00:00:00")
                flags = "offload"
            if (entry_type == 'IP2ME' and action == 'DROP') or               (entry_type == 'NEXT_HOP' and action == 'TRAP'):
                self.hroutes_exceptions.update(set([(vrid, prefix)]))
            if (entry_type == 'IP2ME') or (entry_type == 'NEXT_HOP' and action == 'TRAP'):
                flags = "trap"
            target_table[(vrid, prefix)] = ndx
            if wfi_enabled and not (prefix.startswith("fe80") or prefix.startswith("127") or prefix.startswith('0') or prefix.startswith("::/0") or prefix.startswith("ff00")):
                self.offload_flags.setdefault(str(vrid) + " " + str(prefix) + " " + flags, [])
    def _build_vlan_vfid_mapping(self):
        vfid_to_vlan_xlate_table = "/cumulus/switchd/run/software-tables/16"
        try:
          os.system("echo TRUE > "+vfid_to_vlan_xlate_table)
          stream = open(vfid_to_vlan_xlate_table, "r")
        except:
          return
        data = StringIO()
        vfid_to_vlan_parser = stream.readlines()
        vfid_to_vlan_parser = vfid_to_vlan_parser[3:len(vfid_to_vlan_parser)-2]
        for line in vfid_to_vlan_parser:
            if line.startswith(" vfid-entry:"):
                continue
            data.write(line)
        data.seek(0)
        self.vfid_vlan_map = yaml.safe_load(data)

    def _read_fdb_entries(self, fdbtable):
        for mac, macinfo in fdbtable.iteritems():
            for entry in macinfo:
                vfid = "vfid-%08d"%entry['fid_vid']
                vlan = self.vfid_vlan_map.get(vfid, None)
                if vlan:
                    vlan = int(vlan.split("-")[1])
                if entry['dst_type'] == "nexthop":
                    self.remote_mac_table[(mac, vlan)] = [entry['destination']]
                elif entry['dst_type'] == "ecmp":
                    ecmpId = entry['destination']
                    ecmpinfo = mlx_get_operational_ecmp(ecmpId)
                    nexthopinfo = ecmpinfo[ecmpId]
                    nexthoplist = []
                    for items in nexthopinfo:
                        nexthoplist.append(items['underlay_dip'])
                    self.remote_mac_table[(mac, vlan)] = nexthoplist
                    self.l2_ecmp[ecmpId] = nexthoplist

    def collect_data(self):

        mlx_open_connection()

        # read the ipv4 and ipv6 neighbor entries
        self.neighbor_table = mlx_get_neighbor()
        neighbor_table6 = mlx_get_neighbor(version=6)
        self.neighbor_table.update(neighbor_table6)

        for p, neighbor in self.neighbor_table.iteritems():
            vrid = int(neighbor['vrid'])
            if vrid:
                vrid += 1000
            ndx = self.neighbor_table.keys().index(p)
            rif = int(neighbor['rif'])
            prefix = p.split(' ')[0]
            if ':' in prefix:
                prefix = rfc5952(prefix)
            self.hosts[(vrid, prefix)] = ndx
            self.egresses[ndx] = (rif, neighbor['mac'])

        # read the ip4 route entries
        route_table = mlx_get_uc_route()
        self._read_ip_route_entries(route_table, 'ipv4')
        route_table = mlx_get_uc_route(6)
        self._read_ip_route_entries(route_table, 'ipv6')
        
        self._build_vlan_vfid_mapping()
        fdb_table = mlx_get_fdb(0)
        self._read_fdb_entries(fdb_table)

        self.build_route_dict()

        mlx_close_connection()


class Routing(object):

    def __init__(self):
        self.zebra_bgp_ipv4_prefixes = []
        self.zebra_fib_ipv4_prefixes = []
        self.recursed_nexthops_per_bgp = []
        self.recursed_nexthops_per_zebra = []
        self.zebra_bgp_ipv6_prefixes = []
        self.zebra_fib_ipv6_prefixes = []
        self.bgp_ipv4_prefixes = []
        self.bgp_ipv6_prefixes = []

    def collect_data(self):
        self.collect_data_zebra_ipv4()
        self.collect_data_zebra_ipv6()
        self.collect_data_bgp_ipv4()
        self.collect_data_bgp_ipv6()

    def collect_data_zebra_ipv4(self):
        """
        {
          "10.1.10.0/24": [
            {
              "distance": 200,
              "metric": 0,
              "nexthops": [
                {
                  "active": true,
                  "afi": "ipv4",
                  "ip": "10.1.1.2",
                  "recursive": true
                },
                {
                  "active": true,
                  "afi": "ipv4",
                  "fib": true,
                  "interfaceIndex": 3,
                  "interfaceName": "swp1",
                  "ip": "10.1.2.2"
                }
              ],
              "prefix": "10.1.10.0/24",
              "protocol": "bgp",
              "selected": true,
              "uptime": "00:03:37"
            }
          ],
        """
        zebra_ipv4 = {}

        # Zebra IPv4
        cmd = ["/usr/bin/vtysh", "-c", "show ip route json"]
        try:
            output = check_output(cmd).strip()
        except CalledProcessError:
            self.zebra_bgp_ipv4_prefixes = set()
            self.zebra_fib_ipv4_prefixes = set()
            self.recursed_nexthops_per_bgp = set()
            self.recursed_nexthops_per_zebra = set()
            return

        try:
            zebra_ipv4 = json.loads(output)
        except ValueError:
            zebra_ipv4 = {}
            print "ERROR: The json from '%s' is invalid\n%s" % (' '.join(cmd), output)

        for (prefix, paths) in zebra_ipv4.iteritems():
            prefix = str(prefix)

            # uncomment to fake a failure
            # if prefix == '10.1.9.0/24':
            #     continue

            for path in paths:
                instance = path.get('instance')
                if instance is None:
                    instance = 0

                for nexthop in path.get('nexthops'):
                    if nexthop.get('fib'):
                        ip = None if not nexthop.get('ip') else str(nexthop.get('ip'))

                        # RFC 5549 allows BGP to advertise an IPv4 prefix with
                        # an IPv6 nexthop.  When this happens the route is
                        # installed in zebra with a v6 nexthop but when zebra
                        # installs the route into the kernel it changes the
                        # nexthop to 169.254.0.1.  We must make the same change
                        # here else we will get a "zebra has IPv4 FIB routes
                        # not in the kernel" failure due to the nexthops not matching
                        if ip is not None:
                            try:
                                IPv6Network(ip)
                                ip = '169.254.0.1'
                            except AddressValueError:
                                pass

                        self.zebra_fib_ipv4_prefixes.append((instance, prefix, ip))

                if path.get('protocol') == 'bgp':
                    # If one of the entries is flagged as recursive that is
                    # the nexthop that zebra RXed from BGP, the other entry
                    # will be the resolution of that nexthop
                    if has_recursive_nexthop(path.get('nexthops')):
                        for nexthop in path.get('nexthops'):
                            if nexthop.get('recursive'):
                                ip = None if not nexthop.get('ip') else str(nexthop.get('ip'))
                                self.zebra_bgp_ipv4_prefixes.append((prefix, ip))
                    else:
                        for nexthop in path.get('nexthops'):
                            ip = None if not nexthop.get('ip') else str(nexthop.get('ip'))
                            self.zebra_bgp_ipv4_prefixes.append((prefix, ip))

        self.zebra_bgp_ipv4_prefixes = set(self.zebra_bgp_ipv4_prefixes)
        self.zebra_fib_ipv4_prefixes = set(self.zebra_fib_ipv4_prefixes)
        """
        Catch bugs like this where BGP says that 172.16.0.93 only resolves
        via 172.16.0.88 when in fact it should resolve via 172.16.0.88 and 172.16.0.90

        root@superm-redxp-05[~]# vtysh -c 'show ip route 10.1.1.4/32'
        Routing entry for 10.1.1.4/32
          Known via "bgp", distance 200, metric 0, best
          Last update 00:00:28 ago
            172.16.0.73 (recursive)
          *   172.16.0.68, via swp1
          *   172.16.0.70, via swp2
            172.16.0.93 (recursive)
          *   172.16.0.88, via swp3
            172.16.0.113 (recursive)
          *   172.16.0.108, via swp5
          *   172.16.0.110, via swp6
            172.16.0.133 (recursive)
          *   172.16.0.128, via swp7
          *   172.16.0.130, via swp8

        root@superm-redxp-05[~]#

        root@superm-redxp-05[~]# vtysh -c 'show ip route 172.16.0.93'
        Routing entry for 172.16.0.92/31
          Known via "bgp", distance 200, metric 0, best
          Last update 00:00:39 ago
          * 172.16.0.88, via swp3
          * 172.16.0.90, via swp4

        root@superm-redxp-05[~]#
        """
        for (prefix, paths) in zebra_ipv4.iteritems():
            prefix = str(prefix)

            for path in paths:

                if path.get('protocol') not in ('bgp', ):
                    continue

                instance = path.get('instance')
                if instance is None:
                    instance = 0

                recursive_nexthops = []
                via_nexthops = []
                recursive_ip = None

                for nexthop in path.get('nexthops'):
                    ip = None if not nexthop.get('ip') else str(nexthop.get('ip'))

                    if nexthop.get('recursive') or not nexthop.get('active'):
                        if via_nexthops:
                            via_nexthops = sorted(via_nexthops)

                            if (recursive_ip, via_nexthops) not in self.recursed_nexthops_per_bgp:
                                self.recursed_nexthops_per_bgp.append((recursive_ip, tuple(via_nexthops)))

                            if recursive_ip not in recursive_nexthops:
                                recursive_nexthops.append(recursive_ip)

                            via_nexthops = []
                            recursive_ip = None

                        if nexthop.get('recursive'):
                            recursive_ip = ip
                    else:
                        if recursive_ip:
                            via_nexthops.append(ip)

                if via_nexthops:
                    via_nexthops = sorted(via_nexthops)

                    if (recursive_ip, via_nexthops) not in self.recursed_nexthops_per_bgp:
                        self.recursed_nexthops_per_bgp.append((recursive_ip, tuple(via_nexthops)))

                    if recursive_ip not in recursive_nexthops:
                        recursive_nexthops.append(recursive_ip)

                for nexthop in recursive_nexthops:
                    if nexthop not in self.recursed_nexthops_per_zebra:
                        via_nexthops = []

                        # IPv4 nextop
                        try:
                            IPv4Network(nexthop)
                            cmd = ["/usr/bin/vtysh", "-c", "show ip route %s" % nexthop]
                        except ipaddr.AddressValueError:
                            cmd = ["/usr/bin/vtysh", "-c", "show ipv6 route %s" % nexthop]

                        '''
                        root@superm-redxp-05[~]# vtysh -c 'show ip route 172.16.0.93'
                        Routing entry for 172.16.0.92/31
                          Known via "bgp", distance 200, metric 0, best
                          Last update 00:00:39 ago
                          * 172.16.0.88, via swp3
                          * 172.16.0.90, via swp4

                        root@superm-redxp-05[~]# vtysh -c 'show ip route 1.1.1.1'
                        Routing entry for 0.0.0.0/0
                          Known via "kernel", distance 0, metric 0, best
                          * 10.0.1.2, via eth0

                        root@superm-redxp-05[~]#
                        '''
                        try:
                            output = check_output(cmd)
                            for line in output.splitlines():
                                re_via = re.search('(\S+), via', line)

                                if re_via:
                                    via_nexthops.append(re_via.group(1))
                        except CalledProcessError:
                            pass

                        via_nexthops = sorted(via_nexthops)
                        self.recursed_nexthops_per_zebra.append((nexthop, tuple(via_nexthops)))

        self.recursed_nexthops_per_bgp = set(self.recursed_nexthops_per_bgp)
        self.recursed_nexthops_per_zebra = set(self.recursed_nexthops_per_zebra)

    def collect_data_zebra_ipv6(self):
        zebra_ipv6 = {}

        # Zebra IPv6
        cmd = ["/usr/bin/vtysh", "-c", "show ipv6 route json"]
        try:
            output = check_output(cmd).strip()
        except CalledProcessError:
            self.zebra_bgp_ipv6_prefixes = set()
            self.zebra_fib_ipv6_prefixes = set()
            return

        try:
            zebra_ipv6 = json.loads(output)
        except ValueError:
            zebra_ipv6 = {}
            print "ERROR: The json from '%s' is invalid\n%s" % (' '.join(cmd), output)

        for (prefix, paths) in zebra_ipv6.iteritems():
            prefix = str(prefix)

            # uncomment to fake a failure
            # if prefix == '2001:40:2:1::/64':
            #     continue

            for path in paths:
                instance = path.get('instance')
                if instance is None:
                    instance = 0

                for nexthop in path.get('nexthops'):
                    if nexthop.get('fib'):
                        ip = None if not nexthop.get('ip') else str(nexthop.get('ip'))
                        self.zebra_fib_ipv6_prefixes.append((instance, prefix, ip))

                if path.get('protocol') == 'bgp':
                    # If one of the entries is flagged as recursive that is
                    # the nexthop that zebra RXed from BGP, the other entry
                    # will be the resolution of that nexthop
                    if has_recursive_nexthop(path.get('nexthops')):
                        for nexthop in path.get('nexthops'):
                            if nexthop.get('recursive'):
                                ip = None if not nexthop.get('ip') else str(nexthop.get('ip'))
                                self.zebra_bgp_ipv6_prefixes.append((prefix, ip))
                    else:
                        for nexthop in path.get('nexthops'):
                            ip = None if not nexthop.get('ip') else str(nexthop.get('ip'))
                            self.zebra_bgp_ipv6_prefixes.append((prefix, ip))

        self.zebra_bgp_ipv6_prefixes = set(self.zebra_bgp_ipv6_prefixes)
        self.zebra_fib_ipv6_prefixes = set(self.zebra_fib_ipv6_prefixes)

    def collect_data_bgp_ipv4(self):
        """
        {
          "routerId": "10.1.1.1",
          "routes": {
            "10.1.10.0/24": [
              {
                "aspath": "",
                "bestpath": true,
                "localpref": 100,
                "med": 0,
                "nexthops": [
                  {
                    "afi": "ipv4",
                    "ip": "10.1.1.2",
                    "used": true
                  }
                ],
                "origin": "IGP",
                "pathFrom": "internal",
                "peerId": "10.1.1.2",
                "valid": true,
                "weight": 0
              }
            ],
        """
        bgp_ipv4 = {}
        cmd = ["/usr/bin/vtysh", "-c", "show bgp ipv4 unicast json"]
        try:
            output = check_output(cmd).strip()
        except CalledProcessError:
            return

        try:
            bgp_ipv4 = json.loads(output)
        except ValueError:
            print "ERROR: The json from '%s' is invalid\n%s" % (' '.join(cmd), output)

        if bgp_ipv4.get('routes'):
            for (prefix, paths) in bgp_ipv4['routes'].iteritems():
                prefix = str(prefix)

                # uncomment to fake a failure
                # if prefix == '10.1.8.0/24':
                #     continue

                for path in paths:

                    # The BGP prefix will only be in zebra if there is a bestpath
                    if path.get('bestpath') or path.get('multipath'):

                        # For IPv6 there can be a LL and Global nexthop, the LL one should be marked as 'used'
                        for nexthop in path.get('nexthops'):
                            ip = None if not nexthop.get('ip') else str(nexthop.get('ip'))

                            # If the nexthop is 0.0.0.0 that means we originated the route
                            if nexthop.get('used') and ip != '0.0.0.0':
                                self.bgp_ipv4_prefixes.append((prefix, ip))

        self.bgp_ipv4_prefixes = set(self.bgp_ipv4_prefixes)


    def collect_data_bgp_ipv6(self):
        bgp_ipv6 = {}

        cmd = ["/usr/bin/vtysh", "-c", "show bgp ipv6 unicast json"]
        try:
            output = check_output(cmd).strip()
        except CalledProcessError:
            return

        try:
            bgp_ipv6 = json.loads(output)
        except ValueError:
            print "ERROR: The json from '%s' is invalid\n%s" % (' '.join(cmd), output)

        if bgp_ipv6.get('routes'):
            for (prefix, paths) in bgp_ipv6['routes'].iteritems():
                prefix = str(prefix)

                # uncomment to fake a failure
                # if prefix == '2001:10:2:3::/64':
                #     continue

                for path in paths:

                    # The BGP prefix will only be in zebra if there is a bestpath
                    if path.get('bestpath') or path.get('multipath'):

                        # For IPv6 there can be a LL and Global nexthop, the LL one should be marked as 'used'
                        for nexthop in path.get('nexthops'):
                            if nexthop.get('used'):
                                ip = None if not nexthop.get('ip') else str(nexthop.get('ip'))
                                self.bgp_ipv6_prefixes.append((prefix, ip))

        self.bgp_ipv6_prefixes = set(self.bgp_ipv6_prefixes)


class DiffTestParam(object):

    def __init__(self, set1, set2, exceptions, msg, key):
        self.set1 = set1
        self.set2 = set2
        self.exceptions = exceptions
        self.msg = msg
        self.key = key

        # also store any result diffs here
        self.results = None


class Comparer(object):

    def __init__(self, routing, kernel, asic):
        self.routing = routing
        self.kernel = kernel
        self.asic = asic

        # Assume all tests PASS
        self.ret_code = 0
        self.diff_dict = {}

        self.asic_all_ndxs = []
        self.asic_egress_mac = []
        self.asic_egress_ndxs = []
        self.asic_hosts_ndxs = []
        self.asic_hosts_prefixes = []
        self.asic_hroutes_prefixes = []
        self.asic_mp_egress_ndxs = []
        self.asic_mp_ndxs = []
        self.asic_routes_ecmps = []
        self.asic_routes_ecmps = []
        self.asic_routes_prefixes = []
        self.asic_routes_rifs = []
        self.asic_remote_macs = []
        self.kernel_mac = []
        self.kernel_neigh_prefixes = []
        self.kernel_prefixes = []
        self.recursed_nexthops_per_bgp = []
        self.recursed_nexthops_per_zebra = []
        self.kernel_remote_macs = []
        self.kernel_offload_flags = [] 
        self.asic_offload_flags = []
    def _s_diff_t(self, s, t, exceptions):
        """
        Find differences between set s and set t.
        An exception set can be used to rule out known differences.
        """
        return sorted(s.difference(t).difference(exceptions))

    def s_diff_t(self, diff_test_param):
        return self._s_diff_t(diff_test_param.set1,
                              diff_test_param.set2,
                              diff_test_param.exceptions)

    def run_tests(self, check_tuples, return_bit=0):
        """
        Given list of DiffTestParam objects, run the s_diff_t method for each and populate
        a key-->DiffTestParam dictionary to hold the errors (diffs) if any.
        """

        for i, diff_test_param in enumerate(check_tuples, return_bit):
            diff = self.s_diff_t(diff_test_param)

            if diff:
                diff_test_param.results = diff

                if diff_test_param.key in self.diff_dict:
                    self.diff_dict[diff_test_param.key].append(diff_test_param)
                else:
                    self.diff_dict[diff_test_param.key] = [diff_test_param]

                self.ret_code = 1

    def get_kernel_route_exceptions(self):
        """
        Traverse all of the kernel neighbors and find the ones that the kernel
        also has a host route for. These host routes will be missing from l3 defip
        because they will be in l3table instead.
        """
        kernel_route_exceptions = []

        for ip in self.kernel_neigh_prefixes:

            # IPv4
            if '.' in ip:
                ip = ip + '/32'

            # IPv6
            else:
                ip = ip + '/128'

            if (0, ip) in self.kernel_prefixes:
                kernel_route_exceptions.append((0, ip))

        self.kernel.route_exceptions.update(set(kernel_route_exceptions))
        return self.kernel.route_exceptions

    def compute_kernel_vs_asic_deltas(self):
        #
        # Build a bunch of sets for quick s - t comparisons
        #
        self.asic_hosts_ndxs     = set(self.asic.hosts.itervalues())
        self.asic_egress_ndxs    = set(self.asic.egresses.iterkeys())
        self.asic_routes_rifs    = set(self.asic.route_2_rif.itervalues())
        self.asic_routes_ecmps   = set(self.asic.route_2_ecmp.itervalues())
        self.asic_mp_ndxs        = set(self.asic.multipaths.iterkeys())
        self.asic_mp_egress_ndxs = set([n for v in self.asic.multipaths.itervalues() for n in v])
        self.asic_all_ndxs       = self.asic_egress_ndxs.union(self.asic_mp_ndxs)
        self.asic_hosts_ndxs_adjusted  = set([str(int(item) - 300000) if (int(item) > 400000) else item  for item in self.asic.hosts.itervalues()])
        self.asic_egress_ndxs_adjusted = set([str(int(item) - 300000) if (int(item) > 400000) else item  for item in self.asic.egresses.iterkeys()])
        self.asic_remote_macs    = set(self.asic.remote_mac_table)
        self.kernel_remote_macs  = set(self.kernel.remote_macs)

        self.kernel_neigh_prefixes = set()

        for (ip, iface) in self.kernel.neighs.iterkeys():
            self.kernel_neigh_prefixes.add(ip)

        # The kernel's neighbor table is not VRF aware whereas the asic host table is.
        # The asic pieces together bits of info to figure out which neighbors need to
        # be added to which VRF. For computing deltas we are going to ignore the VRF
        # component of the asic host table.
        self.asic_hosts_prefixes = []
        for (vrf, ip) in self.asic.hosts.iterkeys():
            self.asic_hosts_prefixes.append(ip)
        self.asic_hosts_prefixes = set(self.asic_hosts_prefixes)

        self.asic_hostsipv6_prefixes = set(self.asic.hostsipv6.iterkeys())
        self.asic_routes_prefixes = set(self.asic.ip4_routes.iterkeys())
        self.asic_routes_prefixes.update(set(self.asic.ip6_routes.iterkeys()))
        self.asic_hroutes_prefixes = set(self.asic.hroutes.iterkeys())

        self.kernel_prefixes = set(self.kernel.routes.iterkeys())
        self.asic_offload_flags = set(self.asic.offload_flags.iterkeys())
        self.kernel_offload_flags = set(self.kernel.offload_flags.iterkeys())
        self.kernel_ipv6local_prefixes = set(self.kernel.ipv6_local_routes.iterkeys())

        self.kernel_mac = set(self.kernel.neigh_macs.iterkeys())
        self.asic_egress_mac = set([mac for intf, mac in self.asic.egresses.itervalues()])

        #
        # Exception lists.  There are items we want to exclude in some test.
        #
        adjacency_exceptions = set(["00:00:00:00:00:00"])  # "send to SW" adjacency
        ip4route_exceptions = set([(0, "0.0.0.0/0")])    # default route
        ip6route_exceptions = set([(0, "::/0")])          # default route
        kernel_ip_offload_exceptions = set([("0 ::1/128 "),("0 ff00::/8 ")]) 
        #offload programming skipped for these ipv6 routes
        hroutes_exceptions = ip4route_exceptions
        hroutes_exceptions.update(ip6route_exceptions)
        hroutes_exceptions.update(self.asic.hroutes_exceptions)
        kernel_route_exceptions = self.get_kernel_route_exceptions()
        kernel_neigh_exceptions = self.kernel.neigh_exceptions
        kernel_remote_mac_exceptions = self.kernel.remotemac_exceptions

        # MLX and Broadcom have some differenes in how they handle onlink host
        # routes. MLX will install these in the HW host table while Broadcom
        # will not.  If we are dealing with Broadcom then update kernel_neigh_exceptions
        # so that we do not worry about having a HW host entry if the host is
        # known via a /32 onlink route.
        if isinstance(self.asic, Broadcom):
            for (prefix, paths) in self.kernel.onlinks.iteritems():
                (prefix, prefixlen) = prefix.split('/')

                # If this onlink route is an IPv4 host route
                if prefixlen == '32' and '.' in prefix:

                    # And if this /32 is also in the neighbor table then the HW will
                    # not have an entry in its host table
                    for (iface, _) in paths:
                        if (prefix, iface) in self.kernel.neighs:
                            kernel_neigh_exceptions.update(set([prefix]))

        # Both Broadcom and MLX program vxlan evpn remote host routes
        # in route table and not host table though they are /32 routes.
        # Add these routes to neigh exceptions
        for (prefix, paths) in self.kernel.vxlan_onlinks.iteritems():
            (prefix, prefixlen) = prefix.split('/')
            if prefixlen == '32' and '.' in prefix:
                kernel_neigh_exceptions.update(set([prefix]))
            if prefixlen == '128' and ':' in prefix:
                kernel_neigh_exceptions.update(set([prefix]))

        # The HW installs a default route per vrf
        for (vrf_name, vrf_id) in self.kernel.vrfs.iteritems():
            hroutes_exceptions.update(set([(vrf_id, "0.0.0.0/0"), (vrf_id, "::/0")]))

        #
        # Compare HW tables for consitency
        #
        check_tuples = (
            DiffTestParam(self.asic_mp_egress_ndxs,
                          self.asic_egress_ndxs,
                          set(),
                          "asic multipath table has index(es) not found in asic egress table",
                          "mp_not_in_egress"),
            #DiffTestParam(self.asic_egress_ndxs,
            #              self.asic_hosts_ndxs | self.asic_routes_rifs,
            #              self.asic.egress_exceptions,
            #              "asic egress table has index(es) not found in asic host/route tables",
            #              "egress_not_in_host_route"),
            DiffTestParam(self.asic_mp_ndxs,
                          self.asic_routes_ecmps,
                          set(),
                          "asic multipath table has index(es) not found in asic route tables",
                          "mptable_idx_not_in_routes"),

            DiffTestParam(self.asic_routes_rifs,
                          self.asic_egress_ndxs,
                          set(),
                          "asic route tables have interface index(es) not found in asic egress table",
                          "ip4_ip6_route_idx_not_in_egress"),

            DiffTestParam(self.asic_routes_ecmps,
                          self.asic_mp_ndxs,
                          self.asic.mp_exceptions,
                          "asic route tables have ecmp index(es) not found in asic multipath table",
                          "route_idx_not_in_mptables"),

            DiffTestParam(self.asic_hosts_ndxs_adjusted,
                          self.asic_egress_ndxs_adjusted,
                          set(),
                          "asic host tables have index(es) not found in asic egress table",
                          "host_tables_idx_not_in_egress"),

            DiffTestParam(self.asic_routes_prefixes,
                          self.asic_hroutes_prefixes,
                          hroutes_exceptions,
                          "asic route tables have prefix(es) not found in asic hroutes",
                          "route_prefix_not_in_hroutes"),

            DiffTestParam(self.asic_routes_prefixes,
                          self.kernel_prefixes,
                          hroutes_exceptions,
                          "route tables have prefix(es) not found in kernel routes",
                          "route_prefix_not_in_kernel_routes"),

            DiffTestParam(self.kernel_prefixes,
                          self.asic_routes_prefixes,
                          kernel_route_exceptions,
                          "kernel routes has prefix(es) not found in asic route tables",
                          "kernel_prefix_not_found_in_routes"),

            DiffTestParam(self.asic_egress_mac,
                          self.kernel_mac,
                          adjacency_exceptions,
                          "asic egress table has mac not found in kernel neighbors",
                          "egress_table_mac_not_in_kernel_neighbors"),

            DiffTestParam(self.asic_hosts_prefixes,
                          self.kernel_neigh_prefixes,
                          set(),
                          "asic host table has prefix(es) not found in kernel neigh table",
                          "host_table_prefix_not_in_kernel_neigh_table"),

            # self.kernel.neigh_exceptions
            DiffTestParam(self.kernel_neigh_prefixes,
                          self.asic_hosts_prefixes,
                          kernel_neigh_exceptions,
                          "kernel neigh table has prefix(es) not found in asic host table",
                          "kernel_neigh_table_prefix_not_in_host_table")
            )

        #Remote Macs Consistency Check support is added only for mellanox.  
        if isinstance(self.asic, Mellanox):
            check_tuples = list(check_tuples)
            check_tuples.extend((
            # Evpn Remote Macs
            DiffTestParam(self.asic_remote_macs,
                          self.kernel_remote_macs,
                          kernel_remote_mac_exceptions,
                          "asic fdb table has macs not found in kernel mac table",
                          "asic_remote_mac_not_in_kernel_fdb_table"),

            DiffTestParam(self.kernel_remote_macs,
                          self.asic_remote_macs,
                          set(),
                          "kernel fdb table has macs not found in asic fdb table",
                          "kernel_remote_mac_table_not_in_asic_fdb_table"),
            ))
            if wfi_enabled:
                check_tuples.extend((
                DiffTestParam(self.kernel_offload_flags,
                              self.asic_offload_flags,
                              kernel_ip_offload_exceptions,
                              "kernel offoad flags doesn't match",
                              "kernel offload_flags_doesn't match"),
                DiffTestParam(self.asic_offload_flags,
                              self.kernel_offload_flags,
                              set(),
                              "asic flags doesn't match",
                              "asic offload_flags_doesn't match"),
                ))
            check_tuples = tuple(check_tuples)

        self.run_tests(check_tuples)

        #
        # Compare HW tables with SW tables for consistency
        #
        nhs_lens = [len(nhs) for nhs in self.asic.hroutes.itervalues()]

        ave_len = 0
        if len(nhs_lens) > 0:
            ave_len = sum(nhs_lens) / float(len(nhs_lens))
        doing_ecmp = ave_len > 1.0

        for dest, hnhs in self.asic.hroutes.items():
            if dest in self.kernel.routes and dest not in self.kernel.route_exceptions:
                knhs = self.kernel.routes[dest]
                #knhs = [mac if mac else
                #            "00:00:00:00:00:00" if not self.kernel.ifaces.get(iface)
                #            else self.kernel.ifaces[iface][1] for iface, mac in knhs]

                km = []
                for (iface, mac, nexthop) in knhs:
                    if not mac:
                        continue
                    elif mac == 'no neighbor found':
                        continue
                    km.append(mac)
                knhs = km

                #hnhs = [mac for intf, mac in hnhs]
                hm = []
                for intf, mac in hnhs:
                    if mac != "00:00:00:00:00:00":
                        hm.append(mac)
                hnhs = hm

                # HW has limited space for ecmp routes.  If it looks like we're
                # doing ecmp in HW, and this HW route is single but the matching
                # kernel route is ecmp, assume the HW ecmp table is full and
                # this ecmp kernel route was reduces to a single HW route.
                if doing_ecmp and len(hnhs) == 1 and len(knhs) > 1 and hnhs[0] in knhs:
                    knhs = hnhs

                check_tuples = (
                    DiffTestParam(set(hnhs),
                                  set(knhs),
                                  set(),
                                  "route '%s' has next hop in HW not found in kernel" % pformat(dest),
                                  "route_nexthop_in_hw_not_in_kernel"),

                    DiffTestParam(set(knhs),
                                  set(hnhs),
                                  set(),
                                  "route '%s' has next hop in kernel not found in HW" % pformat(dest),
                                  "route_nexthop_in_kernel_not_in_hw")
                )

                self.run_tests(check_tuples, 11)

        # Check results for switching ASICs that store kernel IPv6 local
        # routes in their IPv6 host table
        if self.asic.split_ipv6_local:
            check_tuples = (
                DiffTestParam(self.asic_hostsipv6_prefixes,
                              self.kernel_ipv6local_prefixes,
                              set(),
                              "IPv6 host table has prefix(es) not found in kernel IPv6 local route table",
                              "ipv6_host_table_prefix_not_in_kernel_ipv6_local_route_table"),

                DiffTestParam(self.kernel_ipv6local_prefixes,
                              self.asic_hostsipv6_prefixes,
                              set(),
                              "kernel IPv6 local route table has prefix(es) not found in IPv6 host table",
                              "kernel_ipv6_local_route_table_prefix_not_in_ipv6_host_table")
            )
            self.run_tests(check_tuples)

    def compute_bgp_vs_zebra_deltas(self):
        """
        Verify all BGP bestpath/multipath prefixes and their nexthop are installed in zebra
        """
        check_tuples = (
            DiffTestParam(self.routing.bgp_ipv4_prefixes,
                          self.routing.zebra_bgp_ipv4_prefixes,
                          set(),
                          "BGP IPv4 (prefix, nexthop) not in zebra",
                          "bgp_ipv4_prefix_nexthop_not_in_zebra"),
            DiffTestParam(self.routing.zebra_bgp_ipv4_prefixes,
                          self.routing.bgp_ipv4_prefixes,
                          set(),
                          "zebra BGP IPv4 (prefix, nexthop) not in BGP",
                          "zebra_bgp_ipv4_prefix_nexthop_not_in_bgp"),
            DiffTestParam(self.routing.bgp_ipv6_prefixes,
                          self.routing.zebra_bgp_ipv6_prefixes,
                          set(),
                          "BGP IPv6 (prefix, nexthop) not in zebra",
                          "bgp_ipv6_prefix_nexthop_not_in_zebra"),
            DiffTestParam(self.routing.zebra_bgp_ipv6_prefixes,
                          self.routing.bgp_ipv6_prefixes,
                          set(),
                          "zebra BGP IPv6 (prefix, nexthop) not in BGP",
                          "zebra_bgp_ipv6_prefix_nexthop_not_in_bgp"),
        )
        self.run_tests(check_tuples)

    def compute_zebra_recursion_deltas(self):
        check_tuples = (
            DiffTestParam(self.routing.recursed_nexthops_per_bgp,
                          self.routing.recursed_nexthops_per_zebra,
                          set(),
                          "BGP has recursive nexthops that do not match in zebra",
                          "bgp_recursive_nexthops_not_in_zebra"),
        )
        self.run_tests(check_tuples)

    def compute_zebra_vs_kernel_deltas(self):
        """
        Verify all zebra routes/nexthops marked as 'fib' are installed in the kernel
        """
        self.kernel_prefixes_with_nexthop = []

        for (key, value) in self.kernel.routes.iteritems():
            table_id = key[0]
            prefix = key[1]

            for (iface, mac, nexthop) in value:
                self.kernel_prefixes_with_nexthop.append((table_id, prefix, nexthop))

        self.kernel_prefixes_with_nexthop = set(self.kernel_prefixes_with_nexthop)

        check_tuples = (
            DiffTestParam(self.routing.zebra_fib_ipv4_prefixes,
                          self.kernel_prefixes_with_nexthop,
                          set(),
                          "zebra has IPv4 FIB routes not in the kernel",
                          "zebra_ipv4_fib_not_in_kernel"),
            DiffTestParam(self.routing.zebra_fib_ipv6_prefixes,
                          self.kernel_prefixes_with_nexthop,
                          set(),
                          "zebra has IPv6 FIB routes not in the kernel",
                          "zebra_ipv6_fib_not_in_kernel"),
        )
        self.run_tests(check_tuples)

    def get_result_json(self):

        if not self.diff_dict:
            return json.dumps({'status': 'success'})

        out_sio = StringIO()
        data_dict = {}

        for diff_key in self.diff_dict:
            if len(self.diff_dict[diff_key]) > 1:
                res_list = []
                for dtparam_obj in self.diff_dict[diff_key]:
                    res_list.extend(dtparam_obj.results)
                data_dict[diff_key] = res_list
            else:
                # If only 1 result, just grab its results directly
                data_dict[diff_key] = self.diff_dict[diff_key][0].results

        out_dict = {'status': 'error',
                    'data': data_dict}
        json.dump(out_dict, out_sio)

        return out_sio.getvalue()

    def get_result_str(self, verbose, very_verbose):
        """
        Handle human readable and JSON output given a dictionary of key-->DiffTestParam objects.
        """
        if not self.diff_dict:
            return ''

        out_sio = StringIO()
        print_missing_neighbors = False

        for diff_obj_list in sorted(self.diff_dict.values()):

            if diff_obj_list:

                if not print_missing_neighbors:
                    print_missing_neighbors = True

                    if self.kernel.missing_neighbors:
                        print >> out_sio, "There are routes via the following nexthops but the kernel does not have neighbors for these:"
                        for (table, iface, via) in self.kernel.missing_neighbors:
                            print >> out_sio, "  table %s has a route via %s with nexthop %s" % (table, iface, via)
                        print >> out_sio, ""

                for diff_obj in diff_obj_list:
                    print >> out_sio, "%s:" % diff_obj.msg

                    for i, diff in enumerate(diff_obj.results):
                        if i > 2 and not (verbose or very_verbose):
                            print >> out_sio, "   more..."
                            break
                        print >> out_sio, "  ", diff

        return out_sio.getvalue()

    def print_all(self):
        """
        Method to print all raw data
        """
        # Routing
        print "bgp_ipv4_prefixes \n%s\n" % pformat(self.routing.bgp_ipv4_prefixes)
        print "zebra_bgp_ipv4_prefixes \n%s\n" % pformat(self.routing.zebra_bgp_ipv4_prefixes)
        print "zebra_fib_ipv4_prefixes \n%s\n" % pformat(self.routing.zebra_fib_ipv4_prefixes)
        print "bgp_ipv6_prefixes \n%s\n" % pformat(self.routing.bgp_ipv6_prefixes)
        print "zebra_bgp_ipv6_prefixes \n%s\n" % pformat(self.routing.zebra_bgp_ipv6_prefixes)
        print "zebra_fib_ipv6_prefixes \n%s\n" % pformat(self.routing.zebra_fib_ipv6_prefixes)
        print "recursed_nexthops_per_bgp\n%s\n" % pformat(self.routing.recursed_nexthops_per_bgp)
        print "recursed_nexthops_per_zebra\n%s\n" % pformat(self.routing.recursed_nexthops_per_zebra)

        # Kernel
        print "kernel_neigh_prefixes\n%s\n" % pformat(self.kernel_neigh_prefixes)
        print "kernel_vrfs\n%s\n" % pformat(self.kernel.vrfs)
        print "kernel_l3_nhgs\n%s\n" % pformat(self.kernel.l3nhgs)
        print "kernel_prefixes\n%s\n" % pformat(self.kernel_prefixes)
        print "kernel_routes\n%s\n" % pformat(self.kernel.routes)
        print "kernel_mac\n%s\n" % pformat(self.kernel_mac)
        print "kernel_remote_macs\n%s\n" % pformat(self.kernel_remote_macs)
        print "kernel_offload_flags\n%s\n" % pformat(self.kernel_offload_flags)
        
        # ASIC
        print "asic_hosts_ndxs\n%s\n" % pformat(self.asic_hosts_ndxs)
        print "asic_routes_rifs\n%s\n" % pformat(self.asic_routes_rifs)
        print "asic_routes_ecmp\n%s\n" % pformat(self.asic_routes_ecmps)
        print "asic_egress_ndxs\n%s\n" % pformat(self.asic_egress_ndxs)
        print "asic_mp_ndxs\n%s\n" % pformat(self.asic_mp_ndxs)
        print "asic_mp_egress_ndxs\n%s\n" % pformat(self.asic_mp_egress_ndxs)
        print "asic_all_ndxs\n%s\n" % pformat(self.asic_all_ndxs)
        print "asic_hosts_prefixes\n%s\n" % pformat(self.asic_hosts_prefixes)
        print "asic_routes_prefixes\n%s\n" % pformat(self.asic_routes_prefixes)
        print "asic_hroutes_prefixes\n%s\n" % pformat(self.asic_hroutes_prefixes)
        print "asic_offload_flags\n%s\n" % pformat(self.asic_offload_flags)
        print "asic_egress_mac\n%s\n" % pformat(self.asic_egress_mac)
        print "asic_remote_macs\n%s\n" % pformat(self.asic_remote_macs)

    def print_very_verbose(self):

        print "BGP IPv4 Prefixes"
        print "-----------------"
        for entry in sort_for_humans((map(str, self.routing.bgp_ipv4_prefixes))):
            print entry
        print

        print "Zebra BGP IPv4 Prefixes"
        print "-----------------------"
        for entry in sort_for_humans((map(str, self.routing.zebra_bgp_ipv4_prefixes))):
            print entry
        print

        print "Zebra FIB IPv4 Prefixes"
        print "-----------------------"
        for entry in sort_for_humans((map(str, self.routing.zebra_fib_ipv4_prefixes))):
            print entry
        print

        print "BGP IPv6 Prefixes"
        print "-----------------"
        for entry in sort_for_humans((map(str, self.routing.bgp_ipv6_prefixes))):
            print entry
        print

        print "Zebra BGP IPv6 Prefixes"
        print "-----------------------"
        for entry in sort_for_humans((map(str, self.routing.zebra_bgp_ipv6_prefixes))):
            print entry
        print

        print "Zebra FIB IPv6 Prefixes"
        print "-----------------------"
        for entry in sort_for_humans((map(str, self.routing.zebra_fib_ipv6_prefixes))):
            print entry
        print

        print "Zebra recursed nexthops per BGP"
        print "-------------------------------"
        for entry in sort_for_humans((map(str, self.routing.recursed_nexthops_per_bgp))):
            print entry
        print

        print "Zebra recursed nexthops per zebra"
        print "---------------------------------"
        for entry in sort_for_humans((map(str, self.routing.recursed_nexthops_per_zebra))):
            print entry
        print

        print "interfaces"
        print "----------"
        for key in sort_for_humans(self.kernel.ifaces.keys()):
            value = self.kernel.ifaces[key]
            print "%r: %r" % (key, value)
        print

        print "kernel neighbors"
        print "----------------"
        for (key, value) in sorted(self.kernel.neighs.iteritems()):
            print "%r: %r" % (key, value)
        print

        print "kernel neighbors exceptions"
        print "---------------------------"
        pprint(self.kernel.neigh_exceptions)
        print

        print "kernel neighbor macs"
        print "--------------------"
        for (key, value) in sorted(self.kernel.neigh_macs.iteritems()):
            print "%r: %r" % (key, value)
        print

        print "kernel vrfs"
        print "-----------"
        for (key, value) in sorted(self.kernel.vrfs.iteritems()):
            print "%r: %r" % (key, value)
        print

        print "kernel l3 nexthop groups"
        print "------------------------"
        for key, values in sorted(self.kernel.l3nhgs.iteritems()):
            nexthoplist = []
            for ids in values:
                nexthop = self.kernel.l3nexthops.get(ids, None)
                if nexthop:
                    nexthoplist.append(nexthop)
            print "%r: %r" % (key, nexthoplist)
        print

        print "kernel routes"
        print "-------------"
        for dest in sorted(self.kernel.routes.iterkeys()):
            print "%r: %r" % (dest, self.kernel.routes[dest])
        print

        print "kernel routes exceptions"
        print "------------------------"
        pprint(self.kernel.route_exceptions)
        print

        print "kernel onlink routes"
        print "--------------------"
        pprint(self.kernel.onlinks)
        print
        print "kernel vxlan onlink routes"
        print "--------------------"
        pprint (self.kernel.vxlan_onlinks)
        print
        print "vxlan interfaces"
        print "----------------"
        for key, values in sorted(self.kernel.vxlan_to_vlan_map.iteritems()):
            print "%r: %r" % (key, values)
        print
        print "kernel l2 nexthop groups"
        print "------------------------"
        for key, values in sorted(self.kernel.l2nhgs.iteritems()):
            nexthoplist = []
            for ids in values:
                nexthop = self.kernel.l2nexthops.get(ids, None)
                if nexthop:
                    nexthoplist.append(nexthop)
            print "%r: %r" % (key, nexthoplist)

        print
        print "kernel remote macs"
        print "------------------"
        for key, values in sorted(self.kernel.remote_macs.iteritems()):
            print "%r: %r" % (key, values)
        print

        print "kernel remote mac exceptions"
        print "----------------------------"
        pprint(self.kernel.remotemac_exceptions)
        print

        if self.asic:
            if self.asic.split_ipv6_local:
                print "kernel IPv6 local routes"
                print "------------------------"
                for dest in sorted(self.kernel.ipv6_local_routes.iterkeys()):
                    print "%r: %r" % (dest, self.kernel.ipv6_local_routes[dest])
                print

            print "HW intfs"
            print "--------"
            for intf in sorted(self.asic.intfs.iterkeys()):
                print "%r: %r" % (intf, self.asic.intfs[intf])
            print

            print "HW hosts"
            print "--------"
            for prefix in sorted(self.asic.hosts.iterkeys()):
                print "%r: %r" % (prefix, self.asic.hosts[prefix])
            print

            if self.asic.split_ipv6_local:
                print "HW IPv6 hosts"
                print "-------------"
                for prefix in sorted(self.asic.hostsipv6.iterkeys()):
                    print "%r: %r" % (prefix, self.asic.hostsipv6[prefix])
                print

            print "HW egress"
            print "---------"
            for egress_ndx in sorted(self.asic.egresses.iterkeys()):
                print "%r: %r" % (egress_ndx, self.asic.egresses[egress_ndx])
            print

            print "HW multipaths"
            print "-------------"
            for mp_ndx in sorted(self.asic.multipaths.iterkeys()):
                print "%r: %r" % (mp_ndx, self.asic.multipaths[mp_ndx])
            print

            print "HW IPv4 routes"
            print "--------------"
            for dest in sorted(self.asic.ip4_routes.iterkeys()):
                print "%r: %r" % (dest, self.asic.ip4_routes[dest])
            print

            print "HW IPv6 routes"
            print "--------------"
            for dest in sorted(self.asic.ip6_routes.iterkeys()):
                print "%r: %r" % (dest, self.asic.ip6_routes[dest])
            print

            print "HW routes"
            print "---------"
            for dest in sorted(self.asic.hroutes.iterkeys()):
                print "%r: %r" % (dest, self.asic.hroutes[dest])
            print

            if isinstance(self.asic, Mellanox):
                print "HW L2 ECMP Groups"
                print "-----------------"
                for key, values in sorted(self.asic.l2_ecmp.iteritems()):
                    print "%r: %r" % (key, values)
                print
            
                print "HW Remote Macs"
                print "--------------"
                for key, values in sorted(self.asic.remote_mac_table.iteritems()):
                    print "%r: %r" % (key, values)
                print
                
            

def get_non_swp_config():

    ignore_non_swps = False
    swp_config_file = '/cumulus/switchd/config/ignore_non_swps'
    try:
        with open(swp_config_file, 'r') as f:
            config_str = f.readline()
    except IOError:
        return True

    config_str = config_str.rstrip()
    if config_str.lower() == 'true':
        ignore_non_swps = True

    return ignore_non_swps


def ignore_iface(iface):
    return (iface.startswith('eth') or iface == "lo" or iface == "mgmt" or iface == "swid0_eth") and ignore_non_swps

def get_wfi_enabled():
    rc = False
    route_offload_config_file = '/etc/cumulus/switchd.d/kernel_route_offload_flags.conf'
    try:
        with open(route_offload_config_file , 'r') as f:
            for line in f:
                if line.startswith("kernel_route_offload_flags"):
                    var =int(line.split('=')[1].strip())
                    if (var == 1 or var == 2):
                        return True
                    else:
                        return False
    except:
        return rc
    return rc
if __name__ == '__main__':

    if os.getuid():
        print "Sorry, need to run with admin privs"
        sys.exit(1)

    # command line args
    parser = argparse.ArgumentParser(
        description="cl-route-check: verify kernel neighbor/route tables vs. hardware neighbor/route tables",
    )

    mode = parser.add_mutually_exclusive_group(required=False)
    mode.add_argument('--hardware', default=True, action='store_true', help='Verify kernel down to hardware')
    mode.add_argument('--layer3', default=False, action='store_true', help='Verify routing down to kernel')
    mode.add_argument('--all', default=False, action='store_true', help='Verify routing down to kernel and kernel down to hardware')

    option = parser.add_mutually_exclusive_group(required=False)
    option.add_argument('-j', '--json', default=False, action='store_true', help='JSON output')
    option.add_argument('-r', '--raw', default=False, action='store_true', help='Raw data for debugging')
    option.add_argument('-v', '--verbose', default=False, action='store_true')
    option.add_argument('-V', '--very-verbose', default=False, action='store_true')
    parser.add_argument('--version', action='version', version='%(prog)s 1.1')
    args = parser.parse_args()

    if args.all:
        check_hardware = True
        check_layer3 = True
    elif args.layer3:
        check_hardware = False
        check_layer3 = True
    else:
        check_hardware = True
        check_layer3 = False

    # read the 'ignore_non_swps'
    ignore_non_swps = get_non_swp_config()
    wfi_enabled  = get_wfi_enabled()
    routing = Routing()
    kernel = Kernel()
    asic = None

    # collect data
    if check_hardware:
        # create the ASIC object
        platform_object = cumulus.platforms.probe()
        chip = platform_object.switch.chip
        if chip.sw_base == 'bcm':
            if chip.dnx:
                asic = BroadcomDNX()
            else:
                asic = Broadcom()
        elif chip.sw_base == 'mlx':
            asic = Mellanox()
            from cumulus.mlx import mlx_open_connection
            from cumulus.mlx import mlx_close_connection
            from cumulus.mlx import mlx_get_intf
            from cumulus.mlx import mlx_get_interface
            from cumulus.mlx import mlx_get_neighbor
            from cumulus.mlx import mlx_get_uc_route
            from cumulus.mlx import mlx_get_ecmp
            from cumulus.mlx import mlx_get_operational_ecmp
            from cumulus.mlx import mlx_get_fdb
        else:
            raise NotImplementedError

        asic.collect_data()

    if check_layer3 and not is_service_running("frr"):
        print "cannot check layer3 as frr is not running\n"
        check_layer3 = False
    if check_layer3:
        routing.collect_data()

    kernel.collect_data()

    # compare data
    comp = Comparer(routing, kernel, asic)

    if check_hardware:
        comp.compute_kernel_vs_asic_deltas()

    if check_layer3:
        comp.compute_bgp_vs_zebra_deltas()
        comp.compute_zebra_recursion_deltas()
        comp.compute_zebra_vs_kernel_deltas()

    if args.json:
        sys.stdout.write(comp.get_result_json() + '\n')
    else:
        sys.stdout.write(comp.get_result_str(args.verbose, args.very_verbose) + '\n')

        if args.raw:
            comp.print_all()
        elif args.very_verbose:
            comp.print_very_verbose()

    sys.exit(comp.ret_code)
