From b6026461b72fd7dc924d91bb742f9002b8fd90bc Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Tue, 17 Sep 2024 13:32:51 +0200 Subject: [PATCH 1/7] refactoring WIP draft ideas --- cosmo/serializer.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/cosmo/serializer.py b/cosmo/serializer.py index b075787..a3a20bb 100644 --- a/cosmo/serializer.py +++ b/cosmo/serializer.py @@ -705,6 +705,30 @@ def serialize(self): return device_stub +class SwitchElement(abc.ABC): + @abc.abstractmethod + def serialize(): + pass + +class SwitchElementCompound(SwitchElement): + # children is a list + def add(c: SwitchElement): + raise NotImplemented + + def remove(c: SwitchElement): + raise NotImplemented + + def getChildren(c: SwitchElement): + raise NotImplemented + + def serialize(): + raise NotImplemented + +class InterfaceSwitchElement(SwitchElement): + def serialize(): + raise NotImplemented + + class SwitchSerializer: def __init__(self, device): self.device = device From 4a168a2e60b92e4896b07890b7d182ea5b294d4d Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Thu, 6 Feb 2025 17:43:53 +0100 Subject: [PATCH 2/7] add object-transformation of netbox data + base visitor --- cosmo/clients/netbox_v4.py | 13 +++++++ cosmo/types.py | 75 ++++++++++++++++++++++++++++++++++++++ cosmo/visitors.py | 50 +++++++++++++++++++++++++ 3 files changed, 138 insertions(+) create mode 100644 cosmo/types.py create mode 100644 cosmo/visitors.py diff --git a/cosmo/clients/netbox_v4.py b/cosmo/clients/netbox_v4.py index 949a929..c872e82 100644 --- a/cosmo/clients/netbox_v4.py +++ b/cosmo/clients/netbox_v4.py @@ -251,24 +251,30 @@ def _fetch_data(self, kwargs): device_list(filters: { name: { i_exact: $device }, }) { + __typename id name serial device_type { + __typename slug } platform { + __typename manufacturer { + __typename slug } slug } primary_ip4 { + __typename address } interfaces { + __typename id name enabled @@ -277,29 +283,36 @@ def _fetch_data(self, kwargs): mtu description vrf { + __typename id } lag { + __typename id } ip_addresses { + __typename address } untagged_vlan { + __typename id name vid } tagged_vlans { + __typename id name vid } tags { + __typename name slug } parent { + __typename id mtu } diff --git a/cosmo/types.py b/cosmo/types.py new file mode 100644 index 0000000..5082998 --- /dev/null +++ b/cosmo/types.py @@ -0,0 +1,75 @@ +import abc + + +class AbstractNetboxType(abc.ABC, dict): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.__mappings = {} + for c in AbstractNetboxType.__subclasses__(): + self.__mappings.update(c.register()) + for k, v in self.items(): + self[k] = self.convert(v) + + def convert(self, item): + if isinstance(item, dict): + if "__typename" in item.keys(): + c = self.__mappings[item["__typename"]] + return c({k: self.convert(v) for k,v in item.items()}) + else: + return item + elif isinstance(item, list): + replacement = [] + for i in item: + replacement.append(self.convert(i)) + return replacement + else: + return item + + @classmethod + def _getNetboxType(cls): + # classes should have the same name as the type name + # if not, you can override in parent class + return cls.__name__ + + @classmethod + def register(cls) -> dict: + return {cls._getNetboxType(): cls} + + def __repr__(self): + return self._getNetboxType() + + +class DeviceType(AbstractNetboxType): + pass + + +class DeviceTypeType(AbstractNetboxType): + pass + + +class PlatformType(AbstractNetboxType): + pass + + +class ManufacturerType(AbstractNetboxType): + pass + + +class IPAddressType(AbstractNetboxType): + pass + + +class InterfaceType(AbstractNetboxType): + pass + + +class VRFType(AbstractNetboxType): + pass + + +class TagType(AbstractNetboxType): + pass + + +class VLANType(AbstractNetboxType): + pass diff --git a/cosmo/visitors.py b/cosmo/visitors.py new file mode 100644 index 0000000..6f13710 --- /dev/null +++ b/cosmo/visitors.py @@ -0,0 +1,50 @@ +import abc +from copy import deepcopy +from functools import singledispatchmethod + + +class AbstractNoopNetboxTypesVisitor(abc.ABC): + def __init__(self, *args, **kwargs): + for c in AbstractNoopNetboxTypesVisitor.__subclasses__(): + self.accept.register( + c, + lambda self, o: self._dictLikeTemplateMethod(o) + ) + + @singledispatchmethod + def accept(self, o): + raise NotImplementedError(f"unsupported type {o}") + + @accept.register + def _(self, o: int): + return o + + @accept.register + def _(self, o: None): + return o + + @accept.register + def _(self, o: str): + return o + + def _dictLikeTemplateMethod(self, o): + breakpoint() + o = deepcopy(o) + keys = list(o.keys()) + for key in keys: + self._mutateDictKVTemplateMethod(o, key) + return o + + def _mutateDictKVTemplateMethod(self, o, key): + o[key] = self.accept(o[key]) + + @accept.register + def _(self, o: dict) -> dict: + return self._dictLikeTemplateMethod(o) + + @accept.register + def _(self, o: list) -> list: + o = deepcopy(o) + for i, v in enumerate(o): + o[i] = self.accept(v) + return o From e1fbf1121e4a4c06836b98be830f7e49fd7d6c3b Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Fri, 7 Feb 2025 10:36:31 +0100 Subject: [PATCH 3/7] remove breakpoint --- cosmo/visitors.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cosmo/visitors.py b/cosmo/visitors.py index 6f13710..907e86a 100644 --- a/cosmo/visitors.py +++ b/cosmo/visitors.py @@ -28,7 +28,6 @@ def _(self, o: str): return o def _dictLikeTemplateMethod(self, o): - breakpoint() o = deepcopy(o) keys = list(o.keys()) for key in keys: From 7697fab2865a04b520118935bd97725ddf38851f Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Fri, 7 Feb 2025 12:29:18 +0100 Subject: [PATCH 4/7] add some properties --- cosmo/types.py | 72 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/cosmo/types.py b/cosmo/types.py index 5082998..d8f008e 100644 --- a/cosmo/types.py +++ b/cosmo/types.py @@ -39,20 +39,86 @@ def __repr__(self): return self._getNetboxType() +class AbstractManufacturerStrategy(abc.ABC): + def matches(self, manuf_slug): + return True if manuf_slug == self.mySlug() else False + @abc.abstractmethod + def mySlug(self): + pass + @abc.abstractmethod + def getRoutingInstance(self): + pass + @abc.abstractmethod + def getManagementInterface(self): + pass + @abc.abstractmethod + def getBmcInterface(self): + pass + +class JuniperManufacturerStrategy(AbstractManufacturerStrategy): + def mySlug(self): + return "juniper" + def getRoutingInstance(self): + return "mgmt_junos" + def getManagementInterface(self): + return "fxp0" + def getBmcInterface(self): + return None + +class RtBrickManufacturerStrategy(AbstractManufacturerStrategy): + def mySlug(self): + return "rtbrick" + def getRoutingInstance(self): + return "mgmt" + def getManagementInterface(self): + return "ma1" + def getBmcInterface(self): + return "bmc0" + +# POJO style store class DeviceType(AbstractNetboxType): - pass + manufacturer_strategy: AbstractManufacturerStrategy = None + + def getPlatformManufacturer(self): + return self.getPlatform().getManufacturer().getSlug() + + def getManufacturerStrategy(self): + if self.manufacturer_strategy: + return self.manufacturer_strategy + else: + slug = self.getPlatformManufacturer() + for c in AbstractManufacturerStrategy.__subclasses__(): + if c().matches(slug): + self.manufacturer_strategy = c(); break + return self.manufacturer_strategy + + def getRoutingInstance(self): + return self.getManufacturerStrategy().getRoutingInstance() + def getManagementInterface(self): + return self.getManufacturerStrategy().getManagementInterface() + + def getBmcInterface(self): + return self.getManufacturerStrategy().getBmcInterface() + + def getDeviceType(self): + return self['device_type'] + + def getPlatform(self): + return self['platform'] class DeviceTypeType(AbstractNetboxType): pass class PlatformType(AbstractNetboxType): - pass + def getManufacturer(self): + return self['manufacturer'] class ManufacturerType(AbstractNetboxType): - pass + def getSlug(self): + return self['slug'] class IPAddressType(AbstractNetboxType): From 3408778afdfaf6ded5976820deffaffab621d072 Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Fri, 7 Feb 2025 15:25:50 +0100 Subject: [PATCH 5/7] more sophisticated getters for special interfaces --- cosmo/common.py | 2 ++ cosmo/types.py | 49 ++++++++++++++++++++++++++++++++++++------------- 2 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 cosmo/common.py diff --git a/cosmo/common.py b/cosmo/common.py new file mode 100644 index 0000000..79804bd --- /dev/null +++ b/cosmo/common.py @@ -0,0 +1,2 @@ +def head(l): + return None if not l else l[0] diff --git a/cosmo/types.py b/cosmo/types.py index d8f008e..1b4cace 100644 --- a/cosmo/types.py +++ b/cosmo/types.py @@ -1,4 +1,5 @@ import abc +from .common import head class AbstractNetboxType(abc.ABC, dict): @@ -46,33 +47,33 @@ def matches(self, manuf_slug): def mySlug(self): pass @abc.abstractmethod - def getRoutingInstance(self): + def getRoutingInstanceName(self): pass @abc.abstractmethod - def getManagementInterface(self): + def getManagementInterfaceName(self): pass @abc.abstractmethod - def getBmcInterface(self): + def getBmcInterfaceName(self): pass class JuniperManufacturerStrategy(AbstractManufacturerStrategy): def mySlug(self): return "juniper" - def getRoutingInstance(self): + def getRoutingInstanceName(self): return "mgmt_junos" - def getManagementInterface(self): + def getManagementInterfaceName(self): return "fxp0" - def getBmcInterface(self): + def getBmcInterfaceName(self): return None class RtBrickManufacturerStrategy(AbstractManufacturerStrategy): def mySlug(self): return "rtbrick" - def getRoutingInstance(self): + def getRoutingInstanceName(self): return "mgmt" - def getManagementInterface(self): + def getManagementInterfaceName(self): return "ma1" - def getBmcInterface(self): + def getBmcInterfaceName(self): return "bmc0" # POJO style store @@ -82,6 +83,7 @@ class DeviceType(AbstractNetboxType): def getPlatformManufacturer(self): return self.getPlatform().getManufacturer().getSlug() + # can't @cache, non-hashable def getManufacturerStrategy(self): if self.manufacturer_strategy: return self.manufacturer_strategy @@ -92,14 +94,27 @@ def getManufacturerStrategy(self): self.manufacturer_strategy = c(); break return self.manufacturer_strategy + def getInterfaceByName(self, name): + l = list( + filter( + lambda i: i.getInterfaceName() == name, + self.getInterfaces() + ) + ) + return l if l != [] else None + def getRoutingInstance(self): - return self.getManufacturerStrategy().getRoutingInstance() + return self.getManufacturerStrategy().getRoutingInstanceName() def getManagementInterface(self): - return self.getManufacturerStrategy().getManagementInterface() + return head(self.getInterfaceByName( + self.getManufacturerStrategy().getManagementInterfaceName() + )) def getBmcInterface(self): - return self.getManufacturerStrategy().getBmcInterface() + return head(self.getInterfaceByName( + self.getManufacturerStrategy().getBmcInterfaceName() + )) def getDeviceType(self): return self['device_type'] @@ -107,6 +122,10 @@ def getDeviceType(self): def getPlatform(self): return self['platform'] + def getInterfaces(self): + return self['interfaces'] + + class DeviceTypeType(AbstractNetboxType): pass @@ -126,7 +145,11 @@ class IPAddressType(AbstractNetboxType): class InterfaceType(AbstractNetboxType): - pass + def __repr__(self): + return super().__repr__() + f"({self.getInterfaceName()})" + + def getInterfaceName(self): + return self['name'] class VRFType(AbstractNetboxType): From b00dfb919431c55911fee3e33307d22cda0cd62d Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Fri, 7 Feb 2025 15:38:35 +0100 Subject: [PATCH 6/7] remove dead code --- cosmo/serializer.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/cosmo/serializer.py b/cosmo/serializer.py index a3a20bb..b075787 100644 --- a/cosmo/serializer.py +++ b/cosmo/serializer.py @@ -705,30 +705,6 @@ def serialize(self): return device_stub -class SwitchElement(abc.ABC): - @abc.abstractmethod - def serialize(): - pass - -class SwitchElementCompound(SwitchElement): - # children is a list - def add(c: SwitchElement): - raise NotImplemented - - def remove(c: SwitchElement): - raise NotImplemented - - def getChildren(c: SwitchElement): - raise NotImplemented - - def serialize(): - raise NotImplemented - -class InterfaceSwitchElement(SwitchElement): - def serialize(): - raise NotImplemented - - class SwitchSerializer: def __init__(self, device): self.device = device From 15be47013b0eb041e9f3faa49a13d5796a411cce Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Fri, 7 Feb 2025 18:07:27 +0100 Subject: [PATCH 7/7] add the possibility to get item parent in composite tree --- cosmo/common.py | 6 ++++++ cosmo/types.py | 23 ++++++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/cosmo/common.py b/cosmo/common.py index 79804bd..b174c2c 100644 --- a/cosmo/common.py +++ b/cosmo/common.py @@ -1,2 +1,8 @@ def head(l): return None if not l else l[0] + + +def without_keys(d, keys) -> dict: + if type(keys) != list: + keys = [keys] + return {k: v for k,v in d.items() if k not in keys} diff --git a/cosmo/types.py b/cosmo/types.py index 1b4cace..95b8252 100644 --- a/cosmo/types.py +++ b/cosmo/types.py @@ -1,26 +1,31 @@ import abc -from .common import head +from copy import deepcopy +from .common import head, without_keys class AbstractNetboxType(abc.ABC, dict): + __parent = None + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__mappings = {} for c in AbstractNetboxType.__subclasses__(): self.__mappings.update(c.register()) - for k, v in self.items(): + for k, v in without_keys(self, "__parent").items(): self[k] = self.convert(v) def convert(self, item): if isinstance(item, dict): if "__typename" in item.keys(): c = self.__mappings[item["__typename"]] - return c({k: self.convert(v) for k,v in item.items()}) + # self descending in tree + return c({k: self.convert(v) for k, v in without_keys(item, "__parent").items()} | {"__parent": self}) else: return item elif isinstance(item, list): replacement = [] for i in item: + # self descending in tree replacement.append(self.convert(i)) return replacement else: @@ -36,6 +41,18 @@ def _getNetboxType(cls): def register(cls) -> dict: return {cls._getNetboxType(): cls} + def getParent(self): + return self['__parent'] + + def __deepcopy__(self, memo): + # I'm using convert because we have to rebuild the circular reference + # tree, since we cannot use the old references and thus __parent + # becomes invalid. Implementing __deepcopy__ is better than implementing + # workarounds for object instances in client code. + return self.__class__().convert(self.__class__( + {k: deepcopy(v,memo) for k, v in without_keys(self, "__parent").items()} + )) + def __repr__(self): return self._getNetboxType()