dev: config migration fixes for 20.12 (#793)

* use enableEnforcedPropogation for lan/vpn/interface/ethernet when >=20.12 * annotate not-defined fields in traffic-policy * Fix aliases for ethernet model * Add cg to reverter * report cli centralized policy as unsupported (instead failed) * Mark PolicySettingsParcel as unsupported for 20.12 * Fix tests. Fix wrong variable assigement * dont create svi for transport feature profile * bump version --------- Co-authored-by: Jakub Krajewski <[email protected]>
cisco-open · Aug 7, 2024 · 1ffb529 · 1ffb529
1 parent 223bedc
commit 1ffb529
Show file tree

Hide file tree

Showing 11 changed files with 102 additions and 24 deletions.
diff --git a/ENDPOINTS.md b/ENDPOINTS.md
@@ -1,6 +1,6 @@
 **THIS FILE WAS AUTO-GENERATED DO NOT EDIT**
 
-Generated for: catalystwan-0.34.0.dev1
+Generated for: catalystwan-0.34.0.dev2
 
 All URIs are relative to */dataservice*
 HTTP request | Supported Versions | Method | Payload Type | Return Type | Tenancy Mode

diff --git a/...lystwan/models/configuration/feature_profile/sdwan/application_priority/traffic_policy.py b/...lystwan/models/configuration/feature_profile/sdwan/application_priority/traffic_policy.py
@@ -1,9 +1,18 @@
 # Copyright 2024 Cisco Systems, Inc. and its affiliates
 from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network
-from typing import List, Literal, Optional, Tuple, Type, TypeVar, Union, overload
+from typing import Any, Dict, List, Literal, Optional, Tuple, Type, TypeVar, Union, overload
 from uuid import UUID
 
-from pydantic import AliasPath, BaseModel, ConfigDict, Field, field_validator
+from pydantic import (
+    AliasPath,
+    BaseModel,
+    ConfigDict,
+    Field,
+    SerializationInfo,
+    SerializerFunctionWrapHandler,
+    field_validator,
+    model_serializer,
+)
 from typing_extensions import Annotated
 
 from catalystwan.api.configuration_groups.parcel import Global, _ParcelBase, as_global, as_optional_global
@@ -477,16 +486,20 @@ class LossCorrection(BaseModel):
 
 class Nat(BaseModel):
     model_config = ConfigDict(populate_by_name=True, extra="forbid")
-    bypass: Optional[Global[bool]] = Field(default=None)
-    dia_interface: Optional[Global[List[str]]] = Field(
+    bypass: Annotated[Optional[Global[bool]], VersionedField(versions="<20.14", forbidden=True)] = Field(default=None)
+    dia_interface: Annotated[Optional[Global[List[str]]], VersionedField(versions="<20.14", forbidden=True)] = Field(
         default=None, validation_alias="diaInterface", serialization_alias="diaInterface"
     )
-    dia_pool: Optional[Global[List[int]]] = Field(
+    dia_pool: Annotated[Optional[Global[List[int]]], VersionedField(versions="<20.14", forbidden=True)] = Field(
         default=None, validation_alias="diaPool", serialization_alias="diaPool"
     )
     fallback: Optional[Global[bool]] = Field(default=None)
     use_vpn: Global[bool] = Field(default=None, validation_alias="useVpn", serialization_alias="useVpn")
 
+    @model_serializer(mode="wrap", when_used="json")
+    def serialize(self, handler: SerializerFunctionWrapHandler, info: SerializationInfo) -> Dict[str, Any]:
+        return VersionedField.dump(self.model_fields, handler(self), info)
+
 
 SecureServiceEdgeInstance = Literal[
     "Cisco-Secure-Access",
@@ -950,13 +963,13 @@ def associate_loss_correction_action(self, type: LossProtectionType, fec: Option
         self._insert_action(LossCorrectionAction(loss_correction=loss_correction))
 
     def associate_nat_action(
-        self, bypass: bool, dia_interface: List[str], dia_pool: List[int], fallback: bool, use_vpn: bool
+        self, bypass: bool, dia_interface: List[str], dia_pool: List[int], fallback: Optional[bool], use_vpn: bool
     ) -> None:
         nat = Nat(
             bypass=as_global(bypass),
             dia_interface=as_global(dia_interface) if dia_interface else None,
             dia_pool=as_global(dia_pool) if dia_pool else None,
-            fallback=as_global(fallback),
+            fallback=as_optional_global(fallback),
             use_vpn=as_global(use_vpn),
         )
         self._insert_action(NatAction(nat=nat))

diff --git a/catalystwan/models/configuration/feature_profile/sdwan/service/lan/ethernet.py b/catalystwan/models/configuration/feature_profile/sdwan/service/lan/ethernet.py
@@ -1,13 +1,23 @@
 # Copyright 2024 Cisco Systems, Inc. and its affiliates
 
 from ipaddress import IPv4Address
-from typing import List, Literal, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, Union
 from uuid import UUID
 
-from pydantic import AliasPath, BaseModel, ConfigDict, Field
+from pydantic import (
+    AliasChoices,
+    AliasPath,
+    BaseModel,
+    ConfigDict,
+    Field,
+    SerializationInfo,
+    SerializerFunctionWrapHandler,
+    model_serializer,
+)
+from typing_extensions import Annotated
 
 from catalystwan.api.configuration_groups.parcel import Default, Global, Variable, _ParcelBase
-from catalystwan.models.common import EthernetDuplexMode, MediaType
+from catalystwan.models.common import EthernetDuplexMode, MediaType, VersionedField
 from catalystwan.models.configuration.feature_profile.common import (
     Arp,
     DynamicDhcpDistance,
@@ -93,7 +103,10 @@ class VrrpIPv6(BaseModel):
 class VrrpIPv4(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True, populate_by_name=True, extra="forbid")
 
-    group_id: Union[Variable, Global[int]] = Field(serialization_alias="groupId", validation_alias="groupId")
+    group_id: Annotated[
+        Union[Variable, Global[int]],
+        VersionedField(versions="<=20.12", serialization_alias="group_id"),
+    ] = Field(serialization_alias="groupId", validation_alias=AliasChoices("groupId", "group_id"))
     priority: Union[Variable, Global[int], Default[int]] = Default[int](value=100)
     timer: Union[Variable, Global[int], Default[int]] = Default[int](value=1000)
     track_omp: Union[Global[bool], Default[bool]] = Field(
@@ -117,26 +130,43 @@ class VrrpIPv4(BaseModel):
         serialization_alias="trackingObject", validation_alias="trackingObject", default=None
     )
 
+    @model_serializer(mode="wrap", when_used="json")
+    def serialize(self, handler: SerializerFunctionWrapHandler, info: SerializationInfo) -> Dict[str, Any]:
+        return VersionedField.dump(self.model_fields, handler(self), info)
+
 
 class Trustsec(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True, populate_by_name=True, extra="forbid")
 
-    enable_sgt_propagation: Union[Global[bool], Default[bool]] = Field(
+    enable_sgt_propagation: Annotated[
+        Union[Global[bool], Default[bool]],
+        VersionedField(versions="<=20.12", serialization_alias="enableSGTPropogation"),
+    ] = Field(
         serialization_alias="enableSGTPropagation",
-        validation_alias="enableSGTPropagation",
+        validation_alias=AliasChoices("enableSGTPropagation", "enableSGTPropogation"),
         default=Default[bool](value=False),
     )
-    propagate: Optional[Union[Global[bool], Default[bool]]] = Default[bool](value=True)
+    propagate: Annotated[
+        Optional[Union[Global[bool], Default[bool]]], VersionedField(versions="<=20.12", forbidden=True)
+    ] = Default[bool](value=True)
     security_group_tag: Optional[Union[Global[int], Variable, Default[None]]] = Field(
         serialization_alias="securityGroupTag", validation_alias="securityGroupTag", default=None
     )
-    enable_enforced_propagation: Union[Global[bool], Default[None]] = Field(
-        serialization_alias="enableEnforcedPropagation", validation_alias="enableEnforcedPropagation"
+    enable_enforced_propagation: Annotated[
+        Union[Global[bool], Default[None]],
+        VersionedField(versions="<=20.12", serialization_alias="enableEnforcedPropogation"),
+    ] = Field(
+        serialization_alias="enableEnforcedPropagation",
+        validation_alias=AliasChoices("enableEnforcedPropagation", "enableEnforcedPropogation"),
     )
     enforced_security_group_tag: Union[Global[int], Variable, Default[None]] = Field(
         serialization_alias="enforcedSecurityGroupTag", validation_alias="enforcedSecurityGroupTag"
     )
 
+    @model_serializer(mode="wrap", when_used="json")
+    def serialize(self, handler: SerializerFunctionWrapHandler, info: SerializationInfo) -> Dict[str, Any]:
+        return VersionedField.dump(self.model_fields, handler(self), info)
+
 
 class AdvancedEthernetAttributes(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True, populate_by_name=True, extra="forbid")

diff --git a/catalystwan/tests/config_migration/test_transform.py b/catalystwan/tests/config_migration/test_transform.py
@@ -1,5 +1,4 @@
 # Copyright 2024 Cisco Systems, Inc. and its affiliates
-# Copyright 2023 Cisco Systems, Inc. and its affiliates
 from typing import List, Optional, Set, TypeVar
 from uuid import UUID, uuid4
 
@@ -12,6 +11,7 @@
     UX1Config,
     UX1Policies,
     UX1Templates,
+    VersionInfo,
 )
 from catalystwan.models.templates import FeatureTemplateInformation
 from catalystwan.tests.config_migration.test_data import (
@@ -546,6 +546,7 @@ def test_when_localized_policy_with_qos_expect_application_priority_feature_prof
             policy_definitions=[qos_map_1, qos_map_2],
         )
     )
+    ux1_config.version = VersionInfo(platform="20.15")  # To include all items in the output)
     # Act
     ux2_config = transform(ux1_config).ux2_config
     # Find application priority feature profile
@@ -591,6 +592,7 @@ def test_policy_profile_merge():
 
     """
     ux1 = UX1Config()
+    ux1.version = VersionInfo(platform="20.15")  # To include all items in the output)
 
     sig_1_uuid = uuid4()
 

diff --git a/catalystwan/utils/config_migration/converters/policy/centralized_policy.py b/catalystwan/utils/config_migration/converters/policy/centralized_policy.py
@@ -15,6 +15,7 @@
     TransformedParcel,
     TransformedTopologyGroup,
     TransformHeader,
+    UnsupportedConversionItem,
     UX1Config,
     UX2Config,
 )
@@ -89,6 +90,7 @@ def __init__(self, ux1: UX1Config, context: PolicyConvertContext, ux2: UX2Config
         self.parcel_lookup: Dict[UUID, List[TransformedParcel]] = dict()
         self._create_parcel_by_policy_id_lookup()
         self.failed_items: List[FailedConversionItem] = list()
+        self.unsupported_items: List[UnsupportedConversionItem] = list()
 
     def _create_parcel_by_policy_id_lookup(self) -> None:
         for policy_definition in self.ux1.policies.policy_definitions:
@@ -218,7 +220,13 @@ def update_groups_and_profiles(self) -> None:
                 if dst_transformed_app_prio_parcels:
                     self.update_app_prio_profiles(centralized_policy, dst_transformed_app_prio_parcels)
             else:
-                problems.append("cli policy definition not supported")
+                self.unsupported_items.append(
+                    UnsupportedConversionItem(
+                        name=centralized_policy.policy_name,
+                        uuid=centralized_policy.policy_id,
+                        type=centralized_policy.policy_type,
+                    )
+                )
             if problems:
                 self.failed_items.append(
                     FailedConversionItem(policy=centralized_policy, exception_message="\n".join(problems))

diff --git a/catalystwan/utils/config_migration/converters/policy/policy_settings.py b/catalystwan/utils/config_migration/converters/policy/policy_settings.py
@@ -1,6 +1,8 @@
 # Copyright 2024 Cisco Systems, Inc. and its affiliates
 from uuid import UUID
 
+from packaging.version import Version
+
 from catalystwan.api.configuration_groups.parcel import as_optional_global
 from catalystwan.models.configuration.config_migration import ConvertResult, PolicyConvertContext
 from catalystwan.models.configuration.feature_profile.sdwan.application_priority.policy_settings import (
@@ -12,6 +14,14 @@
 def convert_localized_policy_settings(
     policy: LocalizedPolicyInfo, uuid: UUID, context: PolicyConvertContext
 ) -> ConvertResult[PolicySettingsParcel]:
+    if context.platform_version < Version("20.13"):
+        return ConvertResult(
+            status="unsupported",
+            info=[
+                f"Localized policy {policy.policy_name} is not supported on platform version {context.platform_version}"
+            ],
+        )
+
     if isinstance(policy.policy_definition, str):
         return ConvertResult(
             output=None,

diff --git a/catalystwan/utils/config_migration/creators/localized_policy_pusher.py b/catalystwan/utils/config_migration/creators/localized_policy_pusher.py
@@ -185,6 +185,9 @@ def push(self):
             profile_type, profile_id, profile_name, transformed_parcels = profile_info
             self._progress(f"Updating {profile_name} profile with policy parcels", i + 1, len(profile_ids))
             for transformed_parcel in transformed_parcels:
+                if isinstance(transformed_parcel.parcel, RoutePolicyParcel):
+                    logger.warning("Route Policy is not supported in localized policy")
+                    continue
                 report = profile_reports[profile_id]
                 error_parcel = transformed_parcel.parcel
                 error_info: Union[ManagerErrorInfo, str, None] = None

diff --git a/catalystwan/utils/config_migration/reverters/config_reverter.py b/catalystwan/utils/config_migration/reverters/config_reverter.py
@@ -23,6 +23,14 @@ def rollback(self, rollback_info: UX2RollbackInfo, progress: Callable[[str, int,
                 all_deleted = False
                 logger.error(f"Error occured during deleting config group {cg_id}: {e}")
 
+        for i, pg in enumerate(rollback_info.policy_group_ids):
+            try:
+                self._session.endpoints.configuration.policy_group.delete(pg)
+                progress("Removing Policy Groups", i + 1, len(rollback_info.policy_group_ids))
+            except CatalystwanException as e:
+                all_deleted = False
+                logger.error(f"Error occured during deleting policy group {pg}: {e}")
+
         for i, tg_id in enumerate(rollback_info.topology_group_ids):
             try:
                 self._session.endpoints.configuration.topology_group.delete(tg_id)

diff --git a/catalystwan/utils/config_migration/steps/constants.py b/catalystwan/utils/config_migration/steps/constants.py
@@ -26,13 +26,16 @@
     "bgp",
     "cisco_ospfv3",
     "cisco_ospf",
-    "ospf"
+    "ospf",
 ]
 
 NO_SUBSTITUTE_ERROR = "NO_SUBSTITUTE_ERROR"
 NO_SUBSTITUTE_VPN_MANAGEMENT_SVI = (
     "NO_SUBSTITUTE_ERROR: UX1.0 -> We can attach SVI to vpn 512, UX2.0 -> There is no SVI parcel for vpn 512"
 )
+NO_SUBSTITUTE_VPN_TRANSPORT_SVI = (
+    "NO_SUBSTITUTE_ERROR: UX1.0 -> We can attach SVI to vpn 0, UX2.0 -> There is no SVI parcel for vpn 0"
+)
 
 MANAGEMENT_VPN_ETHERNET = "management/vpn/interface/ethernet"
 
@@ -75,7 +78,7 @@
             "vpn-vedge-interface-gre": WAN_VPN_GRE,
             "cisco_vpn_interface_gre": WAN_VPN_GRE,
             "cisco_vpn_interface_ipsec": WAN_VPN_IPSEC,
-            "vpn-interface-svi": WAN_VPN_SVI,
+            "vpn-interface-svi": NO_SUBSTITUTE_VPN_TRANSPORT_SVI,
             "cisco_vpn_interface": WAN_VPN_ETHERNET,
             "vpn-vsmart-interface": WAN_VPN_ETHERNET,
             "vpn-vedge-interface": WAN_VPN_ETHERNET,

diff --git a/catalystwan/workflows/config_migration.py b/catalystwan/workflows/config_migration.py
@@ -455,8 +455,8 @@ def transform(ux1: UX1Config, add_suffix: bool = False) -> ConfigTransformResult
         if settings_status == "unsupported":
             transform_result.add_unsupported_item(
                 name=f"{localized_policy.policy_name} Settings",
-                uuid=policy_definition.definition_id,
-                type=policy_definition.type,
+                uuid=localized_policy.policy_id,
+                type=localized_policy.policy_type,
             )
         elif settings_status == "failed":
             transform_result.add_failed_conversion_parcel(
@@ -530,6 +530,7 @@ def transform(ux1: UX1Config, add_suffix: bool = False) -> ConfigTransformResult
     centralized_policy_converter = CentralizedPolicyConverter(ux1=ux1, context=policy_context, ux2=ux2)
     centralized_policy_converter.update_groups_and_profiles()
     transform_result.failed_items.extend(centralized_policy_converter.failed_items)
+    transform_result.unsupported_items.extend(centralized_policy_converter.unsupported_items)
 
     # Add additional objects emmited by the conversion
     ux2.thread_grid_api = policy_context.threat_grid_api

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "catalystwan"
-version = "0.34.0dev1"
+version = "0.34.0dev2"
 description = "Cisco Catalyst WAN SDK for Python"
 authors = ["kagorski <[email protected]>"]
 readme = "README.md"