Feat/dbt v9 (#523)

* bump * adds support for v9 * bump and publish * updated lock --------- Co-authored-by: Edward Louth <[email protected]>
grai-io · Aug 2, 2023 · a424ff1 · a424ff1
1 parent cd05bc4
commit a424ff1
Show file tree

Hide file tree

Showing 14 changed files with 258 additions and 195 deletions.
diff --git a/grai-integrations/source-dbt/poetry.lock b/grai-integrations/source-dbt/poetry.lock
diff --git a/grai-integrations/source-dbt/pyproject.toml b/grai-integrations/source-dbt/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "grai_source_dbt"
-version = "0.3.0"
+version = "0.3.1"
 description = ""
 authors = ["Ian Eaves <[email protected]>", "Edward Louth <[email protected]>"]
 license = "Elastic-2.0"
@@ -18,7 +18,7 @@ python = "^3.8"
 pydantic = "^1.9.1"
 grai-client = "^0.3.0"
 grai-schemas = "^0.2.0"
-dbt-artifacts-parser = "^0.2.4"
+dbt-artifacts-parser = "^v0.4.0-rc1 "
 
 [tool.poetry.group.dev.dependencies]
 isort = "^5.10.1"

diff --git a/grai-integrations/source-dbt/src/grai_source_dbt/__init__.py b/grai-integrations/source-dbt/src/grai_source_dbt/__init__.py
@@ -9,4 +9,4 @@
 )
 from grai_source_dbt.package_definitions import config
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"
diff --git a/grai-integrations/source-dbt/src/grai_source_dbt/loaders/__init__.py b/grai-integrations/source-dbt/src/grai_source_dbt/loaders/__init__.py
@@ -2,16 +2,33 @@
 
 from dbt_artifacts_parser.parsers.version_map import ArtifactTypes
 
-from grai_source_dbt.loaders import base, utils, v1, v2, v3, v4, v5, v6, v7, v8
+from grai_source_dbt.loaders import base, utils, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10
 from grai_source_dbt.loaders.v1 import ManifestLoaderV1
 from grai_source_dbt.loaders.v7 import ManifestLoaderV7
 from grai_source_dbt.utils import full_name, set_extra_fields
 
 ManifestTypes = Union[
-    v1.ManifestV1, v2.ManifestV2, v3.ManifestV3, v4.ManifestV4, v6.ManifestV6, v7.ManifestV7, v8.ManifestV8
+    v1.ManifestV1,
+    v2.ManifestV2,
+    v3.ManifestV3,
+    v4.ManifestV4,
+    v6.ManifestV6,
+    v7.ManifestV7,
+    v8.ManifestV8,
+    v9.ManifestV9,
+    v10.ManifestV10,
 ]
 NodeTypes = Union[
-    v1.NodeTypes, v2.NodeTypes, v3.NodeTypes, v4.NodeTypes, v5.NodeTypes, v6.NodeTypes, v7.NodeTypes, v8.NodeTypes
+    v1.NodeTypes,
+    v2.NodeTypes,
+    v3.NodeTypes,
+    v4.NodeTypes,
+    v5.NodeTypes,
+    v6.NodeTypes,
+    v7.NodeTypes,
+    v8.NodeTypes,
+    v9.NodeTypes,
+    v10.NodeTypes,
 ]
 
 SourceTypes = Union[
@@ -23,6 +40,8 @@
     v6.SourceTypes,
     v7.SourceTypes,
     v8.SourceTypes,
+    v9.SourceTypes,
+    v10.SourceTypes,
 ]
 AllDbtNodeTypes = Union[NodeTypes, SourceTypes]
 AllDbtNodeInstances = get_args(AllDbtNodeTypes)
@@ -68,6 +87,8 @@ def source_full_name(obj: SourceTypes) -> str:
     ArtifactTypes.MANIFEST_V6.value.dbt_schema_version: ManifestLoaderV1,
     ArtifactTypes.MANIFEST_V7.value.dbt_schema_version: ManifestLoaderV7,
     ArtifactTypes.MANIFEST_V8.value.dbt_schema_version: ManifestLoaderV7,
+    ArtifactTypes.MANIFEST_V9.value.dbt_schema_version: ManifestLoaderV7,
+    # ArtifactTypes.MANIFEST_V10.value.dbt_schema_version: ManifestLoaderV7,
 }
 
 

diff --git a/grai-integrations/source-dbt/src/grai_source_dbt/loaders/v10.py b/grai-integrations/source-dbt/src/grai_source_dbt/loaders/v10.py
@@ -0,0 +1,29 @@
+from typing import Union, get_args
+
+from dbt_artifacts_parser.parsers.manifest.manifest_v10 import (
+    AnalysisNode,
+    GenericTestNode,
+    HookNode,
+    ManifestV10,
+    ModelNode,
+    RPCNode,
+    SeedNode,
+    SingularTestNode,
+    SnapshotNode,
+    SourceDefinition,
+    SqlNode,
+)
+
+NodeTypes = Union[
+    AnalysisNode,
+    SingularTestNode,
+    HookNode,
+    ModelNode,
+    RPCNode,
+    SqlNode,
+    GenericTestNode,
+    SnapshotNode,
+    SeedNode,
+]
+
+SourceTypes = Union[SourceDefinition]
diff --git a/grai-integrations/source-dbt/src/grai_source_dbt/loaders/v8.py b/grai-integrations/source-dbt/src/grai_source_dbt/loaders/v8.py
@@ -1,4 +1,4 @@
-from typing import Union, get_args
+from typing import Union
 
 from dbt_artifacts_parser.parsers.manifest.manifest_v8 import (
     AnalysisNode,
@@ -14,8 +14,6 @@
     SqlNode,
 )
 
-from grai_source_dbt.utils import set_extra_fields
-
 NodeTypes = Union[
     AnalysisNode,
     SingularTestNode,

diff --git a/grai-integrations/source-dbt/src/grai_source_dbt/loaders/v9.py b/grai-integrations/source-dbt/src/grai_source_dbt/loaders/v9.py
@@ -0,0 +1,29 @@
+from typing import Union
+
+from dbt_artifacts_parser.parsers.manifest.manifest_v9 import (
+    AnalysisNode,
+    GenericTestNode,
+    HookNode,
+    ManifestV9,
+    ModelNode,
+    RPCNode,
+    SeedNode,
+    SingularTestNode,
+    SnapshotNode,
+    SourceDefinition,
+    SqlNode,
+)
+
+NodeTypes = Union[
+    AnalysisNode,
+    SingularTestNode,
+    HookNode,
+    ModelNode,
+    RPCNode,
+    SqlNode,
+    GenericTestNode,
+    SnapshotNode,
+    SeedNode,
+]
+
+SourceTypes = Union[SourceDefinition]
diff --git a/grai-integrations/source-dbt/src/grai_source_dbt/processor.py b/grai-integrations/source-dbt/src/grai_source_dbt/processor.py
@@ -1,4 +1,5 @@
 import json
+import warnings
 from functools import cached_property
 from typing import List, Union
 
@@ -20,10 +21,11 @@ class ManifestProcessor:
 
     source: SourceSpec
 
-    def __init__(self, loader: BaseManifestLoader, source: SourceSpec):
+    def __init__(self, loader: BaseManifestLoader, source: SourceSpec, strict_mode: bool = False):
         self.loader = loader
         self.namespace = loader.namespace
         self.source = source
+        self.strict_mode = strict_mode
 
     @cached_property
     def adapted_nodes(self) -> List[SourcedNodeV1]:
@@ -91,12 +93,16 @@ def manifest(self) -> ManifestTypes:
         return self.loader.manifest
 
     @classmethod
-    def load(cls, manifest_obj: Union[str, dict], namespace: str, source: SourceSpec) -> "ManifestProcessor":
+    def load(
+        cls, manifest_obj: Union[str, dict], namespace: str, source: SourceSpec, strict_mode: bool = False
+    ) -> "ManifestProcessor":
         """
 
         Args:
-            manifest_obj (Union[str, dict]):
-            namespace (str):
+            manifest_obj:
+            namespace:
+            source:
+            strict_mode:
 
         Returns:
 
@@ -110,10 +116,18 @@ def load(cls, manifest_obj: Union[str, dict], namespace: str, source: SourceSpec
             manifest_dict = manifest_obj
 
         version = get_dbt_schema_version(manifest_dict)
-        if version not in cls.MANIFEST_MAP:
-            message = f"Manifest version {version} not yet supported"
-            raise NotImplementedError(message)
+
+        if version not in cls.MANIFEST_MAP and not strict_mode:
+            latest_supported_version = list(cls.MANIFEST_MAP.keys())[-1]
+            version_label = latest_supported_version.split("/")[-1].split(".")[0]
+            warnings.warn(
+                f"dbt manifest version {version} is not yet supported. We will attempt to parse the manifest as"
+                f"version {version_label} as a fallback. If you see this message please open an issue at"
+                f"www.github.com/grai-io/grai-core/issues/new"
+            )
+            version = latest_supported_version
 
         manifest_obj = parse_manifest(manifest_dict)
         manifest = cls.MANIFEST_MAP[version](manifest_obj, namespace)
+
         return ManifestProcessor(manifest, source)
diff --git a/grai-integrations/source-dbt/tests/resources/v9/jaffle_shop/manifest.json b/grai-integrations/source-dbt/tests/resources/v9/jaffle_shop/manifest.json
diff --git a/grai-integrations/source-dbt/tests/test_load.py b/grai-integrations/source-dbt/tests/test_load.py
@@ -45,21 +45,6 @@ def load_resource(file, source) -> ManifestProcessor:
     return ManifestProcessor.load(file, "default", source)
 
 
-def manifest_ver(id: str) -> ManifestProcessor:
-    """
-
-    Args:
-        id (str):
-
-    Returns:
-
-    Raises:
-
-    """
-    file = resource_path("manifest.json", version=id)
-    return ManifestProcessor.load(file, "default")
-
-
 files = [resource_path("manifest.json", ver) for ver in SUPPORTED_VERSIONS]
 files += [
     resource_path("manifest-seed.json", "v8"),
@@ -73,16 +58,7 @@ class TestProcessors:
     """ """
 
     def test_all_manifest_node_full_names_unique(self, file, mock_source):
-        """
-
-        Args:
-            processor:
-
-        Returns:
-
-        Raises:
-
-        """
+        """"""
         processor = load_resource(file, mock_source)
         node_names = {full_name(node) for node in processor.manifest.nodes.values()}
         assert len(node_names) == len(processor.manifest.nodes)

diff --git a/grai-server/app/Dockerfile b/grai-server/app/Dockerfile
@@ -25,6 +25,7 @@ RUN apt update \
       unixodbc-dev \
     && pip install "poetry==${POETRY_VERSION}" \
     && poetry config virtualenvs.create false \
+    && poetry lock \
     && poetry install --no-root --no-interaction --no-ansi \
     && poetry version -s > /version.txt