fix circular import to allow importing models and metrics directly (#56)

Previously if you tried importing model classes (models.py) or metric classes (metrics.py) directly in your third party code, you would get a circular import error. This PR reorganizes some files to fix that issue. Co-authored-by: Safoora Yousefi <[email protected]>
microsoft · Dec 6, 2024 · 601621e · 601621e
1 parent e971de5
commit 601621e
Show file tree

Hide file tree

Showing 49 changed files with 247 additions and 227 deletions.
diff --git a/eureka_ml_insights/configs/__init__.py b/eureka_ml_insights/configs/__init__.py
@@ -1,4 +1,3 @@
-from .aime import AIME_PIPELINE
 from .config import (
     AggregatorConfig,
     DataJoinConfig,
@@ -11,71 +10,7 @@
     PipelineConfig,
     PromptProcessingConfig,
 )
-from .dna import DNA_PIPELINE
-from .drop import Drop_Experiment_Pipeline
 from .experiment_config import ExperimentConfig, create_logdir
-from .flenqa import FlenQA_Experiment_Pipeline
-from .geometer import GEOMETER_PIPELINE
-from .gpqa import GPQA_Experiment_Pipeline
-from .ifeval import IFEval_PIPELINE
-from .image_understanding.object_detection import (
-    OBJECT_DETECTION_PAIRS_LOCAL_PIPELINE,
-    OBJECT_DETECTION_PAIRS_PIPELINE,
-    OBJECT_DETECTION_SINGLE_LOCAL_PIPELINE,
-    OBJECT_DETECTION_SINGLE_PIPELINE,
-)
-from .image_understanding.object_recognition import (
-    OBJECT_RECOGNITION_PAIRS_LOCAL_PIPELINE,
-    OBJECT_RECOGNITION_PAIRS_PIPELINE,
-    OBJECT_RECOGNITION_SINGLE_LOCAL_PIPELINE,
-    OBJECT_RECOGNITION_SINGLE_PIPELINE,
-)
-from .image_understanding.spatial_reasoning import (
-    SPATIAL_REASONING_PAIRS_LOCAL_PIPELINE,
-    SPATIAL_REASONING_PAIRS_PIPELINE,
-    SPATIAL_REASONING_SINGLE_LOCAL_PIPELINE,
-    SPATIAL_REASONING_SINGLE_PIPELINE,
-)
-from .image_understanding.visual_prompting import (
-    VISUAL_PROMPTING_PAIRS_LOCAL_PIPELINE,
-    VISUAL_PROMPTING_PAIRS_PIPELINE,
-    VISUAL_PROMPTING_SINGLE_LOCAL_PIPELINE,
-    VISUAL_PROMPTING_SINGLE_PIPELINE,
-)
-from .kitab import (
-    GPT35_KITAB_ONE_BOOK_CONSTRAINT_PIPELINE,
-    KITAB_ONE_BOOK_CONSTRAINT_PIPELINE,
-    KITAB_ONE_BOOK_CONSTRAINT_PIPELINE_SELF_CONTEXT,
-    KITAB_ONE_BOOK_CONSTRAINT_PIPELINE_WITH_CONTEXT,
-    KITAB_TWO_BOOK_CONSTRAINT_PIPELINE,
-    KITAB_TWO_BOOK_CONSTRAINT_PIPELINE_WITH_CONTEXT,
-)
-from .mmmu import MMMU_BASELINE_PIPELINE
-from .nondeterminism import (
-    Geo_Nondeterminism,
-    IFEval_Nondeterminism,
-    Kitab_Nondeterminism,
-    MMMU_Nondeterminism,
-)
-from .toxigen import (
-    ToxiGen_Discriminative_PIPELINE,
-    ToxiGen_Generative_PIPELINE,
-)
-from .vision_language.maze import (
-    MAZE_PIPELINE,
-    MAZE_REPORTING_PIPELINE,
-    MAZE_TEXTONLY_PIPELINE,
-)
-from .vision_language.spatial_grid import (
-    SPATIAL_GRID_PIPELINE,
-    SPATIAL_GRID_REPORTING_PIPELINE,
-    SPATIAL_GRID_TEXTONLY_PIPELINE,
-)
-from .vision_language.spatial_map import (
-    SPATIAL_MAP_PIPELINE,
-    SPATIAL_MAP_REPORTING_PIPELINE,
-    SPATIAL_MAP_TEXTONLY_PIPELINE,
-)
 
 __all__ = [
     PipelineConfig,
@@ -89,50 +24,5 @@
     DataSetConfig,
     EvalReportingConfig,
     ExperimentConfig,
-    OBJECT_DETECTION_PAIRS_PIPELINE,
-    OBJECT_DETECTION_SINGLE_PIPELINE,
-    OBJECT_DETECTION_PAIRS_LOCAL_PIPELINE,
-    OBJECT_DETECTION_SINGLE_LOCAL_PIPELINE,
-    OBJECT_RECOGNITION_PAIRS_PIPELINE,
-    OBJECT_RECOGNITION_SINGLE_PIPELINE,
-    OBJECT_RECOGNITION_PAIRS_LOCAL_PIPELINE,
-    OBJECT_RECOGNITION_SINGLE_LOCAL_PIPELINE,
-    SPATIAL_REASONING_PAIRS_PIPELINE,
-    SPATIAL_REASONING_SINGLE_PIPELINE,
-    SPATIAL_REASONING_PAIRS_LOCAL_PIPELINE,
-    SPATIAL_REASONING_SINGLE_LOCAL_PIPELINE,
-    VISUAL_PROMPTING_PAIRS_PIPELINE,
-    VISUAL_PROMPTING_SINGLE_PIPELINE,
-    VISUAL_PROMPTING_PAIRS_LOCAL_PIPELINE,
-    VISUAL_PROMPTING_SINGLE_LOCAL_PIPELINE,
-    SPATIAL_GRID_PIPELINE,
-    SPATIAL_GRID_TEXTONLY_PIPELINE,
-    SPATIAL_GRID_REPORTING_PIPELINE,
-    SPATIAL_MAP_PIPELINE,
-    SPATIAL_MAP_TEXTONLY_PIPELINE,
-    SPATIAL_MAP_REPORTING_PIPELINE,
-    MAZE_PIPELINE,
-    MAZE_TEXTONLY_PIPELINE,
-    MAZE_REPORTING_PIPELINE,
-    IFEval_PIPELINE,
-    FlenQA_Experiment_Pipeline,
-    GPQA_Experiment_Pipeline,
-    Drop_Experiment_Pipeline,
-    GEOMETER_PIPELINE,
-    MMMU_BASELINE_PIPELINE,
-    KITAB_ONE_BOOK_CONSTRAINT_PIPELINE,
-    KITAB_ONE_BOOK_CONSTRAINT_PIPELINE_WITH_CONTEXT,
-    KITAB_ONE_BOOK_CONSTRAINT_PIPELINE_SELF_CONTEXT,
-    KITAB_TWO_BOOK_CONSTRAINT_PIPELINE,
-    KITAB_TWO_BOOK_CONSTRAINT_PIPELINE_WITH_CONTEXT,
-    GPT35_KITAB_ONE_BOOK_CONSTRAINT_PIPELINE,
-    DNA_PIPELINE,
-    ToxiGen_Discriminative_PIPELINE,
-    ToxiGen_Generative_PIPELINE,
-    Geo_Nondeterminism,
-    MMMU_Nondeterminism,
-    IFEval_Nondeterminism,
-    Kitab_Nondeterminism,
-    AIME_PIPELINE,
     create_logdir,
 ]
diff --git a/eureka_ml_insights/configs/model_configs.py b/eureka_ml_insights/configs/model_configs.py
@@ -13,6 +13,7 @@
     LLaVAModel,
     MistralServerlessAzureRestEndpointModel,
     RestEndpointModel,
+    TestModel,
 )
 
 from .config import ModelConfig
@@ -21,6 +22,10 @@
 # in the secret_key_params dictionary. OR you can provide the key name and key vault URL to fetch the key from Azure Key Vault.
 # You don't need to provide both the key_vault_url and local_keys_path. You can provide one of them based on your setup.
 
+
+# Test model
+TEST_MODEL_CONFIG = ModelConfig(TestModel, {})
+
 # OpenAI models
 
 OPENAI_SECRET_KEY_PARAMS = {
@@ -193,4 +198,4 @@
         },
         "model_name": "Mistral-large-2407",
     },
-)
+)
diff --git a/eureka_ml_insights/core/__init__.py b/eureka_ml_insights/core/__init__.py
@@ -1,5 +1,5 @@
 from .data_join import DataJoin
-from .data_processing import DataProcessing, NumpyEncoder
+from .data_processing import DataProcessing
 from .eval_reporting import EvalReporting
 from .inference import Inference
 from .pipeline import Component, Pipeline
@@ -12,6 +12,5 @@
     "EvalReporting",
     "DataProcessing",
     "PromptProcessing",
-    "NumpyEncoder",
     "DataJoin",
 ]
diff --git a/eureka_ml_insights/core/data_processing.py b/eureka_ml_insights/core/data_processing.py
@@ -1,11 +1,10 @@
-import base64
 import json
 import logging
 import os
 from hashlib import md5
 from typing import List, Optional
 
-import numpy as np
+from eureka_ml_insights.data_utils import NumpyEncoder
 
 from .pipeline import Component
 from .reserved_names import (
@@ -21,36 +20,6 @@ def compute_hash(val: str) -> str:
     return md5(val.encode("utf-8")).hexdigest()
 
 
-class NumpyEncoder(json.JSONEncoder):
-    """Special json encoder for numpy types"""
-
-    def default(self, obj):
-        if isinstance(
-            obj,
-            (
-                np.int_,
-                np.intc,
-                np.intp,
-                np.int8,
-                np.int16,
-                np.int32,
-                np.int64,
-                np.uint8,
-                np.uint16,
-                np.uint32,
-                np.uint64,
-            ),
-        ):
-            return int(obj)
-        elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)):
-            return float(obj)
-        elif isinstance(obj, (np.ndarray,)):
-            return obj.tolist()
-        elif isinstance(obj, bytes):
-            return base64.b64encode(obj).decode("ascii")
-        return json.JSONEncoder.default(self, obj)
-
-
 class DataProcessing(Component):
     @classmethod
     def from_config(cls, config):

diff --git a/eureka_ml_insights/core/eval_reporting.py b/eureka_ml_insights/core/eval_reporting.py
@@ -2,9 +2,9 @@
 import json
 import os
 
+from eureka_ml_insights.data_utils import NumpyEncoder
 from eureka_ml_insights.metrics import Reporter
 
-from .data_processing import NumpyEncoder
 from .pipeline import Component
 
 

diff --git a/eureka_ml_insights/data_utils/__init__.py b/eureka_ml_insights/data_utils/__init__.py
@@ -11,8 +11,8 @@
     MMDataLoader,
     TXTWriter,
 )
+from .encoders import NumpyEncoder
 from .prompt_processing import JinjaPromptTemplate
-from .secret_key_utils import GetKey
 from .spatial_utils import (
     ExtractAnswerGrid,
     ExtractAnswerMaze,
@@ -67,11 +67,11 @@
     RegexTransform,
     ASTEvalTransform,
     PrependStringTransform,
-    GetKey,
     ExtractAnswerGrid,
     ExtractAnswerSpatialMap,
     ExtractAnswerMaze,
     ShuffleColumnsTransform,
     ColumnMatchMapTransform,
     TokenCounterTransform,
+    NumpyEncoder,
 ]
diff --git a/eureka_ml_insights/data_utils/data.py b/eureka_ml_insights/data_utils/data.py
@@ -14,9 +14,9 @@
 from PIL import Image
 from tqdm import tqdm
 
-from eureka_ml_insights.core import NumpyEncoder
+from eureka_ml_insights.secret_management import get_secret
 
-from .secret_key_utils import GetKey
+from .encoders import NumpyEncoder
 from .transform import DFTransformBase
 
 log = logging.getLogger("data_reader")
@@ -216,14 +216,14 @@ def get_query_string(self, query_string=None, secret_key_params=None):
         One of the two arguments must be provided.
         args:
             query_string: str, query string to authenticate with Azure Blob Storage.
-            secret_key_params: dict, dictionary containing the paramters to call GetKey with.
+            secret_key_params: dict, dictionary containing the paramters to call get_secret with.
         """
         self.query_string = query_string
         self.secret_key_params = secret_key_params
         if self.query_string is None and self.secret_key_params is None:
             raise ValueError("Either provide query_string or secret_key_params to load data from Azure.")
         if self.query_string is None:
-            self.query_string = GetKey(**secret_key_params)
+            self.query_string = get_secret(**secret_key_params)
 
 
 class AzureMMDataLoader(MMDataLoader):

diff --git a/eureka_ml_insights/data_utils/encoders.py b/eureka_ml_insights/data_utils/encoders.py
@@ -0,0 +1,32 @@
+import json
+import base64
+import numpy as np
+
+class NumpyEncoder(json.JSONEncoder):
+    """Special json encoder for numpy types"""
+
+    def default(self, obj):
+        if isinstance(
+            obj,
+            (
+                np.int_,
+                np.intc,
+                np.intp,
+                np.int8,
+                np.int16,
+                np.int32,
+                np.int64,
+                np.uint8,
+                np.uint16,
+                np.uint32,
+                np.uint64,
+            ),
+        ):
+            return int(obj)
+        elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)):
+            return float(obj)
+        elif isinstance(obj, (np.ndarray,)):
+            return obj.tolist()
+        elif isinstance(obj, bytes):
+            return base64.b64encode(obj).decode("ascii")
+        return json.JSONEncoder.default(self, obj)