Merge branch 'main' into dependabot/pip/jinja2-3.1.5

.github/workflows/connector-tests.yml

@@ -129,11 +129,10 @@ jobs:
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
           POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0"
-        # TODO: Revert below to use `tools.airbyte-ci-binary.install` after Airbyte CI released:
         run: |
           cd airbyte
-          make tools.airbyte-ci-dev.install
-          airbyte-ci-dev \
+          make tools.airbyte-ci-binary.install
+          airbyte-ci \
           --ci-report-bucket-name=airbyte-ci-reports-multi \
           connectors \
           --name ${{matrix.connector}} \

.github/workflows/pypi_publish.yml

@@ -10,7 +10,7 @@ name: Packaging and Publishing
 on:
   push:
     tags:
-      - 'v*'
+      - "v*"
   workflow_dispatch:
     inputs:
       version:

airbyte_cdk/__init__.py

@@ -48,27 +48,46 @@
 # Once those issues are resolved, the below can be sorted with isort.
 import dunamai as _dunamai
 
-from .destinations import Destination
-from .models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type, FailureType, AirbyteStream, AdvancedAuth, DestinationSyncMode, ConnectorSpecification, OAuthConfigSpecification, OrchestratorType, ConfiguredAirbyteStream, SyncMode, AirbyteLogMessage, Level, AirbyteRecordMessage
-
-from .sources import Source
-from .config_observation import create_connector_config_control_message, emit_configuration_as_airbyte_control_message
+from .config_observation import (
+    create_connector_config_control_message,
+    emit_configuration_as_airbyte_control_message,
+)
 from .connector import BaseConnector, Connector
-
-from .entrypoint import launch, AirbyteEntrypoint
-
+from .destinations import Destination
+from .entrypoint import AirbyteEntrypoint, launch
 from .logger import AirbyteLogFormatter, init_logger
-from .sources import AbstractSource
+from .models import (
+    AdvancedAuth,
+    AirbyteConnectionStatus,
+    AirbyteLogMessage,
+    AirbyteMessage,
+    AirbyteRecordMessage,
+    AirbyteStream,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    ConnectorSpecification,
+    DestinationSyncMode,
+    FailureType,
+    Level,
+    OAuthConfigSpecification,
+    OrchestratorType,
+    Status,
+    SyncMode,
+    Type,
+)
+from .sources import AbstractSource, Source
 from .sources.concurrent_source.concurrent_source import ConcurrentSource
 from .sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
 from .sources.config import BaseConfig
-from .sources.types import Config, Record, StreamSlice
 from .sources.connector_state_manager import ConnectorStateManager
 from .sources.declarative.auth import DeclarativeOauth2Authenticator
-from .sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
-from .sources.declarative.auth.declarative_authenticator import NoAuth
+from .sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth
 from .sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator
-from .sources.declarative.auth.token import BasicHttpAuthenticator, BearerAuthenticator, ApiKeyAuthenticator
+from .sources.declarative.auth.token import (
+    ApiKeyAuthenticator,
+    BasicHttpAuthenticator,
+    BearerAuthenticator,
+)
 from .sources.declarative.datetime.min_max_datetime import MinMaxDatetime
 from .sources.declarative.declarative_stream import DeclarativeStream
 from .sources.declarative.decoders import Decoder, JsonDecoder
@@ -77,48 +96,89 @@
 from .sources.declarative.extractors.record_extractor import RecordExtractor
 from .sources.declarative.extractors.record_filter import RecordFilter
 from .sources.declarative.incremental import DatetimeBasedCursor
-from .sources.declarative.interpolation import InterpolatedString, InterpolatedBoolean
+from .sources.declarative.interpolation import InterpolatedBoolean, InterpolatedString
 from .sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
-from .sources.declarative.migrations.legacy_to_per_partition_state_migration import LegacyToPerPartitionStateMigration
-
-from .sources.declarative.partition_routers import CartesianProductStreamSlicer, SinglePartitionRouter, SubstreamPartitionRouter
+from .sources.declarative.migrations.legacy_to_per_partition_state_migration import (
+    LegacyToPerPartitionStateMigration,
+)
+from .sources.declarative.partition_routers import (
+    CartesianProductStreamSlicer,
+    SinglePartitionRouter,
+    SubstreamPartitionRouter,
+)
 from .sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig
-from .sources.declarative.requesters import Requester, HttpRequester
-
+from .sources.declarative.requesters import HttpRequester, Requester
 from .sources.declarative.requesters.error_handlers import BackoffStrategy
 from .sources.declarative.requesters.paginators import DefaultPaginator, PaginationStrategy
-from .sources.declarative.requesters.paginators.strategies import OffsetIncrement, CursorPaginationStrategy, PageIncrement, StopConditionPaginationStrategyDecorator
-
+from .sources.declarative.requesters.paginators.strategies import (
+    CursorPaginationStrategy,
+    OffsetIncrement,
+    PageIncrement,
+    StopConditionPaginationStrategyDecorator,
+)
 from .sources.declarative.requesters.request_option import RequestOption, RequestOptionType
-
-from .sources.declarative.requesters.request_options.default_request_options_provider import DefaultRequestOptionsProvider
-from .sources.declarative.requesters.request_options.interpolated_request_input_provider import InterpolatedRequestInputProvider
+from .sources.declarative.requesters.request_options.default_request_options_provider import (
+    DefaultRequestOptionsProvider,
+)
+from .sources.declarative.requesters.request_options.interpolated_request_input_provider import (
+    InterpolatedRequestInputProvider,
+)
 from .sources.declarative.requesters.requester import HttpMethod
 from .sources.declarative.retrievers import SimpleRetriever
 from .sources.declarative.schema import JsonFileSchemaLoader
-from .sources.declarative.transformations.add_fields import AddFields, AddedFieldDefinition
+from .sources.declarative.transformations.add_fields import AddedFieldDefinition, AddFields
 from .sources.declarative.transformations.transformation import RecordTransformation
 from .sources.declarative.types import FieldPointer
 from .sources.declarative.yaml_declarative_source import YamlDeclarativeSource
 from .sources.message import InMemoryMessageRepository, MessageRepository
 from .sources.source import TState
 from .sources.streams.availability_strategy import AvailabilityStrategy
-from .sources.streams.call_rate import AbstractAPIBudget, HttpAPIBudget, HttpRequestMatcher, MovingWindowCallRatePolicy, Rate, CachedLimiterSession, LimiterSession
+from .sources.streams.call_rate import (
+    AbstractAPIBudget,
+    CachedLimiterSession,
+    HttpAPIBudget,
+    HttpRequestMatcher,
+    LimiterSession,
+    MovingWindowCallRatePolicy,
+    Rate,
+)
 from .sources.streams.checkpoint import Cursor as LegacyCursor
 from .sources.streams.checkpoint import ResumableFullRefreshCursor
 from .sources.streams.concurrent.adapters import StreamFacade
-from .sources.streams.concurrent.cursor import ConcurrentCursor, CursorField, FinalStateCursor
-from .sources.streams.concurrent.cursor import Cursor
-from .sources.streams.concurrent.state_converters.datetime_stream_state_converter import EpochValueConcurrentStreamStateConverter, IsoMillisConcurrentStreamStateConverter
-from .sources.streams.core import Stream, IncrementalMixin, package_name_from_class
+from .sources.streams.concurrent.cursor import (
+    ConcurrentCursor,
+    Cursor,
+    CursorField,
+    FinalStateCursor,
+)
+from .sources.streams.concurrent.state_converters.datetime_stream_state_converter import (
+    EpochValueConcurrentStreamStateConverter,
+    IsoMillisConcurrentStreamStateConverter,
+)
+from .sources.streams.core import IncrementalMixin, Stream, package_name_from_class
 from .sources.streams.http import HttpStream, HttpSubStream
 from .sources.streams.http.availability_strategy import HttpAvailabilityStrategy
-from .sources.streams.http.exceptions import BaseBackoffException, DefaultBackoffException, UserDefinedBackoffException
+from .sources.streams.http.exceptions import (
+    BaseBackoffException,
+    DefaultBackoffException,
+    UserDefinedBackoffException,
+)
 from .sources.streams.http.rate_limiting import default_backoff_handler
-from .sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator, SingleUseRefreshTokenOauth2Authenticator
+from .sources.streams.http.requests_native_auth import (
+    Oauth2Authenticator,
+    SingleUseRefreshTokenOauth2Authenticator,
+    TokenAuthenticator,
+)
 from .sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
+from .sources.types import Config, Record, StreamSlice
 from .sources.utils import casing
-from .sources.utils.schema_helpers import InternalConfig, ResourceSchemaLoader, check_config_against_spec_or_exit, split_config, expand_refs
+from .sources.utils.schema_helpers import (
+    InternalConfig,
+    ResourceSchemaLoader,
+    check_config_against_spec_or_exit,
+    expand_refs,
+    split_config,
+)
 from .sources.utils.transform import TransformConfig, TypeTransformer
 from .utils import AirbyteTracedException, is_cloud_environment
 from .utils.constants import ENV_REQUEST_CACHE_PATH
@@ -127,7 +187,6 @@
 from .utils.spec_schema_transformations import resolve_refs
 from .utils.stream_status_utils import as_airbyte_message
 
-
 __all__ = [
     # Availability strategy
     "AvailabilityStrategy",

airbyte_cdk/cli/source_declarative_manifest/__init__.py

@@ -1,6 +1,5 @@
 from airbyte_cdk.cli.source_declarative_manifest._run import run
 
-
 __all__ = [
     "run",
 ]

airbyte_cdk/entrypoint.py

@@ -5,6 +5,7 @@
 import argparse
 import importlib
 import ipaddress
+import json
 import logging
 import os.path
 import socket
@@ -46,6 +47,7 @@
 
 VALID_URL_SCHEMES = ["https"]
 CLOUD_DEPLOYMENT_MODE = "cloud"
+_HAS_LOGGED_FOR_SERIALIZATION_ERROR = False
 
 
 class AirbyteEntrypoint(object):
@@ -291,7 +293,17 @@ def set_up_secret_filter(config: TConfig, connection_specification: Mapping[str,
 
     @staticmethod
     def airbyte_message_to_string(airbyte_message: AirbyteMessage) -> str:
-        return orjson.dumps(AirbyteMessageSerializer.dump(airbyte_message)).decode()
+        global _HAS_LOGGED_FOR_SERIALIZATION_ERROR
+        serialized_message = AirbyteMessageSerializer.dump(airbyte_message)
+        try:
+            return orjson.dumps(serialized_message).decode()
+        except Exception as exception:
+            if not _HAS_LOGGED_FOR_SERIALIZATION_ERROR:
+                logger.warning(
+                    f"There was an error during the serialization of an AirbyteMessage: `{exception}`. This might impact the sync performances."
+                )
+                _HAS_LOGGED_FOR_SERIALIZATION_ERROR = True
+            return json.dumps(serialized_message)
 
     @classmethod
     def extract_state(cls, args: List[str]) -> Optional[Any]:

airbyte_cdk/models/__init__.py

@@ -7,7 +7,6 @@
 # of airbyte-cdk rather than a standalone package.
 from .airbyte_protocol import (
     AdvancedAuth,
-    AirbyteStateStats,
     AirbyteAnalyticsTraceMessage,
     AirbyteCatalog,
     AirbyteConnectionStatus,
@@ -22,13 +21,14 @@
     AirbyteRecordMessage,
     AirbyteStateBlob,
     AirbyteStateMessage,
+    AirbyteStateStats,
     AirbyteStateType,
     AirbyteStream,
     AirbyteStreamState,
     AirbyteStreamStatus,
-    AirbyteStreamStatusTraceMessage,
     AirbyteStreamStatusReason,
     AirbyteStreamStatusReasonType,
+    AirbyteStreamStatusTraceMessage,
     AirbyteTraceMessage,
     AuthFlowType,
     ConfiguredAirbyteCatalog,
@@ -48,6 +48,14 @@
     TraceType,
     Type,
 )
+from .airbyte_protocol_serializers import (
+    AirbyteMessageSerializer,
+    AirbyteStateMessageSerializer,
+    AirbyteStreamStateSerializer,
+    ConfiguredAirbyteCatalogSerializer,
+    ConfiguredAirbyteStreamSerializer,
+    ConnectorSpecificationSerializer,
+)
 from .well_known_types import (
     BinaryData,
     Boolean,
@@ -61,12 +69,3 @@
     TimeWithoutTimezone,
     TimeWithTimezone,
 )
-
-from .airbyte_protocol_serializers import (
-AirbyteStreamStateSerializer,
-AirbyteStateMessageSerializer,
-AirbyteMessageSerializer,
-ConfiguredAirbyteCatalogSerializer,
-ConfiguredAirbyteStreamSerializer,
-ConnectorSpecificationSerializer,
-)

airbyte_cdk/sources/declarative/async_job/job_orchestrator.py

@@ -20,7 +20,6 @@
     TypeVar,
 )
 
-from airbyte_cdk import StreamSlice
 from airbyte_cdk.logger import lazy_log
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.declarative.async_job.job import AsyncJob
@@ -31,6 +30,7 @@
 from airbyte_cdk.sources.declarative.async_job.repository import AsyncJobRepository
 from airbyte_cdk.sources.declarative.async_job.status import AsyncJobStatus
 from airbyte_cdk.sources.message import MessageRepository
+from airbyte_cdk.sources.types import StreamSlice
 from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 

airbyte_cdk/sources/declarative/auth/__init__.py

@@ -2,10 +2,7 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeOauth2Authenticator
 from airbyte_cdk.sources.declarative.auth.jwt import JwtAuthenticator
+from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeOauth2Authenticator
 
-__all__ = [
-    "DeclarativeOauth2Authenticator",
-    "JwtAuthenticator"
-]
+__all__ = ["DeclarativeOauth2Authenticator", "JwtAuthenticator"]