From 2dbeda7243cfa4b92ca50b3094563e70ee8a7aff Mon Sep 17 00:00:00 2001 From: Luis Yamada Date: Mon, 3 Feb 2025 15:52:14 -0300 Subject: [PATCH] restoring original state of gradle properties --- .../gradle/wrapper/gradle-wrapper.properties | 2 +- .../gradle/wrapper/gradle-wrapper.properties | 2 +- .../spark-extension-interfaces/build.gradle | 8 +- .../gradle/wrapper/gradle-wrapper.properties | 2 +- .../SparkOpenLineageExtensionVisitorTest.java | 166 +++++++++--------- .../spark/agent/NuEventEmitter.java | 3 +- 6 files changed, 92 insertions(+), 91 deletions(-) diff --git a/client/java/gradle/wrapper/gradle-wrapper.properties b/client/java/gradle/wrapper/gradle-wrapper.properties index 9355b41557..09523c0e54 100644 --- a/client/java/gradle/wrapper/gradle-wrapper.properties +++ b/client/java/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/integration/spark-extension-entrypoint/gradle/wrapper/gradle-wrapper.properties b/integration/spark-extension-entrypoint/gradle/wrapper/gradle-wrapper.properties index 9355b41557..09523c0e54 100644 --- a/integration/spark-extension-entrypoint/gradle/wrapper/gradle-wrapper.properties +++ b/integration/spark-extension-entrypoint/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/integration/spark-extension-interfaces/build.gradle b/integration/spark-extension-interfaces/build.gradle index 1815e3e264..97ef204edd 100644 --- a/integration/spark-extension-interfaces/build.gradle +++ b/integration/spark-extension-interfaces/build.gradle @@ -170,10 +170,10 @@ publishing { } // -------- removing the following section to enable the dependencies within the pom file -------- - // // pom.withXml { - // // Node pomNode = asNode() - // // pomNode.remove(pomNode.get("dependencies")) - // // } + // pom.withXml { + // Node pomNode = asNode() + // pomNode.remove(pomNode.get("dependencies")) + // } } } diff --git a/integration/spark-extension-interfaces/gradle/wrapper/gradle-wrapper.properties b/integration/spark-extension-interfaces/gradle/wrapper/gradle-wrapper.properties index e1adfb4938..19cfad969b 100644 --- a/integration/spark-extension-interfaces/gradle/wrapper/gradle-wrapper.properties +++ b/integration/spark-extension-interfaces/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/integration/spark-extension-interfaces/src/test/java/io/openlineage/spark/shade/extension/v1/lifecycle/plan/SparkOpenLineageExtensionVisitorTest.java b/integration/spark-extension-interfaces/src/test/java/io/openlineage/spark/shade/extension/v1/lifecycle/plan/SparkOpenLineageExtensionVisitorTest.java index 5020543ada..0f202e05c3 100644 --- a/integration/spark-extension-interfaces/src/test/java/io/openlineage/spark/shade/extension/v1/lifecycle/plan/SparkOpenLineageExtensionVisitorTest.java +++ b/integration/spark-extension-interfaces/src/test/java/io/openlineage/spark/shade/extension/v1/lifecycle/plan/SparkOpenLineageExtensionVisitorTest.java @@ -31,97 +31,97 @@ class SparkOpenLineageExtensionVisitorTest { SparkOpenLineageExtensionVisitor visitor = new SparkOpenLineageExtensionVisitor(); Map options = Collections.singletonMap("path", "some-path"); -// @Test -// void testDatasetIdentifierReturnedByLineageRelationProvider() { -// // given -// LineageRelationProvider provider = -// (sparkListenerEventName, openLineage, sqlContext, parameters) -> -// new DatasetIdentifier("name", "namespace") -// .withSymlink( -// new DatasetIdentifier.Symlink( -// "name1", "namespace1", DatasetIdentifier.SymlinkType.TABLE)); -// assertThat(visitor.isDefinedAt(provider)).isTrue(); -// Map result = -// visitor.apply(provider, event.getClass().getName(), sqlContext, options); + @Test + void testDatasetIdentifierReturnedByLineageRelationProvider() { + // given + LineageRelationProvider provider = + (sparkListenerEventName, openLineage, sqlContext, parameters) -> + new DatasetIdentifier("name", "namespace") + .withSymlink( + new DatasetIdentifier.Symlink( + "name1", "namespace1", DatasetIdentifier.SymlinkType.TABLE)); + assertThat(visitor.isDefinedAt(provider)).isTrue(); + Map result = + visitor.apply(provider, event.getClass().getName(), sqlContext, options); -// // then -// assertThat(result).extracting("name").isEqualTo("name"); -// assertThat(result).extracting("namespace").isEqualTo("namespace"); -// assertThat(result) -// .extracting("symlinks") -// .isEqualTo( -// list( -// ImmutableMap.builder() -// .put("name", "name1") -// .put("namespace", "namespace1") -// .put("type", "TABLE") -// .build())); -// } + // then + assertThat(result).extracting("name").isEqualTo("name"); + assertThat(result).extracting("namespace").isEqualTo("namespace"); + assertThat(result) + .extracting("symlinks") + .isEqualTo( + list( + ImmutableMap.builder() + .put("name", "name1") + .put("namespace", "namespace1") + .put("type", "TABLE") + .build())); + } -// @Test -// void testDatasetIdentifierReturnedByLineageRelation() { -// // given -// LineageRelation lineageRelation = -// (sparkListenerEventName, openLineage) -> -// new DatasetIdentifier("name", "namespace") -// .withSymlink( -// new DatasetIdentifier.Symlink( -// "name1", "namespace1", DatasetIdentifier.SymlinkType.TABLE)); + @Test + void testDatasetIdentifierReturnedByLineageRelation() { + // given + LineageRelation lineageRelation = + (sparkListenerEventName, openLineage) -> + new DatasetIdentifier("name", "namespace") + .withSymlink( + new DatasetIdentifier.Symlink( + "name1", "namespace1", DatasetIdentifier.SymlinkType.TABLE)); -// // when -// assertThat(visitor.isDefinedAt(lineageRelation)).isTrue(); -// Map result = visitor.apply(lineageRelation, event.getClass().getName()); + // when + assertThat(visitor.isDefinedAt(lineageRelation)).isTrue(); + Map result = visitor.apply(lineageRelation, event.getClass().getName()); -// // then -// assertThat(result).extracting("name").isEqualTo("name"); -// assertThat(result).extracting("namespace").isEqualTo("namespace"); -// assertThat(result) -// .extracting("symlinks") -// .isEqualTo( -// list( -// ImmutableMap.builder() -// .put("name", "name1") -// .put("namespace", "namespace1") -// .put("type", "TABLE") -// .build())); -// } + // then + assertThat(result).extracting("name").isEqualTo("name"); + assertThat(result).extracting("namespace").isEqualTo("namespace"); + assertThat(result) + .extracting("symlinks") + .isEqualTo( + list( + ImmutableMap.builder() + .put("name", "name1") + .put("namespace", "namespace1") + .put("type", "TABLE") + .build())); + } -// @Test -// void testFacetsReturnedByInputLineageNodeWithIdentifier() { -// // given -// InputLineageNode inputLineageNode = -// (sparkListenerEventName, openLineage) -> -// Collections.singletonList( -// new InputDatasetWithIdentifier( -// new DatasetIdentifier("a", "b"), -// openLineage.newDatasetFacetsBuilder(), -// openLineage.newInputDatasetInputFacetsBuilder())); + @Test + void testFacetsReturnedByInputLineageNodeWithIdentifier() { + // given + InputLineageNode inputLineageNode = + (sparkListenerEventName, openLineage) -> + Collections.singletonList( + new InputDatasetWithIdentifier( + new DatasetIdentifier("a", "b"), + openLineage.newDatasetFacetsBuilder(), + openLineage.newInputDatasetInputFacetsBuilder())); -// // when -// assertThat(visitor.isDefinedAt(inputLineageNode)).isTrue(); -// Map result = visitor.apply(inputLineageNode, event.getClass().getName()); + // when + assertThat(visitor.isDefinedAt(inputLineageNode)).isTrue(); + Map result = visitor.apply(inputLineageNode, event.getClass().getName()); -// // then -// assertThat(result).extracting("delegateNodes").isEqualTo(list()); -// } + // then + assertThat(result).extracting("delegateNodes").isEqualTo(list()); + } -// @Test -// void testFacetsReturnedByInputLineageNodeWithDelegates() { -// // given -// LogicalPlan delegate = mock(LogicalPlan.class); -// InputLineageNode inputLineageNode = -// (sparkListenerEventName, openLineage) -> -// Collections.singletonList( -// new InputDatasetWithDelegate( -// delegate, -// openLineage.newDatasetFacetsBuilder(), -// openLineage.newInputDatasetInputFacetsBuilder())); + @Test + void testFacetsReturnedByInputLineageNodeWithDelegates() { + // given + LogicalPlan delegate = mock(LogicalPlan.class); + InputLineageNode inputLineageNode = + (sparkListenerEventName, openLineage) -> + Collections.singletonList( + new InputDatasetWithDelegate( + delegate, + openLineage.newDatasetFacetsBuilder(), + openLineage.newInputDatasetInputFacetsBuilder())); -// // when -// assertThat(visitor.isDefinedAt(inputLineageNode)).isTrue(); -// Map result = visitor.apply(inputLineageNode, event.getClass().getName()); + // when + assertThat(visitor.isDefinedAt(inputLineageNode)).isTrue(); + Map result = visitor.apply(inputLineageNode, event.getClass().getName()); -// // then -// assertThat(result).extracting("delegateNodes").isEqualTo(list(delegate)); -// } + // then + assertThat(result).extracting("delegateNodes").isEqualTo(list(delegate)); + } } diff --git a/integration/spark/app/src/main/java/io/openlineage/spark/agent/NuEventEmitter.java b/integration/spark/app/src/main/java/io/openlineage/spark/agent/NuEventEmitter.java index 3689d79cf1..d91aa1529b 100644 --- a/integration/spark/app/src/main/java/io/openlineage/spark/agent/NuEventEmitter.java +++ b/integration/spark/app/src/main/java/io/openlineage/spark/agent/NuEventEmitter.java @@ -28,7 +28,8 @@ public class NuEventEmitter { private static final Set WANTED_EVENT_NAME_SUBSTRINGS = new HashSet<>( Arrays.asList( ".execute_insert_into_hadoop_fs_relation_command.", - ".adaptive_spark_plan." + ".adaptive_spark_plan.", + "." ) );