Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

source-cockroachdb: adopt CDK 0.20.4 #35234

Merged
merged 2 commits into from
Feb 14, 2024
Merged

source-cockroachdb: adopt CDK 0.20.4 #35234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
de69db6
source-cockroachdb: adopt CDK 0.20.4
postamar Feb 13, 2024
cdb079e
bump version and update changelog
postamar Feb 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 3 additions & 19 deletions airbyte-integrations/connectors/source-cockroachdb/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,36 +1,20 @@
plugins {
id 'application'
id 'airbyte-java-connector'
}

airbyteJavaConnector {
cdkVersionRequired = '0.13.2'
cdkVersionRequired = '0.20.4'
features = ['db-sources']
useLocalCdk = false
}

//remove once upgrading the CDK version to 0.4.x or later
java {
compileTestJava {
options.compilerArgs.remove("-Werror")
}
compileJava {
options.compilerArgs.remove("-Werror")
}
}

airbyteJavaConnector.addCdkDependencies()

application {
mainClass = 'io.airbyte.integrations.source.cockroachdb.CockroachDbSource'
applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
}

dependencies {
implementation 'org.postgresql:postgresql:42.6.0'

implementation 'org.apache.commons:commons-lang3:3.11'
implementation libs.postgresql

testImplementation libs.testcontainers.cockroachdb
testImplementation 'org.apache.commons:commons-lang3:3.11'
testImplementation 'org.testcontainers:cockroachdb:1.19.4'
}
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-cockroachdb/metadata.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ data:
connectorSubtype: database
connectorType: source
definitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003
dockerImageTag: 0.2.1
dockerImageTag: 0.2.2
dockerRepository: airbyte/source-cockroachdb
githubIssueLabel: source-cockroachdb
icon: cockroachdb.svg
Expand Down
1 change: 1 addition & 0 deletions ...ckroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ public Set<String> getExcludedInternalNameSpaces() {
}

@Override
@SuppressWarnings("unchecked")
public Set<JdbcPrivilegeDto> getPrivilegesTableForCurrentUser(final JdbcDatabase database, final String schema) throws SQLException {
try (final Stream<JsonNode> stream = database.unsafeQuery(getPrivileges(database), sourceOperations::rowToJson)) {
return stream.map(this::getPrivilegeDto).collect(Collectors.toSet());
Expand Down
6 changes: 6 additions & 0 deletions ...achdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcDatabase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Stream;

/**
Expand Down Expand Up @@ -81,4 +82,9 @@ public Stream<JsonNode> unsafeQuery(final String sql, final String... params) th

}

@Override
public <T> T executeMetadataQuery(Function<DatabaseMetaData, T> function) throws SQLException {
return database.executeMetadataQuery(function);
}

}
28 changes: 14 additions & 14 deletions ...tion/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,27 +59,27 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc
.put(JdbcUtils.SSL_KEY, false)
.build());

try (final DSLContext dslContext = DSLContextFactory.create(
final DSLContext dslContext = DSLContextFactory.create(
config.get(JdbcUtils.USERNAME_KEY).asText(),
config.get(JdbcUtils.PASSWORD_KEY).asText(),
DatabaseDriver.POSTGRESQL.getDriverClassName(),
String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(),
config.get(JdbcUtils.HOST_KEY).asText(),
config.get(JdbcUtils.PORT_KEY).asInt(),
config.get(JdbcUtils.DATABASE_KEY).asText()),
SQLDialect.POSTGRES)) {
final Database database = new Database(dslContext);

database.query(ctx -> {
ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));");
ctx.fetch(
"INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');");
ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));");
ctx.fetch(
"INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');");
return null;
});
}
SQLDialect.POSTGRES);
final Database database = new Database(dslContext);

database.query(ctx -> {
ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));");
ctx.fetch(
"INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');");
ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));");
ctx.fetch(
"INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');");
return null;
});

}

@Override
Expand Down
1 change: 0 additions & 1 deletion ...ration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,6 @@ protected JsonNode getConfig() {

@Override
protected void tearDown(final TestDestinationEnv testEnv) {
dslContext.close();
container.close();
}

Expand Down
101 changes: 49 additions & 52 deletions ...achdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,32 +103,31 @@ void setup() throws Exception {
dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase();

final JsonNode config = getConfig(PSQL_DB, null);
try (final DSLContext dslContext = getDslContext(config)) {
final Database database = getDatabase(dslContext);
database.query(ctx -> {
ctx.fetch("CREATE DATABASE " + dbName + ";");
ctx.fetch(
"CREATE TABLE " + dbName + ".id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch("CREATE INDEX i1 ON " + dbName + ".id_and_name (id);");
ctx.fetch(
"INSERT INTO " + dbName
+ ".id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');");

ctx.fetch(
"CREATE TABLE " + dbName + ".id_and_name2(id NUMERIC(20, 10), name VARCHAR(200), power double precision);");
ctx.fetch(
"INSERT INTO " + dbName
+ ".id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');");

ctx.fetch(
"CREATE TABLE " + dbName
+ ".names(first_name VARCHAR(200), last_name VARCHAR(200), power double precision, PRIMARY KEY (first_name, last_name));");
ctx.fetch(
"INSERT INTO " + dbName
+ ".names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', 'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');");
return null;
});
}
final DSLContext dslContext = getDslContext(config);
final Database database = getDatabase(dslContext);
database.query(ctx -> {
ctx.fetch("CREATE DATABASE " + dbName + ";");
ctx.fetch(
"CREATE TABLE " + dbName + ".id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch("CREATE INDEX i1 ON " + dbName + ".id_and_name (id);");
ctx.fetch(
"INSERT INTO " + dbName
+ ".id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');");

ctx.fetch(
"CREATE TABLE " + dbName + ".id_and_name2(id NUMERIC(20, 10), name VARCHAR(200), power double precision);");
ctx.fetch(
"INSERT INTO " + dbName
+ ".id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');");

ctx.fetch(
"CREATE TABLE " + dbName
+ ".names(first_name VARCHAR(200), last_name VARCHAR(200), power double precision, PRIMARY KEY (first_name, last_name));");
ctx.fetch(
"INSERT INTO " + dbName
+ ".names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', 'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');");
return null;
});
}

private static Database getDatabase(final DSLContext dslContext) {
Expand Down Expand Up @@ -184,15 +183,14 @@ public void testCanReadUtf8() throws Exception {
// .withCommand("postgres -c client_encoding=sql_ascii")
db.start();
final JsonNode config = getConfig(db);
try (final DSLContext dslContext = getDslContext(config)) {
final Database database = getDatabase(dslContext);
database.query(ctx -> {
ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));");
ctx.fetch(
"INSERT INTO id_and_name (id, name) VALUES (1,E'\\u2013 someutfstring'), (2, E'\\u2215');");
return null;
});
}
final DSLContext dslContext = getDslContext(config);
final Database database = getDatabase(dslContext);
database.query(ctx -> {
ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));");
ctx.fetch(
"INSERT INTO id_and_name (id, name) VALUES (1,E'\\u2013 someutfstring'), (2, E'\\u2215');");
return null;
});

final Set<AirbyteMessage> actualMessages = MoreIterators
.toSet(new CockroachDbSource().read(config, CONFIGURED_CATALOG, null));
Expand Down Expand Up @@ -225,23 +223,22 @@ void testDiscoverWithPk() throws Exception {
@Test
void testDiscoverWithPermissions() throws Exception {
final JsonNode config = getConfig(PSQL_DB, dbName);
try (final DSLContext dslContext = getDslContext(config)) {
final Database database = getDatabase(dslContext);
database.query(ctx -> {
ctx.fetch(
"CREATE USER cock;");
ctx.fetch(
"CREATE TABLE id_and_name_perm1(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch(
"CREATE TABLE id_and_name_perm2(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch(
"CREATE TABLE id_and_name_perm3(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch("grant all on database " + dbName + " to cock;");
ctx.fetch("grant all on table " + dbName + ".public.id_and_name_perm1 to cock;");
ctx.fetch("grant select on table " + dbName + ".public.id_and_name_perm2 to cock;");
return null;
});
}
final DSLContext dslContext = getDslContext(config);
final Database database = getDatabase(dslContext);
database.query(ctx -> {
ctx.fetch(
"CREATE USER cock;");
ctx.fetch(
"CREATE TABLE id_and_name_perm1(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch(
"CREATE TABLE id_and_name_perm2(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch(
"CREATE TABLE id_and_name_perm3(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch("grant all on database " + dbName + " to cock;");
ctx.fetch("grant all on table " + dbName + ".public.id_and_name_perm1 to cock;");
ctx.fetch("grant select on table " + dbName + ".public.id_and_name_perm2 to cock;");
return null;
});

final List<String> expected = List.of("id_and_name_perm1", "id_and_name_perm2");

Expand Down
1 change: 1 addition & 0 deletions docs/integrations/sources/cockroachdb.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ Your database user should now be ready for use with Airbyte.

| Version | Date | Pull Request | Subject |
|:--------|:-----------| :--- |:------------------------------------------------------------------------------------------------------------------------------------------|
| 0.2.2 | 2024-02-13 | [35234](https://github.com/airbytehq/airbyte/pull/35234) | Adopt CDK 0.20.4 |
| 0.2.1 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version |
| 0.2.0 | 2023-12-18 | [33485](https://github.com/airbytehq/airbyte/pull/33485) | Removed LEGACY state |
| 0.1.22 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting |
Expand Down
Loading