diff --git a/.github/workflows/R_CMD_check_Hades.yaml b/.github/workflows/R_CMD_check_Hades.yaml index 73ecbc34..2d22d9fb 100644 --- a/.github/workflows/R_CMD_check_Hades.yaml +++ b/.github/workflows/R_CMD_check_Hades.yaml @@ -111,16 +111,31 @@ jobs: path: check/*.tar.gz - name: Install covr - if: runner.os == 'macOS' + if: runner.os == 'Linux' run: | - install.packages("covr") + remotes::install_cran("covr") + remotes::install_cran("xml2") shell: Rscript {0} - + - name: Test coverage - if: runner.os == 'macOS' - run: covr::codecov() + if: runner.os == 'Linux' + run: | + cov <- covr::package_coverage( + quiet = FALSE, + clean = FALSE, + install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package") + ) + covr::to_cobertura(cov) shell: Rscript {0} + - uses: codecov/codecov-action@v4 + if: runner.os == 'Linux' + with: + file: ./cobertura.xml + plugin: noop + disable_search: true + token: ${{ secrets.CODECOV_TOKEN }} + Release: needs: R-CMD-Check diff --git a/CRAN-SUBMISSION b/CRAN-SUBMISSION index 55064784..7924be52 100644 --- a/CRAN-SUBMISSION +++ b/CRAN-SUBMISSION @@ -1,3 +1,3 @@ -Version: 6.3.2 -Date: 2023-12-11 13:54:30 UTC -SHA: dd511ec8b23927ffb61a17bedd9ee8bb81cbe476 +Version: 6.3.3 +Date: 2025-01-16 00:44:34 UTC +SHA: 7b5548f88a19ff84ca3f1b7c075ab370723e8341 diff --git a/DESCRIPTION b/DESCRIPTION index 0d8d39a2..18807f51 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -2,7 +2,7 @@ Package: DatabaseConnector Type: Package Title: Connecting to Various Database Platforms Version: 6.3.3 -Date: 2024-06-13 +Date: 2025-01-15 Authors@R: c( person("Martijn", "Schuemie", email = "schuemie@ohdsi.org", role = c("aut", "cre")), person("Marc", "Suchard", role = c("aut")), diff --git a/NAMESPACE b/NAMESPACE index a89c645e..e83c41cd 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,9 +1,19 @@ # Generated by roxygen2: do not edit by hand +S3method(compileReconnectCode,DatabaseConnectorDbiConnection) +S3method(compileReconnectCode,default) S3method(disconnect,DatabaseConnectorDbiConnection) S3method(disconnect,default) +S3method(getCatalogs,DatabaseConnectorDbiConnection) +S3method(getCatalogs,default) +S3method(getSchemaNames,DatabaseConnectorDbiConnection) +S3method(getSchemaNames,default) +S3method(getServer,DatabaseConnectorDbiConnection) +S3method(getServer,default) S3method(insertTable,DatabaseConnectorDbiConnection) S3method(insertTable,default) +S3method(listDatabaseConnectorColumns,DatabaseConnectorDbiConnection) +S3method(listDatabaseConnectorColumns,default) S3method(lowLevelExecuteSql,DatabaseConnectorDbiConnection) S3method(lowLevelExecuteSql,default) S3method(lowLevelQuerySql,DatabaseConnectorDbiConnection) diff --git a/NEWS.md b/NEWS.md index de98fff4..97f8159f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -7,6 +7,10 @@ Changes: - Updated Databricks driver to 2.6.36. +- Updated BigQuery driver to 1.6.2. + +- Using `INSERT` with multiple values in `insertTable()` for DataBricks for faster inserts. + DatabaseConnector 6.3.2 ======================= diff --git a/R/Andromeda.R b/R/Andromeda.R index ac4ca261..7dd57878 100644 --- a/R/Andromeda.R +++ b/R/Andromeda.R @@ -1,4 +1,4 @@ -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2025 Observational Health Data Sciences and Informatics # # This file is part of DatabaseConnector # diff --git a/R/BulkLoad.R b/R/BulkLoad.R index e441ecee..31a62d6e 100644 --- a/R/BulkLoad.R +++ b/R/BulkLoad.R @@ -1,6 +1,6 @@ # @file BulkLoad.R # -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2025 Observational Health Data Sciences and Informatics # # This file is part of DatabaseConnector # @@ -299,7 +299,19 @@ bulkLoadPostgres <- function(connection, sqlTableName, sqlFieldNames, sqlDataTyp readr::write_excel_csv(data, csvFileName, na = "") on.exit(unlink(csvFileName)) - hostServerDb <- strsplit(attr(connection, "server")(), "/")[[1]] + server <- attr(connection, "server")() + if (is.null(server)) { + # taken directly from DatabaseConnector R/RStudio.R - getServer.default, could an attr too? + databaseMetaData <- rJava::.jcall( + connection@jConnection, + "Ljava/sql/DatabaseMetaData;", + "getMetaData" + ) + server <- rJava::.jcall(databaseMetaData, "Ljava/lang/String;", "getURL") + server <- strsplit(server, "//")[[1]][2] + } + + hostServerDb <- strsplit(server, "/")[[1]] port <- attr(connection, "port")() user <- attr(connection, "user")() password <- attr(connection, "password")() diff --git a/R/Compression.R b/R/Compression.R index 3d973e0c..11b06b77 100644 --- a/R/Compression.R +++ b/R/Compression.R @@ -1,6 +1,6 @@ # @file InsertTable.R # -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2025 Observational Health Data Sciences and Informatics # # This file is part of DatabaseConnector # diff --git a/R/Connect.R b/R/Connect.R index fbe4740c..8c510f95 100644 --- a/R/Connect.R +++ b/R/Connect.R @@ -1,6 +1,6 @@ # @file Connect.R # -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2025 Observational Health Data Sciences and Informatics # # This file is part of DatabaseConnector # diff --git a/R/CtasHack.R b/R/CtasHack.R index d46acbe8..a659719d 100644 --- a/R/CtasHack.R +++ b/R/CtasHack.R @@ -1,6 +1,6 @@ # @file CtasHack.R # -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2025 Observational Health Data Sciences and Informatics # # This file is part of DatabaseConnector # @@ -178,3 +178,41 @@ ctasHack <- function(connection, sqlTableName, tempTable, sqlFieldNames, sqlData delta <- Sys.time() - startTime inform(paste("Inserting data took", signif(delta, 3), attr(delta, "units"))) } + +multiValuesInsert <- function(connection, sqlTableName, sqlFieldNames, sqlDataTypes, data, progressBar, tempEmulationSchema) { + logTrace(sprintf("Inserting %d rows into table '%s' using multi-values inserts", nrow(data), sqlTableName)) + + assign("noLogging", TRUE, envir = globalVars) + on.exit( + assign("noLogging", NULL, envir = globalVars) + ) + startTime <- Sys.time() + batchSize <- 1000 + + # Insert data in batches using multi-value inserts: + if (progressBar) { + pb <- txtProgressBar(style = 3) + } + for (start in seq(1, nrow(data), by = batchSize)) { + if (progressBar) { + setTxtProgressBar(pb, start / nrow(data)) + } + end <- min(start + batchSize - 1, nrow(data)) + batch <- toStrings(data[start:end, , drop = FALSE], sqlDataTypes) + valuesString <- paste("(", paste(apply(batch, MARGIN = 1, FUN = paste, collapse = ","), collapse = "),("), ")") + + sql <- "INSERT INTO @table (@fields) VALUES @values;" + sql <- SqlRender::render(sql = sql, + table = sqlTableName, + fields = sqlFieldNames, + values = valuesString) + sql <- SqlRender::translate(sql, targetDialect = dbms(connection), tempEmulationSchema = tempEmulationSchema) + executeSql(connection, sql, progressBar = FALSE, reportOverallTime = FALSE) + } + if (progressBar) { + setTxtProgressBar(pb, 1) + close(pb) + } + delta <- Sys.time() - startTime + inform(paste("Inserting data took", signif(delta, 3), attr(delta, "units"))) +} diff --git a/R/DBI.R b/R/DBI.R index 669b8a0a..ff9a956d 100644 --- a/R/DBI.R +++ b/R/DBI.R @@ -1,4 +1,4 @@ -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2025 Observational Health Data Sciences and Informatics # # This file is part of DatabaseConnector # diff --git a/R/DataHash.R b/R/DataHash.R index cc165a29..91c9b6ec 100644 --- a/R/DataHash.R +++ b/R/DataHash.R @@ -1,4 +1,4 @@ -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2025 Observational Health Data Sciences and Informatics # # This file is part of DatabaseConnector # @@ -263,4 +263,4 @@ bulkHashRedShift <- function(connection, databaseSchema, tables) { strings <- sapply(subsets, createStringPerTable) names(strings) <- NULL return(strings) -} \ No newline at end of file +} diff --git a/R/DatabaseConnector.R b/R/DatabaseConnector.R index 726ed2cf..b03b82fe 100644 --- a/R/DatabaseConnector.R +++ b/R/DatabaseConnector.R @@ -1,6 +1,6 @@ # @file DatabaseConnector.R # -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2025 Observational Health Data Sciences and Informatics # # This file is part of DatabaseConnector # @@ -63,12 +63,12 @@ NULL #' # Netezza #' #' Read the instructions -#' [here](https://www.ibm.com/docs/en/SSULQD_7.2.1/com.ibm.nz.datacon.doc/t_datacon_setup_JDBC.html) +#' [here](https://www.ibm.com/docs/en/netezza?topic=dls-installing-configuring-jdbc) #' on how to obtain the Netezza JDBC driver. #' #' # Impala #' -#' Go to [Cloudera's site](https://www.cloudera.com/downloads/connectors/impala/jdbc/2-5-5.html), pick +#' Go to [Cloudera's site](https://www.cloudera.com/downloads/connectors/impala/jdbc.html), pick #' your OS version, and click "GET IT NOW!'. Register, and you should be able to download the driver. #' #' # SQLite diff --git a/R/DbiDateFunctions.R b/R/DbiDateFunctions.R index 4e62d344..0b5db703 100644 --- a/R/DbiDateFunctions.R +++ b/R/DbiDateFunctions.R @@ -1,4 +1,4 @@ -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2025 Observational Health Data Sciences and Informatics # # This file is part of DatabaseConnector # diff --git a/R/Drivers.R b/R/Drivers.R index 581ffef3..7861ba8d 100644 --- a/R/Drivers.R +++ b/R/Drivers.R @@ -1,6 +1,6 @@ # @file Drivers.R # -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2025 Observational Health Data Sciences and Informatics # # This file is part of DatabaseConnector # @@ -98,9 +98,10 @@ downloadJdbcDrivers <- function(dbms, pathToDriver = Sys.getenv("DATABASECONNECT 4,oracle,oracleV19.8.zip,https://ohdsi.github.io/DatabaseConnectorJars/ 5,spark,DatabricksJDBC42-2.6.36.1062.zip,https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.36/ 6,snowflake,snowflake-jdbc-3.16.1.jar,https://repo1.maven.org/maven2/net/snowflake/snowflake-jdbc/3.16.1/ - 7,bigquery,SimbaBigQueryJDBC42-1.3.2.1003.zip,https://storage.googleapis.com/simba-bq-release/jdbc/ + 7,bigquery,SimbaJDBCDriverforGoogleBigQuery42_1.6.2.1003.zip,https://storage.googleapis.com/simba-bq-release/jdbc/ 8,iris,intersystems-jdbc-3.10.2.jar,https://repo1.maven.org/maven2/com/intersystems/intersystems-jdbc/3.10.2/" ) + if (dbms == "all") { dbms <- jdbcDriverSources$dbms } diff --git a/R/HelperFunctions.R b/R/HelperFunctions.R index d777ffba..a49e032e 100644 --- a/R/HelperFunctions.R +++ b/R/HelperFunctions.R @@ -1,4 +1,4 @@ -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2025 Observational Health Data Sciences and Informatics # # This file is part of DatabaseConnector # diff --git a/R/InsertTable.R b/R/InsertTable.R index 9e996695..a4ccb9f4 100644 --- a/R/InsertTable.R +++ b/R/InsertTable.R @@ -1,6 +1,6 @@ # @file InsertTable.R # -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2025 Observational Health Data Sciences and Informatics # # This file is part of DatabaseConnector # @@ -312,6 +312,8 @@ insertTable.default <- function(connection, } else if (useCtasHack) { # Inserting using CTAS hack ---------------------------------------------------------------- ctasHack(connection, sqlTableName, tempTable, sqlFieldNames, sqlDataTypes, data, progressBar, tempEmulationSchema) + } else if (dbms == "spark") { + multiValuesInsert(connection, sqlTableName, sqlFieldNames, sqlDataTypes, data, progressBar, tempEmulationSchema) } else { # Inserting using SQL inserts -------------------------------------------------------------- logTrace(sprintf("Inserting %d rows into table '%s'", nrow(data), sqlTableName)) diff --git a/R/ListTables.R b/R/ListTables.R index 72f8c62c..911d8c6b 100644 --- a/R/ListTables.R +++ b/R/ListTables.R @@ -1,6 +1,6 @@ # @file ListTables.R # -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2025 Observational Health Data Sciences and Informatics # # This file is part of DatabaseConnector # diff --git a/R/RStudio.R b/R/RStudio.R index dc4c83f6..02ef0997 100644 --- a/R/RStudio.R +++ b/R/RStudio.R @@ -1,6 +1,6 @@ # @file RStudio.R # -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2025 Observational Health Data Sciences and Informatics # # This file is part of DatabaseConnector # @@ -87,6 +87,7 @@ listDatabaseConnectorColumns <- function(connection, UseMethod("listDatabaseConnectorColumns", connection) } +#' @exportS3Method listDatabaseConnectorColumns.default <- function(connection, catalog = NULL, schema = NULL, @@ -136,6 +137,7 @@ listDatabaseConnectorColumns.default <- function(connection, return(data.frame(name = fields, type = types, stringsAsFactors = FALSE)) } +#' @exportS3Method listDatabaseConnectorColumns.DatabaseConnectorDbiConnection <- function(connection, catalog = NULL, schema = NULL, @@ -220,6 +222,7 @@ getServer <- function(connection) { UseMethod("getServer", connection) } +#' @exportS3Method getServer.default <- function(connection) { if (dbms(connection) == "hive") { url <- connection@url @@ -235,6 +238,7 @@ getServer.default <- function(connection) { return(server) } +#' @exportS3Method getServer.DatabaseConnectorDbiConnection <- function(connection) { return(connection@server) } @@ -243,6 +247,7 @@ compileReconnectCode <- function(connection) { UseMethod("compileReconnectCode", connection) } +#' @exportS3Method compileReconnectCode.default <- function(connection) { databaseMetaData <- rJava::.jcall( connection@jConnection, @@ -265,6 +270,7 @@ compileReconnectCode.default <- function(connection) { return(code) } +#' @exportS3Method compileReconnectCode.DatabaseConnectorDbiConnection <- function(connection) { code <- sprintf( "library(DatabaseConnector)\ncon <- connect(dbms = \"%s\", server = \"%s\")", @@ -278,6 +284,7 @@ getSchemaNames <- function(conn, catalog = NULL) { UseMethod("getSchemaNames", conn) } +#' @exportS3Method getSchemaNames.default <- function(conn, catalog = NULL) { if (is.null(catalog)) { catalog <- rJava::.jnull("java/lang/String") @@ -301,6 +308,7 @@ getSchemaNames.default <- function(conn, catalog = NULL) { return(schemas) } +#' @exportS3Method getSchemaNames.DatabaseConnectorDbiConnection <- function(conn, catalog = NULL) { if (dbms(conn) %in% c("sqlite", "sqlite extended")) { return("main") @@ -319,6 +327,7 @@ getCatalogs <- function(connection) { UseMethod("getCatalogs", connection) } +#' @exportS3Method getCatalogs.default <- function(connection) { metaData <- rJava::.jcall(connection@jConnection, "Ljava/sql/DatabaseMetaData;", "getMetaData") resultSet <- rJava::.jcall(metaData, "Ljava/sql/ResultSet;", "getCatalogs") @@ -331,6 +340,7 @@ getCatalogs.default <- function(connection) { return(catalogs) } +#' @exportS3Method getCatalogs.DatabaseConnectorDbiConnection <- function(connection) { if (connection@dbms == "duckdb") { sql <- " diff --git a/R/Sql.R b/R/Sql.R index c5687a2a..f1e990bd 100644 --- a/R/Sql.R +++ b/R/Sql.R @@ -1,6 +1,6 @@ # @file Sql.R # -# Copyright 2023 Observational Health Data Sciences and Informatics +# Copyright 2025 Observational Health Data Sciences and Informatics # # This file is part of DatabaseConnector # diff --git a/cran-comments.md b/cran-comments.md index 5bf4c873..66f905aa 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,12 +1,12 @@ -This update includes 1 bugfix. (see NEWS.md). +This update includes 4 minor changes. (see NEWS.md). --- ## Test environments -* Ubuntu 20.04, R 4.3.1 -* MacOS, R 4.2.3 -* MacOS, 4.3.1 -* Windows 10, R 4.3.1 +* Ubuntu 20.04, R 4.4.2 +* MacOS, R 4.4.2 +* MacOS, 4.4.1 +* Windows 10, R 4.4.2 ## R CMD check results diff --git a/docs/404.html b/docs/404.html index d96e7633..1ffed6c0 100644 --- a/docs/404.html +++ b/docs/404.html @@ -6,7 +6,7 @@