Merge branch 'OHDSI-main'

intersystems-community · Jan 27, 2025 · cd08658 · cd08658
2 parents 2ac302c + 2ffbd15
commit cd08658
Show file tree

Hide file tree

Showing 168 changed files with 2,303 additions and 2,168 deletions.
diff --git a/.github/workflows/R_CMD_check_Hades.yaml b/.github/workflows/R_CMD_check_Hades.yaml
@@ -111,16 +111,31 @@ jobs:
           path: check/*.tar.gz
 
       - name: Install covr
-        if: runner.os == 'macOS'
+        if: runner.os == 'Linux'
         run: |
-          install.packages("covr")
+          remotes::install_cran("covr")
+          remotes::install_cran("xml2")
         shell: Rscript {0}
-
+        
       - name: Test coverage
-        if: runner.os == 'macOS'
-        run: covr::codecov()
+        if: runner.os == 'Linux'
+        run: |
+          cov <- covr::package_coverage(
+            quiet = FALSE,
+            clean = FALSE,
+            install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package")
+          )
+          covr::to_cobertura(cov)
         shell: Rscript {0}
 
+      - uses: codecov/codecov-action@v4
+        if: runner.os == 'Linux'
+        with:
+          file: ./cobertura.xml
+          plugin: noop
+          disable_search: true
+          token: ${{ secrets.CODECOV_TOKEN }}
+
   Release:
     needs: R-CMD-Check
 

diff --git a/CRAN-SUBMISSION b/CRAN-SUBMISSION
@@ -1,3 +1,3 @@
-Version: 6.3.2
-Date: 2023-12-11 13:54:30 UTC
-SHA: dd511ec8b23927ffb61a17bedd9ee8bb81cbe476
+Version: 6.3.3
+Date: 2025-01-16 00:44:34 UTC
+SHA: 7b5548f88a19ff84ca3f1b7c075ab370723e8341
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -2,7 +2,7 @@ Package: DatabaseConnector
 Type: Package
 Title: Connecting to Various Database Platforms
 Version: 6.3.3
-Date: 2024-06-13
+Date: 2025-01-15
 Authors@R: c(
     person("Martijn", "Schuemie", email = "[email protected]", role = c("aut", "cre")),
     person("Marc", "Suchard", role = c("aut")),

diff --git a/NAMESPACE b/NAMESPACE
@@ -1,9 +1,19 @@
 # Generated by roxygen2: do not edit by hand
 
+S3method(compileReconnectCode,DatabaseConnectorDbiConnection)
+S3method(compileReconnectCode,default)
 S3method(disconnect,DatabaseConnectorDbiConnection)
 S3method(disconnect,default)
+S3method(getCatalogs,DatabaseConnectorDbiConnection)
+S3method(getCatalogs,default)
+S3method(getSchemaNames,DatabaseConnectorDbiConnection)
+S3method(getSchemaNames,default)
+S3method(getServer,DatabaseConnectorDbiConnection)
+S3method(getServer,default)
 S3method(insertTable,DatabaseConnectorDbiConnection)
 S3method(insertTable,default)
+S3method(listDatabaseConnectorColumns,DatabaseConnectorDbiConnection)
+S3method(listDatabaseConnectorColumns,default)
 S3method(lowLevelExecuteSql,DatabaseConnectorDbiConnection)
 S3method(lowLevelExecuteSql,default)
 S3method(lowLevelQuerySql,DatabaseConnectorDbiConnection)

diff --git a/NEWS.md b/NEWS.md
@@ -7,6 +7,10 @@ Changes:
 
 - Updated Databricks driver to 2.6.36.
 
+- Updated BigQuery driver to 1.6.2.
+
+- Using `INSERT` with multiple values in `insertTable()` for DataBricks for faster inserts.
+
 
 DatabaseConnector 6.3.2
 =======================

diff --git a/R/Andromeda.R b/R/Andromeda.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #

diff --git a/R/BulkLoad.R b/R/BulkLoad.R
@@ -1,6 +1,6 @@
 # @file BulkLoad.R
 #
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #
@@ -299,7 +299,19 @@ bulkLoadPostgres <- function(connection, sqlTableName, sqlFieldNames, sqlDataTyp
   readr::write_excel_csv(data, csvFileName, na = "")
   on.exit(unlink(csvFileName))
 
-  hostServerDb <- strsplit(attr(connection, "server")(), "/")[[1]]
+  server <- attr(connection, "server")()
+  if (is.null(server)) {
+    # taken directly from DatabaseConnector R/RStudio.R - getServer.default, could an attr too?
+    databaseMetaData <- rJava::.jcall(
+      connection@jConnection,
+      "Ljava/sql/DatabaseMetaData;",
+      "getMetaData"
+    )
+    server <- rJava::.jcall(databaseMetaData, "Ljava/lang/String;", "getURL")
+    server <- strsplit(server, "//")[[1]][2]
+  }
+
+  hostServerDb <- strsplit(server, "/")[[1]]
   port <- attr(connection, "port")()
   user <- attr(connection, "user")()
   password <- attr(connection, "password")()

diff --git a/R/Compression.R b/R/Compression.R
@@ -1,6 +1,6 @@
 # @file InsertTable.R
 #
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #

diff --git a/R/Connect.R b/R/Connect.R
@@ -1,6 +1,6 @@
 # @file Connect.R
 #
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #

diff --git a/R/CtasHack.R b/R/CtasHack.R
@@ -1,6 +1,6 @@
 # @file CtasHack.R
 #
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #
@@ -178,3 +178,41 @@ ctasHack <- function(connection, sqlTableName, tempTable, sqlFieldNames, sqlData
   delta <- Sys.time() - startTime
   inform(paste("Inserting data took", signif(delta, 3), attr(delta, "units")))
 }
+
+multiValuesInsert <- function(connection, sqlTableName, sqlFieldNames, sqlDataTypes, data, progressBar, tempEmulationSchema) {
+  logTrace(sprintf("Inserting %d rows into table '%s' using multi-values inserts", nrow(data), sqlTableName))
+
+  assign("noLogging", TRUE, envir = globalVars)
+  on.exit(
+    assign("noLogging", NULL, envir = globalVars)
+  )
+  startTime <- Sys.time()
+  batchSize <- 1000
+
+  # Insert data in batches using multi-value inserts:
+  if (progressBar) {
+    pb <- txtProgressBar(style = 3)
+  }
+  for (start in seq(1, nrow(data), by = batchSize)) {
+    if (progressBar) {
+      setTxtProgressBar(pb, start / nrow(data))
+    }
+    end <- min(start + batchSize - 1, nrow(data))
+    batch <- toStrings(data[start:end, , drop = FALSE], sqlDataTypes)
+    valuesString <- paste("(", paste(apply(batch, MARGIN = 1, FUN = paste, collapse = ","), collapse = "),("), ")")
+
+    sql <- "INSERT INTO @table (@fields) VALUES @values;"
+    sql <- SqlRender::render(sql = sql,
+                             table = sqlTableName,
+                             fields = sqlFieldNames,
+                             values = valuesString)
+    sql <- SqlRender::translate(sql, targetDialect = dbms(connection), tempEmulationSchema = tempEmulationSchema)
+    executeSql(connection, sql, progressBar = FALSE, reportOverallTime = FALSE)
+  }
+  if (progressBar) {
+    setTxtProgressBar(pb, 1)
+    close(pb)
+  }
+  delta <- Sys.time() - startTime
+  inform(paste("Inserting data took", signif(delta, 3), attr(delta, "units")))
+}
diff --git a/R/DBI.R b/R/DBI.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #

diff --git a/R/DataHash.R b/R/DataHash.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #
@@ -263,4 +263,4 @@ bulkHashRedShift <- function(connection, databaseSchema, tables) {
   strings <- sapply(subsets, createStringPerTable)
   names(strings) <- NULL
   return(strings)
-}
+}
diff --git a/R/DatabaseConnector.R b/R/DatabaseConnector.R
@@ -1,6 +1,6 @@
 # @file DatabaseConnector.R
 #
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #
@@ -63,12 +63,12 @@ NULL
 #' # Netezza
 #' 
 #' Read the instructions
-#' [here](https://www.ibm.com/docs/en/SSULQD_7.2.1/com.ibm.nz.datacon.doc/t_datacon_setup_JDBC.html)
+#' [here](https://www.ibm.com/docs/en/netezza?topic=dls-installing-configuring-jdbc)
 #' on how to obtain the Netezza JDBC driver.
 #'
 #' # Impala
 #' 
-#' Go to [Cloudera's site](https://www.cloudera.com/downloads/connectors/impala/jdbc/2-5-5.html), pick
+#' Go to [Cloudera's site](https://www.cloudera.com/downloads/connectors/impala/jdbc.html), pick
 #' your OS version, and click "GET IT NOW!'. Register, and you should be able to download the driver.
 #'
 #' # SQLite

diff --git a/R/DbiDateFunctions.R b/R/DbiDateFunctions.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #

diff --git a/R/Drivers.R b/R/Drivers.R
@@ -1,6 +1,6 @@
 # @file Drivers.R
 #
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #
@@ -98,9 +98,10 @@ downloadJdbcDrivers <- function(dbms, pathToDriver = Sys.getenv("DATABASECONNECT
     4,oracle,oracleV19.8.zip,https://ohdsi.github.io/DatabaseConnectorJars/
     5,spark,DatabricksJDBC42-2.6.36.1062.zip,https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.36/
     6,snowflake,snowflake-jdbc-3.16.1.jar,https://repo1.maven.org/maven2/net/snowflake/snowflake-jdbc/3.16.1/
-    7,bigquery,SimbaBigQueryJDBC42-1.3.2.1003.zip,https://storage.googleapis.com/simba-bq-release/jdbc/
+    7,bigquery,SimbaJDBCDriverforGoogleBigQuery42_1.6.2.1003.zip,https://storage.googleapis.com/simba-bq-release/jdbc/
     8,iris,intersystems-jdbc-3.10.2.jar,https://repo1.maven.org/maven2/com/intersystems/intersystems-jdbc/3.10.2/"
   )
+
   if (dbms == "all") {
     dbms <- jdbcDriverSources$dbms
   }

diff --git a/R/HelperFunctions.R b/R/HelperFunctions.R
@@ -1,4 +1,4 @@
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #

diff --git a/R/InsertTable.R b/R/InsertTable.R
@@ -1,6 +1,6 @@
 # @file InsertTable.R
 #
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #
@@ -312,6 +312,8 @@ insertTable.default <- function(connection,
   } else if (useCtasHack) {
     # Inserting using CTAS hack ----------------------------------------------------------------
     ctasHack(connection, sqlTableName, tempTable, sqlFieldNames, sqlDataTypes, data, progressBar, tempEmulationSchema)
+  } else if (dbms == "spark") {
+    multiValuesInsert(connection, sqlTableName, sqlFieldNames, sqlDataTypes, data, progressBar, tempEmulationSchema)
   } else {
     # Inserting using SQL inserts --------------------------------------------------------------
     logTrace(sprintf("Inserting %d rows into table '%s'", nrow(data), sqlTableName))

diff --git a/R/ListTables.R b/R/ListTables.R
@@ -1,6 +1,6 @@
 # @file ListTables.R
 #
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #

diff --git a/R/RStudio.R b/R/RStudio.R
@@ -1,6 +1,6 @@
 # @file RStudio.R
 #
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #
@@ -87,6 +87,7 @@ listDatabaseConnectorColumns <- function(connection,
   UseMethod("listDatabaseConnectorColumns", connection)
 }
 
+#' @exportS3Method
 listDatabaseConnectorColumns.default <- function(connection,
                                                  catalog = NULL,
                                                  schema = NULL,
@@ -136,6 +137,7 @@ listDatabaseConnectorColumns.default <- function(connection,
   return(data.frame(name = fields, type = types, stringsAsFactors = FALSE))
 }
 
+#' @exportS3Method
 listDatabaseConnectorColumns.DatabaseConnectorDbiConnection <- function(connection,
                                                                         catalog = NULL,
                                                                         schema = NULL,
@@ -220,6 +222,7 @@ getServer <- function(connection) {
   UseMethod("getServer", connection)
 }
 
+#' @exportS3Method
 getServer.default <- function(connection) {
   if (dbms(connection) == "hive") {
     url <- connection@url
@@ -235,6 +238,7 @@ getServer.default <- function(connection) {
   return(server)
 }
 
+#' @exportS3Method
 getServer.DatabaseConnectorDbiConnection <- function(connection) {
   return(connection@server)
 }
@@ -243,6 +247,7 @@ compileReconnectCode <- function(connection) {
   UseMethod("compileReconnectCode", connection)
 }
 
+#' @exportS3Method
 compileReconnectCode.default <- function(connection) {
   databaseMetaData <- rJava::.jcall(
     connection@jConnection,
@@ -265,6 +270,7 @@ compileReconnectCode.default <- function(connection) {
   return(code)
 }
 
+#' @exportS3Method
 compileReconnectCode.DatabaseConnectorDbiConnection <- function(connection) {
   code <- sprintf(
     "library(DatabaseConnector)\ncon <- connect(dbms = \"%s\", server = \"%s\")",
@@ -278,6 +284,7 @@ getSchemaNames <- function(conn, catalog = NULL) {
   UseMethod("getSchemaNames", conn)
 }
 
+#' @exportS3Method
 getSchemaNames.default <- function(conn, catalog = NULL) {
   if (is.null(catalog)) {
     catalog <- rJava::.jnull("java/lang/String")
@@ -301,6 +308,7 @@ getSchemaNames.default <- function(conn, catalog = NULL) {
   return(schemas)
 }
 
+#' @exportS3Method
 getSchemaNames.DatabaseConnectorDbiConnection <- function(conn, catalog = NULL) {
   if (dbms(conn) %in% c("sqlite", "sqlite extended")) {
     return("main")
@@ -319,6 +327,7 @@ getCatalogs <- function(connection) {
   UseMethod("getCatalogs", connection)
 }
 
+#' @exportS3Method
 getCatalogs.default <- function(connection) {
   metaData <- rJava::.jcall(connection@jConnection, "Ljava/sql/DatabaseMetaData;", "getMetaData")
   resultSet <- rJava::.jcall(metaData, "Ljava/sql/ResultSet;", "getCatalogs")
@@ -331,6 +340,7 @@ getCatalogs.default <- function(connection) {
   return(catalogs)
 }
 
+#' @exportS3Method
 getCatalogs.DatabaseConnectorDbiConnection <- function(connection) {
   if (connection@dbms == "duckdb") {
     sql <- "

diff --git a/R/Sql.R b/R/Sql.R
@@ -1,6 +1,6 @@
 # @file Sql.R
 #
-# Copyright 2023 Observational Health Data Sciences and Informatics
+# Copyright 2025 Observational Health Data Sciences and Informatics
 #
 # This file is part of DatabaseConnector
 #

diff --git a/cran-comments.md b/cran-comments.md
@@ -1,12 +1,12 @@
-This update includes 1 bugfix. (see NEWS.md).
+This update includes 4 minor changes. (see NEWS.md).
 
 ---
 
 ## Test environments
-* Ubuntu 20.04, R 4.3.1
-* MacOS, R 4.2.3
-* MacOS, 4.3.1
-* Windows 10, R 4.3.1
+* Ubuntu 20.04, R 4.4.2
+* MacOS, R 4.4.2
+* MacOS, 4.4.1
+* Windows 10, R 4.4.2
 
 ## R CMD check results