Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

sql_insert: support keyword options to add before the INTO clause of the query #2999

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions docs/modules/components/pages/outputs/sql_insert.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ output:
args_mapping: root = [ this.cat.meow, this.doc.woofs[0] ] # No default (required)
prefix: "" # No default (optional)
suffix: ON CONFLICT (name) DO NOTHING # No default (optional)
options: [] # No default (optional)
max_in_flight: 64
init_files: [] # No default (optional)
init_statement: | # No default (optional)
Expand Down Expand Up @@ -276,6 +277,22 @@ An optional suffix to append to the insert query.
suffix: ON CONFLICT (name) DO NOTHING
```

=== `options`

A list of keyword options to add before the INTO clause of the query.


*Type*: `array`


```yml
# Examples

options:
- DELAYED
- IGNORE
```

=== `max_in_flight`

The maximum number of inserts to run in parallel.
Expand Down
17 changes: 17 additions & 0 deletions docs/modules/components/pages/processors/sql_insert.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ sql_insert:
args_mapping: root = [ this.cat.meow, this.doc.woofs[0] ] # No default (required)
prefix: "" # No default (optional)
suffix: ON CONFLICT (name) DO NOTHING # No default (optional)
options: [] # No default (optional)
init_files: [] # No default (optional)
init_statement: | # No default (optional)
CREATE TABLE IF NOT EXISTS some_table (
Expand Down Expand Up @@ -264,6 +265,22 @@ An optional suffix to append to the insert query.
suffix: ON CONFLICT (name) DO NOTHING
```

=== `options`

A list of keyword options to add before the INTO clause of the query.


*Type*: `array`


```yml
# Examples

options:
- DELAYED
- IGNORE
```

=== `init_files`

An optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).
Expand Down
13 changes: 13 additions & 0 deletions internal/impl/sql/output_sql_insert.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,11 @@ func sqlInsertOutputConfig() *service.ConfigSpec {
Optional().
Advanced().
Example("ON CONFLICT (name) DO NOTHING")).
Field(service.NewStringListField("options").
Description("A list of keyword options to add before the INTO clause of the query.").
Optional().
Advanced().
Example([]string{"DELAYED", "IGNORE"})).
Field(service.NewIntField("max_in_flight").
Description("The maximum number of inserts to run in parallel.").
Default(64))
Expand Down Expand Up @@ -198,6 +203,14 @@ func newSQLInsertOutputFromConfig(conf *service.ParsedConfig, mgr *service.Resou
s.builder = s.builder.Suffix(suffixStr)
}

if conf.Contains("options") {
options, err := conf.FieldStringList("options")
if err != nil {
return nil, err
}
s.builder = s.builder.Options(options...)
}

if s.connSettings, err = connSettingsFromParsed(conf, mgr); err != nil {
return nil, err
}
Expand Down
15 changes: 14 additions & 1 deletion internal/impl/sql/processor_sql_insert.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,12 @@ If the insert fails to execute then the message will still remain unchanged and
Description("An optional suffix to append to the insert query.").
Optional().
Advanced().
Example("ON CONFLICT (name) DO NOTHING"))
Example("ON CONFLICT (name) DO NOTHING")).
Field(service.NewStringListField("options").
Description("A list of keyword options to add before the INTO clause of the query.").
Optional().
Advanced().
Example([]string{"DELAYED", "IGNORE"}))

for _, f := range connFields() {
spec = spec.Field(f)
Expand Down Expand Up @@ -187,6 +192,14 @@ func NewSQLInsertProcessorFromConfig(conf *service.ParsedConfig, mgr *service.Re
s.builder = s.builder.Suffix(suffixStr)
}

if conf.Contains("options") {
options, err := conf.FieldStringList("options")
if err != nil {
return nil, err
}
s.builder = s.builder.Options(options...)
}

connSettings, err := connSettingsFromParsed(conf, mgr)
if err != nil {
return nil, err
Expand Down