From b7510a09e2028181679902e8fb6ef70b02693f99 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:07:04 -0600 Subject: [PATCH 01/70] sched: add coroutine support to timer callback Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_scheduler.h | 142 ++++++++++++++++- src/flb_scheduler.c | 237 ++++++++++++++++++++++++++++- 2 files changed, 366 insertions(+), 13 deletions(-) diff --git a/include/fluent-bit/flb_scheduler.h b/include/fluent-bit/flb_scheduler.h index 52c72c019d3..71213b100de 100644 --- a/include/fluent-bit/flb_scheduler.h +++ b/include/fluent-bit/flb_scheduler.h @@ -31,10 +31,13 @@ #define FLB_SCHED_REQUEST_FRAME 10 /* Timer types */ -#define FLB_SCHED_TIMER_REQUEST 1 /* timerfd */ -#define FLB_SCHED_TIMER_FRAME 2 /* timer frame checker */ -#define FLB_SCHED_TIMER_CB_ONESHOT 3 /* one-shot callback timer */ -#define FLB_SCHED_TIMER_CB_PERM 4 /* permanent callback timer */ +#define FLB_SCHED_TIMER_REQUEST 1 /* timerfd */ +#define FLB_SCHED_TIMER_FRAME 2 /* timer frame checker */ +#define FLB_SCHED_TIMER_CB_ONESHOT 3 /* one-shot callback timer */ +#define FLB_SCHED_TIMER_CB_PERM 4 /* permanent callback timer */ + +/* notifications through channels */ +#define FLB_SCHED_TIMER_CORO_RETURN 1 struct flb_sched; @@ -50,6 +53,7 @@ struct flb_sched_timer { struct mk_event event; int active; int type; + int coro; void *data; struct flb_sched *sched; @@ -58,9 +62,11 @@ struct flb_sched_timer { * * - timer_fd = timer file descriptor * - cb = callback to be triggerd upon expiration + * - cb_coroutine_wrapper = coroutine wrapper for the callback */ int timer_fd; void (*cb)(struct flb_config *, void *); + void (*cb_coroutine_wrapper)(struct flb_config *, void *); /* Parent context */ struct flb_config *config; @@ -81,6 +87,7 @@ struct flb_sched_request { /* Scheduler context */ struct flb_sched { + struct mk_event event; /* event context to associate events */ /* * Scheduler lists: @@ -112,13 +119,27 @@ struct flb_sched { */ struct mk_list timers_drop; + /* Linked list of timers*/ + struct cfl_list timer_coro_list; + struct cfl_list timer_coro_list_drop; + /* Frame timer context */ flb_pipefd_t frame_fd; struct mk_event_loop *evl; struct flb_config *config; + + /* + * Every scheduler context have it own file descriptor to receive + * custom notifications from other scheduler components. The primary use + * case is the use of timers running under a co-routine that needs to + * be handled in active event loop. + */ + flb_pipefd_t ch_events[2]; }; + + int flb_sched_request_create(struct flb_config *config, void *data, int tries); int flb_sched_request_destroy(struct flb_sched_request *req); @@ -137,11 +158,20 @@ int flb_sched_request_invalidate(struct flb_config *config, void *data); int flb_sched_timer_cb_create(struct flb_sched *sched, int type, int ms, void (*cb)(struct flb_config *, void *), void *data, struct flb_sched_timer **out_timer); +int flb_sched_timer_coro_cb_create(struct flb_sched *sched, int type, int64_t ms, + void (*cb)(struct flb_config *, void *), + void *data, struct flb_sched_timer **out_timer); + +struct flb_sched_timer_coro *flb_sched_timer_coro_create(struct flb_sched_timer *timer, + struct flb_config *config, + void *data); +int flb_sched_timer_coro_cleanup(struct flb_sched *sched); + int flb_sched_timer_cb_disable(struct flb_sched_timer *timer); int flb_sched_timer_cb_destroy(struct flb_sched_timer *timer); void flb_sched_timer_invalidate(struct flb_sched_timer *timer); int flb_sched_timer_cleanup(struct flb_sched *sched); -int flb_sched_retry_now(struct flb_config *config, +int flb_sched_retry_now(struct flb_config *config, struct flb_task_retry *retry); /* Sched context api for multithread environment */ @@ -149,4 +179,106 @@ void flb_sched_ctx_init(); struct flb_sched *flb_sched_ctx_get(); void flb_sched_ctx_set(struct flb_sched *sched); + +struct flb_sched_timer_coro { + uint32_t id; + struct flb_sched_timer *timer; + struct flb_config *config; + struct flb_coro *coro; + void *data; + + /* link to sched->timer_coro_list */ + struct cfl_list _head; +}; + +/* parameter for timer callback running under a co-routine */ +struct flb_sched_timer_coro_cb_params { + struct flb_sched_timer_coro *stc; + struct flb_config *config; + void *data; + struct flb_coro *coro; +}; + +extern FLB_TLS_DEFINE(struct flb_sched_timer_cb_coro_params, sched_timer_coro_cb_params); + + +struct flb_timer_cb_coro_params { + struct flb_config *config; + void *data; +}; + + +static FLB_INLINE void flb_sched_timer_cb_coro_return() +{ + int n; + uint64_t val; + struct flb_coro *coro; + struct flb_sched *sched; + struct flb_sched_timer_coro *stc; + + coro = flb_coro_get(); + + sched = flb_sched_ctx_get(); + if (!sched) { + flb_error("[sched] invalid scheduler context"); + return; + } + + stc = (struct flb_sched_timer_coro *) coro->data; + if (!stc) { + flb_error("[sched] invalid timer coro context"); + return; + } + + val = FLB_BITS_U64_SET(FLB_SCHED_TIMER_CORO_RETURN, stc->id); + n = flb_pipe_w(sched->ch_events[1], &val, sizeof(val)); + if (n == -1) { + flb_errno(); + } + + flb_coro_yield(coro, FLB_TRUE); +} + +static FLB_INLINE void sched_timer_cb_params_set(struct flb_sched_timer_coro *stc, + struct flb_coro *coro, struct flb_config *config, void *data) +{ + struct flb_sched_timer_coro_cb_params *params; + + params = (struct flb_sched_timer_coro_cb_params *) FLB_TLS_GET(sched_timer_coro_cb_params); + if (!params) { + params = flb_malloc(sizeof(struct flb_sched_timer_coro_cb_params)); + if (!params) { + flb_errno(); + return; + } + } + + params->stc = stc; + params->config = config; + params->data = data; + params->coro = coro; + + FLB_TLS_SET(sched_timer_coro_cb_params, params); + co_switch(coro->callee); +} + +static FLB_INLINE void sched_timer_coro_cb_run(void) +{ + struct flb_coro *coro; + struct flb_sched_timer *timer; + struct flb_sched_timer_coro_cb_params *params; + + params = (struct flb_sched_timer_coro_cb_params *) FLB_TLS_GET(sched_timer_coro_cb_params); + if (!params) { + return; + } + + coro = params->coro; + + co_switch(coro->caller); + + timer = params->stc->timer; + timer->cb(params->config, params->data); +} + #endif diff --git a/src/flb_scheduler.c b/src/flb_scheduler.c index 4bc88c272c1..8dab1fe3222 100644 --- a/src/flb_scheduler.c +++ b/src/flb_scheduler.c @@ -31,10 +31,12 @@ #include FLB_TLS_DEFINE(struct flb_sched, flb_sched_ctx); +FLB_TLS_DEFINE(struct flb_sched_timer_cb_params, sched_timer_coro_cb_params); void flb_sched_ctx_init() { FLB_TLS_INIT(flb_sched_ctx); + FLB_TLS_INIT(sched_timer_coro_cb_params); } struct flb_sched *flb_sched_ctx_get() @@ -393,7 +395,7 @@ int flb_sched_request_invalidate(struct flb_config *config, void *data) * * If we are NOT using kequeue, just read(2) the byte. */ -static inline int timer_consume_byte(int fd) +static inline int event_fd_consume_byte(int fd) { #ifndef FLB_EVENT_LOOP_KQUEUE return consume_byte(fd); @@ -402,14 +404,152 @@ static inline int timer_consume_byte(int fd) return 0; } +/* + * Lookup for the next available 'id' for a struct flb_sched_timer_coro. This is a slow search, + * however is expected we don't have more than a couple dozen active timer_coro contexts under + * the same scheduler context. + * + * We cap this as an uint32_t so we can use the sched->ch_events channels to send the id and + * link it to some notification. + */ +static inline uint32_t sched_timer_coro_get_id(struct flb_sched *sched) +{ + uint32_t id = 0; + int found = FLB_FALSE; + struct cfl_list *head; + struct flb_sched_timer_coro *stc; + + while (id < UINT32_MAX) { + /* check if the proposed id is in use already */ + cfl_list_foreach(head, &sched->timer_coro_list) { + stc = cfl_list_entry(head, struct flb_sched_timer_coro, _head); + if (stc->id == id) { + found = FLB_TRUE; + break; + } + } + + if (!found) { + break; + } + else { + id++; + found = FLB_FALSE; + } + } + + return id; +} + +/* context of a scheduled timer that holds a coroutine context */ +struct flb_sched_timer_coro *flb_sched_timer_coro_create(struct flb_sched_timer *timer, + struct flb_config *config, + void *data) +{ + size_t stack_size; + struct flb_coro *coro; + struct flb_sched *sched; + struct flb_sched_timer_coro *stc; + + /* get scheduler context */ + sched = flb_sched_ctx_get(); + if (!sched) { + flb_error("[sched] no scheduler context available"); + return NULL; + } + + stc = flb_malloc(sizeof(struct flb_sched_timer_coro)); + if (!stc) { + flb_errno(); + return NULL; + } + + stc->id = sched_timer_coro_get_id(sched); + stc->timer = timer; + stc->config = config; + stc->data = data; + + coro = flb_coro_create(stc); + if (!coro) { + flb_free(stc); + return NULL; + } + stc->coro = coro; + + coro->caller = co_active(); + coro->callee = co_create(config->coro_stack_size, + sched_timer_coro_cb_run, &stack_size); + +#ifdef FLB_HAVE_VALGRIND + coro->valgrind_stack_id = VALGRIND_STACK_REGISTER(coro->callee, ((char *) coro->callee) + stack_size); +#endif + cfl_list_add(&stc->_head, &sched->timer_coro_list); + + sched_timer_cb_params_set(stc, coro, config, data); + return stc; +} + +/* + * Create a timer that triggers the defined callback every N milliseconds. + */ +static void timer_cb_coro_trampoline(struct flb_config *config, + struct flb_sched_timer *timer, void *data) +{ + struct flb_sched_timer_coro *stc; + + stc = flb_sched_timer_coro_create(timer, config, data); + if (!stc) { + return; + } + + flb_coro_resume(stc->coro); +} + /* Handle a timeout event set by a previous flb_sched_request_create(...) */ int flb_sched_event_handler(struct flb_config *config, struct mk_event *event) { int ret; + uint64_t val; + uint32_t op; + uint32_t id; + struct flb_sched *sched; struct flb_sched_timer *timer; struct flb_sched_request *req; + struct flb_sched_timer_coro *stc; + + if (event->type == FLB_ENGINE_EV_SCHED_CORO) { + printf("CORO EVENT\n"); + stc = (struct flb_sched_timer_coro *) event; + if (!stc) { + flb_error("[sched] invalid timer coro context"); + return -1; + } + + /* consume the notification */ + val = flb_pipe_r(event->fd, &val, sizeof(val)); + if (val == -1) { + flb_errno(); + return -1; + } + + op = FLB_BITS_U64_HIGH(val); + id = FLB_BITS_U64_LOW(val); + if (op == FLB_SCHED_TIMER_CORO_RETURN) { + /* move stc to the drop list */ + sched = flb_sched_ctx_get(); + cfl_list_del(&stc->_head); + cfl_list_add(&stc->_head, &sched->timer_coro_list_drop); + } + else { + flb_error("[sched] unknown coro event operation %u", op); + } + + return 0; + } + + /* Everything else is just a normal timer handling */ timer = (struct flb_sched_timer *) event; if (timer->active == FLB_FALSE) { return 0; @@ -430,18 +570,24 @@ int flb_sched_event_handler(struct flb_config *config, struct mk_event *event) } else if (timer->type == FLB_SCHED_TIMER_FRAME) { sched = timer->data; - timer_consume_byte(sched->frame_fd); + event_fd_consume_byte(sched->frame_fd); schedule_request_promote(sched); } else if (timer->type == FLB_SCHED_TIMER_CB_ONESHOT) { - timer_consume_byte(timer->timer_fd); + event_fd_consume_byte(timer->timer_fd); flb_sched_timer_cb_disable(timer); timer->cb(config, timer->data); flb_sched_timer_cb_destroy(timer); } else if (timer->type == FLB_SCHED_TIMER_CB_PERM) { - timer_consume_byte(timer->timer_fd); - timer->cb(config, timer->data); + event_fd_consume_byte(timer->timer_fd); + + if (timer->coro == FLB_TRUE) { + timer_cb_coro_trampoline(config, timer, timer->data); + } + else { + timer->cb(config, timer->data); + } } return 0; @@ -511,12 +657,35 @@ int flb_sched_timer_cb_create(struct flb_sched *sched, int type, int ms, return 0; } +/* + * Creates a timer that triggers the defined callback every N milliseconds. The target + * function runs in a coroutine context. + */ +int flb_sched_timer_coro_cb_create(struct flb_sched *sched, int type, int64_t ms, + void (*cb)(struct flb_config *, void *), + void *data, struct flb_sched_timer **out_timer) + +{ + int ret; + struct flb_sched_timer *timer = NULL; + + ret = flb_sched_timer_cb_create(sched, type, ms, cb, data, &timer); + if (ret == -1) { + flb_error("[sched] cannot create timer for coroutine callback"); + return -1; + } + + /* mark that the callback will run inside a coroutine */ + timer->coro = FLB_TRUE; + + return 0; +} + /* Disable notifications, used before to destroy the context */ int flb_sched_timer_cb_disable(struct flb_sched_timer *timer) { if (timer->timer_fd != -1) { mk_event_timeout_destroy(timer->sched->evl, &timer->event); - timer->timer_fd = -1; } @@ -526,7 +695,6 @@ int flb_sched_timer_cb_disable(struct flb_sched_timer *timer) int flb_sched_timer_cb_destroy(struct flb_sched_timer *timer) { flb_sched_timer_destroy(timer); - return 0; } @@ -534,6 +702,7 @@ int flb_sched_timer_cb_destroy(struct flb_sched_timer *timer) struct flb_sched *flb_sched_create(struct flb_config *config, struct mk_event_loop *evl) { + int ret; flb_pipefd_t fd; struct mk_event *event; struct flb_sched *sched; @@ -554,6 +723,9 @@ struct flb_sched *flb_sched_create(struct flb_config *config, mk_list_init(&sched->timers); mk_list_init(&sched->timers_drop); + cfl_list_init(&sched->timer_coro_list); + cfl_list_init(&sched->timer_coro_list_drop); + /* Create the frame timer who enqueue 'requests' for future time */ timer = flb_sched_timer_create(sched); if (!timer) { @@ -580,6 +752,17 @@ struct flb_sched *flb_sched_create(struct flb_config *config, } sched->frame_fd = fd; + /* Creates a channel to handle notifications */ + ret = mk_event_channel_create(sched->evl, + &sched->ch_events[0], + &sched->ch_events[1], + sched); + if (ret == -1) { + flb_sched_destroy(sched); + return NULL; + } + sched->event.type = FLB_ENGINE_EV_SCHED; + /* * Note: mk_event_timeout_create() sets a type = MK_EVENT_NOTIFICATION by * default, we need to overwrite this value so we can do a clean check @@ -652,6 +835,7 @@ struct flb_sched_timer *flb_sched_timer_create(struct flb_sched *sched) timer->config = sched->config; timer->sched = sched; timer->data = NULL; + timer->coro = FLB_FALSE; /* Active timer (not invalidated) */ timer->active = FLB_TRUE; @@ -684,7 +868,8 @@ int flb_sched_timer_destroy(struct flb_sched_timer *timer) /* Used by the engine to cleanup pending timers waiting to be destroyed */ int flb_sched_timer_cleanup(struct flb_sched *sched) { - int c = 0; + int timer_count = 0; + int timer_coro_count = 0; struct mk_list *tmp; struct mk_list *head; struct flb_sched_timer *timer; @@ -692,6 +877,27 @@ int flb_sched_timer_cleanup(struct flb_sched *sched) mk_list_foreach_safe(head, tmp, &sched->timers_drop) { timer = mk_list_entry(head, struct flb_sched_timer, _head); flb_sched_timer_destroy(timer); + timer_count++; + } + + timer_coro_count = flb_sched_timer_coro_cleanup(sched); + flb_trace("[sched] %i timer coroutines destroyed", timer_coro_count); + + return timer_count + timer_coro_count; +} + +int flb_sched_timer_coro_cleanup(struct flb_sched *sched) +{ + int c = 0; + struct cfl_list *tmp; + struct cfl_list *head; + struct flb_sched_timer_coro *stc; + + cfl_list_foreach_safe(head, tmp, &sched->timer_coro_list_drop) { + stc = cfl_list_entry(head, struct flb_sched_timer_coro, _head); + flb_coro_destroy(stc->coro); + cfl_list_del(&stc->_head); + flb_free(stc); c++; } @@ -742,3 +948,18 @@ int flb_sched_retry_now(struct flb_config *config, } return 0; } + +struct flb_sched_timer_coro *flb_sched_timer_coro_get(struct flb_sched *sched, uint32_t id) +{ + struct cfl_list *head; + struct flb_sched_timer_coro *stc; + + cfl_list_foreach(head, &sched->timer_coro_list) { + stc = cfl_list_entry(head, struct flb_sched_timer_coro, _head); + if (stc->id == id) { + return stc; + } + } + + return NULL; +} \ No newline at end of file From 1c93041e13db9247895a9b613236ba17938810a6 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:08:07 -0600 Subject: [PATCH 02/70] input_event: add support for Blob signal Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_input_event.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/fluent-bit/flb_input_event.h b/include/fluent-bit/flb_input_event.h index eae461310d6..ffef798670c 100644 --- a/include/fluent-bit/flb_input_event.h +++ b/include/fluent-bit/flb_input_event.h @@ -20,10 +20,10 @@ #ifndef FLB_INPUT_EVENT_H #define FLB_INPUT_EVENT_H -/* support event types by input plugins*/ - +/* support event types by input plugins */ #define FLB_INPUT_LOGS 0 #define FLB_INPUT_METRICS 1 #define FLB_INPUT_TRACES 2 +#define FLB_INPUT_BLOBS 3 #endif From 67d32968b57470a3db7d67c35d193b8f9c48b173 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:09:22 -0600 Subject: [PATCH 03/70] input_chunk: register chunks blob type Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_input_chunk.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/fluent-bit/flb_input_chunk.h b/include/fluent-bit/flb_input_chunk.h index 8d9156ad002..d801b11e89f 100644 --- a/include/fluent-bit/flb_input_chunk.h +++ b/include/fluent-bit/flb_input_chunk.h @@ -50,6 +50,7 @@ #define FLB_INPUT_CHUNK_TYPE_LOGS 0 #define FLB_INPUT_CHUNK_TYPE_METRICS 1 #define FLB_INPUT_CHUNK_TYPE_TRACES 2 +#define FLB_INPUT_CHUNK_TYPE_BLOBS 3 #ifdef FLB_HAVE_CHUNK_TRACE #define FLB_INPUT_CHUNK_HAS_TRACE 1 << 31 From 2404460ee43b81495bb92c1e074b33a1153ec1d8 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:10:22 -0600 Subject: [PATCH 04/70] input: use FLB_INLINE macro Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_input.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/fluent-bit/flb_input.h b/include/fluent-bit/flb_input.h index 9140b1dc71f..094766a09ad 100644 --- a/include/fluent-bit/flb_input.h +++ b/include/fluent-bit/flb_input.h @@ -577,7 +577,7 @@ static FLB_INLINE int flb_input_is_threaded(struct flb_input_instance *ins) * number of retries, if it has exceeded the 'retry_limit' option, an FLB_ERROR * will be returned instead. */ -static inline void flb_input_return(struct flb_coro *coro) { +static FLB_INLINE void flb_input_return(struct flb_coro *coro) { int n; uint64_t val; struct flb_input_coro *input_coro; @@ -599,7 +599,7 @@ static inline void flb_input_return(struct flb_coro *coro) { flb_input_coro_prepare_destroy(input_coro); } -static inline void flb_input_return_do(int ret) { +static FLB_INLINE void flb_input_return_do(int ret) { struct flb_coro *coro = flb_coro_get(); flb_input_return(coro); From 4996a2608d05d9806ef0c019f67014982f8e65bb Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:10:58 -0600 Subject: [PATCH 05/70] engine_macros: adjust scheduler macros Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_engine_macros.h | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/include/fluent-bit/flb_engine_macros.h b/include/fluent-bit/flb_engine_macros.h index 151dd5ad346..8b79634f7d2 100644 --- a/include/fluent-bit/flb_engine_macros.h +++ b/include/fluent-bit/flb_engine_macros.h @@ -26,16 +26,18 @@ /* Types of events handled by the Server engine */ #define FLB_ENGINE_EV_CORE MK_EVENT_NOTIFICATION #define FLB_ENGINE_EV_CUSTOM MK_EVENT_CUSTOM -#define FLB_ENGINE_EV_THREAD 1024 -#define FLB_ENGINE_EV_SCHED 2048 -#define FLB_ENGINE_EV_SCHED_FRAME (FLB_ENGINE_EV_SCHED + 4096) -#define FLB_ENGINE_EV_INPUT 8192 -#define FLB_ENGINE_EV_THREAD_INPUT 16384 /* reserved, not used yet */ +#define FLB_ENGINE_EV_THREAD (1 << 10) /* 1024 */ +#define FLB_ENGINE_EV_SCHED (1 << 11) /* 2048 */ +#define FLB_ENGINE_EV_SCHED_FRAME (FLB_ENGINE_EV_SCHED | (1 << 12)) /* 2048 | 4096 = 6144 */ +#define FLB_ENGINE_EV_SCHED_CORO (1 << 13) /* 8192 */ -#define FLB_ENGINE_EV_OUTPUT 32768 -#define FLB_ENGINE_EV_THREAD_OUTPUT 65536 -#define FLB_ENGINE_EV_THREAD_ENGINE 131072 +#define FLB_ENGINE_EV_INPUT (1 << 14) /* 16384 */ +#define FLB_ENGINE_EV_THREAD_INPUT (1 << 15) /* 32768 */ + +#define FLB_ENGINE_EV_OUTPUT (1 << 16) /* 65536 */ +#define FLB_ENGINE_EV_THREAD_OUTPUT (1 << 17) /* 131072 */ +#define FLB_ENGINE_EV_THREAD_ENGINE (1 << 18) /* 262144 */ /* Engine events: all engine events set the left 32 bits to '1' */ #define FLB_ENGINE_EV_STARTED FLB_BITS_U64_SET(1, 1) /* Engine started */ From 19618c30da56b956bf6207ff7431903f05577132 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:12:05 -0600 Subject: [PATCH 06/70] event: map event type blobs as chunk type blobs Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_event.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/fluent-bit/flb_event.h b/include/fluent-bit/flb_event.h index f2e44b57187..9b94ea4c980 100644 --- a/include/fluent-bit/flb_event.h +++ b/include/fluent-bit/flb_event.h @@ -28,6 +28,7 @@ #define FLB_EVENT_TYPE_LOGS FLB_INPUT_CHUNK_TYPE_LOGS #define FLB_EVENT_TYPE_METRICS FLB_INPUT_CHUNK_TYPE_METRICS #define FLB_EVENT_TYPE_TRACES FLB_INPUT_CHUNK_TYPE_TRACES +#define FLB_EVENT_TYPE_BLOBS FLB_INPUT_CHUNK_TYPE_BLOBS #define FLB_EVENT_TYPE_HAS_TRACE FLB_INPUT_CHUNK_HAS_TRACE From 86d325bf14395ba8c7e3b50d85f1156fb4d5d8de Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:13:30 -0600 Subject: [PATCH 07/70] output_thread: add plugin callback for worker initialization Signed-off-by: Eduardo Silva --- src/flb_output_thread.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/flb_output_thread.c b/src/flb_output_thread.c index cc75e623742..4baa5181b30 100644 --- a/src/flb_output_thread.c +++ b/src/flb_output_thread.c @@ -243,6 +243,10 @@ static void output_thread(void *data) } event_local.type = FLB_ENGINE_EV_OUTPUT; + if (ins->p->cb_worker_init) { + ret = ins->p->cb_worker_init(ins->context, ins->config); + } + flb_plg_info(th_ins->ins, "worker #%i started", thread_id); /* Thread event loop */ From bee712bb71a76f3f38c1908777d8690d1663eebf Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:14:34 -0600 Subject: [PATCH 08/70] utils: add new function to read files by offset Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_utils.h | 1 + src/flb_utils.c | 53 ++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/include/fluent-bit/flb_utils.h b/include/fluent-bit/flb_utils.h index f1424d78842..b1f06592278 100644 --- a/include/fluent-bit/flb_utils.h +++ b/include/fluent-bit/flb_utils.h @@ -67,6 +67,7 @@ int flb_utils_proxy_url_split(const char *in_url, char **out_protocol, char **out_username, char **out_password, char **out_host, char **out_port); int flb_utils_read_file(char *path, char **out_buf, size_t *out_size); +int flb_utils_read_file_offset(char *path, off_t offset_start, off_t offset_end, char **out_buf, size_t *out_size); char *flb_utils_get_os_name(); int flb_utils_uuid_v4_gen(char *buf); int flb_utils_get_machine_id(char **out_id, size_t *out_size); diff --git a/src/flb_utils.c b/src/flb_utils.c index 37e89d28a4f..b235d8f31e1 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -1282,10 +1282,17 @@ int flb_utils_uuid_v4_gen(char *buf) #endif int flb_utils_read_file(char *path, char **out_buf, size_t *out_size) +{ + return flb_utils_read_file_offset(path, 0, 0, out_buf, out_size); +} + +int flb_utils_read_file_offset(char *path, off_t offset_start, off_t offset_end, char **out_buf, size_t *out_size) { int fd; int ret; size_t bytes; + size_t bytes_to_read; + size_t total_bytes_read = 0; struct stat st; flb_sds_t buf; FILE *fp; @@ -1303,26 +1310,54 @@ int flb_utils_read_file(char *path, char **out_buf, size_t *out_size) return -1; } - buf = flb_calloc(1, st.st_size + 1); - if (!buf) { - flb_errno(); + if (offset_start > st.st_size || offset_end > st.st_size) { + flb_error("offsets exceed file size (%jd bytes)", (intmax_t) st.st_size); fclose(fp); return -1; } - bytes = fread(buf, st.st_size, 1, fp); - if (bytes < 1) { - if (ferror(fp)) { + if (offset_start > 0) { + ret = fseek(fp, offset_start, SEEK_SET); + if (ret != 0) { flb_errno(); + fclose(fp); + return -1; } - flb_free(buf); + } + + if (offset_end == 0) { + offset_end = st.st_size; + } + + bytes_to_read = offset_end - offset_start; + + buf = flb_calloc(1, bytes_to_read + 1); + if (!buf) { + flb_errno(); fclose(fp); return -1; } + + while (total_bytes_read < bytes_to_read) { + bytes = fread(buf + total_bytes_read, 1, bytes_to_read - total_bytes_read, fp); + if (bytes < 1) { + if (feof(fp)) { + break; + } + if (ferror(fp)) { + flb_errno(); + free(buf); + fclose(fp); + return -1; + } + } + total_bytes_read += bytes; + } fclose(fp); *out_buf = buf; - *out_size = st.st_size; + *out_size = total_bytes_read; + return 0; } @@ -1426,7 +1461,7 @@ int flb_utils_get_machine_id(char **out_id, size_t *out_size) memcpy(*out_id, buf, dwBufSize); - /* RegQueryValueEx sets dwBufSize to strlen()+1 for the NULL + /* RegQueryValueEx sets dwBufSize to strlen()+1 for the NULL * terminator, but we only need strlen(). */ *out_size = dwBufSize-1; return 0; From cefd0671f8abb7f4dbb27c7f973c4a0aad917b85 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:15:33 -0600 Subject: [PATCH 09/70] router: allow routing for blob input types Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_router.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/fluent-bit/flb_router.h b/include/fluent-bit/flb_router.h index 2fcd7d6e2f6..dd7efca6bad 100644 --- a/include/fluent-bit/flb_router.h +++ b/include/fluent-bit/flb_router.h @@ -44,6 +44,10 @@ static inline int flb_router_match_type(int in_event_type, !(o_ins->event_type & FLB_OUTPUT_TRACES)) { return FLB_FALSE; } + else if (in_event_type == FLB_INPUT_BLOBS && + !(o_ins->event_type & FLB_OUTPUT_BLOBS)) { + return FLB_FALSE; + } return FLB_TRUE; } From 9cdc593cb32ef2ef6316173f2aa2c76376fbbc7e Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:16:29 -0600 Subject: [PATCH 10/70] sqldb: commit before closing handler Signed-off-by: Eduardo Silva --- src/flb_sqldb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/flb_sqldb.c b/src/flb_sqldb.c index 79c44ad65ad..a4a5df2f490 100644 --- a/src/flb_sqldb.c +++ b/src/flb_sqldb.c @@ -99,6 +99,7 @@ int flb_sqldb_close(struct flb_sqldb *db) parent->users--; } else { + sqlite3_exec(db->handler, "COMMIT;", NULL, NULL, NULL); sqlite3_close(db->handler); } mk_list_del(&db->_head); From 332988f7dda9dc667db1f2ee85fd569beb6e28d7 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:17:04 -0600 Subject: [PATCH 11/70] output: register output blob Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_output.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/fluent-bit/flb_output.h b/include/fluent-bit/flb_output.h index 58433fcb0d5..dd928450c99 100644 --- a/include/fluent-bit/flb_output.h +++ b/include/fluent-bit/flb_output.h @@ -89,6 +89,7 @@ int flb_chunk_trace_output(struct flb_chunk_trace *trace, struct flb_output_inst #define FLB_OUTPUT_LOGS 1 #define FLB_OUTPUT_METRICS 2 #define FLB_OUTPUT_TRACES 4 +#define FLB_OUTPUT_BLOBS 8 #define FLB_OUTPUT_FLUSH_COMPAT_OLD_18() \ const void *data = event_chunk->data; \ @@ -232,6 +233,8 @@ struct flb_output_plugin { /* Default number of worker threads */ int workers; + int (*cb_worker_init) (void *, struct flb_config *); + /* Tests */ struct flb_test_out_formatter test_formatter; From 63c8b8b4d899bc2404357d52013ffcd169a981a1 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:18:02 -0600 Subject: [PATCH 12/70] input_blob: add new interface to register blobs Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_input_blob.h | 36 ++++++ src/flb_input_blob.c | 177 ++++++++++++++++++++++++++++ 2 files changed, 213 insertions(+) create mode 100644 include/fluent-bit/flb_input_blob.h create mode 100644 src/flb_input_blob.c diff --git a/include/fluent-bit/flb_input_blob.h b/include/fluent-bit/flb_input_blob.h new file mode 100644 index 00000000000..859468230cc --- /dev/null +++ b/include/fluent-bit/flb_input_blob.h @@ -0,0 +1,36 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_INPUT_BLOB_H +#define FLB_INPUT_BLOB_H + +#include +#include +#include + +struct flb_blob_file { + cfl_sds_t path; +}; + +int flb_input_blob_file_get_info(msgpack_object map, cfl_sds_t *file_path, size_t *size); +int flb_input_blob_file_register(struct flb_input_instance *ins, + struct flb_log_event_encoder *encoder, + const char *tag, size_t tag_len, + char *file_path, size_t size); +#endif diff --git a/src/flb_input_blob.c b/src/flb_input_blob.c new file mode 100644 index 00000000000..06d97e8f566 --- /dev/null +++ b/src/flb_input_blob.c @@ -0,0 +1,177 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include +#include +#include +#include +#include +#include +#include + +#include + +int flb_input_blob_file_get_info(msgpack_object map, cfl_sds_t *file_path, size_t *size) +{ + cfl_sds_t tmp_file_path; + msgpack_object o; + + if (map.type != MSGPACK_OBJECT_MAP) { + return -1; + } + + if (map.via.map.size < 2) { + return -1; + } + + /* get file_path */ + o = map.via.map.ptr[0].key; + if (o.type != MSGPACK_OBJECT_STR) { + return -1; + } + if (o.via.str.size != 9 || strncmp(o.via.str.ptr, "file_path", 9) != 0) { + return -1; + } + + o = map.via.map.ptr[0].val; + if (o.type != MSGPACK_OBJECT_STR) { + return -1; + } + + tmp_file_path = cfl_sds_create_len(o.via.str.ptr, o.via.str.size); + if (tmp_file_path == NULL) { + return -1; + } + + /* get size */ + o = map.via.map.ptr[1].key; + if (o.type != MSGPACK_OBJECT_STR) { + cfl_sds_destroy(tmp_file_path); + return -1; + } + if (o.via.str.size != 4 || strncmp(o.via.str.ptr, "size", 4) != 0) { + cfl_sds_destroy(tmp_file_path); + return -1; + } + + o = map.via.map.ptr[1].val; + if (o.type != MSGPACK_OBJECT_POSITIVE_INTEGER) { + cfl_sds_destroy(tmp_file_path); + return -1; + } + + *size = o.via.u64; + *file_path = tmp_file_path; + + return 0; +} + +int flb_input_blob_file_register(struct flb_input_instance *ins, + struct flb_log_event_encoder *encoder, + const char *tag, size_t tag_len, + char *file_path, size_t size) +{ + int ret; + struct stat st; + + /* check if the file is readable */ + ret = access(file_path, R_OK); + if (ret == -1) { + flb_plg_error(ins, "file %s is not readable", file_path); + return -1; + } + + /* get file information */ + ret = stat(file_path, &st); + if (ret == -1) { + flb_errno(); + return -1; + } + + /* is the requested file size valid ? */ + if (size > st.st_size) { + flb_error("[blob file registration] requested size %zu for file %s is greater than the file size %zu", + size, file_path, st.st_size); + return -1; + } + + /* Encode the blob file info in msgpack by using the log encoder wrapper */ + ret = flb_log_event_encoder_begin_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[blob file registration] could not begin blob record"); + return -1; + } + + /* add timestamp */ + ret = flb_log_event_encoder_set_current_timestamp(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[blob file registration] could not set timestamp"); + flb_log_event_encoder_reset(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_cstring(encoder, "file_path"); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[blob file registration] could not append path"); + flb_log_event_encoder_reset(encoder); + return -1; + } + ret = flb_log_event_encoder_append_body_cstring(encoder, file_path); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[blob file registration] could not append path"); + flb_log_event_encoder_reset(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_cstring(encoder, "size"); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[blob file registration] could not append path"); + flb_log_event_encoder_reset(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_uint64(encoder, size); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[blob file registration] could not append size"); + flb_log_event_encoder_reset(encoder); + return -1; + } + + ret = flb_log_event_encoder_commit_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[blob file registration] could not commit record"); + flb_log_event_encoder_reset(encoder); + return -1; + } + + /* register entry as a chunk */ + ret = flb_input_chunk_append_raw(ins, FLB_INPUT_BLOBS, 0, + tag, tag_len, + encoder->output_buffer, + encoder->output_length); + if (ret != 0) { + flb_error("[blob file registration] could not append blob record"); + flb_log_event_encoder_reset(encoder); + return -1; + } + + flb_log_event_encoder_reset(encoder); + return ret; +} From 6b831aeefdc054ab7f8c6a221a912d21d7ca426a Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:20:07 -0600 Subject: [PATCH 13/70] input_chunk: event type blob support Signed-off-by: Eduardo Silva --- src/CMakeLists.txt | 1 + src/flb_input_chunk.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 084f2c57d52..ea7403f7b6c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -27,6 +27,7 @@ set(src flb_input_log.c flb_input_metric.c flb_input_trace.c + flb_input_blob.c flb_input_thread.c flb_filter.c flb_output.c diff --git a/src/flb_input_chunk.c b/src/flb_input_chunk.c index ca02e6fca68..1e8a4a09d75 100644 --- a/src/flb_input_chunk.c +++ b/src/flb_input_chunk.c @@ -2049,6 +2049,9 @@ int flb_input_chunk_get_event_type(struct flb_input_chunk *ic) else if (buf[2] == FLB_INPUT_CHUNK_TYPE_TRACES) { type = FLB_INPUT_TRACES; } + else if (buf[2] == FLB_INPUT_CHUNK_TYPE_BLOBS) { + type = FLB_INPUT_BLOBS; + } } else { type = FLB_INPUT_LOGS; From 6901e6fe74fd906b9c57d32be0e46a7dd0105643 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:21:14 -0600 Subject: [PATCH 14/70] in_blob: new plugin to support large binary files Recent patch series add support to process/route large binary files through a zero-copy strategy. This new in_blob plugin allows to scan a path from the file system and register files that matched the pattern. service: flush: 1 log_level: info pipeline: inputs: - name: blob path: '~/logs/blob/*' database_file: blob.db outputs: - name: stdout match: '*' - name: azure_blob match: '*' path: kubernetes container_name: blobs auto_create_container: on database_file: azure.db part_size: 4M upload_parts_timeout: 1s workers: 10 Signed-off-by: Eduardo Silva --- plugins/CMakeLists.txt | 2 + plugins/in_blob/CMakeLists.txt | 12 ++ plugins/in_blob/blob.c | 359 +++++++++++++++++++++++++++++++++ plugins/in_blob/blob.h | 75 +++++++ plugins/in_blob/blob_db.c | 203 +++++++++++++++++++ plugins/in_blob/blob_db.h | 47 +++++ plugins/in_blob/blob_file.c | 123 +++++++++++ plugins/in_blob/blob_file.h | 31 +++ 8 files changed, 852 insertions(+) create mode 100644 plugins/in_blob/CMakeLists.txt create mode 100644 plugins/in_blob/blob.c create mode 100644 plugins/in_blob/blob.h create mode 100644 plugins/in_blob/blob_db.c create mode 100644 plugins/in_blob/blob_db.h create mode 100644 plugins/in_blob/blob_file.c create mode 100644 plugins/in_blob/blob_file.h diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index ce8cae64d97..200a09b449c 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -193,6 +193,8 @@ endmacro() # Custom Plugins REGISTER_CUSTOM_PLUGIN("custom_calyptia") +REGISTER_IN_PLUGIN("in_blob") + # These plugins works only on Linux if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") REGISTER_IN_PLUGIN("in_cpu") diff --git a/plugins/in_blob/CMakeLists.txt b/plugins/in_blob/CMakeLists.txt new file mode 100644 index 00000000000..ef8547ec67c --- /dev/null +++ b/plugins/in_blob/CMakeLists.txt @@ -0,0 +1,12 @@ +# FIXME: there is something wrong when linking objects and this +# static plugin, I should not require to link to a specific symbol +# if the object was already linked from fluent-bit core on src/, also +# jsmn should not be required. + +set(src + blob.c + blob_db.c + blob_file.c + ) + +FLB_PLUGIN(in_blob "${src}" "") diff --git a/plugins/in_blob/blob.c b/plugins/in_blob/blob.c new file mode 100644 index 00000000000..223aa4a13c8 --- /dev/null +++ b/plugins/in_blob/blob.c @@ -0,0 +1,359 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "blob.h" +#include "blob_db.h" +#include "blob_file.h" + +/* Define missing GLOB_TILDE if not exists */ +#ifndef GLOB_TILDE +#define GLOB_TILDE 1<<2 /* use GNU Libc value */ +#define UNSUP_TILDE 1 + +/* we need these extra headers for path resolution */ +#include +#include +#include + +static char *expand_tilde(const char *path) +{ + int len; + char user[256]; + char *p = NULL; + char *dir = NULL; + char *tmp = NULL; + struct passwd *uinfo = NULL; + + if (path[0] == '~') { + p = strchr(path, '/'); + + if (p) { + /* check case '~/' */ + if ((p - path) == 1) { + dir = getenv("HOME"); + if (!dir) { + return path; + } + } + else { + /* + * it refers to a different user: ~user/abc, first step grab + * the user name. + */ + len = (p - path) - 1; + memcpy(user, path + 1, len); + user[len] = '\0'; + + /* use getpwnam() to resolve user information */ + uinfo = getpwnam(user); + if (!uinfo) { + return path; + } + + dir = uinfo->pw_dir; + } + } + else { + dir = getenv("HOME"); + if (!dir) { + return path; + } + } + + if (p) { + tmp = flb_malloc(PATH_MAX); + if (!tmp) { + flb_errno(); + return NULL; + } + snprintf(tmp, PATH_MAX - 1, "%s%s", dir, p); + } + else { + dir = getenv("HOME"); + if (!dir) { + return path; + } + + tmp = flb_strdup(dir); + if (!tmp) { + return path; + } + } + + return tmp; + } + + return path; +} +#endif + + +static inline int do_glob(const char *pattern, int flags, + void *not_used, glob_t *pglob) +{ + int ret; + int new_flags; + char *tmp = NULL; + int tmp_needs_free = FLB_FALSE; + (void) not_used; + + /* Save current values */ + new_flags = flags; + + if (flags & GLOB_TILDE) { +#ifdef UNSUP_TILDE + /* + * Some libc libraries like Musl do not support GLOB_TILDE for tilde + * expansion. A workaround is to use wordexp(3) but looking at it + * implementation in Musl it looks quite expensive: + * + * http://git.musl-libc.org/cgit/musl/tree/src/misc/wordexp.c + * + * the workaround is to do our own tilde expansion in a temporary buffer. + */ + + /* Look for a tilde */ + tmp = expand_tilde(pattern); + if (tmp != pattern) { + /* the path was expanded */ + pattern = tmp; + tmp_needs_free = FLB_TRUE; + } + + /* remove unused flag */ + new_flags &= ~GLOB_TILDE; +#endif + } + + /* invoke glob with new parameters */ + ret = glob(pattern, new_flags, NULL, pglob); + + /* remove temporary buffer, if allocated by expand_tilde above. + * Note that this buffer is only used for libc implementations + * that do not support the GLOB_TILDE flag, like musl. */ + if ((tmp != NULL) && (tmp_needs_free == FLB_TRUE)) { + flb_free(tmp); + } + + return ret; +} + +static int scan_path(struct blob_ctx *ctx) +{ + int i; + int ret; + int count = 0; + glob_t globbuf; + struct stat st; + + flb_plg_debug(ctx->ins, "scanning path %s", ctx->path); + + /* Safe reset for globfree() */ + globbuf.gl_pathv = NULL; + + /* Scan the given path */ + ret = do_glob(ctx->path, GLOB_TILDE | GLOB_ERR, NULL, &globbuf); + if (ret != 0) { + switch (ret) { + case GLOB_NOSPACE: + flb_plg_error(ctx->ins, "no memory space available"); + return -1; + case GLOB_ABORTED: + flb_plg_error(ctx->ins, "read error, check permissions: %s", ctx->path); + return -1; + case GLOB_NOMATCH: + ret = stat(ctx->path, &st); + if (ret == -1) { + flb_plg_debug(ctx->ins, "cannot read info from: %s", ctx->path); + } + else { + ret = access(ctx->path, R_OK); + if (ret == -1 && errno == EACCES) { + flb_plg_error(ctx->ins, "NO read access for path: %s", ctx->path); + } + else { + flb_plg_debug(ctx->ins, "NO matches for path: %s", ctx->path); + } + } + return 0; + } + } + + /* For every entry found, generate an output list */ + for (i = 0; i < globbuf.gl_pathc; i++) { + ret = stat(globbuf.gl_pathv[i], &st); + if (ret != 0) { + flb_plg_debug(ctx->ins, "skip entry=%s", globbuf.gl_pathv[i]); + continue; + } + + if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) { + ret = blob_file_append(ctx, globbuf.gl_pathv[i], &st); + if (ret == 0) { + flb_plg_debug(ctx->ins, "blob scan add: %s, inode %" PRIu64, + globbuf.gl_pathv[i], (uint64_t) st.st_ino); + } + else { + flb_plg_debug(ctx->ins, "blob scan skip: %s", globbuf.gl_pathv[i]); + } + } + else { + flb_plg_debug(ctx->ins, "skip entry=%s", globbuf.gl_pathv[i]); + } + } + + globfree(&globbuf); + return count; +} + +static int cb_scan_path(struct flb_input_instance *ins, + struct flb_config *config, void *in_context) +{ + struct blob_ctx *ctx = in_context; + + return scan_path(ctx); +} + +/* Initialize plugin */ +static int in_blob_init(struct flb_input_instance *ins, + struct flb_config *config, void *data) +{ + int ret; + struct blob_ctx *ctx; + + /* Allocate space for the configuration context */ + ctx = flb_calloc(1, sizeof(struct blob_ctx)); + if (!ctx) { + return -1; + } + ctx->ins = ins; + ctx->config = config; + cfl_list_init(&ctx->files); + + /* laod the config map */ + ret = flb_input_config_map_set(ins, ctx); + if (ret == -1) { + flb_free(ctx); + return -1; + } + + /* associate the context with the instance */ + flb_input_set_context(ins, ctx); + + /* 'path' must be set */ + if (!ctx->path) { + flb_plg_error(ins, "'path' configuration property is not set"); + flb_free(ctx); + return -1; + } + +#ifdef FLB_HAVE_SQLDB + if (ctx->database_file) { + ctx->db = blob_db_open(ctx, ctx->database_file); + if (!ctx->db) { + return -1; + } + } +#endif + + /* create a collector to scan the path of files */ + ret = flb_input_set_collector_time(ins, + cb_scan_path, + ctx->scan_refresh_interval, 0, + config); + if (ret == -1) { + flb_plg_error(ins, "could not create collector"); + return -1; + } + ctx->coll_fd = ret; + + /* initialize the encoder */ + ctx->log_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (ctx->log_encoder == NULL) { + flb_plg_error(ins, "could not initialize event encoder"); + return -1; + } + + return 0; +} + +/* Cleanup serial input */ +static int in_blob_exit(void *in_context, struct flb_config *config) +{ + struct blob_ctx *ctx = in_context; + + if (!ctx) { + return 0; + } + + blob_db_close(ctx); + blob_file_list_remove_all(ctx); + flb_log_event_encoder_destroy(ctx->log_encoder); + flb_free(ctx); + + return 0; +} + +static struct flb_config_map config_map[] = { + { + FLB_CONFIG_MAP_STR, "path", NULL, + 0, FLB_TRUE, offsetof(struct blob_ctx, path), + "Path to scan for blob/binary files" + }, + +#ifdef FLB_HAVE_SQLDB + { + FLB_CONFIG_MAP_STR, "database_file", NULL, + 0, FLB_TRUE, offsetof(struct blob_ctx, database_file), + }, +#endif + + { + FLB_CONFIG_MAP_TIME, "scan_refresh_interval", "2s", + 0, FLB_TRUE, offsetof(struct blob_ctx, scan_refresh_interval), + "Set the interval time to scan for new files" + }, + + /* EOF */ + {0} +}; + +/* Plugin reference */ +struct flb_input_plugin in_blob_plugin = { + .name = "blob", + .description = "Blob (binary) files", + .cb_init = in_blob_init, + .cb_pre_run = NULL, + .cb_collect = NULL, + .cb_flush_buf = NULL, + .cb_exit = in_blob_exit, + .config_map = config_map +}; diff --git a/plugins/in_blob/blob.h b/plugins/in_blob/blob.h new file mode 100644 index 00000000000..13b932e5d5e --- /dev/null +++ b/plugins/in_blob/blob.h @@ -0,0 +1,75 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_IN_BLOB_H +#define FLB_IN_BLOB_H + +#include +#include +#include +#include + +struct blob_file { + /* database reference (id) */ + uint64_t db_id; + + /* file path */ + flb_sds_t path; + + /* file size found when registered */ + size_t size; + + /* link to parent list blob_ctx->files */ + struct cfl_list _head; +}; + +struct blob_ctx { + /* collector for scan_refresh_interval */ + int coll_fd; + + /* + * list of files that has been found and being processed: file as soon as they are found are + * registered with the flb_input_blob_file_register() function. + */ + struct cfl_list files; + + /* Fluent Bit context */ + struct flb_config *config; + + /* input instance */ + struct flb_input_instance *ins; + + /* log encoder */ + struct flb_log_event_encoder *log_encoder; + + /* database */ +#ifdef FLB_HAVE_SQLDB + struct flb_sqldb *db; + sqlite3_stmt *stmt_insert_file; + sqlite3_stmt *stmt_delete_file; + sqlite3_stmt *stmt_get_file; +#endif + + /* config map options */ + flb_sds_t path; + flb_sds_t database_file; + time_t scan_refresh_interval; +}; + +#endif diff --git a/plugins/in_blob/blob_db.c b/plugins/in_blob/blob_db.c new file mode 100644 index 00000000000..df96769212c --- /dev/null +++ b/plugins/in_blob/blob_db.c @@ -0,0 +1,203 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#ifdef FLB_HAVE_SQLDB + +#include + +#include "blob.h" +#include "blob_db.h" + +static int prepare_stmts(struct flb_sqldb *db, struct blob_ctx *ctx) +{ + int ret; + + /* insert */ + ret = sqlite3_prepare_v2(db->handler, SQL_INSERT_FILE, -1, + &ctx->stmt_insert_file, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_INSERT_FILE); + return -1; + } + + /* delete */ + ret = sqlite3_prepare_v2(db->handler, SQL_DELETE_FILE, -1, + &ctx->stmt_delete_file, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_DELETE_FILE); + return -1; + } + + /* get */ + ret = sqlite3_prepare_v2(db->handler, SQL_GET_FILE, -1, + &ctx->stmt_get_file, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_GET_FILE); + return -1; + } + + return 0; +} +// static int my_special_callback(void *unused, int count, char **data, char **columns) +// { +// int idx; + +// printf("There are %d column(s)\n", count); + +// for (idx = 0; idx < count; idx++) { +// printf("The data in column \"%s\" is: %s\n", columns[idx], data[idx]); +// } + +// printf("\n"); + +// return 0; +// } + +struct flb_sqldb *blob_db_open(struct blob_ctx *ctx, char *db_path) +{ + int ret; + struct flb_sqldb *db; + struct flb_input_instance *ins; + + ins = ctx->ins; + + db = flb_sqldb_open(db_path, ins->name, ctx->config); + if (!db) { + flb_plg_error(ctx->ins, "cannot open database %s", db_path); + return NULL; + } + + ret = flb_sqldb_query(db, SQL_CREATE_BLOB_FILES, NULL, NULL); + if (ret != FLB_OK) { + flb_plg_error(ctx->ins, "cannot create table 'in_blob_files'"); + flb_sqldb_close(db); + return NULL; + } + + ret = prepare_stmts(db, ctx); + if (ret == -1) { + flb_sqldb_close(db); + return NULL; + } + + return db; +} + +int blob_db_close(struct blob_ctx *ctx) +{ + /* finalize prepared statements */ + sqlite3_finalize(ctx->stmt_get_file); + sqlite3_finalize(ctx->stmt_insert_file); + sqlite3_finalize(ctx->stmt_delete_file); + + return flb_sqldb_close(ctx->db); +} + +int blob_db_file_exists(struct blob_ctx *ctx, char *path, uint64_t *id) +{ + int ret; + int exists = FLB_FALSE; + + /* Bind parameters */ + sqlite3_bind_text(ctx->stmt_get_file, 1, path, -1, 0); + ret = sqlite3_step(ctx->stmt_get_file); + if (ret == SQLITE_ROW) { + exists = FLB_TRUE; + + /* id: column 0 */ + *id = sqlite3_column_int64(ctx->stmt_get_file, 0); + } + else if (ret == SQLITE_DONE) { + /* all good */ + } + else { + exists = -1; + } + + sqlite3_clear_bindings(ctx->stmt_get_file); + sqlite3_reset(ctx->stmt_get_file); + + return exists; +} + +int64_t blob_db_file_insert(struct blob_ctx *ctx, char *path, size_t size) +{ + int ret; + int64_t id; + time_t created; + + /* Register the file */ + created = time(NULL); + + /* Bind parameters */ + sqlite3_bind_text(ctx->stmt_insert_file, 1, path, -1, 0); + sqlite3_bind_int64(ctx->stmt_insert_file, 2, size); + sqlite3_bind_int64(ctx->stmt_insert_file, 3, created); + + /* Run the insert */ + ret = sqlite3_step(ctx->stmt_insert_file); + if (ret != SQLITE_DONE) { + sqlite3_clear_bindings(ctx->stmt_insert_file); + sqlite3_reset(ctx->stmt_insert_file); + flb_plg_error(ctx->ins, "cannot execute insert file '%s'", path); + return -1; + } + + /* Get the database ID for this file */ + id = flb_sqldb_last_id(ctx->db); + + sqlite3_clear_bindings(ctx->stmt_insert_file); + sqlite3_reset(ctx->stmt_insert_file); + + + flb_plg_trace(ctx->ins, "db: file '%s' inserted with id=%ld", path, id); + return id; +} + +int blob_db_file_delete(struct blob_ctx *ctx, uint64_t id, char *path) +{ + int ret; + + /* Bind parameters */ + sqlite3_bind_int64(ctx->stmt_delete_file, 1, id); + ret = sqlite3_step(ctx->stmt_delete_file); + if (ret != SQLITE_DONE) { + return -1; + } + + sqlite3_clear_bindings(ctx->stmt_delete_file); + sqlite3_reset(ctx->stmt_delete_file); + + if (ret != SQLITE_DONE) { + flb_plg_error(ctx->ins, "db: error deleting entry id=%" PRIu64 + ", path='%s' from database", id, path); + return -1; + } + + flb_plg_debug(ctx->ins, "db: file id=%" PRIu64 + ", path='%s' deleted from database", id, path); + return 0; +} + +#endif \ No newline at end of file diff --git a/plugins/in_blob/blob_db.h b/plugins/in_blob/blob_db.h new file mode 100644 index 00000000000..714dbdb6eac --- /dev/null +++ b/plugins/in_blob/blob_db.h @@ -0,0 +1,47 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IN_BLOB_DB_H +#define IN_BLOB_DB_H + +#define SQL_CREATE_BLOB_FILES \ + "CREATE TABLE IF NOT EXISTS in_blob_files (" \ + " id INTEGER PRIMARY KEY," \ + " path TEXT NOT NULL," \ + " size INTEGER," \ + " created INTEGER" \ + ");" + +#define SQL_INSERT_FILE \ + "INSERT INTO in_blob_files (path, size, created)" \ + " VALUES (@path, @size, @created);" + +#define SQL_DELETE_FILE \ + "DELETE FROM in_blob_files WHERE id=@id;" + +#define SQL_GET_FILE \ + "SELECT * from in_blob_files WHERE path=@path order by id desc;" + +struct flb_sqldb *blob_db_open(struct blob_ctx *ctx, char *db_path); +int blob_db_close(struct blob_ctx *ctx); +int blob_db_file_exists(struct blob_ctx *ctx, char *path, uint64_t *id); +int64_t blob_db_file_insert(struct blob_ctx *ctx, char *path, size_t size); +int blob_db_file_delete(struct blob_ctx *ctx, uint64_t id, char *path); + +#endif \ No newline at end of file diff --git a/plugins/in_blob/blob_file.c b/plugins/in_blob/blob_file.c new file mode 100644 index 00000000000..bf5aaf5caca --- /dev/null +++ b/plugins/in_blob/blob_file.c @@ -0,0 +1,123 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include + +#include "blob.h" +#include "blob_db.h" + +int blob_file_append(struct blob_ctx *ctx, char *path, struct stat *st) +{ + int fd; + int ret; + uint64_t id_found; + struct cfl_list *head; + struct blob_file *bfile; + struct flb_input_instance *ins = ctx->ins; + + /* check if the file already exists in the linked list in memory */ + cfl_list_foreach(head, &ctx->files) { + bfile = cfl_list_entry(head, struct blob_file, _head); + if (strcmp(bfile->path, path) == 0) { + /* file already exists */ + return -1; + } + } + +#ifdef FLB_HAVE_SQLDB + if (ctx->database_file) { + /* the file was already registered, just skipt it */ + if (blob_db_file_exists(ctx, path, &id_found) == FLB_TRUE) { + return 0; + } + } +#endif + + /* try to open the file */ + fd = open(path, O_RDONLY); + if (fd == -1) { + flb_errno(); + flb_plg_error(ctx->ins, "cannot open %s", path); + return -1; + } + close(fd); + + /* create the reference entry */ + bfile = flb_calloc(1, sizeof(struct blob_file)); + if (!bfile) { + flb_errno(); + return -1; + } + + bfile->path = cfl_sds_create(path); + if (!bfile->path) { + flb_free(bfile); + return -1; + } + bfile->size = st->st_size; + +#ifdef FLB_HAVE_SQLDB + /* insert the entry into the database */ + bfile->db_id = blob_db_file_insert(ctx, path, st->st_size); + if (bfile->db_id < 0) { + cfl_sds_destroy(bfile->path); + flb_free(bfile); + return -1; + } +#endif + + ret = flb_input_blob_file_register(ctx->ins, ctx->log_encoder, + ins->tag, ins->tag_len, + bfile->path, bfile->size); + if (ret == -1) { + cfl_sds_destroy(bfile->path); + flb_free(bfile); + return -1; + } + + cfl_list_add(&bfile->_head, &ctx->files); + return 0; +} + +/* release resources of a blob_file */ +void blob_file_list_remove(struct blob_file *bfile) +{ + if (bfile->path) { + cfl_sds_destroy(bfile->path); + } + flb_free(bfile); +} + +/* release all blob_files from the context list */ +void blob_file_list_remove_all(struct blob_ctx *ctx) +{ + struct cfl_list *head; + struct cfl_list *tmp; + struct blob_file *bfile; + + cfl_list_foreach_safe(head, tmp, &ctx->files) { + bfile = cfl_list_entry(head, struct blob_file, _head); + cfl_list_del(&bfile->_head); + blob_file_list_remove(bfile); + } +} \ No newline at end of file diff --git a/plugins/in_blob/blob_file.h b/plugins/in_blob/blob_file.h new file mode 100644 index 00000000000..2c2c5ecfc2e --- /dev/null +++ b/plugins/in_blob/blob_file.h @@ -0,0 +1,31 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IN_BLOB_FILE_H +#define IN_BLOB_FILE_H + +#include + +#include "blob.h" + +int blob_file_append(struct blob_ctx *ctx, char *path, struct stat *st); +void blob_file_list_remove(struct blob_file *bfile); +void blob_file_list_remove_all(struct blob_ctx *ctx); + +#endif \ No newline at end of file From 4a99c4ca7722cb15e6ffb9e2e47edfcf8bccd022 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:24:35 -0600 Subject: [PATCH 15/70] out_azure_blob: add support to upload large binary files The recent changes in Fluent Bit, allows to process Blob signal types which represents large binary files. When a blob arrives to the plugin, it's enqueued and processed through parts and uploaded as a Block Blob. Part sizes are configurable and survives service restart. example usage: service: flush: 1 log_level: info pipeline: inputs: - name: blob path: '~/logs/blob/*' database_file: blob.db outputs: - name: stdout match: '*' - name: azure_blob match: '*' path: kubernetes container_name: blobs auto_create_container: on database_file: azure.db part_size: 4M upload_parts_timeout: 1s workers: 10 account_name: abcdefghijk shared_key: asdkljaskldjaskldjaskldjasioduasoudaskldjaskld tls: on Signed-off-by: Eduardo Silva --- plugins/out_azure_blob/CMakeLists.txt | 1 + plugins/out_azure_blob/azure_blob.c | 650 ++++++++++++++---- plugins/out_azure_blob/azure_blob.h | 34 + plugins/out_azure_blob/azure_blob_blockblob.c | 254 +++++-- plugins/out_azure_blob/azure_blob_blockblob.h | 10 +- plugins/out_azure_blob/azure_blob_conf.c | 14 + plugins/out_azure_blob/azure_blob_db.c | 499 ++++++++++++++ plugins/out_azure_blob/azure_blob_db.h | 128 ++++ 8 files changed, 1403 insertions(+), 187 deletions(-) create mode 100644 plugins/out_azure_blob/azure_blob_db.c create mode 100644 plugins/out_azure_blob/azure_blob_db.h diff --git a/plugins/out_azure_blob/CMakeLists.txt b/plugins/out_azure_blob/CMakeLists.txt index 3624480e64a..f9e2a370741 100644 --- a/plugins/out_azure_blob/CMakeLists.txt +++ b/plugins/out_azure_blob/CMakeLists.txt @@ -3,6 +3,7 @@ set(src azure_blob_uri.c azure_blob_conf.c azure_blob_http.c + azure_blob_db.c azure_blob_appendblob.c azure_blob_blockblob.c ) diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index f35667fb378..b3ecf5dc874 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -25,10 +25,14 @@ #include #include #include +#include +#include +#include #include #include "azure_blob.h" +#include "azure_blob_db.h" #include "azure_blob_uri.h" #include "azure_blob_conf.h" #include "azure_blob_appendblob.h" @@ -37,6 +41,14 @@ #define CREATE_BLOB 1337 +/* thread_local_storage for workers */ + +struct worker_info { + int active_upload; +}; + +FLB_TLS_DEFINE(struct worker_info, worker_info); + static int azure_blob_format(struct flb_config *config, struct flb_input_instance *ins, void *plugin_context, @@ -62,40 +74,16 @@ static int azure_blob_format(struct flb_config *config, return 0; } -static int send_blob(struct flb_config *config, - struct flb_input_instance *i_ins, - struct flb_azure_blob *ctx, char *name, - char *tag, int tag_len, void *data, size_t bytes) +static int create_blob(struct flb_azure_blob *ctx, char *name) { int ret; - int compressed = FLB_FALSE; - int content_encoding = FLB_FALSE; - int content_type = FLB_FALSE; - uint64_t ms = 0; size_t b_sent; - void *out_buf; - size_t out_size; flb_sds_t uri = NULL; - flb_sds_t blockid = NULL; - void *payload_buf; - size_t payload_size; struct flb_http_client *c; struct flb_connection *u_conn; - if (ctx->btype == AZURE_BLOB_APPENDBLOB) { - uri = azb_append_blob_uri(ctx, tag); - } - else if (ctx->btype == AZURE_BLOB_BLOCKBLOB) { - blockid = azb_block_blob_id(&ms); - if (!blockid) { - flb_plg_error(ctx->ins, "could not generate block id"); - return FLB_RETRY; - } - uri = azb_block_blob_uri(ctx, tag, blockid, ms); - } - + uri = azb_uri_create_blob(ctx, name); if (!uri) { - flb_free(blockid); return FLB_RETRY; } @@ -103,44 +91,107 @@ static int send_blob(struct flb_config *config, u_conn = flb_upstream_conn_get(ctx->u); if (!u_conn) { flb_plg_error(ctx->ins, - "cannot create upstream connection for append_blob"); + "cannot create upstream connection for create_append_blob"); flb_sds_destroy(uri); - flb_free(blockid); return FLB_RETRY; } - /* Format the data */ - ret = azure_blob_format(config, i_ins, - ctx, NULL, - FLB_EVENT_TYPE_LOGS, - tag, tag_len, - data, bytes, - &out_buf, &out_size); - if (ret != 0) { + /* Create HTTP client context */ + c = flb_http_client(u_conn, FLB_HTTP_PUT, + uri, + NULL, 0, NULL, 0, NULL, 0); + if (!c) { + flb_plg_error(ctx->ins, "cannot create HTTP client context"); flb_upstream_conn_release(u_conn); flb_sds_destroy(uri); - flb_free(blockid); return FLB_RETRY; } - /* Map buffer */ - payload_buf = out_buf; - payload_size = out_size; + /* Prepare headers and authentication */ + azb_http_client_setup(ctx, c, -1, FLB_TRUE, + AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); - if (ctx->compress_gzip == FLB_TRUE || ctx->compress_blob == FLB_TRUE) { - ret = flb_gzip_compress((void *) out_buf, out_size, - &payload_buf, &payload_size); - if (ret == -1) { - flb_plg_error(ctx->ins, - "cannot gzip payload, disabling compression"); + /* Send HTTP request */ + ret = flb_http_do(c, &b_sent); + flb_sds_destroy(uri); + + if (ret == -1) { + flb_plg_error(ctx->ins, "error sending append_blob"); + flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); + return FLB_RETRY; + } + + if (c->resp.status == 201) { + /* delete "&sig=..." in the c->uri for security */ + char *p = strstr(c->uri, "&sig="); + if (p) { + *p = '\0'; + } + flb_plg_info(ctx->ins, "blob created successfully: %s", c->uri); + } + else { + if (c->resp.payload_size > 0) { + flb_plg_error(ctx->ins, "http_status=%i cannot create append blob\n%s", + c->resp.status, c->resp.payload); } else { + flb_plg_error(ctx->ins, "http_status=%i cannot create append blob", + c->resp.status); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); + return FLB_RETRY; + } + + flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); + return FLB_OK; +} + +static int http_send_blob(struct flb_config *config, struct flb_azure_blob *ctx, + flb_sds_t ref_name, + flb_sds_t uri, + flb_sds_t block_id, + int event_type, + void *data, size_t bytes) +{ + int ret; + int compressed = FLB_FALSE; + int content_encoding = FLB_FALSE; + int content_type = FLB_FALSE; + size_t b_sent; + void *payload_buf; + size_t payload_size; + struct flb_http_client *c; + struct flb_connection *u_conn; + + /* Get upstream connection */ + u_conn = flb_upstream_conn_get(ctx->u); + if (!u_conn) { + flb_plg_error(ctx->ins, + "cannot create TCP upstream connection"); + return FLB_RETRY; + } + + payload_buf = data; + payload_size = bytes; + + /* Handle compression requests */ + if (ctx->compress_gzip == FLB_TRUE || ctx->compress_blob == FLB_TRUE) { + ret = flb_gzip_compress((void *) data, bytes, &payload_buf, &payload_size); + if (ret == 0) { compressed = FLB_TRUE; - /* JSON buffer is not longer needed */ - flb_sds_destroy(out_buf); + } + else { + flb_plg_warn(ctx->ins, + "cannot gzip payload, disabling compression"); + payload_buf = data; + payload_size = bytes; } } + /* set http header flags */ if (ctx->compress_blob == FLB_TRUE) { content_encoding = AZURE_BLOB_CE_NONE; content_type = AZURE_BLOB_CT_GZIP; @@ -156,9 +207,10 @@ static int send_blob(struct flb_config *config, payload_buf, payload_size, NULL, 0, NULL, 0); if (!c) { flb_plg_error(ctx->ins, "cannot create HTTP client context"); - flb_sds_destroy(out_buf); + if (compressed == FLB_TRUE) { + flb_free(payload_buf); + } flb_upstream_conn_release(u_conn); - flb_free(blockid); return FLB_RETRY; } @@ -168,13 +220,9 @@ static int send_blob(struct flb_config *config, /* Send HTTP request */ ret = flb_http_do(c, &b_sent); - flb_sds_destroy(uri); - /* Release */ - if (compressed == FLB_FALSE) { - flb_sds_destroy(out_buf); - } - else { + /* Release compressed buffer */ + if (compressed == FLB_TRUE) { flb_free(payload_buf); } @@ -182,21 +230,13 @@ static int send_blob(struct flb_config *config, /* Validate HTTP status */ if (ret == -1) { - flb_plg_error(ctx->ins, "error sending append_blob"); - flb_free(blockid); + flb_plg_error(ctx->ins, "error sending append_blob for %s", ref_name); return FLB_RETRY; } if (c->resp.status == 201) { - flb_plg_info(ctx->ins, "content appended to blob successfully"); + flb_plg_info(ctx->ins, "content uploaded successfully: %s", ref_name); flb_http_client_destroy(c); - - if (ctx->btype == AZURE_BLOB_BLOCKBLOB) { - ret = azb_block_blob_commit(ctx, blockid, tag, ms); - flb_free(blockid); - return ret; - } - flb_free(blockid); return FLB_OK; } else if (c->resp.status == 404) { @@ -211,94 +251,115 @@ static int send_blob(struct flb_config *config, return CREATE_BLOB; } else if (c->resp.payload_size > 0) { - flb_plg_error(ctx->ins, "cannot append content to blob\n%s", - c->resp.payload); + flb_plg_error(ctx->ins, "http_status=%i cannot append content to blob\n%s", + c->resp.status, c->resp.payload); if (strstr(c->resp.payload, "must be 0 for Create Append")) { flb_http_client_destroy(c); return CREATE_BLOB; } } else { - flb_plg_error(ctx->ins, "cannot append content to blob"); + flb_plg_error(ctx->ins, "cannot upload %s content to blob (http_status=%i)", + ref_name, c->resp.status); } flb_http_client_destroy(c); return FLB_RETRY; } -static int create_blob(struct flb_azure_blob *ctx, char *name) +static int send_blob(struct flb_config *config, + struct flb_input_instance *i_ins, + struct flb_azure_blob *ctx, + int event_type, + int blob_type, char *name, uint64_t part_id, + char *tag, int tag_len, void *data, size_t bytes) { int ret; - size_t b_sent; + uint64_t ms = 0; flb_sds_t uri = NULL; - struct flb_http_client *c; - struct flb_connection *u_conn; + flb_sds_t block_id = NULL; + flb_sds_t ref_name = NULL; + void *payload_buf = data; + size_t payload_size = bytes; - uri = azb_uri_create_blob(ctx, name); - if (!uri) { + ref_name = flb_sds_create_size(256); + if (!ref_name) { return FLB_RETRY; } - /* Get upstream connection */ - u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_plg_error(ctx->ins, - "cannot create upstream connection for create_append_blob"); - flb_sds_destroy(uri); - return FLB_RETRY; + if (blob_type == AZURE_BLOB_APPENDBLOB) { + uri = azb_append_blob_uri(ctx, tag); + } + else if (blob_type == AZURE_BLOB_BLOCKBLOB) { + if (event_type == FLB_EVENT_TYPE_LOGS) { + block_id = azb_block_blob_id_logs(&ms); + if (!block_id) { + flb_plg_error(ctx->ins, "could not generate block id"); + return FLB_RETRY; + } + uri = azb_block_blob_uri(ctx, tag, block_id, ms); + ref_name = flb_sds_printf(&ref_name, "file=%s.%" PRIu64, name, ms); + } + else if (event_type == FLB_EVENT_TYPE_BLOBS) { + block_id = azb_block_blob_id_blob(ctx, name, part_id); + uri = azb_block_blob_uri(ctx, name, block_id, 0); + ref_name = flb_sds_printf(&ref_name, "file=%s:%" PRIu64, name, part_id); + } } - /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_PUT, - uri, - NULL, 0, NULL, 0, NULL, 0); - if (!c) { - flb_plg_error(ctx->ins, "cannot create HTTP client context"); - flb_upstream_conn_release(u_conn); - flb_sds_destroy(uri); + if (!uri) { + flb_free(block_id); + flb_sds_destroy(ref_name); return FLB_RETRY; } - /* Prepare headers and authentication */ - azb_http_client_setup(ctx, c, -1, FLB_TRUE, - AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); + /* Logs: Format the data (msgpack -> JSON) */ + if (event_type == FLB_EVENT_TYPE_LOGS) { + ret = azure_blob_format(config, i_ins, + ctx, NULL, + FLB_EVENT_TYPE_LOGS, + tag, tag_len, + data, bytes, + &payload_buf, &payload_size); + if (ret != 0) { + flb_sds_destroy(uri); + flb_free(block_id); + flb_sds_destroy(ref_name); + return FLB_ERROR; + } + } + else if (event_type == FLB_EVENT_TYPE_BLOBS) { + payload_buf = data; + payload_size = bytes; + } - /* Send HTTP request */ - ret = flb_http_do(c, &b_sent); - flb_sds_destroy(uri); - if (ret == -1) { - flb_plg_error(ctx->ins, "error sending append_blob"); - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - return FLB_RETRY; - } + ret = http_send_blob(config, ctx, ref_name, uri, block_id, event_type, payload_buf, payload_size); + flb_plg_debug(ctx->ins, "http_send_blob()=%i", ret); - if (c->resp.status == 201) { - /* delete "&sig=..." in the c->uri for security */ - char *p = strstr(c->uri, "&sig="); - if (p) { - *p = '\0'; + if (ret == FLB_OK) { + /* For Logs type, we need to commit the block right away */ + if (event_type == FLB_EVENT_TYPE_LOGS) { + ret = azb_block_blob_commit_block(ctx, block_id, tag, ms); + flb_free(block_id); } - flb_plg_info(ctx->ins, "blob created successfully: %s", c->uri); } - else { - if (c->resp.payload_size > 0) { - flb_plg_error(ctx->ins, "http_status=%i cannot create append blob\n%s", - c->resp.status, c->resp.payload); - } - else { - flb_plg_error(ctx->ins, "http_status=%i cannot create append blob", - c->resp.status); + else if (ret == CREATE_BLOB) { + ret = create_blob(ctx, name); + if (ret == FLB_OK) { + ret = http_send_blob(config, ctx, ref_name, uri, block_id, event_type, payload_buf, payload_size); } - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - return FLB_RETRY; } + flb_sds_destroy(ref_name); - flb_http_client_destroy(c); - flb_upstream_conn_release(u_conn); - return FLB_OK; + if (payload_buf != data) { + flb_sds_destroy(payload_buf); + } + + flb_sds_destroy(uri); + flb_free(block_id); + + return ret; } static int create_container(struct flb_azure_blob *ctx, char *name) @@ -456,6 +517,8 @@ static int cb_azure_blob_init(struct flb_output_instance *ins, (void) config; (void) data; + FLB_TLS_INIT(worker_info); + ctx = flb_azure_blob_conf_create(ins, config); if (!ctx) { return -1; @@ -465,36 +528,284 @@ static int cb_azure_blob_init(struct flb_output_instance *ins, return 0; } +static int blob_chunk_register_parts(struct flb_azure_blob *ctx, uint64_t file_id, size_t total_size) +{ + int ret; + int64_t parts = 0; + int64_t id; + size_t offset_start = 0; + size_t offset_end = 0; + + /* generate file parts */ + while (offset_start < total_size) { + offset_end = offset_start + ctx->part_size; + + /* do not exceed maximum size */ + if (offset_end > total_size) { + offset_end = total_size; + } + + /* insert part */ + ret = azb_db_file_part_insert(ctx, file_id, parts, offset_start, offset_end, &id); + if (ret == -1) { + flb_plg_error(ctx->ins, "cannot insert blob file part into database"); + return -1; + } + offset_start = offset_end; + parts++; + } + + return parts; +} + +static int process_blob_chunk(struct flb_azure_blob *ctx, struct flb_event_chunk *event_chunk) +{ + int64_t ret; + int64_t file_id; + cfl_sds_t file_path = NULL; + size_t file_size; + msgpack_object map; + + struct flb_log_event_decoder log_decoder; + struct flb_log_event log_event; + + ret = flb_log_event_decoder_init(&log_decoder, + (char *) event_chunk->data, + event_chunk->size); + + if (ret != FLB_EVENT_DECODER_SUCCESS) { + flb_plg_error(ctx->ins, + "Log event decoder initialization error : %i", (int) ret); + return -1; + + } + + while (flb_log_event_decoder_next(&log_decoder, &log_event) == FLB_EVENT_DECODER_SUCCESS) { + map = *log_event.body; + ret = flb_input_blob_file_get_info(map, &file_path, &file_size); + if (ret == -1) { + flb_plg_error(ctx->ins, "cannot get file info from blob record, skipping"); + continue; + } + + ret = azb_db_file_insert(ctx, file_path, file_size); + if (ret == -1) { + flb_plg_error(ctx->ins, "cannot insert blob file into database: %s (size=%lu)", + file_path, file_size); + cfl_sds_destroy(file_path); + continue; + } + cfl_sds_destroy(file_path); + + /* generate the parts by using the newest id created (ret) */ + file_id = ret; + ret = blob_chunk_register_parts(ctx, file_id, file_size); + if (ret == -1) { + flb_plg_error(ctx->ins, "cannot register blob file '%s 'parts into database", + file_path); + return -1; + } + + flb_plg_debug(ctx->ins, "blob file '%s' (id=%zu) registered with %zu parts", + file_path, file_id, ret); + + + } + + flb_log_event_decoder_destroy(&log_decoder); + return 0; +} + +static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context) +{ + int ret; + char *out_buf = NULL; + size_t out_size; + uint64_t id; + uint64_t file_id; + uint64_t part_id; + off_t offset_start; + off_t offset_end; + cfl_sds_t file_path = NULL; + cfl_sds_t part_ids = NULL; + struct flb_azure_blob *ctx = out_context; + struct worker_info *info; + + info = FLB_TLS_GET(worker_info); + if (info->active_upload) { + flb_plg_trace(ctx->ins, "[worker: file upload] upload already in progress..."); + flb_sched_timer_cb_coro_return(); + } + + info->active_upload = FLB_TRUE; + + /* + * Check if is there any file which has been fully uploaded and we need to commit it with + * the Put Block List operation + */ + + pthread_mutex_lock(&ctx->file_upload_commit_file_parts); + + ret = azb_db_file_oldest_ready(ctx, &file_id, &file_path, &part_ids); + if (ret == 0) { + flb_plg_trace(ctx->ins, "no blob files ready to commit"); + } + else if (ret == -1) { + flb_plg_error(ctx->ins, "cannot get oldest blob file ready to upload"); + } + else if (ret == 1) { + /* one file is ready to be committed */ + flb_plg_debug(ctx->ins, "blob file '%s' (id=%" PRIu64 ") ready to upload", file_path, file_id); + ret = azb_block_blob_commit_file_parts(ctx, file_id, file_path, part_ids); + if (ret == -1) { + flb_plg_error(ctx->ins, "cannot commit blob file parts for file id=%" PRIu64 " path=%s", + file_id, file_path); + + } + else { + flb_plg_info(ctx->ins, "blob file '%s' (id=%" PRIu64 ") committed successfully", file_path, file_id); + /* notify the engine the blob file has been processed */ + /* FIXME! */ + + /* remove the file entry from the database */ + ret = azb_db_file_delete(ctx, file_id, file_path); + if (ret == -1) { + flb_plg_error(ctx->ins, "cannot delete blob file '%s' (id=%" PRIu64 ") from the database", + file_path, file_id); + } + } + } + pthread_mutex_unlock(&ctx->file_upload_commit_file_parts); + + if (file_path) { + cfl_sds_destroy(file_path); + } + if (part_ids) { + cfl_sds_destroy(part_ids); + } + + /* check for a next part file and lock it */ + ret = azb_db_file_part_get_next(ctx, &id, &file_id, &part_id, &offset_start, &offset_end, &file_path); + if (ret == -1) { + flb_plg_error(ctx->ins, "cannot get next blob file part"); + info->active_upload = FLB_FALSE; + flb_sched_timer_cb_coro_return(); + } + else if (ret == 0) { + flb_plg_trace(ctx->ins, "no more blob file parts to process"); + info->active_upload = FLB_FALSE; + flb_sched_timer_cb_coro_return(); + } + else if (ret == 1) { + /* just continue, the row info was retrieved */ + } + + /* read the file content */ + ret = flb_utils_read_file_offset(file_path, offset_start, offset_end, &out_buf, &out_size); + if (ret == -1) { + flb_plg_error(ctx->ins, "cannot read file part %s", file_path); + cfl_sds_destroy(file_path); + info->active_upload = FLB_FALSE; + flb_sched_timer_cb_coro_return(); + } + + flb_plg_debug(ctx->ins, "sending part file %s (id=%" PRIu64 " part_id=%" PRIu64 ")", file_path, id, part_id); + ret = send_blob(config, NULL, ctx, FLB_EVENT_TYPE_BLOBS, + AZURE_BLOB_BLOCKBLOB, file_path, part_id, NULL, 0, out_buf, out_size); + if (ret == FLB_OK) { + ret = azb_db_file_part_uploaded(ctx, id); + if (ret == -1) { + info->active_upload = FLB_FALSE; + flb_sched_timer_cb_coro_return(); + } + } + else if (ret == FLB_RETRY) { + /* FIXME */ + } + info->active_upload = FLB_FALSE; + + if (out_buf) { + flb_free(out_buf); + } + cfl_sds_destroy(file_path); + flb_sched_timer_cb_coro_return(); +} + +static int azb_timer_create(struct flb_azure_blob *ctx) +{ + int ret; + int64_t ms; + struct flb_sched *sched; + + sched = flb_sched_ctx_get(); + + /* convert from seconds to milliseconds (scheduler needs ms) */ + ms = ctx->upload_parts_timeout * 1000; + + ret = flb_sched_timer_coro_cb_create(sched, FLB_SCHED_TIMER_CB_PERM, ms, + cb_azb_blob_file_upload, ctx, NULL); + if (ret == -1) { + flb_plg_error(ctx->ins, "failed to create upload timer"); + return -1; + } + + return 0; +} + static void cb_azure_blob_flush(struct flb_event_chunk *event_chunk, struct flb_output_flush *out_flush, struct flb_input_instance *i_ins, void *out_context, struct flb_config *config) { - int ret; + int ret = FLB_OK; struct flb_azure_blob *ctx = out_context; (void) i_ins; (void) config; - /* Validate the container exists, otherwise just create it */ + /* + * Azure blob requires a container. The following function validate that the container exists, + * otherwise it will be created. Note that that container name is specified by the user + * in the configuration file. + * + * https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blob-container-create#about-container-naming + */ ret = ensure_container(ctx); if (ret == FLB_FALSE) { FLB_OUTPUT_RETURN(FLB_RETRY); } - ret = send_blob(config, i_ins, ctx, - (char *) event_chunk->tag, /* use tag as 'name' */ - (char *) event_chunk->tag, flb_sds_len(event_chunk->tag), - (char *) event_chunk->data, event_chunk->size); - - if (ret == CREATE_BLOB) { - ret = create_blob(ctx, event_chunk->tag); - if (ret == FLB_OK) { - ret = send_blob(config, i_ins, ctx, - (char *) event_chunk->tag, /* use tag as 'name' */ - (char *) event_chunk->tag, - flb_sds_len(event_chunk->tag), - (char *) event_chunk->data, event_chunk->size); + if (event_chunk->type == FLB_EVENT_TYPE_LOGS) { + ret = send_blob(config, i_ins, ctx, + FLB_EVENT_TYPE_LOGS, + ctx->btype, /* blob type per user configuration */ + (char *) event_chunk->tag, /* use tag as 'name' */ + 0, /* part id */ + (char *) event_chunk->tag, flb_sds_len(event_chunk->tag), + (char *) event_chunk->data, event_chunk->size); + + if (ret == CREATE_BLOB) { + ret = create_blob(ctx, event_chunk->tag); + if (ret == FLB_OK) { + ret = send_blob(config, i_ins, ctx, + FLB_EVENT_TYPE_LOGS, + ctx->btype, /* blob type per user configuration */ + (char *) event_chunk->tag, /* use tag as 'name' */ + 0, /* part id */ + (char *) event_chunk->tag, /* use tag as 'name' */ + flb_sds_len(event_chunk->tag), + (char *) event_chunk->data, event_chunk->size); + } + } + } + else if (event_chunk->type == FLB_EVENT_TYPE_BLOBS) { + /* + * For Blob types, we use the flush callback to enqueue the file, then cb_azb_blob_file_upload() + * takes care of the rest like reading the file and uploading it to Azure. + */ + ret = process_blob_chunk(ctx, event_chunk); + if (ret == -1) { + FLB_OUTPUT_RETURN(FLB_RETRY); } } @@ -514,6 +825,36 @@ static int cb_azure_blob_exit(void *data, struct flb_config *config) return 0; } +/* worker initialization, used for our internal timers */ +static int cb_worker_init(void *data, struct flb_config *config) +{ + int ret; + struct worker_info *info; + struct flb_azure_blob *ctx = data; + + flb_plg_info(ctx->ins, "initializing worker"); + + info = FLB_TLS_GET(worker_info); + if (!info) { + /* initialize worker global info */ + info = flb_malloc(sizeof(struct worker_info)); + if (!info) { + flb_errno(); + return -1; + } + info->active_upload = FLB_FALSE; + FLB_TLS_SET(worker_info, info); + } + + ret = azb_timer_create(ctx); + if (ret == -1) { + flb_plg_error(ctx->ins, "failed to create upload timer"); + return -1; + } + + return 0; +} + /* Configuration properties map */ static struct flb_config_map config_map[] = { { @@ -595,23 +936,42 @@ static struct flb_config_map config_map[] = { "Azure Blob SAS token" }, + { + FLB_CONFIG_MAP_STR, "database_file", NULL, + 0, FLB_TRUE, offsetof(struct flb_azure_blob, database_file), + "Absolute path to a database file to be used to store blob files contexts" + }, + + { + FLB_CONFIG_MAP_SIZE, "part_size", "25M", + 0, FLB_TRUE, offsetof(struct flb_azure_blob, part_size), + "Size of each part when uploading blob files" + }, + + { + FLB_CONFIG_MAP_TIME, "upload_parts_timeout", "10M", + 0, FLB_TRUE, offsetof(struct flb_azure_blob, upload_parts_timeout), + "Timeout to upload parts of a blob file" + }, + /* EOF */ {0} }; /* Plugin registration */ struct flb_output_plugin out_azure_blob_plugin = { - .name = "azure_blob", - .description = "Azure Blob Storage", - .cb_init = cb_azure_blob_init, - .cb_flush = cb_azure_blob_flush, - .cb_exit = cb_azure_blob_exit, + .name = "azure_blob", + .description = "Azure Blob Storage", + .cb_init = cb_azure_blob_init, + .cb_flush = cb_azure_blob_flush, + .cb_exit = cb_azure_blob_exit, + .cb_worker_init = cb_worker_init, /* Test */ .test_formatter.callback = azure_blob_format, + .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, + .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_BLOBS, .config_map = config_map, - - /* Plugin flags */ - .flags = FLB_OUTPUT_NET | FLB_IO_OPT_TLS, + .workers = 1, }; diff --git a/plugins/out_azure_blob/azure_blob.h b/plugins/out_azure_blob/azure_blob.h index b1812927adb..68784f0f9fc 100644 --- a/plugins/out_azure_blob/azure_blob.h +++ b/plugins/out_azure_blob/azure_blob.h @@ -23,6 +23,7 @@ #include #include #include +#include /* Content-Type */ #define AZURE_BLOB_CT "Content-Type" @@ -58,6 +59,9 @@ struct flb_azure_blob { flb_sds_t date_key; flb_sds_t auth_type; flb_sds_t sas_token; + flb_sds_t database_file; + size_t part_size; + time_t upload_parts_timeout; /* * Internal use @@ -72,9 +76,39 @@ struct flb_azure_blob { unsigned char *decoded_sk; /* decoded shared key */ size_t decoded_sk_size; /* size of decoded shared key */ +#ifdef FLB_HAVE_SQLDB + /* + * SQLite by default is not built with multi-threading enabled, and + * since we aim to share the database connection and prepared statements + * in the output workers, we need to protect the access to these + * resources using a mutex. + */ + pthread_mutex_t db_lock; + + pthread_mutex_t file_upload_commit_file_parts; + + /* database context */ + struct flb_sqldb *db; + + /* prepared statements: files */ + sqlite3_stmt *stmt_insert_file; + sqlite3_stmt *stmt_delete_file; + sqlite3_stmt *stmt_get_file; + + /* prepared statement: file parts */ + sqlite3_stmt *stmt_insert_file_part; + sqlite3_stmt *stmt_update_file_part_uploaded; + + sqlite3_stmt *stmt_get_next_file_part; + sqlite3_stmt *stmt_update_file_part_in_progress; + sqlite3_stmt *stmt_get_oldest_file_with_parts; +#endif + /* Upstream connection */ struct flb_upstream *u; + struct flb_output_instance *ins; + struct flb_config *config; }; #endif diff --git a/plugins/out_azure_blob/azure_blob_blockblob.c b/plugins/out_azure_blob/azure_blob_blockblob.c index 8a47a700967..5acb03689b9 100644 --- a/plugins/out_azure_blob/azure_blob_blockblob.c +++ b/plugins/out_azure_blob/azure_blob_blockblob.c @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include @@ -29,7 +31,31 @@ #include "azure_blob_uri.h" #include "azure_blob_http.h" -flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *tag, +flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx, char *name) +{ + flb_sds_t uri; + + uri = azb_uri_container(ctx); + if (!uri) { + return NULL; + } + + if (ctx->path) { + flb_sds_printf(&uri, "/%s/%s?comp=blocklist", + ctx->path, name); + } + else { + flb_sds_printf(&uri, "/%s?comp=blocklist", name); + } + + if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) { + flb_sds_printf(&uri, "&%s", ctx->sas_token); + } + + return uri; +} + +flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *name, char *blockid, uint64_t ms) { int len; @@ -57,12 +83,24 @@ flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *tag, } if (ctx->path) { - flb_sds_printf(&uri, "/%s/%s.%" PRIu64 "%s?blockid=%s&comp=block", - ctx->path, tag, ms, ext, encoded_blockid); + if (ms > 0) { + flb_sds_printf(&uri, "/%s/%s.%" PRIu64 "%s?blockid=%s&comp=block", + ctx->path, name, ms, ext, encoded_blockid); + } + else { + flb_sds_printf(&uri, "/%s/%s%s?blockid=%s&comp=block", + ctx->path, name, ext, encoded_blockid); + } } else { - flb_sds_printf(&uri, "/%s.%" PRIu64 "%s?blockid=%s&comp=block", - tag, ms, ext, encoded_blockid); + if (ms > 0) { + flb_sds_printf(&uri, "/%s.%" PRIu64 "%s?blockid=%s&comp=block", + name, ms, ext, encoded_blockid); + } + else { + flb_sds_printf(&uri, "/%s%s?blockid=%s&comp=block", + name, ext, encoded_blockid); + } } if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) { @@ -106,8 +144,11 @@ flb_sds_t azb_block_blob_uri_commit(struct flb_azure_blob *ctx, return uri; } -/* Generate a block id */ -char *azb_block_blob_id(uint64_t *ms) +/* + * Generate a block id for log type events, we always submit one chunk as a block, so + * we just use the current time in milliseconds as a suffix. + */ +char *azb_block_blob_id_logs(uint64_t *ms) { int len; int ret; @@ -148,13 +189,72 @@ char *azb_block_blob_id(uint64_t *ms) return b64; } -int azb_block_blob_commit(struct flb_azure_blob *ctx, char *blockid, char *tag, - uint64_t ms) +/* + * Generate a block id for blob type events: + * + * Azure Blob requires that Blobs IDs do not exceed 64 bytes, so we generate a MD5 + * of the path and append the part number to it, we add some zeros for padding since + * all blocks id MUST have the same length. + */ +char *azb_block_blob_id_blob(struct flb_azure_blob *ctx, char *path, int64_t part_id) +{ + int i; + int len; + int ret; + unsigned char md5[16] = {0}; + char tmp[128]; + flb_sds_t md5_hex; + size_t size; + size_t o_len; + char *b64; + + /* + * block ids in base64 cannot exceed 64 bytes, so we hash the path to avoid + * exceeding the lenght and then just append the part number. + */ + ret = flb_hash_simple(FLB_HASH_MD5, (unsigned char *) path, strlen(path), + md5, sizeof(md5)); + if (ret != 0) { + flb_plg_error(ctx->ins, "cannot hash block id for path %s", path); + return NULL; + } + + /* convert md5 to hex string (32 byte hex string) */ + md5_hex = flb_sds_create_size(32); + if (!md5_hex) { + return NULL; + } + + for (i = 0; i < 16; i++) { + snprintf(md5_hex + (i * 2), 3, "%02x", md5[i]); + } + flb_sds_len_set(md5_hex, 32); + + /* append part number */ + len = snprintf(tmp, sizeof(tmp) - 1, "%s.flb-part.%06ld", md5_hex, part_id); + flb_sds_destroy(md5_hex); + + size = 64 + 1; + b64 = flb_calloc(1, size); + if (!b64) { + return NULL; + } + + /* base64 encode block id */ + ret = flb_base64_encode((unsigned char *) b64, size, &o_len, + (unsigned char *) tmp, len); + if (ret != 0) { + flb_free(b64); + return NULL; + } + + return b64; +} + +int azb_block_blob_put_block_list(struct flb_azure_blob *ctx, flb_sds_t uri, flb_sds_t payload) { int ret; size_t b_sent; - flb_sds_t uri = NULL; - flb_sds_t payload; struct flb_http_client *c; struct flb_connection *u_conn; @@ -166,35 +266,12 @@ int azb_block_blob_commit(struct flb_azure_blob *ctx, char *blockid, char *tag, return FLB_RETRY; } - /* Compose commit URI */ - uri = azb_block_blob_uri_commit(ctx, tag, ms); - if (!uri) { - flb_upstream_conn_release(u_conn); - return FLB_ERROR; - } - - payload = flb_sds_create_size(256); - if (!payload) { - flb_sds_destroy(uri); - flb_upstream_conn_release(u_conn); - return FLB_ERROR; - } - - flb_sds_printf(&payload, - "" - "" - " %s" - "", - blockid); - /* Create HTTP client context */ c = flb_http_client(u_conn, FLB_HTTP_PUT, uri, payload, flb_sds_len(payload), NULL, 0, NULL, 0); if (!c) { flb_plg_error(ctx->ins, "cannot create HTTP client context"); - flb_sds_destroy(payload); - flb_sds_destroy(uri); flb_upstream_conn_release(u_conn); return FLB_RETRY; } @@ -206,8 +283,6 @@ int azb_block_blob_commit(struct flb_azure_blob *ctx, char *blockid, char *tag, /* Send HTTP request */ ret = flb_http_do(c, &b_sent); - flb_sds_destroy(uri); - flb_sds_destroy(payload); /* Validate HTTP status */ if (ret == -1) { @@ -216,7 +291,6 @@ int azb_block_blob_commit(struct flb_azure_blob *ctx, char *blockid, char *tag, } if (c->resp.status == 201) { - flb_plg_info(ctx->ins, "blob id %s committed successfully", blockid); flb_http_client_destroy(c); flb_upstream_conn_release(u_conn); return FLB_OK; @@ -234,8 +308,8 @@ int azb_block_blob_commit(struct flb_azure_blob *ctx, char *blockid, char *tag, return FLB_RETRY; } else if (c->resp.payload_size > 0) { - flb_plg_error(ctx->ins, "cannot commit blob id %s\n%s", - blockid, c->resp.payload); + // flb_plg_error(ctx->ins, "cannot commit blob id %s\n%s", + // blockid, c->resp.payload); if (strstr(c->resp.payload, "must be 0 for Create Append")) { flb_http_client_destroy(c); flb_upstream_conn_release(u_conn); @@ -250,3 +324,105 @@ int azb_block_blob_commit(struct flb_azure_blob *ctx, char *blockid, char *tag, return FLB_OK; } + +/* Commit a single block */ +int azb_block_blob_commit_block(struct flb_azure_blob *ctx, char *blockid, char *tag, uint64_t ms) +{ + int ret; + flb_sds_t uri = NULL; + flb_sds_t payload; + + /* Compose commit URI */ + uri = azb_block_blob_uri_commit(ctx, tag, ms); + if (!uri) { + return FLB_ERROR; + } + + payload = flb_sds_create_size(256); + if (!payload) { + flb_sds_destroy(uri); + return FLB_ERROR; + } + + flb_sds_printf(&payload, + "" + "" + " %s" + "", + blockid); + + ret = azb_block_blob_put_block_list(ctx, uri, payload); + flb_sds_destroy(uri); + flb_sds_destroy(payload); + + if (ret == FLB_OK) { + flb_plg_info(ctx->ins, "blob id %s committed successfully", blockid); + } + + return ret; +} + +int azb_block_blob_commit_file_parts(struct flb_azure_blob *ctx, uint64_t file_id, cfl_sds_t path, cfl_sds_t part_ids) +{ + int ret; + uint64_t id; + char *block_id; + cfl_sds_t payload; + flb_sds_t uri; + struct mk_list *list; + struct mk_list *head; + struct flb_split_entry *sentry; + + /* split parts in a list */ + list = flb_utils_split(part_ids, ',', -1); + if (!list) { + flb_plg_error(ctx->ins, "cannot split parts list for file id=%" PRIu64 " name %s", file_id, path); + return -1; + } + + payload = flb_sds_create_size(1024); + if (!payload) { + flb_utils_split_free(list); + return -1; + } + + /* + * Compose XML with list of blocks + * https://learn.microsoft.com/en-us/rest/api/storageservices/put-block-list?tabs=microsoft-entra-id#request-body + */ + cfl_sds_printf(&payload, + "\n" + "\n" + "\n", + path); + + mk_list_foreach(head, list) { + sentry = mk_list_entry(head, struct flb_split_entry, _head); + + id = atol(sentry->value); + block_id = azb_block_blob_id_blob(ctx, path, id); + + cfl_sds_cat_safe(&payload, " ", 2); + cfl_sds_cat_safe(&payload, "", 13); + cfl_sds_cat_safe(&payload, block_id, strlen(block_id)); + cfl_sds_cat_safe(&payload, "", 14); + cfl_sds_cat_safe(&payload, "\n", 1); + + flb_free(block_id); + } + + cfl_sds_cat_safe(&payload, "", 12); + flb_utils_split_free(list); + + uri = azb_block_blob_blocklist_uri(ctx, path); + if (!uri) { + flb_sds_destroy(payload); + return -1; + } + + ret = azb_block_blob_put_block_list(ctx, uri, payload); + flb_sds_destroy(uri); + flb_sds_destroy(payload); + + return ret; +} diff --git a/plugins/out_azure_blob/azure_blob_blockblob.h b/plugins/out_azure_blob/azure_blob_blockblob.h index 741265b59d5..42d6a970428 100644 --- a/plugins/out_azure_blob/azure_blob_blockblob.h +++ b/plugins/out_azure_blob/azure_blob_blockblob.h @@ -23,10 +23,14 @@ #include #include "azure_blob.h" +flb_sds_t azb_block_blob_blocklist_uri(struct flb_azure_blob *ctx, char *name); flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *tag, char *blockid, uint64_t ms); -char *azb_block_blob_id(uint64_t *ms); -int azb_block_blob_commit(struct flb_azure_blob *ctx, char *blockid, char *tag, - uint64_t ms); +char *azb_block_blob_id_logs(uint64_t *ms); +char *azb_block_blob_id_blob(struct flb_azure_blob *ctx, char *path, uint64_t part_id); + +int azb_block_blob_commit_block(struct flb_azure_blob *ctx, char *blockid, char *tag, uint64_t ms); +int azb_block_blob_commit_file_parts(struct flb_azure_blob *ctx, uint64_t file_id, + cfl_sds_t path, cfl_sds_t part_ids); #endif diff --git a/plugins/out_azure_blob/azure_blob_conf.c b/plugins/out_azure_blob/azure_blob_conf.c index f0f7d420714..c376def322d 100644 --- a/plugins/out_azure_blob/azure_blob_conf.c +++ b/plugins/out_azure_blob/azure_blob_conf.c @@ -22,6 +22,7 @@ #include "azure_blob.h" #include "azure_blob_conf.h" +#include "azure_blob_db.h" #include #include @@ -70,6 +71,7 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in return NULL; } ctx->ins = ins; + ctx->config = config; /* Set context */ flb_output_set_context(ins, ctx); @@ -243,6 +245,16 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in } } + /* database file for blob signal handling */ + if (ctx->database_file) { + ctx->db = azb_db_open(ctx, ctx->database_file); + if (!ctx->db) { + return NULL; + } + } + + pthread_mutex_init(&ctx->file_upload_commit_file_parts, NULL); + flb_plg_info(ctx->ins, "account_name=%s, container_name=%s, blob_type=%s, emulator_mode=%s, endpoint=%s, auth_type=%s", ctx->account_name, ctx->container_name, @@ -275,5 +287,7 @@ void flb_azure_blob_conf_destroy(struct flb_azure_blob *ctx) flb_upstream_destroy(ctx->u); } + + azb_db_close(ctx); flb_free(ctx); } diff --git a/plugins/out_azure_blob/azure_blob_db.c b/plugins/out_azure_blob/azure_blob_db.c new file mode 100644 index 00000000000..f6239f1660a --- /dev/null +++ b/plugins/out_azure_blob/azure_blob_db.c @@ -0,0 +1,499 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#ifdef FLB_HAVE_SQLDB + +#include + +#include "azure_blob_db.h" + +static int prepare_stmts(struct flb_sqldb *db, struct flb_azure_blob *ctx) +{ + int ret; + + /* insert */ + ret = sqlite3_prepare_v2(db->handler, SQL_INSERT_FILE, -1, + &ctx->stmt_insert_file, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_INSERT_FILE); + return -1; + } + + /* delete */ + ret = sqlite3_prepare_v2(db->handler, SQL_DELETE_FILE, -1, + &ctx->stmt_delete_file, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_DELETE_FILE); + return -1; + } + + /* get */ + ret = sqlite3_prepare_v2(db->handler, SQL_GET_FILE, -1, + &ctx->stmt_get_file, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_GET_FILE); + return -1; + } + + /* insert blob file part */ + ret = sqlite3_prepare_v2(db->handler, SQL_INSERT_FILE_PART, -1, + &ctx->stmt_insert_file_part, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_INSERT_FILE_PART); + return -1; + } + + /* update blob part uploaded */ + ret = sqlite3_prepare_v2(db->handler, SQL_UPDATE_FILE_PART_UPLOADED, -1, + &ctx->stmt_update_file_part_uploaded, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_UPDATE_FILE_PART_UPLOADED); + return -1; + } + + ret = sqlite3_prepare_v2(db->handler, SQL_GET_NEXT_FILE_PART, -1, + &ctx->stmt_get_next_file_part, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_GET_NEXT_FILE_PART); + return -1; + } + + ret = sqlite3_prepare_v2(db->handler, SQL_UPDATE_FILE_PART_IN_PROGRESS, -1, + &ctx->stmt_update_file_part_in_progress, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_UPDATE_FILE_PART_IN_PROGRESS); + return -1; + } + + + ret = sqlite3_prepare_v2(db->handler, SQL_GET_OLDEST_FILE_WITH_PARTS_CONCAT, -1, + &ctx->stmt_get_oldest_file_with_parts, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_GET_OLDEST_FILE_WITH_PARTS_CONCAT); + return -1; + } + + return 0; +} + +static int azb_db_lock(struct flb_azure_blob *ctx) +{ + int ret; + + ret = pthread_mutex_lock(&ctx->db_lock); + if (ret != 0) { + flb_plg_error(ctx->ins, "cannot lock database mutex"); + return -1; + } + + return 0; +} + +static int azb_db_unlock(struct flb_azure_blob *ctx) +{ + int ret; + + ret = pthread_mutex_unlock(&ctx->db_lock); + if (ret != 0) { + flb_plg_error(ctx->ins, "cannot unlock database mutex"); + return -1; + } + + return 0; +} + +struct flb_sqldb *azb_db_open(struct flb_azure_blob *ctx, char *db_path) +{ + int ret; + struct flb_sqldb *db; + struct flb_output_instance *ins; + + ins = ctx->ins; + + db = flb_sqldb_open(db_path, ins->name, ctx->config); + if (!db) { + flb_plg_error(ctx->ins, "cannot open database %s", db_path); + return NULL; + } + + ret = flb_sqldb_query(db, SQL_CREATE_AZURE_BLOB_FILES, NULL, NULL); + if (ret != FLB_OK) { + flb_plg_error(ctx->ins, "cannot create database tables"); + flb_sqldb_close(db); + return NULL; + } + + ret = flb_sqldb_query(db, SQL_CREATE_AZURE_BLOB_PARTS, NULL, NULL); + if (ret != FLB_OK) { + flb_plg_error(ctx->ins, "cannot create database table for parts"); + flb_sqldb_close(db); + return NULL; + } + + ret = flb_sqldb_query(db, SQL_PRAGMA_FOREIGN_KEYS, NULL, NULL); + if (ret != FLB_OK) { + flb_plg_error(ctx->ins, "cannot enable foreign keys"); + flb_sqldb_close(db); + return NULL; + } + + ret = prepare_stmts(db, ctx); + if (ret == -1) { + flb_sqldb_close(db); + return NULL; + } + + pthread_mutex_init(&ctx->db_lock, NULL); + return db; +} + +int azb_db_close(struct flb_azure_blob *ctx) +{ + /* finalize prepared statements */ + sqlite3_finalize(ctx->stmt_insert_file); + sqlite3_finalize(ctx->stmt_delete_file); + sqlite3_finalize(ctx->stmt_get_file); + + sqlite3_finalize(ctx->stmt_insert_file_part); + sqlite3_finalize(ctx->stmt_update_file_part_uploaded); + sqlite3_finalize(ctx->stmt_update_file_part_in_progress); + + sqlite3_finalize(ctx->stmt_get_next_file_part); + sqlite3_finalize(ctx->stmt_get_oldest_file_with_parts); + + pthread_mutex_destroy(&ctx->db_lock); + return flb_sqldb_close(ctx->db); +} + +int azb_db_file_exists(struct flb_azure_blob *ctx, char *path, uint64_t *id) +{ + int ret; + int exists = FLB_FALSE; + + azb_db_lock(ctx); + + /* Bind parameters */ + sqlite3_bind_text(ctx->stmt_get_file, 1, path, -1, 0); + ret = sqlite3_step(ctx->stmt_get_file); + if (ret == SQLITE_ROW) { + exists = FLB_TRUE; + + /* id: column 0 */ + *id = sqlite3_column_int64(ctx->stmt_get_file, 0); + } + else if (ret == SQLITE_DONE) { + /* all good */ + } + else { + exists = -1; + } + + sqlite3_clear_bindings(ctx->stmt_get_file); + sqlite3_reset(ctx->stmt_get_file); + + azb_db_unlock(ctx); + + return exists; +} + +int64_t azb_db_file_insert(struct flb_azure_blob *ctx, char *path, size_t size) +{ + int ret; + int64_t id; + time_t created; + + /* Register the file */ + created = time(NULL); + + azb_db_lock(ctx); + + /* Bind parameters */ + sqlite3_bind_text(ctx->stmt_insert_file, 1, path, -1, 0); + sqlite3_bind_int64(ctx->stmt_insert_file, 2, size); + sqlite3_bind_int64(ctx->stmt_insert_file, 3, created); + + /* Run the insert */ + ret = sqlite3_step(ctx->stmt_insert_file); + if (ret != SQLITE_DONE) { + sqlite3_clear_bindings(ctx->stmt_insert_file); + sqlite3_reset(ctx->stmt_insert_file); + flb_plg_error(ctx->ins, "cannot execute insert file '%s'", path); + + azb_db_unlock(ctx); + return -1; + } + + sqlite3_clear_bindings(ctx->stmt_insert_file); + sqlite3_reset(ctx->stmt_insert_file); + + /* Get the database ID for this file */ + id = flb_sqldb_last_id(ctx->db); + flb_plg_trace(ctx->ins, "db: file '%s' inserted with id=%ld", path, id); + + azb_db_unlock(ctx); + + return id; +} + +int azb_db_file_delete(struct flb_azure_blob *ctx, uint64_t id, char *path) +{ + int ret; + + azb_db_lock(ctx); + + /* Bind parameters */ + sqlite3_bind_int64(ctx->stmt_delete_file, 1, id); + ret = sqlite3_step(ctx->stmt_delete_file); + if (ret != SQLITE_DONE) { + sqlite3_clear_bindings(ctx->stmt_delete_file); + sqlite3_reset(ctx->stmt_delete_file); + azb_db_unlock(ctx); + return -1; + } + + sqlite3_clear_bindings(ctx->stmt_delete_file); + sqlite3_reset(ctx->stmt_delete_file); + + if (ret != SQLITE_DONE) { + flb_plg_error(ctx->ins, "db: error deleting entry id=%" PRIu64 + ", path='%s' from database", id, path); + azb_db_unlock(ctx); + return -1; + } + + flb_plg_debug(ctx->ins, "db: file id=%" PRIu64 + ", path='%s' deleted from database", id, path); + + azb_db_unlock(ctx); + return 0; +} + +int azb_db_file_part_insert(struct flb_azure_blob *ctx, uint64_t file_id, + uint64_t part_id, + size_t offset_start, size_t offset_end, + int64_t *out_id) +{ + int ret; + + azb_db_lock(ctx); + + /* Bind parameters */ + sqlite3_bind_int64(ctx->stmt_insert_file_part, 1, file_id); + sqlite3_bind_int64(ctx->stmt_insert_file_part, 2, part_id); + sqlite3_bind_int64(ctx->stmt_insert_file_part, 3, offset_start); + sqlite3_bind_int64(ctx->stmt_insert_file_part, 4, offset_end); + + /* Run the insert */ + ret = sqlite3_step(ctx->stmt_insert_file_part); + if (ret != SQLITE_DONE) { + sqlite3_clear_bindings(ctx->stmt_insert_file_part); + sqlite3_reset(ctx->stmt_insert_file_part); + flb_plg_error(ctx->ins, "cannot execute insert part for file_id=%" PRIu64, + file_id); + + azb_db_unlock(ctx); + return -1; + } + + sqlite3_clear_bindings(ctx->stmt_insert_file_part); + sqlite3_reset(ctx->stmt_insert_file_part); + + azb_db_unlock(ctx); + return 0; +} + +int azb_db_file_part_in_progress(struct flb_azure_blob *ctx, int in_progress, uint64_t id) +{ + int ret; + + /* Bind parameters */ + sqlite3_bind_int(ctx->stmt_update_file_part_in_progress, 1, in_progress); + sqlite3_bind_int64(ctx->stmt_update_file_part_in_progress, 2, id); + + /* Run the update */ + ret = sqlite3_step(ctx->stmt_update_file_part_in_progress); + if (ret != SQLITE_DONE) { + sqlite3_clear_bindings(ctx->stmt_update_file_part_in_progress); + sqlite3_reset(ctx->stmt_update_file_part_in_progress); + flb_plg_error(ctx->ins, "cannot update part with id=%" PRIu64, id); + return -1; + } + + sqlite3_clear_bindings(ctx->stmt_update_file_part_in_progress); + sqlite3_reset(ctx->stmt_update_file_part_in_progress); + + return 0; +} + +/* + * Retrieve the next part file that must be processed. Note tha this function will also lock + * the file into the database to avoid any concurrency issue if multi workers are in use + */ +int azb_db_file_part_get_next(struct flb_azure_blob *ctx, + uint64_t *id, uint64_t *file_id, uint64_t *part_id, + off_t *offset_start, off_t *offset_end, + cfl_sds_t *file_path) +{ + int ret; + char *tmp = NULL; + cfl_sds_t path; + + if (azb_db_lock(ctx) != 0) { + return -1; + } + + *file_path = NULL; + + /* Run the query */ + ret = sqlite3_step(ctx->stmt_get_next_file_part); + if (ret == SQLITE_ROW) { + *id = sqlite3_column_int64(ctx->stmt_get_next_file_part, 0); + *file_id = sqlite3_column_int64(ctx->stmt_get_next_file_part, 1); + *part_id = sqlite3_column_int64(ctx->stmt_get_next_file_part, 2); + *offset_start = sqlite3_column_int64(ctx->stmt_get_next_file_part, 3); + *offset_end = sqlite3_column_int64(ctx->stmt_get_next_file_part, 4); + tmp = (char *) sqlite3_column_text(ctx->stmt_get_next_file_part, 5); + } + else if (ret == SQLITE_DONE) { + /* no records */ + sqlite3_clear_bindings(ctx->stmt_get_next_file_part); + sqlite3_reset(ctx->stmt_get_next_file_part); + azb_db_unlock(ctx); + return 0; + } + else { + sqlite3_clear_bindings(ctx->stmt_get_next_file_part); + sqlite3_reset(ctx->stmt_get_next_file_part); + azb_db_unlock(ctx); + return -1; + } + + path = cfl_sds_create(tmp); + + sqlite3_clear_bindings(ctx->stmt_get_next_file_part); + sqlite3_reset(ctx->stmt_get_next_file_part); + + if (!path) { + azb_db_unlock(ctx); + return -1; + } + + /* set the part flag 'in_progress' to '1' */ + ret = azb_db_file_part_in_progress(ctx, 1, *id); + if (ret == -1) { + cfl_sds_destroy(path); + azb_db_unlock(ctx); + return -1; + } + + *file_path = path; + azb_db_unlock(ctx); + + return 1; +} + +int azb_db_file_part_uploaded(struct flb_azure_blob *ctx, uint64_t id) +{ + int ret; + + azb_db_lock(ctx); + + /* Bind parameters */ + sqlite3_bind_int64(ctx->stmt_update_file_part_uploaded, 1, id); + + /* Run the update */ + ret = sqlite3_step(ctx->stmt_update_file_part_uploaded); + if (ret != SQLITE_DONE) { + sqlite3_clear_bindings(ctx->stmt_update_file_part_uploaded); + sqlite3_reset(ctx->stmt_update_file_part_uploaded); + flb_plg_error(ctx->ins, "cannot update part id=%" PRIu64, id); + azb_db_unlock(ctx); + return -1; + } + + sqlite3_clear_bindings(ctx->stmt_update_file_part_uploaded); + sqlite3_reset(ctx->stmt_update_file_part_uploaded); + + azb_db_unlock(ctx); + + return 0; +} + +int azb_db_file_oldest_ready(struct flb_azure_blob *ctx, + uint64_t *file_id, cfl_sds_t *path, cfl_sds_t *part_ids) +{ + int ret; + char *tmp = NULL; + + azb_db_lock(ctx); + + /* Run the query */ + ret = sqlite3_step(ctx->stmt_get_oldest_file_with_parts); + if (ret == SQLITE_ROW) { + /* file_id */ + *file_id = sqlite3_column_int64(ctx->stmt_get_oldest_file_with_parts, 0); + tmp = (char *) sqlite3_column_text(ctx->stmt_get_oldest_file_with_parts, 1); + + /* path */ + *path = cfl_sds_create(tmp); + if (!*path) { + sqlite3_clear_bindings(ctx->stmt_get_oldest_file_with_parts); + sqlite3_reset(ctx->stmt_get_oldest_file_with_parts); + azb_db_unlock(ctx); + return -1; + } + + /* part_ids */ + tmp = (char *) sqlite3_column_text(ctx->stmt_get_oldest_file_with_parts, 2); + *part_ids = cfl_sds_create(tmp); + if (!*part_ids) { + cfl_sds_destroy(*path); + sqlite3_clear_bindings(ctx->stmt_get_oldest_file_with_parts); + sqlite3_reset(ctx->stmt_get_oldest_file_with_parts); + azb_db_unlock(ctx); + return -1; + } + } + else if (ret == SQLITE_DONE) { + /* no records */ + sqlite3_clear_bindings(ctx->stmt_get_oldest_file_with_parts); + sqlite3_reset(ctx->stmt_get_oldest_file_with_parts); + azb_db_unlock(ctx); + return 0; + } + else { + azb_db_unlock(ctx); + return -1; + } + + azb_db_unlock(ctx); + return 1; +} + +#endif \ No newline at end of file diff --git a/plugins/out_azure_blob/azure_blob_db.h b/plugins/out_azure_blob/azure_blob_db.h new file mode 100644 index 00000000000..d238349bae2 --- /dev/null +++ b/plugins/out_azure_blob/azure_blob_db.h @@ -0,0 +1,128 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OUT_AZURE_BLOB_DB_H +#define OUT_AZURE_BLOB_DB_H + +#include +#include "azure_blob.h" + +#define SQL_PRAGMA_FOREIGN_KEYS "PRAGMA foreign_keys = ON;" + +#define SQL_CREATE_AZURE_BLOB_FILES \ + "CREATE TABLE IF NOT EXISTS out_azure_blob_files (" \ + " id INTEGER PRIMARY KEY," \ + " path TEXT NOT NULL," \ + " size INTEGER," \ + " created INTEGER" \ + ");" + +#define SQL_CREATE_AZURE_BLOB_PARTS \ + "CREATE TABLE IF NOT EXISTS out_azure_blob_parts (" \ + " id INTEGER PRIMARY KEY," \ + " file_id INTEGER NOT NULL," \ + " part_id INTEGER NOT NULL," \ + " uploaded INTEGER DEFAULT 0," \ + " in_progress INTEGER DEFAULT 0," \ + " offset_start INTEGER," \ + " offset_end INTEGER," \ + " FOREIGN KEY (file_id) REFERENCES out_azure_blob_files(id) " \ + " ON DELETE CASCADE" \ + ");" + +#define SQL_INSERT_FILE \ + "INSERT INTO out_azure_blob_files (path, size, created)" \ + " VALUES (@path, @size, @created);" + +/* DELETE a registered file and all it parts */ +#define SQL_DELETE_FILE \ + "DELETE FROM out_azure_blob_files WHERE id=@id;" + +#define SQL_GET_FILE \ + "SELECT * from out_azure_blob_files WHERE path=@path order by id desc;" + +#define SQL_INSERT_FILE_PART \ + "INSERT INTO out_azure_blob_parts (file_id, part_id, offset_start, offset_end)" \ + " VALUES (@file_id, @part_id, @offset_start, @offset_end);" + +#define SQL_UPDATE_FILE_PART_UPLOADED \ + "UPDATE out_azure_blob_parts SET uploaded=1, in_progress=0 WHERE id=@id;" + +#define SQL_UPDATE_FILE_PART_IN_PROGRESS \ + "UPDATE out_azure_blob_parts SET in_progress=@status WHERE id=@id;" + +/* Find the oldest files and retrieve the oldest part ready to be uploaded */ +#define SQL_GET_NEXT_FILE_PART \ + "SELECT p.id, " \ + " p.file_id, " \ + " p.part_id, " \ + " p.offset_start, " \ + " p.offset_end, " \ + " f.path " \ + "FROM out_azure_blob_parts p " \ + " JOIN out_azure_blob_files f " \ + " ON p.file_id = f.id " \ + "WHERE p.uploaded = 0 " \ + " AND p.in_progress = 0 " \ + "ORDER BY f.created ASC, " \ + " p.part_id ASC " \ + "LIMIT 1;" + + +/* + * Query to retrieve the oldest file which all it parts are mark as uploaded, this + * query will group the results in a single record, e.g: + * +* path part_ids + * ---------------- ---------- ------------------------------------------------------------ + * /.../alice29.txt 1726423769 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18, + * 19,20,21,22,23,24,25,26,27,28,29,30 + * + * this query is used to compose + */ +#define SQL_GET_OLDEST_FILE_WITH_PARTS_CONCAT \ + "SELECT f.id, f.path, GROUP_CONCAT(p.part_id ORDER BY p.part_id ASC) AS part_ids " \ + "FROM out_azure_blob_files f " \ + "JOIN out_azure_blob_parts p ON f.id = p.file_id " \ + "WHERE p.uploaded = 1 " \ + "GROUP BY f.id " \ + "HAVING COUNT(p.id) = (SELECT COUNT(p2.id) FROM out_azure_blob_parts p2 WHERE p2.file_id = f.id) " \ + "ORDER BY f.created ASC " \ + "LIMIT 1;" + +struct flb_sqldb *azb_db_open(struct flb_azure_blob *ctx, char *db_path); +int azb_db_close(struct flb_azure_blob *ctx); +int azb_db_file_exists(struct flb_azure_blob *ctx, char *path, uint64_t *id); +int64_t azb_db_file_insert(struct flb_azure_blob *ctx, char *path, size_t size); +int azb_db_file_delete(struct flb_azure_blob *ctx, uint64_t id, char *path); + +int azb_db_file_part_insert(struct flb_azure_blob *ctx, uint64_t file_id, + uint64_t part_id, + size_t offset_start, size_t offset_end, + int64_t *out_id); +int azb_db_file_part_in_progress(struct flb_azure_blob *ctx, int in_progress, uint64_t id); +int azb_db_file_part_get_next(struct flb_azure_blob *ctx, + uint64_t *id, uint64_t *file_id, uint64_t *part_id, + off_t *offset_start, off_t *offset_end, + cfl_sds_t *file_path); +int azb_db_file_part_uploaded(struct flb_azure_blob *ctx, uint64_t id); + +int azb_db_file_oldest_ready(struct flb_azure_blob *ctx, + uint64_t *file_id, cfl_sds_t *path, cfl_sds_t *part_ids); +#endif \ No newline at end of file From 7bd6bd12e5913c75426dd9e6c1af88c5dd657390 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:29:00 -0600 Subject: [PATCH 16/70] build: register in_blob Signed-off-by: Eduardo Silva --- cmake/plugins_options.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/plugins_options.cmake b/cmake/plugins_options.cmake index 5a63bbc7d68..4292ef204b5 100644 --- a/cmake/plugins_options.cmake +++ b/cmake/plugins_options.cmake @@ -11,6 +11,7 @@ option(FLB_MINIMAL "Enable minimal build configuration" No) # Inputs (sources, data collectors) # ================================= +DEFINE_OPTION(FLB_IN_BLOB "Enable Blob input plugin" ON) DEFINE_OPTION(FLB_IN_CALYPTIA_FLEET "Enable Calyptia Fleet input plugin" ON) DEFINE_OPTION(FLB_IN_COLLECTD "Enable Collectd input plugin" ON) DEFINE_OPTION(FLB_IN_CPU "Enable CPU input plugin" ON) From e7defa8009aee031f713e0cce21724e94d736f4c Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 16 Sep 2024 13:29:21 -0600 Subject: [PATCH 17/70] out_stdout: support blob type (print targe file name) Signed-off-by: Eduardo Silva --- plugins/out_stdout/stdout.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/out_stdout/stdout.c b/plugins/out_stdout/stdout.c index 5c126fa075b..625d3d3ec89 100644 --- a/plugins/out_stdout/stdout.c +++ b/plugins/out_stdout/stdout.c @@ -305,6 +305,6 @@ struct flb_output_plugin out_stdout_plugin = { .cb_exit = cb_stdout_exit, .flags = 0, .workers = 1, - .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS | FLB_OUTPUT_TRACES, + .event_type = FLB_OUTPUT_LOGS | FLB_OUTPUT_METRICS | FLB_OUTPUT_TRACES | FLB_OUTPUT_BLOBS, .config_map = config_map }; From 84c84a77e0ceb0f9734a527340001ec3e98c47cb Mon Sep 17 00:00:00 2001 From: leonardo-albertovich Date: Tue, 17 Sep 2024 14:32:23 +0200 Subject: [PATCH 18/70] scheduler: fixed typo Signed-off-by: leonardo-albertovich --- src/flb_scheduler.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/flb_scheduler.c b/src/flb_scheduler.c index 8dab1fe3222..bacc5083e01 100644 --- a/src/flb_scheduler.c +++ b/src/flb_scheduler.c @@ -31,7 +31,7 @@ #include FLB_TLS_DEFINE(struct flb_sched, flb_sched_ctx); -FLB_TLS_DEFINE(struct flb_sched_timer_cb_params, sched_timer_coro_cb_params); +FLB_TLS_DEFINE(struct flb_sched_timer_coro_cb_params, sched_timer_coro_cb_params); void flb_sched_ctx_init() { @@ -962,4 +962,4 @@ struct flb_sched_timer_coro *flb_sched_timer_coro_get(struct flb_sched *sched, u } return NULL; -} \ No newline at end of file +} From 38e7ff0947c36d3ed9a0ecfc8dd32bd3ac23e199 Mon Sep 17 00:00:00 2001 From: leonardo-albertovich Date: Tue, 17 Sep 2024 14:33:04 +0200 Subject: [PATCH 19/70] scheduler: fixed typo Signed-off-by: leonardo-albertovich --- include/fluent-bit/flb_scheduler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/fluent-bit/flb_scheduler.h b/include/fluent-bit/flb_scheduler.h index 71213b100de..c76cda60290 100644 --- a/include/fluent-bit/flb_scheduler.h +++ b/include/fluent-bit/flb_scheduler.h @@ -199,7 +199,7 @@ struct flb_sched_timer_coro_cb_params { struct flb_coro *coro; }; -extern FLB_TLS_DEFINE(struct flb_sched_timer_cb_coro_params, sched_timer_coro_cb_params); +extern FLB_TLS_DEFINE(struct flb_sched_timer_coro_cb_params, sched_timer_coro_cb_params); struct flb_timer_cb_coro_params { From aa6717e48a26d76317c1fab9ac260a520b72e899 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Fri, 20 Sep 2024 13:07:30 +0200 Subject: [PATCH 20/70] http_client: added DELETE support --- include/fluent-bit/flb_http_client.h | 1 + src/flb_http_client.c | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/include/fluent-bit/flb_http_client.h b/include/fluent-bit/flb_http_client.h index 5be84039b07..8d7058622e0 100644 --- a/include/fluent-bit/flb_http_client.h +++ b/include/fluent-bit/flb_http_client.h @@ -36,6 +36,7 @@ #define FLB_HTTP_HEAD 3 #define FLB_HTTP_CONNECT 4 #define FLB_HTTP_PATCH 5 +#define FLB_HTTP_DELETE 6 /* HTTP Flags */ #define FLB_HTTP_10 1 diff --git a/src/flb_http_client.c b/src/flb_http_client.c index cf9bce55064..72a1fadbbd1 100644 --- a/src/flb_http_client.c +++ b/src/flb_http_client.c @@ -385,7 +385,7 @@ static int process_chunked_data(struct flb_http_client *c) if (found_full_chunk == FLB_TRUE) { return FLB_HTTP_CHUNK_AVAILABLE; } - return FLB_HTTP_MORE; + return FLB_HTTP_MORE; } static int process_data(struct flb_http_client *c) @@ -665,6 +665,9 @@ struct flb_http_client *flb_http_client(struct flb_connection *u_conn, case FLB_HTTP_PUT: str_method = "PUT"; break; + case FLB_HTTP_DELETE: + str_method = "DELETE"; + break; case FLB_HTTP_HEAD: str_method = "HEAD"; break; @@ -1182,10 +1185,10 @@ int flb_http_bearer_auth(struct flb_http_client *c, const char *token) /* flb_http_do_request only sends the http request the data. * This is useful for processing the chunked responses on your own. -* If you do not want to process the response on your own or expect +* If you do not want to process the response on your own or expect * all response data before you process data, use flb_http_do instead. */ -int flb_http_do_request(struct flb_http_client *c, size_t *bytes) +int flb_http_do_request(struct flb_http_client *c, size_t *bytes) { int ret; int crlf = 2; @@ -1261,24 +1264,24 @@ int flb_http_do_request(struct flb_http_client *c, size_t *bytes) return FLB_HTTP_MORE; } -int flb_http_get_response_data(struct flb_http_client *c, size_t bytes_consumed) +int flb_http_get_response_data(struct flb_http_client *c, size_t bytes_consumed) { - /* returns + /* returns * FLB_HTTP_MORE - if we are waiting for more data to be received * FLB_HTTP_CHUNK_AVAILABLE - if this is a chunked transfer and one or more chunks * have been received and it is not the end of the stream - * FLB_HTTP_OK - if we have collected all response data and no errors were thrown - * (in chunked transfers this means we've received the end chunk + * FLB_HTTP_OK - if we have collected all response data and no errors were thrown + * (in chunked transfers this means we've received the end chunk * and any remaining data to process from the end of stream, will be * contained in the response payload) * FLB_HTTP_ERROR - for any error */ int ret = FLB_HTTP_MORE; - int r_bytes; + int r_bytes; ssize_t available; size_t out_size; - // if the caller has consumed some of the payload (via bytes_consumed) + // if the caller has consumed some of the payload (via bytes_consumed) // we consume those bytes off the payload if( bytes_consumed > 0 ) { if(bytes_consumed > c->resp.payload_size) { From f7b7b6c55bdb34c502027a7caaa8947be3f9fa7b Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Fri, 20 Sep 2024 13:07:59 +0200 Subject: [PATCH 21/70] input_blob: added source tracking field --- include/fluent-bit/flb_input_blob.h | 3 +- src/flb_input_blob.c | 44 +++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/include/fluent-bit/flb_input_blob.h b/include/fluent-bit/flb_input_blob.h index 859468230cc..0691b8f29aa 100644 --- a/include/fluent-bit/flb_input_blob.h +++ b/include/fluent-bit/flb_input_blob.h @@ -28,7 +28,8 @@ struct flb_blob_file { cfl_sds_t path; }; -int flb_input_blob_file_get_info(msgpack_object map, cfl_sds_t *file_path, size_t *size); +int flb_input_blob_file_get_info(msgpack_object map, cfl_sds_t *source, + cfl_sds_t *file_path, size_t *size); int flb_input_blob_file_register(struct flb_input_instance *ins, struct flb_log_event_encoder *encoder, const char *tag, size_t tag_len, diff --git a/src/flb_input_blob.c b/src/flb_input_blob.c index 06d97e8f566..f0958c215d0 100644 --- a/src/flb_input_blob.c +++ b/src/flb_input_blob.c @@ -28,8 +28,9 @@ #include -int flb_input_blob_file_get_info(msgpack_object map, cfl_sds_t *file_path, size_t *size) +int flb_input_blob_file_get_info(msgpack_object map, cfl_sds_t *source, cfl_sds_t *file_path, size_t *size) { + cfl_sds_t tmp_source; cfl_sds_t tmp_file_path; msgpack_object o; @@ -37,7 +38,7 @@ int flb_input_blob_file_get_info(msgpack_object map, cfl_sds_t *file_path, size_ return -1; } - if (map.via.map.size < 2) { + if (map.via.map.size < 3) { return -1; } @@ -77,8 +78,32 @@ int flb_input_blob_file_get_info(msgpack_object map, cfl_sds_t *file_path, size_ return -1; } + /* get source plugin */ + o = map.via.map.ptr[2].key; + if (o.type != MSGPACK_OBJECT_STR) { + cfl_sds_destroy(tmp_file_path); + return -1; + } + if (o.via.str.size != 6 || strncmp(o.via.str.ptr, "source", 6) != 0) { + cfl_sds_destroy(tmp_file_path); + return -1; + } + + o = map.via.map.ptr[2].val; + if (o.type != MSGPACK_OBJECT_STR) { + cfl_sds_destroy(tmp_file_path); + return -1; + } + + tmp_source = cfl_sds_create_len(o.via.str.ptr, o.via.str.size); + if (tmp_source == NULL) { + cfl_sds_destroy(tmp_file_path); + return -1; + } + *size = o.via.u64; *file_path = tmp_file_path; + *source = tmp_source; return 0; } @@ -133,6 +158,7 @@ int flb_input_blob_file_register(struct flb_input_instance *ins, flb_log_event_encoder_reset(encoder); return -1; } + ret = flb_log_event_encoder_append_body_cstring(encoder, file_path); if (ret != FLB_EVENT_ENCODER_SUCCESS) { flb_error("[blob file registration] could not append path"); @@ -154,6 +180,20 @@ int flb_input_blob_file_register(struct flb_input_instance *ins, return -1; } + ret = flb_log_event_encoder_append_body_cstring(encoder, "source"); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[blob file registration] could not append path"); + flb_log_event_encoder_reset(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_cstring(encoder, flb_input_name(ins)); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[blob file registration] could not append source plugin name"); + flb_log_event_encoder_reset(encoder); + return -1; + } + ret = flb_log_event_encoder_commit_record(encoder); if (ret != FLB_EVENT_ENCODER_SUCCESS) { flb_error("[blob file registration] could not commit record"); From 553bf3ff4a7cb9240ffbd0aac3d811cb5cf126b5 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Fri, 20 Sep 2024 13:08:58 +0200 Subject: [PATCH 22/70] out_azure_blob: added blob delete function and both file and part retry mechanism --- plugins/out_azure_blob/azure_blob.c | 168 +++++++++++++++++++++++++++- plugins/out_azure_blob/azure_blob.h | 13 +++ 2 files changed, 176 insertions(+), 5 deletions(-) diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index b3ecf5dc874..7817b11a3a0 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -149,6 +149,81 @@ static int create_blob(struct flb_azure_blob *ctx, char *name) return FLB_OK; } +static int delete_blob(struct flb_azure_blob *ctx, char *name) +{ + int ret; + size_t b_sent; + flb_sds_t uri = NULL; + struct flb_http_client *c; + struct flb_connection *u_conn; + + uri = azb_uri_create_blob(ctx, name); + if (!uri) { + return FLB_RETRY; + } + + /* Get upstream connection */ + u_conn = flb_upstream_conn_get(ctx->u); + if (!u_conn) { + flb_plg_error(ctx->ins, + "cannot create upstream connection for create_append_blob"); + flb_sds_destroy(uri); + return FLB_RETRY; + } + + /* Create HTTP client context */ + c = flb_http_client(u_conn, FLB_HTTP_DELETE, + uri, + NULL, 0, NULL, 0, NULL, 0); + if (!c) { + flb_plg_error(ctx->ins, "cannot create HTTP client context"); + flb_upstream_conn_release(u_conn); + flb_sds_destroy(uri); + return FLB_RETRY; + } + + /* Prepare headers and authentication */ + azb_http_client_setup(ctx, c, -1, FLB_TRUE, + AZURE_BLOB_CT_NONE, AZURE_BLOB_CE_NONE); + + /* Send HTTP request */ + ret = flb_http_do(c, &b_sent); + flb_sds_destroy(uri); + + if (ret == -1) { + flb_plg_error(ctx->ins, "error sending append_blob"); + flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); + return FLB_RETRY; + } + + if (c->resp.status == 201) { + /* delete "&sig=..." in the c->uri for security */ + char *p = strstr(c->uri, "&sig="); + if (p) { + *p = '\0'; + } + flb_plg_info(ctx->ins, "blob deleted successfully: %s", c->uri); + } + else { + if (c->resp.payload_size > 0) { + flb_plg_error(ctx->ins, "http_status=%i cannot delete append blob\n%s", + c->resp.status, c->resp.payload); + } + else { + flb_plg_error(ctx->ins, "http_status=%i cannot delete append blob", + c->resp.status); + } + flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); + return FLB_RETRY; + } + + flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); + return FLB_OK; +} + static int http_send_blob(struct flb_config *config, struct flb_azure_blob *ctx, flb_sds_t ref_name, flb_sds_t uri, @@ -563,6 +638,7 @@ static int process_blob_chunk(struct flb_azure_blob *ctx, struct flb_event_chunk int64_t ret; int64_t file_id; cfl_sds_t file_path = NULL; + cfl_sds_t source = NULL; size_t file_size; msgpack_object map; @@ -582,20 +658,22 @@ static int process_blob_chunk(struct flb_azure_blob *ctx, struct flb_event_chunk while (flb_log_event_decoder_next(&log_decoder, &log_event) == FLB_EVENT_DECODER_SUCCESS) { map = *log_event.body; - ret = flb_input_blob_file_get_info(map, &file_path, &file_size); + ret = flb_input_blob_file_get_info(map, &source, &file_path, &file_size); if (ret == -1) { flb_plg_error(ctx->ins, "cannot get file info from blob record, skipping"); continue; } - ret = azb_db_file_insert(ctx, file_path, file_size); + ret = azb_db_file_insert(ctx, source, file_path, file_size); if (ret == -1) { flb_plg_error(ctx->ins, "cannot insert blob file into database: %s (size=%lu)", file_path, file_size); cfl_sds_destroy(file_path); + cfl_sds_destroy(source); continue; } cfl_sds_destroy(file_path); + cfl_sds_destroy(source); /* generate the parts by using the newest id created (ret) */ file_id = ret; @@ -608,8 +686,6 @@ static int process_blob_chunk(struct flb_azure_blob *ctx, struct flb_event_chunk flb_plg_debug(ctx->ins, "blob file '%s' (id=%zu) registered with %zu parts", file_path, file_id, ret); - - } flb_log_event_decoder_destroy(&log_decoder); @@ -624,10 +700,13 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context uint64_t id; uint64_t file_id; uint64_t part_id; + uint64_t part_delivery_attempts; + uint64_t file_delivery_attempts; off_t offset_start; off_t offset_end; cfl_sds_t file_path = NULL; cfl_sds_t part_ids = NULL; + cfl_sds_t source = NULL; struct flb_azure_blob *ctx = out_context; struct worker_info *info; @@ -646,6 +725,30 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context pthread_mutex_lock(&ctx->file_upload_commit_file_parts); + ret = azb_db_file_get_next_aborted(ctx, + &file_id, + &file_delivery_attempts, + &file_path, + &source); + + if (ret == 1) { + ret = delete_blob(ctx, file_path); + + if (ctx->file_delivery_attempt_limit != FLB_OUT_RETRY_UNLIMITED && + file_delivery_attempts < ctx->file_delivery_attempt_limit) { + azb_db_file_reset_upload_states(ctx, file_id, file_path); + azb_db_file_set_aborted_state(ctx, file_id, file_path, 0); + } + else { + /* notify the input plugin + */ + ret = azb_db_file_delete(ctx, file_id, file_path); + } + + cfl_sds_destroy(file_path); + cfl_sds_destroy(source); + } + ret = azb_db_file_oldest_ready(ctx, &file_id, &file_path, &part_ids); if (ret == 0) { flb_plg_trace(ctx->ins, "no blob files ready to commit"); @@ -685,7 +788,11 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context } /* check for a next part file and lock it */ - ret = azb_db_file_part_get_next(ctx, &id, &file_id, &part_id, &offset_start, &offset_end, &file_path); + ret = azb_db_file_part_get_next(ctx, &id, &file_id, &part_id, + &offset_start, &offset_end, + &part_delivery_attempts, + &file_delivery_attempts, + &file_path); if (ret == -1) { flb_plg_error(ctx->ins, "cannot get next blob file part"); info->active_upload = FLB_FALSE; @@ -700,6 +807,13 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context /* just continue, the row info was retrieved */ } + /* since this is the first part we want to increment the files + * delivery attempt counter. + */ + if (part_id == 0) { + ret = azb_db_file_delivery_attempts(ctx, file_id, ++file_delivery_attempts); + } + /* read the file content */ ret = flb_utils_read_file_offset(file_path, offset_start, offset_end, &out_buf, &out_size); if (ret == -1) { @@ -709,6 +823,8 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context flb_sched_timer_cb_coro_return(); } + azb_db_file_part_delivery_attempts(ctx, part_id, ++part_delivery_attempts); + flb_plg_debug(ctx->ins, "sending part file %s (id=%" PRIu64 " part_id=%" PRIu64 ")", file_path, id, part_id); ret = send_blob(config, NULL, ctx, FLB_EVENT_TYPE_BLOBS, AZURE_BLOB_BLOCKBLOB, file_path, part_id, NULL, 0, out_buf, out_size); @@ -720,6 +836,12 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context } } else if (ret == FLB_RETRY) { + azb_db_file_part_in_progress(ctx, 0, id); + + if (ctx->part_delivery_attempt_limit != FLB_OUT_RETRY_UNLIMITED && + part_delivery_attempts >= ctx->part_delivery_attempt_limit) { + azb_db_file_set_aborted_state(ctx, file_id, file_path, 1); + } /* FIXME */ } info->active_upload = FLB_FALSE; @@ -948,12 +1070,48 @@ static struct flb_config_map config_map[] = { "Size of each part when uploading blob files" }, + { + FLB_CONFIG_MAP_INT, "file_delivery_attempt_limit", "1", + 0, FLB_TRUE, offsetof(struct flb_azure_blob, file_delivery_attempt_limit), + "File delivery attempt limit" + }, + + { + FLB_CONFIG_MAP_INT, "part_delivery_attempt_limit", "1", + 0, FLB_TRUE, offsetof(struct flb_azure_blob, part_delivery_attempt_limit), + "File part delivery attempt limit" + }, + { FLB_CONFIG_MAP_TIME, "upload_parts_timeout", "10M", 0, FLB_TRUE, offsetof(struct flb_azure_blob, upload_parts_timeout), "Timeout to upload parts of a blob file" }, + { + FLB_CONFIG_MAP_STR, "configuration_endpoint_url", NULL, + 0, FLB_TRUE, offsetof(struct flb_azure_blob, configuration_endpoint_url), + "Configuration endpoint URL" + }, + + { + FLB_CONFIG_MAP_STR, "configuration_endpoint_username", NULL, + 0, FLB_TRUE, offsetof(struct flb_azure_blob, configuration_endpoint_username), + "Configuration endpoint basic authentication username" + }, + + { + FLB_CONFIG_MAP_STR, "configuration_endpoint_password", NULL, + 0, FLB_TRUE, offsetof(struct flb_azure_blob, configuration_endpoint_password), + "Configuration endpoint basic authentication password" + }, + + { + FLB_CONFIG_MAP_STR, "configuration_endpoint_bearer_token", NULL, + 0, FLB_TRUE, offsetof(struct flb_azure_blob, configuration_endpoint_bearer_token), + "Configuration endpoint bearer token" + }, + /* EOF */ {0} }; diff --git a/plugins/out_azure_blob/azure_blob.h b/plugins/out_azure_blob/azure_blob.h index 68784f0f9fc..8b3009e978d 100644 --- a/plugins/out_azure_blob/azure_blob.h +++ b/plugins/out_azure_blob/azure_blob.h @@ -62,6 +62,12 @@ struct flb_azure_blob { flb_sds_t database_file; size_t part_size; time_t upload_parts_timeout; + int file_delivery_attempt_limit; + int part_delivery_attempt_limit; + flb_sds_t configuration_endpoint_url; + flb_sds_t configuration_endpoint_username; + flb_sds_t configuration_endpoint_password; + flb_sds_t configuration_endpoint_bearer_token; /* * Internal use @@ -93,11 +99,18 @@ struct flb_azure_blob { /* prepared statements: files */ sqlite3_stmt *stmt_insert_file; sqlite3_stmt *stmt_delete_file; + sqlite3_stmt *stmt_abort_file; sqlite3_stmt *stmt_get_file; + sqlite3_stmt *stmt_update_file_delivery_attempt_count; + sqlite3_stmt *stmt_set_file_aborted_state; + sqlite3_stmt *stmt_get_next_aborted_file; + sqlite3_stmt *stmt_reset_file_upload_states; + /* prepared statement: file parts */ sqlite3_stmt *stmt_insert_file_part; sqlite3_stmt *stmt_update_file_part_uploaded; + sqlite3_stmt *stmt_update_file_part_delivery_attempt_count; sqlite3_stmt *stmt_get_next_file_part; sqlite3_stmt *stmt_update_file_part_in_progress; From 53a72f619f2a7a799732aa2b55f9a4d5b621a928 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Fri, 20 Sep 2024 13:09:31 +0200 Subject: [PATCH 23/70] out_azure_blob: added remote configuration retrieval functionality --- plugins/out_azure_blob/azure_blob_conf.c | 468 ++++++++++++++++++++++- 1 file changed, 465 insertions(+), 3 deletions(-) diff --git a/plugins/out_azure_blob/azure_blob_conf.c b/plugins/out_azure_blob/azure_blob_conf.c index c376def322d..c6cb7e978aa 100644 --- a/plugins/out_azure_blob/azure_blob_conf.c +++ b/plugins/out_azure_blob/azure_blob_conf.c @@ -19,6 +19,7 @@ #include #include +#include #include "azure_blob.h" #include "azure_blob_conf.h" @@ -56,6 +57,452 @@ static int set_shared_key(struct flb_azure_blob *ctx) return 0; } +static int find_map_entry_by_key(msgpack_object_map *map, + char *key, + size_t match_index, + int case_insensitive) +{ + size_t match_count; + int result; + int index; + + match_count = 0; + + for (index = 0 ; index < (int) map->size ; index++) { + if (map->ptr[index].key.type == MSGPACK_OBJECT_STR) { + if (case_insensitive) { + result = strncasecmp(map->ptr[index].key.via.str.ptr, + key, + map->ptr[index].key.via.str.size); + } + else { + result = strncmp(map->ptr[index].key.via.str.ptr, + key, + map->ptr[index].key.via.str.size); + } + + if (result == 0) { + if (match_count == match_index) { + return index; + } + + match_count++; + } + } + } + + return -1; +} + +static int extract_map_string_entry_by_key(flb_sds_t *output, + msgpack_object_map *map, + char *key, + size_t match_index, + int case_insensitive) +{ + int index; + int result; + + index = find_map_entry_by_key(map, + key, + match_index, + case_insensitive); + + if (index == -1) { + return -1; + } + + if (map->ptr[index].val.type != MSGPACK_OBJECT_STR) { + return -2; + } + + if (*output == NULL) { + *output = flb_sds_create_len(map->ptr[index].val.via.str.ptr, + map->ptr[index].val.via.str.size); + + if (*output == NULL) { + return -3; + } + } + else { + (*output)[0] = '\0'; + + flb_sds_len_set(*output, 0); + + result = flb_sds_cat_safe(output, + map->ptr[index].val.via.str.ptr, + map->ptr[index].val.via.str.size); + + if (result != 0) { + return -4; + } + } + + return 0; +} + +static int flb_azure_blob_process_remote_configuration_payload( + struct flb_azure_blob *context, + char *payload, + size_t payload_size) +{ + size_t msgpack_body_length; + msgpack_object_map *configuration_map; + msgpack_unpacked unpacked_root; + char *msgpack_body; + int root_type; + size_t offset; + int result; + + result = flb_pack_json(payload, + payload_size, + &msgpack_body, + &msgpack_body_length, + &root_type, + NULL); + + if (result != 0) { + flb_plg_error(context->ins, + "JSON to msgpack conversion error"); + + result = -1; + } + else { + msgpack_unpacked_init(&unpacked_root); + + offset = 0; + result = msgpack_unpack_next(&unpacked_root, + msgpack_body, + msgpack_body_length, + &offset); + + if (result != MSGPACK_UNPACK_SUCCESS) { + flb_plg_error(context->ins, "corrupted msgpack data"); + + result = -1; + + goto cleanup; + } + + if (unpacked_root.data.type != MSGPACK_OBJECT_MAP) { + flb_plg_error(context->ins, "unexpected root object type"); + + result = -1; + + goto cleanup; + } + + configuration_map = &unpacked_root.data.via.map; + + result = extract_map_string_entry_by_key(&context->endpoint, + configuration_map, + "host", 0, FLB_TRUE); + + if (result != 0) { + flb_plg_error(context->ins, + "endpoint extraction error : %d", result); + + goto cleanup; + } + + if (context->atype == AZURE_BLOB_AUTH_KEY) { + result = extract_map_string_entry_by_key(&context->shared_key, + configuration_map, + "shared_key", 0, FLB_TRUE); + + if (result != 0) { + flb_plg_error(context->ins, + "neither sas_token nor shared_key " \ + "could be extracted : %d", result); + + goto cleanup; + } + } + else if (context->atype == AZURE_BLOB_AUTH_SAS) { + result = extract_map_string_entry_by_key(&context->sas_token, + configuration_map, + "sas_token", 0, FLB_TRUE); + + if (result != 0) { + flb_plg_error(context->ins, + "sas_token extraction error : %d", result); + + goto cleanup; + } + } + + result = extract_map_string_entry_by_key(&context->container_name, + configuration_map, + "container", 0, FLB_TRUE); + + if (result != 0) { + flb_plg_error(context->ins, + "container extraction error : %d", result); + + goto cleanup; + } + + result = extract_map_string_entry_by_key(&context->path, + configuration_map, + "path", 0, FLB_TRUE); + + if (result != 0) { + flb_plg_error(context->ins, + "path extraction error : %d", result); + + goto cleanup; + } + +cleanup: + if (result != 0) { + result = -1; + } + + msgpack_unpacked_destroy(&unpacked_root); + + flb_free(msgpack_body); + } + + return result; +} + +static int flb_azure_blob_apply_remote_configuration(struct flb_azure_blob *context) +{ + int ret; + size_t b_sent; + struct flb_http_client *http_client; + struct flb_connection *connection; + struct flb_upstream *upstream; + struct flb_tls *tls_context; + char *scheme = NULL; + char *host = NULL; + char *port = NULL; + char *uri = NULL; + uint16_t port_as_short; + + /* Parse and split URL */ + ret = flb_utils_url_split(context->configuration_endpoint_url, + &scheme, &host, &port, &uri); + if (ret == -1) { + flb_plg_error(context->ins, + "Invalid URL: %s", + context->configuration_endpoint_url); + + return -1; + } + + if (port != NULL) { + port_as_short = (uint16_t) strtoul(port, NULL, 10); + } + else { + if (scheme != NULL) { + if (strcasecmp(scheme, "https") == 0) { + port_as_short = 443; + } + else { + port_as_short = 80; + } + } + } + + if (scheme != NULL) { + flb_free(scheme); + scheme = NULL; + } + + if (port != NULL) { + flb_free(port); + port = NULL; + } + + if (host == NULL || uri == NULL) { + flb_plg_error(context->ins, + "Invalid URL: %s", + context->configuration_endpoint_url); + + if (host != NULL) { + flb_free(host); + } + + if (uri != NULL) { + flb_free(uri); + } + + return -2; + } + + tls_context = flb_tls_create(FLB_TLS_CLIENT_MODE, + FLB_FALSE, + FLB_FALSE, + host, + NULL, + NULL, + NULL, + NULL, + NULL); + + if (tls_context == NULL) { + flb_free(host); + flb_free(uri); + + flb_plg_error(context->ins, + "TLS context creation errror"); + + return -2; + } + + upstream = flb_upstream_create_url(context->config, + context->configuration_endpoint_url, + FLB_IO_TCP, + tls_context); + + if (upstream == NULL) { + flb_tls_destroy(tls_context); + flb_free(host); + flb_free(uri); + + flb_plg_error(context->ins, + "Upstream creation errror"); + + return -3; + } + + flb_stream_disable_async_mode(&upstream->base); + + /* Get upstream connection */ + connection = flb_upstream_conn_get(upstream); + if (connection == NULL) { + flb_upstream_destroy(upstream); + flb_tls_destroy(tls_context); + flb_free(host); + flb_free(uri); + + flb_plg_error(context->ins, + "cannot create connection"); + + return -3; + } + + /* Create HTTP client context */ + http_client = flb_http_client(connection, + FLB_HTTP_GET, + uri, + NULL, 0, + host, + (int) port_as_short, + NULL, 0); + if (http_client == NULL) { + flb_upstream_conn_release(connection); + flb_upstream_destroy(upstream); + flb_tls_destroy(tls_context); + flb_free(host); + flb_free(uri); + + flb_plg_error(context->ins, + "cannot create HTTP client"); + + return -4; + } + + flb_http_add_header(http_client, + "Accept", + strlen("Accept"), + "application/json", + 16); + + /* User Agent */ + flb_http_add_header(http_client, + "User-Agent", 10, + "Fluent-Bit", 10); + + if (context->configuration_endpoint_username != NULL && + context->configuration_endpoint_password != NULL) { + flb_http_basic_auth(http_client, + context->configuration_endpoint_username, + context->configuration_endpoint_password); + } + else if (context->configuration_endpoint_bearer_token != NULL) { + flb_http_bearer_auth(http_client, + context->configuration_endpoint_bearer_token); + } + + /* Send HTTP request */ + ret = flb_http_do(http_client, &b_sent); + + if (ret == -1) { + flb_http_client_destroy(http_client); + flb_upstream_conn_release(connection); + flb_upstream_destroy(upstream); + flb_tls_destroy(tls_context); + flb_free(host); + flb_free(uri); + + flb_plg_error(context->ins, + "Error sending configuration request"); + + return -5; + } + + if (http_client->resp.status == 200) { + flb_plg_info(context->ins, + "Configuration retrieved successfully"); + + ret = flb_azure_blob_process_remote_configuration_payload( + context, + http_client->resp.payload, + http_client->resp.payload_size); + + if (ret != 0) { + flb_plg_error(context->ins, + "Configuration payload processing error %d", + ret); + + flb_http_client_destroy(http_client); + flb_upstream_conn_release(connection); + flb_upstream_destroy(upstream); + flb_tls_destroy(tls_context); + flb_free(host); + flb_free(uri); + + return -7; + } + + flb_plg_info(context->ins, + "Configuration applied successfully"); + } + else { + if (http_client->resp.payload_size > 0) { + flb_plg_error(context->ins, + "Configuration retrieval failed with status %i\n%s", + http_client->resp.status, + http_client->resp.payload); + } + else { + flb_plg_error(context->ins, + "Configuration retrieval failed with status %i", + http_client->resp.status); + } + + flb_http_client_destroy(http_client); + flb_upstream_conn_release(connection); + flb_upstream_destroy(upstream); + flb_tls_destroy(tls_context); + flb_free(host); + flb_free(uri); + + return -6; + } + + flb_http_client_destroy(http_client); + flb_upstream_conn_release(connection); + flb_upstream_destroy(upstream); + flb_tls_destroy(tls_context); + flb_free(host); + flb_free(uri); + + return 0; +} + struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *ins, struct flb_config *config) { @@ -79,9 +526,21 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in /* Load config map */ ret = flb_output_config_map_set(ins, (void *) ctx); if (ret == -1) { + flb_free(ctx); + return NULL; } + if (ctx->configuration_endpoint_url != NULL) { + ret = flb_azure_blob_apply_remote_configuration(ctx); + + if (ret != 0) { + flb_free(ctx); + + return NULL; + } + } + if (!ctx->container_name) { flb_plg_error(ctx->ins, "'container_name' has not been set"); return NULL; @@ -104,12 +563,14 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in return NULL; } } - if (ctx->atype == AZURE_BLOB_AUTH_KEY && !ctx->shared_key) { + if (ctx->atype == AZURE_BLOB_AUTH_KEY && + ctx->shared_key == NULL) { flb_plg_error(ctx->ins, "'shared_key' has not been set"); return NULL; } + if (ctx->atype == AZURE_BLOB_AUTH_SAS) { - if (!ctx->sas_token) { + if (ctx->sas_token == NULL) { flb_plg_error(ctx->ins, "'sas_token' has not been set"); return NULL; } @@ -119,7 +580,8 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in } /* If the shared key is set decode it */ - if (ctx->atype == AZURE_BLOB_AUTH_KEY && ctx->shared_key) { + if (ctx->atype == AZURE_BLOB_AUTH_KEY && + ctx->shared_key != NULL) { ret = set_shared_key(ctx); if (ret == -1) { return NULL; From 997aa8179f55ff4c10281e602b75176c00dbddf1 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Fri, 20 Sep 2024 13:09:56 +0200 Subject: [PATCH 24/70] out_azure_blob: added retry and source tracking related functionality --- plugins/out_azure_blob/azure_blob_db.c | 257 ++++++++++++++++++++++++- plugins/out_azure_blob/azure_blob_db.h | 78 ++++++-- 2 files changed, 313 insertions(+), 22 deletions(-) diff --git a/plugins/out_azure_blob/azure_blob_db.c b/plugins/out_azure_blob/azure_blob_db.c index f6239f1660a..b218a3b9dfc 100644 --- a/plugins/out_azure_blob/azure_blob_db.c +++ b/plugins/out_azure_blob/azure_blob_db.c @@ -47,6 +47,26 @@ static int prepare_stmts(struct flb_sqldb *db, struct flb_azure_blob *ctx) return -1; } + /* abort */ + ret = sqlite3_prepare_v2(db->handler, SQL_SET_FILE_ABORTED_STATE, -1, + &ctx->stmt_set_file_aborted_state, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_SET_FILE_ABORTED_STATE); + return -1; + } + + /* delivery attempt counter update */ + ret = sqlite3_prepare_v2(db->handler, + SQL_UPDATE_FILE_DELIVERY_ATTEMPT_COUNT, -1, + &ctx->stmt_update_file_delivery_attempt_count, + NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_UPDATE_FILE_DELIVERY_ATTEMPT_COUNT); + return -1; + } + /* get */ ret = sqlite3_prepare_v2(db->handler, SQL_GET_FILE, -1, &ctx->stmt_get_file, NULL); @@ -56,6 +76,23 @@ static int prepare_stmts(struct flb_sqldb *db, struct flb_azure_blob *ctx) return -1; } + /* get next aborted file */ + ret = sqlite3_prepare_v2(db->handler, SQL_GET_NEXT_ABORTED_FILE, -1, + &ctx->stmt_get_next_aborted_file, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_GET_NEXT_ABORTED_FILE); + return -1; + } + + ret = sqlite3_prepare_v2(db->handler, SQL_RESET_FILE_UPLOAD_STATES, -1, + &ctx->stmt_reset_file_upload_states, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_RESET_FILE_UPLOAD_STATES); + return -1; + } + /* insert blob file part */ ret = sqlite3_prepare_v2(db->handler, SQL_INSERT_FILE_PART, -1, &ctx->stmt_insert_file_part, NULL); @@ -90,6 +127,15 @@ static int prepare_stmts(struct flb_sqldb *db, struct flb_azure_blob *ctx) return -1; } + ret = sqlite3_prepare_v2(db->handler, + SQL_UPDATE_FILE_PART_DELIVERY_ATTEMPT_COUNT, -1, + &ctx->stmt_update_file_part_delivery_attempt_count, + NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_UPDATE_FILE_PART_DELIVERY_ATTEMPT_COUNT); + return -1; + } ret = sqlite3_prepare_v2(db->handler, SQL_GET_OLDEST_FILE_WITH_PARTS_CONCAT, -1, &ctx->stmt_get_oldest_file_with_parts, NULL); @@ -178,11 +224,16 @@ int azb_db_close(struct flb_azure_blob *ctx) /* finalize prepared statements */ sqlite3_finalize(ctx->stmt_insert_file); sqlite3_finalize(ctx->stmt_delete_file); + sqlite3_finalize(ctx->stmt_set_file_aborted_state); sqlite3_finalize(ctx->stmt_get_file); + sqlite3_finalize(ctx->stmt_update_file_delivery_attempt_count); + sqlite3_finalize(ctx->stmt_get_next_aborted_file); + sqlite3_finalize(ctx->stmt_reset_file_upload_states); sqlite3_finalize(ctx->stmt_insert_file_part); sqlite3_finalize(ctx->stmt_update_file_part_uploaded); sqlite3_finalize(ctx->stmt_update_file_part_in_progress); + sqlite3_finalize(ctx->stmt_update_file_part_delivery_attempt_count); sqlite3_finalize(ctx->stmt_get_next_file_part); sqlite3_finalize(ctx->stmt_get_oldest_file_with_parts); @@ -222,7 +273,8 @@ int azb_db_file_exists(struct flb_azure_blob *ctx, char *path, uint64_t *id) return exists; } -int64_t azb_db_file_insert(struct flb_azure_blob *ctx, char *path, size_t size) +int64_t azb_db_file_insert(struct flb_azure_blob *ctx, + char *source, char *path, size_t size) { int ret; int64_t id; @@ -234,9 +286,10 @@ int64_t azb_db_file_insert(struct flb_azure_blob *ctx, char *path, size_t size) azb_db_lock(ctx); /* Bind parameters */ - sqlite3_bind_text(ctx->stmt_insert_file, 1, path, -1, 0); - sqlite3_bind_int64(ctx->stmt_insert_file, 2, size); - sqlite3_bind_int64(ctx->stmt_insert_file, 3, created); + sqlite3_bind_text(ctx->stmt_insert_file, 1, source, -1, 0); + sqlite3_bind_text(ctx->stmt_insert_file, 2, path, -1, 0); + sqlite3_bind_int64(ctx->stmt_insert_file, 3, size); + sqlite3_bind_int64(ctx->stmt_insert_file, 4, created); /* Run the insert */ ret = sqlite3_step(ctx->stmt_insert_file); @@ -294,6 +347,166 @@ int azb_db_file_delete(struct flb_azure_blob *ctx, uint64_t id, char *path) return 0; } +int azb_db_file_set_aborted_state(struct flb_azure_blob *ctx, + uint64_t id, char *path, + uint64_t state) +{ + int ret; + + azb_db_lock(ctx); + + /* Bind parameters */ + sqlite3_bind_int64(ctx->stmt_set_file_aborted_state, 1, state); + sqlite3_bind_int64(ctx->stmt_set_file_aborted_state, 2, id); + ret = sqlite3_step(ctx->stmt_set_file_aborted_state); + if (ret != SQLITE_DONE) { + sqlite3_clear_bindings(ctx->stmt_set_file_aborted_state); + sqlite3_reset(ctx->stmt_set_file_aborted_state); + azb_db_unlock(ctx); + return -1; + } + + sqlite3_clear_bindings(ctx->stmt_set_file_aborted_state); + sqlite3_reset(ctx->stmt_set_file_aborted_state); + + if (ret != SQLITE_DONE) { + flb_plg_error(ctx->ins, "db: error aborting entry id=%" PRIu64 + ", path='%s' from database", id, path); + azb_db_unlock(ctx); + return -1; + } + + flb_plg_debug(ctx->ins, "db: file id=%" PRIu64 + ", path='%s' marked as aborted in the database", id, path); + + azb_db_unlock(ctx); + return 0; +} + +int azb_db_file_delivery_attempts(struct flb_azure_blob *ctx, + uint64_t id, uint64_t attempts) +{ + int ret; + + azb_db_lock(ctx); + + /* Bind parameters */ + sqlite3_bind_int64(ctx->stmt_update_file_delivery_attempt_count, 1, attempts); + sqlite3_bind_int64(ctx->stmt_update_file_delivery_attempt_count, 2, id); + + /* Run the update */ + ret = sqlite3_step(ctx->stmt_update_file_delivery_attempt_count); + + sqlite3_clear_bindings(ctx->stmt_update_file_delivery_attempt_count); + sqlite3_reset(ctx->stmt_update_file_delivery_attempt_count); + + azb_db_unlock(ctx); + + if (ret != SQLITE_DONE) { + flb_plg_error(ctx->ins, + "cannot update delivery attempt " + "count for file id=%" PRIu64, id); + + return -1; + } + + return 0; +} + +int azb_db_file_get_next_aborted(struct flb_azure_blob *ctx, + uint64_t *id, + uint64_t *delivery_attempts, + cfl_sds_t *path, + cfl_sds_t *source) +{ + int ret; + char *tmp_source; + char *tmp_path; + int exists = FLB_FALSE; + + azb_db_lock(ctx); + + /* Bind parameters */ + ret = sqlite3_step(ctx->stmt_get_next_aborted_file); + if (ret == SQLITE_ROW) { + exists = FLB_TRUE; + + /* id: column 0 */ + *id = sqlite3_column_int64(ctx->stmt_get_next_aborted_file, 0); + *delivery_attempts = sqlite3_column_int64(ctx->stmt_get_next_file_part, 1); + tmp_source = (char *) sqlite3_column_text(ctx->stmt_get_next_file_part, 2); + tmp_path = (char *) sqlite3_column_text(ctx->stmt_get_next_file_part, 3); + + *path = cfl_sds_create(tmp_path); + + if (*path == NULL) { + exists = -1; + } + else { + *source = cfl_sds_create(tmp_source); + if (*source == NULL) { + cfl_sds_destroy(*path); + exists = -1; + } + } + } + else if (ret == SQLITE_DONE) { + /* all good */ + } + else { + exists = -1; + } + + sqlite3_clear_bindings(ctx->stmt_get_next_aborted_file); + sqlite3_reset(ctx->stmt_get_next_aborted_file); + + azb_db_unlock(ctx); + + if (exists == -1) { + *id = 0; + *delivery_attempts = 0; + *path = NULL; + *source = NULL; + } + + return exists; +} + +int azb_db_file_reset_upload_states(struct flb_azure_blob *ctx, uint64_t id, char *path) +{ + int ret; + + azb_db_lock(ctx); + + /* Bind parameters */ + sqlite3_bind_int64(ctx->stmt_reset_file_upload_states, 1, id); + ret = sqlite3_step(ctx->stmt_reset_file_upload_states); + if (ret != SQLITE_DONE) { + sqlite3_clear_bindings(ctx->stmt_reset_file_upload_states); + sqlite3_reset(ctx->stmt_reset_file_upload_states); + azb_db_unlock(ctx); + return -1; + } + + sqlite3_clear_bindings(ctx->stmt_reset_file_upload_states); + sqlite3_reset(ctx->stmt_reset_file_upload_states); + + if (ret != SQLITE_DONE) { + flb_plg_error(ctx->ins, "db: error reseting upload " + "states for entry id=%" PRIu64 + ", path='%s'", id, path); + azb_db_unlock(ctx); + return -1; + } + + flb_plg_debug(ctx->ins, "db: file id=%" PRIu64 + ", path='%s' upload states reset", id, path); + + azb_db_unlock(ctx); + + return 0; +} + int azb_db_file_part_insert(struct flb_azure_blob *ctx, uint64_t file_id, uint64_t part_id, size_t offset_start, size_t offset_end, @@ -358,6 +571,8 @@ int azb_db_file_part_in_progress(struct flb_azure_blob *ctx, int in_progress, ui int azb_db_file_part_get_next(struct flb_azure_blob *ctx, uint64_t *id, uint64_t *file_id, uint64_t *part_id, off_t *offset_start, off_t *offset_end, + uint64_t *part_delivery_attempts, + uint64_t *file_delivery_attempts, cfl_sds_t *file_path) { int ret; @@ -378,7 +593,9 @@ int azb_db_file_part_get_next(struct flb_azure_blob *ctx, *part_id = sqlite3_column_int64(ctx->stmt_get_next_file_part, 2); *offset_start = sqlite3_column_int64(ctx->stmt_get_next_file_part, 3); *offset_end = sqlite3_column_int64(ctx->stmt_get_next_file_part, 4); - tmp = (char *) sqlite3_column_text(ctx->stmt_get_next_file_part, 5); + *part_delivery_attempts = sqlite3_column_int64(ctx->stmt_get_next_file_part, 5); + tmp = (char *) sqlite3_column_text(ctx->stmt_get_next_file_part, 6); + *file_delivery_attempts = sqlite3_column_int64(ctx->stmt_get_next_file_part, 7); } else if (ret == SQLITE_DONE) { /* no records */ @@ -445,6 +662,36 @@ int azb_db_file_part_uploaded(struct flb_azure_blob *ctx, uint64_t id) return 0; } +int azb_db_file_part_delivery_attempts(struct flb_azure_blob *ctx, + uint64_t id, uint64_t attempts) +{ + int ret; + + azb_db_lock(ctx); + + /* Bind parameters */ + sqlite3_bind_int64(ctx->stmt_update_file_part_delivery_attempt_count, 1, attempts); + sqlite3_bind_int64(ctx->stmt_update_file_part_delivery_attempt_count, 2, id); + + /* Run the update */ + ret = sqlite3_step(ctx->stmt_update_file_part_delivery_attempt_count); + + sqlite3_clear_bindings(ctx->stmt_update_file_part_delivery_attempt_count); + sqlite3_reset(ctx->stmt_update_file_part_delivery_attempt_count); + + azb_db_unlock(ctx); + + if (ret != SQLITE_DONE) { + flb_plg_error(ctx->ins, + "cannot update delivery attempt " + "count for part id=%" PRIu64, id); + + return -1; + } + + return 0; +} + int azb_db_file_oldest_ready(struct flb_azure_blob *ctx, uint64_t *file_id, cfl_sds_t *path, cfl_sds_t *part_ids) { diff --git a/plugins/out_azure_blob/azure_blob_db.h b/plugins/out_azure_blob/azure_blob_db.h index d238349bae2..1009d08733c 100644 --- a/plugins/out_azure_blob/azure_blob_db.h +++ b/plugins/out_azure_blob/azure_blob_db.h @@ -27,35 +27,56 @@ #define SQL_CREATE_AZURE_BLOB_FILES \ "CREATE TABLE IF NOT EXISTS out_azure_blob_files (" \ - " id INTEGER PRIMARY KEY," \ - " path TEXT NOT NULL," \ - " size INTEGER," \ - " created INTEGER" \ + " id INTEGER PRIMARY KEY," \ + " source TEXT NOT NULL," \ + " path TEXT NOT NULL," \ + " size INTEGER," \ + " created INTEGER," \ + " delivery_attempts INTEGER DEFAULT 0," \ + " aborted INTEGER DEFAULT 0" \ ");" #define SQL_CREATE_AZURE_BLOB_PARTS \ "CREATE TABLE IF NOT EXISTS out_azure_blob_parts (" \ - " id INTEGER PRIMARY KEY," \ - " file_id INTEGER NOT NULL," \ - " part_id INTEGER NOT NULL," \ - " uploaded INTEGER DEFAULT 0," \ - " in_progress INTEGER DEFAULT 0," \ - " offset_start INTEGER," \ - " offset_end INTEGER," \ + " id INTEGER PRIMARY KEY," \ + " file_id INTEGER NOT NULL," \ + " part_id INTEGER NOT NULL," \ + " uploaded INTEGER DEFAULT 0," \ + " in_progress INTEGER DEFAULT 0," \ + " offset_start INTEGER," \ + " offset_end INTEGER," \ + " delivery_attempts INTEGER DEFAULT 0," \ " FOREIGN KEY (file_id) REFERENCES out_azure_blob_files(id) " \ " ON DELETE CASCADE" \ ");" #define SQL_INSERT_FILE \ - "INSERT INTO out_azure_blob_files (path, size, created)" \ - " VALUES (@path, @size, @created);" + "INSERT INTO out_azure_blob_files (source, path, size, created)" \ + " VALUES (@source, @path, @size, @created);" /* DELETE a registered file and all it parts */ #define SQL_DELETE_FILE \ "DELETE FROM out_azure_blob_files WHERE id=@id;" +#define SQL_SET_FILE_ABORTED_STATE \ + "UPDATE out_azure_blob_files SET aborted=@state WHERE id=@id;" + +#define SQL_UPDATE_FILE_DELIVERY_ATTEMPT_COUNT \ + "UPDATE out_azure_blob_files SET delivery_attempts=@delivery_attempts WHERE id=@id;" + #define SQL_GET_FILE \ - "SELECT * from out_azure_blob_files WHERE path=@path order by id desc;" + "SELECT * FROM out_azure_blob_files WHERE path=@path ORDER BY id DESC;" + +#define SQL_GET_NEXT_ABORTED_FILE \ + "SELECT id, delivery_attempts, source, path " \ + " FROM out_azure_blob_files azbf " \ + " WHERE aborted = 1 " \ + " AND (SELECT COUNT(*) " \ + " FROM out_azure_blob_parts azbp " \ + " WHERE azbp.file_id = azbf.id " \ + " AND in_progress = 1) = 0 " \ + "ORDER BY id DESC " \ + "LIMIT 1;" #define SQL_INSERT_FILE_PART \ "INSERT INTO out_azure_blob_parts (file_id, part_id, offset_start, offset_end)" \ @@ -67,6 +88,12 @@ #define SQL_UPDATE_FILE_PART_IN_PROGRESS \ "UPDATE out_azure_blob_parts SET in_progress=@status WHERE id=@id;" +#define SQL_UPDATE_FILE_PART_DELIVERY_ATTEMPT_COUNT \ + "UPDATE out_azure_blob_parts SET delivery_attempts=@delivery_attempts WHERE id=@id;" + +#define SQL_RESET_FILE_UPLOAD_STATES \ + "UPDATE out_azure_blob_parts SET delivery_attempts=0, uploaded=0, in_progress=0 WHERE file_id=@id;" + /* Find the oldest files and retrieve the oldest part ready to be uploaded */ #define SQL_GET_NEXT_FILE_PART \ "SELECT p.id, " \ @@ -74,12 +101,15 @@ " p.part_id, " \ " p.offset_start, " \ " p.offset_end, " \ - " f.path " \ + " p.delivery_attempts, " \ + " f.path, " \ + " f.delivery_attempts " \ "FROM out_azure_blob_parts p " \ " JOIN out_azure_blob_files f " \ " ON p.file_id = f.id " \ "WHERE p.uploaded = 0 " \ - " AND p.in_progress = 0 " \ + " AND p.in_progress = 0 " \ + " AND f.aborted = 0 " \ "ORDER BY f.created ASC, " \ " p.part_id ASC " \ "LIMIT 1;" @@ -109,8 +139,19 @@ struct flb_sqldb *azb_db_open(struct flb_azure_blob *ctx, char *db_path); int azb_db_close(struct flb_azure_blob *ctx); int azb_db_file_exists(struct flb_azure_blob *ctx, char *path, uint64_t *id); -int64_t azb_db_file_insert(struct flb_azure_blob *ctx, char *path, size_t size); +int64_t azb_db_file_insert(struct flb_azure_blob *ctx, + char *source, char *path, size_t size); int azb_db_file_delete(struct flb_azure_blob *ctx, uint64_t id, char *path); +int azb_db_file_set_aborted_state(struct flb_azure_blob *ctx, + uint64_t id, char *path, + uint64_t state); +int azb_db_file_delivery_attempts(struct flb_azure_blob *ctx, uint64_t id, uint64_t attempts); +int azb_db_file_get_next_aborted(struct flb_azure_blob *ctx, + uint64_t *id, + uint64_t *delivery_attempts, + cfl_sds_t *path, + cfl_sds_t *source); +int azb_db_file_reset_upload_states(struct flb_azure_blob *ctx, uint64_t id, char *path); int azb_db_file_part_insert(struct flb_azure_blob *ctx, uint64_t file_id, uint64_t part_id, @@ -120,8 +161,11 @@ int azb_db_file_part_in_progress(struct flb_azure_blob *ctx, int in_progress, ui int azb_db_file_part_get_next(struct flb_azure_blob *ctx, uint64_t *id, uint64_t *file_id, uint64_t *part_id, off_t *offset_start, off_t *offset_end, + uint64_t *part_delivery_attempts, + uint64_t *file_delivery_attempts, cfl_sds_t *file_path); int azb_db_file_part_uploaded(struct flb_azure_blob *ctx, uint64_t id); +int azb_db_file_part_delivery_attempts(struct flb_azure_blob *ctx, uint64_t id, uint64_t attempts); int azb_db_file_oldest_ready(struct flb_azure_blob *ctx, uint64_t *file_id, cfl_sds_t *path, cfl_sds_t *part_ids); From 78adc864137993504809eac952b5f8469fcaf09e Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sat, 21 Sep 2024 19:29:50 +0200 Subject: [PATCH 25/70] in_blob: added fluentd style recursive file discovery Signed-off-by: Leonardo Alminana --- plugins/in_blob/blob.c | 261 ++++++++++++++++++++++++++++++++++------- 1 file changed, 217 insertions(+), 44 deletions(-) diff --git a/plugins/in_blob/blob.c b/plugins/in_blob/blob.c index 223aa4a13c8..895d08c4f29 100644 --- a/plugins/in_blob/blob.c +++ b/plugins/in_blob/blob.c @@ -115,9 +115,10 @@ static char *expand_tilde(const char *path) } #endif - -static inline int do_glob(const char *pattern, int flags, - void *not_used, glob_t *pglob) +static inline int do_glob(const char *pattern, + int flags, + void *not_used, + glob_t *pglob) { int ret; int new_flags; @@ -166,80 +167,252 @@ static inline int do_glob(const char *pattern, int flags, return ret; } -static int scan_path(struct blob_ctx *ctx) +/* This function recursively searches a directory tree using + * a glob compatible pattern that implements the fluentd style + * recursion wildcard **. + * + * Result values : + * - Negative values identify error conditions + * - Zero and positive values represent the match count + * + * Known issues : + * - This function is only able to handle one recursion + * wildcard in its pattern. + * - This function does not follow directory links + * - Pattern generation needs to be optimized to + * minimize garbage pattern generation + */ +static ssize_t recursive_file_search(struct blob_ctx *ctx, + const char *path, + const char *pattern) { - int i; - int ret; - int count = 0; - glob_t globbuf; - struct stat st; + char *recursive_search_token; + int recursive_search_flag; + struct stat fs_entry_metadata; + ssize_t recursion_result; + cfl_sds_t local_pattern; + glob_t glob_context; + ssize_t match_count; + cfl_sds_t local_path; + cfl_sds_t sds_result; + int result; + size_t index; + match_count = 0; + + if (path != NULL) { + local_path = cfl_sds_create(path); + } + else { + local_path = cfl_sds_create(""); + } + + if (local_path == NULL) { + return -1; + } + + if (pattern != NULL) { + local_pattern = cfl_sds_create(pattern); + } + else { + local_pattern = cfl_sds_create(""); + } + + if (local_pattern == NULL) { + cfl_sds_destroy(local_path); + + return -2; + } + + recursive_search_flag = FLB_TRUE; + recursive_search_token = strstr(local_pattern, "/**/"); + + if (recursive_search_token != NULL) { + if (strstr(&recursive_search_token[4], "/**/") != NULL) { + flb_plg_error(ctx->ins, + "a search path with multiple recursivity " \ + "wildcards was provided which is not " \ + "supported. \"%s\"", local_pattern); - flb_plg_debug(ctx->ins, "scanning path %s", ctx->path); + cfl_sds_destroy(local_pattern); + cfl_sds_destroy(local_path); - /* Safe reset for globfree() */ - globbuf.gl_pathv = NULL; + return -3; + } + + recursive_search_token[0] = '\0'; + + sds_result = cfl_sds_cat(local_path, + local_pattern, + strlen(local_pattern)); + + if (sds_result == NULL) { + cfl_sds_destroy(local_pattern); + cfl_sds_destroy(local_path); + + return -4; + } + + local_path = sds_result; + + memmove(local_pattern, + &recursive_search_token[3], + strlen(&recursive_search_token[3]) + 1); + + cfl_sds_len_set(local_pattern, + strlen(&recursive_search_token[3])); + } + else if (cfl_sds_len(local_pattern) == 0) { + recursive_search_flag = FLB_FALSE; + } + + memset(&glob_context, 0, sizeof(glob_t)); /* Scan the given path */ - ret = do_glob(ctx->path, GLOB_TILDE | GLOB_ERR, NULL, &globbuf); - if (ret != 0) { - switch (ret) { + result = do_glob(local_path, GLOB_TILDE | GLOB_ERR, NULL, &glob_context); + if (result != 0) { + switch (result) { case GLOB_NOSPACE: flb_plg_error(ctx->ins, "no memory space available"); - return -1; + return -5; case GLOB_ABORTED: - flb_plg_error(ctx->ins, "read error, check permissions: %s", ctx->path); - return -1; + return 0; case GLOB_NOMATCH: - ret = stat(ctx->path, &st); - if (ret == -1) { - flb_plg_debug(ctx->ins, "cannot read info from: %s", ctx->path); + result = stat(local_path, &fs_entry_metadata); + + if (result == -1) { + flb_plg_debug(ctx->ins, "cannot read info from: %s", local_path); } else { - ret = access(ctx->path, R_OK); - if (ret == -1 && errno == EACCES) { - flb_plg_error(ctx->ins, "NO read access for path: %s", ctx->path); + result = access(local_path, R_OK); + if (result == -1 && errno == EACCES) { + flb_plg_error(ctx->ins, "NO read access for path: %s", local_path); } else { - flb_plg_debug(ctx->ins, "NO matches for path: %s", ctx->path); + flb_plg_debug(ctx->ins, "NO matches for path: %s", local_path); } } - return 0; + + return 6; } } - /* For every entry found, generate an output list */ - for (i = 0; i < globbuf.gl_pathc; i++) { - ret = stat(globbuf.gl_pathv[i], &st); - if (ret != 0) { - flb_plg_debug(ctx->ins, "skip entry=%s", globbuf.gl_pathv[i]); + for (index = 0; index < glob_context.gl_pathc; index++) { + result = stat(glob_context.gl_pathv[index], &fs_entry_metadata); + + if (result != 0) { + flb_plg_debug(ctx->ins, "skip entry=%s", glob_context.gl_pathv[index]); + continue; } - if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) { - ret = blob_file_append(ctx, globbuf.gl_pathv[i], &st); - if (ret == 0) { - flb_plg_debug(ctx->ins, "blob scan add: %s, inode %" PRIu64, - globbuf.gl_pathv[i], (uint64_t) st.st_ino); + if (S_ISDIR(fs_entry_metadata.st_mode)) { + local_path[0] = '\0'; + + cfl_sds_len_set(local_path, 0); + + sds_result = cfl_sds_printf(&local_path, + "%s/*", + glob_context.gl_pathv[index]); + + if (sds_result == NULL) { + cfl_sds_destroy(local_pattern); + cfl_sds_destroy(local_path); + + return -7; } - else { - flb_plg_debug(ctx->ins, "blob scan skip: %s", globbuf.gl_pathv[i]); + + recursion_result = recursive_file_search(ctx, + local_path, + local_pattern); + + if (recursion_result < 0) { + cfl_sds_destroy(local_pattern); + cfl_sds_destroy(local_path); + + return -8; + } + + match_count += recursion_result; + + local_path[0] = '\0'; + + cfl_sds_len_set(local_path, 0); + + sds_result = cfl_sds_printf(&local_path, + "%s%s", + glob_context.gl_pathv[index], + local_pattern); + + if (sds_result == NULL) { + cfl_sds_destroy(local_pattern); + cfl_sds_destroy(local_path); + + return -9; + } + + recursion_result = recursive_file_search(ctx, + local_path, + NULL); + + if (recursion_result < 0) { + cfl_sds_destroy(local_pattern); + cfl_sds_destroy(local_path); + + return -10; } + + match_count += recursion_result; } - else { - flb_plg_debug(ctx->ins, "skip entry=%s", globbuf.gl_pathv[i]); + else if (recursive_search_flag == FLB_FALSE && + (S_ISREG(fs_entry_metadata.st_mode) || + S_ISLNK(fs_entry_metadata.st_mode))) { + result = blob_file_append(ctx, + glob_context.gl_pathv[index], + &fs_entry_metadata); + + if (result == 0) { + flb_plg_debug(ctx->ins, + "blob scan add: %s, inode %" PRIu64, + glob_context.gl_pathv[index], + (uint64_t) fs_entry_metadata.st_ino); + } + else { + flb_plg_debug(ctx->ins, + "blob scan skip: %s", + glob_context.gl_pathv[index]); + } + + match_count++; } } - globfree(&globbuf); - return count; + globfree(&glob_context); + + cfl_sds_destroy(local_pattern); + cfl_sds_destroy(local_path); + + return match_count; } static int cb_scan_path(struct flb_input_instance *ins, struct flb_config *config, void *in_context) { - struct blob_ctx *ctx = in_context; + ssize_t result; + struct blob_ctx *ctx;; + + ctx = (struct blob_ctx *) in_context; + + flb_plg_info(ctx->ins, "scanning path %s", ctx->path); - return scan_path(ctx); + result = recursive_file_search(ctx, NULL, ctx->path); + + if (result < 0) { + flb_plg_trace(ctx->ins, + "path scanning returned error code : %zd", + result); + } + + return 0; } /* Initialize plugin */ From ee429c7a12de49374a3f818192d2478a5c295b06 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 01:26:13 +0200 Subject: [PATCH 26/70] config: added plugin notification channel Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_config.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/include/fluent-bit/flb_config.h b/include/fluent-bit/flb_config.h index 82003386d6e..1cfc6301ff8 100644 --- a/include/fluent-bit/flb_config.h +++ b/include/fluent-bit/flb_config.h @@ -48,11 +48,11 @@ struct flb_config { int is_running; /* service running ? */ double flush; /* Flush timeout */ - /* - * Maximum grace time on shutdown. If set to -1, the engine will + /* + * Maximum grace time on shutdown. If set to -1, the engine will * shutdown when all remaining tasks are flushed */ - int grace; + int grace; int grace_count; /* Count of grace shutdown tries */ flb_pipefd_t flush_fd; /* Timer FD associated to flush */ int convert_nan_to_null; /* convert null to nan ? */ @@ -71,6 +71,10 @@ struct flb_config { flb_pipefd_t ch_self_events[2]; /* channel to recieve thread tasks */ + int notification_channels_initialized; + flb_pipefd_t notification_channels[2]; + struct mk_event notification_event; + /* Channel event loop (just for ch_notif) */ struct mk_event_loop *ch_evl; From 5d98dc54b264907994ac25b6da94cbc44fbe0f09 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 01:26:31 +0200 Subject: [PATCH 27/70] engine_macros: added plugin notification event type Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_engine_macros.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/fluent-bit/flb_engine_macros.h b/include/fluent-bit/flb_engine_macros.h index 8b79634f7d2..348387e3386 100644 --- a/include/fluent-bit/flb_engine_macros.h +++ b/include/fluent-bit/flb_engine_macros.h @@ -39,6 +39,8 @@ #define FLB_ENGINE_EV_THREAD_OUTPUT (1 << 17) /* 131072 */ #define FLB_ENGINE_EV_THREAD_ENGINE (1 << 18) /* 262144 */ +#define FLB_ENGINE_EV_NOTIFICATION (1 << 19) /* 524288 */ + /* Engine events: all engine events set the left 32 bits to '1' */ #define FLB_ENGINE_EV_STARTED FLB_BITS_U64_SET(1, 1) /* Engine started */ #define FLB_ENGINE_EV_FAILED FLB_BITS_U64_SET(1, 2) /* Engine started */ From 2141b5b129c188d6493dfe7e95a812bb81f02439 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 01:27:02 +0200 Subject: [PATCH 28/70] filter: added plugin notification feature Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_filter.h | 4 ++++ src/flb_filter.c | 3 +++ 2 files changed, 7 insertions(+) diff --git a/include/fluent-bit/flb_filter.h b/include/fluent-bit/flb_filter.h index 1b6c2ee7b16..a7e71e0e933 100644 --- a/include/fluent-bit/flb_filter.h +++ b/include/fluent-bit/flb_filter.h @@ -74,6 +74,9 @@ struct flb_filter_plugin { void *, struct flb_config *); int (*cb_exit) (void *, struct flb_config *); + /* Notification: this callback will be invoked anytime a notification is received*/ + int (*cb_notification) (struct flb_filter_instance *, struct flb_config *, void *); + struct mk_list _head; /* Link to parent list (config->filters) */ }; @@ -110,6 +113,7 @@ struct flb_filter_instance { #ifdef FLB_HAVE_METRICS struct flb_metrics *metrics; /* metrics */ #endif + flb_pipefd_t notification_channel; /* Keep a reference to the original context this instance belongs to */ struct flb_config *config; diff --git a/src/flb_filter.c b/src/flb_filter.c index e344336dbad..83b2a1b3328 100644 --- a/src/flb_filter.c +++ b/src/flb_filter.c @@ -614,6 +614,9 @@ int flb_filter_init_all(struct flb_config *config) flb_filter_instance_destroy(ins); return -1; } + + ins->notification_channel = \ + config->notification_channels[1]; } return 0; From dbd0e9bb1c6ced9c452d0cb85a5f8f459b43348c Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 01:27:16 +0200 Subject: [PATCH 29/70] input: added plugin notification feature Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_input.h | 6 ++++++ src/flb_input.c | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/include/fluent-bit/flb_input.h b/include/fluent-bit/flb_input.h index 094766a09ad..fa209b2bef5 100644 --- a/include/fluent-bit/flb_input.h +++ b/include/fluent-bit/flb_input.h @@ -102,6 +102,9 @@ struct flb_input_plugin { /* Collect: every certain amount of time, Fluent Bit trigger this callback */ int (*cb_collect) (struct flb_input_instance *, struct flb_config *, void *); + /* Notification: this callback will be invoked anytime a notification is received*/ + int (*cb_notification) (struct flb_input_instance *, struct flb_config *, void *); + /* * Flush: each plugin during a collection, it does some buffering, * when the Flush timer takes place on the Engine, it will trigger @@ -381,6 +384,8 @@ struct flb_input_instance { struct mk_list *net_config_map; struct mk_list net_properties; + flb_pipefd_t notification_channel; + /* Keep a reference to the original context this instance belongs to */ struct flb_config *config; }; @@ -733,6 +738,7 @@ int flb_input_upstream_set(struct flb_upstream *u, struct flb_input_instance *in int flb_input_downstream_set(struct flb_downstream *stream, struct flb_input_instance *ins); + /* processors */ int flb_input_instance_processors_load(struct flb_input_instance *ins, struct flb_cf_group *processors); diff --git a/src/flb_input.c b/src/flb_input.c index 7b7390b40e8..8b147b4d5ca 100644 --- a/src/flb_input.c +++ b/src/flb_input.c @@ -808,6 +808,7 @@ void flb_input_instance_destroy(struct flb_input_instance *ins) if (ins->processor) { flb_processor_destroy(ins->processor); } + flb_free(ins); } @@ -1201,6 +1202,8 @@ int flb_input_instance_init(struct flb_input_instance *ins, return -1; } + //ins->notification_channel = ins->thi->notification_channels[1]; + /* register the ring buffer */ ret = flb_ring_buffer_add_event_loop(ins->rb, config->evl, FLB_INPUT_RING_BUFFER_WINDOW); if (ret) { @@ -1216,6 +1219,9 @@ int flb_input_instance_init(struct flb_input_instance *ins, flb_error("failed initialize channel events on input %s", ins->name); } + + ins->notification_channel = config->notification_channels[1]; + ret = p->cb_init(ins, config, ins->data); if (ret != 0) { flb_error("failed initialize input %s", @@ -1225,6 +1231,8 @@ int flb_input_instance_init(struct flb_input_instance *ins, } } + ins->processor->notification_channel = ins->notification_channel; + /* initialize processors */ ret = flb_processor_init(ins->processor); if (ret == -1) { @@ -2016,3 +2024,8 @@ int flb_input_downstream_set(struct flb_downstream *stream, return 0; } + +int flb_input_handle_notification(struct flb_input_instance *ins) +{ + return 0; +} From c4892699e548519646cb9cd0e954f62e688b0d75 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 01:27:29 +0200 Subject: [PATCH 30/70] input: added plugin notification feature Signed-off-by: Leonardo Alminana --- src/flb_input_thread.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/flb_input_thread.c b/src/flb_input_thread.c index bf073296de2..43eb1ec905e 100644 --- a/src/flb_input_thread.c +++ b/src/flb_input_thread.c @@ -31,6 +31,7 @@ #include #include #include +#include static int input_thread_instance_set_status(struct flb_input_instance *ins, uint32_t status); static int input_thread_instance_get_status(struct flb_input_instance *ins); @@ -208,6 +209,15 @@ static void input_thread_instance_destroy(struct flb_input_thread_instance *thi) mk_event_closesocket(thi->ch_thread_events[1]); } + if (thi->notification_channels_initialized == FLB_TRUE) { + mk_event_channel_destroy(thi->evl, + thi->notification_channels[0], + thi->notification_channels[1], + &thi->notification_event); + + thi->notification_channels_initialized = FLB_FALSE; + } + flb_tp_destroy(thi->tp); flb_free(thi); } @@ -270,6 +280,24 @@ static struct flb_input_thread_instance *input_thread_instance_create(struct flb } thi->event_local.type = FLB_ENGINE_EV_THREAD_INPUT; + ret = mk_event_channel_create(thi->evl, + &thi->notification_channels[0], + &thi->notification_channels[1], + &thi->notification_event); + if (ret == -1) { + flb_error("could not create notification channel for %s", + flb_input_name(ins)); + + input_thread_instance_destroy(thi); + + return NULL; + } + + thi->notification_channels_initialized = FLB_TRUE; + thi->notification_event.type = FLB_ENGINE_EV_NOTIFICATION; + + ins->notification_channel = thi->notification_channels[1]; + /* create thread pool, just one worker */ thi->tp = flb_tp_create(ins->config); if (!thi->tp) { @@ -296,6 +324,7 @@ static void input_thread(void *data) struct flb_input_plugin *p; struct flb_sched *sched = NULL; struct flb_net_dns dns_ctx = {0}; + struct flb_notification *notification; thi = (struct flb_input_thread_instance *) data; ins = thi->ins; @@ -432,6 +461,15 @@ static void input_thread(void *data) else if (event->type == FLB_ENGINE_EV_THREAD_INPUT) { handle_input_thread_event(event->fd, ins->config); } + else if(event->type == FLB_ENGINE_EV_NOTIFICATION) { + ret = flb_notification_receive(event->fd, ¬ification); + + if (ret == 0) { + ret = flb_notification_deliver(notification); + + flb_notification_cleanup(notification); + } + } } flb_net_dns_lookup_context_cleanup(&dns_ctx); From 5ec1810c657825f036b3b62bf73220dcd2fd51dd Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 01:27:47 +0200 Subject: [PATCH 31/70] output: added plugin notification feature Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_output.h | 5 +++ include/fluent-bit/flb_output_thread.h | 3 ++ src/flb_output.c | 8 +++-- src/flb_output_thread.c | 46 ++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/include/fluent-bit/flb_output.h b/include/fluent-bit/flb_output.h index dd928450c99..bf93381b789 100644 --- a/include/fluent-bit/flb_output.h +++ b/include/fluent-bit/flb_output.h @@ -235,6 +235,9 @@ struct flb_output_plugin { int (*cb_worker_init) (void *, struct flb_config *); + /* Notification: this callback will be invoked anytime a notification is received*/ + int (*cb_notification) (struct flb_output_instance *, struct flb_config *, void *); + /* Tests */ struct flb_test_out_formatter test_formatter; @@ -448,6 +451,8 @@ struct flb_output_instance { struct mk_list flush_list; struct mk_list flush_list_destroy; + flb_pipefd_t notification_channel; + /* Keep a reference to the original context this instance belongs to */ struct flb_config *config; }; diff --git a/include/fluent-bit/flb_output_thread.h b/include/fluent-bit/flb_output_thread.h index 2bf4ca11d77..51cc20465fd 100644 --- a/include/fluent-bit/flb_output_thread.h +++ b/include/fluent-bit/flb_output_thread.h @@ -63,6 +63,9 @@ struct flb_out_thread_instance { struct flb_bucket_queue *evl_bktq; /* bucket queue for evl track event priority */ flb_pipefd_t ch_parent_events[2]; /* channel to receive parent notifications */ flb_pipefd_t ch_thread_events[2]; /* channel to send messages local event loop */ + int notification_channels_initialized; + flb_pipefd_t notification_channels[2]; + struct mk_event notification_event; struct flb_output_instance *ins; /* output plugin instance */ struct flb_config *config; struct flb_tp_thread *th; diff --git a/src/flb_output.c b/src/flb_output.c index 0c9a7cc1333..365708e10ea 100644 --- a/src/flb_output.c +++ b/src/flb_output.c @@ -279,7 +279,7 @@ int flb_output_task_singleplex_enqueue(struct flb_task_queue *queue, if (is_empty) { return flb_output_task_queue_flush_one(out_ins->singleplex_queue); } - + return 0; } @@ -300,7 +300,7 @@ int flb_output_task_singleplex_flush_next(struct flb_task_queue *queue) mk_list_del(&ended_task->_head); flb_free(ended_task); } - + /* Flush if there is a pending task queued */ is_empty = mk_list_is_empty(&queue->pending) == 0; if (!is_empty) { @@ -1311,6 +1311,8 @@ int flb_output_init_all(struct flb_config *config) return -1; } + ins->notification_channel = config->notification_channels[1]; + /* Multi-threading enabled if configured */ ret = flb_output_enable_multi_threading(ins, config); if (ret == -1) { @@ -1319,6 +1321,8 @@ int flb_output_init_all(struct flb_config *config) return -1; } + ins->processor->notification_channel = ins->notification_channel; + /* initialize processors */ ret = flb_processor_init(ins->processor); if (ret == -1) { diff --git a/src/flb_output_thread.c b/src/flb_output_thread.c index 4baa5181b30..330ff4feeec 100644 --- a/src/flb_output_thread.c +++ b/src/flb_output_thread.c @@ -25,6 +25,7 @@ #include #include #include +#include static pthread_once_t local_thread_instance_init = PTHREAD_ONCE_INIT; FLB_TLS_DEFINE(struct flb_out_thread_instance, local_thread_instance); @@ -182,6 +183,7 @@ static void output_thread(void *data) struct flb_out_thread_instance *th_ins = data; struct flb_out_flush_params *params; struct flb_net_dns dns_ctx; + struct flb_notification *notification; /* Register thread instance */ flb_output_thread_instance_set(th_ins); @@ -326,6 +328,15 @@ static void output_thread(void *data) */ handle_output_event(th_ins->config, ins->ch_events[1], event->fd); } + else if(event->type == FLB_ENGINE_EV_NOTIFICATION) { + ret = flb_notification_receive(event->fd, ¬ification); + + if (ret == 0) { + ret = flb_notification_deliver(notification); + + flb_notification_cleanup(notification); + } + } else { flb_plg_warn(ins, "unhandled event type => %i\n", event->type); } @@ -379,6 +390,15 @@ static void output_thread(void *data) th_ins->ch_parent_events[1], th_ins); + if (th_ins->notification_channels_initialized == FLB_TRUE) { + mk_event_channel_destroy(th_ins->evl, + th_ins->notification_channels[0], + th_ins->notification_channels[1], + &th_ins->notification_event); + + th_ins->notification_channels_initialized = FLB_FALSE; + } + mk_event_loop_destroy(th_ins->evl); flb_bucket_queue_destroy(th_ins->evl_bktq); @@ -499,6 +519,32 @@ int flb_output_thread_pool_create(struct flb_config *config, th_ins->event.type = FLB_ENGINE_EV_THREAD_OUTPUT; th_ins->event.priority = FLB_ENGINE_PRIORITY_THREAD; + if (i == 0) { + ret = mk_event_channel_create(th_ins->evl, + &th_ins->notification_channels[0], + &th_ins->notification_channels[1], + &th_ins->notification_event); + if (ret == -1) { + flb_plg_error(th_ins->ins, "could not create notification channel"); + + mk_event_channel_destroy(th_ins->evl, + th_ins->ch_parent_events[0], + th_ins->ch_parent_events[1], + th_ins); + + mk_event_loop_destroy(th_ins->evl); + flb_bucket_queue_destroy(th_ins->evl_bktq); + flb_free(th_ins); + + continue; + } + + th_ins->notification_channels_initialized = FLB_TRUE; + th_ins->notification_event.type = FLB_ENGINE_EV_NOTIFICATION; + + ins->notification_channel = th_ins->notification_channels[1]; + } + /* Spawn the thread */ th = flb_tp_thread_create(tp, output_thread, th_ins, config); if (!th) { From c812d4f363b8373a850544db9107bdd741b6b737 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 01:28:00 +0200 Subject: [PATCH 32/70] input: added plugin notification feature Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_input_thread.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/fluent-bit/flb_input_thread.h b/include/fluent-bit/flb_input_thread.h index abc16123106..4abfc1511ea 100644 --- a/include/fluent-bit/flb_input_thread.h +++ b/include/fluent-bit/flb_input_thread.h @@ -64,6 +64,9 @@ struct flb_input_thread_instance { struct mk_event_loop *evl; /* thread event loop context */ flb_pipefd_t ch_parent_events[2]; /* communication between parent and thread */ flb_pipefd_t ch_thread_events[2]; /* local messages in the thread event loop */ + int notification_channels_initialized; + flb_pipefd_t notification_channels[2]; + struct mk_event notification_event; struct flb_input_instance *ins; /* output plugin instance */ struct flb_tp *tp; struct flb_tp_thread *th; From 6e7e1cabc5dab38c6f983b811a64a3b663e1a27f Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 01:28:31 +0200 Subject: [PATCH 33/70] processor: added plugin notification feature Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_processor.h | 7 +++++++ src/flb_processor.c | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/include/fluent-bit/flb_processor.h b/include/fluent-bit/flb_processor.h index c4abd7b43a0..bc7a9338371 100644 --- a/include/fluent-bit/flb_processor.h +++ b/include/fluent-bit/flb_processor.h @@ -119,6 +119,8 @@ struct flb_processor { void *data; int source_plugin_type; + flb_pipefd_t notification_channel; + /* Fluent Bit context */ struct flb_config *config; }; @@ -155,6 +157,9 @@ struct flb_processor_plugin { int (*cb_exit) (struct flb_processor_instance *, void *); + /* Notification: this callback will be invoked anytime a notification is received*/ + int (*cb_notification) (struct flb_processor_instance *, struct flb_config *, void *); + struct mk_list _head; /* Link to parent list (config->filters) */ }; @@ -179,6 +184,8 @@ struct flb_processor_instance { */ struct cmt *cmt; /* parent context */ + flb_pipefd_t notification_channel; + /* Keep a reference to the original context this instance belongs to */ struct flb_config *config; }; diff --git a/src/flb_processor.c b/src/flb_processor.c index a756940861a..00c892491f4 100644 --- a/src/flb_processor.c +++ b/src/flb_processor.c @@ -342,6 +342,9 @@ int flb_processor_unit_init(struct flb_processor_unit *pu) flb_error("[processor] error initializing unit filter %s", pu->name); return -1; } + + ((struct flb_filter_instance *) pu->ctx)->notification_channel = \ + proc->notification_channel; } else { ret = flb_processor_instance_init( @@ -356,6 +359,9 @@ int flb_processor_unit_init(struct flb_processor_unit *pu) return -1; } + + ((struct flb_processor_instance *) pu->ctx)->notification_channel = \ + proc->notification_channel; } return ret; From 594fe474b3ec86b39ac078219787fc9bfdf22a91 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 01:28:50 +0200 Subject: [PATCH 34/70] build: added plugin notification feature Signed-off-by: Leonardo Alminana --- src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ea7403f7b6c..e24666484a7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -81,6 +81,7 @@ set(src flb_processor.c flb_reload.c flb_msgpack_append_message.c + flb_notification.c ) # Config format From d08ce1f1ab90d1e7429c4f3cba2bae6b4413a8c5 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 01:29:09 +0200 Subject: [PATCH 35/70] engine: added plugin notification feature Signed-off-by: Leonardo Alminana --- src/flb_engine.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/flb_engine.c b/src/flb_engine.c index d85c208531c..475495c03cf 100644 --- a/src/flb_engine.c +++ b/src/flb_engine.c @@ -692,6 +692,7 @@ int flb_engine_start(struct flb_config *config) struct flb_bucket_queue *evl_bktq; struct flb_sched *sched; struct flb_net_dns dns_ctx; + struct flb_notification *notification; /* Initialize the networking layer */ flb_net_lib_init(); @@ -771,6 +772,19 @@ int flb_engine_start(struct flb_config *config) return -1; } + ret = mk_event_channel_create(config->evl, + &config->notification_channels[0], + &config->notification_channels[1], + &config->notification_event); + if (ret == -1) { + flb_error("could not create main notification channel"); + + return -1; + } + + config->notification_channels_initialized = FLB_TRUE; + config->notification_event.type = FLB_ENGINE_EV_NOTIFICATION; + /* Initialize custom plugins */ ret = flb_custom_init_all(config); if (ret == -1) { @@ -1072,6 +1086,15 @@ int flb_engine_start(struct flb_config *config) rb_flush_flag = FLB_TRUE; } + else if(event->type == FLB_ENGINE_EV_NOTIFICATION) { + ret = flb_notification_receive(event->fd, ¬ification); + + if (ret == 0) { + ret = flb_notification_deliver(notification); + + flb_notification_cleanup(notification); + } + } } if (rb_flush_flag) { @@ -1142,6 +1165,15 @@ int flb_engine_shutdown(struct flb_config *config) &config->event_thread_init); } + if (config->notification_channels_initialized == FLB_TRUE) { + mk_event_channel_destroy(config->evl, + config->notification_channels[0], + config->notification_channels[1], + &config->notification_event); + + config->notification_channels_initialized = FLB_FALSE; + } + return 0; } From 689883be6158315b76d50cbd365014091c2daec1 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 01:29:40 +0200 Subject: [PATCH 36/70] input_blob: added delivery notification structure and cleanup function Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_input_blob.h | 10 ++++++++++ src/flb_input_blob.c | 11 +++++++++++ 2 files changed, 21 insertions(+) diff --git a/include/fluent-bit/flb_input_blob.h b/include/fluent-bit/flb_input_blob.h index 0691b8f29aa..4d66ab1d624 100644 --- a/include/fluent-bit/flb_input_blob.h +++ b/include/fluent-bit/flb_input_blob.h @@ -22,12 +22,22 @@ #include #include +#include #include +struct flb_blob_delivery_notification { + struct flb_notification base; + cfl_sds_t path; + int success; +}; + + struct flb_blob_file { cfl_sds_t path; }; +void flb_input_blob_delivery_notification_destroy(void *instance); + int flb_input_blob_file_get_info(msgpack_object map, cfl_sds_t *source, cfl_sds_t *file_path, size_t *size); int flb_input_blob_file_register(struct flb_input_instance *ins, diff --git a/src/flb_input_blob.c b/src/flb_input_blob.c index f0958c215d0..2e74e07fe40 100644 --- a/src/flb_input_blob.c +++ b/src/flb_input_blob.c @@ -215,3 +215,14 @@ int flb_input_blob_file_register(struct flb_input_instance *ins, flb_log_event_encoder_reset(encoder); return ret; } + +void flb_input_blob_delivery_notification_destroy(void *instance) +{ + struct flb_blob_delivery_notification *local_instance; + + local_instance = (struct flb_blob_delivery_notification *) instance; + + if (local_instance->path != NULL) { + cfl_sds_destroy(local_instance->path); + } +} From aee897e653c3f6c5aca9acbf5b04c5307cb4d2e7 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 01:30:00 +0200 Subject: [PATCH 37/70] in_blob: added notification handler Signed-off-by: Leonardo Alminana --- plugins/in_blob/blob.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/plugins/in_blob/blob.c b/plugins/in_blob/blob.c index 895d08c4f29..967a0bfa6fc 100644 --- a/plugins/in_blob/blob.c +++ b/plugins/in_blob/blob.c @@ -402,7 +402,7 @@ static int cb_scan_path(struct flb_input_instance *ins, ctx = (struct blob_ctx *) in_context; - flb_plg_info(ctx->ins, "scanning path %s", ctx->path); + flb_plg_debug(ctx->ins, "scanning path %s", ctx->path); result = recursive_file_search(ctx, NULL, ctx->path); @@ -519,14 +519,25 @@ static struct flb_config_map config_map[] = { {0} }; +static int in_blob_notification(struct flb_input_instance *in_context, + struct flb_config *config, + void *notification) +{ + printf("INPUT PLUGIN : NOTIFICATION RECEIVED %p\n", notification); + return 0; +} + + + /* Plugin reference */ struct flb_input_plugin in_blob_plugin = { - .name = "blob", - .description = "Blob (binary) files", - .cb_init = in_blob_init, - .cb_pre_run = NULL, - .cb_collect = NULL, - .cb_flush_buf = NULL, - .cb_exit = in_blob_exit, - .config_map = config_map + .name = "blob", + .description = "Blob (binary) files", + .cb_init = in_blob_init, + .cb_pre_run = NULL, + .cb_collect = NULL, + .cb_flush_buf = NULL, + .cb_exit = in_blob_exit, + .cb_notification = in_blob_notification, + .config_map = config_map }; From 111a45e049d157b4d6dc2298d1dd58c77921e50d Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 01:30:28 +0200 Subject: [PATCH 38/70] out_azure_blob: added delivery notification feature Signed-off-by: Leonardo Alminana --- plugins/out_azure_blob/azure_blob.c | 54 +++++++++++++++++++++++++- plugins/out_azure_blob/azure_blob_db.c | 14 ++++++- plugins/out_azure_blob/azure_blob_db.h | 4 +- 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index 7817b11a3a0..435e620fff4 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -28,6 +28,8 @@ #include #include #include +#include +#include #include @@ -709,6 +711,7 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context cfl_sds_t source = NULL; struct flb_azure_blob *ctx = out_context; struct worker_info *info; + struct flb_blob_delivery_notification *notification; info = FLB_TLS_GET(worker_info); if (info->active_upload) { @@ -747,9 +750,12 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context cfl_sds_destroy(file_path); cfl_sds_destroy(source); + + file_path = NULL; + source = NULL; } - ret = azb_db_file_oldest_ready(ctx, &file_id, &file_path, &part_ids); + ret = azb_db_file_oldest_ready(ctx, &file_id, &file_path, &part_ids, &source); if (ret == 0) { flb_plg_trace(ctx->ins, "no blob files ready to commit"); } @@ -764,12 +770,55 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context flb_plg_error(ctx->ins, "cannot commit blob file parts for file id=%" PRIu64 " path=%s", file_id, file_path); + notification = flb_calloc(1, + sizeof( + struct flb_blob_delivery_notification)); + + if (notification != NULL) { + notification->base.dynamically_allocated = FLB_TRUE; + notification->base.destructor = flb_input_blob_delivery_notification_destroy; + notification->success = FLB_TRUE; + notification->path = cfl_sds_create(file_path); + + ret = flb_notification_enqueue(FLB_PLUGIN_INPUT, + source, + ¬ification->base, + config); + + if (ret != 0) { + flb_plg_error(ctx->ins, + "blob file '%s' (id=%" PRIu64 ") notification " \ + "delivery error %d", file_path, file_id, ret); + } + } } else { flb_plg_info(ctx->ins, "blob file '%s' (id=%" PRIu64 ") committed successfully", file_path, file_id); /* notify the engine the blob file has been processed */ /* FIXME! */ + notification = flb_calloc(1, + sizeof( + struct flb_blob_delivery_notification)); + + if (notification != NULL) { + notification->base.dynamically_allocated = FLB_TRUE; + notification->base.destructor = flb_input_blob_delivery_notification_destroy; + notification->success = FLB_FALSE; + notification->path = cfl_sds_create(file_path); + + ret = flb_notification_enqueue(FLB_PLUGIN_INPUT, + source, + ¬ification->base, + config); + + if (ret != 0) { + flb_plg_error(ctx->ins, + "blob file '%s' (id=%" PRIu64 ") notification " \ + "delivery error %d", file_path, file_id, ret); + } + } + /* remove the file entry from the database */ ret = azb_db_file_delete(ctx, file_id, file_path); if (ret == -1) { @@ -786,6 +835,9 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context if (part_ids) { cfl_sds_destroy(part_ids); } + if (source) { + cfl_sds_destroy(source); + } /* check for a next part file and lock it */ ret = azb_db_file_part_get_next(ctx, &id, &file_id, &part_id, diff --git a/plugins/out_azure_blob/azure_blob_db.c b/plugins/out_azure_blob/azure_blob_db.c index b218a3b9dfc..586df720744 100644 --- a/plugins/out_azure_blob/azure_blob_db.c +++ b/plugins/out_azure_blob/azure_blob_db.c @@ -693,7 +693,7 @@ int azb_db_file_part_delivery_attempts(struct flb_azure_blob *ctx, } int azb_db_file_oldest_ready(struct flb_azure_blob *ctx, - uint64_t *file_id, cfl_sds_t *path, cfl_sds_t *part_ids) + uint64_t *file_id, cfl_sds_t *path, cfl_sds_t *part_ids, cfl_sds_t *source) { int ret; char *tmp = NULL; @@ -726,6 +726,18 @@ int azb_db_file_oldest_ready(struct flb_azure_blob *ctx, azb_db_unlock(ctx); return -1; } + + /* source */ + tmp = (char *) sqlite3_column_text(ctx->stmt_get_oldest_file_with_parts, 3); + *source = cfl_sds_create(tmp); + if (!*part_ids) { + cfl_sds_destroy(*part_ids); + cfl_sds_destroy(*path); + sqlite3_clear_bindings(ctx->stmt_get_oldest_file_with_parts); + sqlite3_reset(ctx->stmt_get_oldest_file_with_parts); + azb_db_unlock(ctx); + return -1; + } } else if (ret == SQLITE_DONE) { /* no records */ diff --git a/plugins/out_azure_blob/azure_blob_db.h b/plugins/out_azure_blob/azure_blob_db.h index 1009d08733c..0cd682f1b33 100644 --- a/plugins/out_azure_blob/azure_blob_db.h +++ b/plugins/out_azure_blob/azure_blob_db.h @@ -127,7 +127,7 @@ * this query is used to compose */ #define SQL_GET_OLDEST_FILE_WITH_PARTS_CONCAT \ - "SELECT f.id, f.path, GROUP_CONCAT(p.part_id ORDER BY p.part_id ASC) AS part_ids " \ + "SELECT f.id, f.path, GROUP_CONCAT(p.part_id ORDER BY p.part_id ASC) AS part_ids, f.source " \ "FROM out_azure_blob_files f " \ "JOIN out_azure_blob_parts p ON f.id = p.file_id " \ "WHERE p.uploaded = 1 " \ @@ -168,5 +168,5 @@ int azb_db_file_part_uploaded(struct flb_azure_blob *ctx, uint64_t id); int azb_db_file_part_delivery_attempts(struct flb_azure_blob *ctx, uint64_t id, uint64_t attempts); int azb_db_file_oldest_ready(struct flb_azure_blob *ctx, - uint64_t *file_id, cfl_sds_t *path, cfl_sds_t *part_ids); + uint64_t *file_id, cfl_sds_t *path, cfl_sds_t *part_ids, cfl_sds_t *source); #endif \ No newline at end of file From 249fa6e8903b7ea1cebde18ebd2d572ec9496182 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 01:30:55 +0200 Subject: [PATCH 39/70] notification: initial commit of the notification feature Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_notification.h | 49 ++++ src/flb_notification.c | 381 ++++++++++++++++++++++++++ 2 files changed, 430 insertions(+) create mode 100644 include/fluent-bit/flb_notification.h create mode 100644 src/flb_notification.c diff --git a/include/fluent-bit/flb_notification.h b/include/fluent-bit/flb_notification.h new file mode 100644 index 00000000000..539a557199c --- /dev/null +++ b/include/fluent-bit/flb_notification.h @@ -0,0 +1,49 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_NOTIFICATION_H +#define FLB_NOTIFICATION_H + +#include +#include +#include +#include +#include + +struct flb_notification { + int plugin_type; + void *plugin_instance; + int dynamically_allocated; + + void (*destructor)(void *); +}; + +int flb_notification_enqueue(int plugin_type, + char *instance_name, + struct flb_notification *notification, + struct flb_config *config); + +int flb_notification_receive(flb_pipefd_t channel, + struct flb_notification **notification); + +int flb_notification_deliver(struct flb_notification *notification); + +void flb_notification_cleanup(struct flb_notification *notification); + +#endif diff --git a/src/flb_notification.c b/src/flb_notification.c new file mode 100644 index 00000000000..4d546dea9b1 --- /dev/null +++ b/src/flb_notification.c @@ -0,0 +1,381 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +union generic_plugin_instance { + struct flb_input_instance *input; + struct flb_output_instance *output; + struct flb_filter_instance *filter; + struct flb_processor_instance *processor; + void *generic; +}; + +static struct flb_input_instance *find_input_instance( + char *name, + struct flb_config *config) +{ + struct mk_list *iterator; + struct flb_input_instance *instance; + + mk_list_foreach(iterator, &config->inputs) { + instance = mk_list_entry(iterator, + struct flb_input_instance, + _head); + + if (strcmp(flb_input_name(instance), name) == 0) { + return instance; + } + } + + return NULL; +} + +static struct flb_output_instance *find_output_instance( + char *name, + struct flb_config *config) +{ + struct mk_list *iterator; + struct flb_output_instance *instance; + + mk_list_foreach(iterator, &config->outputs) { + instance = mk_list_entry(iterator, + struct flb_output_instance, + _head); + + if (strcmp(flb_output_name(instance), name) == 0) { + return instance; + } + } + + return NULL; +} + +static struct flb_filter_instance *find_filter_instance( + char *name, + struct flb_config *config) +{ + struct mk_list *iterator; + struct flb_filter_instance *instance; + + mk_list_foreach(iterator, &config->filters) { + instance = mk_list_entry(iterator, + struct flb_filter_instance, + _head); + + if (strcmp(flb_filter_name(instance), name) == 0) { + return instance; + } + } + + return NULL; +} + +static void *find_processor_instance_internal_unit_level( + char *name, + int *plugin_type, + struct mk_list *processor_unit_list) +{ + struct mk_list *iterator; + struct flb_processor_unit *processor_unit; + struct flb_filter_instance *filter_instance; + struct flb_processor_instance *processor_instance; + + mk_list_foreach(iterator, processor_unit_list) { + processor_unit = mk_list_entry(iterator, + struct flb_processor_unit, + _head); + + if (processor_unit->unit_type == FLB_PROCESSOR_UNIT_FILTER) { + filter_instance = (struct flb_filter_instance *) \ + processor_unit->ctx; + + if (strcmp(flb_filter_name(filter_instance), name) == 0) { + *plugin_type = FLB_PLUGIN_FILTER; + + return (void *) filter_instance; + } + } + else if (processor_unit->unit_type == FLB_PROCESSOR_UNIT_NATIVE) { + processor_instance = (struct flb_processor_instance *) \ + processor_unit->ctx; + + if (strcmp(flb_processor_instance_get_name(processor_instance), + name) == 0) { + *plugin_type = FLB_PLUGIN_PROCESSOR; + + return (void *) processor_instance; + } + } + } + + return NULL; +} + + +static void *find_processor_instance_internal_processor_level( + char *name, + int *plugin_type, + struct flb_processor *processor) +{ + void *result; + + result = find_processor_instance_internal_unit_level( + name, + plugin_type, + &processor->logs); + + if (result == NULL) { + result = find_processor_instance_internal_unit_level( + name, + plugin_type, + &processor->metrics); + } + + if (result == NULL) { + result = find_processor_instance_internal_unit_level( + name, + plugin_type, + &processor->traces); + } + + return result; +} + +static void *find_processor_instance( + char *name, + int *plugin_type, + struct flb_config *config) +{ + struct flb_output_instance *output_instance; + struct flb_input_instance *input_instance; + struct mk_list *iterator; + void *result; + + mk_list_foreach(iterator, &config->inputs) { + input_instance = mk_list_entry(iterator, + struct flb_input_instance, + _head); + + result = find_processor_instance_internal_processor_level( + name, + plugin_type, + input_instance->processor); + + if (result != NULL) { + return result; + } + } + + mk_list_foreach(iterator, &config->outputs) { + output_instance = mk_list_entry(iterator, + struct flb_output_instance, + _head); + + result = find_processor_instance_internal_processor_level( + name, + plugin_type, + output_instance->processor); + + if (result != NULL) { + return result; + } + } + + return NULL; +} + +int flb_notification_enqueue(int plugin_type, + char *instance_name, + struct flb_notification *notification, + struct flb_config *config) +{ + flb_pipefd_t notification_channel; + union generic_plugin_instance plugin_instance; + int result; + + plugin_instance.generic = NULL; + + if (plugin_instance.generic == NULL && + (plugin_type == FLB_PLUGIN_INPUT || + plugin_type == -1)) { + plugin_instance.input = find_input_instance(instance_name, config); + notification_channel = plugin_instance.input->notification_channel; + plugin_type = FLB_PLUGIN_INPUT; + } + + if (plugin_instance.generic == NULL && + (plugin_type == FLB_PLUGIN_OUTPUT || + plugin_type == -1)) { + plugin_instance.output = find_output_instance(instance_name, config); + notification_channel = plugin_instance.output->notification_channel; + plugin_type = FLB_PLUGIN_OUTPUT; + } + + if (plugin_instance.generic == NULL && + (plugin_type == FLB_PLUGIN_FILTER || + plugin_type == -1)) { + plugin_instance.filter = find_filter_instance(instance_name, config); + notification_channel = plugin_instance.filter->notification_channel; + plugin_type = FLB_PLUGIN_FILTER; + } + + if (plugin_instance.generic == NULL && + (plugin_type == FLB_PLUGIN_FILTER || + plugin_type == -1)) { + plugin_instance.generic = find_processor_instance(instance_name, + &plugin_type, + config); + + if (plugin_instance.generic != NULL) { + if (plugin_type == FLB_PLUGIN_FILTER) { + notification_channel = plugin_instance.filter->notification_channel; + } + else if (plugin_type == FLB_PLUGIN_PROCESSOR) { + notification_channel = plugin_instance.processor->notification_channel; + } + } + } + + if (plugin_instance.generic == NULL) { + flb_error("cannot enqueue notification for plugin \"%s\" with type %d", + instance_name, plugin_type); + + return -1; + } + + notification->plugin_type = plugin_type; + notification->plugin_instance = plugin_instance.generic; + + result = flb_pipe_w(notification_channel, + ¬ification, + sizeof(void *)); + + if (result == -1) { + flb_errno(); + + return -1; + } + + return 0; +} + +int flb_notification_receive(flb_pipefd_t channel, + struct flb_notification **notification) +{ + int result; + + result = flb_pipe_r(channel, notification, sizeof(struct flb_notification *)); + + if (result <= 0) { + flb_errno(); + return -1;; + } + + return 0; +} + +int flb_notification_deliver(struct flb_notification *notification) +{ + int result; + union generic_plugin_instance instance; + + if (notification == NULL) { + flb_error("cannot deliver NULL notification instance"); + + return -1; + } + + instance.generic = notification->plugin_instance; + result = -2; + + switch(notification->plugin_type) { + case FLB_PLUGIN_INPUT: + if (instance.input->p->cb_notification != NULL) { + result = instance.input->p->cb_notification( + instance.input, + instance.input->config, + (void *) notification); + } + else { + result = -3; + } + + break; + + case FLB_PLUGIN_OUTPUT: + if (instance.output->p->cb_notification != NULL) { + result = instance.output->p->cb_notification( + instance.output, + instance.output->config, + (void *) notification); + } + else { + result = -3; + } + + break; + + case FLB_PLUGIN_FILTER: + if (instance.filter->p->cb_notification != NULL) { + result = instance.filter->p->cb_notification( + instance.filter, + instance.filter->config, + (void *) notification); + } + else { + result = -3; + } + + break; + + case FLB_PLUGIN_PROCESSOR: + if (instance.processor->p->cb_notification != NULL) { + result = instance.processor->p->cb_notification( + instance.processor, + instance.processor->config, + (void *) notification); + } + else { + result = -3; + } + + break; + } + + return result; +} + +void flb_notification_cleanup(struct flb_notification *notification) +{ + if (notification->destructor != NULL) { + notification->destructor((void *) notification); + } + + if (notification->dynamically_allocated == FLB_TRUE) { + flb_free(notification); + } +} From a428eac3586ac6d005ad3e279cff89266a31c201 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 18:08:10 +0200 Subject: [PATCH 40/70] notification: added notification type field and global list Also fixed the context pointer provided by the notification callback Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_notification.h | 5 +++++ src/flb_notification.c | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/include/fluent-bit/flb_notification.h b/include/fluent-bit/flb_notification.h index 539a557199c..e634c2d30a9 100644 --- a/include/fluent-bit/flb_notification.h +++ b/include/fluent-bit/flb_notification.h @@ -26,9 +26,14 @@ #include #include + +#define FLB_NOTIFICATION_TYPE_BLOB_DELIVERY 1 + + struct flb_notification { int plugin_type; void *plugin_instance; + int notification_type; int dynamically_allocated; void (*destructor)(void *); diff --git a/src/flb_notification.c b/src/flb_notification.c index 4d546dea9b1..b6e575c4974 100644 --- a/src/flb_notification.c +++ b/src/flb_notification.c @@ -316,7 +316,7 @@ int flb_notification_deliver(struct flb_notification *notification) case FLB_PLUGIN_INPUT: if (instance.input->p->cb_notification != NULL) { result = instance.input->p->cb_notification( - instance.input, + instance.input->context, instance.input->config, (void *) notification); } @@ -329,7 +329,7 @@ int flb_notification_deliver(struct flb_notification *notification) case FLB_PLUGIN_OUTPUT: if (instance.output->p->cb_notification != NULL) { result = instance.output->p->cb_notification( - instance.output, + instance.output->context, instance.output->config, (void *) notification); } @@ -342,7 +342,7 @@ int flb_notification_deliver(struct flb_notification *notification) case FLB_PLUGIN_FILTER: if (instance.filter->p->cb_notification != NULL) { result = instance.filter->p->cb_notification( - instance.filter, + instance.filter->context, instance.filter->config, (void *) notification); } @@ -355,7 +355,7 @@ int flb_notification_deliver(struct flb_notification *notification) case FLB_PLUGIN_PROCESSOR: if (instance.processor->p->cb_notification != NULL) { result = instance.processor->p->cb_notification( - instance.processor, + instance.processor->context, instance.processor->config, (void *) notification); } From 9802a51120280ffe55c1ed953b03ab15f28ac91e Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 18:08:37 +0200 Subject: [PATCH 41/70] in_blob: added file upload result notification handling Signed-off-by: Leonardo Alminana --- plugins/in_blob/blob.c | 331 +++++++++++++++++++++++++++++++++++++++-- plugins/in_blob/blob.h | 15 ++ 2 files changed, 336 insertions(+), 10 deletions(-) diff --git a/plugins/in_blob/blob.c b/plugins/in_blob/blob.c index 967a0bfa6fc..d9d50ffa547 100644 --- a/plugins/in_blob/blob.c +++ b/plugins/in_blob/blob.c @@ -23,11 +23,14 @@ #include #include #include +#include +#include #include #include #include #include +#include #include "blob.h" #include "blob_db.h" @@ -457,6 +460,68 @@ static int in_blob_init(struct flb_input_instance *ins, } #endif + ctx->upload_success_action = POST_UPLOAD_ACTION_NONE; + + if (ctx->upload_success_action_str != NULL) { + if (strcasecmp(ctx->upload_success_action_str, + "delete") == 0) { + ctx->upload_success_action = POST_UPLOAD_ACTION_DELETE; + } + else if (strcasecmp(ctx->upload_success_action_str, + "emit_log") == 0) { + ctx->upload_success_action = POST_UPLOAD_ACTION_EMIT_LOG; + } + else if (strcasecmp(ctx->upload_success_action_str, + "add_suffix") == 0) { + if (ctx->upload_success_suffix == NULL) { + flb_plg_error(ins, + "'upload_success_suffix' configuration " \ + "property is not set"); +#ifdef FLB_HAVE_SQLDB + if (ctx->db != NULL) { + blob_db_close(ctx); + ctx->db = NULL; + } +#endif + flb_free(ctx); + + return -1; + } + ctx->upload_success_action = POST_UPLOAD_ACTION_ADD_SUFFIX; + } + } + + ctx->upload_failure_action = POST_UPLOAD_ACTION_NONE; + + if (ctx->upload_failure_action_str != NULL) { + if (strcasecmp(ctx->upload_failure_action_str, + "delete") == 0) { + ctx->upload_failure_action = POST_UPLOAD_ACTION_DELETE; + } + else if (strcasecmp(ctx->upload_failure_action_str, + "emit_log") == 0) { + ctx->upload_failure_action = POST_UPLOAD_ACTION_EMIT_LOG; + } + else if (strcasecmp(ctx->upload_failure_action_str, + "add_suffix") == 0) { + if (ctx->upload_failure_suffix == NULL) { + flb_plg_error(ins, + "'upload_failure_suffix' configuration " \ + "property is not set"); +#ifdef FLB_HAVE_SQLDB + if (ctx->db != NULL) { + blob_db_close(ctx); + ctx->db = NULL; + } +#endif + flb_free(ctx); + + return -1; + } + ctx->upload_failure_action = POST_UPLOAD_ACTION_ADD_SUFFIX; + } + } + /* create a collector to scan the path of files */ ret = flb_input_set_collector_time(ins, cb_scan_path, @@ -495,6 +560,232 @@ static int in_blob_exit(void *in_context, struct flb_config *config) return 0; } + +static int in_blob_notification(struct flb_input_instance *in_context, + struct flb_config *config, + void *untyped_notification) +{ + struct flb_blob_delivery_notification *notification; + cfl_sds_t new_filename; + cfl_sds_t sds_result; + struct blob_ctx *context; + int result; + + context = (struct blob_ctx *) in_context; + + notification = (struct flb_blob_delivery_notification *) untyped_notification; + + if (notification->base.notification_type != + FLB_NOTIFICATION_TYPE_BLOB_DELIVERY) { + flb_plg_error(context->ins, + "unexpected notification type received : %d", + notification->base.notification_type); + + return -1; + } + + if (notification->success == FLB_TRUE) { + switch (context->upload_success_action) { + case POST_UPLOAD_ACTION_DELETE: + result = unlink(notification->path); + + if (result == -1) { + flb_errno(); + + flb_plg_error(context->ins, + "successfully uploaded file \"%s\" could not be deleted", + notification->path); + } + + break; + + case POST_UPLOAD_ACTION_EMIT_LOG: + flb_log_event_encoder_begin_record(context->log_encoder); + + flb_log_event_encoder_set_current_timestamp(context->log_encoder); + + result = flb_log_event_encoder_append_metadata_values( + context->log_encoder, + FLB_LOG_EVENT_CSTRING_VALUE("path"), + FLB_LOG_EVENT_CSTRING_VALUE(notification->path)); + + if (result != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_rollback_record(context->log_encoder); + + break; + } + + result = flb_log_event_encoder_append_body_values( + context->log_encoder, + FLB_LOG_EVENT_CSTRING_VALUE("message"), + FLB_LOG_EVENT_CSTRING_VALUE(context->upload_success_message)); + + if (result != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_rollback_record(context->log_encoder); + + break; + } + + result = flb_log_event_encoder_commit_record(context->log_encoder); + + if (result != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_rollback_record(context->log_encoder); + + break; + } + + flb_input_log_append(context->ins, NULL, 0, + context->log_encoder->output_buffer, + context->log_encoder->output_length); + + flb_log_event_encoder_reset_record(context->log_encoder); + + break; + + case POST_UPLOAD_ACTION_ADD_SUFFIX: + new_filename = cfl_sds_create(notification->path); + + if (new_filename == NULL) { + flb_plg_error(context->ins, + "successfully uploaded file \"%s\" could not be renamed " \ + "(new filename buffer allocation error)", + notification->path); + + break; + } + + sds_result = cfl_sds_cat(new_filename, + context->upload_success_suffix, + strlen(context->upload_success_suffix)); + + if (sds_result == NULL) { + flb_plg_error(context->ins, + "successfully uploaded file \"%s\" could not be renamed " \ + "(filename suffix concatentation error)", + notification->path); + + break; + } + + new_filename = sds_result; + + result = rename(notification->path, new_filename); + + if (result == -1) { + flb_errno(); + + flb_plg_error(context->ins, + "successfully uploaded file \"%s\" could not be renamed " \ + "(rename operation error)", + notification->path); + } + + break; + } + } + else if (notification->success == FLB_FALSE) { + switch (context->upload_failure_action) { + case POST_UPLOAD_ACTION_DELETE: + result = unlink(notification->path); + + if (result == -1) { + flb_errno(); + + flb_plg_error(context->ins, + "failed to upload file \"%s\" could not be deleted", + notification->path); + } + + break; + + case POST_UPLOAD_ACTION_EMIT_LOG: + flb_log_event_encoder_begin_record(context->log_encoder); + + flb_log_event_encoder_set_current_timestamp(context->log_encoder); + + result = flb_log_event_encoder_append_metadata_values( + context->log_encoder, + FLB_LOG_EVENT_CSTRING_VALUE("path"), + FLB_LOG_EVENT_CSTRING_VALUE(notification->path)); + + if (result != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_rollback_record(context->log_encoder); + + break; + } + + result = flb_log_event_encoder_append_body_values( + context->log_encoder, + FLB_LOG_EVENT_CSTRING_VALUE("message"), + FLB_LOG_EVENT_CSTRING_VALUE(context->upload_failure_message)); + + if (result != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_rollback_record(context->log_encoder); + + break; + } + + result = flb_log_event_encoder_commit_record(context->log_encoder); + + if (result != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_rollback_record(context->log_encoder); + + break; + } + + flb_input_log_append(context->ins, NULL, 0, + context->log_encoder->output_buffer, + context->log_encoder->output_length); + + flb_log_event_encoder_reset_record(context->log_encoder); + + break; + + case POST_UPLOAD_ACTION_ADD_SUFFIX: + new_filename = cfl_sds_create(notification->path); + + if (new_filename == NULL) { + flb_plg_error(context->ins, + "failed to upload file \"%s\" could not be renamed " \ + "(new filename buffer allocation error)", + notification->path); + + break; + } + + sds_result = cfl_sds_cat(new_filename, + context->upload_failure_suffix, + strlen(context->upload_failure_suffix)); + + if (sds_result == NULL) { + flb_plg_error(context->ins, + "failed to upload file \"%s\" could not be renamed " \ + "(filename suffix concatentation error)", + notification->path); + + break; + } + + new_filename = sds_result; + + result = rename(notification->path, new_filename); + + if (result == -1) { + flb_errno(); + + flb_plg_error(context->ins, + "failed to upload file \"%s\" could not be renamed " \ + "(rename operation error)", + notification->path); + } + + break; + } + } + + return 0; +} + static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_STR, "path", NULL, @@ -515,19 +806,39 @@ static struct flb_config_map config_map[] = { "Set the interval time to scan for new files" }, - /* EOF */ - {0} -}; + { + FLB_CONFIG_MAP_STR, "upload_success_action", NULL, + 0, FLB_TRUE, offsetof(struct blob_ctx, upload_success_action_str), + }, -static int in_blob_notification(struct flb_input_instance *in_context, - struct flb_config *config, - void *notification) -{ - printf("INPUT PLUGIN : NOTIFICATION RECEIVED %p\n", notification); - return 0; -} + { + FLB_CONFIG_MAP_STR, "upload_success_suffix", NULL, + 0, FLB_TRUE, offsetof(struct blob_ctx, upload_success_suffix), + }, + + { + FLB_CONFIG_MAP_STR, "upload_success_message", NULL, + 0, FLB_TRUE, offsetof(struct blob_ctx, upload_success_message), + }, + + { + FLB_CONFIG_MAP_STR, "upload_failure_action", NULL, + 0, FLB_TRUE, offsetof(struct blob_ctx, upload_failure_action_str), + }, + { + FLB_CONFIG_MAP_STR, "upload_failure_suffix", NULL, + 0, FLB_TRUE, offsetof(struct blob_ctx, upload_failure_suffix), + }, + { + FLB_CONFIG_MAP_STR, "upload_failure_message", NULL, + 0, FLB_TRUE, offsetof(struct blob_ctx, upload_failure_message), + }, + + /* EOF */ + {0} +}; /* Plugin reference */ struct flb_input_plugin in_blob_plugin = { diff --git a/plugins/in_blob/blob.h b/plugins/in_blob/blob.h index 13b932e5d5e..adedce21d79 100644 --- a/plugins/in_blob/blob.h +++ b/plugins/in_blob/blob.h @@ -25,6 +25,11 @@ #include #include +#define POST_UPLOAD_ACTION_NONE 0 +#define POST_UPLOAD_ACTION_DELETE 1 +#define POST_UPLOAD_ACTION_EMIT_LOG 2 +#define POST_UPLOAD_ACTION_ADD_SUFFIX 3 + struct blob_file { /* database reference (id) */ uint64_t db_id; @@ -70,6 +75,16 @@ struct blob_ctx { flb_sds_t path; flb_sds_t database_file; time_t scan_refresh_interval; + + int upload_success_action; + flb_sds_t upload_success_action_str; + flb_sds_t upload_success_suffix; + flb_sds_t upload_success_message; + + int upload_failure_action; + flb_sds_t upload_failure_action_str; + flb_sds_t upload_failure_suffix; + flb_sds_t upload_failure_message; }; #endif From bf59de9cb729b15fbef013a29016ab4077967d6d Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 18:09:01 +0200 Subject: [PATCH 42/70] input_blob: fixed erroneous file size unpacking Signed-off-by: Leonardo Alminana --- src/flb_input_blob.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/flb_input_blob.c b/src/flb_input_blob.c index 2e74e07fe40..b4a5f8b654c 100644 --- a/src/flb_input_blob.c +++ b/src/flb_input_blob.c @@ -33,6 +33,7 @@ int flb_input_blob_file_get_info(msgpack_object map, cfl_sds_t *source, cfl_sds_ cfl_sds_t tmp_source; cfl_sds_t tmp_file_path; msgpack_object o; + size_t tmp_size; if (map.type != MSGPACK_OBJECT_MAP) { return -1; @@ -78,6 +79,8 @@ int flb_input_blob_file_get_info(msgpack_object map, cfl_sds_t *source, cfl_sds_ return -1; } + tmp_size = o.via.u64; + /* get source plugin */ o = map.via.map.ptr[2].key; if (o.type != MSGPACK_OBJECT_STR) { @@ -101,7 +104,7 @@ int flb_input_blob_file_get_info(msgpack_object map, cfl_sds_t *source, cfl_sds_ return -1; } - *size = o.via.u64; + *size = tmp_size; *file_path = tmp_file_path; *source = tmp_source; From 4a94c2dc574f980f37bd1e65647299c79bb1cc2e Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 18:09:46 +0200 Subject: [PATCH 43/70] out_azure_blob: fixed stored erroneous procedure references and code Signed-off-by: Leonardo Alminana --- plugins/out_azure_blob/azure_blob_db.c | 6 +++--- plugins/out_azure_blob/azure_blob_db.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/out_azure_blob/azure_blob_db.c b/plugins/out_azure_blob/azure_blob_db.c index 586df720744..2a7fd68ad7f 100644 --- a/plugins/out_azure_blob/azure_blob_db.c +++ b/plugins/out_azure_blob/azure_blob_db.c @@ -433,9 +433,9 @@ int azb_db_file_get_next_aborted(struct flb_azure_blob *ctx, /* id: column 0 */ *id = sqlite3_column_int64(ctx->stmt_get_next_aborted_file, 0); - *delivery_attempts = sqlite3_column_int64(ctx->stmt_get_next_file_part, 1); - tmp_source = (char *) sqlite3_column_text(ctx->stmt_get_next_file_part, 2); - tmp_path = (char *) sqlite3_column_text(ctx->stmt_get_next_file_part, 3); + *delivery_attempts = sqlite3_column_int64(ctx->stmt_get_next_aborted_file, 1); + tmp_source = (char *) sqlite3_column_text(ctx->stmt_get_next_aborted_file, 2); + tmp_path = (char *) sqlite3_column_text(ctx->stmt_get_next_aborted_file, 3); *path = cfl_sds_create(tmp_path); diff --git a/plugins/out_azure_blob/azure_blob_db.h b/plugins/out_azure_blob/azure_blob_db.h index 0cd682f1b33..fe067f35013 100644 --- a/plugins/out_azure_blob/azure_blob_db.h +++ b/plugins/out_azure_blob/azure_blob_db.h @@ -68,7 +68,7 @@ "SELECT * FROM out_azure_blob_files WHERE path=@path ORDER BY id DESC;" #define SQL_GET_NEXT_ABORTED_FILE \ - "SELECT id, delivery_attempts, source, path " \ + "SELECT id, azbf.delivery_attempts, source, path " \ " FROM out_azure_blob_files azbf " \ " WHERE aborted = 1 " \ " AND (SELECT COUNT(*) " \ @@ -89,7 +89,7 @@ "UPDATE out_azure_blob_parts SET in_progress=@status WHERE id=@id;" #define SQL_UPDATE_FILE_PART_DELIVERY_ATTEMPT_COUNT \ - "UPDATE out_azure_blob_parts SET delivery_attempts=@delivery_attempts WHERE id=@id;" + "UPDATE out_azure_blob_parts SET delivery_attempts=@delivery_attempts WHERE part_id=@id;" #define SQL_RESET_FILE_UPLOAD_STATES \ "UPDATE out_azure_blob_parts SET delivery_attempts=0, uploaded=0, in_progress=0 WHERE file_id=@id;" From c74cfe4bd79dcaca62422265fef590d40913bbd1 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 22 Sep 2024 18:10:13 +0200 Subject: [PATCH 44/70] out_azure_blob: fixed failure notification producer Signed-off-by: Leonardo Alminana --- plugins/out_azure_blob/azure_blob.c | 59 ++++++++++++++--------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index 435e620fff4..32dceb839ca 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -410,7 +410,6 @@ static int send_blob(struct flb_config *config, payload_size = bytes; } - ret = http_send_blob(config, ctx, ref_name, uri, block_id, event_type, payload_buf, payload_size); flb_plg_debug(ctx->ins, "http_send_blob()=%i", ret); @@ -729,10 +728,10 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context pthread_mutex_lock(&ctx->file_upload_commit_file_parts); ret = azb_db_file_get_next_aborted(ctx, - &file_id, - &file_delivery_attempts, - &file_path, - &source); + &file_id, + &file_delivery_attempts, + &file_path, + &source); if (ret == 1) { ret = delete_blob(ctx, file_path); @@ -743,9 +742,30 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context azb_db_file_set_aborted_state(ctx, file_id, file_path, 0); } else { - /* notify the input plugin - */ ret = azb_db_file_delete(ctx, file_id, file_path); + + notification = flb_calloc(1, + sizeof( + struct flb_blob_delivery_notification)); + + if (notification != NULL) { + notification->base.dynamically_allocated = FLB_TRUE; + notification->base.notification_type = FLB_NOTIFICATION_TYPE_BLOB_DELIVERY; + notification->base.destructor = flb_input_blob_delivery_notification_destroy; + notification->success = FLB_FALSE; + notification->path = cfl_sds_create(file_path); + + ret = flb_notification_enqueue(FLB_PLUGIN_INPUT, + source, + ¬ification->base, + config); + + if (ret != 0) { + flb_plg_error(ctx->ins, + "blob file '%s' (id=%" PRIu64 ") notification " \ + "delivery error %d", file_path, file_id, ret); + } + } } cfl_sds_destroy(file_path); @@ -769,28 +789,6 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context if (ret == -1) { flb_plg_error(ctx->ins, "cannot commit blob file parts for file id=%" PRIu64 " path=%s", file_id, file_path); - - notification = flb_calloc(1, - sizeof( - struct flb_blob_delivery_notification)); - - if (notification != NULL) { - notification->base.dynamically_allocated = FLB_TRUE; - notification->base.destructor = flb_input_blob_delivery_notification_destroy; - notification->success = FLB_TRUE; - notification->path = cfl_sds_create(file_path); - - ret = flb_notification_enqueue(FLB_PLUGIN_INPUT, - source, - ¬ification->base, - config); - - if (ret != 0) { - flb_plg_error(ctx->ins, - "blob file '%s' (id=%" PRIu64 ") notification " \ - "delivery error %d", file_path, file_id, ret); - } - } } else { flb_plg_info(ctx->ins, "blob file '%s' (id=%" PRIu64 ") committed successfully", file_path, file_id); @@ -803,8 +801,9 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context if (notification != NULL) { notification->base.dynamically_allocated = FLB_TRUE; + notification->base.notification_type = FLB_NOTIFICATION_TYPE_BLOB_DELIVERY; notification->base.destructor = flb_input_blob_delivery_notification_destroy; - notification->success = FLB_FALSE; + notification->success = FLB_TRUE; notification->path = cfl_sds_create(file_path); ret = flb_notification_enqueue(FLB_PLUGIN_INPUT, From df3447e39f0d4edc3f808dcca841b5780650a2c8 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 24 Sep 2024 13:15:20 +0200 Subject: [PATCH 45/70] output: added worker cleanup callback missing initialization call Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_output.h | 1 + src/flb_output.c | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/include/fluent-bit/flb_output.h b/include/fluent-bit/flb_output.h index bf93381b789..0dab6a8604d 100644 --- a/include/fluent-bit/flb_output.h +++ b/include/fluent-bit/flb_output.h @@ -234,6 +234,7 @@ struct flb_output_plugin { int workers; int (*cb_worker_init) (void *, struct flb_config *); + int (*cb_worker_exit) (void *, struct flb_config *); /* Notification: this callback will be invoked anytime a notification is received*/ int (*cb_notification) (struct flb_output_instance *, struct flb_config *, void *); diff --git a/src/flb_output.c b/src/flb_output.c index 365708e10ea..90593905960 100644 --- a/src/flb_output.c +++ b/src/flb_output.c @@ -464,6 +464,12 @@ void flb_output_exit(struct flb_config *config) ins = mk_list_entry(head, struct flb_output_instance, _head); p = ins->p; + if (ins->is_threaded == FLB_FALSE) { + if (ins->p->cb_worker_exit) { + ins->p->cb_worker_exit(ins->context, ins->config); + } + } + /* Stop any worker thread */ if (flb_output_is_threaded(ins) == FLB_TRUE) { flb_output_thread_pool_destroy(ins); @@ -479,6 +485,7 @@ void flb_output_exit(struct flb_config *config) params = FLB_TLS_GET(out_flush_params); if (params) { flb_free(params); + FLB_TLS_SET(out_flush_params, NULL); } } @@ -1321,6 +1328,12 @@ int flb_output_init_all(struct flb_config *config) return -1; } + if (ins->is_threaded == FLB_FALSE) { + if (ins->p->cb_worker_init) { + ret = ins->p->cb_worker_init(ins->context, ins->config); + } + } + ins->processor->notification_channel = ins->notification_channel; /* initialize processors */ From 9290bb3e3f52093255d44831c6d2b3b05efde55b Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 24 Sep 2024 13:21:30 +0200 Subject: [PATCH 46/70] scheduler: improved timer coroutine return signal handling Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_scheduler.h | 7 +++- src/flb_scheduler.c | 61 +++++++++++++++++++++++------- 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/include/fluent-bit/flb_scheduler.h b/include/fluent-bit/flb_scheduler.h index c76cda60290..952d23c3d85 100644 --- a/include/fluent-bit/flb_scheduler.h +++ b/include/fluent-bit/flb_scheduler.h @@ -56,6 +56,7 @@ struct flb_sched_timer { int coro; void *data; struct flb_sched *sched; + void *cb_params; /* * Custom timer specific data: @@ -138,7 +139,7 @@ struct flb_sched { flb_pipefd_t ch_events[2]; }; - +struct flb_sched_timer_coro; int flb_sched_request_create(struct flb_config *config, void *data, int tries); @@ -162,6 +163,7 @@ int flb_sched_timer_coro_cb_create(struct flb_sched *sched, int type, int64_t ms void (*cb)(struct flb_config *, void *), void *data, struct flb_sched_timer **out_timer); +void flb_sched_timer_coro_destroy(struct flb_sched_timer_coro *instance); struct flb_sched_timer_coro *flb_sched_timer_coro_create(struct flb_sched_timer *timer, struct flb_config *config, void *data); @@ -246,7 +248,8 @@ static FLB_INLINE void sched_timer_cb_params_set(struct flb_sched_timer_coro *st params = (struct flb_sched_timer_coro_cb_params *) FLB_TLS_GET(sched_timer_coro_cb_params); if (!params) { - params = flb_malloc(sizeof(struct flb_sched_timer_coro_cb_params)); + params = flb_calloc(1, sizeof(struct flb_sched_timer_coro_cb_params)); + if (!params) { flb_errno(); return; diff --git a/src/flb_scheduler.c b/src/flb_scheduler.c index bacc5083e01..fc91c967391 100644 --- a/src/flb_scheduler.c +++ b/src/flb_scheduler.c @@ -441,6 +441,25 @@ static inline uint32_t sched_timer_coro_get_id(struct flb_sched *sched) return id; } +static inline struct flb_sched_timer_coro *sched_timer_coro_get_by_id( + struct flb_sched *sched, + uint32_t id) +{ + struct cfl_list *head; + struct flb_sched_timer_coro *stc; + + /* check if the proposed id is in use already */ + cfl_list_foreach(head, &sched->timer_coro_list) { + stc = cfl_list_entry(head, struct flb_sched_timer_coro, _head); + if (stc->id == id) { + return stc; + } + } + + return NULL; +} + + /* context of a scheduled timer that holds a coroutine context */ struct flb_sched_timer_coro *flb_sched_timer_coro_create(struct flb_sched_timer *timer, struct flb_config *config, @@ -458,7 +477,7 @@ struct flb_sched_timer_coro *flb_sched_timer_coro_create(struct flb_sched_timer return NULL; } - stc = flb_malloc(sizeof(struct flb_sched_timer_coro)); + stc = flb_calloc(1, sizeof(struct flb_sched_timer_coro)); if (!stc) { flb_errno(); return NULL; @@ -489,6 +508,21 @@ struct flb_sched_timer_coro *flb_sched_timer_coro_create(struct flb_sched_timer return stc; } +void flb_sched_timer_coro_destroy(struct flb_sched_timer_coro *instance) +{ + if (instance == NULL) { + return; + } + + if (instance->coro != NULL) { + flb_coro_destroy(instance->coro); + } + + cfl_list_del(&instance->_head); + + flb_free(instance); +} + /* * Create a timer that triggers the defined callback every N milliseconds. */ @@ -519,16 +553,11 @@ int flb_sched_event_handler(struct flb_config *config, struct mk_event *event) struct flb_sched_timer_coro *stc; if (event->type == FLB_ENGINE_EV_SCHED_CORO) { - printf("CORO EVENT\n"); - stc = (struct flb_sched_timer_coro *) event; - if (!stc) { - flb_error("[sched] invalid timer coro context"); - return -1; - } + sched = flb_sched_ctx_get(); /* consume the notification */ - val = flb_pipe_r(event->fd, &val, sizeof(val)); - if (val == -1) { + ret = flb_pipe_r(event->fd, &val, sizeof(val)); + if (ret == -1) { flb_errno(); return -1; } @@ -536,9 +565,15 @@ int flb_sched_event_handler(struct flb_config *config, struct mk_event *event) op = FLB_BITS_U64_HIGH(val); id = FLB_BITS_U64_LOW(val); + stc = sched_timer_coro_get_by_id(sched, id); + + if (stc == NULL) { + flb_error("[sched] invalid timer coroutine id %u", id); + return -1; + } + if (op == FLB_SCHED_TIMER_CORO_RETURN) { /* move stc to the drop list */ - sched = flb_sched_ctx_get(); cfl_list_del(&stc->_head); cfl_list_add(&stc->_head, &sched->timer_coro_list_drop); } @@ -761,7 +796,7 @@ struct flb_sched *flb_sched_create(struct flb_config *config, flb_sched_destroy(sched); return NULL; } - sched->event.type = FLB_ENGINE_EV_SCHED; + sched->event.type = FLB_ENGINE_EV_SCHED_CORO; /* * Note: mk_event_timeout_create() sets a type = MK_EVENT_NOTIFICATION by @@ -895,9 +930,7 @@ int flb_sched_timer_coro_cleanup(struct flb_sched *sched) cfl_list_foreach_safe(head, tmp, &sched->timer_coro_list_drop) { stc = cfl_list_entry(head, struct flb_sched_timer_coro, _head); - flb_coro_destroy(stc->coro); - cfl_list_del(&stc->_head); - flb_free(stc); + flb_sched_timer_coro_destroy(stc); c++; } From e0afbcb30c3fad3f5b351ccbb9b33a36c6607517 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 24 Sep 2024 13:22:08 +0200 Subject: [PATCH 47/70] scheduler: removed erroneously added field Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_scheduler.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/fluent-bit/flb_scheduler.h b/include/fluent-bit/flb_scheduler.h index 952d23c3d85..891d35195f3 100644 --- a/include/fluent-bit/flb_scheduler.h +++ b/include/fluent-bit/flb_scheduler.h @@ -56,7 +56,6 @@ struct flb_sched_timer { int coro; void *data; struct flb_sched *sched; - void *cb_params; /* * Custom timer specific data: From 492fc600369829a6c604807239674a4a6c1d935a Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 24 Sep 2024 13:22:36 +0200 Subject: [PATCH 48/70] engine: added missing scheduler coroutine cleanup Signed-off-by: Leonardo Alminana --- src/flb_engine.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/flb_engine.c b/src/flb_engine.c index 475495c03cf..0efa9ec85db 100644 --- a/src/flb_engine.c +++ b/src/flb_engine.c @@ -1124,6 +1124,7 @@ int flb_engine_start(struct flb_config *config) /* Release all resources associated to the engine */ int flb_engine_shutdown(struct flb_config *config) { + struct flb_sched_timer_coro_cb_params *sched_params; config->is_running = FLB_FALSE; flb_input_pause_all(config); @@ -1143,6 +1144,13 @@ int flb_engine_shutdown(struct flb_config *config) flb_custom_exit(config); flb_input_exit_all(config); + /* scheduler */ + sched_params = (struct flb_sched_timer_coro_cb_params *) FLB_TLS_GET(sched_timer_coro_cb_params); + if (sched_params != NULL) { + flb_free(sched_params); + FLB_TLS_SET(sched_timer_coro_cb_params, NULL); + } + /* Destroy the storage context */ flb_storage_destroy(config); From 6d39093829af08c8f81568854cc02603b755d073 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 24 Sep 2024 13:22:59 +0200 Subject: [PATCH 49/70] output_thread: added missing scheduler coroutine cleanup Signed-off-by: Leonardo Alminana --- src/flb_output_thread.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/flb_output_thread.c b/src/flb_output_thread.c index 330ff4feeec..7e094551b62 100644 --- a/src/flb_output_thread.c +++ b/src/flb_output_thread.c @@ -182,6 +182,7 @@ static void output_thread(void *data) struct flb_output_flush *out_flush; struct flb_out_thread_instance *th_ins = data; struct flb_out_flush_params *params; + struct flb_sched_timer_coro_cb_params *sched_params; struct flb_net_dns dns_ctx; struct flb_notification *notification; @@ -274,6 +275,16 @@ static void output_thread(void *data) */ flb_sched_event_handler(sched->config, event); } + else if (event->type & FLB_ENGINE_EV_SCHED_CORO) { + /* + * Note that this scheduler event handler has more features + * designed to be used from the parent thread, on this specific + * use case we just care about simple timers created on this + * thread or threaded by some output plugin. + */ + flb_sched_event_handler(sched->config, event); + + } else if (event->type == FLB_ENGINE_EV_THREAD_OUTPUT) { /* Read the task reference */ n = flb_pipe_r(event->fd, &task, sizeof(struct flb_task *)); @@ -361,6 +372,10 @@ static void output_thread(void *data) } } + if (ins->p->cb_worker_exit) { + ret = ins->p->cb_worker_exit(ins->context, ins->config); + } + mk_event_channel_destroy(th_ins->evl, th_ins->ch_thread_events[0], th_ins->ch_thread_events[1], @@ -383,8 +398,16 @@ static void output_thread(void *data) params = FLB_TLS_GET(out_flush_params); if (params) { flb_free(params); + FLB_TLS_SET(out_flush_params, NULL); } + sched_params = (struct flb_sched_timer_coro_cb_params *) FLB_TLS_GET(sched_timer_coro_cb_params); + if (sched_params != NULL) { + flb_free(sched_params); + FLB_TLS_SET(sched_timer_coro_cb_params, NULL); + } + + mk_event_channel_destroy(th_ins->evl, th_ins->ch_parent_events[0], th_ins->ch_parent_events[1], From 519393b9f374874dad06b1226a26461ea045615d Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 24 Sep 2024 13:23:42 +0200 Subject: [PATCH 50/70] out_azure_blob: added worker context cleanup and stale file remediation Signed-off-by: Leonardo Alminana --- plugins/out_azure_blob/azure_blob.c | 120 ++++++++++++++++++++-------- plugins/out_azure_blob/azure_blob.h | 9 +++ 2 files changed, 95 insertions(+), 34 deletions(-) diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index 32dceb839ca..88d07fff3f3 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -713,6 +713,7 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context struct flb_blob_delivery_notification *notification; info = FLB_TLS_GET(worker_info); + if (info->active_upload) { flb_plg_trace(ctx->ins, "[worker: file upload] upload already in progress..."); flb_sched_timer_cb_coro_return(); @@ -727,52 +728,77 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context pthread_mutex_lock(&ctx->file_upload_commit_file_parts); - ret = azb_db_file_get_next_aborted(ctx, - &file_id, - &file_delivery_attempts, - &file_path, - &source); + while (1) { + ret = azb_db_file_get_next_stale(ctx, + &file_id, + &file_path); - if (ret == 1) { - ret = delete_blob(ctx, file_path); + if (ret == 1) { + delete_blob(ctx, file_path); - if (ctx->file_delivery_attempt_limit != FLB_OUT_RETRY_UNLIMITED && - file_delivery_attempts < ctx->file_delivery_attempt_limit) { azb_db_file_reset_upload_states(ctx, file_id, file_path); azb_db_file_set_aborted_state(ctx, file_id, file_path, 0); + + cfl_sds_destroy(file_path); + + file_path = NULL; } else { - ret = azb_db_file_delete(ctx, file_id, file_path); - - notification = flb_calloc(1, - sizeof( - struct flb_blob_delivery_notification)); + break; + } + } - if (notification != NULL) { - notification->base.dynamically_allocated = FLB_TRUE; - notification->base.notification_type = FLB_NOTIFICATION_TYPE_BLOB_DELIVERY; - notification->base.destructor = flb_input_blob_delivery_notification_destroy; - notification->success = FLB_FALSE; - notification->path = cfl_sds_create(file_path); + while (1) { + ret = azb_db_file_get_next_aborted(ctx, + &file_id, + &file_delivery_attempts, + &file_path, + &source); - ret = flb_notification_enqueue(FLB_PLUGIN_INPUT, - source, - ¬ification->base, - config); + if (ret == 1) { + ret = delete_blob(ctx, file_path); - if (ret != 0) { - flb_plg_error(ctx->ins, - "blob file '%s' (id=%" PRIu64 ") notification " \ - "delivery error %d", file_path, file_id, ret); + if (ctx->file_delivery_attempt_limit != FLB_OUT_RETRY_UNLIMITED && + file_delivery_attempts < ctx->file_delivery_attempt_limit) { + azb_db_file_reset_upload_states(ctx, file_id, file_path); + azb_db_file_set_aborted_state(ctx, file_id, file_path, 0); + } + else { + ret = azb_db_file_delete(ctx, file_id, file_path); + + notification = flb_calloc(1, + sizeof( + struct flb_blob_delivery_notification)); + + if (notification != NULL) { + notification->base.dynamically_allocated = FLB_TRUE; + notification->base.notification_type = FLB_NOTIFICATION_TYPE_BLOB_DELIVERY; + notification->base.destructor = flb_input_blob_delivery_notification_destroy; + notification->success = FLB_FALSE; + notification->path = cfl_sds_create(file_path); + + ret = flb_notification_enqueue(FLB_PLUGIN_INPUT, + source, + ¬ification->base, + config); + + if (ret != 0) { + flb_plg_error(ctx->ins, + "blob file '%s' (id=%" PRIu64 ") notification " \ + "delivery error %d", file_path, file_id, ret); + } } } - } - cfl_sds_destroy(file_path); - cfl_sds_destroy(source); + cfl_sds_destroy(file_path); + cfl_sds_destroy(source); - file_path = NULL; - source = NULL; + file_path = NULL; + source = NULL; + } + else { + break; + } } ret = azb_db_file_oldest_ready(ctx, &file_id, &file_path, &part_ids, &source); @@ -874,7 +900,7 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context flb_sched_timer_cb_coro_return(); } - azb_db_file_part_delivery_attempts(ctx, part_id, ++part_delivery_attempts); + azb_db_file_part_delivery_attempts(ctx, file_id, part_id, ++part_delivery_attempts); flb_plg_debug(ctx->ins, "sending part file %s (id=%" PRIu64 " part_id=%" PRIu64 ")", file_path, id, part_id); ret = send_blob(config, NULL, ctx, FLB_EVENT_TYPE_BLOBS, @@ -901,6 +927,7 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context flb_free(out_buf); } cfl_sds_destroy(file_path); + flb_sched_timer_cb_coro_return(); } @@ -1028,6 +1055,24 @@ static int cb_worker_init(void *data, struct flb_config *config) return 0; } +/* worker teardown */ +static int cb_worker_exit(void *data, struct flb_config *config) +{ + int ret; + struct worker_info *info; + struct flb_azure_blob *ctx = data; + + flb_plg_info(ctx->ins, "initializing worker"); + + info = FLB_TLS_GET(worker_info); + if (info != NULL) { + flb_free(info); + FLB_TLS_SET(worker_info, NULL); + } + + return 0; +} + /* Configuration properties map */ static struct flb_config_map config_map[] = { { @@ -1139,6 +1184,12 @@ static struct flb_config_map config_map[] = { "Timeout to upload parts of a blob file" }, + { + FLB_CONFIG_MAP_TIME, "upload_part_freshness_limit", "6D", + 0, FLB_TRUE, offsetof(struct flb_azure_blob, upload_parts_freshness_threshold), + "Maximum lifespan of an uncommitted file part" + }, + { FLB_CONFIG_MAP_STR, "configuration_endpoint_url", NULL, 0, FLB_TRUE, offsetof(struct flb_azure_blob, configuration_endpoint_url), @@ -1175,6 +1226,7 @@ struct flb_output_plugin out_azure_blob_plugin = { .cb_flush = cb_azure_blob_flush, .cb_exit = cb_azure_blob_exit, .cb_worker_init = cb_worker_init, + .cb_worker_exit = cb_worker_exit, /* Test */ .test_formatter.callback = azure_blob_format, diff --git a/plugins/out_azure_blob/azure_blob.h b/plugins/out_azure_blob/azure_blob.h index 8b3009e978d..43cb5035792 100644 --- a/plugins/out_azure_blob/azure_blob.h +++ b/plugins/out_azure_blob/azure_blob.h @@ -62,6 +62,7 @@ struct flb_azure_blob { flb_sds_t database_file; size_t part_size; time_t upload_parts_timeout; + time_t upload_parts_freshness_threshold; int file_delivery_attempt_limit; int part_delivery_attempt_limit; flb_sds_t configuration_endpoint_url; @@ -69,6 +70,12 @@ struct flb_azure_blob { flb_sds_t configuration_endpoint_password; flb_sds_t configuration_endpoint_bearer_token; + int endpoint_overriden_flag; + int shared_key_overriden_flag; + int sas_token_overriden_flag; + int container_name_overriden_flag; + int path_overriden_flag; + /* * Internal use */ @@ -104,7 +111,9 @@ struct flb_azure_blob { sqlite3_stmt *stmt_update_file_delivery_attempt_count; sqlite3_stmt *stmt_set_file_aborted_state; sqlite3_stmt *stmt_get_next_aborted_file; + sqlite3_stmt *stmt_get_next_stale_file; sqlite3_stmt *stmt_reset_file_upload_states; + sqlite3_stmt *stmt_reset_file_part_upload_states; /* prepared statement: file parts */ From ee94110da2306f3a425ea4717cb2e7bfa047657e Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 24 Sep 2024 13:24:10 +0200 Subject: [PATCH 51/70] out_azure_blob: fixed leak and double free Signed-off-by: Leonardo Alminana --- plugins/out_azure_blob/azure_blob_conf.c | 66 ++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/plugins/out_azure_blob/azure_blob_conf.c b/plugins/out_azure_blob/azure_blob_conf.c index c6cb7e978aa..a66af75933a 100644 --- a/plugins/out_azure_blob/azure_blob_conf.c +++ b/plugins/out_azure_blob/azure_blob_conf.c @@ -150,6 +150,7 @@ static int flb_azure_blob_process_remote_configuration_payload( msgpack_object_map *configuration_map; msgpack_unpacked unpacked_root; char *msgpack_body; + char *value_backup; int root_type; size_t offset; int result; @@ -194,65 +195,100 @@ static int flb_azure_blob_process_remote_configuration_payload( configuration_map = &unpacked_root.data.via.map; + value_backup = context->endpoint; + context->endpoint = NULL; + result = extract_map_string_entry_by_key(&context->endpoint, configuration_map, "host", 0, FLB_TRUE); if (result != 0) { + context->endpoint = value_backup; + flb_plg_error(context->ins, "endpoint extraction error : %d", result); goto cleanup; } + context->endpoint_overriden_flag = FLB_TRUE; + if (context->atype == AZURE_BLOB_AUTH_KEY) { + value_backup = context->shared_key; + context->shared_key = NULL; + result = extract_map_string_entry_by_key(&context->shared_key, - configuration_map, - "shared_key", 0, FLB_TRUE); + configuration_map, + "shared_key", 0, FLB_TRUE); if (result != 0) { + context->shared_key = value_backup; + flb_plg_error(context->ins, - "neither sas_token nor shared_key " \ - "could be extracted : %d", result); + "neither sas_token nor shared_key " \ + "could be extracted : %d", result); goto cleanup; } + + context->shared_key_overriden_flag = FLB_TRUE; } else if (context->atype == AZURE_BLOB_AUTH_SAS) { + value_backup = context->sas_token; + context->sas_token = NULL; + result = extract_map_string_entry_by_key(&context->sas_token, configuration_map, "sas_token", 0, FLB_TRUE); if (result != 0) { + context->sas_token = value_backup; + flb_plg_error(context->ins, "sas_token extraction error : %d", result); goto cleanup; } + + context->sas_token_overriden_flag = FLB_TRUE; } + value_backup = context->container_name; + context->container_name = NULL; + result = extract_map_string_entry_by_key(&context->container_name, configuration_map, "container", 0, FLB_TRUE); if (result != 0) { + context->container_name = value_backup; + flb_plg_error(context->ins, "container extraction error : %d", result); goto cleanup; } + context->container_name_overriden_flag = FLB_TRUE; + + value_backup = context->path; + context->path = NULL; + result = extract_map_string_entry_by_key(&context->path, configuration_map, "path", 0, FLB_TRUE); if (result != 0) { + context->path = value_backup; + flb_plg_error(context->ins, "path extraction error : %d", result); goto cleanup; } + context->path_overriden_flag = FLB_TRUE; + cleanup: if (result != 0) { result = -1; @@ -729,6 +765,28 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in void flb_azure_blob_conf_destroy(struct flb_azure_blob *ctx) { + + if (ctx->endpoint_overriden_flag == FLB_TRUE) { + flb_sds_destroy(ctx->endpoint); + ctx->endpoint = NULL; + } + if (ctx->shared_key_overriden_flag == FLB_TRUE) { + flb_sds_destroy(ctx->shared_key); + ctx->shared_key = NULL; + } + if (ctx->sas_token_overriden_flag == FLB_TRUE) { + flb_sds_destroy(ctx->sas_token); + ctx->sas_token = NULL; + } + if (ctx->container_name_overriden_flag == FLB_TRUE) { + flb_sds_destroy(ctx->container_name); + ctx->container_name = NULL; + } + if (ctx->path_overriden_flag == FLB_TRUE) { + flb_sds_destroy(ctx->path); + ctx->path = NULL; + } + if (ctx->decoded_sk) { flb_free(ctx->decoded_sk); } From 4d09bb8a9dc5c978ebc8921cb1f8381139dfa344 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 24 Sep 2024 13:24:53 +0200 Subject: [PATCH 52/70] out_azure_blob: added stale file remediation Signed-off-by: Leonardo Alminana --- plugins/out_azure_blob/azure_blob_db.c | 118 ++++++++++++++++++++++++- plugins/out_azure_blob/azure_blob_db.h | 65 +++++++++++--- 2 files changed, 167 insertions(+), 16 deletions(-) diff --git a/plugins/out_azure_blob/azure_blob_db.c b/plugins/out_azure_blob/azure_blob_db.c index 2a7fd68ad7f..3de48b15dea 100644 --- a/plugins/out_azure_blob/azure_blob_db.c +++ b/plugins/out_azure_blob/azure_blob_db.c @@ -85,6 +85,16 @@ static int prepare_stmts(struct flb_sqldb *db, struct flb_azure_blob *ctx) return -1; } + /* get next stale file */ + ret = sqlite3_prepare_v2(db->handler, SQL_GET_NEXT_STALE_FILE, -1, + &ctx->stmt_get_next_stale_file, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_GET_NEXT_STALE_FILE); + return -1; + } + + /* reset file upload progress */ ret = sqlite3_prepare_v2(db->handler, SQL_RESET_FILE_UPLOAD_STATES, -1, &ctx->stmt_reset_file_upload_states, NULL); if (ret != SQLITE_OK) { @@ -93,6 +103,15 @@ static int prepare_stmts(struct flb_sqldb *db, struct flb_azure_blob *ctx) return -1; } + /* reset file part upload progress */ + ret = sqlite3_prepare_v2(db->handler, SQL_RESET_FILE_PART_UPLOAD_STATES, -1, + &ctx->stmt_reset_file_part_upload_states, NULL); + if (ret != SQLITE_OK) { + flb_plg_error(ctx->ins, "cannot prepare SQL statement: %s", + SQL_RESET_FILE_PART_UPLOAD_STATES); + return -1; + } + /* insert blob file part */ ret = sqlite3_prepare_v2(db->handler, SQL_INSERT_FILE_PART, -1, &ctx->stmt_insert_file_part, NULL); @@ -228,7 +247,9 @@ int azb_db_close(struct flb_azure_blob *ctx) sqlite3_finalize(ctx->stmt_get_file); sqlite3_finalize(ctx->stmt_update_file_delivery_attempt_count); sqlite3_finalize(ctx->stmt_get_next_aborted_file); + sqlite3_finalize(ctx->stmt_get_next_stale_file); sqlite3_finalize(ctx->stmt_reset_file_upload_states); + sqlite3_finalize(ctx->stmt_reset_file_part_upload_states); sqlite3_finalize(ctx->stmt_insert_file_part); sqlite3_finalize(ctx->stmt_update_file_part_uploaded); @@ -413,6 +434,56 @@ int azb_db_file_delivery_attempts(struct flb_azure_blob *ctx, return 0; } +int azb_db_file_get_next_stale(struct flb_azure_blob *ctx, + uint64_t *id, + cfl_sds_t *path) +{ + int ret; + char *tmp_path; + int exists = FLB_FALSE; + time_t freshness_threshold; + + freshness_threshold = time(NULL); + freshness_threshold -= ctx->upload_parts_freshness_threshold; + + azb_db_lock(ctx); + + /* Bind parameters */ + sqlite3_bind_int64(ctx->stmt_get_next_stale_file, 1, freshness_threshold); + ret = sqlite3_step(ctx->stmt_get_next_stale_file); + if (ret == SQLITE_ROW) { + exists = FLB_TRUE; + + /* id: column 0 */ + *id = sqlite3_column_int64(ctx->stmt_get_next_stale_file, 0); + tmp_path = (char *) sqlite3_column_text(ctx->stmt_get_next_stale_file, 1); + + *path = cfl_sds_create(tmp_path); + + if (*path == NULL) { + exists = -1; + } + } + else if (ret == SQLITE_DONE) { + /* all good */ + } + else { + exists = -1; + } + + sqlite3_clear_bindings(ctx->stmt_get_next_stale_file); + sqlite3_reset(ctx->stmt_get_next_stale_file); + + azb_db_unlock(ctx); + + if (exists == -1) { + *id = 0; + *path = NULL; + } + + return exists; +} + int azb_db_file_get_next_aborted(struct flb_azure_blob *ctx, uint64_t *id, uint64_t *delivery_attempts, @@ -472,6 +543,42 @@ int azb_db_file_get_next_aborted(struct flb_azure_blob *ctx, return exists; } + +static int azb_db_file_reset_part_upload_states(struct flb_azure_blob *ctx, uint64_t id, char *path) +{ + int ret; + + azb_db_lock(ctx); + + /* Bind parameters */ + sqlite3_bind_int64(ctx->stmt_reset_file_part_upload_states, 1, id); + ret = sqlite3_step(ctx->stmt_reset_file_part_upload_states); + if (ret != SQLITE_DONE) { + sqlite3_clear_bindings(ctx->stmt_reset_file_part_upload_states); + sqlite3_reset(ctx->stmt_reset_file_part_upload_states); + azb_db_unlock(ctx); + return -1; + } + + sqlite3_clear_bindings(ctx->stmt_reset_file_part_upload_states); + sqlite3_reset(ctx->stmt_reset_file_part_upload_states); + + if (ret != SQLITE_DONE) { + flb_plg_error(ctx->ins, "db: error reseting upload " + "states for entry id=%" PRIu64 + ", path='%s'", id, path); + azb_db_unlock(ctx); + return -1; + } + + flb_plg_debug(ctx->ins, "db: file id=%" PRIu64 + ", path='%s' upload states reset", id, path); + + azb_db_unlock(ctx); + + return 0; +} + int azb_db_file_reset_upload_states(struct flb_azure_blob *ctx, uint64_t id, char *path) { int ret; @@ -504,7 +611,7 @@ int azb_db_file_reset_upload_states(struct flb_azure_blob *ctx, uint64_t id, cha azb_db_unlock(ctx); - return 0; + return azb_db_file_reset_part_upload_states(ctx, id, path); } int azb_db_file_part_insert(struct flb_azure_blob *ctx, uint64_t file_id, @@ -663,7 +770,8 @@ int azb_db_file_part_uploaded(struct flb_azure_blob *ctx, uint64_t id) } int azb_db_file_part_delivery_attempts(struct flb_azure_blob *ctx, - uint64_t id, uint64_t attempts) + uint64_t file_id, + uint64_t part_id, uint64_t attempts) { int ret; @@ -671,7 +779,8 @@ int azb_db_file_part_delivery_attempts(struct flb_azure_blob *ctx, /* Bind parameters */ sqlite3_bind_int64(ctx->stmt_update_file_part_delivery_attempt_count, 1, attempts); - sqlite3_bind_int64(ctx->stmt_update_file_part_delivery_attempt_count, 2, id); + sqlite3_bind_int64(ctx->stmt_update_file_part_delivery_attempt_count, 2, file_id); + sqlite3_bind_int64(ctx->stmt_update_file_part_delivery_attempt_count, 3, part_id); /* Run the update */ ret = sqlite3_step(ctx->stmt_update_file_part_delivery_attempt_count); @@ -684,7 +793,8 @@ int azb_db_file_part_delivery_attempts(struct flb_azure_blob *ctx, if (ret != SQLITE_DONE) { flb_plg_error(ctx->ins, "cannot update delivery attempt " - "count for part id=%" PRIu64, id); + "count for part %" PRIu64 ".%" PRIu64, + file_id, part_id); return -1; } diff --git a/plugins/out_azure_blob/azure_blob_db.h b/plugins/out_azure_blob/azure_blob_db.h index fe067f35013..475d5ba9301 100644 --- a/plugins/out_azure_blob/azure_blob_db.h +++ b/plugins/out_azure_blob/azure_blob_db.h @@ -27,13 +27,14 @@ #define SQL_CREATE_AZURE_BLOB_FILES \ "CREATE TABLE IF NOT EXISTS out_azure_blob_files (" \ - " id INTEGER PRIMARY KEY," \ - " source TEXT NOT NULL," \ - " path TEXT NOT NULL," \ - " size INTEGER," \ - " created INTEGER," \ - " delivery_attempts INTEGER DEFAULT 0," \ - " aborted INTEGER DEFAULT 0" \ + " id INTEGER PRIMARY KEY," \ + " source TEXT NOT NULL," \ + " path TEXT NOT NULL," \ + " size INTEGER," \ + " created INTEGER," \ + " delivery_attempts INTEGER DEFAULT 0," \ + " aborted INTEGER DEFAULT 0," \ + " last_delivery_attempt INTEGER DEFAULT 0" \ ");" #define SQL_CREATE_AZURE_BLOB_PARTS \ @@ -62,7 +63,10 @@ "UPDATE out_azure_blob_files SET aborted=@state WHERE id=@id;" #define SQL_UPDATE_FILE_DELIVERY_ATTEMPT_COUNT \ - "UPDATE out_azure_blob_files SET delivery_attempts=@delivery_attempts WHERE id=@id;" + "UPDATE out_azure_blob_files " \ + " SET delivery_attempts=@delivery_attempts, " \ + " last_delivery_attempt=UNIXEPOCH() " \ + " WHERE id=@id;" #define SQL_GET_FILE \ "SELECT * FROM out_azure_blob_files WHERE path=@path ORDER BY id DESC;" @@ -78,6 +82,16 @@ "ORDER BY id DESC " \ "LIMIT 1;" + +#define SQL_GET_NEXT_STALE_FILE \ + "SELECT id, path " \ + " FROM out_azure_blob_files azbf " \ + " WHERE aborted = 0 " \ + " AND last_delivery_attempt > 0 " \ + " AND last_delivery_attempt < @freshness_threshold " \ + "ORDER BY id DESC " \ + "LIMIT 1;" + #define SQL_INSERT_FILE_PART \ "INSERT INTO out_azure_blob_parts (file_id, part_id, offset_start, offset_end)" \ " VALUES (@file_id, @part_id, @offset_start, @offset_end);" @@ -89,10 +103,22 @@ "UPDATE out_azure_blob_parts SET in_progress=@status WHERE id=@id;" #define SQL_UPDATE_FILE_PART_DELIVERY_ATTEMPT_COUNT \ - "UPDATE out_azure_blob_parts SET delivery_attempts=@delivery_attempts WHERE part_id=@id;" + "UPDATE out_azure_blob_parts " \ + " SET delivery_attempts=@delivery_attempts " \ + " WHERE file_id=@file_id " \ + " AND part_id=@part_id;" #define SQL_RESET_FILE_UPLOAD_STATES \ - "UPDATE out_azure_blob_parts SET delivery_attempts=0, uploaded=0, in_progress=0 WHERE file_id=@id;" + "UPDATE out_azure_blob_files " \ + " SET last_delivery_attempt=0 " \ + " WHERE id=@id;" + +#define SQL_RESET_FILE_PART_UPLOAD_STATES \ + "UPDATE out_azure_blob_parts " \ + " SET delivery_attempts=0, " \ + " uploaded=0, " \ + " in_progress=0 " \ + " WHERE file_id=@id;" /* Find the oldest files and retrieve the oldest part ready to be uploaded */ #define SQL_GET_NEXT_FILE_PART \ @@ -103,7 +129,8 @@ " p.offset_end, " \ " p.delivery_attempts, " \ " f.path, " \ - " f.delivery_attempts " \ + " f.delivery_attempts, " \ + " f.last_delivery_attempt " \ "FROM out_azure_blob_parts p " \ " JOIN out_azure_blob_files f " \ " ON p.file_id = f.id " \ @@ -139,18 +166,29 @@ struct flb_sqldb *azb_db_open(struct flb_azure_blob *ctx, char *db_path); int azb_db_close(struct flb_azure_blob *ctx); int azb_db_file_exists(struct flb_azure_blob *ctx, char *path, uint64_t *id); + int64_t azb_db_file_insert(struct flb_azure_blob *ctx, char *source, char *path, size_t size); + int azb_db_file_delete(struct flb_azure_blob *ctx, uint64_t id, char *path); + int azb_db_file_set_aborted_state(struct flb_azure_blob *ctx, uint64_t id, char *path, uint64_t state); + int azb_db_file_delivery_attempts(struct flb_azure_blob *ctx, uint64_t id, uint64_t attempts); + int azb_db_file_get_next_aborted(struct flb_azure_blob *ctx, uint64_t *id, uint64_t *delivery_attempts, cfl_sds_t *path, cfl_sds_t *source); + + +int azb_db_file_get_next_stale(struct flb_azure_blob *ctx, + uint64_t *id, + cfl_sds_t *path); + int azb_db_file_reset_upload_states(struct flb_azure_blob *ctx, uint64_t id, char *path); int azb_db_file_part_insert(struct flb_azure_blob *ctx, uint64_t file_id, @@ -165,7 +203,10 @@ int azb_db_file_part_get_next(struct flb_azure_blob *ctx, uint64_t *file_delivery_attempts, cfl_sds_t *file_path); int azb_db_file_part_uploaded(struct flb_azure_blob *ctx, uint64_t id); -int azb_db_file_part_delivery_attempts(struct flb_azure_blob *ctx, uint64_t id, uint64_t attempts); +int azb_db_file_part_delivery_attempts(struct flb_azure_blob *ctx, + uint64_t file_id, + uint64_t part_id, + uint64_t attempts); int azb_db_file_oldest_ready(struct flb_azure_blob *ctx, uint64_t *file_id, cfl_sds_t *path, cfl_sds_t *part_ids, cfl_sds_t *source); From ddccb4d12997942ca2aae4dd52476662289b9918 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 24 Sep 2024 13:25:11 +0200 Subject: [PATCH 53/70] out_azure_blob: fixed missing http verb support Signed-off-by: Leonardo Alminana --- plugins/out_azure_blob/azure_blob_http.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/out_azure_blob/azure_blob_http.c b/plugins/out_azure_blob/azure_blob_http.c index 5d97d5e25ad..af891c165c0 100644 --- a/plugins/out_azure_blob/azure_blob_http.c +++ b/plugins/out_azure_blob/azure_blob_http.c @@ -179,6 +179,9 @@ flb_sds_t azb_http_canonical_request(struct flb_azure_blob *ctx, case FLB_HTTP_PUT: tmp = flb_sds_cat(can_req, "PUT\n", 4); break; + case FLB_HTTP_DELETE: + tmp = flb_sds_cat(can_req, "DELETE\n", 4); + break; }; if (!tmp) { From 99a9b63abcddf9f73df8552598004438f9b6bc6e Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 24 Sep 2024 13:25:25 +0200 Subject: [PATCH 54/70] in_blob: fixed memory leaks Signed-off-by: Leonardo Alminana --- plugins/in_blob/blob.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/in_blob/blob.c b/plugins/in_blob/blob.c index d9d50ffa547..3aa50021a37 100644 --- a/plugins/in_blob/blob.c +++ b/plugins/in_blob/blob.c @@ -276,8 +276,15 @@ static ssize_t recursive_file_search(struct blob_ctx *ctx, switch (result) { case GLOB_NOSPACE: flb_plg_error(ctx->ins, "no memory space available"); + + cfl_sds_destroy(local_pattern); + cfl_sds_destroy(local_path); + return -5; case GLOB_ABORTED: + cfl_sds_destroy(local_pattern); + cfl_sds_destroy(local_path); + return 0; case GLOB_NOMATCH: result = stat(local_path, &fs_entry_metadata); @@ -295,6 +302,9 @@ static ssize_t recursive_file_search(struct blob_ctx *ctx, } } + cfl_sds_destroy(local_pattern); + cfl_sds_destroy(local_path); + return 6; } } From 2a0020c60a67fa8f75a824ccc6320a1bb1a64a46 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Wed, 25 Sep 2024 17:29:03 +0200 Subject: [PATCH 55/70] in_blob: fixed memory leaks and added temporary win32 compatibility shim Signed-off-by: Leonardo Alminana --- plugins/in_blob/blob.c | 13 + plugins/in_blob/win32_glob.c | 599 +++++++++++++++++++++++++++++++++++ 2 files changed, 612 insertions(+) create mode 100644 plugins/in_blob/win32_glob.c diff --git a/plugins/in_blob/blob.c b/plugins/in_blob/blob.c index 3aa50021a37..206d58947f7 100644 --- a/plugins/in_blob/blob.c +++ b/plugins/in_blob/blob.c @@ -36,6 +36,8 @@ #include "blob_db.h" #include "blob_file.h" +#include "win32_glob.c" + /* Define missing GLOB_TILDE if not exists */ #ifndef GLOB_TILDE #define GLOB_TILDE 1<<2 /* use GNU Libc value */ @@ -200,8 +202,11 @@ static ssize_t recursive_file_search(struct blob_ctx *ctx, cfl_sds_t sds_result; int result; size_t index; + match_count = 0; + memset(&glob_context, 0, sizeof(glob_t)); + if (path != NULL) { local_path = cfl_sds_create(path); } @@ -331,6 +336,8 @@ static ssize_t recursive_file_search(struct blob_ctx *ctx, cfl_sds_destroy(local_pattern); cfl_sds_destroy(local_path); + globfree(&glob_context); + return -7; } @@ -342,6 +349,8 @@ static ssize_t recursive_file_search(struct blob_ctx *ctx, cfl_sds_destroy(local_pattern); cfl_sds_destroy(local_path); + globfree(&glob_context); + return -8; } @@ -360,6 +369,8 @@ static ssize_t recursive_file_search(struct blob_ctx *ctx, cfl_sds_destroy(local_pattern); cfl_sds_destroy(local_path); + globfree(&glob_context); + return -9; } @@ -371,6 +382,8 @@ static ssize_t recursive_file_search(struct blob_ctx *ctx, cfl_sds_destroy(local_pattern); cfl_sds_destroy(local_path); + globfree(&glob_context); + return -10; } diff --git a/plugins/in_blob/win32_glob.c b/plugins/in_blob/win32_glob.c new file mode 100644 index 00000000000..b3828587c0f --- /dev/null +++ b/plugins/in_blob/win32_glob.c @@ -0,0 +1,599 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2021 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if defined(FLB_SYSTEM_WINDOWS) && !defined(FLB_FILE_GLOB_ERROR_SUCCESS) + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#define FLB_FILE_GLOB_ABORT_ON_ERROR (((uint64_t) 1) << 0) +#define FLB_FILE_GLOB_MARK_DIRECTORIES (((uint64_t) 1) << 1) +#define FLB_FILE_GLOB_DO_NOT_SORT (((uint64_t) 1) << 2) +#define FLB_FILE_GLOB_EXPAND_TILDE (((uint64_t) 1) << 3) + +#define FLB_FILE_GLOB_ERROR_SUCCESS 0 +#define FLB_FILE_GLOB_ERROR_ABORTED 1 +#define FLB_FILE_GLOB_ERROR_NO_MEMORY 2 +#define FLB_FILE_GLOB_ERROR_NO_FILE 3 +#define FLB_FILE_GLOB_ERROR_NO_ACCESS 4 +#define FLB_FILE_GLOB_ERROR_NO_MATCHES 5 +#define FLB_FILE_GLOB_ERROR_NO_MORE_RESULTS 6 +#define FLB_FILE_GLOB_ERROR_OVERSIZED_PATH 7 +#define FLB_FILE_GLOB_ERROR_INVALID_ARGUMENT 8 + +#ifndef GLOB_NOSPACE +#define GLOB_NOSPACE FLB_FILE_GLOB_ERROR_NO_MEMORY +#endif + +#ifndef GLOB_ABORTED +#define GLOB_ABORTED FLB_FILE_GLOB_ERROR_ABORTED +#endif + +#ifndef GLOB_NOMATCH +#define GLOB_NOMATCH FLB_FILE_GLOB_ERROR_NO_MATCHES +#endif + +struct flb_file_glob_inner_entry { + char *path; + struct cfl_list _head; +}; + +struct flb_file_glob_inner_context { + struct flb_file_glob_inner_entry *current_entry; + struct cfl_list results; + size_t entries; + size_t index; + uint64_t flags; +}; + +struct flb_file_glob_context { + struct flb_file_glob_inner_context *inner_context; + uint64_t flags; + char *path; +}; + +typedef struct glob_t { + struct flb_file_glob_context inner_context; + char **gl_pathv; + size_t gl_pathc; + +} + +static int flb_file_glob_start(struct flb_file_glob_context *context, + const char *path, + uint64_t flags); + +static void flb_file_glob_clean(struct flb_file_glob_context *context); + +static int flb_file_glob_fetch(struct flb_file_glob_context *context, + char **result); + +static void globfree(glob_t *context) +{ + size_t index; + + if (context->gl_pathv != NULL) { + flb_free(context->gl_pathv); + context->gl_pathv = NULL; + } + + flb_file_glob_clean(context->inner_context); +} + +static int glob(const char *path, + uint64_t flags, + void *unused, + glob_t *context) +{ + size_t entries; + int result; + size_t index; + + (void) unused; + + result = flb_file_glob_start(context, path, flags); + + if (result == FLB_FILE_GLOB_ERROR_SUCCESS) { + entries = cfl_list_size(context->inner_context->results); + + context->gl_pathv = flb_calloc(entries, sizeof(char *)); + + if (context->gl_pathv == NULL) { + globfree(context); + + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + for (index = 0 ; index < entries ; index++) { + result = flb_file_glob_fetch(context->internal_context, + &context->gl_pathv[index]); + + if (result != FLB_FILE_GLOB_ERROR_SUCCESS) { + globfree(context); + + return result; + } + } + } + + return result; +} + + +static void reset_errno() +{ + errno = 0; +} + +static void propagate_last_error_to_errno() +{ + DWORD error_code; + + error_code = GetLastError(); + + switch (error_code) { + case ERROR_INVALID_TARGET_HANDLE: + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + + case ERROR_TOO_MANY_OPEN_FILES: + errno = EMFILE; + break; + + case ERROR_INVALID_FLAG_NUMBER: + case ERROR_INVALID_PARAMETER: + errno = EINVAL; + break; + + case ERROR_NOT_ENOUGH_MEMORY: + case ERROR_OUTOFMEMORY: + errno = ENOMEM; + break; + + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + case ERROR_PATH_BUSY: + case ERROR_BUSY: + errno = EBUSY; + break; + + case ERROR_HANDLE_DISK_FULL: + case ERROR_DISK_FULL: + errno = ENOSPC; + break; + + case ERROR_INVALID_ADDRESS: + errno = EFAULT; + break; + + case ERROR_FILE_TOO_LARGE: + errno = EFBIG; + break; + + case ERROR_ALREADY_EXISTS: + case ERROR_FILE_EXISTS: + errno = EEXIST; + break; + + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_INVALID_DRIVE: + case ERROR_BAD_PATHNAME: + case ERROR_INVALID_NAME: + case ERROR_BAD_UNIT: + errno = ENOENT; + break; + + case ERROR_SEEK_ON_DEVICE: + errno = ESPIPE; + break; + + case ERROR_NEGATIVE_SEEK: + errno = EINVAL; + break; + + case ERROR_ACCESS_DENIED: + errno = EACCES; + break; + + case ERROR_DIR_NOT_EMPTY: + errno = ENOTEMPTY; + break; + + case ERROR_BROKEN_PIPE: + errno = EPIPE; + break; + + case ERROR_GEN_FAILURE: + errno = EIO; + break; + + case ERROR_OPEN_FAILED: + errno = EIO; + break; + + case ERROR_SUCCESS: + errno = 0; + break; + + default: + /* This is just a canary, if you find this + * error then it means we need to expand the + * translation list. + */ + + errno = EOWNERDEAD; + break; + } +} + +static int limited_win32_glob_append_entry( + struct flb_file_glob_inner_context *context, + char *path, + uint16_t mode_filter) +{ + char entry_path_buffer[FLB_FILE_MAX_PATH_LENGTH]; + char *entry_path; + struct flb_file_stat entry_info; + int result; + struct flb_file_glob_inner_entry *entry; + + result = flb_file_stat(path, &entry_info); + + if (result != 0) { + result = FLB_FILE_GLOB_ERROR_NO_FILE; + } + else { + result = FLB_FILE_GLOB_ERROR_SUCCESS; + + if (mode_filter != 0) { + if (!FLB_FILE_ISTYPE(entry_info.mode, mode_filter)) { + result = FLB_FILE_GLOB_ERROR_NO_MATCHES; + } + } + } + + if (result == FLB_FILE_GLOB_ERROR_SUCCESS) { + entry_path = _fullpath(entry_path_buffer, + path, + FLB_FILE_MAX_PATH_LENGTH); + + if (entry_path == NULL) { + result = FLB_FILE_GLOB_ERROR_OVERSIZED_PATH; + } + } + + if (result == FLB_FILE_GLOB_ERROR_SUCCESS) { + entry = flb_calloc(1, sizeof(struct flb_file_glob_inner_entry)); + + if (entry == NULL) { + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + entry->path = flb_strdup(entry_path); + + if (entry->path == NULL) { + flb_free(entry); + + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + cfl_list_append(&entry->_head, &context->results); + + context->entries++; + } + + return result; +} + +/* + * Perform patern match on the given path string. This function + * supports patterns with "nested" wildcards like below. + * + * tail_scan_pattern("C:\fluent-bit\*\*.txt", ctx); + * + * On success, the number of files found is returned (zero indicates + * "no file found"). On error, -1 is returned. + */ +static int limited_win32_glob(struct flb_file_glob_inner_context *context, + char *path) +{ + char *star, *p0, *p1; + char pattern[FLB_FILE_MAX_PATH_LENGTH]; + char buf[FLB_FILE_MAX_PATH_LENGTH]; + int ret; + int n_added = 0; + time_t now; + int64_t mtime; + HANDLE h; + WIN32_FIND_DATA data; + struct flb_file_glob_inner_entry *entry; + int transverse_directory; + struct flb_file_stat entry_info; + + if (strlen(path) >= FLB_FILE_MAX_PATH_LENGTH) { + return FLB_FILE_GLOB_ERROR_OVERSIZED_PATH; + } + + star = strchr(path, '*'); + + if (star == NULL) { + return limited_win32_glob_append_entry(context, path, 0); + } + + /* + * C:\data\tmp\input_*.conf + * 0<-----| + */ + p0 = star; + while (path <= p0 && *p0 != '\\') { + p0--; + } + + /* + * C:\data\tmp\input_*.conf + * |---->1 + */ + p1 = star; + while (*p1 && *p1 != '\\') { + p1++; + } + + memcpy(pattern, path, (p1 - path)); + pattern[p1 - path] = '\0'; + + h = FindFirstFileA(pattern, &data); + + if (h == INVALID_HANDLE_VALUE) { + return FLB_FILE_GLOB_ERROR_NO_MATCHES; + } + + ret = FLB_FILE_GLOB_ERROR_SUCCESS; + + do { + /* Ignore the current and parent dirs */ + if (!strcmp(".", data.cFileName) || + !strcmp("..", data.cFileName)) { + continue; + } + + /* Avoid an infinite loop */ + if (strchr(data.cFileName, '*')) { + continue; + } + + /* Create a path (prefix + filename + suffix) */ + memcpy(buf, path, p0 - path + 1); + buf[p0 - path + 1] = '\0'; + + if ((strlen(buf) + + strlen(data.cFileName) + + strlen(p1)) >= FLB_FILE_MAX_PATH_LENGTH) { + if (context->flags & + FLB_FILE_GLOB_ABORT_ON_ERROR) { + ret = FLB_FILE_GLOB_ERROR_OVERSIZED_PATH; + + break; + } + else { + continue; + } + } + + strcat(buf, data.cFileName); + + if (strchr(p1, '*')) { + transverse_directory = FLB_FALSE; + + if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + transverse_directory = FLB_TRUE; + } + else if (data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + ret = flb_file_stat(data.cFileName, + &entry_info); + + if (ret != 0) { + if (context->flags & + FLB_FILE_GLOB_ABORT_ON_ERROR) { + ret = FLB_FILE_GLOB_ERROR_NO_FILE; + + break; + } + } + + if (FLB_FILE_ISDIR(entry_info.mode)) { + transverse_directory = FLB_TRUE; + } + } + + if (transverse_directory) { + strcat(buf, p1); + + ret = limited_win32_glob(context, buf); + + if (ret != FLB_FILE_GLOB_ERROR_SUCCESS && + ret != FLB_FILE_GLOB_ERROR_NO_FILE && + ret != FLB_FILE_GLOB_ERROR_NO_MATCHES) { + if (context->flags & + FLB_FILE_GLOB_ABORT_ON_ERROR) { + break; + } + } + + continue; + } + } + + strcat(buf, p1); + + ret = limited_win32_glob_append_entry(context, buf, 0); + + if (ret != FLB_FILE_GLOB_ERROR_SUCCESS && + ret != FLB_FILE_GLOB_ERROR_NO_FILE) { + if (context->flags & + FLB_FILE_GLOB_ABORT_ON_ERROR) { + break; + } + } + + ret = FLB_FILE_GLOB_ERROR_SUCCESS; + } while (FindNextFileA(h, &data) != 0); + + FindClose(h); + + if (!(context->flags & + FLB_FILE_GLOB_ABORT_ON_ERROR)) { + ret = FLB_FILE_GLOB_ERROR_SUCCESS; + } + + return ret; +} + +int flb_file_glob_start(struct flb_file_glob_context *context, + const char *path, + uint64_t flags) +{ + + int tilde_expansion_attempted; + struct flb_file_stat path_stat; + int result; + + if (context == NULL) { + return -1; + } + + memset(context, 0, sizeof(struct flb_file_glob_context)); + + context->inner_context = + flb_calloc(1, sizeof(struct flb_file_glob_inner_context)); + + if (context->inner_context == NULL) { + return -2; + } + + cfl_list_init(&context->inner_context->results); + + context->inner_context->flags = 0; + context->flags = flags; + + if (flags & FLB_FILE_GLOB_ABORT_ON_ERROR) { + context->inner_context->flags |= FLB_FILE_GLOB_ABORT_ON_ERROR; + } + + context->path = flb_strdup(path); + + if (context->path == NULL) { + flb_file_glob_clean(context); + + return -3; + } + + return limited_win32_glob(context->inner_context, + context->path); +} + +void flb_file_glob_clean(struct flb_file_glob_context *context) +{ + struct cfl_list *iterator_backup; + struct cfl_list *iterator; + struct flb_file_glob_inner_entry *entry; + + if (context != NULL) { + if (context->path != NULL) { + flb_free(context->path); + } + + if (context->inner_context != NULL) { + cfl_list_foreach_safe(iterator, + iterator_backup, + &context->inner_context->results) { + entry = cfl_list_entry(iterator, + struct flb_file_glob_inner_entry, + _head); + + if (entry->path != NULL) { + flb_free(entry->path); + } + + cfl_list_del(&entry->_head); + + flb_free(entry); + } + + flb_free(context->inner_context); + } + + memset(context, 0, sizeof(struct flb_file_glob_context)); + } + +} + +int flb_file_glob_fetch(struct flb_file_glob_context *context, + char **result) +{ + + if (context == NULL) { + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + if (result == NULL) { + return FLB_FILE_GLOB_ERROR_NO_MEMORY; + } + + *result = NULL; + + if (context->inner_context->index >= + context->inner_context->entries) { + return FLB_FILE_GLOB_ERROR_NO_MORE_RESULTS; + } + + if (context->inner_context->current_entry == NULL) { + context->inner_context->current_entry = + cfl_list_entry_first(&context->inner_context->results, + struct flb_file_glob_inner_entry, + _head); + } + else { + context->inner_context->current_entry = + cfl_list_entry_next(&context->inner_context->current_entry->_head, + struct flb_file_glob_inner_entry, + _head, + &context->inner_context->results); + } + + *result = context->inner_context->current_entry->path; + + context->inner_context->index++; + + return FLB_FILE_GLOB_ERROR_SUCCESS; +} + +#endif \ No newline at end of file From 4d2d5fbcd32ddad932ff78abf250b4bd8745e039 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Wed, 25 Sep 2024 17:55:46 +0200 Subject: [PATCH 56/70] input_thread: fixed use after free (CID 509982) Signed-off-by: Leonardo Alminana --- src/flb_input_thread.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/flb_input_thread.c b/src/flb_input_thread.c index 43eb1ec905e..52966121759 100644 --- a/src/flb_input_thread.c +++ b/src/flb_input_thread.c @@ -189,6 +189,15 @@ static FLB_INLINE int engine_handle_event(flb_pipefd_t fd, int mask, static void input_thread_instance_destroy(struct flb_input_thread_instance *thi) { + if (thi->notification_channels_initialized == FLB_TRUE) { + mk_event_channel_destroy(thi->evl, + thi->notification_channels[0], + thi->notification_channels[1], + &thi->notification_event); + + thi->notification_channels_initialized = FLB_FALSE; + } + if (thi->evl) { mk_event_loop_destroy(thi->evl); } @@ -209,15 +218,6 @@ static void input_thread_instance_destroy(struct flb_input_thread_instance *thi) mk_event_closesocket(thi->ch_thread_events[1]); } - if (thi->notification_channels_initialized == FLB_TRUE) { - mk_event_channel_destroy(thi->evl, - thi->notification_channels[0], - thi->notification_channels[1], - &thi->notification_event); - - thi->notification_channels_initialized = FLB_FALSE; - } - flb_tp_destroy(thi->tp); flb_free(thi); } From 2bd01c16e56ac5775dbc7bf06d811eab173ecd37 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Wed, 25 Sep 2024 18:01:44 +0200 Subject: [PATCH 57/70] in_blob: fixed leak (CID 509979) Signed-off-by: Leonardo Alminana --- plugins/in_blob/blob.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins/in_blob/blob.c b/plugins/in_blob/blob.c index 206d58947f7..66d982ba91d 100644 --- a/plugins/in_blob/blob.c +++ b/plugins/in_blob/blob.c @@ -687,6 +687,8 @@ static int in_blob_notification(struct flb_input_instance *in_context, "(filename suffix concatentation error)", notification->path); + cfl_sds_destroy(new_filename); + break; } @@ -703,6 +705,8 @@ static int in_blob_notification(struct flb_input_instance *in_context, notification->path); } + cfl_sds_destroy(new_filename); + break; } } @@ -786,6 +790,8 @@ static int in_blob_notification(struct flb_input_instance *in_context, "(filename suffix concatentation error)", notification->path); + cfl_sds_destroy(new_filename); + break; } @@ -802,6 +808,8 @@ static int in_blob_notification(struct flb_input_instance *in_context, notification->path); } + cfl_sds_destroy(new_filename); + break; } } From 8eca355c6ded9292840282b84c242a31f21949f3 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Wed, 25 Sep 2024 18:05:27 +0200 Subject: [PATCH 58/70] out_azure_blob: fixed leak (CID 509977) Signed-off-by: Leonardo Alminana --- plugins/out_azure_blob/azure_blob.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index 88d07fff3f3..e0a5af56ed8 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -786,6 +786,8 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context flb_plg_error(ctx->ins, "blob file '%s' (id=%" PRIu64 ") notification " \ "delivery error %d", file_path, file_id, ret); + + flb_notification_cleanup(notification); } } } @@ -841,6 +843,8 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context flb_plg_error(ctx->ins, "blob file '%s' (id=%" PRIu64 ") notification " \ "delivery error %d", file_path, file_id, ret); + + flb_notification_cleanup(notification); } } From d7b9b26ecf07f94f34226405c6b6444b66036136 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Wed, 25 Sep 2024 18:08:19 +0200 Subject: [PATCH 59/70] out_azure_blob: fixed leak (CID 509970) Signed-off-by: Leonardo Alminana --- plugins/out_azure_blob/azure_blob.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index e0a5af56ed8..91247eea305 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -372,6 +372,9 @@ static int send_blob(struct flb_config *config, block_id = azb_block_blob_id_logs(&ms); if (!block_id) { flb_plg_error(ctx->ins, "could not generate block id"); + + cfl_sds_destroy(ref_name); + return FLB_RETRY; } uri = azb_block_blob_uri(ctx, tag, block_id, ms); From 7a867d30b540e9a26dac3d866975f20579109dde Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Thu, 26 Sep 2024 00:09:22 +0200 Subject: [PATCH 60/70] in_blob: fixed windows compatibility issues Signed-off-by: Leonardo Alminana --- plugins/in_blob/blob.c | 15 +++++++++++++-- plugins/in_blob/win32_glob.c | 32 ++++++++++++++++---------------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/plugins/in_blob/blob.c b/plugins/in_blob/blob.c index 66d982ba91d..ab6bff72b48 100644 --- a/plugins/in_blob/blob.c +++ b/plugins/in_blob/blob.c @@ -28,8 +28,12 @@ #include #include + +#ifndef FLB_SYSTEM_WINDOWS #include #include +#endif + #include #include "blob.h" @@ -43,6 +47,7 @@ #define GLOB_TILDE 1<<2 /* use GNU Libc value */ #define UNSUP_TILDE 1 +#ifndef FLB_SYSTEM_WINDOWS /* we need these extra headers for path resolution */ #include #include @@ -118,6 +123,12 @@ static char *expand_tilde(const char *path) return path; } +#else +static char* expand_tilde(const char* path) +{ + return NULL; +} +#endif #endif static inline int do_glob(const char *pattern, @@ -134,8 +145,8 @@ static inline int do_glob(const char *pattern, /* Save current values */ new_flags = flags; +#if defined(UNSUP_TILDE) && !defined(FLB_SYSTEM_WINDOWS) if (flags & GLOB_TILDE) { -#ifdef UNSUP_TILDE /* * Some libc libraries like Musl do not support GLOB_TILDE for tilde * expansion. A workaround is to use wordexp(3) but looking at it @@ -156,8 +167,8 @@ static inline int do_glob(const char *pattern, /* remove unused flag */ new_flags &= ~GLOB_TILDE; -#endif } +#endif /* invoke glob with new parameters */ ret = glob(pattern, new_flags, NULL, pglob); diff --git a/plugins/in_blob/win32_glob.c b/plugins/in_blob/win32_glob.c index b3828587c0f..ef042b34452 100644 --- a/plugins/in_blob/win32_glob.c +++ b/plugins/in_blob/win32_glob.c @@ -60,6 +60,10 @@ #define GLOB_NOMATCH FLB_FILE_GLOB_ERROR_NO_MATCHES #endif +#ifndef GLOB_ERR +#define GLOB_ERR FLB_FILE_GLOB_ABORT_ON_ERROR +#endif + struct flb_file_glob_inner_entry { char *path; struct cfl_list _head; @@ -83,8 +87,9 @@ typedef struct glob_t { struct flb_file_glob_context inner_context; char **gl_pathv; size_t gl_pathc; +}; -} +typedef struct glob_t glob_t; static int flb_file_glob_start(struct flb_file_glob_context *context, const char *path, @@ -104,7 +109,7 @@ static void globfree(glob_t *context) context->gl_pathv = NULL; } - flb_file_glob_clean(context->inner_context); + flb_file_glob_clean(&context->inner_context); } static int glob(const char *path, @@ -121,7 +126,7 @@ static int glob(const char *path, result = flb_file_glob_start(context, path, flags); if (result == FLB_FILE_GLOB_ERROR_SUCCESS) { - entries = cfl_list_size(context->inner_context->results); + entries = cfl_list_size(&context->inner_context.inner_context->results); context->gl_pathv = flb_calloc(entries, sizeof(char *)); @@ -132,7 +137,7 @@ static int glob(const char *path, } for (index = 0 ; index < entries ; index++) { - result = flb_file_glob_fetch(context->internal_context, + result = flb_file_glob_fetch(&context->inner_context, &context->gl_pathv[index]); if (result != FLB_FILE_GLOB_ERROR_SUCCESS) { @@ -262,11 +267,11 @@ static int limited_win32_glob_append_entry( { char entry_path_buffer[FLB_FILE_MAX_PATH_LENGTH]; char *entry_path; - struct flb_file_stat entry_info; + struct stat entry_info; int result; struct flb_file_glob_inner_entry *entry; - result = flb_file_stat(path, &entry_info); + result = stat(path, &entry_info); if (result != 0) { result = FLB_FILE_GLOB_ERROR_NO_FILE; @@ -337,7 +342,7 @@ static int limited_win32_glob(struct flb_file_glob_inner_context *context, WIN32_FIND_DATA data; struct flb_file_glob_inner_entry *entry; int transverse_directory; - struct flb_file_stat entry_info; + struct stat entry_info; if (strlen(path) >= FLB_FILE_MAX_PATH_LENGTH) { return FLB_FILE_GLOB_ERROR_OVERSIZED_PATH; @@ -417,8 +422,7 @@ static int limited_win32_glob(struct flb_file_glob_inner_context *context, transverse_directory = FLB_TRUE; } else if (data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - ret = flb_file_stat(data.cFileName, - &entry_info); + ret = stat(data.cFileName, &entry_info); if (ret != 0) { if (context->flags & @@ -428,10 +432,6 @@ static int limited_win32_glob(struct flb_file_glob_inner_context *context, break; } } - - if (FLB_FILE_ISDIR(entry_info.mode)) { - transverse_directory = FLB_TRUE; - } } if (transverse_directory) { @@ -482,9 +482,9 @@ int flb_file_glob_start(struct flb_file_glob_context *context, uint64_t flags) { - int tilde_expansion_attempted; - struct flb_file_stat path_stat; - int result; + int tilde_expansion_attempted; + struct stat path_stat; + int result; if (context == NULL) { return -1; From 284013c69c6d32c764bc5fed607debeb86bc5c78 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Thu, 26 Sep 2024 00:10:37 +0200 Subject: [PATCH 61/70] out_azure_blob: fixed warnings Signed-off-by: Leonardo Alminana --- plugins/out_azure_blob/azure_blob.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index 91247eea305..ec82ee9278c 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -790,7 +790,7 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context "blob file '%s' (id=%" PRIu64 ") notification " \ "delivery error %d", file_path, file_id, ret); - flb_notification_cleanup(notification); + flb_notification_cleanup(¬ification->base); } } } @@ -847,7 +847,7 @@ static void cb_azb_blob_file_upload(struct flb_config *config, void *out_context "blob file '%s' (id=%" PRIu64 ") notification " \ "delivery error %d", file_path, file_id, ret); - flb_notification_cleanup(notification); + flb_notification_cleanup(¬ification->base); } } @@ -1065,7 +1065,6 @@ static int cb_worker_init(void *data, struct flb_config *config) /* worker teardown */ static int cb_worker_exit(void *data, struct flb_config *config) { - int ret; struct worker_info *info; struct flb_azure_blob *ctx = data; From bce1a8661992c36d6f6a2c39ec451c362756e9fa Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Thu, 26 Sep 2024 18:25:15 +0200 Subject: [PATCH 62/70] out_azure_blob: added missing function Signed-off-by: Leonardo Alminana --- plugins/in_blob/blob.c | 12 +++++++++++- plugins/in_blob/win32_glob.c | 4 ++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/plugins/in_blob/blob.c b/plugins/in_blob/blob.c index ab6bff72b48..54e4fe6ece7 100644 --- a/plugins/in_blob/blob.c +++ b/plugins/in_blob/blob.c @@ -131,6 +131,15 @@ static char* expand_tilde(const char* path) #endif #endif + + +#ifndef FLB_SYSTEM_WINDOWS +static int is_directory(char *path, struct stat *fs_entry_metadata) +{ + return S_ISDIR(fs_entry_metadata->st_mode); +} +#endif + static inline int do_glob(const char *pattern, int flags, void *not_used, @@ -334,7 +343,8 @@ static ssize_t recursive_file_search(struct blob_ctx *ctx, continue; } - if (S_ISDIR(fs_entry_metadata.st_mode)) { + if (is_directory(glob_context.gl_pathv[index], + fs_entry_metadata.st_mode)) { local_path[0] = '\0'; cfl_sds_len_set(local_path, 0); diff --git a/plugins/in_blob/win32_glob.c b/plugins/in_blob/win32_glob.c index ef042b34452..5c7453f5aaf 100644 --- a/plugins/in_blob/win32_glob.c +++ b/plugins/in_blob/win32_glob.c @@ -151,6 +151,10 @@ static int glob(const char *path, return result; } +static int is_directory(char* path, struct stat* fs_entry_metadata) +{ + return (fs_entry_metadata->st_mode & S_IFDIR != 0); +} static void reset_errno() { From 6f1da58de29b72406e5b0ad9860587ec29489584 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Thu, 26 Sep 2024 18:40:08 +0200 Subject: [PATCH 63/70] out_azure_blob: fixed typo and added missing constant Signed-off-by: Leonardo Alminana --- plugins/in_blob/win32_glob.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/in_blob/win32_glob.c b/plugins/in_blob/win32_glob.c index 5c7453f5aaf..af2bf774b09 100644 --- a/plugins/in_blob/win32_glob.c +++ b/plugins/in_blob/win32_glob.c @@ -64,6 +64,8 @@ #define GLOB_ERR FLB_FILE_GLOB_ABORT_ON_ERROR #endif +#define FLB_FILE_MAX_PATH_LENGTH PATH_MAX + struct flb_file_glob_inner_entry { char *path; struct cfl_list _head; @@ -83,7 +85,7 @@ struct flb_file_glob_context { char *path; }; -typedef struct glob_t { +struct glob_t { struct flb_file_glob_context inner_context; char **gl_pathv; size_t gl_pathc; @@ -284,7 +286,7 @@ static int limited_win32_glob_append_entry( result = FLB_FILE_GLOB_ERROR_SUCCESS; if (mode_filter != 0) { - if (!FLB_FILE_ISTYPE(entry_info.mode, mode_filter)) { + if (!FLB_FILE_ISTYPE(entry_info.st_mode, mode_filter)) { result = FLB_FILE_GLOB_ERROR_NO_MATCHES; } } From b0081602d196051ec49ae259f85eea71109e3de9 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Thu, 26 Sep 2024 18:45:00 +0200 Subject: [PATCH 64/70] out_azure_blob: added missing macro Signed-off-by: Leonardo Alminana --- plugins/in_blob/win32_glob.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/in_blob/win32_glob.c b/plugins/in_blob/win32_glob.c index af2bf774b09..3efa3532152 100644 --- a/plugins/in_blob/win32_glob.c +++ b/plugins/in_blob/win32_glob.c @@ -66,6 +66,8 @@ #define FLB_FILE_MAX_PATH_LENGTH PATH_MAX +#define FLB_FILE_ISTYPE(m, t) (((m) & 0170000) == t) + struct flb_file_glob_inner_entry { char *path; struct cfl_list _head; From 3841f0fab6b66171ee75442d3ff036df3fc09ac0 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Fri, 27 Sep 2024 09:37:41 +0200 Subject: [PATCH 65/70] engine: added missing header Signed-off-by: Leonardo Alminana --- src/flb_engine.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/flb_engine.c b/src/flb_engine.c index 0efa9ec85db..ac1d0e3fdc7 100644 --- a/src/flb_engine.c +++ b/src/flb_engine.c @@ -53,6 +53,7 @@ #include #include #include +#include #ifdef FLB_HAVE_METRICS #include From 0bf115c4a73f206aa6573edc41cb27e73c1cac7a Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Fri, 27 Sep 2024 09:37:58 +0200 Subject: [PATCH 66/70] in_blob: fixed typos Signed-off-by: Leonardo Alminana --- plugins/in_blob/blob.c | 4 +--- plugins/in_blob/win32_glob.c | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/in_blob/blob.c b/plugins/in_blob/blob.c index 54e4fe6ece7..c4f5175425d 100644 --- a/plugins/in_blob/blob.c +++ b/plugins/in_blob/blob.c @@ -131,8 +131,6 @@ static char* expand_tilde(const char* path) #endif #endif - - #ifndef FLB_SYSTEM_WINDOWS static int is_directory(char *path, struct stat *fs_entry_metadata) { @@ -344,7 +342,7 @@ static ssize_t recursive_file_search(struct blob_ctx *ctx, } if (is_directory(glob_context.gl_pathv[index], - fs_entry_metadata.st_mode)) { + &fs_entry_metadata)) { local_path[0] = '\0'; cfl_sds_len_set(local_path, 0); diff --git a/plugins/in_blob/win32_glob.c b/plugins/in_blob/win32_glob.c index 3efa3532152..9bba4ca1e1c 100644 --- a/plugins/in_blob/win32_glob.c +++ b/plugins/in_blob/win32_glob.c @@ -155,7 +155,7 @@ static int glob(const char *path, return result; } -static int is_directory(char* path, struct stat* fs_entry_metadata) +static int is_directory(char *path, struct stat *fs_entry_metadata) { return (fs_entry_metadata->st_mode & S_IFDIR != 0); } From c3355e262639f17b10be1edce7249a23a3b60af1 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Fri, 27 Sep 2024 09:39:35 +0200 Subject: [PATCH 67/70] input_blob: fixed warning Signed-off-by: Leonardo Alminana --- src/flb_input_blob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_input_blob.c b/src/flb_input_blob.c index b4a5f8b654c..cef22bae680 100644 --- a/src/flb_input_blob.c +++ b/src/flb_input_blob.c @@ -190,7 +190,7 @@ int flb_input_blob_file_register(struct flb_input_instance *ins, return -1; } - ret = flb_log_event_encoder_append_body_cstring(encoder, flb_input_name(ins)); + ret = flb_log_event_encoder_append_body_cstring(encoder, (char *) flb_input_name(ins)); if (ret != FLB_EVENT_ENCODER_SUCCESS) { flb_error("[blob file registration] could not append source plugin name"); flb_log_event_encoder_reset(encoder); From 5600b5bc40f72f8681010c99aa62035fa57a1408 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Sep 2024 12:06:27 -0600 Subject: [PATCH 68/70] tests: runtime: config_map_opts: initialize scheduler Signed-off-by: Eduardo Silva --- tests/runtime/config_map_opts.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/runtime/config_map_opts.c b/tests/runtime/config_map_opts.c index 57b396bb421..378588e9ffa 100644 --- a/tests/runtime/config_map_opts.c +++ b/tests/runtime/config_map_opts.c @@ -1,5 +1,6 @@ /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ #include +#include #include "flb_tests_runtime.h" /* Test functions */ @@ -17,6 +18,7 @@ void flb_test_config_map_opts(void) int in_ffd, r; flb_init_env(); + flb_sched_ctx_init(); ctx = flb_create(); in_ffd = flb_input(ctx, (char *) "tail", NULL); From 478f91da37be1ace0c5fc01723702877c86c04b3 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Sep 2024 13:14:39 -0600 Subject: [PATCH 69/70] tests: runtime: filter_stdout: initialize scheduler Signed-off-by: Eduardo Silva --- tests/runtime/filter_stdout.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/runtime/filter_stdout.c b/tests/runtime/filter_stdout.c index 835ad8eda7a..21e5c98fbcc 100644 --- a/tests/runtime/filter_stdout.c +++ b/tests/runtime/filter_stdout.c @@ -1,6 +1,8 @@ /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ #include +#include + #include "flb_tests_runtime.h" /* Test data */ @@ -16,7 +18,7 @@ TEST_LIST = { {NULL, NULL} }; -/* +/* * This test case is to check if fluent-bit allows case-insensitive plugin name. * This test is not unique to filter_stdout, but we test here :) , */ @@ -39,6 +41,7 @@ void flb_test_filter_stdout_case_insensitive(void) * In general, macOS requests surely initialization for pthread stuffs. */ flb_init_env(); + flb_sched_ctx_init(); flb_destroy(ctx); } From d801d96ab4e64b4566c2652da567faf612e05e2a Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 27 Sep 2024 13:16:06 -0600 Subject: [PATCH 70/70] tests: runtime: custom_calyptia: initialize scheduler Signed-off-by: Eduardo Silva --- tests/runtime/custom_calyptia_test.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/runtime/custom_calyptia_test.c b/tests/runtime/custom_calyptia_test.c index 5ad30e7a0bf..7a13fa48dd1 100644 --- a/tests/runtime/custom_calyptia_test.c +++ b/tests/runtime/custom_calyptia_test.c @@ -1,6 +1,7 @@ /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ #include +#include #include #include "flb_tests_runtime.h" @@ -17,6 +18,8 @@ void flb_custom_calyptia_pipeline_config_get_test() flb_sds_t cfg; ctx = flb_create(); + flb_sched_ctx_init(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); in_ffd_dummy = flb_input(ctx, (char *) "dummy", NULL); @@ -36,7 +39,7 @@ void flb_custom_calyptia_pipeline_config_get_test() calyptia = flb_custom_new(ctx->config, (char *)"calyptia", NULL); TEST_CHECK(calyptia != NULL); - flb_custom_set_property(calyptia, "api_key", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + flb_custom_set_property(calyptia, "api_key", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); flb_custom_set_property(calyptia, "log_level", "debug"); flb_custom_set_property(calyptia, "log_level", "7DDD2941-3ED6-4B8C-9F84-DD04C4A018A4"); flb_custom_set_property(calyptia, "add_label", "pipeline_id 7DDD2941-3ED6-4B8C-9F84-DD04C4A018A4");