Skip to content

Commit

Permalink
in_ebpf: initial version of the plugin (experimental)
Browse files Browse the repository at this point in the history
This is a proposal to implement a ebpf trace ingestor
plugin to allow sending traces from in kernel functions and
userland through uprobes.

This initial implementation has 3 traces implemented:
bind (tcp), malloc (uprobe) and signals (kernel trace).

Events types are known and defined in the fluent-bit codebase and
those has to be implemented by the ebpf program to follow when submitted
into the ring buffer.

Signed-off-by: Jorge Niedbalski <[email protected]>
  • Loading branch information
niedbalski committed Nov 11, 2024
1 parent 282923d commit cb98794
Show file tree
Hide file tree
Showing 45 changed files with 476,996 additions and 5 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/unit-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ jobs:
- name: Setup environment
run: |
sudo apt-get update
sudo apt-get install -y gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev
sudo apt-get install -y gcc-7 g++-7 clang-6.0 libsystemd-dev gcovr libyaml-dev libbpf-dev linux-tools-common
sudo ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true
- uses: actions/checkout@v4
Expand Down Expand Up @@ -142,7 +142,7 @@ jobs:
- name: Setup environment
run: |
sudo apt-get update
sudo apt-get install -y gcc-9 g++-9 clang-12 cmake flex bison libsystemd-dev gcovr libyaml-dev
sudo apt-get install -y gcc-9 g++-9 clang-12 cmake flex bison libsystemd-dev gcovr libyaml-dev libbpf-dev linux-tools-common
sudo ln -s /usr/bin/llvm-symbolizer-12 /usr/bin/llvm-symbolizer || true
- name: Build and test with actuated runners
Expand Down Expand Up @@ -195,7 +195,7 @@ jobs:
--volume "/etc/machine-id:/etc/machine-id"
install: |
apt-get update
apt-get install -y gcc-7 g++-7 clang-6.0 libyaml-dev cmake flex bison libssl-dev #libsystemd-dev
apt-get install -y gcc-7 g++-7 clang-6.0 libyaml-dev cmake flex bison libssl-dev libbpf-dev linux-tools-common#libsystemd-dev
ln -s /usr/bin/llvm-symbolizer-6.0 /usr/bin/llvm-symbolizer || true
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90
Expand Down
4 changes: 2 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ workflow/
*.key
*.log
*.tar.gz

*.o
# clangd files
.cache/
compile_commands.json
Expand All @@ -40,4 +40,4 @@ workflow/
# examples
examples/wasi_serde_json/target/
# WASM test data
tests/runtime/wasm/go/*.wasm
tests/runtime/wasm/go/*.wasm
30 changes: 30 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1067,6 +1067,36 @@ else()
set(FLB_ARROW OFF)
endif()

# EBPF Support
# ============
if (FLB_IN_EBPF)
find_package(PkgConfig)

# Check for libbpf with pkg-config
pkg_check_modules(LIBBPF libbpf>=0.5.0)

if (LIBBPF_FOUND)
message(STATUS "libbpf found: ${LIBBPF_LIBRARIES}")
include_directories(${LIBBPF_INCLUDE_DIRS})
list(APPEND EXTRA_LIBS ${LIBBPF_LIBRARIES})
else()
# Manually find the library if pkg-config fails
find_library(LIBBPF_LIBRARY NAMES bpf REQUIRED)
if (LIBBPF_LIBRARY)
message(STATUS "Found libbpf: ${LIBBPF_LIBRARY}")
list(APPEND EXTRA_LIBS ${LIBBPF_LIBRARY})
else()
if (FLB_SYSTEM_LINUX)
message(FATAL_ERROR "libbpf is required on Linux. Please install libbpf or ensure it is in your library path.")
else()
message(STATUS "libbpf is not found. Disabling eBPF support.")
set(FLB_IN_EBPF OFF)
endif()
endif()
endif()

endif()

# Pthread Local Storage
# =====================
# By default we expect the compiler already support thread local storage
Expand Down
1 change: 1 addition & 0 deletions cmake/plugins_options.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ DEFINE_OPTION(FLB_IN_WINLOG "Enable Windows Log input plugin"
DEFINE_OPTION(FLB_IN_WINDOWS_EXPORTER_METRICS "Enable windows exporter metrics input plugin" ON)
DEFINE_OPTION(FLB_IN_WINEVTLOG "Enable Windows EvtLog input plugin" OFF)
DEFINE_OPTION(FLB_IN_WINSTAT "Enable Windows Stat input plugin" OFF)
DEFINE_OPTION(FLB_IN_EBPF "Enable Linux eBPF input plugin" OFF)

# Processors
# ==========
Expand Down
1 change: 1 addition & 0 deletions cmake/windows-setup.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ if(FLB_WINDOWS_DEFAULTS)
set(FLB_IN_STORAGE_BACKLOG Yes)
set(FLB_IN_EMITTER Yes)
set(FLB_IN_PODMAN_METRICS No)
set(FLB_IN_EBPF No)
set(FLB_IN_ELASTICSEARCH Yes)
set(FLB_IN_SPLUNK Yes)
set(FLB_IN_PROMETHEUS_REMOTE_WRITE Yes)
Expand Down
1 change: 1 addition & 0 deletions plugins/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,7 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
REGISTER_IN_PLUGIN("in_docker_events")
REGISTER_IN_PLUGIN("in_podman_metrics")
REGISTER_IN_PLUGIN("in_process_exporter_metrics")
REGISTER_IN_PLUGIN("in_ebpf")
endif()

if(${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
Expand Down
96 changes: 96 additions & 0 deletions plugins/in_ebpf/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
cmake_minimum_required(VERSION 3.0)

# Define source files for the main plugin
file(GLOB_RECURSE src
"in_ebpf.c"
"traces/**/handler.c"
)

# Determine architecture and set flags accordingly
if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
set(ARCH_FLAG "-D__TARGET_ARCH_x86_64")
set(VMLINUX_PATH "${CMAKE_SOURCE_DIR}/plugins/in_ebpf/traces/includes/external/gadget/amd64")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
set(ARCH_FLAG "-D__TARGET_ARCH_arm64")
set(VMLINUX_PATH "${CMAKE_SOURCE_DIR}/plugins/in_ebpf/traces/includes/external/gadget/arm64")
else()
message(FATAL_ERROR "Unsupported architecture: ${CMAKE_SYSTEM_PROCESSOR}")
endif()

# Include directories for external headers, common headers, and generated skeletons
include_directories(
${CMAKE_SOURCE_DIR}/plugins/in_ebpf/traces/includes/external
${CMAKE_SOURCE_DIR}/plugins/in_ebpf/traces/includes
)

# Create an interface library for gadget includes
add_library(gadget INTERFACE)
target_include_directories(gadget INTERFACE ${CMAKE_SOURCE_DIR}/plugins/in_ebpf/traces/includes/external/gadget)

# Find all bpf.c files in the traces directory
file(GLOB_RECURSE TRACE_C_FILES ${CMAKE_SOURCE_DIR}/plugins/in_ebpf/traces/*/bpf.c)

# Create a list to hold all the object files and skeleton headers that will be generated
set(TRACE_OBJ_FILES "")
set(TRACE_SKEL_HEADERS "")

# Iterate over each trace bpf.c file to generate corresponding .o and .skel.h files
foreach(TRACE_C_FILE ${TRACE_C_FILES})
# Get the filename and parent directory name (for uniqueness)
get_filename_component(TRACE_FILE_NAME ${TRACE_C_FILE} NAME_WE)
get_filename_component(TRACE_PARENT_DIR ${TRACE_C_FILE} DIRECTORY)
get_filename_component(TRACE_PARENT_DIR_NAME ${TRACE_PARENT_DIR} NAME)

# Ensure the output filenames maintain the original "trace_" prefix
set(TRACE_BASE_NAME "trace_${TRACE_PARENT_DIR_NAME}")

# Set unique names by including the parent directory name in the output paths
set(TRACE_OBJ_FILE ${CMAKE_BINARY_DIR}/plugins/in_ebpf/traces/includes/generated/${TRACE_BASE_NAME}.o)
set(TRACE_SKEL_HEADER ${CMAKE_BINARY_DIR}/plugins/in_ebpf/traces/includes/generated/${TRACE_BASE_NAME}.skel.h)

# Compile each bpf.c file to its corresponding .o file
add_custom_command(
OUTPUT ${TRACE_OBJ_FILE}
COMMAND clang
-target bpf
${ARCH_FLAG} # Use architecture-specific flag
-O2 # Optional: Optimization level
-g # Optional: Debug info
-I${CMAKE_SOURCE_DIR}/plugins/in_ebpf/traces/includes/external
-I${CMAKE_SOURCE_DIR}/plugins/in_ebpf/traces/includes
-I${VMLINUX_PATH} # Include the correct vmlinux.h based on architecture
-c ${TRACE_C_FILE}
-o ${TRACE_OBJ_FILE}
DEPENDS ${TRACE_C_FILE}
)

# Generate skeleton header for each compiled BPF object file
add_custom_command(
OUTPUT ${TRACE_SKEL_HEADER}
COMMAND bpftool gen skeleton ${TRACE_OBJ_FILE} > ${TRACE_SKEL_HEADER}
DEPENDS ${TRACE_OBJ_FILE}
COMMENT "Generating skeleton ${TRACE_SKEL_HEADER} from ${TRACE_OBJ_FILE}"
)

# Add generated object and skeleton files to their respective lists
list(APPEND TRACE_OBJ_FILES ${TRACE_OBJ_FILE})
list(APPEND TRACE_SKEL_HEADERS ${TRACE_SKEL_HEADER})
endforeach()

# Create a custom target specifically for generating eBPF skeletons
add_custom_target(generate_skeletons DEPENDS ${TRACE_SKEL_HEADERS})

# Create a custom target to compile all eBPF programs (all trace bpf.c files)
add_custom_target(compile_ebpf ALL DEPENDS ${TRACE_OBJ_FILES} ${TRACE_SKEL_HEADERS})

# Ensure that the custom target depends on the gadget interface library (for include paths)
add_dependencies(compile_ebpf gadget)

# Include generated skeleton headers in the main plugin
include_directories(${CMAKE_BINARY_DIR}/plugins/in_ebpf/traces/includes/generated)

# Declare the Fluent Bit plugin (using the default compiler for the main plugin)
FLB_PLUGIN(in_ebpf "${src}" "")

# Link necessary libraries
target_link_libraries(flb-plugin-in_ebpf gadget -lbpf -lelf -lz)
Loading

0 comments on commit cb98794

Please sign in to comment.