Merge branch 'master' of github.com:lemire/streamvbyte

fast-pack · Jan 2, 2024 · 09076dc · 09076dc
2 parents d49f2b0 + 57cb691
commit 09076dc
Show file tree

Hide file tree

Showing 28 changed files with 589 additions and 369 deletions.
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -23,26 +23,26 @@ jobs:
           - {compiler: gcc-12}
           - {compiler: clang}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Use cmake (debug)
         run: |
           mkdir builddebug &&
           cd builddebug &&
-          cmake .. -DCMAKE_BUILD_TYPE=buildundefsani -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DCMAKE_C_COMPILER=${{matrix.compiler}} &&
+          cmake .. -DCMAKE_BUILD_TYPE=buildundefsani -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -DCMAKE_C_COMPILER=${{matrix.compiler}} &&
           cmake --build .   &&
           ctest --output-on-failure
       - name: Use cmake (release)
         run: |
           mkdir buildrelease &&
           cd buildrelease &&
-          cmake .. -DCMAKE_BUILD_TYPE=Release -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON &&
+          cmake .. -DCMAKE_BUILD_TYPE=Release -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON &&
           cmake --build .   &&
           ctest --output-on-failure
 
       - name: Use cmake (undefined sanitizer)
         run: |
           mkdir buildundefsani &&
           cd buildundefsani &&
-          cmake .. -DSTREAMVBYTE_SANITIZE_UNDEFINED=ON -DCMAKE_BUILD_TYPE=buildundefsani -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON &&
+          cmake .. -DSTREAMVBYTE_SANITIZE_UNDEFINED=ON -DCMAKE_BUILD_TYPE=buildundefsani -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON &&
           cmake --build .   &&
           ctest --output-on-failure
diff --git a/.github/workflows/ubuntu20.yml b/.github/workflows/ubuntu20.yml
@@ -0,0 +1,48 @@
+name: Ubuntu 20.04 CI (GCC 9.4 and 10, LLVM 10 and 11)
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {compiler: gcc}
+          - {compiler: gcc-10}
+          - {compiler: clang}
+          - {compiler: clang-11}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use cmake (debug)
+        run: |
+          mkdir builddebug &&
+          cd builddebug &&
+          cmake .. -DCMAKE_BUILD_TYPE=buildundefsani -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -DCMAKE_C_COMPILER=${{matrix.compiler}} &&
+          cmake --build .   &&
+          ctest --output-on-failure
+      - name: Use cmake (release)
+        run: |
+          mkdir buildrelease &&
+          cd buildrelease &&
+          cmake .. -DCMAKE_BUILD_TYPE=Release -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON &&
+          cmake --build .   &&
+          ctest --output-on-failure
+
+      - name: Use cmake (undefined sanitizer)
+        run: |
+          mkdir buildundefsani &&
+          cd buildundefsani &&
+          cmake .. -DSTREAMVBYTE_SANITIZE_UNDEFINED=ON -DCMAKE_BUILD_TYPE=buildundefsani -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON &&
+          cmake --build .   &&
+          ctest --output-on-failure
diff --git a/.github/workflows/ubuntu22.yml b/.github/workflows/ubuntu22.yml
@@ -25,26 +25,26 @@ jobs:
           - {compiler: clang-13}
           - {compiler: clang}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Use cmake (debug)
         run: |
           mkdir builddebug &&
           cd builddebug &&
-          cmake .. -DCMAKE_BUILD_TYPE=buildundefsani -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DCMAKE_C_COMPILER=${{matrix.compiler}} &&
+          cmake .. -DCMAKE_BUILD_TYPE=buildundefsani -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DCMAKE_C_COMPILER=${{matrix.compiler}} -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON &&
           cmake --build .   &&
           ctest --output-on-failure
       - name: Use cmake (release)
         run: |
           mkdir buildrelease &&
           cd buildrelease &&
-          cmake .. -DCMAKE_BUILD_TYPE=Release -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON &&
+          cmake .. -DCMAKE_BUILD_TYPE=Release -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON &&
           cmake --build .   &&
           ctest --output-on-failure
 
       - name: Use cmake (undefined sanitizer)
         run: |
           mkdir buildundefsani &&
           cd buildundefsani &&
-          cmake .. -DSTREAMVBYTE_SANITIZE_UNDEFINED=ON -DCMAKE_BUILD_TYPE=buildundefsani -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON &&
+          cmake .. -DSTREAMVBYTE_SANITIZE_UNDEFINED=ON -DCMAKE_BUILD_TYPE=buildundefsani -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON &&
           cmake --build .   &&
           ctest --output-on-failure
diff --git a/.github/workflows/vs.yml b/.github/workflows/vs.yml
@@ -29,10 +29,10 @@ jobs:
           - {gen: Visual Studio 17 2022, mode: -T ClangCL, arch: x64, static: OFF}
     steps:
     - name: checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Configure
       run: |
-        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} "${{matrix.mode}}" -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -B build
+        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} "${{matrix.mode}}" -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -B build
     - name: Build Debug
       run: cmake --build build --config Debug --verbose
     - name: Build Release
@@ -44,4 +44,4 @@ jobs:
     - name: Run Debug tests
       run: |
         cd build
-        ctest -C Debug --output-on-failure
+        ctest -C Debug --output-on-failure
diff --git a/.github/workflows/vs16.yml b/.github/workflows/vs16.yml
@@ -29,10 +29,10 @@ jobs:
           - {gen: Visual Studio 16 2019, mode: -T ClangCL, arch: x64, static: OFF}
     steps:
     - name: checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Configure
       run: |
-        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} "${{matrix.mode}}" -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -B build
+        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} "${{matrix.mode}}" -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -DSTREAMVBYTE_ENABLE_EXAMPLES=ON -DSTREAMVBYTE_ENABLE_TESTS=ON -DSTREAMVBYTE_WALL=ON -DSTREAMVBYTE_WERROR=ON -B build
     - name: Build Debug
       run: cmake --build build --config Debug --verbose
     - name: Build Release
@@ -44,4 +44,4 @@ jobs:
     - name: Run Debug tests
       run: |
         cd build
-        ctest -C Debug --output-on-failure
+        ctest -C Debug --output-on-failure
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,6 +1,4 @@
 cmake_minimum_required(VERSION 3.3)
-
-set(CMAKE_MACOSX_RPATH OFF)
 project(STREAMVBYTE VERSION "1.0.0")
 
 set(STREAMVBYTE_LIB_VERSION "1.0.0" CACHE STRING "streamvbyte library version")
@@ -9,6 +7,59 @@ set(STREAMVBYTE_LIB_SOVERSION "1" CACHE STRING "streamvbyte library soversion")
 set(CMAKE_C_STANDARD 99)
 set(CMAKE_C_STANDARD_REQUIRED ON)
 
+include(CheckCCompilerFlag)
+include(GNUInstallDirs)
+
+option(STREAMVBYTE_WERROR "Treat warnings as errors." OFF)
+option(STREAMVBYTE_WALL "Emit all warnings during compilation." ON)
+
+if(STREAMVBYTE_WERROR)
+    if(MSVC)
+        check_c_compiler_flag("/WX" WERROR_MSVC_SUPPORTED)
+        if(WERROR_MSVC_SUPPORTED)
+            add_compile_options(/WX)
+        endif()
+        # Wall + WX may throw errors from the corecrt headers. Workaround:
+        if(STREAMVBYTE_WALL)
+            check_c_compiler_flag("/W3" W3_MSVC_SUPPORTED)
+            if(W3_MSVC_SUPPORTED)
+                add_compile_options(/W3)
+            endif()
+            check_c_compiler_flag("/w34714" SUPPRESS_C34714_MSVC_SUPPORTED)
+            if(SUPPRESS_C34714_MSVC_SUPPORTED)
+                add_compile_options(/w34714)
+            endif()
+            check_c_compiler_flag("/wd5045" SUPPRESS_SPECTRE_MSVC_SUPPORTED)
+            if(SUPPRESS_SPECTRE_MSVC_SUPPORTED)
+                add_compile_options(/wd5045)
+            endif()
+            check_c_compiler_flag("/sdl" SDL_MSVC_SUPPORTED)
+            if(SDL_MSVC_SUPPORTED)
+                add_compile_options(/sdl)
+            endif()
+        endif()
+    else()
+        check_c_compiler_flag(-Werror WERROR_GNU_SUPPORTED)
+        if(WERROR_GNU_SUPPORTED)
+            add_compile_options(-Werror)
+        endif()
+    endif()
+endif()
+
+if(STREAMVBYTE_WALL AND NOT(STREAMVBYTE_WERROR))
+    if(MSVC)
+        check_c_compiler_flag("/Wall" WALL_MSVC_SUPPORTED)
+        if(WALL_MSVC_SUPPORTED)
+            add_compile_options(/Wall)
+        endif()
+    else()
+        check_c_compiler_flag(-Wall WALL_GNU_SUPPORTED)
+        if(WALL_GNU_SUPPORTED)
+            add_compile_options(-Wall)
+        endif()
+    endif()
+endif()
+
 option(STREAMVBYTE_SANITIZE "Sanitize addresses" OFF)
 
 if(NOT CMAKE_BUILD_TYPE)
@@ -82,9 +133,14 @@ install(
         ${PROJECT_SOURCE_DIR}/include/streamvbyte.h
         ${PROJECT_SOURCE_DIR}/include/streamvbytedelta.h
         ${PROJECT_SOURCE_DIR}/include/streamvbyte_zigzag.h
-    DESTINATION include
+    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+)
+install(
+    TARGETS streamvbyte
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
 )
-install(TARGETS streamvbyte DESTINATION lib)
 
 option(STREAMVBYTE_SANITIZE_UNDEFINED "Sanitize undefined behavior" OFF)
 if(STREAMVBYTE_SANITIZE_UNDEFINED)
@@ -124,6 +180,7 @@ if(STREAMVBYTE_ENABLE_TESTS)
     # unit
     add_executable(unit ${PROJECT_SOURCE_DIR}/tests/unit.c)
     target_link_libraries(unit streamvbyte)
+    target_include_directories(unit PRIVATE ${PROJECT_SOURCE_DIR}/src)
 
     enable_testing()
 

diff --git a/README.md b/README.md
@@ -67,7 +67,10 @@ from the input buffer (these bytes are read but never used).
 
 
 
-## 1. Building with CMake:
+### 1. Building with CMake:
+
+We expect a recent CMake. Please make sure that your version of CMake is up-to-date or you may
+need to adapt our instructions.
 
 The cmake build system also offers a `libstreamvbyte_static` static library
 (`libstreamvbyte_static` under linux) in addition to
@@ -76,42 +79,44 @@ The cmake build system also offers a `libstreamvbyte_static` static library
 `-DCMAKE_INSTALL_PREFIX:PATH=/path/to/install` is optional.
 Defaults to /usr/local{include,lib}
 
+
+
 ```
-mkdir build
-cd build
-cmake .. -DCMAKE_BUILD_TYPE=Release \
+cmake -DCMAKE_BUILD_TYPE=Release \
          -DCMAKE_INSTALL_PREFIX:PATH=/path/to/install \
 	 -DSTREAMVBYTE_ENABLE_EXAMPLES=ON \
-	 -DSTREAMVBYTE_ENABLE_TESTS=ON
-make install
+	 -DSTREAMVBYTE_ENABLE_TESTS=ON -B build
 
+cmake --build build
 # run the tests like:
-ctest -V
+ctest --test-dir build
 
 ```
 
+#### Installation with CMake
 
-## 2. Building with Makefile:
+```
+cmake --install build 
+```
 
-      make
-      ./unit
+#### Benchmarking with CMake
 
-Signed integers
------------------
 
-We do not directly support signed integers, but you can use fast functions to convert signed integers to unsigned integers.
+After building, you may run our benchmark as follows:
 
-```C
+```
+./build/test/perf
+```
 
-#include "streamvbyte_zigzag.h"
+The benchmarks are not currently built under Windows.
 
-zigzag_encode(mysignedints, myunsignedints, number); // mysignedints => myunsignedints
 
-zigzag_decode(myunsignedints, mysignedints, number); // myunsignedints => mysignedints
-```
+### 2. Building with Makefile:
 
-Installation
-----------------
+      make
+      ./unit
+
+#### Installation with Makefile
 
 You can install the library (as a dynamic library) on your machine if you have root access:
 
@@ -123,8 +128,8 @@ To uninstall, simply type:
 
 It is recommended that you try ``make dyntest`` before proceeding.
 
-Benchmarking
------------------
+#### Benchmarking with Makefile
+
 
 You can try to benchmark the speed in this manner:
 
@@ -133,6 +138,21 @@ You can try to benchmark the speed in this manner:
 
 Make sure to run ``make test`` before, as a sanity test.
 
+
+Signed integers
+-----------------
+
+We do not directly support signed integers, but you can use fast functions to convert signed integers to unsigned integers.
+
+```C
+
+#include "streamvbyte_zigzag.h"
+
+zigzag_encode(mysignedints, myunsignedints, number); // mysignedints => myunsignedints
+
+zigzag_decode(myunsignedints, mysignedints, number); // myunsignedints => mysignedints
+```
+
 Technical posts
 ---------------
 
@@ -155,6 +175,7 @@ Stream VByte in other languages
 * There is a [Rust version](https://bitbucket.org/marshallpierce/stream-vbyte-rust) by Marshall Pierce.
 * There is a [Go version](https://github.com/nelz9999/stream-vbyte-go) by Nelz.
 * There is an accelerated [Go version](https://github.com/theMPatel/streamvbyte-simdgo) by Milan Patel.
+* There is a [Zig version](https://github.com/fulcrum-so/streamvbyte-zig) by Nick Gates.
 
 Format Specification
 ---------------------

diff --git a/examples/example.c b/examples/example.c
@@ -4,12 +4,17 @@
 
 #include "streamvbyte.h"
 
-int main() {
-	int N = 5000;
+#ifdef __clang__
+#pragma clang diagnostic ignored "-Wdeclaration-after-statement"
+#pragma clang diagnostic ignored "-Wunused-variable"
+#endif
+
+int main(void) {
+	uint32_t N = 5000U;
 	uint32_t * datain = malloc(N * sizeof(uint32_t));
 	uint8_t * compressedbuffer = malloc(streamvbyte_max_compressedbytes(N));
 	uint32_t * recovdata = malloc(N * sizeof(uint32_t));
-	for (int k = 0; k < N; ++k)
+	for (uint32_t k = 0; k < N; ++k)
 		datain[k] = 120;
 	size_t compsize = streamvbyte_encode(datain, N, compressedbuffer); // encoding
 	// here the result is stored in compressedbuffer using compsize bytes