github-featured/alibaba--zvec
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat: slim down zvec dynamic libraries (#627)

Browse Source
This commit is contained in:
lichen2015
2026-08-20 08:01:06 +08:00
committed by GitHub
parent bd772e0d74
commit ae9e61df30
57 changed files with 5015 additions and 2159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/04-android-build.yml
+10 -36
View File
@@ -71,33 +71,8 @@ jobs:
sdkmanager --install "system-images;android-${{ matrix.api }};google_apis_playstore;x86_64" 2>/dev/null || \
sdkmanager --install "system-images;android-${{ matrix.api }};default;x86_64"
# ── Step 1: build host protoc (using HOST compiler, NOT NDK) ───────
- name: Cache host protoc
uses: actions/cache@v6
with:
path: build_host
key: ${{ runner.os }}-host-protoc-${{ hashFiles('thirdparty/protobuf/**', 'CMakeLists.txt') }}
restore-keys: |
${{ runner.os }}-host-protoc-
- name: 'Step 1: Build host protoc'
shell: bash
run: |
if [ ! -f "build_host/bin/protoc" ]; then
git submodule foreach --recursive 'git stash --include-untracked' 2>/dev/null || true
cmake -S . -B build_host \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_TOOLCHAIN_FILE="" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-G Ninja
cmake --build build_host --target protoc --parallel
else
echo "Using cached host protoc"
fi
# ── Step 2: cross-compile zvec + tests for Android ─────────────────
- name: 'Step 2: Cross-compile zvec and tests'
# ── Step 1: cross-compile zvec + tests for Android ─────────────────────────
- name: 'Step 1: Cross-compile zvec and tests'
shell: bash
env:
BUILD_DIR: build_android_${{ matrix.abi }}
@@ -123,7 +98,6 @@ jobs:
-DAUTO_DETECT_ARCH=OFF \
-DENABLE_WERROR=ON \
-DCMAKE_INSTALL_PREFIX="$BUILD_DIR/install" \
-DGLOBAL_CC_PROTOBUF_PROTOC="$GITHUB_WORKSPACE/build_host/bin/protoc" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
@@ -150,8 +124,8 @@ jobs:
echo "Building ${#TEST_NAMES[@]} test executables..."
ninja -C "$BUILD_DIR" -j$(nproc) "${TEST_NAMES[@]}"
# ── Step 3: start emulator ─────────────────────────────────────────
- name: 'Step 3: Start Android emulator'
# ── Step 2: start emulator ─────────────────────────────────────────
- name: 'Step 2: Start Android emulator'
shell: bash
run: |
AVD_NAME="zvec_test_avd"
@@ -244,8 +218,8 @@ jobs:
echo "Device ABI: $(adb shell getprop ro.product.cpu.abi | tr -d '\r')"
echo "ABI list : $(adb shell getprop ro.product.cpu.abilist | tr -d '\r')"
# ── Step 4: run unit tests on emulator ─────────────────────────────
- name: 'Step 4: Run unit tests on emulator'
# ── Step 3: run unit tests on emulator ─────────────────────────────
- name: 'Step 3: Run unit tests on emulator'
shell: bash
env:
BUILD_DIR: build_android_${{ matrix.abi }}
@@ -385,8 +359,8 @@ jobs:
fi
echo "All tests passed!"
# ── Step 5: build and run examples ─────────────────────────────────
- name: 'Step 5: Build and run examples'
# ── Step 4: build and run examples ─────────────────────────────────
- name: 'Step 4: Build and run examples'
shell: bash
env:
BUILD_DIR: build_android_${{ matrix.abi }}
@@ -405,8 +379,8 @@ jobs:
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
cmake --build "$EXAMPLES_BUILD" --parallel
# Reuse the shared-library directory from Step 4; push again in
# case Step 4 was skipped or the directory was cleaned.
# Reuse the shared-library directory from Step 3; push again in
# case Step 3 was skipped or the directory was cleaned.
DEVICE_LIB_DIR="/data/local/tmp/zvec_tests/lib"
adb shell "mkdir -p $DEVICE_LIB_DIR" 2>/dev/null || true
SO_COUNT=0
.github/workflows/06-ios-build.yml
-20
View File
@@ -37,25 +37,6 @@ jobs:
key: ios-${{ matrix.platform }}
max-size: 150M
- name: Cache host protoc build
uses: actions/cache@v6
with:
path: build_host
key: macos-host-protoc-${{ hashFiles('thirdparty/protobuf/**', 'CMakeLists.txt') }}
restore-keys: |
macos-host-protoc-
- name: Build host protoc
run: |
if [ ! -f "build_host/bin/protoc" ]; then
cmake -S . -B build_host -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
cmake --build build_host --target protoc --parallel $(sysctl -n hw.ncpu)
else
echo "Using cached host protoc"
fi
- name: Configure and Build
run: |
git submodule foreach --recursive 'git stash --include-untracked' || true
@@ -73,7 +54,6 @@ jobs:
-DBUILD_TOOLS=OFF \
-DENABLE_WERROR=ON \
-DCMAKE_INSTALL_PREFIX="./install" \
-DGLOBAL_CC_PROTOBUF_PROTOC="$GITHUB_WORKSPACE/build_host/bin/protoc" \
-DIOS=ON \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
.github/workflows/clang_tidy.yml
+2 -4
View File
@@ -134,8 +134,7 @@ jobs:
path: |
build/external
build/thirdparty
build/src/db/proto
key: clang-tidy-headers-${{ runner.os }}-${{ hashFiles('thirdparty/**/*.cmake', 'thirdparty/**/CMakeLists.txt', 'thirdparty/**/*.patch', 'src/db/proto/*.proto') }}-${{ steps.submodule_hash.outputs.value }}
key: clang-tidy-headers-${{ runner.os }}-${{ hashFiles('thirdparty/**/*.cmake', 'thirdparty/**/CMakeLists.txt', 'thirdparty/**/*.patch') }}-${{ steps.submodule_hash.outputs.value }}
- name: Build generated headers only
if: steps.tidy_files.outputs.any_tidy_files == 'true' && steps.cache_headers.outputs.cache-hit != 'true'
@@ -149,8 +148,7 @@ jobs:
path: |
build/external
build/thirdparty
build/src/db/proto
key: clang-tidy-headers-${{ runner.os }}-${{ hashFiles('thirdparty/**/*.cmake', 'thirdparty/**/CMakeLists.txt', 'thirdparty/**/*.patch', 'src/db/proto/*.proto') }}-${{ steps.submodule_hash.outputs.value }}
key: clang-tidy-headers-${{ runner.os }}-${{ hashFiles('thirdparty/**/*.cmake', 'thirdparty/**/CMakeLists.txt', 'thirdparty/**/*.patch') }}-${{ steps.submodule_hash.outputs.value }}
- name: Run clang-tidy on changed files (parallel)
if: steps.tidy_files.outputs.any_tidy_files == 'true'
.gitmodules
-4
View File
@@ -24,10 +24,6 @@
path = thirdparty/glog/glog-0.5.0
url = https://github.com/google/glog.git
ignore = dirty
[submodule "thirdparty/protobuf/protobuf-3.21.12"]
path = thirdparty/protobuf/protobuf-3.21.12
url = https://github.com/protocolbuffers/protobuf.git
ignore = dirty
[submodule "thirdparty/lz4/lz4-1.9.4"]
path = thirdparty/lz4/lz4-1.9.4
url = https://github.com/lz4/lz4.git
CMakeLists.txt
+19 -1
View File
@@ -39,6 +39,9 @@ if(MSVC)
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror=return-type")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=return-type")
# Fine-grained dead code elimination: place each function/data in its own section
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ffunction-sections -fdata-sections")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffunction-sections -fdata-sections")
endif()
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT IOS)
@@ -66,6 +69,21 @@ if(NOT DEFINED PROJECT_ROOT_DIR OR NOT PROJECT_ROOT_DIR)
set(PROJECT_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE PATH "Root directory of the project" FORCE)
endif()
# LTO (Link-Time Optimization) — helps on Linux with --gc-sections but may
# increase size on macOS due to aggressive inlining. Default OFF; enable
# explicitly with -DZVEC_ENABLE_LTO=ON when targeting Linux.
option(ZVEC_ENABLE_LTO "Enable Link-Time Optimization for Release builds" OFF)
if(ZVEC_ENABLE_LTO AND CMAKE_BUILD_TYPE STREQUAL "Release")
include(CheckIPOSupported)
check_ipo_supported(RESULT IPO_SUPPORTED OUTPUT IPO_OUTPUT)
if(IPO_SUPPORTED)
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON)
message(STATUS "LTO enabled for Release build")
else()
message(STATUS "LTO not supported: ${IPO_OUTPUT}")
endif()
endif()
message(STATUS "PROJECT_ROOT_DIR = ${PROJECT_ROOT_DIR}")
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
@@ -164,7 +182,7 @@ cc_directories(src)
cc_directories(tests)
add_custom_target(clang_tidy_deps DEPENDS zvec_proto ARROW.BUILD glog gflags Lz4.BUILD)
add_custom_target(clang_tidy_deps DEPENDS ARROW.BUILD glog gflags Lz4.BUILD)
if(BUILD_TOOLS)
cc_directories(tools)
cmake/bazel.cmake
+1 -1
View File
@@ -1502,7 +1502,7 @@ function(_find_protobuf _VERSION)
"${protoc_INCLUDE_DIR}" CACHE STRING "Protobuf includes"
)
set(
CC_PROTOBUF_LIBS_${_VERSION} libprotobuf CACHE STRING "Protobuf libraries"
CC_PROTOBUF_LIBS_${_VERSION} libprotobuf-lite CACHE STRING "Protobuf libraries"
)
endfunction()
python/tests/detail/test_collection_open.py
+9 -3
View File
@@ -874,6 +874,10 @@ class TestCollectionOpen:
# Shared variables to collect results from threads
results = []
errors = []
# Successful opens must stay alive until every thread has tried:
# closing inside the thread would release the LOCK file early and let
# a later-starting thread legitimately acquire it.
opened = []
# Lock for thread-safe operations
lock = threading.Lock()
@@ -887,9 +891,7 @@ class TestCollectionOpen:
collection_path_result = coll.path
with lock:
results.append((thread_id, collection_path_result))
# Close the collection if opened successfully
if hasattr(coll, "close") and coll is not None:
coll.close()
opened.append(coll)
except Exception as e:
with lock:
errors.append((thread_id, str(e)))
@@ -913,6 +915,10 @@ class TestCollectionOpen:
f"Expected exactly four failures, but got {len(errors)}"
)
# Release the lock held by the successful open before further checks.
for coll in opened:
coll.close()
# Additional verification: check that the successful open has a valid collection
_, successful_collection_path = results[0]
assert successful_collection_path is not None, (
scripts/build_android.sh
+6 -38
View File
@@ -19,7 +19,7 @@ API_LEVEL=${1:-35}
BUILD_TYPE=${2:-"Release"}
CORE_COUNT=$(sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4)
# ── Android SDK paths (set later, after host protoc build) ───────────
# ── Android SDK paths ────────────────────────────────────────────────
ANDROID_SDK_ROOT=${ANDROID_SDK_ROOT:-$HOME/Library/Android/sdk}
ANDROID_NDK_HOME=${ANDROID_NDK_HOME:-$(ls -d "$ANDROID_SDK_ROOT/ndk/"* 2>/dev/null | sort -V | tail -1)}
@@ -39,40 +39,9 @@ if [ ! -d "$ANDROID_NDK_HOME" ]; then
exit 1
fi
# ── Step 1: build host protoc (using HOST compiler, NOT NDK) ─────────
# ── Step 1: cross-compile zvec + tests for Android ───────────────────
echo ""
echo ">>> Step 1: Building protoc for host..."
HOST_BUILD_DIR="build_host"
git submodule foreach --recursive 'git stash --include-untracked' 2>/dev/null || true
if [ ! -f "$CURRENT_DIR/$HOST_BUILD_DIR/bin/protoc" ]; then
# Explicitly avoid NDK toolchain for host build
cmake -S . -B "$HOST_BUILD_DIR" \
-DCMAKE_BUILD_TYPE="$BUILD_TYPE" \
-DCMAKE_TOOLCHAIN_FILE="" \
-G Ninja
cmake --build "$HOST_BUILD_DIR" --target protoc -j"$CORE_COUNT"
else
echo " (cached — skipping)"
fi
PROTOC_EXECUTABLE=$CURRENT_DIR/$HOST_BUILD_DIR/bin/protoc
echo ">>> Step 1: Done (protoc=$PROTOC_EXECUTABLE)"
# ── Now export Android env vars for cross-compilation ────────────────
export ANDROID_SDK_ROOT
export ANDROID_HOME=$ANDROID_SDK_ROOT
export ANDROID_NDK_HOME
export CMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake
export PATH=$PATH:$ANDROID_SDK_ROOT/cmdline-tools/latest/bin
export PATH=$PATH:$ANDROID_SDK_ROOT/platform-tools
export PATH=$PATH:$ANDROID_SDK_ROOT/emulator
export PATH=$PATH:$ANDROID_NDK_HOME
# ── Step 2: cross-compile zvec + tests for Android ───────────────────
echo ""
echo ">>> Step 2: Cross-compiling zvec for Android ($ABI, API $API_LEVEL)..."
echo ">>> Step 1: Cross-compiling zvec for Android ($ABI, API $API_LEVEL)..."
# reset thirdparty so the cross toolchain can patch cleanly
git submodule foreach --recursive 'git stash --include-untracked' 2>/dev/null || true
@@ -94,7 +63,6 @@ cmake -S . -B "$BUILD_DIR" -G Ninja \
-DENABLE_NATIVE=OFF \
-DAUTO_DETECT_ARCH=OFF \
-DCMAKE_INSTALL_PREFIX="$BUILD_DIR/install" \
-DGLOBAL_CC_PROTOBUF_PROTOC="$PROTOC_EXECUTABLE"
echo " Building library..."
cmake --build "$BUILD_DIR" -j"$CORE_COUNT"
@@ -130,11 +98,11 @@ fi
echo " Building ${#TEST_NAMES[@]} test executables..."
ninja -C "$BUILD_DIR" -j"$CORE_COUNT" "${TEST_NAMES[@]}"
echo ">>> Step 2: Done"
echo ">>> Step 1: Done"
# ── Step 3: collect test binaries ────────────────────────────────────
# ── Step 2: collect test binaries ────────────────────────────────────
echo ""
echo ">>> Step 3: Collecting test binaries..."
echo ">>> Step 2: Collecting test binaries..."
TEST_BINS=()
for name in "${TEST_NAMES[@]}"; do
scripts/build_ios.sh
+6 -23
View File
@@ -34,24 +34,8 @@ echo " Architecture: $ARCH"
echo " Build Type: $BUILD_TYPE"
echo " iOS Deployment Target: $IOS_DEPLOYMENT_TARGET"
# step1: use host env to compile protoc
echo "step1: building protoc for host..."
git submodule foreach --recursive 'git stash --include-untracked'
HOST_BUILD_DIR="build_host"
mkdir -p $HOST_BUILD_DIR
cd $HOST_BUILD_DIR
cmake -DCMAKE_BUILD_TYPE="$BUILD_TYPE" ..
make -j protoc
PROTOC_EXECUTABLE=$CURRENT_DIR/$HOST_BUILD_DIR/bin/protoc
cd $CURRENT_DIR
echo "step1: Done!!!"
# step2: cross build zvec for iOS
echo "step2: building zvec for iOS..."
# step1: cross build zvec for iOS
echo "step1: building zvec for iOS..."
# reset thirdparty directory
git submodule foreach --recursive 'git stash --include-untracked'
@@ -79,7 +63,6 @@ cmake \
-DBUILD_PYTHON_BINDINGS=OFF \
-DBUILD_TOOLS=OFF \
-DCMAKE_INSTALL_PREFIX="./install" \
-DGLOBAL_CC_PROTOBUF_PROTOC=$PROTOC_EXECUTABLE \
-DIOS=ON \
../
@@ -87,11 +70,11 @@ echo "building..."
CORE_COUNT=$(sysctl -n hw.ncpu)
make -j$CORE_COUNT
echo "step2: Done!!!"
echo "step1: Done!!!"
# step3: build and run all unit tests on simulator
# step2: build and run all unit tests on simulator
if [ "$PLATFORM" != "OS" ]; then
echo "step3: building and running unit tests on simulator..."
echo "step2: building and running unit tests on simulator..."
make -j$CORE_COUNT unittest
@@ -163,7 +146,7 @@ sys.exit(1)
exit 1
fi
echo "step3: Done!!!"
echo "step2: Done!!!"
else
echo "Skipping tests (device build cannot run on simulator)"
fi
scripts/gcov.sh
+1 -1
View File
@@ -8,7 +8,7 @@ keep_info=false
script_dir=$(cd "$(dirname "$0")"; pwd)
source_base=$(dirname "$script_dir")
filter_list="'*/tests/*' '*/thirdparty/*' '*/deps/*' '*/proto/*' '*/external/*' '*/sqlengine/antlr/gen/*'"
filter_list="'*/tests/*' '*/thirdparty/*' '*/deps/*' '*/external/*' '*/sqlengine/antlr/gen/*'"
while getopts t:p:o:zk option; do
case "$option" in
src/CMakeLists.txt
+24
View File
@@ -143,6 +143,30 @@ function(zvec_add_all_in_one_shared TARGET_NAME OUTPUT_NAME)
endif()
endif()
# Restrict exported symbols to zvec's own public C++ API (namespace zvec,
# which includes nested zvec::ailego / zvec::core / zvec::turbo /
# zvec::reranker). All third-party symbols (arrow, rocksdb, parquet,
# antlr4) are hidden, and unreferenced code is dead-stripped to
# reduce the shared library size. The public C++ headers never expose
# third-party types, so hiding them is safe for external C++ consumers.
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
if(APPLE)
set(_ZVEC_ALLIN_EXPORTS ${CMAKE_CURRENT_SOURCE_DIR}/exported_symbols_cpp.txt)
target_link_options(${TARGET_NAME} PRIVATE
"-Wl,-exported_symbols_list,${_ZVEC_ALLIN_EXPORTS}"
"-Wl,-dead_strip")
set_property(TARGET ${TARGET_NAME} APPEND PROPERTY
LINK_DEPENDS ${_ZVEC_ALLIN_EXPORTS})
elseif(UNIX)
set(_ZVEC_ALLIN_EXPORTS ${CMAKE_CURRENT_SOURCE_DIR}/exported_symbols_cpp.lds)
target_link_options(${TARGET_NAME} PRIVATE
"-Wl,--version-script,${_ZVEC_ALLIN_EXPORTS}"
"-Wl,--gc-sections")
set_property(TARGET ${TARGET_NAME} APPEND PROPERTY
LINK_DEPENDS ${_ZVEC_ALLIN_EXPORTS})
endif()
endif()
target_include_directories(${TARGET_NAME}
PUBLIC
$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/src/generated>
src/binding/c/CMakeLists.txt
+15
View File
@@ -202,6 +202,21 @@ target_compile_options(zvec_c_api PRIVATE
$<$<CXX_COMPILER_ID:Clang>:-Wall -Wextra -Wpedantic>
)
# Linker-level dead code elimination with explicit export list.
# Only symbols in the export list are kept as global; the linker can then
# aggressively strip all code unreachable from those roots.
if(APPLE)
target_link_options(zvec_c_api PRIVATE
-Wl,-dead_strip
-Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/exported_symbols.txt
)
elseif(UNIX)
target_link_options(zvec_c_api PRIVATE
-Wl,--gc-sections
-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exported_symbols.lds
)
endif()
# Strip symbols in release builds to reduce library size
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
if(UNIX AND NOT APPLE)
src/binding/c/exported_symbols.lds
+376
View File
@@ -0,0 +1,376 @@
{ global:
zvec_bin_create;
zvec_byte_array_create;
zvec_byte_array_destroy;
zvec_check_version;
zvec_clear_error;
zvec_collection_add_column;
zvec_collection_alter_column;
zvec_collection_close;
zvec_collection_create_and_open;
zvec_collection_create_index;
zvec_collection_delete;
zvec_collection_delete_by_filter;
zvec_collection_delete_with_results;
zvec_collection_destroy;
zvec_collection_drop_column;
zvec_collection_drop_index;
zvec_collection_fetch;
zvec_collection_flush;
zvec_collection_get_options;
zvec_collection_get_schema;
zvec_collection_get_stats;
zvec_collection_insert;
zvec_collection_insert_with_results;
zvec_collection_multi_query;
zvec_collection_open;
zvec_collection_optimize;
zvec_collection_options_create;
zvec_collection_options_destroy;
zvec_collection_options_get_enable_mmap;
zvec_collection_options_get_max_buffer_size;
zvec_collection_options_get_read_only;
zvec_collection_options_set_enable_mmap;
zvec_collection_options_set_max_buffer_size;
zvec_collection_options_set_read_only;
zvec_collection_query;
zvec_collection_schema_add_field;
zvec_collection_schema_add_index;
zvec_collection_schema_alter_field;
zvec_collection_schema_create;
zvec_collection_schema_destroy;
zvec_collection_schema_drop_field;
zvec_collection_schema_drop_index;
zvec_collection_schema_get_all_field_names;
zvec_collection_schema_get_field;
zvec_collection_schema_get_forward_field;
zvec_collection_schema_get_forward_field_names;
zvec_collection_schema_get_forward_field_names_with_index;
zvec_collection_schema_get_forward_fields;
zvec_collection_schema_get_forward_fields_with_index;
zvec_collection_schema_get_max_doc_count_per_segment;
zvec_collection_schema_get_name;
zvec_collection_schema_get_vector_field;
zvec_collection_schema_get_vector_fields;
zvec_collection_schema_has_field;
zvec_collection_schema_has_index;
zvec_collection_schema_set_max_doc_count_per_segment;
zvec_collection_schema_set_name;
zvec_collection_schema_validate;
zvec_collection_stats_destroy;
zvec_collection_stats_get_doc_count;
zvec_collection_stats_get_index_completeness;
zvec_collection_stats_get_index_count;
zvec_collection_stats_get_index_name;
zvec_collection_update;
zvec_collection_update_with_results;
zvec_collection_upsert;
zvec_collection_upsert_with_results;
zvec_config_data_create;
zvec_config_data_destroy;
zvec_config_data_get_brute_force_by_keys_ratio;
zvec_config_data_get_fts_brute_force_by_keys_ratio;
zvec_config_data_get_invert_to_forward_scan_ratio;
zvec_config_data_get_jieba_dict_dir;
zvec_config_data_get_log_type;
zvec_config_data_get_memory_limit;
zvec_config_data_get_optimize_thread_count;
zvec_config_data_get_query_thread_count;
zvec_config_data_set_brute_force_by_keys_ratio;
zvec_config_data_set_fts_brute_force_by_keys_ratio;
zvec_config_data_set_invert_to_forward_scan_ratio;
zvec_config_data_set_jieba_dict_dir;
zvec_config_data_set_log_config;
zvec_config_data_set_memory_limit;
zvec_config_data_set_optimize_thread_count;
zvec_config_data_set_query_thread_count;
zvec_config_log_create_console;
zvec_config_log_create_file;
zvec_config_log_destroy;
zvec_config_log_get_basename;
zvec_config_log_get_dir;
zvec_config_log_get_file_size;
zvec_config_log_get_level;
zvec_config_log_get_overdue_days;
zvec_config_log_is_file_type;
zvec_config_log_set_basename;
zvec_config_log_set_dir;
zvec_config_log_set_file_size;
zvec_config_log_set_level;
zvec_config_log_set_overdue_days;
zvec_data_type_to_string;
zvec_doc_add_field_by_struct;
zvec_doc_add_field_by_value;
zvec_doc_clear;
zvec_doc_create;
zvec_doc_deserialize;
zvec_doc_destroy;
zvec_doc_get_doc_id;
zvec_doc_get_field_count;
zvec_doc_get_field_names;
zvec_doc_get_field_value_basic;
zvec_doc_get_field_value_copy;
zvec_doc_get_field_value_pointer;
zvec_doc_get_operator;
zvec_doc_get_pk_copy;
zvec_doc_get_pk_pointer;
zvec_doc_get_score;
zvec_doc_has_field;
zvec_doc_has_field_value;
zvec_doc_is_empty;
zvec_doc_is_field_null;
zvec_doc_memory_usage;
zvec_doc_merge;
zvec_doc_remove_field;
zvec_doc_serialize;
zvec_doc_set_doc_id;
zvec_doc_set_field_null;
zvec_doc_set_operator;
zvec_doc_set_pk;
zvec_doc_set_score;
zvec_doc_to_detail_string;
zvec_docs_free;
zvec_error_code_to_string;
zvec_field_schema_create;
zvec_field_schema_destroy;
zvec_field_schema_get_data_type;
zvec_field_schema_get_dimension;
zvec_field_schema_get_element_data_size;
zvec_field_schema_get_element_data_type;
zvec_field_schema_get_index_params;
zvec_field_schema_get_index_type;
zvec_field_schema_get_name;
zvec_field_schema_has_index;
zvec_field_schema_has_invert_index;
zvec_field_schema_is_array_type;
zvec_field_schema_is_dense_vector;
zvec_field_schema_is_nullable;
zvec_field_schema_is_sparse_vector;
zvec_field_schema_is_vector_field;
zvec_field_schema_set_data_type;
zvec_field_schema_set_dimension;
zvec_field_schema_set_index_params;
zvec_field_schema_set_name;
zvec_field_schema_set_nullable;
zvec_field_schema_validate;
zvec_float_array_create;
zvec_float_array_destroy;
zvec_free;
zvec_free_field_schema;
zvec_free_str_array;
zvec_free_string;
zvec_free_uint8_array;
zvec_fts_create;
zvec_fts_destroy;
zvec_fts_get_match_string;
zvec_fts_get_query_string;
zvec_fts_set_match_string;
zvec_fts_set_query_string;
zvec_get_default_jieba_dict_dir;
zvec_get_io_backend_description;
zvec_get_io_backend_type;
zvec_get_io_backend_type_name;
zvec_get_last_error;
zvec_get_last_error_details;
zvec_get_version;
zvec_get_version_major;
zvec_get_version_minor;
zvec_get_version_patch;
zvec_group_by_vector_query_create;
zvec_group_by_vector_query_destroy;
zvec_group_by_vector_query_get_field_name;
zvec_group_by_vector_query_get_filter;
zvec_group_by_vector_query_get_group_by_field_name;
zvec_group_by_vector_query_get_group_count;
zvec_group_by_vector_query_get_include_vector;
zvec_group_by_vector_query_get_output_fields;
zvec_group_by_vector_query_get_topk_per_group;
zvec_group_by_vector_query_set_diskann_params;
zvec_group_by_vector_query_set_field_name;
zvec_group_by_vector_query_set_filter;
zvec_group_by_vector_query_set_flat_params;
zvec_group_by_vector_query_set_group_by_field_name;
zvec_group_by_vector_query_set_group_count;
zvec_group_by_vector_query_set_hnsw_params;
zvec_group_by_vector_query_set_include_vector;
zvec_group_by_vector_query_set_ivf_params;
zvec_group_by_vector_query_set_ivf_rabitq_params;
zvec_group_by_vector_query_set_output_fields;
zvec_group_by_vector_query_set_query_params;
zvec_group_by_vector_query_set_query_vector;
zvec_group_by_vector_query_set_topk_per_group;
zvec_group_by_vector_query_set_vamana_params;
zvec_index_params_create;
zvec_index_params_destroy;
zvec_index_params_get_diskann_list_size;
zvec_index_params_get_diskann_max_degree;
zvec_index_params_get_diskann_pq_chunk_num;
zvec_index_params_get_fts_params;
zvec_index_params_get_hnsw_ef_construction;
zvec_index_params_get_hnsw_m;
zvec_index_params_get_invert_params;
zvec_index_params_get_ivf_params;
zvec_index_params_get_ivf_rabitq_params;
zvec_index_params_get_metric_type;
zvec_index_params_get_quantize_type;
zvec_index_params_get_quantizer_enable_rotate;
zvec_index_params_get_type;
zvec_index_params_get_vamana_params;
zvec_index_params_get_vamana_two_pass_build;
zvec_index_params_set_diskann_params;
zvec_index_params_set_fts_params;
zvec_index_params_set_hnsw_params;
zvec_index_params_set_invert_params;
zvec_index_params_set_ivf_params;
zvec_index_params_set_ivf_rabitq_params;
zvec_index_params_set_metric_type;
zvec_index_params_set_quantize_type;
zvec_index_params_set_quantizer_enable_rotate;
zvec_index_params_set_vamana_params;
zvec_index_params_set_vamana_two_pass_build;
zvec_index_type_to_string;
zvec_initialize;
zvec_int64_array_create;
zvec_int64_array_destroy;
zvec_is_initialized;
zvec_malloc;
zvec_metric_type_to_string;
zvec_multi_query_add_sub_query;
zvec_multi_query_create;
zvec_multi_query_destroy;
zvec_multi_query_get_filter;
zvec_multi_query_get_include_vector;
zvec_multi_query_get_output_fields;
zvec_multi_query_get_sub_query_count;
zvec_multi_query_get_topk;
zvec_multi_query_set_filter;
zvec_multi_query_set_include_vector;
zvec_multi_query_set_output_fields;
zvec_multi_query_set_rerank_rrf;
zvec_multi_query_set_rerank_weighted;
zvec_multi_query_set_topk;
zvec_query_params_diskann_create;
zvec_query_params_diskann_destroy;
zvec_query_params_diskann_get_is_linear;
zvec_query_params_diskann_get_is_using_refiner;
zvec_query_params_diskann_get_list_size;
zvec_query_params_diskann_get_radius;
zvec_query_params_diskann_set_is_linear;
zvec_query_params_diskann_set_is_using_refiner;
zvec_query_params_diskann_set_list_size;
zvec_query_params_diskann_set_radius;
zvec_query_params_flat_create;
zvec_query_params_flat_destroy;
zvec_query_params_flat_get_is_linear;
zvec_query_params_flat_get_is_using_refiner;
zvec_query_params_flat_get_radius;
zvec_query_params_flat_get_scale_factor;
zvec_query_params_flat_set_is_linear;
zvec_query_params_flat_set_is_using_refiner;
zvec_query_params_flat_set_radius;
zvec_query_params_flat_set_scale_factor;
zvec_query_params_fts_create;
zvec_query_params_fts_destroy;
zvec_query_params_fts_get_default_operator;
zvec_query_params_fts_set_default_operator;
zvec_query_params_hnsw_create;
zvec_query_params_hnsw_destroy;
zvec_query_params_hnsw_get_ef;
zvec_query_params_hnsw_get_is_linear;
zvec_query_params_hnsw_get_is_using_refiner;
zvec_query_params_hnsw_get_radius;
zvec_query_params_hnsw_set_ef;
zvec_query_params_hnsw_set_is_linear;
zvec_query_params_hnsw_set_is_using_refiner;
zvec_query_params_hnsw_set_radius;
zvec_query_params_ivf_create;
zvec_query_params_ivf_destroy;
zvec_query_params_ivf_get_is_linear;
zvec_query_params_ivf_get_is_using_refiner;
zvec_query_params_ivf_get_nprobe;
zvec_query_params_ivf_get_radius;
zvec_query_params_ivf_get_scale_factor;
zvec_query_params_ivf_rabitq_create;
zvec_query_params_ivf_rabitq_destroy;
zvec_query_params_ivf_rabitq_get_is_linear;
zvec_query_params_ivf_rabitq_get_is_using_refiner;
zvec_query_params_ivf_rabitq_get_nprobe;
zvec_query_params_ivf_rabitq_get_radius;
zvec_query_params_ivf_rabitq_get_scale_factor;
zvec_query_params_ivf_rabitq_set_is_linear;
zvec_query_params_ivf_rabitq_set_is_using_refiner;
zvec_query_params_ivf_rabitq_set_nprobe;
zvec_query_params_ivf_rabitq_set_radius;
zvec_query_params_ivf_rabitq_set_scale_factor;
zvec_query_params_ivf_set_is_linear;
zvec_query_params_ivf_set_is_using_refiner;
zvec_query_params_ivf_set_nprobe;
zvec_query_params_ivf_set_radius;
zvec_query_params_ivf_set_scale_factor;
zvec_query_params_vamana_create;
zvec_query_params_vamana_destroy;
zvec_query_params_vamana_get_ef_search;
zvec_query_params_vamana_get_is_linear;
zvec_query_params_vamana_get_is_using_refiner;
zvec_query_params_vamana_get_radius;
zvec_query_params_vamana_set_ef_search;
zvec_query_params_vamana_set_is_linear;
zvec_query_params_vamana_set_is_using_refiner;
zvec_query_params_vamana_set_radius;
zvec_set_default_jieba_dict_dir;
zvec_shutdown;
zvec_string_array_add;
zvec_string_array_create;
zvec_string_array_destroy;
zvec_string_c_str;
zvec_string_compare;
zvec_string_copy;
zvec_string_create;
zvec_string_create_from_view;
zvec_string_length;
zvec_sub_query_create;
zvec_sub_query_destroy;
zvec_sub_query_get_field_name;
zvec_sub_query_get_num_candidates;
zvec_sub_query_set_diskann_params;
zvec_sub_query_set_field_name;
zvec_sub_query_set_flat_params;
zvec_sub_query_set_fts;
zvec_sub_query_set_fts_params;
zvec_sub_query_set_hnsw_params;
zvec_sub_query_set_ivf_params;
zvec_sub_query_set_ivf_rabitq_params;
zvec_sub_query_set_num_candidates;
zvec_sub_query_set_query_vector;
zvec_sub_query_set_sparse_indices;
zvec_sub_query_set_sparse_values;
zvec_sub_query_set_sparse_vector;
zvec_sub_query_set_vamana_params;
zvec_vector_query_create;
zvec_vector_query_destroy;
zvec_vector_query_get_field_name;
zvec_vector_query_get_filter;
zvec_vector_query_get_fts;
zvec_vector_query_get_include_doc_id;
zvec_vector_query_get_include_vector;
zvec_vector_query_get_output_fields;
zvec_vector_query_get_topk;
zvec_vector_query_set_diskann_params;
zvec_vector_query_set_field_name;
zvec_vector_query_set_filter;
zvec_vector_query_set_flat_params;
zvec_vector_query_set_fts;
zvec_vector_query_set_fts_params;
zvec_vector_query_set_hnsw_params;
zvec_vector_query_set_include_doc_id;
zvec_vector_query_set_include_vector;
zvec_vector_query_set_ivf_params;
zvec_vector_query_set_ivf_rabitq_params;
zvec_vector_query_set_output_fields;
zvec_vector_query_set_query_params;
zvec_vector_query_set_query_vector;
zvec_vector_query_set_topk;
zvec_vector_query_set_vamana_params;
zvec_write_results_free;
local: *;
};
src/binding/c/exported_symbols.txt
+373
View File
@@ -0,0 +1,373 @@
_zvec_bin_create
_zvec_byte_array_create
_zvec_byte_array_destroy
_zvec_check_version
_zvec_clear_error
_zvec_collection_add_column
_zvec_collection_alter_column
_zvec_collection_close
_zvec_collection_create_and_open
_zvec_collection_create_index
_zvec_collection_delete
_zvec_collection_delete_by_filter
_zvec_collection_delete_with_results
_zvec_collection_destroy
_zvec_collection_drop_column
_zvec_collection_drop_index
_zvec_collection_fetch
_zvec_collection_flush
_zvec_collection_get_options
_zvec_collection_get_schema
_zvec_collection_get_stats
_zvec_collection_insert
_zvec_collection_insert_with_results
_zvec_collection_multi_query
_zvec_collection_open
_zvec_collection_optimize
_zvec_collection_options_create
_zvec_collection_options_destroy
_zvec_collection_options_get_enable_mmap
_zvec_collection_options_get_max_buffer_size
_zvec_collection_options_get_read_only
_zvec_collection_options_set_enable_mmap
_zvec_collection_options_set_max_buffer_size
_zvec_collection_options_set_read_only
_zvec_collection_query
_zvec_collection_schema_add_field
_zvec_collection_schema_add_index
_zvec_collection_schema_alter_field
_zvec_collection_schema_create
_zvec_collection_schema_destroy
_zvec_collection_schema_drop_field
_zvec_collection_schema_drop_index
_zvec_collection_schema_get_all_field_names
_zvec_collection_schema_get_field
_zvec_collection_schema_get_forward_field
_zvec_collection_schema_get_forward_field_names
_zvec_collection_schema_get_forward_field_names_with_index
_zvec_collection_schema_get_forward_fields
_zvec_collection_schema_get_forward_fields_with_index
_zvec_collection_schema_get_max_doc_count_per_segment
_zvec_collection_schema_get_name
_zvec_collection_schema_get_vector_field
_zvec_collection_schema_get_vector_fields
_zvec_collection_schema_has_field
_zvec_collection_schema_has_index
_zvec_collection_schema_set_max_doc_count_per_segment
_zvec_collection_schema_set_name
_zvec_collection_schema_validate
_zvec_collection_stats_destroy
_zvec_collection_stats_get_doc_count
_zvec_collection_stats_get_index_completeness
_zvec_collection_stats_get_index_count
_zvec_collection_stats_get_index_name
_zvec_collection_update
_zvec_collection_update_with_results
_zvec_collection_upsert
_zvec_collection_upsert_with_results
_zvec_config_data_create
_zvec_config_data_destroy
_zvec_config_data_get_brute_force_by_keys_ratio
_zvec_config_data_get_fts_brute_force_by_keys_ratio
_zvec_config_data_get_invert_to_forward_scan_ratio
_zvec_config_data_get_jieba_dict_dir
_zvec_config_data_get_log_type
_zvec_config_data_get_memory_limit
_zvec_config_data_get_optimize_thread_count
_zvec_config_data_get_query_thread_count
_zvec_config_data_set_brute_force_by_keys_ratio
_zvec_config_data_set_fts_brute_force_by_keys_ratio
_zvec_config_data_set_invert_to_forward_scan_ratio
_zvec_config_data_set_jieba_dict_dir
_zvec_config_data_set_log_config
_zvec_config_data_set_memory_limit
_zvec_config_data_set_optimize_thread_count
_zvec_config_data_set_query_thread_count
_zvec_config_log_create_console
_zvec_config_log_create_file
_zvec_config_log_destroy
_zvec_config_log_get_basename
_zvec_config_log_get_dir
_zvec_config_log_get_file_size
_zvec_config_log_get_level
_zvec_config_log_get_overdue_days
_zvec_config_log_is_file_type
_zvec_config_log_set_basename
_zvec_config_log_set_dir
_zvec_config_log_set_file_size
_zvec_config_log_set_level
_zvec_config_log_set_overdue_days
_zvec_data_type_to_string
_zvec_doc_add_field_by_struct
_zvec_doc_add_field_by_value
_zvec_doc_clear
_zvec_doc_create
_zvec_doc_deserialize
_zvec_doc_destroy
_zvec_doc_get_doc_id
_zvec_doc_get_field_count
_zvec_doc_get_field_names
_zvec_doc_get_field_value_basic
_zvec_doc_get_field_value_copy
_zvec_doc_get_field_value_pointer
_zvec_doc_get_operator
_zvec_doc_get_pk_copy
_zvec_doc_get_pk_pointer
_zvec_doc_get_score
_zvec_doc_has_field
_zvec_doc_has_field_value
_zvec_doc_is_empty
_zvec_doc_is_field_null
_zvec_doc_memory_usage
_zvec_doc_merge
_zvec_doc_remove_field
_zvec_doc_serialize
_zvec_doc_set_doc_id
_zvec_doc_set_field_null
_zvec_doc_set_operator
_zvec_doc_set_pk
_zvec_doc_set_score
_zvec_doc_to_detail_string
_zvec_docs_free
_zvec_error_code_to_string
_zvec_field_schema_create
_zvec_field_schema_destroy
_zvec_field_schema_get_data_type
_zvec_field_schema_get_dimension
_zvec_field_schema_get_element_data_size
_zvec_field_schema_get_element_data_type
_zvec_field_schema_get_index_params
_zvec_field_schema_get_index_type
_zvec_field_schema_get_name
_zvec_field_schema_has_index
_zvec_field_schema_has_invert_index
_zvec_field_schema_is_array_type
_zvec_field_schema_is_dense_vector
_zvec_field_schema_is_nullable
_zvec_field_schema_is_sparse_vector
_zvec_field_schema_is_vector_field
_zvec_field_schema_set_data_type
_zvec_field_schema_set_dimension
_zvec_field_schema_set_index_params
_zvec_field_schema_set_name
_zvec_field_schema_set_nullable
_zvec_field_schema_validate
_zvec_float_array_create
_zvec_float_array_destroy
_zvec_free
_zvec_free_field_schema
_zvec_free_str_array
_zvec_free_string
_zvec_free_uint8_array
_zvec_fts_create
_zvec_fts_destroy
_zvec_fts_get_match_string
_zvec_fts_get_query_string
_zvec_fts_set_match_string
_zvec_fts_set_query_string
_zvec_get_default_jieba_dict_dir
_zvec_get_io_backend_description
_zvec_get_io_backend_type
_zvec_get_io_backend_type_name
_zvec_get_last_error
_zvec_get_last_error_details
_zvec_get_version
_zvec_get_version_major
_zvec_get_version_minor
_zvec_get_version_patch
_zvec_group_by_vector_query_create
_zvec_group_by_vector_query_destroy
_zvec_group_by_vector_query_get_field_name
_zvec_group_by_vector_query_get_filter
_zvec_group_by_vector_query_get_group_by_field_name
_zvec_group_by_vector_query_get_group_count
_zvec_group_by_vector_query_get_include_vector
_zvec_group_by_vector_query_get_output_fields
_zvec_group_by_vector_query_get_topk_per_group
_zvec_group_by_vector_query_set_diskann_params
_zvec_group_by_vector_query_set_field_name
_zvec_group_by_vector_query_set_filter
_zvec_group_by_vector_query_set_flat_params
_zvec_group_by_vector_query_set_group_by_field_name
_zvec_group_by_vector_query_set_group_count
_zvec_group_by_vector_query_set_hnsw_params
_zvec_group_by_vector_query_set_include_vector
_zvec_group_by_vector_query_set_ivf_params
_zvec_group_by_vector_query_set_ivf_rabitq_params
_zvec_group_by_vector_query_set_output_fields
_zvec_group_by_vector_query_set_query_params
_zvec_group_by_vector_query_set_query_vector
_zvec_group_by_vector_query_set_topk_per_group
_zvec_group_by_vector_query_set_vamana_params
_zvec_index_params_create
_zvec_index_params_destroy
_zvec_index_params_get_diskann_list_size
_zvec_index_params_get_diskann_max_degree
_zvec_index_params_get_diskann_pq_chunk_num
_zvec_index_params_get_fts_params
_zvec_index_params_get_hnsw_ef_construction
_zvec_index_params_get_hnsw_m
_zvec_index_params_get_invert_params
_zvec_index_params_get_ivf_params
_zvec_index_params_get_ivf_rabitq_params
_zvec_index_params_get_metric_type
_zvec_index_params_get_quantize_type
_zvec_index_params_get_quantizer_enable_rotate
_zvec_index_params_get_type
_zvec_index_params_get_vamana_params
_zvec_index_params_get_vamana_two_pass_build
_zvec_index_params_set_diskann_params
_zvec_index_params_set_fts_params
_zvec_index_params_set_hnsw_params
_zvec_index_params_set_invert_params
_zvec_index_params_set_ivf_params
_zvec_index_params_set_ivf_rabitq_params
_zvec_index_params_set_metric_type
_zvec_index_params_set_quantize_type
_zvec_index_params_set_quantizer_enable_rotate
_zvec_index_params_set_vamana_params
_zvec_index_params_set_vamana_two_pass_build
_zvec_index_type_to_string
_zvec_initialize
_zvec_int64_array_create
_zvec_int64_array_destroy
_zvec_is_initialized
_zvec_malloc
_zvec_metric_type_to_string
_zvec_multi_query_add_sub_query
_zvec_multi_query_create
_zvec_multi_query_destroy
_zvec_multi_query_get_filter
_zvec_multi_query_get_include_vector
_zvec_multi_query_get_output_fields
_zvec_multi_query_get_sub_query_count
_zvec_multi_query_get_topk
_zvec_multi_query_set_filter
_zvec_multi_query_set_include_vector
_zvec_multi_query_set_output_fields
_zvec_multi_query_set_rerank_rrf
_zvec_multi_query_set_rerank_weighted
_zvec_multi_query_set_topk
_zvec_query_params_diskann_create
_zvec_query_params_diskann_destroy
_zvec_query_params_diskann_get_is_linear
_zvec_query_params_diskann_get_is_using_refiner
_zvec_query_params_diskann_get_list_size
_zvec_query_params_diskann_get_radius
_zvec_query_params_diskann_set_is_linear
_zvec_query_params_diskann_set_is_using_refiner
_zvec_query_params_diskann_set_list_size
_zvec_query_params_diskann_set_radius
_zvec_query_params_flat_create
_zvec_query_params_flat_destroy
_zvec_query_params_flat_get_is_linear
_zvec_query_params_flat_get_is_using_refiner
_zvec_query_params_flat_get_radius
_zvec_query_params_flat_get_scale_factor
_zvec_query_params_flat_set_is_linear
_zvec_query_params_flat_set_is_using_refiner
_zvec_query_params_flat_set_radius
_zvec_query_params_flat_set_scale_factor
_zvec_query_params_fts_create
_zvec_query_params_fts_destroy
_zvec_query_params_fts_get_default_operator
_zvec_query_params_fts_set_default_operator
_zvec_query_params_hnsw_create
_zvec_query_params_hnsw_destroy
_zvec_query_params_hnsw_get_ef
_zvec_query_params_hnsw_get_is_linear
_zvec_query_params_hnsw_get_is_using_refiner
_zvec_query_params_hnsw_get_radius
_zvec_query_params_hnsw_set_ef
_zvec_query_params_hnsw_set_is_linear
_zvec_query_params_hnsw_set_is_using_refiner
_zvec_query_params_hnsw_set_radius
_zvec_query_params_ivf_create
_zvec_query_params_ivf_destroy
_zvec_query_params_ivf_get_is_linear
_zvec_query_params_ivf_get_is_using_refiner
_zvec_query_params_ivf_get_nprobe
_zvec_query_params_ivf_get_radius
_zvec_query_params_ivf_get_scale_factor
_zvec_query_params_ivf_rabitq_create
_zvec_query_params_ivf_rabitq_destroy
_zvec_query_params_ivf_rabitq_get_is_linear
_zvec_query_params_ivf_rabitq_get_is_using_refiner
_zvec_query_params_ivf_rabitq_get_nprobe
_zvec_query_params_ivf_rabitq_get_radius
_zvec_query_params_ivf_rabitq_get_scale_factor
_zvec_query_params_ivf_rabitq_set_is_linear
_zvec_query_params_ivf_rabitq_set_is_using_refiner
_zvec_query_params_ivf_rabitq_set_nprobe
_zvec_query_params_ivf_rabitq_set_radius
_zvec_query_params_ivf_rabitq_set_scale_factor
_zvec_query_params_ivf_set_is_linear
_zvec_query_params_ivf_set_is_using_refiner
_zvec_query_params_ivf_set_nprobe
_zvec_query_params_ivf_set_radius
_zvec_query_params_ivf_set_scale_factor
_zvec_query_params_vamana_create
_zvec_query_params_vamana_destroy
_zvec_query_params_vamana_get_ef_search
_zvec_query_params_vamana_get_is_linear
_zvec_query_params_vamana_get_is_using_refiner
_zvec_query_params_vamana_get_radius
_zvec_query_params_vamana_set_ef_search
_zvec_query_params_vamana_set_is_linear
_zvec_query_params_vamana_set_is_using_refiner
_zvec_query_params_vamana_set_radius
_zvec_set_default_jieba_dict_dir
_zvec_shutdown
_zvec_string_array_add
_zvec_string_array_create
_zvec_string_array_destroy
_zvec_string_c_str
_zvec_string_compare
_zvec_string_copy
_zvec_string_create
_zvec_string_create_from_view
_zvec_string_length
_zvec_sub_query_create
_zvec_sub_query_destroy
_zvec_sub_query_get_field_name
_zvec_sub_query_get_num_candidates
_zvec_sub_query_set_diskann_params
_zvec_sub_query_set_field_name
_zvec_sub_query_set_flat_params
_zvec_sub_query_set_fts
_zvec_sub_query_set_fts_params
_zvec_sub_query_set_hnsw_params
_zvec_sub_query_set_ivf_params
_zvec_sub_query_set_ivf_rabitq_params
_zvec_sub_query_set_num_candidates
_zvec_sub_query_set_query_vector
_zvec_sub_query_set_sparse_indices
_zvec_sub_query_set_sparse_values
_zvec_sub_query_set_sparse_vector
_zvec_sub_query_set_vamana_params
_zvec_vector_query_create
_zvec_vector_query_destroy
_zvec_vector_query_get_field_name
_zvec_vector_query_get_filter
_zvec_vector_query_get_fts
_zvec_vector_query_get_include_doc_id
_zvec_vector_query_get_include_vector
_zvec_vector_query_get_output_fields
_zvec_vector_query_get_topk
_zvec_vector_query_set_diskann_params
_zvec_vector_query_set_field_name
_zvec_vector_query_set_filter
_zvec_vector_query_set_flat_params
_zvec_vector_query_set_fts
_zvec_vector_query_set_fts_params
_zvec_vector_query_set_hnsw_params
_zvec_vector_query_set_include_doc_id
_zvec_vector_query_set_include_vector
_zvec_vector_query_set_ivf_params
_zvec_vector_query_set_ivf_rabitq_params
_zvec_vector_query_set_output_fields
_zvec_vector_query_set_query_params
_zvec_vector_query_set_query_vector
_zvec_vector_query_set_topk
_zvec_vector_query_set_vamana_params
_zvec_write_results_free
src/binding/python/CMakeLists.txt
+5 -5
View File
@@ -68,11 +68,11 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
"LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports.map"
)
# On musl, dynamic linking would force auditwheel to rewrite the ELF via
# patchelf so it can vendor libstdc++/libgcc. The patchelf versions used in
# current build environments (0.17.x/0.18.x) corrupt protobuf's
# protodesc_cold section on musl 1.2.5 x86_64, causing init crashes.
# Statically linking libstdc++ and libgcc avoids patchelf entirely and
# keeps _zvec self-contained.
# patchelf so it can vendor libstdc++/libgcc. The patchelf versions used
# in current build environments (0.17.x/0.18.x) have corrupted binaries
# on musl 1.2.5 x86_64, causing init crashes. Statically linking
# libstdc++ and libgcc avoids patchelf entirely and keeps _zvec
# self-contained.
if(ZVEC_ON_MUSL)
target_link_options(_zvec PRIVATE -static-libstdc++ -static-libgcc)
message(STATUS "_zvec: statically linking libstdc++ and libgcc on musl")
src/db/CMakeLists.txt
+1 -10
View File
@@ -1,12 +1,6 @@
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
cc_proto_library(
NAME zvec_proto STATIC
SRCS proto/*.proto
PROTOROOT ./
)
cc_directory(common)
cc_directory(index)
cc_directory(sqlengine)
@@ -40,7 +34,7 @@ endif()
cc_library(
NAME zvec STATIC STRICT SRCS_NO_GLOB PACKED ${ZVEC_DB_LIBRARY_OPTIONS}
SRCS ${ALL_DB_SRCS} ${CMAKE_CURRENT_BINARY_DIR}/proto/zvec.pb.cc
SRCS ${ALL_DB_SRCS}
INCS . ${CMAKE_CURRENT_BINARY_DIR}
PUBINCS ${PROJECT_ROOT_DIR}/src/include
LIBS
@@ -50,16 +44,13 @@ cc_library(
roaring
rocksdb
antlr4
libprotobuf
FastPFOR
cppjieba
snowball
Arrow::arrow_static
Arrow::parquet_static
Arrow::arrow_compute
Arrow::arrow_dataset
Arrow::arrow_acero
utf8proc
DEPS zvec_proto
VERSION "${PROXIMA_ZVEC_VERSION}"
)
src/db/index/CMakeLists.txt
+1 -3
View File
@@ -19,15 +19,13 @@ endif()
cc_library(
NAME zvec_index STATIC STRICT
SRCS *.cc segment/*.cc column/vector_column/*.cc column/inverted_column/*.cc column/fts_column/*.cc column/fts_column/tokenizer/*.cc column/fts_column/posting/*.cc column/fts_column/iterator/*.cc storage/*.cc storage/wal/*.cc common/*.cc
SRCS *.cc segment/*.cc column/vector_column/*.cc column/inverted_column/*.cc column/fts_column/*.cc column/fts_column/tokenizer/*.cc column/fts_column/posting/*.cc column/fts_column/iterator/*.cc storage/*.cc storage/wal/*.cc common/*.cc common/manifest/*.cc
LIBS zvec_common
zvec_proto
rocksdb
core_interface
Arrow::arrow_static
Arrow::parquet_static
Arrow::arrow_compute
Arrow::arrow_dataset
cppjieba
snowball
FastPFOR
src/db/index/common/manifest/README.md
+74
View File
@@ -0,0 +1,74 @@
# Manifest Wire Format
This directory holds the **implementation details** of the zvec manifest
on-disk format. The public API deliberately lives outside of it:
- Public API: `db/index/common/manifest_codec.h`
(`ManifestData`, `ManifestCodec::Encode/Decode`; the per-message helpers
declared there are exposed only for unit testing).
- Wire enums: `db/index/common/manifest_enum.h`
(shared with the public `type_helper.h`).
- Consumer: `Version::Load/Save` in `db/index/common/version_manager.cc`.
Files in this directory:
- `manifest_codec.cc` — the field numbers (the `f_*` namespaces) and the
encode/decode logic for every message.
- `pb_wire.{h,cc}` — the minimal protobuf wire-format reader/writer used by
the codec (varint, length-delimited, fixed32/fixed64).
## Why not libprotobuf?
The manifest format **is** the protobuf wire format
(https://protobuf.dev/programming-guides/encoding/), so manifests stay
byte-compatible with the previous protobuf-based implementation. The
libprotobuf/protoc dependency was dropped to slim down the shared library;
`manifest_codec.cc` together with `manifest_enum.h` now defines the format.
## Compatibility rules
The manifest is a persisted artifact: files written by one zvec version must
remain readable by later versions, and — thanks to protobuf's skipping of
unknown fields — also by older versions.
### Adding a field (allowed at any time)
1. Pick the next unused field number in the message's `f_*` namespace in
`manifest_codec.cc`. Field numbers are the identity of a field.
2. Encode it with the matching `Writer::Put*` call. Singular fields holding
proto3 default values are skipped on write, so manifests written by older
versions decode to the default automatically.
3. Read it back in the message's `Decode*` function. Unknown fields are
skipped by readers, so older binaries keep working on newer files.
4. Extend the C++ side (`ManifestData`, index params, ...) as needed.
5. Add test coverage. The golden tests in
`tests/db/index/common/manifest_codec_golden_test.cc` pin the exact bytes
produced by the old protobuf implementation: adding a field must not
change existing golden bytes (default-valued fields are not serialized),
and new fields get their own cases.
### Removing a field (allowed, with care)
1. Stop encoding the field. Existing manifests still decode: the field simply
falls back to its default value on load.
2. Keep the field number in the `f_*` namespace, marked as reserved with a
comment. **Never reuse a retired field number** for a different meaning —
old manifest files would silently decode garbage into the new field.
3. If the removed field carried information that is still required, migrate
it to a new field number first.
### Renaming a field
Only field numbers go over the wire; names may change freely.
### Changing enums
Enums are encoded as their numeric values (`manifest_enum.h` mirrors the old
proto numbers). Existing numbers must never change; add new values with new
numbers. Unknown values read from disk are preserved as-is and mapped to
`UNDEFINED` by the CodeBooks in `type_helper.h`.
### Writer constraint
`pbwire::Writer` does not sort fields. Each message's encode function must
write fields in ascending field-number order to produce canonical bytes.
src/db/index/common/manifest/manifest_codec.cc
+934
View File
@@ -0,0 +1,934 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "db/index/common/manifest_codec.h"
#include <cstdint>
#include <memory>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#include "db/index/common/manifest/pb_wire.h"
#include "db/index/common/manifest_enum.h"
#include "db/index/common/type_helper.h"
namespace zvec {
namespace {
using pbwire::Reader;
using pbwire::Writer;
// Field numbers of the manifest wire format. Changing any of these
// breaks compatibility with existing manifest files.
namespace f_quantizer {
constexpr uint32_t kEnableRotate = 1;
}
namespace f_base {
constexpr uint32_t kMetricType = 1;
constexpr uint32_t kQuantizeType = 2;
constexpr uint32_t kQuantizerParam = 4;
} // namespace f_base
namespace f_invert {
constexpr uint32_t kEnableRangeOptimization = 1;
// Added after the protobuf dependency was dropped; old manifests simply lack
// this field and decode to the default (false).
constexpr uint32_t kEnableExtendedWildcard = 2;
} // namespace f_invert
namespace f_hnsw {
constexpr uint32_t kBase = 1;
constexpr uint32_t kM = 2;
constexpr uint32_t kEfConstruction = 3;
constexpr uint32_t kUseContiguousMemory = 4;
} // namespace f_hnsw
namespace f_hnsw_rabitq {
constexpr uint32_t kBase = 1;
constexpr uint32_t kM = 2;
constexpr uint32_t kEfConstruction = 3;
constexpr uint32_t kTotalBits = 4;
constexpr uint32_t kNumClusters = 5;
constexpr uint32_t kSampleCount = 6;
} // namespace f_hnsw_rabitq
namespace f_ivf_rabitq {
constexpr uint32_t kBase = 1;
constexpr uint32_t kNList = 2;
constexpr uint32_t kTotalBits = 3;
constexpr uint32_t kSampleCount = 4;
} // namespace f_ivf_rabitq
namespace f_flat {
constexpr uint32_t kBase = 1;
}
namespace f_ivf {
constexpr uint32_t kBase = 1;
constexpr uint32_t kNList = 2;
constexpr uint32_t kNIters = 3;
constexpr uint32_t kUseSoar = 4;
} // namespace f_ivf
namespace f_diskann {
constexpr uint32_t kBase = 1;
constexpr uint32_t kMaxDegree = 2;
constexpr uint32_t kListSize = 3;
constexpr uint32_t kPqChunkNum = 4;
} // namespace f_diskann
namespace f_vamana {
constexpr uint32_t kBase = 1;
constexpr uint32_t kMaxDegree = 2;
constexpr uint32_t kSearchListSize = 3;
constexpr uint32_t kAlpha = 4;
constexpr uint32_t kSaturateGraph = 5;
constexpr uint32_t kUseContiguousMemory = 6;
constexpr uint32_t kUseIdMap = 7;
constexpr uint32_t kTwoPassBuild = 8;
} // namespace f_vamana
namespace f_fts {
constexpr uint32_t kTokenizerName = 1;
constexpr uint32_t kFilters = 2;
constexpr uint32_t kExtraParams = 3;
} // namespace f_fts
namespace f_index_params {
constexpr uint32_t kInvert = 1;
constexpr uint32_t kHnsw = 2;
constexpr uint32_t kFlat = 3;
constexpr uint32_t kIvf = 4;
constexpr uint32_t kHnswRabitq = 5;
constexpr uint32_t kVamana = 6;
constexpr uint32_t kFts = 7;
constexpr uint32_t kDiskann = 8;
constexpr uint32_t kIvfRabitq = 9;
} // namespace f_index_params
namespace f_field {
constexpr uint32_t kName = 1;
constexpr uint32_t kDataType = 2;
constexpr uint32_t kDimension = 3;
constexpr uint32_t kNullable = 4;
constexpr uint32_t kIndexParams = 5;
} // namespace f_field
namespace f_collection {
constexpr uint32_t kName = 1;
constexpr uint32_t kFields = 2;
constexpr uint32_t kMaxDocCountPerSegment = 3;
} // namespace f_collection
namespace f_block {
constexpr uint32_t kBlockId = 1;
constexpr uint32_t kBlockType = 2;
constexpr uint32_t kMinDocId = 3;
constexpr uint32_t kMaxDocId = 4;
constexpr uint32_t kDocCount = 5;
constexpr uint32_t kColumns = 6;
} // namespace f_block
namespace f_segment {
constexpr uint32_t kSegmentId = 1;
constexpr uint32_t kPersistedBlocks = 2;
constexpr uint32_t kWritingForwardBlock = 3;
constexpr uint32_t kIndexedVectorFields = 4;
} // namespace f_segment
namespace f_manifest {
constexpr uint32_t kVersion = 1;
constexpr uint32_t kSchema = 2;
constexpr uint32_t kEnableMmap = 3;
constexpr uint32_t kPersistedSegmentMetas = 4;
constexpr uint32_t kWritingSegmentMeta = 5;
constexpr uint32_t kIdMapPathSuffix = 6;
constexpr uint32_t kDeleteSnapshotPathSuffix = 7;
constexpr uint32_t kNextSegmentId = 8;
} // namespace f_manifest
//! Mirror of proto BaseIndexParams, shared by all vector index params.
struct BaseParams {
MetricType metric_type{MetricType::UNDEFINED};
QuantizeType quantize_type{QuantizeType::UNDEFINED};
// Whether the quantizer_param sub-message is present. The protobuf-based
// converter called mutable_quantizer_param() for every index type except
// HNSW_RABITQ, so presence must be reproduced exactly to keep the encoded
// bytes identical.
bool has_quantizer_param{false};
bool enable_rotate{false};
};
void EncodeBase(const BaseParams &base, std::string *out) {
Writer w(out);
w.PutVarint(f_base::kMetricType,
static_cast<uint64_t>(
wire::ToNumber(MetricTypeCodeBook::Get(base.metric_type))));
w.PutVarint(f_base::kQuantizeType,
static_cast<uint64_t>(wire::ToNumber(
QuantizeTypeCodeBook::Get(base.quantize_type))));
if (base.has_quantizer_param) {
std::string quantizer;
Writer qw(&quantizer);
qw.PutBool(f_quantizer::kEnableRotate, base.enable_rotate);
w.PutMessage(f_base::kQuantizerParam, quantizer);
}
}
BaseParams DecodeBase(std::string_view buf) {
BaseParams base;
Reader r(buf);
while (r.Next()) {
switch (r.field()) {
case f_base::kMetricType:
base.metric_type = MetricTypeCodeBook::Get(
wire::FromNumber<wire::MetricType>(r.int32_value()));
break;
case f_base::kQuantizeType:
base.quantize_type = QuantizeTypeCodeBook::Get(
wire::FromNumber<wire::QuantizeType>(r.int32_value()));
break;
case f_base::kQuantizerParam: {
base.has_quantizer_param = true;
Reader qr(r.bytes());
while (qr.Next()) {
if (qr.field() == f_quantizer::kEnableRotate) {
base.enable_rotate = qr.bool_value();
}
}
break;
}
default:
break; // unknown field: already consumed by Next()
}
}
return base;
}
//! Builds the base params of an index that carries a quantizer_param.
template <typename Params>
BaseParams MakeBase(const Params *params) {
BaseParams base;
base.metric_type = params->metric_type();
base.quantize_type = params->quantize_type();
base.has_quantizer_param = true;
base.enable_rotate = params->quantizer_param().enable_rotate();
return base;
}
void EncodeHnsw(const HnswIndexParams *params, std::string *out) {
std::string base;
EncodeBase(MakeBase(params), &base);
Writer w(out);
w.PutMessage(f_hnsw::kBase, base);
w.PutVarint(f_hnsw::kM, static_cast<uint64_t>(params->m()));
w.PutVarint(f_hnsw::kEfConstruction,
static_cast<uint64_t>(params->ef_construction()));
w.PutBool(f_hnsw::kUseContiguousMemory, params->use_contiguous_memory());
}
HnswIndexParams::OPtr DecodeHnsw(std::string_view buf) {
BaseParams base;
int32_t m = 0;
int32_t ef_construction = 0;
bool use_contiguous_memory = false;
Reader r(buf);
while (r.Next()) {
switch (r.field()) {
case f_hnsw::kBase:
base = DecodeBase(r.bytes());
break;
case f_hnsw::kM:
m = r.int32_value();
break;
case f_hnsw::kEfConstruction:
ef_construction = r.int32_value();
break;
case f_hnsw::kUseContiguousMemory:
use_contiguous_memory = r.bool_value();
break;
default:
break;
}
}
return std::make_shared<HnswIndexParams>(
base.metric_type, m, ef_construction, base.quantize_type,
use_contiguous_memory, QuantizerParam(base.enable_rotate));
}
void EncodeHnswRabitq(const HnswRabitqIndexParams *params, std::string *out) {
// NOTE: unlike the other vector indexes, the protobuf converter never
// touched quantizer_param here, so the sub-message must stay absent.
BaseParams base;
base.metric_type = params->metric_type();
base.quantize_type = params->quantize_type();
base.has_quantizer_param = false;
std::string base_buf;
EncodeBase(base, &base_buf);
Writer w(out);
w.PutMessage(f_hnsw_rabitq::kBase, base_buf);
w.PutVarint(f_hnsw_rabitq::kM, static_cast<uint64_t>(params->m()));
w.PutVarint(f_hnsw_rabitq::kEfConstruction,
static_cast<uint64_t>(params->ef_construction()));
w.PutVarint(f_hnsw_rabitq::kTotalBits,
static_cast<uint64_t>(params->total_bits()));
w.PutVarint(f_hnsw_rabitq::kNumClusters,
static_cast<uint64_t>(params->num_clusters()));
w.PutVarint(f_hnsw_rabitq::kSampleCount,
static_cast<uint64_t>(params->sample_count()));
}
HnswRabitqIndexParams::OPtr DecodeHnswRabitq(std::string_view buf) {
BaseParams base;
int32_t m = 0;
int32_t ef_construction = 0;
int32_t total_bits = 0;
int32_t num_clusters = 0;
int32_t sample_count = 0;
Reader r(buf);
while (r.Next()) {
switch (r.field()) {
case f_hnsw_rabitq::kBase:
base = DecodeBase(r.bytes());
break;
case f_hnsw_rabitq::kM:
m = r.int32_value();
break;
case f_hnsw_rabitq::kEfConstruction:
ef_construction = r.int32_value();
break;
case f_hnsw_rabitq::kTotalBits:
total_bits = r.int32_value();
break;
case f_hnsw_rabitq::kNumClusters:
num_clusters = r.int32_value();
break;
case f_hnsw_rabitq::kSampleCount:
sample_count = r.int32_value();
break;
default:
break;
}
}
return std::make_shared<HnswRabitqIndexParams>(base.metric_type, total_bits,
num_clusters, m,
ef_construction, sample_count);
}
//! Like HNSW_RABITQ, the protobuf converter never touched quantizer_param
//! here, so the sub-message must stay absent to keep the bytes identical.
void EncodeIvfRabitq(const IvfRabitqIndexParams *params, std::string *out) {
BaseParams base;
base.metric_type = params->metric_type();
base.quantize_type = params->quantize_type();
base.has_quantizer_param = false;
std::string base_buf;
EncodeBase(base, &base_buf);
Writer w(out);
w.PutMessage(f_ivf_rabitq::kBase, base_buf);
w.PutVarint(f_ivf_rabitq::kNList, static_cast<uint64_t>(params->nlist()));
w.PutVarint(f_ivf_rabitq::kTotalBits,
static_cast<uint64_t>(params->total_bits()));
w.PutVarint(f_ivf_rabitq::kSampleCount,
static_cast<uint64_t>(params->sample_count()));
}
IvfRabitqIndexParams::OPtr DecodeIvfRabitq(std::string_view buf) {
BaseParams base;
int32_t nlist = 0;
int32_t total_bits = 0;
int32_t sample_count = 0;
Reader r(buf);
while (r.Next()) {
switch (r.field()) {
case f_ivf_rabitq::kBase:
base = DecodeBase(r.bytes());
break;
case f_ivf_rabitq::kNList:
nlist = r.int32_value();
break;
case f_ivf_rabitq::kTotalBits:
total_bits = r.int32_value();
break;
case f_ivf_rabitq::kSampleCount:
sample_count = r.int32_value();
break;
default:
break;
}
}
return std::make_shared<IvfRabitqIndexParams>(base.metric_type, nlist,
total_bits, sample_count);
}
void EncodeFlat(const FlatIndexParams *params, std::string *out) {
std::string base;
EncodeBase(MakeBase(params), &base);
Writer w(out);
w.PutMessage(f_flat::kBase, base);
}
FlatIndexParams::OPtr DecodeFlat(std::string_view buf) {
BaseParams base;
Reader r(buf);
while (r.Next()) {
if (r.field() == f_flat::kBase) {
base = DecodeBase(r.bytes());
}
}
return std::make_shared<FlatIndexParams>(base.metric_type, base.quantize_type,
QuantizerParam(base.enable_rotate));
}
void EncodeIvf(const IVFIndexParams *params, std::string *out) {
std::string base;
EncodeBase(MakeBase(params), &base);
Writer w(out);
w.PutMessage(f_ivf::kBase, base);
w.PutVarint(f_ivf::kNList, static_cast<uint64_t>(params->n_list()));
w.PutVarint(f_ivf::kNIters, static_cast<uint64_t>(params->n_iters()));
w.PutBool(f_ivf::kUseSoar, params->use_soar());
}
IVFIndexParams::OPtr DecodeIvf(std::string_view buf) {
BaseParams base;
int32_t n_list = 0;
int32_t n_iters = 0;
bool use_soar = false;
Reader r(buf);
while (r.Next()) {
switch (r.field()) {
case f_ivf::kBase:
base = DecodeBase(r.bytes());
break;
case f_ivf::kNList:
n_list = r.int32_value();
break;
case f_ivf::kNIters:
n_iters = r.int32_value();
break;
case f_ivf::kUseSoar:
use_soar = r.bool_value();
break;
default:
break;
}
}
return std::make_shared<IVFIndexParams>(base.metric_type, n_list, n_iters,
use_soar, base.quantize_type,
QuantizerParam(base.enable_rotate));
}
void EncodeDiskAnn(const DiskAnnIndexParams *params, std::string *out) {
std::string base;
EncodeBase(MakeBase(params), &base);
Writer w(out);
w.PutMessage(f_diskann::kBase, base);
w.PutVarint(f_diskann::kMaxDegree,
static_cast<uint64_t>(params->max_degree()));
w.PutVarint(f_diskann::kListSize, static_cast<uint64_t>(params->list_size()));
w.PutVarint(f_diskann::kPqChunkNum,
static_cast<uint64_t>(params->pq_chunk_num()));
}
DiskAnnIndexParams::OPtr DecodeDiskAnn(std::string_view buf) {
BaseParams base;
int32_t max_degree = 0;
int32_t list_size = 0;
int32_t pq_chunk_num = 0;
Reader r(buf);
while (r.Next()) {
switch (r.field()) {
case f_diskann::kBase:
base = DecodeBase(r.bytes());
break;
case f_diskann::kMaxDegree:
max_degree = r.int32_value();
break;
case f_diskann::kListSize:
list_size = r.int32_value();
break;
case f_diskann::kPqChunkNum:
pq_chunk_num = r.int32_value();
break;
default:
break;
}
}
return std::make_shared<DiskAnnIndexParams>(
base.metric_type, max_degree, list_size, pq_chunk_num, base.quantize_type,
QuantizerParam(base.enable_rotate));
}
void EncodeVamana(const VamanaIndexParams *params, std::string *out) {
std::string base;
EncodeBase(MakeBase(params), &base);
Writer w(out);
w.PutMessage(f_vamana::kBase, base);
w.PutVarint(f_vamana::kMaxDegree,
static_cast<uint64_t>(params->max_degree()));
w.PutVarint(f_vamana::kSearchListSize,
static_cast<uint64_t>(params->search_list_size()));
w.PutFloat(f_vamana::kAlpha, params->alpha());
w.PutBool(f_vamana::kSaturateGraph, params->saturate_graph());
w.PutBool(f_vamana::kUseContiguousMemory, params->use_contiguous_memory());
w.PutBool(f_vamana::kUseIdMap, params->use_id_map());
w.PutBool(f_vamana::kTwoPassBuild, params->two_pass_build());
}
VamanaIndexParams::OPtr DecodeVamana(std::string_view buf) {
BaseParams base;
int32_t max_degree = 0;
int32_t search_list_size = 0;
float alpha = 0.0f;
bool saturate_graph = false;
bool use_contiguous_memory = false;
bool use_id_map = false;
bool two_pass_build = false;
Reader r(buf);
while (r.Next()) {
switch (r.field()) {
case f_vamana::kBase:
base = DecodeBase(r.bytes());
break;
case f_vamana::kMaxDegree:
max_degree = r.int32_value();
break;
case f_vamana::kSearchListSize:
search_list_size = r.int32_value();
break;
case f_vamana::kAlpha:
alpha = r.float_value();
break;
case f_vamana::kSaturateGraph:
saturate_graph = r.bool_value();
break;
case f_vamana::kUseContiguousMemory:
use_contiguous_memory = r.bool_value();
break;
case f_vamana::kUseIdMap:
use_id_map = r.bool_value();
break;
case f_vamana::kTwoPassBuild:
two_pass_build = r.bool_value();
break;
default:
break;
}
}
return std::make_shared<VamanaIndexParams>(
base.metric_type, max_degree, search_list_size, alpha, saturate_graph,
use_contiguous_memory, use_id_map, base.quantize_type,
QuantizerParam(base.enable_rotate), two_pass_build);
}
void EncodeInvert(const InvertIndexParams *params, std::string *out) {
Writer w(out);
w.PutBool(f_invert::kEnableRangeOptimization,
params->enable_range_optimization());
w.PutBool(f_invert::kEnableExtendedWildcard,
params->enable_extended_wildcard());
}
InvertIndexParams::OPtr DecodeInvert(std::string_view buf) {
bool enable_range_optimization = false;
bool enable_extended_wildcard = false;
Reader r(buf);
while (r.Next()) {
if (r.field() == f_invert::kEnableRangeOptimization) {
enable_range_optimization = r.bool_value();
} else if (r.field() == f_invert::kEnableExtendedWildcard) {
enable_extended_wildcard = r.bool_value();
}
}
return std::make_shared<InvertIndexParams>(enable_range_optimization,
enable_extended_wildcard);
}
void EncodeFts(const FtsIndexParams *params, std::string *out) {
Writer w(out);
w.PutString(f_fts::kTokenizerName, params->tokenizer_name());
for (const auto &filter : params->filters()) {
w.AddString(f_fts::kFilters, filter);
}
w.PutString(f_fts::kExtraParams, params->extra_params());
}
FtsIndexParams::Ptr DecodeFts(std::string_view buf) {
std::string tokenizer_name;
std::vector<std::string> filters;
std::string extra_params;
Reader r(buf);
while (r.Next()) {
switch (r.field()) {
case f_fts::kTokenizerName:
tokenizer_name = r.string_value();
break;
case f_fts::kFilters:
filters.push_back(r.string_value());
break;
case f_fts::kExtraParams:
extra_params = r.string_value();
break;
default:
break;
}
}
return std::make_shared<FtsIndexParams>(tokenizer_name, std::move(filters),
extra_params);
}
} // namespace
void ManifestCodec::EncodeIndexParams(const IndexParams *params,
std::string *out) {
if (params == nullptr) {
return;
}
Writer w(out);
std::string payload;
switch (params->type()) {
case IndexType::INVERT:
if (auto *p = dynamic_cast<const InvertIndexParams *>(params)) {
EncodeInvert(p, &payload);
w.PutMessage(f_index_params::kInvert, payload);
}
break;
case IndexType::HNSW:
if (auto *p = dynamic_cast<const HnswIndexParams *>(params)) {
EncodeHnsw(p, &payload);
w.PutMessage(f_index_params::kHnsw, payload);
}
break;
case IndexType::FLAT:
if (auto *p = dynamic_cast<const FlatIndexParams *>(params)) {
EncodeFlat(p, &payload);
w.PutMessage(f_index_params::kFlat, payload);
}
break;
case IndexType::IVF:
if (auto *p = dynamic_cast<const IVFIndexParams *>(params)) {
EncodeIvf(p, &payload);
w.PutMessage(f_index_params::kIvf, payload);
}
break;
case IndexType::HNSW_RABITQ:
if (auto *p = dynamic_cast<const HnswRabitqIndexParams *>(params)) {
EncodeHnswRabitq(p, &payload);
w.PutMessage(f_index_params::kHnswRabitq, payload);
}
break;
case IndexType::IVF_RABITQ:
if (auto *p = dynamic_cast<const IvfRabitqIndexParams *>(params)) {
EncodeIvfRabitq(p, &payload);
w.PutMessage(f_index_params::kIvfRabitq, payload);
}
break;
case IndexType::VAMANA:
if (auto *p = dynamic_cast<const VamanaIndexParams *>(params)) {
EncodeVamana(p, &payload);
w.PutMessage(f_index_params::kVamana, payload);
}
break;
case IndexType::FTS:
if (auto *p = dynamic_cast<const FtsIndexParams *>(params)) {
EncodeFts(p, &payload);
w.PutMessage(f_index_params::kFts, payload);
}
break;
case IndexType::DISKANN:
if (auto *p = dynamic_cast<const DiskAnnIndexParams *>(params)) {
EncodeDiskAnn(p, &payload);
w.PutMessage(f_index_params::kDiskann, payload);
}
break;
default:
break;
}
}
IndexParams::Ptr ManifestCodec::DecodeIndexParams(std::string_view buf) {
// oneof semantics: the last branch present on the wire wins.
IndexParams::Ptr params;
Reader r(buf);
while (r.Next()) {
switch (r.field()) {
case f_index_params::kInvert:
params = DecodeInvert(r.bytes());
break;
case f_index_params::kHnsw:
params = DecodeHnsw(r.bytes());
break;
case f_index_params::kFlat:
params = DecodeFlat(r.bytes());
break;
case f_index_params::kIvf:
params = DecodeIvf(r.bytes());
break;
case f_index_params::kHnswRabitq:
params = DecodeHnswRabitq(r.bytes());
break;
case f_index_params::kIvfRabitq:
params = DecodeIvfRabitq(r.bytes());
break;
case f_index_params::kVamana:
params = DecodeVamana(r.bytes());
break;
case f_index_params::kFts:
params = DecodeFts(r.bytes());
break;
case f_index_params::kDiskann:
params = DecodeDiskAnn(r.bytes());
break;
default:
break;
}
}
return params;
}
void ManifestCodec::EncodeFieldSchema(const FieldSchema &field,
std::string *out) {
Writer w(out);
w.PutString(f_field::kName, field.name());
w.PutVarint(f_field::kDataType,
static_cast<uint64_t>(
wire::ToNumber(DataTypeCodeBook::Get(field.data_type()))));
w.PutVarint(f_field::kDimension, field.dimension());
w.PutBool(f_field::kNullable, field.nullable());
auto index_params = field.index_params();
if (index_params) {
std::string payload;
EncodeIndexParams(index_params.get(), &payload);
w.PutMessage(f_field::kIndexParams, payload);
}
}
FieldSchema::Ptr ManifestCodec::DecodeFieldSchema(std::string_view buf) {
auto field = std::make_shared<FieldSchema>();
Reader r(buf);
while (r.Next()) {
switch (r.field()) {
case f_field::kName:
field->set_name(r.string_value());
break;
case f_field::kDataType:
field->set_data_type(DataTypeCodeBook::Get(
wire::FromNumber<wire::DataType>(r.int32_value())));
break;
case f_field::kDimension:
field->set_dimension(r.uint32_value());
break;
case f_field::kNullable:
field->set_nullable(r.bool_value());
break;
case f_field::kIndexParams:
field->set_index_params(DecodeIndexParams(r.bytes()));
break;
default:
break;
}
}
return field;
}
void ManifestCodec::EncodeCollectionSchema(const CollectionSchema &schema,
std::string *out) {
Writer w(out);
w.PutString(f_collection::kName, schema.name());
for (const auto &field : schema.fields()) {
std::string payload;
EncodeFieldSchema(*field, &payload);
w.PutMessage(f_collection::kFields, payload);
}
w.PutVarint(f_collection::kMaxDocCountPerSegment,
schema.max_doc_count_per_segment());
}
CollectionSchema::Ptr ManifestCodec::DecodeCollectionSchema(
std::string_view buf) {
auto schema = std::make_shared<CollectionSchema>();
// The protobuf-based converter read max_doc_count_per_segment straight from
// the wire, so a message without the field decoded to zero rather than to
// the C++ default. Mirror that to stay compatible with such manifests.
schema->set_max_doc_count_per_segment(0);
Reader r(buf);
while (r.Next()) {
switch (r.field()) {
case f_collection::kName:
schema->set_name(r.string_value());
break;
case f_collection::kFields:
schema->add_field(DecodeFieldSchema(r.bytes()));
break;
case f_collection::kMaxDocCountPerSegment:
schema->set_max_doc_count_per_segment(r.varint());
break;
default:
break;
}
}
return schema;
}
void ManifestCodec::EncodeBlockMeta(const BlockMeta &meta, std::string *out) {
Writer w(out);
w.PutVarint(f_block::kBlockId, meta.id());
w.PutVarint(f_block::kBlockType, static_cast<uint64_t>(wire::ToNumber(
BlockTypeCodeBook::Get(meta.type()))));
w.PutVarint(f_block::kMinDocId, meta.min_doc_id());
w.PutVarint(f_block::kMaxDocId, meta.max_doc_id());
w.PutVarint(f_block::kDocCount, meta.doc_count());
for (const auto &column : meta.columns()) {
w.AddString(f_block::kColumns, column);
}
}
BlockMeta::Ptr ManifestCodec::DecodeBlockMeta(std::string_view buf) {
auto meta = std::make_shared<BlockMeta>();
Reader r(buf);
while (r.Next()) {
switch (r.field()) {
case f_block::kBlockId:
meta->set_id(r.uint32_value());
break;
case f_block::kBlockType:
meta->set_type(BlockTypeCodeBook::Get(
wire::FromNumber<wire::BlockType>(r.int32_value())));
break;
case f_block::kMinDocId:
meta->set_min_doc_id(r.varint());
break;
case f_block::kMaxDocId:
meta->set_max_doc_id(r.varint());
break;
case f_block::kDocCount:
meta->set_doc_count(r.uint32_value());
break;
case f_block::kColumns:
meta->add_column(r.string_value());
break;
default:
break;
}
}
return meta;
}
void ManifestCodec::EncodeSegmentMeta(const SegmentMeta &meta,
std::string *out) {
Writer w(out);
w.PutVarint(f_segment::kSegmentId, meta.id());
for (const auto &block : meta.persisted_blocks()) {
std::string payload;
EncodeBlockMeta(block, &payload);
w.PutMessage(f_segment::kPersistedBlocks, payload);
}
if (meta.has_writing_forward_block()) {
std::string payload;
EncodeBlockMeta(meta.writing_forward_block().value(), &payload);
w.PutMessage(f_segment::kWritingForwardBlock, payload);
}
for (const auto &field : meta.indexed_vector_fields()) {
w.AddString(f_segment::kIndexedVectorFields, field);
}
}
SegmentMeta::Ptr ManifestCodec::DecodeSegmentMeta(std::string_view buf) {
auto meta = std::make_shared<SegmentMeta>(0);
Reader r(buf);
while (r.Next()) {
switch (r.field()) {
case f_segment::kSegmentId:
meta->set_id(r.uint32_value());
break;
case f_segment::kPersistedBlocks:
meta->add_persisted_block(*DecodeBlockMeta(r.bytes()));
break;
case f_segment::kWritingForwardBlock:
meta->set_writing_forward_block(*DecodeBlockMeta(r.bytes()));
break;
case f_segment::kIndexedVectorFields:
meta->add_indexed_vector_field(r.string_value());
break;
default:
break;
}
}
return meta;
}
Status ManifestCodec::Encode(const ManifestData &data, std::string *out) {
Writer w(out);
w.PutVarint(f_manifest::kVersion, data.version);
if (data.schema) {
std::string payload;
EncodeCollectionSchema(*data.schema, &payload);
w.PutMessage(f_manifest::kSchema, payload);
}
w.PutBool(f_manifest::kEnableMmap, data.enable_mmap);
for (const auto &meta : data.persisted_segment_metas) {
if (!meta) {
continue;
}
std::string payload;
EncodeSegmentMeta(*meta, &payload);
w.PutMessage(f_manifest::kPersistedSegmentMetas, payload);
}
if (data.writing_segment_meta) {
std::string payload;
EncodeSegmentMeta(*data.writing_segment_meta, &payload);
w.PutMessage(f_manifest::kWritingSegmentMeta, payload);
}
w.PutVarint(f_manifest::kIdMapPathSuffix, data.id_map_path_suffix);
w.PutVarint(f_manifest::kDeleteSnapshotPathSuffix,
data.delete_snapshot_path_suffix);
w.PutVarint(f_manifest::kNextSegmentId, data.next_segment_id);
return Status::OK();
}
Status ManifestCodec::Decode(std::string_view buf, ManifestData *data) {
Reader r(buf);
while (r.Next()) {
switch (r.field()) {
case f_manifest::kVersion:
data->version = r.uint32_value();
break;
case f_manifest::kSchema:
data->schema = DecodeCollectionSchema(r.bytes());
break;
case f_manifest::kEnableMmap:
data->enable_mmap = r.bool_value();
break;
case f_manifest::kPersistedSegmentMetas:
data->persisted_segment_metas.push_back(DecodeSegmentMeta(r.bytes()));
break;
case f_manifest::kWritingSegmentMeta:
data->writing_segment_meta = DecodeSegmentMeta(r.bytes());
break;
case f_manifest::kIdMapPathSuffix:
data->id_map_path_suffix = r.uint32_value();
break;
case f_manifest::kDeleteSnapshotPathSuffix:
data->delete_snapshot_path_suffix = r.uint32_value();
break;
case f_manifest::kNextSegmentId:
data->next_segment_id = r.uint32_value();
break;
default:
break;
}
}
if (!r.ok()) {
return Status::InternalError("Malformed manifest data");
}
if (!data->schema) {
// An absent schema means the file is not a valid manifest; the protobuf
// parser accepted it but downstream code always expects a schema.
// The old reader decoded the default (empty) protobuf message in that
// case, yielding a schema whose max_doc_count_per_segment is zero.
data->schema = std::make_shared<CollectionSchema>();
data->schema->set_max_doc_count_per_segment(0);
}
return Status::OK();
}
} // namespace zvec
src/db/index/common/manifest/pb_wire.cc
+98
View File
@@ -0,0 +1,98 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "db/index/common/manifest/pb_wire.h"
#include <cstdint>
namespace zvec {
namespace pbwire {
bool Reader::Next() {
if (!ok_ || pos_ >= size_) {
return false;
}
uint64_t tag = 0;
if (!ReadVarintRaw(&tag)) {
return Fail();
}
field_ = static_cast<uint32_t>(tag >> 3);
type_ = static_cast<WireType>(tag & 0x7);
if (field_ == 0) {
return Fail(); // field number 0 is illegal
}
switch (type_) {
case kVarint:
return ReadVarintRaw(&varint_) ? true : Fail();
case kFixed64: {
if (size_ - pos_ < 8) {
return Fail();
}
uint64_t bits = 0;
for (int i = 0; i < 8; ++i) {
bits |= static_cast<uint64_t>(static_cast<uint8_t>(data_[pos_ + i]))
<< (8 * i);
}
pos_ += 8;
varint_ = bits;
return true;
}
case kFixed32: {
if (size_ - pos_ < 4) {
return Fail();
}
uint32_t bits = 0;
for (int i = 0; i < 4; ++i) {
bits |= static_cast<uint32_t>(static_cast<uint8_t>(data_[pos_ + i]))
<< (8 * i);
}
pos_ += 4;
fixed32_ = bits;
return true;
}
case kLenDelim: {
uint64_t len = 0;
if (!ReadVarintRaw(&len)) {
return Fail();
}
if (len > size_ - pos_) {
return Fail();
}
bytes_ = std::string_view(data_ + pos_, static_cast<size_t>(len));
pos_ += static_cast<size_t>(len);
return true;
}
default:
return Fail(); // groups and unknown wire types
}
}
bool Reader::ReadVarintRaw(uint64_t *out) {
uint64_t result = 0;
for (int shift = 0; shift < 64; shift += 7) {
if (pos_ >= size_) {
return false;
}
uint8_t byte = static_cast<uint8_t>(data_[pos_++]);
result |= static_cast<uint64_t>(byte & 0x7F) << shift;
if ((byte & 0x80) == 0) {
*out = result;
return true;
}
}
return false; // varint longer than 10 bytes
}
} // namespace pbwire
} // namespace zvec
src/db/index/common/manifest/pb_wire.h
+229
View File
@@ -0,0 +1,229 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Minimal protobuf wire-format reader/writer.
//!
//! zvec persists its manifest in protobuf wire format. This header implements
//! just enough of the encoding to read and write that format so that the
//! project does not need to depend on libprotobuf/protoc. The concrete
//! message layout lives in manifest_codec.cc in this directory (field
//! numbers) and in db/index/common/manifest_enum.h (enum values), which
//! together define the format.
//!
//! Wire format reference:
//! https://protobuf.dev/programming-guides/encoding/
//!
//! The wire types used by the manifest (varint, length-delimited, 32-bit)
//! are supported for reading values, and 64-bit fields are accepted as well
//! for completeness. Deprecated groups (wire types 3 and 4) are rejected as
//! corrupt input.
#pragma once
#include <cstdint>
#include <cstring>
#include <string>
#include <string_view>
namespace zvec {
namespace pbwire {
//! Protobuf wire types.
enum WireType : uint32_t {
kVarint = 0,
kFixed64 = 1,
kLenDelim = 2,
kStartGroup = 3, // deprecated, unsupported
kEndGroup = 4, // deprecated, unsupported
kFixed32 = 5,
};
//! Appends protobuf-encoded fields to a string buffer.
//!
//! Fields must be written in ascending field-number order to match the byte
//! layout produced by the protobuf library.
class Writer {
public:
explicit Writer(std::string *out) : out_(out) {}
//! Writes a varint field. Skipped when the value is zero, matching proto3
//! semantics where default-valued singular fields are not serialized.
void PutVarint(uint32_t field, uint64_t value) {
if (value == 0) {
return;
}
PutVarintAlways(field, value);
}
//! Writes a bool field. Skipped when false (proto3 default).
void PutBool(uint32_t field, bool value) {
if (!value) {
return;
}
PutVarintAlways(field, 1);
}
//! Writes a float field as fixed32. Skipped when the value is +0.0f
//! (proto3 default). Note -0.0f compares equal to 0.0f and is therefore
//! also skipped, which matches the protobuf library behaviour.
void PutFloat(uint32_t field, float value) {
if (value == 0.0f) {
return;
}
PutTag(field, kFixed32);
uint32_t bits;
std::memcpy(&bits, &value, sizeof(bits));
PutLittleEndian32(bits);
}
//! Writes a singular string field. Skipped when empty (proto3 default).
void PutString(uint32_t field, const std::string &value) {
if (value.empty()) {
return;
}
PutLenDelim(field, value);
}
//! Writes one element of a repeated string field. Unlike singular fields,
//! repeated elements are always written, including empty strings.
void AddString(uint32_t field, const std::string &value) {
PutLenDelim(field, value);
}
//! Writes a length-delimited field holding an already encoded sub-message.
//! Always written, including when the payload is empty: an empty but
//! present sub-message is encoded as a zero-length field by protobuf.
void PutMessage(uint32_t field, std::string_view encoded) {
PutLenDelim(field, encoded);
}
private:
void PutTag(uint32_t field, WireType type) {
PutVarintRaw((static_cast<uint64_t>(field) << 3) |
static_cast<uint64_t>(type));
}
void PutVarintAlways(uint32_t field, uint64_t value) {
PutTag(field, kVarint);
PutVarintRaw(value);
}
void PutLenDelim(uint32_t field, std::string_view value) {
PutTag(field, kLenDelim);
PutVarintRaw(value.size());
out_->append(value.data(), value.size());
}
void PutVarintRaw(uint64_t value) {
while (value >= 0x80) {
out_->push_back(static_cast<char>((value & 0x7F) | 0x80));
value >>= 7;
}
out_->push_back(static_cast<char>(value));
}
void PutLittleEndian32(uint32_t value) {
for (int i = 0; i < 4; ++i) {
out_->push_back(static_cast<char>(value & 0xFF));
value >>= 8;
}
}
std::string *out_;
};
//! Iterates over the fields of a protobuf-encoded buffer.
//!
//! Each successful Next() fully consumes one field, so unknown fields are
//! skipped simply by not reading their value. This preserves protobuf's
//! forward compatibility: manifests written by newer versions carrying extra
//! fields remain readable.
class Reader {
public:
Reader(const char *data, size_t size) : data_(data), size_(size) {}
explicit Reader(std::string_view buf) : Reader(buf.data(), buf.size()) {}
//! Advances to the next field. Returns false at end of buffer or on error;
//! use ok() to distinguish the two.
bool Next();
uint32_t field() const {
return field_;
}
WireType wire_type() const {
return type_;
}
//! Value of the current field decoded as a varint.
uint64_t varint() const {
return varint_;
}
uint32_t uint32_value() const {
return static_cast<uint32_t>(varint_);
}
int32_t int32_value() const {
return static_cast<int32_t>(static_cast<uint32_t>(varint_));
}
bool bool_value() const {
return varint_ != 0;
}
//! Value of the current field decoded as a fixed32 float.
float float_value() const {
float value;
std::memcpy(&value, &fixed32_, sizeof(value));
return value;
}
//! Payload of the current length-delimited field. The view points into the
//! buffer passed to the constructor and stays valid as long as it does.
std::string_view bytes() const {
return bytes_;
}
std::string string_value() const {
return std::string(bytes_);
}
//! False once malformed input has been encountered.
bool ok() const {
return ok_;
}
private:
bool Fail() {
ok_ = false;
return false;
}
bool ReadVarintRaw(uint64_t *out);
const char *data_;
size_t size_;
size_t pos_{0};
uint32_t field_{0};
WireType type_{kVarint};
uint64_t varint_{0};
uint32_t fixed32_{0};
std::string_view bytes_{};
bool ok_{true};
};
} // namespace pbwire
} // namespace zvec
src/db/index/common/manifest_codec.h
+79
View File
@@ -0,0 +1,79 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Encoder/decoder for the collection manifest.
//!
//! The on-disk layout is the protobuf wire format. This header and its
//! implementation in manifest/manifest_codec.cc are the authoritative
//! definition of that format: the field numbers assigned there and the enum
//! values in manifest_enum.h are part of the persisted format and must never
//! change (only append).
//!
//! Compared with the generated protobuf code this converts directly between
//! zvec's own C++ types and bytes, without an intermediate message object.
#pragma once
#include <cstdint>
#include <string>
#include <string_view>
#include <vector>
#include <zvec/db/index_params.h>
#include <zvec/db/schema.h>
#include <zvec/db/status.h>
#include "db/index/common/meta.h"
namespace zvec {
//! In-memory representation of the manifest file contents.
struct ManifestData {
uint32_t version{0};
CollectionSchema::Ptr schema;
bool enable_mmap{false};
std::vector<SegmentMeta::Ptr> persisted_segment_metas;
SegmentMeta::Ptr writing_segment_meta; // null when absent
uint32_t id_map_path_suffix{0};
uint32_t delete_snapshot_path_suffix{0};
uint32_t next_segment_id{0};
};
//! Converts between ManifestData and its on-disk byte representation.
struct ManifestCodec {
//! Serializes a manifest. Appends to `out`.
static Status Encode(const ManifestData &data, std::string *out);
//! Parses a manifest. Returns an error for malformed input.
static Status Decode(std::string_view buf, ManifestData *data);
// The helpers below are exposed for unit testing so that each message can
// be checked against the format independently.
static void EncodeIndexParams(const IndexParams *params, std::string *out);
static IndexParams::Ptr DecodeIndexParams(std::string_view buf);
static void EncodeFieldSchema(const FieldSchema &field, std::string *out);
static FieldSchema::Ptr DecodeFieldSchema(std::string_view buf);
static void EncodeCollectionSchema(const CollectionSchema &schema,
std::string *out);
static CollectionSchema::Ptr DecodeCollectionSchema(std::string_view buf);
static void EncodeBlockMeta(const BlockMeta &meta, std::string *out);
static BlockMeta::Ptr DecodeBlockMeta(std::string_view buf);
static void EncodeSegmentMeta(const SegmentMeta &meta, std::string *out);
static SegmentMeta::Ptr DecodeSegmentMeta(std::string_view buf);
};
} // namespace zvec
src/db/index/common/manifest_enum.h
+120
View File
@@ -0,0 +1,120 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! On-disk enum values of the manifest format.
//!
//! These enum values are part of the persisted manifest format. The numeric
//! values must never change; only append new entries.
#pragma once
#include <cstdint>
namespace zvec {
namespace wire {
//! Mirrors proto enum DataType.
enum class DataType : int32_t {
DT_UNDEFINED = 0,
DT_BINARY = 1,
DT_STRING = 2,
DT_BOOL = 3,
DT_INT32 = 4,
DT_INT64 = 5,
DT_UINT32 = 6,
DT_UINT64 = 7,
DT_FLOAT = 8,
DT_DOUBLE = 9,
DT_VECTOR_BINARY32 = 20,
DT_VECTOR_BINARY64 = 21,
DT_VECTOR_FP16 = 22,
DT_VECTOR_FP32 = 23,
DT_VECTOR_FP64 = 24,
DT_VECTOR_INT4 = 25,
DT_VECTOR_INT8 = 26,
DT_VECTOR_INT16 = 27,
DT_SPARSE_VECTOR_FP16 = 30,
DT_SPARSE_VECTOR_FP32 = 31,
DT_ARRAY_BINARY = 40,
DT_ARRAY_STRING = 41,
DT_ARRAY_BOOL = 42,
DT_ARRAY_INT32 = 43,
DT_ARRAY_INT64 = 44,
DT_ARRAY_UINT32 = 45,
DT_ARRAY_UINT64 = 46,
DT_ARRAY_FLOAT = 47,
DT_ARRAY_DOUBLE = 48,
};
//! Mirrors proto enum IndexType.
enum class IndexType : int32_t {
IT_UNDEFINED = 0,
IT_HNSW = 1,
IT_IVF = 2,
IT_FLAT = 3,
IT_HNSW_RABITQ = 4,
IT_VAMANA = 5,
IT_DISKANN = 6,
IT_IVF_RABITQ = 7,
IT_INVERT = 10,
IT_FTS = 11,
};
//! Mirrors proto enum QuantizeType.
enum class QuantizeType : int32_t {
QT_UNDEFINED = 0,
QT_FP16 = 1,
QT_INT8 = 2,
QT_INT4 = 3,
QT_RABITQ = 4,
};
//! Mirrors proto enum MetricType.
enum class MetricType : int32_t {
MT_UNDEFINED = 0,
MT_L2 = 1,
MT_IP = 2,
MT_COSINE = 3,
};
//! Mirrors proto enum BlockType.
enum class BlockType : int32_t {
BT_UNDEFINED = 0,
BT_SCALAR = 1,
BT_SCALAR_INDEX = 2,
BT_VECTOR_INDEX = 3,
BT_VECTOR_INDEX_QUANTIZE = 4,
BT_FTS_INDEX = 5,
};
//! Converts a wire enum to its underlying numeric value for encoding.
template <typename E>
constexpr int32_t ToNumber(E value) {
return static_cast<int32_t>(value);
}
//! Converts a numeric value read off the wire back to a wire enum. Unknown
//! values are preserved as-is; the CodeBook mapping turns them into
//! UNDEFINED, matching the previous protobuf-based behaviour.
template <typename E>
constexpr E FromNumber(int32_t value) {
return static_cast<E>(value);
}
} // namespace wire
} // namespace zvec
src/db/index/common/meta.h
+2 -3
View File
@@ -16,6 +16,7 @@
#include <algorithm>
#include <cstdint>
#include <limits>
#include <memory>
#include <optional>
#include <set>
@@ -364,8 +365,6 @@ class SegmentMeta {
return !(*this == other);
}
// Add these methods to SegmentMeta class in meta.h
std::string to_string() const {
std::ostringstream oss;
oss << "SegmentMeta{"
@@ -442,4 +441,4 @@ class SegmentMeta {
std::set<std::string> indexed_vector_fields_;
};
} // namespace zvec
} // namespace zvec
src/db/index/common/proto_converter.cc
-473
View File
@@ -1,473 +0,0 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "proto_converter.h"
namespace zvec {
HnswIndexParams::OPtr ProtoConverter::FromPb(
const proto::HnswIndexParams &params_pb) {
bool enable_rotate = params_pb.base().quantizer_param().enable_rotate();
auto params = std::make_shared<HnswIndexParams>(
MetricTypeCodeBook::Get(params_pb.base().metric_type()), params_pb.m(),
params_pb.ef_construction(),
QuantizeTypeCodeBook::Get(params_pb.base().quantize_type()),
params_pb.use_contiguous_memory(), QuantizerParam(enable_rotate));
return params;
}
proto::HnswIndexParams ProtoConverter::ToPb(const HnswIndexParams *params) {
proto::HnswIndexParams params_pb;
params_pb.mutable_base()->set_metric_type(
MetricTypeCodeBook::Get(params->metric_type()));
params_pb.mutable_base()->set_quantize_type(
QuantizeTypeCodeBook::Get(params->quantize_type()));
params_pb.mutable_base()->mutable_quantizer_param()->set_enable_rotate(
params->quantizer_param().enable_rotate());
params_pb.set_ef_construction(params->ef_construction());
params_pb.set_m(params->m());
params_pb.set_use_contiguous_memory(params->use_contiguous_memory());
return params_pb;
}
// HnswRabitqIndexParams
HnswRabitqIndexParams::OPtr ProtoConverter::FromPb(
const proto::HnswRabitqIndexParams &params_pb) {
auto params = std::make_shared<HnswRabitqIndexParams>(
MetricTypeCodeBook::Get(params_pb.base().metric_type()),
params_pb.total_bits(), params_pb.num_clusters(), params_pb.m(),
params_pb.ef_construction(), params_pb.sample_count());
return params;
}
proto::HnswRabitqIndexParams ProtoConverter::ToPb(
const HnswRabitqIndexParams *params) {
proto::HnswRabitqIndexParams params_pb;
params_pb.mutable_base()->set_metric_type(
MetricTypeCodeBook::Get(params->metric_type()));
params_pb.mutable_base()->set_quantize_type(
QuantizeTypeCodeBook::Get(params->quantize_type()));
params_pb.set_m(params->m());
params_pb.set_ef_construction(params->ef_construction());
params_pb.set_total_bits(params->total_bits());
params_pb.set_num_clusters(params->num_clusters());
params_pb.set_sample_count(params->sample_count());
return params_pb;
}
// IvfRabitqIndexParams
IvfRabitqIndexParams::OPtr ProtoConverter::FromPb(
const proto::IvfRabitqIndexParams &params_pb) {
auto params = std::make_shared<IvfRabitqIndexParams>(
MetricTypeCodeBook::Get(params_pb.base().metric_type()),
params_pb.nlist(), params_pb.total_bits(), params_pb.sample_count());
return params;
}
proto::IvfRabitqIndexParams ProtoConverter::ToPb(
const IvfRabitqIndexParams *params) {
proto::IvfRabitqIndexParams params_pb;
params_pb.mutable_base()->set_metric_type(
MetricTypeCodeBook::Get(params->metric_type()));
params_pb.mutable_base()->set_quantize_type(
QuantizeTypeCodeBook::Get(params->quantize_type()));
params_pb.set_nlist(params->nlist());
params_pb.set_total_bits(params->total_bits());
params_pb.set_sample_count(params->sample_count());
return params_pb;
}
// FlatIndexParams
FlatIndexParams::OPtr ProtoConverter::FromPb(
const proto::FlatIndexParams &params_pb) {
bool enable_rotate = params_pb.base().quantizer_param().enable_rotate();
return std::make_shared<FlatIndexParams>(
MetricTypeCodeBook::Get(params_pb.base().metric_type()),
QuantizeTypeCodeBook::Get(params_pb.base().quantize_type()),
QuantizerParam(enable_rotate));
}
proto::FlatIndexParams ProtoConverter::ToPb(const FlatIndexParams *params) {
proto::FlatIndexParams params_pb;
params_pb.mutable_base()->set_metric_type(
MetricTypeCodeBook::Get(params->metric_type()));
params_pb.mutable_base()->set_quantize_type(
QuantizeTypeCodeBook::Get(params->quantize_type()));
params_pb.mutable_base()->mutable_quantizer_param()->set_enable_rotate(
params->quantizer_param().enable_rotate());
return params_pb;
}
// IVFIndexParams
IVFIndexParams::OPtr ProtoConverter::FromPb(
const proto::IVFIndexParams &params_pb) {
bool enable_rotate = params_pb.base().quantizer_param().enable_rotate();
return std::make_shared<IVFIndexParams>(
MetricTypeCodeBook::Get(params_pb.base().metric_type()),
params_pb.n_list(), params_pb.n_iters(), params_pb.use_soar(),
QuantizeTypeCodeBook::Get(params_pb.base().quantize_type()),
QuantizerParam(enable_rotate));
}
proto::IVFIndexParams ProtoConverter::ToPb(const IVFIndexParams *params) {
proto::IVFIndexParams params_pb;
params_pb.mutable_base()->set_metric_type(
MetricTypeCodeBook::Get(params->metric_type()));
params_pb.mutable_base()->set_quantize_type(
QuantizeTypeCodeBook::Get(params->quantize_type()));
params_pb.mutable_base()->mutable_quantizer_param()->set_enable_rotate(
params->quantizer_param().enable_rotate());
params_pb.set_n_list(params->n_list());
params_pb.set_n_iters(params->n_iters());
params_pb.set_use_soar(params->use_soar());
return params_pb;
}
// VamanaIndexParams
VamanaIndexParams::OPtr ProtoConverter::FromPb(
const proto::VamanaIndexParams &params_pb) {
bool enable_rotate = params_pb.base().quantizer_param().enable_rotate();
return std::make_shared<VamanaIndexParams>(
MetricTypeCodeBook::Get(params_pb.base().metric_type()),
params_pb.max_degree(), params_pb.search_list_size(), params_pb.alpha(),
params_pb.saturate_graph(), params_pb.use_contiguous_memory(),
params_pb.use_id_map(),
QuantizeTypeCodeBook::Get(params_pb.base().quantize_type()),
QuantizerParam(enable_rotate), params_pb.two_pass_build());
}
proto::VamanaIndexParams ProtoConverter::ToPb(const VamanaIndexParams *params) {
proto::VamanaIndexParams params_pb;
params_pb.mutable_base()->set_metric_type(
MetricTypeCodeBook::Get(params->metric_type()));
params_pb.mutable_base()->set_quantize_type(
QuantizeTypeCodeBook::Get(params->quantize_type()));
params_pb.mutable_base()->mutable_quantizer_param()->set_enable_rotate(
params->quantizer_param().enable_rotate());
params_pb.set_max_degree(params->max_degree());
params_pb.set_search_list_size(params->search_list_size());
params_pb.set_alpha(params->alpha());
params_pb.set_saturate_graph(params->saturate_graph());
params_pb.set_use_contiguous_memory(params->use_contiguous_memory());
params_pb.set_use_id_map(params->use_id_map());
params_pb.set_two_pass_build(params->two_pass_build());
return params_pb;
}
// InvertIndexParams
InvertIndexParams::OPtr ProtoConverter::FromPb(
const proto::InvertIndexParams &params_pb) {
auto params = std::make_shared<InvertIndexParams>(
params_pb.enable_range_optimization());
return params;
}
proto::InvertIndexParams ProtoConverter::ToPb(const InvertIndexParams *params) {
proto::InvertIndexParams params_pb;
params_pb.set_enable_range_optimization(params->enable_range_optimization());
return params_pb;
}
// DiskAnnIndexParams
DiskAnnIndexParams::OPtr ProtoConverter::FromPb(
const proto::DiskAnnIndexParams &params_pb) {
bool enable_rotate = params_pb.base().quantizer_param().enable_rotate();
return std::make_shared<DiskAnnIndexParams>(
MetricTypeCodeBook::Get(params_pb.base().metric_type()),
params_pb.max_degree(), params_pb.list_size(), params_pb.pq_chunk_num(),
QuantizeTypeCodeBook::Get(params_pb.base().quantize_type()),
QuantizerParam(enable_rotate));
}
proto::DiskAnnIndexParams ProtoConverter::ToPb(
const DiskAnnIndexParams *params) {
proto::DiskAnnIndexParams params_pb;
params_pb.mutable_base()->set_metric_type(
MetricTypeCodeBook::Get(params->metric_type()));
params_pb.mutable_base()->set_quantize_type(
QuantizeTypeCodeBook::Get(params->quantize_type()));
params_pb.mutable_base()->mutable_quantizer_param()->set_enable_rotate(
params->quantizer_param().enable_rotate());
params_pb.set_max_degree(params->max_degree());
params_pb.set_list_size(params->list_size());
params_pb.set_pq_chunk_num(params->pq_chunk_num());
return params_pb;
}
// FtsIndexParams
FtsIndexParams::Ptr ProtoConverter::FromPb(
const proto::FtsIndexParams &params_pb) {
std::vector<std::string> filters;
filters.reserve(params_pb.filters_size());
for (const auto &filter : params_pb.filters()) {
filters.push_back(filter);
}
return std::make_shared<FtsIndexParams>(
params_pb.tokenizer_name(), std::move(filters), params_pb.extra_params());
}
proto::FtsIndexParams ProtoConverter::ToPb(const FtsIndexParams *params) {
proto::FtsIndexParams params_pb;
params_pb.set_tokenizer_name(params->tokenizer_name());
for (const auto &filter : params->filters()) {
params_pb.add_filters(filter);
}
params_pb.set_extra_params(params->extra_params());
return params_pb;
}
// FieldSchema
FieldSchema::Ptr ProtoConverter::FromPb(const proto::FieldSchema &schema_pb) {
auto schema = std::make_shared<FieldSchema>();
schema->set_name(schema_pb.name());
schema->set_data_type(DataTypeCodeBook::Get(schema_pb.data_type()));
schema->set_dimension(schema_pb.dimension());
schema->set_nullable(schema_pb.nullable());
if (schema_pb.has_index_params()) {
schema->set_index_params(ProtoConverter::FromPb(schema_pb.index_params()));
}
return schema;
}
proto::FieldSchema ProtoConverter::ToPb(const FieldSchema &schema) {
proto::FieldSchema schema_pb;
schema_pb.set_name(schema.name());
schema_pb.set_data_type(DataTypeCodeBook::Get(schema.data_type()));
schema_pb.set_dimension(schema.dimension());
schema_pb.set_nullable(schema.nullable());
auto index_params = schema.index_params();
if (index_params) {
auto index_params_pb = schema_pb.mutable_index_params();
index_params_pb->MergeFrom(ProtoConverter::ToPb(index_params.get()));
}
return schema_pb;
}
// CollectionSchema
CollectionSchema::Ptr ProtoConverter::FromPb(
const proto::CollectionSchema &schema_pb) {
CollectionSchema::Ptr schema = std::make_shared<CollectionSchema>();
schema->set_name(schema_pb.name());
for (auto &column_schema_pb : schema_pb.fields()) {
FieldSchema::Ptr column_schema = ProtoConverter::FromPb(column_schema_pb);
schema->add_field(column_schema);
}
schema->set_max_doc_count_per_segment(schema_pb.max_doc_count_per_segment());
return schema;
}
proto::CollectionSchema ProtoConverter::ToPb(const CollectionSchema &schema) {
proto::CollectionSchema schema_pb;
schema_pb.set_name(schema.name());
for (auto &column_schema : schema.fields()) {
proto::FieldSchema *column_schema_pb = schema_pb.add_fields();
column_schema_pb->MergeFrom(ProtoConverter::ToPb(*column_schema));
}
schema_pb.set_max_doc_count_per_segment(schema.max_doc_count_per_segment());
return schema_pb;
}
IndexParams::Ptr ProtoConverter::FromPb(const proto::IndexParams &params_pb) {
if (params_pb.has_hnsw()) {
return ProtoConverter::FromPb(params_pb.hnsw());
} else if (params_pb.has_invert()) {
return ProtoConverter::FromPb(params_pb.invert());
} else if (params_pb.has_ivf()) {
return ProtoConverter::FromPb(params_pb.ivf());
} else if (params_pb.has_flat()) {
return ProtoConverter::FromPb(params_pb.flat());
} else if (params_pb.has_hnsw_rabitq()) {
return ProtoConverter::FromPb(params_pb.hnsw_rabitq());
} else if (params_pb.has_ivf_rabitq()) {
return ProtoConverter::FromPb(params_pb.ivf_rabitq());
} else if (params_pb.has_diskann()) {
return ProtoConverter::FromPb(params_pb.diskann());
} else if (params_pb.has_vamana()) {
return ProtoConverter::FromPb(params_pb.vamana());
} else if (params_pb.has_fts()) {
return ProtoConverter::FromPb(params_pb.fts());
}
return nullptr;
}
// BlockMeta
BlockMeta::Ptr ProtoConverter::FromPb(const proto::BlockMeta &meta_pb) {
auto block_meta = std::make_shared<BlockMeta>();
block_meta->set_id(meta_pb.block_id());
block_meta->set_type(BlockTypeCodeBook::Get(meta_pb.block_type()));
block_meta->set_min_doc_id(meta_pb.min_doc_id());
block_meta->set_max_doc_id(meta_pb.max_doc_id());
block_meta->set_doc_count(meta_pb.doc_count());
for (auto &column : meta_pb.columns()) {
block_meta->add_column(column);
}
return block_meta;
}
proto::IndexParams ProtoConverter::ToPb(const IndexParams *params) {
proto::IndexParams params_pb;
switch (params->type()) {
case IndexType::INVERT: {
auto invert_params = dynamic_cast<const InvertIndexParams *>(params);
if (invert_params) {
params_pb.mutable_invert()->CopyFrom(
ProtoConverter::ToPb(invert_params));
}
break;
}
case IndexType::HNSW: {
auto hnsw_params = dynamic_cast<const HnswIndexParams *>(params);
if (hnsw_params) {
params_pb.mutable_hnsw()->CopyFrom(ProtoConverter::ToPb(hnsw_params));
}
break;
}
case IndexType::IVF: {
auto ivf_params = dynamic_cast<const IVFIndexParams *>(params);
if (ivf_params) {
params_pb.mutable_ivf()->CopyFrom(ProtoConverter::ToPb(ivf_params));
}
break;
}
case IndexType::FLAT: {
auto flat_params = dynamic_cast<const FlatIndexParams *>(params);
if (flat_params) {
params_pb.mutable_flat()->CopyFrom(ProtoConverter::ToPb(flat_params));
}
break;
}
case IndexType::HNSW_RABITQ: {
auto hnsw_rabitq_params =
dynamic_cast<const HnswRabitqIndexParams *>(params);
if (hnsw_rabitq_params) {
params_pb.mutable_hnsw_rabitq()->CopyFrom(
ProtoConverter::ToPb(hnsw_rabitq_params));
}
break;
}
case IndexType::IVF_RABITQ: {
auto ivf_rabitq_params =
dynamic_cast<const IvfRabitqIndexParams *>(params);
if (ivf_rabitq_params) {
params_pb.mutable_ivf_rabitq()->CopyFrom(
ProtoConverter::ToPb(ivf_rabitq_params));
}
break;
}
case IndexType::DISKANN: {
auto diskann_params = dynamic_cast<const DiskAnnIndexParams *>(params);
if (diskann_params) {
params_pb.mutable_diskann()->CopyFrom(
ProtoConverter::ToPb(diskann_params));
}
break;
}
case IndexType::VAMANA: {
auto vamana_params = dynamic_cast<const VamanaIndexParams *>(params);
if (vamana_params) {
params_pb.mutable_vamana()->CopyFrom(
ProtoConverter::ToPb(vamana_params));
}
break;
}
case IndexType::FTS: {
auto fts_params = dynamic_cast<const FtsIndexParams *>(params);
if (fts_params) {
params_pb.mutable_fts()->CopyFrom(ProtoConverter::ToPb(fts_params));
}
break;
}
default:
break;
}
return params_pb;
}
proto::BlockMeta ProtoConverter::ToPb(const BlockMeta &meta) {
proto::BlockMeta meta_pb;
meta_pb.set_block_id(meta.id());
meta_pb.set_block_type(BlockTypeCodeBook::Get(meta.type()));
meta_pb.set_min_doc_id(meta.min_doc_id());
meta_pb.set_max_doc_id(meta.max_doc_id());
meta_pb.set_doc_count(meta.doc_count());
for (auto &column : meta.columns()) {
meta_pb.add_columns(column);
}
return meta_pb;
}
// SegmentMeta
SegmentMeta::Ptr ProtoConverter::FromPb(const proto::SegmentMeta &meta_pb) {
auto meta = std::make_shared<SegmentMeta>(meta_pb.segment_id());
auto persisted_blocks = meta_pb.persisted_blocks();
for (auto &persisted_block_pb : persisted_blocks) {
BlockMeta::Ptr persisted_block = ProtoConverter::FromPb(persisted_block_pb);
meta->add_persisted_block(*persisted_block);
}
if (meta_pb.has_writing_forward_block()) {
meta->set_writing_forward_block(
*ProtoConverter::FromPb(meta_pb.writing_forward_block()));
}
auto indexed_vector_fields = meta_pb.indexed_vector_fields();
for (auto &indexed_vector_field : indexed_vector_fields) {
meta->add_indexed_vector_field(indexed_vector_field);
}
return meta;
}
proto::SegmentMeta ProtoConverter::ToPb(const SegmentMeta &meta) {
proto::SegmentMeta meta_pb;
meta_pb.set_segment_id(meta.id());
auto persisted_blocks = meta.persisted_blocks();
for (auto &persisted_block : persisted_blocks) {
auto persisted_block_pb = ProtoConverter::ToPb(persisted_block);
meta_pb.add_persisted_blocks()->MergeFrom(persisted_block_pb);
}
if (meta.has_writing_forward_block()) {
meta_pb.mutable_writing_forward_block()->MergeFrom(
ProtoConverter::ToPb(meta.writing_forward_block().value()));
}
auto indexed_vector_fields = meta.indexed_vector_fields();
for (auto &field : indexed_vector_fields) {
meta_pb.add_indexed_vector_fields(field);
}
return meta_pb;
}
} // namespace zvec
src/db/index/common/proto_converter.h
-86
View File
@@ -1,86 +0,0 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/db/index_params.h>
#include <zvec/db/schema.h>
#include "db/index/common/meta.h"
namespace zvec {
struct ProtoConverter {
// HnswIndexParams
static HnswIndexParams::OPtr FromPb(const proto::HnswIndexParams &params_pb);
static proto::HnswIndexParams ToPb(const HnswIndexParams *params);
// HnswRabitqIndexParams
static HnswRabitqIndexParams::OPtr FromPb(
const proto::HnswRabitqIndexParams &params_pb);
static proto::HnswRabitqIndexParams ToPb(const HnswRabitqIndexParams *params);
// IvfRabitqIndexParams
static IvfRabitqIndexParams::OPtr FromPb(
const proto::IvfRabitqIndexParams &params_pb);
static proto::IvfRabitqIndexParams ToPb(const IvfRabitqIndexParams *params);
// FlatIndexParams
static FlatIndexParams::OPtr FromPb(const proto::FlatIndexParams &params_pb);
static proto::FlatIndexParams ToPb(const FlatIndexParams *params);
// IVFIndexParams
static IVFIndexParams::OPtr FromPb(const proto::IVFIndexParams &params_pb);
static proto::IVFIndexParams ToPb(const IVFIndexParams *params);
// VamanaIndexParams
static VamanaIndexParams::OPtr FromPb(
const proto::VamanaIndexParams &params_pb);
static proto::VamanaIndexParams ToPb(const VamanaIndexParams *params);
// InvertIndexParams
static InvertIndexParams::OPtr FromPb(
const proto::InvertIndexParams &params_pb);
static proto::InvertIndexParams ToPb(const InvertIndexParams *params);
// DiskAnnIndexParams
static DiskAnnIndexParams::OPtr FromPb(
const proto::DiskAnnIndexParams &params_pb);
static proto::DiskAnnIndexParams ToPb(const DiskAnnIndexParams *params);
// FtsIndexParams
static FtsIndexParams::Ptr FromPb(const proto::FtsIndexParams &params_pb);
static proto::FtsIndexParams ToPb(const FtsIndexParams *params);
// IndexParams
static IndexParams::Ptr FromPb(const proto::IndexParams &params_pb);
static proto::IndexParams ToPb(const IndexParams *params);
// FieldSchema
static FieldSchema::Ptr FromPb(const proto::FieldSchema &field_pb);
static proto::FieldSchema ToPb(const FieldSchema &field);
// CollectionSchema
static CollectionSchema::Ptr FromPb(const proto::CollectionSchema &schema_pb);
static proto::CollectionSchema ToPb(const CollectionSchema &schema);
// BlockMeta
static BlockMeta::Ptr FromPb(const proto::BlockMeta &meta_pb);
static proto::BlockMeta ToPb(const BlockMeta &meta);
// SegmentMeta
static SegmentMeta::Ptr FromPb(const proto::SegmentMeta &meta_pb);
static proto::SegmentMeta ToPb(const SegmentMeta &meta);
};
} // namespace zvec
src/db/index/common/type_helper.h
+122 -120
View File
@@ -16,9 +16,11 @@
#include <cstddef>
#include <cstdint>
#include <set>
#include <string>
#include <zvec/core/framework/index_meta.h>
#include <zvec/db/type.h>
#include "proto/zvec.pb.h"
#include "db/index/common/manifest_enum.h"
namespace zvec {
@@ -39,26 +41,26 @@ bool sort_and_find_duplicates(uint32_t *indices, char *values, size_t n,
//! Index Type Codebook
struct IndexTypeCodeBook {
//! convert protobuf IndexType to C++ IndexType
static IndexType Get(proto::IndexType type) {
//! Convert wire-format IndexType to C++ IndexType
static IndexType Get(wire::IndexType type) {
switch (type) {
case proto::IT_HNSW:
case wire::IndexType::IT_HNSW:
return IndexType::HNSW;
case proto::IT_HNSW_RABITQ:
case wire::IndexType::IT_HNSW_RABITQ:
return IndexType::HNSW_RABITQ;
case proto::IT_IVF_RABITQ:
case wire::IndexType::IT_IVF_RABITQ:
return IndexType::IVF_RABITQ;
case proto::IT_FLAT:
case wire::IndexType::IT_FLAT:
return IndexType::FLAT;
case proto::IT_IVF:
case wire::IndexType::IT_IVF:
return IndexType::IVF;
case proto::IT_VAMANA:
case wire::IndexType::IT_VAMANA:
return IndexType::VAMANA;
case proto::IT_INVERT:
case wire::IndexType::IT_INVERT:
return IndexType::INVERT;
case proto::IT_DISKANN:
case wire::IndexType::IT_DISKANN:
return IndexType::DISKANN;
case proto::IT_FTS:
case wire::IndexType::IT_FTS:
return IndexType::FTS;
default:
break;
@@ -66,31 +68,31 @@ struct IndexTypeCodeBook {
return IndexType::UNDEFINED;
}
//! Convert C++ IndexType to protobuf IndexType
static proto::IndexType Get(IndexType type) {
//! Convert C++ IndexType to wire-format IndexType
static wire::IndexType Get(IndexType type) {
switch (type) {
case IndexType::HNSW:
return proto::IT_HNSW;
return wire::IndexType::IT_HNSW;
case IndexType::HNSW_RABITQ:
return proto::IT_HNSW_RABITQ;
return wire::IndexType::IT_HNSW_RABITQ;
case IndexType::IVF_RABITQ:
return proto::IT_IVF_RABITQ;
return wire::IndexType::IT_IVF_RABITQ;
case IndexType::FLAT:
return proto::IT_FLAT;
return wire::IndexType::IT_FLAT;
case IndexType::IVF:
return proto::IT_IVF;
return wire::IndexType::IT_IVF;
case IndexType::VAMANA:
return proto::IT_VAMANA;
return wire::IndexType::IT_VAMANA;
case IndexType::INVERT:
return proto::IT_INVERT;
return wire::IndexType::IT_INVERT;
case IndexType::DISKANN:
return proto::IT_DISKANN;
return wire::IndexType::IT_DISKANN;
case IndexType::FTS:
return proto::IT_FTS;
return wire::IndexType::IT_FTS;
default:
break;
}
return proto::IT_UNDEFINED;
return wire::IndexType::IT_UNDEFINED;
}
//! Convert C++ IndexType to C++ String
@@ -122,96 +124,96 @@ struct IndexTypeCodeBook {
};
struct DataTypeCodeBook {
static bool IsArrayType(proto::DataType type) {
return proto::DataType::DT_ARRAY_BINARY <= type &&
type <= proto::DataType::DT_ARRAY_DOUBLE;
static bool IsArrayType(wire::DataType type) {
return wire::DataType::DT_ARRAY_BINARY <= type &&
type <= wire::DataType::DT_ARRAY_DOUBLE;
}
static DataType Get(proto::DataType type) {
static DataType Get(wire::DataType type) {
DataType data_types = DataType::UNDEFINED;
switch (type) {
case proto::DataType::DT_BINARY:
case wire::DataType::DT_BINARY:
data_types = DataType::BINARY;
break;
case proto::DataType::DT_STRING:
case wire::DataType::DT_STRING:
data_types = DataType::STRING;
break;
case proto::DataType::DT_BOOL:
case wire::DataType::DT_BOOL:
data_types = DataType::BOOL;
break;
case proto::DataType::DT_INT32:
case wire::DataType::DT_INT32:
data_types = DataType::INT32;
break;
case proto::DataType::DT_INT64:
case wire::DataType::DT_INT64:
data_types = DataType::INT64;
break;
case proto::DataType::DT_UINT32:
case wire::DataType::DT_UINT32:
data_types = DataType::UINT32;
break;
case proto::DataType::DT_UINT64:
case wire::DataType::DT_UINT64:
data_types = DataType::UINT64;
break;
case proto::DataType::DT_FLOAT:
case wire::DataType::DT_FLOAT:
data_types = DataType::FLOAT;
break;
case proto::DataType::DT_DOUBLE:
case wire::DataType::DT_DOUBLE:
data_types = DataType::DOUBLE;
break;
case proto::DataType::DT_VECTOR_BINARY32:
case wire::DataType::DT_VECTOR_BINARY32:
data_types = DataType::VECTOR_BINARY32;
break;
case proto::DataType::DT_VECTOR_BINARY64:
case wire::DataType::DT_VECTOR_BINARY64:
data_types = DataType::VECTOR_BINARY64;
break;
case proto::DataType::DT_VECTOR_FP16:
case wire::DataType::DT_VECTOR_FP16:
data_types = DataType::VECTOR_FP16;
break;
case proto::DataType::DT_VECTOR_FP32:
case wire::DataType::DT_VECTOR_FP32:
data_types = DataType::VECTOR_FP32;
break;
case proto::DataType::DT_VECTOR_FP64:
case wire::DataType::DT_VECTOR_FP64:
data_types = DataType::VECTOR_FP64;
break;
case proto::DataType::DT_VECTOR_INT4:
case wire::DataType::DT_VECTOR_INT4:
data_types = DataType::VECTOR_INT4;
break;
case proto::DataType::DT_VECTOR_INT8:
case wire::DataType::DT_VECTOR_INT8:
data_types = DataType::VECTOR_INT8;
break;
case proto::DataType::DT_VECTOR_INT16:
case wire::DataType::DT_VECTOR_INT16:
data_types = DataType::VECTOR_INT16;
break;
case proto::DataType::DT_SPARSE_VECTOR_FP16:
case wire::DataType::DT_SPARSE_VECTOR_FP16:
data_types = DataType::SPARSE_VECTOR_FP16;
break;
case proto::DataType::DT_SPARSE_VECTOR_FP32:
case wire::DataType::DT_SPARSE_VECTOR_FP32:
data_types = DataType::SPARSE_VECTOR_FP32;
break;
case proto::DataType::DT_ARRAY_BINARY:
case wire::DataType::DT_ARRAY_BINARY:
data_types = DataType::ARRAY_BINARY;
break;
case proto::DataType::DT_ARRAY_STRING:
case wire::DataType::DT_ARRAY_STRING:
data_types = DataType::ARRAY_STRING;
break;
case proto::DataType::DT_ARRAY_BOOL:
case wire::DataType::DT_ARRAY_BOOL:
data_types = DataType::ARRAY_BOOL;
break;
case proto::DataType::DT_ARRAY_INT32:
case wire::DataType::DT_ARRAY_INT32:
data_types = DataType::ARRAY_INT32;
break;
case proto::DataType::DT_ARRAY_INT64:
case wire::DataType::DT_ARRAY_INT64:
data_types = DataType::ARRAY_INT64;
break;
case proto::DataType::DT_ARRAY_UINT32:
case wire::DataType::DT_ARRAY_UINT32:
data_types = DataType::ARRAY_UINT32;
break;
case proto::DataType::DT_ARRAY_UINT64:
case wire::DataType::DT_ARRAY_UINT64:
data_types = DataType::ARRAY_UINT64;
break;
case proto::DataType::DT_ARRAY_FLOAT:
case wire::DataType::DT_ARRAY_FLOAT:
data_types = DataType::ARRAY_FLOAT;
break;
case proto::DataType::DT_ARRAY_DOUBLE:
case wire::DataType::DT_ARRAY_DOUBLE:
data_types = DataType::ARRAY_DOUBLE;
break;
@@ -221,92 +223,92 @@ struct DataTypeCodeBook {
return data_types;
}
static proto::DataType Get(const DataType type) {
proto::DataType data_type = proto::DataType::DT_UNDEFINED;
static wire::DataType Get(const DataType type) {
wire::DataType data_type = wire::DataType::DT_UNDEFINED;
switch (type) {
case DataType::BINARY:
data_type = proto::DataType::DT_BINARY;
data_type = wire::DataType::DT_BINARY;
break;
case DataType::STRING:
data_type = proto::DataType::DT_STRING;
data_type = wire::DataType::DT_STRING;
break;
case DataType::BOOL:
data_type = proto::DataType::DT_BOOL;
data_type = wire::DataType::DT_BOOL;
break;
case DataType::INT32:
data_type = proto::DataType::DT_INT32;
data_type = wire::DataType::DT_INT32;
break;
case DataType::INT64:
data_type = proto::DataType::DT_INT64;
data_type = wire::DataType::DT_INT64;
break;
case DataType::UINT32:
data_type = proto::DataType::DT_UINT32;
data_type = wire::DataType::DT_UINT32;
break;
case DataType::UINT64:
data_type = proto::DataType::DT_UINT64;
data_type = wire::DataType::DT_UINT64;
break;
case DataType::FLOAT:
data_type = proto::DataType::DT_FLOAT;
data_type = wire::DataType::DT_FLOAT;
break;
case DataType::DOUBLE:
data_type = proto::DataType::DT_DOUBLE;
data_type = wire::DataType::DT_DOUBLE;
break;
case DataType::VECTOR_BINARY32:
data_type = proto::DataType::DT_VECTOR_BINARY32;
data_type = wire::DataType::DT_VECTOR_BINARY32;
break;
case DataType::VECTOR_BINARY64:
data_type = proto::DataType::DT_VECTOR_BINARY64;
data_type = wire::DataType::DT_VECTOR_BINARY64;
break;
case DataType::VECTOR_FP16:
data_type = proto::DataType::DT_VECTOR_FP16;
data_type = wire::DataType::DT_VECTOR_FP16;
break;
case DataType::VECTOR_FP32:
data_type = proto::DataType::DT_VECTOR_FP32;
data_type = wire::DataType::DT_VECTOR_FP32;
break;
case DataType::VECTOR_FP64:
data_type = proto::DataType::DT_VECTOR_FP64;
data_type = wire::DataType::DT_VECTOR_FP64;
break;
case DataType::VECTOR_INT4:
data_type = proto::DataType::DT_VECTOR_INT4;
data_type = wire::DataType::DT_VECTOR_INT4;
break;
case DataType::VECTOR_INT8:
data_type = proto::DataType::DT_VECTOR_INT8;
data_type = wire::DataType::DT_VECTOR_INT8;
break;
case DataType::VECTOR_INT16:
data_type = proto::DataType::DT_VECTOR_INT16;
data_type = wire::DataType::DT_VECTOR_INT16;
break;
case DataType::SPARSE_VECTOR_FP16:
data_type = proto::DataType::DT_SPARSE_VECTOR_FP16;
data_type = wire::DataType::DT_SPARSE_VECTOR_FP16;
break;
case DataType::SPARSE_VECTOR_FP32:
data_type = proto::DataType::DT_SPARSE_VECTOR_FP32;
data_type = wire::DataType::DT_SPARSE_VECTOR_FP32;
break;
case DataType::ARRAY_BINARY:
data_type = proto::DataType::DT_ARRAY_BINARY;
data_type = wire::DataType::DT_ARRAY_BINARY;
break;
case DataType::ARRAY_BOOL:
data_type = proto::DataType::DT_ARRAY_BOOL;
data_type = wire::DataType::DT_ARRAY_BOOL;
break;
case DataType::ARRAY_DOUBLE:
data_type = proto::DataType::DT_ARRAY_DOUBLE;
data_type = wire::DataType::DT_ARRAY_DOUBLE;
break;
case DataType::ARRAY_FLOAT:
data_type = proto::DataType::DT_ARRAY_FLOAT;
data_type = wire::DataType::DT_ARRAY_FLOAT;
break;
case DataType::ARRAY_INT32:
data_type = proto::DataType::DT_ARRAY_INT32;
data_type = wire::DataType::DT_ARRAY_INT32;
break;
case DataType::ARRAY_INT64:
data_type = proto::DataType::DT_ARRAY_INT64;
data_type = wire::DataType::DT_ARRAY_INT64;
break;
case DataType::ARRAY_STRING:
data_type = proto::DataType::DT_ARRAY_STRING;
data_type = wire::DataType::DT_ARRAY_STRING;
break;
case DataType::ARRAY_UINT32:
data_type = proto::DataType::DT_ARRAY_UINT32;
data_type = wire::DataType::DT_ARRAY_UINT32;
break;
case DataType::ARRAY_UINT64:
data_type = proto::DataType::DT_ARRAY_UINT64;
data_type = wire::DataType::DT_ARRAY_UINT64;
break;
default:
break;
@@ -414,29 +416,29 @@ struct DataTypeCodeBook {
};
struct MetricTypeCodeBook {
static MetricType Get(proto::MetricType type) {
static MetricType Get(wire::MetricType type) {
switch (type) {
case proto::MetricType::MT_IP:
case wire::MetricType::MT_IP:
return MetricType::IP;
case proto::MetricType::MT_L2:
case wire::MetricType::MT_L2:
return MetricType::L2;
case proto::MetricType::MT_COSINE:
case wire::MetricType::MT_COSINE:
return MetricType::COSINE;
default:
return MetricType::UNDEFINED;
}
}
static proto::MetricType Get(MetricType type) {
static wire::MetricType Get(MetricType type) {
switch (type) {
case MetricType::IP:
return proto::MetricType::MT_IP;
return wire::MetricType::MT_IP;
case MetricType::L2:
return proto::MetricType::MT_L2;
return wire::MetricType::MT_L2;
case MetricType::COSINE:
return proto::MetricType::MT_COSINE;
return wire::MetricType::MT_COSINE;
default:
return proto::MetricType::MT_UNDEFINED;
return wire::MetricType::MT_UNDEFINED;
}
}
@@ -455,33 +457,33 @@ struct MetricTypeCodeBook {
};
struct QuantizeTypeCodeBook {
static QuantizeType Get(proto::QuantizeType type) {
static QuantizeType Get(wire::QuantizeType type) {
switch (type) {
case proto::QuantizeType::QT_FP16:
case wire::QuantizeType::QT_FP16:
return QuantizeType::FP16;
case proto::QuantizeType::QT_INT4:
case wire::QuantizeType::QT_INT4:
return QuantizeType::INT4;
case proto::QuantizeType::QT_INT8:
case wire::QuantizeType::QT_INT8:
return QuantizeType::INT8;
case proto::QuantizeType::QT_RABITQ:
case wire::QuantizeType::QT_RABITQ:
return QuantizeType::RABITQ;
default:
return QuantizeType::UNDEFINED;
}
}
static proto::QuantizeType Get(QuantizeType type) {
static wire::QuantizeType Get(QuantizeType type) {
switch (type) {
case QuantizeType::FP16:
return proto::QuantizeType::QT_FP16;
return wire::QuantizeType::QT_FP16;
case QuantizeType::INT4:
return proto::QuantizeType::QT_INT4;
return wire::QuantizeType::QT_INT4;
case QuantizeType::INT8:
return proto::QuantizeType::QT_INT8;
return wire::QuantizeType::QT_INT8;
case QuantizeType::RABITQ:
return proto::QuantizeType::QT_RABITQ;
return wire::QuantizeType::QT_RABITQ;
default:
return proto::QuantizeType::QT_UNDEFINED;
return wire::QuantizeType::QT_UNDEFINED;
}
}
@@ -510,22 +512,22 @@ struct QuantizeTypeCodeBook {
};
struct BlockTypeCodeBook {
static BlockType Get(proto::BlockType type) {
static BlockType Get(wire::BlockType type) {
BlockType block_types = BlockType::UNDEFINED;
switch (type) {
case proto::BlockType::BT_SCALAR:
case wire::BlockType::BT_SCALAR:
block_types = BlockType::SCALAR;
break;
case proto::BlockType::BT_SCALAR_INDEX:
case wire::BlockType::BT_SCALAR_INDEX:
block_types = BlockType::SCALAR_INDEX;
break;
case proto::BlockType::BT_VECTOR_INDEX:
case wire::BlockType::BT_VECTOR_INDEX:
block_types = BlockType::VECTOR_INDEX;
break;
case proto::BlockType::BT_VECTOR_INDEX_QUANTIZE:
case wire::BlockType::BT_VECTOR_INDEX_QUANTIZE:
block_types = BlockType::VECTOR_INDEX_QUANTIZE;
break;
case proto::BlockType::BT_FTS_INDEX:
case wire::BlockType::BT_FTS_INDEX:
block_types = BlockType::FTS_INDEX;
break;
default:
@@ -534,23 +536,23 @@ struct BlockTypeCodeBook {
return block_types;
}
static proto::BlockType Get(BlockType type) {
proto::BlockType block_types = proto::BlockType::BT_UNDEFINED;
static wire::BlockType Get(BlockType type) {
wire::BlockType block_types = wire::BlockType::BT_UNDEFINED;
switch (type) {
case BlockType::SCALAR:
block_types = proto::BlockType::BT_SCALAR;
block_types = wire::BlockType::BT_SCALAR;
break;
case BlockType::SCALAR_INDEX:
block_types = proto::BlockType::BT_SCALAR_INDEX;
block_types = wire::BlockType::BT_SCALAR_INDEX;
break;
case BlockType::VECTOR_INDEX:
block_types = proto::BlockType::BT_VECTOR_INDEX;
block_types = wire::BlockType::BT_VECTOR_INDEX;
break;
case BlockType::VECTOR_INDEX_QUANTIZE:
block_types = proto::BlockType::BT_VECTOR_INDEX_QUANTIZE;
block_types = wire::BlockType::BT_VECTOR_INDEX_QUANTIZE;
break;
case BlockType::FTS_INDEX:
block_types = proto::BlockType::BT_FTS_INDEX;
block_types = wire::BlockType::BT_FTS_INDEX;
break;
default:
break;
@@ -577,4 +579,4 @@ struct BlockTypeCodeBook {
}
};
} // namespace zvec
} // namespace zvec
src/db/index/common/version_manager.cc
+34 -39
View File
@@ -18,17 +18,18 @@
#include <cstring>
#include <filesystem>
#include <fstream>
#include <memory>
#include <mutex>
#include <regex>
#include <sstream>
#include <string>
#include <proto/zvec.pb.h>
#include <zvec/ailego/logger/logger.h>
#include <zvec/ailego/pattern/expected.hpp>
#include <zvec/ailego/utility/string_helper.h>
#include <zvec/db/status.h>
#include "db/common/file_helper.h"
#include "db/common/typedef.h"
#include "db/index/common/proto_converter.h"
#include "db/index/common/manifest_codec.h"
#include "db/index/common/type_helper.h"
namespace zvec {
@@ -41,35 +42,36 @@ Status Version::Load(const std::string &path, Version *version) {
return Status::InternalError("Failed to open file");
}
proto::Manifest manifest;
// Manifests are small (kilobytes), so reading the whole file at once keeps
// the decoder simple.
std::ostringstream buffer;
buffer << ifs.rdbuf();
const std::string encoded = buffer.str();
if (!manifest.ParseFromIstream(&ifs)) {
ManifestData manifest;
auto status = ManifestCodec::Decode(encoded, &manifest);
if (!status.ok()) {
LOG_ERROR("Failed to parse manifest from file: %s", path.c_str());
return Status::InternalError("Failed to parse manifest");
}
CollectionSchema::Ptr schema = ProtoConverter::FromPb(manifest.schema());
version->set_schema(*schema);
version->set_schema(*manifest.schema);
version->set_enable_mmap(manifest.enable_mmap());
version->set_enable_mmap(manifest.enable_mmap);
for (int i = 0; i < manifest.persisted_segment_metas_size(); ++i) {
SegmentMeta::Ptr meta =
ProtoConverter::FromPb(manifest.persisted_segment_metas(i));
for (auto &meta : manifest.persisted_segment_metas) {
version->add_persisted_segment_meta(meta);
}
if (manifest.has_writing_segment_meta()) {
SegmentMeta::Ptr meta =
ProtoConverter::FromPb(manifest.writing_segment_meta());
version->reset_writing_segment_meta(meta);
if (manifest.writing_segment_meta) {
version->reset_writing_segment_meta(manifest.writing_segment_meta);
}
version->set_id_map_path_suffix(manifest.id_map_path_suffix());
version->set_id_map_path_suffix(manifest.id_map_path_suffix);
version->set_delete_snapshot_path_suffix(
manifest.delete_snapshot_path_suffix());
manifest.delete_snapshot_path_suffix);
version->set_next_segment_id(manifest.next_segment_id());
version->set_next_segment_id(manifest.next_segment_id);
return Status::OK();
}
@@ -83,31 +85,24 @@ Status Version::Save(const std::string &path, const Version &version) {
return Status::InternalError("Failed to open file: %s", path.c_str());
}
proto::Manifest manifest;
ManifestData manifest;
manifest.schema = std::make_shared<CollectionSchema>(version.schema());
manifest.enable_mmap = version.enable_mmap();
manifest.persisted_segment_metas = version.persisted_segment_metas();
manifest.writing_segment_meta = version.writing_segment_meta();
manifest.id_map_path_suffix = version.id_map_path_suffix();
manifest.delete_snapshot_path_suffix = version.delete_snapshot_path_suffix();
manifest.next_segment_id = version.next_segment_id();
// set schema
auto schema = ProtoConverter::ToPb(version.schema());
manifest.mutable_schema()->Swap(&schema);
manifest.set_enable_mmap(version.enable_mmap());
// set segments meta
for (auto &meta : version.persisted_segment_metas()) {
auto meta_pb = ProtoConverter::ToPb(*meta);
manifest.add_persisted_segment_metas()->Swap(&meta_pb);
std::string encoded;
auto status = ManifestCodec::Encode(manifest, &encoded);
if (!status.ok()) {
LOG_ERROR("Failed to serialize manifest to file: %s", path.c_str());
return Status::InternalError("Failed to serialize manifest to file");
}
if (version.writing_segment_meta()) {
auto meta_pb = ProtoConverter::ToPb(*version.writing_segment_meta());
manifest.mutable_writing_segment_meta()->Swap(&meta_pb);
}
manifest.set_id_map_path_suffix(version.id_map_path_suffix());
manifest.set_delete_snapshot_path_suffix(
version.delete_snapshot_path_suffix());
manifest.set_next_segment_id(version.next_segment_id());
if (!manifest.SerializeToOstream(&ofs)) {
ofs.write(encoded.data(), static_cast<std::streamsize>(encoded.size()));
if (!ofs.good()) {
LOG_ERROR("Failed to serialize manifest to file: %s", path.c_str());
return Status::InternalError("Failed to serialize manifest to file");
}
src/db/index/segment/segment.cc
+12 -13
View File
@@ -25,8 +25,6 @@
#include <unordered_set>
#include <ailego/parallel/multi_thread_list.h>
#include <ailego/pattern/defer.h>
#include <arrow/dataset/dataset.h>
#include <arrow/dataset/scanner.h>
#include <arrow/ipc/reader.h>
#include <arrow/table.h>
#include <arrow/util/iterator.h>
@@ -3220,14 +3218,14 @@ Status SegmentImpl::add_column(FieldSchema::Ptr column_schema,
}
auto expr = p_result.ValueOrDie();
auto result = ReadBlocksAsDataset(scalar_blocks, path_, segment_meta_->id(),
!options_.enable_mmap_);
auto result = ReadBlocksAsTable(scalar_blocks, path_, segment_meta_->id(),
!options_.enable_mmap_);
if (!result.ok()) {
return Status::InternalError(result.status().message());
}
auto dataset = std::move(result).ValueOrDie();
auto eval_result = EvaluateExpressionWithDataset(
dataset, column_schema->name(), expr, expected_type);
auto table = std::move(result).ValueOrDie();
auto eval_result = EvaluateExpressionOnTable(table, column_schema->name(),
expr, expected_type);
if (!eval_result.ok()) {
return Status::InternalError("evaluate expression failed:",
eval_result.status().message());
@@ -3366,16 +3364,17 @@ Status SegmentImpl::alter_column(const std::string &column_name,
}
}
auto result = ReadBlocksAsDataset(
filter_column_blocks, path_, segment_meta_->id(), !options_.enable_mmap_);
auto result = ReadBlocksAsTable(filter_column_blocks, path_,
segment_meta_->id(), !options_.enable_mmap_);
if (!result.ok()) {
return Status::InternalError(result.status().message());
}
auto dataset = std::move(result).ValueOrDie();
auto table = std::move(result).ValueOrDie();
arrow::Expression expr = arrow::compute::field_ref(old_field_schema->name());
auto eval_result = EvaluateExpressionWithDataset(
dataset, new_column_name, expr, new_arrow_field->type());
arrow::compute::Expression expr =
arrow::compute::field_ref(old_field_schema->name());
auto eval_result = EvaluateExpressionOnTable(table, new_column_name, expr,
new_arrow_field->type());
if (!eval_result.ok()) {
return Status::InternalError("evaluate expression failed:",
eval_result.status().message());
src/db/index/storage/bufferpool_forward_store.cc
+1 -1
View File
@@ -355,7 +355,7 @@ ExecBatchPtr BufferPoolForwardStore::fetch(
scalars.emplace_back(std::move(scalar_result.ValueOrDie()));
}
return std::make_shared<arrow::ExecBatch>(std::move(scalars), 1);
return std::make_shared<arrow::compute::ExecBatch>(std::move(scalars), 1);
}
RecordBatchReaderPtr BufferPoolForwardStore::scan(
src/db/index/storage/memory_forward_store.cc
+1 -1
View File
@@ -463,7 +463,7 @@ ExecBatchPtr MemForwardStore::fetch(const std::vector<std::string> &columns,
scalars.emplace_back(std::move(scalar_result.ValueOrDie()));
}
return std::make_shared<arrow::ExecBatch>(std::move(scalars), 1);
return std::make_shared<arrow::compute::ExecBatch>(std::move(scalars), 1);
}
RecordBatchReaderPtr MemForwardStore::scan(
src/db/index/storage/mmap_forward_store.cc
+2 -2
View File
@@ -378,7 +378,7 @@ ExecBatchPtr MmapForwardStore::FetchParquet(
scalars.emplace_back(std::move(scalar_result.ValueOrDie()));
}
return std::make_shared<arrow::ExecBatch>(std::move(scalars), 1);
return std::make_shared<arrow::compute::ExecBatch>(std::move(scalars), 1);
}
TablePtr MmapForwardStore::FetchIPC(const std::vector<std::string> &columns,
@@ -460,7 +460,7 @@ ExecBatchPtr MmapForwardStore::FetchIPC(const std::vector<std::string> &columns,
}
}
return std::make_shared<arrow::ExecBatch>(std::move(scalars), 1);
return std::make_shared<arrow::compute::ExecBatch>(std::move(scalars), 1);
}
int MmapForwardStore::FindRowGroupForRow(int64_t row) {
src/db/index/storage/mmap_forward_store.h
-1
View File
@@ -22,7 +22,6 @@
#include <arrow/api.h>
#include <arrow/chunked_array.h>
#include <arrow/compute/api.h>
#include <arrow/dataset/api.h>
#include <arrow/filesystem/api.h>
#include <arrow/io/file.h>
#include <arrow/ipc/reader.h>
src/db/index/storage/store_helper.h
+38 -33
View File
@@ -22,7 +22,6 @@
#include <string>
#include <arrow/api.h>
#include <arrow/compute/api.h>
#include <arrow/dataset/api.h>
#include <arrow/filesystem/api.h>
#include <arrow/io/file.h>
#include <arrow/ipc/reader.h>
@@ -758,10 +757,9 @@ inline arrow::Result<std::shared_ptr<arrow::Array>> SelectArrayByIndices(
return arrow::compute::Take(*arr, *indices_array);
}
inline arrow::Result<std::shared_ptr<arrow::dataset::Dataset>>
ReadBlocksAsDataset(const std::vector<BlockMeta> &scalar_blocks,
const std::string &base_path, uint32_t collection_id,
bool use_parquet) {
inline arrow::Result<std::shared_ptr<arrow::Table>> ReadBlocksAsTable(
const std::vector<BlockMeta> &scalar_blocks, const std::string &base_path,
uint32_t collection_id, bool use_parquet) {
auto fs = std::make_shared<arrow::fs::LocalFileSystem>();
auto pool = arrow::default_memory_pool();
@@ -858,45 +856,52 @@ ReadBlocksAsDataset(const std::vector<BlockMeta> &scalar_blocks,
ARROW_ASSIGN_OR_RAISE(auto final_table,
arrow::ConcatenateTables(segment_tables));
auto dataset = std::make_shared<arrow::dataset::InMemoryDataset>(final_table);
return dataset;
return final_table;
}
inline arrow::Result<std::shared_ptr<arrow::Table>>
EvaluateExpressionWithDataset(
const std::shared_ptr<arrow::dataset::Dataset> &dataset,
inline arrow::Result<std::shared_ptr<arrow::Table>> EvaluateExpressionOnTable(
const std::shared_ptr<arrow::Table> &table,
const std::string &new_column_name, const arrow::compute::Expression &expr,
const std::shared_ptr<arrow::DataType> &expected_type) {
auto new_scan_result = dataset->NewScan();
if (!new_scan_result.ok()) {
return arrow::Status::Invalid("Failed to create scanner builder");
}
auto scanner_builder = std::move(new_scan_result.ValueOrDie());
arrow::compute::CastOptions cast_options;
cast_options.to_type = expected_type;
cast_options.allow_int_overflow = true;
cast_options.allow_float_truncate = true;
arrow::Expression cast_expr = call("cast", {expr}, cast_options);
arrow::compute::Expression cast_expr =
arrow::compute::call("cast", {expr}, cast_options);
auto status = scanner_builder->Project({cast_expr}, {new_column_name});
if (!status.ok()) {
return arrow::Status::Invalid("Failed to project expression: ",
status.ToString());
}
auto scanner_result = scanner_builder->Finish();
if (!scanner_result.ok()) {
return arrow::Status::Invalid("Failed to finish scanner builder: ",
scanner_result.status().ToString());
}
auto scanner = std::move(scanner_result.ValueOrDie());
ARROW_ASSIGN_OR_RAISE(auto bound_expr, cast_expr.Bind(*table->schema()));
auto to_table_result = scanner->ToTable();
if (!to_table_result.ok()) {
return arrow::Status::Invalid("Failed to convert scanner to table: ",
to_table_result.status().ToString());
auto batches = arrow::TableBatchReader(*table);
std::vector<std::shared_ptr<arrow::Array>> result_arrays;
std::shared_ptr<arrow::RecordBatch> batch;
while (true) {
ARROW_RETURN_NOT_OK(batches.ReadNext(&batch));
if (!batch) break;
arrow::compute::ExecBatch exec_batch(*batch);
ARROW_ASSIGN_OR_RAISE(auto datum, arrow::compute::ExecuteScalarExpression(
bound_expr, exec_batch));
if (datum.is_array()) {
result_arrays.push_back(datum.make_array());
} else if (datum.is_scalar()) {
ARROW_ASSIGN_OR_RAISE(auto arr, arrow::MakeArrayFromScalar(
*datum.scalar(), batch->num_rows()));
result_arrays.push_back(arr);
} else {
return arrow::Status::Invalid(
"Expression result is neither array nor scalar");
}
}
auto result_table = std::move(to_table_result.ValueOrDie());
auto result_chunked =
std::make_shared<arrow::ChunkedArray>(result_arrays, expected_type);
auto result_field = arrow::field(new_column_name, expected_type);
auto result_schema = arrow::schema({result_field});
auto result_table =
arrow::Table::Make(result_schema, {result_chunked}, table->num_rows());
return result_table;
}
src/db/proto/zvec.proto
-251
View File
@@ -1,251 +0,0 @@
syntax = "proto3";
package zvec.proto;
option cc_enable_arenas = true;
// The Go package name, refers to
// https://developers.google.com/protocol-buffers/docs/reference/go-generated#package
option go_package = "proxima/zvec/proto";
/*! Types of Data
*/
enum DataType {
DT_UNDEFINED = 0;
DT_BINARY = 1;
DT_STRING = 2;
DT_BOOL = 3;
DT_INT32 = 4;
DT_INT64 = 5;
DT_UINT32 = 6;
DT_UINT64 = 7;
DT_FLOAT = 8;
DT_DOUBLE = 9;
DT_VECTOR_BINARY32 = 20;
DT_VECTOR_BINARY64 = 21;
DT_VECTOR_FP16 = 22;
DT_VECTOR_FP32 = 23;
DT_VECTOR_FP64 = 24;
DT_VECTOR_INT4 = 25;
DT_VECTOR_INT8 = 26;
DT_VECTOR_INT16 = 27;
DT_SPARSE_VECTOR_FP16 = 30;
DT_SPARSE_VECTOR_FP32 = 31;
// ARRAY
DT_ARRAY_BINARY = 40;
DT_ARRAY_STRING = 41;
DT_ARRAY_BOOL = 42;
DT_ARRAY_INT32 = 43;
DT_ARRAY_INT64 = 44;
DT_ARRAY_UINT32 = 45;
DT_ARRAY_UINT64 = 46;
DT_ARRAY_FLOAT = 47;
DT_ARRAY_DOUBLE = 48;
};
enum IndexType {
// Undefined
IT_UNDEFINED = 0;
// Proxima HNSW Index
IT_HNSW = 1;
// Proxima IVF Index
IT_IVF = 2;
// Proxima FLAT Index
IT_FLAT = 3;
// Proxima HNSW RABITQ Index
IT_HNSW_RABITQ = 4;
// Proxima Vamana Index
IT_VAMANA = 5;
// Proxima DiskAnn Index
IT_DISKANN = 6;
// Proxima IVF RABITQ Index
IT_IVF_RABITQ = 7;
// Invert Index
IT_INVERT = 10;
// Full-Text Search Index
IT_FTS = 11;
};
enum QuantizeType {
QT_UNDEFINED = 0;
QT_FP16 = 1;
QT_INT8 = 2;
QT_INT4 = 3;
QT_RABITQ = 4;
};
enum MetricType {
MT_UNDEFINED = 0;
MT_L2 = 1;
MT_IP = 2;
MT_COSINE = 3;
};
message InvertIndexParams {
bool enable_range_optimization = 1;
};
// Quantizer-related parameters for vector indexes.
// Designed for future extensibility.
message QuantizerParam {
// When enabled, vectors are rotated before INT8 quantization to reduce
// quantization error. Only effective with quantize_type=INT8.
bool enable_rotate = 1;
};
message BaseIndexParams {
MetricType metric_type = 1;
QuantizeType quantize_type = 2;
// Quantizer parameters (enable_rotate, etc.)
QuantizerParam quantizer_param = 4;
};
message HnswIndexParams {
BaseIndexParams base = 1;
int32 m = 2;
int32 ef_construction = 3;
// When enabled, the HNSW streamer allocates a single contiguous memory
// arena for all graph nodes, which improves cache locality / search
// throughput at the cost of peak memory usage. Defaults to false.
bool use_contiguous_memory = 4;
}
message HnswRabitqIndexParams {
BaseIndexParams base = 1;
int32 m = 2;
int32 ef_construction = 3;
int32 total_bits = 4;
int32 num_clusters = 5;
int32 sample_count = 6;
}
message IvfRabitqIndexParams {
BaseIndexParams base = 1;
int32 nlist = 2;
int32 total_bits = 3;
int32 sample_count = 4;
}
message FlatIndexParams {
BaseIndexParams base = 1;
}
message IVFIndexParams {
BaseIndexParams base = 1;
int32 n_list = 2;
int32 n_iters = 3;
bool use_soar = 4;
}
message DiskAnnIndexParams {
BaseIndexParams base = 1;
int32 max_degree = 2;
int32 list_size = 3;
int32 pq_chunk_num = 4;
}
message VamanaIndexParams {
BaseIndexParams base = 1;
int32 max_degree = 2;
int32 search_list_size = 3;
float alpha = 4;
bool saturate_graph = 5;
// When enabled, the Vamana streamer allocates a single contiguous memory
// arena for all graph nodes, which improves cache locality / search
// throughput at the cost of peak memory usage. Defaults to false.
bool use_contiguous_memory = 6;
bool use_id_map = 7;
// Run the standard alpha=1.0 pass followed by a full-graph pass at alpha.
bool two_pass_build = 8;
}
message FtsIndexParams {
string tokenizer_name = 1;
repeated string filters = 2;
string extra_params = 3;
};
message IndexParams {
oneof params {
InvertIndexParams invert = 1;
HnswIndexParams hnsw = 2;
FlatIndexParams flat = 3;
IVFIndexParams ivf = 4;
HnswRabitqIndexParams hnsw_rabitq = 5;
VamanaIndexParams vamana = 6;
FtsIndexParams fts = 7;
DiskAnnIndexParams diskann = 8;
IvfRabitqIndexParams ivf_rabitq = 9;
};
};
message FieldSchema {
string name = 1;
DataType data_type = 2;
uint32 dimension = 3;
bool nullable = 4;
IndexParams index_params = 5;
};
message CollectionSchema {
string name = 1;
repeated FieldSchema fields = 2;
uint64 max_doc_count_per_segment = 3;
};
enum BlockType {
BT_UNDEFINED = 0;
BT_SCALAR = 1;
BT_SCALAR_INDEX = 2;
BT_VECTOR_INDEX = 3;
BT_VECTOR_INDEX_QUANTIZE = 4;
BT_FTS_INDEX = 5;
};
message BlockMeta {
uint32 block_id = 1;
BlockType block_type = 2; // for getting filename prefix
uint64 min_doc_id = 3;
uint64 max_doc_id = 4;
uint64 doc_count = 5;
repeated string columns = 6; // columns contained in this block
};
// message AlterColumnMeta {
// string old_column_name = 1;
// FieldSchema new_schema = 2;
// };
message SegmentMeta {
uint32 segment_id = 1;
// scalar data, vector data and vector index
repeated BlockMeta persisted_blocks = 2;
BlockMeta writing_forward_block = 3;
// if indexed, index_params can be retrieved from schema
// if not indexed, index_params is default index_params(flat)
repeated string indexed_vector_fields = 4;
// repeated AlterColumnMeta alter_columns = 10;
};
message Manifest {
uint32 version = 1;
CollectionSchema schema = 2;
bool enable_mmap = 3;
repeated SegmentMeta persisted_segment_metas = 4;
SegmentMeta writing_segment_meta = 5;
uint32 id_map_path_suffix = 6;
uint32 delete_snapshot_path_suffix = 7;
uint32 next_segment_id = 8;
};
src/exported_symbols_cpp.lds
+23
View File
@@ -0,0 +1,23 @@
{
global:
extern "C++" {
/* Unquoted = glob pattern matching all demangled zvec:: symbols.
Do NOT add a quoted "zvec::*" entry: quotes mean literal match,
and lld (Android NDK) fails with --no-undefined-version when a
literal entry matches no symbol. */
zvec::*;
};
zvec_*;
/* vtable / typeinfo / VTT / guard variables of zvec:: classes.
Their demangled names start with "vtable for" etc., so the
zvec::* patterns above do not match them; without these the
consumers fail with "undefined reference to vtable". Mirrors
__ZTVN4zvec* etc. in exported_symbols_cpp.txt on macOS. */
_ZTVN4zvec*;
_ZTIN4zvec*;
_ZTSN4zvec*;
_ZTTN4zvec*;
_ZGVN4zvec*;
local:
*;
};
src/exported_symbols_cpp.txt
+11
View File
@@ -0,0 +1,11 @@
__ZN4zvec*
__ZNK4zvec*
__ZZN4zvec*
__ZZNK4zvec*
__ZTVN4zvec*
__ZTIN4zvec*
__ZTSN4zvec*
__ZGVN4zvec*
__ZGVZN4zvec*
__ZGVZNK4zvec*
_zvec_*
tests/db/CMakeLists.txt
-1
View File
@@ -28,7 +28,6 @@ foreach(CC_SRCS ${ALL_TEST_SRCS})
cc_gmock(
NAME ${CC_TARGET} STRICT
LIBS zvec
zvec_proto
core_knn_flat
core_knn_flat_sparse
core_knn_hnsw
tests/db/crash_recovery/CMakeLists.txt
-1
View File
@@ -27,7 +27,6 @@ endif()
# Common libraries
set(CRASH_RECOVERY_COMMON_LIBS
zvec
zvec_proto
core_knn_flat
core_knn_flat_sparse
core_knn_hnsw
tests/db/index/CMakeLists.txt
-2
View File
@@ -38,7 +38,6 @@ foreach(CC_SRCS ${ALL_TEST_SRCS})
NAME ${CC_TARGET} STRICT
LIBS zvec
zvec_ailego
zvec_proto
core_knn_diskann
core_metric_static
core_utility_static
@@ -46,7 +45,6 @@ foreach(CC_SRCS ${ALL_TEST_SRCS})
core_knn_hnsw core_knn_hnsw_sparse sparsehash
core_knn_flat core_knn_flat_sparse core_knn_ivf
core_knn_hnsw_rabitq core_knn_ivf_rabitq core_mix_reducer
Arrow::arrow_dataset
${CMAKE_THREAD_LIBS_INIT}
${CMAKE_DL_LIBS}
SRCS ${CC_SRCS} utils/utils.cc
tests/db/index/common/db_proto_converter_test.cc
-647
View File
@@ -1,647 +0,0 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <gtest/gtest.h>
#include "db/index/common/proto_converter.h"
#include "db/index/common/type_helper.h"
using namespace zvec;
TEST(ConverterTest, InvertIndexParamsConversion) {
// Test conversion from protobuf to C++ InvertIndexParams
proto::InvertIndexParams invert_pb;
invert_pb.set_enable_range_optimization(true);
auto invert_params = ProtoConverter::FromPb(invert_pb);
ASSERT_NE(invert_params, nullptr);
EXPECT_TRUE(invert_params->enable_range_optimization());
EXPECT_EQ(invert_params->type(), IndexType::INVERT);
// Test with false value
proto::InvertIndexParams invert_pb2;
invert_pb2.set_enable_range_optimization(false);
auto invert_params2 = ProtoConverter::FromPb(invert_pb2);
ASSERT_NE(invert_params2, nullptr);
EXPECT_FALSE(invert_params2->enable_range_optimization());
// Test conversion from C++ to protobuf
InvertIndexParams original_params(true);
auto pb_result = ProtoConverter::ToPb(&original_params);
EXPECT_TRUE(pb_result.enable_range_optimization());
}
TEST(ConverterTest, HnswIndexParamsConversion) {
// Test conversion from protobuf to C++ HnswIndexParams
proto::HnswIndexParams hnsw_pb;
auto *base_params = hnsw_pb.mutable_base();
base_params->set_metric_type(proto::MT_L2);
base_params->set_quantize_type(proto::QT_FP16);
hnsw_pb.set_m(16);
hnsw_pb.set_ef_construction(100);
auto hnsw_params = ProtoConverter::FromPb(hnsw_pb);
ASSERT_NE(hnsw_params, nullptr);
EXPECT_EQ(hnsw_params->metric_type(), MetricType::L2);
EXPECT_EQ(hnsw_params->m(), 16);
EXPECT_EQ(hnsw_params->ef_construction(), 100);
EXPECT_EQ(hnsw_params->quantize_type(), QuantizeType::FP16);
EXPECT_EQ(hnsw_params->type(), IndexType::HNSW);
// Test conversion from C++ to protobuf
HnswIndexParams original_params(MetricType::IP, 32, 200, QuantizeType::INT8);
auto pb_result = ProtoConverter::ToPb(&original_params);
EXPECT_EQ(pb_result.base().metric_type(), proto::MT_IP);
EXPECT_EQ(pb_result.m(), 32);
EXPECT_EQ(pb_result.ef_construction(), 200);
EXPECT_EQ(pb_result.base().quantize_type(), proto::QT_INT8);
}
TEST(ConverterTest, FlatIndexParamsConversion) {
// Test conversion from protobuf to C++ FlatIndexParams
proto::FlatIndexParams flat_pb;
auto *base_params = flat_pb.mutable_base();
base_params->set_metric_type(proto::MT_COSINE);
base_params->set_quantize_type(proto::QT_INT4);
auto flat_params = ProtoConverter::FromPb(flat_pb);
ASSERT_NE(flat_params, nullptr);
EXPECT_EQ(flat_params->metric_type(), MetricType::COSINE);
EXPECT_EQ(flat_params->quantize_type(), QuantizeType::INT4);
EXPECT_EQ(flat_params->type(), IndexType::FLAT);
// Test conversion from C++ to protobuf
FlatIndexParams original_params(MetricType::L2, QuantizeType::FP16);
auto pb_result = ProtoConverter::ToPb(&original_params);
EXPECT_EQ(pb_result.base().metric_type(), proto::MT_L2);
EXPECT_EQ(pb_result.base().quantize_type(), proto::QT_FP16);
}
TEST(ConverterTest, IVFIndexParamsConversion) {
// Test conversion from protobuf to C++ IVFIndexParams
proto::IVFIndexParams ivf_pb;
auto *base_params = ivf_pb.mutable_base();
base_params->set_metric_type(proto::MT_IP);
base_params->set_quantize_type(proto::QT_INT8);
ivf_pb.set_n_list(128);
auto ivf_params = ProtoConverter::FromPb(ivf_pb);
ASSERT_NE(ivf_params, nullptr);
EXPECT_EQ(ivf_params->metric_type(), MetricType::IP);
EXPECT_EQ(ivf_params->n_list(), 128);
EXPECT_EQ(ivf_params->quantize_type(), QuantizeType::INT8);
EXPECT_EQ(ivf_params->type(), IndexType::IVF);
// Test conversion from C++ to protobuf
IVFIndexParams original_params(MetricType::COSINE, 256, 10, false,
QuantizeType::INT4);
auto pb_result = ProtoConverter::ToPb(&original_params);
EXPECT_EQ(pb_result.base().metric_type(), proto::MT_COSINE);
EXPECT_EQ(pb_result.n_list(), 256);
EXPECT_EQ(pb_result.n_iters(), 10);
EXPECT_FALSE(pb_result.use_soar());
EXPECT_EQ(pb_result.base().quantize_type(), proto::QT_INT4);
}
#if RABITQ_SUPPORTED
TEST(ConverterTest, IvfRabitqIndexParamsConversion) {
proto::IvfRabitqIndexParams ivf_rabitq_pb;
auto *base_params = ivf_rabitq_pb.mutable_base();
base_params->set_metric_type(proto::MT_COSINE);
base_params->set_quantize_type(proto::QT_RABITQ);
ivf_rabitq_pb.set_nlist(32);
ivf_rabitq_pb.set_total_bits(1);
ivf_rabitq_pb.set_sample_count(5);
auto ivf_rabitq_params = ProtoConverter::FromPb(ivf_rabitq_pb);
ASSERT_NE(ivf_rabitq_params, nullptr);
EXPECT_EQ(ivf_rabitq_params->metric_type(), MetricType::COSINE);
EXPECT_EQ(ivf_rabitq_params->quantize_type(), QuantizeType::RABITQ);
EXPECT_EQ(ivf_rabitq_params->type(), IndexType::IVF_RABITQ);
EXPECT_EQ(ivf_rabitq_params->nlist(), 32);
EXPECT_EQ(ivf_rabitq_params->total_bits(), 1);
EXPECT_EQ(ivf_rabitq_params->sample_count(), 5);
IvfRabitqIndexParams original_params(MetricType::IP, 64, 7, 3);
auto pb_result = ProtoConverter::ToPb(&original_params);
EXPECT_EQ(pb_result.base().metric_type(), proto::MT_IP);
EXPECT_EQ(pb_result.base().quantize_type(), proto::QT_RABITQ);
EXPECT_EQ(pb_result.nlist(), 64);
EXPECT_EQ(pb_result.total_bits(), 7);
EXPECT_EQ(pb_result.sample_count(), 3);
}
#endif
TEST(ConverterTest, VamanaIndexParamsConversion) {
proto::VamanaIndexParams vamana_pb;
auto *base_params = vamana_pb.mutable_base();
base_params->set_metric_type(proto::MT_L2);
base_params->set_quantize_type(proto::QT_FP16);
vamana_pb.set_max_degree(48);
vamana_pb.set_search_list_size(160);
vamana_pb.set_alpha(1.4f);
vamana_pb.set_saturate_graph(true);
vamana_pb.set_use_contiguous_memory(true);
vamana_pb.set_use_id_map(false);
vamana_pb.set_two_pass_build(true);
auto params = ProtoConverter::FromPb(vamana_pb);
ASSERT_NE(params, nullptr);
EXPECT_EQ(params->metric_type(), MetricType::L2);
EXPECT_EQ(params->max_degree(), 48);
EXPECT_EQ(params->search_list_size(), 160);
EXPECT_FLOAT_EQ(params->alpha(), 1.4f);
EXPECT_TRUE(params->saturate_graph());
EXPECT_TRUE(params->use_contiguous_memory());
EXPECT_FALSE(params->use_id_map());
EXPECT_TRUE(params->two_pass_build());
EXPECT_EQ(params->quantize_type(), QuantizeType::FP16);
VamanaIndexParams original(MetricType::COSINE, 32, 100, 1.3f, false, false,
false, QuantizeType::INT8, QuantizerParam(), true);
auto pb_result = ProtoConverter::ToPb(&original);
EXPECT_EQ(pb_result.base().metric_type(), proto::MT_COSINE);
EXPECT_EQ(pb_result.base().quantize_type(), proto::QT_INT8);
EXPECT_EQ(pb_result.max_degree(), 32);
EXPECT_EQ(pb_result.search_list_size(), 100);
EXPECT_FLOAT_EQ(pb_result.alpha(), 1.3f);
EXPECT_TRUE(pb_result.two_pass_build());
}
TEST(ConverterTest, IndexParamsConversion) {
// Test conversion from protobuf to C++ IndexParams for HNSW
proto::IndexParams index_pb;
auto *hnsw_pb = index_pb.mutable_hnsw();
auto *base_params = hnsw_pb->mutable_base();
base_params->set_metric_type(proto::MT_L2);
base_params->set_quantize_type(proto::QT_FP16);
hnsw_pb->set_m(16);
hnsw_pb->set_ef_construction(100);
auto index_params = ProtoConverter::FromPb(index_pb);
ASSERT_NE(index_params, nullptr);
EXPECT_EQ(index_params->type(), IndexType::HNSW);
auto hnsw_cast = std::dynamic_pointer_cast<HnswIndexParams>(index_params);
ASSERT_NE(hnsw_cast, nullptr);
EXPECT_EQ(hnsw_cast->metric_type(), MetricType::L2);
EXPECT_EQ(hnsw_cast->m(), 16);
EXPECT_EQ(hnsw_cast->ef_construction(), 100);
EXPECT_EQ(hnsw_cast->quantize_type(), QuantizeType::FP16);
// Test conversion from C++ HnswIndexParams to protobuf IndexParams
HnswIndexParams hnsw_original(MetricType::IP, 32, 200);
auto pb_result = ProtoConverter::ToPb(&hnsw_original);
EXPECT_EQ(pb_result.base().metric_type(), proto::MT_IP);
EXPECT_EQ(pb_result.m(), 32);
EXPECT_EQ(pb_result.ef_construction(), 200);
// Test conversion from protobuf to C++ IndexParams for FLAT
proto::IndexParams index_pb2;
auto *flat_pb = index_pb2.mutable_flat();
auto *base_params2 = flat_pb->mutable_base();
base_params2->set_metric_type(proto::MT_COSINE);
base_params2->set_quantize_type(proto::QT_INT8);
auto index_params2 = ProtoConverter::FromPb(index_pb2);
ASSERT_NE(index_params2, nullptr);
EXPECT_EQ(index_params2->type(), IndexType::FLAT);
auto flat_cast = std::dynamic_pointer_cast<FlatIndexParams>(index_params2);
ASSERT_NE(flat_cast, nullptr);
EXPECT_EQ(flat_cast->metric_type(), MetricType::COSINE);
EXPECT_EQ(flat_cast->quantize_type(), QuantizeType::INT8);
// Test conversion from C++ FlatIndexParams to protobuf IndexParams
FlatIndexParams flat_original(MetricType::L2);
auto pb_result2 = ProtoConverter::ToPb(&flat_original);
EXPECT_EQ(pb_result2.base().metric_type(), proto::MT_L2);
// Test conversion from protobuf to C++ IndexParams for IVF
proto::IndexParams index_pb3;
auto *ivf_pb = index_pb3.mutable_ivf();
auto *base_params3 = ivf_pb->mutable_base();
base_params3->set_metric_type(proto::MT_IP);
base_params3->set_quantize_type(proto::QT_INT4);
ivf_pb->set_n_list(128);
auto index_params3 = ProtoConverter::FromPb(index_pb3);
ASSERT_NE(index_params3, nullptr);
EXPECT_EQ(index_params3->type(), IndexType::IVF);
auto ivf_cast = std::dynamic_pointer_cast<IVFIndexParams>(index_params3);
ASSERT_NE(ivf_cast, nullptr);
EXPECT_EQ(ivf_cast->metric_type(), MetricType::IP);
EXPECT_EQ(ivf_cast->n_list(), 128);
EXPECT_EQ(ivf_cast->quantize_type(), QuantizeType::INT4);
// Test conversion from C++ IVFIndexParams to protobuf IndexParams
IVFIndexParams ivf_original(MetricType::COSINE, 256);
auto pb_result3 = ProtoConverter::ToPb(&ivf_original);
EXPECT_EQ(pb_result3.base().metric_type(), proto::MT_COSINE);
EXPECT_EQ(pb_result3.n_list(), 256);
#if RABITQ_SUPPORTED
// Test conversion from protobuf to C++ IndexParams for IVF_RABITQ
proto::IndexParams index_pb5;
auto *ivf_rabitq_pb = index_pb5.mutable_ivf_rabitq();
auto *base_params5 = ivf_rabitq_pb->mutable_base();
base_params5->set_metric_type(proto::MT_IP);
base_params5->set_quantize_type(proto::QT_RABITQ);
ivf_rabitq_pb->set_nlist(32);
ivf_rabitq_pb->set_total_bits(1);
ivf_rabitq_pb->set_sample_count(5);
auto index_params5 = ProtoConverter::FromPb(index_pb5);
ASSERT_NE(index_params5, nullptr);
EXPECT_EQ(index_params5->type(), IndexType::IVF_RABITQ);
auto ivf_rabitq_cast =
std::dynamic_pointer_cast<IvfRabitqIndexParams>(index_params5);
ASSERT_NE(ivf_rabitq_cast, nullptr);
EXPECT_EQ(ivf_rabitq_cast->metric_type(), MetricType::IP);
EXPECT_EQ(ivf_rabitq_cast->quantize_type(), QuantizeType::RABITQ);
EXPECT_EQ(ivf_rabitq_cast->nlist(), 32);
EXPECT_EQ(ivf_rabitq_cast->total_bits(), 1);
EXPECT_EQ(ivf_rabitq_cast->sample_count(), 5);
IvfRabitqIndexParams ivf_rabitq_original(MetricType::COSINE, 64, 7, 3);
auto pb_result5 = ProtoConverter::ToPb(&ivf_rabitq_original);
EXPECT_EQ(pb_result5.base().metric_type(), proto::MT_COSINE);
EXPECT_EQ(pb_result5.base().quantize_type(), proto::QT_RABITQ);
EXPECT_EQ(pb_result5.nlist(), 64);
EXPECT_EQ(pb_result5.total_bits(), 7);
EXPECT_EQ(pb_result5.sample_count(), 3);
#endif
// Test conversion from protobuf to C++ IndexParams for INVERT
proto::IndexParams index_pb4;
auto *invert_pb = index_pb4.mutable_invert();
invert_pb->set_enable_range_optimization(true);
auto index_params4 = ProtoConverter::FromPb(index_pb4);
ASSERT_NE(index_params4, nullptr);
EXPECT_EQ(index_params4->type(), IndexType::INVERT);
auto invert_cast =
std::dynamic_pointer_cast<InvertIndexParams>(index_params4);
ASSERT_NE(invert_cast, nullptr);
EXPECT_TRUE(invert_cast->enable_range_optimization());
// Test conversion from C++ InvertIndexParams to protobuf IndexParams
InvertIndexParams invert_original(false);
auto pb_result4 = ProtoConverter::ToPb(&invert_original);
EXPECT_FALSE(pb_result4.enable_range_optimization());
}
TEST(ConverterTest, FieldSchemaConversion) {
// Test conversion from protobuf to C++ FieldSchema
proto::FieldSchema field_pb;
field_pb.set_name("test_field");
field_pb.set_data_type(proto::DT_VECTOR_FP32);
field_pb.set_dimension(128);
field_pb.set_nullable(true);
// Add index params
auto *index_params_pb = field_pb.mutable_index_params();
auto *hnsw_pb = index_params_pb->mutable_hnsw();
auto *base_params = hnsw_pb->mutable_base();
base_params->set_metric_type(proto::MT_L2);
base_params->set_quantize_type(proto::QT_FP16);
hnsw_pb->set_m(16);
hnsw_pb->set_ef_construction(100);
auto field_schema = ProtoConverter::FromPb(field_pb);
ASSERT_NE(field_schema, nullptr);
EXPECT_EQ(field_schema->name(), "test_field");
EXPECT_EQ(field_schema->data_type(), DataType::VECTOR_FP32);
EXPECT_TRUE(field_schema->nullable());
EXPECT_EQ(field_schema->dimension(), 128u);
ASSERT_NE(field_schema->index_params(), nullptr);
EXPECT_EQ(field_schema->index_params()->type(), IndexType::HNSW);
// Test conversion from C++ to protobuf
FieldSchema original_field("another_field", DataType::ARRAY_INT32, 64, false,
nullptr);
auto pb_result = ProtoConverter::ToPb(original_field);
EXPECT_EQ(pb_result.name(), "another_field");
EXPECT_EQ(pb_result.data_type(), proto::DT_ARRAY_INT32);
EXPECT_FALSE(pb_result.nullable());
EXPECT_EQ(pb_result.dimension(), 64u);
}
TEST(ConverterTest, CollectionSchemaConversion) {
// Test conversion from protobuf to C++ CollectionSchema
proto::CollectionSchema schema_pb;
schema_pb.set_name("test_collection");
schema_pb.set_max_doc_count_per_segment(1000000);
auto *field1_pb = schema_pb.add_fields();
field1_pb->set_name("field1");
field1_pb->set_data_type(proto::DT_STRING);
auto *field2_pb = schema_pb.add_fields();
field2_pb->set_name("field2");
field2_pb->set_data_type(proto::DT_VECTOR_FP32);
field2_pb->set_dimension(128);
auto collection_schema = ProtoConverter::FromPb(schema_pb);
ASSERT_NE(collection_schema, nullptr);
EXPECT_EQ(collection_schema->name(), "test_collection");
EXPECT_EQ(collection_schema->fields().size(), 2);
EXPECT_EQ(collection_schema->max_doc_count_per_segment(), 1000000u);
// Test conversion from C++ to protobuf
CollectionSchema original_schema;
original_schema.set_name("original_collection");
auto pb_result = ProtoConverter::ToPb(original_schema);
EXPECT_EQ(pb_result.name(), "original_collection");
}
TEST(ConverterTest, BlockMetaConversion) {
// Test conversion from protobuf to C++ BlockMeta
proto::BlockMeta meta_pb;
meta_pb.set_block_id(1);
meta_pb.set_block_type(proto::BT_SCALAR);
meta_pb.set_min_doc_id(100);
meta_pb.set_max_doc_id(200);
meta_pb.set_doc_count(50);
meta_pb.add_columns("col1");
meta_pb.add_columns("col2");
auto block_meta = ProtoConverter::FromPb(meta_pb);
ASSERT_NE(block_meta, nullptr);
EXPECT_EQ(block_meta->id(), 1u);
EXPECT_EQ(block_meta->type(), BlockType::SCALAR);
EXPECT_EQ(block_meta->min_doc_id(), 100u);
EXPECT_EQ(block_meta->max_doc_id(), 200u);
EXPECT_EQ(block_meta->doc_count(), 50u);
EXPECT_EQ(block_meta->columns().size(), 2);
EXPECT_EQ(block_meta->columns()[0], "col1");
EXPECT_EQ(block_meta->columns()[1], "col2");
// Test conversion from C++ to protobuf
BlockMeta original_meta(2, BlockType::VECTOR_INDEX, 300, 400);
original_meta.set_doc_count(75);
original_meta.add_column("col3");
original_meta.add_column("col4");
auto pb_result = ProtoConverter::ToPb(original_meta);
EXPECT_EQ(pb_result.block_id(), 2u);
EXPECT_EQ(pb_result.block_type(), proto::BT_VECTOR_INDEX);
EXPECT_EQ(pb_result.min_doc_id(), 300u);
EXPECT_EQ(pb_result.max_doc_id(), 400u);
EXPECT_EQ(pb_result.doc_count(), 75u);
EXPECT_EQ(pb_result.columns_size(), 2);
EXPECT_EQ(pb_result.columns(0), "col3");
EXPECT_EQ(pb_result.columns(1), "col4");
}
TEST(ConverterTest, SegmentMetaConversion) {
// Test conversion from protobuf to C++ SegmentMeta
proto::SegmentMeta segment_pb;
segment_pb.set_segment_id(10);
// Add persisted blocks
auto *block1_pb = segment_pb.add_persisted_blocks();
block1_pb->set_block_id(1);
block1_pb->set_block_type(proto::BT_SCALAR);
block1_pb->set_min_doc_id(0);
block1_pb->set_max_doc_id(100);
block1_pb->set_doc_count(50);
block1_pb->add_columns("col1");
block1_pb->add_columns("col2");
auto *block2_pb = segment_pb.add_persisted_blocks();
block2_pb->set_block_id(2);
block2_pb->set_block_type(proto::BT_VECTOR_INDEX);
block2_pb->set_min_doc_id(101);
block2_pb->set_max_doc_id(200);
block2_pb->set_doc_count(75);
block2_pb->add_columns("vec_col");
// Add writing forward block
auto *writing_block_pb = segment_pb.mutable_writing_forward_block();
writing_block_pb->set_block_id(3);
writing_block_pb->set_block_type(proto::BT_SCALAR);
writing_block_pb->set_min_doc_id(201);
writing_block_pb->set_max_doc_id(300);
writing_block_pb->set_doc_count(25);
writing_block_pb->add_columns("col3");
// Add indexed vector fields
segment_pb.add_indexed_vector_fields("vec_col1");
segment_pb.add_indexed_vector_fields("vec_col2");
auto segment_meta = ProtoConverter::FromPb(segment_pb);
ASSERT_NE(segment_meta, nullptr);
EXPECT_EQ(segment_meta->id(), 10u);
EXPECT_EQ(segment_meta->persisted_blocks().size(), 2);
EXPECT_TRUE(segment_meta->has_writing_forward_block());
// Check first persisted block
const auto &block1 = segment_meta->persisted_blocks()[0];
EXPECT_EQ(block1.id(), 1u);
EXPECT_EQ(block1.type(), BlockType::SCALAR);
EXPECT_EQ(block1.min_doc_id(), 0u);
EXPECT_EQ(block1.max_doc_id(), 100u);
EXPECT_EQ(block1.doc_count(), 50u);
EXPECT_EQ(block1.columns().size(), 2);
EXPECT_EQ(block1.columns()[0], "col1");
EXPECT_EQ(block1.columns()[1], "col2");
// Check second persisted block
const auto &block2 = segment_meta->persisted_blocks()[1];
EXPECT_EQ(block2.id(), 2u);
EXPECT_EQ(block2.type(), BlockType::VECTOR_INDEX);
EXPECT_EQ(block2.min_doc_id(), 101u);
EXPECT_EQ(block2.max_doc_id(), 200u);
EXPECT_EQ(block2.doc_count(), 75u);
EXPECT_EQ(block2.columns().size(), 1);
EXPECT_EQ(block2.columns()[0], "vec_col");
// Check writing forward block
const auto &writing_block = segment_meta->writing_forward_block();
EXPECT_EQ(writing_block.value().id(), 3u);
EXPECT_EQ(writing_block.value().type(), BlockType::SCALAR);
EXPECT_EQ(writing_block.value().min_doc_id(), 201u);
EXPECT_EQ(writing_block.value().max_doc_id(), 300u);
EXPECT_EQ(writing_block.value().doc_count(), 25u);
EXPECT_EQ(writing_block.value().columns().size(), 1);
EXPECT_EQ(writing_block.value().columns()[0], "col3");
// Check indexed vector fields
EXPECT_TRUE(segment_meta->vector_indexed("vec_col1"));
EXPECT_TRUE(segment_meta->vector_indexed("vec_col2"));
EXPECT_FALSE(segment_meta->vector_indexed("non_existent_field"));
// Test conversion from C++ to protobuf
SegmentMeta original_meta(20);
// Add persisted blocks
BlockMeta block1_meta(1, BlockType::SCALAR_INDEX, 0, 50);
block1_meta.set_doc_count(25);
block1_meta.add_column("col3");
block1_meta.add_column("col4");
original_meta.add_persisted_block(block1_meta);
BlockMeta block2_meta(2, BlockType::VECTOR_INDEX_QUANTIZE, 51, 100);
block2_meta.set_doc_count(30);
block2_meta.add_column("vec_col2");
original_meta.add_persisted_block(block2_meta);
// Set writing forward block
BlockMeta writing_block_meta(3, BlockType::SCALAR, 101, 150);
writing_block_meta.set_doc_count(40);
writing_block_meta.add_column("col5");
original_meta.set_writing_forward_block(writing_block_meta);
// Add indexed vector fields
original_meta.add_indexed_vector_field("vec_field1");
original_meta.add_indexed_vector_field("vec_field2");
auto pb_result = ProtoConverter::ToPb(original_meta);
EXPECT_EQ(pb_result.segment_id(), 20u);
EXPECT_EQ(pb_result.persisted_blocks_size(), 2);
// Check first persisted block
const auto &pb_block1 = pb_result.persisted_blocks(0);
EXPECT_EQ(pb_block1.block_id(), 1u);
EXPECT_EQ(pb_block1.block_type(), proto::BT_SCALAR_INDEX);
EXPECT_EQ(pb_block1.min_doc_id(), 0u);
EXPECT_EQ(pb_block1.max_doc_id(), 50u);
EXPECT_EQ(pb_block1.doc_count(), 25u);
EXPECT_EQ(pb_block1.columns_size(), 2);
EXPECT_EQ(pb_block1.columns(0), "col3");
EXPECT_EQ(pb_block1.columns(1), "col4");
// Check second persisted block
const auto &pb_block2 = pb_result.persisted_blocks(1);
EXPECT_EQ(pb_block2.block_id(), 2u);
EXPECT_EQ(pb_block2.block_type(), proto::BT_VECTOR_INDEX_QUANTIZE);
EXPECT_EQ(pb_block2.min_doc_id(), 51u);
EXPECT_EQ(pb_block2.max_doc_id(), 100u);
EXPECT_EQ(pb_block2.doc_count(), 30u);
EXPECT_EQ(pb_block2.columns_size(), 1);
EXPECT_EQ(pb_block2.columns(0), "vec_col2");
// Check writing forward block
const auto &pb_writing_block = pb_result.writing_forward_block();
EXPECT_EQ(pb_writing_block.block_id(), 3u);
EXPECT_EQ(pb_writing_block.block_type(), proto::BT_SCALAR);
EXPECT_EQ(pb_writing_block.min_doc_id(), 101u);
EXPECT_EQ(pb_writing_block.max_doc_id(), 150u);
EXPECT_EQ(pb_writing_block.doc_count(), 40u);
EXPECT_EQ(pb_writing_block.columns_size(), 1);
EXPECT_EQ(pb_writing_block.columns(0), "col5");
// Check indexed vector fields
EXPECT_EQ(pb_result.indexed_vector_fields_size(), 2);
EXPECT_EQ(pb_result.indexed_vector_fields(0), "vec_field1");
EXPECT_EQ(pb_result.indexed_vector_fields(1), "vec_field2");
}
TEST(ConverterTest, SegmentMetaWithEmptyFields) {
// Test conversion with minimal data
proto::SegmentMeta segment_pb;
segment_pb.set_segment_id(1);
auto segment_meta = ProtoConverter::FromPb(segment_pb);
ASSERT_NE(segment_meta, nullptr);
EXPECT_EQ(segment_meta->id(), 1u);
EXPECT_EQ(segment_meta->persisted_blocks().size(), 0);
EXPECT_FALSE(segment_meta->has_writing_forward_block());
EXPECT_EQ(segment_meta->indexed_vector_fields().size(), 0);
// Test conversion from C++ to protobuf with minimal data
SegmentMeta original_meta(5);
auto pb_result = ProtoConverter::ToPb(original_meta);
EXPECT_EQ(pb_result.segment_id(), 5u);
EXPECT_EQ(pb_result.persisted_blocks_size(), 0);
EXPECT_FALSE(pb_result.has_writing_forward_block());
EXPECT_EQ(pb_result.indexed_vector_fields_size(), 0);
}
// ==================== enable_rotate roundtrip tests ====================
TEST(ConverterTest, HnswIndexParamsWithEnableRotate) {
// C++ -> PB -> C++ roundtrip with enable_rotate = true
HnswIndexParams original(MetricType::COSINE, 16, 200, QuantizeType::INT8,
false, QuantizerParam(true));
EXPECT_TRUE(original.quantizer_param().enable_rotate());
auto pb = ProtoConverter::ToPb(&original);
EXPECT_TRUE(pb.base().quantizer_param().enable_rotate());
auto restored = ProtoConverter::FromPb(pb);
ASSERT_NE(restored, nullptr);
EXPECT_TRUE(restored->quantizer_param().enable_rotate());
EXPECT_TRUE(restored->enable_rotate()); // convenience getter
EXPECT_EQ(restored->metric_type(), MetricType::COSINE);
EXPECT_EQ(restored->m(), 16);
EXPECT_EQ(restored->ef_construction(), 200);
EXPECT_EQ(restored->quantize_type(), QuantizeType::INT8);
// C++ -> PB -> C++ roundtrip with enable_rotate = false
HnswIndexParams original_no_rot(MetricType::L2, 32, 100, QuantizeType::FP16);
auto pb2 = ProtoConverter::ToPb(&original_no_rot);
EXPECT_FALSE(pb2.base().quantizer_param().enable_rotate());
auto restored2 = ProtoConverter::FromPb(pb2);
ASSERT_NE(restored2, nullptr);
EXPECT_FALSE(restored2->quantizer_param().enable_rotate());
}
TEST(ConverterTest, FlatIndexParamsWithEnableRotate) {
FlatIndexParams original(MetricType::IP, QuantizeType::INT8,
QuantizerParam(true));
EXPECT_TRUE(original.quantizer_param().enable_rotate());
auto pb = ProtoConverter::ToPb(&original);
EXPECT_TRUE(pb.base().quantizer_param().enable_rotate());
auto restored = ProtoConverter::FromPb(pb);
ASSERT_NE(restored, nullptr);
EXPECT_TRUE(restored->quantizer_param().enable_rotate());
EXPECT_EQ(restored->metric_type(), MetricType::IP);
EXPECT_EQ(restored->quantize_type(), QuantizeType::INT8);
// enable_rotate = false
FlatIndexParams original_no_rot(MetricType::L2, QuantizeType::FP16);
auto pb2 = ProtoConverter::ToPb(&original_no_rot);
EXPECT_FALSE(pb2.base().quantizer_param().enable_rotate());
auto restored2 = ProtoConverter::FromPb(pb2);
EXPECT_FALSE(restored2->quantizer_param().enable_rotate());
}
TEST(ConverterTest, IVFIndexParamsWithEnableRotate) {
IVFIndexParams original(MetricType::COSINE, 256, 20, true, QuantizeType::INT8,
QuantizerParam(true));
EXPECT_TRUE(original.quantizer_param().enable_rotate());
auto pb = ProtoConverter::ToPb(&original);
EXPECT_TRUE(pb.base().quantizer_param().enable_rotate());
auto restored = ProtoConverter::FromPb(pb);
ASSERT_NE(restored, nullptr);
EXPECT_TRUE(restored->quantizer_param().enable_rotate());
EXPECT_EQ(restored->metric_type(), MetricType::COSINE);
EXPECT_EQ(restored->n_list(), 256);
EXPECT_EQ(restored->n_iters(), 20);
EXPECT_TRUE(restored->use_soar());
EXPECT_EQ(restored->quantize_type(), QuantizeType::INT8);
// enable_rotate = false
IVFIndexParams original_no_rot(MetricType::L2, 128, 10, false,
QuantizeType::FP16);
auto pb2 = ProtoConverter::ToPb(&original_no_rot);
EXPECT_FALSE(pb2.base().quantizer_param().enable_rotate());
auto restored2 = ProtoConverter::FromPb(pb2);
EXPECT_FALSE(restored2->quantizer_param().enable_rotate());
}
tests/db/index/common/db_type_helper_test.cc
+262 -151
View File
@@ -12,37 +12,67 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <set>
#include <gtest/gtest.h>
#include "db/index/common/type_helper.h"
using namespace zvec;
TEST(IndexTypeCodeBookTest, ProtoToCppConversion) {
// Test conversion from protobuf to C++ IndexType
EXPECT_EQ(IndexTypeCodeBook::Get(proto::IT_HNSW), IndexType::HNSW);
EXPECT_EQ(IndexTypeCodeBook::Get(proto::IT_FLAT), IndexType::FLAT);
EXPECT_EQ(IndexTypeCodeBook::Get(proto::IT_IVF), IndexType::IVF);
EXPECT_EQ(IndexTypeCodeBook::Get(proto::IT_INVERT), IndexType::INVERT);
EXPECT_EQ(IndexTypeCodeBook::Get(proto::IT_UNDEFINED), IndexType::UNDEFINED);
EXPECT_EQ(IndexTypeCodeBook::Get(static_cast<proto::IndexType>(999)),
TEST(IndexTypeCodeBookTest, WireToCppConversion) {
// Test conversion from the wire format to C++ IndexType
EXPECT_EQ(IndexTypeCodeBook::Get(wire::IndexType::IT_HNSW), IndexType::HNSW);
EXPECT_EQ(IndexTypeCodeBook::Get(wire::IndexType::IT_HNSW_RABITQ),
IndexType::HNSW_RABITQ);
EXPECT_EQ(IndexTypeCodeBook::Get(wire::IndexType::IT_IVF_RABITQ),
IndexType::IVF_RABITQ);
EXPECT_EQ(IndexTypeCodeBook::Get(wire::IndexType::IT_FLAT), IndexType::FLAT);
EXPECT_EQ(IndexTypeCodeBook::Get(wire::IndexType::IT_IVF), IndexType::IVF);
EXPECT_EQ(IndexTypeCodeBook::Get(wire::IndexType::IT_VAMANA),
IndexType::VAMANA);
EXPECT_EQ(IndexTypeCodeBook::Get(wire::IndexType::IT_INVERT),
IndexType::INVERT);
EXPECT_EQ(IndexTypeCodeBook::Get(wire::IndexType::IT_DISKANN),
IndexType::DISKANN);
EXPECT_EQ(IndexTypeCodeBook::Get(wire::IndexType::IT_FTS), IndexType::FTS);
EXPECT_EQ(IndexTypeCodeBook::Get(wire::IndexType::IT_UNDEFINED),
IndexType::UNDEFINED);
EXPECT_EQ(IndexTypeCodeBook::Get(static_cast<wire::IndexType>(999)),
IndexType::UNDEFINED);
}
TEST(IndexTypeCodeBookTest, CppToProtoConversion) {
// Test conversion from C++ IndexType to protobuf IndexType
EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::HNSW), proto::IT_HNSW);
EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::FLAT), proto::IT_FLAT);
EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::IVF), proto::IT_IVF);
EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::INVERT), proto::IT_INVERT);
EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::UNDEFINED), proto::IT_UNDEFINED);
TEST(IndexTypeCodeBookTest, CppToWireConversion) {
// Test conversion from C++ IndexType to the wire format
EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::HNSW), wire::IndexType::IT_HNSW);
EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::HNSW_RABITQ),
wire::IndexType::IT_HNSW_RABITQ);
EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::IVF_RABITQ),
wire::IndexType::IT_IVF_RABITQ);
EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::FLAT), wire::IndexType::IT_FLAT);
EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::IVF), wire::IndexType::IT_IVF);
EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::VAMANA),
wire::IndexType::IT_VAMANA);
EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::INVERT),
wire::IndexType::IT_INVERT);
EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::DISKANN),
wire::IndexType::IT_DISKANN);
EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::FTS), wire::IndexType::IT_FTS);
EXPECT_EQ(IndexTypeCodeBook::Get(IndexType::UNDEFINED),
wire::IndexType::IT_UNDEFINED);
EXPECT_EQ(IndexTypeCodeBook::Get(static_cast<IndexType>(999)),
proto::IT_UNDEFINED);
wire::IndexType::IT_UNDEFINED);
}
TEST(IndexTypeCodeBookTest, CppToStringConversion) {
// Test conversion from C++ IndexType to string
EXPECT_EQ(IndexTypeCodeBook::AsString(IndexType::HNSW), "HNSW");
EXPECT_EQ(IndexTypeCodeBook::AsString(IndexType::HNSW_RABITQ), "HNSW_RABITQ");
EXPECT_EQ(IndexTypeCodeBook::AsString(IndexType::IVF_RABITQ), "IVF_RABITQ");
EXPECT_EQ(IndexTypeCodeBook::AsString(IndexType::FLAT), "FLAT");
EXPECT_EQ(IndexTypeCodeBook::AsString(IndexType::IVF), "IVF");
EXPECT_EQ(IndexTypeCodeBook::AsString(IndexType::VAMANA), "VAMANA");
EXPECT_EQ(IndexTypeCodeBook::AsString(IndexType::DISKANN), "DISKANN");
EXPECT_EQ(IndexTypeCodeBook::AsString(IndexType::INVERT), "INVERT");
EXPECT_EQ(IndexTypeCodeBook::AsString(IndexType::FTS), "FTS");
EXPECT_EQ(IndexTypeCodeBook::AsString(IndexType::UNDEFINED), "UNDEFINED");
EXPECT_EQ(IndexTypeCodeBook::AsString(static_cast<IndexType>(999)),
"UNDEFINED");
@@ -50,138 +80,149 @@ TEST(IndexTypeCodeBookTest, CppToStringConversion) {
TEST(DataTypeCodeBookTest, IsArrayType) {
// Test array type detection
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_BINARY));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_STRING));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_BOOL));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_INT32));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_INT64));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_UINT32));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_UINT64));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_FLOAT));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_DOUBLE));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_BINARY32));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_BINARY64));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_FP16));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_FP32));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_FP64));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_INT4));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_INT8));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_VECTOR_INT16));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(proto::DT_SPARSE_VECTOR_FP32));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_BINARY));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_STRING));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_BOOL));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_INT32));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_INT64));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_UINT32));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_UINT64));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_FLOAT));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_DOUBLE));
EXPECT_FALSE(
DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_BINARY32));
EXPECT_FALSE(
DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_BINARY64));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_FP16));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_FP32));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_FP64));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_INT4));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_INT8));
EXPECT_FALSE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_VECTOR_INT16));
EXPECT_FALSE(
DataTypeCodeBook::IsArrayType(wire::DataType::DT_SPARSE_VECTOR_FP32));
EXPECT_FALSE(
DataTypeCodeBook::IsArrayType(wire::DataType::DT_SPARSE_VECTOR_FP16));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_BINARY));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_STRING));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_BOOL));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_INT32));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_INT64));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_UINT32));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_UINT64));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_FLOAT));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(proto::DT_ARRAY_DOUBLE));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_BINARY));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_STRING));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_BOOL));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_INT32));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_INT64));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_UINT32));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_UINT64));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_FLOAT));
EXPECT_TRUE(DataTypeCodeBook::IsArrayType(wire::DataType::DT_ARRAY_DOUBLE));
}
TEST(DataTypeCodeBookTest, ProtoToCppConversion) {
// Test conversion from protobuf to C++ DataType
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_BINARY), DataType::BINARY);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_STRING), DataType::STRING);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_BOOL), DataType::BOOL);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_INT32), DataType::INT32);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_INT64), DataType::INT64);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_UINT32), DataType::UINT32);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_UINT64), DataType::UINT64);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_FLOAT), DataType::FLOAT);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_DOUBLE), DataType::DOUBLE);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_BINARY32),
TEST(DataTypeCodeBookTest, WireToCppConversion) {
// Test conversion from the wire format to C++ DataType
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_BINARY), DataType::BINARY);
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_STRING), DataType::STRING);
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_BOOL), DataType::BOOL);
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_INT32), DataType::INT32);
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_INT64), DataType::INT64);
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_UINT32), DataType::UINT32);
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_UINT64), DataType::UINT64);
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_FLOAT), DataType::FLOAT);
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_DOUBLE), DataType::DOUBLE);
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_BINARY32),
DataType::VECTOR_BINARY32);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_BINARY64),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_BINARY64),
DataType::VECTOR_BINARY64);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_FP16),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_FP16),
DataType::VECTOR_FP16);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_FP32),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_FP32),
DataType::VECTOR_FP32);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_FP64),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_FP64),
DataType::VECTOR_FP64);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_INT4),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_INT4),
DataType::VECTOR_INT4);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_INT8),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_INT8),
DataType::VECTOR_INT8);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_VECTOR_INT16),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_VECTOR_INT16),
DataType::VECTOR_INT16);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_SPARSE_VECTOR_FP32),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_SPARSE_VECTOR_FP16),
DataType::SPARSE_VECTOR_FP16);
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_SPARSE_VECTOR_FP32),
DataType::SPARSE_VECTOR_FP32);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_BINARY),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_BINARY),
DataType::ARRAY_BINARY);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_STRING),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_STRING),
DataType::ARRAY_STRING);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_BOOL), DataType::ARRAY_BOOL);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_INT32),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_BOOL),
DataType::ARRAY_BOOL);
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_INT32),
DataType::ARRAY_INT32);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_INT64),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_INT64),
DataType::ARRAY_INT64);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_UINT32),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_UINT32),
DataType::ARRAY_UINT32);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_UINT64),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_UINT64),
DataType::ARRAY_UINT64);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_FLOAT),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_FLOAT),
DataType::ARRAY_FLOAT);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_ARRAY_DOUBLE),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_ARRAY_DOUBLE),
DataType::ARRAY_DOUBLE);
EXPECT_EQ(DataTypeCodeBook::Get(proto::DT_UNDEFINED), DataType::UNDEFINED);
EXPECT_EQ(DataTypeCodeBook::Get(static_cast<proto::DataType>(999)),
EXPECT_EQ(DataTypeCodeBook::Get(wire::DataType::DT_UNDEFINED),
DataType::UNDEFINED);
EXPECT_EQ(DataTypeCodeBook::Get(static_cast<wire::DataType>(999)),
DataType::UNDEFINED);
}
TEST(DataTypeCodeBookTest, CppToProtoConversion) {
// Test conversion from C++ DataType to protobuf DataType
EXPECT_EQ(DataTypeCodeBook::Get(DataType::BINARY), proto::DT_BINARY);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::STRING), proto::DT_STRING);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::BOOL), proto::DT_BOOL);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::INT32), proto::DT_INT32);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::INT64), proto::DT_INT64);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::UINT32), proto::DT_UINT32);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::UINT64), proto::DT_UINT64);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::FLOAT), proto::DT_FLOAT);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::DOUBLE), proto::DT_DOUBLE);
TEST(DataTypeCodeBookTest, CppToWireConversion) {
// Test conversion from C++ DataType to the wire format
EXPECT_EQ(DataTypeCodeBook::Get(DataType::BINARY), wire::DataType::DT_BINARY);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::STRING), wire::DataType::DT_STRING);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::BOOL), wire::DataType::DT_BOOL);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::INT32), wire::DataType::DT_INT32);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::INT64), wire::DataType::DT_INT64);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::UINT32), wire::DataType::DT_UINT32);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::UINT64), wire::DataType::DT_UINT64);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::FLOAT), wire::DataType::DT_FLOAT);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::DOUBLE), wire::DataType::DT_DOUBLE);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_BINARY32),
proto::DT_VECTOR_BINARY32);
wire::DataType::DT_VECTOR_BINARY32);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_BINARY64),
proto::DT_VECTOR_BINARY64);
wire::DataType::DT_VECTOR_BINARY64);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_FP16),
proto::DT_VECTOR_FP16);
wire::DataType::DT_VECTOR_FP16);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_FP32),
proto::DT_VECTOR_FP32);
wire::DataType::DT_VECTOR_FP32);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_FP64),
proto::DT_VECTOR_FP64);
wire::DataType::DT_VECTOR_FP64);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_INT4),
proto::DT_VECTOR_INT4);
wire::DataType::DT_VECTOR_INT4);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_INT8),
proto::DT_VECTOR_INT8);
wire::DataType::DT_VECTOR_INT8);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::VECTOR_INT16),
proto::DT_VECTOR_INT16);
wire::DataType::DT_VECTOR_INT16);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::SPARSE_VECTOR_FP16),
proto::DT_SPARSE_VECTOR_FP16);
wire::DataType::DT_SPARSE_VECTOR_FP16);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::SPARSE_VECTOR_FP32),
proto::DT_SPARSE_VECTOR_FP32);
wire::DataType::DT_SPARSE_VECTOR_FP32);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_BINARY),
proto::DT_ARRAY_BINARY);
wire::DataType::DT_ARRAY_BINARY);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_STRING),
proto::DT_ARRAY_STRING);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_BOOL), proto::DT_ARRAY_BOOL);
wire::DataType::DT_ARRAY_STRING);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_BOOL),
wire::DataType::DT_ARRAY_BOOL);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_INT32),
proto::DT_ARRAY_INT32);
wire::DataType::DT_ARRAY_INT32);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_INT64),
proto::DT_ARRAY_INT64);
wire::DataType::DT_ARRAY_INT64);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_UINT32),
proto::DT_ARRAY_UINT32);
wire::DataType::DT_ARRAY_UINT32);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_UINT64),
proto::DT_ARRAY_UINT64);
wire::DataType::DT_ARRAY_UINT64);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_FLOAT),
proto::DT_ARRAY_FLOAT);
wire::DataType::DT_ARRAY_FLOAT);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::ARRAY_DOUBLE),
proto::DT_ARRAY_DOUBLE);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::UNDEFINED), proto::DT_UNDEFINED);
wire::DataType::DT_ARRAY_DOUBLE);
EXPECT_EQ(DataTypeCodeBook::Get(DataType::UNDEFINED),
wire::DataType::DT_UNDEFINED);
EXPECT_EQ(DataTypeCodeBook::Get(static_cast<DataType>(999)),
proto::DT_UNDEFINED);
wire::DataType::DT_UNDEFINED);
}
TEST(DataTypeCodeBookTest, CppToStringConversion) {
@@ -205,6 +246,10 @@ TEST(DataTypeCodeBookTest, CppToStringConversion) {
EXPECT_EQ(DataTypeCodeBook::AsString(DataType::VECTOR_INT4), "VECTOR_INT4");
EXPECT_EQ(DataTypeCodeBook::AsString(DataType::VECTOR_INT8), "VECTOR_INT8");
EXPECT_EQ(DataTypeCodeBook::AsString(DataType::VECTOR_INT16), "VECTOR_INT16");
EXPECT_EQ(DataTypeCodeBook::AsString(DataType::SPARSE_VECTOR_FP16),
"SPARSE_VECTOR_FP16");
EXPECT_EQ(DataTypeCodeBook::AsString(DataType::SPARSE_VECTOR_FP32),
"SPARSE_VECTOR_FP32");
EXPECT_EQ(DataTypeCodeBook::AsString(DataType::ARRAY_BINARY), "ARRAY_BINARY");
EXPECT_EQ(DataTypeCodeBook::AsString(DataType::ARRAY_STRING), "ARRAY_STRING");
EXPECT_EQ(DataTypeCodeBook::AsString(DataType::ARRAY_BOOL), "ARRAY_BOOL");
@@ -218,74 +263,140 @@ TEST(DataTypeCodeBookTest, CppToStringConversion) {
EXPECT_EQ(DataTypeCodeBook::AsString(static_cast<DataType>(999)), "");
}
TEST(MetricTypeCodeBookTest, ProtoToCppConversion) {
// Test conversion from protobuf to C++ MetricType
EXPECT_EQ(MetricTypeCodeBook::Get(proto::MT_IP), MetricType::IP);
EXPECT_EQ(MetricTypeCodeBook::Get(proto::MT_L2), MetricType::L2);
EXPECT_EQ(MetricTypeCodeBook::Get(proto::MT_COSINE), MetricType::COSINE);
EXPECT_EQ(MetricTypeCodeBook::Get(proto::MT_UNDEFINED),
TEST(MetricTypeCodeBookTest, WireToCppConversion) {
// Test conversion from the wire format to C++ MetricType
EXPECT_EQ(MetricTypeCodeBook::Get(wire::MetricType::MT_IP), MetricType::IP);
EXPECT_EQ(MetricTypeCodeBook::Get(wire::MetricType::MT_L2), MetricType::L2);
EXPECT_EQ(MetricTypeCodeBook::Get(wire::MetricType::MT_COSINE),
MetricType::COSINE);
EXPECT_EQ(MetricTypeCodeBook::Get(wire::MetricType::MT_UNDEFINED),
MetricType::UNDEFINED);
EXPECT_EQ(MetricTypeCodeBook::Get(static_cast<proto::MetricType>(999)),
EXPECT_EQ(MetricTypeCodeBook::Get(static_cast<wire::MetricType>(999)),
MetricType::UNDEFINED);
}
TEST(MetricTypeCodeBookTest, CppToProtoConversion) {
// Test conversion from C++ MetricType to protobuf MetricType
EXPECT_EQ(MetricTypeCodeBook::Get(MetricType::IP), proto::MT_IP);
EXPECT_EQ(MetricTypeCodeBook::Get(MetricType::L2), proto::MT_L2);
EXPECT_EQ(MetricTypeCodeBook::Get(MetricType::COSINE), proto::MT_COSINE);
TEST(MetricTypeCodeBookTest, CppToWireConversion) {
// Test conversion from C++ MetricType to the wire format
EXPECT_EQ(MetricTypeCodeBook::Get(MetricType::IP), wire::MetricType::MT_IP);
EXPECT_EQ(MetricTypeCodeBook::Get(MetricType::L2), wire::MetricType::MT_L2);
EXPECT_EQ(MetricTypeCodeBook::Get(MetricType::COSINE),
wire::MetricType::MT_COSINE);
// MIPSL2 is a C++-only metric type without a wire-format counterpart.
EXPECT_EQ(MetricTypeCodeBook::Get(MetricType::MIPSL2),
wire::MetricType::MT_UNDEFINED);
EXPECT_EQ(MetricTypeCodeBook::Get(MetricType::UNDEFINED),
proto::MT_UNDEFINED);
wire::MetricType::MT_UNDEFINED);
EXPECT_EQ(MetricTypeCodeBook::Get(static_cast<MetricType>(999)),
proto::MT_UNDEFINED);
wire::MetricType::MT_UNDEFINED);
}
TEST(QuantizeTypeCodeBookTest, ProtoToCppConversion) {
// Test conversion from protobuf to C++ QuantizeType
EXPECT_EQ(QuantizeTypeCodeBook::Get(proto::QT_FP16), QuantizeType::FP16);
EXPECT_EQ(QuantizeTypeCodeBook::Get(proto::QT_INT4), QuantizeType::INT4);
EXPECT_EQ(QuantizeTypeCodeBook::Get(proto::QT_INT8), QuantizeType::INT8);
EXPECT_EQ(QuantizeTypeCodeBook::Get(proto::QT_UNDEFINED),
TEST(MetricTypeCodeBookTest, CppToStringConversion) {
// Test conversion from C++ MetricType to string
EXPECT_EQ(MetricTypeCodeBook::AsString(MetricType::IP), "IP");
EXPECT_EQ(MetricTypeCodeBook::AsString(MetricType::L2), "L2");
EXPECT_EQ(MetricTypeCodeBook::AsString(MetricType::COSINE), "COSINE");
EXPECT_EQ(MetricTypeCodeBook::AsString(MetricType::MIPSL2), "UNDEFINED");
EXPECT_EQ(MetricTypeCodeBook::AsString(MetricType::UNDEFINED), "UNDEFINED");
EXPECT_EQ(MetricTypeCodeBook::AsString(static_cast<MetricType>(999)),
"UNDEFINED");
}
TEST(QuantizeTypeCodeBookTest, WireToCppConversion) {
// Test conversion from the wire format to C++ QuantizeType
EXPECT_EQ(QuantizeTypeCodeBook::Get(wire::QuantizeType::QT_FP16),
QuantizeType::FP16);
EXPECT_EQ(QuantizeTypeCodeBook::Get(wire::QuantizeType::QT_INT4),
QuantizeType::INT4);
EXPECT_EQ(QuantizeTypeCodeBook::Get(wire::QuantizeType::QT_INT8),
QuantizeType::INT8);
EXPECT_EQ(QuantizeTypeCodeBook::Get(wire::QuantizeType::QT_RABITQ),
QuantizeType::RABITQ);
EXPECT_EQ(QuantizeTypeCodeBook::Get(wire::QuantizeType::QT_UNDEFINED),
QuantizeType::UNDEFINED);
EXPECT_EQ(QuantizeTypeCodeBook::Get(static_cast<proto::QuantizeType>(999)),
EXPECT_EQ(QuantizeTypeCodeBook::Get(static_cast<wire::QuantizeType>(999)),
QuantizeType::UNDEFINED);
}
TEST(QuantizeTypeCodeBookTest, CppToProtoConversion) {
// Test conversion from C++ QuantizeType to protobuf QuantizeType
EXPECT_EQ(QuantizeTypeCodeBook::Get(QuantizeType::FP16), proto::QT_FP16);
EXPECT_EQ(QuantizeTypeCodeBook::Get(QuantizeType::INT4), proto::QT_INT4);
EXPECT_EQ(QuantizeTypeCodeBook::Get(QuantizeType::INT8), proto::QT_INT8);
TEST(QuantizeTypeCodeBookTest, CppToWireConversion) {
// Test conversion from C++ QuantizeType to the wire format
EXPECT_EQ(QuantizeTypeCodeBook::Get(QuantizeType::FP16),
wire::QuantizeType::QT_FP16);
EXPECT_EQ(QuantizeTypeCodeBook::Get(QuantizeType::INT4),
wire::QuantizeType::QT_INT4);
EXPECT_EQ(QuantizeTypeCodeBook::Get(QuantizeType::INT8),
wire::QuantizeType::QT_INT8);
EXPECT_EQ(QuantizeTypeCodeBook::Get(QuantizeType::RABITQ),
wire::QuantizeType::QT_RABITQ);
EXPECT_EQ(QuantizeTypeCodeBook::Get(QuantizeType::UNDEFINED),
proto::QT_UNDEFINED);
wire::QuantizeType::QT_UNDEFINED);
EXPECT_EQ(QuantizeTypeCodeBook::Get(static_cast<QuantizeType>(999)),
proto::QT_UNDEFINED);
wire::QuantizeType::QT_UNDEFINED);
}
TEST(BlockTypeCodeBookTest, ProtoToCppConversion) {
// Test conversion from protobuf to C++ BlockType
EXPECT_EQ(BlockTypeCodeBook::Get(proto::BT_SCALAR), BlockType::SCALAR);
EXPECT_EQ(BlockTypeCodeBook::Get(proto::BT_SCALAR_INDEX),
TEST(QuantizeTypeCodeBookTest, CppToStringConversion) {
// Test conversion from C++ QuantizeType to string
EXPECT_EQ(QuantizeTypeCodeBook::AsString(QuantizeType::FP16), "FP16");
EXPECT_EQ(QuantizeTypeCodeBook::AsString(QuantizeType::INT4), "INT4");
EXPECT_EQ(QuantizeTypeCodeBook::AsString(QuantizeType::INT8), "INT8");
EXPECT_EQ(QuantizeTypeCodeBook::AsString(QuantizeType::RABITQ), "RABITQ");
EXPECT_EQ(QuantizeTypeCodeBook::AsString(QuantizeType::UNDEFINED),
"UNDEFINED");
EXPECT_EQ(QuantizeTypeCodeBook::AsString(static_cast<QuantizeType>(999)),
"UNDEFINED");
// The set overload joins the sorted quantize types with commas.
EXPECT_EQ(QuantizeTypeCodeBook::AsString(std::set<QuantizeType>{
QuantizeType::RABITQ, QuantizeType::FP16, QuantizeType::INT8}),
"FP16,INT8,RABITQ");
}
TEST(BlockTypeCodeBookTest, WireToCppConversion) {
// Test conversion from the wire format to C++ BlockType
EXPECT_EQ(BlockTypeCodeBook::Get(wire::BlockType::BT_SCALAR),
BlockType::SCALAR);
EXPECT_EQ(BlockTypeCodeBook::Get(wire::BlockType::BT_SCALAR_INDEX),
BlockType::SCALAR_INDEX);
EXPECT_EQ(BlockTypeCodeBook::Get(proto::BT_VECTOR_INDEX),
EXPECT_EQ(BlockTypeCodeBook::Get(wire::BlockType::BT_VECTOR_INDEX),
BlockType::VECTOR_INDEX);
EXPECT_EQ(BlockTypeCodeBook::Get(proto::BT_VECTOR_INDEX_QUANTIZE),
EXPECT_EQ(BlockTypeCodeBook::Get(wire::BlockType::BT_VECTOR_INDEX_QUANTIZE),
BlockType::VECTOR_INDEX_QUANTIZE);
EXPECT_EQ(BlockTypeCodeBook::Get(proto::BT_UNDEFINED), BlockType::UNDEFINED);
EXPECT_EQ(BlockTypeCodeBook::Get(static_cast<proto::BlockType>(999)),
EXPECT_EQ(BlockTypeCodeBook::Get(wire::BlockType::BT_FTS_INDEX),
BlockType::FTS_INDEX);
EXPECT_EQ(BlockTypeCodeBook::Get(wire::BlockType::BT_UNDEFINED),
BlockType::UNDEFINED);
EXPECT_EQ(BlockTypeCodeBook::Get(static_cast<wire::BlockType>(999)),
BlockType::UNDEFINED);
}
TEST(BlockTypeCodeBookTest, CppToProtoConversion) {
// Test conversion from C++ BlockType to protobuf BlockType
EXPECT_EQ(BlockTypeCodeBook::Get(BlockType::SCALAR), proto::BT_SCALAR);
TEST(BlockTypeCodeBookTest, CppToWireConversion) {
// Test conversion from C++ BlockType to the wire format
EXPECT_EQ(BlockTypeCodeBook::Get(BlockType::SCALAR),
wire::BlockType::BT_SCALAR);
EXPECT_EQ(BlockTypeCodeBook::Get(BlockType::SCALAR_INDEX),
proto::BT_SCALAR_INDEX);
wire::BlockType::BT_SCALAR_INDEX);
EXPECT_EQ(BlockTypeCodeBook::Get(BlockType::VECTOR_INDEX),
proto::BT_VECTOR_INDEX);
wire::BlockType::BT_VECTOR_INDEX);
EXPECT_EQ(BlockTypeCodeBook::Get(BlockType::VECTOR_INDEX_QUANTIZE),
proto::BT_VECTOR_INDEX_QUANTIZE);
EXPECT_EQ(BlockTypeCodeBook::Get(BlockType::UNDEFINED), proto::BT_UNDEFINED);
wire::BlockType::BT_VECTOR_INDEX_QUANTIZE);
EXPECT_EQ(BlockTypeCodeBook::Get(BlockType::FTS_INDEX),
wire::BlockType::BT_FTS_INDEX);
EXPECT_EQ(BlockTypeCodeBook::Get(BlockType::UNDEFINED),
wire::BlockType::BT_UNDEFINED);
EXPECT_EQ(BlockTypeCodeBook::Get(static_cast<BlockType>(999)),
proto::BT_UNDEFINED);
}
wire::BlockType::BT_UNDEFINED);
}
TEST(BlockTypeCodeBookTest, CppToStringConversion) {
// Test conversion from C++ BlockType to string
EXPECT_EQ(BlockTypeCodeBook::AsString(BlockType::SCALAR), "SCALAR");
EXPECT_EQ(BlockTypeCodeBook::AsString(BlockType::SCALAR_INDEX),
"SCALAR_INDEX");
EXPECT_EQ(BlockTypeCodeBook::AsString(BlockType::VECTOR_INDEX),
"VECTOR_INDEX");
EXPECT_EQ(BlockTypeCodeBook::AsString(BlockType::VECTOR_INDEX_QUANTIZE),
"VECTOR_INDEX_QUANTIZE");
EXPECT_EQ(BlockTypeCodeBook::AsString(BlockType::FTS_INDEX), "FTS_INDEX");
EXPECT_EQ(BlockTypeCodeBook::AsString(BlockType::UNDEFINED), "UNDEFINED");
EXPECT_EQ(BlockTypeCodeBook::AsString(static_cast<BlockType>(999)),
"UNDEFINED");
}
tests/db/index/common/manifest_codec_golden_test.cc
+1103
View File
File diff suppressed because it is too large Load Diff
tests/db/index/common/pb_wire_test.cc
+281
View File
@@ -0,0 +1,281 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "db/index/common/manifest/pb_wire.h"
#include <cstdint>
#include <limits>
#include <string>
#include <string_view>
#include <gtest/gtest.h>
namespace zvec {
namespace pbwire {
namespace {
TEST(PbWireWriterTest, VarintRoundTrip) {
const struct {
uint64_t value;
size_t encoded_size; // tag byte + varint bytes
} cases[] = {
{1, 2},
{127, 2},
{128, 3},
{300, 3},
{std::numeric_limits<uint32_t>::max(), 6},
{std::numeric_limits<uint64_t>::max(), 11},
};
for (const auto &test_case : cases) {
std::string buf;
Writer writer(&buf);
writer.PutVarint(1, test_case.value);
EXPECT_EQ(buf.size(), test_case.encoded_size);
Reader reader(buf);
ASSERT_TRUE(reader.Next());
EXPECT_EQ(reader.field(), 1u);
EXPECT_EQ(reader.wire_type(), kVarint);
EXPECT_EQ(reader.varint(), test_case.value);
EXPECT_FALSE(reader.Next());
EXPECT_TRUE(reader.ok());
}
}
TEST(PbWireWriterTest, VarintByteLayoutMatchesProtobuf) {
std::string buf;
Writer writer(&buf);
writer.PutVarint(1, 300);
// Tag: field 1 with wire type 0; varint 300 encodes as 0xAC 0x02.
const std::string expected("\x08\xAC\x02", 3);
EXPECT_EQ(buf, expected);
}
TEST(PbWireWriterTest, DefaultValuesAreSkipped) {
std::string buf;
Writer writer(&buf);
writer.PutVarint(1, 0);
writer.PutBool(2, false);
writer.PutFloat(3, 0.0f);
writer.PutString(4, "");
EXPECT_TRUE(buf.empty());
}
TEST(PbWireWriterTest, BoolRoundTrip) {
std::string buf;
Writer writer(&buf);
writer.PutBool(7, true);
Reader reader(buf);
ASSERT_TRUE(reader.Next());
EXPECT_EQ(reader.field(), 7u);
EXPECT_EQ(reader.wire_type(), kVarint);
EXPECT_TRUE(reader.bool_value());
EXPECT_FALSE(reader.Next());
EXPECT_TRUE(reader.ok());
}
TEST(PbWireWriterTest, FloatRoundTrip) {
std::string buf;
Writer writer(&buf);
writer.PutFloat(5, 3.5f);
writer.PutFloat(6, -1.25f);
Reader reader(buf);
ASSERT_TRUE(reader.Next());
EXPECT_EQ(reader.field(), 5u);
EXPECT_EQ(reader.wire_type(), kFixed32);
EXPECT_FLOAT_EQ(reader.float_value(), 3.5f);
ASSERT_TRUE(reader.Next());
EXPECT_EQ(reader.field(), 6u);
EXPECT_EQ(reader.wire_type(), kFixed32);
EXPECT_FLOAT_EQ(reader.float_value(), -1.25f);
EXPECT_FALSE(reader.Next());
EXPECT_TRUE(reader.ok());
}
TEST(PbWireWriterTest, StringRoundTrip) {
std::string buf;
Writer writer(&buf);
writer.PutString(2, "manifest");
Reader reader(buf);
ASSERT_TRUE(reader.Next());
EXPECT_EQ(reader.field(), 2u);
EXPECT_EQ(reader.wire_type(), kLenDelim);
EXPECT_EQ(reader.bytes(), std::string_view("manifest"));
EXPECT_EQ(reader.string_value(), "manifest");
EXPECT_FALSE(reader.Next());
EXPECT_TRUE(reader.ok());
}
TEST(PbWireWriterTest, AddStringWritesEmptyElements) {
std::string buf;
Writer writer(&buf);
writer.AddString(2, "");
// Tag for field 2 with wire type 2, followed by a zero length.
const std::string expected("\x12\x00", 2);
EXPECT_EQ(buf, expected);
Reader reader(buf);
ASSERT_TRUE(reader.Next());
EXPECT_EQ(reader.field(), 2u);
EXPECT_EQ(reader.wire_type(), kLenDelim);
EXPECT_TRUE(reader.bytes().empty());
EXPECT_TRUE(reader.ok());
}
TEST(PbWireWriterTest, PutMessageWritesEmptyPayload) {
std::string buf;
Writer writer(&buf);
writer.PutMessage(3, "");
// An empty but present sub-message is a zero-length field.
const std::string expected("\x1A\x00", 2);
EXPECT_EQ(buf, expected);
}
TEST(PbWireWriterTest, MultipleFieldsInOrder) {
std::string buf;
Writer writer(&buf);
writer.PutVarint(1, 42);
writer.PutString(2, "zvec");
writer.PutBool(3, true);
Reader reader(buf);
ASSERT_TRUE(reader.Next());
EXPECT_EQ(reader.field(), 1u);
EXPECT_EQ(reader.uint32_value(), 42u);
ASSERT_TRUE(reader.Next());
EXPECT_EQ(reader.field(), 2u);
EXPECT_EQ(reader.string_value(), "zvec");
ASSERT_TRUE(reader.Next());
EXPECT_EQ(reader.field(), 3u);
EXPECT_TRUE(reader.bool_value());
EXPECT_FALSE(reader.Next());
EXPECT_TRUE(reader.ok());
}
TEST(PbWireWriterTest, Int32AndUint32ValueInterpretation) {
std::string buf;
Writer writer(&buf);
// 0xFFFFFFFF is -1 when interpreted as an int32.
writer.PutVarint(1, 0xFFFFFFFFULL);
// Larger than 32 bits: uint32_value keeps the low 32 bits.
writer.PutVarint(2, 0x1FFFFFFFFULL);
Reader reader(buf);
ASSERT_TRUE(reader.Next());
EXPECT_EQ(reader.int32_value(), -1);
ASSERT_TRUE(reader.Next());
EXPECT_EQ(reader.uint32_value(), 0xFFFFFFFFu);
EXPECT_FALSE(reader.Next());
EXPECT_TRUE(reader.ok());
}
TEST(PbWireReaderTest, EmptyBufferEndsImmediately) {
const std::string_view empty;
Reader reader(empty);
EXPECT_FALSE(reader.Next());
EXPECT_TRUE(reader.ok());
}
TEST(PbWireReaderTest, CharSizeConstructorWorks) {
const std::string buf("\x08\x2A", 2);
Reader reader(buf.data(), buf.size());
ASSERT_TRUE(reader.Next());
EXPECT_EQ(reader.field(), 1u);
EXPECT_EQ(reader.uint32_value(), 42u);
EXPECT_FALSE(reader.Next());
EXPECT_TRUE(reader.ok());
}
TEST(PbWireReaderTest, Fixed64IsAccepted) {
// Tag for field 1 with wire type 1, followed by 8 little-endian bytes.
const std::string buf("\x09\x01\x02\x03\x04\x05\x06\x07\x08", 9);
Reader reader(buf);
ASSERT_TRUE(reader.Next());
EXPECT_EQ(reader.field(), 1u);
EXPECT_EQ(reader.wire_type(), kFixed64);
EXPECT_EQ(reader.varint(), 0x0807060504030201ULL);
EXPECT_FALSE(reader.Next());
EXPECT_TRUE(reader.ok());
}
TEST(PbWireReaderTest, TruncatedVarintFails) {
// Continuation bit set, then end of buffer.
const std::string buf("\x08\x80", 2);
Reader reader(buf);
EXPECT_FALSE(reader.Next());
EXPECT_FALSE(reader.ok());
}
TEST(PbWireReaderTest, OverlongVarintFails) {
// Ten bytes that all carry the continuation bit would need an eleventh.
const std::string buf(10, '\xFF');
Reader reader(buf);
EXPECT_FALSE(reader.Next());
EXPECT_FALSE(reader.ok());
}
TEST(PbWireReaderTest, TruncatedFixed32Fails) {
const std::string buf("\x0D\x01\x02", 3);
Reader reader(buf);
EXPECT_FALSE(reader.Next());
EXPECT_FALSE(reader.ok());
}
TEST(PbWireReaderTest, TruncatedFixed64Fails) {
const std::string buf("\x09\x01\x02\x03", 4);
Reader reader(buf);
EXPECT_FALSE(reader.Next());
EXPECT_FALSE(reader.ok());
}
TEST(PbWireReaderTest, TruncatedLengthDelimitedFails) {
// Declares 10 payload bytes but provides only 2.
const std::string buf(
"\x12\x0A"
"ab",
4);
Reader reader(buf);
EXPECT_FALSE(reader.Next());
EXPECT_FALSE(reader.ok());
}
TEST(PbWireReaderTest, TruncatedLengthVarintFails) {
const std::string buf("\x12\x80", 2);
Reader reader(buf);
EXPECT_FALSE(reader.Next());
EXPECT_FALSE(reader.ok());
}
TEST(PbWireReaderTest, FieldNumberZeroFails) {
// A tag whose field number is 0 is illegal in protobuf.
const std::string zero_tag(1, '\x00');
Reader zero_reader(zero_tag);
EXPECT_FALSE(zero_reader.Next());
EXPECT_FALSE(zero_reader.ok());
}
TEST(PbWireReaderTest, GroupAndUnknownWireTypesFail) {
// Field 1 with wire types 3 (start group), 4 (end group), 6 and 7.
for (const char tag : {'\x0B', '\x0C', '\x0E', '\x0F'}) {
const std::string buf(&tag, 1);
Reader reader(buf);
EXPECT_FALSE(reader.Next()) << "wire tag " << static_cast<int>(tag);
EXPECT_FALSE(reader.ok()) << "wire tag " << static_cast<int>(tag);
}
}
} // namespace
} // namespace pbwire
} // namespace zvec
tests/db/index/common/version_manager_test.cc
+87 -21
View File
@@ -18,7 +18,6 @@
#include <gtest/gtest.h>
#include "db/common/file_helper.h"
#include "db/index/common/meta.h"
#include "proto/zvec.pb.h"
#include "zvec/db/schema.h"
namespace zvec {
@@ -89,15 +88,47 @@ TEST_F(VersionManagerTest, VersionLoadSave) {
CollectionSchema schema;
schema.set_name("test_collection");
version.set_schema(schema);
// A vector field carrying HNSW index parameters.
auto vector_field = std::make_shared<FieldSchema>();
vector_field->set_name("vector_field");
vector_field->set_data_type(DataType::VECTOR_FP32);
vector_field->set_dimension(128);
vector_field->set_index_params(
std::make_shared<HnswIndexParams>(MetricType::IP, 32, 200));
schema.add_field(vector_field);
// A scalar field with an inverted index.
auto scalar_field = std::make_shared<FieldSchema>();
scalar_field->set_name("title");
scalar_field->set_data_type(DataType::STRING);
scalar_field->set_index_params(
std::make_shared<InvertIndexParams>(true, true));
schema.add_field(scalar_field);
version.set_schema(schema);
version.set_enable_mmap(true);
// Persisted segment with two blocks and an indexed vector field.
auto segment_meta = std::make_shared<SegmentMeta>(1);
segment_meta->set_id(1);
version.add_persisted_segment_meta(segment_meta);
segment_meta->add_persisted_block(
BlockMeta(1, BlockType::SCALAR, 0, 99, 100, {"vector_field", "title"}));
segment_meta->add_persisted_block(
BlockMeta(2, BlockType::VECTOR_INDEX, 0, 99));
segment_meta->add_indexed_vector_field("vector_field");
EXPECT_TRUE(version.add_persisted_segment_meta(segment_meta).ok());
// Writing segment with an in-flight forward block.
auto writing_meta = std::make_shared<SegmentMeta>(2);
writing_meta->set_id(2);
writing_meta->set_writing_forward_block(
BlockMeta(10, BlockType::SCALAR, 100, 149, 50, {"vector_field"}));
version.reset_writing_segment_meta(writing_meta);
version.set_id_map_path_suffix(100);
version.set_delete_snapshot_path_suffix(200);
version.set_next_segment_id(2);
version.set_next_segment_id(3);
// Save version
EXPECT_TRUE(Version::Save(manifest_path, version).ok());
@@ -108,10 +139,49 @@ TEST_F(VersionManagerTest, VersionLoadSave) {
// Verify loaded version matches original
EXPECT_EQ(loaded_version.schema().name(), "test_collection");
EXPECT_EQ(loaded_version.persisted_segment_metas().size(), 1);
EXPECT_TRUE(loaded_version.enable_mmap());
const auto loaded_fields = loaded_version.schema().fields();
ASSERT_EQ(loaded_fields.size(), 2u);
const auto &loaded_vector_field = loaded_fields[0];
EXPECT_EQ(loaded_vector_field->name(), "vector_field");
EXPECT_EQ(loaded_vector_field->data_type(), DataType::VECTOR_FP32);
EXPECT_EQ(loaded_vector_field->dimension(), 128u);
ASSERT_NE(loaded_vector_field->index_params(), nullptr);
EXPECT_EQ(*loaded_vector_field->index_params(),
HnswIndexParams(MetricType::IP, 32, 200));
const auto &loaded_scalar_field = loaded_fields[1];
EXPECT_EQ(loaded_scalar_field->name(), "title");
EXPECT_EQ(loaded_scalar_field->data_type(), DataType::STRING);
ASSERT_NE(loaded_scalar_field->index_params(), nullptr);
EXPECT_EQ(*loaded_scalar_field->index_params(),
InvertIndexParams(true, true));
ASSERT_EQ(loaded_version.persisted_segment_metas().size(), 1u);
const auto loaded_segments = loaded_version.persisted_segment_metas();
ASSERT_EQ(loaded_segments.size(), 1u);
const auto &loaded_segment = loaded_segments[0];
EXPECT_EQ(loaded_segment->id(), 1u);
ASSERT_EQ(loaded_segment->persisted_blocks().size(), 2u);
EXPECT_EQ(
loaded_segment->persisted_blocks()[0],
BlockMeta(1, BlockType::SCALAR, 0, 99, 100, {"vector_field", "title"}));
EXPECT_EQ(loaded_segment->persisted_blocks()[1],
BlockMeta(2, BlockType::VECTOR_INDEX, 0, 99));
EXPECT_TRUE(loaded_segment->vector_indexed("vector_field"));
ASSERT_NE(loaded_version.writing_segment_meta(), nullptr);
EXPECT_EQ(loaded_version.writing_segment_meta()->id(), 2u);
ASSERT_TRUE(
loaded_version.writing_segment_meta()->has_writing_forward_block());
EXPECT_EQ(
loaded_version.writing_segment_meta()->writing_forward_block().value(),
BlockMeta(10, BlockType::SCALAR, 100, 149, 50, {"vector_field"}));
EXPECT_EQ(loaded_version.id_map_path_suffix(), 100);
EXPECT_EQ(loaded_version.delete_snapshot_path_suffix(), 200);
EXPECT_EQ(loaded_version.next_segment_id(), 2);
EXPECT_EQ(loaded_version.next_segment_id(), 3);
}
// Test VersionManager creation and recovery
@@ -254,27 +324,23 @@ TEST_F(VersionManagerTest, ErrorConditions) {
EXPECT_FALSE(version_manager->remove_persisted_segment_meta(999).ok());
}
// Test conversion between protobuf and internal schema
// Test that a schema round-trips through a Version object.
TEST_F(VersionManagerTest, SchemaConversion) {
// Create protobuf schema
zvec::proto::CollectionSchema pb_schema;
pb_schema.set_name("test_collection");
auto pb_field = pb_schema.add_fields();
pb_field->set_name("vector_field");
pb_field->set_data_type(zvec::proto::DataType::DT_VECTOR_FP32);
pb_field->set_dimension(128);
// Convert to internal schema (this would be done in the Load method)
CollectionSchema internal_schema;
internal_schema.set_name(pb_schema.name());
// In a real implementation, fields would be converted here
internal_schema.set_name("test_collection");
auto field = std::make_shared<FieldSchema>();
field->set_name("vector_field");
field->set_data_type(DataType::VECTOR_FP32);
field->set_dimension(128);
internal_schema.add_field(field);
// Test that we can set and retrieve the schema
Version version;
version.set_schema(internal_schema);
EXPECT_EQ(version.schema().name(), "test_collection");
ASSERT_EQ(version.schema().fields().size(), 1u);
EXPECT_EQ(version.schema().fields()[0]->dimension(), 128u);
}
// Test SegmentMeta functionality
@@ -300,4 +366,4 @@ TEST_F(VersionManagerTest, SegmentMetaOperations) {
EXPECT_EQ(segment_meta.max_doc_id(), 100);
}
} // namespace zvec
} // namespace zvec
tests/db/index/segment/sql_expr_parser_test.cc
+22 -32
View File
@@ -16,9 +16,8 @@
#include "db/index/segment/sql_expr_parser.h"
#include <arrow/array.h>
#include <arrow/compute/api.h>
#include <arrow/dataset/api.h>
#include <arrow/dataset/discovery.h>
#include <arrow/memory_pool.h>
#include <arrow/record_batch.h>
#include <arrow/result.h>
#include <arrow/table.h>
#include <arrow/testing/gtest_util.h>
@@ -26,7 +25,6 @@
#include "utils/utils.h"
using namespace arrow;
using namespace arrow::dataset;
using namespace zvec;
class SqlExprParserTest : public ::testing::Test {
@@ -237,42 +235,37 @@ std::shared_ptr<arrow::Table> MakeTestTable() {
}
// Convert Table to Dataset (for testing)
arrow::Result<std::shared_ptr<arrow::dataset::Dataset>> MakeTestDataset(
const std::shared_ptr<arrow::Table> &table) {
return std::make_shared<arrow::dataset::InMemoryDataset>(table);
}
TEST_F(SqlExprParserTest, ParseAndScanDataSet) {
auto status = arrow::compute::Initialize();
// Execute the parsed expression against in-memory data. This used to go
// through arrow::dataset::Scanner, but Arrow Dataset is disabled in the
// slim build; ExecuteScalarExpression preserves the same intent: verify
// that a parsed expression evaluates correctly on real data.
TEST_F(SqlExprParserTest, ParseAndExecuteExpression) {
ASSERT_OK(arrow::compute::Initialize());
auto schema = arrow::schema({arrow::field("int_col", arrow::int32()),
arrow::field("double_col", arrow::float64()),
arrow::field("string_col", arrow::utf8()),
arrow::field("bool_col", arrow::boolean())});
// Step 1: Create test table
// Step 1: Create test table and turn it into a single record batch
auto table = MakeTestTable();
ASSERT_OK_AND_ASSIGN(auto batch, table->CombineChunksToBatch());
// Step 2: Convert to Dataset
auto dataset = MakeTestDataset(table).ValueOrDie();
// Step 3: Create scanner and project expression A + B
auto scanner_builder = dataset->NewScan().ValueOrDie();
// Step 2: Parse and bind the projection expression A + B
auto expr = ParseToExpression("int_col + double_col", schema).ValueOrDie();
status = scanner_builder->Project({expr}, {"sum"});
ASSERT_OK_AND_ASSIGN(auto bound, expr.Bind(*schema));
auto scanner = scanner_builder->Finish().ValueOrDie();
// Step 3: Execute the expression and get results
ASSERT_OK_AND_ASSIGN(auto datum, arrow::compute::ExecuteScalarExpression(
bound, *schema, arrow::Datum(batch)));
// Step 4: Execute and get results
auto result_table = scanner->ToTable().ValueOrDie();
ASSERT_TRUE(result_table != nullptr);
ASSERT_EQ(result_table->num_rows(), 5);
auto sum_array = datum.make_array();
ASSERT_TRUE(sum_array != nullptr);
ASSERT_EQ(sum_array->length(), 5);
auto int_col = table->column(0); // int_col
auto double_col = table->column(1); // double_col
auto sum_col = result_table->column(0); // sum column
auto int_col = table->column(0); // int_col
auto double_col = table->column(1); // double_col
auto sum_col = std::static_pointer_cast<arrow::DoubleArray>(sum_array);
for (int64_t i = 0; i < table->num_rows(); ++i) {
auto int_value =
@@ -281,10 +274,7 @@ TEST_F(SqlExprParserTest, ParseAndScanDataSet) {
auto double_value =
std::static_pointer_cast<arrow::DoubleArray>(double_col->chunk(0))
->Value(i);
auto sum_value =
std::static_pointer_cast<arrow::DoubleArray>(sum_col->chunk(0))
->Value(i);
ASSERT_NEAR(int_value + double_value, sum_value, 1e-10);
ASSERT_NEAR(int_value + double_value, sum_col->Value(i), 1e-10);
}
}
}
tests/db/index/segment/sql_expr_validator_test.cc
-16
View File
@@ -14,7 +14,6 @@
#include <arrow/array.h>
#include <arrow/builder.h>
#include <arrow/dataset/api.h>
#include <arrow/table.h>
#include <arrow/type.h>
#include <gmock/gmock-matchers.h>
@@ -38,24 +37,9 @@ class ExprValidatorTest : public ::testing::Test {
schema_ = arrow::schema({arrow::field("int32_col", arrow::int32()),
arrow::field("double_col", arrow::float64()),
arrow::field("str_col", arrow::utf8())});
std::vector<std::shared_ptr<arrow::Array>> arrays;
for (const auto &field : schema_->fields()) {
std::unique_ptr<arrow::ArrayBuilder> builder;
ASSERT_TRUE(arrow::MakeBuilder(arrow::default_memory_pool(),
field->type(), &builder)
.ok());
std::shared_ptr<arrow::Array> array;
ASSERT_TRUE(builder->Finish(&array).ok());
arrays.push_back(array);
}
auto table = arrow::Table::Make(schema_, arrays);
dataset_ = std::make_shared<arrow::dataset::InMemoryDataset>(table);
}
std::shared_ptr<arrow::Schema> schema_;
std::shared_ptr<arrow::dataset::Dataset> dataset_;
};
TEST_F(ExprValidatorTest, SingleNumericColumn_Valid) {
tests/db/sqlengine/CMakeLists.txt
-1
View File
@@ -23,7 +23,6 @@ foreach(CC_SRCS ${ALL_TEST_SRCS})
cc_gmock(
NAME ${CC_TARGET} STRICT
LIBS zvec_common
zvec_proto
zvec_sqlengine
zvec
zvec_ailego
tests/db/sqlengine/perf_bench_test.cc
+448
View File
@@ -0,0 +1,448 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Micro benchmarks for the query paths affected by the size-optimization
//! work (third-party -Os for non-hot libraries, slimmed Arrow compute,
//! trimmed RocksDB features):
//!
//! - forward filter : column scan + expression evaluation (Arrow compute)
//! - inverted search : term/range/boolean filters through inverted indexes
//! - FTS search : tokenized match queries (cppjieba/snowball/utf8proc)
//! - insert : segment build throughput (RocksDB id map, Arrow IPC)
//!
//! The benchmarks reuse the recall-test fixtures, so results are comparable
//! against the same data layout as the functional tests.
//!
//! Disabled by default to keep CI fast. Run with:
//!
//! ZVEC_RUN_PERF_BENCH=1 ./build/bin/perf_bench_test
//!
//! Tunables (environment variables):
//! ZVEC_PERF_DOC_COUNT documents per segment (default 100000)
//! ZVEC_PERF_ITERATIONS measured runs per bench (default 30)
#include <algorithm>
#include <chrono>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <functional>
#include <memory>
#include <numeric>
#include <string>
#include <vector>
#include <gtest/gtest.h>
#include "db/sqlengine/sqlengine.h"
#include "zvec/db/query_params.h"
#include "zvec/db/schema.h"
#include "recall_base.h"
namespace zvec::sqlengine {
namespace {
bool PerfEnabled() {
const char *value = std::getenv("ZVEC_RUN_PERF_BENCH");
return value != nullptr && std::string(value) == "1";
}
int64_t DocCount() {
if (const char *value = std::getenv("ZVEC_PERF_DOC_COUNT")) {
return std::atoll(value);
}
return 100000;
}
int Iterations() {
if (const char *value = std::getenv("ZVEC_PERF_ITERATIONS")) {
return std::atoi(value);
}
return 30;
}
// Executes fn once for warm-up, then measures repeated runs and prints a
// one-line [PERF] report with latency percentiles and throughput.
void RunBench(const std::string &name,
const std::function<Result<DocPtrList>()> &fn) {
auto result = fn(); // warm-up also validates the query
if (!result.has_value()) {
std::printf("[PERF] %-34s | ERROR: %s\n", name.c_str(),
result.error().c_str());
return;
}
const size_t rows = result.value().size();
std::vector<double> samples_us;
samples_us.reserve(Iterations());
for (int i = 0; i < Iterations(); ++i) {
const auto start = std::chrono::steady_clock::now();
auto r = fn();
const auto end = std::chrono::steady_clock::now();
if (!r.has_value()) {
std::printf("[PERF] %-34s | ERROR on run %d: %s\n", name.c_str(), i,
r.error().c_str());
return;
}
samples_us.push_back(
std::chrono::duration<double, std::micro>(end - start).count());
}
std::sort(samples_us.begin(), samples_us.end());
const auto percentile = [&](double p) {
const size_t idx = static_cast<size_t>(p * samples_us.size());
return samples_us[std::min(idx, samples_us.size() - 1)];
};
const double sum_us =
std::accumulate(samples_us.begin(), samples_us.end(), 0.0);
const double mean_us = sum_us / static_cast<double>(samples_us.size());
std::printf(
"[PERF] %-34s | rows=%6zu | median=%9.1fus mean=%9.1fus p90=%9.1fus "
"p99=%9.1fus min=%9.1fus | qps=%8.1f\n",
name.c_str(), rows, percentile(0.50), mean_us, percentile(0.90),
percentile(0.99), samples_us.front(), 1e6 / mean_us);
std::fflush(stdout);
}
} // namespace
// ============================================================================
// Forward filter + inverted search benchmarks (recall_base schema)
// ============================================================================
class ScalarPerfBench : public RecallTest {
protected:
static void SetUpTestSuite() {
if (!PerfEnabled()) {
return;
}
FileHelper::RemoveDirectory(seg_path_);
FileHelper::CreateDirectory(seg_path_);
collection_schema_ = GetCollectionSchema();
auto segment = create_segment();
ASSERT_NE(segment, nullptr);
const auto start = std::chrono::steady_clock::now();
auto status = InsertDoc(segment, 0, static_cast<uint64_t>(DocCount()));
const auto end = std::chrono::steady_clock::now();
ASSERT_TRUE(status.ok()) << status.c_str();
segments_.push_back(segment);
const double total_ms =
std::chrono::duration<double, std::milli>(end - start).count();
std::printf(
"[PERF] %-34s | docs=%6lld | total=%9.1fms | docs_per_sec=%9.0f\n",
"insert_build_segment", static_cast<long long>(DocCount()), total_ms,
static_cast<double>(DocCount()) / (total_ms / 1000.0));
std::fflush(stdout);
engine_ = SQLEngine::create(std::make_shared<Profiler>());
}
static void TearDownTestSuite() {
engine_.reset();
RecallTest::TearDownTestSuite();
}
// Builds a filter-only SearchQuery (no vector target) and returns a
// runnable closure for RunBench.
std::function<Result<DocPtrList>()> FilterQuery(const std::string &filter,
int topk = 2000) const {
return [filter, topk]() -> Result<DocPtrList> {
SearchQuery query;
query.filter_ = filter;
query.topk_ = topk;
query.output_fields_ = {"id", "name", "age"};
return engine_->execute(collection_schema_, query, segments_);
};
}
static inline SQLEngine::Ptr engine_;
};
#define PERF_SKIP() \
do { \
if (!PerfEnabled()) { \
GTEST_SKIP() << "set ZVEC_RUN_PERF_BENCH=1 to run perf benchmarks"; \
} \
} while (0)
TEST_F(ScalarPerfBench, ForwardFilterLowSelectivity) {
PERF_SKIP();
// age = doc_id % 100, so age < 5 matches 5% of the docs.
RunBench("forward_filter_age_lt_5", FilterQuery("age < 5"));
}
TEST_F(ScalarPerfBench, ForwardFilterHalfSelectivity) {
PERF_SKIP();
RunBench("forward_filter_age_lt_50", FilterQuery("age < 50"));
}
TEST_F(ScalarPerfBench, ForwardFilterRange) {
PERF_SKIP();
RunBench("forward_filter_age_range", FilterQuery("age >= 25 and age <= 75"));
}
TEST_F(ScalarPerfBench, ForwardFilterString) {
PERF_SKIP();
// name = "user_" + doc_id % 100: full-scan string comparison.
RunBench("forward_filter_name_eq", FilterQuery("name = 'user_5'"));
}
TEST_F(ScalarPerfBench, ForwardFilterDouble) {
PERF_SKIP();
RunBench("forward_filter_score_gt", FilterQuery("score > 50.0"));
}
TEST_F(ScalarPerfBench, InvertTerm) {
PERF_SKIP();
RunBench("invert_term_age_eq_5", FilterQuery("invert_age = 5"));
}
TEST_F(ScalarPerfBench, InvertRange) {
PERF_SKIP();
RunBench("invert_range_age_lt_10", FilterQuery("invert_age < 10"));
}
TEST_F(ScalarPerfBench, InvertStringTerm) {
PERF_SKIP();
RunBench("invert_term_name_eq", FilterQuery("invert_name = 'user_5'"));
}
TEST_F(ScalarPerfBench, InvertWideRange) {
PERF_SKIP();
const std::string filter = "invert_id >= " + std::to_string(DocCount() / 2);
RunBench("invert_range_id_half", FilterQuery(filter));
}
TEST_F(ScalarPerfBench, InvertBooleanOr) {
PERF_SKIP();
RunBench("invert_or_two_terms",
FilterQuery("invert_age = 5 or invert_age = 6"));
}
// Sanity-checks the hit counts that the benchmarks above rely on.
TEST_F(ScalarPerfBench, ExpectedHitCounts) {
PERF_SKIP();
const int64_t n = DocCount();
const auto count_ids_where = [n](std::function<bool(uint64_t)> pred) {
int64_t count = 0;
for (uint64_t id = 0; id < static_cast<uint64_t>(n); ++id) {
if (pred(id)) ++count;
}
return count;
};
auto result = FilterQuery("age < 5")();
ASSERT_TRUE(result.has_value());
const int64_t expected =
count_ids_where([](uint64_t id) { return id % 100 < 5; });
EXPECT_EQ(static_cast<int64_t>(result.value().size()),
std::min<int64_t>(expected, 2000));
result = FilterQuery("invert_age = 5")();
ASSERT_TRUE(result.has_value());
EXPECT_EQ(static_cast<int64_t>(result.value().size()),
std::min<int64_t>(n / 100, 2000));
}
// ============================================================================
// FTS benchmarks (dedicated schema with a tokenized content field)
// ============================================================================
class FtsPerfBench : public ::testing::Test {
protected:
static void SetUpTestSuite() {
if (!PerfEnabled()) {
return;
}
FileHelper::RemoveDirectory(seg_path_);
FileHelper::CreateDirectory(seg_path_);
build_schema();
auto segment = create_segment();
ASSERT_NE(segment, nullptr);
const auto start = std::chrono::steady_clock::now();
insert_docs(segment);
const auto end = std::chrono::steady_clock::now();
segments_.push_back(segment);
const double total_ms =
std::chrono::duration<double, std::milli>(end - start).count();
std::printf(
"[PERF] %-34s | docs=%6lld | total=%9.1fms | docs_per_sec=%9.0f\n",
"fts_insert_build_segment", static_cast<long long>(DocCount()),
total_ms, static_cast<double>(DocCount()) / (total_ms / 1000.0));
std::fflush(stdout);
engine_ = SQLEngine::create(std::make_shared<Profiler>());
}
static void TearDownTestSuite() {
segments_.clear();
engine_.reset();
schema_.reset();
FileHelper::RemoveDirectory(seg_path_);
}
std::function<Result<DocPtrList>()> FtsMatch(
const std::string &match_string, const std::string &default_op = "",
const std::string &filter = "", int topk = 100) const {
return [match_string, default_op, filter, topk]() -> Result<DocPtrList> {
SearchQuery query;
query.topk_ = topk;
query.target_.field_name_ = "content";
FtsClause fts;
fts.match_string_ = match_string;
query.target_.clause_ = fts;
if (!default_op.empty()) {
auto fts_qp = std::make_shared<zvec::FtsQueryParams>();
fts_qp->set_default_operator(default_op);
query.target_.query_params_ = fts_qp;
}
if (!filter.empty()) {
query.filter_ = filter;
}
return engine_->execute(schema_, query, segments_);
};
}
private:
static void build_schema() {
auto fts_params = std::make_shared<FtsIndexParams>(
"whitespace", std::vector<std::string>{"lowercase"}, "");
auto invert_params = std::make_shared<InvertIndexParams>(true);
schema_ = std::make_shared<CollectionSchema>(
"fts_perf_bench",
std::vector<FieldSchema::Ptr>{
std::make_shared<FieldSchema>("content", DataType::STRING, false,
fts_params),
std::make_shared<FieldSchema>("tag", DataType::INT32, false,
invert_params),
// Dummy vector field required for filter parsing path in execute.
std::make_shared<FieldSchema>(
"vec", DataType::VECTOR_FP32, 4, false,
std::make_shared<FlatIndexParams>(MetricType::L2)),
});
}
static Segment::Ptr create_segment() {
auto segment_meta = std::make_shared<SegmentMeta>();
segment_meta->set_id(0);
auto id_map = IDMap::CreateAndOpen("fts_perf_bench", seg_path_ + "/id_map",
true, false);
auto delete_store = std::make_shared<DeleteStore>("fts_perf_bench");
Version v1;
v1.set_schema(*schema_);
std::string v_path = seg_path_ + "/manifest";
FileHelper::CreateDirectory(v_path);
auto vm = VersionManager::Create(v_path, v1);
if (!vm.has_value()) {
return nullptr;
}
BlockMeta mem_block;
mem_block.id_ = 0;
mem_block.type_ = BlockType::SCALAR;
mem_block.min_doc_id_ = 0;
mem_block.max_doc_id_ = 0;
mem_block.doc_count_ = 0;
segment_meta->set_writing_forward_block(mem_block);
SegmentOptions options;
options.read_only_ = false;
options.enable_mmap_ = true;
options.max_buffer_size_ = 256 * 1024;
auto result = Segment::CreateAndOpen(seg_path_, *schema_, 0, 0, id_map,
delete_store, vm.value(), options);
if (!result) {
return nullptr;
}
return result.value();
}
// Generates DocCount() documents with 8 tokens drawn from a 256-word
// vocabulary (each word hits ~3% of docs), plus a 1% word ("mediumword")
// and a 0.1% word ("rareword") for selectivity sweeps.
static void insert_docs(const Segment::Ptr &segment) {
for (int64_t doc_id = 0; doc_id < DocCount(); ++doc_id) {
std::string content;
for (int k = 0; k < 8; ++k) {
if (k > 0) content += ' ';
content += "w" + std::to_string((doc_id + k * 31) % 256);
}
if (doc_id % 100 == 0) {
content += " mediumword";
}
if (doc_id % 1000 == 0) {
content += " rareword";
}
Doc doc;
doc.set_pk("pk_" + std::to_string(doc_id));
doc.set_doc_id(static_cast<uint64_t>(doc_id));
doc.set<std::string>("content", content);
doc.set<int32_t>("tag", static_cast<int32_t>(doc_id % 2));
auto status = segment->Insert(doc);
if (!status.ok()) {
std::printf("[PERF] fts insert failed at doc %lld: %s\n",
static_cast<long long>(doc_id), status.c_str());
return;
}
}
}
protected:
static inline std::string seg_path_ = "./fts_perf_bench_collection";
static inline CollectionSchema::Ptr schema_;
static inline std::vector<Segment::Ptr> segments_;
static inline SQLEngine::Ptr engine_;
};
TEST_F(FtsPerfBench, RareTerm) {
PERF_SKIP();
RunBench("fts_match_rare_0.1pct", FtsMatch("rareword"));
}
TEST_F(FtsPerfBench, MediumTerm) {
PERF_SKIP();
RunBench("fts_match_medium_1pct", FtsMatch("mediumword"));
}
TEST_F(FtsPerfBench, CommonTerm) {
PERF_SKIP();
RunBench("fts_match_common_3pct", FtsMatch("w5"));
}
TEST_F(FtsPerfBench, MultiTermOr) {
PERF_SKIP();
RunBench("fts_match_3term_or", FtsMatch("w1 w2 w3"));
}
TEST_F(FtsPerfBench, MultiTermAnd) {
PERF_SKIP();
// Adjacent generated tokens differ by 31 (mod 256), so w1+w32 co-occur.
RunBench("fts_match_2term_and", FtsMatch("w1 w32", "AND"));
}
TEST_F(FtsPerfBench, MatchWithInvertedFilter) {
PERF_SKIP();
// mediumword only appears in docs whose id % 100 == 0, all of which have
// tag = 0, so filter on tag = 0 keeps every match.
RunBench("fts_match_plus_filter", FtsMatch("mediumword", "", "tag = 0"));
}
} // namespace zvec::sqlengine
tests/db/utf8_acp/CMakeLists.txt
-1
View File
@@ -9,7 +9,6 @@ endif()
set(UTF8_ACP_LIBS
zvec
zvec_proto
core_knn_flat
core_knn_flat_sparse
core_knn_hnsw
thirdparty/CMakeLists.txt
Vendored
+16 -1
View File
@@ -4,6 +4,22 @@ project(thirdparty)
include(${PROJECT_ROOT_DIR}/cmake/utils.cmake)
# Optimize third-party libraries for size instead of speed to reduce the final
# binary size. zvec's own code (built under src/) keeps its default speed
# optimization since these flags are directory-scoped to thirdparty/.
# - GCC/Clang: use -Os in place of -O3
# - MSVC: favor small code (/O1, /Ob1) in place of /O2, /Ob2, while preserving
# the runtime library and other default Release flags
if(NOT MSVC AND CMAKE_BUILD_TYPE STREQUAL "Release")
set(CMAKE_C_FLAGS_RELEASE "-Os -DNDEBUG")
set(CMAKE_CXX_FLAGS_RELEASE "-Os -DNDEBUG")
elseif(MSVC)
string(REPLACE "/O2" "/O1" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
string(REPLACE "/O2" "/O1" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
string(REPLACE "/Ob2" "/Ob1" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
string(REPLACE "/Ob2" "/Ob1" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
endif()
set(CMAKE_MACOSX_RPATH ON)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
set(EXTERNAL_BINARY_DIR ${CMAKE_BINARY_DIR}/external)
@@ -22,7 +38,6 @@ add_subdirectory(arrow arrow EXCLUDE_FROM_ALL)
add_subdirectory(gflags gflags EXCLUDE_FROM_ALL)
add_subdirectory(glog glog EXCLUDE_FROM_ALL)
add_subdirectory(yaml-cpp yaml-cpp EXCLUDE_FROM_ALL)
add_subdirectory(protobuf protobuf EXCLUDE_FROM_ALL)
add_subdirectory(antlr antlr EXCLUDE_FROM_ALL)
add_subdirectory(lz4 lz4 EXCLUDE_FROM_ALL)
add_subdirectory(rocksdb rocksdb EXCLUDE_FROM_ALL)
thirdparty/arrow/CMakeLists.txt
Vendored
+20 -21
View File
@@ -19,6 +19,9 @@ if(MSVC)
set(ARROW_WIN_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/arrow.windows.patch)
apply_patch_once("arrow_windows_crt_fix" "${ARROW_SRC_DIR}" "${ARROW_WIN_PATCH}")
endif()
# Trim unused compute kernels to reduce binary size (~14MB savings)
set(ARROW_SLIM_COMPUTE_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/arrow.slim_compute.patch)
apply_patch_once("arrow_slim_compute" "${ARROW_SRC_DIR}" "${ARROW_SLIM_COMPUTE_PATCH}")
include(ExternalProject)
include(ProcessorCount)
@@ -29,6 +32,15 @@ set(ARROW_CMAKE_BUILD_TYPE "${CMAKE_BUILD_TYPE}")
if(NOT ARROW_CMAKE_BUILD_TYPE)
set(ARROW_CMAKE_BUILD_TYPE Release)
endif()
# Keep Arrow on the Release configuration: benchmarks showed that building
# its compute kernels with -Os slows filter evaluation by ~8-10% while the
# size saving is kept by the slim compute kernel set and gc-sections.
# Pass -ffunction-sections -fdata-sections to Arrow so the linker can
# perform fine-grained dead code elimination on Arrow objects.
if(NOT MSVC)
set(_ARROW_SIZE_FLAGS "-DCMAKE_C_FLAGS=-ffunction-sections -fdata-sections" "-DCMAKE_CXX_FLAGS=-ffunction-sections -fdata-sections")
endif()
if(MSVC)
set(LIB_PARQUET ${EXTERNAL_LIB_DIR}/parquet_static.lib)
@@ -36,14 +48,12 @@ if(MSVC)
set(LIB_COMPUTE ${EXTERNAL_LIB_DIR}/arrow_compute_static.lib)
set(LIB_ACERO ${EXTERNAL_LIB_DIR}/arrow_acero_static.lib)
set(LIB_ARROW_DEPENDS ${EXTERNAL_LIB_DIR}/arrow_bundled_dependencies.lib)
set(LIB_ARROW_DATASET ${EXTERNAL_LIB_DIR}/arrow_dataset_static.lib)
else()
set(LIB_PARQUET ${EXTERNAL_LIB_DIR}/libparquet.a)
set(LIB_ARROW ${EXTERNAL_LIB_DIR}/libarrow.a)
set(LIB_COMPUTE ${EXTERNAL_LIB_DIR}/libarrow_compute.a)
set(LIB_ACERO ${EXTERNAL_LIB_DIR}/libarrow_acero.a)
set(LIB_ARROW_DEPENDS ${EXTERNAL_LIB_DIR}/libarrow_bundled_dependencies.a)
set(LIB_ARROW_DATASET ${EXTERNAL_LIB_DIR}/libarrow_dataset.a)
endif()
set(CONFIGURE_ENV_LIST "")
@@ -85,10 +95,10 @@ if(ANDROID)
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0
DOWNLOAD_COMMAND ""
BUILD_IN_SOURCE false
CONFIGURE_COMMAND env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} -DCMAKE_BUILD_TYPE=${ARROW_CMAKE_BUILD_TYPE} -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=ON -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} -DANDROID_ABI=${ANDROID_ABI} -DANDROID_NATIVE_API_LEVEL=${ANDROID_NATIVE_API_LEVEL} -DARROW_WITH_MUSL=OFF "<SOURCE_DIR>/cpp"
CONFIGURE_COMMAND env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} -DCMAKE_BUILD_TYPE=${ARROW_CMAKE_BUILD_TYPE} -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=OFF -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} -DANDROID_ABI=${ANDROID_ABI} -DANDROID_NATIVE_API_LEVEL=${ANDROID_NATIVE_API_LEVEL} -DARROW_WITH_MUSL=OFF "<SOURCE_DIR>/cpp"
BUILD_COMMAND "${CMAKE_COMMAND}" --build . --target all -- -j ${NPROC}
INSTALL_COMMAND "${CMAKE_COMMAND}" --install "<BINARY_DIR>" --prefix=${EXTERNAL_BINARY_DIR}/usr/local
BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS} ${LIB_ARROW_DATASET}
BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS}
LOG_DOWNLOAD ON
LOG_CONFIGURE ON
LOG_BUILD ON
@@ -112,10 +122,10 @@ elseif(IOS)
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0
DOWNLOAD_COMMAND ""
BUILD_IN_SOURCE false
CONFIGURE_COMMAND env ${_arrow_ios_env} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} -DCMAKE_BUILD_TYPE=${ARROW_CMAKE_BUILD_TYPE} -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=ON -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_SYSTEM_NAME=iOS -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET} -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -DCMAKE_OSX_SYSROOT=${CMAKE_OSX_SYSROOT} -DARROW_CPU_FLAG=${IOS_ARROW_CPU_FLAG} "<SOURCE_DIR>/cpp"
CONFIGURE_COMMAND env ${_arrow_ios_env} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} -DCMAKE_BUILD_TYPE=${ARROW_CMAKE_BUILD_TYPE} -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=OFF -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_SYSTEM_NAME=iOS -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET} -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -DCMAKE_OSX_SYSROOT=${CMAKE_OSX_SYSROOT} -DARROW_CPU_FLAG=${IOS_ARROW_CPU_FLAG} "<SOURCE_DIR>/cpp"
BUILD_COMMAND env "IPHONEOS_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}" "${CMAKE_COMMAND}" --build . --target all -- -j ${NPROC}
INSTALL_COMMAND "${CMAKE_COMMAND}" --install "<BINARY_DIR>" --prefix=${EXTERNAL_BINARY_DIR}/usr/local
BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS} ${LIB_ARROW_DATASET}
BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS}
LOG_DOWNLOAD ON
LOG_CONFIGURE ON
LOG_BUILD ON
@@ -140,10 +150,10 @@ elseif (MSVC)
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0
DOWNLOAD_COMMAND ""
BUILD_IN_SOURCE false
CONFIGURE_COMMAND "${CMAKE_COMMAND}" -E env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${ARROW_EXTRA_CMAKE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} -DCMAKE_BUILD_TYPE=$<CONFIG> -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=ON -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib "<SOURCE_DIR>/cpp"
CONFIGURE_COMMAND "${CMAKE_COMMAND}" -E env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${ARROW_EXTRA_CMAKE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} -DCMAKE_BUILD_TYPE=$<CONFIG> -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=OFF -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib "<SOURCE_DIR>/cpp"
BUILD_COMMAND "${CMAKE_COMMAND}" --build . -j ${NPROC} --config $<CONFIG>
INSTALL_COMMAND "${CMAKE_COMMAND}" --install "<BINARY_DIR>" --prefix=${EXTERNAL_BINARY_DIR}/usr/local --config $<CONFIG>
BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS} ${LIB_ARROW_DATASET}
BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS}
LOG_DOWNLOAD ON
LOG_CONFIGURE ON
LOG_BUILD ON
@@ -155,10 +165,10 @@ else ()
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0
DOWNLOAD_COMMAND ""
BUILD_IN_SOURCE false
CONFIGURE_COMMAND env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} -DCMAKE_BUILD_TYPE=${ARROW_CMAKE_BUILD_TYPE} -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=ON -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib "<SOURCE_DIR>/cpp"
CONFIGURE_COMMAND env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${_ARROW_COMPILER_LAUNCHER_ARGS} ${_ARROW_COMPILER_ARGS} ${_ARROW_SIZE_FLAGS} -DCMAKE_BUILD_TYPE=${ARROW_CMAKE_BUILD_TYPE} -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=OFF -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib "<SOURCE_DIR>/cpp"
BUILD_COMMAND "${CMAKE_COMMAND}" --build . --target all -- -j ${NPROC}
INSTALL_COMMAND "${CMAKE_COMMAND}" --install "<BINARY_DIR>" --prefix=${EXTERNAL_BINARY_DIR}/usr/local
BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS} ${LIB_ARROW_DATASET}
BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS}
LOG_DOWNLOAD ON
LOG_CONFIGURE ON
LOG_BUILD ON
@@ -228,14 +238,3 @@ set_target_properties(
INTERFACE_COMPILE_DEFINITIONS "ARROW_ACERO_STATIC"
)
add_dependencies(Arrow::arrow_acero ARROW.BUILD)
add_library(Arrow::arrow_dataset UNKNOWN IMPORTED GLOBAL)
set_target_properties(
Arrow::arrow_dataset PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES ${EXTERNAL_INC_DIR}
INTERFACE_SYSTEM_INCLUDE_DIRECTORIES ${EXTERNAL_INC_DIR}
IMPORTED_LOCATION "${LIB_ARROW_DATASET}"
INTERFACE_LINK_LIBRARIES "Arrow::arrow_depends;Arrow::arrow_static;Arrow::arrow_compute;Arrow::arrow_acero"
INTERFACE_COMPILE_DEFINITIONS "ARROW_DS_STATIC"
)
add_dependencies(Arrow::arrow_dataset ARROW.BUILD)
thirdparty/arrow/arrow.slim_compute.patch
Vendored
+132
View File
@@ -0,0 +1,132 @@
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index b33acae5b9..6771f4a98d 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -747,45 +747,39 @@ if(ARROW_COMPUTE)
string(APPEND ARROW_COMPUTE_PC_CFLAGS "${ARROW_COMPUTE_PC_CFLAGS_PRIVATE}")
set(ARROW_COMPUTE_PC_CFLAGS_PRIVATE "")
endif()
- # Include the remaining kernels
+ # Include only kernels required by zvec:
+ # - scalar_arithmetic: add/subtract/multiply/divide for add_column expressions
+ # - scalar_boolean: AND/OR/NOT for filter expressions
+ # - scalar_compare: comparison operators for filter expressions
+ # - scalar_if_else: conditional expression support
+ # - scalar_nested: make_struct (dataset scan), list_value_length
+ # - scalar_set_lookup: is_in / index_in for filter
+ # - scalar_string_ascii/utf8: match_like for LIKE queries
+ # - scalar_validity: is_null / is_valid
+ # - vector_array_sort / vector_sort: sort_indices for ORDER BY
+ # - vector_swizzle: Take (index-based row selection)
+ # - util_internal: shared helpers for kernels above
+ # - ree_util_internal: required by vector_selection
+ # Removed: aggregate_*, hash_aggregate_*, pivot_internal, scalar_random,
+ # scalar_round, scalar_temporal_*, vector_cumulative_ops, vector_nested,
+ # vector_pairwise, vector_rank, vector_replace, vector_run_end_encode,
+ # vector_select_k, vector_statistics
list(APPEND
ARROW_COMPUTE_LIB_SRCS
compute/initialize.cc
- compute/kernels/aggregate_basic.cc
- compute/kernels/aggregate_mode.cc
- compute/kernels/aggregate_pivot.cc
- compute/kernels/aggregate_quantile.cc
- compute/kernels/aggregate_tdigest.cc
- compute/kernels/aggregate_var_std.cc
- compute/kernels/hash_aggregate.cc
- compute/kernels/hash_aggregate_numeric.cc
- compute/kernels/hash_aggregate_pivot.cc
- compute/kernels/pivot_internal.cc
compute/kernels/ree_util_internal.cc
compute/kernels/scalar_arithmetic.cc
compute/kernels/scalar_boolean.cc
compute/kernels/scalar_compare.cc
compute/kernels/scalar_if_else.cc
compute/kernels/scalar_nested.cc
- compute/kernels/scalar_random.cc
- compute/kernels/scalar_round.cc
compute/kernels/scalar_set_lookup.cc
compute/kernels/scalar_string_ascii.cc
compute/kernels/scalar_string_utf8.cc
- compute/kernels/scalar_temporal_binary.cc
- compute/kernels/scalar_temporal_unary.cc
compute/kernels/scalar_validity.cc
compute/kernels/util_internal.cc
compute/kernels/vector_array_sort.cc
- compute/kernels/vector_cumulative_ops.cc
- compute/kernels/vector_nested.cc
- compute/kernels/vector_pairwise.cc
- compute/kernels/vector_rank.cc
- compute/kernels/vector_replace.cc
- compute/kernels/vector_run_end_encode.cc
- compute/kernels/vector_select_k.cc
compute/kernels/vector_sort.cc
- compute/kernels/vector_statistics.cc
compute/kernels/vector_swizzle.cc
compute/key_hash_internal.cc
compute/key_map_internal.cc
@@ -798,9 +792,6 @@ if(ARROW_COMPUTE)
compute/util.cc
compute/util_internal.cc)
- append_runtime_avx2_src(ARROW_COMPUTE_LIB_SRCS compute/kernels/aggregate_basic_avx2.cc)
- append_runtime_avx512_src(ARROW_COMPUTE_LIB_SRCS
- compute/kernels/aggregate_basic_avx512.cc)
append_runtime_avx2_src(ARROW_COMPUTE_LIB_SRCS compute/key_hash_internal_avx2.cc)
append_runtime_avx2_bmi2_src(ARROW_COMPUTE_LIB_SRCS compute/key_map_internal_avx2.cc)
append_runtime_avx2_src(ARROW_COMPUTE_LIB_SRCS compute/row/compare_internal_avx2.cc)
diff --git a/cpp/src/arrow/compute/initialize.cc b/cpp/src/arrow/compute/initialize.cc
index d126ac951f..b53967d03d 100644
--- a/cpp/src/arrow/compute/initialize.cc
+++ b/cpp/src/arrow/compute/initialize.cc
@@ -26,47 +26,23 @@ namespace {
Status RegisterComputeKernels() {
auto registry = GetFunctionRegistry();
- // Register additional kernels on libarrow_compute
- // Scalar functions
+ // Only register kernels required by zvec:
+ // Scalar functions needed for expressions, filters, LIKE and dataset scan
internal::RegisterScalarArithmetic(registry);
internal::RegisterScalarBoolean(registry);
internal::RegisterScalarComparison(registry);
internal::RegisterScalarIfElse(registry);
internal::RegisterScalarNested(registry);
- internal::RegisterScalarRandom(registry); // Nullary
- internal::RegisterScalarRoundArithmetic(registry);
internal::RegisterScalarSetLookup(registry);
internal::RegisterScalarStringAscii(registry);
internal::RegisterScalarStringUtf8(registry);
- internal::RegisterScalarTemporalBinary(registry);
- internal::RegisterScalarTemporalUnary(registry);
internal::RegisterScalarValidity(registry);
- // Vector functions
+ // Vector functions needed for row selection and sorting (sort_indices)
internal::RegisterVectorArraySort(registry);
- internal::RegisterVectorCumulativeSum(registry);
- internal::RegisterVectorNested(registry);
- internal::RegisterVectorRank(registry);
- internal::RegisterVectorReplace(registry);
- internal::RegisterVectorSelectK(registry);
internal::RegisterVectorSort(registry);
- internal::RegisterVectorRunEndEncode(registry);
- internal::RegisterVectorRunEndDecode(registry);
- internal::RegisterVectorPairwise(registry);
- internal::RegisterVectorStatistics(registry);
internal::RegisterVectorSwizzle(registry);
- // Aggregate functions
- internal::RegisterHashAggregateBasic(registry);
- internal::RegisterHashAggregateNumeric(registry);
- internal::RegisterHashAggregatePivot(registry);
- internal::RegisterScalarAggregateBasic(registry);
- internal::RegisterScalarAggregateMode(registry);
- internal::RegisterScalarAggregatePivot(registry);
- internal::RegisterScalarAggregateQuantile(registry);
- internal::RegisterScalarAggregateTDigest(registry);
- internal::RegisterScalarAggregateVariance(registry);
-
return Status::OK();
}
thirdparty/protobuf/CMakeLists.txt
Vendored
-76
View File
@@ -1,76 +0,0 @@
set(protobuf_BUILD_TESTS OFF CACHE BOOL "Disable testing in protobuf" FORCE)
set(protobuf_WITH_ZLIB ON CACHE BOOL "Disable zlib support in protobuf" FORCE)
if(MSVC)
set(protobuf_MSVC_STATIC_RUNTIME ${ZVEC_USE_STATIC_CRT} CACHE BOOL "" FORCE)
endif()
if(MSVC)
# Remove /MP for sccache compatibility (sccache cannot cache multi-file compilations).
set(PROTOBUF_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/protobuf-3.21.12)
set(PROTOBUF_WINDOWS_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/protobuf.windows.patch)
apply_patch_once("protobuf_windows_fix" "${PROTOBUF_SRC_DIR}" "${PROTOBUF_WINDOWS_PATCH}")
endif()
set(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY})
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${EXTERNAL_LIB_DIR})
add_subdirectory(protobuf-3.21.12/cmake protobuf-3.21.12)
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY})
unset(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY)
mark_target_includes_system(libprotobuf libprotobuf-lite libprotoc)
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
target_compile_options(libprotobuf PRIVATE
-Wno-deprecated-declarations
-Wno-invalid-noreturn
-Wno-unused-function
)
target_compile_options(libprotoc PRIVATE
-Wno-unused-private-field
-Wno-unused-function
)
target_compile_options(protoc PRIVATE
-Wno-unused-private-field
-Wno-unused-function
)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
target_compile_options(libprotobuf PRIVATE
-Wno-deprecated-declarations
-Wno-unused-function
-Wno-maybe-uninitialized
-Wno-sign-compare
-Wno-return-type
-Wno-stringop-overflow
-Wno-stringop-overread
-Wno-array-bounds
)
target_compile_options(libprotoc PRIVATE
-Wno-unused-private-field
-Wno-unused-function
-Wno-unused-but-set-variable
-Wno-sign-compare
)
target_compile_options(protoc PRIVATE
-Wno-unused-private-field
-Wno-unused-function
-Wno-unused-but-set-variable
-Wno-sign-compare
)
endif()
get_target_property(libprotobuf_SOURCE_DIR libprotobuf SOURCE_DIR)
get_filename_component(libprotobuf_INCLUDE_DIR ${libprotobuf_SOURCE_DIR}/../src ABSOLUTE)
set(PROTOBUF_FOUND TRUE PARENT_SCOPE)
set(PROTOBUF_INCLUDE_DIR ${libprotobuf_INCLUDE_DIR} PARENT_SCOPE)
set(PROTOBUF_INCLUDE_DIRS ${libprotobuf_INCLUDE_DIR} PARENT_SCOPE)
set(PROTOBUF_LIBRARY $<TARGET_FILE:libprotobuf> PARENT_SCOPE)
set(PROTOBUF_LIBRARIES $<TARGET_FILE:libprotobuf> PARENT_SCOPE)
set(PROTOBUF_LITE_LIBRARY $<TARGET_FILE:libprotobuf-lite> PARENT_SCOPE)
set(PROTOBUF_LITE_LIBRARIES $<TARGET_FILE:libprotobuf-lite> PARENT_SCOPE)
set(PROTOBUF_PROTOC_LIBRARY $<TARGET_FILE:libprotoc> PARENT_SCOPE)
set(PROTOBUF_PROTOC_LIBRARIES $<TARGET_FILE:libprotoc> PARENT_SCOPE)
set(PROTOBUF_PROTOC_EXECUTABLE $<TARGET_FILE:protoc> PARENT_SCOPE)
thirdparty/protobuf/protobuf-3.21.12
Vendored
-1
Submodule thirdparty/protobuf/protobuf-3.21.12 deleted from f0dc78d7e6
thirdparty/protobuf/protobuf.windows.patch
Vendored
-15
View File
@@ -1,15 +0,0 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1234567..abcdefg 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -241,10 +241,6 @@
if (MSVC)
- if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
- # Build with multiple processes
- add_definitions(/MP)
- endif()
# Set source file and execution character sets to UTF-8
add_definitions(/utf-8)
# MSVC warning suppressions
add_definitions(
thirdparty/rocksdb/CMakeLists.txt
Vendored
+15
View File
@@ -42,6 +42,9 @@ set(WITH_CORE_TOOLS OFF CACHE BOOL "build with ldb and sst_dump" FORCE)
set(WITH_TOOLS OFF CACHE BOOL "build with tools" FORCE)
set(WITH_LZ4 ON CACHE BOOL "build with lz4" FORCE)
set(USE_RTTI ON CACHE BOOL "build with RTTI" FORCE)
set(WITH_TRACE_TOOLS OFF CACHE BOOL "Disable trace tools to reduce binary size" FORCE)
set(WITH_IOSTATS_CONTEXT OFF CACHE BOOL "Disable IO stats context to reduce binary size" FORCE)
set(WITH_PERF_CONTEXT OFF CACHE BOOL "Disable perf context to reduce binary size" FORCE)
set(WITH_WINDOWS_UTF8_FILENAMES ON CACHE BOOL "use UTF8 as characterset for opening files, regardles of the system code page" FORCE)
# TODO(windows): verify
set(ROCKSDB_SKIP_THIRDPARTY ON CACHE BOOL "skip thirdparty.inc" FORCE)
@@ -69,6 +72,18 @@ endif()
set(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY})
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${EXTERNAL_LIB_DIR})
# thirdparty/ builds with -Os for size, but RocksDB sits on hot paths
# (insert id-map, inverted-index scans) where -Os costs up to ~20%.
# Restore the Release defaults for this subtree to match upstream main.
if(MSVC)
string(REPLACE "/O1" "/O2" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
string(REPLACE "/O1" "/O2" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
string(REPLACE "/Ob1" "/Ob2" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
string(REPLACE "/Ob1" "/Ob2" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
else()
set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG")
set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
endif()
add_subdirectory(rocksdb-8.1.1)
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY})