diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000..16a4f68 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,32 @@ +# Curated, pragmatic check set for a modern C++20 library. The goal is to catch real defects +# (bugprone-*, the static analyzer, performance traps) without the noise that makes a tidy run +# something people learn to ignore. Not run as -warnings-as-errors in CI; it reports on changed +# files (see .github/workflows/format.yml). +Checks: > + bugprone-*, + performance-*, + portability-*, + clang-analyzer-*, + misc-*, + modernize-use-nullptr, + modernize-use-override, + modernize-use-using, + modernize-use-emplace, + modernize-make-unique, + modernize-loop-convert, + readability-braces-around-statements, + readability-redundant-member-init, + readability-misleading-indentation, + -bugprone-easily-swappable-parameters, + -bugprone-exception-escape, + -bugprone-narrowing-conversions, + -misc-no-recursion, + -misc-non-private-member-variables-in-classes, + -misc-include-cleaner, + -performance-avoid-endl, + -clang-analyzer-optin.core.EnumCastOutOfRange + +# Library headers + sources only; never third-party (stb, gtest, pybind11) or generated headers. +HeaderFilterRegex: '(include/tensorrt_cpp_api|src)/[^/]*\.(h|hpp)$' +WarningsAsErrors: '' +FormatStyle: file diff --git a/.cmake-format.yaml b/.cmake-format.yaml new file mode 100644 index 0000000..e95c6ef --- /dev/null +++ b/.cmake-format.yaml @@ -0,0 +1,15 @@ +# cmake-format config (used by the pre-commit cmake-format hook). Tuned to the repo's existing +# CMake style: a generous line width so the hand-wrapped install()/target_* calls are not +# reflowed, lowercase commands, and dangling close-parens for multi-line calls. +format: + line_width: 130 + tab_size: 4 + separate_ctrl_name_with_space: false + separate_fn_name_with_space: false + dangle_parens: true + command_case: lower + keyword_case: upper + max_subgroups_hwrap: 4 + max_pargs_hwrap: 6 +markup: + enable_markup: false diff --git a/.github/actions/setup-trt/action.yml b/.github/actions/setup-trt/action.yml new file mode 100644 index 0000000..58924cf --- /dev/null +++ b/.github/actions/setup-trt/action.yml @@ -0,0 +1,21 @@ +name: Setup CUDA + TensorRT +description: > + Install a consistent CUDA 12.6 + TensorRT 10.7 (cuda12.6) toolchain from NVIDIA's apt repo. + The TensorRT version is PINNED: the unversioned libnvinfer-dev now resolves to TensorRT 11 built + for CUDA 13, which is incompatible with the CUDA 12.6 toolkit. No GPU is needed to compile. +runs: + using: composite + steps: + - shell: bash + run: | + set -eux + wget -qO cuda-keyring.deb \ + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb" + sudo dpkg -i cuda-keyring.deb + sudo apt-get update + TRT=10.7.0.23-1+cuda12.6 + sudo apt-get install -y --no-install-recommends \ + cuda-cudart-dev-12-6 cuda-nvcc-12-6 \ + libnvinfer-dev="$TRT" libnvinfer-headers-dev="$TRT" libnvinfer10="$TRT" \ + libnvonnxparsers-dev="$TRT" libnvonnxparsers10="$TRT" + echo "/usr/local/cuda-12.6/bin" >> "$GITHUB_PATH" diff --git a/.github/consumer/CMakeLists.txt b/.github/consumer/CMakeLists.txt new file mode 100644 index 0000000..daf9df2 --- /dev/null +++ b/.github/consumer/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 3.22) +project(trtcpp_consumer LANGUAGES CXX) + +# Smoke test that an installed tensorrt_cpp_api is consumable via find_package: configure with +# -DCMAKE_PREFIX_PATH=. +find_package(tensorrt_cpp_api REQUIRED) + +add_executable(consumer main.cpp) +target_link_libraries(consumer PRIVATE tensorrt_cpp_api::tensorrt_cpp_api) diff --git a/.github/consumer/main.cpp b/.github/consumer/main.cpp new file mode 100644 index 0000000..55ded44 --- /dev/null +++ b/.github/consumer/main.cpp @@ -0,0 +1,8 @@ +#include + +#include + +int main() { + std::printf("consumer linked trtcpp %s\n", trtcpp::versionString().c_str()); + return trtcpp::libraryVersion().major == 7 ? 0 : 1; +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9fc91c1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,76 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Build + the non-GPU test suite. GitHub-hosted runners have no NVIDIA GPU, so we compile against + # the CUDA/TensorRT headers and run only `ctest -LE gpu`. The CUDA + TensorRT toolchain comes from + # the pinned composite action (.github/actions/setup-trt). The excluded GPU-labeled integration + # tests run via the manual-dispatch `gpu tests` workflow (.github/workflows/gpu-tests.yml). + build-cpu: + name: build + cpu tests (ubuntu-24.04) + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-trt + - name: Configure + run: | + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DTRT_CPP_API_BUILD_TESTS=ON \ + -DTRT_CPP_API_BUILD_PREPROC=ON \ + -DCMAKE_CUDA_COMPILER=/usr/local/cuda-12.6/bin/nvcc + - name: Build + run: cmake --build build -j2 + - name: CPU tests (exclude gpu-labeled) + run: ctest --test-dir build -LE gpu --output-on-failure + - name: Install smoke (find_package consumer) + run: | + cmake --install build --prefix "$PWD/_install" + cmake -S .github/consumer -B build/consumer -DCMAKE_PREFIX_PATH="$PWD/_install" + cmake --build build/consumer + ./build/consumer/consumer + + # AddressSanitizer + UBSan on the CPU test suite (preproc/CUDA off; the CPU tests don't need it). + sanitizers: + name: asan + ubsan (cpu tests) + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-trt + - name: Configure (sanitized) + run: | + cmake -S . -B build-san \ + -DCMAKE_BUILD_TYPE=Debug \ + -DTRT_CPP_API_BUILD_TESTS=ON \ + -DTRT_CPP_API_BUILD_PREPROC=OFF \ + -DCMAKE_CXX_FLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -fno-omit-frame-pointer" + - name: Build CPU tests + run: cmake --build build-san -j2 --target trtcpp_core_tests + - name: Run CPU tests under sanitizers + run: ./build-san/tests/trtcpp_core_tests + + # Python wheel build + import sanity (no GPU needed to import). + python-wheel: + name: python wheel + import + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: ./.github/actions/setup-trt + - name: Build + install the wheel + run: | + python -m pip install --upgrade pip + pip install . -v --config-settings=cmake.define.CMAKE_CUDA_COMPILER=/usr/local/cuda-12.6/bin/nvcc + - name: Import sanity + run: python -c "import trtcpp; print(trtcpp.version_string()); assert trtcpp.library_version()[0] == 7" diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml new file mode 100644 index 0000000..4055103 --- /dev/null +++ b/.github/workflows/format.yml @@ -0,0 +1,56 @@ +name: format + +on: + pull_request: + workflow_dispatch: + +jobs: + clang-format: + name: clang-format + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install clang-format (pinned to match .pre-commit-config.yaml) + run: pip install "clang-format==18.1.8" + - name: Check formatting (library + examples; excludes vendored stb) + run: | + files=$(git ls-files '*.h' '*.hpp' '*.cpp' '*.cu' \ + | grep -vE '(^|/)stb_image(_write)?\.h$') + echo "$files" + clang-format --dry-run --Werror $files + + clang-tidy: + name: clang-tidy (changed files) + runs-on: ubuntu-24.04 + if: ${{ github.event_name == 'pull_request' }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: ./.github/actions/setup-trt + - name: Install clang-tidy + run: sudo apt-get install -y clang-tidy-18 + - name: Configure (export compile_commands.json) + run: | + cmake -S . -B build \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DTRT_CPP_API_BUILD_TESTS=ON \ + -DCMAKE_CUDA_COMPILER=/usr/local/cuda-12.6/bin/nvcc + - name: clang-tidy on changed library sources + run: | + base="${{ github.event.pull_request.base.sha }}" + # Lint changed C++ translation units under src/ (the shipping core). clang-tidy is run + # on source files, never on headers or .cu: a header has no compile command of its own, + # so clang-tidy infers flags from the nearest unit -- which for our headers is the CUDA + # kernel (preproc.cu), making it parse them in CUDA mode and fail to find CUDA toolkit + # headers. Headers are still checked via HeaderFilterRegex when a linted .cpp includes + # them. --diff-filter=d drops files the PR deletes. + changed=$(git diff --name-only --diff-filter=d "$base"...HEAD \ + | grep -E '^src/.*\.(c|cc|cpp)$' \ + | grep -vE '\.cu$' || true) + if [ -z "$changed" ]; then echo "no library source changes"; exit 0; fi + echo "$changed" + clang-tidy-18 -p build $changed diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml new file mode 100644 index 0000000..b9f7fd1 --- /dev/null +++ b/.github/workflows/gpu-tests.yml @@ -0,0 +1,21 @@ +name: gpu tests + +# Manual-dispatch only. GitHub-hosted runners have no NVIDIA GPU, so the GPU-labeled CUDA / +# TensorRT / OpenCV / preproc integration tests are excluded from PR CI (ctest -LE gpu). Run this +# workflow from the Actions tab on a self-hosted runner labeled [self-hosted, gpu] to exercise the +# full suite. It is intentionally not attached to pull_request, so it adds no per-PR check. +on: + workflow_dispatch: + +jobs: + gpu-tests: + name: gpu tests (self-hosted) + runs-on: [self-hosted, gpu] + steps: + - uses: actions/checkout@v4 + - name: Configure + build + run: | + cmake -S . -B build -DTRT_CPP_API_BUILD_TESTS=ON + cmake --build build -j4 + - name: Full test suite + run: ctest --test-dir build --output-on-failure diff --git a/.gitignore b/.gitignore index 8a328d5..8e8026a 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,22 @@ cmake-build-* .vscode/ *.onnx +*.pt + +# Keep the tiny committed test fixtures (override the global *.onnx ignore) +!tests/models/*.onnx + +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +dist/ +wheelhouse/ + +# Generated API docs (doxygen Doxyfile) +docs/api/ + +# CTest working dir (created when running ctest from the repo root) +Testing/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 273d14b..f956278 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,23 @@ +# Run `pre-commit install` once; hooks then run on `git commit`. +# Vendored third-party single-header libs (stb) are excluded from formatting. +exclude: '(^|/)stb_image(_write)?\.h$' + repos: -- repo: https://github.com/pre-commit/mirrors-clang-format - rev: 'v17.0.3' # Use the sha / tag you want to point at - hooks: - - id: clang-format - types_or: [c++, c, cuda] + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: 'v18.1.8' # must match the pip clang-format pinned in .github/workflows/format.yml + hooks: + - id: clang-format + types_or: [c++, c, cuda] + + - repo: https://github.com/cheshirekow/cmake-format-precommit + rev: v0.6.13 + hooks: + - id: cmake-format + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: mixed-line-ending diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..b9b915f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,52 @@ +# AGENTS.md + +Orientation for AI coding agents (Claude Code, Codex, Cursor) working in this repo. Humans should +read [`README.md`](README.md); if the two ever disagree, README wins for humans and this file wins +for agents — keep them in lockstep. + +## What this is + +`tensorrt_cpp_api` is a C++ library that wraps NVIDIA TensorRT for CNN-class vision models: build a +TensorRT engine from ONNX, cache it on disk, and run inference. It has optional fused GPU +preprocessing, optional OpenCV interop, and optional zero-copy Python bindings. **Linux + NVIDIA +GPU only.** It is **not** an LLM/transformer serving framework, and Windows is out of scope. + +Targets TensorRT >= 10 (written to the TensorRT 11 surface, version-gated), CUDA 12, C++20. + +## Build & test + +```bash +cmake -S . -B build -DTRT_CPP_API_BUILD_TESTS=ON # add -DTensorRT_DIR= for a tarball TensorRT +cmake --build build -j +ctest --test-dir build -LE gpu # CPU-only tests; drop -LE gpu to run the full suite (needs a GPU) +``` + +`scripts/install_deps.sh` (one-time, needs sudo) and `scripts/verify_deps.sh` help set up a host. +If `nvcc` is not on `PATH`, pass `-DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc`. + +## Layout + +- `include/tensorrt_cpp_api/` — the public headers (one umbrella `all.h`). No `nvinfer1`, OpenCV, or + spdlog types appear here; the optional `preproc.h` / `opencv_interop.h` are separate. +- `src/` — implementation; `src/detail/` is internal (TensorRT glue, cache, buffers, execution). +- `python/` — the pybind11 `trtcpp` extension. `examples/` — reference programs. `tests/` — GoogleTest. + +## Conventions + +- **Formatting:** `.clang-format` (LLVM-based, 4-space, 140 col) and `.cmake-format.yaml`. Run + `pre-commit run --all-files` (or `clang-format -i`) before committing. CI enforces clang-format. +- **Comments:** default to none; comment only the non-obvious *why*, never narrate code. +- **No emojis** anywhere — source, comments, or commit messages. +- **Public API:** no-throw — every fallible call returns `Status` or `Result` (no exceptions); + name-keyed tensor IO (`unordered_map`); caller-provided CUDA streams; no + third-party types leaked through public headers (PImpl + version-gating). +- **Git:** never force-push, never amend a pushed commit, never skip hooks. + +## More + +- Usage & concepts: [`docs/quickstart.md`](docs/quickstart.md) +- Install options: [`docs/install.md`](docs/install.md) +- Migrating from v6: [`docs/upgrading_from_v6.md`](docs/upgrading_from_v6.md) + +Reference downstream consumers: [`YOLOv8-TensorRT-CPP`](https://github.com/cyrusbehr/YOLOv8-TensorRT-CPP) +and [`YOLOv9-TensorRT-CPP`](https://github.com/cyrusbehr/YOLOv9-TensorRT-CPP). diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..0a332ff --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,79 @@ +# Changelog + +## v7.0.0-rc1 (2026-05-29) + +First release candidate of the v7 rewrite — a ground-up reimplementation into a reusable, +installable C++ library. **This is a clean break from v6 — the API is not source-compatible.** +See [docs/upgrading_from_v6.md](docs/upgrading_from_v6.md). + +### Added +- No-throw API: every fallible call returns `Status` or `Result` (no exceptions); `TRTCPP_TRY` + sugar for propagation. +- Name-keyed IO (`unordered_map`) at the inference boundary; non-templated + `Tensor`/`TensorView` with a runtime `DType` (replaces v6's `Engine` and nested vectors). +- Caller-owned CUDA streams (`Stream`, including `Stream::wrap` for an external handle); explicit + host/device transfers (`Tensor::toHost`/`to`/`copyFrom`) — never an implicit D2H copy. +- Safe engine cache: `EngineBuilder::buildOrLoad`/`buildAndLoad` keyed by ONNX content hash + + build options + TensorRT version + GPU UUID, JSON sidecar, atomic write, stale-cache detection. +- Dynamic shapes via per-input min/opt/max optimization profiles and a `-1`-aware `Shape`. +- `EnginePool` for concurrent multi-stream inference (one optimization profile per context). +- Quantization that is version-aware and never a silent no-op (`Precision::kFp16`/`kInt8Qdq`/ + `kFp8`/…); legacy INT8 calibration is available only when built against TensorRT < 11. +- Optional fused preprocessing sublibrary (`tensorrt_cpp_api::preproc`): a single CUDA kernel for + letterbox-resize → BGR↔RGB → per-channel normalize → HWC→NCHW → cast. +- Optional OpenCV interop header (zero-copy views over `cv::Mat` / `cv::cuda::GpuMat`), strictly + opt-in behind `-DTRT_CPP_API_WITH_OPENCV`. +- Optional Python bindings (`trtcpp`, pybind11 + scikit-build-core) with zero-copy + `__cuda_array_interface__`/DLPack interop, caller streams, and GIL release during inference. +- CMake install/export: `find_package(tensorrt_cpp_api)` with a relocatable package config and a + bundled `FindTensorRT` module (apt or tarball). +- Reference examples (classification, detection, segmentation, zero-copy Python) and GitHub Actions + CI (build + CPU tests, sanitizers, Python wheel, lint). + +### Changed +- Minimum TensorRT is 10.0; the code is written to the TensorRT 11 surface and version-gates the + removed-in-11 features (legacy calibrators, weak typing, `IPluginV2`). +- C++20 (was C++17); namespace `trtcpp`; include root `tensorrt_cpp_api/`; CMake target + `tensorrt_cpp_api::tensorrt_cpp_api`. + +### Removed +- The v6 API in its entirety — no source-compatibility shim (see the upgrade guide): + - the monolithic templated `Engine` and its `.inl`-in-header implementation; + - the single `Options` struct, replaced by `BuildOptions` + `EngineOptions`; + - OpenCV `cv::cuda::GpuMat` in the inference signatures and the triply-nested `std::vector` + inputs/outputs, replaced by name-keyed `TensorView`/`Tensor`; + - `bool`/exception error handling, replaced by `Status`/`Result`; + - per-call stream creation, replaced by caller-owned `Stream`; + - the `run_inference_benchmark` CLI executable and the v6 `include/` header layout. + +--- + +## v6.0 +- Implementation now requires TensorRT >= 10.0. + +## v5.0 +- `Engine` became a class template parameterized on the model's output data type (`float`, + `__half`, `int8_t`, `int32_t`, `bool`, `uint8_t`). +- Added loading a TensorRT engine file directly (without compiling from ONNX). +- Added a command-line parser. + +## v4.1 +- Support for fixed batch size > 1. + +## v4.0 +- Added INT8 precision support. + +## v3.0 +- Updated to the TensorRT 8.6 API (`IExecutionContext::enqueueV3()`). +- Benchmark executable renamed to `run_inference_benchmark`; takes the ONNX path as an argument. +- Auto-detect supported batch sizes; stop limiting workspace memory. + +## v2.2 +- Serialize the model name as part of the engine file. + +## v2.1 +- Support for models with multiple inputs. + +## v2.0 +- Requires OpenCV with CUDA. +- Support for models with more than one output and for non-batchable models; more error checking. diff --git a/CMakeLists.txt b/CMakeLists.txt index 3377253..35d24bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,34 +1,198 @@ -cmake_minimum_required(VERSION 3.18) -project(tensorrt_cpp_api) +cmake_minimum_required(VERSION 3.22) +project(tensorrt_cpp_api VERSION 7.0.0 LANGUAGES CXX) -# Use ccache to speed up rebuilds -include(cmake/ccache.cmake) +# The engine core depends only on CUDA + TensorRT. The preprocessing sublibrary, OpenCV interop, +# Python bindings, tests, and examples are independent, opt-in components; all default OFF except +# the preprocessing sublibrary. -# Set C++ version and optimization level -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Ofast -DNDEBUG -Wno-deprecated-declarations") +option(TRT_CPP_API_BUILD_TESTS "Build unit/integration tests" OFF) +option(TRT_CPP_API_BUILD_EXAMPLES "Build reference examples" OFF) +option(TRT_CPP_API_BUILD_PREPROC "Build the fused preprocessing sublibrary" ON) +option(TRT_CPP_API_BUILD_PYTHON "Build the pybind11 Python bindings" OFF) +option(TRT_CPP_API_WITH_OPENCV "Build the optional OpenCV interop" OFF) +option(TRT_CPP_API_WITH_SPDLOG "Build the optional spdlog logger adapter" OFF) -# For finding FindTensorRT.cmake -set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") -# TODO: Specify the path to TensorRT root dir -if (NOT TensorRT_DIR) - set(TensorRT_DIR /home/cyrus/work/libs/TensorRT-10.0.0.6/) +# Enable CTest at the top level so tests registered in subdirectories (tests/, python/) are +# discoverable. Called early -- before add_subdirectory(python) registers the binding test. +if(TRT_CPP_API_BUILD_TESTS) + enable_testing() endif() -# Use the correct version of CUDA -set(CUDA_TOOLKIT_ROOT_DIR /usr/local/cuda) -# We require CUDA, OpenCV, and TensorRT +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) +endif() + +# Modern ccache hook (replaces v6's generator-specific RULE_LAUNCH_COMPILE). +find_program(CCACHE_PROGRAM ccache) +if(CCACHE_PROGRAM) + set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}") +endif() + +# CUDA runtime: a hard dependency of the engine core (streams, allocation, Tensor cross +# the public API). Plain C++ against the CUDA runtime API -- nvcc is only needed later for +# the preprocessing kernels (.cu), at which point CUDA is enabled as a language. +find_package(CUDAToolkit REQUIRED) + +# TensorRT (apt or tarball). Linked PRIVATE: nvinfer1 types never appear in public headers +# (PImpl/forward-decl), so consumers need TRT at runtime but not at compile time. Pass +# -DTensorRT_DIR= for a tarball install. find_package(TensorRT REQUIRED) -find_package(CUDA REQUIRED) -find_package(OpenCV REQUIRED) -find_package(fmt REQUIRED) -add_library(tensorrt_cpp_api SHARED - src/engine.cpp) +# Threads: the logger and EnginePool use std::mutex/condition_variable. Linked PRIVATE +# (PImpl, so no thread types in public headers); the package config re-resolves it. +find_package(Threads REQUIRED) + +# Generate build_config.h recording the library + TensorRT versions, so public headers can +# version-gate (e.g. the legacy calibrator) without including a TensorRT header. +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/build_config.h.in ${CMAKE_CURRENT_BINARY_DIR}/generated/tensorrt_cpp_api/build_config.h + @ONLY +) + +add_library(tensorrt_cpp_api) +add_library(tensorrt_cpp_api::tensorrt_cpp_api ALIAS tensorrt_cpp_api) +target_sources( + tensorrt_cpp_api + PRIVATE src/core.cpp + src/logger.cpp + src/cuda.cpp + src/allocator.cpp + src/device_tensor.cpp + src/engine_builder.cpp + src/engine.cpp + src/engine_pool.cpp + src/calibrator.cpp + src/detail/buffers.cpp + src/detail/calibrator_bridge.cpp + src/detail/engine_cache.cpp + src/detail/execution.cpp + src/detail/sha256.cpp + src/detail/trt_common.cpp +) +target_include_directories(tensorrt_cpp_api PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) +target_link_libraries(tensorrt_cpp_api PUBLIC CUDA::cudart PRIVATE TensorRT::TensorRT Threads::Threads) +# Build-tree include roots only. The install-tree include dir is supplied once by the +# INCLUDES DESTINATION of install(TARGETS) below (both the public headers and the generated +# build_config.h land under the same /include tree), so no $ here. +target_include_directories( + tensorrt_cpp_api PUBLIC $ + $ +) +target_compile_features(tensorrt_cpp_api PUBLIC cxx_std_20) +# POSITION_INDEPENDENT_CODE: the static core is routinely linked into shared objects -- the Python +# extension and downstream shared libraries such as the YOLO sibling repos' libYoloV8_TRT.so -- so +# it must be PIC. (Negligible cost; standard for an embeddable static library.) +set_target_properties( + tensorrt_cpp_api PROPERTIES CXX_EXTENSIONS OFF POSITION_INDEPENDENT_CODE ON VERSION ${PROJECT_VERSION} + SOVERSION ${PROJECT_VERSION_MAJOR} +) +if(NOT MSVC) + target_compile_options(tensorrt_cpp_api PRIVATE -Wall -Wextra) +endif() + +# Optional spdlog logger adapter. PUBLIC so makeSpdlogLogger() is declared and +# spdlog is linked for consumers of a spdlog-enabled install. +if(TRT_CPP_API_WITH_SPDLOG) + find_package(spdlog REQUIRED) + target_link_libraries(tensorrt_cpp_api PUBLIC spdlog::spdlog) + target_compile_definitions(tensorrt_cpp_api PUBLIC TRT_CPP_API_WITH_SPDLOG) +endif() + +# Optional OpenCV interop. Requests only the `core` module (which carries cv::Mat +# and cv::cuda::GpuMat) to avoid pulling opencv_dnn and its cuDNN dependency. Strictly +# opt-in; the engine core has no OpenCV in any non-gated header. +if(TRT_CPP_API_WITH_OPENCV) + find_package(OpenCV REQUIRED COMPONENTS core) + target_sources(tensorrt_cpp_api PRIVATE src/opencv_interop.cpp) + target_link_libraries(tensorrt_cpp_api PUBLIC ${OpenCV_LIBS}) + target_include_directories(tensorrt_cpp_api PUBLIC ${OpenCV_INCLUDE_DIRS}) + target_compile_definitions(tensorrt_cpp_api PUBLIC TRT_CPP_API_WITH_OPENCV) +endif() + +# Optional preprocessing sublibrary: one fused CUDA (.cu) kernel. Separate target -- +# the engine core does NOT depend on it. Enabling it turns on the CUDA language (needs nvcc). +if(TRT_CPP_API_BUILD_PREPROC) + # Architectures must be set BEFORE enable_language(CUDA): CMake validates the variable + # during language init, and an unset value on hosts where nvcc can't self-detect a default + # aborts configuration. + if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + set(CMAKE_CUDA_ARCHITECTURES 75-real 80-real 86-real 89-real 90-virtual) + endif() + enable_language(CUDA) + add_library(tensorrt_cpp_api_preproc src/preproc.cu) + add_library(tensorrt_cpp_api::preproc ALIAS tensorrt_cpp_api_preproc) + target_link_libraries(tensorrt_cpp_api_preproc PUBLIC tensorrt_cpp_api::tensorrt_cpp_api CUDA::cudart) + target_compile_features(tensorrt_cpp_api_preproc PUBLIC cxx_std_20) + set_target_properties( + tensorrt_cpp_api_preproc + PROPERTIES CUDA_STANDARD 20 + CUDA_STANDARD_REQUIRED ON + CXX_EXTENSIONS OFF + POSITION_INDEPENDENT_CODE ON # embeddable into shared libs (Python ext, sibling .so) + EXPORT_NAME preproc # consumers link tensorrt_cpp_api::preproc (matches the in-tree alias) + VERSION ${PROJECT_VERSION} + SOVERSION ${PROJECT_VERSION_MAJOR} + ) +endif() + +# Python bindings. Added after the core/preproc targets exist so the extension can link +# them. The subdirectory builds trtcpp._core. +if(TRT_CPP_API_BUILD_PYTHON) + add_subdirectory(python) +endif() + +# Reference examples. Normally built standalone against the installed package; this in-tree +# path is a convenience that reuses the just-built targets. +if(TRT_CPP_API_BUILD_EXAMPLES) + add_subdirectory(examples) +endif() + +# ---- install / export (find_package(tensorrt_cpp_api) consumable) ---- +# Skipped under scikit-build-core (SKBUILD): a Python wheel must contain only the Python package +# and the compiled extension, not the C++ dev headers / static libs / CMake package config. +if(NOT SKBUILD) + include(GNUInstallDirs) + include(CMakePackageConfigHelpers) + set(TRTCPP_INSTALL_CMAKEDIR ${CMAKE_INSTALL_LIBDIR}/cmake/tensorrt_cpp_api) + + set(TRTCPP_INSTALL_TARGETS tensorrt_cpp_api) + if(TARGET tensorrt_cpp_api_preproc) + list(APPEND TRTCPP_INSTALL_TARGETS tensorrt_cpp_api_preproc) + endif() + + install( + TARGETS ${TRTCPP_INSTALL_TARGETS} + EXPORT tensorrt_cpp_apiTargets + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + INCLUDES + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + ) -target_include_directories(tensorrt_cpp_api PUBLIC ${OpenCV_INCLUDE_DIRS} ${CUDA_INCLUDE_DIRS} ${TensorRT_INCLUDE_DIRS} include include/interfaces) -target_link_libraries(tensorrt_cpp_api PUBLIC ${OpenCV_LIBS} ${CUDA_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${TensorRT_LIBRARIES} fmt::fmt) + install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/generated/tensorrt_cpp_api/build_config.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/tensorrt_cpp_api + ) + install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindTensorRT.cmake DESTINATION ${TRTCPP_INSTALL_CMAKEDIR}) -add_executable(run_inference_benchmark src/main.cpp) -target_link_libraries(run_inference_benchmark tensorrt_cpp_api fmt::fmt) \ No newline at end of file + install(EXPORT tensorrt_cpp_apiTargets NAMESPACE tensorrt_cpp_api:: DESTINATION ${TRTCPP_INSTALL_CMAKEDIR}) + + configure_package_config_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/tensorrt_cpp_api-config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/tensorrt_cpp_api-config.cmake INSTALL_DESTINATION ${TRTCPP_INSTALL_CMAKEDIR} + ) + write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/tensorrt_cpp_api-config-version.cmake VERSION ${PROJECT_VERSION} + COMPATIBILITY SameMajorVersion + ) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/tensorrt_cpp_api-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/tensorrt_cpp_api-config-version.cmake DESTINATION ${TRTCPP_INSTALL_CMAKEDIR} + ) +endif() # NOT SKBUILD + +if(TRT_CPP_API_BUILD_TESTS) + add_subdirectory(tests) +endif() diff --git a/Doxyfile b/Doxyfile new file mode 100644 index 0000000..836bbe9 --- /dev/null +++ b/Doxyfile @@ -0,0 +1,36 @@ +# Doxygen config for the tensorrt_cpp_api public API reference. +# doxygen Doxyfile # -> HTML in docs/api/html/index.html +# Only the public headers are documented; src/ (impl + detail) and third-party are excluded. + +PROJECT_NAME = "tensorrt_cpp_api" +PROJECT_NUMBER = 7.0.0 +PROJECT_BRIEF = "A no-throw C++ TensorRT inference library for CNN models" +OUTPUT_DIRECTORY = docs/api + +INPUT = include README.md docs/quickstart.md +USE_MDFILE_AS_MAINPAGE = README.md +RECURSIVE = YES +FILE_PATTERNS = *.h *.hpp +EXCLUDE_PATTERNS = */build_config.h # generated at build time + +# The headers use /// and /** */ comments; treat the first sentence as the brief. +JAVADOC_AUTOBRIEF = YES +QT_AUTOBRIEF = YES +EXTRACT_ALL = YES +EXTRACT_STATIC = YES +HIDE_UNDOC_MEMBERS = NO +SHOW_NAMESPACES = YES + +# C++20; let Doxygen see the project's own macros without a full build. +ENABLE_PREPROCESSING = YES +MACRO_EXPANSION = YES +EXPAND_ONLY_PREDEF = NO +PREDEFINED = TRT_CPP_API_WITH_OPENCV=1 \ + TRT_CPP_API_WITH_SPDLOG=1 + +GENERATE_HTML = YES +GENERATE_LATEX = NO +GENERATE_TREEVIEW = YES +SORT_MEMBER_DOCS = NO +QUIET = YES +WARN_IF_UNDOCUMENTED = NO diff --git a/README.md b/README.md index 741190e..83583f6 100644 --- a/README.md +++ b/README.md @@ -1,178 +1,146 @@ [![Stargazers][stars-shield]][stars-url] - -[![All Contributors](https://img.shields.io/badge/all_contributors-3-orange.svg?style=flat-square)](#contributors-) - [![Issues][issues-shield]][issues-url] [![LinkedIn][linkedin-shield]][linkedin-url] -

logo -

TensorRT C++ API Tutorial

+

TensorRT C++ API

- - How to use TensorRT C++ API for high performance GPU machine-learning inference. - -
- Supports models with single / multiple inputs and single / multiple outputs with batching. -
-
- Project Overview Video - . - Code Deep-Dive Video + A modern, no-throw C++ library for high-performance GPU inference of CNN models with NVIDIA TensorRT — with optional zero-copy Python bindings.

-## Looking for Maintainers 🚀 - -This project is actively seeking maintainers to help guide its growth and improvement. If you're passionate about this project and interested in contributing, I’d love to hear from you! - -Please feel free to reach out via [LinkedIn](https://www.linkedin.com/in/cyrus-behroozi/) to discuss how you can get involved. - - -# TensorRT C++ Tutorial -*I read all the NVIDIA TensorRT docs so that you don't have to!* - -This project demonstrates how to use the TensorRT C++ API for high performance GPU inference on image data. It covers how to do the following: -- How to install TensorRT 10 on Ubuntu 20.04 / 22.04. -- How to generate a TensorRT engine file optimized for your GPU. -- How to specify a simple optimization profile. -- How to run FP32, FP16, or INT8 precision inference. -- How to read / write data from / into GPU memory and work with GPU images. -- How to use cuda stream to run async inference and later synchronize. -- How to work with models with static and dynamic batch sizes. -- How to work with models with single or multiple output tensors. -- How to work with models with multiple inputs. -- Includes a [Video walkthrough](https://youtu.be/Z0n5aLmcRHQ) where I explain every line of code. -- The code can be used as a base for any model which takes a fixed size image / images as input, including [Insightface](https://github.com/deepinsight/insightface) [ArcFace](https://github.com/onnx/models/tree/main/vision/body_analysis/arcface), [YoloV8](https://github.com/ultralytics/ultralytics), [SCRFD](https://insightface.ai/scrfd) face detection. - - You will just need to implement the appropriate post-processing code. -- TODO: Add support for models with dynamic input shapes. -- TODO: Add support for Windows - -## Getting Started -The following instructions assume you are using Ubuntu 20.04 or 22.04. -You will need to supply your own onnx model for this sample code or you can download the sample model (see Sanity Check section below). - -### Prerequisites -- Tested and working on Ubuntu 20.04 and 22.04 (Windows is **not** supported at this time) -- Install CUDA 11 or 12, instructions [here](https://developer.nvidia.com/cuda-downloads). - - Recommended >= 12.0 - - Required >= 11.0 -- Install cudnn, instructions [here](https://docs.nvidia.com/deeplearning/cudnn/install-guide/index.html#download). - - Required >= 8 - - Required < 9 (OpenCV GPU does not yet support) -- `sudo apt install build-essential` -- `sudo snap install cmake --classic` -- `sudo apt install libspdlog-dev libfmt-dev` (for logging) -- Install OpenCV with cuda support. To compile OpenCV from source, run the `build_opencv.sh` script provided in `./scripts/`. - - If you use the provided script and you have installed cuDNN to a non-standard location, you must modify the `CUDNN_INCLUDE_DIR` and `CUDNN_LIBRARY` variables in the script. - - Recommended >= 4.8 -- Download TensorRT 10 from [here](https://developer.nvidia.com/tensorrt/download/10x). - - Required >= 10.0 -- Navigate to the `CMakeLists.txt` file and replace the `TODO` with the path to your TensorRT installation. - -### Building the Library -- `mkdir build` -- `cd build` -- `cmake ..` -- `make -j$(nproc)` - -### Running the Executable -- Navigate to the build directory -- Run the executable and provide the path to your onnx model. -- ex. `./run_inference_benchmark --onnx_model ../models/yolov8n.onnx` - - Note: See sanity check section below for instructions on how to obtain the yolov8n model. -- The first time you run the executable for a given model and options, a TensorRT engine file will be built from your onnx model. This process is fairly slow and can take 5+ minutes for some models (ex. yolo models). -- Alternatively, you can choose to supply your own TensorRT engine file directly: -- ex. `./run_inference_benchmark --trt_model ../models/yolov8n.engine.NVIDIAGeForceRTX3080LaptopGPU.fp16.1.1` - - Note: See V5.0 changelog below for warnings when supply your own TensorRT engine file. - -### Sanity Check -- To perform a sanity check, download the `YOLOv8n` model from [here](https://github.com/ultralytics/ultralytics#models). -- Next, convert it from pytorch to onnx using the following script: - - You will need to run `pip3 install ultralytics` first. - -```python -from ultralytics import YOLO -model = YOLO("./yolov8n.pt") -model.fuse() -model.info(verbose=False) # Print model information -model.export(format="onnx", opset=12) # Export the model to onnx using opset 12 +--- + +`tensorrt_cpp_api` turns an ONNX model into a cached, optimized TensorRT engine and runs it with +a small, leak-free API: name-keyed tensors at the boundary, caller-owned CUDA streams, explicit +host/device transfers, and a `Status`/`Result` error model — no exceptions, no OpenCV or +TensorRT types in the public headers. It targets **TensorRT ≥ 10** (built to the TensorRT 11 +surface), CUDA 12, C++20, Linux. + +```cpp +#include +using namespace trtcpp; + +int main() { + // Build an FP16 engine from ONNX, or load it from the on-disk cache if one is already current. + BuildOptions opt; + opt.precision = Precision::kFp16; + opt.engineCacheDir = "engines"; + auto engine = EngineBuilder{}.buildAndLoad("model.onnx", opt); + if (!engine) { + std::fprintf(stderr, "%s\n", engine.status().message().c_str()); + return 1; + } + + Stream stream; // owns a CUDA stream — or Stream::wrap(existingHandle) to use yours + auto input = Tensor::allocate(DType::kFloat32, Shape{1, 3, 640, 640}, Device::kCuda).value(); + // ... fill `input` (e.g. via the fused preproc kernel) ... + + auto output = engine->inferSingle({{engine->inputNames().front(), input.view()}}, stream); + if (!output) return 1; + + auto host = output->toHost(stream).value(); // explicit D2H + sync; never implicit + std::span scores = host.as().value(); + // ... post-process `scores` ... +} ``` -- Place the resulting onnx model, `yolov8n.onnx`, in the `./models/` directory. -- Running inference using said model and the image located in `./inputs/team.jpg` should produce the following feature vector: - - Note: The feature vector will not be identical (but very similar) as [TensorRT is not deterministic](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#determinism). -```text -3.41113 16.5312 20.8828 29.8984 43.7266 54.9609 62.0625 65.8594 70.0312 72.9531 ... -``` +## Features + +- **Engine cache that's actually safe.** Build-or-load keyed by ONNX content hash + build options + + TensorRT version + GPU UUID, with a JSON sidecar and atomic writes. A stale cache (changed + model, options, driver, or GPU) is detected and rebuilt instead of silently misused. +- **Dynamic shapes, done right.** Per-input min/opt/max optimization profiles; `-1`-aware `Shape`; + one optimization profile per execution context for concurrent dynamic-shape inference. +- **Concurrency.** `EnginePool` leases execution contexts for multi-stream inference; every call + runs on a caller-provided `Stream`. The engine is thread-compatible; the pool is thread-safe. +- **No leaky abstractions.** No `nvinfer1`, OpenCV, or spdlog types in any public header (PImpl + + version-gated, generated `build_config.h`). Consumers need TensorRT at runtime, not compile time. +- **Quantization without surprises.** `Precision::kFp16` / `kInt8Qdq` / `kFp8` …; precision is + version-aware and never a silent no-op (it errors clearly when a mode isn't achievable). +- **Optional fused preprocessing** (`tensorrt_cpp_api::preproc`): one CUDA kernel does + letterbox-resize → BGR↔RGB → per-channel normalize → HWC→NCHW → cast, no intermediate buffers. +- **Optional zero-copy Python bindings** (`trtcpp`): feed CuPy / PyTorch / Numba GPU arrays in and + get them back via `__cuda_array_interface__` / DLPack — no host round-trips, GIL released during + inference. See [`examples/python`](examples/python). +- **Installable.** `cmake --install` produces a `find_package(tensorrt_cpp_api)`-consumable package. + +## Performance + +Single-stream inference latency on an **RTX 3080 Laptop GPU** (preallocated, zero-copy `enqueue` +loop — `examples/benchmark`), TensorRT 10: + +| Model | Precision | Latency | Throughput | +|---|---|---|---| +| YOLOv8n | FP16 | 1.07 ms | 937 inf/s | +| YOLOv8n | FP32 | 2.00 ms | 499 inf/s | +| MobileNetV2 | FP16 | 0.31 ms | 3199 inf/s | + +Inference time is TensorRT-bound — it is the `enqueueV3` cost of the engine, so the wrapper adds +**no** inference overhead (v6 and v7 run the identical engine on identical hardware in the same +time). v7's gains are on the host side and in safety: zero-copy name-keyed IO with no per-call +allocations or nested-vector copies, a stream-ordered allocator, and the no-throw `Status`/`Result` +API. The Python bindings run the same path within ~13% of C++ (`examples/python/benchmark_parity.py`). + +> For reference, v6's published figures (a weaker RTX 3050 Ti Laptop GPU) were YOLOv8n FP16 +> 2.49 ms / FP32 4.73 ms; the headline difference above is the GPU, not the wrapper. + +## Install + +TensorRT and CUDA are system/externally provided. In brief: -### INT8 Inference -Enabling INT8 precision can further speed up inference at the cost of accuracy reduction due to reduced dynamic range. -For INT8 precision, the user must supply calibration data which is representative of real data the model will see. -It is advised to use 1K+ calibration images. To enable INT8 inference with the YoloV8 sanity check model, the following steps must be taken: -- Change `options.precision = Precision::FP16;` to `options.precision = Precision::INT8;` in `main.cpp` -- `options.calibrationDataDirectoryPath = "";` must be changed in `main.cpp` to specify path containing calibration data. - - If using the YoloV8 model, it is advised to used the COCO validation dataset, which can be downloaded with `wget http://images.cocodataset.org/zips/val2017.zip` -- Make sure the resizing code in the `Int8EntropyCalibrator2::getBatch` method in `engine.cpp` (see `TODO`) is correct for your model. - - If using the YoloV8 model, the preprocessing code is correct and does not need to be changed. -- Recompile, run the executable. -- The calibration cache will be written to disk (`.calibration` extension) so that on subsequent model optimizations it can be reused. If you'd like to regenerate the calibration data, you must delete this cache file. -- If you get an "out of memory in function allocate" error, then you must reduce `Options.calibrationBatchSize` so that the entire batch can fit in your GPU memory. - -### Benchmarks -Benchmarks run on RTX 3050 Ti Laptop GPU, 11th Gen Intel(R) Core(TM) i9-11900H @ 2.50GHz. - -| Model | Precision | Batch Size | Avg Inference Time | -|---------|-----------|------------|--------------------| -| yolov8n | FP32 | 1 | 4.732 ms | -| yolov8n | FP16 | 1 | 2.493 ms | -| yolov8n | INT8 | 1 | 2.009 ms | -| yolov8x | FP32 | 1 | 76.63 ms | -| yolov8x | FP16 | 1 | 25.08 ms | -| yolov8x | INT8 | 1 | 11.62 ms | - -### Sample Integration -Wondering how to integrate this library into your project? Or perhaps how to read the outputs of the YoloV8 model to extract meaningful information? -If so, check out my two latest projects, [YOLOv8-TensorRT-CPP](https://github.com/cyrusbehr/YOLOv8-TensorRT-CPP) and [YOLOv9-TensorRT-CPP](https://github.com/cyrusbehr/YOLOv9-TensorRT-CPP), which demonstrate how to use the TensorRT C++ API to run YoloV8/9 inference (supports object detection, semantic segmentation, and body pose estimation). They make use of this project in the backend! - -### Project Structure ```sh -project-root/ -├── include/ -│ ├── engine/ -│ │ ├── EngineRunInference.inl -│ │ ├── EngineUtilities.inl -│ │ └── EngineBuildLoadNetwork.inl -│ ├── util/... -│ ├── ... -├── src/ -| ├── ... -│ ├── engine.cpp -│ ├── engine.h -│ └── main.cpp -├── CMakeLists.txt -└── README.md +cmake -S . -B build -DTRT_CPP_API_BUILD_PREPROC=ON # add -DTensorRT_DIR= for a tarball +cmake --build build -j$(nproc) +cmake --install build --prefix /opt/trtcpp ``` -### Understanding the Code -- The bulk of the implementation is located in `include/engine`. I have written lots of comments all throughout the code which should make it easy to understand what is going on. -- The inference code is located in `include/engine/EngineRunInference.inl`. -- The building and loading of the TensorRT engine file is located in `include/engine/EngineBuildLoadNetwork.inl`. -- You can also check out my [deep-dive video](https://youtu.be/Z0n5aLmcRHQ) in which I explain every line of code. +Then in a downstream project: + +```cmake +find_package(tensorrt_cpp_api REQUIRED) +target_link_libraries(myapp PRIVATE tensorrt_cpp_api::tensorrt_cpp_api tensorrt_cpp_api::preproc) +``` + +Python: `pip install .` (builds the `trtcpp` wheel via scikit-build-core). Full details — +apt vs tarball TensorRT, build options, Python — are in [`docs/install.md`](docs/install.md). + +## Examples + +[`examples/`](examples) has four runnable reference programs, each consuming the installed package: +**classification** (ImageNet top-5), **detection** (YOLOv8n + NMS), **segmentation** (DeepLabV3), +and a **zero-copy Python** demo with a C++/Python perf-parity benchmark. `examples/download_models.sh` +fetches the models. + +## Documentation + +- [Quickstart & core concepts](docs/quickstart.md) +- [Installation](docs/install.md) +- [Upgrading from v6](docs/upgrading_from_v6.md) +- API reference: `doxygen Doxyfile` (HTML in `docs/api/`) + +## Sister projects + +This library is the inference backend for [YOLOv8-TensorRT-CPP](https://github.com/cyrusbehr/YOLOv8-TensorRT-CPP) +and [YOLOv9-TensorRT-CPP](https://github.com/cyrusbehr/YOLOv9-TensorRT-CPP) (object detection, +segmentation, pose). -### How to Debug -- The implementation uses the `spdlog` library for logging. You can change the log level by setting the environment variable `LOG_LEVEL` to one of the following values: `trace`, `debug`, `info`, `warn`, `error`, `critical`, `off`. +## Scope -- If you have issues creating the TensorRT engine file from the onnx model, consider setting the environment variable `LOG_LEVEL` to `trace` and re-run the application. This should give you more information on where exactly the build process is failing. +Linux, CUDA 12, TensorRT ≥ 10, CNN-style vision models. Windows and LLM/transformer-specific +features are out of scope. -### Show your Appreciation -If this project was helpful to you, I would appreciate if you could give it a star. That will encourage me to ensure it's up to date and solve issues quickly. I also do consulting work if you require more specific help. Connect with me on [LinkedIn](https://www.linkedin.com/in/cyrus-behroozi/). +## Contributing + +Issues and PRs welcome. Install the hooks with `pre-commit install` (clang-format + cmake-format); +CI runs the build, the CPU test suite, sanitizers, and a Python wheel build. If this project helps +you, a ⭐ is appreciated — connect on [LinkedIn](https://www.linkedin.com/in/cyrus-behroozi/). ### Contributors @@ -188,76 +156,20 @@ If this project was helpful to you, I would appreciate if you could give it a st - - -### Changelog - -**V6.0** - -- Implementation now requires TensorRT >= 10.0. - -**V5.0** - -- `Engine` class has been modified to take a template parameter which specifies the models output data type. The implementation now supports outputs of type `float`, `__half`, `int8_t`, `int32_t`, `bool`, and `uint8_t`. -- Added support for loading TensorRT engine file directly without needing to compile from onnx model. Howver, it is highly recommended that you use the API provided to build the engine file from the onnx model, instead of loading a TensorRT model directly. If you choose to load a TensorRT model file directly, you must hand-check that the `Options` have been set correctly for your model (for example, if your model has been compiled for FP32 but you try running FP16 inference, it will fail, potentially without a verbose error). -- Added command line parser. - -**V4.1** - -- Added support for fixed batch size > 1. +This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. -**V4.0** +## License -- Added support for INT8 precision. - - -**V3.0** - -- Implementation has been updated to use TensorRT 8.6 API (ex. `IExecutionContext::enqueueV3()`). -- Executable has renamed from `driver` to `run_inference_benchmark` and now must be passed path to onnx model as command line argument. -- Removed `Options.doesSupportDynamicBatchSize`. Implementation now auto-detects supported batch sizes. -- Removed `Options.maxWorkspaceSize`. Implementation now does not limit GPU memory during model constructions, allowing implementation to use as much of memory pool as is available for intermediate layers. - -**v2.2** - -- Serialize model name as part of engine file. - -**V2.1** - -- Added support for models with multiple inputs. Implementation now supports models with single inputs, multiple inputs, single outputs, multiple outputs, and batching. - -**V2.0** - -- Requires OpenCV cuda to be installed. To install, follow instructions [here](https://gist.github.com/raulqf/f42c718a658cddc16f9df07ecc627be7). -- `Options.optBatchSizes` has been removed, replaced by `Options.optBatchSize`. -- Support models with more than a single output (ex. SCRFD). -- Added support for models which do not support batch inference (first input dimension is fixed). -- More error checking. -- Fixed a bunch of common issues people were running into with the original V1.0 version. -- Remove whitespace from GPU device name +See [LICENSE](LICENSE). Version history is in [CHANGELOG.md](CHANGELOG.md). - [stars-shield]: https://img.shields.io/github/stars/cyrusbehr/tensorrt-cpp-api.svg?style=flat-square [stars-url]: https://github.com/cyrusbehr/tensorrt-cpp-api/stargazers [issues-shield]: https://img.shields.io/github/issues/cyrusbehr/tensorrt-cpp-api.svg?style=flat-square [issues-url]: https://github.com/cyrusbehr/tensorrt-cpp-api/issues [linkedin-shield]: https://img.shields.io/badge/-LinkedIn-black.svg?style=flat-square&logo=linkedin&colorB=555 [linkedin-url]: https://linkedin.com/in/cyrus-behroozi/ - -## Contributors ✨ - -Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): - - - - - - - - -This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! diff --git a/cmake/FindTensorRT.cmake b/cmake/FindTensorRT.cmake index adfa71a..f89ce2f 100644 --- a/cmake/FindTensorRT.cmake +++ b/cmake/FindTensorRT.cmake @@ -1,81 +1,72 @@ -# source: -# https://github.com/NVIDIA/tensorrt-laboratory/blob/master/cmake/FindTensorRT.cmake - -# This module defines the following variables: -# -# :: -# -# TensorRT_INCLUDE_DIRS -# TensorRT_LIBRARIES -# TensorRT_FOUND -# -# :: +# FindTensorRT.cmake -- locate a TensorRT install (NVIDIA apt repo OR tarball) and expose +# the imported target TensorRT::TensorRT (nvinfer + nvonnxparser + headers). Supports +# TensorRT 10.0 through 11.x and errors clearly otherwise. Relocatable: it bakes no +# build-tree paths, so it can be installed alongside the package config (Phase E14/H). # -# TensorRT_VERSION_STRING - version (x.y.z) -# TensorRT_VERSION_MAJOR - major version (x) -# TensorRT_VERSION_MINOR - minor version (y) -# TensorRT_VERSION_PATCH - patch version (z) -# -# Hints -# ^^^^^ -# A user may set ``TensorRT_DIR`` to an installation root to tell this module where to look. -# -set(_TensorRT_SEARCHES) +# Hints: set -DTensorRT_DIR= (or the env var) to point at a tarball; on a +# host with libnvinfer-dev from the NVIDIA apt repo no hint is needed. +set(_trt_hints) if(TensorRT_DIR) - set(_TensorRT_SEARCH_ROOT PATHS ${TensorRT_DIR} NO_DEFAULT_PATH) - list(APPEND _TensorRT_SEARCHES _TensorRT_SEARCH_ROOT) + list(APPEND _trt_hints "${TensorRT_DIR}") endif() - -# appends some common paths -set(_TensorRT_SEARCH_NORMAL - PATHS "/usr" - ) -list(APPEND _TensorRT_SEARCHES _TensorRT_SEARCH_NORMAL) - -# Include dir -foreach(search ${_TensorRT_SEARCHES}) - find_path(TensorRT_INCLUDE_DIR NAMES NvInfer.h ${${search}} PATH_SUFFIXES include) -endforeach() - -if(NOT TensorRT_LIBRARY) - foreach(search ${_TensorRT_SEARCHES}) - find_library(TensorRT_LIBRARY NAMES nvinfer ${${search}} PATH_SUFFIXES lib) - endforeach() +if(DEFINED ENV{TensorRT_DIR}) + list(APPEND _trt_hints "$ENV{TensorRT_DIR}") endif() -if(NOT TensorRT_NVONNXPARSER_LIBRARY) - foreach(search ${_TensorRT_SEARCHES}) - find_library(TensorRT_NVONNXPARSER_LIBRARY NAMES nvonnxparser ${${search}} PATH_SUFFIXES lib) - endforeach() -endif() +find_path(TensorRT_INCLUDE_DIR + NAMES NvInfer.h + HINTS ${_trt_hints} + PATH_SUFFIXES include + PATHS /usr/include/x86_64-linux-gnu /usr/include /usr/local/include /usr/local/tensorrt/include) -mark_as_advanced(TensorRT_INCLUDE_DIR) +find_library(TensorRT_nvinfer_LIBRARY + NAMES nvinfer + HINTS ${_trt_hints} + PATH_SUFFIXES lib lib64 targets/x86_64-linux/lib + PATHS /usr/lib/x86_64-linux-gnu /usr/lib /usr/local/lib) -if(TensorRT_INCLUDE_DIR AND EXISTS "${TensorRT_INCLUDE_DIR}/NvInfer.h") - file(STRINGS "${TensorRT_INCLUDE_DIR}/NvInfer.h" TensorRT_MAJOR REGEX "^#define NV_TENSORRT_MAJOR [0-9]+.*$") - file(STRINGS "${TensorRT_INCLUDE_DIR}/NvInfer.h" TensorRT_MINOR REGEX "^#define NV_TENSORRT_MINOR [0-9]+.*$") - file(STRINGS "${TensorRT_INCLUDE_DIR}/NvInfer.h" TensorRT_PATCH REGEX "^#define NV_TENSORRT_PATCH [0-9]+.*$") +find_library(TensorRT_nvonnxparser_LIBRARY + NAMES nvonnxparser + HINTS ${_trt_hints} + PATH_SUFFIXES lib lib64 targets/x86_64-linux/lib + PATHS /usr/lib/x86_64-linux-gnu /usr/lib /usr/local/lib) - string(REGEX REPLACE "^#define NV_TENSORRT_MAJOR ([0-9]+).*$" "\\1" TensorRT_VERSION_MAJOR "${TensorRT_MAJOR}") - string(REGEX REPLACE "^#define NV_TENSORRT_MINOR ([0-9]+).*$" "\\1" TensorRT_VERSION_MINOR "${TensorRT_MINOR}") - string(REGEX REPLACE "^#define NV_TENSORRT_PATCH ([0-9]+).*$" "\\1" TensorRT_VERSION_PATCH "${TensorRT_PATCH}") - set(TensorRT_VERSION_STRING "${TensorRT_VERSION_MAJOR}.${TensorRT_VERSION_MINOR}.${TensorRT_VERSION_PATCH}") +if(TensorRT_INCLUDE_DIR AND EXISTS "${TensorRT_INCLUDE_DIR}/NvInferVersion.h") + file(STRINGS "${TensorRT_INCLUDE_DIR}/NvInferVersion.h" _trt_ver_lines REGEX "#define NV_TENSORRT_(MAJOR|MINOR|PATCH) ") + string(REGEX REPLACE ".*NV_TENSORRT_MAJOR ([0-9]+).*" "\\1" TensorRT_VERSION_MAJOR "${_trt_ver_lines}") + string(REGEX REPLACE ".*NV_TENSORRT_MINOR ([0-9]+).*" "\\1" TensorRT_VERSION_MINOR "${_trt_ver_lines}") + string(REGEX REPLACE ".*NV_TENSORRT_PATCH ([0-9]+).*" "\\1" TensorRT_VERSION_PATCH "${_trt_ver_lines}") + set(TensorRT_VERSION "${TensorRT_VERSION_MAJOR}.${TensorRT_VERSION_MINOR}.${TensorRT_VERSION_PATCH}") endif() include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(TensorRT REQUIRED_VARS TensorRT_LIBRARY TensorRT_INCLUDE_DIR VERSION_VAR TensorRT_VERSION_STRING) +find_package_handle_standard_args(TensorRT + REQUIRED_VARS TensorRT_nvinfer_LIBRARY TensorRT_nvonnxparser_LIBRARY TensorRT_INCLUDE_DIR + VERSION_VAR TensorRT_VERSION) if(TensorRT_FOUND) - set(TensorRT_INCLUDE_DIRS ${TensorRT_INCLUDE_DIR}) - - if(NOT TensorRT_LIBRARIES) - set(TensorRT_LIBRARIES ${TensorRT_LIBRARY} ${TensorRT_NVONNXPARSER_LIBRARY} ${TensorRT_NVPARSERS_LIBRARY}) + if(TensorRT_VERSION VERSION_LESS "10.0" OR NOT TensorRT_VERSION VERSION_LESS "12.0") + message(FATAL_ERROR + "tensorrt_cpp_api requires TensorRT 10.0 - 11.x, but found ${TensorRT_VERSION} at " + "${TensorRT_INCLUDE_DIR}.\n" + " Point -DTensorRT_DIR= at a supported tarball, or install libnvinfer-dev from " + "the NVIDIA apt repo (scripts/install_deps.sh).") endif() + if(NOT TARGET TensorRT::nvonnxparser) + add_library(TensorRT::nvonnxparser UNKNOWN IMPORTED) + set_target_properties(TensorRT::nvonnxparser PROPERTIES IMPORTED_LOCATION "${TensorRT_nvonnxparser_LIBRARY}") + endif() if(NOT TARGET TensorRT::TensorRT) add_library(TensorRT::TensorRT UNKNOWN IMPORTED) - set_target_properties(TensorRT::TensorRT PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIRS}") - set_property(TARGET TensorRT::TensorRT APPEND PROPERTY IMPORTED_LOCATION "${TensorRT_LIBRARY}") + set_target_properties(TensorRT::TensorRT PROPERTIES + IMPORTED_LOCATION "${TensorRT_nvinfer_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES TensorRT::nvonnxparser) endif() -endif() \ No newline at end of file + set(TensorRT_LIBRARIES ${TensorRT_nvinfer_LIBRARY} ${TensorRT_nvonnxparser_LIBRARY}) + set(TensorRT_INCLUDE_DIRS ${TensorRT_INCLUDE_DIR}) +endif() + +mark_as_advanced(TensorRT_INCLUDE_DIR TensorRT_nvinfer_LIBRARY TensorRT_nvonnxparser_LIBRARY) diff --git a/cmake/build_config.h.in b/cmake/build_config.h.in new file mode 100644 index 0000000..186db94 --- /dev/null +++ b/cmake/build_config.h.in @@ -0,0 +1,12 @@ +#pragma once + +// Generated by CMake (configure_file). Records the library version and the TensorRT/CUDA +// versions the library was BUILT against, so public headers can version-gate (e.g. the +// legacy calibrator) without including any TensorRT header. + +#define TRT_CPP_API_VERSION_MAJOR @PROJECT_VERSION_MAJOR@ +#define TRT_CPP_API_VERSION_MINOR @PROJECT_VERSION_MINOR@ +#define TRT_CPP_API_VERSION_PATCH @PROJECT_VERSION_PATCH@ + +#define TRT_CPP_API_TENSORRT_VERSION_MAJOR @TensorRT_VERSION_MAJOR@ +#define TRT_CPP_API_TENSORRT_VERSION_MINOR @TensorRT_VERSION_MINOR@ diff --git a/cmake/tensorrt_cpp_api-config.cmake.in b/cmake/tensorrt_cpp_api-config.cmake.in new file mode 100644 index 0000000..a37061b --- /dev/null +++ b/cmake/tensorrt_cpp_api-config.cmake.in @@ -0,0 +1,20 @@ +@PACKAGE_INIT@ + +# Re-resolve transitive dependencies in the consumer's scope (the exported targets reference +# the imported CUDA/TensorRT/OpenCV/spdlog targets, which must exist downstream). +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") # installed FindTensorRT.cmake +include(CMakeFindDependencyMacro) + +find_dependency(CUDAToolkit) +find_dependency(TensorRT) # pass -DTensorRT_DIR= for a tarball install +find_dependency(Threads) +if(@TRT_CPP_API_WITH_OPENCV@) + find_dependency(OpenCV) +endif() +if(@TRT_CPP_API_WITH_SPDLOG@) + find_dependency(spdlog) +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/tensorrt_cpp_apiTargets.cmake") + +check_required_components(tensorrt_cpp_api) diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 0000000..bbc25c9 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,90 @@ +# Installation + +`tensorrt_cpp_api` is a Linux / CUDA 12 / TensorRT ≥ 10 / C++20 library. TensorRT and the CUDA +toolkit are **system/externally provided** — they are not fetched by the build. + +## Prerequisites + +- **CMake ≥ 3.22** and a C++20 compiler (GCC ≥ 11 or Clang ≥ 14). +- **CUDA Toolkit 12.x** (`nvcc` is needed for the preprocessing kernel; the engine core itself + only needs the CUDA runtime). CUDA 13 requires driver ≥ 580. +- **TensorRT 10.0 – 11.x**, installed one of two ways: + - **apt** (NVIDIA CUDA network repo): `libnvinfer-dev` + `libnvonnxparsers-dev`. The bundled + `FindTensorRT` module searches the standard `/usr/include/x86_64-linux-gnu` + + `/usr/lib/x86_64-linux-gnu` layout automatically. + - **tarball**: download from NVIDIA and pass `-DTensorRT_DIR=/path/to/TensorRT-10.x` at configure. +- Optional: **spdlog** (`-DTRT_CPP_API_WITH_SPDLOG=ON`), **OpenCV** core (`-DTRT_CPP_API_WITH_OPENCV=ON`), + **pybind11** + a Python 3.9–3.13 dev environment (for the bindings). + +## Build options + +| Option | Default | Effect | +|---|---|---| +| `TRT_CPP_API_BUILD_PREPROC` | `ON` | Build the fused preprocessing sublibrary (`::preproc`). | +| `TRT_CPP_API_BUILD_TESTS` | `OFF` | Build the GoogleTest suite (`ctest`; GPU tests are labeled `gpu`). | +| `TRT_CPP_API_BUILD_EXAMPLES` | `OFF` | Build the reference examples in-tree. | +| `TRT_CPP_API_BUILD_PYTHON` | `OFF` | Build the `trtcpp` pybind11 extension. | +| `TRT_CPP_API_WITH_OPENCV` | `OFF` | Build the optional OpenCV interop header/source. | +| `TRT_CPP_API_WITH_SPDLOG` | `OFF` | Build the optional spdlog logger adapter. | +| `CMAKE_CUDA_ARCHITECTURES` | `75;80;86;89;90` | Override for your target GPUs. | + +## Build & install (C++) + +```sh +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DTRT_CPP_API_BUILD_PREPROC=ON + # add -DTensorRT_DIR=/opt/TensorRT-10.x for a tarball TensorRT + # add -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc if nvcc is not on PATH +cmake --build build -j$(nproc) +cmake --install build --prefix /opt/trtcpp +``` + +The install lays down `include/tensorrt_cpp_api/…`, the static libraries, and a CMake package +(`lib/cmake/tensorrt_cpp_api/`) including the bundled `FindTensorRT` module. + +## Consume it downstream + +```cmake +find_package(tensorrt_cpp_api REQUIRED) # set CMAKE_PREFIX_PATH=/opt/trtcpp (and TensorRT_DIR if tarball) +add_executable(myapp main.cpp) +target_link_libraries(myapp PRIVATE + tensorrt_cpp_api::tensorrt_cpp_api # core + tensorrt_cpp_api::preproc) # optional preprocessing +``` + +The package config re-resolves CUDA, TensorRT, Threads (and OpenCV/spdlog if the install was built +with them), so the only thing a consumer must provide is the location of a system TensorRT. + +## Python bindings + +```sh +pip install . # builds the trtcpp wheel via scikit-build-core +# tarball TensorRT: pip install . --config-settings=cmake.define.TensorRT_DIR=/opt/TensorRT-10.x +python -c "import trtcpp; print(trtcpp.version_string())" +``` + +For zero-copy GPU interop install a matching CuPy (`pip install cupy-cuda12x`); see +[`examples/python`](../examples/python). + +## Verifying + +```sh +cmake -S . -B build -DTRT_CPP_API_BUILD_TESTS=ON +cmake --build build -j$(nproc) +ctest --test-dir build -LE gpu # CPU-only tests (no GPU needed) +ctest --test-dir build # full suite (needs an NVIDIA GPU) +``` + +## Troubleshooting + +- **`Could NOT find CUDA: ... required is exact version "12.0"`** when building examples/consumers. + This comes from a CUDA-enabled OpenCV whose CMake config pins an exact CUDA version that differs + from your toolkit. The library core does **not** use OpenCV; build with the default + `-DTRT_CPP_API_WITH_OPENCV=OFF`, and prefer the stb-based examples (no OpenCV). +- **`Failed to detect a default CUDA architecture` / `nvcc` not found.** Pass + `-DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc`, or put `nvcc` on `PATH`. +- **`Could NOT find TensorRT`.** Install `libnvinfer-dev`/`libnvonnxparsers-dev`, or pass + `-DTensorRT_DIR=`. TensorRT must be in the 10.0–11.x range. +- **Stale-engine rebuilds.** Changing the ONNX, build options, driver/TensorRT version, or GPU + invalidates a cached engine on purpose; delete `engineCacheDir` to force a clean rebuild. diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 0000000..1c69ed2 --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,139 @@ +# Quickstart & core concepts + +This walks through the mental model and the common flows. For end-to-end programs see +[`examples/`](../examples); for installation see [install.md](install.md). + +## The five types you'll use + +| Type | Role | +|---|---| +| `Status` / `Result` | No-throw error handling. `Status` is ok-or-error; `Result` is value-or-error. `if (!r) … r.status()`. | +| `Shape` | A dynamic-aware shape (`-1` = dynamic). `Shape{1,3,640,640}`. | +| `Tensor` | An **owning** device or pinned-host buffer (RAII, move-only). The library returns these for outputs. | +| `TensorView` | A **non-owning** view over memory you (or another library) own. The zero-copy boundary type. | +| `Stream` | A CUDA stream you own (or wrap). Every async call takes one; you control synchronization. | + +Everything lives in `namespace trtcpp` and is reachable via `#include ` +(the optional `preproc.h` / `opencv_interop.h` are included separately). + +## Error handling + +Every fallible call returns `Status` or `Result` — never throws. + +```cpp +auto engine = EngineBuilder{}.buildAndLoad("model.onnx", opt); +if (!engine) { + std::fprintf(stderr, "%s\n", engine.status().message().c_str()); + return 1; +} +engine->inputNames(); // operator-> / operator* to reach the value + +// Or propagate with the helper (the enclosing function must return Status or a Result): +TRTCPP_TRY(auto eng, EngineBuilder{}.buildAndLoad("model.onnx", opt)); +``` + +## Build (or load) an engine + +`EngineBuilder` turns an ONNX model into an optimized engine and caches it on disk. `buildAndLoad` +is the one-call entry point: it builds if there's no current cache, otherwise reuses it, then +deserializes into a ready `Engine`. + +```cpp +BuildOptions opt; +opt.precision = Precision::kFp16; // kFp32 / kFp16 / kInt8Qdq / kFp8 … +opt.engineCacheDir = "engines"; // where the .engine + .json sidecar live +auto engine = EngineBuilder{}.buildAndLoad("model.onnx", opt); +``` + +The cache key is the ONNX content hash + build options + TensorRT version + GPU UUID. Change any of +them (new model, different precision, driver upgrade, different GPU) and the stale cache is detected +and rebuilt — it is never silently reused. To build and deserialize separately, use +`buildOrLoad` (returns a path) + `Engine::loadFromFile`. + +## Run inference + +Inputs and outputs are **name-keyed** maps of `TensorView`. There are three entry points: + +```cpp +Stream stream; // owns a non-blocking stream; or Stream::wrap(yourCudaStreamHandle) + +// 1) Library allocates outputs for you: +auto outputs = engine->infer({{"images", inputView}}, stream); // Result> + +// 2) Single-output shortcut (classifier/detector common case): +auto out = engine->inferSingle({{"images", inputView}}, stream); // Result + +// 3) Caller-allocated, fully zero-copy (no allocation in the hot path): +engine->enqueue({{"images", inputView}}, {{"output", outView}}, stream); // Status +``` + +None of these synchronize. Read results back explicitly: + +```cpp +auto host = out->toHost(stream).value(); // D2H copy AND stream sync +std::span v = host.as().value(); +``` + +`as()` dtype-checks and errors on a device tensor (it never does an implicit D2H). Use +`engine->outputShapes(inputs)` to size your own output buffers for the `enqueue` path. + +## Preprocess on the GPU (optional) + +The `preproc` sublibrary fuses the usual CNN front-end into one kernel — no intermediate buffers: + +```cpp +#include + +// src: HWC uint8 device tensor [H,W,C]; dst: NCHW float device tensor [1,C,Hout,Wout] +preproc::PreprocSpec spec; // out = (pixel - mean) * scale, per channel +spec.swapRB = true; // BGR -> RGB (if your decode gives BGR) +spec.keepAspectRatioPad = true; // letterbox; pads right/bottom +spec.scale = {1.f/255, 1.f/255, 1.f/255, 1.f}; +preproc::letterboxToTensor(src.view(), dst.view(), spec, stream); +``` + +## Concurrency + +`Engine` is thread-compatible (use it from one thread at a time). For concurrent inference across +threads/streams, use `EnginePool`, which hands out execution-context leases: + +```cpp +auto pool = EnginePool::create("model.engine", /*contexts=*/4).value(); +auto lease = pool.acquire(); // blocks until one is free +auto out = lease.infer({{"images", inputView}}, myStream); +``` + +For a **dynamic-shape** engine, each concurrently-used context needs its own optimization profile, +so build the engine with at least `contexts` profiles (TensorRT 11 rule; `create` enforces it). + +## Dynamic shapes + +```cpp +ProfileShape p; +p.inputName = "images"; +p.min = Shape{1,3,640,640}; p.opt = Shape{1,3,640,640}; p.max = Shape{8,3,640,640}; +opt.profiles = {OptimizationProfile{{p}}}; +// at inference time, the input view's shape selects the actual dims within [min,max] +``` + +## Streams & lifetime — the contract + +- Inference is **asynchronous** on the stream you pass and **never implicitly synchronizes**. +- Keep input buffers alive until you've synchronized the stream (`stream.synchronize()` or + `Tensor::toHost`, which syncs). +- A `Stream` you construct owns its CUDA stream; `Stream::wrap(handle)` adopts an existing one + (e.g. a PyTorch/CuPy stream) without owning it. + +## Python + +The same model, zero-copy, from Python: + +```python +import cupy as cp, trtcpp +eng = trtcpp.EngineBuilder().build_and_load("model.onnx", trtcpp.BuildOptions()) +stream = trtcpp.Stream.wrap(cp.cuda.get_current_stream().ptr) +out = eng.infer_single({"images": cp_gpu_array}, stream) # cupy array in, zero-copy +y = cp.asarray(out); stream.synchronize() # cupy view of the output, zero-copy +``` + +See [`examples/python`](../examples/python) and the API reference (`doxygen Doxyfile`). diff --git a/docs/upgrading_from_v6.md b/docs/upgrading_from_v6.md new file mode 100644 index 0000000..57f52cd --- /dev/null +++ b/docs/upgrading_from_v6.md @@ -0,0 +1,116 @@ +# Upgrading from v6 + +v7 is a **clean break**: there is no source-compatibility shim. v6's API — a templated +`Engine`, OpenCV `cv::cuda::GpuMat` in the signatures, triply-nested `std::vector` outputs, +`bool`/throw error handling, and preprocessing baked into the engine — is fundamentally +incompatible with v7's leak-free, name-keyed, no-throw design. This guide maps every v6 concept to +its v7 equivalent and shows a full before/after. + +## What changed, and why + +| v6 | v7 | Why | +|---|---|---| +| `Engine` (templated on output type) | `Engine` (non-templated) + runtime `DType` on `Tensor`/`TensorView` | One type handles mixed-dtype, multi-output models; no template bloat. | +| `Options` (one struct for everything) | `BuildOptions` (build) + `EngineOptions` (load) | Build-time vs runtime concerns are separate. | +| `Precision::FP16` / `INT8` | `Precision::kFp16` / `kInt8Qdq` (or `kInt8CalibLegacy`, TRT < 11) | INT8 defaults to explicit-QDQ; legacy calibration is gated out on TensorRT 11. | +| `buildLoadNetwork(onnx, subVals, divVals, normalize)` | `EngineBuilder::buildAndLoad(onnx, BuildOptions)` + separate `preproc::letterboxToTensor(...)` | Preprocessing is no longer fused into the engine; it's an explicit, reusable GPU kernel. | +| `loadNetwork(trtPath, …)` | `Engine::loadFromFile(path, EngineOptions)` / `loadFromMemory(bytes, …)` | — | +| `runInference(inputs: vector>, out: vector>>)` | `engine.infer(map, Stream)` → `map`, or `enqueue(in, out, Stream)` for caller-allocated zero-copy | Name-keyed, zero-copy, single-allocation; no nested vectors. | +| Engine creates/destroys a stream per call | Caller passes a `Stream` (own it, or `Stream::wrap(handle)`) | Caller controls stream ordering and overlap. | +| Implicit device→host copy of outputs | Explicit `Tensor::toHost(stream)` (copies **and** syncs) | No hidden D2H; you decide when to pay for it. | +| `bool` returns / exceptions | `Status` / `Result` (+ `TRTCPP_TRY`) | Uniform, no-throw error handling. | +| `cv::cuda::GpuMat` inputs | `TensorView` over device memory (or `opencv_interop::viewOf(gpuMat)` with `-DTRT_CPP_API_WITH_OPENCV=ON`) | OpenCV is optional, never in the core API. | +| `subVals` / `divVals` / `normalize` | `preproc::PreprocSpec{ mean, scale, swapRB, keepAspectRatioPad }` (`scale = 1/divVals`, `mean = subVals`) | Per-channel, with letterbox, in one fused kernel. | +| `optBatchSize` / `maxBatchSize` / input-width options | `BuildOptions.profiles` = `OptimizationProfile`/`ProfileShape` (per-input min/opt/max) | Real dynamic-shape support, not just batch. | +| `getInputDims()` / `getOutputDims()` | `engine.inputNames()` / `outputNames()` / `tensorShape(name)` / `tensors()` | Name-keyed introspection. | +| Global spdlog logger | Injectable `EngineOptions.logger` (`ILogger`); spdlog adapter behind `-DTRT_CPP_API_WITH_SPDLOG=ON` | No forced logging dependency. | +| `#include "engine.h"`, `Engine`/`Util`/`Stopwatch` in the global namespace | `#include `, everything under `namespace trtcpp` | Clean include root + namespace. | +| Header `.inl` implementation (TRT/CUDA leak into every TU) | PImpl; no `nvinfer1`/OpenCV/spdlog in public headers | Fast, decoupled compiles. | + +## Before / after + +**v6:** +```cpp +#include "engine.h" + +Options options; +options.precision = Precision::FP16; +options.optBatchSize = 1; +options.maxBatchSize = 1; + +Engine engine(options); +std::array subVals{0.f, 0.f, 0.f}, divVals{1.f, 1.f, 1.f}; +engine.buildLoadNetwork("model.onnx", subVals, divVals, /*normalize=*/true); + +std::vector> inputs = /* one GpuMat per input, per batch */; +std::vector>> featureVectors; +engine.runInference(inputs, featureVectors); // returns bool; D2H happens internally +float first = featureVectors[0][0][0]; +``` + +**v7:** +```cpp +#include +#include +using namespace trtcpp; + +BuildOptions opt; +opt.precision = Precision::kFp16; +opt.engineCacheDir = "engines"; + +auto engine = EngineBuilder{}.buildAndLoad("model.onnx", opt); +if (!engine) return /* handle */ engine.status(); + +Stream stream; +// Preprocess on the GPU into an NCHW tensor (replaces subVals/divVals/normalize): +auto input = Tensor::allocate(DType::kFloat32, Shape{1, 3, H, W}, Device::kCuda).value(); +preproc::PreprocSpec spec; // out = (pixel - mean) * scale +spec.scale = {1.f / 255, 1.f / 255, 1.f / 255, 1.f}; +preproc::letterboxToTensor(srcHWCDeviceView, input.view(), spec, stream); + +auto out = engine->inferSingle({{engine->inputNames().front(), input.view()}}, stream); +if (!out) return out.status(); +auto host = out->toHost(stream).value(); // explicit D2H + sync +std::span scores = host.as().value(); +float first = scores[0]; +``` + +## INT8 / calibration + +- **Preferred (works on TensorRT 10 and 11):** export an explicit Q/DQ ONNX and build with + `Precision::kInt8Qdq`. No calibration data, no precision flags. +- **Legacy calibration** (v6 `calibrationDataDirectoryPath`): only available when the library is + built against **TensorRT < 11**. Use `Precision::kInt8CalibLegacy` and set + `BuildOptions.calibrator` to an `ICalibrator` (see `tensorrt_cpp_api/calibrator.h`). On + TensorRT ≥ 11 this path is rejected with a clear error rather than silently downgrading. + +## Dynamic shapes + +v6 exposed only batch knobs. In v7, give one `OptimizationProfile` per concurrent context: +```cpp +ProfileShape p; +p.inputName = "images"; +p.min = Shape{1, 3, 640, 640}; +p.opt = Shape{1, 3, 640, 640}; +p.max = Shape{8, 3, 640, 640}; +opt.profiles = {OptimizationProfile{{p}}}; +``` +For concurrent multi-stream dynamic inference, build with ≥ N profiles and use `EnginePool` +(one profile per leased context). + +## Migration checklist + +1. Replace `#include "engine.h"` with `#include `; add `using namespace trtcpp;`. +2. Split `Options` into `BuildOptions` (precision, profiles, cache dir) and `EngineOptions` + (device, logger, plugins). +3. Drop the `Engine` template parameter; read outputs through `Tensor`/`as()`. +4. Move preprocessing out of the engine into `preproc::letterboxToTensor` (or your own kernel); + translate `subVals → mean`, `divVals → 1/scale`. +5. Replace `runInference(GpuMat, nested-vector)` with `infer`/`inferSingle`/`enqueue` over a + name-keyed `TensorView` map and a caller `Stream`; read back with `toHost`. +6. Replace `bool` checks with `if (!result) … result.status()`. +7. (Optional) Turn on `-DTRT_CPP_API_WITH_OPENCV=ON` and use `opencv_interop::viewOf` if you still + feed `cv::cuda::GpuMat`. + +See the [reference examples](../examples) for complete, runnable detection/classification/segmentation +pipelines built this way. diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt new file mode 100644 index 0000000..cf33405 --- /dev/null +++ b/examples/CMakeLists.txt @@ -0,0 +1,28 @@ +cmake_minimum_required(VERSION 3.22) + +# These examples are meant to be built against the INSTALLED library: +# cmake -S examples -B build/examples -DCMAKE_PREFIX_PATH= +# When this directory is instead pulled into the parent build (-DTRT_CPP_API_BUILD_EXAMPLES=ON), +# the library targets already exist, so we declare no project and skip find_package. +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + project(tensorrt_cpp_api_examples LANGUAGES CXX) + find_package(tensorrt_cpp_api REQUIRED) +endif() + +# Image decode/encode is handled by the vendored stb single-header libs (common/), so the +# examples have NO third-party image dependency -- they build anywhere the library does. +function(add_example name) + add_executable(${name} ${name}/main.cpp ${CMAKE_CURRENT_SOURCE_DIR}/common/stb_impl.cpp) + target_link_libraries(${name} PRIVATE tensorrt_cpp_api::tensorrt_cpp_api tensorrt_cpp_api::preproc) + target_compile_features(${name} PRIVATE cxx_std_20) +endfunction() + +add_example(classification) +add_example(detection) +add_example(segmentation) + +# Inference-throughput benchmark (no image I/O, so no stb): the C++ baseline for the Python +# zero-copy perf-parity check. +add_executable(benchmark benchmark/main.cpp) +target_link_libraries(benchmark PRIVATE tensorrt_cpp_api::tensorrt_cpp_api) +target_compile_features(benchmark PRIVATE cxx_std_20) diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..2b07c06 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,54 @@ +# Examples + +Four reference programs covering the common CNN inference tasks, each built against the +**installed** `tensorrt_cpp_api` package via `find_package`. They share one trait: the library +does the GPU work (build/cache the engine, fused preprocessing, inference) and the example does +only I/O and task-specific post-processing. + +| Example | Model | Pipeline | +|---|---|---| +| `classification/` | MobileNetV2 / ResNet50 (ImageNet) | decode → preproc → infer → softmax top-5 | +| `detection/` | YOLOv8n | decode → letterbox preproc → infer → decode `[1,84,8400]` → NMS → draw | +| `segmentation/` | DeepLabV3-MobileNetV3 | decode → preproc → infer → per-pixel argmax → colorize/blend | +| `benchmark/` | any | preallocated zero-copy `enqueue` loop, throughput baseline | + +Image decode/encode uses the vendored **stb** single-header libraries (`common/`), so the +examples have **no third-party image dependency** — they build wherever the library does. (If you +prefer OpenCV, the library ships an optional `opencv_interop.h`; swap `common/image_io.h` +accordingly.) + +## Get the models + +```sh +./download_models.sh # writes ../models/{mobilenetv2-7,yolov8n,deeplabv3_mobilenetv3}.onnx +``` + +Detection/segmentation export needs `ultralytics` and `torch`/`torchvision` (CPU is fine — see the +script header). Classification needs only `curl`. + +## Build (against an installed library) + +```sh +# 1. install the library somewhere +cmake -S . -B build -DTRT_CPP_API_BUILD_PREPROC=ON +cmake --build build -j && cmake --install build --prefix /opt/trtcpp + +# 2. build the examples against it +cmake -S examples -B build/examples -DCMAKE_PREFIX_PATH=/opt/trtcpp +cmake --build build/examples -j +``` + +(For a tarball TensorRT, add `-DTensorRT_DIR=/path/to/TensorRT-10.x` to both.) + +Alternatively build them in-tree with the library: `cmake -S . -B build -DTRT_CPP_API_BUILD_EXAMPLES=ON`. + +## Run + +```sh +./build/examples/classification models/mobilenetv2-7.onnx image.jpg +./build/examples/detection models/yolov8n.onnx image.jpg 0.25 out.jpg +./build/examples/segmentation models/deeplabv3_mobilenetv3.onnx image.jpg out.jpg +``` + +The first run builds and caches a TensorRT engine next to the working directory (`engines/`); +later runs load the cache. See `python/` for the zero-copy CuPy example and the perf-parity check. diff --git a/examples/benchmark/main.cpp b/examples/benchmark/main.cpp new file mode 100644 index 0000000..fee49e6 --- /dev/null +++ b/examples/benchmark/main.cpp @@ -0,0 +1,69 @@ +// Minimal C++ inference-throughput benchmark, the baseline for the Python zero-copy +// perf-parity check (examples/python/benchmark_parity.py). Uses the zero-copy enqueue path with +// preallocated device IO so it measures inference + launch overhead, not allocation. +// +// Output (stdout): a single line "cpp_latency_ms " plus a human-readable summary on +// stderr, so the Python harness can parse the number. +// +// Usage: benchmark [iters=200] + +#include +#include +#include +#include + +#include + +using namespace trtcpp; + +int main(int argc, char **argv) { + if (argc < 2) { + std::fprintf(stderr, "usage: %s [iters=200] [precision=fp16|fp32|int8]\n", argv[0]); + return 2; + } + const std::string modelPath = argv[1]; + const int iters = argc > 2 ? std::atoi(argv[2]) : 200; + const std::string prec = argc > 3 ? argv[3] : "fp16"; + + BuildOptions bo; + bo.precision = prec == "fp32" ? Precision::kFp32 : prec == "int8" ? Precision::kInt8Qdq : Precision::kFp16; + bo.engineCacheDir = "engines"; + auto engine = EngineBuilder{}.buildAndLoad(modelPath, bo); + if (!engine) { + std::fprintf(stderr, "engine: %s\n", engine.status().message().c_str()); + return 1; + } + + const std::string inName = engine->inputNames().front(); + const std::string outName = engine->outputNames().front(); + auto inShape = engine->tensorShape(inName).value(); + auto outShape = engine->tensorShape(outName).value(); + + Stream stream; + auto in = Tensor::allocate(DType::kFloat32, inShape, Device::kCuda).value(); + auto out = Tensor::allocate(engine->tensorDType(outName).value(), outShape, Device::kCuda).value(); + std::unordered_map inputs{{inName, in.view()}}; + std::unordered_map outputs{{outName, out.view()}}; + + auto once = [&]() { + if (auto s = engine->enqueue(inputs, outputs, stream); !s) { + std::fprintf(stderr, "enqueue: %s\n", s.message().c_str()); + std::exit(1); + } + stream.synchronize(); + }; + + for (int i = 0; i < 20; ++i) { + once(); // warmup + } + const auto t0 = std::chrono::steady_clock::now(); + for (int i = 0; i < iters; ++i) { + once(); + } + const auto t1 = std::chrono::steady_clock::now(); + const double ms = std::chrono::duration(t1 - t0).count() / iters; + + std::fprintf(stderr, "C++ : %.3f ms/infer (%.0f inf/s) over %d iters\n", ms, 1000.0 / ms, iters); + std::printf("cpp_latency_ms %.4f\n", ms); + return 0; +} diff --git a/examples/classification/main.cpp b/examples/classification/main.cpp new file mode 100644 index 0000000..f2e8f90 --- /dev/null +++ b/examples/classification/main.cpp @@ -0,0 +1,87 @@ +// Image classification (ImageNet) with tensorrt_cpp_api. +// +// Pipeline: decode (stb, CPU) -> upload HWC-uint8 to GPU -> fused resize/normalize/NCHW +// (preproc sublib) -> infer -> softmax + top-5. Works with any ImageNet ONNX classifier whose +// input is [1,3,H,W] and output is [1,N] logits (e.g. MobileNetV2, ResNet50). +// +// Usage: classification [topk] + +#include +#include +#include +#include +#include +#include + +#include "../common/image_io.h" + +using namespace trtcpp; + +int main(int argc, char **argv) { + if (argc < 3) { + std::fprintf(stderr, "usage: %s [topk]\n", argv[0]); + return 2; + } + const std::string modelPath = argv[1]; + const std::string imagePath = argv[2]; + const int topk = argc > 3 ? std::atoi(argv[3]) : 5; + + BuildOptions bo; + bo.precision = Precision::kFp16; + bo.engineCacheDir = "engines"; + auto engine = EngineBuilder{}.buildAndLoad(modelPath, bo); + if (!engine) { + std::fprintf(stderr, "engine: %s\n", engine.status().message().c_str()); + return 1; + } + + const std::string inName = engine->inputNames().front(); + auto inShape = engine->tensorShape(inName).value(); // [1,3,H,W] + const int inH = static_cast(inShape[2]); + const int inW = static_cast(inShape[3]); + + examples::Image img = examples::decodeImage(imagePath); + if (img.empty()) { + std::fprintf(stderr, "could not read image: %s\n", imagePath.c_str()); + return 1; + } + + Stream stream; + auto src = examples::uploadHWC(img, stream); + if (!src) { + std::fprintf(stderr, "upload: %s\n", src.status().message().c_str()); + return 1; + } + auto dst = Tensor::allocate(DType::kFloat32, Shape{1, 3, inH, inW}, Device::kCuda).value(); + if (auto s = preproc::letterboxToTensor(src->view(), dst.view(), examples::imagenetSpec(), stream); !s) { + std::fprintf(stderr, "preproc: %s\n", s.message().c_str()); + return 1; + } + + auto output = engine->inferSingle({{inName, dst.view()}}, stream); + if (!output) { + std::fprintf(stderr, "infer: %s\n", output.status().message().c_str()); + return 1; + } + auto host = output->toHost(stream).value(); + auto logits = host.as().value(); // [1, N] + + // numerically-stable softmax + top-k + const float maxLogit = *std::max_element(logits.begin(), logits.end()); + std::vector probs(logits.size()); + double sum = 0.0; + for (std::size_t i = 0; i < logits.size(); ++i) { + probs[i] = std::exp(logits[i] - maxLogit); + sum += probs[i]; + } + std::vector idx(logits.size()); + std::iota(idx.begin(), idx.end(), 0); + const int k = std::min(topk, static_cast(idx.size())); + std::partial_sort(idx.begin(), idx.begin() + k, idx.end(), [&](int a, int b) { return probs[a] > probs[b]; }); + + std::printf("top-%d of %zu classes:\n", k, logits.size()); + for (int i = 0; i < k; ++i) { + std::printf(" class %4d %6.2f%%\n", idx[i], 100.0 * probs[idx[i]] / sum); + } + return 0; +} diff --git a/examples/common/image_io.h b/examples/common/image_io.h new file mode 100644 index 0000000..df35ef7 --- /dev/null +++ b/examples/common/image_io.h @@ -0,0 +1,109 @@ +#pragma once + +// Dependency-free image I/O for the reference examples, via the vendored stb single-header +// libraries. The library itself has no image-codec dependency; the examples decode on the CPU, +// upload raw HWC-uint8 to the GPU, do all tensor work there (preproc sublib + engine), and write +// results back. This keeps the examples buildable anywhere -- no OpenCV / cuDNN / system-codec +// version coupling. (The library separately offers an optional OpenCV interop header for callers +// who already have cv::Mat / cv::cuda::GpuMat.) +// +// stb_image decodes to RGB, so the preproc specs here use swapRB=false (the ONNX classifiers / +// detectors expect RGB). + +#include +#include +#include +#include + +#include "stb_image.h" +#include "stb_image_write.h" + +#include +#include + +namespace examples { + +/// An 8-bit RGB image in HWC layout (channels = 3). +struct Image { + std::vector data; + int width = 0; + int height = 0; + int channels = 3; + bool empty() const { return data.empty(); } +}; + +inline Image decodeImage(const std::string &path) { + int w = 0, h = 0, c = 0; + std::uint8_t *pixels = stbi_load(path.c_str(), &w, &h, &c, 3); // force RGB + Image img; + if (pixels) { + img.width = w; + img.height = h; + img.channels = 3; + img.data.assign(pixels, pixels + static_cast(w) * h * 3); + stbi_image_free(pixels); + } + return img; +} + +inline bool writeJpg(const std::string &path, const Image &img, int quality = 92) { + return stbi_write_jpg(path.c_str(), img.width, img.height, img.channels, img.data.data(), quality) != 0; +} + +/// Upload a decoded HWC-uint8 RGB image to a device Tensor [H, W, 3]. Async on `stream`. +inline trtcpp::Result uploadHWC(const Image &img, const trtcpp::Stream &stream) { + if (img.empty()) { + return trtcpp::Status{trtcpp::StatusCode::kInvalidArgument, "image is empty"}; + } + const trtcpp::Shape shape{img.height, img.width, 3}; + auto dev = trtcpp::Tensor::allocate(trtcpp::DType::kUInt8, shape, trtcpp::Device::kCuda); + if (!dev) { + return dev.status(); + } + const trtcpp::TensorView host{const_cast(img.data.data()), trtcpp::DType::kUInt8, shape, trtcpp::Device::kHost}; + if (auto s = dev->copyFrom(host, stream); !s) { + return s; + } + return dev; +} + +/// ImageNet preprocessing for torchvision/ONNX-zoo models: /255 then (x-mean)/std with the +/// standard ImageNet statistics. The preproc kernel computes out = (pixel - mean)*scale, so +/// mean = 255*imagenet_mean and scale = 1/(255*imagenet_std), in RGB channel order. +inline trtcpp::preproc::PreprocSpec imagenetSpec() { + trtcpp::preproc::PreprocSpec spec; + spec.swapRB = false; // stb already gives RGB + spec.keepAspectRatioPad = false; // plain resize to the square input + spec.mean = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f, 0.f}; + spec.scale = {1.f / (0.229f * 255.f), 1.f / (0.224f * 255.f), 1.f / (0.225f * 255.f), 1.f}; + return spec; +} + +/// Draw an axis-aligned rectangle outline (clamped to the image) in RGB. +inline void drawRect(Image &img, int x0, int y0, int x1, int y1, std::uint8_t r, std::uint8_t g, std::uint8_t b, int thickness = 2) { + x0 = std::clamp(x0, 0, img.width - 1); + y0 = std::clamp(y0, 0, img.height - 1); + x1 = std::clamp(x1, 0, img.width - 1); + y1 = std::clamp(y1, 0, img.height - 1); + auto put = [&](int x, int y) { + if (x < 0 || y < 0 || x >= img.width || y >= img.height) { + return; + } + auto *p = &img.data[(static_cast(y) * img.width + x) * 3]; + p[0] = r; + p[1] = g; + p[2] = b; + }; + for (int t = 0; t < thickness; ++t) { + for (int x = x0; x <= x1; ++x) { + put(x, y0 + t); + put(x, y1 - t); + } + for (int y = y0; y <= y1; ++y) { + put(x0 + t, y); + put(x1 - t, y); + } + } +} + +} // namespace examples diff --git a/examples/common/stb_image.h b/examples/common/stb_image.h new file mode 100644 index 0000000..9eedabe --- /dev/null +++ b/examples/common/stb_image.h @@ -0,0 +1,7988 @@ +/* stb_image - v2.30 - public domain image loader - http://nothings.org/stb + no warranty implied; use at your own risk + + Do this: + #define STB_IMAGE_IMPLEMENTATION + before you include this file in *one* C or C++ file to create the implementation. + + // i.e. it should look like this: + #include ... + #include ... + #include ... + #define STB_IMAGE_IMPLEMENTATION + #include "stb_image.h" + + You can #define STBI_ASSERT(x) before the #include to avoid using assert.h. + And #define STBI_MALLOC, STBI_REALLOC, and STBI_FREE to avoid using malloc,realloc,free + + + QUICK NOTES: + Primarily of interest to game developers and other people who can + avoid problematic images and only need the trivial interface + + JPEG baseline & progressive (12 bpc/arithmetic not supported, same as stock IJG lib) + PNG 1/2/4/8/16-bit-per-channel + + TGA (not sure what subset, if a subset) + BMP non-1bpp, non-RLE + PSD (composited view only, no extra channels, 8/16 bit-per-channel) + + GIF (*comp always reports as 4-channel) + HDR (radiance rgbE format) + PIC (Softimage PIC) + PNM (PPM and PGM binary only) + + Animated GIF still needs a proper API, but here's one way to do it: + http://gist.github.com/urraka/685d9a6340b26b830d49 + + - decode from memory or through FILE (define STBI_NO_STDIO to remove code) + - decode from arbitrary I/O callbacks + - SIMD acceleration on x86/x64 (SSE2) and ARM (NEON) + + Full documentation under "DOCUMENTATION" below. + + +LICENSE + + See end of file for license information. + +RECENT REVISION HISTORY: + + 2.30 (2024-05-31) avoid erroneous gcc warning + 2.29 (2023-05-xx) optimizations + 2.28 (2023-01-29) many error fixes, security errors, just tons of stuff + 2.27 (2021-07-11) document stbi_info better, 16-bit PNM support, bug fixes + 2.26 (2020-07-13) many minor fixes + 2.25 (2020-02-02) fix warnings + 2.24 (2020-02-02) fix warnings; thread-local failure_reason and flip_vertically + 2.23 (2019-08-11) fix clang static analysis warning + 2.22 (2019-03-04) gif fixes, fix warnings + 2.21 (2019-02-25) fix typo in comment + 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs + 2.19 (2018-02-11) fix warning + 2.18 (2018-01-30) fix warnings + 2.17 (2018-01-29) bugfix, 1-bit BMP, 16-bitness query, fix warnings + 2.16 (2017-07-23) all functions have 16-bit variants; optimizations; bugfixes + 2.15 (2017-03-18) fix png-1,2,4; all Imagenet JPGs; no runtime SSE detection on GCC + 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs + 2.13 (2016-12-04) experimental 16-bit API, only for PNG so far; fixes + 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes + 2.11 (2016-04-02) 16-bit PNGS; enable SSE2 in non-gcc x64 + RGB-format JPEG; remove white matting in PSD; + allocate large structures on the stack; + correct channel count for PNG & BMP + 2.10 (2016-01-22) avoid warning introduced in 2.09 + 2.09 (2016-01-16) 16-bit TGA; comments in PNM files; STBI_REALLOC_SIZED + + See end of file for full revision history. + + + ============================ Contributors ========================= + + Image formats Extensions, features + Sean Barrett (jpeg, png, bmp) Jetro Lauha (stbi_info) + Nicolas Schulz (hdr, psd) Martin "SpartanJ" Golini (stbi_info) + Jonathan Dummer (tga) James "moose2000" Brown (iPhone PNG) + Jean-Marc Lienher (gif) Ben "Disch" Wenger (io callbacks) + Tom Seddon (pic) Omar Cornut (1/2/4-bit PNG) + Thatcher Ulrich (psd) Nicolas Guillemot (vertical flip) + Ken Miller (pgm, ppm) Richard Mitton (16-bit PSD) + github:urraka (animated gif) Junggon Kim (PNM comments) + Christopher Forseth (animated gif) Daniel Gibson (16-bit TGA) + socks-the-fox (16-bit PNG) + Jeremy Sawicki (handle all ImageNet JPGs) + Optimizations & bugfixes Mikhail Morozov (1-bit BMP) + Fabian "ryg" Giesen Anael Seghezzi (is-16-bit query) + Arseny Kapoulkine Simon Breuss (16-bit PNM) + John-Mark Allen + Carmelo J Fdez-Aguera + + Bug & warning fixes + Marc LeBlanc David Woo Guillaume George Martins Mozeiko + Christpher Lloyd Jerry Jansson Joseph Thomson Blazej Dariusz Roszkowski + Phil Jordan Dave Moore Roy Eltham + Hayaki Saito Nathan Reed Won Chun + Luke Graham Johan Duparc Nick Verigakis the Horde3D community + Thomas Ruf Ronny Chevalier github:rlyeh + Janez Zemva John Bartholomew Michal Cichon github:romigrou + Jonathan Blow Ken Hamada Tero Hanninen github:svdijk + Eugene Golushkov Laurent Gomila Cort Stratton github:snagar + Aruelien Pocheville Sergio Gonzalez Thibault Reuille github:Zelex + Cass Everitt Ryamond Barbiero github:grim210 + Paul Du Bois Engin Manap Aldo Culquicondor github:sammyhw + Philipp Wiesemann Dale Weiler Oriol Ferrer Mesia github:phprus + Josh Tobin Neil Bickford Matthew Gregan github:poppolopoppo + Julian Raschke Gregory Mullen Christian Floisand github:darealshinji + Baldur Karlsson Kevin Schmidt JR Smith github:Michaelangel007 + Brad Weinberger Matvey Cherevko github:mosra + Luca Sas Alexander Veselov Zack Middleton [reserved] + Ryan C. Gordon [reserved] [reserved] + DO NOT ADD YOUR NAME HERE + + Jacko Dirks + + To add your name to the credits, pick a random blank space in the middle and fill it. + 80% of merge conflicts on stb PRs are due to people adding their name at the end + of the credits. +*/ + +#ifndef STBI_INCLUDE_STB_IMAGE_H +#define STBI_INCLUDE_STB_IMAGE_H + +// DOCUMENTATION +// +// Limitations: +// - no 12-bit-per-channel JPEG +// - no JPEGs with arithmetic coding +// - GIF always returns *comp=4 +// +// Basic usage (see HDR discussion below for HDR usage): +// int x,y,n; +// unsigned char *data = stbi_load(filename, &x, &y, &n, 0); +// // ... process data if not NULL ... +// // ... x = width, y = height, n = # 8-bit components per pixel ... +// // ... replace '0' with '1'..'4' to force that many components per pixel +// // ... but 'n' will always be the number that it would have been if you said 0 +// stbi_image_free(data); +// +// Standard parameters: +// int *x -- outputs image width in pixels +// int *y -- outputs image height in pixels +// int *channels_in_file -- outputs # of image components in image file +// int desired_channels -- if non-zero, # of image components requested in result +// +// The return value from an image loader is an 'unsigned char *' which points +// to the pixel data, or NULL on an allocation failure or if the image is +// corrupt or invalid. The pixel data consists of *y scanlines of *x pixels, +// with each pixel consisting of N interleaved 8-bit components; the first +// pixel pointed to is top-left-most in the image. There is no padding between +// image scanlines or between pixels, regardless of format. The number of +// components N is 'desired_channels' if desired_channels is non-zero, or +// *channels_in_file otherwise. If desired_channels is non-zero, +// *channels_in_file has the number of components that _would_ have been +// output otherwise. E.g. if you set desired_channels to 4, you will always +// get RGBA output, but you can check *channels_in_file to see if it's trivially +// opaque because e.g. there were only 3 channels in the source image. +// +// An output image with N components has the following components interleaved +// in this order in each pixel: +// +// N=#comp components +// 1 grey +// 2 grey, alpha +// 3 red, green, blue +// 4 red, green, blue, alpha +// +// If image loading fails for any reason, the return value will be NULL, +// and *x, *y, *channels_in_file will be unchanged. The function +// stbi_failure_reason() can be queried for an extremely brief, end-user +// unfriendly explanation of why the load failed. Define STBI_NO_FAILURE_STRINGS +// to avoid compiling these strings at all, and STBI_FAILURE_USERMSG to get slightly +// more user-friendly ones. +// +// Paletted PNG, BMP, GIF, and PIC images are automatically depalettized. +// +// To query the width, height and component count of an image without having to +// decode the full file, you can use the stbi_info family of functions: +// +// int x,y,n,ok; +// ok = stbi_info(filename, &x, &y, &n); +// // returns ok=1 and sets x, y, n if image is a supported format, +// // 0 otherwise. +// +// Note that stb_image pervasively uses ints in its public API for sizes, +// including sizes of memory buffers. This is now part of the API and thus +// hard to change without causing breakage. As a result, the various image +// loaders all have certain limits on image size; these differ somewhat +// by format but generally boil down to either just under 2GB or just under +// 1GB. When the decoded image would be larger than this, stb_image decoding +// will fail. +// +// Additionally, stb_image will reject image files that have any of their +// dimensions set to a larger value than the configurable STBI_MAX_DIMENSIONS, +// which defaults to 2**24 = 16777216 pixels. Due to the above memory limit, +// the only way to have an image with such dimensions load correctly +// is for it to have a rather extreme aspect ratio. Either way, the +// assumption here is that such larger images are likely to be malformed +// or malicious. If you do need to load an image with individual dimensions +// larger than that, and it still fits in the overall size limit, you can +// #define STBI_MAX_DIMENSIONS on your own to be something larger. +// +// =========================================================================== +// +// UNICODE: +// +// If compiling for Windows and you wish to use Unicode filenames, compile +// with +// #define STBI_WINDOWS_UTF8 +// and pass utf8-encoded filenames. Call stbi_convert_wchar_to_utf8 to convert +// Windows wchar_t filenames to utf8. +// +// =========================================================================== +// +// Philosophy +// +// stb libraries are designed with the following priorities: +// +// 1. easy to use +// 2. easy to maintain +// 3. good performance +// +// Sometimes I let "good performance" creep up in priority over "easy to maintain", +// and for best performance I may provide less-easy-to-use APIs that give higher +// performance, in addition to the easy-to-use ones. Nevertheless, it's important +// to keep in mind that from the standpoint of you, a client of this library, +// all you care about is #1 and #3, and stb libraries DO NOT emphasize #3 above all. +// +// Some secondary priorities arise directly from the first two, some of which +// provide more explicit reasons why performance can't be emphasized. +// +// - Portable ("ease of use") +// - Small source code footprint ("easy to maintain") +// - No dependencies ("ease of use") +// +// =========================================================================== +// +// I/O callbacks +// +// I/O callbacks allow you to read from arbitrary sources, like packaged +// files or some other source. Data read from callbacks are processed +// through a small internal buffer (currently 128 bytes) to try to reduce +// overhead. +// +// The three functions you must define are "read" (reads some bytes of data), +// "skip" (skips some bytes of data), "eof" (reports if the stream is at the end). +// +// =========================================================================== +// +// SIMD support +// +// The JPEG decoder will try to automatically use SIMD kernels on x86 when +// supported by the compiler. For ARM Neon support, you must explicitly +// request it. +// +// (The old do-it-yourself SIMD API is no longer supported in the current +// code.) +// +// On x86, SSE2 will automatically be used when available based on a run-time +// test; if not, the generic C versions are used as a fall-back. On ARM targets, +// the typical path is to have separate builds for NEON and non-NEON devices +// (at least this is true for iOS and Android). Therefore, the NEON support is +// toggled by a build flag: define STBI_NEON to get NEON loops. +// +// If for some reason you do not want to use any of SIMD code, or if +// you have issues compiling it, you can disable it entirely by +// defining STBI_NO_SIMD. +// +// =========================================================================== +// +// HDR image support (disable by defining STBI_NO_HDR) +// +// stb_image supports loading HDR images in general, and currently the Radiance +// .HDR file format specifically. You can still load any file through the existing +// interface; if you attempt to load an HDR file, it will be automatically remapped +// to LDR, assuming gamma 2.2 and an arbitrary scale factor defaulting to 1; +// both of these constants can be reconfigured through this interface: +// +// stbi_hdr_to_ldr_gamma(2.2f); +// stbi_hdr_to_ldr_scale(1.0f); +// +// (note, do not use _inverse_ constants; stbi_image will invert them +// appropriately). +// +// Additionally, there is a new, parallel interface for loading files as +// (linear) floats to preserve the full dynamic range: +// +// float *data = stbi_loadf(filename, &x, &y, &n, 0); +// +// If you load LDR images through this interface, those images will +// be promoted to floating point values, run through the inverse of +// constants corresponding to the above: +// +// stbi_ldr_to_hdr_scale(1.0f); +// stbi_ldr_to_hdr_gamma(2.2f); +// +// Finally, given a filename (or an open file or memory block--see header +// file for details) containing image data, you can query for the "most +// appropriate" interface to use (that is, whether the image is HDR or +// not), using: +// +// stbi_is_hdr(char *filename); +// +// =========================================================================== +// +// iPhone PNG support: +// +// We optionally support converting iPhone-formatted PNGs (which store +// premultiplied BGRA) back to RGB, even though they're internally encoded +// differently. To enable this conversion, call +// stbi_convert_iphone_png_to_rgb(1). +// +// Call stbi_set_unpremultiply_on_load(1) as well to force a divide per +// pixel to remove any premultiplied alpha *only* if the image file explicitly +// says there's premultiplied data (currently only happens in iPhone images, +// and only if iPhone convert-to-rgb processing is on). +// +// =========================================================================== +// +// ADDITIONAL CONFIGURATION +// +// - You can suppress implementation of any of the decoders to reduce +// your code footprint by #defining one or more of the following +// symbols before creating the implementation. +// +// STBI_NO_JPEG +// STBI_NO_PNG +// STBI_NO_BMP +// STBI_NO_PSD +// STBI_NO_TGA +// STBI_NO_GIF +// STBI_NO_HDR +// STBI_NO_PIC +// STBI_NO_PNM (.ppm and .pgm) +// +// - You can request *only* certain decoders and suppress all other ones +// (this will be more forward-compatible, as addition of new decoders +// doesn't require you to disable them explicitly): +// +// STBI_ONLY_JPEG +// STBI_ONLY_PNG +// STBI_ONLY_BMP +// STBI_ONLY_PSD +// STBI_ONLY_TGA +// STBI_ONLY_GIF +// STBI_ONLY_HDR +// STBI_ONLY_PIC +// STBI_ONLY_PNM (.ppm and .pgm) +// +// - If you use STBI_NO_PNG (or _ONLY_ without PNG), and you still +// want the zlib decoder to be available, #define STBI_SUPPORT_ZLIB +// +// - If you define STBI_MAX_DIMENSIONS, stb_image will reject images greater +// than that size (in either width or height) without further processing. +// This is to let programs in the wild set an upper bound to prevent +// denial-of-service attacks on untrusted data, as one could generate a +// valid image of gigantic dimensions and force stb_image to allocate a +// huge block of memory and spend disproportionate time decoding it. By +// default this is set to (1 << 24), which is 16777216, but that's still +// very big. + +#ifndef STBI_NO_STDIO +#include +#endif // STBI_NO_STDIO + +#define STBI_VERSION 1 + +enum +{ + STBI_default = 0, // only used for desired_channels + + STBI_grey = 1, + STBI_grey_alpha = 2, + STBI_rgb = 3, + STBI_rgb_alpha = 4 +}; + +#include +typedef unsigned char stbi_uc; +typedef unsigned short stbi_us; + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef STBIDEF +#ifdef STB_IMAGE_STATIC +#define STBIDEF static +#else +#define STBIDEF extern +#endif +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// PRIMARY API - works on images of any type +// + +// +// load image by filename, open file, or memory buffer +// + +typedef struct +{ + int (*read) (void *user,char *data,int size); // fill 'data' with 'size' bytes. return number of bytes actually read + void (*skip) (void *user,int n); // skip the next 'n' bytes, or 'unget' the last -n bytes if negative + int (*eof) (void *user); // returns nonzero if we are at end of file/data +} stbi_io_callbacks; + +//////////////////////////////////// +// +// 8-bits-per-channel interface +// + +STBIDEF stbi_uc *stbi_load_from_memory (stbi_uc const *buffer, int len , int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk , void *user, int *x, int *y, int *channels_in_file, int desired_channels); + +#ifndef STBI_NO_STDIO +STBIDEF stbi_uc *stbi_load (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_uc *stbi_load_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); +// for stbi_load_from_file, file pointer is left pointing immediately after image +#endif + +#ifndef STBI_NO_GIF +STBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp); +#endif + +#ifdef STBI_WINDOWS_UTF8 +STBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input); +#endif + +//////////////////////////////////// +// +// 16-bits-per-channel interface +// + +STBIDEF stbi_us *stbi_load_16_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels); + +#ifndef STBI_NO_STDIO +STBIDEF stbi_us *stbi_load_16 (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_us *stbi_load_from_file_16(FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); +#endif + +//////////////////////////////////// +// +// float-per-channel interface +// +#ifndef STBI_NO_LINEAR + STBIDEF float *stbi_loadf_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels); + STBIDEF float *stbi_loadf_from_callbacks (stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels); + + #ifndef STBI_NO_STDIO + STBIDEF float *stbi_loadf (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); + STBIDEF float *stbi_loadf_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); + #endif +#endif + +#ifndef STBI_NO_HDR + STBIDEF void stbi_hdr_to_ldr_gamma(float gamma); + STBIDEF void stbi_hdr_to_ldr_scale(float scale); +#endif // STBI_NO_HDR + +#ifndef STBI_NO_LINEAR + STBIDEF void stbi_ldr_to_hdr_gamma(float gamma); + STBIDEF void stbi_ldr_to_hdr_scale(float scale); +#endif // STBI_NO_LINEAR + +// stbi_is_hdr is always defined, but always returns false if STBI_NO_HDR +STBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user); +STBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len); +#ifndef STBI_NO_STDIO +STBIDEF int stbi_is_hdr (char const *filename); +STBIDEF int stbi_is_hdr_from_file(FILE *f); +#endif // STBI_NO_STDIO + + +// get a VERY brief reason for failure +// on most compilers (and ALL modern mainstream compilers) this is threadsafe +STBIDEF const char *stbi_failure_reason (void); + +// free the loaded image -- this is just free() +STBIDEF void stbi_image_free (void *retval_from_stbi_load); + +// get image dimensions & components without fully decoding +STBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp); +STBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp); +STBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len); +STBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *clbk, void *user); + +#ifndef STBI_NO_STDIO +STBIDEF int stbi_info (char const *filename, int *x, int *y, int *comp); +STBIDEF int stbi_info_from_file (FILE *f, int *x, int *y, int *comp); +STBIDEF int stbi_is_16_bit (char const *filename); +STBIDEF int stbi_is_16_bit_from_file(FILE *f); +#endif + + + +// for image formats that explicitly notate that they have premultiplied alpha, +// we just return the colors as stored in the file. set this flag to force +// unpremultiplication. results are undefined if the unpremultiply overflow. +STBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply); + +// indicate whether we should process iphone images back to canonical format, +// or just pass them through "as-is" +STBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert); + +// flip the image vertically, so the first pixel in the output array is the bottom left +STBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip); + +// as above, but only applies to images loaded on the thread that calls the function +// this function is only available if your compiler supports thread-local variables; +// calling it will fail to link if your compiler doesn't +STBIDEF void stbi_set_unpremultiply_on_load_thread(int flag_true_if_should_unpremultiply); +STBIDEF void stbi_convert_iphone_png_to_rgb_thread(int flag_true_if_should_convert); +STBIDEF void stbi_set_flip_vertically_on_load_thread(int flag_true_if_should_flip); + +// ZLIB client - used by PNG, available for other purposes + +STBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen); +STBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header); +STBIDEF char *stbi_zlib_decode_malloc(const char *buffer, int len, int *outlen); +STBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, const char *ibuffer, int ilen); + +STBIDEF char *stbi_zlib_decode_noheader_malloc(const char *buffer, int len, int *outlen); +STBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen); + + +#ifdef __cplusplus +} +#endif + +// +// +//// end header file ///////////////////////////////////////////////////// +#endif // STBI_INCLUDE_STB_IMAGE_H + +#ifdef STB_IMAGE_IMPLEMENTATION + +#if defined(STBI_ONLY_JPEG) || defined(STBI_ONLY_PNG) || defined(STBI_ONLY_BMP) \ + || defined(STBI_ONLY_TGA) || defined(STBI_ONLY_GIF) || defined(STBI_ONLY_PSD) \ + || defined(STBI_ONLY_HDR) || defined(STBI_ONLY_PIC) || defined(STBI_ONLY_PNM) \ + || defined(STBI_ONLY_ZLIB) + #ifndef STBI_ONLY_JPEG + #define STBI_NO_JPEG + #endif + #ifndef STBI_ONLY_PNG + #define STBI_NO_PNG + #endif + #ifndef STBI_ONLY_BMP + #define STBI_NO_BMP + #endif + #ifndef STBI_ONLY_PSD + #define STBI_NO_PSD + #endif + #ifndef STBI_ONLY_TGA + #define STBI_NO_TGA + #endif + #ifndef STBI_ONLY_GIF + #define STBI_NO_GIF + #endif + #ifndef STBI_ONLY_HDR + #define STBI_NO_HDR + #endif + #ifndef STBI_ONLY_PIC + #define STBI_NO_PIC + #endif + #ifndef STBI_ONLY_PNM + #define STBI_NO_PNM + #endif +#endif + +#if defined(STBI_NO_PNG) && !defined(STBI_SUPPORT_ZLIB) && !defined(STBI_NO_ZLIB) +#define STBI_NO_ZLIB +#endif + + +#include +#include // ptrdiff_t on osx +#include +#include +#include + +#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) +#include // ldexp, pow +#endif + +#ifndef STBI_NO_STDIO +#include +#endif + +#ifndef STBI_ASSERT +#include +#define STBI_ASSERT(x) assert(x) +#endif + +#ifdef __cplusplus +#define STBI_EXTERN extern "C" +#else +#define STBI_EXTERN extern +#endif + + +#ifndef _MSC_VER + #ifdef __cplusplus + #define stbi_inline inline + #else + #define stbi_inline + #endif +#else + #define stbi_inline __forceinline +#endif + +#ifndef STBI_NO_THREAD_LOCALS + #if defined(__cplusplus) && __cplusplus >= 201103L + #define STBI_THREAD_LOCAL thread_local + #elif defined(__GNUC__) && __GNUC__ < 5 + #define STBI_THREAD_LOCAL __thread + #elif defined(_MSC_VER) + #define STBI_THREAD_LOCAL __declspec(thread) + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) + #define STBI_THREAD_LOCAL _Thread_local + #endif + + #ifndef STBI_THREAD_LOCAL + #if defined(__GNUC__) + #define STBI_THREAD_LOCAL __thread + #endif + #endif +#endif + +#if defined(_MSC_VER) || defined(__SYMBIAN32__) +typedef unsigned short stbi__uint16; +typedef signed short stbi__int16; +typedef unsigned int stbi__uint32; +typedef signed int stbi__int32; +#else +#include +typedef uint16_t stbi__uint16; +typedef int16_t stbi__int16; +typedef uint32_t stbi__uint32; +typedef int32_t stbi__int32; +#endif + +// should produce compiler error if size is wrong +typedef unsigned char validate_uint32[sizeof(stbi__uint32)==4 ? 1 : -1]; + +#ifdef _MSC_VER +#define STBI_NOTUSED(v) (void)(v) +#else +#define STBI_NOTUSED(v) (void)sizeof(v) +#endif + +#ifdef _MSC_VER +#define STBI_HAS_LROTL +#endif + +#ifdef STBI_HAS_LROTL + #define stbi_lrot(x,y) _lrotl(x,y) +#else + #define stbi_lrot(x,y) (((x) << (y)) | ((x) >> (-(y) & 31))) +#endif + +#if defined(STBI_MALLOC) && defined(STBI_FREE) && (defined(STBI_REALLOC) || defined(STBI_REALLOC_SIZED)) +// ok +#elif !defined(STBI_MALLOC) && !defined(STBI_FREE) && !defined(STBI_REALLOC) && !defined(STBI_REALLOC_SIZED) +// ok +#else +#error "Must define all or none of STBI_MALLOC, STBI_FREE, and STBI_REALLOC (or STBI_REALLOC_SIZED)." +#endif + +#ifndef STBI_MALLOC +#define STBI_MALLOC(sz) malloc(sz) +#define STBI_REALLOC(p,newsz) realloc(p,newsz) +#define STBI_FREE(p) free(p) +#endif + +#ifndef STBI_REALLOC_SIZED +#define STBI_REALLOC_SIZED(p,oldsz,newsz) STBI_REALLOC(p,newsz) +#endif + +// x86/x64 detection +#if defined(__x86_64__) || defined(_M_X64) +#define STBI__X64_TARGET +#elif defined(__i386) || defined(_M_IX86) +#define STBI__X86_TARGET +#endif + +#if defined(__GNUC__) && defined(STBI__X86_TARGET) && !defined(__SSE2__) && !defined(STBI_NO_SIMD) +// gcc doesn't support sse2 intrinsics unless you compile with -msse2, +// which in turn means it gets to use SSE2 everywhere. This is unfortunate, +// but previous attempts to provide the SSE2 functions with runtime +// detection caused numerous issues. The way architecture extensions are +// exposed in GCC/Clang is, sadly, not really suited for one-file libs. +// New behavior: if compiled with -msse2, we use SSE2 without any +// detection; if not, we don't use it at all. +#define STBI_NO_SIMD +#endif + +#if defined(__MINGW32__) && defined(STBI__X86_TARGET) && !defined(STBI_MINGW_ENABLE_SSE2) && !defined(STBI_NO_SIMD) +// Note that __MINGW32__ doesn't actually mean 32-bit, so we have to avoid STBI__X64_TARGET +// +// 32-bit MinGW wants ESP to be 16-byte aligned, but this is not in the +// Windows ABI and VC++ as well as Windows DLLs don't maintain that invariant. +// As a result, enabling SSE2 on 32-bit MinGW is dangerous when not +// simultaneously enabling "-mstackrealign". +// +// See https://github.com/nothings/stb/issues/81 for more information. +// +// So default to no SSE2 on 32-bit MinGW. If you've read this far and added +// -mstackrealign to your build settings, feel free to #define STBI_MINGW_ENABLE_SSE2. +#define STBI_NO_SIMD +#endif + +#if !defined(STBI_NO_SIMD) && (defined(STBI__X86_TARGET) || defined(STBI__X64_TARGET)) +#define STBI_SSE2 +#include + +#ifdef _MSC_VER + +#if _MSC_VER >= 1400 // not VC6 +#include // __cpuid +static int stbi__cpuid3(void) +{ + int info[4]; + __cpuid(info,1); + return info[3]; +} +#else +static int stbi__cpuid3(void) +{ + int res; + __asm { + mov eax,1 + cpuid + mov res,edx + } + return res; +} +#endif + +#define STBI_SIMD_ALIGN(type, name) __declspec(align(16)) type name + +#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2) +static int stbi__sse2_available(void) +{ + int info3 = stbi__cpuid3(); + return ((info3 >> 26) & 1) != 0; +} +#endif + +#else // assume GCC-style if not VC++ +#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16))) + +#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2) +static int stbi__sse2_available(void) +{ + // If we're even attempting to compile this on GCC/Clang, that means + // -msse2 is on, which means the compiler is allowed to use SSE2 + // instructions at will, and so are we. + return 1; +} +#endif + +#endif +#endif + +// ARM NEON +#if defined(STBI_NO_SIMD) && defined(STBI_NEON) +#undef STBI_NEON +#endif + +#ifdef STBI_NEON +#include +#ifdef _MSC_VER +#define STBI_SIMD_ALIGN(type, name) __declspec(align(16)) type name +#else +#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16))) +#endif +#endif + +#ifndef STBI_SIMD_ALIGN +#define STBI_SIMD_ALIGN(type, name) type name +#endif + +#ifndef STBI_MAX_DIMENSIONS +#define STBI_MAX_DIMENSIONS (1 << 24) +#endif + +/////////////////////////////////////////////// +// +// stbi__context struct and start_xxx functions + +// stbi__context structure is our basic context used by all images, so it +// contains all the IO context, plus some basic image information +typedef struct +{ + stbi__uint32 img_x, img_y; + int img_n, img_out_n; + + stbi_io_callbacks io; + void *io_user_data; + + int read_from_callbacks; + int buflen; + stbi_uc buffer_start[128]; + int callback_already_read; + + stbi_uc *img_buffer, *img_buffer_end; + stbi_uc *img_buffer_original, *img_buffer_original_end; +} stbi__context; + + +static void stbi__refill_buffer(stbi__context *s); + +// initialize a memory-decode context +static void stbi__start_mem(stbi__context *s, stbi_uc const *buffer, int len) +{ + s->io.read = NULL; + s->read_from_callbacks = 0; + s->callback_already_read = 0; + s->img_buffer = s->img_buffer_original = (stbi_uc *) buffer; + s->img_buffer_end = s->img_buffer_original_end = (stbi_uc *) buffer+len; +} + +// initialize a callback-based context +static void stbi__start_callbacks(stbi__context *s, stbi_io_callbacks *c, void *user) +{ + s->io = *c; + s->io_user_data = user; + s->buflen = sizeof(s->buffer_start); + s->read_from_callbacks = 1; + s->callback_already_read = 0; + s->img_buffer = s->img_buffer_original = s->buffer_start; + stbi__refill_buffer(s); + s->img_buffer_original_end = s->img_buffer_end; +} + +#ifndef STBI_NO_STDIO + +static int stbi__stdio_read(void *user, char *data, int size) +{ + return (int) fread(data,1,size,(FILE*) user); +} + +static void stbi__stdio_skip(void *user, int n) +{ + int ch; + fseek((FILE*) user, n, SEEK_CUR); + ch = fgetc((FILE*) user); /* have to read a byte to reset feof()'s flag */ + if (ch != EOF) { + ungetc(ch, (FILE *) user); /* push byte back onto stream if valid. */ + } +} + +static int stbi__stdio_eof(void *user) +{ + return feof((FILE*) user) || ferror((FILE *) user); +} + +static stbi_io_callbacks stbi__stdio_callbacks = +{ + stbi__stdio_read, + stbi__stdio_skip, + stbi__stdio_eof, +}; + +static void stbi__start_file(stbi__context *s, FILE *f) +{ + stbi__start_callbacks(s, &stbi__stdio_callbacks, (void *) f); +} + +//static void stop_file(stbi__context *s) { } + +#endif // !STBI_NO_STDIO + +static void stbi__rewind(stbi__context *s) +{ + // conceptually rewind SHOULD rewind to the beginning of the stream, + // but we just rewind to the beginning of the initial buffer, because + // we only use it after doing 'test', which only ever looks at at most 92 bytes + s->img_buffer = s->img_buffer_original; + s->img_buffer_end = s->img_buffer_original_end; +} + +enum +{ + STBI_ORDER_RGB, + STBI_ORDER_BGR +}; + +typedef struct +{ + int bits_per_channel; + int num_channels; + int channel_order; +} stbi__result_info; + +#ifndef STBI_NO_JPEG +static int stbi__jpeg_test(stbi__context *s); +static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PNG +static int stbi__png_test(stbi__context *s); +static void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__png_info(stbi__context *s, int *x, int *y, int *comp); +static int stbi__png_is16(stbi__context *s); +#endif + +#ifndef STBI_NO_BMP +static int stbi__bmp_test(stbi__context *s); +static void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_TGA +static int stbi__tga_test(stbi__context *s); +static void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PSD +static int stbi__psd_test(stbi__context *s); +static void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc); +static int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp); +static int stbi__psd_is16(stbi__context *s); +#endif + +#ifndef STBI_NO_HDR +static int stbi__hdr_test(stbi__context *s); +static float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PIC +static int stbi__pic_test(stbi__context *s); +static void *stbi__pic_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_GIF +static int stbi__gif_test(stbi__context *s); +static void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp); +static int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PNM +static int stbi__pnm_test(stbi__context *s); +static void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp); +static int stbi__pnm_is16(stbi__context *s); +#endif + +static +#ifdef STBI_THREAD_LOCAL +STBI_THREAD_LOCAL +#endif +const char *stbi__g_failure_reason; + +STBIDEF const char *stbi_failure_reason(void) +{ + return stbi__g_failure_reason; +} + +#ifndef STBI_NO_FAILURE_STRINGS +static int stbi__err(const char *str) +{ + stbi__g_failure_reason = str; + return 0; +} +#endif + +static void *stbi__malloc(size_t size) +{ + return STBI_MALLOC(size); +} + +// stb_image uses ints pervasively, including for offset calculations. +// therefore the largest decoded image size we can support with the +// current code, even on 64-bit targets, is INT_MAX. this is not a +// significant limitation for the intended use case. +// +// we do, however, need to make sure our size calculations don't +// overflow. hence a few helper functions for size calculations that +// multiply integers together, making sure that they're non-negative +// and no overflow occurs. + +// return 1 if the sum is valid, 0 on overflow. +// negative terms are considered invalid. +static int stbi__addsizes_valid(int a, int b) +{ + if (b < 0) return 0; + // now 0 <= b <= INT_MAX, hence also + // 0 <= INT_MAX - b <= INTMAX. + // And "a + b <= INT_MAX" (which might overflow) is the + // same as a <= INT_MAX - b (no overflow) + return a <= INT_MAX - b; +} + +// returns 1 if the product is valid, 0 on overflow. +// negative factors are considered invalid. +static int stbi__mul2sizes_valid(int a, int b) +{ + if (a < 0 || b < 0) return 0; + if (b == 0) return 1; // mul-by-0 is always safe + // portable way to check for no overflows in a*b + return a <= INT_MAX/b; +} + +#if !defined(STBI_NO_JPEG) || !defined(STBI_NO_PNG) || !defined(STBI_NO_TGA) || !defined(STBI_NO_HDR) +// returns 1 if "a*b + add" has no negative terms/factors and doesn't overflow +static int stbi__mad2sizes_valid(int a, int b, int add) +{ + return stbi__mul2sizes_valid(a, b) && stbi__addsizes_valid(a*b, add); +} +#endif + +// returns 1 if "a*b*c + add" has no negative terms/factors and doesn't overflow +static int stbi__mad3sizes_valid(int a, int b, int c, int add) +{ + return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) && + stbi__addsizes_valid(a*b*c, add); +} + +// returns 1 if "a*b*c*d + add" has no negative terms/factors and doesn't overflow +#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) || !defined(STBI_NO_PNM) +static int stbi__mad4sizes_valid(int a, int b, int c, int d, int add) +{ + return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) && + stbi__mul2sizes_valid(a*b*c, d) && stbi__addsizes_valid(a*b*c*d, add); +} +#endif + +#if !defined(STBI_NO_JPEG) || !defined(STBI_NO_PNG) || !defined(STBI_NO_TGA) || !defined(STBI_NO_HDR) +// mallocs with size overflow checking +static void *stbi__malloc_mad2(int a, int b, int add) +{ + if (!stbi__mad2sizes_valid(a, b, add)) return NULL; + return stbi__malloc(a*b + add); +} +#endif + +static void *stbi__malloc_mad3(int a, int b, int c, int add) +{ + if (!stbi__mad3sizes_valid(a, b, c, add)) return NULL; + return stbi__malloc(a*b*c + add); +} + +#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) || !defined(STBI_NO_PNM) +static void *stbi__malloc_mad4(int a, int b, int c, int d, int add) +{ + if (!stbi__mad4sizes_valid(a, b, c, d, add)) return NULL; + return stbi__malloc(a*b*c*d + add); +} +#endif + +// returns 1 if the sum of two signed ints is valid (between -2^31 and 2^31-1 inclusive), 0 on overflow. +static int stbi__addints_valid(int a, int b) +{ + if ((a >= 0) != (b >= 0)) return 1; // a and b have different signs, so no overflow + if (a < 0 && b < 0) return a >= INT_MIN - b; // same as a + b >= INT_MIN; INT_MIN - b cannot overflow since b < 0. + return a <= INT_MAX - b; +} + +// returns 1 if the product of two ints fits in a signed short, 0 on overflow. +static int stbi__mul2shorts_valid(int a, int b) +{ + if (b == 0 || b == -1) return 1; // multiplication by 0 is always 0; check for -1 so SHRT_MIN/b doesn't overflow + if ((a >= 0) == (b >= 0)) return a <= SHRT_MAX/b; // product is positive, so similar to mul2sizes_valid + if (b < 0) return a <= SHRT_MIN / b; // same as a * b >= SHRT_MIN + return a >= SHRT_MIN / b; +} + +// stbi__err - error +// stbi__errpf - error returning pointer to float +// stbi__errpuc - error returning pointer to unsigned char + +#ifdef STBI_NO_FAILURE_STRINGS + #define stbi__err(x,y) 0 +#elif defined(STBI_FAILURE_USERMSG) + #define stbi__err(x,y) stbi__err(y) +#else + #define stbi__err(x,y) stbi__err(x) +#endif + +#define stbi__errpf(x,y) ((float *)(size_t) (stbi__err(x,y)?NULL:NULL)) +#define stbi__errpuc(x,y) ((unsigned char *)(size_t) (stbi__err(x,y)?NULL:NULL)) + +STBIDEF void stbi_image_free(void *retval_from_stbi_load) +{ + STBI_FREE(retval_from_stbi_load); +} + +#ifndef STBI_NO_LINEAR +static float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp); +#endif + +#ifndef STBI_NO_HDR +static stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp); +#endif + +static int stbi__vertically_flip_on_load_global = 0; + +STBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip) +{ + stbi__vertically_flip_on_load_global = flag_true_if_should_flip; +} + +#ifndef STBI_THREAD_LOCAL +#define stbi__vertically_flip_on_load stbi__vertically_flip_on_load_global +#else +static STBI_THREAD_LOCAL int stbi__vertically_flip_on_load_local, stbi__vertically_flip_on_load_set; + +STBIDEF void stbi_set_flip_vertically_on_load_thread(int flag_true_if_should_flip) +{ + stbi__vertically_flip_on_load_local = flag_true_if_should_flip; + stbi__vertically_flip_on_load_set = 1; +} + +#define stbi__vertically_flip_on_load (stbi__vertically_flip_on_load_set \ + ? stbi__vertically_flip_on_load_local \ + : stbi__vertically_flip_on_load_global) +#endif // STBI_THREAD_LOCAL + +static void *stbi__load_main(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc) +{ + memset(ri, 0, sizeof(*ri)); // make sure it's initialized if we add new fields + ri->bits_per_channel = 8; // default is 8 so most paths don't have to be changed + ri->channel_order = STBI_ORDER_RGB; // all current input & output are this, but this is here so we can add BGR order + ri->num_channels = 0; + + // test the formats with a very explicit header first (at least a FOURCC + // or distinctive magic number first) + #ifndef STBI_NO_PNG + if (stbi__png_test(s)) return stbi__png_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_BMP + if (stbi__bmp_test(s)) return stbi__bmp_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_GIF + if (stbi__gif_test(s)) return stbi__gif_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_PSD + if (stbi__psd_test(s)) return stbi__psd_load(s,x,y,comp,req_comp, ri, bpc); + #else + STBI_NOTUSED(bpc); + #endif + #ifndef STBI_NO_PIC + if (stbi__pic_test(s)) return stbi__pic_load(s,x,y,comp,req_comp, ri); + #endif + + // then the formats that can end up attempting to load with just 1 or 2 + // bytes matching expectations; these are prone to false positives, so + // try them later + #ifndef STBI_NO_JPEG + if (stbi__jpeg_test(s)) return stbi__jpeg_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_PNM + if (stbi__pnm_test(s)) return stbi__pnm_load(s,x,y,comp,req_comp, ri); + #endif + + #ifndef STBI_NO_HDR + if (stbi__hdr_test(s)) { + float *hdr = stbi__hdr_load(s, x,y,comp,req_comp, ri); + return stbi__hdr_to_ldr(hdr, *x, *y, req_comp ? req_comp : *comp); + } + #endif + + #ifndef STBI_NO_TGA + // test tga last because it's a crappy test! + if (stbi__tga_test(s)) + return stbi__tga_load(s,x,y,comp,req_comp, ri); + #endif + + return stbi__errpuc("unknown image type", "Image not of any known type, or corrupt"); +} + +static stbi_uc *stbi__convert_16_to_8(stbi__uint16 *orig, int w, int h, int channels) +{ + int i; + int img_len = w * h * channels; + stbi_uc *reduced; + + reduced = (stbi_uc *) stbi__malloc(img_len); + if (reduced == NULL) return stbi__errpuc("outofmem", "Out of memory"); + + for (i = 0; i < img_len; ++i) + reduced[i] = (stbi_uc)((orig[i] >> 8) & 0xFF); // top half of each byte is sufficient approx of 16->8 bit scaling + + STBI_FREE(orig); + return reduced; +} + +static stbi__uint16 *stbi__convert_8_to_16(stbi_uc *orig, int w, int h, int channels) +{ + int i; + int img_len = w * h * channels; + stbi__uint16 *enlarged; + + enlarged = (stbi__uint16 *) stbi__malloc(img_len*2); + if (enlarged == NULL) return (stbi__uint16 *) stbi__errpuc("outofmem", "Out of memory"); + + for (i = 0; i < img_len; ++i) + enlarged[i] = (stbi__uint16)((orig[i] << 8) + orig[i]); // replicate to high and low byte, maps 0->0, 255->0xffff + + STBI_FREE(orig); + return enlarged; +} + +static void stbi__vertical_flip(void *image, int w, int h, int bytes_per_pixel) +{ + int row; + size_t bytes_per_row = (size_t)w * bytes_per_pixel; + stbi_uc temp[2048]; + stbi_uc *bytes = (stbi_uc *)image; + + for (row = 0; row < (h>>1); row++) { + stbi_uc *row0 = bytes + row*bytes_per_row; + stbi_uc *row1 = bytes + (h - row - 1)*bytes_per_row; + // swap row0 with row1 + size_t bytes_left = bytes_per_row; + while (bytes_left) { + size_t bytes_copy = (bytes_left < sizeof(temp)) ? bytes_left : sizeof(temp); + memcpy(temp, row0, bytes_copy); + memcpy(row0, row1, bytes_copy); + memcpy(row1, temp, bytes_copy); + row0 += bytes_copy; + row1 += bytes_copy; + bytes_left -= bytes_copy; + } + } +} + +#ifndef STBI_NO_GIF +static void stbi__vertical_flip_slices(void *image, int w, int h, int z, int bytes_per_pixel) +{ + int slice; + int slice_size = w * h * bytes_per_pixel; + + stbi_uc *bytes = (stbi_uc *)image; + for (slice = 0; slice < z; ++slice) { + stbi__vertical_flip(bytes, w, h, bytes_per_pixel); + bytes += slice_size; + } +} +#endif + +static unsigned char *stbi__load_and_postprocess_8bit(stbi__context *s, int *x, int *y, int *comp, int req_comp) +{ + stbi__result_info ri; + void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 8); + + if (result == NULL) + return NULL; + + // it is the responsibility of the loaders to make sure we get either 8 or 16 bit. + STBI_ASSERT(ri.bits_per_channel == 8 || ri.bits_per_channel == 16); + + if (ri.bits_per_channel != 8) { + result = stbi__convert_16_to_8((stbi__uint16 *) result, *x, *y, req_comp == 0 ? *comp : req_comp); + ri.bits_per_channel = 8; + } + + // @TODO: move stbi__convert_format to here + + if (stbi__vertically_flip_on_load) { + int channels = req_comp ? req_comp : *comp; + stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi_uc)); + } + + return (unsigned char *) result; +} + +static stbi__uint16 *stbi__load_and_postprocess_16bit(stbi__context *s, int *x, int *y, int *comp, int req_comp) +{ + stbi__result_info ri; + void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 16); + + if (result == NULL) + return NULL; + + // it is the responsibility of the loaders to make sure we get either 8 or 16 bit. + STBI_ASSERT(ri.bits_per_channel == 8 || ri.bits_per_channel == 16); + + if (ri.bits_per_channel != 16) { + result = stbi__convert_8_to_16((stbi_uc *) result, *x, *y, req_comp == 0 ? *comp : req_comp); + ri.bits_per_channel = 16; + } + + // @TODO: move stbi__convert_format16 to here + // @TODO: special case RGB-to-Y (and RGBA-to-YA) for 8-bit-to-16-bit case to keep more precision + + if (stbi__vertically_flip_on_load) { + int channels = req_comp ? req_comp : *comp; + stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi__uint16)); + } + + return (stbi__uint16 *) result; +} + +#if !defined(STBI_NO_HDR) && !defined(STBI_NO_LINEAR) +static void stbi__float_postprocess(float *result, int *x, int *y, int *comp, int req_comp) +{ + if (stbi__vertically_flip_on_load && result != NULL) { + int channels = req_comp ? req_comp : *comp; + stbi__vertical_flip(result, *x, *y, channels * sizeof(float)); + } +} +#endif + +#ifndef STBI_NO_STDIO + +#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8) +STBI_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar(unsigned int cp, unsigned long flags, const char *str, int cbmb, wchar_t *widestr, int cchwide); +STBI_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, char *str, int cbmb, const char *defchar, int *used_default); +#endif + +#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8) +STBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input) +{ + return WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, (int) bufferlen, NULL, NULL); +} +#endif + +static FILE *stbi__fopen(char const *filename, char const *mode) +{ + FILE *f; +#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8) + wchar_t wMode[64]; + wchar_t wFilename[1024]; + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, sizeof(wFilename)/sizeof(*wFilename))) + return 0; + + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, sizeof(wMode)/sizeof(*wMode))) + return 0; + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != _wfopen_s(&f, wFilename, wMode)) + f = 0; +#else + f = _wfopen(wFilename, wMode); +#endif + +#elif defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != fopen_s(&f, filename, mode)) + f=0; +#else + f = fopen(filename, mode); +#endif + return f; +} + + +STBIDEF stbi_uc *stbi_load(char const *filename, int *x, int *y, int *comp, int req_comp) +{ + FILE *f = stbi__fopen(filename, "rb"); + unsigned char *result; + if (!f) return stbi__errpuc("can't fopen", "Unable to open file"); + result = stbi_load_from_file(f,x,y,comp,req_comp); + fclose(f); + return result; +} + +STBIDEF stbi_uc *stbi_load_from_file(FILE *f, int *x, int *y, int *comp, int req_comp) +{ + unsigned char *result; + stbi__context s; + stbi__start_file(&s,f); + result = stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); + if (result) { + // need to 'unget' all the characters in the IO buffer + fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR); + } + return result; +} + +STBIDEF stbi__uint16 *stbi_load_from_file_16(FILE *f, int *x, int *y, int *comp, int req_comp) +{ + stbi__uint16 *result; + stbi__context s; + stbi__start_file(&s,f); + result = stbi__load_and_postprocess_16bit(&s,x,y,comp,req_comp); + if (result) { + // need to 'unget' all the characters in the IO buffer + fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR); + } + return result; +} + +STBIDEF stbi_us *stbi_load_16(char const *filename, int *x, int *y, int *comp, int req_comp) +{ + FILE *f = stbi__fopen(filename, "rb"); + stbi__uint16 *result; + if (!f) return (stbi_us *) stbi__errpuc("can't fopen", "Unable to open file"); + result = stbi_load_from_file_16(f,x,y,comp,req_comp); + fclose(f); + return result; +} + + +#endif //!STBI_NO_STDIO + +STBIDEF stbi_us *stbi_load_16_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels); +} + +STBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *)clbk, user); + return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels); +} + +STBIDEF stbi_uc *stbi_load_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); +} + +STBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); + return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); +} + +#ifndef STBI_NO_GIF +STBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp) +{ + unsigned char *result; + stbi__context s; + stbi__start_mem(&s,buffer,len); + + result = (unsigned char*) stbi__load_gif_main(&s, delays, x, y, z, comp, req_comp); + if (stbi__vertically_flip_on_load) { + stbi__vertical_flip_slices( result, *x, *y, *z, *comp ); + } + + return result; +} +#endif + +#ifndef STBI_NO_LINEAR +static float *stbi__loadf_main(stbi__context *s, int *x, int *y, int *comp, int req_comp) +{ + unsigned char *data; + #ifndef STBI_NO_HDR + if (stbi__hdr_test(s)) { + stbi__result_info ri; + float *hdr_data = stbi__hdr_load(s,x,y,comp,req_comp, &ri); + if (hdr_data) + stbi__float_postprocess(hdr_data,x,y,comp,req_comp); + return hdr_data; + } + #endif + data = stbi__load_and_postprocess_8bit(s, x, y, comp, req_comp); + if (data) + return stbi__ldr_to_hdr(data, *x, *y, req_comp ? req_comp : *comp); + return stbi__errpf("unknown image type", "Image not of any known type, or corrupt"); +} + +STBIDEF float *stbi_loadf_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__loadf_main(&s,x,y,comp,req_comp); +} + +STBIDEF float *stbi_loadf_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); + return stbi__loadf_main(&s,x,y,comp,req_comp); +} + +#ifndef STBI_NO_STDIO +STBIDEF float *stbi_loadf(char const *filename, int *x, int *y, int *comp, int req_comp) +{ + float *result; + FILE *f = stbi__fopen(filename, "rb"); + if (!f) return stbi__errpf("can't fopen", "Unable to open file"); + result = stbi_loadf_from_file(f,x,y,comp,req_comp); + fclose(f); + return result; +} + +STBIDEF float *stbi_loadf_from_file(FILE *f, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_file(&s,f); + return stbi__loadf_main(&s,x,y,comp,req_comp); +} +#endif // !STBI_NO_STDIO + +#endif // !STBI_NO_LINEAR + +// these is-hdr-or-not is defined independent of whether STBI_NO_LINEAR is +// defined, for API simplicity; if STBI_NO_LINEAR is defined, it always +// reports false! + +STBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len) +{ + #ifndef STBI_NO_HDR + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__hdr_test(&s); + #else + STBI_NOTUSED(buffer); + STBI_NOTUSED(len); + return 0; + #endif +} + +#ifndef STBI_NO_STDIO +STBIDEF int stbi_is_hdr (char const *filename) +{ + FILE *f = stbi__fopen(filename, "rb"); + int result=0; + if (f) { + result = stbi_is_hdr_from_file(f); + fclose(f); + } + return result; +} + +STBIDEF int stbi_is_hdr_from_file(FILE *f) +{ + #ifndef STBI_NO_HDR + long pos = ftell(f); + int res; + stbi__context s; + stbi__start_file(&s,f); + res = stbi__hdr_test(&s); + fseek(f, pos, SEEK_SET); + return res; + #else + STBI_NOTUSED(f); + return 0; + #endif +} +#endif // !STBI_NO_STDIO + +STBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user) +{ + #ifndef STBI_NO_HDR + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); + return stbi__hdr_test(&s); + #else + STBI_NOTUSED(clbk); + STBI_NOTUSED(user); + return 0; + #endif +} + +#ifndef STBI_NO_LINEAR +static float stbi__l2h_gamma=2.2f, stbi__l2h_scale=1.0f; + +STBIDEF void stbi_ldr_to_hdr_gamma(float gamma) { stbi__l2h_gamma = gamma; } +STBIDEF void stbi_ldr_to_hdr_scale(float scale) { stbi__l2h_scale = scale; } +#endif + +static float stbi__h2l_gamma_i=1.0f/2.2f, stbi__h2l_scale_i=1.0f; + +STBIDEF void stbi_hdr_to_ldr_gamma(float gamma) { stbi__h2l_gamma_i = 1/gamma; } +STBIDEF void stbi_hdr_to_ldr_scale(float scale) { stbi__h2l_scale_i = 1/scale; } + + +////////////////////////////////////////////////////////////////////////////// +// +// Common code used by all image loaders +// + +enum +{ + STBI__SCAN_load=0, + STBI__SCAN_type, + STBI__SCAN_header +}; + +static void stbi__refill_buffer(stbi__context *s) +{ + int n = (s->io.read)(s->io_user_data,(char*)s->buffer_start,s->buflen); + s->callback_already_read += (int) (s->img_buffer - s->img_buffer_original); + if (n == 0) { + // at end of file, treat same as if from memory, but need to handle case + // where s->img_buffer isn't pointing to safe memory, e.g. 0-byte file + s->read_from_callbacks = 0; + s->img_buffer = s->buffer_start; + s->img_buffer_end = s->buffer_start+1; + *s->img_buffer = 0; + } else { + s->img_buffer = s->buffer_start; + s->img_buffer_end = s->buffer_start + n; + } +} + +stbi_inline static stbi_uc stbi__get8(stbi__context *s) +{ + if (s->img_buffer < s->img_buffer_end) + return *s->img_buffer++; + if (s->read_from_callbacks) { + stbi__refill_buffer(s); + return *s->img_buffer++; + } + return 0; +} + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_HDR) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM) +// nothing +#else +stbi_inline static int stbi__at_eof(stbi__context *s) +{ + if (s->io.read) { + if (!(s->io.eof)(s->io_user_data)) return 0; + // if feof() is true, check if buffer = end + // special case: we've only got the special 0 character at the end + if (s->read_from_callbacks == 0) return 1; + } + + return s->img_buffer >= s->img_buffer_end; +} +#endif + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) +// nothing +#else +static void stbi__skip(stbi__context *s, int n) +{ + if (n == 0) return; // already there! + if (n < 0) { + s->img_buffer = s->img_buffer_end; + return; + } + if (s->io.read) { + int blen = (int) (s->img_buffer_end - s->img_buffer); + if (blen < n) { + s->img_buffer = s->img_buffer_end; + (s->io.skip)(s->io_user_data, n - blen); + return; + } + } + s->img_buffer += n; +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_TGA) && defined(STBI_NO_HDR) && defined(STBI_NO_PNM) +// nothing +#else +static int stbi__getn(stbi__context *s, stbi_uc *buffer, int n) +{ + if (s->io.read) { + int blen = (int) (s->img_buffer_end - s->img_buffer); + if (blen < n) { + int res, count; + + memcpy(buffer, s->img_buffer, blen); + + count = (s->io.read)(s->io_user_data, (char*) buffer + blen, n - blen); + res = (count == (n-blen)); + s->img_buffer = s->img_buffer_end; + return res; + } + } + + if (s->img_buffer+n <= s->img_buffer_end) { + memcpy(buffer, s->img_buffer, n); + s->img_buffer += n; + return 1; + } else + return 0; +} +#endif + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_PSD) && defined(STBI_NO_PIC) +// nothing +#else +static int stbi__get16be(stbi__context *s) +{ + int z = stbi__get8(s); + return (z << 8) + stbi__get8(s); +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD) && defined(STBI_NO_PIC) +// nothing +#else +static stbi__uint32 stbi__get32be(stbi__context *s) +{ + stbi__uint32 z = stbi__get16be(s); + return (z << 16) + stbi__get16be(s); +} +#endif + +#if defined(STBI_NO_BMP) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) +// nothing +#else +static int stbi__get16le(stbi__context *s) +{ + int z = stbi__get8(s); + return z + (stbi__get8(s) << 8); +} +#endif + +#ifndef STBI_NO_BMP +static stbi__uint32 stbi__get32le(stbi__context *s) +{ + stbi__uint32 z = stbi__get16le(s); + z += (stbi__uint32)stbi__get16le(s) << 16; + return z; +} +#endif + +#define STBI__BYTECAST(x) ((stbi_uc) ((x) & 255)) // truncate int to byte without warnings + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM) +// nothing +#else +////////////////////////////////////////////////////////////////////////////// +// +// generic converter from built-in img_n to req_comp +// individual types do this automatically as much as possible (e.g. jpeg +// does all cases internally since it needs to colorspace convert anyway, +// and it never has alpha, so very few cases ). png can automatically +// interleave an alpha=255 channel, but falls back to this for other cases +// +// assume data buffer is malloced, so malloc a new one and free that one +// only failure mode is malloc failing + +static stbi_uc stbi__compute_y(int r, int g, int b) +{ + return (stbi_uc) (((r*77) + (g*150) + (29*b)) >> 8); +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM) +// nothing +#else +static unsigned char *stbi__convert_format(unsigned char *data, int img_n, int req_comp, unsigned int x, unsigned int y) +{ + int i,j; + unsigned char *good; + + if (req_comp == img_n) return data; + STBI_ASSERT(req_comp >= 1 && req_comp <= 4); + + good = (unsigned char *) stbi__malloc_mad3(req_comp, x, y, 0); + if (good == NULL) { + STBI_FREE(data); + return stbi__errpuc("outofmem", "Out of memory"); + } + + for (j=0; j < (int) y; ++j) { + unsigned char *src = data + j * x * img_n ; + unsigned char *dest = good + j * x * req_comp; + + #define STBI__COMBO(a,b) ((a)*8+(b)) + #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b) + // convert source image with img_n components to one with req_comp components; + // avoid switch per pixel, so use switch per scanline and massive macros + switch (STBI__COMBO(img_n, req_comp)) { + STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=255; } break; + STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=255; } break; + STBI__CASE(2,1) { dest[0]=src[0]; } break; + STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break; + STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=255; } break; + STBI__CASE(3,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break; + STBI__CASE(3,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = 255; } break; + STBI__CASE(4,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break; + STBI__CASE(4,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = src[3]; } break; + STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break; + default: STBI_ASSERT(0); STBI_FREE(data); STBI_FREE(good); return stbi__errpuc("unsupported", "Unsupported format conversion"); + } + #undef STBI__CASE + } + + STBI_FREE(data); + return good; +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD) +// nothing +#else +static stbi__uint16 stbi__compute_y_16(int r, int g, int b) +{ + return (stbi__uint16) (((r*77) + (g*150) + (29*b)) >> 8); +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD) +// nothing +#else +static stbi__uint16 *stbi__convert_format16(stbi__uint16 *data, int img_n, int req_comp, unsigned int x, unsigned int y) +{ + int i,j; + stbi__uint16 *good; + + if (req_comp == img_n) return data; + STBI_ASSERT(req_comp >= 1 && req_comp <= 4); + + good = (stbi__uint16 *) stbi__malloc(req_comp * x * y * 2); + if (good == NULL) { + STBI_FREE(data); + return (stbi__uint16 *) stbi__errpuc("outofmem", "Out of memory"); + } + + for (j=0; j < (int) y; ++j) { + stbi__uint16 *src = data + j * x * img_n ; + stbi__uint16 *dest = good + j * x * req_comp; + + #define STBI__COMBO(a,b) ((a)*8+(b)) + #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b) + // convert source image with img_n components to one with req_comp components; + // avoid switch per pixel, so use switch per scanline and massive macros + switch (STBI__COMBO(img_n, req_comp)) { + STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=0xffff; } break; + STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=0xffff; } break; + STBI__CASE(2,1) { dest[0]=src[0]; } break; + STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break; + STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=0xffff; } break; + STBI__CASE(3,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break; + STBI__CASE(3,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = 0xffff; } break; + STBI__CASE(4,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break; + STBI__CASE(4,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = src[3]; } break; + STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break; + default: STBI_ASSERT(0); STBI_FREE(data); STBI_FREE(good); return (stbi__uint16*) stbi__errpuc("unsupported", "Unsupported format conversion"); + } + #undef STBI__CASE + } + + STBI_FREE(data); + return good; +} +#endif + +#ifndef STBI_NO_LINEAR +static float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp) +{ + int i,k,n; + float *output; + if (!data) return NULL; + output = (float *) stbi__malloc_mad4(x, y, comp, sizeof(float), 0); + if (output == NULL) { STBI_FREE(data); return stbi__errpf("outofmem", "Out of memory"); } + // compute number of non-alpha components + if (comp & 1) n = comp; else n = comp-1; + for (i=0; i < x*y; ++i) { + for (k=0; k < n; ++k) { + output[i*comp + k] = (float) (pow(data[i*comp+k]/255.0f, stbi__l2h_gamma) * stbi__l2h_scale); + } + } + if (n < comp) { + for (i=0; i < x*y; ++i) { + output[i*comp + n] = data[i*comp + n]/255.0f; + } + } + STBI_FREE(data); + return output; +} +#endif + +#ifndef STBI_NO_HDR +#define stbi__float2int(x) ((int) (x)) +static stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp) +{ + int i,k,n; + stbi_uc *output; + if (!data) return NULL; + output = (stbi_uc *) stbi__malloc_mad3(x, y, comp, 0); + if (output == NULL) { STBI_FREE(data); return stbi__errpuc("outofmem", "Out of memory"); } + // compute number of non-alpha components + if (comp & 1) n = comp; else n = comp-1; + for (i=0; i < x*y; ++i) { + for (k=0; k < n; ++k) { + float z = (float) pow(data[i*comp+k]*stbi__h2l_scale_i, stbi__h2l_gamma_i) * 255 + 0.5f; + if (z < 0) z = 0; + if (z > 255) z = 255; + output[i*comp + k] = (stbi_uc) stbi__float2int(z); + } + if (k < comp) { + float z = data[i*comp+k] * 255 + 0.5f; + if (z < 0) z = 0; + if (z > 255) z = 255; + output[i*comp + k] = (stbi_uc) stbi__float2int(z); + } + } + STBI_FREE(data); + return output; +} +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// "baseline" JPEG/JFIF decoder +// +// simple implementation +// - doesn't support delayed output of y-dimension +// - simple interface (only one output format: 8-bit interleaved RGB) +// - doesn't try to recover corrupt jpegs +// - doesn't allow partial loading, loading multiple at once +// - still fast on x86 (copying globals into locals doesn't help x86) +// - allocates lots of intermediate memory (full size of all components) +// - non-interleaved case requires this anyway +// - allows good upsampling (see next) +// high-quality +// - upsampled channels are bilinearly interpolated, even across blocks +// - quality integer IDCT derived from IJG's 'slow' +// performance +// - fast huffman; reasonable integer IDCT +// - some SIMD kernels for common paths on targets with SSE2/NEON +// - uses a lot of intermediate memory, could cache poorly + +#ifndef STBI_NO_JPEG + +// huffman decoding acceleration +#define FAST_BITS 9 // larger handles more cases; smaller stomps less cache + +typedef struct +{ + stbi_uc fast[1 << FAST_BITS]; + // weirdly, repacking this into AoS is a 10% speed loss, instead of a win + stbi__uint16 code[256]; + stbi_uc values[256]; + stbi_uc size[257]; + unsigned int maxcode[18]; + int delta[17]; // old 'firstsymbol' - old 'firstcode' +} stbi__huffman; + +typedef struct +{ + stbi__context *s; + stbi__huffman huff_dc[4]; + stbi__huffman huff_ac[4]; + stbi__uint16 dequant[4][64]; + stbi__int16 fast_ac[4][1 << FAST_BITS]; + +// sizes for components, interleaved MCUs + int img_h_max, img_v_max; + int img_mcu_x, img_mcu_y; + int img_mcu_w, img_mcu_h; + +// definition of jpeg image component + struct + { + int id; + int h,v; + int tq; + int hd,ha; + int dc_pred; + + int x,y,w2,h2; + stbi_uc *data; + void *raw_data, *raw_coeff; + stbi_uc *linebuf; + short *coeff; // progressive only + int coeff_w, coeff_h; // number of 8x8 coefficient blocks + } img_comp[4]; + + stbi__uint32 code_buffer; // jpeg entropy-coded buffer + int code_bits; // number of valid bits + unsigned char marker; // marker seen while filling entropy buffer + int nomore; // flag if we saw a marker so must stop + + int progressive; + int spec_start; + int spec_end; + int succ_high; + int succ_low; + int eob_run; + int jfif; + int app14_color_transform; // Adobe APP14 tag + int rgb; + + int scan_n, order[4]; + int restart_interval, todo; + +// kernels + void (*idct_block_kernel)(stbi_uc *out, int out_stride, short data[64]); + void (*YCbCr_to_RGB_kernel)(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step); + stbi_uc *(*resample_row_hv_2_kernel)(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs); +} stbi__jpeg; + +static int stbi__build_huffman(stbi__huffman *h, int *count) +{ + int i,j,k=0; + unsigned int code; + // build size list for each symbol (from JPEG spec) + for (i=0; i < 16; ++i) { + for (j=0; j < count[i]; ++j) { + h->size[k++] = (stbi_uc) (i+1); + if(k >= 257) return stbi__err("bad size list","Corrupt JPEG"); + } + } + h->size[k] = 0; + + // compute actual symbols (from jpeg spec) + code = 0; + k = 0; + for(j=1; j <= 16; ++j) { + // compute delta to add to code to compute symbol id + h->delta[j] = k - code; + if (h->size[k] == j) { + while (h->size[k] == j) + h->code[k++] = (stbi__uint16) (code++); + if (code-1 >= (1u << j)) return stbi__err("bad code lengths","Corrupt JPEG"); + } + // compute largest code + 1 for this size, preshifted as needed later + h->maxcode[j] = code << (16-j); + code <<= 1; + } + h->maxcode[j] = 0xffffffff; + + // build non-spec acceleration table; 255 is flag for not-accelerated + memset(h->fast, 255, 1 << FAST_BITS); + for (i=0; i < k; ++i) { + int s = h->size[i]; + if (s <= FAST_BITS) { + int c = h->code[i] << (FAST_BITS-s); + int m = 1 << (FAST_BITS-s); + for (j=0; j < m; ++j) { + h->fast[c+j] = (stbi_uc) i; + } + } + } + return 1; +} + +// build a table that decodes both magnitude and value of small ACs in +// one go. +static void stbi__build_fast_ac(stbi__int16 *fast_ac, stbi__huffman *h) +{ + int i; + for (i=0; i < (1 << FAST_BITS); ++i) { + stbi_uc fast = h->fast[i]; + fast_ac[i] = 0; + if (fast < 255) { + int rs = h->values[fast]; + int run = (rs >> 4) & 15; + int magbits = rs & 15; + int len = h->size[fast]; + + if (magbits && len + magbits <= FAST_BITS) { + // magnitude code followed by receive_extend code + int k = ((i << len) & ((1 << FAST_BITS) - 1)) >> (FAST_BITS - magbits); + int m = 1 << (magbits - 1); + if (k < m) k += (~0U << magbits) + 1; + // if the result is small enough, we can fit it in fast_ac table + if (k >= -128 && k <= 127) + fast_ac[i] = (stbi__int16) ((k * 256) + (run * 16) + (len + magbits)); + } + } + } +} + +static void stbi__grow_buffer_unsafe(stbi__jpeg *j) +{ + do { + unsigned int b = j->nomore ? 0 : stbi__get8(j->s); + if (b == 0xff) { + int c = stbi__get8(j->s); + while (c == 0xff) c = stbi__get8(j->s); // consume fill bytes + if (c != 0) { + j->marker = (unsigned char) c; + j->nomore = 1; + return; + } + } + j->code_buffer |= b << (24 - j->code_bits); + j->code_bits += 8; + } while (j->code_bits <= 24); +} + +// (1 << n) - 1 +static const stbi__uint32 stbi__bmask[17]={0,1,3,7,15,31,63,127,255,511,1023,2047,4095,8191,16383,32767,65535}; + +// decode a jpeg huffman value from the bitstream +stbi_inline static int stbi__jpeg_huff_decode(stbi__jpeg *j, stbi__huffman *h) +{ + unsigned int temp; + int c,k; + + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + + // look at the top FAST_BITS and determine what symbol ID it is, + // if the code is <= FAST_BITS + c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); + k = h->fast[c]; + if (k < 255) { + int s = h->size[k]; + if (s > j->code_bits) + return -1; + j->code_buffer <<= s; + j->code_bits -= s; + return h->values[k]; + } + + // naive test is to shift the code_buffer down so k bits are + // valid, then test against maxcode. To speed this up, we've + // preshifted maxcode left so that it has (16-k) 0s at the + // end; in other words, regardless of the number of bits, it + // wants to be compared against something shifted to have 16; + // that way we don't need to shift inside the loop. + temp = j->code_buffer >> 16; + for (k=FAST_BITS+1 ; ; ++k) + if (temp < h->maxcode[k]) + break; + if (k == 17) { + // error! code not found + j->code_bits -= 16; + return -1; + } + + if (k > j->code_bits) + return -1; + + // convert the huffman code to the symbol id + c = ((j->code_buffer >> (32 - k)) & stbi__bmask[k]) + h->delta[k]; + if(c < 0 || c >= 256) // symbol id out of bounds! + return -1; + STBI_ASSERT((((j->code_buffer) >> (32 - h->size[c])) & stbi__bmask[h->size[c]]) == h->code[c]); + + // convert the id to a symbol + j->code_bits -= k; + j->code_buffer <<= k; + return h->values[c]; +} + +// bias[n] = (-1<code_bits < n) stbi__grow_buffer_unsafe(j); + if (j->code_bits < n) return 0; // ran out of bits from stream, return 0s intead of continuing + + sgn = j->code_buffer >> 31; // sign bit always in MSB; 0 if MSB clear (positive), 1 if MSB set (negative) + k = stbi_lrot(j->code_buffer, n); + j->code_buffer = k & ~stbi__bmask[n]; + k &= stbi__bmask[n]; + j->code_bits -= n; + return k + (stbi__jbias[n] & (sgn - 1)); +} + +// get some unsigned bits +stbi_inline static int stbi__jpeg_get_bits(stbi__jpeg *j, int n) +{ + unsigned int k; + if (j->code_bits < n) stbi__grow_buffer_unsafe(j); + if (j->code_bits < n) return 0; // ran out of bits from stream, return 0s intead of continuing + k = stbi_lrot(j->code_buffer, n); + j->code_buffer = k & ~stbi__bmask[n]; + k &= stbi__bmask[n]; + j->code_bits -= n; + return k; +} + +stbi_inline static int stbi__jpeg_get_bit(stbi__jpeg *j) +{ + unsigned int k; + if (j->code_bits < 1) stbi__grow_buffer_unsafe(j); + if (j->code_bits < 1) return 0; // ran out of bits from stream, return 0s intead of continuing + k = j->code_buffer; + j->code_buffer <<= 1; + --j->code_bits; + return k & 0x80000000; +} + +// given a value that's at position X in the zigzag stream, +// where does it appear in the 8x8 matrix coded as row-major? +static const stbi_uc stbi__jpeg_dezigzag[64+15] = +{ + 0, 1, 8, 16, 9, 2, 3, 10, + 17, 24, 32, 25, 18, 11, 4, 5, + 12, 19, 26, 33, 40, 48, 41, 34, + 27, 20, 13, 6, 7, 14, 21, 28, + 35, 42, 49, 56, 57, 50, 43, 36, + 29, 22, 15, 23, 30, 37, 44, 51, + 58, 59, 52, 45, 38, 31, 39, 46, + 53, 60, 61, 54, 47, 55, 62, 63, + // let corrupt input sample past end + 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 63 +}; + +// decode one 64-entry block-- +static int stbi__jpeg_decode_block(stbi__jpeg *j, short data[64], stbi__huffman *hdc, stbi__huffman *hac, stbi__int16 *fac, int b, stbi__uint16 *dequant) +{ + int diff,dc,k; + int t; + + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + t = stbi__jpeg_huff_decode(j, hdc); + if (t < 0 || t > 15) return stbi__err("bad huffman code","Corrupt JPEG"); + + // 0 all the ac values now so we can do it 32-bits at a time + memset(data,0,64*sizeof(data[0])); + + diff = t ? stbi__extend_receive(j, t) : 0; + if (!stbi__addints_valid(j->img_comp[b].dc_pred, diff)) return stbi__err("bad delta","Corrupt JPEG"); + dc = j->img_comp[b].dc_pred + diff; + j->img_comp[b].dc_pred = dc; + if (!stbi__mul2shorts_valid(dc, dequant[0])) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + data[0] = (short) (dc * dequant[0]); + + // decode AC components, see JPEG spec + k = 1; + do { + unsigned int zig; + int c,r,s; + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); + r = fac[c]; + if (r) { // fast-AC path + k += (r >> 4) & 15; // run + s = r & 15; // combined length + if (s > j->code_bits) return stbi__err("bad huffman code", "Combined length longer than code bits available"); + j->code_buffer <<= s; + j->code_bits -= s; + // decode into unzigzag'd location + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) ((r >> 8) * dequant[zig]); + } else { + int rs = stbi__jpeg_huff_decode(j, hac); + if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (rs != 0xf0) break; // end block + k += 16; + } else { + k += r; + // decode into unzigzag'd location + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) (stbi__extend_receive(j,s) * dequant[zig]); + } + } + } while (k < 64); + return 1; +} + +static int stbi__jpeg_decode_block_prog_dc(stbi__jpeg *j, short data[64], stbi__huffman *hdc, int b) +{ + int diff,dc; + int t; + if (j->spec_end != 0) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + + if (j->succ_high == 0) { + // first scan for DC coefficient, must be first + memset(data,0,64*sizeof(data[0])); // 0 all the ac values now + t = stbi__jpeg_huff_decode(j, hdc); + if (t < 0 || t > 15) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + diff = t ? stbi__extend_receive(j, t) : 0; + + if (!stbi__addints_valid(j->img_comp[b].dc_pred, diff)) return stbi__err("bad delta", "Corrupt JPEG"); + dc = j->img_comp[b].dc_pred + diff; + j->img_comp[b].dc_pred = dc; + if (!stbi__mul2shorts_valid(dc, 1 << j->succ_low)) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + data[0] = (short) (dc * (1 << j->succ_low)); + } else { + // refinement scan for DC coefficient + if (stbi__jpeg_get_bit(j)) + data[0] += (short) (1 << j->succ_low); + } + return 1; +} + +// @OPTIMIZE: store non-zigzagged during the decode passes, +// and only de-zigzag when dequantizing +static int stbi__jpeg_decode_block_prog_ac(stbi__jpeg *j, short data[64], stbi__huffman *hac, stbi__int16 *fac) +{ + int k; + if (j->spec_start == 0) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + + if (j->succ_high == 0) { + int shift = j->succ_low; + + if (j->eob_run) { + --j->eob_run; + return 1; + } + + k = j->spec_start; + do { + unsigned int zig; + int c,r,s; + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); + r = fac[c]; + if (r) { // fast-AC path + k += (r >> 4) & 15; // run + s = r & 15; // combined length + if (s > j->code_bits) return stbi__err("bad huffman code", "Combined length longer than code bits available"); + j->code_buffer <<= s; + j->code_bits -= s; + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) ((r >> 8) * (1 << shift)); + } else { + int rs = stbi__jpeg_huff_decode(j, hac); + if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (r < 15) { + j->eob_run = (1 << r); + if (r) + j->eob_run += stbi__jpeg_get_bits(j, r); + --j->eob_run; + break; + } + k += 16; + } else { + k += r; + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) (stbi__extend_receive(j,s) * (1 << shift)); + } + } + } while (k <= j->spec_end); + } else { + // refinement scan for these AC coefficients + + short bit = (short) (1 << j->succ_low); + + if (j->eob_run) { + --j->eob_run; + for (k = j->spec_start; k <= j->spec_end; ++k) { + short *p = &data[stbi__jpeg_dezigzag[k]]; + if (*p != 0) + if (stbi__jpeg_get_bit(j)) + if ((*p & bit)==0) { + if (*p > 0) + *p += bit; + else + *p -= bit; + } + } + } else { + k = j->spec_start; + do { + int r,s; + int rs = stbi__jpeg_huff_decode(j, hac); // @OPTIMIZE see if we can use the fast path here, advance-by-r is so slow, eh + if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (r < 15) { + j->eob_run = (1 << r) - 1; + if (r) + j->eob_run += stbi__jpeg_get_bits(j, r); + r = 64; // force end of block + } else { + // r=15 s=0 should write 16 0s, so we just do + // a run of 15 0s and then write s (which is 0), + // so we don't have to do anything special here + } + } else { + if (s != 1) return stbi__err("bad huffman code", "Corrupt JPEG"); + // sign bit + if (stbi__jpeg_get_bit(j)) + s = bit; + else + s = -bit; + } + + // advance by r + while (k <= j->spec_end) { + short *p = &data[stbi__jpeg_dezigzag[k++]]; + if (*p != 0) { + if (stbi__jpeg_get_bit(j)) + if ((*p & bit)==0) { + if (*p > 0) + *p += bit; + else + *p -= bit; + } + } else { + if (r == 0) { + *p = (short) s; + break; + } + --r; + } + } + } while (k <= j->spec_end); + } + } + return 1; +} + +// take a -128..127 value and stbi__clamp it and convert to 0..255 +stbi_inline static stbi_uc stbi__clamp(int x) +{ + // trick to use a single test to catch both cases + if ((unsigned int) x > 255) { + if (x < 0) return 0; + if (x > 255) return 255; + } + return (stbi_uc) x; +} + +#define stbi__f2f(x) ((int) (((x) * 4096 + 0.5))) +#define stbi__fsh(x) ((x) * 4096) + +// derived from jidctint -- DCT_ISLOW +#define STBI__IDCT_1D(s0,s1,s2,s3,s4,s5,s6,s7) \ + int t0,t1,t2,t3,p1,p2,p3,p4,p5,x0,x1,x2,x3; \ + p2 = s2; \ + p3 = s6; \ + p1 = (p2+p3) * stbi__f2f(0.5411961f); \ + t2 = p1 + p3*stbi__f2f(-1.847759065f); \ + t3 = p1 + p2*stbi__f2f( 0.765366865f); \ + p2 = s0; \ + p3 = s4; \ + t0 = stbi__fsh(p2+p3); \ + t1 = stbi__fsh(p2-p3); \ + x0 = t0+t3; \ + x3 = t0-t3; \ + x1 = t1+t2; \ + x2 = t1-t2; \ + t0 = s7; \ + t1 = s5; \ + t2 = s3; \ + t3 = s1; \ + p3 = t0+t2; \ + p4 = t1+t3; \ + p1 = t0+t3; \ + p2 = t1+t2; \ + p5 = (p3+p4)*stbi__f2f( 1.175875602f); \ + t0 = t0*stbi__f2f( 0.298631336f); \ + t1 = t1*stbi__f2f( 2.053119869f); \ + t2 = t2*stbi__f2f( 3.072711026f); \ + t3 = t3*stbi__f2f( 1.501321110f); \ + p1 = p5 + p1*stbi__f2f(-0.899976223f); \ + p2 = p5 + p2*stbi__f2f(-2.562915447f); \ + p3 = p3*stbi__f2f(-1.961570560f); \ + p4 = p4*stbi__f2f(-0.390180644f); \ + t3 += p1+p4; \ + t2 += p2+p3; \ + t1 += p2+p4; \ + t0 += p1+p3; + +static void stbi__idct_block(stbi_uc *out, int out_stride, short data[64]) +{ + int i,val[64],*v=val; + stbi_uc *o; + short *d = data; + + // columns + for (i=0; i < 8; ++i,++d, ++v) { + // if all zeroes, shortcut -- this avoids dequantizing 0s and IDCTing + if (d[ 8]==0 && d[16]==0 && d[24]==0 && d[32]==0 + && d[40]==0 && d[48]==0 && d[56]==0) { + // no shortcut 0 seconds + // (1|2|3|4|5|6|7)==0 0 seconds + // all separate -0.047 seconds + // 1 && 2|3 && 4|5 && 6|7: -0.047 seconds + int dcterm = d[0]*4; + v[0] = v[8] = v[16] = v[24] = v[32] = v[40] = v[48] = v[56] = dcterm; + } else { + STBI__IDCT_1D(d[ 0],d[ 8],d[16],d[24],d[32],d[40],d[48],d[56]) + // constants scaled things up by 1<<12; let's bring them back + // down, but keep 2 extra bits of precision + x0 += 512; x1 += 512; x2 += 512; x3 += 512; + v[ 0] = (x0+t3) >> 10; + v[56] = (x0-t3) >> 10; + v[ 8] = (x1+t2) >> 10; + v[48] = (x1-t2) >> 10; + v[16] = (x2+t1) >> 10; + v[40] = (x2-t1) >> 10; + v[24] = (x3+t0) >> 10; + v[32] = (x3-t0) >> 10; + } + } + + for (i=0, v=val, o=out; i < 8; ++i,v+=8,o+=out_stride) { + // no fast case since the first 1D IDCT spread components out + STBI__IDCT_1D(v[0],v[1],v[2],v[3],v[4],v[5],v[6],v[7]) + // constants scaled things up by 1<<12, plus we had 1<<2 from first + // loop, plus horizontal and vertical each scale by sqrt(8) so together + // we've got an extra 1<<3, so 1<<17 total we need to remove. + // so we want to round that, which means adding 0.5 * 1<<17, + // aka 65536. Also, we'll end up with -128 to 127 that we want + // to encode as 0..255 by adding 128, so we'll add that before the shift + x0 += 65536 + (128<<17); + x1 += 65536 + (128<<17); + x2 += 65536 + (128<<17); + x3 += 65536 + (128<<17); + // tried computing the shifts into temps, or'ing the temps to see + // if any were out of range, but that was slower + o[0] = stbi__clamp((x0+t3) >> 17); + o[7] = stbi__clamp((x0-t3) >> 17); + o[1] = stbi__clamp((x1+t2) >> 17); + o[6] = stbi__clamp((x1-t2) >> 17); + o[2] = stbi__clamp((x2+t1) >> 17); + o[5] = stbi__clamp((x2-t1) >> 17); + o[3] = stbi__clamp((x3+t0) >> 17); + o[4] = stbi__clamp((x3-t0) >> 17); + } +} + +#ifdef STBI_SSE2 +// sse2 integer IDCT. not the fastest possible implementation but it +// produces bit-identical results to the generic C version so it's +// fully "transparent". +static void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64]) +{ + // This is constructed to match our regular (generic) integer IDCT exactly. + __m128i row0, row1, row2, row3, row4, row5, row6, row7; + __m128i tmp; + + // dot product constant: even elems=x, odd elems=y + #define dct_const(x,y) _mm_setr_epi16((x),(y),(x),(y),(x),(y),(x),(y)) + + // out(0) = c0[even]*x + c0[odd]*y (c0, x, y 16-bit, out 32-bit) + // out(1) = c1[even]*x + c1[odd]*y + #define dct_rot(out0,out1, x,y,c0,c1) \ + __m128i c0##lo = _mm_unpacklo_epi16((x),(y)); \ + __m128i c0##hi = _mm_unpackhi_epi16((x),(y)); \ + __m128i out0##_l = _mm_madd_epi16(c0##lo, c0); \ + __m128i out0##_h = _mm_madd_epi16(c0##hi, c0); \ + __m128i out1##_l = _mm_madd_epi16(c0##lo, c1); \ + __m128i out1##_h = _mm_madd_epi16(c0##hi, c1) + + // out = in << 12 (in 16-bit, out 32-bit) + #define dct_widen(out, in) \ + __m128i out##_l = _mm_srai_epi32(_mm_unpacklo_epi16(_mm_setzero_si128(), (in)), 4); \ + __m128i out##_h = _mm_srai_epi32(_mm_unpackhi_epi16(_mm_setzero_si128(), (in)), 4) + + // wide add + #define dct_wadd(out, a, b) \ + __m128i out##_l = _mm_add_epi32(a##_l, b##_l); \ + __m128i out##_h = _mm_add_epi32(a##_h, b##_h) + + // wide sub + #define dct_wsub(out, a, b) \ + __m128i out##_l = _mm_sub_epi32(a##_l, b##_l); \ + __m128i out##_h = _mm_sub_epi32(a##_h, b##_h) + + // butterfly a/b, add bias, then shift by "s" and pack + #define dct_bfly32o(out0, out1, a,b,bias,s) \ + { \ + __m128i abiased_l = _mm_add_epi32(a##_l, bias); \ + __m128i abiased_h = _mm_add_epi32(a##_h, bias); \ + dct_wadd(sum, abiased, b); \ + dct_wsub(dif, abiased, b); \ + out0 = _mm_packs_epi32(_mm_srai_epi32(sum_l, s), _mm_srai_epi32(sum_h, s)); \ + out1 = _mm_packs_epi32(_mm_srai_epi32(dif_l, s), _mm_srai_epi32(dif_h, s)); \ + } + + // 8-bit interleave step (for transposes) + #define dct_interleave8(a, b) \ + tmp = a; \ + a = _mm_unpacklo_epi8(a, b); \ + b = _mm_unpackhi_epi8(tmp, b) + + // 16-bit interleave step (for transposes) + #define dct_interleave16(a, b) \ + tmp = a; \ + a = _mm_unpacklo_epi16(a, b); \ + b = _mm_unpackhi_epi16(tmp, b) + + #define dct_pass(bias,shift) \ + { \ + /* even part */ \ + dct_rot(t2e,t3e, row2,row6, rot0_0,rot0_1); \ + __m128i sum04 = _mm_add_epi16(row0, row4); \ + __m128i dif04 = _mm_sub_epi16(row0, row4); \ + dct_widen(t0e, sum04); \ + dct_widen(t1e, dif04); \ + dct_wadd(x0, t0e, t3e); \ + dct_wsub(x3, t0e, t3e); \ + dct_wadd(x1, t1e, t2e); \ + dct_wsub(x2, t1e, t2e); \ + /* odd part */ \ + dct_rot(y0o,y2o, row7,row3, rot2_0,rot2_1); \ + dct_rot(y1o,y3o, row5,row1, rot3_0,rot3_1); \ + __m128i sum17 = _mm_add_epi16(row1, row7); \ + __m128i sum35 = _mm_add_epi16(row3, row5); \ + dct_rot(y4o,y5o, sum17,sum35, rot1_0,rot1_1); \ + dct_wadd(x4, y0o, y4o); \ + dct_wadd(x5, y1o, y5o); \ + dct_wadd(x6, y2o, y5o); \ + dct_wadd(x7, y3o, y4o); \ + dct_bfly32o(row0,row7, x0,x7,bias,shift); \ + dct_bfly32o(row1,row6, x1,x6,bias,shift); \ + dct_bfly32o(row2,row5, x2,x5,bias,shift); \ + dct_bfly32o(row3,row4, x3,x4,bias,shift); \ + } + + __m128i rot0_0 = dct_const(stbi__f2f(0.5411961f), stbi__f2f(0.5411961f) + stbi__f2f(-1.847759065f)); + __m128i rot0_1 = dct_const(stbi__f2f(0.5411961f) + stbi__f2f( 0.765366865f), stbi__f2f(0.5411961f)); + __m128i rot1_0 = dct_const(stbi__f2f(1.175875602f) + stbi__f2f(-0.899976223f), stbi__f2f(1.175875602f)); + __m128i rot1_1 = dct_const(stbi__f2f(1.175875602f), stbi__f2f(1.175875602f) + stbi__f2f(-2.562915447f)); + __m128i rot2_0 = dct_const(stbi__f2f(-1.961570560f) + stbi__f2f( 0.298631336f), stbi__f2f(-1.961570560f)); + __m128i rot2_1 = dct_const(stbi__f2f(-1.961570560f), stbi__f2f(-1.961570560f) + stbi__f2f( 3.072711026f)); + __m128i rot3_0 = dct_const(stbi__f2f(-0.390180644f) + stbi__f2f( 2.053119869f), stbi__f2f(-0.390180644f)); + __m128i rot3_1 = dct_const(stbi__f2f(-0.390180644f), stbi__f2f(-0.390180644f) + stbi__f2f( 1.501321110f)); + + // rounding biases in column/row passes, see stbi__idct_block for explanation. + __m128i bias_0 = _mm_set1_epi32(512); + __m128i bias_1 = _mm_set1_epi32(65536 + (128<<17)); + + // load + row0 = _mm_load_si128((const __m128i *) (data + 0*8)); + row1 = _mm_load_si128((const __m128i *) (data + 1*8)); + row2 = _mm_load_si128((const __m128i *) (data + 2*8)); + row3 = _mm_load_si128((const __m128i *) (data + 3*8)); + row4 = _mm_load_si128((const __m128i *) (data + 4*8)); + row5 = _mm_load_si128((const __m128i *) (data + 5*8)); + row6 = _mm_load_si128((const __m128i *) (data + 6*8)); + row7 = _mm_load_si128((const __m128i *) (data + 7*8)); + + // column pass + dct_pass(bias_0, 10); + + { + // 16bit 8x8 transpose pass 1 + dct_interleave16(row0, row4); + dct_interleave16(row1, row5); + dct_interleave16(row2, row6); + dct_interleave16(row3, row7); + + // transpose pass 2 + dct_interleave16(row0, row2); + dct_interleave16(row1, row3); + dct_interleave16(row4, row6); + dct_interleave16(row5, row7); + + // transpose pass 3 + dct_interleave16(row0, row1); + dct_interleave16(row2, row3); + dct_interleave16(row4, row5); + dct_interleave16(row6, row7); + } + + // row pass + dct_pass(bias_1, 17); + + { + // pack + __m128i p0 = _mm_packus_epi16(row0, row1); // a0a1a2a3...a7b0b1b2b3...b7 + __m128i p1 = _mm_packus_epi16(row2, row3); + __m128i p2 = _mm_packus_epi16(row4, row5); + __m128i p3 = _mm_packus_epi16(row6, row7); + + // 8bit 8x8 transpose pass 1 + dct_interleave8(p0, p2); // a0e0a1e1... + dct_interleave8(p1, p3); // c0g0c1g1... + + // transpose pass 2 + dct_interleave8(p0, p1); // a0c0e0g0... + dct_interleave8(p2, p3); // b0d0f0h0... + + // transpose pass 3 + dct_interleave8(p0, p2); // a0b0c0d0... + dct_interleave8(p1, p3); // a4b4c4d4... + + // store + _mm_storel_epi64((__m128i *) out, p0); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p0, 0x4e)); out += out_stride; + _mm_storel_epi64((__m128i *) out, p2); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p2, 0x4e)); out += out_stride; + _mm_storel_epi64((__m128i *) out, p1); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p1, 0x4e)); out += out_stride; + _mm_storel_epi64((__m128i *) out, p3); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p3, 0x4e)); + } + +#undef dct_const +#undef dct_rot +#undef dct_widen +#undef dct_wadd +#undef dct_wsub +#undef dct_bfly32o +#undef dct_interleave8 +#undef dct_interleave16 +#undef dct_pass +} + +#endif // STBI_SSE2 + +#ifdef STBI_NEON + +// NEON integer IDCT. should produce bit-identical +// results to the generic C version. +static void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64]) +{ + int16x8_t row0, row1, row2, row3, row4, row5, row6, row7; + + int16x4_t rot0_0 = vdup_n_s16(stbi__f2f(0.5411961f)); + int16x4_t rot0_1 = vdup_n_s16(stbi__f2f(-1.847759065f)); + int16x4_t rot0_2 = vdup_n_s16(stbi__f2f( 0.765366865f)); + int16x4_t rot1_0 = vdup_n_s16(stbi__f2f( 1.175875602f)); + int16x4_t rot1_1 = vdup_n_s16(stbi__f2f(-0.899976223f)); + int16x4_t rot1_2 = vdup_n_s16(stbi__f2f(-2.562915447f)); + int16x4_t rot2_0 = vdup_n_s16(stbi__f2f(-1.961570560f)); + int16x4_t rot2_1 = vdup_n_s16(stbi__f2f(-0.390180644f)); + int16x4_t rot3_0 = vdup_n_s16(stbi__f2f( 0.298631336f)); + int16x4_t rot3_1 = vdup_n_s16(stbi__f2f( 2.053119869f)); + int16x4_t rot3_2 = vdup_n_s16(stbi__f2f( 3.072711026f)); + int16x4_t rot3_3 = vdup_n_s16(stbi__f2f( 1.501321110f)); + +#define dct_long_mul(out, inq, coeff) \ + int32x4_t out##_l = vmull_s16(vget_low_s16(inq), coeff); \ + int32x4_t out##_h = vmull_s16(vget_high_s16(inq), coeff) + +#define dct_long_mac(out, acc, inq, coeff) \ + int32x4_t out##_l = vmlal_s16(acc##_l, vget_low_s16(inq), coeff); \ + int32x4_t out##_h = vmlal_s16(acc##_h, vget_high_s16(inq), coeff) + +#define dct_widen(out, inq) \ + int32x4_t out##_l = vshll_n_s16(vget_low_s16(inq), 12); \ + int32x4_t out##_h = vshll_n_s16(vget_high_s16(inq), 12) + +// wide add +#define dct_wadd(out, a, b) \ + int32x4_t out##_l = vaddq_s32(a##_l, b##_l); \ + int32x4_t out##_h = vaddq_s32(a##_h, b##_h) + +// wide sub +#define dct_wsub(out, a, b) \ + int32x4_t out##_l = vsubq_s32(a##_l, b##_l); \ + int32x4_t out##_h = vsubq_s32(a##_h, b##_h) + +// butterfly a/b, then shift using "shiftop" by "s" and pack +#define dct_bfly32o(out0,out1, a,b,shiftop,s) \ + { \ + dct_wadd(sum, a, b); \ + dct_wsub(dif, a, b); \ + out0 = vcombine_s16(shiftop(sum_l, s), shiftop(sum_h, s)); \ + out1 = vcombine_s16(shiftop(dif_l, s), shiftop(dif_h, s)); \ + } + +#define dct_pass(shiftop, shift) \ + { \ + /* even part */ \ + int16x8_t sum26 = vaddq_s16(row2, row6); \ + dct_long_mul(p1e, sum26, rot0_0); \ + dct_long_mac(t2e, p1e, row6, rot0_1); \ + dct_long_mac(t3e, p1e, row2, rot0_2); \ + int16x8_t sum04 = vaddq_s16(row0, row4); \ + int16x8_t dif04 = vsubq_s16(row0, row4); \ + dct_widen(t0e, sum04); \ + dct_widen(t1e, dif04); \ + dct_wadd(x0, t0e, t3e); \ + dct_wsub(x3, t0e, t3e); \ + dct_wadd(x1, t1e, t2e); \ + dct_wsub(x2, t1e, t2e); \ + /* odd part */ \ + int16x8_t sum15 = vaddq_s16(row1, row5); \ + int16x8_t sum17 = vaddq_s16(row1, row7); \ + int16x8_t sum35 = vaddq_s16(row3, row5); \ + int16x8_t sum37 = vaddq_s16(row3, row7); \ + int16x8_t sumodd = vaddq_s16(sum17, sum35); \ + dct_long_mul(p5o, sumodd, rot1_0); \ + dct_long_mac(p1o, p5o, sum17, rot1_1); \ + dct_long_mac(p2o, p5o, sum35, rot1_2); \ + dct_long_mul(p3o, sum37, rot2_0); \ + dct_long_mul(p4o, sum15, rot2_1); \ + dct_wadd(sump13o, p1o, p3o); \ + dct_wadd(sump24o, p2o, p4o); \ + dct_wadd(sump23o, p2o, p3o); \ + dct_wadd(sump14o, p1o, p4o); \ + dct_long_mac(x4, sump13o, row7, rot3_0); \ + dct_long_mac(x5, sump24o, row5, rot3_1); \ + dct_long_mac(x6, sump23o, row3, rot3_2); \ + dct_long_mac(x7, sump14o, row1, rot3_3); \ + dct_bfly32o(row0,row7, x0,x7,shiftop,shift); \ + dct_bfly32o(row1,row6, x1,x6,shiftop,shift); \ + dct_bfly32o(row2,row5, x2,x5,shiftop,shift); \ + dct_bfly32o(row3,row4, x3,x4,shiftop,shift); \ + } + + // load + row0 = vld1q_s16(data + 0*8); + row1 = vld1q_s16(data + 1*8); + row2 = vld1q_s16(data + 2*8); + row3 = vld1q_s16(data + 3*8); + row4 = vld1q_s16(data + 4*8); + row5 = vld1q_s16(data + 5*8); + row6 = vld1q_s16(data + 6*8); + row7 = vld1q_s16(data + 7*8); + + // add DC bias + row0 = vaddq_s16(row0, vsetq_lane_s16(1024, vdupq_n_s16(0), 0)); + + // column pass + dct_pass(vrshrn_n_s32, 10); + + // 16bit 8x8 transpose + { +// these three map to a single VTRN.16, VTRN.32, and VSWP, respectively. +// whether compilers actually get this is another story, sadly. +#define dct_trn16(x, y) { int16x8x2_t t = vtrnq_s16(x, y); x = t.val[0]; y = t.val[1]; } +#define dct_trn32(x, y) { int32x4x2_t t = vtrnq_s32(vreinterpretq_s32_s16(x), vreinterpretq_s32_s16(y)); x = vreinterpretq_s16_s32(t.val[0]); y = vreinterpretq_s16_s32(t.val[1]); } +#define dct_trn64(x, y) { int16x8_t x0 = x; int16x8_t y0 = y; x = vcombine_s16(vget_low_s16(x0), vget_low_s16(y0)); y = vcombine_s16(vget_high_s16(x0), vget_high_s16(y0)); } + + // pass 1 + dct_trn16(row0, row1); // a0b0a2b2a4b4a6b6 + dct_trn16(row2, row3); + dct_trn16(row4, row5); + dct_trn16(row6, row7); + + // pass 2 + dct_trn32(row0, row2); // a0b0c0d0a4b4c4d4 + dct_trn32(row1, row3); + dct_trn32(row4, row6); + dct_trn32(row5, row7); + + // pass 3 + dct_trn64(row0, row4); // a0b0c0d0e0f0g0h0 + dct_trn64(row1, row5); + dct_trn64(row2, row6); + dct_trn64(row3, row7); + +#undef dct_trn16 +#undef dct_trn32 +#undef dct_trn64 + } + + // row pass + // vrshrn_n_s32 only supports shifts up to 16, we need + // 17. so do a non-rounding shift of 16 first then follow + // up with a rounding shift by 1. + dct_pass(vshrn_n_s32, 16); + + { + // pack and round + uint8x8_t p0 = vqrshrun_n_s16(row0, 1); + uint8x8_t p1 = vqrshrun_n_s16(row1, 1); + uint8x8_t p2 = vqrshrun_n_s16(row2, 1); + uint8x8_t p3 = vqrshrun_n_s16(row3, 1); + uint8x8_t p4 = vqrshrun_n_s16(row4, 1); + uint8x8_t p5 = vqrshrun_n_s16(row5, 1); + uint8x8_t p6 = vqrshrun_n_s16(row6, 1); + uint8x8_t p7 = vqrshrun_n_s16(row7, 1); + + // again, these can translate into one instruction, but often don't. +#define dct_trn8_8(x, y) { uint8x8x2_t t = vtrn_u8(x, y); x = t.val[0]; y = t.val[1]; } +#define dct_trn8_16(x, y) { uint16x4x2_t t = vtrn_u16(vreinterpret_u16_u8(x), vreinterpret_u16_u8(y)); x = vreinterpret_u8_u16(t.val[0]); y = vreinterpret_u8_u16(t.val[1]); } +#define dct_trn8_32(x, y) { uint32x2x2_t t = vtrn_u32(vreinterpret_u32_u8(x), vreinterpret_u32_u8(y)); x = vreinterpret_u8_u32(t.val[0]); y = vreinterpret_u8_u32(t.val[1]); } + + // sadly can't use interleaved stores here since we only write + // 8 bytes to each scan line! + + // 8x8 8-bit transpose pass 1 + dct_trn8_8(p0, p1); + dct_trn8_8(p2, p3); + dct_trn8_8(p4, p5); + dct_trn8_8(p6, p7); + + // pass 2 + dct_trn8_16(p0, p2); + dct_trn8_16(p1, p3); + dct_trn8_16(p4, p6); + dct_trn8_16(p5, p7); + + // pass 3 + dct_trn8_32(p0, p4); + dct_trn8_32(p1, p5); + dct_trn8_32(p2, p6); + dct_trn8_32(p3, p7); + + // store + vst1_u8(out, p0); out += out_stride; + vst1_u8(out, p1); out += out_stride; + vst1_u8(out, p2); out += out_stride; + vst1_u8(out, p3); out += out_stride; + vst1_u8(out, p4); out += out_stride; + vst1_u8(out, p5); out += out_stride; + vst1_u8(out, p6); out += out_stride; + vst1_u8(out, p7); + +#undef dct_trn8_8 +#undef dct_trn8_16 +#undef dct_trn8_32 + } + +#undef dct_long_mul +#undef dct_long_mac +#undef dct_widen +#undef dct_wadd +#undef dct_wsub +#undef dct_bfly32o +#undef dct_pass +} + +#endif // STBI_NEON + +#define STBI__MARKER_none 0xff +// if there's a pending marker from the entropy stream, return that +// otherwise, fetch from the stream and get a marker. if there's no +// marker, return 0xff, which is never a valid marker value +static stbi_uc stbi__get_marker(stbi__jpeg *j) +{ + stbi_uc x; + if (j->marker != STBI__MARKER_none) { x = j->marker; j->marker = STBI__MARKER_none; return x; } + x = stbi__get8(j->s); + if (x != 0xff) return STBI__MARKER_none; + while (x == 0xff) + x = stbi__get8(j->s); // consume repeated 0xff fill bytes + return x; +} + +// in each scan, we'll have scan_n components, and the order +// of the components is specified by order[] +#define STBI__RESTART(x) ((x) >= 0xd0 && (x) <= 0xd7) + +// after a restart interval, stbi__jpeg_reset the entropy decoder and +// the dc prediction +static void stbi__jpeg_reset(stbi__jpeg *j) +{ + j->code_bits = 0; + j->code_buffer = 0; + j->nomore = 0; + j->img_comp[0].dc_pred = j->img_comp[1].dc_pred = j->img_comp[2].dc_pred = j->img_comp[3].dc_pred = 0; + j->marker = STBI__MARKER_none; + j->todo = j->restart_interval ? j->restart_interval : 0x7fffffff; + j->eob_run = 0; + // no more than 1<<31 MCUs if no restart_interal? that's plenty safe, + // since we don't even allow 1<<30 pixels +} + +static int stbi__parse_entropy_coded_data(stbi__jpeg *z) +{ + stbi__jpeg_reset(z); + if (!z->progressive) { + if (z->scan_n == 1) { + int i,j; + STBI_SIMD_ALIGN(short, data[64]); + int n = z->order[0]; + // non-interleaved data, we just need to process one block at a time, + // in trivial scanline order + // number of blocks to do just depends on how many actual "pixels" this + // component has, independent of interleaved MCU blocking and such + int w = (z->img_comp[n].x+7) >> 3; + int h = (z->img_comp[n].y+7) >> 3; + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) { + int ha = z->img_comp[n].ha; + if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0; + z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data); + // every data block is an MCU, so countdown the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + // if it's NOT a restart, then just bail, so we get corrupt data + // rather than no data + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } else { // interleaved + int i,j,k,x,y; + STBI_SIMD_ALIGN(short, data[64]); + for (j=0; j < z->img_mcu_y; ++j) { + for (i=0; i < z->img_mcu_x; ++i) { + // scan an interleaved mcu... process scan_n components in order + for (k=0; k < z->scan_n; ++k) { + int n = z->order[k]; + // scan out an mcu's worth of this component; that's just determined + // by the basic H and V specified for the component + for (y=0; y < z->img_comp[n].v; ++y) { + for (x=0; x < z->img_comp[n].h; ++x) { + int x2 = (i*z->img_comp[n].h + x)*8; + int y2 = (j*z->img_comp[n].v + y)*8; + int ha = z->img_comp[n].ha; + if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0; + z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*y2+x2, z->img_comp[n].w2, data); + } + } + } + // after all interleaved components, that's an interleaved MCU, + // so now count down the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } + } else { + if (z->scan_n == 1) { + int i,j; + int n = z->order[0]; + // non-interleaved data, we just need to process one block at a time, + // in trivial scanline order + // number of blocks to do just depends on how many actual "pixels" this + // component has, independent of interleaved MCU blocking and such + int w = (z->img_comp[n].x+7) >> 3; + int h = (z->img_comp[n].y+7) >> 3; + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) { + short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w); + if (z->spec_start == 0) { + if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n)) + return 0; + } else { + int ha = z->img_comp[n].ha; + if (!stbi__jpeg_decode_block_prog_ac(z, data, &z->huff_ac[ha], z->fast_ac[ha])) + return 0; + } + // every data block is an MCU, so countdown the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } else { // interleaved + int i,j,k,x,y; + for (j=0; j < z->img_mcu_y; ++j) { + for (i=0; i < z->img_mcu_x; ++i) { + // scan an interleaved mcu... process scan_n components in order + for (k=0; k < z->scan_n; ++k) { + int n = z->order[k]; + // scan out an mcu's worth of this component; that's just determined + // by the basic H and V specified for the component + for (y=0; y < z->img_comp[n].v; ++y) { + for (x=0; x < z->img_comp[n].h; ++x) { + int x2 = (i*z->img_comp[n].h + x); + int y2 = (j*z->img_comp[n].v + y); + short *data = z->img_comp[n].coeff + 64 * (x2 + y2 * z->img_comp[n].coeff_w); + if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n)) + return 0; + } + } + } + // after all interleaved components, that's an interleaved MCU, + // so now count down the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } + } +} + +static void stbi__jpeg_dequantize(short *data, stbi__uint16 *dequant) +{ + int i; + for (i=0; i < 64; ++i) + data[i] *= dequant[i]; +} + +static void stbi__jpeg_finish(stbi__jpeg *z) +{ + if (z->progressive) { + // dequantize and idct the data + int i,j,n; + for (n=0; n < z->s->img_n; ++n) { + int w = (z->img_comp[n].x+7) >> 3; + int h = (z->img_comp[n].y+7) >> 3; + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) { + short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w); + stbi__jpeg_dequantize(data, z->dequant[z->img_comp[n].tq]); + z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data); + } + } + } + } +} + +static int stbi__process_marker(stbi__jpeg *z, int m) +{ + int L; + switch (m) { + case STBI__MARKER_none: // no marker found + return stbi__err("expected marker","Corrupt JPEG"); + + case 0xDD: // DRI - specify restart interval + if (stbi__get16be(z->s) != 4) return stbi__err("bad DRI len","Corrupt JPEG"); + z->restart_interval = stbi__get16be(z->s); + return 1; + + case 0xDB: // DQT - define quantization table + L = stbi__get16be(z->s)-2; + while (L > 0) { + int q = stbi__get8(z->s); + int p = q >> 4, sixteen = (p != 0); + int t = q & 15,i; + if (p != 0 && p != 1) return stbi__err("bad DQT type","Corrupt JPEG"); + if (t > 3) return stbi__err("bad DQT table","Corrupt JPEG"); + + for (i=0; i < 64; ++i) + z->dequant[t][stbi__jpeg_dezigzag[i]] = (stbi__uint16)(sixteen ? stbi__get16be(z->s) : stbi__get8(z->s)); + L -= (sixteen ? 129 : 65); + } + return L==0; + + case 0xC4: // DHT - define huffman table + L = stbi__get16be(z->s)-2; + while (L > 0) { + stbi_uc *v; + int sizes[16],i,n=0; + int q = stbi__get8(z->s); + int tc = q >> 4; + int th = q & 15; + if (tc > 1 || th > 3) return stbi__err("bad DHT header","Corrupt JPEG"); + for (i=0; i < 16; ++i) { + sizes[i] = stbi__get8(z->s); + n += sizes[i]; + } + if(n > 256) return stbi__err("bad DHT header","Corrupt JPEG"); // Loop over i < n would write past end of values! + L -= 17; + if (tc == 0) { + if (!stbi__build_huffman(z->huff_dc+th, sizes)) return 0; + v = z->huff_dc[th].values; + } else { + if (!stbi__build_huffman(z->huff_ac+th, sizes)) return 0; + v = z->huff_ac[th].values; + } + for (i=0; i < n; ++i) + v[i] = stbi__get8(z->s); + if (tc != 0) + stbi__build_fast_ac(z->fast_ac[th], z->huff_ac + th); + L -= n; + } + return L==0; + } + + // check for comment block or APP blocks + if ((m >= 0xE0 && m <= 0xEF) || m == 0xFE) { + L = stbi__get16be(z->s); + if (L < 2) { + if (m == 0xFE) + return stbi__err("bad COM len","Corrupt JPEG"); + else + return stbi__err("bad APP len","Corrupt JPEG"); + } + L -= 2; + + if (m == 0xE0 && L >= 5) { // JFIF APP0 segment + static const unsigned char tag[5] = {'J','F','I','F','\0'}; + int ok = 1; + int i; + for (i=0; i < 5; ++i) + if (stbi__get8(z->s) != tag[i]) + ok = 0; + L -= 5; + if (ok) + z->jfif = 1; + } else if (m == 0xEE && L >= 12) { // Adobe APP14 segment + static const unsigned char tag[6] = {'A','d','o','b','e','\0'}; + int ok = 1; + int i; + for (i=0; i < 6; ++i) + if (stbi__get8(z->s) != tag[i]) + ok = 0; + L -= 6; + if (ok) { + stbi__get8(z->s); // version + stbi__get16be(z->s); // flags0 + stbi__get16be(z->s); // flags1 + z->app14_color_transform = stbi__get8(z->s); // color transform + L -= 6; + } + } + + stbi__skip(z->s, L); + return 1; + } + + return stbi__err("unknown marker","Corrupt JPEG"); +} + +// after we see SOS +static int stbi__process_scan_header(stbi__jpeg *z) +{ + int i; + int Ls = stbi__get16be(z->s); + z->scan_n = stbi__get8(z->s); + if (z->scan_n < 1 || z->scan_n > 4 || z->scan_n > (int) z->s->img_n) return stbi__err("bad SOS component count","Corrupt JPEG"); + if (Ls != 6+2*z->scan_n) return stbi__err("bad SOS len","Corrupt JPEG"); + for (i=0; i < z->scan_n; ++i) { + int id = stbi__get8(z->s), which; + int q = stbi__get8(z->s); + for (which = 0; which < z->s->img_n; ++which) + if (z->img_comp[which].id == id) + break; + if (which == z->s->img_n) return 0; // no match + z->img_comp[which].hd = q >> 4; if (z->img_comp[which].hd > 3) return stbi__err("bad DC huff","Corrupt JPEG"); + z->img_comp[which].ha = q & 15; if (z->img_comp[which].ha > 3) return stbi__err("bad AC huff","Corrupt JPEG"); + z->order[i] = which; + } + + { + int aa; + z->spec_start = stbi__get8(z->s); + z->spec_end = stbi__get8(z->s); // should be 63, but might be 0 + aa = stbi__get8(z->s); + z->succ_high = (aa >> 4); + z->succ_low = (aa & 15); + if (z->progressive) { + if (z->spec_start > 63 || z->spec_end > 63 || z->spec_start > z->spec_end || z->succ_high > 13 || z->succ_low > 13) + return stbi__err("bad SOS", "Corrupt JPEG"); + } else { + if (z->spec_start != 0) return stbi__err("bad SOS","Corrupt JPEG"); + if (z->succ_high != 0 || z->succ_low != 0) return stbi__err("bad SOS","Corrupt JPEG"); + z->spec_end = 63; + } + } + + return 1; +} + +static int stbi__free_jpeg_components(stbi__jpeg *z, int ncomp, int why) +{ + int i; + for (i=0; i < ncomp; ++i) { + if (z->img_comp[i].raw_data) { + STBI_FREE(z->img_comp[i].raw_data); + z->img_comp[i].raw_data = NULL; + z->img_comp[i].data = NULL; + } + if (z->img_comp[i].raw_coeff) { + STBI_FREE(z->img_comp[i].raw_coeff); + z->img_comp[i].raw_coeff = 0; + z->img_comp[i].coeff = 0; + } + if (z->img_comp[i].linebuf) { + STBI_FREE(z->img_comp[i].linebuf); + z->img_comp[i].linebuf = NULL; + } + } + return why; +} + +static int stbi__process_frame_header(stbi__jpeg *z, int scan) +{ + stbi__context *s = z->s; + int Lf,p,i,q, h_max=1,v_max=1,c; + Lf = stbi__get16be(s); if (Lf < 11) return stbi__err("bad SOF len","Corrupt JPEG"); // JPEG + p = stbi__get8(s); if (p != 8) return stbi__err("only 8-bit","JPEG format not supported: 8-bit only"); // JPEG baseline + s->img_y = stbi__get16be(s); if (s->img_y == 0) return stbi__err("no header height", "JPEG format not supported: delayed height"); // Legal, but we don't handle it--but neither does IJG + s->img_x = stbi__get16be(s); if (s->img_x == 0) return stbi__err("0 width","Corrupt JPEG"); // JPEG requires + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + c = stbi__get8(s); + if (c != 3 && c != 1 && c != 4) return stbi__err("bad component count","Corrupt JPEG"); + s->img_n = c; + for (i=0; i < c; ++i) { + z->img_comp[i].data = NULL; + z->img_comp[i].linebuf = NULL; + } + + if (Lf != 8+3*s->img_n) return stbi__err("bad SOF len","Corrupt JPEG"); + + z->rgb = 0; + for (i=0; i < s->img_n; ++i) { + static const unsigned char rgb[3] = { 'R', 'G', 'B' }; + z->img_comp[i].id = stbi__get8(s); + if (s->img_n == 3 && z->img_comp[i].id == rgb[i]) + ++z->rgb; + q = stbi__get8(s); + z->img_comp[i].h = (q >> 4); if (!z->img_comp[i].h || z->img_comp[i].h > 4) return stbi__err("bad H","Corrupt JPEG"); + z->img_comp[i].v = q & 15; if (!z->img_comp[i].v || z->img_comp[i].v > 4) return stbi__err("bad V","Corrupt JPEG"); + z->img_comp[i].tq = stbi__get8(s); if (z->img_comp[i].tq > 3) return stbi__err("bad TQ","Corrupt JPEG"); + } + + if (scan != STBI__SCAN_load) return 1; + + if (!stbi__mad3sizes_valid(s->img_x, s->img_y, s->img_n, 0)) return stbi__err("too large", "Image too large to decode"); + + for (i=0; i < s->img_n; ++i) { + if (z->img_comp[i].h > h_max) h_max = z->img_comp[i].h; + if (z->img_comp[i].v > v_max) v_max = z->img_comp[i].v; + } + + // check that plane subsampling factors are integer ratios; our resamplers can't deal with fractional ratios + // and I've never seen a non-corrupted JPEG file actually use them + for (i=0; i < s->img_n; ++i) { + if (h_max % z->img_comp[i].h != 0) return stbi__err("bad H","Corrupt JPEG"); + if (v_max % z->img_comp[i].v != 0) return stbi__err("bad V","Corrupt JPEG"); + } + + // compute interleaved mcu info + z->img_h_max = h_max; + z->img_v_max = v_max; + z->img_mcu_w = h_max * 8; + z->img_mcu_h = v_max * 8; + // these sizes can't be more than 17 bits + z->img_mcu_x = (s->img_x + z->img_mcu_w-1) / z->img_mcu_w; + z->img_mcu_y = (s->img_y + z->img_mcu_h-1) / z->img_mcu_h; + + for (i=0; i < s->img_n; ++i) { + // number of effective pixels (e.g. for non-interleaved MCU) + z->img_comp[i].x = (s->img_x * z->img_comp[i].h + h_max-1) / h_max; + z->img_comp[i].y = (s->img_y * z->img_comp[i].v + v_max-1) / v_max; + // to simplify generation, we'll allocate enough memory to decode + // the bogus oversized data from using interleaved MCUs and their + // big blocks (e.g. a 16x16 iMCU on an image of width 33); we won't + // discard the extra data until colorspace conversion + // + // img_mcu_x, img_mcu_y: <=17 bits; comp[i].h and .v are <=4 (checked earlier) + // so these muls can't overflow with 32-bit ints (which we require) + z->img_comp[i].w2 = z->img_mcu_x * z->img_comp[i].h * 8; + z->img_comp[i].h2 = z->img_mcu_y * z->img_comp[i].v * 8; + z->img_comp[i].coeff = 0; + z->img_comp[i].raw_coeff = 0; + z->img_comp[i].linebuf = NULL; + z->img_comp[i].raw_data = stbi__malloc_mad2(z->img_comp[i].w2, z->img_comp[i].h2, 15); + if (z->img_comp[i].raw_data == NULL) + return stbi__free_jpeg_components(z, i+1, stbi__err("outofmem", "Out of memory")); + // align blocks for idct using mmx/sse + z->img_comp[i].data = (stbi_uc*) (((size_t) z->img_comp[i].raw_data + 15) & ~15); + if (z->progressive) { + // w2, h2 are multiples of 8 (see above) + z->img_comp[i].coeff_w = z->img_comp[i].w2 / 8; + z->img_comp[i].coeff_h = z->img_comp[i].h2 / 8; + z->img_comp[i].raw_coeff = stbi__malloc_mad3(z->img_comp[i].w2, z->img_comp[i].h2, sizeof(short), 15); + if (z->img_comp[i].raw_coeff == NULL) + return stbi__free_jpeg_components(z, i+1, stbi__err("outofmem", "Out of memory")); + z->img_comp[i].coeff = (short*) (((size_t) z->img_comp[i].raw_coeff + 15) & ~15); + } + } + + return 1; +} + +// use comparisons since in some cases we handle more than one case (e.g. SOF) +#define stbi__DNL(x) ((x) == 0xdc) +#define stbi__SOI(x) ((x) == 0xd8) +#define stbi__EOI(x) ((x) == 0xd9) +#define stbi__SOF(x) ((x) == 0xc0 || (x) == 0xc1 || (x) == 0xc2) +#define stbi__SOS(x) ((x) == 0xda) + +#define stbi__SOF_progressive(x) ((x) == 0xc2) + +static int stbi__decode_jpeg_header(stbi__jpeg *z, int scan) +{ + int m; + z->jfif = 0; + z->app14_color_transform = -1; // valid values are 0,1,2 + z->marker = STBI__MARKER_none; // initialize cached marker to empty + m = stbi__get_marker(z); + if (!stbi__SOI(m)) return stbi__err("no SOI","Corrupt JPEG"); + if (scan == STBI__SCAN_type) return 1; + m = stbi__get_marker(z); + while (!stbi__SOF(m)) { + if (!stbi__process_marker(z,m)) return 0; + m = stbi__get_marker(z); + while (m == STBI__MARKER_none) { + // some files have extra padding after their blocks, so ok, we'll scan + if (stbi__at_eof(z->s)) return stbi__err("no SOF", "Corrupt JPEG"); + m = stbi__get_marker(z); + } + } + z->progressive = stbi__SOF_progressive(m); + if (!stbi__process_frame_header(z, scan)) return 0; + return 1; +} + +static stbi_uc stbi__skip_jpeg_junk_at_end(stbi__jpeg *j) +{ + // some JPEGs have junk at end, skip over it but if we find what looks + // like a valid marker, resume there + while (!stbi__at_eof(j->s)) { + stbi_uc x = stbi__get8(j->s); + while (x == 0xff) { // might be a marker + if (stbi__at_eof(j->s)) return STBI__MARKER_none; + x = stbi__get8(j->s); + if (x != 0x00 && x != 0xff) { + // not a stuffed zero or lead-in to another marker, looks + // like an actual marker, return it + return x; + } + // stuffed zero has x=0 now which ends the loop, meaning we go + // back to regular scan loop. + // repeated 0xff keeps trying to read the next byte of the marker. + } + } + return STBI__MARKER_none; +} + +// decode image to YCbCr format +static int stbi__decode_jpeg_image(stbi__jpeg *j) +{ + int m; + for (m = 0; m < 4; m++) { + j->img_comp[m].raw_data = NULL; + j->img_comp[m].raw_coeff = NULL; + } + j->restart_interval = 0; + if (!stbi__decode_jpeg_header(j, STBI__SCAN_load)) return 0; + m = stbi__get_marker(j); + while (!stbi__EOI(m)) { + if (stbi__SOS(m)) { + if (!stbi__process_scan_header(j)) return 0; + if (!stbi__parse_entropy_coded_data(j)) return 0; + if (j->marker == STBI__MARKER_none ) { + j->marker = stbi__skip_jpeg_junk_at_end(j); + // if we reach eof without hitting a marker, stbi__get_marker() below will fail and we'll eventually return 0 + } + m = stbi__get_marker(j); + if (STBI__RESTART(m)) + m = stbi__get_marker(j); + } else if (stbi__DNL(m)) { + int Ld = stbi__get16be(j->s); + stbi__uint32 NL = stbi__get16be(j->s); + if (Ld != 4) return stbi__err("bad DNL len", "Corrupt JPEG"); + if (NL != j->s->img_y) return stbi__err("bad DNL height", "Corrupt JPEG"); + m = stbi__get_marker(j); + } else { + if (!stbi__process_marker(j, m)) return 1; + m = stbi__get_marker(j); + } + } + if (j->progressive) + stbi__jpeg_finish(j); + return 1; +} + +// static jfif-centered resampling (across block boundaries) + +typedef stbi_uc *(*resample_row_func)(stbi_uc *out, stbi_uc *in0, stbi_uc *in1, + int w, int hs); + +#define stbi__div4(x) ((stbi_uc) ((x) >> 2)) + +static stbi_uc *resample_row_1(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + STBI_NOTUSED(out); + STBI_NOTUSED(in_far); + STBI_NOTUSED(w); + STBI_NOTUSED(hs); + return in_near; +} + +static stbi_uc* stbi__resample_row_v_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate two samples vertically for every one in input + int i; + STBI_NOTUSED(hs); + for (i=0; i < w; ++i) + out[i] = stbi__div4(3*in_near[i] + in_far[i] + 2); + return out; +} + +static stbi_uc* stbi__resample_row_h_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate two samples horizontally for every one in input + int i; + stbi_uc *input = in_near; + + if (w == 1) { + // if only one sample, can't do any interpolation + out[0] = out[1] = input[0]; + return out; + } + + out[0] = input[0]; + out[1] = stbi__div4(input[0]*3 + input[1] + 2); + for (i=1; i < w-1; ++i) { + int n = 3*input[i]+2; + out[i*2+0] = stbi__div4(n+input[i-1]); + out[i*2+1] = stbi__div4(n+input[i+1]); + } + out[i*2+0] = stbi__div4(input[w-2]*3 + input[w-1] + 2); + out[i*2+1] = input[w-1]; + + STBI_NOTUSED(in_far); + STBI_NOTUSED(hs); + + return out; +} + +#define stbi__div16(x) ((stbi_uc) ((x) >> 4)) + +static stbi_uc *stbi__resample_row_hv_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate 2x2 samples for every one in input + int i,t0,t1; + if (w == 1) { + out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2); + return out; + } + + t1 = 3*in_near[0] + in_far[0]; + out[0] = stbi__div4(t1+2); + for (i=1; i < w; ++i) { + t0 = t1; + t1 = 3*in_near[i]+in_far[i]; + out[i*2-1] = stbi__div16(3*t0 + t1 + 8); + out[i*2 ] = stbi__div16(3*t1 + t0 + 8); + } + out[w*2-1] = stbi__div4(t1+2); + + STBI_NOTUSED(hs); + + return out; +} + +#if defined(STBI_SSE2) || defined(STBI_NEON) +static stbi_uc *stbi__resample_row_hv_2_simd(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate 2x2 samples for every one in input + int i=0,t0,t1; + + if (w == 1) { + out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2); + return out; + } + + t1 = 3*in_near[0] + in_far[0]; + // process groups of 8 pixels for as long as we can. + // note we can't handle the last pixel in a row in this loop + // because we need to handle the filter boundary conditions. + for (; i < ((w-1) & ~7); i += 8) { +#if defined(STBI_SSE2) + // load and perform the vertical filtering pass + // this uses 3*x + y = 4*x + (y - x) + __m128i zero = _mm_setzero_si128(); + __m128i farb = _mm_loadl_epi64((__m128i *) (in_far + i)); + __m128i nearb = _mm_loadl_epi64((__m128i *) (in_near + i)); + __m128i farw = _mm_unpacklo_epi8(farb, zero); + __m128i nearw = _mm_unpacklo_epi8(nearb, zero); + __m128i diff = _mm_sub_epi16(farw, nearw); + __m128i nears = _mm_slli_epi16(nearw, 2); + __m128i curr = _mm_add_epi16(nears, diff); // current row + + // horizontal filter works the same based on shifted vers of current + // row. "prev" is current row shifted right by 1 pixel; we need to + // insert the previous pixel value (from t1). + // "next" is current row shifted left by 1 pixel, with first pixel + // of next block of 8 pixels added in. + __m128i prv0 = _mm_slli_si128(curr, 2); + __m128i nxt0 = _mm_srli_si128(curr, 2); + __m128i prev = _mm_insert_epi16(prv0, t1, 0); + __m128i next = _mm_insert_epi16(nxt0, 3*in_near[i+8] + in_far[i+8], 7); + + // horizontal filter, polyphase implementation since it's convenient: + // even pixels = 3*cur + prev = cur*4 + (prev - cur) + // odd pixels = 3*cur + next = cur*4 + (next - cur) + // note the shared term. + __m128i bias = _mm_set1_epi16(8); + __m128i curs = _mm_slli_epi16(curr, 2); + __m128i prvd = _mm_sub_epi16(prev, curr); + __m128i nxtd = _mm_sub_epi16(next, curr); + __m128i curb = _mm_add_epi16(curs, bias); + __m128i even = _mm_add_epi16(prvd, curb); + __m128i odd = _mm_add_epi16(nxtd, curb); + + // interleave even and odd pixels, then undo scaling. + __m128i int0 = _mm_unpacklo_epi16(even, odd); + __m128i int1 = _mm_unpackhi_epi16(even, odd); + __m128i de0 = _mm_srli_epi16(int0, 4); + __m128i de1 = _mm_srli_epi16(int1, 4); + + // pack and write output + __m128i outv = _mm_packus_epi16(de0, de1); + _mm_storeu_si128((__m128i *) (out + i*2), outv); +#elif defined(STBI_NEON) + // load and perform the vertical filtering pass + // this uses 3*x + y = 4*x + (y - x) + uint8x8_t farb = vld1_u8(in_far + i); + uint8x8_t nearb = vld1_u8(in_near + i); + int16x8_t diff = vreinterpretq_s16_u16(vsubl_u8(farb, nearb)); + int16x8_t nears = vreinterpretq_s16_u16(vshll_n_u8(nearb, 2)); + int16x8_t curr = vaddq_s16(nears, diff); // current row + + // horizontal filter works the same based on shifted vers of current + // row. "prev" is current row shifted right by 1 pixel; we need to + // insert the previous pixel value (from t1). + // "next" is current row shifted left by 1 pixel, with first pixel + // of next block of 8 pixels added in. + int16x8_t prv0 = vextq_s16(curr, curr, 7); + int16x8_t nxt0 = vextq_s16(curr, curr, 1); + int16x8_t prev = vsetq_lane_s16(t1, prv0, 0); + int16x8_t next = vsetq_lane_s16(3*in_near[i+8] + in_far[i+8], nxt0, 7); + + // horizontal filter, polyphase implementation since it's convenient: + // even pixels = 3*cur + prev = cur*4 + (prev - cur) + // odd pixels = 3*cur + next = cur*4 + (next - cur) + // note the shared term. + int16x8_t curs = vshlq_n_s16(curr, 2); + int16x8_t prvd = vsubq_s16(prev, curr); + int16x8_t nxtd = vsubq_s16(next, curr); + int16x8_t even = vaddq_s16(curs, prvd); + int16x8_t odd = vaddq_s16(curs, nxtd); + + // undo scaling and round, then store with even/odd phases interleaved + uint8x8x2_t o; + o.val[0] = vqrshrun_n_s16(even, 4); + o.val[1] = vqrshrun_n_s16(odd, 4); + vst2_u8(out + i*2, o); +#endif + + // "previous" value for next iter + t1 = 3*in_near[i+7] + in_far[i+7]; + } + + t0 = t1; + t1 = 3*in_near[i] + in_far[i]; + out[i*2] = stbi__div16(3*t1 + t0 + 8); + + for (++i; i < w; ++i) { + t0 = t1; + t1 = 3*in_near[i]+in_far[i]; + out[i*2-1] = stbi__div16(3*t0 + t1 + 8); + out[i*2 ] = stbi__div16(3*t1 + t0 + 8); + } + out[w*2-1] = stbi__div4(t1+2); + + STBI_NOTUSED(hs); + + return out; +} +#endif + +static stbi_uc *stbi__resample_row_generic(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // resample with nearest-neighbor + int i,j; + STBI_NOTUSED(in_far); + for (i=0; i < w; ++i) + for (j=0; j < hs; ++j) + out[i*hs+j] = in_near[i]; + return out; +} + +// this is a reduced-precision calculation of YCbCr-to-RGB introduced +// to make sure the code produces the same results in both SIMD and scalar +#define stbi__float2fixed(x) (((int) ((x) * 4096.0f + 0.5f)) << 8) +static void stbi__YCbCr_to_RGB_row(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step) +{ + int i; + for (i=0; i < count; ++i) { + int y_fixed = (y[i] << 20) + (1<<19); // rounding + int r,g,b; + int cr = pcr[i] - 128; + int cb = pcb[i] - 128; + r = y_fixed + cr* stbi__float2fixed(1.40200f); + g = y_fixed + (cr*-stbi__float2fixed(0.71414f)) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000); + b = y_fixed + cb* stbi__float2fixed(1.77200f); + r >>= 20; + g >>= 20; + b >>= 20; + if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; } + if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; } + if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; } + out[0] = (stbi_uc)r; + out[1] = (stbi_uc)g; + out[2] = (stbi_uc)b; + out[3] = 255; + out += step; + } +} + +#if defined(STBI_SSE2) || defined(STBI_NEON) +static void stbi__YCbCr_to_RGB_simd(stbi_uc *out, stbi_uc const *y, stbi_uc const *pcb, stbi_uc const *pcr, int count, int step) +{ + int i = 0; + +#ifdef STBI_SSE2 + // step == 3 is pretty ugly on the final interleave, and i'm not convinced + // it's useful in practice (you wouldn't use it for textures, for example). + // so just accelerate step == 4 case. + if (step == 4) { + // this is a fairly straightforward implementation and not super-optimized. + __m128i signflip = _mm_set1_epi8(-0x80); + __m128i cr_const0 = _mm_set1_epi16( (short) ( 1.40200f*4096.0f+0.5f)); + __m128i cr_const1 = _mm_set1_epi16( - (short) ( 0.71414f*4096.0f+0.5f)); + __m128i cb_const0 = _mm_set1_epi16( - (short) ( 0.34414f*4096.0f+0.5f)); + __m128i cb_const1 = _mm_set1_epi16( (short) ( 1.77200f*4096.0f+0.5f)); + __m128i y_bias = _mm_set1_epi8((char) (unsigned char) 128); + __m128i xw = _mm_set1_epi16(255); // alpha channel + + for (; i+7 < count; i += 8) { + // load + __m128i y_bytes = _mm_loadl_epi64((__m128i *) (y+i)); + __m128i cr_bytes = _mm_loadl_epi64((__m128i *) (pcr+i)); + __m128i cb_bytes = _mm_loadl_epi64((__m128i *) (pcb+i)); + __m128i cr_biased = _mm_xor_si128(cr_bytes, signflip); // -128 + __m128i cb_biased = _mm_xor_si128(cb_bytes, signflip); // -128 + + // unpack to short (and left-shift cr, cb by 8) + __m128i yw = _mm_unpacklo_epi8(y_bias, y_bytes); + __m128i crw = _mm_unpacklo_epi8(_mm_setzero_si128(), cr_biased); + __m128i cbw = _mm_unpacklo_epi8(_mm_setzero_si128(), cb_biased); + + // color transform + __m128i yws = _mm_srli_epi16(yw, 4); + __m128i cr0 = _mm_mulhi_epi16(cr_const0, crw); + __m128i cb0 = _mm_mulhi_epi16(cb_const0, cbw); + __m128i cb1 = _mm_mulhi_epi16(cbw, cb_const1); + __m128i cr1 = _mm_mulhi_epi16(crw, cr_const1); + __m128i rws = _mm_add_epi16(cr0, yws); + __m128i gwt = _mm_add_epi16(cb0, yws); + __m128i bws = _mm_add_epi16(yws, cb1); + __m128i gws = _mm_add_epi16(gwt, cr1); + + // descale + __m128i rw = _mm_srai_epi16(rws, 4); + __m128i bw = _mm_srai_epi16(bws, 4); + __m128i gw = _mm_srai_epi16(gws, 4); + + // back to byte, set up for transpose + __m128i brb = _mm_packus_epi16(rw, bw); + __m128i gxb = _mm_packus_epi16(gw, xw); + + // transpose to interleave channels + __m128i t0 = _mm_unpacklo_epi8(brb, gxb); + __m128i t1 = _mm_unpackhi_epi8(brb, gxb); + __m128i o0 = _mm_unpacklo_epi16(t0, t1); + __m128i o1 = _mm_unpackhi_epi16(t0, t1); + + // store + _mm_storeu_si128((__m128i *) (out + 0), o0); + _mm_storeu_si128((__m128i *) (out + 16), o1); + out += 32; + } + } +#endif + +#ifdef STBI_NEON + // in this version, step=3 support would be easy to add. but is there demand? + if (step == 4) { + // this is a fairly straightforward implementation and not super-optimized. + uint8x8_t signflip = vdup_n_u8(0x80); + int16x8_t cr_const0 = vdupq_n_s16( (short) ( 1.40200f*4096.0f+0.5f)); + int16x8_t cr_const1 = vdupq_n_s16( - (short) ( 0.71414f*4096.0f+0.5f)); + int16x8_t cb_const0 = vdupq_n_s16( - (short) ( 0.34414f*4096.0f+0.5f)); + int16x8_t cb_const1 = vdupq_n_s16( (short) ( 1.77200f*4096.0f+0.5f)); + + for (; i+7 < count; i += 8) { + // load + uint8x8_t y_bytes = vld1_u8(y + i); + uint8x8_t cr_bytes = vld1_u8(pcr + i); + uint8x8_t cb_bytes = vld1_u8(pcb + i); + int8x8_t cr_biased = vreinterpret_s8_u8(vsub_u8(cr_bytes, signflip)); + int8x8_t cb_biased = vreinterpret_s8_u8(vsub_u8(cb_bytes, signflip)); + + // expand to s16 + int16x8_t yws = vreinterpretq_s16_u16(vshll_n_u8(y_bytes, 4)); + int16x8_t crw = vshll_n_s8(cr_biased, 7); + int16x8_t cbw = vshll_n_s8(cb_biased, 7); + + // color transform + int16x8_t cr0 = vqdmulhq_s16(crw, cr_const0); + int16x8_t cb0 = vqdmulhq_s16(cbw, cb_const0); + int16x8_t cr1 = vqdmulhq_s16(crw, cr_const1); + int16x8_t cb1 = vqdmulhq_s16(cbw, cb_const1); + int16x8_t rws = vaddq_s16(yws, cr0); + int16x8_t gws = vaddq_s16(vaddq_s16(yws, cb0), cr1); + int16x8_t bws = vaddq_s16(yws, cb1); + + // undo scaling, round, convert to byte + uint8x8x4_t o; + o.val[0] = vqrshrun_n_s16(rws, 4); + o.val[1] = vqrshrun_n_s16(gws, 4); + o.val[2] = vqrshrun_n_s16(bws, 4); + o.val[3] = vdup_n_u8(255); + + // store, interleaving r/g/b/a + vst4_u8(out, o); + out += 8*4; + } + } +#endif + + for (; i < count; ++i) { + int y_fixed = (y[i] << 20) + (1<<19); // rounding + int r,g,b; + int cr = pcr[i] - 128; + int cb = pcb[i] - 128; + r = y_fixed + cr* stbi__float2fixed(1.40200f); + g = y_fixed + cr*-stbi__float2fixed(0.71414f) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000); + b = y_fixed + cb* stbi__float2fixed(1.77200f); + r >>= 20; + g >>= 20; + b >>= 20; + if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; } + if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; } + if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; } + out[0] = (stbi_uc)r; + out[1] = (stbi_uc)g; + out[2] = (stbi_uc)b; + out[3] = 255; + out += step; + } +} +#endif + +// set up the kernels +static void stbi__setup_jpeg(stbi__jpeg *j) +{ + j->idct_block_kernel = stbi__idct_block; + j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_row; + j->resample_row_hv_2_kernel = stbi__resample_row_hv_2; + +#ifdef STBI_SSE2 + if (stbi__sse2_available()) { + j->idct_block_kernel = stbi__idct_simd; + j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd; + j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd; + } +#endif + +#ifdef STBI_NEON + j->idct_block_kernel = stbi__idct_simd; + j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd; + j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd; +#endif +} + +// clean up the temporary component buffers +static void stbi__cleanup_jpeg(stbi__jpeg *j) +{ + stbi__free_jpeg_components(j, j->s->img_n, 0); +} + +typedef struct +{ + resample_row_func resample; + stbi_uc *line0,*line1; + int hs,vs; // expansion factor in each axis + int w_lores; // horizontal pixels pre-expansion + int ystep; // how far through vertical expansion we are + int ypos; // which pre-expansion row we're on +} stbi__resample; + +// fast 0..255 * 0..255 => 0..255 rounded multiplication +static stbi_uc stbi__blinn_8x8(stbi_uc x, stbi_uc y) +{ + unsigned int t = x*y + 128; + return (stbi_uc) ((t + (t >>8)) >> 8); +} + +static stbi_uc *load_jpeg_image(stbi__jpeg *z, int *out_x, int *out_y, int *comp, int req_comp) +{ + int n, decode_n, is_rgb; + z->s->img_n = 0; // make stbi__cleanup_jpeg safe + + // validate req_comp + if (req_comp < 0 || req_comp > 4) return stbi__errpuc("bad req_comp", "Internal error"); + + // load a jpeg image from whichever source, but leave in YCbCr format + if (!stbi__decode_jpeg_image(z)) { stbi__cleanup_jpeg(z); return NULL; } + + // determine actual number of components to generate + n = req_comp ? req_comp : z->s->img_n >= 3 ? 3 : 1; + + is_rgb = z->s->img_n == 3 && (z->rgb == 3 || (z->app14_color_transform == 0 && !z->jfif)); + + if (z->s->img_n == 3 && n < 3 && !is_rgb) + decode_n = 1; + else + decode_n = z->s->img_n; + + // nothing to do if no components requested; check this now to avoid + // accessing uninitialized coutput[0] later + if (decode_n <= 0) { stbi__cleanup_jpeg(z); return NULL; } + + // resample and color-convert + { + int k; + unsigned int i,j; + stbi_uc *output; + stbi_uc *coutput[4] = { NULL, NULL, NULL, NULL }; + + stbi__resample res_comp[4]; + + for (k=0; k < decode_n; ++k) { + stbi__resample *r = &res_comp[k]; + + // allocate line buffer big enough for upsampling off the edges + // with upsample factor of 4 + z->img_comp[k].linebuf = (stbi_uc *) stbi__malloc(z->s->img_x + 3); + if (!z->img_comp[k].linebuf) { stbi__cleanup_jpeg(z); return stbi__errpuc("outofmem", "Out of memory"); } + + r->hs = z->img_h_max / z->img_comp[k].h; + r->vs = z->img_v_max / z->img_comp[k].v; + r->ystep = r->vs >> 1; + r->w_lores = (z->s->img_x + r->hs-1) / r->hs; + r->ypos = 0; + r->line0 = r->line1 = z->img_comp[k].data; + + if (r->hs == 1 && r->vs == 1) r->resample = resample_row_1; + else if (r->hs == 1 && r->vs == 2) r->resample = stbi__resample_row_v_2; + else if (r->hs == 2 && r->vs == 1) r->resample = stbi__resample_row_h_2; + else if (r->hs == 2 && r->vs == 2) r->resample = z->resample_row_hv_2_kernel; + else r->resample = stbi__resample_row_generic; + } + + // can't error after this so, this is safe + output = (stbi_uc *) stbi__malloc_mad3(n, z->s->img_x, z->s->img_y, 1); + if (!output) { stbi__cleanup_jpeg(z); return stbi__errpuc("outofmem", "Out of memory"); } + + // now go ahead and resample + for (j=0; j < z->s->img_y; ++j) { + stbi_uc *out = output + n * z->s->img_x * j; + for (k=0; k < decode_n; ++k) { + stbi__resample *r = &res_comp[k]; + int y_bot = r->ystep >= (r->vs >> 1); + coutput[k] = r->resample(z->img_comp[k].linebuf, + y_bot ? r->line1 : r->line0, + y_bot ? r->line0 : r->line1, + r->w_lores, r->hs); + if (++r->ystep >= r->vs) { + r->ystep = 0; + r->line0 = r->line1; + if (++r->ypos < z->img_comp[k].y) + r->line1 += z->img_comp[k].w2; + } + } + if (n >= 3) { + stbi_uc *y = coutput[0]; + if (z->s->img_n == 3) { + if (is_rgb) { + for (i=0; i < z->s->img_x; ++i) { + out[0] = y[i]; + out[1] = coutput[1][i]; + out[2] = coutput[2][i]; + out[3] = 255; + out += n; + } + } else { + z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); + } + } else if (z->s->img_n == 4) { + if (z->app14_color_transform == 0) { // CMYK + for (i=0; i < z->s->img_x; ++i) { + stbi_uc m = coutput[3][i]; + out[0] = stbi__blinn_8x8(coutput[0][i], m); + out[1] = stbi__blinn_8x8(coutput[1][i], m); + out[2] = stbi__blinn_8x8(coutput[2][i], m); + out[3] = 255; + out += n; + } + } else if (z->app14_color_transform == 2) { // YCCK + z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); + for (i=0; i < z->s->img_x; ++i) { + stbi_uc m = coutput[3][i]; + out[0] = stbi__blinn_8x8(255 - out[0], m); + out[1] = stbi__blinn_8x8(255 - out[1], m); + out[2] = stbi__blinn_8x8(255 - out[2], m); + out += n; + } + } else { // YCbCr + alpha? Ignore the fourth channel for now + z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); + } + } else + for (i=0; i < z->s->img_x; ++i) { + out[0] = out[1] = out[2] = y[i]; + out[3] = 255; // not used if n==3 + out += n; + } + } else { + if (is_rgb) { + if (n == 1) + for (i=0; i < z->s->img_x; ++i) + *out++ = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]); + else { + for (i=0; i < z->s->img_x; ++i, out += 2) { + out[0] = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]); + out[1] = 255; + } + } + } else if (z->s->img_n == 4 && z->app14_color_transform == 0) { + for (i=0; i < z->s->img_x; ++i) { + stbi_uc m = coutput[3][i]; + stbi_uc r = stbi__blinn_8x8(coutput[0][i], m); + stbi_uc g = stbi__blinn_8x8(coutput[1][i], m); + stbi_uc b = stbi__blinn_8x8(coutput[2][i], m); + out[0] = stbi__compute_y(r, g, b); + out[1] = 255; + out += n; + } + } else if (z->s->img_n == 4 && z->app14_color_transform == 2) { + for (i=0; i < z->s->img_x; ++i) { + out[0] = stbi__blinn_8x8(255 - coutput[0][i], coutput[3][i]); + out[1] = 255; + out += n; + } + } else { + stbi_uc *y = coutput[0]; + if (n == 1) + for (i=0; i < z->s->img_x; ++i) out[i] = y[i]; + else + for (i=0; i < z->s->img_x; ++i) { *out++ = y[i]; *out++ = 255; } + } + } + } + stbi__cleanup_jpeg(z); + *out_x = z->s->img_x; + *out_y = z->s->img_y; + if (comp) *comp = z->s->img_n >= 3 ? 3 : 1; // report original components, not output + return output; + } +} + +static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + unsigned char* result; + stbi__jpeg* j = (stbi__jpeg*) stbi__malloc(sizeof(stbi__jpeg)); + if (!j) return stbi__errpuc("outofmem", "Out of memory"); + memset(j, 0, sizeof(stbi__jpeg)); + STBI_NOTUSED(ri); + j->s = s; + stbi__setup_jpeg(j); + result = load_jpeg_image(j, x,y,comp,req_comp); + STBI_FREE(j); + return result; +} + +static int stbi__jpeg_test(stbi__context *s) +{ + int r; + stbi__jpeg* j = (stbi__jpeg*)stbi__malloc(sizeof(stbi__jpeg)); + if (!j) return stbi__err("outofmem", "Out of memory"); + memset(j, 0, sizeof(stbi__jpeg)); + j->s = s; + stbi__setup_jpeg(j); + r = stbi__decode_jpeg_header(j, STBI__SCAN_type); + stbi__rewind(s); + STBI_FREE(j); + return r; +} + +static int stbi__jpeg_info_raw(stbi__jpeg *j, int *x, int *y, int *comp) +{ + if (!stbi__decode_jpeg_header(j, STBI__SCAN_header)) { + stbi__rewind( j->s ); + return 0; + } + if (x) *x = j->s->img_x; + if (y) *y = j->s->img_y; + if (comp) *comp = j->s->img_n >= 3 ? 3 : 1; + return 1; +} + +static int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp) +{ + int result; + stbi__jpeg* j = (stbi__jpeg*) (stbi__malloc(sizeof(stbi__jpeg))); + if (!j) return stbi__err("outofmem", "Out of memory"); + memset(j, 0, sizeof(stbi__jpeg)); + j->s = s; + result = stbi__jpeg_info_raw(j, x, y, comp); + STBI_FREE(j); + return result; +} +#endif + +// public domain zlib decode v0.2 Sean Barrett 2006-11-18 +// simple implementation +// - all input must be provided in an upfront buffer +// - all output is written to a single output buffer (can malloc/realloc) +// performance +// - fast huffman + +#ifndef STBI_NO_ZLIB + +// fast-way is faster to check than jpeg huffman, but slow way is slower +#define STBI__ZFAST_BITS 9 // accelerate all cases in default tables +#define STBI__ZFAST_MASK ((1 << STBI__ZFAST_BITS) - 1) +#define STBI__ZNSYMS 288 // number of symbols in literal/length alphabet + +// zlib-style huffman encoding +// (jpegs packs from left, zlib from right, so can't share code) +typedef struct +{ + stbi__uint16 fast[1 << STBI__ZFAST_BITS]; + stbi__uint16 firstcode[16]; + int maxcode[17]; + stbi__uint16 firstsymbol[16]; + stbi_uc size[STBI__ZNSYMS]; + stbi__uint16 value[STBI__ZNSYMS]; +} stbi__zhuffman; + +stbi_inline static int stbi__bitreverse16(int n) +{ + n = ((n & 0xAAAA) >> 1) | ((n & 0x5555) << 1); + n = ((n & 0xCCCC) >> 2) | ((n & 0x3333) << 2); + n = ((n & 0xF0F0) >> 4) | ((n & 0x0F0F) << 4); + n = ((n & 0xFF00) >> 8) | ((n & 0x00FF) << 8); + return n; +} + +stbi_inline static int stbi__bit_reverse(int v, int bits) +{ + STBI_ASSERT(bits <= 16); + // to bit reverse n bits, reverse 16 and shift + // e.g. 11 bits, bit reverse and shift away 5 + return stbi__bitreverse16(v) >> (16-bits); +} + +static int stbi__zbuild_huffman(stbi__zhuffman *z, const stbi_uc *sizelist, int num) +{ + int i,k=0; + int code, next_code[16], sizes[17]; + + // DEFLATE spec for generating codes + memset(sizes, 0, sizeof(sizes)); + memset(z->fast, 0, sizeof(z->fast)); + for (i=0; i < num; ++i) + ++sizes[sizelist[i]]; + sizes[0] = 0; + for (i=1; i < 16; ++i) + if (sizes[i] > (1 << i)) + return stbi__err("bad sizes", "Corrupt PNG"); + code = 0; + for (i=1; i < 16; ++i) { + next_code[i] = code; + z->firstcode[i] = (stbi__uint16) code; + z->firstsymbol[i] = (stbi__uint16) k; + code = (code + sizes[i]); + if (sizes[i]) + if (code-1 >= (1 << i)) return stbi__err("bad codelengths","Corrupt PNG"); + z->maxcode[i] = code << (16-i); // preshift for inner loop + code <<= 1; + k += sizes[i]; + } + z->maxcode[16] = 0x10000; // sentinel + for (i=0; i < num; ++i) { + int s = sizelist[i]; + if (s) { + int c = next_code[s] - z->firstcode[s] + z->firstsymbol[s]; + stbi__uint16 fastv = (stbi__uint16) ((s << 9) | i); + z->size [c] = (stbi_uc ) s; + z->value[c] = (stbi__uint16) i; + if (s <= STBI__ZFAST_BITS) { + int j = stbi__bit_reverse(next_code[s],s); + while (j < (1 << STBI__ZFAST_BITS)) { + z->fast[j] = fastv; + j += (1 << s); + } + } + ++next_code[s]; + } + } + return 1; +} + +// zlib-from-memory implementation for PNG reading +// because PNG allows splitting the zlib stream arbitrarily, +// and it's annoying structurally to have PNG call ZLIB call PNG, +// we require PNG read all the IDATs and combine them into a single +// memory buffer + +typedef struct +{ + stbi_uc *zbuffer, *zbuffer_end; + int num_bits; + int hit_zeof_once; + stbi__uint32 code_buffer; + + char *zout; + char *zout_start; + char *zout_end; + int z_expandable; + + stbi__zhuffman z_length, z_distance; +} stbi__zbuf; + +stbi_inline static int stbi__zeof(stbi__zbuf *z) +{ + return (z->zbuffer >= z->zbuffer_end); +} + +stbi_inline static stbi_uc stbi__zget8(stbi__zbuf *z) +{ + return stbi__zeof(z) ? 0 : *z->zbuffer++; +} + +static void stbi__fill_bits(stbi__zbuf *z) +{ + do { + if (z->code_buffer >= (1U << z->num_bits)) { + z->zbuffer = z->zbuffer_end; /* treat this as EOF so we fail. */ + return; + } + z->code_buffer |= (unsigned int) stbi__zget8(z) << z->num_bits; + z->num_bits += 8; + } while (z->num_bits <= 24); +} + +stbi_inline static unsigned int stbi__zreceive(stbi__zbuf *z, int n) +{ + unsigned int k; + if (z->num_bits < n) stbi__fill_bits(z); + k = z->code_buffer & ((1 << n) - 1); + z->code_buffer >>= n; + z->num_bits -= n; + return k; +} + +static int stbi__zhuffman_decode_slowpath(stbi__zbuf *a, stbi__zhuffman *z) +{ + int b,s,k; + // not resolved by fast table, so compute it the slow way + // use jpeg approach, which requires MSbits at top + k = stbi__bit_reverse(a->code_buffer, 16); + for (s=STBI__ZFAST_BITS+1; ; ++s) + if (k < z->maxcode[s]) + break; + if (s >= 16) return -1; // invalid code! + // code size is s, so: + b = (k >> (16-s)) - z->firstcode[s] + z->firstsymbol[s]; + if (b >= STBI__ZNSYMS) return -1; // some data was corrupt somewhere! + if (z->size[b] != s) return -1; // was originally an assert, but report failure instead. + a->code_buffer >>= s; + a->num_bits -= s; + return z->value[b]; +} + +stbi_inline static int stbi__zhuffman_decode(stbi__zbuf *a, stbi__zhuffman *z) +{ + int b,s; + if (a->num_bits < 16) { + if (stbi__zeof(a)) { + if (!a->hit_zeof_once) { + // This is the first time we hit eof, insert 16 extra padding btis + // to allow us to keep going; if we actually consume any of them + // though, that is invalid data. This is caught later. + a->hit_zeof_once = 1; + a->num_bits += 16; // add 16 implicit zero bits + } else { + // We already inserted our extra 16 padding bits and are again + // out, this stream is actually prematurely terminated. + return -1; + } + } else { + stbi__fill_bits(a); + } + } + b = z->fast[a->code_buffer & STBI__ZFAST_MASK]; + if (b) { + s = b >> 9; + a->code_buffer >>= s; + a->num_bits -= s; + return b & 511; + } + return stbi__zhuffman_decode_slowpath(a, z); +} + +static int stbi__zexpand(stbi__zbuf *z, char *zout, int n) // need to make room for n bytes +{ + char *q; + unsigned int cur, limit, old_limit; + z->zout = zout; + if (!z->z_expandable) return stbi__err("output buffer limit","Corrupt PNG"); + cur = (unsigned int) (z->zout - z->zout_start); + limit = old_limit = (unsigned) (z->zout_end - z->zout_start); + if (UINT_MAX - cur < (unsigned) n) return stbi__err("outofmem", "Out of memory"); + while (cur + n > limit) { + if(limit > UINT_MAX / 2) return stbi__err("outofmem", "Out of memory"); + limit *= 2; + } + q = (char *) STBI_REALLOC_SIZED(z->zout_start, old_limit, limit); + STBI_NOTUSED(old_limit); + if (q == NULL) return stbi__err("outofmem", "Out of memory"); + z->zout_start = q; + z->zout = q + cur; + z->zout_end = q + limit; + return 1; +} + +static const int stbi__zlength_base[31] = { + 3,4,5,6,7,8,9,10,11,13, + 15,17,19,23,27,31,35,43,51,59, + 67,83,99,115,131,163,195,227,258,0,0 }; + +static const int stbi__zlength_extra[31]= +{ 0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0 }; + +static const int stbi__zdist_base[32] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193, +257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577,0,0}; + +static const int stbi__zdist_extra[32] = +{ 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13}; + +static int stbi__parse_huffman_block(stbi__zbuf *a) +{ + char *zout = a->zout; + for(;;) { + int z = stbi__zhuffman_decode(a, &a->z_length); + if (z < 256) { + if (z < 0) return stbi__err("bad huffman code","Corrupt PNG"); // error in huffman codes + if (zout >= a->zout_end) { + if (!stbi__zexpand(a, zout, 1)) return 0; + zout = a->zout; + } + *zout++ = (char) z; + } else { + stbi_uc *p; + int len,dist; + if (z == 256) { + a->zout = zout; + if (a->hit_zeof_once && a->num_bits < 16) { + // The first time we hit zeof, we inserted 16 extra zero bits into our bit + // buffer so the decoder can just do its speculative decoding. But if we + // actually consumed any of those bits (which is the case when num_bits < 16), + // the stream actually read past the end so it is malformed. + return stbi__err("unexpected end","Corrupt PNG"); + } + return 1; + } + if (z >= 286) return stbi__err("bad huffman code","Corrupt PNG"); // per DEFLATE, length codes 286 and 287 must not appear in compressed data + z -= 257; + len = stbi__zlength_base[z]; + if (stbi__zlength_extra[z]) len += stbi__zreceive(a, stbi__zlength_extra[z]); + z = stbi__zhuffman_decode(a, &a->z_distance); + if (z < 0 || z >= 30) return stbi__err("bad huffman code","Corrupt PNG"); // per DEFLATE, distance codes 30 and 31 must not appear in compressed data + dist = stbi__zdist_base[z]; + if (stbi__zdist_extra[z]) dist += stbi__zreceive(a, stbi__zdist_extra[z]); + if (zout - a->zout_start < dist) return stbi__err("bad dist","Corrupt PNG"); + if (len > a->zout_end - zout) { + if (!stbi__zexpand(a, zout, len)) return 0; + zout = a->zout; + } + p = (stbi_uc *) (zout - dist); + if (dist == 1) { // run of one byte; common in images. + stbi_uc v = *p; + if (len) { do *zout++ = v; while (--len); } + } else { + if (len) { do *zout++ = *p++; while (--len); } + } + } + } +} + +static int stbi__compute_huffman_codes(stbi__zbuf *a) +{ + static const stbi_uc length_dezigzag[19] = { 16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15 }; + stbi__zhuffman z_codelength; + stbi_uc lencodes[286+32+137];//padding for maximum single op + stbi_uc codelength_sizes[19]; + int i,n; + + int hlit = stbi__zreceive(a,5) + 257; + int hdist = stbi__zreceive(a,5) + 1; + int hclen = stbi__zreceive(a,4) + 4; + int ntot = hlit + hdist; + + memset(codelength_sizes, 0, sizeof(codelength_sizes)); + for (i=0; i < hclen; ++i) { + int s = stbi__zreceive(a,3); + codelength_sizes[length_dezigzag[i]] = (stbi_uc) s; + } + if (!stbi__zbuild_huffman(&z_codelength, codelength_sizes, 19)) return 0; + + n = 0; + while (n < ntot) { + int c = stbi__zhuffman_decode(a, &z_codelength); + if (c < 0 || c >= 19) return stbi__err("bad codelengths", "Corrupt PNG"); + if (c < 16) + lencodes[n++] = (stbi_uc) c; + else { + stbi_uc fill = 0; + if (c == 16) { + c = stbi__zreceive(a,2)+3; + if (n == 0) return stbi__err("bad codelengths", "Corrupt PNG"); + fill = lencodes[n-1]; + } else if (c == 17) { + c = stbi__zreceive(a,3)+3; + } else if (c == 18) { + c = stbi__zreceive(a,7)+11; + } else { + return stbi__err("bad codelengths", "Corrupt PNG"); + } + if (ntot - n < c) return stbi__err("bad codelengths", "Corrupt PNG"); + memset(lencodes+n, fill, c); + n += c; + } + } + if (n != ntot) return stbi__err("bad codelengths","Corrupt PNG"); + if (!stbi__zbuild_huffman(&a->z_length, lencodes, hlit)) return 0; + if (!stbi__zbuild_huffman(&a->z_distance, lencodes+hlit, hdist)) return 0; + return 1; +} + +static int stbi__parse_uncompressed_block(stbi__zbuf *a) +{ + stbi_uc header[4]; + int len,nlen,k; + if (a->num_bits & 7) + stbi__zreceive(a, a->num_bits & 7); // discard + // drain the bit-packed data into header + k = 0; + while (a->num_bits > 0) { + header[k++] = (stbi_uc) (a->code_buffer & 255); // suppress MSVC run-time check + a->code_buffer >>= 8; + a->num_bits -= 8; + } + if (a->num_bits < 0) return stbi__err("zlib corrupt","Corrupt PNG"); + // now fill header the normal way + while (k < 4) + header[k++] = stbi__zget8(a); + len = header[1] * 256 + header[0]; + nlen = header[3] * 256 + header[2]; + if (nlen != (len ^ 0xffff)) return stbi__err("zlib corrupt","Corrupt PNG"); + if (a->zbuffer + len > a->zbuffer_end) return stbi__err("read past buffer","Corrupt PNG"); + if (a->zout + len > a->zout_end) + if (!stbi__zexpand(a, a->zout, len)) return 0; + memcpy(a->zout, a->zbuffer, len); + a->zbuffer += len; + a->zout += len; + return 1; +} + +static int stbi__parse_zlib_header(stbi__zbuf *a) +{ + int cmf = stbi__zget8(a); + int cm = cmf & 15; + /* int cinfo = cmf >> 4; */ + int flg = stbi__zget8(a); + if (stbi__zeof(a)) return stbi__err("bad zlib header","Corrupt PNG"); // zlib spec + if ((cmf*256+flg) % 31 != 0) return stbi__err("bad zlib header","Corrupt PNG"); // zlib spec + if (flg & 32) return stbi__err("no preset dict","Corrupt PNG"); // preset dictionary not allowed in png + if (cm != 8) return stbi__err("bad compression","Corrupt PNG"); // DEFLATE required for png + // window = 1 << (8 + cinfo)... but who cares, we fully buffer output + return 1; +} + +static const stbi_uc stbi__zdefault_length[STBI__ZNSYMS] = +{ + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8 +}; +static const stbi_uc stbi__zdefault_distance[32] = +{ + 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 +}; +/* +Init algorithm: +{ + int i; // use <= to match clearly with spec + for (i=0; i <= 143; ++i) stbi__zdefault_length[i] = 8; + for ( ; i <= 255; ++i) stbi__zdefault_length[i] = 9; + for ( ; i <= 279; ++i) stbi__zdefault_length[i] = 7; + for ( ; i <= 287; ++i) stbi__zdefault_length[i] = 8; + + for (i=0; i <= 31; ++i) stbi__zdefault_distance[i] = 5; +} +*/ + +static int stbi__parse_zlib(stbi__zbuf *a, int parse_header) +{ + int final, type; + if (parse_header) + if (!stbi__parse_zlib_header(a)) return 0; + a->num_bits = 0; + a->code_buffer = 0; + a->hit_zeof_once = 0; + do { + final = stbi__zreceive(a,1); + type = stbi__zreceive(a,2); + if (type == 0) { + if (!stbi__parse_uncompressed_block(a)) return 0; + } else if (type == 3) { + return 0; + } else { + if (type == 1) { + // use fixed code lengths + if (!stbi__zbuild_huffman(&a->z_length , stbi__zdefault_length , STBI__ZNSYMS)) return 0; + if (!stbi__zbuild_huffman(&a->z_distance, stbi__zdefault_distance, 32)) return 0; + } else { + if (!stbi__compute_huffman_codes(a)) return 0; + } + if (!stbi__parse_huffman_block(a)) return 0; + } + } while (!final); + return 1; +} + +static int stbi__do_zlib(stbi__zbuf *a, char *obuf, int olen, int exp, int parse_header) +{ + a->zout_start = obuf; + a->zout = obuf; + a->zout_end = obuf + olen; + a->z_expandable = exp; + + return stbi__parse_zlib(a, parse_header); +} + +STBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen) +{ + stbi__zbuf a; + char *p = (char *) stbi__malloc(initial_size); + if (p == NULL) return NULL; + a.zbuffer = (stbi_uc *) buffer; + a.zbuffer_end = (stbi_uc *) buffer + len; + if (stbi__do_zlib(&a, p, initial_size, 1, 1)) { + if (outlen) *outlen = (int) (a.zout - a.zout_start); + return a.zout_start; + } else { + STBI_FREE(a.zout_start); + return NULL; + } +} + +STBIDEF char *stbi_zlib_decode_malloc(char const *buffer, int len, int *outlen) +{ + return stbi_zlib_decode_malloc_guesssize(buffer, len, 16384, outlen); +} + +STBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header) +{ + stbi__zbuf a; + char *p = (char *) stbi__malloc(initial_size); + if (p == NULL) return NULL; + a.zbuffer = (stbi_uc *) buffer; + a.zbuffer_end = (stbi_uc *) buffer + len; + if (stbi__do_zlib(&a, p, initial_size, 1, parse_header)) { + if (outlen) *outlen = (int) (a.zout - a.zout_start); + return a.zout_start; + } else { + STBI_FREE(a.zout_start); + return NULL; + } +} + +STBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, char const *ibuffer, int ilen) +{ + stbi__zbuf a; + a.zbuffer = (stbi_uc *) ibuffer; + a.zbuffer_end = (stbi_uc *) ibuffer + ilen; + if (stbi__do_zlib(&a, obuffer, olen, 0, 1)) + return (int) (a.zout - a.zout_start); + else + return -1; +} + +STBIDEF char *stbi_zlib_decode_noheader_malloc(char const *buffer, int len, int *outlen) +{ + stbi__zbuf a; + char *p = (char *) stbi__malloc(16384); + if (p == NULL) return NULL; + a.zbuffer = (stbi_uc *) buffer; + a.zbuffer_end = (stbi_uc *) buffer+len; + if (stbi__do_zlib(&a, p, 16384, 1, 0)) { + if (outlen) *outlen = (int) (a.zout - a.zout_start); + return a.zout_start; + } else { + STBI_FREE(a.zout_start); + return NULL; + } +} + +STBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen) +{ + stbi__zbuf a; + a.zbuffer = (stbi_uc *) ibuffer; + a.zbuffer_end = (stbi_uc *) ibuffer + ilen; + if (stbi__do_zlib(&a, obuffer, olen, 0, 0)) + return (int) (a.zout - a.zout_start); + else + return -1; +} +#endif + +// public domain "baseline" PNG decoder v0.10 Sean Barrett 2006-11-18 +// simple implementation +// - only 8-bit samples +// - no CRC checking +// - allocates lots of intermediate memory +// - avoids problem of streaming data between subsystems +// - avoids explicit window management +// performance +// - uses stb_zlib, a PD zlib implementation with fast huffman decoding + +#ifndef STBI_NO_PNG +typedef struct +{ + stbi__uint32 length; + stbi__uint32 type; +} stbi__pngchunk; + +static stbi__pngchunk stbi__get_chunk_header(stbi__context *s) +{ + stbi__pngchunk c; + c.length = stbi__get32be(s); + c.type = stbi__get32be(s); + return c; +} + +static int stbi__check_png_header(stbi__context *s) +{ + static const stbi_uc png_sig[8] = { 137,80,78,71,13,10,26,10 }; + int i; + for (i=0; i < 8; ++i) + if (stbi__get8(s) != png_sig[i]) return stbi__err("bad png sig","Not a PNG"); + return 1; +} + +typedef struct +{ + stbi__context *s; + stbi_uc *idata, *expanded, *out; + int depth; +} stbi__png; + + +enum { + STBI__F_none=0, + STBI__F_sub=1, + STBI__F_up=2, + STBI__F_avg=3, + STBI__F_paeth=4, + // synthetic filter used for first scanline to avoid needing a dummy row of 0s + STBI__F_avg_first +}; + +static stbi_uc first_row_filter[5] = +{ + STBI__F_none, + STBI__F_sub, + STBI__F_none, + STBI__F_avg_first, + STBI__F_sub // Paeth with b=c=0 turns out to be equivalent to sub +}; + +static int stbi__paeth(int a, int b, int c) +{ + // This formulation looks very different from the reference in the PNG spec, but is + // actually equivalent and has favorable data dependencies and admits straightforward + // generation of branch-free code, which helps performance significantly. + int thresh = c*3 - (a + b); + int lo = a < b ? a : b; + int hi = a < b ? b : a; + int t0 = (hi <= thresh) ? lo : c; + int t1 = (thresh <= lo) ? hi : t0; + return t1; +} + +static const stbi_uc stbi__depth_scale_table[9] = { 0, 0xff, 0x55, 0, 0x11, 0,0,0, 0x01 }; + +// adds an extra all-255 alpha channel +// dest == src is legal +// img_n must be 1 or 3 +static void stbi__create_png_alpha_expand8(stbi_uc *dest, stbi_uc *src, stbi__uint32 x, int img_n) +{ + int i; + // must process data backwards since we allow dest==src + if (img_n == 1) { + for (i=x-1; i >= 0; --i) { + dest[i*2+1] = 255; + dest[i*2+0] = src[i]; + } + } else { + STBI_ASSERT(img_n == 3); + for (i=x-1; i >= 0; --i) { + dest[i*4+3] = 255; + dest[i*4+2] = src[i*3+2]; + dest[i*4+1] = src[i*3+1]; + dest[i*4+0] = src[i*3+0]; + } + } +} + +// create the png data from post-deflated data +static int stbi__create_png_image_raw(stbi__png *a, stbi_uc *raw, stbi__uint32 raw_len, int out_n, stbi__uint32 x, stbi__uint32 y, int depth, int color) +{ + int bytes = (depth == 16 ? 2 : 1); + stbi__context *s = a->s; + stbi__uint32 i,j,stride = x*out_n*bytes; + stbi__uint32 img_len, img_width_bytes; + stbi_uc *filter_buf; + int all_ok = 1; + int k; + int img_n = s->img_n; // copy it into a local for later + + int output_bytes = out_n*bytes; + int filter_bytes = img_n*bytes; + int width = x; + + STBI_ASSERT(out_n == s->img_n || out_n == s->img_n+1); + a->out = (stbi_uc *) stbi__malloc_mad3(x, y, output_bytes, 0); // extra bytes to write off the end into + if (!a->out) return stbi__err("outofmem", "Out of memory"); + + // note: error exits here don't need to clean up a->out individually, + // stbi__do_png always does on error. + if (!stbi__mad3sizes_valid(img_n, x, depth, 7)) return stbi__err("too large", "Corrupt PNG"); + img_width_bytes = (((img_n * x * depth) + 7) >> 3); + if (!stbi__mad2sizes_valid(img_width_bytes, y, img_width_bytes)) return stbi__err("too large", "Corrupt PNG"); + img_len = (img_width_bytes + 1) * y; + + // we used to check for exact match between raw_len and img_len on non-interlaced PNGs, + // but issue #276 reported a PNG in the wild that had extra data at the end (all zeros), + // so just check for raw_len < img_len always. + if (raw_len < img_len) return stbi__err("not enough pixels","Corrupt PNG"); + + // Allocate two scan lines worth of filter workspace buffer. + filter_buf = (stbi_uc *) stbi__malloc_mad2(img_width_bytes, 2, 0); + if (!filter_buf) return stbi__err("outofmem", "Out of memory"); + + // Filtering for low-bit-depth images + if (depth < 8) { + filter_bytes = 1; + width = img_width_bytes; + } + + for (j=0; j < y; ++j) { + // cur/prior filter buffers alternate + stbi_uc *cur = filter_buf + (j & 1)*img_width_bytes; + stbi_uc *prior = filter_buf + (~j & 1)*img_width_bytes; + stbi_uc *dest = a->out + stride*j; + int nk = width * filter_bytes; + int filter = *raw++; + + // check filter type + if (filter > 4) { + all_ok = stbi__err("invalid filter","Corrupt PNG"); + break; + } + + // if first row, use special filter that doesn't sample previous row + if (j == 0) filter = first_row_filter[filter]; + + // perform actual filtering + switch (filter) { + case STBI__F_none: + memcpy(cur, raw, nk); + break; + case STBI__F_sub: + memcpy(cur, raw, filter_bytes); + for (k = filter_bytes; k < nk; ++k) + cur[k] = STBI__BYTECAST(raw[k] + cur[k-filter_bytes]); + break; + case STBI__F_up: + for (k = 0; k < nk; ++k) + cur[k] = STBI__BYTECAST(raw[k] + prior[k]); + break; + case STBI__F_avg: + for (k = 0; k < filter_bytes; ++k) + cur[k] = STBI__BYTECAST(raw[k] + (prior[k]>>1)); + for (k = filter_bytes; k < nk; ++k) + cur[k] = STBI__BYTECAST(raw[k] + ((prior[k] + cur[k-filter_bytes])>>1)); + break; + case STBI__F_paeth: + for (k = 0; k < filter_bytes; ++k) + cur[k] = STBI__BYTECAST(raw[k] + prior[k]); // prior[k] == stbi__paeth(0,prior[k],0) + for (k = filter_bytes; k < nk; ++k) + cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k-filter_bytes], prior[k], prior[k-filter_bytes])); + break; + case STBI__F_avg_first: + memcpy(cur, raw, filter_bytes); + for (k = filter_bytes; k < nk; ++k) + cur[k] = STBI__BYTECAST(raw[k] + (cur[k-filter_bytes] >> 1)); + break; + } + + raw += nk; + + // expand decoded bits in cur to dest, also adding an extra alpha channel if desired + if (depth < 8) { + stbi_uc scale = (color == 0) ? stbi__depth_scale_table[depth] : 1; // scale grayscale values to 0..255 range + stbi_uc *in = cur; + stbi_uc *out = dest; + stbi_uc inb = 0; + stbi__uint32 nsmp = x*img_n; + + // expand bits to bytes first + if (depth == 4) { + for (i=0; i < nsmp; ++i) { + if ((i & 1) == 0) inb = *in++; + *out++ = scale * (inb >> 4); + inb <<= 4; + } + } else if (depth == 2) { + for (i=0; i < nsmp; ++i) { + if ((i & 3) == 0) inb = *in++; + *out++ = scale * (inb >> 6); + inb <<= 2; + } + } else { + STBI_ASSERT(depth == 1); + for (i=0; i < nsmp; ++i) { + if ((i & 7) == 0) inb = *in++; + *out++ = scale * (inb >> 7); + inb <<= 1; + } + } + + // insert alpha=255 values if desired + if (img_n != out_n) + stbi__create_png_alpha_expand8(dest, dest, x, img_n); + } else if (depth == 8) { + if (img_n == out_n) + memcpy(dest, cur, x*img_n); + else + stbi__create_png_alpha_expand8(dest, cur, x, img_n); + } else if (depth == 16) { + // convert the image data from big-endian to platform-native + stbi__uint16 *dest16 = (stbi__uint16*)dest; + stbi__uint32 nsmp = x*img_n; + + if (img_n == out_n) { + for (i = 0; i < nsmp; ++i, ++dest16, cur += 2) + *dest16 = (cur[0] << 8) | cur[1]; + } else { + STBI_ASSERT(img_n+1 == out_n); + if (img_n == 1) { + for (i = 0; i < x; ++i, dest16 += 2, cur += 2) { + dest16[0] = (cur[0] << 8) | cur[1]; + dest16[1] = 0xffff; + } + } else { + STBI_ASSERT(img_n == 3); + for (i = 0; i < x; ++i, dest16 += 4, cur += 6) { + dest16[0] = (cur[0] << 8) | cur[1]; + dest16[1] = (cur[2] << 8) | cur[3]; + dest16[2] = (cur[4] << 8) | cur[5]; + dest16[3] = 0xffff; + } + } + } + } + } + + STBI_FREE(filter_buf); + if (!all_ok) return 0; + + return 1; +} + +static int stbi__create_png_image(stbi__png *a, stbi_uc *image_data, stbi__uint32 image_data_len, int out_n, int depth, int color, int interlaced) +{ + int bytes = (depth == 16 ? 2 : 1); + int out_bytes = out_n * bytes; + stbi_uc *final; + int p; + if (!interlaced) + return stbi__create_png_image_raw(a, image_data, image_data_len, out_n, a->s->img_x, a->s->img_y, depth, color); + + // de-interlacing + final = (stbi_uc *) stbi__malloc_mad3(a->s->img_x, a->s->img_y, out_bytes, 0); + if (!final) return stbi__err("outofmem", "Out of memory"); + for (p=0; p < 7; ++p) { + int xorig[] = { 0,4,0,2,0,1,0 }; + int yorig[] = { 0,0,4,0,2,0,1 }; + int xspc[] = { 8,8,4,4,2,2,1 }; + int yspc[] = { 8,8,8,4,4,2,2 }; + int i,j,x,y; + // pass1_x[4] = 0, pass1_x[5] = 1, pass1_x[12] = 1 + x = (a->s->img_x - xorig[p] + xspc[p]-1) / xspc[p]; + y = (a->s->img_y - yorig[p] + yspc[p]-1) / yspc[p]; + if (x && y) { + stbi__uint32 img_len = ((((a->s->img_n * x * depth) + 7) >> 3) + 1) * y; + if (!stbi__create_png_image_raw(a, image_data, image_data_len, out_n, x, y, depth, color)) { + STBI_FREE(final); + return 0; + } + for (j=0; j < y; ++j) { + for (i=0; i < x; ++i) { + int out_y = j*yspc[p]+yorig[p]; + int out_x = i*xspc[p]+xorig[p]; + memcpy(final + out_y*a->s->img_x*out_bytes + out_x*out_bytes, + a->out + (j*x+i)*out_bytes, out_bytes); + } + } + STBI_FREE(a->out); + image_data += img_len; + image_data_len -= img_len; + } + } + a->out = final; + + return 1; +} + +static int stbi__compute_transparency(stbi__png *z, stbi_uc tc[3], int out_n) +{ + stbi__context *s = z->s; + stbi__uint32 i, pixel_count = s->img_x * s->img_y; + stbi_uc *p = z->out; + + // compute color-based transparency, assuming we've + // already got 255 as the alpha value in the output + STBI_ASSERT(out_n == 2 || out_n == 4); + + if (out_n == 2) { + for (i=0; i < pixel_count; ++i) { + p[1] = (p[0] == tc[0] ? 0 : 255); + p += 2; + } + } else { + for (i=0; i < pixel_count; ++i) { + if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2]) + p[3] = 0; + p += 4; + } + } + return 1; +} + +static int stbi__compute_transparency16(stbi__png *z, stbi__uint16 tc[3], int out_n) +{ + stbi__context *s = z->s; + stbi__uint32 i, pixel_count = s->img_x * s->img_y; + stbi__uint16 *p = (stbi__uint16*) z->out; + + // compute color-based transparency, assuming we've + // already got 65535 as the alpha value in the output + STBI_ASSERT(out_n == 2 || out_n == 4); + + if (out_n == 2) { + for (i = 0; i < pixel_count; ++i) { + p[1] = (p[0] == tc[0] ? 0 : 65535); + p += 2; + } + } else { + for (i = 0; i < pixel_count; ++i) { + if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2]) + p[3] = 0; + p += 4; + } + } + return 1; +} + +static int stbi__expand_png_palette(stbi__png *a, stbi_uc *palette, int len, int pal_img_n) +{ + stbi__uint32 i, pixel_count = a->s->img_x * a->s->img_y; + stbi_uc *p, *temp_out, *orig = a->out; + + p = (stbi_uc *) stbi__malloc_mad2(pixel_count, pal_img_n, 0); + if (p == NULL) return stbi__err("outofmem", "Out of memory"); + + // between here and free(out) below, exitting would leak + temp_out = p; + + if (pal_img_n == 3) { + for (i=0; i < pixel_count; ++i) { + int n = orig[i]*4; + p[0] = palette[n ]; + p[1] = palette[n+1]; + p[2] = palette[n+2]; + p += 3; + } + } else { + for (i=0; i < pixel_count; ++i) { + int n = orig[i]*4; + p[0] = palette[n ]; + p[1] = palette[n+1]; + p[2] = palette[n+2]; + p[3] = palette[n+3]; + p += 4; + } + } + STBI_FREE(a->out); + a->out = temp_out; + + STBI_NOTUSED(len); + + return 1; +} + +static int stbi__unpremultiply_on_load_global = 0; +static int stbi__de_iphone_flag_global = 0; + +STBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply) +{ + stbi__unpremultiply_on_load_global = flag_true_if_should_unpremultiply; +} + +STBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert) +{ + stbi__de_iphone_flag_global = flag_true_if_should_convert; +} + +#ifndef STBI_THREAD_LOCAL +#define stbi__unpremultiply_on_load stbi__unpremultiply_on_load_global +#define stbi__de_iphone_flag stbi__de_iphone_flag_global +#else +static STBI_THREAD_LOCAL int stbi__unpremultiply_on_load_local, stbi__unpremultiply_on_load_set; +static STBI_THREAD_LOCAL int stbi__de_iphone_flag_local, stbi__de_iphone_flag_set; + +STBIDEF void stbi_set_unpremultiply_on_load_thread(int flag_true_if_should_unpremultiply) +{ + stbi__unpremultiply_on_load_local = flag_true_if_should_unpremultiply; + stbi__unpremultiply_on_load_set = 1; +} + +STBIDEF void stbi_convert_iphone_png_to_rgb_thread(int flag_true_if_should_convert) +{ + stbi__de_iphone_flag_local = flag_true_if_should_convert; + stbi__de_iphone_flag_set = 1; +} + +#define stbi__unpremultiply_on_load (stbi__unpremultiply_on_load_set \ + ? stbi__unpremultiply_on_load_local \ + : stbi__unpremultiply_on_load_global) +#define stbi__de_iphone_flag (stbi__de_iphone_flag_set \ + ? stbi__de_iphone_flag_local \ + : stbi__de_iphone_flag_global) +#endif // STBI_THREAD_LOCAL + +static void stbi__de_iphone(stbi__png *z) +{ + stbi__context *s = z->s; + stbi__uint32 i, pixel_count = s->img_x * s->img_y; + stbi_uc *p = z->out; + + if (s->img_out_n == 3) { // convert bgr to rgb + for (i=0; i < pixel_count; ++i) { + stbi_uc t = p[0]; + p[0] = p[2]; + p[2] = t; + p += 3; + } + } else { + STBI_ASSERT(s->img_out_n == 4); + if (stbi__unpremultiply_on_load) { + // convert bgr to rgb and unpremultiply + for (i=0; i < pixel_count; ++i) { + stbi_uc a = p[3]; + stbi_uc t = p[0]; + if (a) { + stbi_uc half = a / 2; + p[0] = (p[2] * 255 + half) / a; + p[1] = (p[1] * 255 + half) / a; + p[2] = ( t * 255 + half) / a; + } else { + p[0] = p[2]; + p[2] = t; + } + p += 4; + } + } else { + // convert bgr to rgb + for (i=0; i < pixel_count; ++i) { + stbi_uc t = p[0]; + p[0] = p[2]; + p[2] = t; + p += 4; + } + } + } +} + +#define STBI__PNG_TYPE(a,b,c,d) (((unsigned) (a) << 24) + ((unsigned) (b) << 16) + ((unsigned) (c) << 8) + (unsigned) (d)) + +static int stbi__parse_png_file(stbi__png *z, int scan, int req_comp) +{ + stbi_uc palette[1024], pal_img_n=0; + stbi_uc has_trans=0, tc[3]={0}; + stbi__uint16 tc16[3]; + stbi__uint32 ioff=0, idata_limit=0, i, pal_len=0; + int first=1,k,interlace=0, color=0, is_iphone=0; + stbi__context *s = z->s; + + z->expanded = NULL; + z->idata = NULL; + z->out = NULL; + + if (!stbi__check_png_header(s)) return 0; + + if (scan == STBI__SCAN_type) return 1; + + for (;;) { + stbi__pngchunk c = stbi__get_chunk_header(s); + switch (c.type) { + case STBI__PNG_TYPE('C','g','B','I'): + is_iphone = 1; + stbi__skip(s, c.length); + break; + case STBI__PNG_TYPE('I','H','D','R'): { + int comp,filter; + if (!first) return stbi__err("multiple IHDR","Corrupt PNG"); + first = 0; + if (c.length != 13) return stbi__err("bad IHDR len","Corrupt PNG"); + s->img_x = stbi__get32be(s); + s->img_y = stbi__get32be(s); + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + z->depth = stbi__get8(s); if (z->depth != 1 && z->depth != 2 && z->depth != 4 && z->depth != 8 && z->depth != 16) return stbi__err("1/2/4/8/16-bit only","PNG not supported: 1/2/4/8/16-bit only"); + color = stbi__get8(s); if (color > 6) return stbi__err("bad ctype","Corrupt PNG"); + if (color == 3 && z->depth == 16) return stbi__err("bad ctype","Corrupt PNG"); + if (color == 3) pal_img_n = 3; else if (color & 1) return stbi__err("bad ctype","Corrupt PNG"); + comp = stbi__get8(s); if (comp) return stbi__err("bad comp method","Corrupt PNG"); + filter= stbi__get8(s); if (filter) return stbi__err("bad filter method","Corrupt PNG"); + interlace = stbi__get8(s); if (interlace>1) return stbi__err("bad interlace method","Corrupt PNG"); + if (!s->img_x || !s->img_y) return stbi__err("0-pixel image","Corrupt PNG"); + if (!pal_img_n) { + s->img_n = (color & 2 ? 3 : 1) + (color & 4 ? 1 : 0); + if ((1 << 30) / s->img_x / s->img_n < s->img_y) return stbi__err("too large", "Image too large to decode"); + } else { + // if paletted, then pal_n is our final components, and + // img_n is # components to decompress/filter. + s->img_n = 1; + if ((1 << 30) / s->img_x / 4 < s->img_y) return stbi__err("too large","Corrupt PNG"); + } + // even with SCAN_header, have to scan to see if we have a tRNS + break; + } + + case STBI__PNG_TYPE('P','L','T','E'): { + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (c.length > 256*3) return stbi__err("invalid PLTE","Corrupt PNG"); + pal_len = c.length / 3; + if (pal_len * 3 != c.length) return stbi__err("invalid PLTE","Corrupt PNG"); + for (i=0; i < pal_len; ++i) { + palette[i*4+0] = stbi__get8(s); + palette[i*4+1] = stbi__get8(s); + palette[i*4+2] = stbi__get8(s); + palette[i*4+3] = 255; + } + break; + } + + case STBI__PNG_TYPE('t','R','N','S'): { + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (z->idata) return stbi__err("tRNS after IDAT","Corrupt PNG"); + if (pal_img_n) { + if (scan == STBI__SCAN_header) { s->img_n = 4; return 1; } + if (pal_len == 0) return stbi__err("tRNS before PLTE","Corrupt PNG"); + if (c.length > pal_len) return stbi__err("bad tRNS len","Corrupt PNG"); + pal_img_n = 4; + for (i=0; i < c.length; ++i) + palette[i*4+3] = stbi__get8(s); + } else { + if (!(s->img_n & 1)) return stbi__err("tRNS with alpha","Corrupt PNG"); + if (c.length != (stbi__uint32) s->img_n*2) return stbi__err("bad tRNS len","Corrupt PNG"); + has_trans = 1; + // non-paletted with tRNS = constant alpha. if header-scanning, we can stop now. + if (scan == STBI__SCAN_header) { ++s->img_n; return 1; } + if (z->depth == 16) { + for (k = 0; k < s->img_n && k < 3; ++k) // extra loop test to suppress false GCC warning + tc16[k] = (stbi__uint16)stbi__get16be(s); // copy the values as-is + } else { + for (k = 0; k < s->img_n && k < 3; ++k) + tc[k] = (stbi_uc)(stbi__get16be(s) & 255) * stbi__depth_scale_table[z->depth]; // non 8-bit images will be larger + } + } + break; + } + + case STBI__PNG_TYPE('I','D','A','T'): { + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (pal_img_n && !pal_len) return stbi__err("no PLTE","Corrupt PNG"); + if (scan == STBI__SCAN_header) { + // header scan definitely stops at first IDAT + if (pal_img_n) + s->img_n = pal_img_n; + return 1; + } + if (c.length > (1u << 30)) return stbi__err("IDAT size limit", "IDAT section larger than 2^30 bytes"); + if ((int)(ioff + c.length) < (int)ioff) return 0; + if (ioff + c.length > idata_limit) { + stbi__uint32 idata_limit_old = idata_limit; + stbi_uc *p; + if (idata_limit == 0) idata_limit = c.length > 4096 ? c.length : 4096; + while (ioff + c.length > idata_limit) + idata_limit *= 2; + STBI_NOTUSED(idata_limit_old); + p = (stbi_uc *) STBI_REALLOC_SIZED(z->idata, idata_limit_old, idata_limit); if (p == NULL) return stbi__err("outofmem", "Out of memory"); + z->idata = p; + } + if (!stbi__getn(s, z->idata+ioff,c.length)) return stbi__err("outofdata","Corrupt PNG"); + ioff += c.length; + break; + } + + case STBI__PNG_TYPE('I','E','N','D'): { + stbi__uint32 raw_len, bpl; + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (scan != STBI__SCAN_load) return 1; + if (z->idata == NULL) return stbi__err("no IDAT","Corrupt PNG"); + // initial guess for decoded data size to avoid unnecessary reallocs + bpl = (s->img_x * z->depth + 7) / 8; // bytes per line, per component + raw_len = bpl * s->img_y * s->img_n /* pixels */ + s->img_y /* filter mode per row */; + z->expanded = (stbi_uc *) stbi_zlib_decode_malloc_guesssize_headerflag((char *) z->idata, ioff, raw_len, (int *) &raw_len, !is_iphone); + if (z->expanded == NULL) return 0; // zlib should set error + STBI_FREE(z->idata); z->idata = NULL; + if ((req_comp == s->img_n+1 && req_comp != 3 && !pal_img_n) || has_trans) + s->img_out_n = s->img_n+1; + else + s->img_out_n = s->img_n; + if (!stbi__create_png_image(z, z->expanded, raw_len, s->img_out_n, z->depth, color, interlace)) return 0; + if (has_trans) { + if (z->depth == 16) { + if (!stbi__compute_transparency16(z, tc16, s->img_out_n)) return 0; + } else { + if (!stbi__compute_transparency(z, tc, s->img_out_n)) return 0; + } + } + if (is_iphone && stbi__de_iphone_flag && s->img_out_n > 2) + stbi__de_iphone(z); + if (pal_img_n) { + // pal_img_n == 3 or 4 + s->img_n = pal_img_n; // record the actual colors we had + s->img_out_n = pal_img_n; + if (req_comp >= 3) s->img_out_n = req_comp; + if (!stbi__expand_png_palette(z, palette, pal_len, s->img_out_n)) + return 0; + } else if (has_trans) { + // non-paletted image with tRNS -> source image has (constant) alpha + ++s->img_n; + } + STBI_FREE(z->expanded); z->expanded = NULL; + // end of PNG chunk, read and skip CRC + stbi__get32be(s); + return 1; + } + + default: + // if critical, fail + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if ((c.type & (1 << 29)) == 0) { + #ifndef STBI_NO_FAILURE_STRINGS + // not threadsafe + static char invalid_chunk[] = "XXXX PNG chunk not known"; + invalid_chunk[0] = STBI__BYTECAST(c.type >> 24); + invalid_chunk[1] = STBI__BYTECAST(c.type >> 16); + invalid_chunk[2] = STBI__BYTECAST(c.type >> 8); + invalid_chunk[3] = STBI__BYTECAST(c.type >> 0); + #endif + return stbi__err(invalid_chunk, "PNG not supported: unknown PNG chunk type"); + } + stbi__skip(s, c.length); + break; + } + // end of PNG chunk, read and skip CRC + stbi__get32be(s); + } +} + +static void *stbi__do_png(stbi__png *p, int *x, int *y, int *n, int req_comp, stbi__result_info *ri) +{ + void *result=NULL; + if (req_comp < 0 || req_comp > 4) return stbi__errpuc("bad req_comp", "Internal error"); + if (stbi__parse_png_file(p, STBI__SCAN_load, req_comp)) { + if (p->depth <= 8) + ri->bits_per_channel = 8; + else if (p->depth == 16) + ri->bits_per_channel = 16; + else + return stbi__errpuc("bad bits_per_channel", "PNG not supported: unsupported color depth"); + result = p->out; + p->out = NULL; + if (req_comp && req_comp != p->s->img_out_n) { + if (ri->bits_per_channel == 8) + result = stbi__convert_format((unsigned char *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y); + else + result = stbi__convert_format16((stbi__uint16 *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y); + p->s->img_out_n = req_comp; + if (result == NULL) return result; + } + *x = p->s->img_x; + *y = p->s->img_y; + if (n) *n = p->s->img_n; + } + STBI_FREE(p->out); p->out = NULL; + STBI_FREE(p->expanded); p->expanded = NULL; + STBI_FREE(p->idata); p->idata = NULL; + + return result; +} + +static void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi__png p; + p.s = s; + return stbi__do_png(&p, x,y,comp,req_comp, ri); +} + +static int stbi__png_test(stbi__context *s) +{ + int r; + r = stbi__check_png_header(s); + stbi__rewind(s); + return r; +} + +static int stbi__png_info_raw(stbi__png *p, int *x, int *y, int *comp) +{ + if (!stbi__parse_png_file(p, STBI__SCAN_header, 0)) { + stbi__rewind( p->s ); + return 0; + } + if (x) *x = p->s->img_x; + if (y) *y = p->s->img_y; + if (comp) *comp = p->s->img_n; + return 1; +} + +static int stbi__png_info(stbi__context *s, int *x, int *y, int *comp) +{ + stbi__png p; + p.s = s; + return stbi__png_info_raw(&p, x, y, comp); +} + +static int stbi__png_is16(stbi__context *s) +{ + stbi__png p; + p.s = s; + if (!stbi__png_info_raw(&p, NULL, NULL, NULL)) + return 0; + if (p.depth != 16) { + stbi__rewind(p.s); + return 0; + } + return 1; +} +#endif + +// Microsoft/Windows BMP image + +#ifndef STBI_NO_BMP +static int stbi__bmp_test_raw(stbi__context *s) +{ + int r; + int sz; + if (stbi__get8(s) != 'B') return 0; + if (stbi__get8(s) != 'M') return 0; + stbi__get32le(s); // discard filesize + stbi__get16le(s); // discard reserved + stbi__get16le(s); // discard reserved + stbi__get32le(s); // discard data offset + sz = stbi__get32le(s); + r = (sz == 12 || sz == 40 || sz == 56 || sz == 108 || sz == 124); + return r; +} + +static int stbi__bmp_test(stbi__context *s) +{ + int r = stbi__bmp_test_raw(s); + stbi__rewind(s); + return r; +} + + +// returns 0..31 for the highest set bit +static int stbi__high_bit(unsigned int z) +{ + int n=0; + if (z == 0) return -1; + if (z >= 0x10000) { n += 16; z >>= 16; } + if (z >= 0x00100) { n += 8; z >>= 8; } + if (z >= 0x00010) { n += 4; z >>= 4; } + if (z >= 0x00004) { n += 2; z >>= 2; } + if (z >= 0x00002) { n += 1;/* >>= 1;*/ } + return n; +} + +static int stbi__bitcount(unsigned int a) +{ + a = (a & 0x55555555) + ((a >> 1) & 0x55555555); // max 2 + a = (a & 0x33333333) + ((a >> 2) & 0x33333333); // max 4 + a = (a + (a >> 4)) & 0x0f0f0f0f; // max 8 per 4, now 8 bits + a = (a + (a >> 8)); // max 16 per 8 bits + a = (a + (a >> 16)); // max 32 per 8 bits + return a & 0xff; +} + +// extract an arbitrarily-aligned N-bit value (N=bits) +// from v, and then make it 8-bits long and fractionally +// extend it to full full range. +static int stbi__shiftsigned(unsigned int v, int shift, int bits) +{ + static unsigned int mul_table[9] = { + 0, + 0xff/*0b11111111*/, 0x55/*0b01010101*/, 0x49/*0b01001001*/, 0x11/*0b00010001*/, + 0x21/*0b00100001*/, 0x41/*0b01000001*/, 0x81/*0b10000001*/, 0x01/*0b00000001*/, + }; + static unsigned int shift_table[9] = { + 0, 0,0,1,0,2,4,6,0, + }; + if (shift < 0) + v <<= -shift; + else + v >>= shift; + STBI_ASSERT(v < 256); + v >>= (8-bits); + STBI_ASSERT(bits >= 0 && bits <= 8); + return (int) ((unsigned) v * mul_table[bits]) >> shift_table[bits]; +} + +typedef struct +{ + int bpp, offset, hsz; + unsigned int mr,mg,mb,ma, all_a; + int extra_read; +} stbi__bmp_data; + +static int stbi__bmp_set_mask_defaults(stbi__bmp_data *info, int compress) +{ + // BI_BITFIELDS specifies masks explicitly, don't override + if (compress == 3) + return 1; + + if (compress == 0) { + if (info->bpp == 16) { + info->mr = 31u << 10; + info->mg = 31u << 5; + info->mb = 31u << 0; + } else if (info->bpp == 32) { + info->mr = 0xffu << 16; + info->mg = 0xffu << 8; + info->mb = 0xffu << 0; + info->ma = 0xffu << 24; + info->all_a = 0; // if all_a is 0 at end, then we loaded alpha channel but it was all 0 + } else { + // otherwise, use defaults, which is all-0 + info->mr = info->mg = info->mb = info->ma = 0; + } + return 1; + } + return 0; // error +} + +static void *stbi__bmp_parse_header(stbi__context *s, stbi__bmp_data *info) +{ + int hsz; + if (stbi__get8(s) != 'B' || stbi__get8(s) != 'M') return stbi__errpuc("not BMP", "Corrupt BMP"); + stbi__get32le(s); // discard filesize + stbi__get16le(s); // discard reserved + stbi__get16le(s); // discard reserved + info->offset = stbi__get32le(s); + info->hsz = hsz = stbi__get32le(s); + info->mr = info->mg = info->mb = info->ma = 0; + info->extra_read = 14; + + if (info->offset < 0) return stbi__errpuc("bad BMP", "bad BMP"); + + if (hsz != 12 && hsz != 40 && hsz != 56 && hsz != 108 && hsz != 124) return stbi__errpuc("unknown BMP", "BMP type not supported: unknown"); + if (hsz == 12) { + s->img_x = stbi__get16le(s); + s->img_y = stbi__get16le(s); + } else { + s->img_x = stbi__get32le(s); + s->img_y = stbi__get32le(s); + } + if (stbi__get16le(s) != 1) return stbi__errpuc("bad BMP", "bad BMP"); + info->bpp = stbi__get16le(s); + if (hsz != 12) { + int compress = stbi__get32le(s); + if (compress == 1 || compress == 2) return stbi__errpuc("BMP RLE", "BMP type not supported: RLE"); + if (compress >= 4) return stbi__errpuc("BMP JPEG/PNG", "BMP type not supported: unsupported compression"); // this includes PNG/JPEG modes + if (compress == 3 && info->bpp != 16 && info->bpp != 32) return stbi__errpuc("bad BMP", "bad BMP"); // bitfields requires 16 or 32 bits/pixel + stbi__get32le(s); // discard sizeof + stbi__get32le(s); // discard hres + stbi__get32le(s); // discard vres + stbi__get32le(s); // discard colorsused + stbi__get32le(s); // discard max important + if (hsz == 40 || hsz == 56) { + if (hsz == 56) { + stbi__get32le(s); + stbi__get32le(s); + stbi__get32le(s); + stbi__get32le(s); + } + if (info->bpp == 16 || info->bpp == 32) { + if (compress == 0) { + stbi__bmp_set_mask_defaults(info, compress); + } else if (compress == 3) { + info->mr = stbi__get32le(s); + info->mg = stbi__get32le(s); + info->mb = stbi__get32le(s); + info->extra_read += 12; + // not documented, but generated by photoshop and handled by mspaint + if (info->mr == info->mg && info->mg == info->mb) { + // ?!?!? + return stbi__errpuc("bad BMP", "bad BMP"); + } + } else + return stbi__errpuc("bad BMP", "bad BMP"); + } + } else { + // V4/V5 header + int i; + if (hsz != 108 && hsz != 124) + return stbi__errpuc("bad BMP", "bad BMP"); + info->mr = stbi__get32le(s); + info->mg = stbi__get32le(s); + info->mb = stbi__get32le(s); + info->ma = stbi__get32le(s); + if (compress != 3) // override mr/mg/mb unless in BI_BITFIELDS mode, as per docs + stbi__bmp_set_mask_defaults(info, compress); + stbi__get32le(s); // discard color space + for (i=0; i < 12; ++i) + stbi__get32le(s); // discard color space parameters + if (hsz == 124) { + stbi__get32le(s); // discard rendering intent + stbi__get32le(s); // discard offset of profile data + stbi__get32le(s); // discard size of profile data + stbi__get32le(s); // discard reserved + } + } + } + return (void *) 1; +} + + +static void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi_uc *out; + unsigned int mr=0,mg=0,mb=0,ma=0, all_a; + stbi_uc pal[256][4]; + int psize=0,i,j,width; + int flip_vertically, pad, target; + stbi__bmp_data info; + STBI_NOTUSED(ri); + + info.all_a = 255; + if (stbi__bmp_parse_header(s, &info) == NULL) + return NULL; // error code already set + + flip_vertically = ((int) s->img_y) > 0; + s->img_y = abs((int) s->img_y); + + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + mr = info.mr; + mg = info.mg; + mb = info.mb; + ma = info.ma; + all_a = info.all_a; + + if (info.hsz == 12) { + if (info.bpp < 24) + psize = (info.offset - info.extra_read - 24) / 3; + } else { + if (info.bpp < 16) + psize = (info.offset - info.extra_read - info.hsz) >> 2; + } + if (psize == 0) { + // accept some number of extra bytes after the header, but if the offset points either to before + // the header ends or implies a large amount of extra data, reject the file as malformed + int bytes_read_so_far = s->callback_already_read + (int)(s->img_buffer - s->img_buffer_original); + int header_limit = 1024; // max we actually read is below 256 bytes currently. + int extra_data_limit = 256*4; // what ordinarily goes here is a palette; 256 entries*4 bytes is its max size. + if (bytes_read_so_far <= 0 || bytes_read_so_far > header_limit) { + return stbi__errpuc("bad header", "Corrupt BMP"); + } + // we established that bytes_read_so_far is positive and sensible. + // the first half of this test rejects offsets that are either too small positives, or + // negative, and guarantees that info.offset >= bytes_read_so_far > 0. this in turn + // ensures the number computed in the second half of the test can't overflow. + if (info.offset < bytes_read_so_far || info.offset - bytes_read_so_far > extra_data_limit) { + return stbi__errpuc("bad offset", "Corrupt BMP"); + } else { + stbi__skip(s, info.offset - bytes_read_so_far); + } + } + + if (info.bpp == 24 && ma == 0xff000000) + s->img_n = 3; + else + s->img_n = ma ? 4 : 3; + if (req_comp && req_comp >= 3) // we can directly decode 3 or 4 + target = req_comp; + else + target = s->img_n; // if they want monochrome, we'll post-convert + + // sanity-check size + if (!stbi__mad3sizes_valid(target, s->img_x, s->img_y, 0)) + return stbi__errpuc("too large", "Corrupt BMP"); + + out = (stbi_uc *) stbi__malloc_mad3(target, s->img_x, s->img_y, 0); + if (!out) return stbi__errpuc("outofmem", "Out of memory"); + if (info.bpp < 16) { + int z=0; + if (psize == 0 || psize > 256) { STBI_FREE(out); return stbi__errpuc("invalid", "Corrupt BMP"); } + for (i=0; i < psize; ++i) { + pal[i][2] = stbi__get8(s); + pal[i][1] = stbi__get8(s); + pal[i][0] = stbi__get8(s); + if (info.hsz != 12) stbi__get8(s); + pal[i][3] = 255; + } + stbi__skip(s, info.offset - info.extra_read - info.hsz - psize * (info.hsz == 12 ? 3 : 4)); + if (info.bpp == 1) width = (s->img_x + 7) >> 3; + else if (info.bpp == 4) width = (s->img_x + 1) >> 1; + else if (info.bpp == 8) width = s->img_x; + else { STBI_FREE(out); return stbi__errpuc("bad bpp", "Corrupt BMP"); } + pad = (-width)&3; + if (info.bpp == 1) { + for (j=0; j < (int) s->img_y; ++j) { + int bit_offset = 7, v = stbi__get8(s); + for (i=0; i < (int) s->img_x; ++i) { + int color = (v>>bit_offset)&0x1; + out[z++] = pal[color][0]; + out[z++] = pal[color][1]; + out[z++] = pal[color][2]; + if (target == 4) out[z++] = 255; + if (i+1 == (int) s->img_x) break; + if((--bit_offset) < 0) { + bit_offset = 7; + v = stbi__get8(s); + } + } + stbi__skip(s, pad); + } + } else { + for (j=0; j < (int) s->img_y; ++j) { + for (i=0; i < (int) s->img_x; i += 2) { + int v=stbi__get8(s),v2=0; + if (info.bpp == 4) { + v2 = v & 15; + v >>= 4; + } + out[z++] = pal[v][0]; + out[z++] = pal[v][1]; + out[z++] = pal[v][2]; + if (target == 4) out[z++] = 255; + if (i+1 == (int) s->img_x) break; + v = (info.bpp == 8) ? stbi__get8(s) : v2; + out[z++] = pal[v][0]; + out[z++] = pal[v][1]; + out[z++] = pal[v][2]; + if (target == 4) out[z++] = 255; + } + stbi__skip(s, pad); + } + } + } else { + int rshift=0,gshift=0,bshift=0,ashift=0,rcount=0,gcount=0,bcount=0,acount=0; + int z = 0; + int easy=0; + stbi__skip(s, info.offset - info.extra_read - info.hsz); + if (info.bpp == 24) width = 3 * s->img_x; + else if (info.bpp == 16) width = 2*s->img_x; + else /* bpp = 32 and pad = 0 */ width=0; + pad = (-width) & 3; + if (info.bpp == 24) { + easy = 1; + } else if (info.bpp == 32) { + if (mb == 0xff && mg == 0xff00 && mr == 0x00ff0000 && ma == 0xff000000) + easy = 2; + } + if (!easy) { + if (!mr || !mg || !mb) { STBI_FREE(out); return stbi__errpuc("bad masks", "Corrupt BMP"); } + // right shift amt to put high bit in position #7 + rshift = stbi__high_bit(mr)-7; rcount = stbi__bitcount(mr); + gshift = stbi__high_bit(mg)-7; gcount = stbi__bitcount(mg); + bshift = stbi__high_bit(mb)-7; bcount = stbi__bitcount(mb); + ashift = stbi__high_bit(ma)-7; acount = stbi__bitcount(ma); + if (rcount > 8 || gcount > 8 || bcount > 8 || acount > 8) { STBI_FREE(out); return stbi__errpuc("bad masks", "Corrupt BMP"); } + } + for (j=0; j < (int) s->img_y; ++j) { + if (easy) { + for (i=0; i < (int) s->img_x; ++i) { + unsigned char a; + out[z+2] = stbi__get8(s); + out[z+1] = stbi__get8(s); + out[z+0] = stbi__get8(s); + z += 3; + a = (easy == 2 ? stbi__get8(s) : 255); + all_a |= a; + if (target == 4) out[z++] = a; + } + } else { + int bpp = info.bpp; + for (i=0; i < (int) s->img_x; ++i) { + stbi__uint32 v = (bpp == 16 ? (stbi__uint32) stbi__get16le(s) : stbi__get32le(s)); + unsigned int a; + out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mr, rshift, rcount)); + out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mg, gshift, gcount)); + out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mb, bshift, bcount)); + a = (ma ? stbi__shiftsigned(v & ma, ashift, acount) : 255); + all_a |= a; + if (target == 4) out[z++] = STBI__BYTECAST(a); + } + } + stbi__skip(s, pad); + } + } + + // if alpha channel is all 0s, replace with all 255s + if (target == 4 && all_a == 0) + for (i=4*s->img_x*s->img_y-1; i >= 0; i -= 4) + out[i] = 255; + + if (flip_vertically) { + stbi_uc t; + for (j=0; j < (int) s->img_y>>1; ++j) { + stbi_uc *p1 = out + j *s->img_x*target; + stbi_uc *p2 = out + (s->img_y-1-j)*s->img_x*target; + for (i=0; i < (int) s->img_x*target; ++i) { + t = p1[i]; p1[i] = p2[i]; p2[i] = t; + } + } + } + + if (req_comp && req_comp != target) { + out = stbi__convert_format(out, target, req_comp, s->img_x, s->img_y); + if (out == NULL) return out; // stbi__convert_format frees input on failure + } + + *x = s->img_x; + *y = s->img_y; + if (comp) *comp = s->img_n; + return out; +} +#endif + +// Targa Truevision - TGA +// by Jonathan Dummer +#ifndef STBI_NO_TGA +// returns STBI_rgb or whatever, 0 on error +static int stbi__tga_get_comp(int bits_per_pixel, int is_grey, int* is_rgb16) +{ + // only RGB or RGBA (incl. 16bit) or grey allowed + if (is_rgb16) *is_rgb16 = 0; + switch(bits_per_pixel) { + case 8: return STBI_grey; + case 16: if(is_grey) return STBI_grey_alpha; + // fallthrough + case 15: if(is_rgb16) *is_rgb16 = 1; + return STBI_rgb; + case 24: // fallthrough + case 32: return bits_per_pixel/8; + default: return 0; + } +} + +static int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp) +{ + int tga_w, tga_h, tga_comp, tga_image_type, tga_bits_per_pixel, tga_colormap_bpp; + int sz, tga_colormap_type; + stbi__get8(s); // discard Offset + tga_colormap_type = stbi__get8(s); // colormap type + if( tga_colormap_type > 1 ) { + stbi__rewind(s); + return 0; // only RGB or indexed allowed + } + tga_image_type = stbi__get8(s); // image type + if ( tga_colormap_type == 1 ) { // colormapped (paletted) image + if (tga_image_type != 1 && tga_image_type != 9) { + stbi__rewind(s); + return 0; + } + stbi__skip(s,4); // skip index of first colormap entry and number of entries + sz = stbi__get8(s); // check bits per palette color entry + if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) { + stbi__rewind(s); + return 0; + } + stbi__skip(s,4); // skip image x and y origin + tga_colormap_bpp = sz; + } else { // "normal" image w/o colormap - only RGB or grey allowed, +/- RLE + if ( (tga_image_type != 2) && (tga_image_type != 3) && (tga_image_type != 10) && (tga_image_type != 11) ) { + stbi__rewind(s); + return 0; // only RGB or grey allowed, +/- RLE + } + stbi__skip(s,9); // skip colormap specification and image x/y origin + tga_colormap_bpp = 0; + } + tga_w = stbi__get16le(s); + if( tga_w < 1 ) { + stbi__rewind(s); + return 0; // test width + } + tga_h = stbi__get16le(s); + if( tga_h < 1 ) { + stbi__rewind(s); + return 0; // test height + } + tga_bits_per_pixel = stbi__get8(s); // bits per pixel + stbi__get8(s); // ignore alpha bits + if (tga_colormap_bpp != 0) { + if((tga_bits_per_pixel != 8) && (tga_bits_per_pixel != 16)) { + // when using a colormap, tga_bits_per_pixel is the size of the indexes + // I don't think anything but 8 or 16bit indexes makes sense + stbi__rewind(s); + return 0; + } + tga_comp = stbi__tga_get_comp(tga_colormap_bpp, 0, NULL); + } else { + tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3) || (tga_image_type == 11), NULL); + } + if(!tga_comp) { + stbi__rewind(s); + return 0; + } + if (x) *x = tga_w; + if (y) *y = tga_h; + if (comp) *comp = tga_comp; + return 1; // seems to have passed everything +} + +static int stbi__tga_test(stbi__context *s) +{ + int res = 0; + int sz, tga_color_type; + stbi__get8(s); // discard Offset + tga_color_type = stbi__get8(s); // color type + if ( tga_color_type > 1 ) goto errorEnd; // only RGB or indexed allowed + sz = stbi__get8(s); // image type + if ( tga_color_type == 1 ) { // colormapped (paletted) image + if (sz != 1 && sz != 9) goto errorEnd; // colortype 1 demands image type 1 or 9 + stbi__skip(s,4); // skip index of first colormap entry and number of entries + sz = stbi__get8(s); // check bits per palette color entry + if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd; + stbi__skip(s,4); // skip image x and y origin + } else { // "normal" image w/o colormap + if ( (sz != 2) && (sz != 3) && (sz != 10) && (sz != 11) ) goto errorEnd; // only RGB or grey allowed, +/- RLE + stbi__skip(s,9); // skip colormap specification and image x/y origin + } + if ( stbi__get16le(s) < 1 ) goto errorEnd; // test width + if ( stbi__get16le(s) < 1 ) goto errorEnd; // test height + sz = stbi__get8(s); // bits per pixel + if ( (tga_color_type == 1) && (sz != 8) && (sz != 16) ) goto errorEnd; // for colormapped images, bpp is size of an index + if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd; + + res = 1; // if we got this far, everything's good and we can return 1 instead of 0 + +errorEnd: + stbi__rewind(s); + return res; +} + +// read 16bit value and convert to 24bit RGB +static void stbi__tga_read_rgb16(stbi__context *s, stbi_uc* out) +{ + stbi__uint16 px = (stbi__uint16)stbi__get16le(s); + stbi__uint16 fiveBitMask = 31; + // we have 3 channels with 5bits each + int r = (px >> 10) & fiveBitMask; + int g = (px >> 5) & fiveBitMask; + int b = px & fiveBitMask; + // Note that this saves the data in RGB(A) order, so it doesn't need to be swapped later + out[0] = (stbi_uc)((r * 255)/31); + out[1] = (stbi_uc)((g * 255)/31); + out[2] = (stbi_uc)((b * 255)/31); + + // some people claim that the most significant bit might be used for alpha + // (possibly if an alpha-bit is set in the "image descriptor byte") + // but that only made 16bit test images completely translucent.. + // so let's treat all 15 and 16bit TGAs as RGB with no alpha. +} + +static void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + // read in the TGA header stuff + int tga_offset = stbi__get8(s); + int tga_indexed = stbi__get8(s); + int tga_image_type = stbi__get8(s); + int tga_is_RLE = 0; + int tga_palette_start = stbi__get16le(s); + int tga_palette_len = stbi__get16le(s); + int tga_palette_bits = stbi__get8(s); + int tga_x_origin = stbi__get16le(s); + int tga_y_origin = stbi__get16le(s); + int tga_width = stbi__get16le(s); + int tga_height = stbi__get16le(s); + int tga_bits_per_pixel = stbi__get8(s); + int tga_comp, tga_rgb16=0; + int tga_inverted = stbi__get8(s); + // int tga_alpha_bits = tga_inverted & 15; // the 4 lowest bits - unused (useless?) + // image data + unsigned char *tga_data; + unsigned char *tga_palette = NULL; + int i, j; + unsigned char raw_data[4] = {0}; + int RLE_count = 0; + int RLE_repeating = 0; + int read_next_pixel = 1; + STBI_NOTUSED(ri); + STBI_NOTUSED(tga_x_origin); // @TODO + STBI_NOTUSED(tga_y_origin); // @TODO + + if (tga_height > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (tga_width > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + // do a tiny bit of precessing + if ( tga_image_type >= 8 ) + { + tga_image_type -= 8; + tga_is_RLE = 1; + } + tga_inverted = 1 - ((tga_inverted >> 5) & 1); + + // If I'm paletted, then I'll use the number of bits from the palette + if ( tga_indexed ) tga_comp = stbi__tga_get_comp(tga_palette_bits, 0, &tga_rgb16); + else tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3), &tga_rgb16); + + if(!tga_comp) // shouldn't really happen, stbi__tga_test() should have ensured basic consistency + return stbi__errpuc("bad format", "Can't find out TGA pixelformat"); + + // tga info + *x = tga_width; + *y = tga_height; + if (comp) *comp = tga_comp; + + if (!stbi__mad3sizes_valid(tga_width, tga_height, tga_comp, 0)) + return stbi__errpuc("too large", "Corrupt TGA"); + + tga_data = (unsigned char*)stbi__malloc_mad3(tga_width, tga_height, tga_comp, 0); + if (!tga_data) return stbi__errpuc("outofmem", "Out of memory"); + + // skip to the data's starting position (offset usually = 0) + stbi__skip(s, tga_offset ); + + if ( !tga_indexed && !tga_is_RLE && !tga_rgb16 ) { + for (i=0; i < tga_height; ++i) { + int row = tga_inverted ? tga_height -i - 1 : i; + stbi_uc *tga_row = tga_data + row*tga_width*tga_comp; + stbi__getn(s, tga_row, tga_width * tga_comp); + } + } else { + // do I need to load a palette? + if ( tga_indexed) + { + if (tga_palette_len == 0) { /* you have to have at least one entry! */ + STBI_FREE(tga_data); + return stbi__errpuc("bad palette", "Corrupt TGA"); + } + + // any data to skip? (offset usually = 0) + stbi__skip(s, tga_palette_start ); + // load the palette + tga_palette = (unsigned char*)stbi__malloc_mad2(tga_palette_len, tga_comp, 0); + if (!tga_palette) { + STBI_FREE(tga_data); + return stbi__errpuc("outofmem", "Out of memory"); + } + if (tga_rgb16) { + stbi_uc *pal_entry = tga_palette; + STBI_ASSERT(tga_comp == STBI_rgb); + for (i=0; i < tga_palette_len; ++i) { + stbi__tga_read_rgb16(s, pal_entry); + pal_entry += tga_comp; + } + } else if (!stbi__getn(s, tga_palette, tga_palette_len * tga_comp)) { + STBI_FREE(tga_data); + STBI_FREE(tga_palette); + return stbi__errpuc("bad palette", "Corrupt TGA"); + } + } + // load the data + for (i=0; i < tga_width * tga_height; ++i) + { + // if I'm in RLE mode, do I need to get a RLE stbi__pngchunk? + if ( tga_is_RLE ) + { + if ( RLE_count == 0 ) + { + // yep, get the next byte as a RLE command + int RLE_cmd = stbi__get8(s); + RLE_count = 1 + (RLE_cmd & 127); + RLE_repeating = RLE_cmd >> 7; + read_next_pixel = 1; + } else if ( !RLE_repeating ) + { + read_next_pixel = 1; + } + } else + { + read_next_pixel = 1; + } + // OK, if I need to read a pixel, do it now + if ( read_next_pixel ) + { + // load however much data we did have + if ( tga_indexed ) + { + // read in index, then perform the lookup + int pal_idx = (tga_bits_per_pixel == 8) ? stbi__get8(s) : stbi__get16le(s); + if ( pal_idx >= tga_palette_len ) { + // invalid index + pal_idx = 0; + } + pal_idx *= tga_comp; + for (j = 0; j < tga_comp; ++j) { + raw_data[j] = tga_palette[pal_idx+j]; + } + } else if(tga_rgb16) { + STBI_ASSERT(tga_comp == STBI_rgb); + stbi__tga_read_rgb16(s, raw_data); + } else { + // read in the data raw + for (j = 0; j < tga_comp; ++j) { + raw_data[j] = stbi__get8(s); + } + } + // clear the reading flag for the next pixel + read_next_pixel = 0; + } // end of reading a pixel + + // copy data + for (j = 0; j < tga_comp; ++j) + tga_data[i*tga_comp+j] = raw_data[j]; + + // in case we're in RLE mode, keep counting down + --RLE_count; + } + // do I need to invert the image? + if ( tga_inverted ) + { + for (j = 0; j*2 < tga_height; ++j) + { + int index1 = j * tga_width * tga_comp; + int index2 = (tga_height - 1 - j) * tga_width * tga_comp; + for (i = tga_width * tga_comp; i > 0; --i) + { + unsigned char temp = tga_data[index1]; + tga_data[index1] = tga_data[index2]; + tga_data[index2] = temp; + ++index1; + ++index2; + } + } + } + // clear my palette, if I had one + if ( tga_palette != NULL ) + { + STBI_FREE( tga_palette ); + } + } + + // swap RGB - if the source data was RGB16, it already is in the right order + if (tga_comp >= 3 && !tga_rgb16) + { + unsigned char* tga_pixel = tga_data; + for (i=0; i < tga_width * tga_height; ++i) + { + unsigned char temp = tga_pixel[0]; + tga_pixel[0] = tga_pixel[2]; + tga_pixel[2] = temp; + tga_pixel += tga_comp; + } + } + + // convert to target component count + if (req_comp && req_comp != tga_comp) + tga_data = stbi__convert_format(tga_data, tga_comp, req_comp, tga_width, tga_height); + + // the things I do to get rid of an error message, and yet keep + // Microsoft's C compilers happy... [8^( + tga_palette_start = tga_palette_len = tga_palette_bits = + tga_x_origin = tga_y_origin = 0; + STBI_NOTUSED(tga_palette_start); + // OK, done + return tga_data; +} +#endif + +// ************************************************************************************************* +// Photoshop PSD loader -- PD by Thatcher Ulrich, integration by Nicolas Schulz, tweaked by STB + +#ifndef STBI_NO_PSD +static int stbi__psd_test(stbi__context *s) +{ + int r = (stbi__get32be(s) == 0x38425053); + stbi__rewind(s); + return r; +} + +static int stbi__psd_decode_rle(stbi__context *s, stbi_uc *p, int pixelCount) +{ + int count, nleft, len; + + count = 0; + while ((nleft = pixelCount - count) > 0) { + len = stbi__get8(s); + if (len == 128) { + // No-op. + } else if (len < 128) { + // Copy next len+1 bytes literally. + len++; + if (len > nleft) return 0; // corrupt data + count += len; + while (len) { + *p = stbi__get8(s); + p += 4; + len--; + } + } else if (len > 128) { + stbi_uc val; + // Next -len+1 bytes in the dest are replicated from next source byte. + // (Interpret len as a negative 8-bit int.) + len = 257 - len; + if (len > nleft) return 0; // corrupt data + val = stbi__get8(s); + count += len; + while (len) { + *p = val; + p += 4; + len--; + } + } + } + + return 1; +} + +static void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc) +{ + int pixelCount; + int channelCount, compression; + int channel, i; + int bitdepth; + int w,h; + stbi_uc *out; + STBI_NOTUSED(ri); + + // Check identifier + if (stbi__get32be(s) != 0x38425053) // "8BPS" + return stbi__errpuc("not PSD", "Corrupt PSD image"); + + // Check file type version. + if (stbi__get16be(s) != 1) + return stbi__errpuc("wrong version", "Unsupported version of PSD image"); + + // Skip 6 reserved bytes. + stbi__skip(s, 6 ); + + // Read the number of channels (R, G, B, A, etc). + channelCount = stbi__get16be(s); + if (channelCount < 0 || channelCount > 16) + return stbi__errpuc("wrong channel count", "Unsupported number of channels in PSD image"); + + // Read the rows and columns of the image. + h = stbi__get32be(s); + w = stbi__get32be(s); + + if (h > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (w > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + // Make sure the depth is 8 bits. + bitdepth = stbi__get16be(s); + if (bitdepth != 8 && bitdepth != 16) + return stbi__errpuc("unsupported bit depth", "PSD bit depth is not 8 or 16 bit"); + + // Make sure the color mode is RGB. + // Valid options are: + // 0: Bitmap + // 1: Grayscale + // 2: Indexed color + // 3: RGB color + // 4: CMYK color + // 7: Multichannel + // 8: Duotone + // 9: Lab color + if (stbi__get16be(s) != 3) + return stbi__errpuc("wrong color format", "PSD is not in RGB color format"); + + // Skip the Mode Data. (It's the palette for indexed color; other info for other modes.) + stbi__skip(s,stbi__get32be(s) ); + + // Skip the image resources. (resolution, pen tool paths, etc) + stbi__skip(s, stbi__get32be(s) ); + + // Skip the reserved data. + stbi__skip(s, stbi__get32be(s) ); + + // Find out if the data is compressed. + // Known values: + // 0: no compression + // 1: RLE compressed + compression = stbi__get16be(s); + if (compression > 1) + return stbi__errpuc("bad compression", "PSD has an unknown compression format"); + + // Check size + if (!stbi__mad3sizes_valid(4, w, h, 0)) + return stbi__errpuc("too large", "Corrupt PSD"); + + // Create the destination image. + + if (!compression && bitdepth == 16 && bpc == 16) { + out = (stbi_uc *) stbi__malloc_mad3(8, w, h, 0); + ri->bits_per_channel = 16; + } else + out = (stbi_uc *) stbi__malloc(4 * w*h); + + if (!out) return stbi__errpuc("outofmem", "Out of memory"); + pixelCount = w*h; + + // Initialize the data to zero. + //memset( out, 0, pixelCount * 4 ); + + // Finally, the image data. + if (compression) { + // RLE as used by .PSD and .TIFF + // Loop until you get the number of unpacked bytes you are expecting: + // Read the next source byte into n. + // If n is between 0 and 127 inclusive, copy the next n+1 bytes literally. + // Else if n is between -127 and -1 inclusive, copy the next byte -n+1 times. + // Else if n is 128, noop. + // Endloop + + // The RLE-compressed data is preceded by a 2-byte data count for each row in the data, + // which we're going to just skip. + stbi__skip(s, h * channelCount * 2 ); + + // Read the RLE data by channel. + for (channel = 0; channel < 4; channel++) { + stbi_uc *p; + + p = out+channel; + if (channel >= channelCount) { + // Fill this channel with default data. + for (i = 0; i < pixelCount; i++, p += 4) + *p = (channel == 3 ? 255 : 0); + } else { + // Read the RLE data. + if (!stbi__psd_decode_rle(s, p, pixelCount)) { + STBI_FREE(out); + return stbi__errpuc("corrupt", "bad RLE data"); + } + } + } + + } else { + // We're at the raw image data. It's each channel in order (Red, Green, Blue, Alpha, ...) + // where each channel consists of an 8-bit (or 16-bit) value for each pixel in the image. + + // Read the data by channel. + for (channel = 0; channel < 4; channel++) { + if (channel >= channelCount) { + // Fill this channel with default data. + if (bitdepth == 16 && bpc == 16) { + stbi__uint16 *q = ((stbi__uint16 *) out) + channel; + stbi__uint16 val = channel == 3 ? 65535 : 0; + for (i = 0; i < pixelCount; i++, q += 4) + *q = val; + } else { + stbi_uc *p = out+channel; + stbi_uc val = channel == 3 ? 255 : 0; + for (i = 0; i < pixelCount; i++, p += 4) + *p = val; + } + } else { + if (ri->bits_per_channel == 16) { // output bpc + stbi__uint16 *q = ((stbi__uint16 *) out) + channel; + for (i = 0; i < pixelCount; i++, q += 4) + *q = (stbi__uint16) stbi__get16be(s); + } else { + stbi_uc *p = out+channel; + if (bitdepth == 16) { // input bpc + for (i = 0; i < pixelCount; i++, p += 4) + *p = (stbi_uc) (stbi__get16be(s) >> 8); + } else { + for (i = 0; i < pixelCount; i++, p += 4) + *p = stbi__get8(s); + } + } + } + } + } + + // remove weird white matte from PSD + if (channelCount >= 4) { + if (ri->bits_per_channel == 16) { + for (i=0; i < w*h; ++i) { + stbi__uint16 *pixel = (stbi__uint16 *) out + 4*i; + if (pixel[3] != 0 && pixel[3] != 65535) { + float a = pixel[3] / 65535.0f; + float ra = 1.0f / a; + float inv_a = 65535.0f * (1 - ra); + pixel[0] = (stbi__uint16) (pixel[0]*ra + inv_a); + pixel[1] = (stbi__uint16) (pixel[1]*ra + inv_a); + pixel[2] = (stbi__uint16) (pixel[2]*ra + inv_a); + } + } + } else { + for (i=0; i < w*h; ++i) { + unsigned char *pixel = out + 4*i; + if (pixel[3] != 0 && pixel[3] != 255) { + float a = pixel[3] / 255.0f; + float ra = 1.0f / a; + float inv_a = 255.0f * (1 - ra); + pixel[0] = (unsigned char) (pixel[0]*ra + inv_a); + pixel[1] = (unsigned char) (pixel[1]*ra + inv_a); + pixel[2] = (unsigned char) (pixel[2]*ra + inv_a); + } + } + } + } + + // convert to desired output format + if (req_comp && req_comp != 4) { + if (ri->bits_per_channel == 16) + out = (stbi_uc *) stbi__convert_format16((stbi__uint16 *) out, 4, req_comp, w, h); + else + out = stbi__convert_format(out, 4, req_comp, w, h); + if (out == NULL) return out; // stbi__convert_format frees input on failure + } + + if (comp) *comp = 4; + *y = h; + *x = w; + + return out; +} +#endif + +// ************************************************************************************************* +// Softimage PIC loader +// by Tom Seddon +// +// See http://softimage.wiki.softimage.com/index.php/INFO:_PIC_file_format +// See http://ozviz.wasp.uwa.edu.au/~pbourke/dataformats/softimagepic/ + +#ifndef STBI_NO_PIC +static int stbi__pic_is4(stbi__context *s,const char *str) +{ + int i; + for (i=0; i<4; ++i) + if (stbi__get8(s) != (stbi_uc)str[i]) + return 0; + + return 1; +} + +static int stbi__pic_test_core(stbi__context *s) +{ + int i; + + if (!stbi__pic_is4(s,"\x53\x80\xF6\x34")) + return 0; + + for(i=0;i<84;++i) + stbi__get8(s); + + if (!stbi__pic_is4(s,"PICT")) + return 0; + + return 1; +} + +typedef struct +{ + stbi_uc size,type,channel; +} stbi__pic_packet; + +static stbi_uc *stbi__readval(stbi__context *s, int channel, stbi_uc *dest) +{ + int mask=0x80, i; + + for (i=0; i<4; ++i, mask>>=1) { + if (channel & mask) { + if (stbi__at_eof(s)) return stbi__errpuc("bad file","PIC file too short"); + dest[i]=stbi__get8(s); + } + } + + return dest; +} + +static void stbi__copyval(int channel,stbi_uc *dest,const stbi_uc *src) +{ + int mask=0x80,i; + + for (i=0;i<4; ++i, mask>>=1) + if (channel&mask) + dest[i]=src[i]; +} + +static stbi_uc *stbi__pic_load_core(stbi__context *s,int width,int height,int *comp, stbi_uc *result) +{ + int act_comp=0,num_packets=0,y,chained; + stbi__pic_packet packets[10]; + + // this will (should...) cater for even some bizarre stuff like having data + // for the same channel in multiple packets. + do { + stbi__pic_packet *packet; + + if (num_packets==sizeof(packets)/sizeof(packets[0])) + return stbi__errpuc("bad format","too many packets"); + + packet = &packets[num_packets++]; + + chained = stbi__get8(s); + packet->size = stbi__get8(s); + packet->type = stbi__get8(s); + packet->channel = stbi__get8(s); + + act_comp |= packet->channel; + + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (reading packets)"); + if (packet->size != 8) return stbi__errpuc("bad format","packet isn't 8bpp"); + } while (chained); + + *comp = (act_comp & 0x10 ? 4 : 3); // has alpha channel? + + for(y=0; ytype) { + default: + return stbi__errpuc("bad format","packet has bad compression type"); + + case 0: {//uncompressed + int x; + + for(x=0;xchannel,dest)) + return 0; + break; + } + + case 1://Pure RLE + { + int left=width, i; + + while (left>0) { + stbi_uc count,value[4]; + + count=stbi__get8(s); + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (pure read count)"); + + if (count > left) + count = (stbi_uc) left; + + if (!stbi__readval(s,packet->channel,value)) return 0; + + for(i=0; ichannel,dest,value); + left -= count; + } + } + break; + + case 2: {//Mixed RLE + int left=width; + while (left>0) { + int count = stbi__get8(s), i; + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (mixed read count)"); + + if (count >= 128) { // Repeated + stbi_uc value[4]; + + if (count==128) + count = stbi__get16be(s); + else + count -= 127; + if (count > left) + return stbi__errpuc("bad file","scanline overrun"); + + if (!stbi__readval(s,packet->channel,value)) + return 0; + + for(i=0;ichannel,dest,value); + } else { // Raw + ++count; + if (count>left) return stbi__errpuc("bad file","scanline overrun"); + + for(i=0;ichannel,dest)) + return 0; + } + left-=count; + } + break; + } + } + } + } + + return result; +} + +static void *stbi__pic_load(stbi__context *s,int *px,int *py,int *comp,int req_comp, stbi__result_info *ri) +{ + stbi_uc *result; + int i, x,y, internal_comp; + STBI_NOTUSED(ri); + + if (!comp) comp = &internal_comp; + + for (i=0; i<92; ++i) + stbi__get8(s); + + x = stbi__get16be(s); + y = stbi__get16be(s); + + if (y > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (x > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (pic header)"); + if (!stbi__mad3sizes_valid(x, y, 4, 0)) return stbi__errpuc("too large", "PIC image too large to decode"); + + stbi__get32be(s); //skip `ratio' + stbi__get16be(s); //skip `fields' + stbi__get16be(s); //skip `pad' + + // intermediate buffer is RGBA + result = (stbi_uc *) stbi__malloc_mad3(x, y, 4, 0); + if (!result) return stbi__errpuc("outofmem", "Out of memory"); + memset(result, 0xff, x*y*4); + + if (!stbi__pic_load_core(s,x,y,comp, result)) { + STBI_FREE(result); + result=0; + } + *px = x; + *py = y; + if (req_comp == 0) req_comp = *comp; + result=stbi__convert_format(result,4,req_comp,x,y); + + return result; +} + +static int stbi__pic_test(stbi__context *s) +{ + int r = stbi__pic_test_core(s); + stbi__rewind(s); + return r; +} +#endif + +// ************************************************************************************************* +// GIF loader -- public domain by Jean-Marc Lienher -- simplified/shrunk by stb + +#ifndef STBI_NO_GIF +typedef struct +{ + stbi__int16 prefix; + stbi_uc first; + stbi_uc suffix; +} stbi__gif_lzw; + +typedef struct +{ + int w,h; + stbi_uc *out; // output buffer (always 4 components) + stbi_uc *background; // The current "background" as far as a gif is concerned + stbi_uc *history; + int flags, bgindex, ratio, transparent, eflags; + stbi_uc pal[256][4]; + stbi_uc lpal[256][4]; + stbi__gif_lzw codes[8192]; + stbi_uc *color_table; + int parse, step; + int lflags; + int start_x, start_y; + int max_x, max_y; + int cur_x, cur_y; + int line_size; + int delay; +} stbi__gif; + +static int stbi__gif_test_raw(stbi__context *s) +{ + int sz; + if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8') return 0; + sz = stbi__get8(s); + if (sz != '9' && sz != '7') return 0; + if (stbi__get8(s) != 'a') return 0; + return 1; +} + +static int stbi__gif_test(stbi__context *s) +{ + int r = stbi__gif_test_raw(s); + stbi__rewind(s); + return r; +} + +static void stbi__gif_parse_colortable(stbi__context *s, stbi_uc pal[256][4], int num_entries, int transp) +{ + int i; + for (i=0; i < num_entries; ++i) { + pal[i][2] = stbi__get8(s); + pal[i][1] = stbi__get8(s); + pal[i][0] = stbi__get8(s); + pal[i][3] = transp == i ? 0 : 255; + } +} + +static int stbi__gif_header(stbi__context *s, stbi__gif *g, int *comp, int is_info) +{ + stbi_uc version; + if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8') + return stbi__err("not GIF", "Corrupt GIF"); + + version = stbi__get8(s); + if (version != '7' && version != '9') return stbi__err("not GIF", "Corrupt GIF"); + if (stbi__get8(s) != 'a') return stbi__err("not GIF", "Corrupt GIF"); + + stbi__g_failure_reason = ""; + g->w = stbi__get16le(s); + g->h = stbi__get16le(s); + g->flags = stbi__get8(s); + g->bgindex = stbi__get8(s); + g->ratio = stbi__get8(s); + g->transparent = -1; + + if (g->w > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + if (g->h > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + + if (comp != 0) *comp = 4; // can't actually tell whether it's 3 or 4 until we parse the comments + + if (is_info) return 1; + + if (g->flags & 0x80) + stbi__gif_parse_colortable(s,g->pal, 2 << (g->flags & 7), -1); + + return 1; +} + +static int stbi__gif_info_raw(stbi__context *s, int *x, int *y, int *comp) +{ + stbi__gif* g = (stbi__gif*) stbi__malloc(sizeof(stbi__gif)); + if (!g) return stbi__err("outofmem", "Out of memory"); + if (!stbi__gif_header(s, g, comp, 1)) { + STBI_FREE(g); + stbi__rewind( s ); + return 0; + } + if (x) *x = g->w; + if (y) *y = g->h; + STBI_FREE(g); + return 1; +} + +static void stbi__out_gif_code(stbi__gif *g, stbi__uint16 code) +{ + stbi_uc *p, *c; + int idx; + + // recurse to decode the prefixes, since the linked-list is backwards, + // and working backwards through an interleaved image would be nasty + if (g->codes[code].prefix >= 0) + stbi__out_gif_code(g, g->codes[code].prefix); + + if (g->cur_y >= g->max_y) return; + + idx = g->cur_x + g->cur_y; + p = &g->out[idx]; + g->history[idx / 4] = 1; + + c = &g->color_table[g->codes[code].suffix * 4]; + if (c[3] > 128) { // don't render transparent pixels; + p[0] = c[2]; + p[1] = c[1]; + p[2] = c[0]; + p[3] = c[3]; + } + g->cur_x += 4; + + if (g->cur_x >= g->max_x) { + g->cur_x = g->start_x; + g->cur_y += g->step; + + while (g->cur_y >= g->max_y && g->parse > 0) { + g->step = (1 << g->parse) * g->line_size; + g->cur_y = g->start_y + (g->step >> 1); + --g->parse; + } + } +} + +static stbi_uc *stbi__process_gif_raster(stbi__context *s, stbi__gif *g) +{ + stbi_uc lzw_cs; + stbi__int32 len, init_code; + stbi__uint32 first; + stbi__int32 codesize, codemask, avail, oldcode, bits, valid_bits, clear; + stbi__gif_lzw *p; + + lzw_cs = stbi__get8(s); + if (lzw_cs > 12) return NULL; + clear = 1 << lzw_cs; + first = 1; + codesize = lzw_cs + 1; + codemask = (1 << codesize) - 1; + bits = 0; + valid_bits = 0; + for (init_code = 0; init_code < clear; init_code++) { + g->codes[init_code].prefix = -1; + g->codes[init_code].first = (stbi_uc) init_code; + g->codes[init_code].suffix = (stbi_uc) init_code; + } + + // support no starting clear code + avail = clear+2; + oldcode = -1; + + len = 0; + for(;;) { + if (valid_bits < codesize) { + if (len == 0) { + len = stbi__get8(s); // start new block + if (len == 0) + return g->out; + } + --len; + bits |= (stbi__int32) stbi__get8(s) << valid_bits; + valid_bits += 8; + } else { + stbi__int32 code = bits & codemask; + bits >>= codesize; + valid_bits -= codesize; + // @OPTIMIZE: is there some way we can accelerate the non-clear path? + if (code == clear) { // clear code + codesize = lzw_cs + 1; + codemask = (1 << codesize) - 1; + avail = clear + 2; + oldcode = -1; + first = 0; + } else if (code == clear + 1) { // end of stream code + stbi__skip(s, len); + while ((len = stbi__get8(s)) > 0) + stbi__skip(s,len); + return g->out; + } else if (code <= avail) { + if (first) { + return stbi__errpuc("no clear code", "Corrupt GIF"); + } + + if (oldcode >= 0) { + p = &g->codes[avail++]; + if (avail > 8192) { + return stbi__errpuc("too many codes", "Corrupt GIF"); + } + + p->prefix = (stbi__int16) oldcode; + p->first = g->codes[oldcode].first; + p->suffix = (code == avail) ? p->first : g->codes[code].first; + } else if (code == avail) + return stbi__errpuc("illegal code in raster", "Corrupt GIF"); + + stbi__out_gif_code(g, (stbi__uint16) code); + + if ((avail & codemask) == 0 && avail <= 0x0FFF) { + codesize++; + codemask = (1 << codesize) - 1; + } + + oldcode = code; + } else { + return stbi__errpuc("illegal code in raster", "Corrupt GIF"); + } + } + } +} + +// this function is designed to support animated gifs, although stb_image doesn't support it +// two back is the image from two frames ago, used for a very specific disposal format +static stbi_uc *stbi__gif_load_next(stbi__context *s, stbi__gif *g, int *comp, int req_comp, stbi_uc *two_back) +{ + int dispose; + int first_frame; + int pi; + int pcount; + STBI_NOTUSED(req_comp); + + // on first frame, any non-written pixels get the background colour (non-transparent) + first_frame = 0; + if (g->out == 0) { + if (!stbi__gif_header(s, g, comp,0)) return 0; // stbi__g_failure_reason set by stbi__gif_header + if (!stbi__mad3sizes_valid(4, g->w, g->h, 0)) + return stbi__errpuc("too large", "GIF image is too large"); + pcount = g->w * g->h; + g->out = (stbi_uc *) stbi__malloc(4 * pcount); + g->background = (stbi_uc *) stbi__malloc(4 * pcount); + g->history = (stbi_uc *) stbi__malloc(pcount); + if (!g->out || !g->background || !g->history) + return stbi__errpuc("outofmem", "Out of memory"); + + // image is treated as "transparent" at the start - ie, nothing overwrites the current background; + // background colour is only used for pixels that are not rendered first frame, after that "background" + // color refers to the color that was there the previous frame. + memset(g->out, 0x00, 4 * pcount); + memset(g->background, 0x00, 4 * pcount); // state of the background (starts transparent) + memset(g->history, 0x00, pcount); // pixels that were affected previous frame + first_frame = 1; + } else { + // second frame - how do we dispose of the previous one? + dispose = (g->eflags & 0x1C) >> 2; + pcount = g->w * g->h; + + if ((dispose == 3) && (two_back == 0)) { + dispose = 2; // if I don't have an image to revert back to, default to the old background + } + + if (dispose == 3) { // use previous graphic + for (pi = 0; pi < pcount; ++pi) { + if (g->history[pi]) { + memcpy( &g->out[pi * 4], &two_back[pi * 4], 4 ); + } + } + } else if (dispose == 2) { + // restore what was changed last frame to background before that frame; + for (pi = 0; pi < pcount; ++pi) { + if (g->history[pi]) { + memcpy( &g->out[pi * 4], &g->background[pi * 4], 4 ); + } + } + } else { + // This is a non-disposal case eithe way, so just + // leave the pixels as is, and they will become the new background + // 1: do not dispose + // 0: not specified. + } + + // background is what out is after the undoing of the previou frame; + memcpy( g->background, g->out, 4 * g->w * g->h ); + } + + // clear my history; + memset( g->history, 0x00, g->w * g->h ); // pixels that were affected previous frame + + for (;;) { + int tag = stbi__get8(s); + switch (tag) { + case 0x2C: /* Image Descriptor */ + { + stbi__int32 x, y, w, h; + stbi_uc *o; + + x = stbi__get16le(s); + y = stbi__get16le(s); + w = stbi__get16le(s); + h = stbi__get16le(s); + if (((x + w) > (g->w)) || ((y + h) > (g->h))) + return stbi__errpuc("bad Image Descriptor", "Corrupt GIF"); + + g->line_size = g->w * 4; + g->start_x = x * 4; + g->start_y = y * g->line_size; + g->max_x = g->start_x + w * 4; + g->max_y = g->start_y + h * g->line_size; + g->cur_x = g->start_x; + g->cur_y = g->start_y; + + // if the width of the specified rectangle is 0, that means + // we may not see *any* pixels or the image is malformed; + // to make sure this is caught, move the current y down to + // max_y (which is what out_gif_code checks). + if (w == 0) + g->cur_y = g->max_y; + + g->lflags = stbi__get8(s); + + if (g->lflags & 0x40) { + g->step = 8 * g->line_size; // first interlaced spacing + g->parse = 3; + } else { + g->step = g->line_size; + g->parse = 0; + } + + if (g->lflags & 0x80) { + stbi__gif_parse_colortable(s,g->lpal, 2 << (g->lflags & 7), g->eflags & 0x01 ? g->transparent : -1); + g->color_table = (stbi_uc *) g->lpal; + } else if (g->flags & 0x80) { + g->color_table = (stbi_uc *) g->pal; + } else + return stbi__errpuc("missing color table", "Corrupt GIF"); + + o = stbi__process_gif_raster(s, g); + if (!o) return NULL; + + // if this was the first frame, + pcount = g->w * g->h; + if (first_frame && (g->bgindex > 0)) { + // if first frame, any pixel not drawn to gets the background color + for (pi = 0; pi < pcount; ++pi) { + if (g->history[pi] == 0) { + g->pal[g->bgindex][3] = 255; // just in case it was made transparent, undo that; It will be reset next frame if need be; + memcpy( &g->out[pi * 4], &g->pal[g->bgindex], 4 ); + } + } + } + + return o; + } + + case 0x21: // Comment Extension. + { + int len; + int ext = stbi__get8(s); + if (ext == 0xF9) { // Graphic Control Extension. + len = stbi__get8(s); + if (len == 4) { + g->eflags = stbi__get8(s); + g->delay = 10 * stbi__get16le(s); // delay - 1/100th of a second, saving as 1/1000ths. + + // unset old transparent + if (g->transparent >= 0) { + g->pal[g->transparent][3] = 255; + } + if (g->eflags & 0x01) { + g->transparent = stbi__get8(s); + if (g->transparent >= 0) { + g->pal[g->transparent][3] = 0; + } + } else { + // don't need transparent + stbi__skip(s, 1); + g->transparent = -1; + } + } else { + stbi__skip(s, len); + break; + } + } + while ((len = stbi__get8(s)) != 0) { + stbi__skip(s, len); + } + break; + } + + case 0x3B: // gif stream termination code + return (stbi_uc *) s; // using '1' causes warning on some compilers + + default: + return stbi__errpuc("unknown code", "Corrupt GIF"); + } + } +} + +static void *stbi__load_gif_main_outofmem(stbi__gif *g, stbi_uc *out, int **delays) +{ + STBI_FREE(g->out); + STBI_FREE(g->history); + STBI_FREE(g->background); + + if (out) STBI_FREE(out); + if (delays && *delays) STBI_FREE(*delays); + return stbi__errpuc("outofmem", "Out of memory"); +} + +static void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp) +{ + if (stbi__gif_test(s)) { + int layers = 0; + stbi_uc *u = 0; + stbi_uc *out = 0; + stbi_uc *two_back = 0; + stbi__gif g; + int stride; + int out_size = 0; + int delays_size = 0; + + STBI_NOTUSED(out_size); + STBI_NOTUSED(delays_size); + + memset(&g, 0, sizeof(g)); + if (delays) { + *delays = 0; + } + + do { + u = stbi__gif_load_next(s, &g, comp, req_comp, two_back); + if (u == (stbi_uc *) s) u = 0; // end of animated gif marker + + if (u) { + *x = g.w; + *y = g.h; + ++layers; + stride = g.w * g.h * 4; + + if (out) { + void *tmp = (stbi_uc*) STBI_REALLOC_SIZED( out, out_size, layers * stride ); + if (!tmp) + return stbi__load_gif_main_outofmem(&g, out, delays); + else { + out = (stbi_uc*) tmp; + out_size = layers * stride; + } + + if (delays) { + int *new_delays = (int*) STBI_REALLOC_SIZED( *delays, delays_size, sizeof(int) * layers ); + if (!new_delays) + return stbi__load_gif_main_outofmem(&g, out, delays); + *delays = new_delays; + delays_size = layers * sizeof(int); + } + } else { + out = (stbi_uc*)stbi__malloc( layers * stride ); + if (!out) + return stbi__load_gif_main_outofmem(&g, out, delays); + out_size = layers * stride; + if (delays) { + *delays = (int*) stbi__malloc( layers * sizeof(int) ); + if (!*delays) + return stbi__load_gif_main_outofmem(&g, out, delays); + delays_size = layers * sizeof(int); + } + } + memcpy( out + ((layers - 1) * stride), u, stride ); + if (layers >= 2) { + two_back = out - 2 * stride; + } + + if (delays) { + (*delays)[layers - 1U] = g.delay; + } + } + } while (u != 0); + + // free temp buffer; + STBI_FREE(g.out); + STBI_FREE(g.history); + STBI_FREE(g.background); + + // do the final conversion after loading everything; + if (req_comp && req_comp != 4) + out = stbi__convert_format(out, 4, req_comp, layers * g.w, g.h); + + *z = layers; + return out; + } else { + return stbi__errpuc("not GIF", "Image was not as a gif type."); + } +} + +static void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi_uc *u = 0; + stbi__gif g; + memset(&g, 0, sizeof(g)); + STBI_NOTUSED(ri); + + u = stbi__gif_load_next(s, &g, comp, req_comp, 0); + if (u == (stbi_uc *) s) u = 0; // end of animated gif marker + if (u) { + *x = g.w; + *y = g.h; + + // moved conversion to after successful load so that the same + // can be done for multiple frames. + if (req_comp && req_comp != 4) + u = stbi__convert_format(u, 4, req_comp, g.w, g.h); + } else if (g.out) { + // if there was an error and we allocated an image buffer, free it! + STBI_FREE(g.out); + } + + // free buffers needed for multiple frame loading; + STBI_FREE(g.history); + STBI_FREE(g.background); + + return u; +} + +static int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp) +{ + return stbi__gif_info_raw(s,x,y,comp); +} +#endif + +// ************************************************************************************************* +// Radiance RGBE HDR loader +// originally by Nicolas Schulz +#ifndef STBI_NO_HDR +static int stbi__hdr_test_core(stbi__context *s, const char *signature) +{ + int i; + for (i=0; signature[i]; ++i) + if (stbi__get8(s) != signature[i]) + return 0; + stbi__rewind(s); + return 1; +} + +static int stbi__hdr_test(stbi__context* s) +{ + int r = stbi__hdr_test_core(s, "#?RADIANCE\n"); + stbi__rewind(s); + if(!r) { + r = stbi__hdr_test_core(s, "#?RGBE\n"); + stbi__rewind(s); + } + return r; +} + +#define STBI__HDR_BUFLEN 1024 +static char *stbi__hdr_gettoken(stbi__context *z, char *buffer) +{ + int len=0; + char c = '\0'; + + c = (char) stbi__get8(z); + + while (!stbi__at_eof(z) && c != '\n') { + buffer[len++] = c; + if (len == STBI__HDR_BUFLEN-1) { + // flush to end of line + while (!stbi__at_eof(z) && stbi__get8(z) != '\n') + ; + break; + } + c = (char) stbi__get8(z); + } + + buffer[len] = 0; + return buffer; +} + +static void stbi__hdr_convert(float *output, stbi_uc *input, int req_comp) +{ + if ( input[3] != 0 ) { + float f1; + // Exponent + f1 = (float) ldexp(1.0f, input[3] - (int)(128 + 8)); + if (req_comp <= 2) + output[0] = (input[0] + input[1] + input[2]) * f1 / 3; + else { + output[0] = input[0] * f1; + output[1] = input[1] * f1; + output[2] = input[2] * f1; + } + if (req_comp == 2) output[1] = 1; + if (req_comp == 4) output[3] = 1; + } else { + switch (req_comp) { + case 4: output[3] = 1; /* fallthrough */ + case 3: output[0] = output[1] = output[2] = 0; + break; + case 2: output[1] = 1; /* fallthrough */ + case 1: output[0] = 0; + break; + } + } +} + +static float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + char buffer[STBI__HDR_BUFLEN]; + char *token; + int valid = 0; + int width, height; + stbi_uc *scanline; + float *hdr_data; + int len; + unsigned char count, value; + int i, j, k, c1,c2, z; + const char *headerToken; + STBI_NOTUSED(ri); + + // Check identifier + headerToken = stbi__hdr_gettoken(s,buffer); + if (strcmp(headerToken, "#?RADIANCE") != 0 && strcmp(headerToken, "#?RGBE") != 0) + return stbi__errpf("not HDR", "Corrupt HDR image"); + + // Parse header + for(;;) { + token = stbi__hdr_gettoken(s,buffer); + if (token[0] == 0) break; + if (strcmp(token, "FORMAT=32-bit_rle_rgbe") == 0) valid = 1; + } + + if (!valid) return stbi__errpf("unsupported format", "Unsupported HDR format"); + + // Parse width and height + // can't use sscanf() if we're not using stdio! + token = stbi__hdr_gettoken(s,buffer); + if (strncmp(token, "-Y ", 3)) return stbi__errpf("unsupported data layout", "Unsupported HDR format"); + token += 3; + height = (int) strtol(token, &token, 10); + while (*token == ' ') ++token; + if (strncmp(token, "+X ", 3)) return stbi__errpf("unsupported data layout", "Unsupported HDR format"); + token += 3; + width = (int) strtol(token, NULL, 10); + + if (height > STBI_MAX_DIMENSIONS) return stbi__errpf("too large","Very large image (corrupt?)"); + if (width > STBI_MAX_DIMENSIONS) return stbi__errpf("too large","Very large image (corrupt?)"); + + *x = width; + *y = height; + + if (comp) *comp = 3; + if (req_comp == 0) req_comp = 3; + + if (!stbi__mad4sizes_valid(width, height, req_comp, sizeof(float), 0)) + return stbi__errpf("too large", "HDR image is too large"); + + // Read data + hdr_data = (float *) stbi__malloc_mad4(width, height, req_comp, sizeof(float), 0); + if (!hdr_data) + return stbi__errpf("outofmem", "Out of memory"); + + // Load image data + // image data is stored as some number of sca + if ( width < 8 || width >= 32768) { + // Read flat data + for (j=0; j < height; ++j) { + for (i=0; i < width; ++i) { + stbi_uc rgbe[4]; + main_decode_loop: + stbi__getn(s, rgbe, 4); + stbi__hdr_convert(hdr_data + j * width * req_comp + i * req_comp, rgbe, req_comp); + } + } + } else { + // Read RLE-encoded data + scanline = NULL; + + for (j = 0; j < height; ++j) { + c1 = stbi__get8(s); + c2 = stbi__get8(s); + len = stbi__get8(s); + if (c1 != 2 || c2 != 2 || (len & 0x80)) { + // not run-length encoded, so we have to actually use THIS data as a decoded + // pixel (note this can't be a valid pixel--one of RGB must be >= 128) + stbi_uc rgbe[4]; + rgbe[0] = (stbi_uc) c1; + rgbe[1] = (stbi_uc) c2; + rgbe[2] = (stbi_uc) len; + rgbe[3] = (stbi_uc) stbi__get8(s); + stbi__hdr_convert(hdr_data, rgbe, req_comp); + i = 1; + j = 0; + STBI_FREE(scanline); + goto main_decode_loop; // yes, this makes no sense + } + len <<= 8; + len |= stbi__get8(s); + if (len != width) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("invalid decoded scanline length", "corrupt HDR"); } + if (scanline == NULL) { + scanline = (stbi_uc *) stbi__malloc_mad2(width, 4, 0); + if (!scanline) { + STBI_FREE(hdr_data); + return stbi__errpf("outofmem", "Out of memory"); + } + } + + for (k = 0; k < 4; ++k) { + int nleft; + i = 0; + while ((nleft = width - i) > 0) { + count = stbi__get8(s); + if (count > 128) { + // Run + value = stbi__get8(s); + count -= 128; + if ((count == 0) || (count > nleft)) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("corrupt", "bad RLE data in HDR"); } + for (z = 0; z < count; ++z) + scanline[i++ * 4 + k] = value; + } else { + // Dump + if ((count == 0) || (count > nleft)) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("corrupt", "bad RLE data in HDR"); } + for (z = 0; z < count; ++z) + scanline[i++ * 4 + k] = stbi__get8(s); + } + } + } + for (i=0; i < width; ++i) + stbi__hdr_convert(hdr_data+(j*width + i)*req_comp, scanline + i*4, req_comp); + } + if (scanline) + STBI_FREE(scanline); + } + + return hdr_data; +} + +static int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp) +{ + char buffer[STBI__HDR_BUFLEN]; + char *token; + int valid = 0; + int dummy; + + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + + if (stbi__hdr_test(s) == 0) { + stbi__rewind( s ); + return 0; + } + + for(;;) { + token = stbi__hdr_gettoken(s,buffer); + if (token[0] == 0) break; + if (strcmp(token, "FORMAT=32-bit_rle_rgbe") == 0) valid = 1; + } + + if (!valid) { + stbi__rewind( s ); + return 0; + } + token = stbi__hdr_gettoken(s,buffer); + if (strncmp(token, "-Y ", 3)) { + stbi__rewind( s ); + return 0; + } + token += 3; + *y = (int) strtol(token, &token, 10); + while (*token == ' ') ++token; + if (strncmp(token, "+X ", 3)) { + stbi__rewind( s ); + return 0; + } + token += 3; + *x = (int) strtol(token, NULL, 10); + *comp = 3; + return 1; +} +#endif // STBI_NO_HDR + +#ifndef STBI_NO_BMP +static int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp) +{ + void *p; + stbi__bmp_data info; + + info.all_a = 255; + p = stbi__bmp_parse_header(s, &info); + if (p == NULL) { + stbi__rewind( s ); + return 0; + } + if (x) *x = s->img_x; + if (y) *y = s->img_y; + if (comp) { + if (info.bpp == 24 && info.ma == 0xff000000) + *comp = 3; + else + *comp = info.ma ? 4 : 3; + } + return 1; +} +#endif + +#ifndef STBI_NO_PSD +static int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp) +{ + int channelCount, dummy, depth; + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + if (stbi__get32be(s) != 0x38425053) { + stbi__rewind( s ); + return 0; + } + if (stbi__get16be(s) != 1) { + stbi__rewind( s ); + return 0; + } + stbi__skip(s, 6); + channelCount = stbi__get16be(s); + if (channelCount < 0 || channelCount > 16) { + stbi__rewind( s ); + return 0; + } + *y = stbi__get32be(s); + *x = stbi__get32be(s); + depth = stbi__get16be(s); + if (depth != 8 && depth != 16) { + stbi__rewind( s ); + return 0; + } + if (stbi__get16be(s) != 3) { + stbi__rewind( s ); + return 0; + } + *comp = 4; + return 1; +} + +static int stbi__psd_is16(stbi__context *s) +{ + int channelCount, depth; + if (stbi__get32be(s) != 0x38425053) { + stbi__rewind( s ); + return 0; + } + if (stbi__get16be(s) != 1) { + stbi__rewind( s ); + return 0; + } + stbi__skip(s, 6); + channelCount = stbi__get16be(s); + if (channelCount < 0 || channelCount > 16) { + stbi__rewind( s ); + return 0; + } + STBI_NOTUSED(stbi__get32be(s)); + STBI_NOTUSED(stbi__get32be(s)); + depth = stbi__get16be(s); + if (depth != 16) { + stbi__rewind( s ); + return 0; + } + return 1; +} +#endif + +#ifndef STBI_NO_PIC +static int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp) +{ + int act_comp=0,num_packets=0,chained,dummy; + stbi__pic_packet packets[10]; + + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + + if (!stbi__pic_is4(s,"\x53\x80\xF6\x34")) { + stbi__rewind(s); + return 0; + } + + stbi__skip(s, 88); + + *x = stbi__get16be(s); + *y = stbi__get16be(s); + if (stbi__at_eof(s)) { + stbi__rewind( s); + return 0; + } + if ( (*x) != 0 && (1 << 28) / (*x) < (*y)) { + stbi__rewind( s ); + return 0; + } + + stbi__skip(s, 8); + + do { + stbi__pic_packet *packet; + + if (num_packets==sizeof(packets)/sizeof(packets[0])) + return 0; + + packet = &packets[num_packets++]; + chained = stbi__get8(s); + packet->size = stbi__get8(s); + packet->type = stbi__get8(s); + packet->channel = stbi__get8(s); + act_comp |= packet->channel; + + if (stbi__at_eof(s)) { + stbi__rewind( s ); + return 0; + } + if (packet->size != 8) { + stbi__rewind( s ); + return 0; + } + } while (chained); + + *comp = (act_comp & 0x10 ? 4 : 3); + + return 1; +} +#endif + +// ************************************************************************************************* +// Portable Gray Map and Portable Pixel Map loader +// by Ken Miller +// +// PGM: http://netpbm.sourceforge.net/doc/pgm.html +// PPM: http://netpbm.sourceforge.net/doc/ppm.html +// +// Known limitations: +// Does not support comments in the header section +// Does not support ASCII image data (formats P2 and P3) + +#ifndef STBI_NO_PNM + +static int stbi__pnm_test(stbi__context *s) +{ + char p, t; + p = (char) stbi__get8(s); + t = (char) stbi__get8(s); + if (p != 'P' || (t != '5' && t != '6')) { + stbi__rewind( s ); + return 0; + } + return 1; +} + +static void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi_uc *out; + STBI_NOTUSED(ri); + + ri->bits_per_channel = stbi__pnm_info(s, (int *)&s->img_x, (int *)&s->img_y, (int *)&s->img_n); + if (ri->bits_per_channel == 0) + return 0; + + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + *x = s->img_x; + *y = s->img_y; + if (comp) *comp = s->img_n; + + if (!stbi__mad4sizes_valid(s->img_n, s->img_x, s->img_y, ri->bits_per_channel / 8, 0)) + return stbi__errpuc("too large", "PNM too large"); + + out = (stbi_uc *) stbi__malloc_mad4(s->img_n, s->img_x, s->img_y, ri->bits_per_channel / 8, 0); + if (!out) return stbi__errpuc("outofmem", "Out of memory"); + if (!stbi__getn(s, out, s->img_n * s->img_x * s->img_y * (ri->bits_per_channel / 8))) { + STBI_FREE(out); + return stbi__errpuc("bad PNM", "PNM file truncated"); + } + + if (req_comp && req_comp != s->img_n) { + if (ri->bits_per_channel == 16) { + out = (stbi_uc *) stbi__convert_format16((stbi__uint16 *) out, s->img_n, req_comp, s->img_x, s->img_y); + } else { + out = stbi__convert_format(out, s->img_n, req_comp, s->img_x, s->img_y); + } + if (out == NULL) return out; // stbi__convert_format frees input on failure + } + return out; +} + +static int stbi__pnm_isspace(char c) +{ + return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r'; +} + +static void stbi__pnm_skip_whitespace(stbi__context *s, char *c) +{ + for (;;) { + while (!stbi__at_eof(s) && stbi__pnm_isspace(*c)) + *c = (char) stbi__get8(s); + + if (stbi__at_eof(s) || *c != '#') + break; + + while (!stbi__at_eof(s) && *c != '\n' && *c != '\r' ) + *c = (char) stbi__get8(s); + } +} + +static int stbi__pnm_isdigit(char c) +{ + return c >= '0' && c <= '9'; +} + +static int stbi__pnm_getinteger(stbi__context *s, char *c) +{ + int value = 0; + + while (!stbi__at_eof(s) && stbi__pnm_isdigit(*c)) { + value = value*10 + (*c - '0'); + *c = (char) stbi__get8(s); + if((value > 214748364) || (value == 214748364 && *c > '7')) + return stbi__err("integer parse overflow", "Parsing an integer in the PPM header overflowed a 32-bit int"); + } + + return value; +} + +static int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp) +{ + int maxv, dummy; + char c, p, t; + + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + + stbi__rewind(s); + + // Get identifier + p = (char) stbi__get8(s); + t = (char) stbi__get8(s); + if (p != 'P' || (t != '5' && t != '6')) { + stbi__rewind(s); + return 0; + } + + *comp = (t == '6') ? 3 : 1; // '5' is 1-component .pgm; '6' is 3-component .ppm + + c = (char) stbi__get8(s); + stbi__pnm_skip_whitespace(s, &c); + + *x = stbi__pnm_getinteger(s, &c); // read width + if(*x == 0) + return stbi__err("invalid width", "PPM image header had zero or overflowing width"); + stbi__pnm_skip_whitespace(s, &c); + + *y = stbi__pnm_getinteger(s, &c); // read height + if (*y == 0) + return stbi__err("invalid width", "PPM image header had zero or overflowing width"); + stbi__pnm_skip_whitespace(s, &c); + + maxv = stbi__pnm_getinteger(s, &c); // read max value + if (maxv > 65535) + return stbi__err("max value > 65535", "PPM image supports only 8-bit and 16-bit images"); + else if (maxv > 255) + return 16; + else + return 8; +} + +static int stbi__pnm_is16(stbi__context *s) +{ + if (stbi__pnm_info(s, NULL, NULL, NULL) == 16) + return 1; + return 0; +} +#endif + +static int stbi__info_main(stbi__context *s, int *x, int *y, int *comp) +{ + #ifndef STBI_NO_JPEG + if (stbi__jpeg_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PNG + if (stbi__png_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_GIF + if (stbi__gif_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_BMP + if (stbi__bmp_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PSD + if (stbi__psd_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PIC + if (stbi__pic_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PNM + if (stbi__pnm_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_HDR + if (stbi__hdr_info(s, x, y, comp)) return 1; + #endif + + // test tga last because it's a crappy test! + #ifndef STBI_NO_TGA + if (stbi__tga_info(s, x, y, comp)) + return 1; + #endif + return stbi__err("unknown image type", "Image not of any known type, or corrupt"); +} + +static int stbi__is_16_main(stbi__context *s) +{ + #ifndef STBI_NO_PNG + if (stbi__png_is16(s)) return 1; + #endif + + #ifndef STBI_NO_PSD + if (stbi__psd_is16(s)) return 1; + #endif + + #ifndef STBI_NO_PNM + if (stbi__pnm_is16(s)) return 1; + #endif + return 0; +} + +#ifndef STBI_NO_STDIO +STBIDEF int stbi_info(char const *filename, int *x, int *y, int *comp) +{ + FILE *f = stbi__fopen(filename, "rb"); + int result; + if (!f) return stbi__err("can't fopen", "Unable to open file"); + result = stbi_info_from_file(f, x, y, comp); + fclose(f); + return result; +} + +STBIDEF int stbi_info_from_file(FILE *f, int *x, int *y, int *comp) +{ + int r; + stbi__context s; + long pos = ftell(f); + stbi__start_file(&s, f); + r = stbi__info_main(&s,x,y,comp); + fseek(f,pos,SEEK_SET); + return r; +} + +STBIDEF int stbi_is_16_bit(char const *filename) +{ + FILE *f = stbi__fopen(filename, "rb"); + int result; + if (!f) return stbi__err("can't fopen", "Unable to open file"); + result = stbi_is_16_bit_from_file(f); + fclose(f); + return result; +} + +STBIDEF int stbi_is_16_bit_from_file(FILE *f) +{ + int r; + stbi__context s; + long pos = ftell(f); + stbi__start_file(&s, f); + r = stbi__is_16_main(&s); + fseek(f,pos,SEEK_SET); + return r; +} +#endif // !STBI_NO_STDIO + +STBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__info_main(&s,x,y,comp); +} + +STBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *c, void *user, int *x, int *y, int *comp) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user); + return stbi__info_main(&s,x,y,comp); +} + +STBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__is_16_main(&s); +} + +STBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *c, void *user) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user); + return stbi__is_16_main(&s); +} + +#endif // STB_IMAGE_IMPLEMENTATION + +/* + revision history: + 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs + 2.19 (2018-02-11) fix warning + 2.18 (2018-01-30) fix warnings + 2.17 (2018-01-29) change sbti__shiftsigned to avoid clang -O2 bug + 1-bit BMP + *_is_16_bit api + avoid warnings + 2.16 (2017-07-23) all functions have 16-bit variants; + STBI_NO_STDIO works again; + compilation fixes; + fix rounding in unpremultiply; + optimize vertical flip; + disable raw_len validation; + documentation fixes + 2.15 (2017-03-18) fix png-1,2,4 bug; now all Imagenet JPGs decode; + warning fixes; disable run-time SSE detection on gcc; + uniform handling of optional "return" values; + thread-safe initialization of zlib tables + 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs + 2.13 (2016-11-29) add 16-bit API, only supported for PNG right now + 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes + 2.11 (2016-04-02) allocate large structures on the stack + remove white matting for transparent PSD + fix reported channel count for PNG & BMP + re-enable SSE2 in non-gcc 64-bit + support RGB-formatted JPEG + read 16-bit PNGs (only as 8-bit) + 2.10 (2016-01-22) avoid warning introduced in 2.09 by STBI_REALLOC_SIZED + 2.09 (2016-01-16) allow comments in PNM files + 16-bit-per-pixel TGA (not bit-per-component) + info() for TGA could break due to .hdr handling + info() for BMP to shares code instead of sloppy parse + can use STBI_REALLOC_SIZED if allocator doesn't support realloc + code cleanup + 2.08 (2015-09-13) fix to 2.07 cleanup, reading RGB PSD as RGBA + 2.07 (2015-09-13) fix compiler warnings + partial animated GIF support + limited 16-bpc PSD support + #ifdef unused functions + bug with < 92 byte PIC,PNM,HDR,TGA + 2.06 (2015-04-19) fix bug where PSD returns wrong '*comp' value + 2.05 (2015-04-19) fix bug in progressive JPEG handling, fix warning + 2.04 (2015-04-15) try to re-enable SIMD on MinGW 64-bit + 2.03 (2015-04-12) extra corruption checking (mmozeiko) + stbi_set_flip_vertically_on_load (nguillemot) + fix NEON support; fix mingw support + 2.02 (2015-01-19) fix incorrect assert, fix warning + 2.01 (2015-01-17) fix various warnings; suppress SIMD on gcc 32-bit without -msse2 + 2.00b (2014-12-25) fix STBI_MALLOC in progressive JPEG + 2.00 (2014-12-25) optimize JPG, including x86 SSE2 & NEON SIMD (ryg) + progressive JPEG (stb) + PGM/PPM support (Ken Miller) + STBI_MALLOC,STBI_REALLOC,STBI_FREE + GIF bugfix -- seemingly never worked + STBI_NO_*, STBI_ONLY_* + 1.48 (2014-12-14) fix incorrectly-named assert() + 1.47 (2014-12-14) 1/2/4-bit PNG support, both direct and paletted (Omar Cornut & stb) + optimize PNG (ryg) + fix bug in interlaced PNG with user-specified channel count (stb) + 1.46 (2014-08-26) + fix broken tRNS chunk (colorkey-style transparency) in non-paletted PNG + 1.45 (2014-08-16) + fix MSVC-ARM internal compiler error by wrapping malloc + 1.44 (2014-08-07) + various warning fixes from Ronny Chevalier + 1.43 (2014-07-15) + fix MSVC-only compiler problem in code changed in 1.42 + 1.42 (2014-07-09) + don't define _CRT_SECURE_NO_WARNINGS (affects user code) + fixes to stbi__cleanup_jpeg path + added STBI_ASSERT to avoid requiring assert.h + 1.41 (2014-06-25) + fix search&replace from 1.36 that messed up comments/error messages + 1.40 (2014-06-22) + fix gcc struct-initialization warning + 1.39 (2014-06-15) + fix to TGA optimization when req_comp != number of components in TGA; + fix to GIF loading because BMP wasn't rewinding (whoops, no GIFs in my test suite) + add support for BMP version 5 (more ignored fields) + 1.38 (2014-06-06) + suppress MSVC warnings on integer casts truncating values + fix accidental rename of 'skip' field of I/O + 1.37 (2014-06-04) + remove duplicate typedef + 1.36 (2014-06-03) + convert to header file single-file library + if de-iphone isn't set, load iphone images color-swapped instead of returning NULL + 1.35 (2014-05-27) + various warnings + fix broken STBI_SIMD path + fix bug where stbi_load_from_file no longer left file pointer in correct place + fix broken non-easy path for 32-bit BMP (possibly never used) + TGA optimization by Arseny Kapoulkine + 1.34 (unknown) + use STBI_NOTUSED in stbi__resample_row_generic(), fix one more leak in tga failure case + 1.33 (2011-07-14) + make stbi_is_hdr work in STBI_NO_HDR (as specified), minor compiler-friendly improvements + 1.32 (2011-07-13) + support for "info" function for all supported filetypes (SpartanJ) + 1.31 (2011-06-20) + a few more leak fixes, bug in PNG handling (SpartanJ) + 1.30 (2011-06-11) + added ability to load files via callbacks to accomidate custom input streams (Ben Wenger) + removed deprecated format-specific test/load functions + removed support for installable file formats (stbi_loader) -- would have been broken for IO callbacks anyway + error cases in bmp and tga give messages and don't leak (Raymond Barbiero, grisha) + fix inefficiency in decoding 32-bit BMP (David Woo) + 1.29 (2010-08-16) + various warning fixes from Aurelien Pocheville + 1.28 (2010-08-01) + fix bug in GIF palette transparency (SpartanJ) + 1.27 (2010-08-01) + cast-to-stbi_uc to fix warnings + 1.26 (2010-07-24) + fix bug in file buffering for PNG reported by SpartanJ + 1.25 (2010-07-17) + refix trans_data warning (Won Chun) + 1.24 (2010-07-12) + perf improvements reading from files on platforms with lock-heavy fgetc() + minor perf improvements for jpeg + deprecated type-specific functions so we'll get feedback if they're needed + attempt to fix trans_data warning (Won Chun) + 1.23 fixed bug in iPhone support + 1.22 (2010-07-10) + removed image *writing* support + stbi_info support from Jetro Lauha + GIF support from Jean-Marc Lienher + iPhone PNG-extensions from James Brown + warning-fixes from Nicolas Schulz and Janez Zemva (i.stbi__err. Janez (U+017D)emva) + 1.21 fix use of 'stbi_uc' in header (reported by jon blow) + 1.20 added support for Softimage PIC, by Tom Seddon + 1.19 bug in interlaced PNG corruption check (found by ryg) + 1.18 (2008-08-02) + fix a threading bug (local mutable static) + 1.17 support interlaced PNG + 1.16 major bugfix - stbi__convert_format converted one too many pixels + 1.15 initialize some fields for thread safety + 1.14 fix threadsafe conversion bug + header-file-only version (#define STBI_HEADER_FILE_ONLY before including) + 1.13 threadsafe + 1.12 const qualifiers in the API + 1.11 Support installable IDCT, colorspace conversion routines + 1.10 Fixes for 64-bit (don't use "unsigned long") + optimized upsampling by Fabian "ryg" Giesen + 1.09 Fix format-conversion for PSD code (bad global variables!) + 1.08 Thatcher Ulrich's PSD code integrated by Nicolas Schulz + 1.07 attempt to fix C++ warning/errors again + 1.06 attempt to fix C++ warning/errors again + 1.05 fix TGA loading to return correct *comp and use good luminance calc + 1.04 default float alpha is 1, not 255; use 'void *' for stbi_image_free + 1.03 bugfixes to STBI_NO_STDIO, STBI_NO_HDR + 1.02 support for (subset of) HDR files, float interface for preferred access to them + 1.01 fix bug: possible bug in handling right-side up bmps... not sure + fix bug: the stbi__bmp_load() and stbi__tga_load() functions didn't work at all + 1.00 interface to zlib that skips zlib header + 0.99 correct handling of alpha in palette + 0.98 TGA loader by lonesock; dynamically add loaders (untested) + 0.97 jpeg errors on too large a file; also catch another malloc failure + 0.96 fix detection of invalid v value - particleman@mollyrocket forum + 0.95 during header scan, seek to markers in case of padding + 0.94 STBI_NO_STDIO to disable stdio usage; rename all #defines the same + 0.93 handle jpegtran output; verbose errors + 0.92 read 4,8,16,24,32-bit BMP files of several formats + 0.91 output 24-bit Windows 3.0 BMP files + 0.90 fix a few more warnings; bump version number to approach 1.0 + 0.61 bugfixes due to Marc LeBlanc, Christopher Lloyd + 0.60 fix compiling as c++ + 0.59 fix warnings: merge Dave Moore's -Wall fixes + 0.58 fix bug: zlib uncompressed mode len/nlen was wrong endian + 0.57 fix bug: jpg last huffman symbol before marker was >9 bits but less than 16 available + 0.56 fix bug: zlib uncompressed mode len vs. nlen + 0.55 fix bug: restart_interval not initialized to 0 + 0.54 allow NULL for 'int *comp' + 0.53 fix bug in png 3->4; speedup png decoding + 0.52 png handles req_comp=3,4 directly; minor cleanup; jpeg comments + 0.51 obey req_comp requests, 1-component jpegs return as 1-component, + on 'test' only check type, not whether we support this variant + 0.50 (2006-11-19) + first released version +*/ + + +/* +------------------------------------------------------------------------------ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2017 Sean Barrett +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------ +*/ diff --git a/examples/common/stb_image_write.h b/examples/common/stb_image_write.h new file mode 100644 index 0000000..e4b32ed --- /dev/null +++ b/examples/common/stb_image_write.h @@ -0,0 +1,1724 @@ +/* stb_image_write - v1.16 - public domain - http://nothings.org/stb + writes out PNG/BMP/TGA/JPEG/HDR images to C stdio - Sean Barrett 2010-2015 + no warranty implied; use at your own risk + + Before #including, + + #define STB_IMAGE_WRITE_IMPLEMENTATION + + in the file that you want to have the implementation. + + Will probably not work correctly with strict-aliasing optimizations. + +ABOUT: + + This header file is a library for writing images to C stdio or a callback. + + The PNG output is not optimal; it is 20-50% larger than the file + written by a decent optimizing implementation; though providing a custom + zlib compress function (see STBIW_ZLIB_COMPRESS) can mitigate that. + This library is designed for source code compactness and simplicity, + not optimal image file size or run-time performance. + +BUILDING: + + You can #define STBIW_ASSERT(x) before the #include to avoid using assert.h. + You can #define STBIW_MALLOC(), STBIW_REALLOC(), and STBIW_FREE() to replace + malloc,realloc,free. + You can #define STBIW_MEMMOVE() to replace memmove() + You can #define STBIW_ZLIB_COMPRESS to use a custom zlib-style compress function + for PNG compression (instead of the builtin one), it must have the following signature: + unsigned char * my_compress(unsigned char *data, int data_len, int *out_len, int quality); + The returned data will be freed with STBIW_FREE() (free() by default), + so it must be heap allocated with STBIW_MALLOC() (malloc() by default), + +UNICODE: + + If compiling for Windows and you wish to use Unicode filenames, compile + with + #define STBIW_WINDOWS_UTF8 + and pass utf8-encoded filenames. Call stbiw_convert_wchar_to_utf8 to convert + Windows wchar_t filenames to utf8. + +USAGE: + + There are five functions, one for each image file format: + + int stbi_write_png(char const *filename, int w, int h, int comp, const void *data, int stride_in_bytes); + int stbi_write_bmp(char const *filename, int w, int h, int comp, const void *data); + int stbi_write_tga(char const *filename, int w, int h, int comp, const void *data); + int stbi_write_jpg(char const *filename, int w, int h, int comp, const void *data, int quality); + int stbi_write_hdr(char const *filename, int w, int h, int comp, const float *data); + + void stbi_flip_vertically_on_write(int flag); // flag is non-zero to flip data vertically + + There are also five equivalent functions that use an arbitrary write function. You are + expected to open/close your file-equivalent before and after calling these: + + int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data, int stride_in_bytes); + int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); + int stbi_write_tga_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); + int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const float *data); + int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality); + + where the callback is: + void stbi_write_func(void *context, void *data, int size); + + You can configure it with these global variables: + int stbi_write_tga_with_rle; // defaults to true; set to 0 to disable RLE + int stbi_write_png_compression_level; // defaults to 8; set to higher for more compression + int stbi_write_force_png_filter; // defaults to -1; set to 0..5 to force a filter mode + + + You can define STBI_WRITE_NO_STDIO to disable the file variant of these + functions, so the library will not use stdio.h at all. However, this will + also disable HDR writing, because it requires stdio for formatted output. + + Each function returns 0 on failure and non-0 on success. + + The functions create an image file defined by the parameters. The image + is a rectangle of pixels stored from left-to-right, top-to-bottom. + Each pixel contains 'comp' channels of data stored interleaved with 8-bits + per channel, in the following order: 1=Y, 2=YA, 3=RGB, 4=RGBA. (Y is + monochrome color.) The rectangle is 'w' pixels wide and 'h' pixels tall. + The *data pointer points to the first byte of the top-left-most pixel. + For PNG, "stride_in_bytes" is the distance in bytes from the first byte of + a row of pixels to the first byte of the next row of pixels. + + PNG creates output files with the same number of components as the input. + The BMP format expands Y to RGB in the file format and does not + output alpha. + + PNG supports writing rectangles of data even when the bytes storing rows of + data are not consecutive in memory (e.g. sub-rectangles of a larger image), + by supplying the stride between the beginning of adjacent rows. The other + formats do not. (Thus you cannot write a native-format BMP through the BMP + writer, both because it is in BGR order and because it may have padding + at the end of the line.) + + PNG allows you to set the deflate compression level by setting the global + variable 'stbi_write_png_compression_level' (it defaults to 8). + + HDR expects linear float data. Since the format is always 32-bit rgb(e) + data, alpha (if provided) is discarded, and for monochrome data it is + replicated across all three channels. + + TGA supports RLE or non-RLE compressed data. To use non-RLE-compressed + data, set the global variable 'stbi_write_tga_with_rle' to 0. + + JPEG does ignore alpha channels in input data; quality is between 1 and 100. + Higher quality looks better but results in a bigger image. + JPEG baseline (no JPEG progressive). + +CREDITS: + + + Sean Barrett - PNG/BMP/TGA + Baldur Karlsson - HDR + Jean-Sebastien Guay - TGA monochrome + Tim Kelsey - misc enhancements + Alan Hickman - TGA RLE + Emmanuel Julien - initial file IO callback implementation + Jon Olick - original jo_jpeg.cpp code + Daniel Gibson - integrate JPEG, allow external zlib + Aarni Koskela - allow choosing PNG filter + + bugfixes: + github:Chribba + Guillaume Chereau + github:jry2 + github:romigrou + Sergio Gonzalez + Jonas Karlsson + Filip Wasil + Thatcher Ulrich + github:poppolopoppo + Patrick Boettcher + github:xeekworx + Cap Petschulat + Simon Rodriguez + Ivan Tikhonov + github:ignotion + Adam Schackart + Andrew Kensler + +LICENSE + + See end of file for license information. + +*/ + +#ifndef INCLUDE_STB_IMAGE_WRITE_H +#define INCLUDE_STB_IMAGE_WRITE_H + +#include + +// if STB_IMAGE_WRITE_STATIC causes problems, try defining STBIWDEF to 'inline' or 'static inline' +#ifndef STBIWDEF +#ifdef STB_IMAGE_WRITE_STATIC +#define STBIWDEF static +#else +#ifdef __cplusplus +#define STBIWDEF extern "C" +#else +#define STBIWDEF extern +#endif +#endif +#endif + +#ifndef STB_IMAGE_WRITE_STATIC // C++ forbids static forward declarations +STBIWDEF int stbi_write_tga_with_rle; +STBIWDEF int stbi_write_png_compression_level; +STBIWDEF int stbi_write_force_png_filter; +#endif + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_png(char const *filename, int w, int h, int comp, const void *data, int stride_in_bytes); +STBIWDEF int stbi_write_bmp(char const *filename, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_tga(char const *filename, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_hdr(char const *filename, int w, int h, int comp, const float *data); +STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void *data, int quality); + +#ifdef STBIW_WINDOWS_UTF8 +STBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input); +#endif +#endif + +typedef void stbi_write_func(void *context, void *data, int size); + +STBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data, int stride_in_bytes); +STBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const float *data); +STBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality); + +STBIWDEF void stbi_flip_vertically_on_write(int flip_boolean); + +#endif//INCLUDE_STB_IMAGE_WRITE_H + +#ifdef STB_IMAGE_WRITE_IMPLEMENTATION + +#ifdef _WIN32 + #ifndef _CRT_SECURE_NO_WARNINGS + #define _CRT_SECURE_NO_WARNINGS + #endif + #ifndef _CRT_NONSTDC_NO_DEPRECATE + #define _CRT_NONSTDC_NO_DEPRECATE + #endif +#endif + +#ifndef STBI_WRITE_NO_STDIO +#include +#endif // STBI_WRITE_NO_STDIO + +#include +#include +#include +#include + +#if defined(STBIW_MALLOC) && defined(STBIW_FREE) && (defined(STBIW_REALLOC) || defined(STBIW_REALLOC_SIZED)) +// ok +#elif !defined(STBIW_MALLOC) && !defined(STBIW_FREE) && !defined(STBIW_REALLOC) && !defined(STBIW_REALLOC_SIZED) +// ok +#else +#error "Must define all or none of STBIW_MALLOC, STBIW_FREE, and STBIW_REALLOC (or STBIW_REALLOC_SIZED)." +#endif + +#ifndef STBIW_MALLOC +#define STBIW_MALLOC(sz) malloc(sz) +#define STBIW_REALLOC(p,newsz) realloc(p,newsz) +#define STBIW_FREE(p) free(p) +#endif + +#ifndef STBIW_REALLOC_SIZED +#define STBIW_REALLOC_SIZED(p,oldsz,newsz) STBIW_REALLOC(p,newsz) +#endif + + +#ifndef STBIW_MEMMOVE +#define STBIW_MEMMOVE(a,b,sz) memmove(a,b,sz) +#endif + + +#ifndef STBIW_ASSERT +#include +#define STBIW_ASSERT(x) assert(x) +#endif + +#define STBIW_UCHAR(x) (unsigned char) ((x) & 0xff) + +#ifdef STB_IMAGE_WRITE_STATIC +static int stbi_write_png_compression_level = 8; +static int stbi_write_tga_with_rle = 1; +static int stbi_write_force_png_filter = -1; +#else +int stbi_write_png_compression_level = 8; +int stbi_write_tga_with_rle = 1; +int stbi_write_force_png_filter = -1; +#endif + +static int stbi__flip_vertically_on_write = 0; + +STBIWDEF void stbi_flip_vertically_on_write(int flag) +{ + stbi__flip_vertically_on_write = flag; +} + +typedef struct +{ + stbi_write_func *func; + void *context; + unsigned char buffer[64]; + int buf_used; +} stbi__write_context; + +// initialize a callback-based context +static void stbi__start_write_callbacks(stbi__write_context *s, stbi_write_func *c, void *context) +{ + s->func = c; + s->context = context; +} + +#ifndef STBI_WRITE_NO_STDIO + +static void stbi__stdio_write(void *context, void *data, int size) +{ + fwrite(data,1,size,(FILE*) context); +} + +#if defined(_WIN32) && defined(STBIW_WINDOWS_UTF8) +#ifdef __cplusplus +#define STBIW_EXTERN extern "C" +#else +#define STBIW_EXTERN extern +#endif +STBIW_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar(unsigned int cp, unsigned long flags, const char *str, int cbmb, wchar_t *widestr, int cchwide); +STBIW_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, char *str, int cbmb, const char *defchar, int *used_default); + +STBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input) +{ + return WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, (int) bufferlen, NULL, NULL); +} +#endif + +static FILE *stbiw__fopen(char const *filename, char const *mode) +{ + FILE *f; +#if defined(_WIN32) && defined(STBIW_WINDOWS_UTF8) + wchar_t wMode[64]; + wchar_t wFilename[1024]; + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, sizeof(wFilename)/sizeof(*wFilename))) + return 0; + + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, sizeof(wMode)/sizeof(*wMode))) + return 0; + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != _wfopen_s(&f, wFilename, wMode)) + f = 0; +#else + f = _wfopen(wFilename, wMode); +#endif + +#elif defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != fopen_s(&f, filename, mode)) + f=0; +#else + f = fopen(filename, mode); +#endif + return f; +} + +static int stbi__start_write_file(stbi__write_context *s, const char *filename) +{ + FILE *f = stbiw__fopen(filename, "wb"); + stbi__start_write_callbacks(s, stbi__stdio_write, (void *) f); + return f != NULL; +} + +static void stbi__end_write_file(stbi__write_context *s) +{ + fclose((FILE *)s->context); +} + +#endif // !STBI_WRITE_NO_STDIO + +typedef unsigned int stbiw_uint32; +typedef int stb_image_write_test[sizeof(stbiw_uint32)==4 ? 1 : -1]; + +static void stbiw__writefv(stbi__write_context *s, const char *fmt, va_list v) +{ + while (*fmt) { + switch (*fmt++) { + case ' ': break; + case '1': { unsigned char x = STBIW_UCHAR(va_arg(v, int)); + s->func(s->context,&x,1); + break; } + case '2': { int x = va_arg(v,int); + unsigned char b[2]; + b[0] = STBIW_UCHAR(x); + b[1] = STBIW_UCHAR(x>>8); + s->func(s->context,b,2); + break; } + case '4': { stbiw_uint32 x = va_arg(v,int); + unsigned char b[4]; + b[0]=STBIW_UCHAR(x); + b[1]=STBIW_UCHAR(x>>8); + b[2]=STBIW_UCHAR(x>>16); + b[3]=STBIW_UCHAR(x>>24); + s->func(s->context,b,4); + break; } + default: + STBIW_ASSERT(0); + return; + } + } +} + +static void stbiw__writef(stbi__write_context *s, const char *fmt, ...) +{ + va_list v; + va_start(v, fmt); + stbiw__writefv(s, fmt, v); + va_end(v); +} + +static void stbiw__write_flush(stbi__write_context *s) +{ + if (s->buf_used) { + s->func(s->context, &s->buffer, s->buf_used); + s->buf_used = 0; + } +} + +static void stbiw__putc(stbi__write_context *s, unsigned char c) +{ + s->func(s->context, &c, 1); +} + +static void stbiw__write1(stbi__write_context *s, unsigned char a) +{ + if ((size_t)s->buf_used + 1 > sizeof(s->buffer)) + stbiw__write_flush(s); + s->buffer[s->buf_used++] = a; +} + +static void stbiw__write3(stbi__write_context *s, unsigned char a, unsigned char b, unsigned char c) +{ + int n; + if ((size_t)s->buf_used + 3 > sizeof(s->buffer)) + stbiw__write_flush(s); + n = s->buf_used; + s->buf_used = n+3; + s->buffer[n+0] = a; + s->buffer[n+1] = b; + s->buffer[n+2] = c; +} + +static void stbiw__write_pixel(stbi__write_context *s, int rgb_dir, int comp, int write_alpha, int expand_mono, unsigned char *d) +{ + unsigned char bg[3] = { 255, 0, 255}, px[3]; + int k; + + if (write_alpha < 0) + stbiw__write1(s, d[comp - 1]); + + switch (comp) { + case 2: // 2 pixels = mono + alpha, alpha is written separately, so same as 1-channel case + case 1: + if (expand_mono) + stbiw__write3(s, d[0], d[0], d[0]); // monochrome bmp + else + stbiw__write1(s, d[0]); // monochrome TGA + break; + case 4: + if (!write_alpha) { + // composite against pink background + for (k = 0; k < 3; ++k) + px[k] = bg[k] + ((d[k] - bg[k]) * d[3]) / 255; + stbiw__write3(s, px[1 - rgb_dir], px[1], px[1 + rgb_dir]); + break; + } + /* FALLTHROUGH */ + case 3: + stbiw__write3(s, d[1 - rgb_dir], d[1], d[1 + rgb_dir]); + break; + } + if (write_alpha > 0) + stbiw__write1(s, d[comp - 1]); +} + +static void stbiw__write_pixels(stbi__write_context *s, int rgb_dir, int vdir, int x, int y, int comp, void *data, int write_alpha, int scanline_pad, int expand_mono) +{ + stbiw_uint32 zero = 0; + int i,j, j_end; + + if (y <= 0) + return; + + if (stbi__flip_vertically_on_write) + vdir *= -1; + + if (vdir < 0) { + j_end = -1; j = y-1; + } else { + j_end = y; j = 0; + } + + for (; j != j_end; j += vdir) { + for (i=0; i < x; ++i) { + unsigned char *d = (unsigned char *) data + (j*x+i)*comp; + stbiw__write_pixel(s, rgb_dir, comp, write_alpha, expand_mono, d); + } + stbiw__write_flush(s); + s->func(s->context, &zero, scanline_pad); + } +} + +static int stbiw__outfile(stbi__write_context *s, int rgb_dir, int vdir, int x, int y, int comp, int expand_mono, void *data, int alpha, int pad, const char *fmt, ...) +{ + if (y < 0 || x < 0) { + return 0; + } else { + va_list v; + va_start(v, fmt); + stbiw__writefv(s, fmt, v); + va_end(v); + stbiw__write_pixels(s,rgb_dir,vdir,x,y,comp,data,alpha,pad, expand_mono); + return 1; + } +} + +static int stbi_write_bmp_core(stbi__write_context *s, int x, int y, int comp, const void *data) +{ + if (comp != 4) { + // write RGB bitmap + int pad = (-x*3) & 3; + return stbiw__outfile(s,-1,-1,x,y,comp,1,(void *) data,0,pad, + "11 4 22 4" "4 44 22 444444", + 'B', 'M', 14+40+(x*3+pad)*y, 0,0, 14+40, // file header + 40, x,y, 1,24, 0,0,0,0,0,0); // bitmap header + } else { + // RGBA bitmaps need a v4 header + // use BI_BITFIELDS mode with 32bpp and alpha mask + // (straight BI_RGB with alpha mask doesn't work in most readers) + return stbiw__outfile(s,-1,-1,x,y,comp,1,(void *)data,1,0, + "11 4 22 4" "4 44 22 444444 4444 4 444 444 444 444", + 'B', 'M', 14+108+x*y*4, 0, 0, 14+108, // file header + 108, x,y, 1,32, 3,0,0,0,0,0, 0xff0000,0xff00,0xff,0xff000000u, 0, 0,0,0, 0,0,0, 0,0,0, 0,0,0); // bitmap V4 header + } +} + +STBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data) +{ + stbi__write_context s = { 0 }; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_bmp_core(&s, x, y, comp, data); +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_bmp(char const *filename, int x, int y, int comp, const void *data) +{ + stbi__write_context s = { 0 }; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_bmp_core(&s, x, y, comp, data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif //!STBI_WRITE_NO_STDIO + +static int stbi_write_tga_core(stbi__write_context *s, int x, int y, int comp, void *data) +{ + int has_alpha = (comp == 2 || comp == 4); + int colorbytes = has_alpha ? comp-1 : comp; + int format = colorbytes < 2 ? 3 : 2; // 3 color channels (RGB/RGBA) = 2, 1 color channel (Y/YA) = 3 + + if (y < 0 || x < 0) + return 0; + + if (!stbi_write_tga_with_rle) { + return stbiw__outfile(s, -1, -1, x, y, comp, 0, (void *) data, has_alpha, 0, + "111 221 2222 11", 0, 0, format, 0, 0, 0, 0, 0, x, y, (colorbytes + has_alpha) * 8, has_alpha * 8); + } else { + int i,j,k; + int jend, jdir; + + stbiw__writef(s, "111 221 2222 11", 0,0,format+8, 0,0,0, 0,0,x,y, (colorbytes + has_alpha) * 8, has_alpha * 8); + + if (stbi__flip_vertically_on_write) { + j = 0; + jend = y; + jdir = 1; + } else { + j = y-1; + jend = -1; + jdir = -1; + } + for (; j != jend; j += jdir) { + unsigned char *row = (unsigned char *) data + j * x * comp; + int len; + + for (i = 0; i < x; i += len) { + unsigned char *begin = row + i * comp; + int diff = 1; + len = 1; + + if (i < x - 1) { + ++len; + diff = memcmp(begin, row + (i + 1) * comp, comp); + if (diff) { + const unsigned char *prev = begin; + for (k = i + 2; k < x && len < 128; ++k) { + if (memcmp(prev, row + k * comp, comp)) { + prev += comp; + ++len; + } else { + --len; + break; + } + } + } else { + for (k = i + 2; k < x && len < 128; ++k) { + if (!memcmp(begin, row + k * comp, comp)) { + ++len; + } else { + break; + } + } + } + } + + if (diff) { + unsigned char header = STBIW_UCHAR(len - 1); + stbiw__write1(s, header); + for (k = 0; k < len; ++k) { + stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin + k * comp); + } + } else { + unsigned char header = STBIW_UCHAR(len - 129); + stbiw__write1(s, header); + stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin); + } + } + } + stbiw__write_flush(s); + } + return 1; +} + +STBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data) +{ + stbi__write_context s = { 0 }; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_tga_core(&s, x, y, comp, (void *) data); +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_tga(char const *filename, int x, int y, int comp, const void *data) +{ + stbi__write_context s = { 0 }; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_tga_core(&s, x, y, comp, (void *) data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif + +// ************************************************************************************************* +// Radiance RGBE HDR writer +// by Baldur Karlsson + +#define stbiw__max(a, b) ((a) > (b) ? (a) : (b)) + +#ifndef STBI_WRITE_NO_STDIO + +static void stbiw__linear_to_rgbe(unsigned char *rgbe, float *linear) +{ + int exponent; + float maxcomp = stbiw__max(linear[0], stbiw__max(linear[1], linear[2])); + + if (maxcomp < 1e-32f) { + rgbe[0] = rgbe[1] = rgbe[2] = rgbe[3] = 0; + } else { + float normalize = (float) frexp(maxcomp, &exponent) * 256.0f/maxcomp; + + rgbe[0] = (unsigned char)(linear[0] * normalize); + rgbe[1] = (unsigned char)(linear[1] * normalize); + rgbe[2] = (unsigned char)(linear[2] * normalize); + rgbe[3] = (unsigned char)(exponent + 128); + } +} + +static void stbiw__write_run_data(stbi__write_context *s, int length, unsigned char databyte) +{ + unsigned char lengthbyte = STBIW_UCHAR(length+128); + STBIW_ASSERT(length+128 <= 255); + s->func(s->context, &lengthbyte, 1); + s->func(s->context, &databyte, 1); +} + +static void stbiw__write_dump_data(stbi__write_context *s, int length, unsigned char *data) +{ + unsigned char lengthbyte = STBIW_UCHAR(length); + STBIW_ASSERT(length <= 128); // inconsistent with spec but consistent with official code + s->func(s->context, &lengthbyte, 1); + s->func(s->context, data, length); +} + +static void stbiw__write_hdr_scanline(stbi__write_context *s, int width, int ncomp, unsigned char *scratch, float *scanline) +{ + unsigned char scanlineheader[4] = { 2, 2, 0, 0 }; + unsigned char rgbe[4]; + float linear[3]; + int x; + + scanlineheader[2] = (width&0xff00)>>8; + scanlineheader[3] = (width&0x00ff); + + /* skip RLE for images too small or large */ + if (width < 8 || width >= 32768) { + for (x=0; x < width; x++) { + switch (ncomp) { + case 4: /* fallthrough */ + case 3: linear[2] = scanline[x*ncomp + 2]; + linear[1] = scanline[x*ncomp + 1]; + linear[0] = scanline[x*ncomp + 0]; + break; + default: + linear[0] = linear[1] = linear[2] = scanline[x*ncomp + 0]; + break; + } + stbiw__linear_to_rgbe(rgbe, linear); + s->func(s->context, rgbe, 4); + } + } else { + int c,r; + /* encode into scratch buffer */ + for (x=0; x < width; x++) { + switch(ncomp) { + case 4: /* fallthrough */ + case 3: linear[2] = scanline[x*ncomp + 2]; + linear[1] = scanline[x*ncomp + 1]; + linear[0] = scanline[x*ncomp + 0]; + break; + default: + linear[0] = linear[1] = linear[2] = scanline[x*ncomp + 0]; + break; + } + stbiw__linear_to_rgbe(rgbe, linear); + scratch[x + width*0] = rgbe[0]; + scratch[x + width*1] = rgbe[1]; + scratch[x + width*2] = rgbe[2]; + scratch[x + width*3] = rgbe[3]; + } + + s->func(s->context, scanlineheader, 4); + + /* RLE each component separately */ + for (c=0; c < 4; c++) { + unsigned char *comp = &scratch[width*c]; + + x = 0; + while (x < width) { + // find first run + r = x; + while (r+2 < width) { + if (comp[r] == comp[r+1] && comp[r] == comp[r+2]) + break; + ++r; + } + if (r+2 >= width) + r = width; + // dump up to first run + while (x < r) { + int len = r-x; + if (len > 128) len = 128; + stbiw__write_dump_data(s, len, &comp[x]); + x += len; + } + // if there's a run, output it + if (r+2 < width) { // same test as what we break out of in search loop, so only true if we break'd + // find next byte after run + while (r < width && comp[r] == comp[x]) + ++r; + // output run up to r + while (x < r) { + int len = r-x; + if (len > 127) len = 127; + stbiw__write_run_data(s, len, comp[x]); + x += len; + } + } + } + } + } +} + +static int stbi_write_hdr_core(stbi__write_context *s, int x, int y, int comp, float *data) +{ + if (y <= 0 || x <= 0 || data == NULL) + return 0; + else { + // Each component is stored separately. Allocate scratch space for full output scanline. + unsigned char *scratch = (unsigned char *) STBIW_MALLOC(x*4); + int i, len; + char buffer[128]; + char header[] = "#?RADIANCE\n# Written by stb_image_write.h\nFORMAT=32-bit_rle_rgbe\n"; + s->func(s->context, header, sizeof(header)-1); + +#ifdef __STDC_LIB_EXT1__ + len = sprintf_s(buffer, sizeof(buffer), "EXPOSURE= 1.0000000000000\n\n-Y %d +X %d\n", y, x); +#else + len = sprintf(buffer, "EXPOSURE= 1.0000000000000\n\n-Y %d +X %d\n", y, x); +#endif + s->func(s->context, buffer, len); + + for(i=0; i < y; i++) + stbiw__write_hdr_scanline(s, x, comp, scratch, data + comp*x*(stbi__flip_vertically_on_write ? y-1-i : i)); + STBIW_FREE(scratch); + return 1; + } +} + +STBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const float *data) +{ + stbi__write_context s = { 0 }; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_hdr_core(&s, x, y, comp, (float *) data); +} + +STBIWDEF int stbi_write_hdr(char const *filename, int x, int y, int comp, const float *data) +{ + stbi__write_context s = { 0 }; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_hdr_core(&s, x, y, comp, (float *) data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif // STBI_WRITE_NO_STDIO + + +////////////////////////////////////////////////////////////////////////////// +// +// PNG writer +// + +#ifndef STBIW_ZLIB_COMPRESS +// stretchy buffer; stbiw__sbpush() == vector<>::push_back() -- stbiw__sbcount() == vector<>::size() +#define stbiw__sbraw(a) ((int *) (void *) (a) - 2) +#define stbiw__sbm(a) stbiw__sbraw(a)[0] +#define stbiw__sbn(a) stbiw__sbraw(a)[1] + +#define stbiw__sbneedgrow(a,n) ((a)==0 || stbiw__sbn(a)+n >= stbiw__sbm(a)) +#define stbiw__sbmaybegrow(a,n) (stbiw__sbneedgrow(a,(n)) ? stbiw__sbgrow(a,n) : 0) +#define stbiw__sbgrow(a,n) stbiw__sbgrowf((void **) &(a), (n), sizeof(*(a))) + +#define stbiw__sbpush(a, v) (stbiw__sbmaybegrow(a,1), (a)[stbiw__sbn(a)++] = (v)) +#define stbiw__sbcount(a) ((a) ? stbiw__sbn(a) : 0) +#define stbiw__sbfree(a) ((a) ? STBIW_FREE(stbiw__sbraw(a)),0 : 0) + +static void *stbiw__sbgrowf(void **arr, int increment, int itemsize) +{ + int m = *arr ? 2*stbiw__sbm(*arr)+increment : increment+1; + void *p = STBIW_REALLOC_SIZED(*arr ? stbiw__sbraw(*arr) : 0, *arr ? (stbiw__sbm(*arr)*itemsize + sizeof(int)*2) : 0, itemsize * m + sizeof(int)*2); + STBIW_ASSERT(p); + if (p) { + if (!*arr) ((int *) p)[1] = 0; + *arr = (void *) ((int *) p + 2); + stbiw__sbm(*arr) = m; + } + return *arr; +} + +static unsigned char *stbiw__zlib_flushf(unsigned char *data, unsigned int *bitbuffer, int *bitcount) +{ + while (*bitcount >= 8) { + stbiw__sbpush(data, STBIW_UCHAR(*bitbuffer)); + *bitbuffer >>= 8; + *bitcount -= 8; + } + return data; +} + +static int stbiw__zlib_bitrev(int code, int codebits) +{ + int res=0; + while (codebits--) { + res = (res << 1) | (code & 1); + code >>= 1; + } + return res; +} + +static unsigned int stbiw__zlib_countm(unsigned char *a, unsigned char *b, int limit) +{ + int i; + for (i=0; i < limit && i < 258; ++i) + if (a[i] != b[i]) break; + return i; +} + +static unsigned int stbiw__zhash(unsigned char *data) +{ + stbiw_uint32 hash = data[0] + (data[1] << 8) + (data[2] << 16); + hash ^= hash << 3; + hash += hash >> 5; + hash ^= hash << 4; + hash += hash >> 17; + hash ^= hash << 25; + hash += hash >> 6; + return hash; +} + +#define stbiw__zlib_flush() (out = stbiw__zlib_flushf(out, &bitbuf, &bitcount)) +#define stbiw__zlib_add(code,codebits) \ + (bitbuf |= (code) << bitcount, bitcount += (codebits), stbiw__zlib_flush()) +#define stbiw__zlib_huffa(b,c) stbiw__zlib_add(stbiw__zlib_bitrev(b,c),c) +// default huffman tables +#define stbiw__zlib_huff1(n) stbiw__zlib_huffa(0x30 + (n), 8) +#define stbiw__zlib_huff2(n) stbiw__zlib_huffa(0x190 + (n)-144, 9) +#define stbiw__zlib_huff3(n) stbiw__zlib_huffa(0 + (n)-256,7) +#define stbiw__zlib_huff4(n) stbiw__zlib_huffa(0xc0 + (n)-280,8) +#define stbiw__zlib_huff(n) ((n) <= 143 ? stbiw__zlib_huff1(n) : (n) <= 255 ? stbiw__zlib_huff2(n) : (n) <= 279 ? stbiw__zlib_huff3(n) : stbiw__zlib_huff4(n)) +#define stbiw__zlib_huffb(n) ((n) <= 143 ? stbiw__zlib_huff1(n) : stbiw__zlib_huff2(n)) + +#define stbiw__ZHASH 16384 + +#endif // STBIW_ZLIB_COMPRESS + +STBIWDEF unsigned char * stbi_zlib_compress(unsigned char *data, int data_len, int *out_len, int quality) +{ +#ifdef STBIW_ZLIB_COMPRESS + // user provided a zlib compress implementation, use that + return STBIW_ZLIB_COMPRESS(data, data_len, out_len, quality); +#else // use builtin + static unsigned short lengthc[] = { 3,4,5,6,7,8,9,10,11,13,15,17,19,23,27,31,35,43,51,59,67,83,99,115,131,163,195,227,258, 259 }; + static unsigned char lengtheb[]= { 0,0,0,0,0,0,0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0 }; + static unsigned short distc[] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193,257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577, 32768 }; + static unsigned char disteb[] = { 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13 }; + unsigned int bitbuf=0; + int i,j, bitcount=0; + unsigned char *out = NULL; + unsigned char ***hash_table = (unsigned char***) STBIW_MALLOC(stbiw__ZHASH * sizeof(unsigned char**)); + if (hash_table == NULL) + return NULL; + if (quality < 5) quality = 5; + + stbiw__sbpush(out, 0x78); // DEFLATE 32K window + stbiw__sbpush(out, 0x5e); // FLEVEL = 1 + stbiw__zlib_add(1,1); // BFINAL = 1 + stbiw__zlib_add(1,2); // BTYPE = 1 -- fixed huffman + + for (i=0; i < stbiw__ZHASH; ++i) + hash_table[i] = NULL; + + i=0; + while (i < data_len-3) { + // hash next 3 bytes of data to be compressed + int h = stbiw__zhash(data+i)&(stbiw__ZHASH-1), best=3; + unsigned char *bestloc = 0; + unsigned char **hlist = hash_table[h]; + int n = stbiw__sbcount(hlist); + for (j=0; j < n; ++j) { + if (hlist[j]-data > i-32768) { // if entry lies within window + int d = stbiw__zlib_countm(hlist[j], data+i, data_len-i); + if (d >= best) { best=d; bestloc=hlist[j]; } + } + } + // when hash table entry is too long, delete half the entries + if (hash_table[h] && stbiw__sbn(hash_table[h]) == 2*quality) { + STBIW_MEMMOVE(hash_table[h], hash_table[h]+quality, sizeof(hash_table[h][0])*quality); + stbiw__sbn(hash_table[h]) = quality; + } + stbiw__sbpush(hash_table[h],data+i); + + if (bestloc) { + // "lazy matching" - check match at *next* byte, and if it's better, do cur byte as literal + h = stbiw__zhash(data+i+1)&(stbiw__ZHASH-1); + hlist = hash_table[h]; + n = stbiw__sbcount(hlist); + for (j=0; j < n; ++j) { + if (hlist[j]-data > i-32767) { + int e = stbiw__zlib_countm(hlist[j], data+i+1, data_len-i-1); + if (e > best) { // if next match is better, bail on current match + bestloc = NULL; + break; + } + } + } + } + + if (bestloc) { + int d = (int) (data+i - bestloc); // distance back + STBIW_ASSERT(d <= 32767 && best <= 258); + for (j=0; best > lengthc[j+1]-1; ++j); + stbiw__zlib_huff(j+257); + if (lengtheb[j]) stbiw__zlib_add(best - lengthc[j], lengtheb[j]); + for (j=0; d > distc[j+1]-1; ++j); + stbiw__zlib_add(stbiw__zlib_bitrev(j,5),5); + if (disteb[j]) stbiw__zlib_add(d - distc[j], disteb[j]); + i += best; + } else { + stbiw__zlib_huffb(data[i]); + ++i; + } + } + // write out final bytes + for (;i < data_len; ++i) + stbiw__zlib_huffb(data[i]); + stbiw__zlib_huff(256); // end of block + // pad with 0 bits to byte boundary + while (bitcount) + stbiw__zlib_add(0,1); + + for (i=0; i < stbiw__ZHASH; ++i) + (void) stbiw__sbfree(hash_table[i]); + STBIW_FREE(hash_table); + + // store uncompressed instead if compression was worse + if (stbiw__sbn(out) > data_len + 2 + ((data_len+32766)/32767)*5) { + stbiw__sbn(out) = 2; // truncate to DEFLATE 32K window and FLEVEL = 1 + for (j = 0; j < data_len;) { + int blocklen = data_len - j; + if (blocklen > 32767) blocklen = 32767; + stbiw__sbpush(out, data_len - j == blocklen); // BFINAL = ?, BTYPE = 0 -- no compression + stbiw__sbpush(out, STBIW_UCHAR(blocklen)); // LEN + stbiw__sbpush(out, STBIW_UCHAR(blocklen >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(~blocklen)); // NLEN + stbiw__sbpush(out, STBIW_UCHAR(~blocklen >> 8)); + memcpy(out+stbiw__sbn(out), data+j, blocklen); + stbiw__sbn(out) += blocklen; + j += blocklen; + } + } + + { + // compute adler32 on input + unsigned int s1=1, s2=0; + int blocklen = (int) (data_len % 5552); + j=0; + while (j < data_len) { + for (i=0; i < blocklen; ++i) { s1 += data[j+i]; s2 += s1; } + s1 %= 65521; s2 %= 65521; + j += blocklen; + blocklen = 5552; + } + stbiw__sbpush(out, STBIW_UCHAR(s2 >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(s2)); + stbiw__sbpush(out, STBIW_UCHAR(s1 >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(s1)); + } + *out_len = stbiw__sbn(out); + // make returned pointer freeable + STBIW_MEMMOVE(stbiw__sbraw(out), out, *out_len); + return (unsigned char *) stbiw__sbraw(out); +#endif // STBIW_ZLIB_COMPRESS +} + +static unsigned int stbiw__crc32(unsigned char *buffer, int len) +{ +#ifdef STBIW_CRC32 + return STBIW_CRC32(buffer, len); +#else + static unsigned int crc_table[256] = + { + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, + 0x0eDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, + 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, + 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, + 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, + 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, + 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, + 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, + 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, + 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, + 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, + 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, + 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, + 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, + 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, + 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, + 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, + 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D + }; + + unsigned int crc = ~0u; + int i; + for (i=0; i < len; ++i) + crc = (crc >> 8) ^ crc_table[buffer[i] ^ (crc & 0xff)]; + return ~crc; +#endif +} + +#define stbiw__wpng4(o,a,b,c,d) ((o)[0]=STBIW_UCHAR(a),(o)[1]=STBIW_UCHAR(b),(o)[2]=STBIW_UCHAR(c),(o)[3]=STBIW_UCHAR(d),(o)+=4) +#define stbiw__wp32(data,v) stbiw__wpng4(data, (v)>>24,(v)>>16,(v)>>8,(v)); +#define stbiw__wptag(data,s) stbiw__wpng4(data, s[0],s[1],s[2],s[3]) + +static void stbiw__wpcrc(unsigned char **data, int len) +{ + unsigned int crc = stbiw__crc32(*data - len - 4, len+4); + stbiw__wp32(*data, crc); +} + +static unsigned char stbiw__paeth(int a, int b, int c) +{ + int p = a + b - c, pa = abs(p-a), pb = abs(p-b), pc = abs(p-c); + if (pa <= pb && pa <= pc) return STBIW_UCHAR(a); + if (pb <= pc) return STBIW_UCHAR(b); + return STBIW_UCHAR(c); +} + +// @OPTIMIZE: provide an option that always forces left-predict or paeth predict +static void stbiw__encode_png_line(unsigned char *pixels, int stride_bytes, int width, int height, int y, int n, int filter_type, signed char *line_buffer) +{ + static int mapping[] = { 0,1,2,3,4 }; + static int firstmap[] = { 0,1,0,5,6 }; + int *mymap = (y != 0) ? mapping : firstmap; + int i; + int type = mymap[filter_type]; + unsigned char *z = pixels + stride_bytes * (stbi__flip_vertically_on_write ? height-1-y : y); + int signed_stride = stbi__flip_vertically_on_write ? -stride_bytes : stride_bytes; + + if (type==0) { + memcpy(line_buffer, z, width*n); + return; + } + + // first loop isn't optimized since it's just one pixel + for (i = 0; i < n; ++i) { + switch (type) { + case 1: line_buffer[i] = z[i]; break; + case 2: line_buffer[i] = z[i] - z[i-signed_stride]; break; + case 3: line_buffer[i] = z[i] - (z[i-signed_stride]>>1); break; + case 4: line_buffer[i] = (signed char) (z[i] - stbiw__paeth(0,z[i-signed_stride],0)); break; + case 5: line_buffer[i] = z[i]; break; + case 6: line_buffer[i] = z[i]; break; + } + } + switch (type) { + case 1: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - z[i-n]; break; + case 2: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - z[i-signed_stride]; break; + case 3: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - ((z[i-n] + z[i-signed_stride])>>1); break; + case 4: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - stbiw__paeth(z[i-n], z[i-signed_stride], z[i-signed_stride-n]); break; + case 5: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - (z[i-n]>>1); break; + case 6: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - stbiw__paeth(z[i-n], 0,0); break; + } +} + +STBIWDEF unsigned char *stbi_write_png_to_mem(const unsigned char *pixels, int stride_bytes, int x, int y, int n, int *out_len) +{ + int force_filter = stbi_write_force_png_filter; + int ctype[5] = { -1, 0, 4, 2, 6 }; + unsigned char sig[8] = { 137,80,78,71,13,10,26,10 }; + unsigned char *out,*o, *filt, *zlib; + signed char *line_buffer; + int j,zlen; + + if (stride_bytes == 0) + stride_bytes = x * n; + + if (force_filter >= 5) { + force_filter = -1; + } + + filt = (unsigned char *) STBIW_MALLOC((x*n+1) * y); if (!filt) return 0; + line_buffer = (signed char *) STBIW_MALLOC(x * n); if (!line_buffer) { STBIW_FREE(filt); return 0; } + for (j=0; j < y; ++j) { + int filter_type; + if (force_filter > -1) { + filter_type = force_filter; + stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, force_filter, line_buffer); + } else { // Estimate the best filter by running through all of them: + int best_filter = 0, best_filter_val = 0x7fffffff, est, i; + for (filter_type = 0; filter_type < 5; filter_type++) { + stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, filter_type, line_buffer); + + // Estimate the entropy of the line using this filter; the less, the better. + est = 0; + for (i = 0; i < x*n; ++i) { + est += abs((signed char) line_buffer[i]); + } + if (est < best_filter_val) { + best_filter_val = est; + best_filter = filter_type; + } + } + if (filter_type != best_filter) { // If the last iteration already got us the best filter, don't redo it + stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, best_filter, line_buffer); + filter_type = best_filter; + } + } + // when we get here, filter_type contains the filter type, and line_buffer contains the data + filt[j*(x*n+1)] = (unsigned char) filter_type; + STBIW_MEMMOVE(filt+j*(x*n+1)+1, line_buffer, x*n); + } + STBIW_FREE(line_buffer); + zlib = stbi_zlib_compress(filt, y*( x*n+1), &zlen, stbi_write_png_compression_level); + STBIW_FREE(filt); + if (!zlib) return 0; + + // each tag requires 12 bytes of overhead + out = (unsigned char *) STBIW_MALLOC(8 + 12+13 + 12+zlen + 12); + if (!out) return 0; + *out_len = 8 + 12+13 + 12+zlen + 12; + + o=out; + STBIW_MEMMOVE(o,sig,8); o+= 8; + stbiw__wp32(o, 13); // header length + stbiw__wptag(o, "IHDR"); + stbiw__wp32(o, x); + stbiw__wp32(o, y); + *o++ = 8; + *o++ = STBIW_UCHAR(ctype[n]); + *o++ = 0; + *o++ = 0; + *o++ = 0; + stbiw__wpcrc(&o,13); + + stbiw__wp32(o, zlen); + stbiw__wptag(o, "IDAT"); + STBIW_MEMMOVE(o, zlib, zlen); + o += zlen; + STBIW_FREE(zlib); + stbiw__wpcrc(&o, zlen); + + stbiw__wp32(o,0); + stbiw__wptag(o, "IEND"); + stbiw__wpcrc(&o,0); + + STBIW_ASSERT(o == out + *out_len); + + return out; +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_png(char const *filename, int x, int y, int comp, const void *data, int stride_bytes) +{ + FILE *f; + int len; + unsigned char *png = stbi_write_png_to_mem((const unsigned char *) data, stride_bytes, x, y, comp, &len); + if (png == NULL) return 0; + + f = stbiw__fopen(filename, "wb"); + if (!f) { STBIW_FREE(png); return 0; } + fwrite(png, 1, len, f); + fclose(f); + STBIW_FREE(png); + return 1; +} +#endif + +STBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int stride_bytes) +{ + int len; + unsigned char *png = stbi_write_png_to_mem((const unsigned char *) data, stride_bytes, x, y, comp, &len); + if (png == NULL) return 0; + func(context, png, len); + STBIW_FREE(png); + return 1; +} + + +/* *************************************************************************** + * + * JPEG writer + * + * This is based on Jon Olick's jo_jpeg.cpp: + * public domain Simple, Minimalistic JPEG writer - http://www.jonolick.com/code.html + */ + +static const unsigned char stbiw__jpg_ZigZag[] = { 0,1,5,6,14,15,27,28,2,4,7,13,16,26,29,42,3,8,12,17,25,30,41,43,9,11,18, + 24,31,40,44,53,10,19,23,32,39,45,52,54,20,22,33,38,46,51,55,60,21,34,37,47,50,56,59,61,35,36,48,49,57,58,62,63 }; + +static void stbiw__jpg_writeBits(stbi__write_context *s, int *bitBufP, int *bitCntP, const unsigned short *bs) { + int bitBuf = *bitBufP, bitCnt = *bitCntP; + bitCnt += bs[1]; + bitBuf |= bs[0] << (24 - bitCnt); + while(bitCnt >= 8) { + unsigned char c = (bitBuf >> 16) & 255; + stbiw__putc(s, c); + if(c == 255) { + stbiw__putc(s, 0); + } + bitBuf <<= 8; + bitCnt -= 8; + } + *bitBufP = bitBuf; + *bitCntP = bitCnt; +} + +static void stbiw__jpg_DCT(float *d0p, float *d1p, float *d2p, float *d3p, float *d4p, float *d5p, float *d6p, float *d7p) { + float d0 = *d0p, d1 = *d1p, d2 = *d2p, d3 = *d3p, d4 = *d4p, d5 = *d5p, d6 = *d6p, d7 = *d7p; + float z1, z2, z3, z4, z5, z11, z13; + + float tmp0 = d0 + d7; + float tmp7 = d0 - d7; + float tmp1 = d1 + d6; + float tmp6 = d1 - d6; + float tmp2 = d2 + d5; + float tmp5 = d2 - d5; + float tmp3 = d3 + d4; + float tmp4 = d3 - d4; + + // Even part + float tmp10 = tmp0 + tmp3; // phase 2 + float tmp13 = tmp0 - tmp3; + float tmp11 = tmp1 + tmp2; + float tmp12 = tmp1 - tmp2; + + d0 = tmp10 + tmp11; // phase 3 + d4 = tmp10 - tmp11; + + z1 = (tmp12 + tmp13) * 0.707106781f; // c4 + d2 = tmp13 + z1; // phase 5 + d6 = tmp13 - z1; + + // Odd part + tmp10 = tmp4 + tmp5; // phase 2 + tmp11 = tmp5 + tmp6; + tmp12 = tmp6 + tmp7; + + // The rotator is modified from fig 4-8 to avoid extra negations. + z5 = (tmp10 - tmp12) * 0.382683433f; // c6 + z2 = tmp10 * 0.541196100f + z5; // c2-c6 + z4 = tmp12 * 1.306562965f + z5; // c2+c6 + z3 = tmp11 * 0.707106781f; // c4 + + z11 = tmp7 + z3; // phase 5 + z13 = tmp7 - z3; + + *d5p = z13 + z2; // phase 6 + *d3p = z13 - z2; + *d1p = z11 + z4; + *d7p = z11 - z4; + + *d0p = d0; *d2p = d2; *d4p = d4; *d6p = d6; +} + +static void stbiw__jpg_calcBits(int val, unsigned short bits[2]) { + int tmp1 = val < 0 ? -val : val; + val = val < 0 ? val-1 : val; + bits[1] = 1; + while(tmp1 >>= 1) { + ++bits[1]; + } + bits[0] = val & ((1<0)&&(DU[end0pos]==0); --end0pos) { + } + // end0pos = first element in reverse order !=0 + if(end0pos == 0) { + stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB); + return DU[0]; + } + for(i = 1; i <= end0pos; ++i) { + int startpos = i; + int nrzeroes; + unsigned short bits[2]; + for (; DU[i]==0 && i<=end0pos; ++i) { + } + nrzeroes = i-startpos; + if ( nrzeroes >= 16 ) { + int lng = nrzeroes>>4; + int nrmarker; + for (nrmarker=1; nrmarker <= lng; ++nrmarker) + stbiw__jpg_writeBits(s, bitBuf, bitCnt, M16zeroes); + nrzeroes &= 15; + } + stbiw__jpg_calcBits(DU[i], bits); + stbiw__jpg_writeBits(s, bitBuf, bitCnt, HTAC[(nrzeroes<<4)+bits[1]]); + stbiw__jpg_writeBits(s, bitBuf, bitCnt, bits); + } + if(end0pos != 63) { + stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB); + } + return DU[0]; +} + +static int stbi_write_jpg_core(stbi__write_context *s, int width, int height, int comp, const void* data, int quality) { + // Constants that don't pollute global namespace + static const unsigned char std_dc_luminance_nrcodes[] = {0,0,1,5,1,1,1,1,1,1,0,0,0,0,0,0,0}; + static const unsigned char std_dc_luminance_values[] = {0,1,2,3,4,5,6,7,8,9,10,11}; + static const unsigned char std_ac_luminance_nrcodes[] = {0,0,2,1,3,3,2,4,3,5,5,4,4,0,0,1,0x7d}; + static const unsigned char std_ac_luminance_values[] = { + 0x01,0x02,0x03,0x00,0x04,0x11,0x05,0x12,0x21,0x31,0x41,0x06,0x13,0x51,0x61,0x07,0x22,0x71,0x14,0x32,0x81,0x91,0xa1,0x08, + 0x23,0x42,0xb1,0xc1,0x15,0x52,0xd1,0xf0,0x24,0x33,0x62,0x72,0x82,0x09,0x0a,0x16,0x17,0x18,0x19,0x1a,0x25,0x26,0x27,0x28, + 0x29,0x2a,0x34,0x35,0x36,0x37,0x38,0x39,0x3a,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x53,0x54,0x55,0x56,0x57,0x58,0x59, + 0x5a,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x83,0x84,0x85,0x86,0x87,0x88,0x89, + 0x8a,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xb2,0xb3,0xb4,0xb5,0xb6, + 0xb7,0xb8,0xb9,0xba,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xe1,0xe2, + 0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa + }; + static const unsigned char std_dc_chrominance_nrcodes[] = {0,0,3,1,1,1,1,1,1,1,1,1,0,0,0,0,0}; + static const unsigned char std_dc_chrominance_values[] = {0,1,2,3,4,5,6,7,8,9,10,11}; + static const unsigned char std_ac_chrominance_nrcodes[] = {0,0,2,1,2,4,4,3,4,7,5,4,4,0,1,2,0x77}; + static const unsigned char std_ac_chrominance_values[] = { + 0x00,0x01,0x02,0x03,0x11,0x04,0x05,0x21,0x31,0x06,0x12,0x41,0x51,0x07,0x61,0x71,0x13,0x22,0x32,0x81,0x08,0x14,0x42,0x91, + 0xa1,0xb1,0xc1,0x09,0x23,0x33,0x52,0xf0,0x15,0x62,0x72,0xd1,0x0a,0x16,0x24,0x34,0xe1,0x25,0xf1,0x17,0x18,0x19,0x1a,0x26, + 0x27,0x28,0x29,0x2a,0x35,0x36,0x37,0x38,0x39,0x3a,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x53,0x54,0x55,0x56,0x57,0x58, + 0x59,0x5a,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x82,0x83,0x84,0x85,0x86,0x87, + 0x88,0x89,0x8a,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xb2,0xb3,0xb4, + 0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda, + 0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa + }; + // Huffman tables + static const unsigned short YDC_HT[256][2] = { {0,2},{2,3},{3,3},{4,3},{5,3},{6,3},{14,4},{30,5},{62,6},{126,7},{254,8},{510,9}}; + static const unsigned short UVDC_HT[256][2] = { {0,2},{1,2},{2,2},{6,3},{14,4},{30,5},{62,6},{126,7},{254,8},{510,9},{1022,10},{2046,11}}; + static const unsigned short YAC_HT[256][2] = { + {10,4},{0,2},{1,2},{4,3},{11,4},{26,5},{120,7},{248,8},{1014,10},{65410,16},{65411,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {12,4},{27,5},{121,7},{502,9},{2038,11},{65412,16},{65413,16},{65414,16},{65415,16},{65416,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {28,5},{249,8},{1015,10},{4084,12},{65417,16},{65418,16},{65419,16},{65420,16},{65421,16},{65422,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {58,6},{503,9},{4085,12},{65423,16},{65424,16},{65425,16},{65426,16},{65427,16},{65428,16},{65429,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {59,6},{1016,10},{65430,16},{65431,16},{65432,16},{65433,16},{65434,16},{65435,16},{65436,16},{65437,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {122,7},{2039,11},{65438,16},{65439,16},{65440,16},{65441,16},{65442,16},{65443,16},{65444,16},{65445,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {123,7},{4086,12},{65446,16},{65447,16},{65448,16},{65449,16},{65450,16},{65451,16},{65452,16},{65453,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {250,8},{4087,12},{65454,16},{65455,16},{65456,16},{65457,16},{65458,16},{65459,16},{65460,16},{65461,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {504,9},{32704,15},{65462,16},{65463,16},{65464,16},{65465,16},{65466,16},{65467,16},{65468,16},{65469,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {505,9},{65470,16},{65471,16},{65472,16},{65473,16},{65474,16},{65475,16},{65476,16},{65477,16},{65478,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {506,9},{65479,16},{65480,16},{65481,16},{65482,16},{65483,16},{65484,16},{65485,16},{65486,16},{65487,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {1017,10},{65488,16},{65489,16},{65490,16},{65491,16},{65492,16},{65493,16},{65494,16},{65495,16},{65496,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {1018,10},{65497,16},{65498,16},{65499,16},{65500,16},{65501,16},{65502,16},{65503,16},{65504,16},{65505,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {2040,11},{65506,16},{65507,16},{65508,16},{65509,16},{65510,16},{65511,16},{65512,16},{65513,16},{65514,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {65515,16},{65516,16},{65517,16},{65518,16},{65519,16},{65520,16},{65521,16},{65522,16},{65523,16},{65524,16},{0,0},{0,0},{0,0},{0,0},{0,0}, + {2041,11},{65525,16},{65526,16},{65527,16},{65528,16},{65529,16},{65530,16},{65531,16},{65532,16},{65533,16},{65534,16},{0,0},{0,0},{0,0},{0,0},{0,0} + }; + static const unsigned short UVAC_HT[256][2] = { + {0,2},{1,2},{4,3},{10,4},{24,5},{25,5},{56,6},{120,7},{500,9},{1014,10},{4084,12},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {11,4},{57,6},{246,8},{501,9},{2038,11},{4085,12},{65416,16},{65417,16},{65418,16},{65419,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {26,5},{247,8},{1015,10},{4086,12},{32706,15},{65420,16},{65421,16},{65422,16},{65423,16},{65424,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {27,5},{248,8},{1016,10},{4087,12},{65425,16},{65426,16},{65427,16},{65428,16},{65429,16},{65430,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {58,6},{502,9},{65431,16},{65432,16},{65433,16},{65434,16},{65435,16},{65436,16},{65437,16},{65438,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {59,6},{1017,10},{65439,16},{65440,16},{65441,16},{65442,16},{65443,16},{65444,16},{65445,16},{65446,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {121,7},{2039,11},{65447,16},{65448,16},{65449,16},{65450,16},{65451,16},{65452,16},{65453,16},{65454,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {122,7},{2040,11},{65455,16},{65456,16},{65457,16},{65458,16},{65459,16},{65460,16},{65461,16},{65462,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {249,8},{65463,16},{65464,16},{65465,16},{65466,16},{65467,16},{65468,16},{65469,16},{65470,16},{65471,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {503,9},{65472,16},{65473,16},{65474,16},{65475,16},{65476,16},{65477,16},{65478,16},{65479,16},{65480,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {504,9},{65481,16},{65482,16},{65483,16},{65484,16},{65485,16},{65486,16},{65487,16},{65488,16},{65489,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {505,9},{65490,16},{65491,16},{65492,16},{65493,16},{65494,16},{65495,16},{65496,16},{65497,16},{65498,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {506,9},{65499,16},{65500,16},{65501,16},{65502,16},{65503,16},{65504,16},{65505,16},{65506,16},{65507,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {2041,11},{65508,16},{65509,16},{65510,16},{65511,16},{65512,16},{65513,16},{65514,16},{65515,16},{65516,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {16352,14},{65517,16},{65518,16},{65519,16},{65520,16},{65521,16},{65522,16},{65523,16},{65524,16},{65525,16},{0,0},{0,0},{0,0},{0,0},{0,0}, + {1018,10},{32707,15},{65526,16},{65527,16},{65528,16},{65529,16},{65530,16},{65531,16},{65532,16},{65533,16},{65534,16},{0,0},{0,0},{0,0},{0,0},{0,0} + }; + static const int YQT[] = {16,11,10,16,24,40,51,61,12,12,14,19,26,58,60,55,14,13,16,24,40,57,69,56,14,17,22,29,51,87,80,62,18,22, + 37,56,68,109,103,77,24,35,55,64,81,104,113,92,49,64,78,87,103,121,120,101,72,92,95,98,112,100,103,99}; + static const int UVQT[] = {17,18,24,47,99,99,99,99,18,21,26,66,99,99,99,99,24,26,56,99,99,99,99,99,47,66,99,99,99,99,99,99, + 99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99}; + static const float aasf[] = { 1.0f * 2.828427125f, 1.387039845f * 2.828427125f, 1.306562965f * 2.828427125f, 1.175875602f * 2.828427125f, + 1.0f * 2.828427125f, 0.785694958f * 2.828427125f, 0.541196100f * 2.828427125f, 0.275899379f * 2.828427125f }; + + int row, col, i, k, subsample; + float fdtbl_Y[64], fdtbl_UV[64]; + unsigned char YTable[64], UVTable[64]; + + if(!data || !width || !height || comp > 4 || comp < 1) { + return 0; + } + + quality = quality ? quality : 90; + subsample = quality <= 90 ? 1 : 0; + quality = quality < 1 ? 1 : quality > 100 ? 100 : quality; + quality = quality < 50 ? 5000 / quality : 200 - quality * 2; + + for(i = 0; i < 64; ++i) { + int uvti, yti = (YQT[i]*quality+50)/100; + YTable[stbiw__jpg_ZigZag[i]] = (unsigned char) (yti < 1 ? 1 : yti > 255 ? 255 : yti); + uvti = (UVQT[i]*quality+50)/100; + UVTable[stbiw__jpg_ZigZag[i]] = (unsigned char) (uvti < 1 ? 1 : uvti > 255 ? 255 : uvti); + } + + for(row = 0, k = 0; row < 8; ++row) { + for(col = 0; col < 8; ++col, ++k) { + fdtbl_Y[k] = 1 / (YTable [stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]); + fdtbl_UV[k] = 1 / (UVTable[stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]); + } + } + + // Write Headers + { + static const unsigned char head0[] = { 0xFF,0xD8,0xFF,0xE0,0,0x10,'J','F','I','F',0,1,1,0,0,1,0,1,0,0,0xFF,0xDB,0,0x84,0 }; + static const unsigned char head2[] = { 0xFF,0xDA,0,0xC,3,1,0,2,0x11,3,0x11,0,0x3F,0 }; + const unsigned char head1[] = { 0xFF,0xC0,0,0x11,8,(unsigned char)(height>>8),STBIW_UCHAR(height),(unsigned char)(width>>8),STBIW_UCHAR(width), + 3,1,(unsigned char)(subsample?0x22:0x11),0,2,0x11,1,3,0x11,1,0xFF,0xC4,0x01,0xA2,0 }; + s->func(s->context, (void*)head0, sizeof(head0)); + s->func(s->context, (void*)YTable, sizeof(YTable)); + stbiw__putc(s, 1); + s->func(s->context, UVTable, sizeof(UVTable)); + s->func(s->context, (void*)head1, sizeof(head1)); + s->func(s->context, (void*)(std_dc_luminance_nrcodes+1), sizeof(std_dc_luminance_nrcodes)-1); + s->func(s->context, (void*)std_dc_luminance_values, sizeof(std_dc_luminance_values)); + stbiw__putc(s, 0x10); // HTYACinfo + s->func(s->context, (void*)(std_ac_luminance_nrcodes+1), sizeof(std_ac_luminance_nrcodes)-1); + s->func(s->context, (void*)std_ac_luminance_values, sizeof(std_ac_luminance_values)); + stbiw__putc(s, 1); // HTUDCinfo + s->func(s->context, (void*)(std_dc_chrominance_nrcodes+1), sizeof(std_dc_chrominance_nrcodes)-1); + s->func(s->context, (void*)std_dc_chrominance_values, sizeof(std_dc_chrominance_values)); + stbiw__putc(s, 0x11); // HTUACinfo + s->func(s->context, (void*)(std_ac_chrominance_nrcodes+1), sizeof(std_ac_chrominance_nrcodes)-1); + s->func(s->context, (void*)std_ac_chrominance_values, sizeof(std_ac_chrominance_values)); + s->func(s->context, (void*)head2, sizeof(head2)); + } + + // Encode 8x8 macroblocks + { + static const unsigned short fillBits[] = {0x7F, 7}; + int DCY=0, DCU=0, DCV=0; + int bitBuf=0, bitCnt=0; + // comp == 2 is grey+alpha (alpha is ignored) + int ofsG = comp > 2 ? 1 : 0, ofsB = comp > 2 ? 2 : 0; + const unsigned char *dataR = (const unsigned char *)data; + const unsigned char *dataG = dataR + ofsG; + const unsigned char *dataB = dataR + ofsB; + int x, y, pos; + if(subsample) { + for(y = 0; y < height; y += 16) { + for(x = 0; x < width; x += 16) { + float Y[256], U[256], V[256]; + for(row = y, pos = 0; row < y+16; ++row) { + // row >= height => use last input row + int clamped_row = (row < height) ? row : height - 1; + int base_p = (stbi__flip_vertically_on_write ? (height-1-clamped_row) : clamped_row)*width*comp; + for(col = x; col < x+16; ++col, ++pos) { + // if col >= width => use pixel from last input column + int p = base_p + ((col < width) ? col : (width-1))*comp; + float r = dataR[p], g = dataG[p], b = dataB[p]; + Y[pos]= +0.29900f*r + 0.58700f*g + 0.11400f*b - 128; + U[pos]= -0.16874f*r - 0.33126f*g + 0.50000f*b; + V[pos]= +0.50000f*r - 0.41869f*g - 0.08131f*b; + } + } + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+0, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+8, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+128, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+136, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + + // subsample U,V + { + float subU[64], subV[64]; + int yy, xx; + for(yy = 0, pos = 0; yy < 8; ++yy) { + for(xx = 0; xx < 8; ++xx, ++pos) { + int j = yy*32+xx*2; + subU[pos] = (U[j+0] + U[j+1] + U[j+16] + U[j+17]) * 0.25f; + subV[pos] = (V[j+0] + V[j+1] + V[j+16] + V[j+17]) * 0.25f; + } + } + DCU = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, subU, 8, fdtbl_UV, DCU, UVDC_HT, UVAC_HT); + DCV = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, subV, 8, fdtbl_UV, DCV, UVDC_HT, UVAC_HT); + } + } + } + } else { + for(y = 0; y < height; y += 8) { + for(x = 0; x < width; x += 8) { + float Y[64], U[64], V[64]; + for(row = y, pos = 0; row < y+8; ++row) { + // row >= height => use last input row + int clamped_row = (row < height) ? row : height - 1; + int base_p = (stbi__flip_vertically_on_write ? (height-1-clamped_row) : clamped_row)*width*comp; + for(col = x; col < x+8; ++col, ++pos) { + // if col >= width => use pixel from last input column + int p = base_p + ((col < width) ? col : (width-1))*comp; + float r = dataR[p], g = dataG[p], b = dataB[p]; + Y[pos]= +0.29900f*r + 0.58700f*g + 0.11400f*b - 128; + U[pos]= -0.16874f*r - 0.33126f*g + 0.50000f*b; + V[pos]= +0.50000f*r - 0.41869f*g - 0.08131f*b; + } + } + + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y, 8, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCU = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, U, 8, fdtbl_UV, DCU, UVDC_HT, UVAC_HT); + DCV = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, V, 8, fdtbl_UV, DCV, UVDC_HT, UVAC_HT); + } + } + } + + // Do the bit alignment of the EOI marker + stbiw__jpg_writeBits(s, &bitBuf, &bitCnt, fillBits); + } + + // EOI + stbiw__putc(s, 0xFF); + stbiw__putc(s, 0xD9); + + return 1; +} + +STBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality) +{ + stbi__write_context s = { 0 }; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_jpg_core(&s, x, y, comp, (void *) data, quality); +} + + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void *data, int quality) +{ + stbi__write_context s = { 0 }; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_jpg_core(&s, x, y, comp, data, quality); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif + +#endif // STB_IMAGE_WRITE_IMPLEMENTATION + +/* Revision history + 1.16 (2021-07-11) + make Deflate code emit uncompressed blocks when it would otherwise expand + support writing BMPs with alpha channel + 1.15 (2020-07-13) unknown + 1.14 (2020-02-02) updated JPEG writer to downsample chroma channels + 1.13 + 1.12 + 1.11 (2019-08-11) + + 1.10 (2019-02-07) + support utf8 filenames in Windows; fix warnings and platform ifdefs + 1.09 (2018-02-11) + fix typo in zlib quality API, improve STB_I_W_STATIC in C++ + 1.08 (2018-01-29) + add stbi__flip_vertically_on_write, external zlib, zlib quality, choose PNG filter + 1.07 (2017-07-24) + doc fix + 1.06 (2017-07-23) + writing JPEG (using Jon Olick's code) + 1.05 ??? + 1.04 (2017-03-03) + monochrome BMP expansion + 1.03 ??? + 1.02 (2016-04-02) + avoid allocating large structures on the stack + 1.01 (2016-01-16) + STBIW_REALLOC_SIZED: support allocators with no realloc support + avoid race-condition in crc initialization + minor compile issues + 1.00 (2015-09-14) + installable file IO function + 0.99 (2015-09-13) + warning fixes; TGA rle support + 0.98 (2015-04-08) + added STBIW_MALLOC, STBIW_ASSERT etc + 0.97 (2015-01-18) + fixed HDR asserts, rewrote HDR rle logic + 0.96 (2015-01-17) + add HDR output + fix monochrome BMP + 0.95 (2014-08-17) + add monochrome TGA output + 0.94 (2014-05-31) + rename private functions to avoid conflicts with stb_image.h + 0.93 (2014-05-27) + warning fixes + 0.92 (2010-08-01) + casts to unsigned char to fix warnings + 0.91 (2010-07-17) + first public release + 0.90 first internal release +*/ + +/* +------------------------------------------------------------------------------ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2017 Sean Barrett +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------ +*/ diff --git a/examples/common/stb_impl.cpp b/examples/common/stb_impl.cpp new file mode 100644 index 0000000..e3b5482 --- /dev/null +++ b/examples/common/stb_impl.cpp @@ -0,0 +1,5 @@ +// Single translation unit that emits the stb implementations (linked into each example). +#define STB_IMAGE_IMPLEMENTATION +#define STB_IMAGE_WRITE_IMPLEMENTATION +#include "stb_image.h" +#include "stb_image_write.h" diff --git a/examples/detection/main.cpp b/examples/detection/main.cpp new file mode 100644 index 0000000..9abba04 --- /dev/null +++ b/examples/detection/main.cpp @@ -0,0 +1,156 @@ +// Object detection with YOLOv8 + tensorrt_cpp_api. +// +// Pipeline: decode (stb) -> upload -> letterbox/normalize/NCHW (preproc) -> infer -> +// decode the [1, 84, 8400] anchor-free output -> class-aware NMS -> draw boxes. +// +// The model is a stock ultralytics export: yolo export model=yolov8n.pt format=onnx imgsz=640 +// Output layout: 84 = 4 box (cx,cy,w,h, in 640-input pixels) + 80 COCO class scores (already +// sigmoid-activated); 8400 anchors. Box coords/scores need no extra activation -- only NMS. +// +// Usage: detection [conf] [out.jpg] + +#include +#include +#include +#include +#include + +#include "../common/image_io.h" + +using namespace trtcpp; + +namespace { + +constexpr int kInput = 640; +constexpr int kNumClasses = 80; + +const std::array kCoco = { + "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", + "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", + "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", + "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", + "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", + "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", + "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", + "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", + "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", + "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", + "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", + "teddy bear", "hair drier", "toothbrush"}; + +struct Detection { + int x0, y0, x1, y1; + float score; + int cls; +}; + +float iou(const Detection &a, const Detection &b) { + const int ix0 = std::max(a.x0, b.x0), iy0 = std::max(a.y0, b.y0); + const int ix1 = std::min(a.x1, b.x1), iy1 = std::min(a.y1, b.y1); + const float inter = static_cast(std::max(0, ix1 - ix0)) * std::max(0, iy1 - iy0); + const float areaA = static_cast(a.x1 - a.x0) * (a.y1 - a.y0); + const float areaB = static_cast(b.x1 - b.x0) * (b.y1 - b.y0); + const float uni = areaA + areaB - inter; + return uni > 0 ? inter / uni : 0.f; +} + +// Greedy class-aware NMS. +std::vector nms(std::vector dets, float iouThresh) { + std::sort(dets.begin(), dets.end(), [](const Detection &a, const Detection &b) { return a.score > b.score; }); + std::vector keep; + std::vector removed(dets.size(), false); + for (std::size_t i = 0; i < dets.size(); ++i) { + if (removed[i]) { + continue; + } + keep.push_back(dets[i]); + for (std::size_t j = i + 1; j < dets.size(); ++j) { + if (!removed[j] && dets[j].cls == dets[i].cls && iou(dets[i], dets[j]) > iouThresh) { + removed[j] = true; + } + } + } + return keep; +} + +} // namespace + +int main(int argc, char **argv) { + if (argc < 3) { + std::fprintf(stderr, "usage: %s [conf=0.25] [out.jpg]\n", argv[0]); + return 2; + } + const std::string modelPath = argv[1]; + const std::string imagePath = argv[2]; + const float conf = argc > 3 ? static_cast(std::atof(argv[3])) : 0.25f; + const std::string outPath = argc > 4 ? argv[4] : "detections.jpg"; + + BuildOptions bo; + bo.precision = Precision::kFp16; + bo.engineCacheDir = "engines"; + auto engine = EngineBuilder{}.buildAndLoad(modelPath, bo); + if (!engine) { + std::fprintf(stderr, "engine: %s\n", engine.status().message().c_str()); + return 1; + } + + examples::Image img = examples::decodeImage(imagePath); + if (img.empty()) { + std::fprintf(stderr, "could not read image: %s\n", imagePath.c_str()); + return 1; + } + + // --- inference (the ergonomic core) --- + Stream stream; + auto src = examples::uploadHWC(img, stream).value(); + auto dst = Tensor::allocate(DType::kFloat32, Shape{1, 3, kInput, kInput}, Device::kCuda).value(); + preproc::PreprocSpec spec; // letterbox: keep aspect, pad right/bottom, /255 (stb is already RGB) + spec.keepAspectRatioPad = true; + spec.scale = {1.f / 255.f, 1.f / 255.f, 1.f / 255.f, 1.f}; + if (auto s = preproc::letterboxToTensor(src.view(), dst.view(), spec, stream); !s) { + std::fprintf(stderr, "preproc: %s\n", s.message().c_str()); + return 1; + } + auto out = engine->inferSingle({{engine->inputNames().front(), dst.view()}}, stream); + if (!out) { + std::fprintf(stderr, "infer: %s\n", out.status().message().c_str()); + return 1; + } + auto host = out->toHost(stream).value(); + auto data = host.as().value(); // [1, 84, 8400], channels-first + // --- end inference core --- + + const int nAnchors = static_cast(host.shape()[2]); // 8400 + const float r = std::min(static_cast(kInput) / img.width, static_cast(kInput) / img.height); + + std::vector dets; + for (int a = 0; a < nAnchors; ++a) { + int bestCls = 0; + float bestScore = 0.f; + for (int c = 0; c < kNumClasses; ++c) { + const float s = data[(4 + c) * nAnchors + a]; + if (s > bestScore) { + bestScore = s; + bestCls = c; + } + } + if (bestScore < conf) { + continue; + } + const float cx = data[0 * nAnchors + a], cy = data[1 * nAnchors + a]; + const float w = data[2 * nAnchors + a], h = data[3 * nAnchors + a]; + // 640-input pixels -> original pixels (letterbox is top-left aligned, pad right/bottom) + dets.push_back({static_cast((cx - w / 2) / r), static_cast((cy - h / 2) / r), static_cast((cx + w / 2) / r), + static_cast((cy + h / 2) / r), bestScore, bestCls}); + } + + const auto kept = nms(std::move(dets), 0.45f); + std::printf("%zu detections (conf >= %.2f):\n", kept.size(), conf); + for (const auto &d : kept) { + std::printf(" %-14s %5.1f%% [%d,%d %dx%d]\n", kCoco[d.cls], 100.f * d.score, d.x0, d.y0, d.x1 - d.x0, d.y1 - d.y0); + examples::drawRect(img, d.x0, d.y0, d.x1, d.y1, 0, 220, 0, 3); + } + examples::writeJpg(outPath, img); + std::printf("wrote %s\n", outPath.c_str()); + return 0; +} diff --git a/examples/download_models.sh b/examples/download_models.sh new file mode 100755 index 0000000..0f08f81 --- /dev/null +++ b/examples/download_models.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Fetch/export the ONNX models the reference examples use, into ../models/. +# classification : MobileNetV2 (ONNX Model Zoo) -> mobilenetv2-7.onnx +# detection : YOLOv8n (ultralytics export) -> yolov8n.onnx +# segmentation : DeepLabV3-MobileNetV3 (torchvision export)-> deeplabv3_mobilenetv3.onnx +# +# Classification needs only curl. Detection/segmentation need Python with `ultralytics` and +# `torch`/`torchvision` respectively (CPU is fine): +# pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu +# pip install ultralytics onnx onnxscript +set -euo pipefail +cd "$(dirname "$0")/../models" + +echo "[1/3] classification: mobilenetv2-7.onnx" +if [ ! -f mobilenetv2-7.onnx ]; then + curl -fL -o mobilenetv2-7.onnx \ + "https://media.githubusercontent.com/media/onnx/models/main/validated/vision/classification/mobilenet/model/mobilenetv2-7.onnx" +fi + +echo "[2/3] detection: yolov8n.onnx" +if [ ! -f yolov8n.onnx ]; then + if command -v yolo >/dev/null 2>&1; then + yolo export model=yolov8n.pt format=onnx opset=12 imgsz=640 + else + echo " ! 'yolo' (ultralytics) not found. Install it, then:" + echo " yolo export model=yolov8n.pt format=onnx opset=12 imgsz=640" + fi +fi + +echo "[3/3] segmentation: deeplabv3_mobilenetv3.onnx" +if [ ! -f deeplabv3_mobilenetv3.onnx ]; then + python3 - <<'PY' || echo " ! torchvision export failed; install torch/torchvision/onnx/onnxscript" +import torch, torchvision +m = torchvision.models.segmentation.deeplabv3_mobilenet_v3_large(weights="DEFAULT").eval() +class Wrap(torch.nn.Module): + def __init__(self, inner): super().__init__(); self.inner = inner + def forward(self, x): return self.inner(x)["out"] +torch.onnx.export(Wrap(m), torch.randn(1, 3, 520, 520), "deeplabv3_mobilenetv3.onnx", + opset_version=12, input_names=["input"], output_names=["out"], dynamo=False) +print(" exported deeplabv3_mobilenetv3.onnx") +PY +fi + +echo "done. models in $(pwd):" +ls -1 *.onnx 2>/dev/null || true diff --git a/examples/python/README.md b/examples/python/README.md new file mode 100644 index 0000000..7c0c27c --- /dev/null +++ b/examples/python/README.md @@ -0,0 +1,33 @@ +# Python examples + +Zero-copy GPU inference with the `trtcpp` bindings and [CuPy](https://cupy.dev/). + +```sh +pip install cupy-cuda12x # match your CUDA major +pip install . # build/install the trtcpp wheel (from the repo root) +``` + +## `classify_zerocopy.py` — zero-copy in and out + +```sh +python classify_zerocopy.py ../../models/mobilenetv2-7.onnx +``` + +A CuPy array on the GPU is fed to the engine with **no host↔device copy** (via +`__cuda_array_interface__`), inference runs on CuPy's current stream with the GIL released, and +the output `Tensor` is wrapped back into a CuPy array (again zero-copy) for a GPU-side softmax. +Only the final five class indices are ever moved to the host. + +The contract: the library **never implicitly synchronizes or copies to host**. Keep your input +arrays alive and call `stream.synchronize()` before reading outputs. + +## `benchmark_parity.py` — Python vs C++ + +```sh +python benchmark_parity.py ../../models/mobilenetv2-7.onnx ../../build/examples/benchmark 300 +``` + +Times the same preallocated zero-copy `enqueue` loop from Python and from the C++ `benchmark` +example, and asserts the Python path is within 20% of C++. The Python overhead is just one GIL +release/acquire plus a dict→`TensorView` build per call — negligible next to the GPU work. +Measured on an RTX 3080 (MobileNetV2, FP16): C++ ≈ 0.28 ms/infer, Python ≈ 0.32 ms (≈ +13%). diff --git a/examples/python/benchmark_parity.py b/examples/python/benchmark_parity.py new file mode 100644 index 0000000..00b13ec --- /dev/null +++ b/examples/python/benchmark_parity.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Perf-parity check: Python zero-copy inference vs the C++ baseline. + +Times the same zero-copy enqueue loop (preallocated device IO, per-iteration stream sync) from +Python and compares against the C++ `benchmark` example on the identical engine. Asserts the +Python path is within a tolerance (default 20%) of C++ -- i.e. the binding overhead (one GIL +release/acquire + a dict->TensorView build per call) is negligible next to the GPU work. + + python benchmark_parity.py [iters=200] [tol=0.20] +""" +import subprocess +import sys +import time + +import cupy as cp + +import trtcpp + + +def main() -> int: + if len(sys.argv) < 3: + print(f"usage: {sys.argv[0]} [iters=200] [tol=0.20]", file=sys.stderr) + return 2 + model_path = sys.argv[1] + cpp_bin = sys.argv[2] + iters = int(sys.argv[3]) if len(sys.argv) > 3 else 200 + tol = float(sys.argv[4]) if len(sys.argv) > 4 else 0.20 + + bo = trtcpp.BuildOptions() + bo.precision = trtcpp.Precision.Fp16 + bo.engine_cache_dir = "engines" + engine = trtcpp.EngineBuilder().build_and_load(model_path, bo) + + in_name = engine.input_names()[0] + out_name = engine.output_names()[0] + in_dims = engine.tensor_shape(in_name).dims() + out_dims = engine.tensor_shape(out_name).dims() + + stream = trtcpp.Stream.wrap(cp.cuda.get_current_stream().ptr) + gpu_in = cp.zeros(in_dims, dtype=cp.float32) + # Preallocated output Tensor reused across iterations (zero-copy enqueue, mirrors the C++ bench). + out = trtcpp.Tensor.allocate(trtcpp.DType.Float32, trtcpp.Shape(out_dims), trtcpp.Device.Cuda) + inputs = {in_name: gpu_in} + outputs = {out_name: out} + + def once(): + engine.enqueue(inputs, outputs, stream) + stream.synchronize() + + for _ in range(20): # warmup + once() + t0 = time.perf_counter() + for _ in range(iters): + once() + py_ms = (time.perf_counter() - t0) * 1000.0 / iters + + # C++ baseline. + res = subprocess.run([cpp_bin, model_path, str(iters)], capture_output=True, text=True) + cpp_ms = None + for line in res.stdout.splitlines(): + if line.startswith("cpp_latency_ms"): + cpp_ms = float(line.split()[1]) + if cpp_ms is None: + print("could not parse C++ benchmark output:\n" + res.stdout + res.stderr, file=sys.stderr) + return 1 + + overhead = (py_ms - cpp_ms) / cpp_ms + print(f"C++ : {cpp_ms:.3f} ms/infer ({1000 / cpp_ms:.0f} inf/s)") + print(f"Python: {py_ms:.3f} ms/infer ({1000 / py_ms:.0f} inf/s)") + print(f"overhead: {overhead * 100:+.1f}% (tolerance +-{tol * 100:.0f}%)") + if abs(overhead) <= tol: + print("PARITY OK") + return 0 + print("PARITY FAIL") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/python/classify_zerocopy.py b/examples/python/classify_zerocopy.py new file mode 100644 index 0000000..8d8bde7 --- /dev/null +++ b/examples/python/classify_zerocopy.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Zero-copy GPU inference with trtcpp + CuPy. + +Demonstrates the binding's zero-copy contract end to end: a CuPy array on the GPU is fed to the +engine with no host<->device copy (via __cuda_array_interface__), inference runs on the caller's +CUDA stream with the GIL released, and the output Tensor is wrapped back into a CuPy array (again +zero-copy) for GPU-side post-processing. The only host transfer is the final tiny top-5 result. + + python classify_zerocopy.py [image.npy] + +If an image .npy ([3,H,W] or [1,3,H,W] float32, ImageNet-normalized) is given it is used; +otherwise a random input is generated. (Decoding/normalizing an image file is exactly what the +preproc sublib + the C++ examples do; this script focuses on the zero-copy Python plumbing.) +""" +import sys + +import cupy as cp +import numpy as np + +import trtcpp + + +def main() -> int: + if len(sys.argv) < 2: + print(f"usage: {sys.argv[0]} [image.npy]", file=sys.stderr) + return 2 + model_path = sys.argv[1] + + bo = trtcpp.BuildOptions() + bo.precision = trtcpp.Precision.Fp16 + bo.engine_cache_dir = "engines" + engine = trtcpp.EngineBuilder().build_and_load(model_path, bo) + + in_name = engine.input_names()[0] + _, _, in_h, in_w = engine.tensor_shape(in_name).dims() + + # Build the input as a CuPy array ON THE GPU (no host buffer in the hot path). + if len(sys.argv) > 2: + arr = np.load(sys.argv[2]).astype(np.float32).reshape(1, 3, in_h, in_w) + gpu_input = cp.asarray(arr) + else: + gpu_input = cp.random.randn(1, 3, in_h, in_w, dtype=cp.float32) + + # Adopt CuPy's current stream so trtcpp and CuPy order work on the same stream. + stream = trtcpp.Stream.wrap(cp.cuda.get_current_stream().ptr) + + # Zero-copy in: gpu_input exposes __cuda_array_interface__, so the engine binds its device + # pointer directly -- no copy. GIL released during inference. + out = engine.infer_single({in_name: gpu_input}, stream) + + # Zero-copy out: wrap the output Tensor as a CuPy array (shares device memory), then do the + # softmax/top-k on the GPU. The output Tensor must stay alive while gpu_out aliases it. + gpu_out = cp.asarray(out).reshape(-1) + stream.synchronize() # the engine never implicitly syncs; do it before reading + probs = cp.exp(gpu_out - gpu_out.max()) + probs /= probs.sum() + topk = cp.asnumpy(cp.argsort(probs)[::-1][:5]) # only the 5 indices come back to the host + + print(f"input {gpu_input.shape} on {gpu_input.device}, zero-copy in/out") + print("top-5:") + for cls in topk: + print(f" class {int(cls):4d} {float(probs[int(cls)]) * 100:6.2f}%") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/segmentation/main.cpp b/examples/segmentation/main.cpp new file mode 100644 index 0000000..6dfe08e --- /dev/null +++ b/examples/segmentation/main.cpp @@ -0,0 +1,134 @@ +// Semantic segmentation with DeepLabV3-MobileNetV3 + tensorrt_cpp_api. +// +// Pipeline: decode (stb) -> upload -> resize/normalize/NCHW (preproc) -> infer -> +// per-pixel argmax over the [1, 21, H, W] class logits -> colorize (Pascal VOC palette) -> +// blend with the input -> write. +// +// Export the model with torchvision (fixed 520x520): +// deeplabv3_mobilenet_v3_large(weights="DEFAULT"); torch.onnx.export(..., 1x3x520x520) +// +// Usage: segmentation [out.jpg] + +#include +#include +#include +#include +#include + +#include "../common/image_io.h" + +using namespace trtcpp; + +namespace { + +struct Rgb { + std::uint8_t r, g, b; +}; + +// Standard Pascal VOC colormap (the bit-reversal palette torchvision/VOC use). +std::array vocPalette() { + std::array palette{}; + for (int i = 0; i < 256; ++i) { + int r = 0, g = 0, b = 0, c = i; + for (int j = 0; j < 8; ++j) { + r |= ((c >> 0) & 1) << (7 - j); + g |= ((c >> 1) & 1) << (7 - j); + b |= ((c >> 2) & 1) << (7 - j); + c >>= 3; + } + palette[static_cast(i)] = {static_cast(r), static_cast(g), static_cast(b)}; + } + return palette; +} + +} // namespace + +int main(int argc, char **argv) { + if (argc < 3) { + std::fprintf(stderr, "usage: %s [out.jpg]\n", argv[0]); + return 2; + } + const std::string modelPath = argv[1]; + const std::string imagePath = argv[2]; + const std::string outPath = argc > 3 ? argv[3] : "segmentation.jpg"; + + BuildOptions bo; + bo.precision = Precision::kFp16; + bo.engineCacheDir = "engines"; + auto engine = EngineBuilder{}.buildAndLoad(modelPath, bo); + if (!engine) { + std::fprintf(stderr, "engine: %s\n", engine.status().message().c_str()); + return 1; + } + const std::string inName = engine->inputNames().front(); + auto inShape = engine->tensorShape(inName).value(); // [1,3,H,W] + const int inH = static_cast(inShape[2]); + const int inW = static_cast(inShape[3]); + + examples::Image img = examples::decodeImage(imagePath); + if (img.empty()) { + std::fprintf(stderr, "could not read image: %s\n", imagePath.c_str()); + return 1; + } + + Stream stream; + auto src = examples::uploadHWC(img, stream).value(); + auto dst = Tensor::allocate(DType::kFloat32, Shape{1, 3, inH, inW}, Device::kCuda).value(); + if (auto s = preproc::letterboxToTensor(src.view(), dst.view(), examples::imagenetSpec(), stream); !s) { + std::fprintf(stderr, "preproc: %s\n", s.message().c_str()); + return 1; + } + auto out = engine->inferSingle({{inName, dst.view()}}, stream); + if (!out) { + std::fprintf(stderr, "infer: %s\n", out.status().message().c_str()); + return 1; + } + auto host = out->toHost(stream).value(); + auto logits = host.as().value(); // [1, C, H, W], channels-first + const int C = static_cast(host.shape()[1]); + const int H = static_cast(host.shape()[2]); + const int W = static_cast(host.shape()[3]); + const int plane = H * W; + + // per-pixel argmax over the C class logits + std::vector classMap(static_cast(plane)); + std::vector hist(static_cast(C), 0); + for (int p = 0; p < plane; ++p) { + int best = 0; + float bestVal = logits[p]; + for (int c = 1; c < C; ++c) { + const float v = logits[static_cast(c) * plane + p]; + if (v > bestVal) { + bestVal = v; + best = c; + } + } + classMap[static_cast(p)] = static_cast(best); + ++hist[static_cast(best)]; + } + + // colorize at network resolution, nearest-neighbor resize to the original, blend 50/50 + const auto palette = vocPalette(); + examples::Image overlay = img; // copy; we blend in place + for (int y = 0; y < img.height; ++y) { + const int sy = y * H / img.height; + for (int x = 0; x < img.width; ++x) { + const int sx = x * W / img.width; + const Rgb col = palette[classMap[static_cast(sy) * W + sx]]; + auto *px = &overlay.data[(static_cast(y) * img.width + x) * 3]; + px[0] = static_cast((px[0] + col.r) / 2); + px[1] = static_cast((px[1] + col.g) / 2); + px[2] = static_cast((px[2] + col.b) / 2); + } + } + examples::writeJpg(outPath, overlay); + + std::printf("segmented %dx%d image into %d classes; present classes:\n", img.width, img.height, C); + for (int c = 0; c < C; ++c) { + if (hist[static_cast(c)] > 0) { + std::printf(" class %2d : %5.1f%% of pixels\n", c, 100.0 * hist[static_cast(c)] / plane); + } + } + std::printf("wrote %s\n", outPath.c_str()); + return 0; +} diff --git a/include/Int8Calibrator.h b/include/Int8Calibrator.h deleted file mode 100644 index 6ab0ac4..0000000 --- a/include/Int8Calibrator.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once -#include "NvInfer.h" - -// Class used for int8 calibration -class Int8EntropyCalibrator2 : public nvinfer1::IInt8EntropyCalibrator2 { -public: - Int8EntropyCalibrator2(int32_t batchSize, int32_t inputW, int32_t inputH, const std::string &calibDataDirPath, - const std::string &calibTableName, const std::string &inputBlobName, - const std::array &subVals = {0.f, 0.f, 0.f}, const std::array &divVals = {1.f, 1.f, 1.f}, - bool normalize = true, bool readCache = true); - virtual ~Int8EntropyCalibrator2(); - // Abstract base class methods which must be implemented - int32_t getBatchSize() const noexcept override; - bool getBatch(void *bindings[], char const *names[], int32_t nbBindings) noexcept override; - void const *readCalibrationCache(std::size_t &length) noexcept override; - void writeCalibrationCache(void const *ptr, std::size_t length) noexcept override; - -private: - const int32_t m_batchSize; - const int32_t m_inputW; - const int32_t m_inputH; - int32_t m_imgIdx; - std::vector m_imgPaths; - size_t m_inputCount; - const std::string m_calibTableName; - const std::string m_inputBlobName; - const std::array m_subVals; - const std::array m_divVals; - const bool m_normalize; - const bool m_readCache; - void *m_deviceInput; - std::vector m_calibCache; -}; \ No newline at end of file diff --git a/include/engine/EngineBuildLoadNetwork.inl b/include/engine/EngineBuildLoadNetwork.inl deleted file mode 100644 index 73c9a30..0000000 --- a/include/engine/EngineBuildLoadNetwork.inl +++ /dev/null @@ -1,396 +0,0 @@ -#pragma once -#include -#include -#include "util/Util.h" - -template -bool Engine::buildLoadNetwork(std::string onnxModelPath, const std::array &subVals, const std::array &divVals, - bool normalize) { - const auto engineName = serializeEngineOptions(m_options, onnxModelPath); - const auto engineDir = std::filesystem::path(m_options.engineFileDir); - std::filesystem::path enginePath = engineDir / engineName; - spdlog::info("Searching for engine file with name: {}", enginePath.string()); - - if (Util::doesFileExist(enginePath)) { - spdlog::info("Engine found, not regenerating..."); - } else { - if (!Util::doesFileExist(onnxModelPath)) { - auto msg = "Could not find ONNX model at path: " + onnxModelPath; - spdlog::error(msg); - throw std::runtime_error(msg); - } - - spdlog::info("Engine not found, generating. This could take a while..."); - if (!std::filesystem::exists(engineDir)) { - std::filesystem::create_directories(engineDir); - spdlog::info("Created directory: {}", engineDir.string()); - } - - auto ret = build(onnxModelPath, subVals, divVals, normalize); - if (!ret) { - return false; - } - } - - return loadNetwork(enginePath, subVals, divVals, normalize); -} - -template -bool Engine::loadNetwork(std::string trtModelPath, const std::array &subVals, const std::array &divVals, - bool normalize) { - m_subVals = subVals; - m_divVals = divVals; - m_normalize = normalize; - - // Read the serialized model from disk - if (!Util::doesFileExist(trtModelPath)) { - auto msg = "Error, unable to read TensorRT model at path: " + trtModelPath; - spdlog::error(msg); - return false; - } else { - auto msg = "Loading TensorRT engine file at path: " + trtModelPath; - spdlog::info(msg); - } - - std::ifstream file(trtModelPath, std::ios::binary | std::ios::ate); - std::streamsize size = file.tellg(); - file.seekg(0, std::ios::beg); - - std::vector buffer(size); - if (!file.read(buffer.data(), size)) { - auto msg = "Error, unable to read engine file"; - spdlog::error(msg); - throw std::runtime_error(msg); - } - - // Create a runtime to deserialize the engine file. - m_runtime = std::unique_ptr{nvinfer1::createInferRuntime(m_logger)}; - if (!m_runtime) { - return false; - } - - // Set the device index - auto ret = cudaSetDevice(m_options.deviceIndex); - if (ret != 0) { - int numGPUs; - cudaGetDeviceCount(&numGPUs); - auto errMsg = "Unable to set GPU device index to: " + std::to_string(m_options.deviceIndex) + ". Note, your device has " + - std::to_string(numGPUs) + " CUDA-capable GPU(s)."; - spdlog::error(errMsg); - throw std::runtime_error(errMsg); - } - - // Create an engine, a representation of the optimized model. - m_engine = std::unique_ptr(m_runtime->deserializeCudaEngine(buffer.data(), buffer.size())); - if (!m_engine) { - return false; - } - - // The execution context contains all of the state associated with a - // particular invocation - m_context = std::unique_ptr(m_engine->createExecutionContext()); - if (!m_context) { - return false; - } - - // Storage for holding the input and output buffers - // This will be passed to TensorRT for inference - clearGpuBuffers(); - m_buffers.resize(m_engine->getNbIOTensors()); - - m_outputLengths.clear(); - m_inputDims.clear(); - m_outputDims.clear(); - m_IOTensorNames.clear(); - - // Create a cuda stream - cudaStream_t stream; - Util::checkCudaErrorCode(cudaStreamCreate(&stream)); - - // Allocate GPU memory for input and output buffers - m_outputLengths.clear(); - for (int i = 0; i < m_engine->getNbIOTensors(); ++i) { - const auto tensorName = m_engine->getIOTensorName(i); - m_IOTensorNames.emplace_back(tensorName); - const auto tensorType = m_engine->getTensorIOMode(tensorName); - const auto tensorShape = m_engine->getTensorShape(tensorName); - const auto tensorDataType = m_engine->getTensorDataType(tensorName); - - if (tensorType == nvinfer1::TensorIOMode::kINPUT) { - // The implementation currently only supports inputs of type float - if (m_engine->getTensorDataType(tensorName) != nvinfer1::DataType::kFLOAT) { - auto msg = "Error, the implementation currently only supports float inputs"; - spdlog::error(msg); - throw std::runtime_error(msg); - } - - // Don't need to allocate memory for inputs as we will be using the OpenCV - // GpuMat buffer directly. - - // Store the input dims for later use - m_inputDims.emplace_back(tensorShape.d[1], tensorShape.d[2], tensorShape.d[3]); - m_inputBatchSize = tensorShape.d[0]; - } else if (tensorType == nvinfer1::TensorIOMode::kOUTPUT) { - // Ensure the model output data type matches the template argument - // specified by the user - if (tensorDataType == nvinfer1::DataType::kFLOAT && !std::is_same::value) { - auto msg = "Error, the model has expected output of type float. Engine class template parameter must be adjusted."; - spdlog::error(msg); - throw std::runtime_error(msg); - } else if (tensorDataType == nvinfer1::DataType::kHALF && !std::is_same<__half, T>::value) { - auto msg = "Error, the model has expected output of type __half. Engine class template parameter must be adjusted."; - spdlog::error(msg); - throw std::runtime_error(msg); - } else if (tensorDataType == nvinfer1::DataType::kINT8 && !std::is_same::value) { - auto msg = "Error, the model has expected output of type int8_t. Engine class template parameter must be adjusted."; - spdlog::error(msg); - throw std::runtime_error(msg); - } else if (tensorDataType == nvinfer1::DataType::kINT32 && !std::is_same::value) { - auto msg = "Error, the model has expected output of type int32_t. Engine class template parameter must be adjusted."; - spdlog::error(msg); - throw std::runtime_error(msg); - } else if (tensorDataType == nvinfer1::DataType::kBOOL && !std::is_same::value) { - auto msg = "Error, the model has expected output of type bool. Engine class template parameter must be adjusted."; - spdlog::error(msg); - throw std::runtime_error(msg); - } else if (tensorDataType == nvinfer1::DataType::kUINT8 && !std::is_same::value) { - auto msg = "Error, the model has expected output of type uint8_t. Engine class template parameter must be adjusted."; - spdlog::error(msg); - throw std::runtime_error(msg); - } else if (tensorDataType == nvinfer1::DataType::kFP8) { - auto msg = "Error, the model has expected output of type kFP8. This is not supported by the Engine class."; - spdlog::error(msg); - throw std::runtime_error(msg); - } - - // The binding is an output - uint32_t outputLength = 1; - m_outputDims.push_back(tensorShape); - - for (int j = 1; j < tensorShape.nbDims; ++j) { - // We ignore j = 0 because that is the batch size, and we will take that - // into account when sizing the buffer - outputLength *= tensorShape.d[j]; - } - - m_outputLengths.push_back(outputLength); - // Now size the output buffer appropriately, taking into account the max - // possible batch size (although we could actually end up using less - // memory) - Util::checkCudaErrorCode(cudaMallocAsync(&m_buffers[i], outputLength * m_options.maxBatchSize * sizeof(T), stream)); - } else { - auto msg = "Error, IO Tensor is neither an input or output!"; - spdlog::error(msg); - throw std::runtime_error(msg); - } - } - - // Synchronize and destroy the cuda stream - Util::checkCudaErrorCode(cudaStreamSynchronize(stream)); - Util::checkCudaErrorCode(cudaStreamDestroy(stream)); - - return true; -} - - -template -bool Engine::build(std::string onnxModelPath, const std::array &subVals, const std::array &divVals, bool normalize) { - // Create our engine builder. - auto builder = std::unique_ptr(nvinfer1::createInferBuilder(m_logger)); - if (!builder) { - return false; - } - - // Define an explicit batch size and then create the network (implicit batch - // size is deprecated). More info here: - // https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#explicit-implicit-batch - auto explicitBatch = 1U << static_cast(nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH); - auto network = std::unique_ptr(builder->createNetworkV2(explicitBatch)); - if (!network) { - return false; - } - - // Create a parser for reading the onnx file. - auto parser = std::unique_ptr(nvonnxparser::createParser(*network, m_logger)); - if (!parser) { - return false; - } - - // We are going to first read the onnx file into memory, then pass that buffer - // to the parser. Had our onnx model file been encrypted, this approach would - // allow us to first decrypt the buffer. - std::ifstream file(onnxModelPath, std::ios::binary | std::ios::ate); - std::streamsize size = file.tellg(); - file.seekg(0, std::ios::beg); - - std::vector buffer(size); - if (!file.read(buffer.data(), size)) { - auto msg = "Error, unable to read engine file"; - spdlog::error(msg); - throw std::runtime_error(msg); - } - - // Parse the buffer we read into memory. - auto parsed = parser->parse(buffer.data(), buffer.size()); - if (!parsed) { - return false; - } - - // Ensure that all the inputs have the same batch size - const auto numInputs = network->getNbInputs(); - if (numInputs < 1) { - auto msg = "Error, model needs at least 1 input!"; - spdlog::error(msg); - throw std::runtime_error(msg); - } - const auto input0Batch = network->getInput(0)->getDimensions().d[0]; - for (int32_t i = 1; i < numInputs; ++i) { - if (network->getInput(i)->getDimensions().d[0] != input0Batch) { - auto msg = "Error, the model has multiple inputs, each with differing batch sizes!"; - spdlog::error(msg); - throw std::runtime_error(msg); - } - } - - // Check to see if the model supports dynamic batch size or not - bool doesSupportDynamicBatch = false; - if (input0Batch == -1) { - doesSupportDynamicBatch = true; - spdlog::info("Model supports dynamic batch size"); - } else { - spdlog::info("Model only supports fixed batch size of {}", input0Batch); - // If the model supports a fixed batch size, ensure that the maxBatchSize - // and optBatchSize were set correctly. - if (m_options.optBatchSize != input0Batch || m_options.maxBatchSize != input0Batch) { - auto msg = "Error, model only supports a fixed batch size of " + std::to_string(input0Batch) + - ". Must set Options.optBatchSize and Options.maxBatchSize to 1"; - spdlog::error(msg); - throw std::runtime_error(msg); - } - } - - const auto input3Batch = network->getInput(0)->getDimensions().d[3]; - bool doesSupportDynamicWidth = false; - if (input3Batch == -1) { - doesSupportDynamicWidth = true; - spdlog::info("Model supports dynamic width. Using Options.maxInputWidth, Options.minInputWidth, and Options.optInputWidth to set the input width."); - - // Check that the values of maxInputWidth, minInputWidth, and optInputWidth are valid - if (m_options.maxInputWidth < m_options.minInputWidth || m_options.maxInputWidth < m_options.optInputWidth || - m_options.minInputWidth > m_options.optInputWidth - || m_options.maxInputWidth < 1 || m_options.minInputWidth < 1 || m_options.optInputWidth < 1) { - auto msg = "Error, invalid values for Options.maxInputWidth, Options.minInputWidth, and Options.optInputWidth"; - spdlog::error(msg); - throw std::runtime_error(msg); - } - } - - - auto config = std::unique_ptr(builder->createBuilderConfig()); - if (!config) { - return false; - } - - // Register a single optimization profile - nvinfer1::IOptimizationProfile *optProfile = builder->createOptimizationProfile(); - for (int32_t i = 0; i < numInputs; ++i) { - // Must specify dimensions for all the inputs the model expects. - const auto input = network->getInput(i); - const auto inputName = input->getName(); - const auto inputDims = input->getDimensions(); - int32_t inputC = inputDims.d[1]; - int32_t inputH = inputDims.d[2]; - int32_t inputW = inputDims.d[3]; - - int32_t minInputWidth = std::max(m_options.minInputWidth, inputW); - - // Specify the optimization profile` - if (doesSupportDynamicBatch) { - optProfile->setDimensions(inputName, nvinfer1::OptProfileSelector::kMIN, nvinfer1::Dims4(1, inputC, inputH, minInputWidth)); - } else { - optProfile->setDimensions(inputName, nvinfer1::OptProfileSelector::kMIN, - nvinfer1::Dims4(m_options.optBatchSize, inputC, inputH, minInputWidth)); - } - - if (doesSupportDynamicWidth) { - optProfile->setDimensions(inputName, nvinfer1::OptProfileSelector::kOPT, - nvinfer1::Dims4(m_options.optBatchSize, inputC, inputH, m_options.optInputWidth)); - optProfile->setDimensions(inputName, nvinfer1::OptProfileSelector::kMAX, - nvinfer1::Dims4(m_options.maxBatchSize, inputC, inputH, m_options.maxInputWidth)); - } else { - optProfile->setDimensions(inputName, nvinfer1::OptProfileSelector::kOPT, - nvinfer1::Dims4(m_options.optBatchSize, inputC, inputH, inputW)); - optProfile->setDimensions(inputName, nvinfer1::OptProfileSelector::kMAX, - nvinfer1::Dims4(m_options.maxBatchSize, inputC, inputH, inputW)); - } - } - config->addOptimizationProfile(optProfile); - - // Set the precision level - const auto engineName = serializeEngineOptions(m_options, onnxModelPath); - if (m_options.precision == Precision::FP16) { - // Ensure the GPU supports FP16 inference - if (!builder->platformHasFastFp16()) { - auto msg = "Error: GPU does not support FP16 precision"; - spdlog::error(msg); - throw std::runtime_error(msg); - } - config->setFlag(nvinfer1::BuilderFlag::kFP16); - } else if (m_options.precision == Precision::INT8) { - if (numInputs > 1) { - auto msg = "Error, this implementation currently only supports INT8 " - "quantization for single input models"; - spdlog::error(msg); - throw std::runtime_error(msg); - } - - // Ensure the GPU supports INT8 Quantization - if (!builder->platformHasFastInt8()) { - auto msg = "Error: GPU does not support INT8 precision"; - spdlog::error(msg); - throw std::runtime_error(msg); - } - - // Ensure the user has provided path to calibration data directory - if (m_options.calibrationDataDirectoryPath.empty()) { - auto msg = "Error: If INT8 precision is selected, must provide path to " - "calibration data directory to Engine::build method"; - throw std::runtime_error(msg); - } - - config->setFlag((nvinfer1::BuilderFlag::kINT8)); - - const auto input = network->getInput(0); - const auto inputName = input->getName(); - const auto inputDims = input->getDimensions(); - const auto calibrationFileName = engineName + ".calibration"; - - m_calibrator = std::make_unique(m_options.calibrationBatchSize, inputDims.d[3], inputDims.d[2], - m_options.calibrationDataDirectoryPath, calibrationFileName, inputName, - subVals, divVals, normalize); - config->setInt8Calibrator(m_calibrator.get()); - } - - // CUDA stream used for profiling by the builder. - cudaStream_t profileStream; - Util::checkCudaErrorCode(cudaStreamCreate(&profileStream)); - config->setProfileStream(profileStream); - - // Build the engine - // If this call fails, it is suggested to increase the logger verbosity to - // kVERBOSE and try rebuilding the engine. Doing so will provide you with more - // information on why exactly it is failing. - std::unique_ptr plan{builder->buildSerializedNetwork(*network, *config)}; - if (!plan) { - return false; - } - - // Write the engine to disk - const auto enginePath = std::filesystem::path(m_options.engineFileDir) / engineName; - std::ofstream outfile(enginePath, std::ofstream::binary); - outfile.write(reinterpret_cast(plan->data()), plan->size()); - spdlog::info("Success, saved engine to {}", enginePath.string()); - - Util::checkCudaErrorCode(cudaStreamDestroy(profileStream)); - return true; -} \ No newline at end of file diff --git a/include/engine/EngineRunInference.inl b/include/engine/EngineRunInference.inl deleted file mode 100644 index 4542a42..0000000 --- a/include/engine/EngineRunInference.inl +++ /dev/null @@ -1,132 +0,0 @@ -#pragma once -#include -#include -#include "util/Util.h" - -template -bool Engine::runInference(const std::vector> &inputs, - std::vector>> &featureVectors) { - // First we do some error checking - if (inputs.empty() || inputs[0].empty()) { - spdlog::error("Provided input vector is empty!"); - return false; - } - - const auto numInputs = m_inputDims.size(); - if (inputs.size() != numInputs) { - spdlog::error("Incorrect number of inputs provided!"); - return false; - } - - // Ensure the batch size does not exceed the max - if (inputs[0].size() > static_cast(m_options.maxBatchSize)) { - spdlog::error("===== Error ====="); - spdlog::error("The batch size is larger than the model expects!"); - spdlog::error("Model max batch size: {}", m_options.maxBatchSize); - spdlog::error("Batch size provided to call to runInference: {}", inputs[0].size()); - return false; - } - - // Ensure that if the model has a fixed batch size that is greater than 1, the - // input has the correct length - if (m_inputBatchSize != -1 && inputs[0].size() != static_cast(m_inputBatchSize)) { - spdlog::error("===== Error ====="); - spdlog::error("The batch size is different from what the model expects!"); - spdlog::error("Model batch size: {}", m_inputBatchSize); - spdlog::error("Batch size provided to call to runInference: {}", inputs[0].size()); - return false; - } - - const auto batchSize = static_cast(inputs[0].size()); - // Make sure the same batch size was provided for all inputs - for (size_t i = 1; i < inputs.size(); ++i) { - if (inputs[i].size() != static_cast(batchSize)) { - spdlog::error("===== Error ====="); - spdlog::error("The batch size is different for each input!"); - return false; - } - } - - // Create the cuda stream that will be used for inference - cudaStream_t inferenceCudaStream; - Util::checkCudaErrorCode(cudaStreamCreate(&inferenceCudaStream)); - - std::vector preprocessedInputs; - - // Preprocess all the inputs - for (size_t i = 0; i < numInputs; ++i) { - const auto &batchInput = inputs[i]; - const auto &dims = m_inputDims[i]; - - auto &input = batchInput[0]; - if (input.channels() != dims.d[0] || input.rows != dims.d[1] || input.cols != dims.d[2]) { - spdlog::error("===== Error ====="); - spdlog::error("Input does not have correct size!"); - spdlog::error("Expected: ({}, {}, {})", dims.d[0], dims.d[1], dims.d[2]); - spdlog::error("Got: ({}, {}, {})", input.channels(), input.rows, input.cols); - spdlog::error("Ensure you resize your input image to the correct size"); - return false; - } - - nvinfer1::Dims4 inputDims = {batchSize, dims.d[0], dims.d[1], dims.d[2]}; - m_context->setInputShape(m_IOTensorNames[i].c_str(), - inputDims); // Define the batch size - - // OpenCV reads images into memory in NHWC format, while TensorRT expects - // images in NCHW format. The following method converts NHWC to NCHW. Even - // though TensorRT expects NCHW at IO, during optimization, it can - // internally use NHWC to optimize cuda kernels See: - // https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#data-layout - // Copy over the input data and perform the preprocessing - auto mfloat = blobFromGpuMats(batchInput, m_subVals, m_divVals, m_normalize); - preprocessedInputs.push_back(mfloat); - m_buffers[i] = mfloat.ptr(); - } - - // Ensure all dynamic bindings have been defined. - if (!m_context->allInputDimensionsSpecified()) { - auto msg = "Error, not all required dimensions specified."; - spdlog::error(msg); - throw std::runtime_error(msg); - } - - // Set the address of the input and output buffers - for (size_t i = 0; i < m_buffers.size(); ++i) { - bool status = m_context->setTensorAddress(m_IOTensorNames[i].c_str(), m_buffers[i]); - if (!status) { - return false; - } - } - - // Run inference. - bool status = m_context->enqueueV3(inferenceCudaStream); - if (!status) { - return false; - } - - // Copy the outputs back to CPU - featureVectors.clear(); - - for (int batch = 0; batch < batchSize; ++batch) { - // Batch - std::vector> batchOutputs{}; - for (int32_t outputBinding = numInputs; outputBinding < m_engine->getNbIOTensors(); ++outputBinding) { - // We start at index m_inputDims.size() to account for the inputs in our - // m_buffers - std::vector output; - auto outputLength = m_outputLengths[outputBinding - numInputs]; - output.resize(outputLength); - // Copy the output - Util::checkCudaErrorCode(cudaMemcpyAsync(output.data(), - static_cast(m_buffers[outputBinding]) + (batch * sizeof(T) * outputLength), - outputLength * sizeof(T), cudaMemcpyDeviceToHost, inferenceCudaStream)); - batchOutputs.emplace_back(std::move(output)); - } - featureVectors.emplace_back(std::move(batchOutputs)); - } - - // Synchronize the cuda stream - Util::checkCudaErrorCode(cudaStreamSynchronize(inferenceCudaStream)); - Util::checkCudaErrorCode(cudaStreamDestroy(inferenceCudaStream)); - return true; -} diff --git a/include/engine/EngineUtilities.inl b/include/engine/EngineUtilities.inl deleted file mode 100644 index d469ee6..0000000 --- a/include/engine/EngineUtilities.inl +++ /dev/null @@ -1,143 +0,0 @@ -#pragma once -#include -#include - -template -void Engine::transformOutput(std::vector>> &input, std::vector> &output) { - if (input.size() != 1) { - auto msg = "The feature vector has incorrect dimensions!"; - spdlog::error(msg); - throw std::logic_error(msg); - } - - output = std::move(input[0]); -} - -template void Engine::transformOutput(std::vector>> &input, std::vector &output) { - if (input.size() != 1 || input[0].size() != 1) { - auto msg = "The feature vector has incorrect dimensions!"; - spdlog::error(msg); - throw std::logic_error(msg); - } - - output = std::move(input[0][0]); -} - -template -cv::cuda::GpuMat Engine::resizeKeepAspectRatioPadRightBottom(const cv::cuda::GpuMat &input, size_t height, size_t width, - const cv::Scalar &bgcolor) { - float r = std::min(width / (input.cols * 1.0), height / (input.rows * 1.0)); - int unpad_w = r * input.cols; - int unpad_h = r * input.rows; - cv::cuda::GpuMat re(unpad_h, unpad_w, CV_8UC3); - cv::cuda::resize(input, re, re.size()); - cv::cuda::GpuMat out(height, width, CV_8UC3, bgcolor); - re.copyTo(out(cv::Rect(0, 0, re.cols, re.rows))); - return out; -} - -template void Engine::getDeviceNames(std::vector &deviceNames) { - int numGPUs; - cudaGetDeviceCount(&numGPUs); - - for (int device = 0; device < numGPUs; device++) { - cudaDeviceProp prop; - cudaGetDeviceProperties(&prop, device); - - deviceNames.push_back(std::string(prop.name)); - } -} - -template std::string Engine::serializeEngineOptions(const Options &options, const std::string &onnxModelPath) { - const auto filenamePos = onnxModelPath.find_last_of('/') + 1; - std::string engineName = onnxModelPath.substr(filenamePos, onnxModelPath.find_last_of('.') - filenamePos) + ".engine"; - - // Add the GPU device name to the file to ensure that the model is only used - // on devices with the exact same GPU - std::vector deviceNames; - getDeviceNames(deviceNames); - - if (static_cast(options.deviceIndex) >= deviceNames.size()) { - auto msg = "Error, provided device index is out of range!"; - spdlog::error(msg); - throw std::runtime_error(msg); - } - - auto deviceName = deviceNames[options.deviceIndex]; - // Remove spaces from the device name - deviceName.erase(std::remove_if(deviceName.begin(), deviceName.end(), ::isspace), deviceName.end()); - - engineName += "." + deviceName; - - // Serialize the specified options into the filename - if (options.precision == Precision::FP16) { - engineName += ".fp16"; - } else if (options.precision == Precision::FP32) { - engineName += ".fp32"; - } else { - engineName += ".int8"; - } - - engineName += "." + std::to_string(options.maxBatchSize); - engineName += "." + std::to_string(options.optBatchSize); - engineName += "." + std::to_string(options.minInputWidth); - engineName += "." + std::to_string(options.optInputWidth); - engineName += "." + std::to_string(options.maxInputWidth); - - spdlog::info("Engine name: {}", engineName); - return engineName; -} - -template -cv::cuda::GpuMat Engine::blobFromGpuMats(const std::vector &batchInput, const std::array &subVals, - const std::array &divVals, bool normalize, bool swapRB) { - - CHECK(!batchInput.empty()) - CHECK(batchInput[0].channels() == 3) - - cv::cuda::GpuMat gpu_dst(1, batchInput[0].rows * batchInput[0].cols * batchInput.size(), CV_8UC3); - - size_t width = batchInput[0].cols * batchInput[0].rows; - if (swapRB) { - for (size_t img = 0; img < batchInput.size(); ++img) { - std::vector input_channels{ - cv::cuda::GpuMat(batchInput[0].rows, batchInput[0].cols, CV_8U, &(gpu_dst.ptr()[width * 2 + width * 3 * img])), - cv::cuda::GpuMat(batchInput[0].rows, batchInput[0].cols, CV_8U, &(gpu_dst.ptr()[width + width * 3 * img])), - cv::cuda::GpuMat(batchInput[0].rows, batchInput[0].cols, CV_8U, &(gpu_dst.ptr()[0 + width * 3 * img]))}; - cv::cuda::split(batchInput[img], input_channels); // HWC -> CHW - } - } else { - for (size_t img = 0; img < batchInput.size(); ++img) { - std::vector input_channels{ - cv::cuda::GpuMat(batchInput[0].rows, batchInput[0].cols, CV_8U, &(gpu_dst.ptr()[0 + width * 3 * img])), - cv::cuda::GpuMat(batchInput[0].rows, batchInput[0].cols, CV_8U, &(gpu_dst.ptr()[width + width * 3 * img])), - cv::cuda::GpuMat(batchInput[0].rows, batchInput[0].cols, CV_8U, &(gpu_dst.ptr()[width * 2 + width * 3 * img]))}; - cv::cuda::split(batchInput[img], input_channels); // HWC -> CHW - } - } - cv::cuda::GpuMat mfloat; - if (normalize) { - // [0.f, 1.f] - gpu_dst.convertTo(mfloat, CV_32FC3, 1.f / 255.f); - } else { - // [0.f, 255.f] - gpu_dst.convertTo(mfloat, CV_32FC3); - } - - // Apply scaling and mean subtraction - cv::cuda::subtract(mfloat, cv::Scalar(subVals[0], subVals[1], subVals[2]), mfloat, cv::noArray(), -1); - cv::cuda::divide(mfloat, cv::Scalar(divVals[0], divVals[1], divVals[2]), mfloat, 1, -1); - - return mfloat; -} - -template void Engine::clearGpuBuffers() { - if (!m_buffers.empty()) { - // Free GPU memory of outputs - const auto numInputs = m_inputDims.size(); - for (int32_t outputBinding = numInputs; outputBinding < m_engine->getNbIOTensors(); ++outputBinding) { - Util::checkCudaErrorCode(cudaFree(m_buffers[outputBinding])); - } - m_buffers.clear(); - } -} diff --git a/include/interfaces/IEngine.h b/include/interfaces/IEngine.h deleted file mode 100644 index 06e1730..0000000 --- a/include/interfaces/IEngine.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include -#include -#include -#include "NvInfer.h" // Include for nvinfer1::Dims and nvinfer1::Dims3 - -template -class IEngine { -public: - virtual ~IEngine() = default; - virtual bool buildLoadNetwork(std::string onnxModelPath, const std::array &subVals = {0.f, 0.f, 0.f}, - const std::array &divVals = {1.f, 1.f, 1.f}, bool normalize = true) = 0; - virtual bool loadNetwork(std::string trtModelPath, const std::array &subVals = {0.f, 0.f, 0.f}, - const std::array &divVals = {1.f, 1.f, 1.f}, bool normalize = true) = 0; - virtual bool runInference(const std::vector> &inputs, - std::vector>> &featureVectors) = 0; - virtual const std::vector &getInputDims() const = 0; - virtual const std::vector &getOutputDims() const = 0; -}; diff --git a/include/logger.h b/include/logger.h deleted file mode 100644 index 027ba0b..0000000 --- a/include/logger.h +++ /dev/null @@ -1,75 +0,0 @@ -#pragma once - -#include -#include -#include - -enum class LogLevel { - Trace, - Debug, - Info, - Warn, - Error, - Critical, - Off, - Unknown -}; - - -// Get the log level string from the environment variable -inline std::string getLogLevelFromEnvironment() { - const char* envValue = std::getenv("LOG_LEVEL"); - if (envValue) { - return std::string(envValue); - } else { - spdlog::warn("LOG_LEVEL environment variable not set. Using default log level (info)."); - return "info"; - } -} - -// Convert log level string to LogLevel enum -inline LogLevel parseLogLevel(const std::string& logLevelStr) { - if (logLevelStr == "trace") { - return LogLevel::Trace; - } else if (logLevelStr == "debug") { - return LogLevel::Debug; - } else if (logLevelStr == "info") { - return LogLevel::Info; - } else if (logLevelStr == "warn" || logLevelStr == "warning") { - return LogLevel::Warn; - } else if (logLevelStr == "err" || logLevelStr == "error") { - return LogLevel::Error; - } else if (logLevelStr == "critical") { - return LogLevel::Critical; - } else if (logLevelStr == "off") { - return LogLevel::Off; - } else { - spdlog::warn("Unknown log level string: {}. Defaulting to 'info' level.", logLevelStr); - return LogLevel::Unknown; - } -} - -// Convert LogLevel enum to spdlog::level::level_enum -inline spdlog::level::level_enum toSpdlogLevel(const std::string& logLevelStr) { - LogLevel logLevel = parseLogLevel(logLevelStr); - - switch (logLevel) { - case LogLevel::Trace: - return spdlog::level::trace; - case LogLevel::Debug: - return spdlog::level::debug; - case LogLevel::Info: - return spdlog::level::info; - case LogLevel::Warn: - return spdlog::level::warn; - case LogLevel::Error: - return spdlog::level::err; - case LogLevel::Critical: - return spdlog::level::critical; - case LogLevel::Off: - return spdlog::level::off; - default: - spdlog::warn("Unknown log level. Using default log level (info)."); - return spdlog::level::info; - } -} diff --git a/include/macros.h b/include/macros.h deleted file mode 100644 index 53294d7..0000000 --- a/include/macros.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include - -#define CHECK(condition) \ - do { \ - if (!(condition)) { \ - spdlog::error("Assertion failed: ({}), function {}, file {}, line {}.", #condition, __FUNCTION__, __FILE__, __LINE__); \ - abort(); \ - } \ - } while (false); diff --git a/include/tensorrt_cpp_api/all.h b/include/tensorrt_cpp_api/all.h new file mode 100644 index 0000000..710b7e6 --- /dev/null +++ b/include/tensorrt_cpp_api/all.h @@ -0,0 +1,27 @@ +#pragma once + +// Umbrella header: includes the whole core public API in one shot. The optional sublibrary +// headers (preproc.h, opencv_interop.h) are intentionally NOT pulled in here -- include +// them explicitly when you build/link those modules. + +#include "tensorrt_cpp_api/build_config.h" + +#include "tensorrt_cpp_api/dtype.h" +#include "tensorrt_cpp_api/layout.h" +#include "tensorrt_cpp_api/shape.h" +#include "tensorrt_cpp_api/status.h" +#include "tensorrt_cpp_api/tensor.h" +#include "tensorrt_cpp_api/version.h" + +#include "tensorrt_cpp_api/logger.h" + +#include "tensorrt_cpp_api/allocator.h" +#include "tensorrt_cpp_api/cuda.h" +#include "tensorrt_cpp_api/device_tensor.h" + +#include "tensorrt_cpp_api/build_options.h" +#include "tensorrt_cpp_api/calibrator.h" // self-gates out on TensorRT >= 11 +#include "tensorrt_cpp_api/engine.h" +#include "tensorrt_cpp_api/engine_builder.h" +#include "tensorrt_cpp_api/engine_pool.h" +#include "tensorrt_cpp_api/quant.h" diff --git a/include/tensorrt_cpp_api/allocator.h b/include/tensorrt_cpp_api/allocator.h new file mode 100644 index 0000000..f73a613 --- /dev/null +++ b/include/tensorrt_cpp_api/allocator.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +#include "tensorrt_cpp_api/cuda.h" + +namespace trtcpp { + +/// Device-memory allocator abstraction so a host application can hand TensorRT its own +/// arena. The default (defaultDeviceAllocator) is stream-ordered (cudaMallocAsync) from a +/// private pool. Used by the engine's buffer management; the standalone owning Tensor +/// uses synchronous cudaMalloc since it has no stream at construction. +class IDeviceAllocator { +public: + virtual ~IDeviceAllocator() = default; + /// Returns device memory ordered on `stream`, or nullptr on failure. `alignment` is a + /// hint; the default allocator already satisfies 256-byte alignment. + virtual void *allocate(std::size_t bytes, std::size_t alignment, const Stream &stream) = 0; + /// Frees memory ordered on `stream`. Must be called on a stream after which the memory + /// is no longer used. + virtual void deallocate(void *ptr, const Stream &stream) noexcept = 0; +}; + +/// Stream-ordered allocator backed by a private CUDA memory pool with the release +/// threshold set to UINT64_MAX (freed memory is retained for reuse across iterations +/// rather than returned to the OS on every sync -- the documented best practice). +std::shared_ptr defaultDeviceAllocator(int deviceIndex = 0); + +} // namespace trtcpp diff --git a/include/tensorrt_cpp_api/build_options.h b/include/tensorrt_cpp_api/build_options.h new file mode 100644 index 0000000..4b7082c --- /dev/null +++ b/include/tensorrt_cpp_api/build_options.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "tensorrt_cpp_api/quant.h" +#include "tensorrt_cpp_api/shape.h" + +namespace trtcpp { + +class ICalibrator; // forward-declared (defined in the calibration module, gated < TRT 11) + +/// One input's min/opt/max extents for an optimization profile. Multi-input, +/// multi-dimension, multi-profile -- supports dynamic shapes. +struct ProfileShape { + std::string inputName; + Shape min; + Shape opt; + Shape max; +}; + +/// A complete optimization profile: one ProfileShape per dynamic input. For multi-stream +/// concurrent dynamic-shape inference, build one profile per execution context (EnginePool). +struct OptimizationProfile { + std::vector inputs; +}; + +struct BuildOptions { + Precision precision = Precision::kFp16; + /// Empty => static shapes taken from the ONNX. Otherwise one profile per concurrent + /// context you intend to run. + std::vector profiles; + int deviceIndex = 0; + int dlaCore = -1; ///< -1 = GPU; >= 0 selects a DLA core + std::optional workspaceBytes; ///< setMemoryPoolLimit(kWORKSPACE); unset => TRT default + /// nullopt => auto: true for kInt8Qdq and on TRT >= 11; false for kFp16/kFp32 on + /// TRT < 11 (so the weak-typed precision flag is honored). + std::optional stronglyTyped; + bool versionCompatible = false; ///< kVERSION_COMPATIBLE (relaxes cache staleness) + bool hardwareCompatible = false; ///< Ampere-plus hardware compatibility (relaxes cache staleness) + std::string engineCacheDir = "."; + std::string timingCachePath; ///< empty => engineCacheDir/.timing + std::vector pluginLibraries; + /// kInt8CalibLegacy only; ignored otherwise. Forward-declared and inert on TRT >= 11 + /// so BuildOptions stays ABI-stable across the calibrator-removal boundary. + std::shared_ptr calibrator; +}; + +} // namespace trtcpp diff --git a/include/tensorrt_cpp_api/calibrator.h b/include/tensorrt_cpp_api/calibrator.h new file mode 100644 index 0000000..1b608de --- /dev/null +++ b/include/tensorrt_cpp_api/calibrator.h @@ -0,0 +1,49 @@ +#pragma once + +#include "tensorrt_cpp_api/build_config.h" + +// Legacy INT8 post-training calibration. TensorRT 11 removed the IInt8Calibrator family, so +// this whole header compiles out when the library is built against TRT >= 11. The forward +// path is Precision::kInt8Qdq with a modelopt-quantized ONNX (no calibrator). +#if TRT_CPP_API_TENSORRT_VERSION_MAJOR < 11 + +#include +#include +#include +#include +#include +#include +#include + +#include "tensorrt_cpp_api/cuda.h" +#include "tensorrt_cpp_api/shape.h" +#include "tensorrt_cpp_api/tensor.h" + +namespace trtcpp { + +/// Supplies calibration batches for legacy INT8 PTQ. Implement nextBatch to fill the named +/// input device buffers (views over TensorRT's calibration buffers) with PRE-PROCESSED data +/// -- using the SAME preprocessing as inference, to avoid the v6 calibration/inference +/// mismatch. +class ICalibrator { +public: + virtual ~ICalibrator() = default; + virtual int batchSize() const = 0; + /// Fill `inputs` with the next batch on `stream`; return false when exhausted. + virtual bool nextBatch(const std::unordered_map &inputs, const Stream &stream) = 0; + /// Optional calibration-cache persistence (return nullopt to recompute). + virtual std::optional> readCache() = 0; + virtual void writeCache(std::span) = 0; +}; + +/// A single-input calibrator that streams pre-processed raw batches (contiguous NCHW float, +/// one binary file per batch of shape `batchShape`) from disk -- no image decoding, so it +/// needs neither OpenCV nor the preprocessing sublibrary. (The decode+preprocess +/// "image directory" convenience lives in the OpenCV interop module.) `cachePath`, if set, +/// persists the calibration table. +std::shared_ptr makeRawBatchCalibrator(std::string inputName, Shape batchShape, std::vector batchFiles, + std::string cachePath = {}); + +} // namespace trtcpp + +#endif // TRT_CPP_API_TENSORRT_VERSION_MAJOR < 11 diff --git a/include/tensorrt_cpp_api/cuda.h b/include/tensorrt_cpp_api/cuda.h new file mode 100644 index 0000000..58137bf --- /dev/null +++ b/include/tensorrt_cpp_api/cuda.h @@ -0,0 +1,65 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "tensorrt_cpp_api/status.h" + +// CUDA is a hard dependency of the engine core (caller-provided streams cross the public +// API), so this header includes . OpenCV/spdlog/TensorRT, by contrast, +// never appear in public headers. + +namespace trtcpp { + +/// Convert a CUDA error code into a Status (kOk on cudaSuccess). `context` prefixes the +/// message, e.g. cudaToStatus(err, "cudaMalloc"). +Status cudaToStatus(cudaError_t code, std::string_view context = {}); + +/// RAII CUDA stream. Either OWNS a stream it created, or WRAPS a caller-provided one +/// (non-owning) -- the model that callers asked for and the Python bindings +/// require (an integer handle from torch/cupy). Replaces v6's per-call create/destroy. +/// +/// A default-constructed Stream owns a new non-blocking stream; if creation fails (only +/// under catastrophic OOM) it degrades to the CUDA default stream (non-owning). Use +/// Stream::create() when you need to observe that failure. +class Stream { +public: + Stream(); + static Result create(); ///< owned non-blocking stream, error-checked + static Stream wrap(cudaStream_t existing) noexcept; ///< non-owning + static Stream wrap(std::uintptr_t existing) noexcept; ///< non-owning, for language bindings + ~Stream(); + + Stream(Stream &&other) noexcept; + Stream &operator=(Stream &&other) noexcept; + Stream(const Stream &) = delete; + Stream &operator=(const Stream &) = delete; + + cudaStream_t handle() const noexcept { return stream_; } ///< nullptr == CUDA default stream + std::uintptr_t raw() const noexcept { return reinterpret_cast(stream_); } + bool owns() const noexcept { return owns_; } + Status synchronize() const noexcept; + +private: + Stream(cudaStream_t stream, bool owns) noexcept : stream_(stream), owns_(owns) {} + cudaStream_t stream_ = nullptr; + bool owns_ = false; +}; + +struct DeviceInfo { + int index = 0; + std::string name; + std::string uuid; ///< 32-char hex + int computeMajor = 0; + int computeMinor = 0; + std::size_t totalMemoryBytes = 0; +}; + +Result deviceCount(); +Result queryDevice(int index); + +} // namespace trtcpp diff --git a/include/tensorrt_cpp_api/device_tensor.h b/include/tensorrt_cpp_api/device_tensor.h new file mode 100644 index 0000000..64c955a --- /dev/null +++ b/include/tensorrt_cpp_api/device_tensor.h @@ -0,0 +1,66 @@ +#pragma once + +#include +#include + +#include "tensorrt_cpp_api/cuda.h" +#include "tensorrt_cpp_api/tensor.h" + +namespace trtcpp { + +/// An owning tensor: a RAII device buffer (cudaMalloc) or pinned host buffer +/// (cudaMallocHost). Move-only. Returned when the library allocates outputs for the +/// caller, and the type the Python bindings own. The owning counterpart +/// to the non-owning TensorView. +class Tensor { +public: + Tensor() = default; + ~Tensor(); + + /// Allocate a contiguous buffer for `shape` of `dtype` on the given device. Device + /// memory is synchronous cudaMalloc; host memory is pinned (cudaMallocHost) for fast + /// async transfers. Errors (kInvalidArgument) on a dynamic shape. + static Result allocate(DType dtype, Shape shape, Device device, int deviceId = 0); + + Tensor(Tensor &&other) noexcept; + Tensor &operator=(Tensor &&other) noexcept; + Tensor(const Tensor &) = delete; + Tensor &operator=(const Tensor &) = delete; + + TensorView view() const noexcept; + void *data() const noexcept { return data_; } + DType dtype() const noexcept { return dtype_; } + const Shape &shape() const noexcept { return shape_; } + Device device() const noexcept { return device_; } + int deviceId() const noexcept { return deviceId_; } + bool empty() const noexcept { return data_ == nullptr; } + std::size_t nbytes() const noexcept { return byteSizeOf(dtype_, static_cast(shape_.numel())); } + + /// Async copy from `src` into this tensor on `stream`. The byte size and dtype must + /// match. The transfer direction (H2D/D2H/D2D/H2H) is inferred from the devices. + /// Cross-device (different deviceId, both CUDA) copies are rejected with kUnsupported + /// (v7 targets single-GPU inference; stage through host for multi-GPU). + Status copyFrom(TensorView src, const Stream &stream); + + /// Allocate a new tensor on the target device and async-copy this one into it. + Result to(Device device, int deviceId, const Stream &stream) const; + /// Materialize on the host: copy to a pinned host tensor AND synchronize `stream`, so + /// the result is immediately readable. The common readback path. + Result toHost(const Stream &stream) const; + + /// Typed, dtype-checked host span (errors on a device tensor); pair with toHost(). + template Result> as() const { return view().as(); } + +private: + Tensor(void *data, DType dtype, Shape shape, Device device, int deviceId) noexcept + : data_(data), dtype_(dtype), shape_(std::move(shape)), device_(device), deviceId_(deviceId) {} + void reset() noexcept; + + void *data_ = nullptr; + DType dtype_ = DType::kFloat32; + Shape shape_; + Device device_ = Device::kCuda; + int deviceId_ = 0; +}; + +} // namespace trtcpp diff --git a/include/tensorrt_cpp_api/dtype.h b/include/tensorrt_cpp_api/dtype.h new file mode 100644 index 0000000..fb222c4 --- /dev/null +++ b/include/tensorrt_cpp_api/dtype.h @@ -0,0 +1,84 @@ +#pragma once + +#include +#include +#include + +namespace trtcpp { + +/// Element type at the API boundary. Library-owned so consumers never include NvInfer.h. +/// Mirrors nvinfer1::DataType; kFp8/kInt4 are type tags whose acceleration is gated on +/// recent TensorRT and GPU architecture (validated at engine build time, not here). +enum class DType : std::uint8_t { + kFloat32, + kFloat16, + kBFloat16, + kInt32, + kInt64, + kInt8, + kUInt8, + kBool, + kFp8, + kInt4, +}; + +/// Bit width of a single element. kInt4 is sub-byte (4 bits). +constexpr int bitsPerElement(DType t) noexcept { + switch (t) { + case DType::kFloat32: + case DType::kInt32: + return 32; + case DType::kInt64: + return 64; + case DType::kFloat16: + case DType::kBFloat16: + return 16; + case DType::kInt8: + case DType::kUInt8: + case DType::kBool: + case DType::kFp8: + return 8; + case DType::kInt4: + return 4; + } + return 0; +} + +/// Storage bytes for one element, rounded up. For sub-byte types prefer byteSizeOf, +/// which accounts for packing (kInt4 stores two elements per byte). +constexpr std::size_t byteSize(DType t) noexcept { return static_cast((bitsPerElement(t) + 7) / 8); } + +/// Total bytes to store `count` elements, honoring sub-byte packing. Assumes the result +/// is representable in std::size_t; callers that allocate from untrusted counts must +/// bounds-check first (the engine's buffer allocation does so before cudaMalloc). +constexpr std::size_t byteSizeOf(DType t, std::size_t count) noexcept { + return (static_cast(bitsPerElement(t)) * count + 7) / 8; +} + +std::string_view toString(DType t) noexcept; + +/// Maps a C++ scalar type to its DType so typed accessors can dtype-check. Left +/// undefined for unsupported T (using as() with an unsupported T is a compile error). +/// fp16/bf16/fp8 have no portable dep-free C++ scalar; their specializations are defined by +/// the buffer/interop layer that needs them, in a CUDA-including header. +template struct DTypeOf; +template <> struct DTypeOf { + static constexpr DType value = DType::kFloat32; +}; +template <> struct DTypeOf { + static constexpr DType value = DType::kInt32; +}; +template <> struct DTypeOf { + static constexpr DType value = DType::kInt64; +}; +template <> struct DTypeOf { + static constexpr DType value = DType::kInt8; +}; +template <> struct DTypeOf { + static constexpr DType value = DType::kUInt8; +}; +template <> struct DTypeOf { + static constexpr DType value = DType::kBool; +}; + +} // namespace trtcpp diff --git a/include/tensorrt_cpp_api/engine.h b/include/tensorrt_cpp_api/engine.h new file mode 100644 index 0000000..f156bfa --- /dev/null +++ b/include/tensorrt_cpp_api/engine.h @@ -0,0 +1,84 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "tensorrt_cpp_api/cuda.h" +#include "tensorrt_cpp_api/device_tensor.h" +#include "tensorrt_cpp_api/dtype.h" +#include "tensorrt_cpp_api/logger.h" +#include "tensorrt_cpp_api/shape.h" +#include "tensorrt_cpp_api/status.h" +#include "tensorrt_cpp_api/tensor.h" + +namespace trtcpp { + +struct EngineOptions { + int deviceIndex = 0; + std::shared_ptr logger = defaultLogger(); + std::vector pluginLibraries; ///< dlopen + register (IPluginV3) before deserialize +}; + +/// Metadata for one IO tensor (always name-keyed; never index-keyed). +struct TensorInfo { + std::string name; + bool isInput = false; + DType dtype = DType::kFloat32; + Shape shape; ///< build-time shape; dynamic axes are -1 +}; + +/// A loaded TensorRT engine. Thread-COMPATIBLE, not thread-safe: for concurrent inference +/// use EnginePool. TensorRT types are hidden behind a PImpl, so this header pulls in no +/// nvinfer1. The owning ICudaEngine/IExecutionContext/IRuntime are released on destruction. +class Engine { +public: + Engine(Engine &&) noexcept; + Engine &operator=(Engine &&) noexcept; + Engine(const Engine &) = delete; + Engine &operator=(const Engine &) = delete; + ~Engine(); + + static Result loadFromFile(const std::string &enginePath, const EngineOptions &options = {}); + static Result loadFromMemory(std::span engineData, const EngineOptions &options = {}); + + std::vector tensors() const; + std::vector inputNames() const; + std::vector outputNames() const; + int nbOptimizationProfiles() const; + Result tensorShape(const std::string &name) const; ///< build-time (may be -1 dynamic) + Result tensorDType(const std::string &name) const; + + /// Caller-allocated, zero-copy, no implicit sync. Selects `profileIndex` + /// (setOptimizationProfileAsync on `stream`) BEFORE setInputShape from the input views, + /// then setTensorAddress for every IO tensor + enqueueV3 -- all on `stream`. The caller + /// owns the sync. Out-of-range profileIndex returns kInvalidArgument. + Status enqueue(const std::unordered_map &inputs, const std::unordered_map &outputs, + const Stream &stream, int profileIndex = 0); + + /// Library-allocated convenience: resolves output shapes from the inputs, allocates + /// owning device Tensors sized from getTensorShape AFTER setInputShape (never build-time + /// -1 dims), enqueues, and returns them. No implicit sync -- read back via Tensor::toHost. + /// For concurrent use, prefer EnginePool. Engine itself is thread-compatible. + Result> infer(const std::unordered_map &inputs, const Stream &stream, + int profileIndex = 0); + + /// Single-output shortcut (the classifier/detector common case): returns the sole + /// output, or kInvalidArgument if the engine has != 1 output. Avoids hard-coding a name. + Result inferSingle(const std::unordered_map &inputs, const Stream &stream, int profileIndex = 0); + + /// Resolve ALL output shapes for a set of input views + profile (sets input shapes on + /// the execution context, reads getTensorShape -- dynamic-aware). Use to pre-allocate + /// output buffers for the enqueue path. + Result> outputShapes(const std::unordered_map &inputs, + int profileIndex = 0); + +private: + Engine(); + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace trtcpp diff --git a/include/tensorrt_cpp_api/engine_builder.h b/include/tensorrt_cpp_api/engine_builder.h new file mode 100644 index 0000000..5bdfe92 --- /dev/null +++ b/include/tensorrt_cpp_api/engine_builder.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "tensorrt_cpp_api/build_options.h" +#include "tensorrt_cpp_api/engine.h" +#include "tensorrt_cpp_api/logger.h" +#include "tensorrt_cpp_api/status.h" + +namespace trtcpp { + +/// Builds a serialized TensorRT engine from an ONNX model. Stateless apart from the +/// logger; one builder can build many engines. The cache-backed buildOrLoad and the +/// build+deserialize buildAndLoad convenience entry points are provided by the cache +/// and runtime modules respectively. +class EngineBuilder { +public: + explicit EngineBuilder(std::shared_ptr logger = defaultLogger()); + + /// Parse the ONNX at `onnxPath` and build a serialized engine for `options`. + Result> buildFromOnnxFile(const std::string &onnxPath, const BuildOptions &options); + /// Parse ONNX bytes (e.g. an in-memory or decrypted model) and build a serialized engine. + Result> buildFromOnnxBytes(std::span onnx, const BuildOptions &options); + + /// Build the engine if no fresh cached one exists in options.engineCacheDir, otherwise + /// reuse the cache. Returns the engine file path. The cache is keyed by ONNX content + /// hash + TRT version + GPU UUID + build options, with a JSON sidecar and atomic write; + /// a stale cache (changed ONNX/options/version/GPU) is detected and rebuilt. + Result buildOrLoad(const std::string &onnxPath, const BuildOptions &options); + + /// One-shot: buildOrLoad + deserialize into a ready-to-run Engine. The common entry point. + Result buildAndLoad(const std::string &onnxPath, const BuildOptions &options, const EngineOptions &engineOptions = {}); + +private: + std::shared_ptr logger_; +}; + +} // namespace trtcpp diff --git a/include/tensorrt_cpp_api/engine_pool.h b/include/tensorrt_cpp_api/engine_pool.h new file mode 100644 index 0000000..b3cae81 --- /dev/null +++ b/include/tensorrt_cpp_api/engine_pool.h @@ -0,0 +1,82 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "tensorrt_cpp_api/cuda.h" +#include "tensorrt_cpp_api/device_tensor.h" +#include "tensorrt_cpp_api/engine.h" // EngineOptions, TensorInfo +#include "tensorrt_cpp_api/shape.h" +#include "tensorrt_cpp_api/status.h" +#include "tensorrt_cpp_api/tensor.h" + +namespace trtcpp { + +namespace detail { +struct PoolState; // defined in engine_pool.cpp +} + +/// One deserialized engine, N execution contexts, ONE optimization profile per context. +/// Lease-based acquisition for concurrent multi-stream dynamic-shape inference. TensorRT +/// 11 requires each concurrently-used context to bind a distinct optimization profile, so a +/// dynamic-shape engine must be built with >= `contexts` profiles (create() rejects +/// otherwise). Each lease runs on the caller's stream. +class EnginePool { +public: + /// A borrowed execution context (+ its bound profile). Returns the context to the pool + /// on destruction. Move-only. + class Lease { + public: + Lease() = default; + Lease(Lease &&other) noexcept; + Lease &operator=(Lease &&other) noexcept; + Lease(const Lease &) = delete; + Lease &operator=(const Lease &) = delete; + ~Lease(); + + bool valid() const noexcept { return slot_ >= 0; } + int profileIndex() const noexcept { return profile_; } + + /// Caller-allocated, zero-copy (mirrors Engine::enqueue) on this lease's context+profile. + Status enqueue(const std::unordered_map &inputs, + const std::unordered_map &outputs, const Stream &stream); + /// Library-allocated convenience (mirrors Engine::infer). + Result> infer(const std::unordered_map &inputs, + const Stream &stream); + Result inferSingle(const std::unordered_map &inputs, const Stream &stream); + Result> outputShapes(const std::unordered_map &inputs); + + private: + friend class EnginePool; + Lease(std::shared_ptr state, int slot, int profile) : state_(std::move(state)), slot_(slot), profile_(profile) {} + std::shared_ptr state_; + int slot_ = -1; + int profile_ = 0; + }; + + EnginePool(EnginePool &&) noexcept; + EnginePool &operator=(EnginePool &&) noexcept; + EnginePool(const EnginePool &) = delete; + EnginePool &operator=(const EnginePool &) = delete; + ~EnginePool(); + + static Result create(const std::string &enginePath, int contexts, const EngineOptions &options = {}); + + Result acquire(); ///< blocks until a context is free + std::optional tryAcquire(); ///< returns nullopt if none free + int size() const noexcept; ///< number of contexts (== profiles for dynamic engines) + + std::vector tensors() const; + std::vector inputNames() const; + std::vector outputNames() const; + +private: + EnginePool() = default; + std::shared_ptr state_; +}; + +} // namespace trtcpp diff --git a/include/tensorrt_cpp_api/layout.h b/include/tensorrt_cpp_api/layout.h new file mode 100644 index 0000000..9538753 --- /dev/null +++ b/include/tensorrt_cpp_api/layout.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +namespace trtcpp { + +/// Memory layout hint for interop and preprocessing. TensorRT IO is layout-agnostic (it +/// sees raw dims); Layout drives the preprocessing kernels and OpenCV adapters only. +enum class Layout : std::uint8_t { + kNCHW, + kNHWC, + kLinear, + kUnknown, +}; + +/// Where a tensor's data lives. +enum class Device : std::uint8_t { + kHost, + kCuda, +}; + +std::string_view toString(Layout l) noexcept; +std::string_view toString(Device d) noexcept; + +} // namespace trtcpp diff --git a/include/tensorrt_cpp_api/logger.h b/include/tensorrt_cpp_api/logger.h new file mode 100644 index 0000000..5b367bd --- /dev/null +++ b/include/tensorrt_cpp_api/logger.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include + +namespace trtcpp { + +/// Log severity, ordered ascending by importance (kVerbose=0 .. kInternalError=4). Each +/// value corresponds to an nvinfer1::ILogger::Severity BY NAME, not by integer: nvinfer1's +/// integer order is the reverse (kINTERNAL_ERROR=0 .. kVERBOSE=4), so the TensorRT +/// bridge must translate per enumerator and never via a plain static_cast (that would +/// invert every level). +enum class Severity : std::uint8_t { + kVerbose, + kInfo, + kWarning, + kError, + kInternalError, +}; + +std::string_view toString(Severity s) noexcept; + +/// Injectable logger. The library NEVER logs to a global -- every component takes an +/// ILogger (or accepts the default), fixing v6's hard-wired global spdlog. Implementations +/// must be thread-safe and must not throw. +class ILogger { +public: + virtual ~ILogger() = default; + virtual void log(Severity severity, std::string_view message) noexcept = 0; +}; + +/// Process-wide thread-safe stderr logger. Its threshold is read once from the +/// TRTCPP_LOG_LEVEL env var (verbose|info|warning|error|off; default info), with +/// LOG_LEVEL honored as a v6-compatibility fallback. +std::shared_ptr defaultLogger(); + +/// A stderr logger with an explicit severity threshold (messages below it are dropped). +std::shared_ptr makeStderrLogger(Severity threshold); + +#ifdef TRT_CPP_API_WITH_SPDLOG +/// Routes log records through spdlog's default logger. Only declared/available when the +/// library is built with -DTRT_CPP_API_WITH_SPDLOG=ON (which links spdlog::spdlog and +/// defines this macro on the public interface). +std::shared_ptr makeSpdlogLogger(); +#endif + +} // namespace trtcpp diff --git a/include/tensorrt_cpp_api/opencv_interop.h b/include/tensorrt_cpp_api/opencv_interop.h new file mode 100644 index 0000000..3717d49 --- /dev/null +++ b/include/tensorrt_cpp_api/opencv_interop.h @@ -0,0 +1,36 @@ +#pragma once + +// Optional OpenCV interop (TRT_CPP_API_WITH_OPENCV). Strictly opt-in: the engine core knows +// nothing about OpenCV, and this header is empty unless the option is on (which links OpenCV +// and defines the macro). Adapters between cv::Mat / cv::cuda::GpuMat and the library types. +#ifdef TRT_CPP_API_WITH_OPENCV + +#include +#include + +#include "tensorrt_cpp_api/cuda.h" +#include "tensorrt_cpp_api/device_tensor.h" +#include "tensorrt_cpp_api/dtype.h" +#include "tensorrt_cpp_api/status.h" +#include "tensorrt_cpp_api/tensor.h" + +namespace trtcpp::opencv { + +/// Map an OpenCV element depth (CV_8U, CV_32F, ...) to a DType. +Result dtypeOfCvDepth(int cvDepth); + +/// Zero-copy device view over a CONTINUOUS GpuMat as an HWC tensor [rows, cols, channels]. +/// Errors (kUnsupported) on a padded (non-continuous) GpuMat -- GpuMat rows are usually pitched, +/// and a TensorView has no row stride, so copy into a cv::cuda::createContinuous() buffer first +/// (clone() does not help; it is pitched too). Device is the current CUDA device. +Result viewOf(const cv::cuda::GpuMat &mat, Layout layout = Layout::kNHWC); + +/// Host view over a continuous cv::Mat as an HWC tensor [rows, cols, channels]. +Result viewOf(const cv::Mat &mat, Layout layout = Layout::kNHWC); + +/// Upload a cv::Mat to a freshly allocated device Tensor (async on `stream`). +Result upload(const cv::Mat &mat, const Stream &stream); + +} // namespace trtcpp::opencv + +#endif // TRT_CPP_API_WITH_OPENCV diff --git a/include/tensorrt_cpp_api/preproc.h b/include/tensorrt_cpp_api/preproc.h new file mode 100644 index 0000000..699d7a1 --- /dev/null +++ b/include/tensorrt_cpp_api/preproc.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include + +#include "tensorrt_cpp_api/cuda.h" +#include "tensorrt_cpp_api/dtype.h" +#include "tensorrt_cpp_api/status.h" +#include "tensorrt_cpp_api/tensor.h" + +// Optional preprocessing sublibrary (tensorrt_cpp_api::preproc). A single fused CUDA kernel +// covers the ~5 ops a CNN TRT input needs, so the engine core links nothing here and there +// is no OpenCV dependency. Build it with -DTRT_CPP_API_BUILD_PREPROC=ON (default ON). + +namespace trtcpp::preproc { + +/// How to turn a decoded HWC uint8 image into a TRT-ready NCHW tensor. The normalization is +/// out = (pixel - mean[c]) * scale[c] applied per channel (fixes the v6 broken per-channel +/// path): e.g. [0,1] scaling is mean=0, scale=1/255; ImageNet folds 1/(255*std) into scale. +/// The output channel count and dtype are taken from `dst` (not duplicated here). Resize, +/// layout, and channel copy handle any channel count; per-channel mean/scale cover up to 4 +/// channels (channels >= 4 use identity normalization). +struct PreprocSpec { + std::array mean{0.0f, 0.0f, 0.0f, 0.0f}; + std::array scale{1.0f, 1.0f, 1.0f, 1.0f}; + bool swapRB = false; ///< swap channels 0 and 2 (BGR<->RGB), 3-channel only + bool keepAspectRatioPad = true; ///< letterbox (resize keeping aspect, pad right/bottom) vs stretch + std::uint8_t padValue = 0; ///< fill for the letterbox padding (then normalized like a pixel) +}; + +/// Fused: letterbox-resize -> optional BGR<->RGB -> (pixel-mean)*scale -> HWC->NCHW -> cast, +/// writing `dst` with no intermediate buffers. Async on `stream` (no implicit sync). +/// src: HWC uint8 device tensor, shape [H,W,C] or [1,H,W,C]. +/// dst: pre-allocated NCHW device tensor [1, C, Hout, Wout], dtype kFloat32 or kFloat16. +Status letterboxToTensor(TensorView src, TensorView dst, const PreprocSpec &spec, const Stream &stream); + +} // namespace trtcpp::preproc diff --git a/include/tensorrt_cpp_api/quant.h b/include/tensorrt_cpp_api/quant.h new file mode 100644 index 0000000..a4db2e1 --- /dev/null +++ b/include/tensorrt_cpp_api/quant.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include + +namespace trtcpp { + +/// Precision / quantization mode for engine building. Version-aware and never a silent no-op +/// (an unachievable mode fails the build with a clear error rather than downgrading): +/// kFp32 - full precision. +/// kFp16 - half precision. On TRT < 11 via the weak-typed kFP16 flag; on +/// TRT >= 11 requires an FP16/QDQ ONNX (else kUnsupported). +/// kInt8Qdq - explicit Q/DQ ONNX, strongly typed, no precision flags. The +/// forward path that compiles on both TRT 10 and 11. Default for +/// quantized models. +/// kInt8CalibLegacy- legacy calibrator PTQ. Only available when built against TRT < 11 +/// (wired up in the calibration module); rejected on TRT >= 11. +/// kFp8 / kNvfp4 - explicit-QDQ FP8 / NVFP4; require Ada (8.9+) / Blackwell (10.0+) +/// hardware respectively (validated at build). +enum class Precision : std::uint8_t { + kFp32, + kFp16, + kInt8Qdq, + kInt8CalibLegacy, + kFp8, + kNvfp4, +}; + +std::string_view toString(Precision p) noexcept; + +} // namespace trtcpp diff --git a/include/tensorrt_cpp_api/shape.h b/include/tensorrt_cpp_api/shape.h new file mode 100644 index 0000000..5fff7e0 --- /dev/null +++ b/include/tensorrt_cpp_api/shape.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace trtcpp { + +/// A dynamic-aware tensor shape with fixed inline capacity (no heap). A dim of -1 means +/// "dynamic / unresolved" (TensorRT's convention). Rank 0 is a scalar (numel 1). +class Shape { +public: + /// Matches nvinfer1::Dims::MAX_DIMS; static_assert'd against it once TRT is linked. + static constexpr int kMaxRank = 8; + + Shape() = default; + Shape(std::initializer_list dims); + explicit Shape(std::span dims); + + int rank() const noexcept { return rank_; } + std::int64_t operator[](int i) const noexcept { return dims_[static_cast(i)]; } + std::int64_t at(int i) const; ///< throws std::out_of_range + + bool isDynamic() const noexcept; ///< any dim < 0 + /// Element count; returns 0 if any dim is dynamic (resolve the shape first). Assumes + /// the product is representable in int64 (true for any real tensor on current GPUs). + std::int64_t numel() const noexcept; + + std::span dims() const noexcept { return {dims_.data(), static_cast(rank_)}; } + + bool operator==(const Shape &other) const noexcept; + bool operator!=(const Shape &other) const noexcept { return !(*this == other); } + + std::string toString() const; ///< e.g. "[1,3,640,640]" + +private: + std::array dims_{}; + int rank_ = 0; +}; + +} // namespace trtcpp diff --git a/include/tensorrt_cpp_api/status.h b/include/tensorrt_cpp_api/status.h new file mode 100644 index 0000000..824537e --- /dev/null +++ b/include/tensorrt_cpp_api/status.h @@ -0,0 +1,120 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace trtcpp { + +/// The public API is no-throw: every fallible call returns Status or Result. +enum class StatusCode { + kOk, + kInvalidArgument, + kNotFound, + kIoError, + kCudaError, + kTensorRtError, + kShapeMismatch, + kDtypeMismatch, + kUnsupported, + kStaleCache, + kInternal, +}; + +std::string_view toString(StatusCode code) noexcept; + +class Status { +public: + Status() noexcept = default; ///< default-constructed Status is kOk + Status(StatusCode code, std::string message) : code_(code), message_(std::move(message)) {} + + bool ok() const noexcept { return code_ == StatusCode::kOk; } + StatusCode code() const noexcept { return code_; } + const std::string &message() const noexcept { return message_; } + explicit operator bool() const noexcept { return ok(); } + +private: + StatusCode code_ = StatusCode::kOk; + std::string message_; +}; + +/// Value-or-Status, std::expected-style but no exceptions. Move-friendly. +template class Result { +public: + Result(T value) : value_(std::move(value)) {} + Result(Status error) : status_(std::move(error)) { assert(!status_.ok() && "Result(Status) must be an error"); } + + bool ok() const noexcept { return status_.ok(); } + explicit operator bool() const noexcept { return ok(); } + + T &value() & { + assert(ok()); + return *value_; + } + const T &value() const & { + assert(ok()); + return *value_; + } + T &&value() && { + assert(ok()); + return std::move(*value_); + } + + T *operator->() { + assert(ok()); + return &*value_; + } + const T *operator->() const { + assert(ok()); + return &*value_; + } + T &operator*() & { + assert(ok()); + return *value_; + } + const T &operator*() const & { + assert(ok()); + return *value_; + } + + const Status &status() const noexcept { return status_; } + T value_or(T fallback) const { return ok() ? *value_ : std::move(fallback); } + + /// F: T& -> Result. Propagates this error if !ok(). + template auto and_then(F &&f) -> std::invoke_result_t { + using R = std::invoke_result_t; + if (ok()) { + return std::forward(f)(*value_); + } + return R(status_); + } + /// F: T& -> U. Wraps the result; propagates this error if !ok(). + template auto transform(F &&f) -> Result> { + using U = std::invoke_result_t; + if (ok()) { + return Result(std::forward(f)(*value_)); + } + return Result(status_); + } + +private: + Status status_; // default-constructed = ok + std::optional value_; +}; + +#define TRTCPP_CONCAT_(a, b) a##b +#define TRTCPP_CONCAT(a, b) TRTCPP_CONCAT_(a, b) +/// Propagate-on-error sugar: `TRTCPP_TRY(auto engine, Engine::loadFromFile(p, {}));` +/// On error, returns the Status from the enclosing function (which must return Status or +/// a Result<...>, since Result is constructible from Status). One use per source line. +#define TRTCPP_TRY(decl, expr) \ + auto TRTCPP_CONCAT(trtcpp_try_, __LINE__) = (expr); \ + if (!TRTCPP_CONCAT(trtcpp_try_, __LINE__)) { \ + return TRTCPP_CONCAT(trtcpp_try_, __LINE__).status(); \ + } \ + decl = std::move(TRTCPP_CONCAT(trtcpp_try_, __LINE__)).value() + +} // namespace trtcpp diff --git a/include/tensorrt_cpp_api/tensor.h b/include/tensorrt_cpp_api/tensor.h new file mode 100644 index 0000000..e96a57e --- /dev/null +++ b/include/tensorrt_cpp_api/tensor.h @@ -0,0 +1,90 @@ +#pragma once + +#include +#include // SIZE_MAX +#include +#include + +#include "tensorrt_cpp_api/dtype.h" +#include "tensorrt_cpp_api/layout.h" +#include "tensorrt_cpp_api/shape.h" +#include "tensorrt_cpp_api/status.h" + +namespace trtcpp { + +/// A non-owning view over host or device memory. The primary input/output type at the +/// enqueue boundary -- zero-copy by construction. Carries enough to drive +/// setTensorAddress + setInputShape and to build DLPack / __cuda_array_interface__. +/// +/// The owning Tensor (allocate/toHost/copyFrom) is CUDA-backed. +class TensorView { +public: + struct Desc { + void *data = nullptr; + DType dtype = DType::kFloat32; + Shape shape; + Device device = Device::kCuda; + int deviceId = 0; + Layout layout = Layout::kLinear; + }; + + TensorView() = default; + explicit TensorView(Desc d) : d_(std::move(d)) {} + TensorView(void *data, DType dtype, Shape shape, Device device, int deviceId = 0, Layout layout = Layout::kLinear) + : d_{data, dtype, std::move(shape), device, deviceId, layout} {} + + void *data() const noexcept { return d_.data; } + DType dtype() const noexcept { return d_.dtype; } + const Shape &shape() const noexcept { return d_.shape; } + Device device() const noexcept { return d_.device; } + int deviceId() const noexcept { return d_.deviceId; } + Layout layout() const noexcept { return d_.layout; } + bool isCuda() const noexcept { return d_.device == Device::kCuda; } + /// Byte size of the data. A dynamic (unresolved) shape has numel 0, so this returns + /// 0 -- resolve the shape (set concrete dims) before using nbytes for allocation. + std::size_t nbytes() const noexcept { return byteSizeOf(d_.dtype, static_cast(d_.shape.numel())); } + + /// Typed, dtype-checked host span. Errors (kInvalidArgument) on a device view -- + /// never an implicit D2H copy -- or (kDtypeMismatch) if T does not match dtype(). + /// A dynamic shape yields an ok, length-0 span (numel 0): resolve the shape first. + template Result> as() const; + +private: + Desc d_; +}; + +/// Byte size of a resolved (dtype, shape), with overflow checking -- the guard the buffer +/// layer owes before any allocation. Errors (kInvalidArgument) on a dynamic shape or if +/// the element count / byte size would overflow std::size_t. (GCC/Clang builtins; the +/// library is Linux/GCC-Clang only.) +inline Result checkedByteSize(DType dtype, const Shape &shape) { + if (shape.isDynamic()) { + return Status{StatusCode::kInvalidArgument, "checkedByteSize requires a fully resolved (non-dynamic) shape"}; + } + std::size_t count = 1; + for (int i = 0; i < shape.rank(); ++i) { + if (__builtin_mul_overflow(count, static_cast(shape[i]), &count)) { + return Status{StatusCode::kInvalidArgument, "shape element count overflows std::size_t"}; + } + } + std::size_t totalBits = 0; + if (__builtin_mul_overflow(count, static_cast(bitsPerElement(dtype)), &totalBits)) { + return Status{StatusCode::kInvalidArgument, "tensor byte size overflows std::size_t"}; + } + if (totalBits > SIZE_MAX - 7) { // guard the round-up below from wrapping past SIZE_MAX + return Status{StatusCode::kInvalidArgument, "tensor byte size overflows std::size_t"}; + } + return (totalBits + 7) / 8; +} + +template Result> TensorView::as() const { + if (d_.device != Device::kHost) { + return Status{StatusCode::kInvalidArgument, "TensorView::as() requires a host tensor; copy to host first (Tensor::toHost)"}; + } + if (d_.dtype != DTypeOf::value) { + return Status{StatusCode::kDtypeMismatch, "TensorView::as() dtype does not match the tensor's dtype"}; + } + return std::span(static_cast(d_.data), static_cast(d_.shape.numel())); +} + +} // namespace trtcpp diff --git a/include/tensorrt_cpp_api/version.h b/include/tensorrt_cpp_api/version.h new file mode 100644 index 0000000..9d4da0c --- /dev/null +++ b/include/tensorrt_cpp_api/version.h @@ -0,0 +1,31 @@ +#pragma once + +#include "tensorrt_cpp_api/build_config.h" + +#include + +namespace trtcpp { + +struct Version { + int major = 0; + int minor = 0; + int patch = 0; +}; + +/// The tensorrt_cpp_api library version. +constexpr Version libraryVersion() noexcept { + return Version{TRT_CPP_API_VERSION_MAJOR, TRT_CPP_API_VERSION_MINOR, TRT_CPP_API_VERSION_PATCH}; +} + +/// The TensorRT major/minor the library was BUILT against (compile-time, from FindTensorRT). +constexpr int tensorrtBuildMajor() noexcept { return TRT_CPP_API_TENSORRT_VERSION_MAJOR; } +constexpr int tensorrtBuildMinor() noexcept { return TRT_CPP_API_TENSORRT_VERSION_MINOR; } + +/// Human-readable, e.g. "7.0.0 (built against TensorRT 10.0)". +inline std::string versionString() { + return std::to_string(TRT_CPP_API_VERSION_MAJOR) + "." + std::to_string(TRT_CPP_API_VERSION_MINOR) + "." + + std::to_string(TRT_CPP_API_VERSION_PATCH) + " (built against TensorRT " + std::to_string(TRT_CPP_API_TENSORRT_VERSION_MAJOR) + + "." + std::to_string(TRT_CPP_API_TENSORRT_VERSION_MINOR) + ")"; +} + +} // namespace trtcpp diff --git a/include/util/Stopwatch.h b/include/util/Stopwatch.h deleted file mode 100644 index 621ee9d..0000000 --- a/include/util/Stopwatch.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once -#include - -// Utility Timer -template class Stopwatch { - typename Clock::time_point start_point; - -public: - Stopwatch() : start_point(Clock::now()) {} - - // Returns elapsed time - template Rep elapsedTime() const { - std::atomic_thread_fence(std::memory_order_relaxed); - auto counted_time = std::chrono::duration_cast(Clock::now() - start_point).count(); - std::atomic_thread_fence(std::memory_order_relaxed); - return static_cast(counted_time); - } -}; - -using preciseStopwatch = Stopwatch<>; \ No newline at end of file diff --git a/include/util/Util.h b/include/util/Util.h deleted file mode 100644 index d378589..0000000 --- a/include/util/Util.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -namespace Util { - // Checks if a file exists at the given file path - bool doesFileExist(const std::string &filepath); - - // Checks and logs CUDA error codes - void checkCudaErrorCode(cudaError_t code); - - // Retrieves a list of file names in the specified directory - std::vector getFilesInDirectory(const std::string &dirPath); -} - -#include "Util.inl" diff --git a/include/util/Util.inl b/include/util/Util.inl deleted file mode 100644 index 37317ee..0000000 --- a/include/util/Util.inl +++ /dev/null @@ -1,30 +0,0 @@ -#pragma once - -#include - -namespace Util { - -inline bool doesFileExist(const std::string &filepath) { - std::ifstream f(filepath.c_str()); - return f.good(); -} - -inline void checkCudaErrorCode(cudaError_t code) { - if (code != cudaSuccess) { - std::string errMsg = "CUDA operation failed with code: " + std::to_string(code) + " (" + cudaGetErrorName(code) + - "), with message: " + cudaGetErrorString(code); - spdlog::error(errMsg); - throw std::runtime_error(errMsg); - } -} - -inline std::vector getFilesInDirectory(const std::string &dirPath) { - std::vector fileNames; - for (const auto &entry : std::filesystem::directory_iterator(dirPath)) { - if (entry.is_regular_file()) { - fileNames.push_back(entry.path().string()); - } - } - return fileNames; - } -} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..184224d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["scikit-build-core>=0.10", "pybind11>=2.12"] +build-backend = "scikit_build_core.build" + +[project] +name = "tensorrt-cpp-api" +version = "7.0.0" +description = "A no-throw C++ TensorRT inference library for CNN models, with zero-copy Python bindings." +readme = "README.md" +requires-python = ">=3.9" +license = { file = "LICENSE" } +authors = [{ name = "Cyrus Behroozi" }] +keywords = ["tensorrt", "inference", "cuda", "gpu", "deep-learning", "cnn"] +classifiers = [ + "Programming Language :: C++", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Environment :: GPU :: NVIDIA CUDA", + "Operating System :: POSIX :: Linux", +] + +[project.urls] +Homepage = "https://github.com/cyrusbehr/tensorrt-cpp-api" +Repository = "https://github.com/cyrusbehr/tensorrt-cpp-api" + +# scikit-build-core drives the project's own CMakeLists with the bindings turned on. TensorRT +# and CUDA are system/externally provided; point the build at a tarball install with, e.g.: +# pip install . --config-settings=cmake.define.TensorRT_DIR=/opt/TensorRT-10.x +[tool.scikit-build] +minimum-version = "build-system.requires" +cmake.version = ">=3.22" +cmake.source-dir = "." +wheel.packages = ["python/trtcpp"] +build-dir = "build/skbuild/{wheel_tag}" + +[tool.scikit-build.cmake.define] +TRT_CPP_API_BUILD_PYTHON = "ON" +TRT_CPP_API_BUILD_PREPROC = "ON" +TRT_CPP_API_BUILD_TESTS = "OFF" +TRT_CPP_API_BUILD_EXAMPLES = "OFF" + +[tool.scikit-build.sdist] +exclude = [".github", "build", "engines", "models", "inputs", "images"] diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt new file mode 100644 index 0000000..f5d2815 --- /dev/null +++ b/python/CMakeLists.txt @@ -0,0 +1,45 @@ +# Python bindings: the trtcpp._core extension. Built two ways: +# * in-tree, with -DTRT_CPP_API_BUILD_PYTHON=ON (added by the root CMakeLists), for dev/test; +# * standalone, by scikit-build-core from the top-level pyproject.toml (the wheel path), which +# configures the whole project with BUILD_PYTHON=ON. +# Either way the core library targets (tensorrt_cpp_api[, ::preproc]) already exist in this +# build tree, so the extension just links them. The static core is built position-independent +# (set on its target) so it links cleanly into this module .so. + +find_package(Python REQUIRED COMPONENTS Interpreter Development.Module) +find_package(pybind11 CONFIG REQUIRED) + +pybind11_add_module(_core MODULE src/bindings.cpp) +target_compile_features(_core PRIVATE cxx_std_20) +target_link_libraries(_core PRIVATE tensorrt_cpp_api::tensorrt_cpp_api CUDA::cudart) + +if(TARGET tensorrt_cpp_api::preproc) + target_link_libraries(_core PRIVATE tensorrt_cpp_api::preproc) + target_compile_definitions(_core PRIVATE TRT_CPP_API_PYTHON_WITH_PREPROC) +endif() + +if(DEFINED SKBUILD) + # Wheel build: install the extension into the package dir inside the wheel. + install(TARGETS _core DESTINATION trtcpp) +else() + # In-tree dev build: stage the extension inside the source package so `import trtcpp` + # (with PYTHONPATH=/python) resolves `from ._core import ...`. The staged .so is + # gitignored (*.so). + add_custom_command( + TARGET _core POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ + ${CMAKE_CURRENT_SOURCE_DIR}/trtcpp/$ + COMMENT "Staging trtcpp._core into the source package for in-tree pytest" + ) + + if(TRT_CPP_API_BUILD_TESTS) + find_program(PYTEST_EXECUTABLE NAMES pytest) + if(PYTEST_EXECUTABLE) + add_test(NAME python_bindings COMMAND ${PYTEST_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests -q) + set_tests_properties( + python_bindings PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}" LABELS "gpu;python" + ) + else() + message(STATUS "pytest not found; skipping the python_bindings test") + endif() + endif() +endif() diff --git a/python/src/bindings.cpp b/python/src/bindings.cpp new file mode 100644 index 0000000..1b9c09d --- /dev/null +++ b/python/src/bindings.cpp @@ -0,0 +1,779 @@ +// pybind11 bindings for tensorrt_cpp_api (module trtcpp._core). +// +// Design: +// * The C++ API is no-throw (Status/Result). Here that maps to Python exceptions: every +// fallible call is unwrapped, raising trtcpp.TrtcppError (carrying .code and message) on +// failure and returning the value otherwise. +// * Zero-copy in: inputs are dicts {name: array} where array is a trtcpp.Tensor/TensorView +// or any object exposing __cuda_array_interface__ (CuPy, Numba, PyTorch CUDA tensors) or +// __dlpack__. No host<->device copy happens at the boundary. +// * Zero-copy out: the owning Tensor exposes __cuda_array_interface__ (device) / +// __array_interface__ (host) AND __dlpack__/__dlpack_device__, so torch.from_dlpack(out) +// and cupy.asarray(out) alias the device memory. +// * Caller stream: trtcpp.Stream wraps a torch/cupy stream handle (Stream.wrap(int)). +// * No implicit D2H: readback is the explicit Tensor.to_host(stream). The GIL is released +// around the actual inference/build call (not the Python<->TensorView conversion). +// +// Lifetime contract (documented on the Python side): because inference is asynchronous on the +// caller's stream and never implicitly synchronizes, the caller MUST keep input arrays alive +// and synchronize the stream before consuming outputs. + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "tensorrt_cpp_api/all.h" +#ifdef TRT_CPP_API_PYTHON_WITH_PREPROC +#include "tensorrt_cpp_api/preproc.h" +#endif + +#include "dlpack.h" + +namespace py = pybind11; +using namespace trtcpp; + +namespace { + +// ---- error mapping --------------------------------------------------------------------- + +py::object &errorClass() { + static py::object cls; + return cls; +} + +[[noreturn]] void raiseStatus(const Status &s) { + py::object exc = errorClass()(s.message()); + exc.attr("code") = py::cast(s.code()); + PyErr_SetObject(errorClass().ptr(), exc.ptr()); + throw py::error_already_set(); +} + +void check(const Status &s) { + if (!s) { + raiseStatus(s); + } +} + +template T unwrap(Result &&r) { + if (!r) { + raiseStatus(r.status()); + } + return std::move(r).value(); +} + +// ---- dtype <-> numpy typestr / DLPack -------------------------------------------------- + +std::string typestrOf(DType t) { + switch (t) { + case DType::kFloat32: + return "','|','='), then kind, then size. + if (ts.size() < 3) { + throw py::value_error("malformed __cuda_array_interface__ typestr: " + ts); + } + const char kind = ts[1]; + const int size = std::stoi(ts.substr(2)); + if (kind == 'f' && size == 4) + return DType::kFloat32; + if (kind == 'f' && size == 2) + return DType::kFloat16; + if (kind == 'i' && size == 4) + return DType::kInt32; + if (kind == 'i' && size == 8) + return DType::kInt64; + if (kind == 'i' && size == 1) + return DType::kInt8; + if (kind == 'u' && size == 1) + return DType::kUInt8; + if (kind == 'b' && size == 1) + return DType::kBool; + throw py::value_error("unsupported __cuda_array_interface__ typestr for TensorRT IO: " + ts); +} + +DLDataType toDLDataType(DType t) { + switch (t) { + case DType::kFloat32: + return {kDLFloat, 32, 1}; + case DType::kFloat16: + return {kDLFloat, 16, 1}; + case DType::kBFloat16: + return {kDLBfloat, 16, 1}; + case DType::kInt32: + return {kDLInt, 32, 1}; + case DType::kInt64: + return {kDLInt, 64, 1}; + case DType::kInt8: + return {kDLInt, 8, 1}; + case DType::kUInt8: + return {kDLUInt, 8, 1}; + case DType::kBool: + return {kDLBool, 8, 1}; + default: + break; + } + throw py::value_error("dtype " + std::string(toString(t)) + " has no DLPack mapping"); +} + +DType dtypeFromDL(DLDataType d) { + if (d.lanes != 1) { + throw py::value_error("vectorized DLPack dtypes (lanes != 1) are unsupported"); + } + if (d.code == kDLFloat && d.bits == 32) + return DType::kFloat32; + if (d.code == kDLFloat && d.bits == 16) + return DType::kFloat16; + if (d.code == kDLBfloat && d.bits == 16) + return DType::kBFloat16; + if (d.code == kDLInt && d.bits == 32) + return DType::kInt32; + if (d.code == kDLInt && d.bits == 64) + return DType::kInt64; + if (d.code == kDLInt && d.bits == 8) + return DType::kInt8; + if (d.code == kDLUInt && d.bits == 8) + return DType::kUInt8; + if (d.code == kDLBool && d.bits == 8) + return DType::kBool; + throw py::value_error("unsupported DLPack dtype (code/bits) for TensorRT IO"); +} + +Shape makeShape(const std::vector &dims) { return Shape(std::span(dims.data(), dims.size())); } + +// Resolve the CUDA device a device pointer lives on (avoids assuming the current device). +int deviceOfPointer(const void *ptr) { + cudaPointerAttributes attr{}; + if (cudaPointerGetAttributes(&attr, ptr) != cudaSuccess) { + cudaGetLastError(); // clear + int cur = 0; + cudaGetDevice(&cur); + return cur; + } + return attr.device; +} + +// Require C-contiguous strides. CAI strides are in BYTES; None => contiguous. `strides` is +// untrusted (it comes straight from a foreign object's __cuda_array_interface__), so its +// length is validated against the shape before indexing. +void requireContiguousBytes(const std::vector &dims, DType dtype, const std::vector &strides) { + if (strides.size() != dims.size()) { + throw py::value_error("__cuda_array_interface__ strides length does not match shape rank"); + } + const auto itemsize = static_cast(byteSize(dtype)); + std::int64_t expected = itemsize; + for (int i = static_cast(dims.size()) - 1; i >= 0; --i) { + if (strides[static_cast(i)] != expected) { + throw py::value_error("TensorRT IO requires a C-contiguous array (non-contiguous strides given)"); + } + // Overflow-checked: a malformed/huge shape from a foreign __cuda_array_interface__ must not + // wrap `expected` into a value that spuriously matches a later stride. + if (__builtin_mul_overflow(expected, dims[static_cast(i)], &expected)) { + throw py::value_error("__cuda_array_interface__ shape is too large (stride overflow)"); + } + } +} + +// ---- Python object -> TensorView (zero-copy) ------------------------------------------- + +TensorView viewFromCAI(const py::dict &cai) { + auto dataTuple = cai["data"].cast(); + auto ptr = reinterpret_cast(dataTuple[0].cast()); + const DType dtype = dtypeFromTypestr(cai["typestr"].cast()); + + std::vector dims; + for (auto d : cai["shape"].cast()) { + dims.push_back(d.cast()); + } + if (cai.contains("strides") && !cai["strides"].is_none()) { + std::vector strides; + for (auto s : cai["strides"].cast()) { + strides.push_back(s.cast()); + } + requireContiguousBytes(dims, dtype, strides); + } + return TensorView(ptr, dtype, makeShape(dims), Device::kCuda, deviceOfPointer(ptr)); +} + +TensorView viewFromDlpack(py::handle obj) { + py::object cap = obj.attr("__dlpack__")(); + if (!PyCapsule_IsValid(cap.ptr(), "dltensor")) { + throw py::type_error("__dlpack__ did not return a valid 'dltensor' capsule"); + } + auto *mt = static_cast(PyCapsule_GetPointer(cap.ptr(), "dltensor")); + // Standard DLPack consumer protocol: rename to "used_dltensor" so the producer's capsule + // destructor will NOT also invoke the deleter when `cap` is GC'd -- we are now responsible + // for the single deleter call below. This is what prevents a double free. + PyCapsule_SetName(cap.ptr(), "used_dltensor"); + const DLTensor &dt = mt->dl_tensor; + + const Device dev = dt.device.device_type == kDLCUDA ? Device::kCuda : Device::kHost; + const DType dtype = dtypeFromDL(dt.dtype); + std::vector dims(dt.shape, dt.shape + dt.ndim); + if (dt.strides) { + // DLPack strides are in ELEMENTS. + std::int64_t expected = 1; + for (int i = dt.ndim - 1; i >= 0; --i) { + if (dt.strides[i] != expected || __builtin_mul_overflow(expected, dims[static_cast(i)], &expected)) { + if (mt->deleter) { + mt->deleter(mt); + } + throw py::value_error("TensorRT IO requires a C-contiguous DLPack tensor (or shape too large)"); + } + } + } + void *data = static_cast(dt.data) + dt.byte_offset; + const int deviceId = dt.device.device_id; + TensorView v(data, dtype, makeShape(dims), dev, deviceId); + // Release the borrow now: the address stays valid via the (caller-kept-alive) source object. + if (mt->deleter) { + mt->deleter(mt); + } + return v; +} + +TensorView toTensorView(py::handle obj) { + if (py::isinstance(obj)) { + return obj.cast().view(); + } + if (py::isinstance(obj)) { + return obj.cast(); + } + if (py::hasattr(obj, "__cuda_array_interface__")) { + return viewFromCAI(obj.attr("__cuda_array_interface__").cast()); + } + if (py::hasattr(obj, "__dlpack__")) { + return viewFromDlpack(obj); + } + throw py::type_error("expected a trtcpp.Tensor/TensorView or an object exposing " + "__cuda_array_interface__ or __dlpack__"); +} + +std::unordered_map toViewMap(const py::dict &d) { + std::unordered_map m; + for (auto item : d) { + m.emplace(item.first.cast(), toTensorView(item.second)); + } + return m; +} + +py::dict tensorMapToDict(std::unordered_map &&m) { + py::dict out; + for (auto &kv : m) { + out[py::str(kv.first)] = py::cast(std::move(kv.second)); + } + return out; +} + +// ---- DLPack export --------------------------------------------------------------------- + +struct DlpackCtx { + std::vector shape; + py::object owner; // keeps the producing trtcpp.Tensor alive while the capsule lives +}; + +void dlpackDeleter(DLManagedTensor *self) { + auto *ctx = static_cast(self->manager_ctx); + { + // The consumer may invoke this from a thread that does or does not hold the GIL; + // gil_scoped_acquire (PyGILState_Ensure) is reentrant-safe for both, and is required + // because dropping the py::object owner touches the Python refcount. + py::gil_scoped_acquire gil; + ctx->owner = py::object(); + } + delete ctx; // frees ctx->shape, which backed self->dl_tensor.shape for the capsule's life + delete self; +} + +void dlpackCapsuleDestructor(PyObject *capsule) { + // Only fires if the consumer never renamed/consumed the capsule. + if (PyCapsule_IsValid(capsule, "dltensor")) { + auto *mt = static_cast(PyCapsule_GetPointer(capsule, "dltensor")); + if (mt && mt->deleter) { + mt->deleter(mt); + } + } +} + +py::capsule tensorDlpack(py::object self) { + auto &t = self.cast(); + if (t.empty()) { + throw py::value_error("cannot export an empty Tensor via DLPack"); + } + auto *ctx = new DlpackCtx{}; + ctx->shape.reserve(static_cast(t.shape().rank())); + for (int i = 0; i < t.shape().rank(); ++i) { + ctx->shape.push_back(t.shape()[i]); + } + ctx->owner = self; + + auto *mt = new DLManagedTensor{}; + mt->dl_tensor.data = t.data(); + mt->dl_tensor.device = DLDevice{t.device() == Device::kCuda ? kDLCUDA : kDLCPU, t.deviceId()}; + mt->dl_tensor.ndim = t.shape().rank(); + mt->dl_tensor.dtype = toDLDataType(t.dtype()); + mt->dl_tensor.shape = ctx->shape.data(); + mt->dl_tensor.strides = nullptr; // C-contiguous + mt->dl_tensor.byte_offset = 0; + mt->manager_ctx = ctx; + mt->deleter = dlpackDeleter; + return py::capsule(mt, "dltensor", dlpackCapsuleDestructor); +} + +py::dict cudaArrayInterface(const Tensor &t) { + if (t.device() != Device::kCuda) { + throw py::attribute_error("__cuda_array_interface__ is only available on CUDA tensors; use __array_interface__"); + } + py::tuple shape(t.shape().rank()); + for (int i = 0; i < t.shape().rank(); ++i) { + shape[i] = t.shape()[i]; + } + py::dict d; + d["shape"] = shape; + d["typestr"] = typestrOf(t.dtype()); + d["data"] = py::make_tuple(reinterpret_cast(t.data()), false); + d["strides"] = py::none(); // C-contiguous + d["version"] = 3; + // No "stream" key: the producing stream is the caller's and never implicitly synced. + // Synchronize that stream before handing this to another library (documented). + return d; +} + +py::dict arrayInterface(const Tensor &t) { + if (t.device() != Device::kHost) { + throw py::attribute_error("__array_interface__ is only available on host tensors; use __cuda_array_interface__"); + } + py::tuple shape(t.shape().rank()); + for (int i = 0; i < t.shape().rank(); ++i) { + shape[i] = t.shape()[i]; + } + py::dict d; + d["shape"] = shape; + d["typestr"] = typestrOf(t.dtype()); + d["data"] = py::make_tuple(reinterpret_cast(t.data()), false); + d["strides"] = py::none(); + d["version"] = 3; + return d; +} + +} // namespace + +PYBIND11_MODULE(_core, m) { + m.doc() = "tensorrt_cpp_api: a no-throw C++ TensorRT inference library (Python bindings)"; + + errorClass() = py::reinterpret_steal(PyErr_NewException("trtcpp._core.TrtcppError", PyExc_RuntimeError, nullptr)); + m.attr("TrtcppError") = errorClass(); + + // ---- enums ---- + py::enum_(m, "StatusCode") + .value("Ok", StatusCode::kOk) + .value("InvalidArgument", StatusCode::kInvalidArgument) + .value("NotFound", StatusCode::kNotFound) + .value("IoError", StatusCode::kIoError) + .value("CudaError", StatusCode::kCudaError) + .value("TensorRtError", StatusCode::kTensorRtError) + .value("ShapeMismatch", StatusCode::kShapeMismatch) + .value("DtypeMismatch", StatusCode::kDtypeMismatch) + .value("Unsupported", StatusCode::kUnsupported) + .value("StaleCache", StatusCode::kStaleCache) + .value("Internal", StatusCode::kInternal); + + py::enum_(m, "DType") + .value("Float32", DType::kFloat32) + .value("Float16", DType::kFloat16) + .value("BFloat16", DType::kBFloat16) + .value("Int32", DType::kInt32) + .value("Int64", DType::kInt64) + .value("Int8", DType::kInt8) + .value("UInt8", DType::kUInt8) + .value("Bool", DType::kBool) + .value("Fp8", DType::kFp8) + .value("Int4", DType::kInt4); + m.def("dtype_to_string", [](DType t) { return std::string(toString(t)); }); + + py::enum_(m, "Device").value("Cuda", Device::kCuda).value("Host", Device::kHost); + py::enum_(m, "Layout") + .value("NCHW", Layout::kNCHW) + .value("NHWC", Layout::kNHWC) + .value("Linear", Layout::kLinear) + .value("Unknown", Layout::kUnknown); + py::enum_(m, "Precision") + .value("Fp32", Precision::kFp32) + .value("Fp16", Precision::kFp16) + .value("Int8Qdq", Precision::kInt8Qdq) + .value("Int8CalibLegacy", Precision::kInt8CalibLegacy) + .value("Fp8", Precision::kFp8) + .value("Nvfp4", Precision::kNvfp4); + + // ---- Shape ---- + py::class_(m, "Shape") + .def(py::init<>()) + .def(py::init([](const std::vector &dims) { return makeShape(dims); }), py::arg("dims")) + .def("rank", &Shape::rank) + .def("numel", &Shape::numel) + .def("is_dynamic", &Shape::isDynamic) + .def("dims", [](const Shape &s) { return std::vector(s.dims().begin(), s.dims().end()); }) + .def("__len__", &Shape::rank) + .def("__getitem__", [](const Shape &s, int i) { return s.at(i); }) + .def("__eq__", [](const Shape &a, const Shape &b) { return a == b; }) + .def("__repr__", [](const Shape &s) { return "Shape(" + s.toString() + ")"; }); + + // ---- Stream ---- + py::class_(m, "Stream") + .def(py::init([]() { return unwrap(Stream::create()); }), "Create and own a new non-blocking CUDA stream.") + .def_static( + "wrap", [](std::uintptr_t handle) { return Stream::wrap(handle); }, py::arg("handle"), + "Wrap an existing stream handle (e.g. torch.cuda.Stream.cuda_stream); non-owning.") + .def_property_readonly("handle", &Stream::raw, "The raw stream handle as an integer.") + .def_property_readonly("owns", &Stream::owns) + .def("synchronize", [](const Stream &s) { check(s.synchronize()); }); + + // ---- device query ---- + py::class_(m, "DeviceInfo") + .def_readonly("index", &DeviceInfo::index) + .def_readonly("name", &DeviceInfo::name) + .def_readonly("uuid", &DeviceInfo::uuid) + .def_readonly("compute_major", &DeviceInfo::computeMajor) + .def_readonly("compute_minor", &DeviceInfo::computeMinor) + .def_readonly("total_memory_bytes", &DeviceInfo::totalMemoryBytes) + .def("__repr__", [](const DeviceInfo &d) { + return "DeviceInfo(index=" + std::to_string(d.index) + ", name='" + d.name + "', sm=" + std::to_string(d.computeMajor) + "." + + std::to_string(d.computeMinor) + ")"; + }); + m.def("device_count", []() { return unwrap(deviceCount()); }); + m.def("query_device", [](int index) { return unwrap(queryDevice(index)); }, py::arg("index")); + + // ---- Tensor (owning) ---- + py::class_(m, "Tensor") + .def_static( + "allocate", + [](DType dtype, const Shape &shape, Device device, int deviceId) { + return unwrap(Tensor::allocate(dtype, shape, device, deviceId)); + }, + py::arg("dtype"), py::arg("shape"), py::arg("device"), py::arg("device_id") = 0) + .def_property_readonly("dtype", &Tensor::dtype) + .def_property_readonly("shape", &Tensor::shape) + .def_property_readonly("device", &Tensor::device) + .def_property_readonly("device_id", &Tensor::deviceId) + .def_property_readonly("nbytes", &Tensor::nbytes) + .def_property_readonly("data_ptr", [](const Tensor &t) { return reinterpret_cast(t.data()); }) + .def( + "copy_from", [](Tensor &t, py::handle src, const Stream &s) { check(t.copyFrom(toTensorView(src), s)); }, py::arg("src"), + py::arg("stream")) + .def( + "to", [](const Tensor &t, Device d, int deviceId, const Stream &s) { return unwrap(t.to(d, deviceId, s)); }, py::arg("device"), + py::arg("device_id"), py::arg("stream")) + .def( + "to_host", [](const Tensor &t, const Stream &s) { return unwrap(t.toHost(s)); }, py::arg("stream"), + "Copy to a pinned host Tensor AND synchronize the stream; the result is immediately readable.") + .def_property_readonly("__cuda_array_interface__", &cudaArrayInterface) + .def_property_readonly("__array_interface__", &arrayInterface) + .def( + "__dlpack__", [](py::object self, py::object /*stream*/) { return tensorDlpack(std::move(self)); }, + py::arg("stream") = py::none()) + .def("__dlpack_device__", [](const Tensor &t) { + return py::make_tuple(static_cast(t.device() == Device::kCuda ? kDLCUDA : kDLCPU), t.deviceId()); + }); + + // ---- TensorView (non-owning) ---- + py::class_(m, "TensorView") + .def_static( + "from_array", [](py::handle obj) { return toTensorView(obj); }, py::arg("array"), + "Build a zero-copy view over an object exposing __cuda_array_interface__ or __dlpack__.") + .def_property_readonly("dtype", &TensorView::dtype) + .def_property_readonly("shape", &TensorView::shape) + .def_property_readonly("device", &TensorView::device) + .def_property_readonly("device_id", &TensorView::deviceId) + .def_property_readonly("nbytes", &TensorView::nbytes) + .def_property_readonly("data_ptr", [](const TensorView &v) { return reinterpret_cast(v.data()); }); + + // ---- TensorInfo ---- + py::class_(m, "TensorInfo") + .def_readonly("name", &TensorInfo::name) + .def_readonly("is_input", &TensorInfo::isInput) + .def_readonly("dtype", &TensorInfo::dtype) + .def_readonly("shape", &TensorInfo::shape) + .def("__repr__", [](const TensorInfo &i) { + return "TensorInfo(name='" + i.name + "', input=" + (i.isInput ? "True" : "False") + ", shape=" + i.shape.toString() + ")"; + }); + + // ---- build options ---- + py::class_(m, "ProfileShape") + .def(py::init<>()) + .def_readwrite("input_name", &ProfileShape::inputName) + .def_readwrite("min", &ProfileShape::min) + .def_readwrite("opt", &ProfileShape::opt) + .def_readwrite("max", &ProfileShape::max); + py::class_(m, "OptimizationProfile").def(py::init<>()).def_readwrite("inputs", &OptimizationProfile::inputs); + py::class_(m, "BuildOptions") + .def(py::init<>()) + .def_readwrite("precision", &BuildOptions::precision) + .def_readwrite("profiles", &BuildOptions::profiles) + .def_readwrite("device_index", &BuildOptions::deviceIndex) + .def_readwrite("dla_core", &BuildOptions::dlaCore) + .def_readwrite("workspace_bytes", &BuildOptions::workspaceBytes) + .def_readwrite("strongly_typed", &BuildOptions::stronglyTyped) + .def_readwrite("version_compatible", &BuildOptions::versionCompatible) + .def_readwrite("hardware_compatible", &BuildOptions::hardwareCompatible) + .def_readwrite("engine_cache_dir", &BuildOptions::engineCacheDir) + .def_readwrite("timing_cache_path", &BuildOptions::timingCachePath) + .def_readwrite("plugin_libraries", &BuildOptions::pluginLibraries); + py::class_(m, "EngineOptions") + .def(py::init<>()) + .def_readwrite("device_index", &EngineOptions::deviceIndex) + .def_readwrite("plugin_libraries", &EngineOptions::pluginLibraries); + + // ---- EngineBuilder ---- + py::class_(m, "EngineBuilder") + .def(py::init<>()) + .def( + "build_from_onnx_file", + [](EngineBuilder &b, const std::string &path, const BuildOptions &opts) { + std::optional>> r; + { + py::gil_scoped_release rel; + r.emplace(b.buildFromOnnxFile(path, opts)); + } + auto bytes = unwrap(std::move(*r)); + return py::bytes(reinterpret_cast(bytes.data()), bytes.size()); + }, + py::arg("onnx_path"), py::arg("options")) + .def( + "build_or_load", + [](EngineBuilder &b, const std::string &path, const BuildOptions &opts) { + std::optional> r; + { + py::gil_scoped_release rel; + r.emplace(b.buildOrLoad(path, opts)); + } + return unwrap(std::move(*r)); + }, + py::arg("onnx_path"), py::arg("options")) + .def( + "build_and_load", + [](EngineBuilder &b, const std::string &path, const BuildOptions &opts, const EngineOptions &eopts) { + std::optional> r; + { + py::gil_scoped_release rel; + r.emplace(b.buildAndLoad(path, opts, eopts)); + } + return unwrap(std::move(*r)); + }, + py::arg("onnx_path"), py::arg("options"), py::arg("engine_options") = EngineOptions{}); + + // ---- Engine ---- + py::class_(m, "Engine") + .def_static( + "load_from_file", + [](const std::string &path, const EngineOptions &opts) { + std::optional> r; + { + py::gil_scoped_release rel; + r.emplace(Engine::loadFromFile(path, opts)); + } + return unwrap(std::move(*r)); + }, + py::arg("engine_path"), py::arg("options") = EngineOptions{}) + .def_static( + "load_from_memory", + [](py::bytes data, const EngineOptions &opts) { + char *buf = nullptr; + ssize_t len = 0; + PYBIND11_BYTES_AS_STRING_AND_SIZE(data.ptr(), &buf, &len); + std::span span(reinterpret_cast(buf), static_cast(len)); + std::optional> r; + { + py::gil_scoped_release rel; + r.emplace(Engine::loadFromMemory(span, opts)); + } + return unwrap(std::move(*r)); + }, + py::arg("engine_data"), py::arg("options") = EngineOptions{}) + .def("tensors", &Engine::tensors) + .def("input_names", &Engine::inputNames) + .def("output_names", &Engine::outputNames) + .def("nb_optimization_profiles", &Engine::nbOptimizationProfiles) + .def( + "tensor_shape", [](const Engine &e, const std::string &n) { return unwrap(e.tensorShape(n)); }, py::arg("name")) + .def( + "tensor_dtype", [](const Engine &e, const std::string &n) { return unwrap(e.tensorDType(n)); }, py::arg("name")) + .def( + "enqueue", + [](Engine &e, const py::dict &inputs, const py::dict &outputs, const Stream &s, int profile) { + auto in = toViewMap(inputs); + auto out = toViewMap(outputs); + Status st; + { + py::gil_scoped_release rel; + st = e.enqueue(in, out, s, profile); + } + check(st); + }, + py::arg("inputs"), py::arg("outputs"), py::arg("stream"), py::arg("profile_index") = 0) + .def( + "infer", + [](Engine &e, const py::dict &inputs, const Stream &s, int profile) { + auto in = toViewMap(inputs); + std::optional>> r; + { + py::gil_scoped_release rel; + r.emplace(e.infer(in, s, profile)); + } + return tensorMapToDict(unwrap(std::move(*r))); + }, + py::arg("inputs"), py::arg("stream"), py::arg("profile_index") = 0) + .def( + "infer_single", + [](Engine &e, const py::dict &inputs, const Stream &s, int profile) { + auto in = toViewMap(inputs); + std::optional> r; + { + py::gil_scoped_release rel; + r.emplace(e.inferSingle(in, s, profile)); + } + return unwrap(std::move(*r)); + }, + py::arg("inputs"), py::arg("stream"), py::arg("profile_index") = 0) + .def( + "output_shapes", + [](Engine &e, const py::dict &inputs, int profile) { + auto in = toViewMap(inputs); + return unwrap(e.outputShapes(in, profile)); + }, + py::arg("inputs"), py::arg("profile_index") = 0); + + // ---- EnginePool + Lease ---- + py::class_(m, "Lease") + .def_property_readonly("valid", &EnginePool::Lease::valid) + .def_property_readonly("profile_index", &EnginePool::Lease::profileIndex) + .def( + "enqueue", + [](EnginePool::Lease &l, const py::dict &inputs, const py::dict &outputs, const Stream &s) { + auto in = toViewMap(inputs); + auto out = toViewMap(outputs); + Status st; + { + py::gil_scoped_release rel; + st = l.enqueue(in, out, s); + } + check(st); + }, + py::arg("inputs"), py::arg("outputs"), py::arg("stream")) + .def( + "infer", + [](EnginePool::Lease &l, const py::dict &inputs, const Stream &s) { + auto in = toViewMap(inputs); + std::optional>> r; + { + py::gil_scoped_release rel; + r.emplace(l.infer(in, s)); + } + return tensorMapToDict(unwrap(std::move(*r))); + }, + py::arg("inputs"), py::arg("stream")) + .def( + "infer_single", + [](EnginePool::Lease &l, const py::dict &inputs, const Stream &s) { + auto in = toViewMap(inputs); + std::optional> r; + { + py::gil_scoped_release rel; + r.emplace(l.inferSingle(in, s)); + } + return unwrap(std::move(*r)); + }, + py::arg("inputs"), py::arg("stream")); + + py::class_(m, "EnginePool") + .def_static( + "create", + [](const std::string &path, int contexts, const EngineOptions &opts) { + std::optional> r; + { + py::gil_scoped_release rel; + r.emplace(EnginePool::create(path, contexts, opts)); + } + return unwrap(std::move(*r)); + }, + py::arg("engine_path"), py::arg("contexts"), py::arg("options") = EngineOptions{}) + .def( + "acquire", + [](EnginePool &p) { + std::optional> r; + { + py::gil_scoped_release rel; + r.emplace(p.acquire()); + } + return unwrap(std::move(*r)); + }, + "Block until a context is free, then return a Lease.") + .def("try_acquire", + [](EnginePool &p) -> py::object { + auto lease = p.tryAcquire(); + if (!lease) { + return py::none(); + } + return py::cast(std::move(*lease)); + }) + .def("size", &EnginePool::size) + .def("tensors", &EnginePool::tensors) + .def("input_names", &EnginePool::inputNames) + .def("output_names", &EnginePool::outputNames); + + // ---- version ---- + m.def("version_string", &versionString); + m.def("library_version", []() { + auto v = libraryVersion(); + return py::make_tuple(v.major, v.minor, v.patch); + }); + m.def("tensorrt_build_version", []() { return py::make_tuple(tensorrtBuildMajor(), tensorrtBuildMinor()); }); + +#ifdef TRT_CPP_API_PYTHON_WITH_PREPROC + auto pre = m.def_submodule("preproc", "Fused GPU preprocessing (letterbox-resize -> normalize -> NCHW)."); + py::class_(pre, "PreprocSpec") + .def(py::init<>()) + .def_readwrite("mean", &preproc::PreprocSpec::mean) + .def_readwrite("scale", &preproc::PreprocSpec::scale) + .def_readwrite("swap_rb", &preproc::PreprocSpec::swapRB) + .def_readwrite("keep_aspect_ratio_pad", &preproc::PreprocSpec::keepAspectRatioPad) + .def_readwrite("pad_value", &preproc::PreprocSpec::padValue); + pre.def( + "letterbox_to_tensor", + [](py::handle src, py::handle dst, const preproc::PreprocSpec &spec, const Stream &s) { + auto srcView = toTensorView(src); + auto dstView = toTensorView(dst); + Status st; + { + py::gil_scoped_release rel; + st = preproc::letterboxToTensor(srcView, dstView, spec, s); + } + check(st); + }, + py::arg("src"), py::arg("dst"), py::arg("spec"), py::arg("stream")); +#endif +} diff --git a/python/src/dlpack.h b/python/src/dlpack.h new file mode 100644 index 0000000..f901169 --- /dev/null +++ b/python/src/dlpack.h @@ -0,0 +1,53 @@ +#pragma once + +// Minimal vendored subset of the DLPack ABI (https://github.com/dmlc/dlpack), the stable +// cross-framework tensor exchange used by PyTorch/CuPy/JAX. We ship only the structures the +// bindings need to PRODUCE and CONSUME the classic "dltensor" capsule; the ABI is stable, so +// vendoring avoids a build dependency on the dlpack headers. + +#include + +extern "C" { + +typedef enum { + kDLCPU = 1, + kDLCUDA = 2, + kDLCUDAHost = 3, +} DLDeviceType; + +typedef struct { + int32_t device_type; + int32_t device_id; +} DLDevice; + +typedef enum { + kDLInt = 0U, + kDLUInt = 1U, + kDLFloat = 2U, + kDLBfloat = 4U, + kDLBool = 6U, +} DLDataTypeCode; + +typedef struct { + uint8_t code; + uint8_t bits; + uint16_t lanes; +} DLDataType; + +typedef struct { + void *data; + DLDevice device; + int32_t ndim; + DLDataType dtype; + int64_t *shape; + int64_t *strides; // nullptr => compact row-major (C-contiguous) + uint64_t byte_offset; +} DLTensor; + +typedef struct DLManagedTensor { + DLTensor dl_tensor; + void *manager_ctx; + void (*deleter)(struct DLManagedTensor *self); +} DLManagedTensor; + +} // extern "C" diff --git a/python/tests/test_bindings.py b/python/tests/test_bindings.py new file mode 100644 index 0000000..3acb195 --- /dev/null +++ b/python/tests/test_bindings.py @@ -0,0 +1,172 @@ +"""Tests for the trtcpp Python bindings. + +CPU/pure tests run anywhere; GPU tests skip when no CUDA device is visible. Run with: + PYTHONPATH=python pytest python/tests -q +(or, against an installed wheel, just `pytest python/tests`). +""" + +import os + +import numpy as np +import pytest + +import trtcpp +from trtcpp import BuildOptions, DType, Device, EngineBuilder, EnginePool, Precision, Shape, Stream, Tensor, TensorView, TrtcppError + +MODELS = os.path.join(os.path.dirname(__file__), "..", "..", "tests", "models") +RELU = os.path.join(MODELS, "relu_1x3x8x8.onnx") + +has_gpu = trtcpp.device_count() > 0 +gpu = pytest.mark.skipif(not has_gpu, reason="no CUDA device") + + +# ---- pure / CPU ---- + +def test_version(): + assert trtcpp.library_version() == (7, 0, 0) + assert trtcpp.tensorrt_build_version()[0] >= 10 + assert "7.0.0" in trtcpp.version_string() + assert trtcpp.__version__ == "7.0.0" + + +def test_shape(): + s = Shape([1, 3, 224, 224]) + assert s.rank() == 4 + assert len(s) == 4 + assert s[2] == 224 + assert s.numel() == 1 * 3 * 224 * 224 + assert not s.is_dynamic() + assert Shape([-1, 3]).is_dynamic() + assert s.dims() == [1, 3, 224, 224] + assert Shape([1, 2]) == Shape([1, 2]) + + +def test_enums(): + assert trtcpp.dtype_to_string(DType.Float16) == "float16" + assert DType.Float32 != DType.Float16 + assert Precision.Int8Qdq is not None + + +def test_error_type(): + assert issubclass(TrtcppError, Exception) + with pytest.raises(TrtcppError) as ei: + trtcpp.Engine.load_from_file("/nonexistent/does_not_exist.engine") + assert ei.value.code == trtcpp.StatusCode.NotFound + + +# ---- GPU ---- + +@gpu +def test_device_query(): + info = trtcpp.query_device(0) + assert info.compute_major >= 5 + assert info.total_memory_bytes > 0 + assert info.name + + +@gpu +def test_tensor_alloc_and_interfaces(): + t = Tensor.allocate(DType.Float32, Shape([2, 3]), Device.Cuda) + assert t.nbytes == 2 * 3 * 4 + cai = t.__cuda_array_interface__ + assert cai["shape"] == (2, 3) + assert cai["typestr"] == " host view (DLPack) -> H2D + back = np.asarray(dev.to_host(s)) # D2H + sync, then __array_interface__ + assert np.array_equal(back, src) + + +@gpu +def test_cuda_array_interface_import_is_zero_copy(): + t = Tensor.allocate(DType.Float32, Shape([2, 5]), Device.Cuda) + + class Producer: + def __init__(self, src): + self.__cuda_array_interface__ = src.__cuda_array_interface__ + + view = TensorView.from_array(Producer(t)) + assert view.data_ptr == t.data_ptr + assert view.shape.dims() == [2, 5] + assert view.device == Device.Cuda + + +@pytest.fixture(scope="module") +def relu_engine_path(tmp_path_factory): + cache = str(tmp_path_factory.mktemp("engines")) + bo = BuildOptions() + bo.precision = Precision.Fp32 + bo.engine_cache_dir = cache + return EngineBuilder().build_or_load(RELU, bo) + + +@gpu +def test_engine_infer_relu(relu_engine_path): + s = Stream() + eng = trtcpp.Engine.load_from_file(relu_engine_path) + assert eng.input_names() == ["input"] + assert eng.output_names() == ["output"] + x = np.random.randn(1, 3, 8, 8).astype(np.float32) + dx = Tensor.allocate(DType.Float32, Shape([1, 3, 8, 8]), Device.Cuda) + dx.copy_from(x, s) + out = eng.infer_single({"input": dx}, s) + y = np.asarray(out.to_host(s)) + assert y.shape == (1, 3, 8, 8) + assert np.allclose(y, np.maximum(x, 0), atol=1e-4) + + +@gpu +def test_engine_infer_dict_and_output_shapes(relu_engine_path): + s = Stream() + eng = trtcpp.Engine.load_from_file(relu_engine_path) + x = np.zeros((1, 3, 8, 8), dtype=np.float32) + dx = Tensor.allocate(DType.Float32, Shape([1, 3, 8, 8]), Device.Cuda) + dx.copy_from(x, s) + shapes = eng.output_shapes({"input": dx}) + assert shapes["output"].dims() == [1, 3, 8, 8] + outs = eng.infer({"input": dx}, s) + assert set(outs) == {"output"} + assert np.asarray(outs["output"].to_host(s)).shape == (1, 3, 8, 8) + + +@gpu +def test_engine_pool(relu_engine_path): + pool = EnginePool.create(relu_engine_path, contexts=2) + assert pool.size() == 2 + s = Stream() + x = np.full((1, 3, 8, 8), -1.0, dtype=np.float32) + dx = Tensor.allocate(DType.Float32, Shape([1, 3, 8, 8]), Device.Cuda) + dx.copy_from(x, s) + lease = pool.acquire() + assert lease.valid + out = lease.infer_single({"input": dx}, s) + y = np.asarray(out.to_host(s)) + assert np.allclose(y, 0.0) # relu of all-negative + + +@gpu +@pytest.mark.skipif(trtcpp.preproc is None, reason="preproc sublib not built") +def test_preproc_letterbox(): + s = Stream() + # HWC uint8 image, resize/normalize to NCHW float32. + img = (np.random.rand(16, 16, 3) * 255).astype(np.uint8) + src = Tensor.allocate(DType.UInt8, Shape([16, 16, 3]), Device.Cuda) + src.copy_from(img, s) + dst = Tensor.allocate(DType.Float32, Shape([1, 3, 8, 8]), Device.Cuda) + spec = trtcpp.preproc.PreprocSpec() + spec.scale = [1.0 / 255, 1.0 / 255, 1.0 / 255, 1.0] + trtcpp.preproc.letterbox_to_tensor(src, dst, spec, s) + y = np.asarray(dst.to_host(s)) + assert y.shape == (1, 3, 8, 8) + assert 0.0 <= y.min() and y.max() <= 1.0 + 1e-4 diff --git a/python/trtcpp/__init__.py b/python/trtcpp/__init__.py new file mode 100644 index 0000000..bd0119b --- /dev/null +++ b/python/trtcpp/__init__.py @@ -0,0 +1,84 @@ +"""tensorrt_cpp_api - a no-throw C++ TensorRT inference library, exposed to Python. + +The whole API lives in the compiled ``_core`` extension; this package re-exports it and adds +the package version. The design mirrors the C++ surface: a no-throw core where every fallible +call raises :class:`TrtcppError` (carrying ``.code`` and the message) instead of returning a +status. + +Zero-copy contract: + * Inputs are dicts ``{name: array}`` where ``array`` is a :class:`Tensor` / :class:`TensorView` + or any object exposing ``__cuda_array_interface__`` (CuPy, Numba, PyTorch CUDA tensors) or + ``__dlpack__``. No host<->device copy happens at the boundary. + * Outputs are :class:`Tensor` objects exposing ``__cuda_array_interface__`` (device) / + ``__array_interface__`` (host) and ``__dlpack__`` / ``__dlpack_device__`` -- so + ``torch.from_dlpack(out)`` and ``cupy.asarray(out)`` alias the device memory. + +Because inference is asynchronous on the caller's stream and never implicitly synchronizes, +you MUST keep input arrays alive and synchronize the stream before consuming outputs. +""" + +from ._core import ( # noqa: F401 + BuildOptions, + DType, + Device, + DeviceInfo, + Engine, + EngineBuilder, + EngineOptions, + EnginePool, + Layout, + Lease, + OptimizationProfile, + Precision, + ProfileShape, + Shape, + StatusCode, + Stream, + Tensor, + TensorInfo, + TensorView, + TrtcppError, + device_count, + dtype_to_string, + library_version, + query_device, + tensorrt_build_version, + version_string, +) + +try: + from ._core import preproc # noqa: F401 +except ImportError: # built without the preprocessing sublibrary + preproc = None + +__all__ = [ + "BuildOptions", + "DType", + "Device", + "DeviceInfo", + "Engine", + "EngineBuilder", + "EngineOptions", + "EnginePool", + "Layout", + "Lease", + "OptimizationProfile", + "Precision", + "ProfileShape", + "Shape", + "StatusCode", + "Stream", + "Tensor", + "TensorInfo", + "TensorView", + "TrtcppError", + "device_count", + "dtype_to_string", + "library_version", + "preproc", + "query_device", + "tensorrt_build_version", + "version_string", +] + +__version__ = ".".join(str(v) for v in library_version()) diff --git a/scripts/install_deps.sh b/scripts/install_deps.sh new file mode 100755 index 0000000..fa02dd6 --- /dev/null +++ b/scripts/install_deps.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash +set -euo pipefail + +# install_deps.sh — authoritative dependency installer for tensorrt-cpp-api (v7) +# +# Installs the system packages needed to BUILD the library on a fresh Ubuntu +# 22.04 or 24.04 x86_64 host: CUDA Toolkit, TensorRT dev, OpenCV dev, spdlog, +# fmt, and the usual build tooling. Optionally installs Python build deps. +# +# It does NOT install the NVIDIA kernel driver (too host-specific) and it does +# NOT build CUDA-enabled OpenCV from source (see docs/install/from_source.md and +# scripts/build_opencv.sh for that). It verifies a compatible driver is present +# and fails fast with a pointer if not. +# +# Safe to re-run: every apt operation is idempotent. + +# --------------------------------------------------------------------------- # +# Defaults / flags +# --------------------------------------------------------------------------- # +CUDA_VERSION="" # e.g. 12.6 or 13.0; empty => auto-detect from driver +WITH_PYTHON=0 +DRY_RUN=0 +NO_SUDO=0 + +usage() { + cat <<'EOF' +Usage: scripts/install_deps.sh [OPTIONS] + +Installs the build dependencies for tensorrt-cpp-api on Ubuntu 22.04/24.04. + +Options: + --cuda-version= CUDA toolkit version to install (e.g. 12.6, 13.0). + Default: auto-detect the newest version the installed + NVIDIA driver supports. + --with-python Also install Python build deps (python3-dev, venv, pip, + pybind11-dev) for the optional Python bindings. + --dry-run Print the commands that would run; change nothing. + --no-sudo Print the privileged commands instead of running them + (for hosts with custom sudo policies). Implies the + caller will run them manually. + -h, --help Show this help and exit. + +Examples: + scripts/install_deps.sh # auto CUDA, no Python + scripts/install_deps.sh --with-python # + Python bindings deps + scripts/install_deps.sh --cuda-version=12.6 # pin CUDA 12.6 + scripts/install_deps.sh --no-sudo # print privileged commands + +Notes: + * Does NOT install the NVIDIA driver. CUDA 13.x needs driver >= 580; CUDA 12.x + needs driver >= 525. If the driver is too old this script stops with a + pointer to https://www.nvidia.com/Download/index.aspx . + * apt `libopencv-dev` does NOT include the CUDA (cv::cuda) modules. For those, + build OpenCV from source: docs/install/from_source.md. +EOF +} + +for arg in "$@"; do + case "$arg" in + --cuda-version=*) CUDA_VERSION="${arg#*=}" ;; + --with-python) WITH_PYTHON=1 ;; + --dry-run) DRY_RUN=1 ;; + --no-sudo) NO_SUDO=1 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $arg" >&2; usage >&2; exit 2 ;; + esac +done + +# --------------------------------------------------------------------------- # +# Helpers +# --------------------------------------------------------------------------- # +banner() { printf '\n\033[1;36m[%s] ==== %s ====\033[0m\n' "$(date '+%H:%M:%S')" "$*"; } +info() { printf '\033[0;32m[%s] %s\033[0m\n' "$(date '+%H:%M:%S')" "$*"; } +warn() { printf '\033[0;33m[%s] WARN: %s\033[0m\n' "$(date '+%H:%M:%S')" "$*" >&2; } +die() { printf '\033[0;31m[%s] ERROR: %s\033[0m\n' "$(date '+%H:%M:%S')" "$*" >&2; exit 1; } + +# Run a privileged command, honoring --dry-run / --no-sudo. +SUDO="sudo" +if [[ "$(id -u)" -eq 0 ]]; then SUDO=""; fi +priv() { + if [[ "$DRY_RUN" -eq 1 || "$NO_SUDO" -eq 1 ]]; then + echo " + ${SUDO:+sudo }$*" + else + # shellcheck disable=SC2086 + $SUDO "$@" + fi +} + +# --------------------------------------------------------------------------- # +# Detect distro +# --------------------------------------------------------------------------- # +banner "Detecting host" +[[ -r /etc/os-release ]] || die "/etc/os-release not found; this script targets Ubuntu." +# shellcheck disable=SC1091 +. /etc/os-release +[[ "${ID:-}" == "ubuntu" ]] || die "Unsupported distro '${ID:-?}'. This script supports Ubuntu 22.04/24.04." +case "${VERSION_ID:-}" in + 22.04) NV_DISTRO="ubuntu2204" ;; + 24.04) NV_DISTRO="ubuntu2404" ;; + *) die "Unsupported Ubuntu ${VERSION_ID:-?}. Supported: 22.04, 24.04." ;; +esac +ARCH="$(uname -m)" +[[ "$ARCH" == "x86_64" ]] || warn "Architecture is $ARCH; the NVIDIA apt repo path below targets x86_64. For Jetson/ARM64 see docs/install/from_source.md." +info "Ubuntu ${VERSION_ID} (${NV_DISTRO}), arch ${ARCH}" + +# --------------------------------------------------------------------------- # +# Detect driver, decide CUDA version, fail fast on incompatibility +# --------------------------------------------------------------------------- # +banner "Checking NVIDIA driver" +command -v nvidia-smi >/dev/null 2>&1 || die "nvidia-smi not found. Install an NVIDIA driver first (this script does not): https://www.nvidia.com/Download/index.aspx" + +DRIVER_VER="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -1 || true)" +DRIVER_MAJOR="${DRIVER_VER%%.*}" +# Max CUDA the driver supports, as printed by nvidia-smi ("CUDA Version: X.Y"). +DRIVER_MAX_CUDA="$(nvidia-smi 2>/dev/null | sed -n 's/.*CUDA Version: \([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -1 || true)" +info "Driver ${DRIVER_VER:-unknown} supports up to CUDA ${DRIVER_MAX_CUDA:-unknown}" + +if [[ -z "$CUDA_VERSION" ]]; then + if [[ -n "$DRIVER_MAX_CUDA" ]]; then + # Default to the driver's max supported CUDA (a known-good toolkit minor + # for that major is chosen below). + CUDA_VERSION="$DRIVER_MAX_CUDA" + else + CUDA_VERSION="12.6" + warn "Could not read max CUDA from driver; defaulting to ${CUDA_VERSION}" + fi + info "Auto-selected CUDA ${CUDA_VERSION}" +fi +CUDA_MAJOR="${CUDA_VERSION%%.*}" +CUDA_MINOR="${CUDA_VERSION#*.}" + +# Driver-major floor per CUDA major (Linux). Conservative. +case "$CUDA_MAJOR" in + 13) MIN_DRIVER=580 ;; + 12) MIN_DRIVER=525 ;; + *) MIN_DRIVER=0; warn "Unrecognized CUDA major ${CUDA_MAJOR}; skipping driver floor check." ;; +esac +if [[ "${DRIVER_MAJOR:-0}" -lt "$MIN_DRIVER" ]]; then + die "Driver ${DRIVER_VER} is too old for CUDA ${CUDA_VERSION} (needs >= ${MIN_DRIVER}.x). + Either upgrade the NVIDIA driver (https://www.nvidia.com/Download/index.aspx) + or pin a lower CUDA with --cuda-version= (your driver supports up to ${DRIVER_MAX_CUDA:-?})." +fi + +# Map CUDA major -> a stable toolkit minor available in the apt repo, unless the +# caller pinned an explicit minor. +APT_CUDA_PKG="cuda-toolkit-${CUDA_MAJOR}-${CUDA_MINOR}" +info "Will install ${APT_CUDA_PKG}" + +# --------------------------------------------------------------------------- # +# Add the NVIDIA CUDA apt repo (idempotent). This repo also serves TensorRT and +# cuDNN packages and does NOT require an NVIDIA developer login. +# --------------------------------------------------------------------------- # +banner "Adding NVIDIA apt repo (cuda-keyring)" +KEYRING_DEB="cuda-keyring_1.1-1_all.deb" +REPO_BASE="https://developer.download.nvidia.com/compute/cuda/repos/${NV_DISTRO}/x86_64" +if dpkg -s cuda-keyring >/dev/null 2>&1; then + info "cuda-keyring already installed; skipping repo add." +else + info "Fetching ${REPO_BASE}/${KEYRING_DEB}" + if [[ "$DRY_RUN" -eq 1 ]]; then + echo " + wget ${REPO_BASE}/${KEYRING_DEB} && sudo dpkg -i ${KEYRING_DEB}" + else + TMP="$(mktemp -d)" + wget -q -O "${TMP}/${KEYRING_DEB}" "${REPO_BASE}/${KEYRING_DEB}" || die "Failed to download cuda-keyring." + priv dpkg -i "${TMP}/${KEYRING_DEB}" + rm -rf "$TMP" + fi +fi + +banner "apt update" +priv apt-get update + +# --------------------------------------------------------------------------- # +# Install packages +# --------------------------------------------------------------------------- # +BUILD_PKGS=(build-essential git pkg-config cmake ninja-build ccache) +NV_PKGS=("$APT_CUDA_PKG" + libnvinfer-dev libnvinfer-headers-dev libnvinfer-plugin-dev + libnvinfer-headers-plugin-dev libnvonnxparsers-dev) +LIB_PKGS=(libopencv-dev libspdlog-dev libfmt-dev) +PY_PKGS=() +if [[ "$WITH_PYTHON" -eq 1 ]]; then + PY_PKGS=(python3-dev python3-pip python3-venv pybind11-dev) +fi + +banner "Installing build tooling" +priv apt-get install -y "${BUILD_PKGS[@]}" + +banner "Installing CUDA toolkit + TensorRT" +priv apt-get install -y "${NV_PKGS[@]}" + +banner "Installing OpenCV / spdlog / fmt" +priv apt-get install -y "${LIB_PKGS[@]}" + +if [[ "$WITH_PYTHON" -eq 1 ]]; then + banner "Installing Python build deps" + priv apt-get install -y "${PY_PKGS[@]}" +fi + +# --------------------------------------------------------------------------- # +# Verify +# --------------------------------------------------------------------------- # +if [[ "$DRY_RUN" -eq 1 || "$NO_SUDO" -eq 1 ]]; then + banner "Done (no changes were made)" + info "Re-run without --dry-run/--no-sudo to actually install." + exit 0 +fi + +banner "Verifying install" +RC=0 +if [[ -x "/usr/local/cuda-${CUDA_MAJOR}.${CUDA_MINOR}/bin/nvcc" ]]; then + "/usr/local/cuda-${CUDA_MAJOR}.${CUDA_MINOR}/bin/nvcc" --version | tail -1 +elif command -v nvcc >/dev/null 2>&1; then + nvcc --version | tail -1 +else + warn "nvcc not on PATH. Add /usr/local/cuda-${CUDA_MAJOR}.${CUDA_MINOR}/bin to PATH." +fi +dpkg -s libnvinfer-dev >/dev/null 2>&1 && info "libnvinfer-dev: $(dpkg-query -W -f='${Version}' libnvinfer-dev)" || { warn "libnvinfer-dev not installed"; RC=1; } +dpkg -s libspdlog-dev >/dev/null 2>&1 && info "libspdlog-dev OK" || { warn "libspdlog-dev not installed"; RC=1; } +dpkg -s libfmt-dev >/dev/null 2>&1 && info "libfmt-dev OK" || { warn "libfmt-dev not installed"; RC=1; } +if [[ "$WITH_PYTHON" -eq 1 ]]; then + if python3 -c 'import pybind11; print("pybind11", pybind11.__version__)' 2>/dev/null; then :; + else warn "pybind11 not importable from system python (ok: scikit-build-core provides it in an isolated build env)."; fi +fi + +if [[ "$RC" -ne 0 ]]; then + die "One or more required packages are missing. See warnings above and docs/install/troubleshooting.md." +fi +banner "All dependencies installed. Next: cmake -B build -G Ninja && cmake --build build" diff --git a/scripts/verify_deps.sh b/scripts/verify_deps.sh new file mode 100755 index 0000000..fcb05c6 --- /dev/null +++ b/scripts/verify_deps.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -uo pipefail + +# verify_deps.sh — non-destructive readiness check for building tensorrt-cpp-api. +# Exits 0 if the host looks ready to build, 1 otherwise. Installs nothing. +# Useful in CI and for users diagnosing install problems. + +ok() { printf ' \033[0;32mOK\033[0m %s\n' "$*"; } +bad() { printf ' \033[0;31mFAIL\033[0m %s\n' "$*"; FAILED=1; } +note() { printf ' \033[0;33m..\033[0m %s\n' "$*"; } +FAILED=0 + +echo "tensorrt-cpp-api dependency check" +echo "=================================" + +# Compilers / build tools -------------------------------------------------- # +echo "Toolchain:" +if command -v cmake >/dev/null 2>&1; then + CM_VER="$(cmake --version | head -1 | awk '{print $3}')" + if printf '%s\n3.22\n' "$CM_VER" | sort -V | head -1 | grep -qx "3.22"; then + ok "cmake $CM_VER (>= 3.22)" + else + bad "cmake $CM_VER is older than the required 3.22" + fi +else + bad "cmake not found" +fi +command -v g++ >/dev/null 2>&1 && ok "g++ $(g++ -dumpversion)" || bad "g++ not found" +command -v ninja >/dev/null 2>&1 && ok "ninja $(ninja --version)" || note "ninja not found (optional; Make works too)" +command -v ccache >/dev/null 2>&1 && ok "ccache present" || note "ccache not found (optional)" + +# CUDA --------------------------------------------------------------------- # +echo "CUDA:" +NVCC="" +if command -v nvcc >/dev/null 2>&1; then NVCC="nvcc"; +elif [[ -x /usr/local/cuda/bin/nvcc ]]; then NVCC="/usr/local/cuda/bin/nvcc"; fi +if [[ -n "$NVCC" ]]; then + ok "nvcc $($NVCC --version | sed -n 's/.*release \([0-9.]*\).*/\1/p' | head -1) ($NVCC)" +else + bad "nvcc not found (add /usr/local/cuda/bin to PATH, or install CUDA toolkit)" +fi +if command -v nvidia-smi >/dev/null 2>&1; then + ok "driver $(nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -1), GPU $(nvidia-smi --query-gpu=name --format=csv,noheader | head -1)" +else + note "nvidia-smi not found (no GPU visible; CPU-only build/lint still possible)" +fi + +# TensorRT ----------------------------------------------------------------- # +echo "TensorRT:" +TRT_HDR="" +for d in /usr/include/x86_64-linux-gnu /usr/include "${TensorRT_DIR:-}/include"; do + [[ -n "$d" && -f "$d/NvInferVersion.h" ]] && { TRT_HDR="$d/NvInferVersion.h"; break; } +done +if [[ -n "$TRT_HDR" ]]; then + TRT_MAJ=$(sed -n 's/^#define NV_TENSORRT_MAJOR \([0-9]*\).*/\1/p' "$TRT_HDR") + TRT_MIN=$(sed -n 's/^#define NV_TENSORRT_MINOR \([0-9]*\).*/\1/p' "$TRT_HDR") + if [[ "${TRT_MAJ:-0}" -ge 10 ]]; then + ok "TensorRT ${TRT_MAJ}.${TRT_MIN} headers ($TRT_HDR)" + else + bad "TensorRT ${TRT_MAJ}.${TRT_MIN} found, but >= 10.0 is required" + fi +else + bad "TensorRT headers not found (set TensorRT_DIR or install libnvinfer-headers-dev)" +fi + +# OpenCV ------------------------------------------------------------------- # +echo "OpenCV (optional interop):" +OCV_CFG="$(find /usr/local/lib/cmake/opencv4 /usr/lib/*/cmake/opencv4 -name 'OpenCVConfig.cmake' 2>/dev/null | head -1)" +if [[ -n "$OCV_CFG" ]]; then + ok "OpenCV cmake config: $OCV_CFG" + if find "$(dirname "$OCV_CFG")/../../.." -name 'libopencv_cudaarithm*' 2>/dev/null | grep -q .; then + ok "OpenCV has CUDA modules (cv::cuda available)" + else + note "OpenCV present but WITHOUT CUDA modules (cv::cuda interop unavailable; see from_source.md)" + fi +else + note "OpenCV not found (engine core does not require it; needed only for OpenCV interop)" +fi + +# spdlog / fmt ------------------------------------------------------------- # +echo "Logging (optional spdlog adapter):" +if find /usr/include /usr/local/include -path '*spdlog/spdlog.h' 2>/dev/null | grep -q .; then + ok "spdlog headers present" +else + note "spdlog not found (engine core uses a built-in stderr logger by default)" +fi + +echo "=================================" +if [[ "$FAILED" -eq 0 ]]; then + echo "Result: host is READY to build tensorrt-cpp-api." + exit 0 +else + echo "Result: host is MISSING required dependencies. Run scripts/install_deps.sh" + echo " or see docs/install/troubleshooting.md." + exit 1 +fi diff --git a/src/allocator.cpp b/src/allocator.cpp new file mode 100644 index 0000000..cbdcb29 --- /dev/null +++ b/src/allocator.cpp @@ -0,0 +1,57 @@ +#include "tensorrt_cpp_api/allocator.h" + +#include + +namespace trtcpp { +namespace { + +/// Stream-ordered allocator over a private CUDA memory pool. The release threshold is +/// UINT64_MAX so freed blocks are retained for reuse instead of being returned to the OS +/// on each sync. A private pool (never the device default pool) keeps the library from +/// perturbing process-wide allocation state. +class StreamOrderedAllocator final : public IDeviceAllocator { +public: + explicit StreamOrderedAllocator(int deviceIndex) { + cudaMemPoolProps props{}; + props.allocType = cudaMemAllocationTypePinned; + props.handleTypes = cudaMemHandleTypeNone; + props.location.type = cudaMemLocationTypeDevice; + props.location.id = deviceIndex; + if (cudaMemPoolCreate(&pool_, &props) == cudaSuccess && pool_ != nullptr) { + std::uint64_t threshold = UINT64_MAX; + cudaMemPoolSetAttribute(pool_, cudaMemPoolAttrReleaseThreshold, &threshold); + } else { + pool_ = nullptr; + } + } + + ~StreamOrderedAllocator() override { + if (pool_ != nullptr) { + cudaMemPoolDestroy(pool_); + } + } + + void *allocate(std::size_t bytes, std::size_t /*alignment*/, const Stream &stream) override { + void *ptr = nullptr; + // If pool creation failed (catastrophic only), degrade to the device default pool + // -- a documented fallback rather than the private pool the class otherwise uses. + cudaError_t code = (pool_ != nullptr) ? cudaMallocFromPoolAsync(&ptr, bytes, pool_, stream.handle()) + : cudaMallocAsync(&ptr, bytes, stream.handle()); + return (code == cudaSuccess) ? ptr : nullptr; + } + + void deallocate(void *ptr, const Stream &stream) noexcept override { + if (ptr != nullptr) { + cudaFreeAsync(ptr, stream.handle()); + } + } + +private: + cudaMemPool_t pool_ = nullptr; +}; + +} // namespace + +std::shared_ptr defaultDeviceAllocator(int deviceIndex) { return std::make_shared(deviceIndex); } + +} // namespace trtcpp diff --git a/src/calibrator.cpp b/src/calibrator.cpp new file mode 100644 index 0000000..87994d2 --- /dev/null +++ b/src/calibrator.cpp @@ -0,0 +1,82 @@ +#include "tensorrt_cpp_api/calibrator.h" + +#if TRT_CPP_API_TENSORRT_VERSION_MAJOR < 11 + +#include "detail/engine_cache.h" // detail::readFile / writeAtomic + +#include + +#include +#include + +namespace trtcpp { +namespace { + +class RawBatchCalibrator final : public ICalibrator { +public: + RawBatchCalibrator(std::string inputName, Shape batchShape, std::vector batchFiles, std::string cachePath) + : inputName_(std::move(inputName)), batchShape_(std::move(batchShape)), batchFiles_(std::move(batchFiles)), + cachePath_(std::move(cachePath)) {} + + int batchSize() const override { return batchShape_.rank() > 0 ? static_cast(batchShape_[0]) : 1; } + + bool nextBatch(const std::unordered_map &inputs, const Stream &stream) override { + if (index_ >= batchFiles_.size()) { + return false; + } + auto it = inputs.find(inputName_); + if (it == inputs.end()) { + return false; + } + auto bytes = detail::readFile(batchFiles_[index_]); + if (!bytes || bytes.value().size() != it->second.nbytes()) { + return false; // missing file or size mismatch with the calibration buffer + } + if (cudaMemcpyAsync(it->second.data(), bytes.value().data(), bytes.value().size(), cudaMemcpyHostToDevice, stream.handle()) != + cudaSuccess) { + return false; + } + // Synchronize before the local host buffer is freed -- the async copy reads it, and a + // pageable-source H2D copy is not contractually host-synchronous. + if (cudaStreamSynchronize(stream.handle()) != cudaSuccess) { + return false; + } + ++index_; + return true; + } + + std::optional> readCache() override { + if (cachePath_.empty()) { + return std::nullopt; + } + auto bytes = detail::readFile(cachePath_); + if (!bytes) { + return std::nullopt; + } + return std::move(bytes.value()); + } + + void writeCache(std::span data) override { + if (!cachePath_.empty()) { + detail::writeAtomic(cachePath_, data); + } + } + +private: + std::string inputName_; + Shape batchShape_; + std::vector batchFiles_; + std::string cachePath_; + std::size_t index_ = 0; +}; + +} // namespace + +std::shared_ptr makeRawBatchCalibrator(std::string inputName, Shape batchShape, std::vector batchFiles, + std::string cachePath) { + return std::make_shared(std::move(inputName), std::move(batchShape), std::move(batchFiles), std::move(cachePath)); +} + +} // namespace trtcpp + +#endif // TRT_CPP_API_TENSORRT_VERSION_MAJOR < 11 diff --git a/src/cmd_line_parser.h b/src/cmd_line_parser.h deleted file mode 100644 index 3b1c0b5..0000000 --- a/src/cmd_line_parser.h +++ /dev/null @@ -1,98 +0,0 @@ -#pragma once -#include "engine.h" -#include - -struct CommandLineArguments { - std::string onnxModelPath = ""; - std::string trtModelPath = ""; -}; - -inline void showHelp(char *argv[]) { - std::cout << "Usage: " << argv[0] << " [OPTIONS]" << std::endl << std::endl; - - std::cout << "Options:" << std::endl; - std::cout << "--onnx_model Path to the ONNX model. " - "(Either onnx_model or trt_model must be provided)" - << std::endl; - std::cout << "--trt_model Path to the TensorRT model. " - "(Either onnx_model or trt_model must be provided)" - << std::endl; - - std::cout << "Example usage:" << std::endl; - std::cout << argv[0] << " --onnx_model model.onnx" << std::endl; -}; - -inline bool tryGetNextArgument(int argc, char *argv[], int ¤tIndex, std::string &value, std::string flag, bool printErrors = true) { - if (currentIndex + 1 >= argc) { - if (printErrors) - std::cout << "Error: No arguments provided for flag '" << flag << "'" << std::endl; - return false; - } - - std::string nextArgument = argv[currentIndex + 1]; - if (nextArgument.substr(0, 2) == "--") { - if (printErrors) - std::cout << "Error: No arguments provided for flag '" << flag << "'" << std::endl; - return false; - } - - value = argv[++currentIndex]; - return true; -}; - -inline bool parseArguments(int argc, char *argv[], CommandLineArguments &arguments) { - if (argc == 1) { - showHelp(argv); - return false; - } - - for (int i = 1; i < argc; i++) { - std::string argument = argv[i]; - - if (argument.substr(0, 2) == "--") { - std::string flag = argument.substr(2); - std::string nextArgument; - - if (flag == "onnx_model") { - if (!tryGetNextArgument(argc, argv, i, nextArgument, flag)) - return false; - - if (!Util::doesFileExist(nextArgument)) { - std::cout << "Error: Unable to find model at path '" << nextArgument << "' for flag '" << flag << "'" << std::endl; - return false; - } - - arguments.onnxModelPath = nextArgument; - } - - else if (flag == "trt_model") { - if (!tryGetNextArgument(argc, argv, i, nextArgument, flag)) - return false; - - if (!Util::doesFileExist(nextArgument)) { - std::cout << "Error: Unable to find model at path '" << nextArgument << "' for flag '" << flag << "'" << std::endl; - return false; - } - - arguments.trtModelPath = nextArgument; - } - - else { - std::cout << "Error: Unknown flag '" << flag << "'" << std::endl; - showHelp(argv); - return false; - } - } else { - std::cout << "Error: Unknown argument '" << argument << "'" << std::endl; - showHelp(argv); - return false; - } - } - - if (arguments.onnxModelPath.empty() && arguments.trtModelPath.empty()) { - std::cout << "Error: Must specify either 'onnx_model' or 'trt_model'" << std::endl; - return false; - } - - return true; -} diff --git a/src/core.cpp b/src/core.cpp new file mode 100644 index 0000000..4f9dc49 --- /dev/null +++ b/src/core.cpp @@ -0,0 +1,150 @@ +#include "tensorrt_cpp_api/dtype.h" +#include "tensorrt_cpp_api/layout.h" +#include "tensorrt_cpp_api/shape.h" +#include "tensorrt_cpp_api/status.h" + +#include +#include + +namespace trtcpp { + +std::string_view toString(DType t) noexcept { + switch (t) { + case DType::kFloat32: + return "float32"; + case DType::kFloat16: + return "float16"; + case DType::kBFloat16: + return "bfloat16"; + case DType::kInt32: + return "int32"; + case DType::kInt64: + return "int64"; + case DType::kInt8: + return "int8"; + case DType::kUInt8: + return "uint8"; + case DType::kBool: + return "bool"; + case DType::kFp8: + return "fp8"; + case DType::kInt4: + return "int4"; + } + return "unknown"; +} + +std::string_view toString(Layout l) noexcept { + switch (l) { + case Layout::kNCHW: + return "NCHW"; + case Layout::kNHWC: + return "NHWC"; + case Layout::kLinear: + return "linear"; + case Layout::kUnknown: + return "unknown"; + } + return "unknown"; +} + +std::string_view toString(Device d) noexcept { + switch (d) { + case Device::kHost: + return "host"; + case Device::kCuda: + return "cuda"; + } + return "unknown"; +} + +std::string_view toString(StatusCode code) noexcept { + switch (code) { + case StatusCode::kOk: + return "ok"; + case StatusCode::kInvalidArgument: + return "invalid_argument"; + case StatusCode::kNotFound: + return "not_found"; + case StatusCode::kIoError: + return "io_error"; + case StatusCode::kCudaError: + return "cuda_error"; + case StatusCode::kTensorRtError: + return "tensorrt_error"; + case StatusCode::kShapeMismatch: + return "shape_mismatch"; + case StatusCode::kDtypeMismatch: + return "dtype_mismatch"; + case StatusCode::kUnsupported: + return "unsupported"; + case StatusCode::kStaleCache: + return "stale_cache"; + case StatusCode::kInternal: + return "internal"; + } + return "unknown"; +} + +Shape::Shape(std::initializer_list dims) : Shape(std::span(dims.begin(), dims.size())) {} + +Shape::Shape(std::span dims) { + assert(dims.size() <= static_cast(kMaxRank) && "Shape rank exceeds kMaxRank"); + rank_ = static_cast(dims.size() < static_cast(kMaxRank) ? dims.size() : static_cast(kMaxRank)); + for (int i = 0; i < rank_; ++i) { + dims_[static_cast(i)] = dims[static_cast(i)]; + } +} + +std::int64_t Shape::at(int i) const { + if (i < 0 || i >= rank_) { + throw std::out_of_range("Shape::at index out of range"); + } + return dims_[static_cast(i)]; +} + +bool Shape::isDynamic() const noexcept { + for (int i = 0; i < rank_; ++i) { + if (dims_[static_cast(i)] < 0) { + return true; + } + } + return false; +} + +std::int64_t Shape::numel() const noexcept { + if (isDynamic()) { + return 0; + } + std::int64_t n = 1; + for (int i = 0; i < rank_; ++i) { + n *= dims_[static_cast(i)]; + } + return n; +} + +bool Shape::operator==(const Shape &other) const noexcept { + if (rank_ != other.rank_) { + return false; + } + for (int i = 0; i < rank_; ++i) { + if (dims_[static_cast(i)] != other.dims_[static_cast(i)]) { + return false; + } + } + return true; +} + +std::string Shape::toString() const { + std::string s = "["; + for (int i = 0; i < rank_; ++i) { + if (i != 0) { + s += ','; + } + s += std::to_string(dims_[static_cast(i)]); + } + s += ']'; + return s; +} + +} // namespace trtcpp diff --git a/src/cuda.cpp b/src/cuda.cpp new file mode 100644 index 0000000..58c890d --- /dev/null +++ b/src/cuda.cpp @@ -0,0 +1,103 @@ +#include "tensorrt_cpp_api/cuda.h" + +#include +#include + +namespace trtcpp { + +Status cudaToStatus(cudaError_t code, std::string_view context) { + if (code == cudaSuccess) { + return Status{}; + } + std::string message; + if (!context.empty()) { + message.append(context); + message.append(": "); + } + message.append(cudaGetErrorName(code)); + message.append(" ("); + message.append(cudaGetErrorString(code)); + message.append(")"); + return Status{StatusCode::kCudaError, std::move(message)}; +} + +Stream::Stream() { + cudaStream_t stream = nullptr; + if (cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking) == cudaSuccess) { + stream_ = stream; + owns_ = true; + } else { + stream_ = nullptr; // degrade to the default stream + owns_ = false; + } +} + +Result Stream::create() { + cudaStream_t stream = nullptr; + cudaError_t code = cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking); + if (code != cudaSuccess) { + return cudaToStatus(code, "cudaStreamCreateWithFlags"); + } + return Stream{stream, true}; +} + +Stream Stream::wrap(cudaStream_t existing) noexcept { return Stream{existing, false}; } + +Stream Stream::wrap(std::uintptr_t existing) noexcept { return Stream{reinterpret_cast(existing), false}; } + +Stream::~Stream() { + if (owns_ && stream_ != nullptr) { + cudaStreamDestroy(stream_); + } +} + +Stream::Stream(Stream &&other) noexcept : stream_(other.stream_), owns_(other.owns_) { + other.stream_ = nullptr; + other.owns_ = false; +} + +Stream &Stream::operator=(Stream &&other) noexcept { + if (this != &other) { + if (owns_ && stream_ != nullptr) { + cudaStreamDestroy(stream_); + } + stream_ = other.stream_; + owns_ = other.owns_; + other.stream_ = nullptr; + other.owns_ = false; + } + return *this; +} + +Status Stream::synchronize() const noexcept { return cudaToStatus(cudaStreamSynchronize(stream_), "cudaStreamSynchronize"); } + +Result deviceCount() { + int count = 0; + cudaError_t code = cudaGetDeviceCount(&count); + if (code != cudaSuccess) { + return cudaToStatus(code, "cudaGetDeviceCount"); + } + return count; +} + +Result queryDevice(int index) { + cudaDeviceProp prop{}; + cudaError_t code = cudaGetDeviceProperties(&prop, index); + if (code != cudaSuccess) { + return cudaToStatus(code, "cudaGetDeviceProperties"); + } + DeviceInfo info; + info.index = index; + info.name = prop.name; + info.computeMajor = prop.major; + info.computeMinor = prop.minor; + info.totalMemoryBytes = prop.totalGlobalMem; + std::array hex{}; + for (int i = 0; i < 16; ++i) { + std::snprintf(hex.data() + i * 2, 3, "%02x", static_cast(prop.uuid.bytes[i])); + } + info.uuid.assign(hex.data(), 32); + return info; +} + +} // namespace trtcpp diff --git a/src/detail/buffers.cpp b/src/detail/buffers.cpp new file mode 100644 index 0000000..e79e5ee --- /dev/null +++ b/src/detail/buffers.cpp @@ -0,0 +1,52 @@ +#include "detail/buffers.h" + +#include +#include + +namespace trtcpp::detail { + +Result NamedBuffers::ensure(const std::string &name, DType dtype, Shape shape, int deviceId) { + auto needBytes = checkedByteSize(dtype, shape); // rejects dynamic shapes and overflow + if (!needBytes) { + return needBytes.status(); + } + const std::size_t need = needBytes.value(); + Slot &slot = slots_[name]; + const bool mustGrow = need > slot.capacity; + const bool wrongDevice = need > 0 && (slot.buffer.empty() || slot.deviceId != deviceId); + if (mustGrow || wrongDevice) { + auto buffer = Tensor::allocate(DType::kUInt8, Shape{static_cast(need)}, Device::kCuda, deviceId); + if (!buffer) { + return buffer.status(); + } + slot.buffer = std::move(buffer).value(); + slot.capacity = need; + slot.deviceId = deviceId; + } + return TensorView{slot.buffer.data(), dtype, std::move(shape), Device::kCuda, deviceId, Layout::kLinear}; +} + +void *NamedBuffers::address(const std::string &name) const noexcept { + auto it = slots_.find(name); + return it == slots_.end() ? nullptr : it->second.buffer.data(); +} + +bool NamedBuffers::has(const std::string &name) const noexcept { return slots_.find(name) != slots_.end(); } + +std::vector NamedBuffers::names() const { + std::vector result; + result.reserve(slots_.size()); + for (const auto &entry : slots_) { + result.push_back(entry.first); + } + return result; +} + +std::size_t NamedBuffers::capacity(const std::string &name) const noexcept { + auto it = slots_.find(name); + return it == slots_.end() ? 0 : it->second.capacity; +} + +void NamedBuffers::clear() noexcept { slots_.clear(); } + +} // namespace trtcpp::detail diff --git a/src/detail/buffers.h b/src/detail/buffers.h new file mode 100644 index 0000000..d1da4e4 --- /dev/null +++ b/src/detail/buffers.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include + +#include "tensorrt_cpp_api/device_tensor.h" +#include "tensorrt_cpp_api/tensor.h" + +namespace trtcpp::detail { + +// Name-keyed reusable device buffers for an engine's IO tensors. Each slot owns a byte +// buffer that is reallocated only when a tensor's required size grows (or its device +// changes), so a fixed-shape inference loop allocates once and reuses thereafter -- +// addressing the dynamic-output-size and per-call-allocation issues. +// +// Backed by owning Tensors (synchronous cudaMalloc/cudaFree), so a slot is destruction-safe +// without a stream; reuse amortizes the synchronous allocation cost across inferences. +// +// Device buffers only -- host staging is not managed here; the readback path already pins +// host memory via Tensor::toHost, so a separate pinned-host pool is not provided. +// has()/names() report that a slot exists, not that it holds a live (non-zero) allocation. +class NamedBuffers { +public: + NamedBuffers() = default; + + // Ensure the buffer named `name` is large enough for (dtype, shape) on `deviceId`, + // reallocating only if it must grow or the device changed. Returns a TensorView of + // (dtype, shape) over the buffer. `shape` must be fully resolved (not dynamic). + Result ensure(const std::string &name, DType dtype, Shape shape, int deviceId = 0); + + void *address(const std::string &name) const noexcept; // device address, or nullptr if absent + bool has(const std::string &name) const noexcept; + std::vector names() const; + std::size_t capacity(const std::string &name) const noexcept; // bytes; 0 if absent + void clear() noexcept; + +private: + struct Slot { + Tensor buffer; // a kUInt8 device buffer of `capacity` bytes + std::size_t capacity = 0; + int deviceId = 0; + }; + std::unordered_map slots_; +}; + +} // namespace trtcpp::detail diff --git a/src/detail/calibrator_bridge.cpp b/src/detail/calibrator_bridge.cpp new file mode 100644 index 0000000..5aed26c --- /dev/null +++ b/src/detail/calibrator_bridge.cpp @@ -0,0 +1,64 @@ +#include "detail/calibrator_bridge.h" + +#if TRT_CPP_API_TENSORRT_VERSION_MAJOR < 11 + +#include +#include + +namespace trtcpp::detail { + +Int8CalibratorBridge::Int8CalibratorBridge(std::shared_ptr calibrator, std::string inputName, Shape batchShape) + : calibrator_(std::move(calibrator)), inputName_(std::move(inputName)), batchShape_(std::move(batchShape)) {} + +int32_t Int8CalibratorBridge::getBatchSize() const noexcept { return calibrator_ ? calibrator_->batchSize() : 0; } + +bool Int8CalibratorBridge::getBatch(void *bindings[], const char *names[], int32_t nbBindings) noexcept { + try { + std::unordered_map inputs; + for (int32_t i = 0; i < nbBindings; ++i) { + if (inputName_ == names[i]) { + inputs.emplace(inputName_, TensorView{bindings[i], DType::kFloat32, batchShape_, Device::kCuda, 0, Layout::kLinear}); + } + } + if (inputs.empty()) { + return false; // calibrator's input name not among the bindings + } + if (!calibrator_->nextBatch(inputs, stream_)) { + // No more batches (or the provider errored). nextBatch may have queued async work on + // stream_ before failing, so drain it before returning -- otherwise a late DMA could + // write into a binding buffer TRT no longer expects to be touched. + stream_.synchronize(); + return false; + } + return stream_.synchronize().ok(); // the buffer must be filled before TRT reads it + } catch (...) { + return false; + } +} + +const void *Int8CalibratorBridge::readCalibrationCache(std::size_t &length) noexcept { + try { + auto cache = calibrator_->readCache(); + if (!cache.has_value() || cache->empty()) { + length = 0; + return nullptr; + } + cache_ = std::move(*cache); + length = cache_.size(); + return cache_.data(); + } catch (...) { + length = 0; + return nullptr; + } +} + +void Int8CalibratorBridge::writeCalibrationCache(const void *ptr, std::size_t length) noexcept { + try { + calibrator_->writeCache(std::span(static_cast(ptr), length)); + } catch (...) { + } +} + +} // namespace trtcpp::detail + +#endif // TRT_CPP_API_TENSORRT_VERSION_MAJOR < 11 diff --git a/src/detail/calibrator_bridge.h b/src/detail/calibrator_bridge.h new file mode 100644 index 0000000..3117ca2 --- /dev/null +++ b/src/detail/calibrator_bridge.h @@ -0,0 +1,42 @@ +#pragma once + +#include "tensorrt_cpp_api/build_config.h" + +#if TRT_CPP_API_TENSORRT_VERSION_MAJOR < 11 + +#include +#include +#include +#include + +#include + +#include "tensorrt_cpp_api/calibrator.h" +#include "tensorrt_cpp_api/cuda.h" +#include "tensorrt_cpp_api/shape.h" + +namespace trtcpp::detail { + +// Adapts a trtcpp::ICalibrator to the legacy nvinfer1::IInt8EntropyCalibrator2 the TRT < 11 +// builder consumes. Single-input only (the legacy calibrator contract). Owns a stream for +// the per-batch uploads and synchronizes before each getBatch returns. +class Int8CalibratorBridge final : public nvinfer1::IInt8EntropyCalibrator2 { +public: + Int8CalibratorBridge(std::shared_ptr calibrator, std::string inputName, Shape batchShape); + + int32_t getBatchSize() const noexcept override; + bool getBatch(void *bindings[], const char *names[], int32_t nbBindings) noexcept override; + const void *readCalibrationCache(std::size_t &length) noexcept override; + void writeCalibrationCache(const void *ptr, std::size_t length) noexcept override; + +private: + std::shared_ptr calibrator_; + std::string inputName_; + Shape batchShape_; + Stream stream_; + std::vector cache_; +}; + +} // namespace trtcpp::detail + +#endif // TRT_CPP_API_TENSORRT_VERSION_MAJOR < 11 diff --git a/src/detail/engine_cache.cpp b/src/detail/engine_cache.cpp new file mode 100644 index 0000000..733f0bd --- /dev/null +++ b/src/detail/engine_cache.cpp @@ -0,0 +1,216 @@ +#include "detail/engine_cache.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace trtcpp::detail { +namespace { + +std::string jsonEscape(const std::string &value) { + std::string out; + out.reserve(value.size() + 2); + for (char c : value) { + if (c == '"' || c == '\\') { + out.push_back('\\'); + } + out.push_back(c); + } + return out; +} + +// Targeted extractor for our own flat sidecar format ("key": "value" or "key": bool/num). +std::optional jsonStringField(const std::string &content, const std::string &key) { + const std::string needle = "\"" + key + "\""; + std::size_t pos = content.find(needle); + if (pos == std::string::npos) { + return std::nullopt; + } + pos = content.find(':', pos + needle.size()); + if (pos == std::string::npos) { + return std::nullopt; + } + pos = content.find('"', pos); + if (pos == std::string::npos) { + return std::nullopt; + } + std::string value; + for (std::size_t i = pos + 1; i < content.size(); ++i) { + if (content[i] == '\\' && i + 1 < content.size()) { + value.push_back(content[++i]); + } else if (content[i] == '"') { + return value; + } else { + value.push_back(content[i]); + } + } + return std::nullopt; +} + +bool jsonBoolField(const std::string &content, const std::string &key) { + const std::string needle = "\"" + key + "\""; + std::size_t pos = content.find(needle); + if (pos == std::string::npos) { + return false; + } + pos = content.find(':', pos + needle.size()); + return pos != std::string::npos && content.find("true", pos) == content.find_first_not_of(" \t", pos + 1); +} + +// No-throw: from_chars never throws (unlike stoi), and leaves the component 0 on an empty +// or overflowing field, so a corrupt sidecar version is treated as a mismatch (stale) -- the +// intended behavior -- rather than crashing the no-throw API. +std::tuple parseVersion(const std::string &version) { + int parts[3] = {0, 0, 0}; + int index = 0; + std::string current; + auto flush = [&]() { + if (index < 3) { + int value = 0; + std::from_chars(current.data(), current.data() + current.size(), value); + parts[index++] = value; + } + current.clear(); + }; + for (char c : version + ".") { + if (c == '.') { + flush(); + } else if (c >= '0' && c <= '9') { + current.push_back(c); + } + } + return {parts[0], parts[1], parts[2]}; +} + +} // namespace + +Result> readFile(const std::string &path) { + std::error_code ec; + if (!std::filesystem::is_regular_file(path, ec)) { + return Status{StatusCode::kNotFound, "not a readable regular file: " + path}; + } + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) { + return Status{StatusCode::kNotFound, "cannot open file: " + path}; + } + const std::streamsize size = file.tellg(); + if (size < 0) { + return Status{StatusCode::kIoError, "cannot size file: " + path}; + } + file.seekg(0, std::ios::beg); + std::vector bytes(static_cast(size)); + if (size > 0 && !file.read(reinterpret_cast(bytes.data()), size)) { + return Status{StatusCode::kIoError, "cannot read file: " + path}; + } + return bytes; +} + +std::string cacheFileName(const std::string &onnxStem, const std::string &onnxSha256, const std::string &trtVersion, + const std::string &gpuUuid, const std::string &precision) { + const std::string sha8 = onnxSha256.substr(0, std::min(8, onnxSha256.size())); + const std::string uuid8 = gpuUuid.substr(0, std::min(8, gpuUuid.size())); + return onnxStem + "." + sha8 + ".trt" + trtVersion + "." + uuid8 + "." + precision + ".engine"; +} + +Status writeAtomic(const std::string &path, std::span bytes) { + std::error_code ec; + const std::filesystem::path target(path); + if (target.has_parent_path()) { + std::filesystem::create_directories(target.parent_path(), ec); + } + // PID-suffixed temp so concurrent builders writing the same cache entry don't collide. + const std::filesystem::path tmp = target.string() + ".tmp." + std::to_string(::getpid()); + { + std::ofstream out(tmp, std::ios::binary | std::ios::trunc); + if (!out) { + return Status{StatusCode::kIoError, "cannot open for write: " + tmp.string()}; + } + if (!bytes.empty()) { + out.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + } + // Flush + close explicitly and check the result BEFORE the rename: a buffered write can + // succeed here yet fail on the destructor's implicit flush (e.g. disk full), which would + // otherwise rename a truncated temp file over the cache. Catch that failure first. + out.close(); + if (!out) { + std::filesystem::remove(tmp, ec); + return Status{StatusCode::kIoError, "write/flush failed: " + tmp.string()}; + } + } + std::filesystem::rename(tmp, target, ec); + if (ec) { + std::filesystem::remove(tmp, ec); + return Status{StatusCode::kIoError, "atomic rename failed: " + path}; + } + return Status{}; +} + +Status writeSidecar(const std::string &path, const CacheMeta &meta) { + std::string json = "{\n"; + json += " \"onnx_sha256\": \"" + jsonEscape(meta.onnxSha256) + "\",\n"; + json += " \"trt_version\": \"" + jsonEscape(meta.trtVersion) + "\",\n"; + json += " \"cuda_version\": \"" + jsonEscape(meta.cudaVersion) + "\",\n"; + json += " \"gpu_name\": \"" + jsonEscape(meta.gpuName) + "\",\n"; + json += " \"gpu_uuid\": \"" + jsonEscape(meta.gpuUuid) + "\",\n"; + json += " \"precision\": \"" + jsonEscape(meta.precision) + "\",\n"; + json += " \"build_options_digest\": \"" + jsonEscape(meta.buildOptionsDigest) + "\",\n"; + json += " \"version_compatible\": " + std::string(meta.versionCompatible ? "true" : "false") + ",\n"; + json += " \"hardware_compatible\": " + std::string(meta.hardwareCompatible ? "true" : "false") + ",\n"; + json += " \"created_unix\": " + std::to_string(meta.createdUnix) + "\n"; + json += "}\n"; + const auto *data = reinterpret_cast(json.data()); + return writeAtomic(path, std::span(data, json.size())); +} + +Result readSidecar(const std::string &path) { + std::ifstream in(path, std::ios::binary); + if (!in) { + return Status{StatusCode::kNotFound, "cannot read sidecar: " + path}; + } + const std::string content((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + CacheMeta meta; + meta.onnxSha256 = jsonStringField(content, "onnx_sha256").value_or(""); + meta.trtVersion = jsonStringField(content, "trt_version").value_or(""); + meta.cudaVersion = jsonStringField(content, "cuda_version").value_or(""); + meta.gpuName = jsonStringField(content, "gpu_name").value_or(""); + meta.gpuUuid = jsonStringField(content, "gpu_uuid").value_or(""); + meta.precision = jsonStringField(content, "precision").value_or(""); + meta.buildOptionsDigest = jsonStringField(content, "build_options_digest").value_or(""); + meta.versionCompatible = jsonBoolField(content, "version_compatible"); + meta.hardwareCompatible = jsonBoolField(content, "hardware_compatible"); + if (meta.onnxSha256.empty()) { + return Status{StatusCode::kStaleCache, "sidecar missing onnx_sha256: " + path}; + } + return meta; +} + +bool isFresh(const CacheMeta &sidecar, const CacheMeta &expected, bool versionCompatible, bool hardwareCompatible) { + if (sidecar.onnxSha256 != expected.onnxSha256) { + return false; + } + if (sidecar.buildOptionsDigest != expected.buildOptionsDigest) { + return false; + } + if (versionCompatible) { + const auto cached = parseVersion(sidecar.trtVersion); + const auto runtime = parseVersion(expected.trtVersion); + // Same major, and the runtime is the same or newer (version compatibility is forward-only). + if (std::get<0>(cached) != std::get<0>(runtime) || runtime < cached) { + return false; + } + } else if (sidecar.trtVersion != expected.trtVersion) { + return false; + } + if (!hardwareCompatible && sidecar.gpuUuid != expected.gpuUuid) { + return false; + } + return true; +} + +} // namespace trtcpp::detail diff --git a/src/detail/engine_cache.h b/src/detail/engine_cache.h new file mode 100644 index 0000000..86b4f8d --- /dev/null +++ b/src/detail/engine_cache.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include + +#include "tensorrt_cpp_api/status.h" + +namespace trtcpp::detail { + +// Read a regular file fully into memory. Rejects missing paths and directories (whose +// tellg() returns a huge sentinel that would otherwise trigger a giant allocation), so it +// never throws out of the no-throw API. Shared by the builder and the engine loader. +Result> readFile(const std::string &path); + +// Metadata recorded in the JSON sidecar next to a cached engine, used to detect stale +// caches across ONNX changes, TensorRT versions, GPUs, and build options. +struct CacheMeta { + std::string onnxSha256; + std::string trtVersion; // e.g. "10.0.0" + std::string cudaVersion; // runtime version, e.g. "12060" + std::string gpuName; + std::string gpuUuid; + std::string precision; + std::string buildOptionsDigest; + bool versionCompatible = false; + bool hardwareCompatible = false; + long long createdUnix = 0; +}; + +// Engine cache filename: ..trt...engine +std::string cacheFileName(const std::string &onnxStem, const std::string &onnxSha256, const std::string &trtVersion, + const std::string &gpuUuid, const std::string &precision); + +// Write `bytes` to `path` atomically (temp file + rename), creating parent dirs. +Status writeAtomic(const std::string &path, std::span bytes); + +// Write/read the JSON sidecar. +Status writeSidecar(const std::string &path, const CacheMeta &meta); +Result readSidecar(const std::string &path); + +// A cached engine is fresh iff the ONNX hash and build-options digest match, the TRT +// version matches (only the major when versionCompatible, requiring runtime >= cached), +// and the GPU UUID matches (skipped when hardwareCompatible). +bool isFresh(const CacheMeta &sidecar, const CacheMeta &expected, bool versionCompatible, bool hardwareCompatible); + +} // namespace trtcpp::detail diff --git a/src/detail/execution.cpp b/src/detail/execution.cpp new file mode 100644 index 0000000..46620c5 --- /dev/null +++ b/src/detail/execution.cpp @@ -0,0 +1,183 @@ +#include "detail/execution.h" + +#include +#include + +namespace trtcpp::detail { +namespace { + +DType fromTrtDataType(nvinfer1::DataType type) { + switch (type) { + case nvinfer1::DataType::kFLOAT: + return DType::kFloat32; + case nvinfer1::DataType::kHALF: + return DType::kFloat16; + case nvinfer1::DataType::kINT8: + return DType::kInt8; + case nvinfer1::DataType::kINT32: + return DType::kInt32; + case nvinfer1::DataType::kBOOL: + return DType::kBool; + case nvinfer1::DataType::kUINT8: + return DType::kUInt8; + case nvinfer1::DataType::kFP8: + return DType::kFp8; + case nvinfer1::DataType::kBF16: + return DType::kBFloat16; + case nvinfer1::DataType::kINT64: + return DType::kInt64; + case nvinfer1::DataType::kINT4: + return DType::kInt4; + } + return DType::kFloat32; +} + +static_assert(Shape::kMaxRank == nvinfer1::Dims::MAX_DIMS, "Shape::kMaxRank must match nvinfer1::Dims::MAX_DIMS"); + +nvinfer1::Dims toDims(const Shape &shape) { + nvinfer1::Dims dims; + dims.nbDims = shape.rank(); + for (int i = 0; i < shape.rank(); ++i) { + dims.d[i] = shape[i]; + } + return dims; +} + +Shape fromDims(const nvinfer1::Dims &dims) { + if (dims.nbDims < 0) { + return Shape{}; // TensorRT signals invalid dims with nbDims == -1 + } + std::array values{}; + for (int i = 0; i < dims.nbDims; ++i) { + values[static_cast(i)] = dims.d[i]; + } + return Shape{std::span(values.data(), static_cast(dims.nbDims))}; +} + +// Select the profile and set every input's shape (and optionally its address) on `context`. +// Ordering -- profile then input shapes -- is the TRT requirement; the profile-set stream +// must match the eventual enqueue stream. +Status bindInputs(nvinfer1::IExecutionContext &context, const EngineMeta &meta, const TensorMap &inputs, int profileIndex, + cudaStream_t stream, bool setAddresses) { + const int profileCeiling = meta.nbProfiles > 0 ? meta.nbProfiles : 1; + if (profileIndex < 0 || profileIndex >= profileCeiling) { + return Status{StatusCode::kInvalidArgument, "profileIndex out of range"}; + } + // Only dynamic engines need per-context profile selection; for static engines the + // default profile 0 is shared safely across concurrent contexts. + if (meta.hasDynamic) { + if (!context.setOptimizationProfileAsync(profileIndex, stream)) { + return Status{StatusCode::kInvalidArgument, "setOptimizationProfileAsync failed for profile " + std::to_string(profileIndex)}; + } + } + for (const std::string &name : meta.inputNames) { + auto it = inputs.find(name); + if (it == inputs.end()) { + return Status{StatusCode::kInvalidArgument, "missing input tensor: " + name}; + } + if (!context.setInputShape(name.c_str(), toDims(it->second.shape()))) { + return Status{StatusCode::kShapeMismatch, "setInputShape rejected the shape for input: " + name}; + } + if (setAddresses && !context.setTensorAddress(name.c_str(), it->second.data())) { + return Status{StatusCode::kTensorRtError, "setTensorAddress failed for input: " + name}; + } + } + if (!context.allInputDimensionsSpecified()) { + return Status{StatusCode::kInvalidArgument, "not all input dimensions were specified"}; + } + return Status{}; +} + +} // namespace + +EngineMeta introspectEngine(nvinfer1::ICudaEngine &engine, int deviceIndex) { + EngineMeta meta; + meta.deviceIndex = deviceIndex; + meta.nbProfiles = engine.getNbOptimizationProfiles(); + const int nbTensors = engine.getNbIOTensors(); + for (int i = 0; i < nbTensors; ++i) { + const char *name = engine.getIOTensorName(i); + TensorInfo info; + info.name = name; + info.isInput = engine.getTensorIOMode(name) == nvinfer1::TensorIOMode::kINPUT; + info.dtype = fromTrtDataType(engine.getTensorDataType(name)); + info.shape = fromDims(engine.getTensorShape(name)); + if (info.isInput) { + meta.inputNames.push_back(info.name); + } else { + meta.outputNames.push_back(info.name); + } + if (info.shape.isDynamic()) { + meta.hasDynamic = true; + } + meta.tensors.push_back(std::move(info)); + } + return meta; +} + +Status enqueue(nvinfer1::IExecutionContext &context, const EngineMeta &meta, const TensorMap &inputs, const TensorMap &outputs, + const Stream &stream, int profileIndex) { + if (Status status = bindInputs(context, meta, inputs, profileIndex, stream.handle(), /*setAddresses=*/true); !status) { + return status; + } + for (const std::string &name : meta.outputNames) { + auto it = outputs.find(name); + if (it == outputs.end()) { + return Status{StatusCode::kInvalidArgument, "missing output tensor: " + name}; + } + if (!context.setTensorAddress(name.c_str(), it->second.data())) { + return Status{StatusCode::kTensorRtError, "setTensorAddress failed for output: " + name}; + } + } + if (!context.enqueueV3(stream.handle())) { + return Status{StatusCode::kTensorRtError, "enqueueV3 failed"}; + } + return Status{}; +} + +Result> infer(nvinfer1::IExecutionContext &context, const EngineMeta &meta, const TensorMap &inputs, + const Stream &stream, int profileIndex) { + if (Status status = bindInputs(context, meta, inputs, profileIndex, stream.handle(), /*setAddresses=*/true); !status) { + return status; + } + std::unordered_map outputs; + for (const TensorInfo &info : meta.tensors) { + if (info.isInput) { + continue; + } + const Shape shape = fromDims(context.getTensorShape(info.name.c_str())); + if (shape.isDynamic()) { + return Status{StatusCode::kInternal, "output shape unresolved after setInputShape: " + info.name}; + } + auto tensor = Tensor::allocate(info.dtype, shape, Device::kCuda, meta.deviceIndex); + if (!tensor) { + return tensor.status(); + } + if (!context.setTensorAddress(info.name.c_str(), tensor.value().data())) { + return Status{StatusCode::kTensorRtError, "setTensorAddress failed for output: " + info.name}; + } + outputs.emplace(info.name, std::move(tensor).value()); + } + if (!context.enqueueV3(stream.handle())) { + return Status{StatusCode::kTensorRtError, "enqueueV3 failed"}; + } + return outputs; +} + +Result> outputShapes(nvinfer1::IExecutionContext &context, const EngineMeta &meta, + const TensorMap &inputs, int profileIndex) { + Stream stream; + if (Status status = bindInputs(context, meta, inputs, profileIndex, stream.handle(), /*setAddresses=*/false); !status) { + return status; + } + if (Status status = stream.synchronize(); !status) { + return status; + } + std::unordered_map result; + for (const std::string &name : meta.outputNames) { + result.emplace(name, fromDims(context.getTensorShape(name.c_str()))); + } + return result; +} + +} // namespace trtcpp::detail diff --git a/src/detail/execution.h b/src/detail/execution.h new file mode 100644 index 0000000..5c5fd98 --- /dev/null +++ b/src/detail/execution.h @@ -0,0 +1,50 @@ +#pragma once + +// Internal: per-execution-context inference logic shared by Engine (one context) and +// EnginePool::Lease (N contexts, one optimization profile each). Includes NvInfer.h, so it +// is never installed. + +#include +#include +#include + +#include + +#include + +#include "tensorrt_cpp_api/cuda.h" +#include "tensorrt_cpp_api/device_tensor.h" +#include "tensorrt_cpp_api/engine.h" // TensorInfo +#include "tensorrt_cpp_api/shape.h" +#include "tensorrt_cpp_api/status.h" +#include "tensorrt_cpp_api/tensor.h" + +namespace trtcpp::detail { + +// Name-keyed engine metadata, introspected once and shared across contexts. +struct EngineMeta { + std::vector tensors; + std::vector inputNames; + std::vector outputNames; + int nbProfiles = 0; + int deviceIndex = 0; + bool hasDynamic = false; ///< any IO tensor has a dynamic (-1) dim; static engines report a profile count of 1 +}; + +EngineMeta introspectEngine(nvinfer1::ICudaEngine &engine, int deviceIndex); + +using TensorMap = std::unordered_map; + +// Caller-allocated: bind profile + input shapes/addresses, bind output addresses, enqueueV3. +Status enqueue(nvinfer1::IExecutionContext &context, const EngineMeta &meta, const TensorMap &inputs, const TensorMap &outputs, + const Stream &stream, int profileIndex); + +// Library-allocated: bind inputs, allocate outputs sized from the resolved shapes, enqueueV3. +Result> infer(nvinfer1::IExecutionContext &context, const EngineMeta &meta, const TensorMap &inputs, + const Stream &stream, int profileIndex); + +// Resolve all output shapes for the given inputs + profile (on a private stream). +Result> outputShapes(nvinfer1::IExecutionContext &context, const EngineMeta &meta, + const TensorMap &inputs, int profileIndex); + +} // namespace trtcpp::detail diff --git a/src/detail/sha256.cpp b/src/detail/sha256.cpp new file mode 100644 index 0000000..445bd21 --- /dev/null +++ b/src/detail/sha256.cpp @@ -0,0 +1,85 @@ +#include "detail/sha256.h" + +#include +#include + +namespace trtcpp::detail { +namespace { + +constexpr std::uint32_t kRoundConstants[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, + 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, + 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, + 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, + 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +inline std::uint32_t rotr(std::uint32_t x, std::uint32_t n) { return (x >> n) | (x << (32 - n)); } + +} // namespace + +std::string sha256Hex(const void *data, std::size_t length) { + std::uint32_t h[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + + const auto *message = static_cast(data); + std::vector buffer(message, message + length); + const std::uint64_t bitLength = static_cast(length) * 8; + buffer.push_back(0x80); + while (buffer.size() % 64 != 56) { + buffer.push_back(0x00); + } + for (int i = 7; i >= 0; --i) { + buffer.push_back(static_cast((bitLength >> (i * 8)) & 0xff)); + } + + for (std::size_t offset = 0; offset < buffer.size(); offset += 64) { + std::uint32_t w[64]; + for (int i = 0; i < 16; ++i) { + w[i] = (static_cast(buffer[offset + i * 4]) << 24) | + (static_cast(buffer[offset + i * 4 + 1]) << 16) | + (static_cast(buffer[offset + i * 4 + 2]) << 8) | (static_cast(buffer[offset + i * 4 + 3])); + } + for (int i = 16; i < 64; ++i) { + const std::uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + const std::uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + std::uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5], g = h[6], hh = h[7]; + for (int i = 0; i < 64; ++i) { + const std::uint32_t S1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25); + const std::uint32_t ch = (e & f) ^ ((~e) & g); + const std::uint32_t t1 = hh + S1 + ch + kRoundConstants[i] + w[i]; + const std::uint32_t S0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22); + const std::uint32_t maj = (a & b) ^ (a & c) ^ (b & c); + const std::uint32_t t2 = S0 + maj; + hh = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + h[0] += a; + h[1] += b; + h[2] += c; + h[3] += d; + h[4] += e; + h[5] += f; + h[6] += g; + h[7] += hh; + } + + static const char digits[] = "0123456789abcdef"; + std::string out; + out.reserve(64); + for (int i = 0; i < 8; ++i) { + for (int j = 7; j >= 0; --j) { + out.push_back(digits[(h[i] >> (j * 4)) & 0xf]); + } + } + return out; +} + +} // namespace trtcpp::detail diff --git a/src/detail/sha256.h b/src/detail/sha256.h new file mode 100644 index 0000000..3c888e0 --- /dev/null +++ b/src/detail/sha256.h @@ -0,0 +1,16 @@ +#pragma once + +#include +#include +#include + +namespace trtcpp::detail { + +// SHA-256 of `length` bytes at `data`, returned as a 64-char lowercase hex string. Used +// for engine-cache content hashing (not security); dependency-free so the core library +// pulls in no crypto library. +std::string sha256Hex(const void *data, std::size_t length); + +inline std::string sha256Hex(std::span bytes) { return sha256Hex(bytes.data(), bytes.size()); } + +} // namespace trtcpp::detail diff --git a/src/detail/trt_common.cpp b/src/detail/trt_common.cpp new file mode 100644 index 0000000..6fa1c0a --- /dev/null +++ b/src/detail/trt_common.cpp @@ -0,0 +1,68 @@ +#include "detail/trt_common.h" + +#include + +namespace trtcpp::detail { + +Severity fromNvSeverity(nvinfer1::ILogger::Severity severity) noexcept { + switch (severity) { + case nvinfer1::ILogger::Severity::kINTERNAL_ERROR: + return Severity::kInternalError; + case nvinfer1::ILogger::Severity::kERROR: + return Severity::kError; + case nvinfer1::ILogger::Severity::kWARNING: + return Severity::kWarning; + case nvinfer1::ILogger::Severity::kINFO: + return Severity::kInfo; + case nvinfer1::ILogger::Severity::kVERBOSE: + return Severity::kVerbose; + } + return Severity::kInfo; +} + +nvinfer1::ILogger::Severity toNvSeverity(Severity severity) noexcept { + switch (severity) { + case Severity::kInternalError: + return nvinfer1::ILogger::Severity::kINTERNAL_ERROR; + case Severity::kError: + return nvinfer1::ILogger::Severity::kERROR; + case Severity::kWarning: + return nvinfer1::ILogger::Severity::kWARNING; + case Severity::kInfo: + return nvinfer1::ILogger::Severity::kINFO; + case Severity::kVerbose: + return nvinfer1::ILogger::Severity::kVERBOSE; + } + return nvinfer1::ILogger::Severity::kINFO; +} + +TrtLoggerBridge::TrtLoggerBridge(std::shared_ptr logger) : logger_(std::move(logger)) { + if (!logger_) { + logger_ = defaultLogger(); + } +} + +void TrtLoggerBridge::log(nvinfer1::ILogger::Severity severity, const char *msg) noexcept { + logger_->log(fromNvSeverity(severity), msg != nullptr ? msg : ""); +} + +Status loadPluginLibraries(const std::vector &paths) { + if (paths.empty()) { + return Status{}; + } + nvinfer1::IPluginRegistry *registry = ::getPluginRegistry(); // global free function, not a member + if (registry == nullptr) { + return Status{StatusCode::kTensorRtError, "getPluginRegistry() returned null"}; + } + for (const std::string &path : paths) { + // loadLibrary returns null for a missing/unreadable library OR one whose plugins + // are already registered -- both are failures to propagate. + if (registry->loadLibrary(path.c_str()) == nullptr) { + return Status{StatusCode::kTensorRtError, + "failed to load plugin library (missing, unreadable, or already registered): " + path}; + } + } + return Status{}; +} + +} // namespace trtcpp::detail diff --git a/src/detail/trt_common.h b/src/detail/trt_common.h new file mode 100644 index 0000000..2e9a095 --- /dev/null +++ b/src/detail/trt_common.h @@ -0,0 +1,45 @@ +#pragma once + +// Internal TensorRT glue. This header includes and is NOT installed -- it is +// included only by .cpp files in src/, keeping nvinfer1 types out of every public header. + +#include +#include +#include + +#include + +#include "tensorrt_cpp_api/logger.h" +#include "tensorrt_cpp_api/status.h" + +namespace trtcpp::detail { + +// TensorRT 10/11 destroy API objects via `delete` (the old destroy() is gone), so a plain +// unique_ptr with the default deleter is the correct RAII wrapper. +template using TrtUniquePtr = std::unique_ptr; + +// Severity translation BY NAME. nvinfer1::ILogger::Severity is numerically the REVERSE of +// trtcpp::Severity (kINTERNAL_ERROR=0 .. kVERBOSE=4), so these must switch on the +// enumerator and must never static_cast between the two. +Severity fromNvSeverity(nvinfer1::ILogger::Severity severity) noexcept; +nvinfer1::ILogger::Severity toNvSeverity(Severity severity) noexcept; + +// Adapts a trtcpp::ILogger to the nvinfer1::ILogger TensorRT requires. Holds the trtcpp +// logger alive; pass nv() (or the bridge itself) to createInferBuilder/createInferRuntime. +class TrtLoggerBridge final : public nvinfer1::ILogger { +public: + // trtcpp:: is required: inside a class derived from nvinfer1::ILogger, an unqualified + // ILogger/Severity would resolve to the base class's names. + explicit TrtLoggerBridge(std::shared_ptr logger); + void log(nvinfer1::ILogger::Severity severity, const char *msg) noexcept override; + nvinfer1::ILogger &nv() noexcept { return *this; } + +private: + std::shared_ptr logger_; +}; + +// Load custom-plugin shared libraries into the global plugin registry (IPluginV3 path, +// via IPluginRegistry::loadLibrary). Returns the first failure, or ok if all loaded. +Status loadPluginLibraries(const std::vector &paths); + +} // namespace trtcpp::detail diff --git a/src/device_tensor.cpp b/src/device_tensor.cpp new file mode 100644 index 0000000..e0b67b8 --- /dev/null +++ b/src/device_tensor.cpp @@ -0,0 +1,128 @@ +#include "tensorrt_cpp_api/device_tensor.h" + +#include + +namespace trtcpp { +namespace { + +cudaMemcpyKind memcpyKind(Device src, Device dst) { + if (src == Device::kHost && dst == Device::kCuda) { + return cudaMemcpyHostToDevice; + } + if (src == Device::kCuda && dst == Device::kHost) { + return cudaMemcpyDeviceToHost; + } + if (src == Device::kCuda && dst == Device::kCuda) { + return cudaMemcpyDeviceToDevice; + } + return cudaMemcpyHostToHost; +} + +} // namespace + +Result Tensor::allocate(DType dtype, Shape shape, Device device, int deviceId) { + auto byteSize = checkedByteSize(dtype, shape); // rejects dynamic shapes and overflow + if (!byteSize) { + return byteSize.status(); + } + const std::size_t bytes = byteSize.value(); + void *ptr = nullptr; + if (bytes > 0) { + if (device == Device::kCuda) { + // Set the target device only around the allocation and restore the caller's + // current device, so allocate() leaves no thread-local device-state side effect. + int previousDevice = 0; + cudaGetDevice(&previousDevice); + if (cudaError_t code = cudaSetDevice(deviceId); code != cudaSuccess) { + cudaSetDevice(previousDevice); + return cudaToStatus(code, "cudaSetDevice"); + } + cudaError_t code = cudaMalloc(&ptr, bytes); + cudaSetDevice(previousDevice); + if (code != cudaSuccess) { + return cudaToStatus(code, "cudaMalloc"); + } + } else { + if (cudaError_t code = cudaMallocHost(&ptr, bytes); code != cudaSuccess) { + return cudaToStatus(code, "cudaMallocHost"); + } + } + } + return Tensor{ptr, dtype, std::move(shape), device, deviceId}; +} + +void Tensor::reset() noexcept { + if (data_ != nullptr) { + if (device_ == Device::kCuda) { + cudaFree(data_); + } else { + cudaFreeHost(data_); + } + data_ = nullptr; + } +} + +Tensor::~Tensor() { reset(); } + +Tensor::Tensor(Tensor &&other) noexcept + : data_(other.data_), dtype_(other.dtype_), shape_(std::move(other.shape_)), device_(other.device_), deviceId_(other.deviceId_) { + other.data_ = nullptr; +} + +Tensor &Tensor::operator=(Tensor &&other) noexcept { + if (this != &other) { + reset(); + data_ = other.data_; + dtype_ = other.dtype_; + shape_ = std::move(other.shape_); + device_ = other.device_; + deviceId_ = other.deviceId_; + other.data_ = nullptr; + } + return *this; +} + +TensorView Tensor::view() const noexcept { return TensorView{data_, dtype_, shape_, device_, deviceId_, Layout::kLinear}; } + +Status Tensor::copyFrom(TensorView src, const Stream &stream) { + if (src.dtype() != dtype_) { + return Status{StatusCode::kDtypeMismatch, "Tensor::copyFrom dtype mismatch"}; + } + if (src.nbytes() != nbytes()) { + return Status{StatusCode::kShapeMismatch, "Tensor::copyFrom byte-size mismatch"}; + } + if (src.device() == Device::kCuda && device_ == Device::kCuda && src.deviceId() != deviceId_) { + // Cross-device (peer) copies need peer-access setup; not supported in v7 (CNN + // inference is single-GPU). Stage through host explicitly if you need this. + return Status{StatusCode::kUnsupported, "Tensor::copyFrom does not support cross-device copies; stage through host"}; + } + if (nbytes() == 0) { + return Status{}; + } + cudaError_t code = cudaMemcpyAsync(data_, src.data(), nbytes(), memcpyKind(src.device(), device_), stream.handle()); + return cudaToStatus(code, "cudaMemcpyAsync"); +} + +Result Tensor::to(Device device, int deviceId, const Stream &stream) const { + auto dst = Tensor::allocate(dtype_, shape_, device, deviceId); + if (!dst) { + return dst.status(); + } + if (Status status = dst.value().copyFrom(view(), stream); !status) { + return status; + } + return std::move(dst).value(); +} + +Result Tensor::toHost(const Stream &stream) const { + auto dst = to(Device::kHost, 0, stream); + if (!dst) { + return dst; + } + if (Status status = stream.synchronize(); !status) { + return status; + } + return dst; +} + +} // namespace trtcpp diff --git a/src/engine.cpp b/src/engine.cpp index c607d06..3512867 100644 --- a/src/engine.cpp +++ b/src/engine.cpp @@ -1,136 +1,117 @@ -#include "engine.h" -#include -#include -#include -#include -#include -#include - -using namespace nvinfer1; -using namespace Util; - -void Logger::log(Severity severity, const char *msg) noexcept { - switch (severity) { - case Severity::kVERBOSE: - spdlog::debug(msg); - break; - case Severity::kINFO: - spdlog::info(msg); - break; - case Severity::kWARNING: - spdlog::warn(msg); - break; - case Severity::kERROR: - spdlog::error(msg); - break; - case Severity::kINTERNAL_ERROR: - spdlog::critical(msg); - break; - default: - spdlog::info("Unexpected severity level"); - } -} +#include "tensorrt_cpp_api/engine.h" -Int8EntropyCalibrator2::Int8EntropyCalibrator2(int32_t batchSize, int32_t inputW, int32_t inputH, const std::string &calibDataDirPath, - const std::string &calibTableName, const std::string &inputBlobName, - const std::array &subVals, const std::array &divVals, bool normalize, - bool readCache) - : m_batchSize(batchSize), m_inputW(inputW), m_inputH(inputH), m_imgIdx(0), m_calibTableName(calibTableName), - m_inputBlobName(inputBlobName), m_subVals(subVals), m_divVals(divVals), m_normalize(normalize), m_readCache(readCache) { - - // Allocate GPU memory to hold the entire batch - m_inputCount = 3 * inputW * inputH * batchSize; - checkCudaErrorCode(cudaMalloc(&m_deviceInput, m_inputCount * sizeof(float))); - - // Read the name of all the files in the specified directory. - if (!doesFileExist(calibDataDirPath)) { - auto msg = "Error, directory at provided path does not exist: " + calibDataDirPath; - spdlog::error(msg); - throw std::runtime_error(msg); - } +#include "detail/engine_cache.h" +#include "detail/execution.h" +#include "detail/trt_common.h" - m_imgPaths = getFilesInDirectory(calibDataDirPath); - if (m_imgPaths.size() < static_cast(batchSize)) { - auto msg = "Error, there are fewer calibration images than the specified batch size!"; - spdlog::error(msg); - throw std::runtime_error(msg); - } +#include - // Randomize the calibration data - auto rd = std::random_device{}; - auto rng = std::default_random_engine{rd()}; - std::shuffle(std::begin(m_imgPaths), std::end(m_imgPaths), rng); -} +#include -int32_t Int8EntropyCalibrator2::getBatchSize() const noexcept { - // Return the batch size - return m_batchSize; -} +#include -bool Int8EntropyCalibrator2::getBatch(void **bindings, const char **names, int32_t nbBindings) noexcept { - // This method will read a batch of images into GPU memory, and place the - // pointer to the GPU memory in the bindings variable. +namespace trtcpp { - if (m_imgIdx + m_batchSize > static_cast(m_imgPaths.size())) { - // There are not enough images left to satisfy an entire batch - return false; - } +struct Engine::Impl { + EngineOptions options; + detail::TrtLoggerBridge bridge; + detail::TrtUniquePtr runtime; + detail::TrtUniquePtr engine; + detail::TrtUniquePtr context; + detail::EngineMeta meta; - // Read the calibration images into memory for the current batch - std::vector inputImgs; - for (int i = m_imgIdx; i < m_imgIdx + m_batchSize; i++) { - spdlog::info("Reading image {}: {}", i, m_imgPaths[i]); - auto cpuImg = cv::imread(m_imgPaths[i]); - if (cpuImg.empty()) { - spdlog::error("Fatal error: Unable to read image at path: " + m_imgPaths[i]); - return false; - } + explicit Impl(EngineOptions opts) : options(std::move(opts)), bridge(options.logger) {} +}; - cv::cuda::GpuMat gpuImg; - gpuImg.upload(cpuImg); - //cv::cuda::cvtColor(gpuImg, gpuImg, cv::COLOR_BGR2RGB); +Engine::Engine() = default; +Engine::Engine(Engine &&) noexcept = default; +Engine &Engine::operator=(Engine &&) noexcept = default; +Engine::~Engine() = default; - // TODO: Define any preprocessing code here, such as resizing - auto resized = Engine::resizeKeepAspectRatioPadRightBottom(gpuImg, m_inputH, m_inputW); +Result Engine::loadFromFile(const std::string &enginePath, const EngineOptions &options) { + auto data = detail::readFile(enginePath); // hardened: rejects directories/missing without throwing + if (!data) { + return data.status(); + } + return loadFromMemory(data.value(), options); +} - inputImgs.emplace_back(std::move(resized)); +Result Engine::loadFromMemory(std::span engineData, const EngineOptions &options) { + if (cudaError_t code = cudaSetDevice(options.deviceIndex); code != cudaSuccess) { + return cudaToStatus(code, "cudaSetDevice"); } + if (Status status = detail::loadPluginLibraries(options.pluginLibraries); !status) { + return status; + } + + Engine result; + result.impl_ = std::make_unique(options); + Impl &impl = *result.impl_; - // Convert the batch from NHWC to NCHW - // ALso apply normalization, scaling, and mean subtraction - auto mfloat = Engine::blobFromGpuMats(inputImgs, m_subVals, m_divVals, m_normalize, true); - auto *dataPointer = mfloat.ptr(); + impl.runtime.reset(nvinfer1::createInferRuntime(impl.bridge.nv())); + if (!impl.runtime) { + return Status{StatusCode::kTensorRtError, "createInferRuntime failed"}; + } + impl.engine.reset(impl.runtime->deserializeCudaEngine(engineData.data(), engineData.size())); + if (!impl.engine) { + return Status{StatusCode::kTensorRtError, "deserializeCudaEngine failed (corrupt or incompatible engine)"}; + } + impl.context.reset(impl.engine->createExecutionContext()); + if (!impl.context) { + return Status{StatusCode::kTensorRtError, "createExecutionContext failed"}; + } + impl.meta = detail::introspectEngine(*impl.engine, options.deviceIndex); + return result; +} - // Copy the GPU buffer to member variable so that it persists - checkCudaErrorCode(cudaMemcpyAsync(m_deviceInput, dataPointer, m_inputCount * sizeof(float), cudaMemcpyDeviceToDevice)); +std::vector Engine::tensors() const { return impl_->meta.tensors; } +std::vector Engine::inputNames() const { return impl_->meta.inputNames; } +std::vector Engine::outputNames() const { return impl_->meta.outputNames; } +int Engine::nbOptimizationProfiles() const { return impl_->meta.nbProfiles; } - m_imgIdx += m_batchSize; - if (std::string(names[0]) != m_inputBlobName) { - spdlog::error("Error: Incorrect input name provided!"); - return false; +Result Engine::tensorShape(const std::string &name) const { + for (const TensorInfo &info : impl_->meta.tensors) { + if (info.name == name) { + return info.shape; + } } - bindings[0] = m_deviceInput; - return true; + return Status{StatusCode::kNotFound, "no such tensor: " + name}; } -void const *Int8EntropyCalibrator2::readCalibrationCache(size_t &length) noexcept { - spdlog::info("Searching for calibration cache: {}", m_calibTableName); - m_calibCache.clear(); - std::ifstream input(m_calibTableName, std::ios::binary); - input >> std::noskipws; - if (m_readCache && input.good()) { - spdlog::info("Reading calibration cache: {}", m_calibTableName); - std::copy(std::istream_iterator(input), std::istream_iterator(), std::back_inserter(m_calibCache)); +Result Engine::tensorDType(const std::string &name) const { + for (const TensorInfo &info : impl_->meta.tensors) { + if (info.name == name) { + return info.dtype; + } } - length = m_calibCache.size(); - return length ? m_calibCache.data() : nullptr; + return Status{StatusCode::kNotFound, "no such tensor: " + name}; +} + +Status Engine::enqueue(const std::unordered_map &inputs, + const std::unordered_map &outputs, const Stream &stream, int profileIndex) { + return detail::enqueue(*impl_->context, impl_->meta, inputs, outputs, stream, profileIndex); } -void Int8EntropyCalibrator2::writeCalibrationCache(const void *ptr, std::size_t length) noexcept { - spdlog::info("Writing calibration cache: {}", m_calibTableName); - spdlog::info("Calibration cache size: {} bytes", length); - std::ofstream output(m_calibTableName, std::ios::binary); - output.write(reinterpret_cast(ptr), length); +Result> Engine::outputShapes(const std::unordered_map &inputs, + int profileIndex) { + return detail::outputShapes(*impl_->context, impl_->meta, inputs, profileIndex); +} + +Result> Engine::infer(const std::unordered_map &inputs, + const Stream &stream, int profileIndex) { + return detail::infer(*impl_->context, impl_->meta, inputs, stream, profileIndex); +} + +Result Engine::inferSingle(const std::unordered_map &inputs, const Stream &stream, int profileIndex) { + if (impl_->meta.outputNames.size() != 1) { + return Status{StatusCode::kInvalidArgument, + "inferSingle requires exactly one output; this engine has " + std::to_string(impl_->meta.outputNames.size())}; + } + auto outputs = infer(inputs, stream, profileIndex); + if (!outputs) { + return outputs.status(); + } + return std::move(outputs.value().begin()->second); } -Int8EntropyCalibrator2::~Int8EntropyCalibrator2() { checkCudaErrorCode(cudaFree(m_deviceInput)); }; +} // namespace trtcpp diff --git a/src/engine.h b/src/engine.h deleted file mode 100644 index d9caa0e..0000000 --- a/src/engine.h +++ /dev/null @@ -1,165 +0,0 @@ -#pragma once - -#include "NvOnnxParser.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "IEngine.h" -#include "logger.h" -#include "Int8Calibrator.h" -#include "util/Util.h" -#include "util/Stopwatch.h" -#include "macros.h" - -// Precision used for GPU inference -enum class Precision { - // Full precision floating point value - FP32, - // Half prevision floating point value - FP16, - // Int8 quantization. - // Has reduced dynamic range, may result in slight loss in accuracy. - // If INT8 is selected, must provide path to calibration dataset directory. - INT8, -}; - -// Options for the network -struct Options { - // Precision to use for GPU inference. - Precision precision = Precision::FP16; - // If INT8 precision is selected, must provide path to calibration dataset - // directory. - std::string calibrationDataDirectoryPath; - // The batch size to be used when computing calibration data for INT8 - // inference. Should be set to as large a batch number as your GPU will - // support. - int32_t calibrationBatchSize = 128; - // The batch size which should be optimized for. - int32_t optBatchSize = 1; - // Maximum allowable batch size - int32_t maxBatchSize = 16; - // GPU device index - int deviceIndex = 0; - // Directory where the engine file should be saved - std::string engineFileDir = "."; - // Maximum allowed input width - int32_t maxInputWidth = -1; // Default to -1 --> expecting fixed input size - // Minimum allowed input width - int32_t minInputWidth = -1; // Default to -1 --> expecting fixed input size - // Optimal input width - int32_t optInputWidth = -1; // Default to -1 --> expecting fixed input size -}; - -// Class to extend TensorRT logger -class Logger : public nvinfer1::ILogger { - void log(Severity severity, const char *msg) noexcept override; -}; - -template -class Engine : public IEngine { -public: - Engine(const Options &options); - ~Engine(); - - // Build the onnx model into a TensorRT engine file, cache the model to disk - // (to avoid rebuilding in future), and then load the model into memory The - // default implementation will normalize values between [0.f, 1.f] Setting the - // normalize flag to false will leave values between [0.f, 255.f] (some - // converted models may require this). If the model requires values to be - // normalized between [-1.f, 1.f], use the following params: - // subVals = {0.5f, 0.5f, 0.5f}; - // divVals = {0.5f, 0.5f, 0.5f}; - // normalize = true; - bool buildLoadNetwork(std::string onnxModelPath, const std::array &subVals = {0.f, 0.f, 0.f}, - const std::array &divVals = {1.f, 1.f, 1.f}, bool normalize = true) override; - - // Load a TensorRT engine file from disk into memory - // The default implementation will normalize values between [0.f, 1.f] - // Setting the normalize flag to false will leave values between [0.f, 255.f] - // (some converted models may require this). If the model requires values to - // be normalized between [-1.f, 1.f], use the following params: - // subVals = {0.5f, 0.5f, 0.5f}; - // divVals = {0.5f, 0.5f, 0.5f}; - // normalize = true; - bool loadNetwork(std::string trtModelPath, const std::array &subVals = {0.f, 0.f, 0.f}, - const std::array &divVals = {1.f, 1.f, 1.f}, bool normalize = true) override; - - // Run inference. - // Input format [input][batch][cv::cuda::GpuMat] - // Output format [batch][output][feature_vector] - bool runInference(const std::vector> &inputs, std::vector>> &featureVectors) override; - - // Utility method for resizing an image while maintaining the aspect ratio by - // adding padding to smaller dimension after scaling While letterbox padding - // normally adds padding to top & bottom, or left & right sides, this - // implementation only adds padding to the right or bottom side This is done - // so that it's easier to convert detected coordinates (ex. YOLO model) back - // to the original reference frame. - static cv::cuda::GpuMat resizeKeepAspectRatioPadRightBottom(const cv::cuda::GpuMat &input, size_t height, size_t width, - const cv::Scalar &bgcolor = cv::Scalar(0, 0, 0)); - - [[nodiscard]] const std::vector &getInputDims() const override { return m_inputDims; }; - [[nodiscard]] const std::vector &getOutputDims() const override { return m_outputDims; }; - - // Utility method for transforming triple nested output array into 2D array - // Should be used when the output batch size is 1, but there are multiple - // output feature vectors - static void transformOutput(std::vector>> &input, std::vector> &output); - - // Utility method for transforming triple nested output array into single - // array Should be used when the output batch size is 1, and there is only a - // single output feature vector - static void transformOutput(std::vector>> &input, std::vector &output); - // Convert NHWC to NCHW and apply scaling and mean subtraction - static cv::cuda::GpuMat blobFromGpuMats(const std::vector &batchInput, const std::array &subVals, - const std::array &divVals, bool normalize, bool swapRB = false); - -private: - // Build the network - bool build(std::string onnxModelPath, const std::array &subVals, const std::array &divVals, bool normalize); - - // Converts the engine options into a string - std::string serializeEngineOptions(const Options &options, const std::string &onnxModelPath); - - void getDeviceNames(std::vector &deviceNames); - - void clearGpuBuffers(); - - // Normalization, scaling, and mean subtraction of inputs - std::array m_subVals{}; - std::array m_divVals{}; - bool m_normalize; - - // Holds pointers to the input and output GPU buffers - std::vector m_buffers; - std::vector m_outputLengths{}; - std::vector m_inputDims; - std::vector m_outputDims; - std::vector m_IOTensorNames; - int32_t m_inputBatchSize; - - // Must keep IRuntime around for inference, see: - // https://forums.developer.nvidia.com/t/is-it-safe-to-deallocate-nvinfer1-iruntime-after-creating-an-nvinfer1-icudaengine-but-before-running-inference-with-said-icudaengine/255381/2?u=cyruspk4w6 - std::unique_ptr m_runtime = nullptr; - std::unique_ptr m_calibrator = nullptr; - std::unique_ptr m_engine = nullptr; - std::unique_ptr m_context = nullptr; - const Options m_options; - Logger m_logger; -}; - -template Engine::Engine(const Options &options) : m_options(options) {} - -template Engine::~Engine() { clearGpuBuffers(); } - -// Include inline implementations -#include "engine/EngineRunInference.inl" -#include "engine/EngineUtilities.inl" -#include "engine/EngineBuildLoadNetwork.inl" diff --git a/src/engine_builder.cpp b/src/engine_builder.cpp new file mode 100644 index 0000000..5b1199a --- /dev/null +++ b/src/engine_builder.cpp @@ -0,0 +1,415 @@ +#include "tensorrt_cpp_api/engine_builder.h" + +#include "detail/calibrator_bridge.h" +#include "detail/engine_cache.h" +#include "detail/sha256.h" +#include "detail/trt_common.h" + +#include "tensorrt_cpp_api/calibrator.h" + +#include + +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "tensorrt_cpp_api/cuda.h" + +namespace trtcpp { + +std::string_view toString(Precision p) noexcept { + switch (p) { + case Precision::kFp32: + return "fp32"; + case Precision::kFp16: + return "fp16"; + case Precision::kInt8Qdq: + return "int8_qdq"; + case Precision::kInt8CalibLegacy: + return "int8_calib_legacy"; + case Precision::kFp8: + return "fp8"; + case Precision::kNvfp4: + return "nvfp4"; + } + return "unknown"; +} + +namespace { + +static_assert(Shape::kMaxRank == nvinfer1::Dims::MAX_DIMS, "Shape::kMaxRank must match nvinfer1::Dims::MAX_DIMS"); + +nvinfer1::Dims toDims(const Shape &shape) { + nvinfer1::Dims dims; + dims.nbDims = shape.rank(); + for (int i = 0; i < shape.rank(); ++i) { + dims.d[i] = shape[i]; + } + return dims; +} + +// Validate the requested precision against the device's compute capability and the linked +// TensorRT version, so unsupported requests fail fast with a clear message rather than a +// cryptic builder error or a wrong-precision engine. +Status validatePrecision(const BuildOptions &options) { + if (options.precision == Precision::kInt8CalibLegacy) { +#if NV_TENSORRT_MAJOR >= 11 + return Status{StatusCode::kUnsupported, "kInt8CalibLegacy is removed in TensorRT 11; use kInt8Qdq with a modelopt-quantized ONNX"}; +#else + if (!options.calibrator) { + return Status{StatusCode::kInvalidArgument, "kInt8CalibLegacy requires options.calibrator (see makeRawBatchCalibrator)"}; + } +#endif + } + if (options.precision == Precision::kFp8 || options.precision == Precision::kNvfp4) { + cudaDeviceProp prop{}; + if (cudaError_t code = cudaGetDeviceProperties(&prop, options.deviceIndex); code != cudaSuccess) { + return cudaToStatus(code, "cudaGetDeviceProperties"); + } + const int sm = prop.major * 10 + prop.minor; + if (options.precision == Precision::kFp8 && sm < 89) { + return Status{StatusCode::kUnsupported, "FP8 requires compute capability >= 8.9 (Ada/Hopper/Blackwell)"}; + } + if (options.precision == Precision::kNvfp4 && prop.major < 10) { + return Status{StatusCode::kUnsupported, "NVFP4 requires compute capability >= 10.0 (Blackwell)"}; + } + } + return Status{}; +} + +bool resolveStronglyTyped(const BuildOptions &options) { + if (options.stronglyTyped.has_value()) { + return options.stronglyTyped.value(); + } +#if NV_TENSORRT_MAJOR >= 11 + return true; // strongly typed is mandatory/default in TRT 11 +#else + return options.precision == Precision::kInt8Qdq || options.precision == Precision::kFp8 || options.precision == Precision::kNvfp4; +#endif +} + +std::vector readFileBytes(const std::string &path, Status &status) { + std::error_code ec; + if (!std::filesystem::is_regular_file(path, ec)) { + // Rejects missing paths AND directories (whose tellg() returns a huge sentinel + // that would otherwise trigger a multi-exabyte allocation). + status = Status{StatusCode::kNotFound, "not a readable regular file: " + path}; + return {}; + } + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) { + status = Status{StatusCode::kNotFound, "cannot open file: " + path}; + return {}; + } + const std::streamsize size = file.tellg(); + if (size < 0) { + status = Status{StatusCode::kIoError, "failed to determine size of file: " + path}; + return {}; + } + file.seekg(0, std::ios::beg); + std::vector bytes(static_cast(size)); + if (size > 0 && !file.read(reinterpret_cast(bytes.data()), size)) { + status = Status{StatusCode::kIoError, "failed to read file: " + path}; + return {}; + } + status = Status{}; + return bytes; +} + +std::string trtVersionString() { + return std::to_string(NV_TENSORRT_MAJOR) + "." + std::to_string(NV_TENSORRT_MINOR) + "." + std::to_string(NV_TENSORRT_PATCH); +} + +std::string cudaVersionString() { + int version = 0; + cudaRuntimeGetVersion(&version); + return std::to_string(version); +} + +// A digest of the build options that affect the produced engine, so changing any of them +// invalidates the cache. +std::string buildOptionsDigest(const BuildOptions &options) { + std::string s = "p="; + s += toString(options.precision); + s += ";dla=" + std::to_string(options.dlaCore); + s += ";ws=" + std::to_string(options.workspaceBytes.value_or(0)); + s += ";st=" + std::string(options.stronglyTyped.has_value() ? (options.stronglyTyped.value() ? "1" : "0") : "auto"); + s += ";vc=" + std::to_string(static_cast(options.versionCompatible)); + s += ";hc=" + std::to_string(static_cast(options.hardwareCompatible)); + for (const std::string &lib : options.pluginLibraries) { + s += ";plugin=" + lib; + } + for (const OptimizationProfile &profile : options.profiles) { + s += "|prof"; + for (const ProfileShape &input : profile.inputs) { + s += ":" + input.inputName + "[" + input.min.toString() + input.opt.toString() + input.max.toString() + "]"; + } + } + return detail::sha256Hex(s.data(), s.size()); +} + +} // namespace + +EngineBuilder::EngineBuilder(std::shared_ptr logger) : logger_(std::move(logger)) { + if (!logger_) { + logger_ = defaultLogger(); + } +} + +Result> EngineBuilder::buildFromOnnxFile(const std::string &onnxPath, const BuildOptions &options) { + auto bytes = detail::readFile(onnxPath); + if (!bytes) { + return bytes.status(); + } + return buildFromOnnxBytes(bytes.value(), options); +} + +Result EngineBuilder::buildAndLoad(const std::string &onnxPath, const BuildOptions &options, const EngineOptions &engineOptions) { + auto path = buildOrLoad(onnxPath, options); + if (!path) { + return path.status(); + } + return Engine::loadFromFile(path.value(), engineOptions); +} + +Result EngineBuilder::buildOrLoad(const std::string &onnxPath, const BuildOptions &options) { + auto onnxResult = detail::readFile(onnxPath); + if (!onnxResult) { + return onnxResult.status(); + } + std::vector &onnx = onnxResult.value(); + + detail::CacheMeta expected; + expected.onnxSha256 = detail::sha256Hex(onnx.data(), onnx.size()); + expected.trtVersion = trtVersionString(); + expected.cudaVersion = cudaVersionString(); + auto device = queryDevice(options.deviceIndex); + if (!device) { + return device.status(); + } + expected.gpuName = device.value().name; + expected.gpuUuid = device.value().uuid; + expected.precision = std::string(toString(options.precision)); + expected.buildOptionsDigest = buildOptionsDigest(options); + expected.versionCompatible = options.versionCompatible; + expected.hardwareCompatible = options.hardwareCompatible; + + const std::string stem = std::filesystem::path(onnxPath).stem().string(); + // Relax the filename for portable engines so the relaxed sidecar check can actually + // find them: major-only TRT for version-compatible, a fixed token for hardware-compatible. + const std::string fileTrtVersion = options.versionCompatible ? std::to_string(NV_TENSORRT_MAJOR) : expected.trtVersion; + const std::string fileGpuUuid = options.hardwareCompatible ? std::string("hwcompat") : expected.gpuUuid; + const std::string fileName = detail::cacheFileName(stem, expected.onnxSha256, fileTrtVersion, fileGpuUuid, expected.precision); + const std::filesystem::path enginePath = std::filesystem::path(options.engineCacheDir) / fileName; + std::filesystem::path sidecarPath = enginePath; + sidecarPath += ".json"; + + if (std::filesystem::is_regular_file(enginePath) && std::filesystem::is_regular_file(sidecarPath)) { + auto meta = detail::readSidecar(sidecarPath.string()); + // A hardware-compatible (Ampere-plus) engine must not be reused on a pre-Ampere GPU + // even though the UUID check is relaxed -- gate on the loading GPU's compute capability. + // Gate on whether the CACHED engine was built hardware-compatible (the sidecar flag), not + // the current request: the safety property belongs to the artifact on disk, independent of + // the filename convention that led us here. + const bool hardwareFloorOk = !meta || !meta.value().hardwareCompatible || device.value().computeMajor >= 8; + if (meta && hardwareFloorOk && detail::isFresh(meta.value(), expected, options.versionCompatible, options.hardwareCompatible)) { + return enginePath.string(); // fresh cache hit + } + } + + auto engine = buildFromOnnxBytes(onnx, options); + if (!engine) { + return engine.status(); + } + if (Status written = detail::writeAtomic(enginePath.string(), engine.value()); !written) { + return written; + } + expected.createdUnix = static_cast(std::time(nullptr)); + if (Status written = detail::writeSidecar(sidecarPath.string(), expected); !written) { + return written; + } + return enginePath.string(); +} + +Result> EngineBuilder::buildFromOnnxBytes(std::span onnx, const BuildOptions &options) { + using detail::TrtLoggerBridge; + using detail::TrtUniquePtr; + + if (Status status = validatePrecision(options); !status) { + return status; + } + if (Status status = detail::loadPluginLibraries(options.pluginLibraries); !status) { + return status; + } + int previousDevice = 0; + cudaGetDevice(&previousDevice); + if (cudaError_t code = cudaSetDevice(options.deviceIndex); code != cudaSuccess) { + return cudaToStatus(code, "cudaSetDevice"); + } + // Restore the caller's current device on every exit path. + struct DeviceRestore { + int device; + ~DeviceRestore() { cudaSetDevice(device); } + } deviceRestore{previousDevice}; + + TrtLoggerBridge bridge(logger_); + TrtUniquePtr builder{nvinfer1::createInferBuilder(bridge.nv())}; + if (!builder) { + return Status{StatusCode::kTensorRtError, "createInferBuilder failed"}; + } + + const bool stronglyTyped = resolveStronglyTyped(options); + uint32_t networkFlags = 0; + if (stronglyTyped) { + networkFlags |= 1U << static_cast(nvinfer1::NetworkDefinitionCreationFlag::kSTRONGLY_TYPED); + } + TrtUniquePtr network{builder->createNetworkV2(networkFlags)}; + if (!network) { + return Status{StatusCode::kTensorRtError, "createNetworkV2 failed"}; + } + + TrtUniquePtr parser{nvonnxparser::createParser(*network, bridge.nv())}; + if (!parser) { + return Status{StatusCode::kTensorRtError, "createParser failed"}; + } + if (!parser->parse(onnx.data(), onnx.size())) { + std::string message = "ONNX parse failed"; + for (int i = 0; i < parser->getNbErrors(); ++i) { + message += "\n "; + message += parser->getError(i)->desc(); + } + return Status{StatusCode::kTensorRtError, std::move(message)}; + } + +#if NV_TENSORRT_MAJOR >= 11 + if (options.precision == Precision::kFp16) { + // Strongly typed (TRT 11): precision comes from the graph. Honor an FP16/mixed ONNX; + // a plain FP32 ONNX cannot be downcast here -- fail loudly with the remedy. + // Look for ANY kHALF tensor (inputs or layer outputs) -- keep_io_types FP16 graphs + // keep FP32 IO with internal half tensors, so checking only inputs is too strict. + bool hasHalf = false; + for (int i = 0; i < network->getNbInputs() && !hasHalf; ++i) { + hasHalf = network->getInput(i)->getType() == nvinfer1::DataType::kHALF; + } + for (int i = 0; i < network->getNbLayers() && !hasHalf; ++i) { + nvinfer1::ILayer *layer = network->getLayer(i); + for (int j = 0; j < layer->getNbOutputs() && !hasHalf; ++j) { + hasHalf = layer->getOutput(j)->getType() == nvinfer1::DataType::kHALF; + } + } + if (!hasHalf) { + return Status{StatusCode::kUnsupported, + "kFp16 on TensorRT >= 11 requires an FP16/QDQ ONNX; cast the model offline (modelopt/onnx fp16)"}; + } + } +#endif + + TrtUniquePtr config{builder->createBuilderConfig()}; + if (!config) { + return Status{StatusCode::kTensorRtError, "createBuilderConfig failed"}; + } + + if (options.workspaceBytes.has_value()) { + config->setMemoryPoolLimit(nvinfer1::MemoryPoolType::kWORKSPACE, options.workspaceBytes.value()); + } + +#if NV_TENSORRT_MAJOR < 11 + // Weak-typing path (precision flags). On TRT 11 these flags are removed; strong typing + // carries precision instead. The calibrator bridge must outlive buildSerializedNetwork. + std::unique_ptr calibratorBridge; + if (!stronglyTyped) { + if (options.precision == Precision::kFp16) { + config->setFlag(nvinfer1::BuilderFlag::kFP16); + } else if (options.precision == Precision::kInt8CalibLegacy) { + if (!builder->platformHasFastInt8()) { + return Status{StatusCode::kUnsupported, "GPU does not support fast INT8"}; + } + if (network->getNbInputs() != 1) { + return Status{StatusCode::kUnsupported, "legacy INT8 calibration supports single-input models only"}; + } + config->setFlag(nvinfer1::BuilderFlag::kINT8); + const nvinfer1::ITensor *input = network->getInput(0); + const nvinfer1::Dims inputDims = input->getDimensions(); + for (int i = 0; i < inputDims.nbDims; ++i) { + if (inputDims.d[i] < 0) { + return Status{StatusCode::kUnsupported, "legacy INT8 calibration requires a static input shape"}; + } + } + std::vector batchDims; + batchDims.push_back(options.calibrator->batchSize()); + for (int i = 1; i < inputDims.nbDims; ++i) { + batchDims.push_back(inputDims.d[i]); + } + Shape batchShape{std::span(batchDims.data(), batchDims.size())}; + calibratorBridge = std::make_unique(options.calibrator, input->getName(), batchShape); + config->setInt8Calibrator(calibratorBridge.get()); + } + } +#endif + + if (options.versionCompatible) { + config->setFlag(nvinfer1::BuilderFlag::kVERSION_COMPATIBLE); + } + if (options.hardwareCompatible) { + config->setHardwareCompatibilityLevel(nvinfer1::HardwareCompatibilityLevel::kAMPERE_PLUS); + } + if (options.dlaCore >= 0) { + config->setDefaultDeviceType(nvinfer1::DeviceType::kDLA); + config->setDLACore(options.dlaCore); + config->setFlag(nvinfer1::BuilderFlag::kGPU_FALLBACK); + } + + for (const OptimizationProfile &profile : options.profiles) { + nvinfer1::IOptimizationProfile *trtProfile = builder->createOptimizationProfile(); + for (const ProfileShape &input : profile.inputs) { + trtProfile->setDimensions(input.inputName.c_str(), nvinfer1::OptProfileSelector::kMIN, toDims(input.min)); + trtProfile->setDimensions(input.inputName.c_str(), nvinfer1::OptProfileSelector::kOPT, toDims(input.opt)); + trtProfile->setDimensions(input.inputName.c_str(), nvinfer1::OptProfileSelector::kMAX, toDims(input.max)); + } + config->addOptimizationProfile(trtProfile); + } + + // Timing cache: reuse tactic timings across cold builds. + const std::string timingPath = options.timingCachePath.empty() ? (options.engineCacheDir + "/timing.cache") : options.timingCachePath; + Status timingStatus; + std::vector timingSeed = readFileBytes(timingPath, timingStatus); // missing is fine + TrtUniquePtr timingCache{ + config->createTimingCache(timingSeed.data(), timingStatus.ok() ? timingSeed.size() : 0)}; + if (timingCache) { + config->setTimingCache(*timingCache, /*ignoreMismatch=*/true); // reuse cross-build seeds + } + + Stream profileStream; + config->setProfileStream(profileStream.handle()); + + TrtUniquePtr plan{builder->buildSerializedNetwork(*network, *config)}; + if (!plan) { + return Status{StatusCode::kTensorRtError, "buildSerializedNetwork failed"}; + } + + if (timingCache) { + if (TrtUniquePtr serialized{timingCache->serialize()}) { + std::error_code mkec; + std::filesystem::create_directories(std::filesystem::path(timingPath).parent_path(), mkec); + std::ofstream out(timingPath, std::ios::binary); + if (out) { + out.write(static_cast(serialized->data()), static_cast(serialized->size())); + } + } + } + + const auto *data = static_cast(plan->data()); + return std::vector(data, data + plan->size()); +} + +} // namespace trtcpp diff --git a/src/engine_pool.cpp b/src/engine_pool.cpp new file mode 100644 index 0000000..bfe7fe9 --- /dev/null +++ b/src/engine_pool.cpp @@ -0,0 +1,200 @@ +#include "tensorrt_cpp_api/engine_pool.h" + +#include "detail/engine_cache.h" +#include "detail/execution.h" +#include "detail/trt_common.h" + +#include + +#include + +#include +#include +#include + +namespace trtcpp { + +namespace detail { + +struct PoolState { + EngineOptions options; + TrtLoggerBridge bridge; + TrtUniquePtr runtime; + TrtUniquePtr engine; + std::vector> contexts; + EngineMeta meta; + + std::mutex mutex; + std::condition_variable cv; + std::vector inUse; + + explicit PoolState(EngineOptions opts) : options(std::move(opts)), bridge(options.logger) {} + + int profileForSlot(int slot) const { return meta.hasDynamic ? slot : 0; } + + void release(int slot) { + { + std::lock_guard lock(mutex); + inUse[static_cast(slot)] = false; + } + cv.notify_one(); + } +}; + +} // namespace detail + +// ---- Lease ---------------------------------------------------------------------------- + +EnginePool::Lease::Lease(Lease &&other) noexcept : state_(std::move(other.state_)), slot_(other.slot_), profile_(other.profile_) { + other.slot_ = -1; +} + +EnginePool::Lease &EnginePool::Lease::operator=(Lease &&other) noexcept { + if (this != &other) { + if (slot_ >= 0 && state_) { + state_->release(slot_); + } + state_ = std::move(other.state_); + slot_ = other.slot_; + profile_ = other.profile_; + other.slot_ = -1; + } + return *this; +} + +EnginePool::Lease::~Lease() { + if (slot_ >= 0 && state_) { + state_->release(slot_); + } +} + +Status EnginePool::Lease::enqueue(const std::unordered_map &inputs, + const std::unordered_map &outputs, const Stream &stream) { + if (!valid()) { + return Status{StatusCode::kInvalidArgument, "lease is not valid (default-constructed or moved-from)"}; + } + return detail::enqueue(*state_->contexts[static_cast(slot_)], state_->meta, inputs, outputs, stream, profile_); +} + +Result> EnginePool::Lease::infer(const std::unordered_map &inputs, + const Stream &stream) { + if (!valid()) { + return Status{StatusCode::kInvalidArgument, "lease is not valid (default-constructed or moved-from)"}; + } + return detail::infer(*state_->contexts[static_cast(slot_)], state_->meta, inputs, stream, profile_); +} + +Result EnginePool::Lease::inferSingle(const std::unordered_map &inputs, const Stream &stream) { + if (!valid()) { + return Status{StatusCode::kInvalidArgument, "lease is not valid (default-constructed or moved-from)"}; + } + if (state_->meta.outputNames.size() != 1) { + return Status{StatusCode::kInvalidArgument, + "inferSingle requires exactly one output; this engine has " + std::to_string(state_->meta.outputNames.size())}; + } + auto outputs = infer(inputs, stream); + if (!outputs) { + return outputs.status(); + } + return std::move(outputs.value().begin()->second); +} + +Result> EnginePool::Lease::outputShapes(const std::unordered_map &inputs) { + if (!valid()) { + return Status{StatusCode::kInvalidArgument, "lease is not valid (default-constructed or moved-from)"}; + } + return detail::outputShapes(*state_->contexts[static_cast(slot_)], state_->meta, inputs, profile_); +} + +// ---- EnginePool ----------------------------------------------------------------------- + +EnginePool::EnginePool(EnginePool &&) noexcept = default; +EnginePool &EnginePool::operator=(EnginePool &&) noexcept = default; +EnginePool::~EnginePool() = default; + +Result EnginePool::create(const std::string &enginePath, int contexts, const EngineOptions &options) { + if (contexts < 1) { + return Status{StatusCode::kInvalidArgument, "EnginePool requires at least one context"}; + } + auto data = detail::readFile(enginePath); + if (!data) { + return data.status(); + } + if (cudaError_t code = cudaSetDevice(options.deviceIndex); code != cudaSuccess) { + return cudaToStatus(code, "cudaSetDevice"); + } + if (Status status = detail::loadPluginLibraries(options.pluginLibraries); !status) { + return status; + } + + auto state = std::make_shared(options); + state->runtime.reset(nvinfer1::createInferRuntime(state->bridge.nv())); + if (!state->runtime) { + return Status{StatusCode::kTensorRtError, "createInferRuntime failed"}; + } + state->engine.reset(state->runtime->deserializeCudaEngine(data.value().data(), data.value().size())); + if (!state->engine) { + return Status{StatusCode::kTensorRtError, "deserializeCudaEngine failed (corrupt or incompatible engine)"}; + } + state->meta = detail::introspectEngine(*state->engine, options.deviceIndex); + + // Dynamic-shape engines need a distinct optimization profile per concurrent context; + // static engines share the default profile across any number of contexts. + if (state->meta.hasDynamic && contexts > state->meta.nbProfiles) { + return Status{StatusCode::kInvalidArgument, "EnginePool needs one optimization profile per context: engine has " + + std::to_string(state->meta.nbProfiles) + " profiles but " + + std::to_string(contexts) + " contexts were requested"}; + } + + for (int i = 0; i < contexts; ++i) { + // Default (kSTATIC) allocation: each context pre-allocates its own activation memory, + // which is what makes concurrent enqueueV3 on distinct contexts safe (no shared scratch). + detail::TrtUniquePtr context{state->engine->createExecutionContext()}; + if (!context) { + return Status{StatusCode::kTensorRtError, "createExecutionContext failed"}; + } + state->contexts.push_back(std::move(context)); + } + state->inUse.assign(static_cast(contexts), false); + + EnginePool pool; + pool.state_ = std::move(state); + return pool; +} + +Result EnginePool::acquire() { + std::unique_lock lock(state_->mutex); + int slot = -1; + // The predicate records the free slot; safe because condition_variable evaluates it + // under the held lock, and the lock is still held when it returns true. + state_->cv.wait(lock, [&] { + for (std::size_t i = 0; i < state_->inUse.size(); ++i) { + if (!state_->inUse[i]) { + slot = static_cast(i); + return true; + } + } + return false; + }); + state_->inUse[static_cast(slot)] = true; + return Lease{state_, slot, state_->profileForSlot(slot)}; +} + +std::optional EnginePool::tryAcquire() { + std::lock_guard lock(state_->mutex); + for (std::size_t i = 0; i < state_->inUse.size(); ++i) { + if (!state_->inUse[i]) { + state_->inUse[i] = true; + const int slot = static_cast(i); + return Lease{state_, slot, state_->profileForSlot(slot)}; + } + } + return std::nullopt; +} + +int EnginePool::size() const noexcept { return static_cast(state_->contexts.size()); } +std::vector EnginePool::tensors() const { return state_->meta.tensors; } +std::vector EnginePool::inputNames() const { return state_->meta.inputNames; } +std::vector EnginePool::outputNames() const { return state_->meta.outputNames; } + +} // namespace trtcpp diff --git a/src/logger.cpp b/src/logger.cpp new file mode 100644 index 0000000..79c26ef --- /dev/null +++ b/src/logger.cpp @@ -0,0 +1,124 @@ +#include "tensorrt_cpp_api/logger.h" + +#include +#include +#include +#include +#include + +#ifdef TRT_CPP_API_WITH_SPDLOG +#include +#endif + +namespace trtcpp { + +std::string_view toString(Severity s) noexcept { + switch (s) { + case Severity::kVerbose: + return "VERBOSE"; + case Severity::kInfo: + return "INFO"; + case Severity::kWarning: + return "WARNING"; + case Severity::kError: + return "ERROR"; + case Severity::kInternalError: + return "INTERNAL_ERROR"; + } + return "UNKNOWN"; +} + +namespace { + +// threshold_ is an int (not a Severity) so the "off" level can sit one past +// kInternalError (value 5) and drop every real message, including internal errors. +class StderrLogger final : public ILogger { +public: + explicit StderrLogger(int threshold) : threshold_(threshold) {} + + void log(Severity severity, std::string_view message) noexcept override { + if (static_cast(severity) < threshold_) { + return; + } + const std::string_view tag = toString(severity); + std::lock_guard lock(mutex_); + std::fprintf(stderr, "[trtcpp] [%.*s] %.*s\n", static_cast(tag.size()), tag.data(), static_cast(message.size()), + message.data()); + std::fflush(stderr); + } + +private: + int threshold_; + std::mutex mutex_; +}; + +constexpr int kSilentThreshold = static_cast(Severity::kInternalError) + 1; + +int thresholdFromEnv() { + const char *value = std::getenv("TRTCPP_LOG_LEVEL"); + if (value == nullptr) { + value = std::getenv("LOG_LEVEL"); // v6 compatibility + } + std::string level = (value != nullptr) ? value : "info"; + for (char &c : level) { + c = static_cast(std::tolower(static_cast(c))); + } + if (level == "verbose" || level == "trace" || level == "debug") { + return static_cast(Severity::kVerbose); + } + if (level == "warning" || level == "warn") { + return static_cast(Severity::kWarning); + } + if (level == "error" || level == "err") { + return static_cast(Severity::kError); + } + if (level == "off") { + return kSilentThreshold; // suppress everything, including internal errors + } + return static_cast(Severity::kInfo); +} + +} // namespace + +std::shared_ptr makeStderrLogger(Severity threshold) { return std::make_shared(static_cast(threshold)); } + +std::shared_ptr defaultLogger() { + static std::shared_ptr instance = std::make_shared(thresholdFromEnv()); + return instance; +} + +#ifdef TRT_CPP_API_WITH_SPDLOG +namespace { +class SpdlogLogger final : public ILogger { +public: + void log(Severity severity, std::string_view message) noexcept override { + // ILogger::log is noexcept; never let a logging-backend exception escape, even if + // spdlog/fmt were built to throw a non-std::exception type. + try { + switch (severity) { + case Severity::kVerbose: + spdlog::debug("{}", message); + break; + case Severity::kInfo: + spdlog::info("{}", message); + break; + case Severity::kWarning: + spdlog::warn("{}", message); + break; + case Severity::kError: + spdlog::error("{}", message); + break; + case Severity::kInternalError: + spdlog::critical("{}", message); + break; + } + } catch (...) { + } + } +}; +} // namespace + +std::shared_ptr makeSpdlogLogger() { return std::make_shared(); } +#endif + +} // namespace trtcpp diff --git a/src/main.cpp b/src/main.cpp deleted file mode 100644 index d3883a2..0000000 --- a/src/main.cpp +++ /dev/null @@ -1,173 +0,0 @@ -#include "cmd_line_parser.h" -#include "logger.h" -#include "engine.h" -#include -#include -#include - -int main(int argc, char *argv[]) { - CommandLineArguments arguments; - - std::string logLevelStr = getLogLevelFromEnvironment(); - spdlog::level::level_enum logLevel = toSpdlogLevel(logLevelStr); - spdlog::set_level(logLevel); - - // Parse the command line arguments - if (!parseArguments(argc, argv, arguments)) { - return -1; - } - - // Specify our GPU inference configuration options - Options options; - // Specify what precision to use for inference - // FP16 is approximately twice as fast as FP32. - options.precision = Precision::FP16; - // If using INT8 precision, must specify path to directory containing - // calibration data. - options.calibrationDataDirectoryPath = ""; - // Specify the batch size to optimize for. - options.optBatchSize = 1; - // Specify the maximum batch size we plan on running. - options.maxBatchSize = 1; - // Specify the directory where you want the model engine model file saved. - options.engineFileDir = "."; - - Engine engine(options); - - // Define our preprocessing code - // The default Engine::build method will normalize values between [0.f, 1.f] - // Setting the normalize flag to false will leave values between [0.f, 255.f] - // (some converted models may require this). - - // For our YoloV8 model, we need the values to be normalized between - // [0.f, 1.f] so we use the following params - std::array subVals{0.f, 0.f, 0.f}; - std::array divVals{1.f, 1.f, 1.f}; - bool normalize = true; - // Note, we could have also used the default values. - - // If the model requires values to be normalized between [-1.f, 1.f], use the - // following params: - // subVals = {0.5f, 0.5f, 0.5f}; - // divVals = {0.5f, 0.5f, 0.5f}; - // normalize = true; - - if (!arguments.onnxModelPath.empty()) { - // Build the onnx model into a TensorRT engine file, and load the TensorRT - // engine file into memory. - bool succ = engine.buildLoadNetwork(arguments.onnxModelPath, subVals, divVals, normalize); - if (!succ) { - throw std::runtime_error("Unable to build or load TensorRT engine."); - } - } else { - // Load the TensorRT engine file directly - bool succ = engine.loadNetwork(arguments.trtModelPath, subVals, divVals, normalize); - if (!succ) { - const std::string msg = "Unable to load TensorRT engine."; - spdlog::error(msg); - throw std::runtime_error(msg); - } - } - - // Read the input image - // TODO: You will need to read the input image required for your model - const std::string inputImage = "../inputs/team.jpg"; - auto cpuImg = cv::imread(inputImage); - if (cpuImg.empty()) { - const std::string msg = "Unable to read image at path: " + inputImage; - spdlog::error(msg); - throw std::runtime_error(msg); - } - - // Upload the image GPU memory - cv::cuda::GpuMat img; - img.upload(cpuImg); - - // The model expects RGB input - cv::cuda::cvtColor(img, img, cv::COLOR_BGR2RGB); - - // In the following section we populate the input vectors to later pass for - // inference - const auto &inputDims = engine.getInputDims(); - std::vector> inputs; - - // Let's use a batch size which matches that which we set the - // Options.optBatchSize option - size_t batchSize = options.optBatchSize; - - // TODO: - // For the sake of the demo, we will be feeding the same image to all the - // inputs You should populate your inputs appropriately. - for (const auto &inputDim : inputDims) { // For each of the model inputs... - std::vector input; - for (size_t j = 0; j < batchSize; ++j) { // For each element we want to add to the batch... - // TODO: - // You can choose to resize by scaling, adding padding, or a combination - // of the two in order to maintain the aspect ratio You can use the - // Engine::resizeKeepAspectRatioPadRightBottom to resize to a square while - // maintain the aspect ratio (adds padding where necessary to achieve - // this). - auto resized = Engine::resizeKeepAspectRatioPadRightBottom(img, inputDim.d[1], inputDim.d[2]); - // You could also perform a resize operation without maintaining aspect - // ratio with the use of padding by using the following instead: - // cv::cuda::resize(img, resized, cv::Size(inputDim.d[2], - // inputDim.d[1])); // TRT dims are (height, width) whereas - // OpenCV is (width, height) - input.emplace_back(std::move(resized)); - } - inputs.emplace_back(std::move(input)); - } - - // Warm up the network before we begin the benchmark - spdlog::info("Warming up the network..."); - std::vector>> featureVectors; - for (int i = 0; i < 100; ++i) { - bool succ = engine.runInference(inputs, featureVectors); - if (!succ) { - const std::string msg = "Unable to run inference."; - spdlog::error(msg); - throw std::runtime_error(msg); - } - } - - // Benchmark the inference time - size_t numIterations = 1000; - spdlog::info("Running benchmarks ({} iterations)...", numIterations); - preciseStopwatch stopwatch; - for (size_t i = 0; i < numIterations; ++i) { - featureVectors.clear(); - engine.runInference(inputs, featureVectors); - } - auto totalElapsedTimeMs = stopwatch.elapsedTime(); - auto avgElapsedTimeMs = totalElapsedTimeMs / numIterations / static_cast(inputs[0].size()); - - spdlog::info("Benchmarking complete!"); - spdlog::info("======================"); - spdlog::info("Avg time per sample: "); - spdlog::info("Avg time per sample: {} ms", avgElapsedTimeMs); - spdlog::info("Batch size: {}", inputs[0].size()); - spdlog::info("Avg FPS: {} fps", static_cast(1000 / avgElapsedTimeMs)); - spdlog::info("======================\n"); - - // Print the feature vectors - for (size_t batch = 0; batch < featureVectors.size(); ++batch) { - for (size_t outputNum = 0; outputNum < featureVectors[batch].size(); ++outputNum) { - spdlog::info("Batch {}, output {}", batch, outputNum); - std::string output; - int i = 0; - for (const auto &e : featureVectors[batch][outputNum]) { - output += std::to_string(e) + " "; - if (++i == 10) { - output += "..."; - break; - } - } - spdlog::info("{}", output); - } - } - - // TODO: If your model requires post processing (ex. convert feature vector - // into bounding boxes) then you would do so here. - - return 0; -} diff --git a/src/opencv_interop.cpp b/src/opencv_interop.cpp new file mode 100644 index 0000000..d3c2406 --- /dev/null +++ b/src/opencv_interop.cpp @@ -0,0 +1,83 @@ +#include "tensorrt_cpp_api/opencv_interop.h" + +#ifdef TRT_CPP_API_WITH_OPENCV + +#include +#include + +namespace trtcpp::opencv { + +Result dtypeOfCvDepth(int cvDepth) { + switch (cvDepth) { + case CV_8U: + return DType::kUInt8; + case CV_8S: + return DType::kInt8; + case CV_32S: + return DType::kInt32; + case CV_32F: + return DType::kFloat32; + case CV_16F: + return DType::kFloat16; + default: + return Status{StatusCode::kUnsupported, "unsupported OpenCV element depth"}; + } +} + +Result viewOf(const cv::cuda::GpuMat &mat, Layout layout) { + if (mat.empty()) { + return Status{StatusCode::kInvalidArgument, "empty GpuMat"}; + } + auto dtype = dtypeOfCvDepth(mat.depth()); + if (!dtype) { + return dtype.status(); + } + const std::size_t rowBytes = static_cast(mat.cols) * mat.elemSize(); + if (mat.step != rowBytes) { + // GpuMat rows are usually pitched (padded for alignment); a TensorView has no row stride. + // Note clone() does NOT help (it is also pitched) -- copy into a cv::cuda::createContinuous + // buffer first. + return Status{StatusCode::kUnsupported, "non-continuous GpuMat (padded rows); copy into a cv::cuda::createContinuous buffer first"}; + } + Shape shape{mat.rows, mat.cols, mat.channels()}; + // The GpuMat lives on the current CUDA device (OpenCV allocates on it); record that rather + // than hardcoding device 0, so a multi-GPU caller's downstream copyFrom/enqueue targets the + // right device. + int device = 0; + cudaGetDevice(&device); + return TensorView{mat.cudaPtr(), dtype.value(), std::move(shape), Device::kCuda, device, layout}; +} + +Result viewOf(const cv::Mat &mat, Layout layout) { + if (mat.empty()) { + return Status{StatusCode::kInvalidArgument, "empty cv::Mat"}; + } + if (!mat.isContinuous()) { + return Status{StatusCode::kUnsupported, "non-continuous cv::Mat; clone() to a continuous buffer first"}; + } + auto dtype = dtypeOfCvDepth(mat.depth()); + if (!dtype) { + return dtype.status(); + } + Shape shape{mat.rows, mat.cols, mat.channels()}; + return TensorView{const_cast(mat.data), dtype.value(), std::move(shape), Device::kHost, 0, layout}; +} + +Result upload(const cv::Mat &mat, const Stream &stream) { + auto hostView = viewOf(mat); + if (!hostView) { + return hostView.status(); + } + auto device = Tensor::allocate(hostView.value().dtype(), hostView.value().shape(), Device::kCuda); + if (!device) { + return device.status(); + } + if (Status status = device.value().copyFrom(hostView.value(), stream); !status) { + return status; + } + return std::move(device).value(); +} + +} // namespace trtcpp::opencv + +#endif // TRT_CPP_API_WITH_OPENCV diff --git a/src/preproc.cu b/src/preproc.cu new file mode 100644 index 0000000..71cbfa1 --- /dev/null +++ b/src/preproc.cu @@ -0,0 +1,148 @@ +#include "tensorrt_cpp_api/preproc.h" + +#include +#include + +#include + +namespace trtcpp::preproc { +namespace { + +template __device__ T toOut(float x); +template <> __device__ float toOut(float x) { return x; } +template <> __device__ __half toOut<__half>(float x) { return __float2half(x); } + +struct Norm { + float mean[4]; + float scale[4]; +}; + +template +__global__ void letterboxKernel(const unsigned char *src, int srcH, int srcW, int channels, OutT *dst, int outC, int outH, int outW, + int newW, int newH, float invScaleX, float invScaleY, Norm norm, bool swapRB, unsigned char padValue) { + const int index = blockIdx.x * blockDim.x + threadIdx.x; + const int total = outC * outH * outW; + if (index >= total) { + return; + } + const int ox = index % outW; + const int oy = (index / outW) % outH; + const int oc = index / (outW * outH); + const float mean = oc < 4 ? norm.mean[oc] : 0.0f; + const float scale = oc < 4 ? norm.scale[oc] : 1.0f; + + float value; + if (ox < newW && oy < newH) { + const float sx = (ox + 0.5f) * invScaleX - 0.5f; + const float sy = (oy + 0.5f) * invScaleY - 0.5f; + int x0 = static_cast(floorf(sx)); + int y0 = static_cast(floorf(sy)); + const float ax = sx - x0; + const float ay = sy - y0; + const int x1 = min(x0 + 1, srcW - 1); + const int y1 = min(y0 + 1, srcH - 1); + x0 = max(0, min(x0, srcW - 1)); + y0 = max(0, min(y0, srcH - 1)); + // BGR<->RGB swap only applies to the first 3 channels; for an output channel >= 3 (or a + // non-3-channel source) keep identity. Clamp to [0, channels-1] so a swapped index can + // never go negative or out of bounds (e.g. oc>=3 would make 2-oc negative). + int ic = (swapRB && channels == 3 && oc < 3) ? (2 - oc) : oc; + ic = max(0, min(ic, channels - 1)); + const float p00 = src[(y0 * srcW + x0) * channels + ic]; + const float p01 = src[(y0 * srcW + x1) * channels + ic]; + const float p10 = src[(y1 * srcW + x0) * channels + ic]; + const float p11 = src[(y1 * srcW + x1) * channels + ic]; + const float top = p00 * (1.0f - ax) + p01 * ax; + const float bottom = p10 * (1.0f - ax) + p11 * ax; + value = top * (1.0f - ay) + bottom * ay; + } else { + value = static_cast(padValue); + } + dst[oc * outH * outW + oy * outW + ox] = toOut((value - mean) * scale); +} + +// Read (H, W, C) from a TensorView whose shape is [H,W,C] or [1,H,W,C]. +bool readHwc(const TensorView &src, int &h, int &w, int &c) { + const Shape &shape = src.shape(); + if (shape.rank() == 3) { + h = static_cast(shape[0]); + w = static_cast(shape[1]); + c = static_cast(shape[2]); + return true; + } + if (shape.rank() == 4 && shape[0] == 1) { + h = static_cast(shape[1]); + w = static_cast(shape[2]); + c = static_cast(shape[3]); + return true; + } + return false; +} + +} // namespace + +Status letterboxToTensor(TensorView src, TensorView dst, const PreprocSpec &spec, const Stream &stream) { + if (!src.isCuda() || !dst.isCuda()) { + return Status{StatusCode::kInvalidArgument, "letterboxToTensor requires device tensors"}; + } + if (src.dtype() != DType::kUInt8) { + return Status{StatusCode::kDtypeMismatch, "letterboxToTensor source must be uint8 HWC"}; + } + if (dst.dtype() != DType::kFloat32 && dst.dtype() != DType::kFloat16) { + return Status{StatusCode::kDtypeMismatch, "letterboxToTensor destination must be float32 or float16"}; + } + if (dst.shape().rank() != 4 || dst.shape()[0] != 1) { + return Status{StatusCode::kInvalidArgument, "letterboxToTensor destination must be NCHW with N==1"}; + } + int srcH = 0, srcW = 0, channels = 0; + if (!readHwc(src, srcH, srcW, channels) || srcH <= 0 || srcW <= 0 || channels <= 0) { + return Status{StatusCode::kInvalidArgument, "letterboxToTensor source must be HWC ([H,W,C] or [1,H,W,C])"}; + } + const int outC = static_cast(dst.shape()[1]); + const int outH = static_cast(dst.shape()[2]); + const int outW = static_cast(dst.shape()[3]); + + int newW = outW; + int newH = outH; + float invScaleX = static_cast(srcW) / outW; + float invScaleY = static_cast(srcH) / outH; + if (spec.keepAspectRatioPad) { + const float scale = std::min(static_cast(outW) / srcW, static_cast(outH) / srcH); + newW = static_cast(std::lround(srcW * scale)); + newH = static_cast(std::lround(srcH * scale)); + newW = std::min(newW, outW); + newH = std::min(newH, outH); + invScaleX = invScaleY = 1.0f / scale; + } + + Norm norm{}; + for (int i = 0; i < 4; ++i) { + norm.mean[i] = spec.mean[static_cast(i)]; + norm.scale[i] = spec.scale[static_cast(i)]; + } + + // Guard the element count against 32-bit overflow: both the grid size here and the kernel's + // internal flat index are int, so an outC*outH*outW exceeding INT_MAX would silently launch + // zero blocks (leaving the output unwritten). Reject it cleanly instead. + const long long total64 = static_cast(outC) * outH * outW; + if (total64 > 2147483647LL) { + return Status{StatusCode::kInvalidArgument, "letterboxToTensor output tensor has too many elements (> INT_MAX)"}; + } + const int total = static_cast(total64); + const int blockSize = 256; + const int gridSize = (total + blockSize - 1) / blockSize; + const auto *srcPtr = static_cast(src.data()); + + if (dst.dtype() == DType::kFloat32) { + letterboxKernel<<>>(srcPtr, srcH, srcW, channels, static_cast(dst.data()), + outC, outH, outW, newW, newH, invScaleX, invScaleY, norm, + spec.swapRB, spec.padValue); + } else { + letterboxKernel<__half><<>>(srcPtr, srcH, srcW, channels, + static_cast<__half *>(dst.data()), outC, outH, outW, newW, + newH, invScaleX, invScaleY, norm, spec.swapRB, spec.padValue); + } + return cudaToStatus(cudaGetLastError(), "letterboxToTensor kernel launch"); +} + +} // namespace trtcpp::preproc diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..b36b9f9 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,61 @@ +# Prefer a system GoogleTest; fall back to fetching it (keeps CI/offline-ish hosts +# working without an extra apt dependency). +find_package(GTest QUIET) +if(NOT GTest_FOUND) + include(FetchContent) + fetchcontent_declare(googletest GIT_REPOSITORY https://github.com/google/googletest.git GIT_TAG v1.15.2) + set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) # don't leak gtest into a consumer's install prefix + fetchcontent_makeavailable(googletest) +endif() + +include(GoogleTest) + +# CPU-only unit tests (no GPU required). +add_executable( + trtcpp_core_tests + test_dtype.cpp + test_shape.cpp + test_status.cpp + test_tensorview.cpp + test_logger.cpp + test_sha256.cpp + test_engine_cache.cpp + test_version.cpp +) +target_include_directories(trtcpp_core_tests PRIVATE ${CMAKE_SOURCE_DIR}/src) # internal detail headers (CPU-only) +target_link_libraries(trtcpp_core_tests PRIVATE tensorrt_cpp_api::tensorrt_cpp_api GTest::gtest_main) +gtest_discover_tests(trtcpp_core_tests) + +# GPU-required integration tests, labeled "gpu" so CI can exclude them (ctest -LE gpu). +add_executable(trtcpp_cuda_tests test_cuda.cpp test_device_tensor.cpp test_buffers.cpp) +target_include_directories(trtcpp_cuda_tests PRIVATE ${CMAKE_SOURCE_DIR}/src) # internal detail headers +target_link_libraries(trtcpp_cuda_tests PRIVATE tensorrt_cpp_api::tensorrt_cpp_api GTest::gtest_main) +gtest_discover_tests(trtcpp_cuda_tests PROPERTIES LABELS "gpu") + +# White-box tests of the internal TensorRT glue (logger bridge, plugin loader). Need the +# TRT runtime + headers and src/ on the include path; labeled "gpu". +if(TARGET TensorRT::TensorRT) + add_executable( + trtcpp_trt_tests test_trt_logger.cpp test_trt_plugin.cpp test_engine_builder.cpp test_engine.cpp test_engine_pool.cpp + test_calibrator.cpp + ) + target_include_directories(trtcpp_trt_tests PRIVATE ${CMAKE_SOURCE_DIR}/src) + target_compile_definitions(trtcpp_trt_tests PRIVATE TRTCPP_TEST_MODEL_DIR="${CMAKE_SOURCE_DIR}/tests/models") + target_link_libraries(trtcpp_trt_tests PRIVATE tensorrt_cpp_api::tensorrt_cpp_api TensorRT::TensorRT GTest::gtest_main) + gtest_discover_tests(trtcpp_trt_tests PROPERTIES LABELS "gpu") +endif() + +# Optional OpenCV interop tests (only when WITH_OPENCV; need a working OpenCV-CUDA + GPU). +if(TRT_CPP_API_WITH_OPENCV) + add_executable(trtcpp_opencv_tests test_opencv_interop.cpp) + target_link_libraries(trtcpp_opencv_tests PRIVATE tensorrt_cpp_api::tensorrt_cpp_api ${OpenCV_LIBS} GTest::gtest_main) + gtest_discover_tests(trtcpp_opencv_tests PROPERTIES LABELS "gpu") +endif() + +# Preprocessing kernel tests (need a GPU; no TRT). +if(TARGET tensorrt_cpp_api::preproc) + add_executable(trtcpp_preproc_tests test_preproc.cpp) + target_link_libraries(trtcpp_preproc_tests PRIVATE tensorrt_cpp_api::preproc GTest::gtest_main) + gtest_discover_tests(trtcpp_preproc_tests PROPERTIES LABELS "gpu") +endif() diff --git a/tests/models/conv_1x3x8x8.onnx b/tests/models/conv_1x3x8x8.onnx new file mode 100644 index 0000000..b5d4afd Binary files /dev/null and b/tests/models/conv_1x3x8x8.onnx differ diff --git a/tests/models/gen_models.py b/tests/models/gen_models.py new file mode 100644 index 0000000..74ed09e --- /dev/null +++ b/tests/models/gen_models.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Generate the tiny ONNX fixtures used by the engine tests. + +These are deliberately trivial (a single Relu) so a build + inference roundtrip is fast +and the expected output is obvious (max(x, 0)). The generated .onnx files are committed +(see the tests/models/.gitignore exception) so CI does not need onnx installed. + +Regenerate with: python3 tests/models/gen_models.py +""" +import os + +import numpy as np +import onnx +from onnx import TensorProto, helper, numpy_helper + +HERE = os.path.dirname(os.path.abspath(__file__)) +OPSET = 17 + + +def relu_model(path, batch): + """input[batch,3,8,8] -> Relu -> output[batch,3,8,8]. batch may be an int or 'N'.""" + shape = [batch, 3, 8, 8] + x = helper.make_tensor_value_info("input", TensorProto.FLOAT, shape) + y = helper.make_tensor_value_info("output", TensorProto.FLOAT, shape) + node = helper.make_node("Relu", ["input"], ["output"]) + graph = helper.make_graph([node], "relu", [x], [y]) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", OPSET)]) + model.ir_version = 9 # broad runtime compatibility + onnx.checker.check_model(model) + onnx.save(model, os.path.join(HERE, path)) + print("wrote", path) + + +def conv_model(path): + """input[1,3,8,8] -> Conv(3->4, 3x3, pad 1) -> output[1,4,8,8]. Has a quantizable + weighted layer, so it is suitable for INT8 calibration (a lone Relu is folded away).""" + x = helper.make_tensor_value_info("input", TensorProto.FLOAT, [1, 3, 8, 8]) + y = helper.make_tensor_value_info("output", TensorProto.FLOAT, [1, 4, 8, 8]) + rng = np.random.RandomState(0) + weight = numpy_helper.from_array(rng.randn(4, 3, 3, 3).astype(np.float32), "W") + bias = numpy_helper.from_array(np.zeros(4, dtype=np.float32), "B") + node = helper.make_node("Conv", ["input", "W", "B"], ["output"], kernel_shape=[3, 3], pads=[1, 1, 1, 1]) + graph = helper.make_graph([node], "conv", [x], [y], initializer=[weight, bias]) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", OPSET)]) + model.ir_version = 9 + onnx.checker.check_model(model) + onnx.save(model, os.path.join(HERE, path)) + print("wrote", path) + + +if __name__ == "__main__": + relu_model("relu_1x3x8x8.onnx", 1) + relu_model("relu_dynamic_batch.onnx", "N") + conv_model("conv_1x3x8x8.onnx") diff --git a/tests/models/relu_1x3x8x8.onnx b/tests/models/relu_1x3x8x8.onnx new file mode 100644 index 0000000..36fe7e6 Binary files /dev/null and b/tests/models/relu_1x3x8x8.onnx differ diff --git a/tests/models/relu_dynamic_batch.onnx b/tests/models/relu_dynamic_batch.onnx new file mode 100644 index 0000000..15739e2 Binary files /dev/null and b/tests/models/relu_dynamic_batch.onnx differ diff --git a/tests/test_buffers.cpp b/tests/test_buffers.cpp new file mode 100644 index 0000000..c84afc6 --- /dev/null +++ b/tests/test_buffers.cpp @@ -0,0 +1,90 @@ +#include "detail/buffers.h" + +#include + +using namespace trtcpp; +using namespace trtcpp::detail; + +TEST(NamedBuffers, AllocatesAndReports) { + NamedBuffers buffers; + auto view = buffers.ensure("out", DType::kFloat32, Shape{1, 100}); + ASSERT_TRUE(view.ok()) << view.status().message(); + EXPECT_NE(view.value().data(), nullptr); + EXPECT_EQ(view.value().dtype(), DType::kFloat32); + EXPECT_EQ(view.value().shape(), (Shape{1, 100})); + EXPECT_TRUE(buffers.has("out")); + EXPECT_EQ(buffers.capacity("out"), 400u); + EXPECT_EQ(buffers.address("out"), view.value().data()); +} + +TEST(NamedBuffers, ReusesWhenNotGrowing) { + NamedBuffers buffers; + auto first = buffers.ensure("out", DType::kFloat32, Shape{1, 100}); + ASSERT_TRUE(first.ok()); + void *ptr = first.value().data(); + std::size_t cap = buffers.capacity("out"); + + // Smaller request reuses the same allocation (no realloc, capacity unchanged). + auto second = buffers.ensure("out", DType::kFloat32, Shape{1, 50}); + ASSERT_TRUE(second.ok()); + EXPECT_EQ(second.value().data(), ptr); + EXPECT_EQ(buffers.capacity("out"), cap); + EXPECT_EQ(second.value().shape(), (Shape{1, 50})); +} + +TEST(NamedBuffers, GrowsWhenLarger) { + NamedBuffers buffers; + ASSERT_TRUE(buffers.ensure("out", DType::kFloat32, Shape{1, 50}).ok()); + EXPECT_EQ(buffers.capacity("out"), 200u); + ASSERT_TRUE(buffers.ensure("out", DType::kFloat32, Shape{1, 200}).ok()); + EXPECT_EQ(buffers.capacity("out"), 800u); // grew +} + +TEST(NamedBuffers, RejectsDynamicShape) { + NamedBuffers buffers; + auto view = buffers.ensure("out", DType::kFloat32, Shape{-1, 100}); + EXPECT_FALSE(view.ok()); + EXPECT_EQ(view.status().code(), StatusCode::kInvalidArgument); +} + +TEST(NamedBuffers, RejectsOverflowingShape) { + NamedBuffers buffers; + auto view = buffers.ensure("out", DType::kFloat32, Shape{1LL << 40, 1LL << 40}); + EXPECT_FALSE(view.ok()); + EXPECT_EQ(view.status().code(), StatusCode::kInvalidArgument); +} + +TEST(NamedBuffers, ZeroSizeShapeAllocatesNothing) { + NamedBuffers buffers; + auto view = buffers.ensure("z", DType::kFloat32, Shape{0}); + ASSERT_TRUE(view.ok()); + EXPECT_EQ(view.value().shape().numel(), 0); + EXPECT_EQ(buffers.capacity("z"), 0u); + EXPECT_EQ(buffers.address("z"), nullptr); +} + +TEST(NamedBuffers, DeviceChangeReallocates) { + auto count = deviceCount(); + if (!count.ok() || count.value() < 2) { + GTEST_SKIP() << "requires >= 2 CUDA devices"; + } + NamedBuffers buffers; + auto onZero = buffers.ensure("x", DType::kFloat32, Shape{4}, 0); + ASSERT_TRUE(onZero.ok()); + void *ptr0 = onZero.value().data(); + auto onOne = buffers.ensure("x", DType::kFloat32, Shape{4}, 1); + ASSERT_TRUE(onOne.ok()); + EXPECT_NE(onOne.value().data(), ptr0); // reallocated on the other device +} + +TEST(NamedBuffers, MultipleNamesAndClear) { + NamedBuffers buffers; + ASSERT_TRUE(buffers.ensure("a", DType::kFloat32, Shape{4}).ok()); + ASSERT_TRUE(buffers.ensure("b", DType::kInt8, Shape{8}).ok()); + EXPECT_EQ(buffers.names().size(), 2u); + EXPECT_NE(buffers.address("a"), buffers.address("b")); + EXPECT_EQ(buffers.address("missing"), nullptr); + buffers.clear(); + EXPECT_FALSE(buffers.has("a")); + EXPECT_EQ(buffers.capacity("a"), 0u); +} diff --git a/tests/test_calibrator.cpp b/tests/test_calibrator.cpp new file mode 100644 index 0000000..1995937 --- /dev/null +++ b/tests/test_calibrator.cpp @@ -0,0 +1,130 @@ +#include "tensorrt_cpp_api/calibrator.h" +#include "tensorrt_cpp_api/cuda.h" +#include "tensorrt_cpp_api/device_tensor.h" +#include "tensorrt_cpp_api/engine_builder.h" + +#include + +#include + +#include +#include +#include +#include +#include +#include + +// The legacy calibrator path only exists on TRT < 11 (calibrator.h compiles out otherwise). +#if TRT_CPP_API_TENSORRT_VERSION_MAJOR < 11 + +using namespace trtcpp; + +namespace { + +std::string modelPath(const char *name) { return std::string(TRTCPP_TEST_MODEL_DIR) + "/" + name; } + +std::vector writeRawBatches(const std::filesystem::path &dir, int count, int floatsPerBatch) { + std::filesystem::create_directories(dir); + std::vector paths; + for (int b = 0; b < count; ++b) { + std::vector data(static_cast(floatsPerBatch)); + for (int i = 0; i < floatsPerBatch; ++i) { + data[static_cast(i)] = static_cast((i + b * 7) % 11) - 5.0f; + } + const std::string path = (dir / ("batch_" + std::to_string(b) + ".bin")).string(); + std::ofstream out(path, std::ios::binary); + out.write(reinterpret_cast(data.data()), static_cast(data.size() * sizeof(float))); + paths.push_back(path); + } + return paths; +} + +// A calibrator that records how many batches TensorRT requested and fills each with varying +// data, so we can assert the build actually drove calibration through our bridge. +class CountingCalibrator final : public ICalibrator { +public: + explicit CountingCalibrator(int maxBatches) : maxBatches_(maxBatches) {} + int batchSize() const override { return 1; } + bool nextBatch(const std::unordered_map &inputs, const Stream &stream) override { + ++calls; + if (calls > maxBatches_ || inputs.empty()) { + return false; + } + const TensorView &view = inputs.begin()->second; + std::vector host(view.nbytes() / sizeof(float)); + for (std::size_t i = 0; i < host.size(); ++i) { + host[i] = static_cast((i + static_cast(calls) * 3) % 13) - 6.0f; + } + cudaMemcpyAsync(view.data(), host.data(), view.nbytes(), cudaMemcpyHostToDevice, stream.handle()); + cudaStreamSynchronize(stream.handle()); // keep host alive until the copy completes + return true; + } + std::optional> readCache() override { return std::nullopt; } + void writeCache(std::span) override { ++cacheWrites; } + + int calls = 0; + int cacheWrites = 0; + +private: + int maxBatches_; +}; + +} // namespace + +TEST(Calibrator, Int8RequiresCalibrator) { + EngineBuilder builder; + BuildOptions options; + options.precision = Precision::kInt8CalibLegacy; + options.engineCacheDir = (std::filesystem::temp_directory_path() / "trtcpp_e10").string(); + auto engine = builder.buildFromOnnxFile(modelPath("conv_1x3x8x8.onnx"), options); + EXPECT_FALSE(engine.ok()); + EXPECT_EQ(engine.status().code(), StatusCode::kInvalidArgument); // missing calibrator +} + +TEST(Calibrator, CalibratorInvokedDuringInt8Build) { + const auto dir = std::filesystem::temp_directory_path() / "trtcpp_e10_build"; + std::filesystem::remove_all(dir); + auto calibrator = std::make_shared(8); + + EngineBuilder builder; + BuildOptions options; + options.precision = Precision::kInt8CalibLegacy; + options.calibrator = calibrator; + options.engineCacheDir = dir.string(); + // The build drives calibration through the bridge. (Whether TRT ultimately emits an INT8 + // engine depends on the model; end-to-end INT8 emission is validated later on a real model.) + builder.buildFromOnnxFile(modelPath("conv_1x3x8x8.onnx"), options); + EXPECT_GT(calibrator->calls, 0); // TensorRT requested batches through our IInt8 bridge + + std::filesystem::remove_all(dir); +} + +TEST(Calibrator, RawBatchCalibratorMechanics) { + const auto dir = std::filesystem::temp_directory_path() / "trtcpp_e10_raw"; + std::filesystem::remove_all(dir); + auto batches = writeRawBatches(dir, 3, 1 * 3 * 8 * 8); + const std::string cachePath = (dir / "calib.cache").string(); + auto calibrator = makeRawBatchCalibrator("input", Shape{1, 3, 8, 8}, batches, cachePath); + + EXPECT_EQ(calibrator->batchSize(), 1); + auto buffer = Tensor::allocate(DType::kFloat32, Shape{1, 3, 8, 8}, Device::kCuda); + ASSERT_TRUE(buffer.ok()); + Stream stream; + std::unordered_map inputs{{"input", buffer.value().view()}}; + + EXPECT_TRUE(calibrator->nextBatch(inputs, stream)); + EXPECT_TRUE(calibrator->nextBatch(inputs, stream)); + EXPECT_TRUE(calibrator->nextBatch(inputs, stream)); + EXPECT_FALSE(calibrator->nextBatch(inputs, stream)); // exhausted after 3 files + + EXPECT_FALSE(calibrator->readCache().has_value()); // nothing written yet + const std::vector table{std::byte{1}, std::byte{2}, std::byte{3}}; + calibrator->writeCache(table); + auto read = calibrator->readCache(); + ASSERT_TRUE(read.has_value()); + EXPECT_EQ(read->size(), 3u); + + std::filesystem::remove_all(dir); +} + +#endif // TRT_CPP_API_TENSORRT_VERSION_MAJOR < 11 diff --git a/tests/test_cuda.cpp b/tests/test_cuda.cpp new file mode 100644 index 0000000..10573e4 --- /dev/null +++ b/tests/test_cuda.cpp @@ -0,0 +1,63 @@ +#include "tensorrt_cpp_api/allocator.h" +#include "tensorrt_cpp_api/cuda.h" + +#include + +#include + +using namespace trtcpp; + +TEST(Cuda, DeviceCountAndQuery) { + auto count = deviceCount(); + ASSERT_TRUE(count.ok()) << count.status().message(); + ASSERT_GE(count.value(), 1); + + auto info = queryDevice(0); + ASSERT_TRUE(info.ok()) << info.status().message(); + EXPECT_FALSE(info.value().name.empty()); + EXPECT_EQ(info.value().uuid.size(), 32u); + EXPECT_GE(info.value().computeMajor, 1); + EXPECT_GT(info.value().totalMemoryBytes, 0u); +} + +TEST(Cuda, DefaultStreamSynchronizes) { + Stream stream; + EXPECT_TRUE(stream.synchronize().ok()); +} + +TEST(Cuda, CreateStreamOwns) { + auto stream = Stream::create(); + ASSERT_TRUE(stream.ok()) << stream.status().message(); + EXPECT_TRUE(stream.value().owns()); + EXPECT_NE(stream.value().handle(), nullptr); + EXPECT_TRUE(stream.value().synchronize().ok()); +} + +TEST(Cuda, WrapStreamIsNonOwning) { + auto owned = Stream::create(); + ASSERT_TRUE(owned.ok()); + Stream wrapped = Stream::wrap(owned.value().handle()); + EXPECT_FALSE(wrapped.owns()); + EXPECT_EQ(wrapped.handle(), owned.value().handle()); + // wrapped's destructor must not destroy the stream still owned by `owned`. +} + +TEST(Cuda, StreamMoveTransfersOwnership) { + auto a = Stream::create(); + ASSERT_TRUE(a.ok()); + cudaStream_t handle = a.value().handle(); + Stream b = std::move(a.value()); + EXPECT_TRUE(b.owns()); + EXPECT_EQ(b.handle(), handle); + EXPECT_FALSE(a.value().owns()); // moved-from no longer owns +} + +TEST(Cuda, StreamOrderedAllocatorRoundtrip) { + auto allocator = defaultDeviceAllocator(0); + ASSERT_NE(allocator, nullptr); + Stream stream; + void *ptr = allocator->allocate(1024, 256, stream); + EXPECT_NE(ptr, nullptr); + allocator->deallocate(ptr, stream); + EXPECT_TRUE(stream.synchronize().ok()); +} diff --git a/tests/test_device_tensor.cpp b/tests/test_device_tensor.cpp new file mode 100644 index 0000000..6eb7502 --- /dev/null +++ b/tests/test_device_tensor.cpp @@ -0,0 +1,125 @@ +#include "tensorrt_cpp_api/device_tensor.h" + +#include + +#include +#include +#include + +using namespace trtcpp; + +TEST(Tensor, AllocateDeviceAndHost) { + auto device = Tensor::allocate(DType::kFloat32, Shape{1, 3, 4, 4}, Device::kCuda); + ASSERT_TRUE(device.ok()) << device.status().message(); + EXPECT_FALSE(device.value().empty()); + EXPECT_EQ(device.value().nbytes(), 48u * 4); + EXPECT_EQ(device.value().device(), Device::kCuda); + + auto host = Tensor::allocate(DType::kFloat32, Shape{4}, Device::kHost); + ASSERT_TRUE(host.ok()) << host.status().message(); + EXPECT_EQ(host.value().device(), Device::kHost); +} + +TEST(Tensor, RejectsDynamicShape) { + auto t = Tensor::allocate(DType::kFloat32, Shape{-1, 3}, Device::kCuda); + EXPECT_FALSE(t.ok()); + EXPECT_EQ(t.status().code(), StatusCode::kInvalidArgument); +} + +TEST(Tensor, HostToDeviceToHostRoundtrip) { + std::array src{1, 2, 3, 4, 5, 6}; + TensorView srcView{src.data(), DType::kFloat32, Shape{2, 3}, Device::kHost}; + Stream stream; + + auto device = Tensor::allocate(DType::kFloat32, Shape{2, 3}, Device::kCuda); + ASSERT_TRUE(device.ok()); + ASSERT_TRUE(device.value().copyFrom(srcView, stream).ok()); + + auto host = device.value().toHost(stream); + ASSERT_TRUE(host.ok()); + auto span = host.value().as(); + ASSERT_TRUE(span.ok()); + ASSERT_EQ(span.value().size(), 6u); + for (std::size_t i = 0; i < 6; ++i) { + EXPECT_FLOAT_EQ(span.value()[i], src[i]); + } +} + +TEST(Tensor, MoveTransfersBuffer) { + auto a = Tensor::allocate(DType::kFloat32, Shape{8}, Device::kCuda); + ASSERT_TRUE(a.ok()); + void *ptr = a.value().data(); + Tensor b = std::move(a.value()); + EXPECT_EQ(b.data(), ptr); + EXPECT_TRUE(a.value().empty()); // moved-from is empty +} + +TEST(Tensor, CopyFromRejectsSizeMismatch) { + std::array src{}; + TensorView srcView{src.data(), DType::kFloat32, Shape{4}, Device::kHost}; + Stream stream; + auto device = Tensor::allocate(DType::kFloat32, Shape{2}, Device::kCuda); + ASSERT_TRUE(device.ok()); + Status status = device.value().copyFrom(srcView, stream); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.code(), StatusCode::kShapeMismatch); +} + +TEST(Tensor, CopyFromRejectsDtypeMismatch) { + std::array src{}; + TensorView srcView{src.data(), DType::kInt32, Shape{4}, Device::kHost}; + Stream stream; + auto device = Tensor::allocate(DType::kFloat32, Shape{4}, Device::kCuda); + ASSERT_TRUE(device.ok()); + Status status = device.value().copyFrom(srcView, stream); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.code(), StatusCode::kDtypeMismatch); +} + +TEST(Tensor, CopyFromRejectsCrossDevice) { + Stream stream; + auto device = Tensor::allocate(DType::kFloat32, Shape{3}, Device::kCuda); // deviceId 0 + ASSERT_TRUE(device.ok()); + // A (synthetic) view claiming to live on a different CUDA device; rejected before any copy. + TensorView otherDevice{device.value().data(), DType::kFloat32, Shape{3}, Device::kCuda, 1}; + Status status = device.value().copyFrom(otherDevice, stream); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.code(), StatusCode::kUnsupported); +} + +TEST(Tensor, MoveAssignmentTransfersBuffer) { + auto a = Tensor::allocate(DType::kFloat32, Shape{8}, Device::kCuda); + auto b = Tensor::allocate(DType::kFloat32, Shape{2}, Device::kCuda); + ASSERT_TRUE(a.ok()); + ASSERT_TRUE(b.ok()); + void *ptr = a.value().data(); + b.value() = std::move(a.value()); // must free b's old buffer and take a's + EXPECT_EQ(b.value().data(), ptr); + EXPECT_TRUE(a.value().empty()); + EXPECT_EQ(b.value().shape(), (Shape{8})); +} + +TEST(Tensor, MoveFromEmptyIsSafe) { + Tensor empty; // default-constructed, no buffer + EXPECT_TRUE(empty.empty()); + Tensor moved = std::move(empty); + EXPECT_TRUE(moved.empty()); + // both destruct here without freeing a null buffer +} + +TEST(Tensor, ToDeviceToDevice) { + std::array src{7, 8, 9}; + TensorView srcView{src.data(), DType::kFloat32, Shape{3}, Device::kHost}; + Stream stream; + auto device = Tensor::allocate(DType::kFloat32, Shape{3}, Device::kCuda); + ASSERT_TRUE(device.ok()); + ASSERT_TRUE(device.value().copyFrom(srcView, stream).ok()); + + auto device2 = device.value().to(Device::kCuda, 0, stream); // device-to-device + ASSERT_TRUE(device2.ok()); + auto host = device2.value().toHost(stream); + ASSERT_TRUE(host.ok()); + auto span = host.value().as(); + ASSERT_TRUE(span.ok()); + EXPECT_FLOAT_EQ(span.value()[2], 9.f); +} diff --git a/tests/test_dtype.cpp b/tests/test_dtype.cpp new file mode 100644 index 0000000..c0910a5 --- /dev/null +++ b/tests/test_dtype.cpp @@ -0,0 +1,47 @@ +#include "tensorrt_cpp_api/dtype.h" + +#include + +using namespace trtcpp; + +TEST(DType, BitsPerElement) { + EXPECT_EQ(bitsPerElement(DType::kFloat32), 32); + EXPECT_EQ(bitsPerElement(DType::kInt64), 64); + EXPECT_EQ(bitsPerElement(DType::kFloat16), 16); + EXPECT_EQ(bitsPerElement(DType::kBFloat16), 16); + EXPECT_EQ(bitsPerElement(DType::kInt8), 8); + EXPECT_EQ(bitsPerElement(DType::kUInt8), 8); + EXPECT_EQ(bitsPerElement(DType::kBool), 8); + EXPECT_EQ(bitsPerElement(DType::kFp8), 8); + EXPECT_EQ(bitsPerElement(DType::kInt4), 4); +} + +TEST(DType, ByteSizeRoundsUp) { + EXPECT_EQ(byteSize(DType::kFloat32), 4u); + EXPECT_EQ(byteSize(DType::kFloat16), 2u); + EXPECT_EQ(byteSize(DType::kInt8), 1u); + EXPECT_EQ(byteSize(DType::kInt4), 1u); // a single sub-byte element rounds up to 1 +} + +TEST(DType, ByteSizeOfPacksSubByte) { + EXPECT_EQ(byteSizeOf(DType::kFloat32, 10), 40u); + EXPECT_EQ(byteSizeOf(DType::kInt8, 10), 10u); + EXPECT_EQ(byteSizeOf(DType::kInt4, 10), 5u); // two int4 per byte + EXPECT_EQ(byteSizeOf(DType::kInt4, 9), 5u); // 9 elements -> ceil(36/8) = 5 bytes + EXPECT_EQ(byteSizeOf(DType::kInt4, 0), 0u); +} + +TEST(DType, ToString) { + EXPECT_EQ(toString(DType::kFloat32), "float32"); + EXPECT_EQ(toString(DType::kInt4), "int4"); + EXPECT_EQ(toString(DType::kBool), "bool"); +} + +TEST(DType, DTypeOfMapping) { + EXPECT_EQ(DTypeOf::value, DType::kFloat32); + EXPECT_EQ(DTypeOf::value, DType::kInt32); + EXPECT_EQ(DTypeOf::value, DType::kInt64); + EXPECT_EQ(DTypeOf::value, DType::kInt8); + EXPECT_EQ(DTypeOf::value, DType::kUInt8); + EXPECT_EQ(DTypeOf::value, DType::kBool); +} diff --git a/tests/test_engine.cpp b/tests/test_engine.cpp new file mode 100644 index 0000000..1b71811 --- /dev/null +++ b/tests/test_engine.cpp @@ -0,0 +1,174 @@ +#include "tensorrt_cpp_api/cuda.h" +#include "tensorrt_cpp_api/device_tensor.h" +#include "tensorrt_cpp_api/engine.h" +#include "tensorrt_cpp_api/engine_builder.h" + +#include + +#include +#include +#include +#include + +using namespace trtcpp; + +namespace { + +std::string modelPath(const char *name) { return std::string(TRTCPP_TEST_MODEL_DIR) + "/" + name; } + +std::string cacheDir() { return (std::filesystem::temp_directory_path() / "trtcpp_e8_cache").string(); } + +Result buildAndLoad(const char *model, BuildOptions options) { + options.engineCacheDir = cacheDir(); + EngineBuilder builder; + auto path = builder.buildOrLoad(modelPath(model), options); + if (!path) { + return path.status(); + } + return Engine::loadFromFile(path.value()); +} + +} // namespace + +TEST(Engine, IntrospectsIO) { + BuildOptions options; + options.precision = Precision::kFp32; + auto engine = buildAndLoad("relu_1x3x8x8.onnx", options); + ASSERT_TRUE(engine.ok()) << engine.status().message(); + EXPECT_EQ(engine.value().inputNames(), (std::vector{"input"})); + EXPECT_EQ(engine.value().outputNames(), (std::vector{"output"})); + auto shape = engine.value().tensorShape("input"); + ASSERT_TRUE(shape.ok()); + EXPECT_EQ(shape.value(), (Shape{1, 3, 8, 8})); + auto dtype = engine.value().tensorDType("output"); + ASSERT_TRUE(dtype.ok()); + EXPECT_EQ(dtype.value(), DType::kFloat32); + EXPECT_FALSE(engine.value().tensorShape("nope").ok()); +} + +TEST(Engine, ReluInferenceRoundtrip) { + BuildOptions options; + options.precision = Precision::kFp32; + auto engine = buildAndLoad("relu_1x3x8x8.onnx", options); + ASSERT_TRUE(engine.ok()) << engine.status().message(); + + Stream stream; + std::vector hostInput(1 * 3 * 8 * 8); + for (std::size_t i = 0; i < hostInput.size(); ++i) { + hostInput[i] = (i % 2 == 0) ? -static_cast(i) : static_cast(i); + } + auto deviceInput = Tensor::allocate(DType::kFloat32, Shape{1, 3, 8, 8}, Device::kCuda); + ASSERT_TRUE(deviceInput.ok()); + TensorView hostView{hostInput.data(), DType::kFloat32, Shape{1, 3, 8, 8}, Device::kHost}; + ASSERT_TRUE(deviceInput.value().copyFrom(hostView, stream).ok()); + + auto output = engine.value().inferSingle({{"input", deviceInput.value().view()}}, stream); + ASSERT_TRUE(output.ok()) << output.status().message(); + + auto host = output.value().toHost(stream); + ASSERT_TRUE(host.ok()); + auto values = host.value().as(); + ASSERT_TRUE(values.ok()); + ASSERT_EQ(values.value().size(), hostInput.size()); + for (std::size_t i = 0; i < hostInput.size(); ++i) { + EXPECT_FLOAT_EQ(values.value()[i], std::max(hostInput[i], 0.0f)); // Relu + } +} + +TEST(Engine, EnqueueCallerAllocated) { + BuildOptions options; + options.precision = Precision::kFp32; + auto engine = buildAndLoad("relu_1x3x8x8.onnx", options); + ASSERT_TRUE(engine.ok()); + + Stream stream; + std::vector hostInput(1 * 3 * 8 * 8, -2.0f); // all negative -> Relu 0 + auto deviceInput = Tensor::allocate(DType::kFloat32, Shape{1, 3, 8, 8}, Device::kCuda); + ASSERT_TRUE(deviceInput.ok()); + TensorView hostView{hostInput.data(), DType::kFloat32, Shape{1, 3, 8, 8}, Device::kHost}; + ASSERT_TRUE(deviceInput.value().copyFrom(hostView, stream).ok()); + + auto shapes = engine.value().outputShapes({{"input", deviceInput.value().view()}}); + ASSERT_TRUE(shapes.ok()); + ASSERT_EQ(shapes.value().at("output"), (Shape{1, 3, 8, 8})); + + auto deviceOutput = Tensor::allocate(DType::kFloat32, shapes.value().at("output"), Device::kCuda); + ASSERT_TRUE(deviceOutput.ok()); + ASSERT_TRUE(engine.value().enqueue({{"input", deviceInput.value().view()}}, {{"output", deviceOutput.value().view()}}, stream).ok()); + + auto host = deviceOutput.value().toHost(stream); + ASSERT_TRUE(host.ok()); + auto values = host.value().as(); + ASSERT_TRUE(values.ok()); + for (float v : values.value()) { + EXPECT_FLOAT_EQ(v, 0.0f); + } +} + +TEST(Engine, DynamicBatchInference) { + BuildOptions options; + options.precision = Precision::kFp32; + OptimizationProfile profile; + profile.inputs.push_back({"input", Shape{1, 3, 8, 8}, Shape{2, 3, 8, 8}, Shape{4, 3, 8, 8}}); + options.profiles.push_back(profile); + auto engine = buildAndLoad("relu_dynamic_batch.onnx", options); + ASSERT_TRUE(engine.ok()) << engine.status().message(); + + Stream stream; + const int batch = 2; // the #1 community bug: batch > 1 must work + std::vector hostInput(static_cast(batch) * 3 * 8 * 8, -1.0f); + auto deviceInput = Tensor::allocate(DType::kFloat32, Shape{batch, 3, 8, 8}, Device::kCuda); + ASSERT_TRUE(deviceInput.ok()); + TensorView hostView{hostInput.data(), DType::kFloat32, Shape{batch, 3, 8, 8}, Device::kHost}; + ASSERT_TRUE(deviceInput.value().copyFrom(hostView, stream).ok()); + + auto output = engine.value().inferSingle({{"input", deviceInput.value().view()}}, stream); + ASSERT_TRUE(output.ok()) << output.status().message(); + EXPECT_EQ(output.value().shape(), (Shape{batch, 3, 8, 8})); // dynamic batch resolved + + auto host = output.value().toHost(stream); + ASSERT_TRUE(host.ok()); + auto values = host.value().as(); + ASSERT_TRUE(values.ok()); + ASSERT_EQ(values.value().size(), hostInput.size()); + for (float v : values.value()) { + EXPECT_FLOAT_EQ(v, 0.0f); // Relu(-1) == 0 + } +} + +TEST(Engine, RejectsOutOfRangeProfile) { + BuildOptions options; + options.precision = Precision::kFp32; + auto engine = buildAndLoad("relu_1x3x8x8.onnx", options); + ASSERT_TRUE(engine.ok()); + Stream stream; + auto input = Tensor::allocate(DType::kFloat32, Shape{1, 3, 8, 8}, Device::kCuda); + ASSERT_TRUE(input.ok()); + auto output = engine.value().infer({{"input", input.value().view()}}, stream, /*profileIndex=*/5); + EXPECT_FALSE(output.ok()); + EXPECT_EQ(output.status().code(), StatusCode::kInvalidArgument); +} + +TEST(EngineBuilder, BuildAndLoadOneShot) { + EngineBuilder builder; + BuildOptions options; + options.precision = Precision::kFp32; + options.engineCacheDir = cacheDir(); + auto engine = builder.buildAndLoad(modelPath("relu_1x3x8x8.onnx"), options); + ASSERT_TRUE(engine.ok()) << engine.status().message(); + EXPECT_EQ(engine.value().inputNames().size(), 1u); + EXPECT_EQ(engine.value().outputNames().size(), 1u); +} + +TEST(Engine, EnqueueRejectsMissingOutput) { + BuildOptions options; + options.precision = Precision::kFp32; + auto engine = buildAndLoad("relu_1x3x8x8.onnx", options); + ASSERT_TRUE(engine.ok()); + Stream stream; + auto deviceInput = Tensor::allocate(DType::kFloat32, Shape{1, 3, 8, 8}, Device::kCuda); + ASSERT_TRUE(deviceInput.ok()); + Status status = engine.value().enqueue({{"input", deviceInput.value().view()}}, {}, stream); // no output buffer + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.code(), StatusCode::kInvalidArgument); +} diff --git a/tests/test_engine_builder.cpp b/tests/test_engine_builder.cpp new file mode 100644 index 0000000..58c35aa --- /dev/null +++ b/tests/test_engine_builder.cpp @@ -0,0 +1,118 @@ +#include "detail/engine_cache.h" +#include "tensorrt_cpp_api/cuda.h" +#include "tensorrt_cpp_api/engine_builder.h" + +#include + +#include +#include +#include +#include + +using namespace trtcpp; + +namespace { +std::string modelPath(const char *name) { return std::string(TRTCPP_TEST_MODEL_DIR) + "/" + name; } +} // namespace + +TEST(EngineBuilder, BuildsFixedShapeFp32) { + EngineBuilder builder; + BuildOptions options; + options.precision = Precision::kFp32; + auto engine = builder.buildFromOnnxFile(modelPath("relu_1x3x8x8.onnx"), options); + ASSERT_TRUE(engine.ok()) << engine.status().message(); + EXPECT_GT(engine.value().size(), 0u); +} + +TEST(EngineBuilder, BuildsFixedShapeFp16) { + EngineBuilder builder; + BuildOptions options; + options.precision = Precision::kFp16; // weak-typed kFP16 path on TRT 10 + auto engine = builder.buildFromOnnxFile(modelPath("relu_1x3x8x8.onnx"), options); + ASSERT_TRUE(engine.ok()) << engine.status().message(); + EXPECT_GT(engine.value().size(), 0u); +} + +TEST(EngineBuilder, BuildsDynamicBatchWithProfile) { + EngineBuilder builder; + BuildOptions options; + options.precision = Precision::kFp32; + OptimizationProfile profile; + profile.inputs.push_back({"input", Shape{1, 3, 8, 8}, Shape{2, 3, 8, 8}, Shape{4, 3, 8, 8}}); + options.profiles.push_back(profile); + auto engine = builder.buildFromOnnxFile(modelPath("relu_dynamic_batch.onnx"), options); + ASSERT_TRUE(engine.ok()) << engine.status().message(); + EXPECT_GT(engine.value().size(), 0u); +} + +TEST(EngineBuilder, MissingFileIsNotFound) { + EngineBuilder builder; + auto engine = builder.buildFromOnnxFile(modelPath("does_not_exist.onnx"), BuildOptions{}); + EXPECT_FALSE(engine.ok()); + EXPECT_EQ(engine.status().code(), StatusCode::kNotFound); +} + +TEST(EngineBuilder, DirectoryPathDoesNotCrash) { + EngineBuilder builder; + auto engine = builder.buildFromOnnxFile(TRTCPP_TEST_MODEL_DIR, BuildOptions{}); // a directory, not a file + EXPECT_FALSE(engine.ok()); + EXPECT_EQ(engine.status().code(), StatusCode::kNotFound); +} + +TEST(EngineBuilder, GarbageOnnxFailsToParse) { + EngineBuilder builder; + const char junk[] = "not a real onnx model"; + auto engine = builder.buildFromOnnxBytes(std::as_bytes(std::span(junk, sizeof(junk))), BuildOptions{}); + EXPECT_FALSE(engine.ok()); + EXPECT_EQ(engine.status().code(), StatusCode::kTensorRtError); +} + +TEST(EngineBuilder, Fp8RejectedOnAmpere) { + auto info = queryDevice(0); + if (info.ok() && (info.value().computeMajor * 10 + info.value().computeMinor) >= 89) { + GTEST_SKIP() << "device supports FP8"; + } + EngineBuilder builder; + BuildOptions options; + options.precision = Precision::kFp8; + auto engine = builder.buildFromOnnxFile(modelPath("relu_1x3x8x8.onnx"), options); + EXPECT_FALSE(engine.ok()); + EXPECT_EQ(engine.status().code(), StatusCode::kUnsupported); +} + +TEST(EngineBuilder, BuildOrLoadCacheHitMissStale) { + const auto dir = std::filesystem::temp_directory_path() / "trtcpp_engine_cache"; + std::filesystem::remove_all(dir); + + EngineBuilder builder; + BuildOptions options; + options.precision = Precision::kFp32; + options.engineCacheDir = dir.string(); + + // Miss -> build, engine + sidecar created. + auto first = builder.buildOrLoad(modelPath("relu_1x3x8x8.onnx"), options); + ASSERT_TRUE(first.ok()) << first.status().message(); + ASSERT_TRUE(std::filesystem::exists(first.value())); + ASSERT_TRUE(std::filesystem::exists(first.value() + ".json")); + const auto builtTime = std::filesystem::last_write_time(first.value()); + + // Hit -> same path, engine file NOT rewritten. + auto second = builder.buildOrLoad(modelPath("relu_1x3x8x8.onnx"), options); + ASSERT_TRUE(second.ok()); + EXPECT_EQ(second.value(), first.value()); + EXPECT_EQ(std::filesystem::last_write_time(second.value()), builtTime); + + // Stale -> corrupt the sidecar's ONNX hash; next call detects it and rebuilds. + { + std::ofstream sidecar(first.value() + ".json", std::ios::trunc); + sidecar << R"({ "onnx_sha256": "deadbeef" })"; + } + auto third = builder.buildOrLoad(modelPath("relu_1x3x8x8.onnx"), options); + ASSERT_TRUE(third.ok()); + EXPECT_EQ(third.value(), first.value()); + auto meta = trtcpp::detail::readSidecar(third.value() + ".json"); + ASSERT_TRUE(meta.ok()); + EXPECT_NE(meta.value().onnxSha256, "deadbeef"); // sidecar was rewritten with the real hash + + std::filesystem::remove_all(dir); +} diff --git a/tests/test_engine_cache.cpp b/tests/test_engine_cache.cpp new file mode 100644 index 0000000..5eb0c5c --- /dev/null +++ b/tests/test_engine_cache.cpp @@ -0,0 +1,100 @@ +#include "detail/engine_cache.h" + +#include + +#include +#include +#include +#include +#include + +using namespace trtcpp; +using namespace trtcpp::detail; + +TEST(EngineCache, FileNameFormat) { + EXPECT_EQ(cacheFileName("yolov8n", "deadbeefcafef00d0011", "10.0.0", "0123456789abcdef", "fp16"), + "yolov8n.deadbeef.trt10.0.0.01234567.fp16.engine"); +} + +TEST(EngineCache, SidecarRoundtrip) { + CacheMeta meta; + meta.onnxSha256 = "aabbcc"; + meta.trtVersion = "10.0.0"; + meta.cudaVersion = "12060"; + meta.gpuName = "NVIDIA GeForce RTX 3080 Laptop GPU"; + meta.gpuUuid = "0123456789abcdef0123456789abcdef"; + meta.precision = "fp16"; + meta.buildOptionsDigest = "digest123"; + meta.versionCompatible = true; + meta.hardwareCompatible = false; + meta.createdUnix = 1234567890; + + const auto dir = std::filesystem::temp_directory_path() / "trtcpp_cache_roundtrip"; + std::filesystem::create_directories(dir); + const std::string path = (dir / "engine.json").string(); + ASSERT_TRUE(writeSidecar(path, meta).ok()); + + auto read = readSidecar(path); + ASSERT_TRUE(read.ok()) << read.status().message(); + EXPECT_EQ(read.value().onnxSha256, "aabbcc"); + EXPECT_EQ(read.value().trtVersion, "10.0.0"); + EXPECT_EQ(read.value().gpuName, meta.gpuName); + EXPECT_EQ(read.value().buildOptionsDigest, "digest123"); + EXPECT_TRUE(read.value().versionCompatible); + EXPECT_FALSE(read.value().hardwareCompatible); + std::filesystem::remove_all(dir); +} + +TEST(EngineCache, FreshnessChecks) { + CacheMeta expected; + expected.onnxSha256 = "hash1"; + expected.trtVersion = "10.0.0"; + expected.gpuUuid = "gpuA"; + expected.buildOptionsDigest = "d1"; + + EXPECT_TRUE(isFresh(expected, expected, false, false)); + + auto changed = expected; + changed.onnxSha256 = "hash2"; + EXPECT_FALSE(isFresh(changed, expected, false, false)); + + changed = expected; + changed.buildOptionsDigest = "d2"; + EXPECT_FALSE(isFresh(changed, expected, false, false)); + + changed = expected; + changed.trtVersion = "10.2.0"; + EXPECT_FALSE(isFresh(changed, expected, false, false)); // exact TRT mismatch + + // version-compatible: cached 10.0.0, runtime 10.2.0 (same major, runtime newer) -> fresh + { + CacheMeta cached = expected; + cached.trtVersion = "10.0.0"; + CacheMeta runtime = expected; + runtime.trtVersion = "10.2.0"; + EXPECT_TRUE(isFresh(cached, runtime, true, false)); + // different major -> stale even when version-compatible + cached.trtVersion = "11.0.0"; + EXPECT_FALSE(isFresh(cached, runtime, true, false)); + } + + changed = expected; + changed.gpuUuid = "gpuB"; + EXPECT_FALSE(isFresh(changed, expected, false, false)); // GPU mismatch + EXPECT_TRUE(isFresh(changed, expected, false, true)); // hardware-compatible ignores UUID +} + +TEST(EngineCache, WriteAtomicCreatesParentDirs) { + const auto dir = std::filesystem::temp_directory_path() / "trtcpp_cache_atomic"; + std::filesystem::remove_all(dir); + const std::string path = (dir / "nested" / "engine.bin").string(); + const std::string data = "hello-engine"; + auto bytes = std::span(reinterpret_cast(data.data()), data.size()); + ASSERT_TRUE(writeAtomic(path, bytes).ok()); + ASSERT_TRUE(std::filesystem::exists(path)); + + std::ifstream in(path, std::ios::binary); + const std::string back((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + EXPECT_EQ(back, data); + std::filesystem::remove_all(dir); +} diff --git a/tests/test_engine_pool.cpp b/tests/test_engine_pool.cpp new file mode 100644 index 0000000..16c1e52 --- /dev/null +++ b/tests/test_engine_pool.cpp @@ -0,0 +1,127 @@ +#include "tensorrt_cpp_api/cuda.h" +#include "tensorrt_cpp_api/device_tensor.h" +#include "tensorrt_cpp_api/engine_builder.h" +#include "tensorrt_cpp_api/engine_pool.h" + +#include + +#include +#include +#include +#include +#include + +using namespace trtcpp; + +namespace { + +std::string modelPath(const char *name) { return std::string(TRTCPP_TEST_MODEL_DIR) + "/" + name; } +std::string cacheDir() { return (std::filesystem::temp_directory_path() / "trtcpp_e9_cache").string(); } + +// Build the dynamic-batch engine with `profiles` identical profiles and return its path. +Result buildDynamic(int profiles) { + BuildOptions options; + options.precision = Precision::kFp32; + options.engineCacheDir = cacheDir(); + for (int i = 0; i < profiles; ++i) { + OptimizationProfile profile; + profile.inputs.push_back({"input", Shape{1, 3, 8, 8}, Shape{2, 3, 8, 8}, Shape{4, 3, 8, 8}}); + options.profiles.push_back(profile); + } + return EngineBuilder{}.buildOrLoad(modelPath("relu_dynamic_batch.onnx"), options); +} + +Result buildStatic() { + BuildOptions options; + options.precision = Precision::kFp32; + options.engineCacheDir = cacheDir(); + return EngineBuilder{}.buildOrLoad(modelPath("relu_1x3x8x8.onnx"), options); +} + +} // namespace + +TEST(EnginePool, Metadata) { + auto path = buildStatic(); + ASSERT_TRUE(path.ok()) << path.status().message(); + auto pool = EnginePool::create(path.value(), 2); + ASSERT_TRUE(pool.ok()) << pool.status().message(); + EXPECT_EQ(pool.value().size(), 2); + EXPECT_EQ(pool.value().inputNames(), (std::vector{"input"})); + EXPECT_EQ(pool.value().outputNames(), (std::vector{"output"})); +} + +TEST(EnginePool, RejectsMoreContextsThanProfiles) { + auto path = buildDynamic(2); + ASSERT_TRUE(path.ok()); + auto pool = EnginePool::create(path.value(), 4); // engine has only 2 profiles + EXPECT_FALSE(pool.ok()); + EXPECT_EQ(pool.status().code(), StatusCode::kInvalidArgument); +} + +TEST(EnginePool, TryAcquireExhaustionAndRelease) { + auto path = buildStatic(); + ASSERT_TRUE(path.ok()); + auto pool = EnginePool::create(path.value(), 2); + ASSERT_TRUE(pool.ok()); + + auto a = pool.value().tryAcquire(); + auto b = pool.value().tryAcquire(); + ASSERT_TRUE(a.has_value()); + ASSERT_TRUE(b.has_value()); + EXPECT_FALSE(pool.value().tryAcquire().has_value()); // both contexts in use + + a.reset(); // return one lease + auto c = pool.value().tryAcquire(); + EXPECT_TRUE(c.has_value()); +} + +TEST(EnginePool, ConcurrentInferenceFourContexts) { + auto path = buildDynamic(4); + ASSERT_TRUE(path.ok()) << path.status().message(); + auto pool = EnginePool::create(path.value(), 4); + ASSERT_TRUE(pool.ok()) << pool.status().message(); + EXPECT_EQ(pool.value().size(), 4); + + std::atomic successes{0}; + std::vector threads; + for (int t = 0; t < 8; ++t) { + threads.emplace_back([&pool, &successes] { + auto lease = pool.value().acquire(); // blocks until a context is free + if (!lease.ok()) { + return; + } + Stream stream; + std::vector hostInput(2 * 3 * 8 * 8, -1.0f); // batch 2, all negative + auto deviceInput = Tensor::allocate(DType::kFloat32, Shape{2, 3, 8, 8}, Device::kCuda); + if (!deviceInput.ok()) { + return; + } + TensorView hostView{hostInput.data(), DType::kFloat32, Shape{2, 3, 8, 8}, Device::kHost}; + if (!deviceInput.value().copyFrom(hostView, stream).ok()) { + return; + } + auto output = lease.value().inferSingle({{"input", deviceInput.value().view()}}, stream); + if (!output.ok()) { + return; + } + auto host = output.value().toHost(stream); + if (!host.ok()) { + return; + } + auto values = host.value().as(); + if (!values.ok() || values.value().size() != hostInput.size()) { + return; + } + for (float v : values.value()) { + if (v != 0.0f) { + return; + } + } + successes.fetch_add(1); + }); + } + for (std::thread &thread : threads) { + thread.join(); + } + EXPECT_EQ(successes.load(), 8); // all 8 jobs ran correctly across 4 concurrent contexts +} diff --git a/tests/test_logger.cpp b/tests/test_logger.cpp new file mode 100644 index 0000000..0234b25 --- /dev/null +++ b/tests/test_logger.cpp @@ -0,0 +1,93 @@ +#include "tensorrt_cpp_api/logger.h" + +#include + +#include +#include +#include +#include +#include + +using namespace trtcpp; + +namespace { + +class RecordingLogger final : public ILogger { +public: + struct Record { + Severity severity; + std::string message; + }; + void log(Severity severity, std::string_view message) noexcept override { records.push_back({severity, std::string(message)}); } + std::vector records; +}; + +// Capture everything written to the C stderr stream during fn() (Linux-only test util). +std::string captureStderr(const std::function &fn) { + std::fflush(stderr); + int saved = dup(fileno(stderr)); + FILE *tmp = std::tmpfile(); + dup2(fileno(tmp), fileno(stderr)); + fn(); + std::fflush(stderr); + dup2(saved, fileno(stderr)); + close(saved); + std::rewind(tmp); + std::string out; + char buf[512]; + size_t n = 0; + while ((n = std::fread(buf, 1, sizeof(buf), tmp)) > 0) { + out.append(buf, n); + } + std::fclose(tmp); + return out; +} + +} // namespace + +TEST(Logger, SeverityToString) { + EXPECT_EQ(toString(Severity::kVerbose), "VERBOSE"); + EXPECT_EQ(toString(Severity::kInfo), "INFO"); + EXPECT_EQ(toString(Severity::kInternalError), "INTERNAL_ERROR"); +} + +TEST(Logger, InjectableAndPolymorphic) { + RecordingLogger recorder; + ILogger &logger = recorder; // used through the interface, as components will + logger.log(Severity::kWarning, "hello"); + logger.log(Severity::kError, "world"); + ASSERT_EQ(recorder.records.size(), 2u); + EXPECT_EQ(recorder.records[0].severity, Severity::kWarning); + EXPECT_EQ(recorder.records[0].message, "hello"); + EXPECT_EQ(recorder.records[1].severity, Severity::kError); + EXPECT_EQ(recorder.records[1].message, "world"); +} + +TEST(Logger, DefaultLoggerIsNonNullSingleton) { + auto logger = defaultLogger(); + ASSERT_NE(logger, nullptr); + EXPECT_EQ(logger, defaultLogger()); +} + +TEST(Logger, StderrThresholdFilters) { + auto logger = makeStderrLogger(Severity::kWarning); + std::string out = captureStderr([&] { + logger->log(Severity::kInfo, "dropped-info"); + logger->log(Severity::kWarning, "kept-warning"); + logger->log(Severity::kError, "kept-error"); + }); + EXPECT_EQ(out.find("dropped-info"), std::string::npos); + EXPECT_NE(out.find("kept-warning"), std::string::npos); + EXPECT_NE(out.find("kept-error"), std::string::npos); + EXPECT_NE(out.find("[WARNING]"), std::string::npos); +} + +TEST(Logger, StderrTopThresholdKeepsOnlyInternalError) { + auto logger = makeStderrLogger(Severity::kInternalError); + std::string out = captureStderr([&] { + logger->log(Severity::kError, "dropped-error"); + logger->log(Severity::kInternalError, "kept-internal"); + }); + EXPECT_EQ(out.find("dropped-error"), std::string::npos); + EXPECT_NE(out.find("kept-internal"), std::string::npos); +} diff --git a/tests/test_opencv_interop.cpp b/tests/test_opencv_interop.cpp new file mode 100644 index 0000000..376c111 --- /dev/null +++ b/tests/test_opencv_interop.cpp @@ -0,0 +1,66 @@ +#include "tensorrt_cpp_api/opencv_interop.h" + +#ifdef TRT_CPP_API_WITH_OPENCV + +#include "tensorrt_cpp_api/cuda.h" +#include "tensorrt_cpp_api/device_tensor.h" + +#include + +#include +#include + +using namespace trtcpp; + +TEST(OpenCVInterop, MatHostViewIsHwc) { + cv::Mat mat(4, 5, CV_8UC3, cv::Scalar(1, 2, 3)); + auto view = opencv::viewOf(mat); + ASSERT_TRUE(view.ok()) << view.status().message(); + EXPECT_EQ(view.value().dtype(), DType::kUInt8); + EXPECT_EQ(view.value().shape(), (Shape{4, 5, 3})); + EXPECT_EQ(view.value().device(), Device::kHost); + EXPECT_EQ(view.value().data(), mat.data); +} + +TEST(OpenCVInterop, UnsupportedDepthRejected) { + cv::Mat mat(2, 2, CV_64FC1); + auto view = opencv::viewOf(mat); + EXPECT_FALSE(view.ok()); + EXPECT_EQ(view.status().code(), StatusCode::kUnsupported); +} + +TEST(OpenCVInterop, UploadRoundtrip) { + cv::Mat mat(2, 2, CV_32FC1); + mat.at(0, 0) = 1.0f; + mat.at(0, 1) = 2.0f; + mat.at(1, 0) = 3.0f; + mat.at(1, 1) = 4.0f; + + Stream stream; + auto device = opencv::upload(mat, stream); + ASSERT_TRUE(device.ok()) << device.status().message(); + EXPECT_EQ(device.value().device(), Device::kCuda); + EXPECT_EQ(device.value().shape(), (Shape{2, 2, 1})); + + auto host = device.value().toHost(stream); + ASSERT_TRUE(host.ok()); + auto values = host.value().as(); + ASSERT_TRUE(values.ok()); + ASSERT_EQ(values.value().size(), 4u); + EXPECT_FLOAT_EQ(values.value()[0], 1.0f); + EXPECT_FLOAT_EQ(values.value()[3], 4.0f); +} + +TEST(OpenCVInterop, ContinuousGpuMatView) { + cv::cuda::GpuMat gpu(4, 8, CV_8UC3); + auto view = opencv::viewOf(gpu); + // A padded GpuMat is rejected (kUnsupported); a continuous one yields an HWC device view. + if (view.ok()) { + EXPECT_EQ(view.value().device(), Device::kCuda); + EXPECT_EQ(view.value().shape(), (Shape{4, 8, 3})); + } else { + EXPECT_EQ(view.status().code(), StatusCode::kUnsupported); + } +} + +#endif // TRT_CPP_API_WITH_OPENCV diff --git a/tests/test_preproc.cpp b/tests/test_preproc.cpp new file mode 100644 index 0000000..2cb384c --- /dev/null +++ b/tests/test_preproc.cpp @@ -0,0 +1,126 @@ +#include "tensorrt_cpp_api/cuda.h" +#include "tensorrt_cpp_api/device_tensor.h" +#include "tensorrt_cpp_api/preproc.h" + +#include + +#include +#include + +using namespace trtcpp; +using namespace trtcpp::preproc; + +namespace { + +// Build an HWC uint8 image with a constant per-channel value, uploaded to the device. +Result constantImage(int h, int w, int c, const std::vector &channelValues) { + std::vector host(static_cast(h * w * c)); + for (int y = 0; y < h; ++y) { + for (int x = 0; x < w; ++x) { + for (int ch = 0; ch < c; ++ch) { + host[static_cast((y * w + x) * c + ch)] = channelValues[static_cast(ch)]; + } + } + } + auto device = Tensor::allocate(DType::kUInt8, Shape{h, w, c}, Device::kCuda); + if (!device) { + return device.status(); + } + Stream stream; + TensorView view{host.data(), DType::kUInt8, Shape{h, w, c}, Device::kHost}; + if (Status status = device.value().copyFrom(view, stream); !status) { + return status; + } + if (Status status = stream.synchronize(); !status) { // keep host alive until uploaded + return status; + } + return std::move(device).value(); +} + +} // namespace + +TEST(Preproc, ConstantImageNoResize) { + auto src = constantImage(4, 4, 3, {10, 20, 30}); + ASSERT_TRUE(src.ok()) << src.status().message(); + auto dst = Tensor::allocate(DType::kFloat32, Shape{1, 3, 4, 4}, Device::kCuda); + ASSERT_TRUE(dst.ok()); + Stream stream; + ASSERT_TRUE(letterboxToTensor(src.value().view(), dst.value().view(), PreprocSpec{}, stream).ok()); + + auto host = dst.value().toHost(stream); + ASSERT_TRUE(host.ok()); + auto values = host.value().as(); + ASSERT_TRUE(values.ok()); + for (int x = 0; x < 16; ++x) { + EXPECT_FLOAT_EQ(values.value()[0 * 16 + x], 10.0f); // channel 0 + EXPECT_FLOAT_EQ(values.value()[1 * 16 + x], 20.0f); // channel 1 + EXPECT_FLOAT_EQ(values.value()[2 * 16 + x], 30.0f); // channel 2 + } +} + +TEST(Preproc, SwapRB) { + auto src = constantImage(4, 4, 3, {10, 20, 30}); + ASSERT_TRUE(src.ok()); + auto dst = Tensor::allocate(DType::kFloat32, Shape{1, 3, 4, 4}, Device::kCuda); + ASSERT_TRUE(dst.ok()); + Stream stream; + PreprocSpec spec; + spec.swapRB = true; + ASSERT_TRUE(letterboxToTensor(src.value().view(), dst.value().view(), spec, stream).ok()); + + auto host = dst.value().toHost(stream); + ASSERT_TRUE(host.ok()); + auto values = host.value().as(); + ASSERT_TRUE(values.ok()); + EXPECT_FLOAT_EQ(values.value()[0 * 16], 30.0f); // channel 0 now holds the old channel 2 + EXPECT_FLOAT_EQ(values.value()[2 * 16], 10.0f); +} + +TEST(Preproc, PerChannelNormalize) { + auto src = constantImage(4, 4, 3, {10, 20, 30}); + ASSERT_TRUE(src.ok()); + auto dst = Tensor::allocate(DType::kFloat32, Shape{1, 3, 4, 4}, Device::kCuda); + ASSERT_TRUE(dst.ok()); + Stream stream; + PreprocSpec spec; + spec.mean = {10.0f, 20.0f, 30.0f, 0.0f}; + spec.scale = {1.0f, 1.0f, 1.0f, 1.0f}; + ASSERT_TRUE(letterboxToTensor(src.value().view(), dst.value().view(), spec, stream).ok()); + + auto host = dst.value().toHost(stream); + ASSERT_TRUE(host.ok()); + auto values = host.value().as(); + ASSERT_TRUE(values.ok()); + for (float v : values.value()) { + EXPECT_FLOAT_EQ(v, 0.0f); // (channel - mean[channel]) == 0 + } +} + +TEST(Preproc, LetterboxPadsRightBottom) { + auto src = constantImage(2, 4, 3, {50, 50, 50}); // H=2, W=4 -> scale 1, pads rows 2..3 + ASSERT_TRUE(src.ok()); + auto dst = Tensor::allocate(DType::kFloat32, Shape{1, 3, 4, 4}, Device::kCuda); + ASSERT_TRUE(dst.ok()); + Stream stream; + ASSERT_TRUE(letterboxToTensor(src.value().view(), dst.value().view(), PreprocSpec{}, stream).ok()); + + auto host = dst.value().toHost(stream); + ASSERT_TRUE(host.ok()); + auto values = host.value().as(); + ASSERT_TRUE(values.ok()); + auto at = [&](int c, int y, int x) { return values.value()[c * 16 + y * 4 + x]; }; + EXPECT_FLOAT_EQ(at(0, 0, 0), 50.0f); // resized region + EXPECT_FLOAT_EQ(at(0, 1, 3), 50.0f); + EXPECT_FLOAT_EQ(at(0, 2, 0), 0.0f); // padded region (rows 2..3) + EXPECT_FLOAT_EQ(at(0, 3, 3), 0.0f); +} + +TEST(Preproc, Float16Runs) { + auto src = constantImage(4, 4, 3, {10, 20, 30}); + ASSERT_TRUE(src.ok()); + auto dst = Tensor::allocate(DType::kFloat16, Shape{1, 3, 4, 4}, Device::kCuda); + ASSERT_TRUE(dst.ok()); + Stream stream; + EXPECT_TRUE(letterboxToTensor(src.value().view(), dst.value().view(), PreprocSpec{}, stream).ok()); + EXPECT_TRUE(stream.synchronize().ok()); +} diff --git a/tests/test_sha256.cpp b/tests/test_sha256.cpp new file mode 100644 index 0000000..368111c --- /dev/null +++ b/tests/test_sha256.cpp @@ -0,0 +1,24 @@ +#include "detail/sha256.h" + +#include + +#include + +using trtcpp::detail::sha256Hex; + +TEST(Sha256, KnownVectors) { + EXPECT_EQ(sha256Hex("", 0), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + + const std::string abc = "abc"; + EXPECT_EQ(sha256Hex(abc.data(), abc.size()), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + + // NIST two-block (>56 byte) vector, exercising multi-block padding. + const std::string two = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; + EXPECT_EQ(sha256Hex(two.data(), two.size()), "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); +} + +TEST(Sha256, DiffersByContent) { + const std::string a = "model-a"; + const std::string b = "model-b"; + EXPECT_NE(sha256Hex(a.data(), a.size()), sha256Hex(b.data(), b.size())); +} diff --git a/tests/test_shape.cpp b/tests/test_shape.cpp new file mode 100644 index 0000000..f1bc2fd --- /dev/null +++ b/tests/test_shape.cpp @@ -0,0 +1,61 @@ +#include "tensorrt_cpp_api/shape.h" + +#include + +#include + +using namespace trtcpp; + +TEST(Shape, DefaultIsScalar) { + Shape s; + EXPECT_EQ(s.rank(), 0); + EXPECT_FALSE(s.isDynamic()); + EXPECT_EQ(s.numel(), 1); // rank-0 scalar + EXPECT_EQ(s.toString(), "[]"); +} + +TEST(Shape, InitializerListAndAccess) { + Shape s{1, 3, 640, 640}; + EXPECT_EQ(s.rank(), 4); + EXPECT_EQ(s[0], 1); + EXPECT_EQ(s[3], 640); + EXPECT_EQ(s.at(1), 3); + EXPECT_EQ(s.numel(), 1LL * 3 * 640 * 640); + EXPECT_EQ(s.toString(), "[1,3,640,640]"); +} + +TEST(Shape, AtThrowsOutOfRange) { + Shape s{2, 2}; + EXPECT_THROW((void)s.at(-1), std::out_of_range); + EXPECT_THROW((void)s.at(2), std::out_of_range); + EXPECT_NO_THROW((void)s.at(0)); +} + +TEST(Shape, DynamicDims) { + Shape s{-1, 3, -1, -1}; + EXPECT_TRUE(s.isDynamic()); + EXPECT_EQ(s.numel(), 0); // dynamic -> 0 + EXPECT_EQ(s.toString(), "[-1,3,-1,-1]"); +} + +TEST(Shape, Equality) { + EXPECT_EQ((Shape{1, 3, 224, 224}), (Shape{1, 3, 224, 224})); + EXPECT_NE((Shape{1, 3, 224, 224}), (Shape{1, 3, 225, 224})); + EXPECT_NE((Shape{1, 3, 224}), (Shape{1, 3, 224, 224})); // different rank +} + +TEST(Shape, SpanConstruction) { + const std::int64_t dims[] = {8, 16}; + Shape s{std::span(dims, 2)}; + EXPECT_EQ(s.rank(), 2); + EXPECT_EQ(s.numel(), 128); + EXPECT_EQ(s.dims().size(), 2u); +} + +TEST(Shape, MaxRankEightDims) { + Shape s{1, 2, 3, 4, 5, 6, 7, 8}; + EXPECT_EQ(s.rank(), Shape::kMaxRank); + EXPECT_EQ(s.rank(), 8); + EXPECT_EQ(s[7], 8); + EXPECT_EQ(s.numel(), 1LL * 2 * 3 * 4 * 5 * 6 * 7 * 8); +} diff --git a/tests/test_status.cpp b/tests/test_status.cpp new file mode 100644 index 0000000..62f04de --- /dev/null +++ b/tests/test_status.cpp @@ -0,0 +1,102 @@ +#include "tensorrt_cpp_api/status.h" + +#include + +#include +#include + +using namespace trtcpp; + +TEST(Status, DefaultIsOk) { + Status s; + EXPECT_TRUE(s.ok()); + EXPECT_TRUE(static_cast(s)); + EXPECT_EQ(s.code(), StatusCode::kOk); +} + +TEST(Status, ErrorCarriesMessage) { + Status s{StatusCode::kInvalidArgument, "bad arg"}; + EXPECT_FALSE(s.ok()); + EXPECT_EQ(s.code(), StatusCode::kInvalidArgument); + EXPECT_EQ(s.message(), "bad arg"); + EXPECT_EQ(toString(s.code()), "invalid_argument"); +} + +TEST(Result, HoldsValue) { + Result r{42}; + ASSERT_TRUE(r.ok()); + EXPECT_EQ(r.value(), 42); + EXPECT_EQ(*r, 42); + EXPECT_EQ(r.value_or(-1), 42); +} + +TEST(Result, HoldsError) { + Result r{Status{StatusCode::kNotFound, "missing"}}; + EXPECT_FALSE(r.ok()); + EXPECT_EQ(r.status().code(), StatusCode::kNotFound); + EXPECT_EQ(r.value_or(-1), -1); +} + +TEST(Result, ArrowOperator) { + Result r{std::string("hello")}; + ASSERT_TRUE(r.ok()); + EXPECT_EQ(r->size(), 5u); +} + +TEST(Result, AndThenChains) { + Result r{10}; + auto doubled = r.and_then([](int &v) { return Result{v * 2}; }); + ASSERT_TRUE(doubled.ok()); + EXPECT_EQ(doubled.value(), 20); + + Result err{Status{StatusCode::kInternal, "boom"}}; + auto chained = err.and_then([](int &v) { return Result{v * 2}; }); + EXPECT_FALSE(chained.ok()); + EXPECT_EQ(chained.status().code(), StatusCode::kInternal); +} + +TEST(Result, TransformMaps) { + Result r{7}; + auto s = r.transform([](int &v) { return std::to_string(v); }); + ASSERT_TRUE(s.ok()); + EXPECT_EQ(s.value(), "7"); + + Result err{Status{StatusCode::kIoError, "io"}}; + auto s2 = err.transform([](int &v) { return std::to_string(v); }); + EXPECT_FALSE(s2.ok()); + EXPECT_EQ(s2.status().code(), StatusCode::kIoError); +} + +namespace { +Status consumePositive(Result r) { + TRTCPP_TRY(int v, std::move(r)); + if (v < 0) { + return Status{StatusCode::kInvalidArgument, "negative"}; + } + return Status{}; +} +} // namespace + +TEST(Result, TryMacroPropagatesError) { + EXPECT_TRUE(consumePositive(Result{5}).ok()); + + Status propagated = consumePositive(Result{Status{StatusCode::kCudaError, "cuda"}}); + EXPECT_FALSE(propagated.ok()); + EXPECT_EQ(propagated.code(), StatusCode::kCudaError); +} + +TEST(Result, MoveOnlyValue) { + Result> r{std::make_unique(7)}; + ASSERT_TRUE(r.ok()); + EXPECT_EQ(*r.value(), 7); + auto ptr = std::move(r).value(); // rvalue value() moves the unique_ptr out + ASSERT_TRUE(ptr); + EXPECT_EQ(*ptr, 7); +} + +TEST(Result, TransformOnMoveOnly) { + Result> r{std::make_unique(5)}; + auto deref = r.transform([](std::unique_ptr &p) { return *p; }); + ASSERT_TRUE(deref.ok()); + EXPECT_EQ(deref.value(), 5); +} diff --git a/tests/test_tensorview.cpp b/tests/test_tensorview.cpp new file mode 100644 index 0000000..b8ca2e7 --- /dev/null +++ b/tests/test_tensorview.cpp @@ -0,0 +1,83 @@ +#include "tensorrt_cpp_api/tensor.h" + +#include + +#include +#include + +using namespace trtcpp; + +TEST(TensorView, DefaultIsEmpty) { + TensorView v; + EXPECT_EQ(v.data(), nullptr); + EXPECT_EQ(v.dtype(), DType::kFloat32); + EXPECT_EQ(v.shape().rank(), 0); +} + +TEST(TensorView, PositionalCtorAndAccessors) { + int dummy = 0; + TensorView v{&dummy, DType::kFloat32, Shape{1, 3, 4, 4}, Device::kCuda, 0, Layout::kNCHW}; + EXPECT_EQ(v.data(), &dummy); + EXPECT_EQ(v.dtype(), DType::kFloat32); + EXPECT_EQ(v.shape(), (Shape{1, 3, 4, 4})); + EXPECT_TRUE(v.isCuda()); + EXPECT_EQ(v.layout(), Layout::kNCHW); + EXPECT_EQ(v.nbytes(), 1u * 3 * 4 * 4 * 4); // 48 floats * 4 bytes +} + +TEST(TensorView, DescCtor) { + float buf[6] = {}; + TensorView v{TensorView::Desc{.data = buf, .dtype = DType::kFloat32, .shape = Shape{2, 3}, .device = Device::kHost}}; + EXPECT_EQ(v.device(), Device::kHost); + EXPECT_FALSE(v.isCuda()); + EXPECT_EQ(v.nbytes(), 24u); +} + +TEST(TensorView, NbytesPacksInt4) { + TensorView v{nullptr, DType::kInt4, Shape{1, 10}, Device::kCuda}; + EXPECT_EQ(v.nbytes(), 5u); // 10 int4 -> 5 bytes +} + +TEST(TensorView, AsHostTypedSpan) { + std::array buf{1.f, 2.f, 3.f, 4.f}; + TensorView v{buf.data(), DType::kFloat32, Shape{4}, Device::kHost}; + auto span = v.as(); + ASSERT_TRUE(span.ok()); + ASSERT_EQ(span.value().size(), 4u); + EXPECT_FLOAT_EQ(span.value()[2], 3.f); +} + +TEST(TensorView, AsRejectsDeviceTensor) { + int dummy = 0; + TensorView v{&dummy, DType::kFloat32, Shape{1}, Device::kCuda}; + auto span = v.as(); + EXPECT_FALSE(span.ok()); + EXPECT_EQ(span.status().code(), StatusCode::kInvalidArgument); +} + +TEST(TensorView, AsRejectsDtypeMismatch) { + std::array buf{1.f, 2.f}; + TensorView v{buf.data(), DType::kFloat32, Shape{2}, Device::kHost}; + auto span = v.as(); + EXPECT_FALSE(span.ok()); + EXPECT_EQ(span.status().code(), StatusCode::kDtypeMismatch); +} + +TEST(TensorView, AsBoolHostSpan) { + std::array buf{true, false, true}; + TensorView v{buf.data(), DType::kBool, Shape{3}, Device::kHost}; + auto span = v.as(); + ASSERT_TRUE(span.ok()); + ASSERT_EQ(span.value().size(), 3u); + EXPECT_TRUE(span.value()[0]); + EXPECT_FALSE(span.value()[1]); +} + +TEST(TensorView, DynamicShapeIsUnresolved) { + std::array buf{}; + TensorView v{buf.data(), DType::kFloat32, Shape{-1, 4}, Device::kHost}; + EXPECT_EQ(v.nbytes(), 0u); // unresolved dynamic shape -> 0 bytes + auto span = v.as(); + ASSERT_TRUE(span.ok()); // ok, but a length-0 span until the shape is resolved + EXPECT_EQ(span.value().size(), 0u); +} diff --git a/tests/test_trt_logger.cpp b/tests/test_trt_logger.cpp new file mode 100644 index 0000000..431b505 --- /dev/null +++ b/tests/test_trt_logger.cpp @@ -0,0 +1,65 @@ +#include "detail/trt_common.h" + +#include + +#include + +#include +#include +#include + +using namespace trtcpp; +using namespace trtcpp::detail; + +namespace { +class RecordingLogger final : public ILogger { +public: + struct Record { + Severity severity; + std::string message; + }; + void log(Severity severity, std::string_view message) noexcept override { records.push_back({severity, std::string(message)}); } + std::vector records; +}; +} // namespace + +TEST(TrtLogger, SeverityMapsByNameNotInteger) { + EXPECT_EQ(fromNvSeverity(nvinfer1::ILogger::Severity::kINTERNAL_ERROR), Severity::kInternalError); + EXPECT_EQ(fromNvSeverity(nvinfer1::ILogger::Severity::kERROR), Severity::kError); + EXPECT_EQ(fromNvSeverity(nvinfer1::ILogger::Severity::kWARNING), Severity::kWarning); + EXPECT_EQ(fromNvSeverity(nvinfer1::ILogger::Severity::kINFO), Severity::kInfo); + EXPECT_EQ(fromNvSeverity(nvinfer1::ILogger::Severity::kVERBOSE), Severity::kVerbose); + + // Direct forward assertions (a round-trip alone could mask a two-to-one toNv bug). + EXPECT_EQ(toNvSeverity(Severity::kInternalError), nvinfer1::ILogger::Severity::kINTERNAL_ERROR); + EXPECT_EQ(toNvSeverity(Severity::kError), nvinfer1::ILogger::Severity::kERROR); + EXPECT_EQ(toNvSeverity(Severity::kWarning), nvinfer1::ILogger::Severity::kWARNING); + EXPECT_EQ(toNvSeverity(Severity::kInfo), nvinfer1::ILogger::Severity::kINFO); + EXPECT_EQ(toNvSeverity(Severity::kVerbose), nvinfer1::ILogger::Severity::kVERBOSE); + + for (Severity s : {Severity::kVerbose, Severity::kInfo, Severity::kWarning, Severity::kError, Severity::kInternalError}) { + EXPECT_EQ(fromNvSeverity(toNvSeverity(s)), s); + } + + // The reason the mapping must be by-name: the integer values genuinely differ, so a + // plain static_cast would invert every level. + EXPECT_NE(static_cast(nvinfer1::ILogger::Severity::kINTERNAL_ERROR), static_cast(Severity::kInternalError)); +} + +TEST(TrtLogger, BridgeForwardsToTrtcppLogger) { + auto recorder = std::make_shared(); + TrtLoggerBridge bridge(recorder); + bridge.log(nvinfer1::ILogger::Severity::kWARNING, "trt-warn"); + bridge.log(nvinfer1::ILogger::Severity::kVERBOSE, "trt-verbose"); + ASSERT_EQ(recorder->records.size(), 2u); + EXPECT_EQ(recorder->records[0].severity, Severity::kWarning); + EXPECT_EQ(recorder->records[0].message, "trt-warn"); + EXPECT_EQ(recorder->records[1].severity, Severity::kVerbose); +} + +TEST(TrtLogger, BridgeNullLoggerFallsBackToDefault) { + TrtLoggerBridge bridge(nullptr); // must not crash + bridge.log(nvinfer1::ILogger::Severity::kINFO, "ok"); + bridge.log(nvinfer1::ILogger::Severity::kERROR, nullptr); // null message tolerated + SUCCEED(); +} diff --git a/tests/test_trt_plugin.cpp b/tests/test_trt_plugin.cpp new file mode 100644 index 0000000..0e78b3c --- /dev/null +++ b/tests/test_trt_plugin.cpp @@ -0,0 +1,25 @@ +#include "detail/trt_common.h" + +#include "tensorrt_cpp_api/logger.h" + +#include + +#include + +using namespace trtcpp; +using namespace trtcpp::detail; + +TEST(TrtPlugin, EmptyListIsOk) { EXPECT_TRUE(loadPluginLibraries({}).ok()); } + +TEST(TrtPlugin, BadPathErrors) { + Status status = loadPluginLibraries({"/nonexistent/path/libfake_trt_plugin.so"}); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.code(), StatusCode::kTensorRtError); +} + +TEST(TrtUniquePtr, BuilderCreateSmoke) { + TrtLoggerBridge bridge(defaultLogger()); + TrtUniquePtr builder{nvinfer1::createInferBuilder(bridge.nv())}; + ASSERT_NE(builder, nullptr); + // RAII: builder is deleted here via the default deleter (TRT 10/11 use `delete`). +} diff --git a/tests/test_version.cpp b/tests/test_version.cpp new file mode 100644 index 0000000..8ba377f --- /dev/null +++ b/tests/test_version.cpp @@ -0,0 +1,22 @@ +// Also exercises that the umbrella header is self-contained and compiles standalone. +#include "tensorrt_cpp_api/all.h" + +#include + +using namespace trtcpp; + +TEST(Version, LibraryVersionIsSeven) { + EXPECT_EQ(libraryVersion().major, 7); + EXPECT_EQ(libraryVersion().minor, 0); +} + +TEST(Version, TensorRTBuildVersionRecorded) { + EXPECT_GE(tensorrtBuildMajor(), 10); // built against TensorRT >= 10 + EXPECT_LT(tensorrtBuildMajor(), 12); +} + +TEST(Version, StringMentionsBothVersions) { + const std::string s = versionString(); + EXPECT_NE(s.find("7.0.0"), std::string::npos); + EXPECT_NE(s.find("TensorRT"), std::string::npos); +}