Skip to content

[Bug] Windows LLP64 compatibility: 2GB file limit (long vs int64_t) and missing std::ios::binary #208

@StasEfremovMSU

Description

@StasEfremovMSU

Hello,

While building and using the library via VCPKG on Windows, I encountered a couple of critical issues:

  • Corrupted binary files (e.g., zip archives): Downloaded binary files get corrupted because std::ofstream is opened in text mode by default on Windows. Adding the std::ios::binary flag fixes this.
  • Crashes on files > 2GB: The library uses the long type for file sizes. On Windows (LLP64 data model), long is strictly 32-bit, which leads to a "stol argument out of range" exception when processing large files. Changing long to long long (and stol to stoll) resolves this.

I am attaching my custom VCPKG portfile.cmake containing the necessary string replacements as a workaround for anyone facing the same issues.

Click to expand: portfile.cmake
vcpkg_check_linkage(ONLY_STATIC_LIBRARY)

vcpkg_from_github(
    OUT_SOURCE_PATH SOURCE_PATH
    REPO minio/minio-cpp
    REF bc08d87a8a0001fd32f998e2af33ca1961454dc8
    SHA512 82fb94e3042a73abe610fa3413b64af6f322a897f5060f3c25e48b5d68846ec9283a86785e1e03161dd3b80c2e633790195e5bc7e983f37061b8932568ff647c
    HEAD_REF main
)

vcpkg_replace_string(
    "${SOURCE_PATH}/src/utils.cc"
    "append_child(pugi::node_pcdata).set_value(value)"
    "append_child(pugi::node_pcdata).set_value(value.c_str())"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/src/client.cc"
    "long object_size = args.object_size;"
    "long long object_size = args.object_size;"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/include/miniocpp/args.h"
    "long object_size = -1;"
    "long long object_size = -1;"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/include/miniocpp/args.h"
    "long object_size_ = -1;"
    "long long object_size_ = -1;"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/include/miniocpp/args.h"
    "PutObjectArgs(std::istream& stream, long object_size, long part_size);"
    "PutObjectArgs(std::istream& stream, long long object_size, long part_size);"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/src/args.cc"
    "PutObjectArgs::PutObjectArgs(std::istream& istream, long object_size,"
    "PutObjectArgs::PutObjectArgs(std::istream& istream, long long object_size,"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/src/args.cc" 
    "object_size_ = static_cast<long>(object_size);"
    "object_size_ = static_cast<long long>(object_size);"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/src/utils.cc"
    "error::Error CalcPartInfo(long object_size, size_t& part_size,"
    "error::Error CalcPartInfo(long long object_size, size_t& part_size,"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/include/miniocpp/utils.h"
    "error::Error CalcPartInfo(long object_size, size_t& part_size,"
    "error::Error CalcPartInfo(long long object_size, size_t& part_size,"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/src/args.cc"
    "object_size = static_cast<long>(obj_size);"
    "object_size = static_cast<long long>(obj_size);"
)

vcpkg_cmake_configure(
    SOURCE_PATH "${SOURCE_PATH}"
    DISABLE_PARALLEL_CONFIGURE
)

vcpkg_replace_string(
    "${SOURCE_PATH}/src/client.cc"
    "std::ofstream fout(temp_filename, std::ios::trunc | std::ios::out);"
    "std::ofstream fout(temp_filename, std::ios::trunc | std::ios::out | std::ios::binary);"
)


vcpkg_replace_string(
    "${SOURCE_PATH}/src/select.cc" 
    "if (!value.empty()) bytes_scanned = std::stol(value);"
    "if (!value.empty()) bytes_scanned = std::stoll(value);"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/src/select.cc" 
    "if (!value.empty()) bytes_processed = std::stol(value);"
    "if (!value.empty()) bytes_processed = std::stoll(value);"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/src/select.cc" 
    "if (!value.empty()) bytes_returned = std::stol(value);"
    "if (!value.empty()) bytes_returned = std::stoll(value);"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/src/select.cc" 
    " long int bytes_scanned = -1;"
    " long long bytes_scanned = -1;"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/src/select.cc" 
    "long int bytes_processed = -1;"
    "long long bytes_processed = -1;"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/src/select.cc" 
    "long int bytes_returned = -1;"
    "long long bytes_returned = -1;"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/src/client.cc" 
    "if (!value.empty()) resp.size = std::stol(value)"
    "if (!value.empty()) resp.size = std::stoll(value)"
)

vcpkg_replace_string(
    "${SOURCE_PATH}/src/baseclient.cc" 
    "if (!value.empty()) resp.size = std::stol(value);"
    "if (!value.empty()) resp.size = std::stoll(value);"
)

vcpkg_cmake_install()
vcpkg_cmake_config_fixup(PACKAGE_NAME miniocpp CONFIG_PATH "lib/cmake/miniocpp")

vcpkg_copy_pdbs()
vcpkg_fixup_pkgconfig()

file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include")
vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE")

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions