From 8c1a0a49e731be1eda7ac7e3849925985b029921 Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 24 Feb 2026 13:56:17 +0100 Subject: [PATCH 01/49] Add lfs curl resume --- third_party/libgit2/lfs.patch | 41 +++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 6139df9e44..8e07425928 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -312,10 +312,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..484811a0c +index 000000000..a682cdbef --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,567 @@ +@@ -0,0 +1,604 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -824,6 +824,43 @@ index 000000000..484811a0c + print_download_info(la->full_path, get_digit(la->lfs_size)); + /* Perform the request, res gets the return code */ + res = curl_easy_perform(dl_curl); ++ /* Check for resume of partial download error */ ++ if (res == CURLE_PARTIAL_FILE) { ++ curl_off_t resume_from = 0; ++ curl_easy_getinfo( ++ dl_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, ++ &resume_from); ++ ++ if (resume_from == -1) { ++ fprintf(stderr, ++ "curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); ++ } else { ++ fprintf(stderr, ++ "curl_easy_perform() failed with transferred a partial file error and trying to resume\n"); ++ curl_off_t offset = 0; ++ if (ftpfile.stream) { ++ fseek(ftpfile.stream, 0, SEEK_END); ++ offset = ftell(ftpfile.stream); ++ } else { ++ ftpfile.stream = ++ fopen(ftpfile.filename, "ab+"); ++ if (ftpfile.stream) { ++ fseek(ftpfile.stream, 0, ++ SEEK_END); ++ offset = ftell(ftpfile.stream); ++ } ++ } ++ ++ // Tell libcurl to resume ++ curl_easy_setopt( ++ dl_curl, CURLOPT_RESUME_FROM_LARGE, ++ offset); ++ /* Perform the request, res gets the return code ++ */ ++ res = curl_easy_perform(dl_curl); ++ } ++ } ++ + /* Check for errors */ + if (res != CURLE_OK) { + fprintf(stderr, "curl_easy_perform() failed: %s\n", From 43c8aac2b8ca1baf48849771134f1855ae5a10ae Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 25 Feb 2026 12:05:53 +0100 Subject: [PATCH 02/49] Fix linux build --- third_party/libgit2/lfs.patch | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 8e07425928..9d09b48943 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -312,10 +312,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..a682cdbef +index 000000000..3b42a7ba4 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,604 @@ +@@ -0,0 +1,603 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -851,12 +851,11 @@ index 000000000..a682cdbef + } + } + -+ // Tell libcurl to resume ++ /* Tell libcurl to resume */ + curl_easy_setopt( + dl_curl, CURLOPT_RESUME_FROM_LARGE, + offset); -+ /* Perform the request, res gets the return code -+ */ ++ /* Perform the request, res gets the return code */ + res = curl_easy_perform(dl_curl); + } + } From f7e1c852fb709cd7aedf5e23dbbd60c7974734e0 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Fri, 27 Feb 2026 11:05:06 +0100 Subject: [PATCH 03/49] Git status --- src/pull_module/libgit2.cpp | 187 ++++++++++++++++++++++++++++++++++++ src/pull_module/libgit2.hpp | 2 + src/status.cpp | 3 + src/status.hpp | 2 + 4 files changed, 194 insertions(+) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 1566805c74..395d461a7f 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -179,6 +179,184 @@ Status HfDownloader::RemoveReadonlyFileAttributeFromDir(const std::string& direc return StatusCode::OK; } +Status HfDownloader::CheckRepositoryStatus() { + git_repository *repo = NULL; + int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); + if (error < 0) { + const git_error *err = git_error_last(); + if (err) + SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); + else + SPDLOG_ERROR("Repository open failed: {}", error); + if (repo) git_repository_free(repo); + + return StatusCode::HF_GIT_STATUS_FAILED; + } + // HEAD state info + bool is_detached = git_repository_head_detached(repo) == 1; + bool is_unborn = git_repository_head_unborn(repo) == 1; + + // Collect status (staged/unstaged/untracked) + git_status_options opts = GIT_STATUS_OPTIONS_INIT; + + opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR; + opts.flags = GIT_STATUS_OPT_INCLUDE_UNTRACKED // include untracked files // | GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX // detect renames HEAD->index - not required currently and impacts performance + | GIT_STATUS_OPT_SORT_CASE_SENSITIVELY; + + + git_status_list* status_list = nullptr; + error = git_status_list_new(&status_list, repo, &opts); + if (error != 0) { + return StatusCode::HF_GIT_STATUS_FAILED; + } + + size_t staged = 0, unstaged = 0, untracked = 0, conflicted = 0; + const size_t n = git_status_list_entrycount(status_list); // iterate entries + for (size_t i = 0; i < n; ++i) { + const git_status_entry* e = git_status_byindex(status_list, i); + unsigned s = e->status; + + // Staged (index) changes + if (s & (GIT_STATUS_INDEX_NEW | + GIT_STATUS_INDEX_MODIFIED| + GIT_STATUS_INDEX_DELETED | + GIT_STATUS_INDEX_RENAMED | + GIT_STATUS_INDEX_TYPECHANGE)) + ++staged; + + // Unstaged (workdir) changes + if (s & (GIT_STATUS_WT_MODIFIED | + GIT_STATUS_WT_DELETED | + GIT_STATUS_WT_RENAMED | + GIT_STATUS_WT_TYPECHANGE)) + ++unstaged; + + // Untracked + if (s & GIT_STATUS_WT_NEW) + ++untracked; + + // libgit2 will also flag conflicted entries via status/diff machinery + if (s & GIT_STATUS_CONFLICTED) + ++conflicted; + } + + std::stringstream ss; + ss << "HEAD state : " + << (is_unborn ? "unborn (no commits)" : (is_detached ? "detached" : "attached")) + << "\n"; + ss << "Staged changes : " << staged << "\n"; + ss << "Unstaged changes: " << unstaged << "\n"; + ss << "Untracked files : " << untracked << "\n"; + if (conflicted) ss << " (" << conflicted << " paths flagged)"; + + SPDLOG_DEBUG(ss.str()); + git_status_list_free(status_list); + + if (is_unborn || is_detached || staged || unstaged || untracked || conflicted) { + return StatusCode::HF_GIT_STATUS_UNCLEAN; + } + return StatusCode::OK; +} + +static int print_changed_and_untracked(git_repository *repo) { + int error = 0; + git_status_list *statuslist = NULL; + + git_status_options opts; + error = git_status_options_init(&opts, GIT_STATUS_OPTIONS_VERSION); + if (error < 0) return error; + + // Choose what to include + opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR; // consider both index and working dir + opts.flags = + GIT_STATUS_OPT_INCLUDE_UNTRACKED | // include untracked files + GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS | // recurse into untracked dirs + GIT_STATUS_OPT_INCLUDE_IGNORED | // (optional) include ignored if you want to see them + GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX | // detect renames in index + GIT_STATUS_OPT_RENAMES_INDEX_TO_WORKDIR | // detect renames in workdir + GIT_STATUS_OPT_SORT_CASE_SENSITIVELY; // stable ordering + + // If you want to limit to certain paths/patterns, set opts.pathspec here. + + if ((error = git_status_list_new(&statuslist, repo, &opts)) < 0) + return error; + + size_t count = git_status_list_entrycount(statuslist); + for (size_t i = 0; i < count; i++) { + const git_status_entry *e = git_status_byindex(statuslist, i); + if (!e) continue; + + unsigned int s = e->status; + + // Consider “changed” as anything that’s not current in HEAD/INDEX/WT: + // You can tailor this to your exact definition. + int is_untracked = + (s & GIT_STATUS_WT_NEW) != 0; // working tree new (untracked) + int is_workdir_changed = + (s & (GIT_STATUS_WT_MODIFIED | + GIT_STATUS_WT_DELETED | + GIT_STATUS_WT_RENAMED | + GIT_STATUS_WT_TYPECHANGE)) != 0; + int is_index_changed = + (s & (GIT_STATUS_INDEX_NEW | + GIT_STATUS_INDEX_MODIFIED | + GIT_STATUS_INDEX_DELETED | + GIT_STATUS_INDEX_RENAMED | + GIT_STATUS_INDEX_TYPECHANGE)) != 0; + + if (!(is_untracked || is_workdir_changed || is_index_changed)) + continue; + + // Prefer the most relevant delta for the path + const git_diff_delta *delta = NULL; + if (is_workdir_changed && e->index_to_workdir) + delta = e->index_to_workdir; + else if (is_index_changed && e->head_to_index) + delta = e->head_to_index; + else if (is_untracked && e->index_to_workdir) + delta = e->index_to_workdir; + + if (!delta) continue; + + // For renames, old_file and new_file may differ; typically you want new_file.path + const char *path = delta->new_file.path ? delta->new_file.path + : delta->old_file.path; + + // Print or collect the filename + SPDLOG_INFO("is_untracked {} is_workdir_changed {} is_index_changed {} File {} ", is_untracked, is_workdir_changed, is_index_changed, path); + } + + git_status_list_free(statuslist); + return 0; +} + +int HfDownloader::CheckRepositoryForResume() { + git_repository *repo = NULL; + int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); + if (error < 0) { + const git_error *err = git_error_last(); + if (err) + SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); + else + SPDLOG_ERROR("Repository open failed: {}", error); + if (repo) git_repository_free(repo); + + return error; + } + + error = print_changed_and_untracked(repo); + if (error < 0) { + const git_error *err = git_error_last(); + if (err) + SPDLOG_ERROR("Print changed files failed: {} {}", err->klass, err->message); + else + SPDLOG_ERROR("Print changed files failed: {}", error); + } + + if (repo) git_repository_free(repo); + return error; +} + Status HfDownloader::downloadModel() { if (FileSystem::isPathEscaped(this->downloadPath)) { SPDLOG_ERROR("Path {} escape with .. is forbidden.", this->downloadPath); @@ -187,6 +365,9 @@ Status HfDownloader::downloadModel() { // Repository exists and we do not want to overwrite if (std::filesystem::is_directory(this->downloadPath) && !this->overwriteModels) { + CheckRepositoryForResume(); + + std::cout << "Path already exists on local filesystem. Skipping download to path: " << this->downloadPath << std::endl; return StatusCode::OK; } @@ -231,6 +412,12 @@ Status HfDownloader::downloadModel() { git_repository_free(cloned_repo); } + SPDLOG_DEBUG("Checking repository status."); + status = CheckRepositoryStatus(); + if (!status.ok()) { + return status; + } + // libgit2 clone sets readonly attributes status = RemoveReadonlyFileAttributeFromDir(this->downloadPath); if (!status.ok()) { diff --git a/src/pull_module/libgit2.hpp b/src/pull_module/libgit2.hpp index 943f3cf725..755a84e89e 100644 --- a/src/pull_module/libgit2.hpp +++ b/src/pull_module/libgit2.hpp @@ -62,5 +62,7 @@ class HfDownloader : public IModelDownloader { std::string GetRepositoryUrlWithPassword(); bool CheckIfProxySet(); Status RemoveReadonlyFileAttributeFromDir(const std::string& directoryPath); + Status CheckRepositoryStatus(); + int CheckRepositoryForResume(); }; } // namespace ovms diff --git a/src/status.cpp b/src/status.cpp index 97a92b9d30..b83b4f7a45 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -348,6 +348,9 @@ const std::unordered_map Status::statusMessageMap = { {StatusCode::HF_RUN_OPTIMUM_CLI_EXPORT_FAILED, "Failed to run optimum-cli export command"}, {StatusCode::HF_RUN_CONVERT_TOKENIZER_EXPORT_FAILED, "Failed to run convert-tokenizer export command"}, {StatusCode::HF_GIT_CLONE_FAILED, "Failed in libgit2 execution of clone method"}, + {StatusCode::HF_GIT_STATUS_FAILED, "Failed in libgit2 execution of status method"}, + {StatusCode::HF_GIT_STATUS_UNCLEAN, "Unclean status detected in libgit2 cloned repository"}, + {StatusCode::PARTIAL_END, "Request has finished and no further communication is needed"}, {StatusCode::NONEXISTENT_PATH, "Nonexistent path"}, diff --git a/src/status.hpp b/src/status.hpp index 18a2b093b5..2f72532cfd 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -360,6 +360,8 @@ enum class StatusCode { HF_RUN_OPTIMUM_CLI_EXPORT_FAILED, HF_RUN_CONVERT_TOKENIZER_EXPORT_FAILED, HF_GIT_CLONE_FAILED, + HF_GIT_STATUS_FAILED, + HF_GIT_STATUS_UNCLEAN, PARTIAL_END, NONEXISTENT_PATH, From 7dcf65916712243492c696defb9f0c7574619889 Mon Sep 17 00:00:00 2001 From: rasapala Date: Fri, 27 Feb 2026 11:52:26 +0100 Subject: [PATCH 04/49] Works on windows --- third_party/libgit2/lfs.patch | 110 +++++++++++++++++++++++++--------- 1 file changed, 81 insertions(+), 29 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 9d09b48943..02a81c57e4 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -312,10 +312,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..3b42a7ba4 +index 000000000..5a005f903 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,603 @@ +@@ -0,0 +1,655 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -337,6 +337,7 @@ index 000000000..3b42a7ba4 + +#include +#include "git2/sys/filter.h" ++#include "oid.h" +#include "filter.h" +#include "str.h" +#include "repository.h" @@ -360,27 +361,49 @@ index 000000000..3b42a7ba4 + size_t number = strtoull(buffer, &endptr, 10); + + if (errno == ERANGE) { -+ fprintf(stderr, "Conversion error\n"); ++ fprintf(stderr, "\n[ERROR] Conversion error\n"); + } + if (endptr == buffer) { -+ fprintf(stderr, "No digits were found\n"); ++ fprintf(stderr, "\n[ERROR] No digits were found\n"); + } else if (*endptr != '\0') { -+ fprintf(stderr, "Additional characters after number: %s\n", endptr); ++ fprintf(stderr, "\n[ERROR] Additional characters after number: %s\n", endptr); + } + + return number; +} + -+char *append_char_to_buffer(char *existingBuffer, char additionalChar) ++/** ++ * Appends a C-string `suffix` to `existingBuffer` by allocating a new buffer. ++ * The original `existingBuffer` is not modified. ++ * ++ * Returns: ++ * - Newly allocated buffer containing the concatenation, or ++ * - NULL on allocation failure or if inputs are invalid. ++ * ++ * Note: Caller is responsible for freeing the returned buffer. ++ */ ++char *append_cstr_to_buffer(const char *existingBuffer, const char *suffix) +{ ++ if (existingBuffer == NULL || suffix == NULL) { ++ return NULL; ++ } ++ + size_t existingLength = strlen(existingBuffer); -+ char *newBuffer = (char *)malloc((existingLength + 2) * sizeof(char)); ++ size_t suffixLength = strlen(suffix); ++ ++ // +1 for the null terminator ++ size_t newSize = existingLength + suffixLength + 1; ++ ++ char *newBuffer = (char *)malloc(newSize); + if (newBuffer == NULL) { + return NULL; + } -+ strcpy(newBuffer, existingBuffer); -+ newBuffer[existingLength] = additionalChar; -+ newBuffer[existingLength + 1] = '\0'; ++ ++ // Copy existing and then append suffix ++ memcpy(newBuffer, existingBuffer, existingLength); ++ memcpy(newBuffer + existingLength, suffix, suffixLength); ++ newBuffer[newSize - 1] = '\0'; ++ + return newBuffer; +} + @@ -412,6 +435,17 @@ index 000000000..3b42a7ba4 + return -1; +} + ++void print_src_oid(const git_filter_source *src) ++{ ++ const git_oid *oid = git_filter_source_id(src); ++ ++ if (oid) { ++ printf("\nsrc->git_oid %s\n", git_oid_tostr_s(oid)); ++ } else { ++ printf("\nsrc has no OID (e.g., not a blob-backed source or unavailable)\n"); ++ } ++} ++ +static int lfs_insert_id( + git_str *to, const git_str *from, const git_filter_source *src, void** payload) +{ @@ -427,18 +461,31 @@ index 000000000..3b42a7ba4 + + const char *obj_regexp = "\noid sha256:(.*)\n"; + const char *size_regexp = "\nsize (.*)\n"; -+ if (get_lfs_info_match(&lfs_oid, obj_regexp) < 0) ++ ++ print_src_oid(src); ++ if (get_lfs_info_match(&lfs_oid, obj_regexp) < 0) { ++ fprintf(stderr,"\n[ERROR] failure, cannot find lfs oid in: %s\n", ++ lfs_oid.ptr); + return -1; ++ } + -+ if (get_lfs_info_match(&lfs_size, size_regexp) < 0) ++ if (get_lfs_info_match(&lfs_size, size_regexp) < 0) { ++ fprintf(stderr, ++ "\n[ERROR] failure, cannot find lfs size in: %s\n", ++ lfs_size.ptr); + return -1; ++ } + + git_repository *repo = git_filter_source_repo(src); + const char *path = git_filter_source_path(src); + + git_str full_path = GIT_STR_INIT; -+ if (git_repository_workdir_path(&full_path, repo, path) < 0) ++ if (git_repository_workdir_path(&full_path, repo, path) < 0) { ++ fprintf(stderr, ++ "\n[ERROR] failure, cannot get repository path: %s\n", ++ path); + return -1; ++ } + + size_t workdir_size = strlen(git_repository_workdir(repo)); + @@ -623,7 +670,7 @@ index 000000000..3b42a7ba4 + /* open file for writing */ + out->stream = fopen(out->filename, "wb"); + if (!out->stream) { -+ fprintf(stderr, "failure, cannot open file to write: %s\n", ++ fprintf(stderr, "\n[ERROR] failure, cannot open file to write: %s\n", + out->filename); + return 0; /* failure, cannot open file to write */ + } @@ -690,11 +737,11 @@ index 000000000..3b42a7ba4 +{ + GIT_UNUSED(self); + if (!payload) { -+ fprintf(stderr, "lfs payload not initialized"); ++ fprintf(stderr, "\n[ERROR] lfs payload not initialized\n"); + return; + } + struct lfs_attrs *la = (struct lfs_attrs *)payload; -+ char *tmp_out_file = append_char_to_buffer(la->full_path, '2'); ++ char *tmp_out_file = append_cstr_to_buffer(la->full_path, "lfs_part"); + + CURL *info_curl,*dl_curl; + CURLcode res = CURLE_OK; @@ -709,7 +756,7 @@ index 000000000..3b42a7ba4 + &lfs_info_url, '.', + la->url, + "git/info/lfs/objects/batch") < 0) { -+ fprintf(stderr, "failed to create url '%s'", ++ fprintf(stderr, "\n[ERROR] failed to create url '%s'\n", + la->full_path); + goto on_error; + } @@ -726,7 +773,7 @@ index 000000000..3b42a7ba4 + CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_URL, lfs_info_url.ptr)); + + if (status != CURLE_OK) { -+ fprintf(stderr, "curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); ++ fprintf(stderr, "\n[ERROR] curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); + goto info_cleaup; + } + git_str lfs_info_data = GIT_STR_INIT; @@ -739,7 +786,7 @@ index 000000000..3b42a7ba4 + ",\"size\":", + la->lfs_size, + "}]}" ) < 0) { -+ fprintf(stderr, "failed to create url '%s'", ++ fprintf(stderr, "\n[ERROR] failed to create url '%s'\n", + la->full_path); + /* always cleanup */ + curl_easy_cleanup(info_curl); @@ -759,14 +806,14 @@ index 000000000..3b42a7ba4 + CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_WRITEDATA, (void *)&response)); + + if (status != CURLE_OK) { -+ fprintf(stderr, "curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); ++ fprintf(stderr, "\n[ERROR] curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); + goto info_cleaup; + } + /* Perform the request, res gets the return code */ + res = curl_easy_perform(info_curl); + /* Check for errors */ + if (res != CURLE_OK) { -+ fprintf(stderr, "curl_easy_perform() failed: %s\n", ++ fprintf(stderr, "\n[ERROR] curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + /* always cleanup */ + curl_easy_cleanup(info_curl); @@ -817,7 +864,7 @@ index 000000000..3b42a7ba4 + CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_XFERINFODATA, &progress_d)); + + if (status != CURLE_OK) { -+ fprintf(stderr, "curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); ++ fprintf(stderr, "\n[ERROR] curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); + curl_easy_cleanup(dl_curl); + goto on_error; + } @@ -833,10 +880,10 @@ index 000000000..3b42a7ba4 + + if (resume_from == -1) { + fprintf(stderr, -+ "curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); ++ "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); + } else { + fprintf(stderr, -+ "curl_easy_perform() failed with transferred a partial file error and trying to resume\n"); ++ "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and trying to resume\n"); + curl_off_t offset = 0; + if (ftpfile.stream) { + fseek(ftpfile.stream, 0, SEEK_END); @@ -862,12 +909,15 @@ index 000000000..3b42a7ba4 + + /* Check for errors */ + if (res != CURLE_OK) { -+ fprintf(stderr, "curl_easy_perform() failed: %s\n", ++ fprintf(stderr, "\n[ERROR] curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + if (ftpfile.stream) + fclose(ftpfile.stream); + /* always cleanup */ + curl_easy_cleanup(dl_curl); ++ ++ /* Add partial file status */ ++ + goto on_error; + } + @@ -879,20 +929,22 @@ index 000000000..3b42a7ba4 + + /* Remove lfs file and rename downloaded file to oryginal lfs filename */ + if (p_unlink(la->full_path) < 0) { -+ fprintf(stderr, "failed to delete file '%s'", la->full_path); ++ fprintf(stderr, "\n[ERROR] failed to delete file '%s'\n", la->full_path); + goto on_error; + } + + if (p_rename(tmp_out_file, la->full_path) < 0) { -+ fprintf(stderr, "failed to rename file to '%s'", la->full_path); ++ fprintf(stderr, "\n[ERROR] failed to rename file to '%s'\n", la->full_path); + goto on_error; + } ++ free(tmp_out_file); + git__free(payload); + return; + -+on_error: ++ on_error: ++ free(tmp_out_file); + git__free(payload); -+ fprintf(stderr, "LFS download failed for file %s\n", la->full_path); ++ fprintf(stderr, "\n[ERROR] LFS download failed for file %s\n", la->full_path); + return; +} + From 1ea3962b947ea2189073699bd57443e869a935a7 Mon Sep 17 00:00:00 2001 From: rasapala Date: Fri, 27 Feb 2026 11:57:15 +0100 Subject: [PATCH 05/49] Fix lin --- third_party/libgit2/lfs.patch | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 02a81c57e4..3b2e8a5024 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -312,7 +312,7 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..5a005f903 +index 000000000..7c1111c34 --- /dev/null +++ b/src/libgit2/lfs_filter.c @@ -0,0 +1,655 @@ @@ -391,7 +391,7 @@ index 000000000..5a005f903 + size_t existingLength = strlen(existingBuffer); + size_t suffixLength = strlen(suffix); + -+ // +1 for the null terminator ++ /* +1 for the null terminator */ + size_t newSize = existingLength + suffixLength + 1; + + char *newBuffer = (char *)malloc(newSize); @@ -399,7 +399,7 @@ index 000000000..5a005f903 + return NULL; + } + -+ // Copy existing and then append suffix ++ /* Copy existing and then append suffix */ + memcpy(newBuffer, existingBuffer, existingLength); + memcpy(newBuffer + existingLength, suffix, suffixLength); + newBuffer[newSize - 1] = '\0'; From 3d7a6090a77242f3f9a508780327c31f3d839bd4 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Mon, 2 Mar 2026 18:13:15 +0100 Subject: [PATCH 06/49] Debug --- src/pull_module/libgit2.cpp | 473 +++++++++++++++++++++++++++++++++++- 1 file changed, 472 insertions(+), 1 deletion(-) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 395d461a7f..423320b1af 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -45,6 +45,7 @@ #endif namespace ovms { +namespace fs = std::filesystem; // Callback for clone authentication - will be used when password is not set in repo_url // Does not work with LFS download as it requires additional authentication when password is not set in repository url @@ -330,6 +331,339 @@ static int print_changed_and_untracked(git_repository *repo) { return 0; } +#define CHECK(call) do { \ + int _err = (call); \ + if (_err < 0) { \ + const git_error *e = git_error_last(); \ + fprintf(stderr, "Error %d: %s (%s:%d)\n", _err, e && e->message ? e->message : "no message", __FILE__, __LINE__); \ + return; \ + } \ +} while (0) + +// Fetch from remote and update FETCH_HEAD +static void do_fetch(git_repository *repo, const char *remote_name, const char *proxy) +{ + git_remote *remote = NULL; + git_fetch_options fetch_opts = GIT_FETCH_OPTIONS_INIT; + + fetch_opts.prune = GIT_FETCH_PRUNE_UNSPECIFIED; + fetch_opts.update_fetchhead = 1; + fetch_opts.download_tags = GIT_REMOTE_DOWNLOAD_TAGS_ALL; + fetch_opts.callbacks = (git_remote_callbacks) GIT_REMOTE_CALLBACKS_INIT; + fetch_opts.callbacks.credentials = cred_acquire_cb; + if (proxy) { + fetch_opts.proxy_opts.type = GIT_PROXY_SPECIFIED; + fetch_opts.proxy_opts.url = proxy; + } + + CHECK(git_remote_lookup(&remote, repo, remote_name)); + + printf("Fetching from %s...\n", remote_name); + CHECK(git_remote_fetch(remote, NULL, &fetch_opts, NULL)); + + // Optional: update remote-tracking branches' default refspec tips + // (git_remote_fetch already updates tips if update_fetchhead=1; explicit + // update_tips is not required in recent libgit2 versions.) + + git_remote_free(remote); +} + +// Fast-forward the local branch to target OID +static void do_fast_forward(git_repository *repo, + git_reference *local_branch, + const git_oid *target_oid) +{ + git_object *target = NULL; + git_checkout_options co_opts = GIT_CHECKOUT_OPTIONS_INIT; + git_reference *updated_ref = NULL; + + CHECK(git_object_lookup(&target, repo, target_oid, GIT_OBJECT_COMMIT)); + + // Update the branch reference to point to the target commit + CHECK(git_reference_set_target(&updated_ref, local_branch, target_oid, "Fast-forward")); + + // Make sure HEAD points to that branch (it normally already does) + CHECK(git_repository_set_head(repo, git_reference_name(updated_ref))); + + // Checkout files to match the target tree + co_opts.checkout_strategy = GIT_CHECKOUT_SAFE; + CHECK(git_checkout_tree(repo, target, &co_opts)); + + printf("Fast-forwarded %s to %s\n", + git_reference_shorthand(local_branch), + git_oid_tostr_s(target_oid)); + + git_object_free(target); + git_reference_free(updated_ref); +} + +// Perform a normal merge and create a merge commit if no conflicts +static void do_normal_merge(git_repository *repo, + git_reference *local_branch, + git_annotated_commit *their_head) +{ + git_merge_options merge_opts = GIT_MERGE_OPTIONS_INIT; + git_checkout_options co_opts = GIT_CHECKOUT_OPTIONS_INIT; + + merge_opts.file_favor = GIT_MERGE_FILE_FAVOR_NORMAL; + co_opts.checkout_strategy = GIT_CHECKOUT_SAFE | GIT_CHECKOUT_RECREATE_MISSING; + + const git_annotated_commit *their_heads[1] = { their_head }; + + printf("Merging...\n"); + CHECK(git_merge(repo, their_heads, 1, &merge_opts, &co_opts)); + + // Check for conflicts + git_index *index = NULL; + CHECK(git_repository_index(&index, repo)); + + if (git_index_has_conflicts(index)) { + printf("Merge has conflicts. Please resolve them and create the merge commit manually.\n"); + git_index_free(index); + return; // Leave repository in merging state + } + + // Write index to tree + git_oid tree_oid; + CHECK(git_index_write_tree(&tree_oid, index)); + CHECK(git_index_write(index)); + git_index_free(index); + + git_tree *tree = NULL; + CHECK(git_tree_lookup(&tree, repo, &tree_oid)); + + // Prepare signature (from config if available) + git_signature *sig = NULL; + int err = git_signature_default(&sig, repo); + if (err == GIT_ENOTFOUND || sig == NULL) { + // Fallback if user.name/email not set in config + CHECK(git_signature_now(&sig, "Your Name", "you@example.com")); + } else { + CHECK(err); + } + + // Get current HEAD (our) commit and their commit to be parents + git_reference *head = NULL, *resolved_branch = NULL; + CHECK(git_repository_head(&head, repo)); + CHECK(git_reference_resolve(&resolved_branch, head)); // ensure direct ref + + const git_oid *our_oid = git_reference_target(resolved_branch); + git_commit *our_commit = NULL; + git_commit *their_commit = NULL; + CHECK(git_commit_lookup(&our_commit, repo, our_oid)); + CHECK(git_commit_lookup(&their_commit, repo, git_annotated_commit_id(their_head))); + + const git_commit *parents[2] = { our_commit, their_commit }; + git_oid merge_commit_oid; + + // Create merge commit on the current branch ref + CHECK(git_commit_create(&merge_commit_oid, + repo, + git_reference_name(resolved_branch), + sig, sig, + NULL /* message_encoding */, + "Merge remote-tracking branch", + tree, + 2, parents)); + + printf("Created merge commit %s on %s\n", + git_oid_tostr_s(&merge_commit_oid), + git_reference_shorthand(resolved_branch)); + + // Cleanup + git_signature_free(sig); + git_tree_free(tree); + git_commit_free(our_commit); + git_commit_free(their_commit); + git_reference_free(head); + git_reference_free(resolved_branch); +} + +// Main pull routine: fetch + merge (fast-forward if possible) +static void pull(git_repository *repo, const char *remote_name, const char *proxy) +{ + // Ensure we are on a branch (not detached HEAD) + git_reference *head = NULL; + int head_res = git_repository_head(&head, repo); + // HEAD state info + bool is_detached = git_repository_head_detached(repo) == 1; + bool is_unborn = git_repository_head_unborn(repo) == 1; + if (is_unborn) { + fprintf(stderr, "Repository has no HEAD yet (unborn branch).\n"); + return; + } else if (is_detached) { + fprintf(stderr, "HEAD is detached; cannot pull safely. Checkout a branch first.\n"); + return; + } + CHECK(head_res); + + // Resolve symbolic HEAD to direct branch ref (refs/heads/…) + git_reference *local_branch = NULL; + CHECK(git_reference_resolve(&local_branch, head)); + + // Find the upstream tracking branch (refs/remotes//) + git_reference *upstream = NULL; + int up_ok = git_branch_upstream(&upstream, local_branch); + if (up_ok != 0 || upstream == NULL) { + fprintf(stderr, "Current branch has no upstream. Set it with:\n" + " git branch --set-upstream-to=%s/ \n", remote_name); + return; + } + + // Verify upstream belongs to the requested remote; not strictly required for fetch + // but we fetch from the chosen remote anyway. + do_fetch(repo, remote_name, proxy); + + // Prepare "their" commit as annotated commit from upstream + git_annotated_commit *their_head = NULL; + CHECK(git_annotated_commit_from_ref(&their_head, repo, upstream)); + + // Merge analysis + git_merge_analysis_t analysis; + git_merge_preference_t preference; + CHECK(git_merge_analysis(&analysis, &preference, repo, + (const git_annotated_commit **)&their_head, 1)); + + if (analysis & GIT_MERGE_ANALYSIS_UP_TO_DATE) { + printf("Already up to date.\n"); + } else if (analysis & GIT_MERGE_ANALYSIS_FASTFORWARD) { + const git_oid *target_oid = git_annotated_commit_id(their_head); + do_fast_forward(repo, local_branch, target_oid); + } else if (analysis & GIT_MERGE_ANALYSIS_NORMAL) { + do_normal_merge(repo, local_branch, their_head); + } else { + printf("No merge action taken (analysis=%u, preference=%u).\n", + (unsigned)analysis, (unsigned)preference); + } + + // Cleanup + git_annotated_commit_free(their_head); + git_reference_free(upstream); + git_reference_free(local_branch); + git_reference_free(head); +} + + +// Trim trailing '\r' (for CRLF files) and surrounding spaces +static inline void rtrimCrLfWhitespace(std::string& s) { + if (!s.empty() && s.back() == '\r') s.pop_back(); // remove trailing '\r' + while (!s.empty() && std::isspace(static_cast(s.back()))) s.pop_back(); // trailing ws + size_t i = 0; + while (i < s.size() && std::isspace(static_cast(s[i]))) ++i; // leading ws + if (i > 0) s.erase(0, i); +} + +// Case-insensitive substring search: returns true if 'needle' is found in 'hay' +static bool containsCaseInsensitive(const std::string& hay, const std::string& needle) { + auto toLower = [](std::string v) { + std::transform(v.begin(), v.end(), v.begin(), + [](unsigned char c){ return static_cast(std::tolower(c)); }); + return v; + }; + std::string hayLower = toLower(hay); + std::string needleLower = toLower(needle); + return hayLower.find(needleLower) != std::string::npos; +} + +// Read at most the first 3 lines of a file, with a per-line cap to avoid huge reads. +// Returns true if successful (even if <3 lines exist; vector will just be shorter). +static bool readFirstThreeLines(const fs::path& p, std::vector& outLines) { + outLines.clear(); + std::ifstream in(p, std::ios::in | std::ios::binary); + if (!in) return false; + + constexpr std::streamsize kMaxPerLine = 8192; + + std::string line; + line.reserve(static_cast(kMaxPerLine)); + for (int i = 0; i < 3 && in.good(); ++i) { + line.clear(); + std::streamsize count = 0; + char ch; + bool gotNewline = false; + while (count < kMaxPerLine && in.get(ch)) { + if (ch == '\n') { gotNewline = true; break; } + line.push_back(ch); + ++count; + } + // If we hit kMaxPerLine without encountering '\n', drain until newline to resync + if (count == kMaxPerLine && !gotNewline) { + while (in.get(ch)) { + if (ch == '\n') break; + } + } + + if (!in && line.empty()) { + // EOF with no data accumulated; if previous lines were read, that's fine. + break; + } + rtrimCrLfWhitespace(line); + outLines.push_back(line); + if (!in) break; // Handle EOF gracefully + } + return true; +} + +// Check if the first 3 lines contain required keywords in positional order: +// line1 -> "version", line2 -> "oid", line3 -> "size" (case-insensitive). +static bool fileHasLfsKeywordsFirst3Positional(const fs::path& p) { + std::error_code ec; + if (!fs::is_regular_file(p, ec)) return false; + + std::vector lines; + if (!readFirstThreeLines(p, lines)) return false; + + if (lines.size() < 3) return false; + + return containsCaseInsensitive(lines[0], "version") && + containsCaseInsensitive(lines[1], "oid") && + containsCaseInsensitive(lines[2], "size"); +} + + +// Helper: make path relative to base (best-effort, non-throwing). +static fs::path makeRelativeToBase(const fs::path& path, const fs::path& base) { + std::error_code ec; + // Try fs::relative first (handles canonical comparisons, may fail if on different roots) + fs::path rel = fs::relative(path, base, ec); + if (!ec && !rel.empty()) return rel; + + // Fallback: purely lexical relative (doesn't access filesystem) + rel = path.lexically_relative(base); + if (!rel.empty()) return rel; + + // Last resort: return filename only (better than absolute when nothing else works) + if (path.has_filename()) return path.filename(); + return path; +} + +// Find all files under 'directory' that satisfy the first-3-lines LFS keyword check. +std::vector findLfsLikeFiles(const std::string& directory, bool recursive = true) { + std::vector matches; + std::error_code ec; + + if (!fs::exists(directory, ec) || !fs::is_directory(directory, ec)) { + return matches; + } + + if (recursive) { + for (fs::recursive_directory_iterator it(directory, ec), end; !ec && it != end; ++it) { + const auto& p = it->path(); + if (fileHasLfsKeywordsFirst3Positional(p)) { + matches.push_back(makeRelativeToBase(p, directory)); + } + } + } else { + for (fs::directory_iterator it(directory, ec), end; !ec && it != end; ++it) { + const auto& p = it->path(); + if (fileHasLfsKeywordsFirst3Positional(p)) { + matches.push_back(makeRelativeToBase(p, directory)); + } + } + } + return matches; +} + int HfDownloader::CheckRepositoryForResume() { git_repository *repo = NULL; int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); @@ -357,6 +691,107 @@ int HfDownloader::CheckRepositoryForResume() { return error; } + +/* + * checkout_one_file: Check out a single path from a treeish (commit/ref) + * into the working directory, applying filters (smudge, EOL) just like clone. + * + * repo_path : filesystem path to the existing (non-bare) repository + * treeish : e.g., "HEAD", "origin/main", a full commit SHA, etc. + * path_in_repo : repo-relative path (e.g., "src/main.c") + * + * Returns 0 on success; <0 (libgit2 error code) on failure. + */ +void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume, const std::string& repositoryPath) { + int error = 0; + + // Remove existing lfs pointer file from repository + std::string fullPath = FileSystem::joinPath({repositoryPath, fileToResume.string()}); + std::filesystem::path filePath(fullPath); + if (!std::filesystem::remove(filePath)) { + SPDLOG_ERROR("Removing lfs file pointer error {}", fullPath); + return; + } + + const char *path_in_repo = fileToResume.string().c_str(); + const char *treeish = "HEAD"; + git_object *target = NULL; + git_strarray paths = {0}; + git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT; + opts.disable_filters = 0; // default; ensure you are NOT disabling filters + + if (git_repository_is_bare(repo)) { + SPDLOG_ERROR("Repository is bare; cannot checkout to working directory {}", fileToResume.string()); + error = GIT_EBAREREPO; + goto done; + } + + if ((error = git_revparse_single(&target, repo, treeish)) < 0) { + SPDLOG_ERROR("git_revparse_single failed {}", fileToResume.string()); + goto done; + } + + // Restrict checkout to a single path + paths.count = 1; + paths.strings = (char **)&path_in_repo; + + opts.paths = paths; + + // Strategy: SAFER defaults — apply filters, write new files, update existing file + // You can add GIT_CHECKOUT_FORCE if you want to overwrite conflicts. + // opts.checkout_strategy = GIT_CHECKOUT_SAFE | GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH; + opts.checkout_strategy = GIT_CHECKOUT_FORCE; // This makes sure BLOB update with filter is called + // This actually writes the filtered content to the working directory + error = git_checkout_tree(repo, target, &opts); + if (error < 0) { + SPDLOG_ERROR("git_checkout_tree failed {}", fileToResume.string()); + } + +done: + if (target) git_object_free(target); + return; +} + +/* Does not work +void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume) { + git_object *obj = NULL; + git_tree *tree = NULL; + git_tree_entry *entry = NULL; + git_blob *blob = NULL; + git_buf out = GIT_BUF_INIT; + // Configure filter behavior + git_blob_filter_options opts = GIT_BLOB_FILTER_OPTIONS_INIT; + // Choose direction: + // GIT_BLOB_FILTER_TO_WORKTREE : apply smudge (as if writing to working tree) + // GIT_BLOB_FILTER_TO_ODB : apply clean (as if writing to ODB) + // opts.flags = GIT_FILTER_TO_WORKTREE; + + const char *file_path_in_repo = fileToResume.string().c_str(); // relative to repo root + + // Resolve HEAD tree + CHECK(git_revparse_single(&obj, repo, "HEAD^{tree}") != 0); + tree = (git_tree *)obj; + + // Find the tree entry and get the blob + CHECK(git_tree_entry_bypath(&entry, tree, file_path_in_repo) != 0); + CHECK(git_tree_entry_type(entry) != GIT_OBJECT_BLOB); + + CHECK(git_blob_lookup(&blob, repo, git_tree_entry_id(entry)) != 0); + + // Apply filters based on .gitattributes for this path + CHECK(git_blob_filter(&out, blob, file_path_in_repo, &opts) != 0); + + // out.ptr now contains the filtered content + fwrite(out.ptr, 1, out.size, stdout); + + git_buf_dispose(&out); + if (blob) git_blob_free(blob); + if (entry) git_tree_entry_free(entry); + if (tree) git_tree_free(tree); + if (obj) git_object_free(obj); + return; +} */ + Status HfDownloader::downloadModel() { if (FileSystem::isPathEscaped(this->downloadPath)) { SPDLOG_ERROR("Path {} escape with .. is forbidden.", this->downloadPath); @@ -365,8 +800,44 @@ Status HfDownloader::downloadModel() { // Repository exists and we do not want to overwrite if (std::filesystem::is_directory(this->downloadPath) && !this->overwriteModels) { - CheckRepositoryForResume(); + auto matches = findLfsLikeFiles(this->downloadPath, true); + + if (matches.empty()) { + std::cout << "No files with LFS-like keywords in the first 3 lines were found.\n"; + } else { + std::cout << "Found " << matches.size() << " matching file(s):\n"; + for (const auto& p : matches) { + std::cout << " " << p.string() << "\n"; + } + } + git_repository *repo = NULL; + int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); + if (error < 0) { + const git_error *err = git_error_last(); + if (err) + SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); + else + SPDLOG_ERROR("Repository open failed: {}", error); + if (repo) git_repository_free(repo); + + std::cout << "Path already exists on local filesystem. And is not a git repository: " << this->downloadPath << std::endl; + return StatusCode::HF_GIT_CLONE_FAILED; + } + + for (const auto& p : matches) { + std::cout << " Resuming " << p.string() << "\n"; + resumeLfsDownloadForFile(repo, p, this->downloadPath); + } + + // Use proxy + if (CheckIfProxySet()) { + SPDLOG_DEBUG("Download using https_proxy settings"); + //pull(repo, "origin", this->httpProxy.c_str()); + } else { + SPDLOG_DEBUG("Download with https_proxy not set"); + pull(repo, "origin", nullptr); + } std::cout << "Path already exists on local filesystem. Skipping download to path: " << this->downloadPath << std::endl; return StatusCode::OK; From 417b95c65c4f28570e609b2df79f6a8fe523ea42 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Tue, 3 Mar 2026 16:12:48 +0100 Subject: [PATCH 07/49] Blob filter works --- src/pull_module/libgit2.cpp | 102 ++++++++++++++++++++++++++++++++---- 1 file changed, 92 insertions(+), 10 deletions(-) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 423320b1af..8b3944e846 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -702,7 +702,7 @@ int HfDownloader::CheckRepositoryForResume() { * * Returns 0 on success; <0 (libgit2 error code) on failure. */ -void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume, const std::string& repositoryPath) { +void resumeLfsDownloadForFile2(git_repository *repo, fs::path fileToResume, const std::string& repositoryPath) { int error = 0; // Remove existing lfs pointer file from repository @@ -714,7 +714,8 @@ void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume, const } const char *path_in_repo = fileToResume.string().c_str(); - const char *treeish = "HEAD"; + // TODO: make sure we are on 'origin/main'. + const char *treeish = "origin/main^{tree}"; git_object *target = NULL; git_strarray paths = {0}; git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT; @@ -740,7 +741,7 @@ void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume, const // Strategy: SAFER defaults — apply filters, write new files, update existing file // You can add GIT_CHECKOUT_FORCE if you want to overwrite conflicts. // opts.checkout_strategy = GIT_CHECKOUT_SAFE | GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH; - opts.checkout_strategy = GIT_CHECKOUT_FORCE; // This makes sure BLOB update with filter is called + opts.checkout_strategy = GIT_CHECKOUT_FORCE | GIT_CHECKOUT_SAFE; // This makes sure BLOB update with filter is called // This actually writes the filtered content to the working directory error = git_checkout_tree(repo, target, &opts); if (error < 0) { @@ -752,8 +753,7 @@ void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume, const return; } -/* Does not work -void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume) { +void resumeLfsDownloadForFile1(git_repository *repo, const char *file_path_in_repo) { git_object *obj = NULL; git_tree *tree = NULL; git_tree_entry *entry = NULL; @@ -766,10 +766,8 @@ void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume) { // GIT_BLOB_FILTER_TO_ODB : apply clean (as if writing to ODB) // opts.flags = GIT_FILTER_TO_WORKTREE; - const char *file_path_in_repo = fileToResume.string().c_str(); // relative to repo root - // Resolve HEAD tree - CHECK(git_revparse_single(&obj, repo, "HEAD^{tree}") != 0); + CHECK(git_revparse_single(&obj, repo, "origin/main^{tree}") != 0); tree = (git_tree *)obj; // Find the tree entry and get the blob @@ -790,7 +788,83 @@ void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume) { if (tree) git_tree_free(tree); if (obj) git_object_free(obj); return; -} */ +} + + +static int on_notify( + git_checkout_notify_t why, const char *path, + const git_diff_file *baseline, const git_diff_file *target, const git_diff_file *workdir, + void *payload) +{ + (void)baseline; (void)target; (void)workdir; (void)payload; + fprintf(stderr, "[checkout notify] why=%u path=%s\n", why, path ? path : "(null)"); + return 0; // non-zero would cancel +} + +void checkout_one_from_origin_master(git_repository *repo, const char *path_rel) { + int err = 0; + git_object *commitobj = NULL; + git_commit *commit = NULL; + git_tree *tree = NULL; + git_tree_entry *te = NULL; + git_diff *diff = NULL; + git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT; + git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT; + + + printf("Path to resume '%s' \n", path_rel); + /* 1) Resolve origin/master to a commit, then get its tree */ + if ((err = git_revparse_single(&commitobj, repo, "refs/remotes/origin/main^{commit}")) < 0) return; + commit = (git_commit *)commitobj; + if ((err = git_commit_tree(&tree, commit)) < 0) return; + + /* 2) Sanity-check: does the path exist in the target tree? */ + if ((err = git_tree_entry_bypath(&te, tree, path_rel)) == GIT_ENOTFOUND) { + fprintf(stderr, "Path '%s' not found in origin/main\n", path_rel); + err = 0; return; // nothing to do + } else if (err < 0) { + return; + } + git_tree_entry_free(te); te = NULL; + + /* 3) Diff target tree -> workdir (with index) for the one path */ + diffopts.pathspec.count = 1; + diffopts.pathspec.strings = (char **)&path_rel; + if ((err = git_diff_tree_to_workdir_with_index(&diff, repo, tree, &diffopts)) < 0) return; + + size_t n = git_diff_num_deltas(diff); + fprintf(stderr, "[pre-checkout] deltas for %s: %zu\n", path_rel, n); + git_diff_free(diff); diff = NULL; + + if (n == 0) { + fprintf(stderr, "No changes to apply for %s (already matches target or not selected)\n", path_rel); + /* fall through: we can still attempt checkout to let planner confirm */ + } + + /* 4) Configure checkout for a single literal path and creation allowed */ + const char *paths[] = { path_rel }; + opts.checkout_strategy = GIT_CHECKOUT_FORCE | GIT_CHECKOUT_SAFE | GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH; // or GIT_CHECKOUT_FORCE to overwrite local edits + opts.paths.strings = (char **)paths; + opts.paths.count = 1; + opts.notify_flags = GIT_CHECKOUT_NOTIFY_ALL; + opts.notify_cb = on_notify; + + /* Optional: ensure baseline reflects current HEAD */ + // git_object *head = NULL; git_tree *head_tree = NULL; + // if (git_revparse_single(&head, repo, "HEAD^{commit}") == 0) { + // git_commit_tree(&head_tree, (git_commit *)head); + // opts.baseline = head_tree; + // } + + /* 5) Only the selected path will be considered; planner will create/update it */ + err = git_checkout_tree(repo, (git_object *)tree, &opts); + + git_tree_free(tree); + git_commit_free(commit); + git_object_free(commitobj); + return; +} + Status HfDownloader::downloadModel() { if (FileSystem::isPathEscaped(this->downloadPath)) { @@ -827,7 +901,15 @@ Status HfDownloader::downloadModel() { for (const auto& p : matches) { std::cout << " Resuming " << p.string() << "\n"; - resumeLfsDownloadForFile(repo, p, this->downloadPath); + // Remove existing lfs pointer file from repository + std::string fullPath = FileSystem::joinPath({this->downloadPath, p.string()}); + std::filesystem::path filePath(fullPath); + if (!std::filesystem::remove(filePath)) { + SPDLOG_ERROR("Removing lfs file pointer error {}", fullPath); + return StatusCode::HF_GIT_CLONE_FAILED; + } + std::string path = p.string(); + resumeLfsDownloadForFile1(repo, path.c_str()); } // Use proxy From 50b2fcf393189059cb2f1df1d597429489764e98 Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 3 Mar 2026 16:14:43 +0100 Subject: [PATCH 08/49] Add resume in libgit2 on existing file --- third_party/libgit2/lfs.patch | 84 ++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 3b2e8a5024..c70f0bbe37 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -312,10 +312,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..7c1111c34 +index 000000000..9551461e7 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,655 @@ +@@ -0,0 +1,665 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -715,6 +715,38 @@ index 000000000..7c1111c34 + status = setopt; \ + } + ++int get_curl_resume_url(CURL *dl_curl, struct FtpFile* ftpfile) ++{ ++ curl_off_t resume_from = 0; ++ curl_easy_getinfo( ++ dl_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &resume_from); ++ ++ if (resume_from == -1) { ++ fprintf(stderr, ++ "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); ++ } else { ++ printf("\n[INFO] curl_easy_perform() trying to resume file download\n"); ++ curl_off_t offset = 0; ++ if (ftpfile->stream) { ++ fseek(ftpfile->stream, 0, SEEK_END); ++ offset = ftell(ftpfile->stream); ++ } else { ++ ftpfile->stream = fopen(ftpfile->filename, "ab+"); ++ if (ftpfile->stream) { ++ fseek(ftpfile->stream, 0, SEEK_END); ++ offset = ftell(ftpfile->stream); ++ } ++ } ++ ++ /* Tell libcurl to resume */ ++ curl_easy_setopt(dl_curl, CURLOPT_RESUME_FROM_LARGE, offset); ++ /* Perform the request, res gets the return code */ ++ return curl_easy_perform(dl_curl); ++ } ++ ++ return resume_from; ++} ++ +/** + * lfs_download - Downloads a file using the LFS (Large File Storage) mechanism. + * @@ -868,43 +900,21 @@ index 000000000..7c1111c34 + curl_easy_cleanup(dl_curl); + goto on_error; + } -+ print_download_info(la->full_path, get_digit(la->lfs_size)); -+ /* Perform the request, res gets the return code */ -+ res = curl_easy_perform(dl_curl); ++ ++ /* Check for resume if previous download failed and we have the partial file on disk */ ++ if (fopen(ftpfile.filename, "r") != NULL) { ++ fclose(ftpfile.filename); ++ res = get_curl_resume_url(dl_curl, &ftpfile); ++ } else { ++ print_download_info( ++ la->full_path, get_digit(la->lfs_size)); ++ /* Perform the request, res gets the return code */ ++ res = curl_easy_perform(dl_curl); ++ } ++ + /* Check for resume of partial download error */ + if (res == CURLE_PARTIAL_FILE) { -+ curl_off_t resume_from = 0; -+ curl_easy_getinfo( -+ dl_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, -+ &resume_from); -+ -+ if (resume_from == -1) { -+ fprintf(stderr, -+ "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); -+ } else { -+ fprintf(stderr, -+ "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and trying to resume\n"); -+ curl_off_t offset = 0; -+ if (ftpfile.stream) { -+ fseek(ftpfile.stream, 0, SEEK_END); -+ offset = ftell(ftpfile.stream); -+ } else { -+ ftpfile.stream = -+ fopen(ftpfile.filename, "ab+"); -+ if (ftpfile.stream) { -+ fseek(ftpfile.stream, 0, -+ SEEK_END); -+ offset = ftell(ftpfile.stream); -+ } -+ } -+ -+ /* Tell libcurl to resume */ -+ curl_easy_setopt( -+ dl_curl, CURLOPT_RESUME_FROM_LARGE, -+ offset); -+ /* Perform the request, res gets the return code */ -+ res = curl_easy_perform(dl_curl); -+ } ++ res = get_curl_resume_url(dl_curl, &ftpfile); + } + + /* Check for errors */ From fdc0309d5a5493e9251dd92f37f41a97c972535d Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 3 Mar 2026 16:33:26 +0100 Subject: [PATCH 09/49] Fix segfault --- third_party/libgit2/lfs.patch | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index c70f0bbe37..7b542c4563 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -312,10 +312,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..9551461e7 +index 000000000..f74666a10 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,665 @@ +@@ -0,0 +1,666 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -952,9 +952,10 @@ index 000000000..9551461e7 + return; + + on_error: ++ fprintf(stderr, "\n[ERROR] LFS download failed for file %s\n", ++ la->full_path); + free(tmp_out_file); + git__free(payload); -+ fprintf(stderr, "\n[ERROR] LFS download failed for file %s\n", la->full_path); + return; +} + From 58f07d0a4118a6b826b10be06592d7fa78ee5601 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Tue, 3 Mar 2026 17:03:29 +0100 Subject: [PATCH 10/49] Find way to set the url --- src/pull_module/libgit2.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 8b3944e846..6953abc5c1 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -899,6 +899,11 @@ Status HfDownloader::downloadModel() { return StatusCode::HF_GIT_CLONE_FAILED; } + // Set repository url + std::string passRepoUrl = GetRepositoryUrlWithPassword(); + const char* url = passRepoUrl.c_str(); + repo->url = url; + for (const auto& p : matches) { std::cout << " Resuming " << p.string() << "\n"; // Remove existing lfs pointer file from repository From acc90984f48590b38730eabd92777b6594a7ecfe Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 4 Mar 2026 09:35:03 +0100 Subject: [PATCH 11/49] Set repositroy url --- third_party/libgit2/lfs.patch | 38 ++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 7b542c4563..52c57b0480 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -24,6 +24,26 @@ index 31da49a88..d61c9735e 100644 if(BUILD_EXAMPLES) add_subdirectory(examples) endif() +diff --git a/include/git2/repository.h b/include/git2/repository.h +index b203576af..26309dd3f 100644 +--- a/include/git2/repository.h ++++ b/include/git2/repository.h +@@ -184,6 +184,15 @@ GIT_EXTERN(int) git_repository_open_ext( + unsigned int flags, + const char *ceiling_dirs); + ++/** ++ * Set repository url member ++ * ++ * ++ * @param repo repository handle to update. If NULL nothing occurs. ++ * @param url the remote repository to clone or run checkout against. ++ */ ++GIT_EXTERN(int) git_repository_set_url(git_repository *repo, const char *url); ++ + /** + * Open a bare repository on the serverside. + * diff --git a/include/git2/sys/filter.h b/include/git2/sys/filter.h index 60466d173..a35ad5f98 100644 --- a/include/git2/sys/filter.h @@ -983,7 +1003,7 @@ index 000000000..f74666a10 + return f; +} diff --git a/src/libgit2/repository.c b/src/libgit2/repository.c -index 73876424a..6c267bc98 100644 +index 73876424a..6c9bd0b75 100644 --- a/src/libgit2/repository.c +++ b/src/libgit2/repository.c @@ -190,6 +190,7 @@ void git_repository_free(git_repository *repo) @@ -994,6 +1014,22 @@ index 73876424a..6c267bc98 100644 git__memzero(repo, sizeof(*repo)); git__free(repo); +@@ -1104,6 +1105,15 @@ static int repo_is_worktree(unsigned *out, const git_repository *repo) + return error; + } + ++int git_repository_set_url( ++ git_repository *repo, ++ const char *url) ++{ ++ GIT_ASSERT_ARG(repo); ++ GIT_ASSERT_ARG(url); ++ repo->url = git__strdup(url); ++} ++ + int git_repository_open_ext( + git_repository **repo_ptr, + const char *start_path, diff --git a/src/libgit2/repository.h b/src/libgit2/repository.h index fbf143894..1890c61c1 100644 --- a/src/libgit2/repository.h From ee4462d4125a4c459a160fcddbe80fb7a701ca0f Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 4 Mar 2026 09:37:29 +0100 Subject: [PATCH 12/49] Update --- third_party/libgit2/lfs.patch | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 52c57b0480..de948749b5 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -1003,7 +1003,7 @@ index 000000000..f74666a10 + return f; +} diff --git a/src/libgit2/repository.c b/src/libgit2/repository.c -index 73876424a..6c9bd0b75 100644 +index 73876424a..f374d7f51 100644 --- a/src/libgit2/repository.c +++ b/src/libgit2/repository.c @@ -190,6 +190,7 @@ void git_repository_free(git_repository *repo) @@ -1014,7 +1014,7 @@ index 73876424a..6c9bd0b75 100644 git__memzero(repo, sizeof(*repo)); git__free(repo); -@@ -1104,6 +1105,15 @@ static int repo_is_worktree(unsigned *out, const git_repository *repo) +@@ -1104,6 +1105,16 @@ static int repo_is_worktree(unsigned *out, const git_repository *repo) return error; } @@ -1025,6 +1025,7 @@ index 73876424a..6c9bd0b75 100644 + GIT_ASSERT_ARG(repo); + GIT_ASSERT_ARG(url); + repo->url = git__strdup(url); ++ return 0; +} + int git_repository_open_ext( From 553458ea31e13e81970e2c377560df6bb19b9794 Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 4 Mar 2026 09:48:00 +0100 Subject: [PATCH 13/49] Fix file --- third_party/libgit2/lfs.patch | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index de948749b5..79159dcf1e 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -332,10 +332,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..f74666a10 +index 000000000..cc540d0c7 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,666 @@ +@@ -0,0 +1,669 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -922,8 +922,10 @@ index 000000000..f74666a10 + } + + /* Check for resume if previous download failed and we have the partial file on disk */ -+ if (fopen(ftpfile.filename, "r") != NULL) { -+ fclose(ftpfile.filename); ++ ftpfile.stream = fopen(ftpfile.filename, "r"); ++ if (ftpfile.stream != NULL) ++ { ++ fclose(ftpfile.stream); + res = get_curl_resume_url(dl_curl, &ftpfile); + } else { + print_download_info( @@ -974,6 +976,7 @@ index 000000000..f74666a10 + on_error: + fprintf(stderr, "\n[ERROR] LFS download failed for file %s\n", + la->full_path); ++ fflush(stderr); + free(tmp_out_file); + git__free(payload); + return; From e9b08a7bbf7a246c32e4d758ae5aec6ddf9e8db9 Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 4 Mar 2026 10:29:17 +0100 Subject: [PATCH 14/49] Force resume --- third_party/libgit2/lfs.patch | 55 ++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 79159dcf1e..c4b6723d4b 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -332,10 +332,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..cc540d0c7 +index 000000000..f76f80a73 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,669 @@ +@@ -0,0 +1,682 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -737,6 +737,7 @@ index 000000000..cc540d0c7 + +int get_curl_resume_url(CURL *dl_curl, struct FtpFile* ftpfile) +{ ++ /* + curl_off_t resume_from = 0; + curl_easy_getinfo( + dl_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &resume_from); @@ -745,26 +746,24 @@ index 000000000..cc540d0c7 + fprintf(stderr, + "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); + } else { -+ printf("\n[INFO] curl_easy_perform() trying to resume file download\n"); -+ curl_off_t offset = 0; ++ */ ++ printf("\n[INFO] curl_easy_perform() trying to resume file download\n"); ++ curl_off_t offset = 0; ++ if (ftpfile->stream) { ++ fseek(ftpfile->stream, 0, SEEK_END); ++ offset = ftell(ftpfile->stream); ++ } else { ++ ftpfile->stream = fopen(ftpfile->filename, "ab+"); + if (ftpfile->stream) { + fseek(ftpfile->stream, 0, SEEK_END); + offset = ftell(ftpfile->stream); -+ } else { -+ ftpfile->stream = fopen(ftpfile->filename, "ab+"); -+ if (ftpfile->stream) { -+ fseek(ftpfile->stream, 0, SEEK_END); -+ offset = ftell(ftpfile->stream); -+ } + } -+ -+ /* Tell libcurl to resume */ -+ curl_easy_setopt(dl_curl, CURLOPT_RESUME_FROM_LARGE, offset); -+ /* Perform the request, res gets the return code */ -+ return curl_easy_perform(dl_curl); + } + -+ return resume_from; ++ /* Tell libcurl to resume */ ++ curl_easy_setopt(dl_curl, CURLOPT_RESUME_FROM_LARGE, offset); ++ /* Perform the request, res gets the return code */ ++ return curl_easy_perform(dl_curl); +} + +/** @@ -921,11 +920,24 @@ index 000000000..cc540d0c7 + goto on_error; + } + ++ curl_off_t resume_from = 0; ++ curl_easy_getinfo( ++ dl_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, ++ &resume_from); ++ ++ if (resume_from == -1) { ++ fprintf(stderr, ++ "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); ++ } else { ++ printf("\n[INFO] curl_easy_perform() trying to resume file download\n"); ++ } ++ + /* Check for resume if previous download failed and we have the partial file on disk */ + ftpfile.stream = fopen(ftpfile.filename, "r"); + if (ftpfile.stream != NULL) + { + fclose(ftpfile.stream); ++ ftpfile.stream = NULL; + res = get_curl_resume_url(dl_curl, &ftpfile); + } else { + print_download_info( @@ -943,18 +955,19 @@ index 000000000..cc540d0c7 + if (res != CURLE_OK) { + fprintf(stderr, "\n[ERROR] curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); -+ if (ftpfile.stream) ++ if (ftpfile.stream) { + fclose(ftpfile.stream); ++ ftpfile.stream = NULL; ++ } + /* always cleanup */ + curl_easy_cleanup(dl_curl); -+ -+ /* Add partial file status */ -+ + goto on_error; + } + -+ if (ftpfile.stream) ++ if (ftpfile.stream) { + fclose(ftpfile.stream); ++ ftpfile.stream = NULL; ++ } + /* always cleanup */ + curl_easy_cleanup(dl_curl); + } From 10780320d36434e19a45edcb1bb63aae0dfcffaa Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 5 Mar 2026 12:46:51 +0100 Subject: [PATCH 15/49] Unit tests for resume --- src/pull_module/libgit2.cpp | 12 ++++- src/test/pull_hf_model_test.cpp | 80 +++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 6953abc5c1..0523c06df0 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -902,7 +902,17 @@ Status HfDownloader::downloadModel() { // Set repository url std::string passRepoUrl = GetRepositoryUrlWithPassword(); const char* url = passRepoUrl.c_str(); - repo->url = url; + error = git_repository_set_url(repo, url); + if (error < 0) { + const git_error *err = git_error_last(); + if (err) + SPDLOG_ERROR("Repository set url failed: {} {}", err->klass, err->message); + else + SPDLOG_ERROR("Repository set url failed: {}", error); + if (repo) git_repository_free(repo); + std::cout << "Path already exists on local filesystem. And set git repository url failed: " << this->downloadPath << std::endl; + return StatusCode::HF_GIT_CLONE_FAILED; + } for (const auto& p : matches) { std::cout << " Resuming " << p.string() << "\n"; diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index b29bbee326..d5b368dba9 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -39,6 +39,8 @@ #include "environment.hpp" +namespace fs = std::filesystem; + class HfDownloaderPullHfModel : public TestWithTempDir { protected: ovms::Server& server = ovms::Server::instance(); @@ -168,6 +170,84 @@ TEST_F(HfDownloaderPullHfModel, PositiveDownload) { ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; } +// Truncate the file to half its size, keeping the first half. +bool removeSecondHalf(const std::string& filrStr) { + const fs::path& file(filrStr); + std::error_code ec; + ec.clear(); + + if (!fs::exists(file, ec) || !fs::is_regular_file(file, ec)) { + if (!ec) ec = std::make_error_code(std::errc::no_such_file_or_directory); + return false; + } + + const std::uintmax_t size = fs::file_size(file, ec); + if (ec) return false; + + const std::uintmax_t newSize = size / 2; // floor(size/2) + fs::resize_file(file, newSize, ec); + return !ec; +} + +bool createGitLfsPointerFile(const std::string& path) { + std::ofstream file(path, std::ios::binary); + if (!file.is_open()) { + return false; + } + + file << + "version https://git-lfs.github.com/spec/v1\n" + "oid sha256:59f24bc922e1a48bb3feeba18b23f0e9622a7ee07166d925650d7a933283f8b1\n" + "size 123882252\n"; + + return true; +} + +TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndResumeFromPArtialDownload) { + std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; + std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); + std::string task = "text_generation"; + this->ServerPullHfModel(modelName, downloadPath, task); + server.setShutdownRequest(1); + if (t) + t->join(); + server.setShutdownRequest(0); + + std::string ovModelName = "openvino_model.bin"; + std::string basePath = ovms::FileSystem::joinPath({this->directoryPath, "repository", "OpenVINO", "Phi-3-mini-FastDraft-50M-int8-ov"}); + std::string modelPath = ovms::FileSystem::appendSlash(basePath) + ovModelName; + std::string graphPath = ovms::FileSystem::appendSlash(basePath) + "graph.pbtxt"; + + ASSERT_EQ(std::filesystem::exists(modelPath), true) << modelPath; + ASSERT_EQ(std::filesystem::exists(graphPath), true) << graphPath; + ASSERT_EQ(std::filesystem::file_size(modelPath), 52417240); + std::string graphContents = GetFileContents(graphPath); + + ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; + + // Prepare a git repository with a lfs_part file and lfs pointer file to simulate partial download error of a big model + ASSERT_EQ(removeSecondHalf(modelPath), true); + ASSERT_EQ(std::filesystem::file_size(modelPath), 26208620); + std::error_code ec; + ec.clear(); + std::string ovModelPartLfsName = "openvino_model.binlfs_part"; + std::string ovModelPartLfsPath = ovms::FileSystem::appendSlash(basePath) + ovModelPartLfsName; + fs::rename(modelPath, ovModelPartLfsPath, ec); + ASSERT_EQ(ec, std::errc()); + ASSERT_EQ(std::filesystem::file_size(ovModelPartLfsPath), 26208620); + ASSERT_EQ(createGitLfsPointerFile(modelPath), true); + + // Call ovms pull to resume the file + this->ServerPullHfModel(modelName, downloadPath, task); + + ASSERT_EQ(std::filesystem::exists(modelPath), true) << modelPath; + ASSERT_EQ(std::filesystem::exists(graphPath), true) << graphPath; + ASSERT_EQ(std::filesystem::file_size(modelPath), 52417240); + graphContents = GetFileContents(graphPath); + + ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; +} + TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndStart) { SKIP_AND_EXIT_IF_NOT_RUNNING_UNSTABLE(); // CVS-180127 // EnvGuard guard; From 7d4498198238cf667fda921eb11156b28595b4d0 Mon Sep 17 00:00:00 2001 From: rasapala Date: Thu, 5 Mar 2026 12:47:29 +0100 Subject: [PATCH 16/49] Fix rename --- third_party/libgit2/lfs.patch | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index c4b6723d4b..0197a68564 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -332,10 +332,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..f76f80a73 +index 000000000..f877c4afa --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,682 @@ +@@ -0,0 +1,677 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -887,6 +887,7 @@ index 000000000..f76f80a73 + } + + /* get a curl handle */ ++ bool resumingFileByBlobFilter = false; + dl_curl = curl_easy_init(); + if (dl_curl) { + struct FtpFile ftpfile = { tmp_out_file, NULL }; @@ -920,22 +921,11 @@ index 000000000..f76f80a73 + goto on_error; + } + -+ curl_off_t resume_from = 0; -+ curl_easy_getinfo( -+ dl_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, -+ &resume_from); -+ -+ if (resume_from == -1) { -+ fprintf(stderr, -+ "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); -+ } else { -+ printf("\n[INFO] curl_easy_perform() trying to resume file download\n"); -+ } -+ + /* Check for resume if previous download failed and we have the partial file on disk */ + ftpfile.stream = fopen(ftpfile.filename, "r"); + if (ftpfile.stream != NULL) + { ++ resumingFileByBlobFilter = true; + fclose(ftpfile.stream); + ftpfile.stream = NULL; + res = get_curl_resume_url(dl_curl, &ftpfile); @@ -972,10 +962,15 @@ index 000000000..f76f80a73 + curl_easy_cleanup(dl_curl); + } + -+ /* Remove lfs file and rename downloaded file to oryginal lfs filename */ -+ if (p_unlink(la->full_path) < 0) { -+ fprintf(stderr, "\n[ERROR] failed to delete file '%s'\n", la->full_path); -+ goto on_error; ++ /* Remove lfs file and rename downloaded file to oryginal lfs filename */ ++ if (!resumingFileByBlobFilter) { ++ /* File does not exist when using blob filters */ ++ if (p_unlink(la->full_path) < 0) { ++ fprintf(stderr, ++ "\n[ERROR] failed to delete file '%s'\n", ++ la->full_path); ++ goto on_error; ++ } + } + + if (p_rename(tmp_out_file, la->full_path) < 0) { From 5aae45443410f167f7896d10f604a828701a1f47 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Mon, 9 Mar 2026 14:59:13 +0100 Subject: [PATCH 17/49] Unit test --- src/pull_module/libgit2.cpp | 485 ++------------------------------ src/pull_module/libgit2.hpp | 2 +- src/test/pull_hf_model_test.cpp | 67 ++++- 3 files changed, 80 insertions(+), 474 deletions(-) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 0523c06df0..570ecd08fe 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -180,7 +180,7 @@ Status HfDownloader::RemoveReadonlyFileAttributeFromDir(const std::string& direc return StatusCode::OK; } -Status HfDownloader::CheckRepositoryStatus() { +Status HfDownloader::CheckRepositoryStatus(bool checkUntracked) { git_repository *repo = NULL; int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); if (error < 0) { @@ -253,84 +253,13 @@ Status HfDownloader::CheckRepositoryStatus() { SPDLOG_DEBUG(ss.str()); git_status_list_free(status_list); - if (is_unborn || is_detached || staged || unstaged || untracked || conflicted) { + // We do not care about untracked until after git clone + if (is_unborn || is_detached || staged || unstaged || conflicted || (checkUntracked && untracked)) { return StatusCode::HF_GIT_STATUS_UNCLEAN; } return StatusCode::OK; } -static int print_changed_and_untracked(git_repository *repo) { - int error = 0; - git_status_list *statuslist = NULL; - - git_status_options opts; - error = git_status_options_init(&opts, GIT_STATUS_OPTIONS_VERSION); - if (error < 0) return error; - - // Choose what to include - opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR; // consider both index and working dir - opts.flags = - GIT_STATUS_OPT_INCLUDE_UNTRACKED | // include untracked files - GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS | // recurse into untracked dirs - GIT_STATUS_OPT_INCLUDE_IGNORED | // (optional) include ignored if you want to see them - GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX | // detect renames in index - GIT_STATUS_OPT_RENAMES_INDEX_TO_WORKDIR | // detect renames in workdir - GIT_STATUS_OPT_SORT_CASE_SENSITIVELY; // stable ordering - - // If you want to limit to certain paths/patterns, set opts.pathspec here. - - if ((error = git_status_list_new(&statuslist, repo, &opts)) < 0) - return error; - - size_t count = git_status_list_entrycount(statuslist); - for (size_t i = 0; i < count; i++) { - const git_status_entry *e = git_status_byindex(statuslist, i); - if (!e) continue; - - unsigned int s = e->status; - - // Consider “changed” as anything that’s not current in HEAD/INDEX/WT: - // You can tailor this to your exact definition. - int is_untracked = - (s & GIT_STATUS_WT_NEW) != 0; // working tree new (untracked) - int is_workdir_changed = - (s & (GIT_STATUS_WT_MODIFIED | - GIT_STATUS_WT_DELETED | - GIT_STATUS_WT_RENAMED | - GIT_STATUS_WT_TYPECHANGE)) != 0; - int is_index_changed = - (s & (GIT_STATUS_INDEX_NEW | - GIT_STATUS_INDEX_MODIFIED | - GIT_STATUS_INDEX_DELETED | - GIT_STATUS_INDEX_RENAMED | - GIT_STATUS_INDEX_TYPECHANGE)) != 0; - - if (!(is_untracked || is_workdir_changed || is_index_changed)) - continue; - - // Prefer the most relevant delta for the path - const git_diff_delta *delta = NULL; - if (is_workdir_changed && e->index_to_workdir) - delta = e->index_to_workdir; - else if (is_index_changed && e->head_to_index) - delta = e->head_to_index; - else if (is_untracked && e->index_to_workdir) - delta = e->index_to_workdir; - - if (!delta) continue; - - // For renames, old_file and new_file may differ; typically you want new_file.path - const char *path = delta->new_file.path ? delta->new_file.path - : delta->old_file.path; - - // Print or collect the filename - SPDLOG_INFO("is_untracked {} is_workdir_changed {} is_index_changed {} File {} ", is_untracked, is_workdir_changed, is_index_changed, path); - } - - git_status_list_free(statuslist); - return 0; -} - #define CHECK(call) do { \ int _err = (call); \ if (_err < 0) { \ @@ -340,210 +269,6 @@ static int print_changed_and_untracked(git_repository *repo) { } \ } while (0) -// Fetch from remote and update FETCH_HEAD -static void do_fetch(git_repository *repo, const char *remote_name, const char *proxy) -{ - git_remote *remote = NULL; - git_fetch_options fetch_opts = GIT_FETCH_OPTIONS_INIT; - - fetch_opts.prune = GIT_FETCH_PRUNE_UNSPECIFIED; - fetch_opts.update_fetchhead = 1; - fetch_opts.download_tags = GIT_REMOTE_DOWNLOAD_TAGS_ALL; - fetch_opts.callbacks = (git_remote_callbacks) GIT_REMOTE_CALLBACKS_INIT; - fetch_opts.callbacks.credentials = cred_acquire_cb; - if (proxy) { - fetch_opts.proxy_opts.type = GIT_PROXY_SPECIFIED; - fetch_opts.proxy_opts.url = proxy; - } - - CHECK(git_remote_lookup(&remote, repo, remote_name)); - - printf("Fetching from %s...\n", remote_name); - CHECK(git_remote_fetch(remote, NULL, &fetch_opts, NULL)); - - // Optional: update remote-tracking branches' default refspec tips - // (git_remote_fetch already updates tips if update_fetchhead=1; explicit - // update_tips is not required in recent libgit2 versions.) - - git_remote_free(remote); -} - -// Fast-forward the local branch to target OID -static void do_fast_forward(git_repository *repo, - git_reference *local_branch, - const git_oid *target_oid) -{ - git_object *target = NULL; - git_checkout_options co_opts = GIT_CHECKOUT_OPTIONS_INIT; - git_reference *updated_ref = NULL; - - CHECK(git_object_lookup(&target, repo, target_oid, GIT_OBJECT_COMMIT)); - - // Update the branch reference to point to the target commit - CHECK(git_reference_set_target(&updated_ref, local_branch, target_oid, "Fast-forward")); - - // Make sure HEAD points to that branch (it normally already does) - CHECK(git_repository_set_head(repo, git_reference_name(updated_ref))); - - // Checkout files to match the target tree - co_opts.checkout_strategy = GIT_CHECKOUT_SAFE; - CHECK(git_checkout_tree(repo, target, &co_opts)); - - printf("Fast-forwarded %s to %s\n", - git_reference_shorthand(local_branch), - git_oid_tostr_s(target_oid)); - - git_object_free(target); - git_reference_free(updated_ref); -} - -// Perform a normal merge and create a merge commit if no conflicts -static void do_normal_merge(git_repository *repo, - git_reference *local_branch, - git_annotated_commit *their_head) -{ - git_merge_options merge_opts = GIT_MERGE_OPTIONS_INIT; - git_checkout_options co_opts = GIT_CHECKOUT_OPTIONS_INIT; - - merge_opts.file_favor = GIT_MERGE_FILE_FAVOR_NORMAL; - co_opts.checkout_strategy = GIT_CHECKOUT_SAFE | GIT_CHECKOUT_RECREATE_MISSING; - - const git_annotated_commit *their_heads[1] = { their_head }; - - printf("Merging...\n"); - CHECK(git_merge(repo, their_heads, 1, &merge_opts, &co_opts)); - - // Check for conflicts - git_index *index = NULL; - CHECK(git_repository_index(&index, repo)); - - if (git_index_has_conflicts(index)) { - printf("Merge has conflicts. Please resolve them and create the merge commit manually.\n"); - git_index_free(index); - return; // Leave repository in merging state - } - - // Write index to tree - git_oid tree_oid; - CHECK(git_index_write_tree(&tree_oid, index)); - CHECK(git_index_write(index)); - git_index_free(index); - - git_tree *tree = NULL; - CHECK(git_tree_lookup(&tree, repo, &tree_oid)); - - // Prepare signature (from config if available) - git_signature *sig = NULL; - int err = git_signature_default(&sig, repo); - if (err == GIT_ENOTFOUND || sig == NULL) { - // Fallback if user.name/email not set in config - CHECK(git_signature_now(&sig, "Your Name", "you@example.com")); - } else { - CHECK(err); - } - - // Get current HEAD (our) commit and their commit to be parents - git_reference *head = NULL, *resolved_branch = NULL; - CHECK(git_repository_head(&head, repo)); - CHECK(git_reference_resolve(&resolved_branch, head)); // ensure direct ref - - const git_oid *our_oid = git_reference_target(resolved_branch); - git_commit *our_commit = NULL; - git_commit *their_commit = NULL; - CHECK(git_commit_lookup(&our_commit, repo, our_oid)); - CHECK(git_commit_lookup(&their_commit, repo, git_annotated_commit_id(their_head))); - - const git_commit *parents[2] = { our_commit, their_commit }; - git_oid merge_commit_oid; - - // Create merge commit on the current branch ref - CHECK(git_commit_create(&merge_commit_oid, - repo, - git_reference_name(resolved_branch), - sig, sig, - NULL /* message_encoding */, - "Merge remote-tracking branch", - tree, - 2, parents)); - - printf("Created merge commit %s on %s\n", - git_oid_tostr_s(&merge_commit_oid), - git_reference_shorthand(resolved_branch)); - - // Cleanup - git_signature_free(sig); - git_tree_free(tree); - git_commit_free(our_commit); - git_commit_free(their_commit); - git_reference_free(head); - git_reference_free(resolved_branch); -} - -// Main pull routine: fetch + merge (fast-forward if possible) -static void pull(git_repository *repo, const char *remote_name, const char *proxy) -{ - // Ensure we are on a branch (not detached HEAD) - git_reference *head = NULL; - int head_res = git_repository_head(&head, repo); - // HEAD state info - bool is_detached = git_repository_head_detached(repo) == 1; - bool is_unborn = git_repository_head_unborn(repo) == 1; - if (is_unborn) { - fprintf(stderr, "Repository has no HEAD yet (unborn branch).\n"); - return; - } else if (is_detached) { - fprintf(stderr, "HEAD is detached; cannot pull safely. Checkout a branch first.\n"); - return; - } - CHECK(head_res); - - // Resolve symbolic HEAD to direct branch ref (refs/heads/…) - git_reference *local_branch = NULL; - CHECK(git_reference_resolve(&local_branch, head)); - - // Find the upstream tracking branch (refs/remotes//) - git_reference *upstream = NULL; - int up_ok = git_branch_upstream(&upstream, local_branch); - if (up_ok != 0 || upstream == NULL) { - fprintf(stderr, "Current branch has no upstream. Set it with:\n" - " git branch --set-upstream-to=%s/ \n", remote_name); - return; - } - - // Verify upstream belongs to the requested remote; not strictly required for fetch - // but we fetch from the chosen remote anyway. - do_fetch(repo, remote_name, proxy); - - // Prepare "their" commit as annotated commit from upstream - git_annotated_commit *their_head = NULL; - CHECK(git_annotated_commit_from_ref(&their_head, repo, upstream)); - - // Merge analysis - git_merge_analysis_t analysis; - git_merge_preference_t preference; - CHECK(git_merge_analysis(&analysis, &preference, repo, - (const git_annotated_commit **)&their_head, 1)); - - if (analysis & GIT_MERGE_ANALYSIS_UP_TO_DATE) { - printf("Already up to date.\n"); - } else if (analysis & GIT_MERGE_ANALYSIS_FASTFORWARD) { - const git_oid *target_oid = git_annotated_commit_id(their_head); - do_fast_forward(repo, local_branch, target_oid); - } else if (analysis & GIT_MERGE_ANALYSIS_NORMAL) { - do_normal_merge(repo, local_branch, their_head); - } else { - printf("No merge action taken (analysis=%u, preference=%u).\n", - (unsigned)analysis, (unsigned)preference); - } - - // Cleanup - git_annotated_commit_free(their_head); - git_reference_free(upstream); - git_reference_free(local_branch); - git_reference_free(head); -} - - // Trim trailing '\r' (for CRLF files) and surrounding spaces static inline void rtrimCrLfWhitespace(std::string& s) { if (!s.empty() && s.back() == '\r') s.pop_back(); // remove trailing '\r' @@ -664,96 +389,7 @@ std::vector findLfsLikeFiles(const std::string& directory, bool recurs return matches; } -int HfDownloader::CheckRepositoryForResume() { - git_repository *repo = NULL; - int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); - if (error < 0) { - const git_error *err = git_error_last(); - if (err) - SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); - else - SPDLOG_ERROR("Repository open failed: {}", error); - if (repo) git_repository_free(repo); - - return error; - } - - error = print_changed_and_untracked(repo); - if (error < 0) { - const git_error *err = git_error_last(); - if (err) - SPDLOG_ERROR("Print changed files failed: {} {}", err->klass, err->message); - else - SPDLOG_ERROR("Print changed files failed: {}", error); - } - - if (repo) git_repository_free(repo); - return error; -} - - -/* - * checkout_one_file: Check out a single path from a treeish (commit/ref) - * into the working directory, applying filters (smudge, EOL) just like clone. - * - * repo_path : filesystem path to the existing (non-bare) repository - * treeish : e.g., "HEAD", "origin/main", a full commit SHA, etc. - * path_in_repo : repo-relative path (e.g., "src/main.c") - * - * Returns 0 on success; <0 (libgit2 error code) on failure. - */ -void resumeLfsDownloadForFile2(git_repository *repo, fs::path fileToResume, const std::string& repositoryPath) { - int error = 0; - - // Remove existing lfs pointer file from repository - std::string fullPath = FileSystem::joinPath({repositoryPath, fileToResume.string()}); - std::filesystem::path filePath(fullPath); - if (!std::filesystem::remove(filePath)) { - SPDLOG_ERROR("Removing lfs file pointer error {}", fullPath); - return; - } - - const char *path_in_repo = fileToResume.string().c_str(); - // TODO: make sure we are on 'origin/main'. - const char *treeish = "origin/main^{tree}"; - git_object *target = NULL; - git_strarray paths = {0}; - git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT; - opts.disable_filters = 0; // default; ensure you are NOT disabling filters - - if (git_repository_is_bare(repo)) { - SPDLOG_ERROR("Repository is bare; cannot checkout to working directory {}", fileToResume.string()); - error = GIT_EBAREREPO; - goto done; - } - - if ((error = git_revparse_single(&target, repo, treeish)) < 0) { - SPDLOG_ERROR("git_revparse_single failed {}", fileToResume.string()); - goto done; - } - - // Restrict checkout to a single path - paths.count = 1; - paths.strings = (char **)&path_in_repo; - - opts.paths = paths; - - // Strategy: SAFER defaults — apply filters, write new files, update existing file - // You can add GIT_CHECKOUT_FORCE if you want to overwrite conflicts. - // opts.checkout_strategy = GIT_CHECKOUT_SAFE | GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH; - opts.checkout_strategy = GIT_CHECKOUT_FORCE | GIT_CHECKOUT_SAFE; // This makes sure BLOB update with filter is called - // This actually writes the filtered content to the working directory - error = git_checkout_tree(repo, target, &opts); - if (error < 0) { - SPDLOG_ERROR("git_checkout_tree failed {}", fileToResume.string()); - } - -done: - if (target) git_object_free(target); - return; -} - -void resumeLfsDownloadForFile1(git_repository *repo, const char *file_path_in_repo) { +void resumeLfsDownloadForFile(git_repository *repo, const char *file_path_in_repo) { git_object *obj = NULL; git_tree *tree = NULL; git_tree_entry *entry = NULL; @@ -779,9 +415,6 @@ void resumeLfsDownloadForFile1(git_repository *repo, const char *file_path_in_re // Apply filters based on .gitattributes for this path CHECK(git_blob_filter(&out, blob, file_path_in_repo, &opts) != 0); - // out.ptr now contains the filtered content - fwrite(out.ptr, 1, out.size, stdout); - git_buf_dispose(&out); if (blob) git_blob_free(blob); if (entry) git_tree_entry_free(entry); @@ -790,82 +423,6 @@ void resumeLfsDownloadForFile1(git_repository *repo, const char *file_path_in_re return; } - -static int on_notify( - git_checkout_notify_t why, const char *path, - const git_diff_file *baseline, const git_diff_file *target, const git_diff_file *workdir, - void *payload) -{ - (void)baseline; (void)target; (void)workdir; (void)payload; - fprintf(stderr, "[checkout notify] why=%u path=%s\n", why, path ? path : "(null)"); - return 0; // non-zero would cancel -} - -void checkout_one_from_origin_master(git_repository *repo, const char *path_rel) { - int err = 0; - git_object *commitobj = NULL; - git_commit *commit = NULL; - git_tree *tree = NULL; - git_tree_entry *te = NULL; - git_diff *diff = NULL; - git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT; - git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT; - - - printf("Path to resume '%s' \n", path_rel); - /* 1) Resolve origin/master to a commit, then get its tree */ - if ((err = git_revparse_single(&commitobj, repo, "refs/remotes/origin/main^{commit}")) < 0) return; - commit = (git_commit *)commitobj; - if ((err = git_commit_tree(&tree, commit)) < 0) return; - - /* 2) Sanity-check: does the path exist in the target tree? */ - if ((err = git_tree_entry_bypath(&te, tree, path_rel)) == GIT_ENOTFOUND) { - fprintf(stderr, "Path '%s' not found in origin/main\n", path_rel); - err = 0; return; // nothing to do - } else if (err < 0) { - return; - } - git_tree_entry_free(te); te = NULL; - - /* 3) Diff target tree -> workdir (with index) for the one path */ - diffopts.pathspec.count = 1; - diffopts.pathspec.strings = (char **)&path_rel; - if ((err = git_diff_tree_to_workdir_with_index(&diff, repo, tree, &diffopts)) < 0) return; - - size_t n = git_diff_num_deltas(diff); - fprintf(stderr, "[pre-checkout] deltas for %s: %zu\n", path_rel, n); - git_diff_free(diff); diff = NULL; - - if (n == 0) { - fprintf(stderr, "No changes to apply for %s (already matches target or not selected)\n", path_rel); - /* fall through: we can still attempt checkout to let planner confirm */ - } - - /* 4) Configure checkout for a single literal path and creation allowed */ - const char *paths[] = { path_rel }; - opts.checkout_strategy = GIT_CHECKOUT_FORCE | GIT_CHECKOUT_SAFE | GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH; // or GIT_CHECKOUT_FORCE to overwrite local edits - opts.paths.strings = (char **)paths; - opts.paths.count = 1; - opts.notify_flags = GIT_CHECKOUT_NOTIFY_ALL; - opts.notify_cb = on_notify; - - /* Optional: ensure baseline reflects current HEAD */ - // git_object *head = NULL; git_tree *head_tree = NULL; - // if (git_revparse_single(&head, repo, "HEAD^{commit}") == 0) { - // git_commit_tree(&head_tree, (git_commit *)head); - // opts.baseline = head_tree; - // } - - /* 5) Only the selected path will be considered; planner will create/update it */ - err = git_checkout_tree(repo, (git_object *)tree, &opts); - - git_tree_free(tree); - git_commit_free(commit); - git_object_free(commitobj); - return; -} - - Status HfDownloader::downloadModel() { if (FileSystem::isPathEscaped(this->downloadPath)) { SPDLOG_ERROR("Path {} escape with .. is forbidden.", this->downloadPath); @@ -874,12 +431,15 @@ Status HfDownloader::downloadModel() { // Repository exists and we do not want to overwrite if (std::filesystem::is_directory(this->downloadPath) && !this->overwriteModels) { + // Checking if the download was partially finished for any files in repository auto matches = findLfsLikeFiles(this->downloadPath, true); if (matches.empty()) { - std::cout << "No files with LFS-like keywords in the first 3 lines were found.\n"; + std::cout << "No files to resume download found.\n"; + std::cout << "Path already exists on local filesystem. Skipping download to path: " << this->downloadPath << std::endl; + return StatusCode::OK; } else { - std::cout << "Found " << matches.size() << " matching file(s):\n"; + std::cout << "Found " << matches.size() << " file(s) to resume partial download:\n"; for (const auto& p : matches) { std::cout << " " << p.string() << "\n"; } @@ -915,28 +475,17 @@ Status HfDownloader::downloadModel() { } for (const auto& p : matches) { - std::cout << " Resuming " << p.string() << "\n"; - // Remove existing lfs pointer file from repository - std::string fullPath = FileSystem::joinPath({this->downloadPath, p.string()}); - std::filesystem::path filePath(fullPath); - if (!std::filesystem::remove(filePath)) { - SPDLOG_ERROR("Removing lfs file pointer error {}", fullPath); - return StatusCode::HF_GIT_CLONE_FAILED; - } + std::cout << " Resuming " << p.string() << "...\n"; std::string path = p.string(); - resumeLfsDownloadForFile1(repo, path.c_str()); + resumeLfsDownloadForFile(repo, path.c_str()); } - - // Use proxy - if (CheckIfProxySet()) { - SPDLOG_DEBUG("Download using https_proxy settings"); - //pull(repo, "origin", this->httpProxy.c_str()); - } else { - SPDLOG_DEBUG("Download with https_proxy not set"); - pull(repo, "origin", nullptr); + + SPDLOG_DEBUG("Checking repository status."); + auto status = CheckRepositoryStatus(false); + if (!status.ok()) { + return status; } - std::cout << "Path already exists on local filesystem. Skipping download to path: " << this->downloadPath << std::endl; return StatusCode::OK; } @@ -981,7 +530,7 @@ Status HfDownloader::downloadModel() { } SPDLOG_DEBUG("Checking repository status."); - status = CheckRepositoryStatus(); + status = CheckRepositoryStatus(true); if (!status.ok()) { return status; } diff --git a/src/pull_module/libgit2.hpp b/src/pull_module/libgit2.hpp index 755a84e89e..b8dacac0e9 100644 --- a/src/pull_module/libgit2.hpp +++ b/src/pull_module/libgit2.hpp @@ -62,7 +62,7 @@ class HfDownloader : public IModelDownloader { std::string GetRepositoryUrlWithPassword(); bool CheckIfProxySet(); Status RemoveReadonlyFileAttributeFromDir(const std::string& directoryPath); - Status CheckRepositoryStatus(); + Status CheckRepositoryStatus(bool checkUntracked); int CheckRepositoryForResume(); }; } // namespace ovms diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index d5b368dba9..9eba14b103 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -14,6 +14,7 @@ // limitations under the License. //***************************************************************************** #include +#include #include #include @@ -152,7 +153,7 @@ const std::string expectedGraphContentsDraft = R"( )"; TEST_F(HfDownloaderPullHfModel, PositiveDownload) { - GTEST_SKIP() << "Skipping test in CI - PositiveDownloadAndStart has full scope testing."; + // GTEST_SKIP() << "Skipping test in CI - PositiveDownloadAndStart has full scope testing."; std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); std::string task = "text_generation"; @@ -197,12 +198,60 @@ bool createGitLfsPointerFile(const std::string& path) { file << "version https://git-lfs.github.com/spec/v1\n" - "oid sha256:59f24bc922e1a48bb3feeba18b23f0e9622a7ee07166d925650d7a933283f8b1\n" - "size 123882252\n"; + "oid sha256:cecf0224201415144c00cf3a6cf3350306f9c78888d631eb590939a63722fefa\n" + "size 52417240\n"; return true; } +// Returns lowercase hex SHA-256 string on success, empty string on failure. +std::string sha256File(std::string_view path, std::error_code& ec) { + ec.clear(); + + std::ifstream ifs(std::string(path), std::ios::binary); + if (!ifs) { + ec = std::make_error_code(std::errc::no_such_file_or_directory); + return {}; + } + + SHA256_CTX ctx; + if (SHA256_Init(&ctx) != 1) { + ec = std::make_error_code(std::errc::io_error); + return {}; + } + + // Read in chunks to support large files without high memory usage. + std::vector buffer(1 << 20); // 1 MiB + while (ifs) { + ifs.read(reinterpret_cast(buffer.data()), static_cast(buffer.size())); + std::streamsize got = ifs.gcount(); + if (got > 0) { + if (SHA256_Update(&ctx, buffer.data(), static_cast(got)) != 1) { + ec = std::make_error_code(std::errc::io_error); + return {}; + } + } + } + if (!ifs.eof()) { // read failed not due to EOF + ec = std::make_error_code(std::errc::io_error); + return {}; + } + + std::array digest{}; + if (SHA256_Final(digest.data(), &ctx) != 1) { + ec = std::make_error_code(std::errc::io_error); + return {}; + } + + // Convert to lowercase hex + std::ostringstream oss; + oss << std::hex << std::setfill('0') << std::nouppercase; + for (unsigned char b : digest) { + oss << std::setw(2) << static_cast(b); + } + return oss.str(); +} + TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndResumeFromPArtialDownload) { std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); @@ -225,11 +274,14 @@ TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndResumeFromPArtialDownload) { ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; + std::error_code ec; + ec.clear(); + std::string expectedDigest = sha256File(modelPath, ec); + ASSERT_EQ(ec, std::errc()); // Prepare a git repository with a lfs_part file and lfs pointer file to simulate partial download error of a big model ASSERT_EQ(removeSecondHalf(modelPath), true); ASSERT_EQ(std::filesystem::file_size(modelPath), 26208620); - std::error_code ec; - ec.clear(); + std::string ovModelPartLfsName = "openvino_model.binlfs_part"; std::string ovModelPartLfsPath = ovms::FileSystem::appendSlash(basePath) + ovModelPartLfsName; fs::rename(modelPath, ovModelPartLfsPath, ec); @@ -240,12 +292,17 @@ TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndResumeFromPArtialDownload) { // Call ovms pull to resume the file this->ServerPullHfModel(modelName, downloadPath, task); + ASSERT_EQ(std::filesystem::exists(ovModelPartLfsPath), false) << modelPath; ASSERT_EQ(std::filesystem::exists(modelPath), true) << modelPath; ASSERT_EQ(std::filesystem::exists(graphPath), true) << graphPath; ASSERT_EQ(std::filesystem::file_size(modelPath), 52417240); graphContents = GetFileContents(graphPath); ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; + + std::string resumedDigest = sha256File(modelPath, ec); + ASSERT_EQ(ec, std::errc()); + ASSERT_EQ(expectedDigest, resumedDigest); } TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndStart) { From 3d8323b51634d22c2b8832cfd12e904cc888161d Mon Sep 17 00:00:00 2001 From: rasapala Date: Mon, 9 Mar 2026 14:59:57 +0100 Subject: [PATCH 18/49] Pass through --- third_party/libgit2/lfs.patch | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 0197a68564..8d4d43217e 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -332,10 +332,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..f877c4afa +index 000000000..6dc8f9650 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,677 @@ +@@ -0,0 +1,679 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -488,6 +488,7 @@ index 000000000..f877c4afa + lfs_oid.ptr); + return -1; + } ++ printf("\nfrom->lfs_oid %s\n", lfs_oid.ptr); + + if (get_lfs_info_match(&lfs_size, size_regexp) < 0) { + fprintf(stderr, @@ -531,8 +532,9 @@ index 000000000..f877c4afa + + if (git_filter_source_mode(src) == GIT_FILTER_SMUDGE) + return lfs_insert_id(to, from, src, payload); -+ /*else -+ * PATH for upload lfs files not needed ++ else ++ return GIT_PASSTHROUGH; ++ /* PATH for upload lfs files not needed + return lfs_remove_id(to, from); + */ + return 0; From 356dd4ed984c9a55cc4d100a9d0daf64c73555b5 Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 10 Mar 2026 11:40:11 +0100 Subject: [PATCH 19/49] Lfs upload --- third_party/libgit2/lfs.patch | 86 ++++++++++++++++++++++++++++++++--- 1 file changed, 79 insertions(+), 7 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 8d4d43217e..d7244c377e 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -24,6 +24,25 @@ index 31da49a88..d61c9735e 100644 if(BUILD_EXAMPLES) add_subdirectory(examples) endif() +diff --git a/include/git2/oid.h b/include/git2/oid.h +index 0af9737a0..6d9a8b08a 100644 +--- a/include/git2/oid.h ++++ b/include/git2/oid.h +@@ -22,14 +22,8 @@ GIT_BEGIN_DECL + + /** The type of object id. */ + typedef enum { +- +-#ifdef GIT_EXPERIMENTAL_SHA256 + GIT_OID_SHA1 = 1, /**< SHA1 */ + GIT_OID_SHA256 = 2 /**< SHA256 */ +-#else +- GIT_OID_SHA1 = 1 /**< SHA1 */ +-#endif +- + } git_oid_t; + + /* diff --git a/include/git2/repository.h b/include/git2/repository.h index b203576af..26309dd3f 100644 --- a/include/git2/repository.h @@ -332,10 +351,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..6dc8f9650 +index 000000000..280bcc7ab --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,679 @@ +@@ -0,0 +1,732 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -357,6 +376,7 @@ index 000000000..6dc8f9650 + +#include +#include "git2/sys/filter.h" ++#include "hash.h" +#include "oid.h" +#include "filter.h" +#include "str.h" @@ -466,6 +486,59 @@ index 000000000..6dc8f9650 + } +} + ++int git_oid_sha256_from_git_str_blob(git_oid *out, const struct git_str *input) ++{ ++ int error = -1; ++ git_hash_ctx *ctx = NULL; ++ ++ if (!out || !input || !input->ptr) ++ return -1; ++ ++ /* 1) Build "blob \\0" header (size = payload length in bytes) */ ++ char header[64]; /* plenty for "blob " + decimal size + \\0 */ ++ int hdrlen = snprintf(header, sizeof(header), "blob %zu", input->size); ++ if (hdrlen < 0 || (size_t)hdrlen + 1 >= sizeof(header)) ++ return -1; /* impossible header size for normal blob lengths */ ++ ++ git_hash_algorithm_t algorithm; ++ algorithm = git_oid_algorithm(GIT_OID_SHA256); ++ /* 2) Init SHA-256 hashing context (internal API) */ ++ if (git_hash_ctx_init(&ctx, algorithm) < 0) ++ goto done; ++ ++ /* 3) Feed header + NUL, then data */ ++ if (git_hash_update(ctx, header, (size_t)hdrlen) < 0) ++ goto done; ++ if (git_hash_update(ctx, "\0", 1) < 0) ++ goto done; ++ if (input->size > 0 && ++ git_hash_update(ctx, input->ptr, input->size) < 0) ++ goto done; ++ ++ /* 4) Finalize into git_oid (32-byte raw digest) */ ++ if (git_hash_final(out->id, ctx) < 0) ++ goto done; ++ ++ error = 0; ++ ++done: ++ if (ctx) ++ git_hash_ctx_cleanup(ctx); ++ return error; ++} ++ ++static int lfs_remove_id( ++ git_str *to, ++ const git_str *from) ++{ ++ git_oid lfs_oid; ++ git_oid_sha256_from_git_str_blob(&lfs_oid, from); ++ ++ fprintf("Size: %d", from->size); ++ fprintf("Oid sha256: %s", lfs_oid.id); ++ return 0; ++} ++ +static int lfs_insert_id( + git_str *to, const git_str *from, const git_filter_source *src, void** payload) +{ @@ -482,13 +555,11 @@ index 000000000..6dc8f9650 + const char *obj_regexp = "\noid sha256:(.*)\n"; + const char *size_regexp = "\nsize (.*)\n"; + -+ print_src_oid(src); + if (get_lfs_info_match(&lfs_oid, obj_regexp) < 0) { + fprintf(stderr,"\n[ERROR] failure, cannot find lfs oid in: %s\n", + lfs_oid.ptr); + return -1; + } -+ printf("\nfrom->lfs_oid %s\n", lfs_oid.ptr); + + if (get_lfs_info_match(&lfs_size, size_regexp) < 0) { + fprintf(stderr, @@ -497,6 +568,9 @@ index 000000000..6dc8f9650 + return -1; + } + ++ printf("\ndownload from->lfs_oid %s\n", lfs_oid.ptr); ++ printf("\ndownload from->lfs_size %s\n", lfs_size.ptr); ++ + git_repository *repo = git_filter_source_repo(src); + const char *path = git_filter_source_path(src); + @@ -533,10 +607,8 @@ index 000000000..6dc8f9650 + if (git_filter_source_mode(src) == GIT_FILTER_SMUDGE) + return lfs_insert_id(to, from, src, payload); + else -+ return GIT_PASSTHROUGH; -+ /* PATH for upload lfs files not needed ++ /* for upload of the lfs pointer files */ + return lfs_remove_id(to, from); -+ */ + return 0; +} + From 30072a9e9024ace837c4bcf879109f0a9c8f7a57 Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 10 Mar 2026 11:46:00 +0100 Subject: [PATCH 20/49] Fix --- third_party/libgit2/lfs.patch | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index d7244c377e..596c981ce6 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -351,7 +351,7 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..280bcc7ab +index 000000000..795f08b29 --- /dev/null +++ b/src/libgit2/lfs_filter.c @@ -0,0 +1,732 @@ @@ -534,8 +534,8 @@ index 000000000..280bcc7ab + git_oid lfs_oid; + git_oid_sha256_from_git_str_blob(&lfs_oid, from); + -+ fprintf("Size: %d", from->size); -+ fprintf("Oid sha256: %s", lfs_oid.id); ++ printf("Size: %d", from->size); ++ printf("Oid sha256: %s", lfs_oid.id); + return 0; +} + From 39661c8fb8b9c6051509e2a21038e226fe42bf33 Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 10 Mar 2026 11:59:34 +0100 Subject: [PATCH 21/49] Fix 2 --- third_party/libgit2/lfs.patch | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 596c981ce6..9825cf3703 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -351,10 +351,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..795f08b29 +index 000000000..141d8e682 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,732 @@ +@@ -0,0 +1,736 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -532,10 +532,14 @@ index 000000000..795f08b29 + const git_str *from) +{ + git_oid lfs_oid; -+ git_oid_sha256_from_git_str_blob(&lfs_oid, from); ++ if (git_oid_sha256_from_git_str_blob(&lfs_oid, from) < 0) { ++ fprintf(stderr, ++ "\n[ERROR] failure, cannot calculate sha256: %s\n"); ++ return -1; ++ } + -+ printf("Size: %d", from->size); -+ printf("Oid sha256: %s", lfs_oid.id); ++ printf("\nSize: %d\n", from->size); ++ printf("\nOid sha256: %s\n", lfs_oid.id); + return 0; +} + From c2e8e8fdb80bf728752fc544bafbf633c816500e Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 10 Mar 2026 12:15:12 +0100 Subject: [PATCH 22/49] Fix 3 --- third_party/libgit2/lfs.patch | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 9825cf3703..a71e15d9f8 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -351,10 +351,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..141d8e682 +index 000000000..6111fb556 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,736 @@ +@@ -0,0 +1,734 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -500,10 +500,8 @@ index 000000000..141d8e682 + if (hdrlen < 0 || (size_t)hdrlen + 1 >= sizeof(header)) + return -1; /* impossible header size for normal blob lengths */ + -+ git_hash_algorithm_t algorithm; -+ algorithm = git_oid_algorithm(GIT_OID_SHA256); + /* 2) Init SHA-256 hashing context (internal API) */ -+ if (git_hash_ctx_init(&ctx, algorithm) < 0) ++ if (git_hash_ctx_init(&ctx, GIT_HASH_ALGORITHM_SHA256) < 0) + goto done; + + /* 3) Feed header + NUL, then data */ @@ -534,7 +532,7 @@ index 000000000..141d8e682 + git_oid lfs_oid; + if (git_oid_sha256_from_git_str_blob(&lfs_oid, from) < 0) { + fprintf(stderr, -+ "\n[ERROR] failure, cannot calculate sha256: %s\n"); ++ "\n[ERROR] failure, cannot calculate sha256\n"); + return -1; + } + From ca6e1c128b33767564bb36e5f96ed2c683155a57 Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 10 Mar 2026 16:54:58 +0100 Subject: [PATCH 23/49] Working sha --- third_party/libgit2/lfs.patch | 168 ++++++++++++++++++++++++++++------ 1 file changed, 141 insertions(+), 27 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index a71e15d9f8..88d1b97e75 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -100,7 +100,7 @@ index d121c588a..b54a01a4b 100644 # diff --git a/src/cli/cmd_clone.c b/src/cli/cmd_clone.c -index c18cb28d4..286fa7153 100644 +index c18cb28d4..6d23dcbb1 100644 --- a/src/cli/cmd_clone.c +++ b/src/cli/cmd_clone.c @@ -146,6 +146,7 @@ int cmd_clone(int argc, char **argv) @@ -111,6 +111,76 @@ index c18cb28d4..286fa7153 100644 if (!checkout) clone_opts.checkout_opts.checkout_strategy = GIT_CHECKOUT_NONE; +@@ -182,6 +183,69 @@ int cmd_clone(int argc, char **argv) + + cli_progress_finish(&progress); + ++ ++ git_repository *repo2 = NULL; ++ int error = git_repository_open_ext(&repo2, local_path, 0, NULL); ++ // HEAD state info ++ bool is_detached = git_repository_head_detached(repo2) == 1; ++ bool is_unborn = git_repository_head_unborn(repo2) == 1; ++ ++ // Collect status (staged/unstaged/untracked) ++ git_status_options opts = GIT_STATUS_OPTIONS_INIT; ++ ++ opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR; ++ opts.flags = GIT_STATUS_OPT_INCLUDE_UNTRACKED // include untracked files ++ // // | ++ // GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX ++ // // detect renames ++ // HEAD->index - not ++ // required currently and ++ // impacts performance ++ | GIT_STATUS_OPT_SORT_CASE_SENSITIVELY; ++ ++ git_status_list *status_list = NULL; ++ ret = git_status_list_new(&status_list, repo2, &opts); ++ ++ size_t staged = 0, unstaged = 0, untracked = 0, conflicted = 0; ++ const size_t n = git_status_list_entrycount(status_list); ++ ++ for (size_t i = 0; i < n; ++i) { ++ const git_status_entry *e = git_status_byindex(status_list, i); ++ if (!e) ++ continue; ++ unsigned s = e->status; ++ ++ // Staged (index) changes ++ if (s & (GIT_STATUS_INDEX_NEW | GIT_STATUS_INDEX_MODIFIED | ++ GIT_STATUS_INDEX_DELETED | GIT_STATUS_INDEX_RENAMED | ++ GIT_STATUS_INDEX_TYPECHANGE)) ++ ++staged; ++ ++ // Unstaged (workdir) changes ++ if (s & (GIT_STATUS_WT_MODIFIED | GIT_STATUS_WT_DELETED | ++ GIT_STATUS_WT_RENAMED | GIT_STATUS_WT_TYPECHANGE)) ++ ++unstaged; ++ ++ // Untracked ++ if (s & GIT_STATUS_WT_NEW) ++ ++untracked; ++ ++ // Conflicted ++ if (s & GIT_STATUS_CONFLICTED) ++ ++conflicted; ++ } ++ ++ // Print summary (mirrors your original stream output) ++ printf("HEAD state : %s\n", ++ is_unborn ? "unborn (no commits)" : ++ (is_detached ? "detached" : "attached")); ++ printf("Staged changes : %zu\n", staged); ++ printf("Unstaged changes: %zu\n", unstaged); ++ printf("Untracked files : %zu", untracked); ++ if (conflicted) { ++ printf(" (%zu paths flagged)", conflicted); ++ } ++ printf("\n"); + done: + cli_progress_dispose(&progress); + git__free(computed_path); diff --git a/src/cli/progress.h b/src/cli/progress.h index f08d68f19..0344304ec 100644 --- a/src/cli/progress.h @@ -351,11 +421,11 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..6111fb556 +index 000000000..876628fd2 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,734 @@ -+/* +@@ -0,0 +1,778 @@ ++/* +/ Copyright 2025 Intel Corporation +/ +/ Licensed under the Apache License, Version 2.0 (the "License"); @@ -486,42 +556,57 @@ index 000000000..6111fb556 + } +} + -+int git_oid_sha256_from_git_str_blob(git_oid *out, const struct git_str *input) ++int git_oid_sha256_from_git_str_blob( ++ git_oid *out, ++ const struct git_str *input, ++ char *pointer_line, ++ size_t pointer_line_cap) +{ + int error = -1; -+ git_hash_ctx *ctx = NULL; ++ git_hash_ctx ctx; + + if (!out || !input || !input->ptr) + return -1; + -+ /* 1) Build "blob \\0" header (size = payload length in bytes) */ -+ char header[64]; /* plenty for "blob " + decimal size + \\0 */ -+ int hdrlen = snprintf(header, sizeof(header), "blob %zu", input->size); -+ if (hdrlen < 0 || (size_t)hdrlen + 1 >= sizeof(header)) -+ return -1; /* impossible header size for normal blob lengths */ -+ -+ /* 2) Init SHA-256 hashing context (internal API) */ ++ /* 1) Init SHA-256 hashing context (internal API) */ + if (git_hash_ctx_init(&ctx, GIT_HASH_ALGORITHM_SHA256) < 0) + goto done; + -+ /* 3) Feed header + NUL, then data */ -+ if (git_hash_update(ctx, header, (size_t)hdrlen) < 0) -+ goto done; -+ if (git_hash_update(ctx, "\0", 1) < 0) -+ goto done; -+ if (input->size > 0 && -+ git_hash_update(ctx, input->ptr, input->size) < 0) -+ goto done; ++ /* 2) Stream the payload in chunks — hash *only* the file bytes. */ ++ const size_t CHUNK = 4 * 1024 * 1024; /* 4 MiB */ ++ const unsigned char *p = (const unsigned char *)input->ptr; ++ size_t remaining = input->size; ++ ++ while (remaining > 0) { ++ size_t n = remaining > CHUNK ? CHUNK : remaining; ++ if (git_hash_update(&ctx, p, n) < 0) ++ goto done; ++ p += n; ++ remaining -= n; ++ } + -+ /* 4) Finalize into git_oid (32-byte raw digest) */ -+ if (git_hash_final(out->id, ctx) < 0) ++ /* 3) Finalize into git_oid (32-byte raw digest for SHA-256). */ ++ if (git_hash_final(out->id, &ctx) < 0) + goto done; + ++ /* 4) Optionally format "oid sha256:" for the LFS pointer file. */ ++ if (pointer_line && ++ pointer_line_cap >= (size_t)(strlen("oid sha256:") + 64 + 1)) { ++ char hex[64 + 1]; ++ /* Formats full hex; no NUL added. */ ++ if (git_oid_fmt(hex, out) < 0) { ++ fprintf(stderr, ++ "\n[ERROR] failure, git_oid_fmt failed\n"); ++ return -1; ++ } ++ ++ hex[64] = '\0'; ++ snprintf(pointer_line, pointer_line_cap, "oid sha256:%s", hex); ++ } ++ + error = 0; + +done: -+ if (ctx) -+ git_hash_ctx_cleanup(ctx); + return error; +} + @@ -529,15 +614,44 @@ index 000000000..6111fb556 + git_str *to, + const git_str *from) +{ ++ int error = 0; ++ ++ if(!from) return -1; ++ ++ /* Use lib git oid to get lfs sha256 */ + git_oid lfs_oid; -+ if (git_oid_sha256_from_git_str_blob(&lfs_oid, from) < 0) { ++ lfs_oid.type = GIT_OID_SHA256; ++ char line[80]; /* 75+ is enough */ ++ if (git_oid_sha256_from_git_str_blob(&lfs_oid, from, line, sizeof(line)) < 0) ++ { + fprintf(stderr, + "\n[ERROR] failure, cannot calculate sha256\n"); + return -1; + } + + printf("\nSize: %d\n", from->size); -+ printf("\nOid sha256: %s\n", lfs_oid.id); ++ printf("\nOid sha256: %s\n", line); ++ ++ git_str_init(to, 0); ++ ++ /* 1) version line (LFS spec requires this literal string) */ ++ if ((error = git_str_puts(to, "version https://git-lfs.github.com/spec/v1\n")) < 0) ++ return error; ++ ++ ++ /* 2) the oid line passed by caller (must end with '\n') */ ++ if ((error = git_str_puts(to, line)) < 0) ++ return error; ++ ++ if (line[strlen(line) - 1] != '\n') { ++ if ((error = git_str_putc(to, '\n')) < 0) ++ return error; ++ } ++ ++ /* 3) size line from the original file size */ ++ if ((error = git_str_printf(to, "size %zu\n", from->size)) < 0) ++ return error; ++ + return 0; +} + From 110c4e2fdb7dca32437b5c45e1b0cb3ffc13ad76 Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 11 Mar 2026 10:01:54 +0100 Subject: [PATCH 24/49] Experimentasl cmake off --- third_party/libgit2/lfs.patch | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 88d1b97e75..300a6ef986 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -24,6 +24,17 @@ index 31da49a88..d61c9735e 100644 if(BUILD_EXAMPLES) add_subdirectory(examples) endif() +diff --git a/cmake/ExperimentalFeatures.cmake b/cmake/ExperimentalFeatures.cmake +index 7eff40bdb..5562acc77 100644 +--- a/cmake/ExperimentalFeatures.cmake ++++ b/cmake/ExperimentalFeatures.cmake +@@ -18,6 +18,3 @@ else() + add_feature_info("SHA256 API" OFF "experimental SHA256 APIs") + endif() + +-if(EXPERIMENTAL) +- set(LIBGIT2_FILENAME "${LIBGIT2_FILENAME}-experimental") +-endif() diff --git a/include/git2/oid.h b/include/git2/oid.h index 0af9737a0..6d9a8b08a 100644 --- a/include/git2/oid.h From 18c37bf25261d47cc5cdc5dd8233bb3d9cd69f77 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 11 Mar 2026 10:36:11 +0100 Subject: [PATCH 25/49] Experimental flag --- third_party/libgit2/libgit2_engine.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/third_party/libgit2/libgit2_engine.bzl b/third_party/libgit2/libgit2_engine.bzl index 68a5037aff..5df5667a5c 100644 --- a/third_party/libgit2/libgit2_engine.bzl +++ b/third_party/libgit2/libgit2_engine.bzl @@ -51,6 +51,7 @@ def _impl(repository_ctx): out_static = "out_interface_libs = [\"{lib_name}.lib\"],".format(lib_name=lib_name) out_libs = "out_shared_libs = [\"{lib_name}.dll\"],".format(lib_name=lib_name) cache_entries = """ + "EXPERIMENTAL_SHA256": "ON", "CMAKE_POSITION_INDEPENDENT_CODE": "ON", "CMAKE_CXX_FLAGS": " /guard:cf /GS -s -D_GLIBCXX_USE_CXX11_ABI=1", "CMAKE_LIBRARY_OUTPUT_DIRECTORY": "Debug", @@ -66,6 +67,7 @@ def _impl(repository_ctx): out_static = "" out_libs = "out_shared_libs = [\"{lib_name}.so\"],".format(lib_name=lib_name) cache_entries = """ + "EXPERIMENTAL_SHA256": "ON", "CMAKE_POSITION_INDEPENDENT_CODE": "ON", "CMAKE_CXX_FLAGS": " /guard:cf -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized", "CMAKE_ARCHIVE_OUTPUT_DIRECTORY": "lib", From 2bae83dbcf18b33f41cb9342715a8e8885ecc9e9 Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 11 Mar 2026 10:36:45 +0100 Subject: [PATCH 26/49] Cleanup --- third_party/libgit2/lfs.patch | 94 +++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 300a6ef986..b6daae42ae 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -432,10 +432,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..876628fd2 +index 000000000..fc3120cca --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,778 @@ +@@ -0,0 +1,810 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -473,6 +473,7 @@ index 000000000..876628fd2 + const char *lfs_oid; + const char *lfs_size; + const char *url; ++ bool is_download; +} lfs_attrs; + +static size_t get_digit(const char *buffer) @@ -576,12 +577,20 @@ index 000000000..876628fd2 + int error = -1; + git_hash_ctx ctx; + -+ if (!out || !input || !input->ptr) ++ if (!out || !input || !input->ptr) { + return -1; ++ } ++ ++ if (!pointer_line || ++ pointer_line_cap < (size_t)(strlen("oid sha256:") + 64 + 1)) { ++ return -1; ++ } + + /* 1) Init SHA-256 hashing context (internal API) */ -+ if (git_hash_ctx_init(&ctx, GIT_HASH_ALGORITHM_SHA256) < 0) -+ goto done; ++ if (git_hash_ctx_init(&ctx, GIT_HASH_ALGORITHM_SHA256) < 0) { ++ fprintf(stderr, "\n[ERROR] git_hash_ctx_init failed\n"); ++ return -1; ++ } + + /* 2) Stream the payload in chunks — hash *only* the file bytes. */ + const size_t CHUNK = 4 * 1024 * 1024; /* 4 MiB */ @@ -590,15 +599,19 @@ index 000000000..876628fd2 + + while (remaining > 0) { + size_t n = remaining > CHUNK ? CHUNK : remaining; -+ if (git_hash_update(&ctx, p, n) < 0) -+ goto done; ++ if (git_hash_update(&ctx, p, n) < 0) { ++ fprintf(stderr, "\n[ERROR] git_hash_update failed\n"); ++ return -1; ++ } + p += n; + remaining -= n; + } + + /* 3) Finalize into git_oid (32-byte raw digest for SHA-256). */ -+ if (git_hash_final(out->id, &ctx) < 0) -+ goto done; ++ if (git_hash_final(out->id, &ctx) < 0) { ++ fprintf(stderr, "\n[ERROR] git_hash_final failed\n"); ++ return -1; ++ } + + /* 4) Optionally format "oid sha256:" for the LFS pointer file. */ + if (pointer_line && @@ -615,53 +628,67 @@ index 000000000..876628fd2 + snprintf(pointer_line, pointer_line_cap, "oid sha256:%s", hex); + } + -+ error = 0; -+ -+done: -+ return error; ++ return 0; +} + +static int lfs_remove_id( + git_str *to, -+ const git_str *from) ++ const git_str *from, ++ void **payload) +{ + int error = 0; ++ /* Init the lfs attrs to indicate git lfs clean, currently only diff support no upload of lfs file supported */ ++ struct lfs_attrs la = { NULL, NULL, NULL, NULL, NULL, NULL, false }; ++ *payload = git__malloc(sizeof(la)); ++ GIT_ERROR_CHECK_ALLOC(*payload); ++ memcpy(*payload, &la, sizeof(la)); + + if(!from) return -1; + ++ /* lfs spec - return empty pointer when the file is empty */ ++ if (from->size == 0) { ++ git_str_init(to, 0); ++ return 0; ++ } ++ + /* Use lib git oid to get lfs sha256 */ + git_oid lfs_oid; + lfs_oid.type = GIT_OID_SHA256; + char line[80]; /* 75+ is enough */ -+ if (git_oid_sha256_from_git_str_blob(&lfs_oid, from, line, sizeof(line)) < 0) -+ { ++ if (git_oid_sha256_from_git_str_blob(&lfs_oid, from, line, sizeof(line)) < 0) { + fprintf(stderr, + "\n[ERROR] failure, cannot calculate sha256\n"); + return -1; + } + -+ printf("\nSize: %d\n", from->size); -+ printf("\nOid sha256: %s\n", line); -+ + git_str_init(to, 0); + + /* 1) version line (LFS spec requires this literal string) */ -+ if ((error = git_str_puts(to, "version https://git-lfs.github.com/spec/v1\n")) < 0) -+ return error; ++ if ((error = git_str_puts( ++ to, "version https://git-lfs.github.com/spec/v1\n")) < 0) { ++ fprintf(stderr, "\n[ERROR] git_str_puts failed\n"); ++ return error; ++ } ++ + -+ + /* 2) the oid line passed by caller (must end with '\n') */ -+ if ((error = git_str_puts(to, line)) < 0) ++ if ((error = git_str_puts(to, line)) < 0) { ++ fprintf(stderr, "\n[ERROR] git_str_puts failed\n"); + return error; ++ } + + if (line[strlen(line) - 1] != '\n') { -+ if ((error = git_str_putc(to, '\n')) < 0) ++ if ((error = git_str_putc(to, '\n')) < 0) { ++ fprintf(stderr, "\n[ERROR] git_str_putc failed\n"); + return error; ++ } + } + + /* 3) size line from the original file size */ -+ if ((error = git_str_printf(to, "size %zu\n", from->size)) < 0) ++ if ((error = git_str_printf(to, "size %zu\n", from->size)) < 0) { ++ fprintf(stderr, "\n[ERROR] git_str_printf failed\n"); + return error; ++ } + + return 0; +} @@ -695,9 +722,6 @@ index 000000000..876628fd2 + return -1; + } + -+ printf("\ndownload from->lfs_oid %s\n", lfs_oid.ptr); -+ printf("\ndownload from->lfs_size %s\n", lfs_size.ptr); -+ + git_repository *repo = git_filter_source_repo(src); + const char *path = git_filter_source_path(src); + @@ -712,7 +736,7 @@ index 000000000..876628fd2 + size_t workdir_size = strlen(git_repository_workdir(repo)); + + const char *workdir = git_repository_workdir(repo); -+ struct lfs_attrs la = { path, full_path.ptr, workdir, lfs_oid.ptr, lfs_size.ptr, repo->url }; ++ struct lfs_attrs la = { path, full_path.ptr, workdir, lfs_oid.ptr, lfs_size.ptr, repo->url, true }; + + *payload = git__malloc(sizeof(la)); + GIT_ERROR_CHECK_ALLOC(*payload); @@ -731,11 +755,12 @@ index 000000000..876628fd2 +{ + GIT_UNUSED(self); GIT_UNUSED(payload); + ++ /* for download of the lfs pointer files */ + if (git_filter_source_mode(src) == GIT_FILTER_SMUDGE) + return lfs_insert_id(to, from, src, payload); + else -+ /* for upload of the lfs pointer files */ -+ return lfs_remove_id(to, from); ++ /* for upload or diff of the lfs pointer files */ ++ return lfs_remove_id(to, from, payload); + return 0; +} + @@ -993,6 +1018,13 @@ index 000000000..876628fd2 + return; + } + struct lfs_attrs *la = (struct lfs_attrs *)payload; ++ ++ /* Currently only download is supoprted, no lfs file upload */ ++ if (!la->is_download) { ++ git__free(payload); ++ return; ++ } ++ + char *tmp_out_file = append_cstr_to_buffer(la->full_path, "lfs_part"); + + CURL *info_curl,*dl_curl; From ad313b772bfd4cfdc852277e8c3b630f470ec52e Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 11 Mar 2026 11:34:08 +0100 Subject: [PATCH 27/49] Cleanup --- src/pull_module/libgit2.cpp | 332 +++++++++++++++++++++----------- src/pull_module/libgit2.hpp | 1 - src/status.cpp | 1 - src/test/pull_hf_model_test.cpp | 21 +- 4 files changed, 233 insertions(+), 122 deletions(-) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 570ecd08fe..14f7bcdb89 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -15,9 +15,11 @@ //***************************************************************************** #include "libgit2.hpp" +#include #include -#include #include +#include +#include #include #include @@ -69,16 +71,16 @@ int cred_acquire_cb(git_credential** out, password = _strdup(username); #endif } else { - fprintf(stderr, "HF_TOKEN env variable is not set.\n"); + fprintf(stderr, "[ERROR] HF_TOKEN env variable is not set.\n"); return -1; } error = git_credential_userpass_plaintext_new(out, username, password); if (error < 0) { - fprintf(stderr, "Creating credentials failed.\n"); + fprintf(stderr, "[ERROR] Creating credentials failed.\n"); error = -1; } } else { - fprintf(stderr, "Only USERPASS_PLAINTEXT supported in OVMS.\n"); + fprintf(stderr, "[ERROR] Only USERPASS_PLAINTEXT supported in OVMS.\n"); return 1; } @@ -180,56 +182,94 @@ Status HfDownloader::RemoveReadonlyFileAttributeFromDir(const std::string& direc return StatusCode::OK; } -Status HfDownloader::CheckRepositoryStatus(bool checkUntracked) { - git_repository *repo = NULL; - int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); - if (error < 0) { - const git_error *err = git_error_last(); - if (err) - SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); - else - SPDLOG_ERROR("Repository open failed: {}", error); - if (repo) git_repository_free(repo); +class GitRepositoryGuard { +public: + git_repository* repo = nullptr; + + GitRepositoryGuard(const std::string& path) { + int error = git_repository_open_ext(&repo, path.c_str(), 0, nullptr); + if (error < 0) { + const git_error* err = git_error_last(); + if (err) + SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); + else + SPDLOG_ERROR("Repository open failed: {}", error); + if (repo) + git_repository_free(repo); + } + } + + ~GitRepositoryGuard() { + if (repo) { + git_repository_free(repo); + } + } + // Allow implicit access to the raw pointer + git_repository* get() const { return repo; } + operator git_repository*() const { return repo; } + + // Non-copyable + GitRepositoryGuard(const GitRepositoryGuard&) = delete; + GitRepositoryGuard& operator=(const GitRepositoryGuard&) = delete; + + // Movable + GitRepositoryGuard(GitRepositoryGuard&& other) noexcept { + repo = other.repo; + other.repo = nullptr; + } + GitRepositoryGuard& operator=(GitRepositoryGuard&& other) noexcept { + if (this != &other) { + if (repo) + git_repository_free(repo); + repo = other.repo; + other.repo = nullptr; + } + return *this; + } +}; + +Status HfDownloader::CheckRepositoryStatus(bool checkUntracked) { + GitRepositoryGuard repoGuard(this->downloadPath); + if (!repoGuard.get()) { return StatusCode::HF_GIT_STATUS_FAILED; } // HEAD state info - bool is_detached = git_repository_head_detached(repo) == 1; - bool is_unborn = git_repository_head_unborn(repo) == 1; - + bool is_detached = git_repository_head_detached(repoGuard.get()) == 1; + bool is_unborn = git_repository_head_unborn(repoGuard.get()) == 1; + // Collect status (staged/unstaged/untracked) git_status_options opts = GIT_STATUS_OPTIONS_INIT; - - opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR; - opts.flags = GIT_STATUS_OPT_INCLUDE_UNTRACKED // include untracked files // | GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX // detect renames HEAD->index - not required currently and impacts performance - | GIT_STATUS_OPT_SORT_CASE_SENSITIVELY; - + opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR; + opts.flags = GIT_STATUS_OPT_INCLUDE_UNTRACKED // include untracked files // | GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX // detect renames HEAD->index - not required currently and impacts performance + | GIT_STATUS_OPT_SORT_CASE_SENSITIVELY; + git_status_list* status_list = nullptr; - error = git_status_list_new(&status_list, repo, &opts); + int error = git_status_list_new(&status_list, repoGuard.get(), &opts); if (error != 0) { return StatusCode::HF_GIT_STATUS_FAILED; } size_t staged = 0, unstaged = 0, untracked = 0, conflicted = 0; - const size_t n = git_status_list_entrycount(status_list); // iterate entries + const size_t n = git_status_list_entrycount(status_list); // iterate entries for (size_t i = 0; i < n; ++i) { const git_status_entry* e = git_status_byindex(status_list, i); unsigned s = e->status; // Staged (index) changes - if (s & (GIT_STATUS_INDEX_NEW | - GIT_STATUS_INDEX_MODIFIED| - GIT_STATUS_INDEX_DELETED | - GIT_STATUS_INDEX_RENAMED | - GIT_STATUS_INDEX_TYPECHANGE)) + if (s & (GIT_STATUS_INDEX_NEW | + GIT_STATUS_INDEX_MODIFIED | + GIT_STATUS_INDEX_DELETED | + GIT_STATUS_INDEX_RENAMED | + GIT_STATUS_INDEX_TYPECHANGE)) ++staged; // Unstaged (workdir) changes - if (s & (GIT_STATUS_WT_MODIFIED | - GIT_STATUS_WT_DELETED | - GIT_STATUS_WT_RENAMED | - GIT_STATUS_WT_TYPECHANGE)) + if (s & (GIT_STATUS_WT_MODIFIED | + GIT_STATUS_WT_DELETED | + GIT_STATUS_WT_RENAMED | + GIT_STATUS_WT_TYPECHANGE)) ++unstaged; // Untracked @@ -243,46 +283,52 @@ Status HfDownloader::CheckRepositoryStatus(bool checkUntracked) { std::stringstream ss; ss << "HEAD state : " - << (is_unborn ? "unborn (no commits)" : (is_detached ? "detached" : "attached")) - << "\n"; - ss << "Staged changes : " << staged << "\n"; - ss << "Unstaged changes: " << unstaged << "\n"; - ss << "Untracked files : " << untracked << "\n"; - if (conflicted) ss << " (" << conflicted << " paths flagged)"; + << (is_unborn ? "unborn (no commits)" : (is_detached ? "detached" : "attached")) + << "\n"; + ss << "Staged changes : " << staged << "\n"; + ss << "Unstaged changes: " << unstaged << "\n"; + ss << "Untracked files : " << untracked << "\n"; + if (conflicted) + ss << " (" << conflicted << " paths flagged)"; SPDLOG_DEBUG(ss.str()); git_status_list_free(status_list); // We do not care about untracked until after git clone if (is_unborn || is_detached || staged || unstaged || conflicted || (checkUntracked && untracked)) { - return StatusCode::HF_GIT_STATUS_UNCLEAN; + return StatusCode::HF_GIT_STATUS_UNCLEAN; } return StatusCode::OK; } -#define CHECK(call) do { \ - int _err = (call); \ - if (_err < 0) { \ - const git_error *e = git_error_last(); \ - fprintf(stderr, "Error %d: %s (%s:%d)\n", _err, e && e->message ? e->message : "no message", __FILE__, __LINE__); \ - return; \ - } \ -} while (0) +#define CHECK(call) \ + do { \ + int _err = (call); \ + if (_err < 0) { \ + const git_error* e = git_error_last(); \ + fprintf(stderr, "[ERROR] %d: %s (%s:%d)\n", _err, e && e->message ? e->message : "no message", __FILE__, __LINE__); \ + return; \ + } \ + } while (0) // Trim trailing '\r' (for CRLF files) and surrounding spaces static inline void rtrimCrLfWhitespace(std::string& s) { - if (!s.empty() && s.back() == '\r') s.pop_back(); // remove trailing '\r' - while (!s.empty() && std::isspace(static_cast(s.back()))) s.pop_back(); // trailing ws + if (!s.empty() && s.back() == '\r') + s.pop_back(); // remove trailing '\r' + while (!s.empty() && std::isspace(static_cast(s.back()))) + s.pop_back(); // trailing ws size_t i = 0; - while (i < s.size() && std::isspace(static_cast(s[i]))) ++i; // leading ws - if (i > 0) s.erase(0, i); + while (i < s.size() && std::isspace(static_cast(s[i]))) + ++i; // leading ws + if (i > 0) + s.erase(0, i); } // Case-insensitive substring search: returns true if 'needle' is found in 'hay' static bool containsCaseInsensitive(const std::string& hay, const std::string& needle) { auto toLower = [](std::string v) { std::transform(v.begin(), v.end(), v.begin(), - [](unsigned char c){ return static_cast(std::tolower(c)); }); + [](unsigned char c) { return static_cast(std::tolower(c)); }); return v; }; std::string hayLower = toLower(hay); @@ -295,7 +341,8 @@ static bool containsCaseInsensitive(const std::string& hay, const std::string& n static bool readFirstThreeLines(const fs::path& p, std::vector& outLines) { outLines.clear(); std::ifstream in(p, std::ios::in | std::ios::binary); - if (!in) return false; + if (!in) + return false; constexpr std::streamsize kMaxPerLine = 8192; @@ -307,14 +354,18 @@ static bool readFirstThreeLines(const fs::path& p, std::vector& out char ch; bool gotNewline = false; while (count < kMaxPerLine && in.get(ch)) { - if (ch == '\n') { gotNewline = true; break; } + if (ch == '\n') { + gotNewline = true; + break; + } line.push_back(ch); ++count; } // If we hit kMaxPerLine without encountering '\n', drain until newline to resync if (count == kMaxPerLine && !gotNewline) { while (in.get(ch)) { - if (ch == '\n') break; + if (ch == '\n') + break; } } @@ -324,7 +375,8 @@ static bool readFirstThreeLines(const fs::path& p, std::vector& out } rtrimCrLfWhitespace(line); outLines.push_back(line); - if (!in) break; // Handle EOF gracefully + if (!in) + break; // Handle EOF gracefully } return true; } @@ -333,37 +385,42 @@ static bool readFirstThreeLines(const fs::path& p, std::vector& out // line1 -> "version", line2 -> "oid", line3 -> "size" (case-insensitive). static bool fileHasLfsKeywordsFirst3Positional(const fs::path& p) { std::error_code ec; - if (!fs::is_regular_file(p, ec)) return false; + if (!fs::is_regular_file(p, ec)) + return false; std::vector lines; - if (!readFirstThreeLines(p, lines)) return false; + if (!readFirstThreeLines(p, lines)) + return false; - if (lines.size() < 3) return false; + if (lines.size() < 3) + return false; return containsCaseInsensitive(lines[0], "version") && containsCaseInsensitive(lines[1], "oid") && containsCaseInsensitive(lines[2], "size"); } - // Helper: make path relative to base (best-effort, non-throwing). static fs::path makeRelativeToBase(const fs::path& path, const fs::path& base) { std::error_code ec; // Try fs::relative first (handles canonical comparisons, may fail if on different roots) fs::path rel = fs::relative(path, base, ec); - if (!ec && !rel.empty()) return rel; + if (!ec && !rel.empty()) + return rel; // Fallback: purely lexical relative (doesn't access filesystem) rel = path.lexically_relative(base); - if (!rel.empty()) return rel; + if (!rel.empty()) + return rel; // Last resort: return filename only (better than absolute when nothing else works) - if (path.has_filename()) return path.filename(); + if (path.has_filename()) + return path.filename(); return path; } // Find all files under 'directory' that satisfy the first-3-lines LFS keyword check. -std::vector findLfsLikeFiles(const std::string& directory, bool recursive = true) { +static std::vector findLfsLikeFiles(const std::string& directory, bool recursive = true) { std::vector matches; std::error_code ec; @@ -389,38 +446,102 @@ std::vector findLfsLikeFiles(const std::string& directory, bool recurs return matches; } -void resumeLfsDownloadForFile(git_repository *repo, const char *file_path_in_repo) { - git_object *obj = NULL; - git_tree *tree = NULL; - git_tree_entry *entry = NULL; - git_blob *blob = NULL; - git_buf out = GIT_BUF_INIT; +// pick the right entry pointer type for your libgit2 +#if defined(GIT_LIBGIT2_VER_MAJOR) +// libgit2 ≥ 1.0 generally has const-correct free() (accepts const*) +using git_tree_entry_ptr = const git_tree_entry*; +#else +using git_tree_entry_ptr = git_tree_entry*; +#endif + +// Single guard that owns all temporaries used in resumeLfsDownloadForFile +struct GitScope { + git_object* tree_obj = nullptr; // owns the tree as a generic git_object + git_tree_entry_ptr entry = nullptr; // owns the entry + git_blob* blob = nullptr; // owns the blob + git_buf out = GIT_BUF_INIT; // owns the buffer + + GitScope() = default; + ~GitScope() { cleanup(); } + + GitScope(const GitScope&) = delete; + GitScope& operator=(const GitScope&) = delete; + + GitScope(GitScope&& other) noexcept : + tree_obj(other.tree_obj), + entry(other.entry), + blob(other.blob), + out(other.out) { + other.tree_obj = nullptr; + other.entry = nullptr; + other.blob = nullptr; + other.out = GIT_BUF_INIT; + } + GitScope& operator=(GitScope&& other) noexcept { + if (this != &other) { + cleanup(); + tree_obj = other.tree_obj; + entry = other.entry; + blob = other.blob; + out = other.out; + other.tree_obj = nullptr; + other.entry = nullptr; + other.blob = nullptr; + other.out = GIT_BUF_INIT; + } + return *this; + } + + git_tree* tree() const { return reinterpret_cast(tree_obj); } + +private: + void cleanup() noexcept { + git_buf_dispose(&out); + if (blob) { + git_blob_free(blob); + blob = nullptr; + } + if (entry) { + git_tree_entry_free(entry); + entry = nullptr; + } + if (tree_obj) { + git_object_free(tree_obj); + tree_obj = nullptr; + } + } +}; + +void resumeLfsDownloadForFile(git_repository* repo, const char* filePathInRepo) { + GitScope g; + + // Resolve HEAD tree (origin/main^{tree}) + CHECK(git_revparse_single(&g.tree_obj, repo, "origin/main^{tree}")); + + // Find the tree entry by path + CHECK(git_tree_entry_bypath(&g.entry, g.tree(), filePathInRepo)); + + // Ensure it's a blob + if (git_tree_entry_type(g.entry) != GIT_OBJECT_BLOB) { + fprintf(stderr, "[ERROR] Path is not a blob: %s\n", filePathInRepo); + return; // Guard cleans up + } + + // Lookup the blob + CHECK(git_blob_lookup(&g.blob, repo, git_tree_entry_id(g.entry))); + // Configure filter behavior git_blob_filter_options opts = GIT_BLOB_FILTER_OPTIONS_INIT; // Choose direction: // GIT_BLOB_FILTER_TO_WORKTREE : apply smudge (as if writing to working tree) // GIT_BLOB_FILTER_TO_ODB : apply clean (as if writing to ODB) - // opts.flags = GIT_FILTER_TO_WORKTREE; + // opts.flags = GIT_BLOB_FILTER_TO_WORKTREE; - // Resolve HEAD tree - CHECK(git_revparse_single(&obj, repo, "origin/main^{tree}") != 0); - tree = (git_tree *)obj; + // Apply filters based on .gitattributes for this path (triggers LFS smudge/clean) + CHECK(git_blob_filter(&g.out, g.blob, filePathInRepo, &opts)); - // Find the tree entry and get the blob - CHECK(git_tree_entry_bypath(&entry, tree, file_path_in_repo) != 0); - CHECK(git_tree_entry_type(entry) != GIT_OBJECT_BLOB); - - CHECK(git_blob_lookup(&blob, repo, git_tree_entry_id(entry)) != 0); - - // Apply filters based on .gitattributes for this path - CHECK(git_blob_filter(&out, blob, file_path_in_repo, &opts) != 0); - - git_buf_dispose(&out); - if (blob) git_blob_free(blob); - if (entry) git_tree_entry_free(entry); - if (tree) git_tree_free(tree); - if (obj) git_object_free(obj); - return; + // We don't need the buffer contents; the filter side-effects are enough. + // All resources (out, blob, entry, tree_obj) will be freed automatically here. } Status HfDownloader::downloadModel() { @@ -433,7 +554,7 @@ Status HfDownloader::downloadModel() { if (std::filesystem::is_directory(this->downloadPath) && !this->overwriteModels) { // Checking if the download was partially finished for any files in repository auto matches = findLfsLikeFiles(this->downloadPath, true); - + if (matches.empty()) { std::cout << "No files to resume download found.\n"; std::cout << "Path already exists on local filesystem. Skipping download to path: " << this->downloadPath << std::endl; @@ -445,40 +566,31 @@ Status HfDownloader::downloadModel() { } } - git_repository *repo = NULL; - int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); - if (error < 0) { - const git_error *err = git_error_last(); - if (err) - SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); - else - SPDLOG_ERROR("Repository open failed: {}", error); - if (repo) git_repository_free(repo); - + GitRepositoryGuard repoGuard(this->downloadPath); + if (!repoGuard.get()) { std::cout << "Path already exists on local filesystem. And is not a git repository: " << this->downloadPath << std::endl; - return StatusCode::HF_GIT_CLONE_FAILED; + return StatusCode::HF_GIT_STATUS_FAILED; } // Set repository url std::string passRepoUrl = GetRepositoryUrlWithPassword(); const char* url = passRepoUrl.c_str(); - error = git_repository_set_url(repo, url); + int error = git_repository_set_url(repoGuard.get(), url); if (error < 0) { - const git_error *err = git_error_last(); + const git_error* err = git_error_last(); if (err) SPDLOG_ERROR("Repository set url failed: {} {}", err->klass, err->message); else SPDLOG_ERROR("Repository set url failed: {}", error); - if (repo) git_repository_free(repo); std::cout << "Path already exists on local filesystem. And set git repository url failed: " << this->downloadPath << std::endl; return StatusCode::HF_GIT_CLONE_FAILED; } for (const auto& p : matches) { - std::cout << " Resuming " << p.string() << "...\n"; - std::string path = p.string(); - resumeLfsDownloadForFile(repo, path.c_str()); - } + std::cout << " Resuming " << p.string() << "...\n"; + std::string path = p.string(); + resumeLfsDownloadForFile(repoGuard.get(), path.c_str()); + } SPDLOG_DEBUG("Checking repository status."); auto status = CheckRepositoryStatus(false); @@ -495,6 +607,7 @@ Status HfDownloader::downloadModel() { } SPDLOG_DEBUG("Downloading to path: {}", this->downloadPath); + git_repository* cloned_repo = NULL; // clone_opts for progress reporting set in libgit2 lib by patch git_clone_options clone_opts = GIT_CLONE_OPTIONS_INIT; @@ -523,7 +636,6 @@ Status HfDownloader::downloadModel() { SPDLOG_ERROR("Libgit2 clone error: {} message: {}", err->klass, err->message); else SPDLOG_ERROR("Libgit2 clone error: {}", error); - return StatusCode::HF_GIT_CLONE_FAILED; } else if (cloned_repo) { git_repository_free(cloned_repo); diff --git a/src/pull_module/libgit2.hpp b/src/pull_module/libgit2.hpp index b8dacac0e9..fb5549b16d 100644 --- a/src/pull_module/libgit2.hpp +++ b/src/pull_module/libgit2.hpp @@ -63,6 +63,5 @@ class HfDownloader : public IModelDownloader { bool CheckIfProxySet(); Status RemoveReadonlyFileAttributeFromDir(const std::string& directoryPath); Status CheckRepositoryStatus(bool checkUntracked); - int CheckRepositoryForResume(); }; } // namespace ovms diff --git a/src/status.cpp b/src/status.cpp index b83b4f7a45..a7750498e9 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -350,7 +350,6 @@ const std::unordered_map Status::statusMessageMap = { {StatusCode::HF_GIT_CLONE_FAILED, "Failed in libgit2 execution of clone method"}, {StatusCode::HF_GIT_STATUS_FAILED, "Failed in libgit2 execution of status method"}, {StatusCode::HF_GIT_STATUS_UNCLEAN, "Unclean status detected in libgit2 cloned repository"}, - {StatusCode::PARTIAL_END, "Request has finished and no further communication is needed"}, {StatusCode::NONEXISTENT_PATH, "Nonexistent path"}, diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index 9eba14b103..6043ea380e 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -153,7 +153,7 @@ const std::string expectedGraphContentsDraft = R"( )"; TEST_F(HfDownloaderPullHfModel, PositiveDownload) { - // GTEST_SKIP() << "Skipping test in CI - PositiveDownloadAndStart has full scope testing."; + GTEST_SKIP() << "Skipping test in CI - PositiveDownloadAndStart has full scope testing."; std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); std::string task = "text_generation"; @@ -178,14 +178,16 @@ bool removeSecondHalf(const std::string& filrStr) { ec.clear(); if (!fs::exists(file, ec) || !fs::is_regular_file(file, ec)) { - if (!ec) ec = std::make_error_code(std::errc::no_such_file_or_directory); + if (!ec) + ec = std::make_error_code(std::errc::no_such_file_or_directory); return false; } const std::uintmax_t size = fs::file_size(file, ec); - if (ec) return false; + if (ec) + return false; - const std::uintmax_t newSize = size / 2; // floor(size/2) + const std::uintmax_t newSize = size / 2; // floor(size/2) fs::resize_file(file, newSize, ec); return !ec; } @@ -196,10 +198,9 @@ bool createGitLfsPointerFile(const std::string& path) { return false; } - file << - "version https://git-lfs.github.com/spec/v1\n" - "oid sha256:cecf0224201415144c00cf3a6cf3350306f9c78888d631eb590939a63722fefa\n" - "size 52417240\n"; + file << "version https://git-lfs.github.com/spec/v1\n" + "oid sha256:cecf0224201415144c00cf3a6cf3350306f9c78888d631eb590939a63722fefa\n" + "size 52417240\n"; return true; } @@ -221,7 +222,7 @@ std::string sha256File(std::string_view path, std::error_code& ec) { } // Read in chunks to support large files without high memory usage. - std::vector buffer(1 << 20); // 1 MiB + std::vector buffer(1 << 20); // 1 MiB while (ifs) { ifs.read(reinterpret_cast(buffer.data()), static_cast(buffer.size())); std::streamsize got = ifs.gcount(); @@ -232,7 +233,7 @@ std::string sha256File(std::string_view path, std::error_code& ec) { } } } - if (!ifs.eof()) { // read failed not due to EOF + if (!ifs.eof()) { // read failed not due to EOF ec = std::make_error_code(std::errc::io_error); return {}; } From 17fcd6122fe52e48a39f36a8acfde41a88e4938b Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 11 Mar 2026 13:10:42 +0100 Subject: [PATCH 28/49] Use VS cmake --- windows_test.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows_test.bat b/windows_test.bat index 2fd5dcbfa7..f1a5a27b05 100644 --- a/windows_test.bat +++ b/windows_test.bat @@ -47,7 +47,7 @@ set "runTest=%cd%\bazel-bin\src\ovms_test.exe --gtest_filter=!gtestFilter! > win :: Setting PATH environment variable based on default windows node settings: Added ovms_windows specific python settings and c:/opt and removed unused Nvidia and OCL specific tools. :: When changing the values here you can print the node default PATH value and base your changes on it. -set "setPath=C:\opt;C:\opt\Python312\;C:\opt\Python312\Scripts\;C:\opt\msys64\usr\bin\;C:\opt\curl-8.18.0_4-win64-mingw\bin;%PATH%;" +set "setPath=C:\opt;C:\opt\Python312\;C:\opt\Python312\Scripts\;C:\opt\msys64\usr\bin\;C:\opt\curl-8.18.0_4-win64-mingw\bin;c:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\;%PATH%;" set "setPythonPath=%cd%\bazel-out\x64_windows-opt\bin\src\python\binding" set "BAZEL_SH=C:\opt\msys64\usr\bin\bash.exe" From 10419500d2ec940b7d944cedef18a98e39f640fa Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 11 Mar 2026 16:21:33 +0100 Subject: [PATCH 29/49] Short name --- src/test/pull_hf_model_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index 6043ea380e..0290e6bf25 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -253,7 +253,7 @@ std::string sha256File(std::string_view path, std::error_code& ec) { return oss.str(); } -TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndResumeFromPArtialDownload) { +TEST_F(HfDownloaderPullHfModel, Resume) { std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); std::string task = "text_generation"; From 5a1d74051363931f4ebd6a3c4c81b7b98977b5d0 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 11 Mar 2026 17:19:42 +0100 Subject: [PATCH 30/49] Unit tests --- src/BUILD | 1 + src/pull_module/libgit2.cpp | 14 +- src/pull_module/libgit2.hpp | 12 +- src/test/libgit2_test.cpp | 334 ++++++++++++++++++++++++++++++++++++ 4 files changed, 353 insertions(+), 8 deletions(-) create mode 100644 src/test/libgit2_test.cpp diff --git a/src/BUILD b/src/BUILD index d3e5af3861..b9fb30e69b 100644 --- a/src/BUILD +++ b/src/BUILD @@ -2470,6 +2470,7 @@ cc_test( "test/kfs_rest_test.cpp", "test/kfs_rest_parser_test.cpp", "test/layout_test.cpp", + "test/libgit2_test.cpp", "test/metric_config_test.cpp", "test/metrics_test.cpp", "test/metrics_flow_test.cpp", diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 14f7bcdb89..9b9b29e1f0 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -312,7 +312,7 @@ Status HfDownloader::CheckRepositoryStatus(bool checkUntracked) { } while (0) // Trim trailing '\r' (for CRLF files) and surrounding spaces -static inline void rtrimCrLfWhitespace(std::string& s) { +void rtrimCrLfWhitespace(std::string& s) { if (!s.empty() && s.back() == '\r') s.pop_back(); // remove trailing '\r' while (!s.empty() && std::isspace(static_cast(s.back()))) @@ -325,7 +325,7 @@ static inline void rtrimCrLfWhitespace(std::string& s) { } // Case-insensitive substring search: returns true if 'needle' is found in 'hay' -static bool containsCaseInsensitive(const std::string& hay, const std::string& needle) { +bool containsCaseInsensitive(const std::string& hay, const std::string& needle) { auto toLower = [](std::string v) { std::transform(v.begin(), v.end(), v.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); @@ -338,7 +338,7 @@ static bool containsCaseInsensitive(const std::string& hay, const std::string& n // Read at most the first 3 lines of a file, with a per-line cap to avoid huge reads. // Returns true if successful (even if <3 lines exist; vector will just be shorter). -static bool readFirstThreeLines(const fs::path& p, std::vector& outLines) { +bool readFirstThreeLines(const fs::path& p, std::vector& outLines) { outLines.clear(); std::ifstream in(p, std::ios::in | std::ios::binary); if (!in) @@ -383,7 +383,7 @@ static bool readFirstThreeLines(const fs::path& p, std::vector& out // Check if the first 3 lines contain required keywords in positional order: // line1 -> "version", line2 -> "oid", line3 -> "size" (case-insensitive). -static bool fileHasLfsKeywordsFirst3Positional(const fs::path& p) { +bool fileHasLfsKeywordsFirst3Positional(const fs::path& p) { std::error_code ec; if (!fs::is_regular_file(p, ec)) return false; @@ -401,7 +401,7 @@ static bool fileHasLfsKeywordsFirst3Positional(const fs::path& p) { } // Helper: make path relative to base (best-effort, non-throwing). -static fs::path makeRelativeToBase(const fs::path& path, const fs::path& base) { +fs::path makeRelativeToBase(const fs::path& path, const fs::path& base) { std::error_code ec; // Try fs::relative first (handles canonical comparisons, may fail if on different roots) fs::path rel = fs::relative(path, base, ec); @@ -419,8 +419,8 @@ static fs::path makeRelativeToBase(const fs::path& path, const fs::path& base) { return path; } -// Find all files under 'directory' that satisfy the first-3-lines LFS keyword check. -static std::vector findLfsLikeFiles(const std::string& directory, bool recursive = true) { +// Find all files under 'directory' that satisfy the first-3-lines LFS keyword check. Default: bool recursive = true +std::vector findLfsLikeFiles(const std::string& directory, bool recursive) { std::vector matches; std::error_code ec; diff --git a/src/pull_module/libgit2.hpp b/src/pull_module/libgit2.hpp index fb5549b16d..f2da9f875f 100644 --- a/src/pull_module/libgit2.hpp +++ b/src/pull_module/libgit2.hpp @@ -15,8 +15,10 @@ // limitations under the License. //***************************************************************************** #pragma once -#include +#include #include +#include +#include #include #include @@ -31,6 +33,7 @@ namespace ovms { class Status; +namespace fs = std::filesystem; /* * libgit2 options. 0 is the default value @@ -64,4 +67,11 @@ class HfDownloader : public IModelDownloader { Status RemoveReadonlyFileAttributeFromDir(const std::string& directoryPath); Status CheckRepositoryStatus(bool checkUntracked); }; + +void rtrimCrLfWhitespace(std::string& s); +bool containsCaseInsensitive(const std::string& hay, const std::string& needle); +bool readFirstThreeLines(const fs::path& p, std::vector& outLines); +bool fileHasLfsKeywordsFirst3Positional(const fs::path& p); +fs::path makeRelativeToBase(const fs::path& path, const fs::path& base); +std::vector findLfsLikeFiles(const std::string& directory, bool recursive = true); } // namespace ovms diff --git a/src/test/libgit2_test.cpp b/src/test/libgit2_test.cpp new file mode 100644 index 0000000000..0a2e1f6328 --- /dev/null +++ b/src/test/libgit2_test.cpp @@ -0,0 +1,334 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include + +#include +#include + +#include "src/pull_module/libgit2.hpp" + +#include "environment.hpp" + +namespace fs = std::filesystem; + +TEST(LibGit2RtrimCrLfWhitespace, EmptyString) { + std::string s; + ovms::rtrimCrLfWhitespace(s); + EXPECT_TRUE(s.empty()); +} + +TEST(LibGit2RtrimCrLfWhitespace, NoWhitespace) { + std::string s = "abc"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, OnlySpaces) { + std::string s = " "; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, ""); +} + +TEST(LibGit2RtrimCrLfWhitespace, LeadingSpacesOnly) { + std::string s = " abc"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, TrailingSpacesOnly) { + std::string s = "abc "; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, LeadingAndTrailingSpaces) { + std::string s = " abc "; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, TabsAndNewlinesAround) { + std::string s = "\t\n abc \n\t"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, AllCWhitespaceAround) { + // Include space, tab, newline, vertical tab, form feed, carriage return + std::string s = " \t\n\v\f\rabc\r\f\v\n\t "; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, PreserveInternalSpaces) { + std::string s = " a b c "; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "a b c"); +} + +TEST(LibGit2RtrimCrLfWhitespace, TrailingCRLF) { + // Windows-style line ending: "\r\n" + std::string s = "abc\r\n"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, TrailingCROnly) { + std::string s = "abc\r"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, TrailingLFOnly) { + std::string s = "abc\n"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, MultipleTrailingCRs) { + // Only one trailing '\r' is specially removed first, but then trailing + // whitespace loop will remove any remaining CRs (since isspace('\r') == true). + std::string s = "abc\r\r\r"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, LeadingCRLFAndSpaces) { + std::string s = "\r\n abc"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, InternalCRLFShouldRemainIfNotLeadingOrTrailing) { + // Internal whitespace should be preserved + std::string s = "a\r\nb"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "a\r\nb"); +} + +TEST(LibGit2RtrimCrLfWhitespace, OnlyCRLFAndWhitespace) { + std::string s = "\r\n\t \r"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, ""); +} + +TEST(LibGit2RtrimCrLfWhitespace, NonAsciiBytesAreNotTrimmedByIsspace) { + // 0xC2 0xA0 is UTF-8 for NO-BREAK SPACE; bytes individually are not ASCII spaces. + // isspace() on unsigned char typically returns false for these bytes in the "C" locale. + // So they should remain unless at edges and recognized by the current locale (usually not). + std::string s = "\xC2""\xA0""abc""\xC2""\xA0"; + ovms::rtrimCrLfWhitespace(s); + // Expect unchanged because these bytes are not recognized by std::isspace in C locale + EXPECT_EQ(s, "\xC2""\xA0""abc""\xC2""\xA0"); +} + +TEST(LibGit2RtrimCrLfWhitespace, Idempotent) { + std::string s = " abc \n"; + ovms::rtrimCrLfWhitespace(s); + auto once = s; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, once); +} + + +TEST(LibGit2ContainsCaseInsensitiveTest, ExactMatch) { + EXPECT_TRUE(ovms::containsCaseInsensitive("hello", "hello")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, MixedCaseMatch) { + EXPECT_TRUE(ovms::containsCaseInsensitive("HeLLo WoRLD", "world")); + EXPECT_TRUE(ovms::containsCaseInsensitive("HeLLo WoRLD", "HELLO")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, NoMatch) { + EXPECT_FALSE(ovms::containsCaseInsensitive("abcdef", "gh")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, EmptyNeedleReturnsTrue) { + // Consistent with std::string::find("") → 0 + EXPECT_TRUE(ovms::containsCaseInsensitive("something", "")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, EmptyHaystackNonEmptyNeedleReturnsFalse) { + EXPECT_FALSE(ovms::containsCaseInsensitive("", "abc")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, BothEmptyReturnsTrue) { + EXPECT_TRUE(ovms::containsCaseInsensitive("", "")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, SubstringAtBeginning) { + EXPECT_TRUE(ovms::containsCaseInsensitive("HelloWorld", "hello")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, SubstringInMiddle) { + EXPECT_TRUE(ovms::containsCaseInsensitive("abcHELLOxyz", "hello")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, SubstringAtEnd) { + EXPECT_TRUE(ovms::containsCaseInsensitive("testCASE", "case")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, NoFalsePositives) { + EXPECT_FALSE(ovms::containsCaseInsensitive("aaaaa", "b")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, UnicodeCharactersSafeButNotSpecialHandled) { + // std::tolower only reliably handles unsigned char range. + // This ensures your implementation does not crash or behave strangely. + EXPECT_FALSE(ovms::containsCaseInsensitive("ĄĆĘŁ", "ę")); // depends on locale; ASCII-only expected false +} + + +// A helper for writing test files. +static fs::path writeTempFile(const std::string& filename, + const std::string& content) { + fs::path p = fs::temp_directory_path() / filename; + std::ofstream out(p, std::ios::binary); + out << content; + return p; +} + +TEST(LibGit2ReadFirstThreeLinesTest, FileNotFoundReturnsFalse) { + std::vector lines; + fs::path p = fs::temp_directory_path() / "nonexistent_12345.txt"; + EXPECT_FALSE(ovms::readFirstThreeLines(p, lines)); + EXPECT_TRUE(lines.empty()); +} + +TEST(LibGit2ReadFirstThreeLinesTest, ReadsExactlyThreeLines) { + fs::path p = writeTempFile("three_lines.txt", + "line1\n" + "line2\n" + "line3\n" + "extra\n"); // should be ignored + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + ASSERT_EQ(out.size(), 3u); + EXPECT_EQ(out[0], "line1"); + EXPECT_EQ(out[1], "line2"); + EXPECT_EQ(out[2], "line3"); +} + +TEST(LibGit2ReadFirstThreeLinesTest, ReadsFewerThanThreeLines) { + fs::path p = writeTempFile("two_lines.txt", + "alpha\n" + "beta\n"); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + ASSERT_EQ(out.size(), 2u); + EXPECT_EQ(out[0], "alpha"); + EXPECT_EQ(out[1], "beta"); +} + +TEST(LibGit2ReadFirstThreeLinesTest, ReadsOneLineOnly) { + fs::path p = writeTempFile("one_line.txt", "solo\n"); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + ASSERT_EQ(out.size(), 1u); + EXPECT_EQ(out[0], "solo"); +} + +TEST(LibGit2ReadFirstThreeLinesTest, EmptyFileProducesZeroLinesAndReturnsTrue) { + fs::path p = writeTempFile("empty.txt", ""); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + EXPECT_TRUE(out.empty()); +} + +TEST(LibGit2ReadFirstThreeLinesTest, CRLFIsTrimmedCorrectly) { + fs::path p = writeTempFile("crlf.txt", + "hello\r\n" + "world\r\n"); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + ASSERT_EQ(out.size(), 2u); + EXPECT_EQ(out[0], "hello"); + EXPECT_EQ(out[1], "world"); +} + +TEST(LibGit2ReadFirstThreeLinesTest, LoneCRAndLFAreTrimmed) { + fs::path p = writeTempFile("mixed_newlines.txt", + "a\r" + "b\n" + "c\r\n"); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + + ASSERT_EQ(out.size(), 3u); + EXPECT_EQ(out[0], "a"); + EXPECT_EQ(out[1], "b"); + EXPECT_EQ(out[2], "c"); +} + +TEST(LibGit2ReadFirstThreeLinesTest, VeryLongLineTriggersDrainLogic) { + constexpr size_t kMax = 8192; + std::string longLine(kMax, 'x'); + std::string content = longLine + "OVERFLOWTHATSHOULDBEDISCARDED\n" // should be truncated + "line2\n" + "line3\n"; + + fs::path p = writeTempFile("long_line.txt", content); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + + ASSERT_EQ(out.size(), 3u); + + // First line should be exactly kMax chars of 'x' + ASSERT_EQ(out[0].size(), kMax); + EXPECT_EQ(out[0], std::string(kMax, 'x')); + + EXPECT_EQ(out[1], "line2"); + EXPECT_EQ(out[2], "line3"); +} + +TEST(LibGit2ReadFirstThreeLinesTest, HandlesEOFWithoutNewlineAtEnd) { + fs::path p = writeTempFile("eof_no_newline.txt", + "first\n" + "second\n" + "third_without_newline"); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + + ASSERT_EQ(out.size(), 3u); + EXPECT_EQ(out[0], "first"); + EXPECT_EQ(out[1], "second"); + EXPECT_EQ(out[2], "third_without_newline"); +} + +TEST(LibGit2ReadFirstThreeLinesTest, TrailingWhitespacePreservedExceptCRLF) { + fs::path p = writeTempFile("spaces.txt", + "abc \n" + "def\t\t\n"); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + + ASSERT_EQ(out.size(), 2u); + EXPECT_EQ(out[0], "abc "); // spaces preserved + EXPECT_EQ(out[1], "def\t\t"); // tabs preserved +} From 214161c1f8584335731923562b1fb50e7e45f390 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 12 Mar 2026 17:15:00 +0100 Subject: [PATCH 31/49] Unit tests2 --- src/pull_module/libgit2.cpp | 69 +++-- src/test/libgit2_test.cpp | 508 +++++++++++++++++++++++++++++++++--- 2 files changed, 505 insertions(+), 72 deletions(-) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 9b9b29e1f0..292530264a 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -338,46 +339,45 @@ bool containsCaseInsensitive(const std::string& hay, const std::string& needle) // Read at most the first 3 lines of a file, with a per-line cap to avoid huge reads. // Returns true if successful (even if <3 lines exist; vector will just be shorter). -bool readFirstThreeLines(const fs::path& p, std::vector& outLines) { - outLines.clear(); - std::ifstream in(p, std::ios::in | std::ios::binary); + +bool readFirstThreeLines(const std::filesystem::path& p, std::vector& out) { + out.clear(); + + std::ifstream in(p, std::ios::binary); if (!in) return false; - constexpr std::streamsize kMaxPerLine = 8192; - std::string line; - line.reserve(static_cast(kMaxPerLine)); - for (int i = 0; i < 3 && in.good(); ++i) { - line.clear(); - std::streamsize count = 0; - char ch; - bool gotNewline = false; - while (count < kMaxPerLine && in.get(ch)) { - if (ch == '\n') { - gotNewline = true; - break; - } - line.push_back(ch); - ++count; - } - // If we hit kMaxPerLine without encountering '\n', drain until newline to resync - if (count == kMaxPerLine && !gotNewline) { - while (in.get(ch)) { - if (ch == '\n') - break; + line.reserve(256); // small optimization + int c; + + while (out.size() < 3 && (c = in.get()) != EOF) { + if (c == '\r') { + // Handle CR or CRLF as one line ending + int next = in.peek(); + if (next == '\n') { + in.get(); // consume '\n' } + // finalize current line + rtrimCrLfWhitespace(line); + out.push_back(std::move(line)); + line.clear(); + } else if (c == '\n') { + // LF line ending + rtrimCrLfWhitespace(line); + out.push_back(std::move(line)); + line.clear(); + } else { + line.push_back(static_cast(c)); } + } - if (!in && line.empty()) { - // EOF with no data accumulated; if previous lines were read, that's fine. - break; - } + // Handle the last line if file did not end with EOL + if (!line.empty() && out.size() < 3) { rtrimCrLfWhitespace(line); - outLines.push_back(line); - if (!in) - break; // Handle EOF gracefully + out.push_back(std::move(line)); } + return true; } @@ -593,12 +593,7 @@ Status HfDownloader::downloadModel() { } SPDLOG_DEBUG("Checking repository status."); - auto status = CheckRepositoryStatus(false); - if (!status.ok()) { - return status; - } - - return StatusCode::OK; + return CheckRepositoryStatus(false); } auto status = IModelDownloader::checkIfOverwriteAndRemove(); diff --git a/src/test/libgit2_test.cpp b/src/test/libgit2_test.cpp index 0a2e1f6328..b221ee9c54 100644 --- a/src/test/libgit2_test.cpp +++ b/src/test/libgit2_test.cpp @@ -15,7 +15,9 @@ //***************************************************************************** #include #include +#include #include +#include #include #include @@ -132,10 +134,18 @@ TEST(LibGit2RtrimCrLfWhitespace, NonAsciiBytesAreNotTrimmedByIsspace) { // 0xC2 0xA0 is UTF-8 for NO-BREAK SPACE; bytes individually are not ASCII spaces. // isspace() on unsigned char typically returns false for these bytes in the "C" locale. // So they should remain unless at edges and recognized by the current locale (usually not). - std::string s = "\xC2""\xA0""abc""\xC2""\xA0"; + std::string s = "\xC2" + "\xA0" + "abc" + "\xC2" + "\xA0"; ovms::rtrimCrLfWhitespace(s); // Expect unchanged because these bytes are not recognized by std::isspace in C locale - EXPECT_EQ(s, "\xC2""\xA0""abc""\xC2""\xA0"); + EXPECT_EQ(s, "\xC2" + "\xA0" + "abc" + "\xC2" + "\xA0"); } TEST(LibGit2RtrimCrLfWhitespace, Idempotent) { @@ -146,7 +156,6 @@ TEST(LibGit2RtrimCrLfWhitespace, Idempotent) { EXPECT_EQ(s, once); } - TEST(LibGit2ContainsCaseInsensitiveTest, ExactMatch) { EXPECT_TRUE(ovms::containsCaseInsensitive("hello", "hello")); } @@ -192,13 +201,12 @@ TEST(LibGit2ContainsCaseInsensitiveTest, NoFalsePositives) { TEST(LibGit2ContainsCaseInsensitiveTest, UnicodeCharactersSafeButNotSpecialHandled) { // std::tolower only reliably handles unsigned char range. // This ensures your implementation does not crash or behave strangely. - EXPECT_FALSE(ovms::containsCaseInsensitive("ĄĆĘŁ", "ę")); // depends on locale; ASCII-only expected false + EXPECT_FALSE(ovms::containsCaseInsensitive("ĄĆĘŁ", "ę")); // depends on locale; ASCII-only expected false } - // A helper for writing test files. static fs::path writeTempFile(const std::string& filename, - const std::string& content) { + const std::string& content) { fs::path p = fs::temp_directory_path() / filename; std::ofstream out(p, std::ios::binary); out << content; @@ -217,7 +225,7 @@ TEST(LibGit2ReadFirstThreeLinesTest, ReadsExactlyThreeLines) { "line1\n" "line2\n" "line3\n" - "extra\n"); // should be ignored + "extra\n"); // should be ignored std::vector out; EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); @@ -283,33 +291,11 @@ TEST(LibGit2ReadFirstThreeLinesTest, LoneCRAndLFAreTrimmed) { EXPECT_EQ(out[2], "c"); } -TEST(LibGit2ReadFirstThreeLinesTest, VeryLongLineTriggersDrainLogic) { - constexpr size_t kMax = 8192; - std::string longLine(kMax, 'x'); - std::string content = longLine + "OVERFLOWTHATSHOULDBEDISCARDED\n" // should be truncated - "line2\n" - "line3\n"; - - fs::path p = writeTempFile("long_line.txt", content); - - std::vector out; - EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); - - ASSERT_EQ(out.size(), 3u); - - // First line should be exactly kMax chars of 'x' - ASSERT_EQ(out[0].size(), kMax); - EXPECT_EQ(out[0], std::string(kMax, 'x')); - - EXPECT_EQ(out[1], "line2"); - EXPECT_EQ(out[2], "line3"); -} - TEST(LibGit2ReadFirstThreeLinesTest, HandlesEOFWithoutNewlineAtEnd) { fs::path p = writeTempFile("eof_no_newline.txt", - "first\n" - "second\n" - "third_without_newline"); + "first\n" + "second\n" + "third_without_newline"); std::vector out; EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); @@ -320,7 +306,7 @@ TEST(LibGit2ReadFirstThreeLinesTest, HandlesEOFWithoutNewlineAtEnd) { EXPECT_EQ(out[2], "third_without_newline"); } -TEST(LibGit2ReadFirstThreeLinesTest, TrailingWhitespacePreservedExceptCRLF) { +TEST(LibGit2ReadFirstThreeLinesTest, TrailingWhitespaceNotPreserved) { fs::path p = writeTempFile("spaces.txt", "abc \n" "def\t\t\n"); @@ -329,6 +315,458 @@ TEST(LibGit2ReadFirstThreeLinesTest, TrailingWhitespacePreservedExceptCRLF) { EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); ASSERT_EQ(out.size(), 2u); - EXPECT_EQ(out[0], "abc "); // spaces preserved - EXPECT_EQ(out[1], "def\t\t"); // tabs preserved + EXPECT_EQ(out[0], "abc"); // spaces preserved + EXPECT_EQ(out[1], "def"); // tabs preserved +} + +// Optional: If you need to call readFirstThreeLines in any test-specific checks, +// declare it too (remove if unused here). +// bool readFirstThreeLines(const fs::path& p, std::vector& out); + +// ---- Test Utilities ---- + +// Create a unique temporary directory inside the system temp directory. +static fs::path createTempDir() { + const fs::path base = fs::temp_directory_path(); + std::random_device rd; + std::mt19937_64 gen(rd()); + std::uniform_int_distribution dist; + + // Try a reasonable number of times to avoid rare collisions + for (int attempt = 0; attempt < 100; ++attempt) { + auto candidate = base / ("lfs_kw_tests_" + std::to_string(dist(gen))); + std::error_code ec; + if (fs::create_directory(candidate, ec)) { + return candidate; + } + // If creation failed due to existing path, loop and try another name + // Otherwise (e.g., permissions), fall through and try again up to limit + } + + throw std::runtime_error("Failed to create a unique temporary directory"); +} + +static fs::path writeFile(const fs::path& dir, const std::string& name, const std::string& content) { + fs::path p = dir / name; + std::ofstream out(p, std::ios::binary); + if (!out) + throw std::runtime_error("Failed to create file: " + p.string()); + out.write(content.data(), static_cast(content.size())); + return p; +} + +// A simple RAII for a temp directory +struct TempDir { + fs::path dir; + TempDir() : + dir(createTempDir()) { + if (dir.empty()) + throw std::runtime_error("Failed to create temp directory"); + } + ~TempDir() { + std::error_code ec; + fs::remove_all(dir, ec); + } +}; + +class LibGit2FileHasLfsKeywordsFirst3PositionalTest : public ::testing::Test { +protected: + TempDir td; +}; + +// ---- Tests ---- + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, ReturnsFalseForNonExistingFile) { + fs::path p = td.dir / "does_not_exist.txt"; + EXPECT_FALSE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, ReturnsFalseForDirectoryPath) { + // Passing the directory itself (not a regular file) + EXPECT_FALSE(ovms::fileHasLfsKeywordsFirst3Positional(td.dir)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, ReturnsFalseForEmptyFile) { + auto p = writeFile(td.dir, "empty.txt", ""); + EXPECT_FALSE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, ReturnsFalseForLessThanThreeLines) { + { + auto p = writeFile(td.dir, "one_line.txt", "version something\n"); + EXPECT_FALSE(ovms::fileHasLfsKeywordsFirst3Positional(p)); + } + { + auto p = writeFile(td.dir, "two_lines.txt", "version x\n" + "oid y\n"); + EXPECT_FALSE(ovms::fileHasLfsKeywordsFirst3Positional(p)); + } +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, HappyPathCaseInsensitiveAndExtraContent) { + // Lines contain the keywords somewhere (case-insensitive), extra content is okay. + const std::string content = + " VeRsIoN https://git-lfs.github.com/spec/v1 \n" + "\toid Sha256:abcdef1234567890\n" + "size 999999 \t \n"; + auto p = writeFile(td.dir, "ok.txt", content); + EXPECT_TRUE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, WrongOrderShouldFail) { + // Put keywords in wrong lines + const std::string content = + "size 100\n" + "version something\n" + "oid abc\n"; + auto p = writeFile(td.dir, "wrong_order.txt", content); + EXPECT_FALSE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, MissingKeywordShouldFail) { + // Line1 has version, line2 missing oid, line3 has size + const std::string content = + "version v1\n" + "hash sha256:abc\n" + "size 42\n"; + auto p = writeFile(td.dir, "missing_keyword.txt", content); + EXPECT_FALSE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, MixedNewlines_CR_LF_CRLF_ShouldPass) { + // Requires readFirstThreeLines to treat \r, \n, and \r\n as line breaks. + const std::string content = + "version one\r" + "oid two\n" + "size three\r\n"; + auto p = writeFile(td.dir, "mixed_newlines.txt", content); + EXPECT_TRUE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, LeadingAndTrailingWhitespaceDoesNotBreak) { + // Assuming readFirstThreeLines trims edge whitespace; otherwise contains() still works + const std::string content = + " version \n" + "\t oid\t\n" + " size \t\n"; + auto p = writeFile(td.dir, "whitespace.txt", content); + EXPECT_TRUE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, KeywordsMayAppearWithinLongerTextOnEachLine) { + const std::string content = + "prefix-version-suffix\n" + "some_oid_here\n" + "the_size_is_here\n"; + auto p = writeFile(td.dir, "contains_substrings.txt", content); + EXPECT_TRUE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, CaseInsensitiveCheck) { + const std::string content = + "VerSiOn 1\n" + "OID something\n" + "SiZe 123\n"; + auto p = writeFile(td.dir, "case_insensitive.txt", content); + EXPECT_TRUE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, ExtraLinesAfterFirstThreeDoNotMatter) { + const std::string content = + "version v1\n" + "oid abc\n" + "size 42\n" + "EXTRA LINE THAT SHOULD NOT AFFECT RESULT\n"; + auto p = writeFile(td.dir, "extra_lines.txt", content); + EXPECT_TRUE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +class LibGit2MakeRelativeToBaseTest : public ::testing::Test { +protected: + TempDir td; +}; + +// Base is an ancestor of path → should return the relative tail. +TEST_F(LibGit2MakeRelativeToBaseTest, BaseIsAncestor) { + fs::path base = td.dir / "root"; + fs::path sub = base / "a" / "b" / "file.txt"; + + std::error_code ec; + fs::create_directories(sub.parent_path(), ec); + + fs::path rel = ovms::makeRelativeToBase(sub, base); + // Expected: "a/b/file.txt" (platform-correct separators) + EXPECT_EQ(rel, fs::path("a") / "b" / "file.txt"); +} + +// Path equals base → fs::relative returns "." (non-empty), we keep it. +TEST_F(LibGit2MakeRelativeToBaseTest, PathEqualsBase) { + fs::path base = td.dir / "same"; + std::error_code ec; + fs::create_directories(base, ec); + + fs::path rel = ovms::makeRelativeToBase(base, base); + EXPECT_EQ(rel, fs::path(".")); +} + +// Sibling subtree: base is ancestor of both; result is still relative path from base. +TEST_F(LibGit2MakeRelativeToBaseTest, SiblingSubtree) { + fs::path base = td.dir / "root2"; + fs::path a = base / "a" / "deep" / "fileA.txt"; + fs::path b = base / "b"; + + std::error_code ec; + fs::create_directories(a.parent_path(), ec); + fs::create_directories(b, ec); + + fs::path rel = ovms::makeRelativeToBase(a, base); + EXPECT_EQ(rel, fs::path("a") / "deep" / "fileA.txt"); +} + +// Base is not an ancestor but on same root → return a proper upward relative like "../x/y". +TEST_F(LibGit2MakeRelativeToBaseTest, BaseIsNotAncestorButSameRoot) { + fs::path base = td.dir / "top" / "left"; + fs::path path = td.dir / "top" / "right" / "x" / "y.txt"; + + std::error_code ec; + fs::create_directories(base, ec); + fs::create_directories(path.parent_path(), ec); + + fs::path rel = ovms::makeRelativeToBase(path, base); + // From .../top/left to .../top/right/x/y.txt → "../right/x/y.txt" + EXPECT_EQ(rel, fs::path("..") / "right" / "x" / "y.txt"); +} + +// Works even if paths do not exist (lexical computation should still yield a sensible result) +TEST_F(LibGit2MakeRelativeToBaseTest, NonExistingPathsLexicalStillWorks) { + fs::path base = td.dir / "ghost" / "base"; + fs::path path = td.dir / "ghost" / "base" / "sub" / "file.dat"; + // No directories created + + fs::path rel = ovms::makeRelativeToBase(path, base); + EXPECT_EQ(rel, fs::path("sub") / "file.dat"); +} + +// Last resort on Windows: different drive letters → fs::relative fails, +// lexically_relative returns empty → function should return filename only. +#ifdef _WIN32 +TEST_F(LibGit2MakeRelativeToBaseTest, DifferentDrivesReturnsFilenameOnly) { + // NOTE: We don't touch the filesystem; we only test the path logic. + // Choose typical drive letters; test won't fail if the drive doesn't exist + // because we don't access the filesystem in lexically_relative path. + fs::path path = fs::path("D:\\folder\\file.txt"); + fs::path base = fs::path("C:\\another\\base"); + + fs::path rel = ovms::makeRelativeToBase(path, base); + EXPECT_EQ(rel, fs::path("file.txt")); +} +#endif + +// If path has no filename (e.g., it's a root), last resort returns path itself. +// On POSIX, "/" has no filename; on Windows, "C:\\" has no filename either. +TEST_F(LibGit2MakeRelativeToBaseTest, NoFilenameEdgeCaseReturnsPathItself) { + fs::path base = td.dir; // arbitrary +#if defined(_WIN32) + // Construct a path that has no filename: root-name + root-directory + // We can't know the system drive at compile time; use a generic root directory. + // For the test, we simulate a root-only path lexically. + fs::path path = fs::path("C:\\"); // has no filename +#else + fs::path path = fs::path("../.."); // has no filename +#endif + + fs::path rel = ovms::makeRelativeToBase(path, base); + EXPECT_EQ(rel, path); +} + +static void mkdirs(const fs::path& p) { + std::error_code ec; + fs::create_directories(p, ec); +} + +class LibGit2FindLfsLikeFilesTest : public ::testing::Test { +protected: + TempDir td; + + // Utility: sort paths lexicographically for deterministic comparison + static void sortPaths(std::vector& v) { + std::sort(v.begin(), v.end(), [](const fs::path& a, const fs::path& b) { + return a.generic_string() < b.generic_string(); + }); + } +}; + +// --- Tests --- + +TEST_F(LibGit2FindLfsLikeFilesTest, NonExistingDirectoryReturnsEmpty) { + fs::path nonexist = td.dir / "does_not_exist"; + auto matches = ovms::findLfsLikeFiles(nonexist.string(), /*recursive=*/true); + EXPECT_TRUE(matches.empty()); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, EmptyDirectoryReturnsEmpty) { + auto matches = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/true); + EXPECT_TRUE(matches.empty()); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, NonRecursiveFindsOnlyTopLevelMatches) { + // Layout: + // td.dir/ + // match_top.txt (should match) + // nomatch_top.txt (should not match) + // sub/ + // match_nested.txt (should match but NOT included in non-recursive) + // Matching condition: lines[0] contains "version", lines[1] contains "oid", lines[2] contains "size" + + // Create top-level files + writeFile(td.dir, "match_top.txt", + "version v1\n" + "oid sha256:abc\n" + "size 123\n"); + + writeFile(td.dir, "nomatch_top.txt", + "version v1\n" + "hash something\n" // missing "oid" on line 2 + "size 123\n"); + + // Create nested directory and file + fs::path sub = td.dir / "sub"; + mkdirs(sub); + writeFile(sub, "match_nested.txt", + " VERSION v1 \n" + "\toid: 123\n" + "size: 42\n"); + + auto matches = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/false); + sortPaths(matches); + + std::vector expected = {fs::path("match_top.txt")}; + sortPaths(expected); + + EXPECT_EQ(matches, expected); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, RecursiveFindsNestedMatches) { + // Same layout as previous test but recursive = true; should include nested match as relative path + writeFile(td.dir, "top_match.txt", + "version spec\n" + "oid hash\n" + "size 1\n"); + + fs::path sub = td.dir / "a" / "b"; + mkdirs(sub); + writeFile(sub, "nested_match.txt", + "VeRsIoN\n" + "OID x\n" + "SiZe y\n"); + + // Add a deeper non-match to ensure it is ignored + fs::path deeper = td.dir / "a" / "b" / "c"; + mkdirs(deeper); + writeFile(deeper, "deep_nomatch.txt", + "hello\n" + "world\n" + "!\n"); + + auto matches = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/true); + sortPaths(matches); + + std::vector expected = { + fs::path("top_match.txt"), + fs::path("a") / "b" / "nested_match.txt"}; + sortPaths(expected); + + EXPECT_EQ(matches, expected); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, MixedNewlinesInMatchingFilesAreHandled) { + // Requires underlying readFirstThreeLines + fileHasLfsKeywordsFirst3Positional to handle \r, \n, \r\n + writeFile(td.dir, "mixed1.txt", + "version one\r" + "oid two\n" + "size three\r\n"); + + auto matches = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/false); + + ASSERT_EQ(matches.size(), 1u); + EXPECT_EQ(matches[0], fs::path("mixed1.txt")); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, WrongOrderOrMissingKeywordsAreNotIncluded) { + writeFile(td.dir, "wrong_order.txt", + "size 1\n" + "version 2\n" + "oid 3\n"); // wrong order → should not match + + writeFile(td.dir, "missing_second.txt", + "version v1\n" + "hash something\n" // missing "oid" + "size 3\n"); + + auto matches = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/false); + EXPECT_TRUE(matches.empty()); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, OnlyRegularFilesConsidered) { + // Create a directory with LFS-like name to ensure it isn't treated as a file + fs::path lfsdir = td.dir / "version_oid_size_dir"; + mkdirs(lfsdir); + + // No files → nothing should match + auto matches = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/true); + EXPECT_TRUE(matches.empty()); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, ReturnsPathsRelativeToBaseDirectory) { + // Ensure results are made relative to the provided base dir. + writeFile(td.dir, "root_match.txt", + "version v\n" + "oid o\n" + "size s\n"); + fs::path sub = td.dir / "x" / "y"; + mkdirs(sub); + writeFile(sub, "nested_match.txt", + "version v\n" + "oid o\n" + "size s\n"); + + auto matches = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/true); + sortPaths(matches); + + std::vector expected = { + fs::path("root_match.txt"), + fs::path("x") / "y" / "nested_match.txt"}; + sortPaths(expected); + + EXPECT_EQ(matches, expected); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, NonRecursiveDoesNotDescendButStillUsesRelativePaths) { + fs::path sub = td.dir / "subdir"; + mkdirs(sub); + + writeFile(td.dir, "toplevel.txt", + "version a\n" + "oid b\n" + "size c\n"); + + writeFile(sub, "nested.txt", + "version a\n" + "oid b\n" + "size c\n"); + + auto matches_nonrec = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/false); + auto matches_rec = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/true); + + // Non-recursive: only top-level + ASSERT_EQ(matches_nonrec.size(), 1u); + EXPECT_EQ(matches_nonrec[0], fs::path("toplevel.txt")); + + // Recursive: both, relative to base dir + sortPaths(matches_rec); + std::vector expected = { + fs::path("toplevel.txt"), + fs::path("subdir") / "nested.txt"}; + sortPaths(expected); + EXPECT_EQ(matches_rec, expected); } From 78fff0273342844a5a3e419fe958fc3920064b0a Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Fri, 13 Mar 2026 14:06:08 +0100 Subject: [PATCH 32/49] More tests --- src/pull_module/hf_pull_model_module.hpp | 4 +- src/pull_module/libgit2.cpp | 20 ++++++-- src/status.cpp | 4 +- src/status.hpp | 2 + src/test/pull_hf_model_test.cpp | 59 ++++++++++++++++++------ 5 files changed, 70 insertions(+), 19 deletions(-) diff --git a/src/pull_module/hf_pull_model_module.hpp b/src/pull_module/hf_pull_model_module.hpp index 2742ac23ca..46b92b8b57 100644 --- a/src/pull_module/hf_pull_model_module.hpp +++ b/src/pull_module/hf_pull_model_module.hpp @@ -21,7 +21,7 @@ #include "../capi_frontend/server_settings.hpp" namespace ovms { - +class Libgt2InitGuard; class HfPullModelModule : public Module { protected: HFSettingsImpl hfSettings; @@ -40,4 +40,6 @@ class HfPullModelModule : public Module { static const std::string GIT_SERVER_TIMEOUT_ENV; static const std::string GIT_SSL_CERT_LOCATIONS_ENV; }; + +std::variant> createGuard(); } // namespace ovms diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 292530264a..a0bedcc593 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -186,14 +186,16 @@ Status HfDownloader::RemoveReadonlyFileAttributeFromDir(const std::string& direc class GitRepositoryGuard { public: git_repository* repo = nullptr; + int git_error_class = 0; GitRepositoryGuard(const std::string& path) { int error = git_repository_open_ext(&repo, path.c_str(), 0, nullptr); if (error < 0) { const git_error* err = git_error_last(); - if (err) + if (err) { SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); - else + git_error_class = err->klass; + } else SPDLOG_ERROR("Repository open failed: {}", error); if (repo) git_repository_free(repo); @@ -233,7 +235,12 @@ class GitRepositoryGuard { Status HfDownloader::CheckRepositoryStatus(bool checkUntracked) { GitRepositoryGuard repoGuard(this->downloadPath); if (!repoGuard.get()) { - return StatusCode::HF_GIT_STATUS_FAILED; + if (repoGuard.git_error_class == 2) + return StatusCode::HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH; + else if (repoGuard.git_error_class == 3) + return StatusCode::HF_GIT_LIGIT2_NOT_INITIALIZED; + else + return StatusCode::HF_GIT_STATUS_FAILED; } // HEAD state info bool is_detached = git_repository_head_detached(repoGuard.get()) == 1; @@ -569,7 +576,12 @@ Status HfDownloader::downloadModel() { GitRepositoryGuard repoGuard(this->downloadPath); if (!repoGuard.get()) { std::cout << "Path already exists on local filesystem. And is not a git repository: " << this->downloadPath << std::endl; - return StatusCode::HF_GIT_STATUS_FAILED; + if (repoGuard.git_error_class == 2) + return StatusCode::HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH; + else if (repoGuard.git_error_class == 3) + return StatusCode::HF_GIT_LIGIT2_NOT_INITIALIZED; + else + return StatusCode::HF_GIT_STATUS_FAILED; } // Set repository url diff --git a/src/status.cpp b/src/status.cpp index a7750498e9..0cc057042b 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -349,7 +349,9 @@ const std::unordered_map Status::statusMessageMap = { {StatusCode::HF_RUN_CONVERT_TOKENIZER_EXPORT_FAILED, "Failed to run convert-tokenizer export command"}, {StatusCode::HF_GIT_CLONE_FAILED, "Failed in libgit2 execution of clone method"}, {StatusCode::HF_GIT_STATUS_FAILED, "Failed in libgit2 execution of status method"}, - {StatusCode::HF_GIT_STATUS_UNCLEAN, "Unclean status detected in libgit2 cloned repository"}, + {StatusCode::HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH, "Failed in libgit2 to check repository status for a given path"}, + {StatusCode::HF_GIT_LIGIT2_NOT_INITIALIZED, "Libgit2 was not initialized"}, + {StatusCode::HF_GIT_STATUS_UNCLEAN, "Unclean status detected in libgit2 repository path"}, {StatusCode::PARTIAL_END, "Request has finished and no further communication is needed"}, {StatusCode::NONEXISTENT_PATH, "Nonexistent path"}, diff --git a/src/status.hpp b/src/status.hpp index 2f72532cfd..02b42886a5 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -361,6 +361,8 @@ enum class StatusCode { HF_RUN_CONVERT_TOKENIZER_EXPORT_FAILED, HF_GIT_CLONE_FAILED, HF_GIT_STATUS_FAILED, + HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH, + HF_GIT_LIGIT2_NOT_INITIALIZED, HF_GIT_STATUS_UNCLEAN, PARTIAL_END, diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index 0290e6bf25..5092f49dbf 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -253,6 +253,20 @@ std::string sha256File(std::string_view path, std::error_code& ec) { return oss.str(); } +class TestHfDownloader : public ovms::HfDownloader { +public: + TestHfDownloader(const std::string& sourceModel, const std::string& downloadPath, const std::string& hfEndpoint, const std::string& hfToken, const std::string& httpProxy, bool overwrite) : + HfDownloader(sourceModel, downloadPath, hfEndpoint, hfToken, httpProxy, overwrite) {} + std::string GetRepoUrl() { return HfDownloader::GetRepoUrl(); } + std::string GetRepositoryUrlWithPassword() { return HfDownloader::GetRepositoryUrlWithPassword(); } + bool CheckIfProxySet() { return HfDownloader::CheckIfProxySet(); } + const std::string& getEndpoint() { return this->hfEndpoint; } + const std::string& getProxy() { return this->httpProxy; } + std::string getGraphDirectory(const std::string& downloadPath, const std::string& sourceModel) { return IModelDownloader::getGraphDirectory(downloadPath, sourceModel); } + std::string getGraphDirectory() { return HfDownloader::getGraphDirectory(); } + ovms::Status CheckRepositoryStatus(bool checkUntracked) { return HfDownloader::CheckRepositoryStatus(checkUntracked); } +}; + TEST_F(HfDownloaderPullHfModel, Resume) { std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); @@ -275,6 +289,12 @@ TEST_F(HfDownloaderPullHfModel, Resume) { ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; + // Check status function + std::unique_ptr hfDownloader = std::make_unique(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), "", "", "", false); + + // Fails because we want clean and it has the graph.pbtxt after download + ASSERT_EQ(hfDownloader->CheckRepositoryStatus(true).getCode(), ovms::StatusCode::HF_GIT_STATUS_UNCLEAN); + std::error_code ec; ec.clear(); std::string expectedDigest = sha256File(modelPath, ec); @@ -439,19 +459,6 @@ class TestOptimumDownloader : public ovms::OptimumDownloader { bool checkIfTokenizerFileIsExported() { return ovms::OptimumDownloader::checkIfTokenizerFileIsExported(); } }; -class TestHfDownloader : public ovms::HfDownloader { -public: - TestHfDownloader(const std::string& sourceModel, const std::string& downloadPath, const std::string& hfEndpoint, const std::string& hfToken, const std::string& httpProxy, bool overwrite) : - HfDownloader(sourceModel, downloadPath, hfEndpoint, hfToken, httpProxy, overwrite) {} - std::string GetRepoUrl() { return HfDownloader::GetRepoUrl(); } - std::string GetRepositoryUrlWithPassword() { return HfDownloader::GetRepositoryUrlWithPassword(); } - bool CheckIfProxySet() { return HfDownloader::CheckIfProxySet(); } - const std::string& getEndpoint() { return this->hfEndpoint; } - const std::string& getProxy() { return this->httpProxy; } - std::string getGraphDirectory(const std::string& downloadPath, const std::string& sourceModel) { return IModelDownloader::getGraphDirectory(downloadPath, sourceModel); } - std::string getGraphDirectory() { return HfDownloader::getGraphDirectory(); } -}; - TEST(HfDownloaderClassTest, Methods) { std::string modelName = "model/name"; std::string downloadPath = "/path/to/Download"; @@ -475,6 +482,32 @@ TEST(HfDownloaderClassTest, Methods) { ASSERT_EQ(hfDownloader->getGraphDirectory(), expectedPath); } +TEST(HfDownloaderClassTest, RepositoryStatusCheckErrors) { + std::string modelName = "model/name"; + std::string downloadPath = "/path/to/Download"; + std::string hfEndpoint = "www.new_hf.com/"; + std::string hfToken = "123$$o_O123!AAbb"; + std::string httpProxy = "https://proxy_test1:123"; + std::unique_ptr hfDownloader = std::make_unique(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, httpProxy, false); + + // Fails without libgit init + ASSERT_EQ(hfDownloader->CheckRepositoryStatus(true).getCode(), ovms::StatusCode::HF_GIT_LIGIT2_NOT_INITIALIZED); + ASSERT_EQ(hfDownloader->CheckRepositoryStatus(false).getCode(), ovms::StatusCode::HF_GIT_LIGIT2_NOT_INITIALIZED); + + auto guardOrError = ovms::createGuard(); + ASSERT_EQ(std::holds_alternative(guardOrError), false); + + // Path does not exist + ASSERT_EQ(hfDownloader->CheckRepositoryStatus(true).getCode(), ovms::StatusCode::HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH); + ASSERT_EQ(hfDownloader->CheckRepositoryStatus(false).getCode(), ovms::StatusCode::HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH); + + // Path not a git repository + downloadPath = getGenericFullPathForSrcTest("/tmp/"); + std::unique_ptr existingHfDownloader = std::make_unique(modelName, downloadPath, hfEndpoint, hfToken, httpProxy, false); + ASSERT_EQ(existingHfDownloader->CheckRepositoryStatus(true).getCode(), ovms::StatusCode::HF_GIT_STATUS_FAILED); + ASSERT_EQ(existingHfDownloader->CheckRepositoryStatus(false).getCode(), ovms::StatusCode::HF_GIT_STATUS_FAILED); +} + class TestOptimumDownloaderSetup : public ::testing::Test { public: ovms::HFSettingsImpl inHfSettings; From abacfa8931cce211657f22154b194dadba7aa43e Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Fri, 13 Mar 2026 14:16:53 +0100 Subject: [PATCH 33/49] Style --- src/pull_module/libgit2.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index a0bedcc593..d2985ded17 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -195,8 +195,9 @@ class GitRepositoryGuard { if (err) { SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); git_error_class = err->klass; - } else + } else { SPDLOG_ERROR("Repository open failed: {}", error); + } if (repo) git_repository_free(repo); } From 600d0d8ea0514b1ad13f0d69126d4d576d71ab90 Mon Sep 17 00:00:00 2001 From: rasapala Date: Mon, 16 Mar 2026 12:50:40 +0100 Subject: [PATCH 34/49] Resume interval and attemps --- third_party/libgit2/lfs.patch | 404 +++++++++++++++++++++++++++++++--- 1 file changed, 371 insertions(+), 33 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index b6daae42ae..49d7e43205 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -111,9 +111,21 @@ index d121c588a..b54a01a4b 100644 # diff --git a/src/cli/cmd_clone.c b/src/cli/cmd_clone.c -index c18cb28d4..6d23dcbb1 100644 +index c18cb28d4..9f89cd1b3 100644 --- a/src/cli/cmd_clone.c +++ b/src/cli/cmd_clone.c +@@ -4,9 +4,9 @@ + * This file is part of libgit2, distributed under the GNU GPL v2 with + * a Linking Exception. For full terms see the included COPYING file. + */ +- +-#include + #include ++#include ++ + #include "common.h" + #include "cmd.h" + #include "error.h" @@ -146,6 +146,7 @@ int cmd_clone(int argc, char **argv) clone_opts.bare = !!bare; clone_opts.checkout_branch = branch; @@ -126,7 +138,7 @@ index c18cb28d4..6d23dcbb1 100644 cli_progress_finish(&progress); -+ ++ // Code below for testing resume in native libgit2 + git_repository *repo2 = NULL; + int error = git_repository_open_ext(&repo2, local_path, 0, NULL); + // HEAD state info @@ -432,10 +444,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..fc3120cca +index 000000000..bb2d49082 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,810 @@ +@@ -0,0 +1,1136 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -452,10 +464,17 @@ index 000000000..fc3120cca +/ limitations under the License. +*/ + ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ +#include "array.h" +#include "common.h" -+ -+#include +#include "git2/sys/filter.h" +#include "hash.h" +#include "oid.h" @@ -465,6 +484,68 @@ index 000000000..fc3120cca +#include "regexp.h" +#include "time.h" + ++ ++#define LFS_RESUME_ATTEMPTS_DEFAULT 5 ++#define LFS_RESUME_ATTEMPTS_MAX 100 ++#define LFS_RESUME_INTERVAL_DEFAULT 10 ++#define LFS_RESUME_INTERVAL_MAX 3600 ++ ++/* Configure how many resume attempts and how long to wait between them */ ++static int g_lfs_resume_attempts = 5; /* <-- make configurable */ ++static unsigned int g_lfs_resume_interval_secs = 10; /* <-- make configurable */ ++ ++/* Initialize globals from env exactly once */ ++static void lfs_resume_env_init(void) ++{ ++ parse_env_nonneg_int( ++ "GIT_LFS_RESUME_ATTEMPTS", LFS_RESUME_ATTEMPTS_DEFAULT, 0, ++ LFS_RESUME_ATTEMPTS_MAX, &g_lfs_resume_attempts); ++ ++ parse_env_nonneg_uint( ++ "GIT_LFS_RESUME_INTERVAL", LFS_RESUME_INTERVAL_DEFAULT, 0, ++ LFS_RESUME_INTERVAL_MAX, &g_lfs_resume_interval_secs); ++ ++ /* log resolved config */ ++ fprintf(stderr, "[INFO] LFS resume: attempts=%d interval=%u s\n", ++ g_lfs_resume_attempts, g_lfs_resume_interval_secs); ++} ++ ++#ifdef _WIN32 ++#include ++#define fseeko _fseeki64 ++#define ftello _ftelli64 ++static void sleep_seconds(unsigned int seconds) ++{ ++ Sleep(seconds * 1000); ++} ++ ++#define LFS_ONCE_INIT INIT_ONCE_STATIC_INIT ++static INIT_ONCE lfs_once = LFS_ONCE_INIT; ++static BOOL CALLBACK lfs_once_cb_win(PINIT_ONCE once, PVOID param, PVOID *ctx) ++{ ++ (void)once; ++ (void)param; ++ (void)ctx; ++ lfs_resume_env_init(); ++ return TRUE; ++} ++#elif defined(__ANDROID__) ++// Android may require _FILE_OFFSET_BITS=64 and proper headers ++#else ++// POSIX systems (Linux, macOS) ++#include ++#include ++static pthread_once_t lfs_once = PTHREAD_ONCE_INIT; ++static void lfs_once_cb_posix(void) ++{ ++ lfs_resume_env_init(); ++} ++static void sleep_seconds(unsigned int seconds) ++{ ++ sleep(seconds); ++} ++#endif ++ +typedef struct lfs_attrs +{ + const char *path; @@ -476,10 +557,120 @@ index 000000000..fc3120cca + bool is_download; +} lfs_attrs; + ++/* Parse a non-negative integer from env, with bounds and default fallback */ ++static int parse_env_nonneg_int( ++ const char *env_name, ++ int default_value, ++ int min_value, ++ int max_value, ++ int *out_value) ++{ ++ const char *s = getenv(env_name); ++ if (!s || !*s) { ++ *out_value = default_value; ++ return 0; ++ } ++ ++ /* Trim leading spaces */ ++ while (isspace((unsigned char)*s)) ++ s++; ++ ++ errno = 0; ++ char *end = NULL; ++ unsigned long long val = strtoull(s, &end, 10); ++ ++ if (errno == ERANGE || end == s) { ++ fprintf(stderr, "[WARN] %s: invalid number, using default=%d\n", ++ env_name, default_value); ++ *out_value = default_value; ++ return -1; ++ } ++ /* Check for trailing junk */ ++ while (isspace((unsigned char)*end)) ++ end++; ++ if (*end != '\0') { ++ fprintf(stderr, ++ "[WARN] %s: trailing characters ignored, using default=%d\n", ++ env_name, default_value); ++ *out_value = default_value; ++ return -1; ++ } ++ ++ if (val > (unsigned long long)INT_MAX) { ++ fprintf(stderr, "[WARN] %s: value too large, capping to %d\n", ++ env_name, max_value); ++ val = (unsigned long long)max_value; ++ } ++ ++ int ival = (int)val; ++ if (ival < min_value) ++ ival = min_value; ++ if (ival > max_value) ++ ival = max_value; ++ ++ *out_value = ival; ++ return 0; ++} ++ ++/* Convenience for unsigned interval */ ++static int parse_env_nonneg_uint( ++ const char *env_name, ++ unsigned int default_value, ++ unsigned int min_value, ++ unsigned int max_value, ++ unsigned int *out_value) ++{ ++ const char *s = getenv(env_name); ++ if (!s || !*s) { ++ *out_value = default_value; ++ return 0; ++ } ++ ++ while (isspace((unsigned char)*s)) ++ s++; ++ ++ errno = 0; ++ char *end = NULL; ++ unsigned long long val = strtoull(s, &end, 10); ++ ++ if (errno == ERANGE || end == s) { ++ fprintf(stderr, "[WARN] %s: invalid number, using default=%u\n", ++ env_name, default_value); ++ *out_value = default_value; ++ return -1; ++ } ++ while (isspace((unsigned char)*end)) ++ end++; ++ if (*end != '\0') { ++ fprintf(stderr, ++ "[WARN] %s: trailing characters ignored, using default=%u\n", ++ env_name, default_value); ++ *out_value = default_value; ++ return -1; ++ } ++ ++ if (val > (unsigned long long)UINT_MAX) ++ val = (unsigned long long)UINT_MAX; ++ ++ unsigned int uval = (unsigned int)val; ++ if (uval < min_value) ++ uval = min_value; ++ if (uval > max_value) ++ uval = max_value; ++ ++ *out_value = uval; ++ return 0; ++} ++ +static size_t get_digit(const char *buffer) +{ + char *endptr; + errno = 0; ++ if (buffer == NULL) { ++ fprintf(stderr, "\n[ERROR] get_digit on NULL\n"); ++ return 0; ++ } ++ + size_t number = strtoull(buffer, &endptr, 10); + + if (errno == ERANGE) { @@ -530,7 +721,7 @@ index 000000000..fc3120cca +} + +int get_lfs_info_match( -+ const git_str *output, ++ git_str *output, + const char *regexp) +{ + int result; @@ -589,7 +780,7 @@ index 000000000..fc3120cca + /* 1) Init SHA-256 hashing context (internal API) */ + if (git_hash_ctx_init(&ctx, GIT_HASH_ALGORITHM_SHA256) < 0) { + fprintf(stderr, "\n[ERROR] git_hash_ctx_init failed\n"); -+ return -1; ++ goto error; + } + + /* 2) Stream the payload in chunks — hash *only* the file bytes. */ @@ -601,7 +792,7 @@ index 000000000..fc3120cca + size_t n = remaining > CHUNK ? CHUNK : remaining; + if (git_hash_update(&ctx, p, n) < 0) { + fprintf(stderr, "\n[ERROR] git_hash_update failed\n"); -+ return -1; ++ goto error; + } + p += n; + remaining -= n; @@ -610,7 +801,7 @@ index 000000000..fc3120cca + /* 3) Finalize into git_oid (32-byte raw digest for SHA-256). */ + if (git_hash_final(out->id, &ctx) < 0) { + fprintf(stderr, "\n[ERROR] git_hash_final failed\n"); -+ return -1; ++ goto error; + } + + /* 4) Optionally format "oid sha256:" for the LFS pointer file. */ @@ -621,15 +812,19 @@ index 000000000..fc3120cca + if (git_oid_fmt(hex, out) < 0) { + fprintf(stderr, + "\n[ERROR] failure, git_oid_fmt failed\n"); -+ return -1; ++ goto error; + } + + hex[64] = '\0'; + snprintf(pointer_line, pointer_line_cap, "oid sha256:%s", hex); + } + ++ git_hash_ctx_cleanup(&ctx); + return 0; -+} ++ error: ++ git_hash_ctx_cleanup(&ctx); ++ return -1; ++ } + +static int lfs_remove_id( + git_str *to, @@ -733,10 +928,17 @@ index 000000000..fc3120cca + return -1; + } + -+ size_t workdir_size = strlen(git_repository_workdir(repo)); -+ + const char *workdir = git_repository_workdir(repo); -+ struct lfs_attrs la = { path, full_path.ptr, workdir, lfs_oid.ptr, lfs_size.ptr, repo->url, true }; ++ /* Setup memory for payload struct ownership */ ++ char *full_path_dup = git__strdup(full_path.ptr); ++ git_str_dispose(&full_path); ++ char *path_dup = git__strdup(path); ++ char *workdir_dup = git__strdup(workdir); ++ char *url_dup = git__strdup(repo->url); ++ ++ struct lfs_attrs la = { path_dup, full_path_dup, workdir_dup, ++ lfs_oid.ptr, lfs_size.ptr, url_dup, ++ true }; + + *payload = git__malloc(sizeof(la)); + GIT_ERROR_CHECK_ALLOC(*payload); @@ -841,6 +1043,14 @@ index 000000000..fc3120cca + bool first_run, + size_t elapsed_time) +{ ++ if (max == 0) { ++ // Print received bytes + rate without percentage bar ++ printf("\rProgress: [unknown size] "); ++ print_download_speed_info(count, elapsed_time); ++ fflush(stdout); ++ return; ++ } ++ + float progress = (float)count / max; + if (!first_run && progress < 0.01 && count > 0) + return; @@ -921,8 +1131,9 @@ index 000000000..fc3120cca + return 0; /* failure, cannot open file to write */ + } + } -+ -+ return fwrite(buffer, size, nmemb, out->stream); ++ ++ size_t written_items = fwrite(buffer, size, nmemb, out->stream); ++ return written_items * size; // return BYTES written +} + +static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp) @@ -961,9 +1172,47 @@ index 000000000..fc3120cca + status = setopt; \ + } + -+int get_curl_resume_url(CURL *dl_curl, struct FtpFile* ftpfile) ++ ++/* Repeatedly attempts resume with a fixed wait interval. ++ - max_retries: total number of resume attempts to try (e.g., 3) ++ - interval_seconds: wait time between attempts (e.g., 2 seconds) ++ Returns the final CURLcode of the last attempt. */ ++static CURLcode download_with_resume( ++ CURL *dl_curl, ++ struct FtpFile *ftpfile, ++ int max_retries, ++ unsigned int interval_seconds) +{ -+ /* ++ CURLcode res = CURLE_OK; ++ ++ for (int attempt = 1; attempt <= max_retries; ++attempt) { ++ res = curl_resume_url_execute(dl_curl, ftpfile); ++ ++ if (res == CURLE_OK) { ++ /* Success */ ++ if (attempt > 1) ++ printf("[INFO] Resume attempt %d succeeded\n", ++ attempt); ++ return CURLE_OK; ++ } ++ ++ fprintf(stderr, "[WARN] Resume attempt %d/%d failed: %s\n", ++ attempt, max_retries, curl_easy_strerror(res)); ++ ++ if (attempt < max_retries) { ++ printf("[INFO] Waiting %u seconds before next resume attempt...\n", ++ interval_seconds); ++ fflush(stdout); ++ sleep_seconds(interval_seconds); ++ } ++ } ++ ++ return res; /* last result (failure) */ ++} ++ ++int curl_resume_url_execute(CURL *dl_curl, struct FtpFile *ftpfile) ++{ ++ /* This helper check does not work but the resume mechanism still works + curl_off_t resume_from = 0; + curl_easy_getinfo( + dl_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &resume_from); @@ -975,21 +1224,39 @@ index 000000000..fc3120cca + */ + printf("\n[INFO] curl_easy_perform() trying to resume file download\n"); + curl_off_t offset = 0; -+ if (ftpfile->stream) { -+ fseek(ftpfile->stream, 0, SEEK_END); -+ offset = ftell(ftpfile->stream); -+ } else { -+ ftpfile->stream = fopen(ftpfile->filename, "ab+"); -+ if (ftpfile->stream) { -+ fseek(ftpfile->stream, 0, SEEK_END); -+ offset = ftell(ftpfile->stream); ++ { ++ FILE *f = fopen(ftpfile->filename, "ab+"); ++ if (f) { ++ if (fseeko(f, 0, SEEK_END) == 0) { ++ off_t pos = ftello(f); ++ if (pos > 0) ++ offset = (curl_off_t)pos; ++ } ++ fclose(f); + } + } + + /* Tell libcurl to resume */ + curl_easy_setopt(dl_curl, CURLOPT_RESUME_FROM_LARGE, offset); + /* Perform the request, res gets the return code */ -+ return curl_easy_perform(dl_curl); ++ ++ /* Perform the request, res gets the return code */ ++ CURLcode res = curl_easy_perform(dl_curl); ++ ++ /* Validate that server honored Range (206) when offset > 0 */ ++ if (res == CURLE_OK && offset > 0) { ++ long http_code = 0; ++ curl_easy_getinfo(dl_curl, CURLINFO_RESPONSE_CODE, &http_code); ++ if (http_code != 206) { ++ /* Server did not honor resume — caller may choose to ++ * retry or restart */ ++ fprintf(stderr, ++ "\n[WARN] Server did not return 206 for resumed request (HTTP %ld)\n", ++ http_code); ++ } ++ } ++ ++ return res; +} + +/** @@ -1026,6 +1293,10 @@ index 000000000..fc3120cca + } + + char *tmp_out_file = append_cstr_to_buffer(la->full_path, "lfs_part"); ++ if (tmp_out_file == NULL) { ++ fprintf(stderr, "\n[ERROR] lfs create temp filename failed\n"); ++ goto on_error2; ++ } + + CURL *info_curl,*dl_curl; + CURLcode res = CURLE_OK; @@ -1042,6 +1313,7 @@ index 000000000..fc3120cca + "git/info/lfs/objects/batch") < 0) { + fprintf(stderr, "\n[ERROR] failed to create url '%s'\n", + la->full_path); ++ git_str_dispose(&lfs_info_url); + goto on_error; + } + @@ -1055,6 +1327,15 @@ index 000000000..fc3120cca + can just as well be an https:// URL if that is what should + receive the data. */ + CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_URL, lfs_info_url.ptr)); ++ /* Add cURL resiliency */ ++ /* unlimited data */ ++ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_CONNECTTIMEOUT, 30L)); ++ /* timeout */ ++ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_TIMEOUT, 0L)); ++ /* low speed 1KB/s */ ++ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_LOW_SPEED_LIMIT, 1024L)); ++ /* for 30s */ ++ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_LOW_SPEED_TIME, 30L)); + + if (status != CURLE_OK) { + fprintf(stderr, "\n[ERROR] curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); @@ -1076,6 +1357,7 @@ index 000000000..fc3120cca + curl_easy_cleanup(info_curl); + /* free the custom headers */ + curl_slist_free_all(chunk); ++ git_str_dispose(&lfs_info_data); + goto on_error; + } + @@ -1103,6 +1385,8 @@ index 000000000..fc3120cca + curl_easy_cleanup(info_curl); + /* free the custom headers */ + curl_slist_free_all(chunk); ++ git_str_dispose(&lfs_info_url); ++ git_str_dispose(&lfs_info_data); + goto on_error; + } + @@ -1110,11 +1394,14 @@ index 000000000..fc3120cca + res_str.asize = response.size; + res_str.size = response.size; + res_str.ptr = git__strdup(response.response); ++ free(response.response); + info_cleaup: + /* always cleanup */ + curl_easy_cleanup(info_curl); + /* free the custom headers */ + curl_slist_free_all(chunk); ++ git_str_dispose(&lfs_info_url); ++ git_str_dispose(&lfs_info_data); + if (status != CURLE_OK) + goto on_error; + } @@ -1129,6 +1416,7 @@ index 000000000..fc3120cca + if (get_lfs_info_match(&res_str, href_regexp) < 0) { + /* always cleanup */ + curl_easy_cleanup(dl_curl); ++ git_str_dispose(&res_str); + goto on_error; + } + /* First set the URL that is about to receive our POST. This URL @@ -1148,9 +1436,19 @@ index 000000000..fc3120cca + CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_XFERINFOFUNCTION, progress_callback)); + CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_XFERINFODATA, &progress_d)); + ++ /* Add cURL resiliency */ ++ /* unlimited data */ ++ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_CONNECTTIMEOUT, 30L)); ++ /* timeout */ ++ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_TIMEOUT, 0L)); ++ /* low speed 1KB/s */ ++ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_LOW_SPEED_LIMIT, 1024L)); ++ /* for 30s */ ++ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_LOW_SPEED_TIME, 30L)); + if (status != CURLE_OK) { + fprintf(stderr, "\n[ERROR] curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); + curl_easy_cleanup(dl_curl); ++ git_str_dispose(&res_str); + goto on_error; + } + @@ -1161,7 +1459,11 @@ index 000000000..fc3120cca + resumingFileByBlobFilter = true; + fclose(ftpfile.stream); + ftpfile.stream = NULL; -+ res = get_curl_resume_url(dl_curl, &ftpfile); ++ ++ /* First try a resume sequence */ ++ res = download_with_resume( ++ dl_curl, &ftpfile, g_lfs_resume_attempts, ++ g_lfs_resume_interval_secs); + } else { + print_download_info( + la->full_path, get_digit(la->lfs_size)); @@ -1171,7 +1473,11 @@ index 000000000..fc3120cca + + /* Check for resume of partial download error */ + if (res == CURLE_PARTIAL_FILE) { -+ res = get_curl_resume_url(dl_curl, &ftpfile); ++ fprintf(stderr, ++ "[WARN] Got CURLE_PARTIAL_FILE, attempting resume sequence\n"); ++ res = download_with_resume( ++ dl_curl, &ftpfile, g_lfs_resume_attempts, ++ g_lfs_resume_interval_secs); + } + + /* Check for errors */ @@ -1211,14 +1517,33 @@ index 000000000..fc3120cca + goto on_error; + } + free(tmp_out_file); ++ if (la) { ++ free((char *)la->path); ++ free((char *)la->full_path); ++ free((char *)la->workdir); ++ free((char *)la->lfs_oid); ++ free((char *)la->lfs_size); ++ free((char *)la->url); ++ } + git__free(payload); ++ git_str_dispose(&res_str); + return; + + on_error: ++ free(tmp_out_file); ++ on_error2: + fprintf(stderr, "\n[ERROR] LFS download failed for file %s\n", + la->full_path); + fflush(stderr); -+ free(tmp_out_file); ++ git_str_dispose(&res_str); ++ if (la) { ++ free((char *)la->path); ++ free((char *)la->full_path); ++ free((char *)la->workdir); ++ free((char *)la->lfs_oid); ++ free((char *)la->lfs_size); ++ free((char *)la->url); ++ } + git__free(payload); + return; +} @@ -1229,6 +1554,16 @@ index 000000000..fc3120cca + git__free(filter); +} + ++/*----public - ish trigger(call this in git_lfs_filter_new)-- --*/ ++static void lfs_resume_env_init_once(void) ++{ ++#ifdef _WIN32 ++ InitOnceExecuteOnce(&lfs_once, lfs_once_cb_win, NULL, NULL); ++#else ++ pthread_once(&lfs_once, lfs_once_cb_posix); ++#endif ++} ++ +git_filter *git_lfs_filter_new(void) +{ + /* In Windows, this inits the Winsock stuff */ @@ -1237,10 +1572,13 @@ index 000000000..fc3120cca + if (f == NULL) + return NULL; + ++ /* Initialize env-config exactly once per process */ ++ lfs_resume_env_init_once(); ++ + f->version = GIT_FILTER_VERSION; + f->attributes = "lfs"; -+ f->shutdown = git_lfs_filter_free; -+ f->stream = lfs_stream; ++ f->shutdown = git_lfs_filter_free; /* will free cfg and filter */ ++ f->stream = lfs_stream; + f->check = lfs_check; + f->cleanup = lfs_download; + From cebb5838443f1bcd53f173a75c9248b440c009cd Mon Sep 17 00:00:00 2001 From: rasapala Date: Mon, 16 Mar 2026 13:27:32 +0100 Subject: [PATCH 35/49] Fix linux compile warnings --- third_party/libgit2/lfs.patch | 343 ++++++++++++++++++---------------- 1 file changed, 179 insertions(+), 164 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 49d7e43205..27f04caffb 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -444,10 +444,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..bb2d49082 +index 000000000..84796d4b0 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,1136 @@ +@@ -0,0 +1,1151 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -494,68 +494,6 @@ index 000000000..bb2d49082 +static int g_lfs_resume_attempts = 5; /* <-- make configurable */ +static unsigned int g_lfs_resume_interval_secs = 10; /* <-- make configurable */ + -+/* Initialize globals from env exactly once */ -+static void lfs_resume_env_init(void) -+{ -+ parse_env_nonneg_int( -+ "GIT_LFS_RESUME_ATTEMPTS", LFS_RESUME_ATTEMPTS_DEFAULT, 0, -+ LFS_RESUME_ATTEMPTS_MAX, &g_lfs_resume_attempts); -+ -+ parse_env_nonneg_uint( -+ "GIT_LFS_RESUME_INTERVAL", LFS_RESUME_INTERVAL_DEFAULT, 0, -+ LFS_RESUME_INTERVAL_MAX, &g_lfs_resume_interval_secs); -+ -+ /* log resolved config */ -+ fprintf(stderr, "[INFO] LFS resume: attempts=%d interval=%u s\n", -+ g_lfs_resume_attempts, g_lfs_resume_interval_secs); -+} -+ -+#ifdef _WIN32 -+#include -+#define fseeko _fseeki64 -+#define ftello _ftelli64 -+static void sleep_seconds(unsigned int seconds) -+{ -+ Sleep(seconds * 1000); -+} -+ -+#define LFS_ONCE_INIT INIT_ONCE_STATIC_INIT -+static INIT_ONCE lfs_once = LFS_ONCE_INIT; -+static BOOL CALLBACK lfs_once_cb_win(PINIT_ONCE once, PVOID param, PVOID *ctx) -+{ -+ (void)once; -+ (void)param; -+ (void)ctx; -+ lfs_resume_env_init(); -+ return TRUE; -+} -+#elif defined(__ANDROID__) -+// Android may require _FILE_OFFSET_BITS=64 and proper headers -+#else -+// POSIX systems (Linux, macOS) -+#include -+#include -+static pthread_once_t lfs_once = PTHREAD_ONCE_INIT; -+static void lfs_once_cb_posix(void) -+{ -+ lfs_resume_env_init(); -+} -+static void sleep_seconds(unsigned int seconds) -+{ -+ sleep(seconds); -+} -+#endif -+ -+typedef struct lfs_attrs -+{ -+ const char *path; -+ const char* full_path; -+ const char* workdir; -+ const char *lfs_oid; -+ const char *lfs_size; -+ const char *url; -+ bool is_download; -+} lfs_attrs; + +/* Parse a non-negative integer from env, with bounds and default fallback */ +static int parse_env_nonneg_int( @@ -620,7 +558,12 @@ index 000000000..bb2d49082 + unsigned int max_value, + unsigned int *out_value) +{ -+ const char *s = getenv(env_name); ++ char *end; ++ unsigned long long val; ++ unsigned int uval; ++ const char *s; ++ end = NULL; ++ s = getenv(env_name); + if (!s || !*s) { + *out_value = default_value; + return 0; @@ -630,8 +573,7 @@ index 000000000..bb2d49082 + s++; + + errno = 0; -+ char *end = NULL; -+ unsigned long long val = strtoull(s, &end, 10); ++ val = strtoull(s, &end, 10); + + if (errno == ERANGE || end == s) { + fprintf(stderr, "[WARN] %s: invalid number, using default=%u\n", @@ -652,7 +594,7 @@ index 000000000..bb2d49082 + if (val > (unsigned long long)UINT_MAX) + val = (unsigned long long)UINT_MAX; + -+ unsigned int uval = (unsigned int)val; ++ uval = (unsigned int)val; + if (uval < min_value) + uval = min_value; + if (uval > max_value) @@ -662,16 +604,80 @@ index 000000000..bb2d49082 + return 0; +} + ++/* Initialize globals from env exactly once */ ++static void lfs_resume_env_init(void) ++{ ++ parse_env_nonneg_int( ++ "GIT_LFS_RESUME_ATTEMPTS", LFS_RESUME_ATTEMPTS_DEFAULT, 0, ++ LFS_RESUME_ATTEMPTS_MAX, &g_lfs_resume_attempts); ++ ++ parse_env_nonneg_uint( ++ "GIT_LFS_RESUME_INTERVAL", LFS_RESUME_INTERVAL_DEFAULT, 0, ++ LFS_RESUME_INTERVAL_MAX, &g_lfs_resume_interval_secs); ++ ++ /* log resolved config */ ++ fprintf(stderr, "[INFO] LFS resume: attempts=%d interval=%u s\n", ++ g_lfs_resume_attempts, g_lfs_resume_interval_secs); ++} ++ ++#ifdef _WIN32 ++#include ++#define fseeko _fseeki64 ++#define ftello _ftelli64 ++static void sleep_seconds(unsigned int seconds) ++{ ++ Sleep(seconds * 1000); ++} ++ ++#define LFS_ONCE_INIT INIT_ONCE_STATIC_INIT ++static INIT_ONCE lfs_once = LFS_ONCE_INIT; ++static BOOL CALLBACK lfs_once_cb_win(PINIT_ONCE once, PVOID param, PVOID *ctx) ++{ ++ (void)once; ++ (void)param; ++ (void)ctx; ++ lfs_resume_env_init(); ++ return TRUE; ++} ++#elif defined(__ANDROID__) ++/* Android may require _FILE_OFFSET_BITS = 64 and proper headers */ ++#else ++/* POSIX systems (Linux, macOS) */ ++#include ++#include ++static pthread_once_t lfs_once = PTHREAD_ONCE_INIT; ++static void lfs_once_cb_posix(void) ++{ ++ lfs_resume_env_init(); ++} ++static void sleep_seconds(unsigned int seconds) ++{ ++ sleep(seconds); ++} ++#endif ++ ++typedef struct lfs_attrs ++{ ++ const char *path; ++ const char* full_path; ++ const char* workdir; ++ const char *lfs_oid; ++ const char *lfs_size; ++ const char *url; ++ bool is_download; ++} lfs_attrs; ++ +static size_t get_digit(const char *buffer) +{ + char *endptr; + errno = 0; ++ size_t number; + if (buffer == NULL) { + fprintf(stderr, "\n[ERROR] get_digit on NULL\n"); + return 0; + } + -+ size_t number = strtoull(buffer, &endptr, 10); ++ number = strtoull(buffer, &endptr, 10); + + if (errno == ERANGE) { + fprintf(stderr, "\n[ERROR] Conversion error\n"); @@ -695,7 +701,7 @@ index 000000000..bb2d49082 + * + * Note: Caller is responsible for freeing the returned buffer. + */ -+char *append_cstr_to_buffer(const char *existingBuffer, const char *suffix) ++static char *append_cstr_to_buffer(const char *existingBuffer, const char *suffix) +{ + if (existingBuffer == NULL || suffix == NULL) { + return NULL; @@ -720,20 +726,21 @@ index 000000000..bb2d49082 + return newBuffer; +} + -+int get_lfs_info_match( ++static int get_lfs_info_match( + git_str *output, + const char *regexp) +{ + int result; -+ git_regexp preg = GIT_REGEX_INIT; ++ git_regexp preg; ++ size_t i; ++ git_regmatch pmatch[2]; ++ ++ preg = GIT_REGEX_INIT; + if ((result = git_regexp_compile(&preg, regexp, 0)) < 0) { + git_regexp_dispose(&preg); + return result; + } + -+ size_t i; -+ git_regmatch pmatch[2]; -+ + if (!git_regexp_search(&preg, output->ptr, 2, pmatch)) { + /* use pmatch data to trim line data */ + i = (pmatch[1].start >= 0) ? 1 : 0; @@ -748,7 +755,7 @@ index 000000000..bb2d49082 + return -1; +} + -+void print_src_oid(const git_filter_source *src) ++static void print_src_oid(const git_filter_source *src) +{ + const git_oid *oid = git_filter_source_id(src); + @@ -759,7 +766,7 @@ index 000000000..bb2d49082 + } +} + -+int git_oid_sha256_from_git_str_blob( ++static int git_oid_sha256_from_git_str_blob( + git_oid *out, + const struct git_str *input, + char *pointer_line, @@ -767,6 +774,9 @@ index 000000000..bb2d49082 +{ + int error = -1; + git_hash_ctx ctx; ++ size_t CHUNK; ++ unsigned char *p; ++ size_t remaining; + + if (!out || !input || !input->ptr) { + return -1; @@ -784,9 +794,9 @@ index 000000000..bb2d49082 + } + + /* 2) Stream the payload in chunks — hash *only* the file bytes. */ -+ const size_t CHUNK = 4 * 1024 * 1024; /* 4 MiB */ -+ const unsigned char *p = (const unsigned char *)input->ptr; -+ size_t remaining = input->size; ++ CHUNK = 4 * 1024 * 1024; /* 4 MiB */ ++ *p = (const unsigned char *)input->ptr; ++ remaining = input->size; + + while (remaining > 0) { + size_t n = remaining > CHUNK ? CHUNK : remaining; @@ -832,6 +842,7 @@ index 000000000..bb2d49082 + void **payload) +{ + int error = 0; ++ git_oid lfs_oid; + /* Init the lfs attrs to indicate git lfs clean, currently only diff support no upload of lfs file supported */ + struct lfs_attrs la = { NULL, NULL, NULL, NULL, NULL, NULL, false }; + *payload = git__malloc(sizeof(la)); @@ -847,7 +858,6 @@ index 000000000..bb2d49082 + } + + /* Use lib git oid to get lfs sha256 */ -+ git_oid lfs_oid; + lfs_oid.type = GIT_OID_SHA256; + char line[80]; /* 75+ is enough */ + if (git_oid_sha256_from_git_str_blob(&lfs_oid, from, line, sizeof(line)) < 0) { @@ -893,6 +903,17 @@ index 000000000..bb2d49082 +{ + git_str lfs_oid = GIT_STR_INIT; + git_str lfs_size = GIT_STR_INIT; ++ git_str full_path = GIT_STR_INIT; ++ const char *obj_regexp = "\noid sha256:(.*)\n"; ++ const char *size_regexp = "\nsize (.*)\n"; ++ git_repository *repo = git_filter_source_repo(src); ++ const char *path = git_filter_source_path(src); ++ const char *workdir = git_repository_workdir(repo); ++ /* Setup memory for payload struct ownership */ ++ char *full_path_dup; ++ char *path_dup = git__strdup(path); ++ char *workdir_dup = git__strdup(workdir); ++ char *url_dup = git__strdup(repo->url); + + lfs_oid.size = from->size; + lfs_oid.asize = from->asize; @@ -901,9 +922,6 @@ index 000000000..bb2d49082 + lfs_size.asize = from->asize; + lfs_size.ptr = git__strdup(from->ptr); + -+ const char *obj_regexp = "\noid sha256:(.*)\n"; -+ const char *size_regexp = "\nsize (.*)\n"; -+ + if (get_lfs_info_match(&lfs_oid, obj_regexp) < 0) { + fprintf(stderr,"\n[ERROR] failure, cannot find lfs oid in: %s\n", + lfs_oid.ptr); @@ -917,24 +935,14 @@ index 000000000..bb2d49082 + return -1; + } + -+ git_repository *repo = git_filter_source_repo(src); -+ const char *path = git_filter_source_path(src); -+ -+ git_str full_path = GIT_STR_INIT; + if (git_repository_workdir_path(&full_path, repo, path) < 0) { + fprintf(stderr, + "\n[ERROR] failure, cannot get repository path: %s\n", + path); + return -1; + } -+ -+ const char *workdir = git_repository_workdir(repo); -+ /* Setup memory for payload struct ownership */ -+ char *full_path_dup = git__strdup(full_path.ptr); ++ full_path_dup = git__strdup(full_path.ptr); + git_str_dispose(&full_path); -+ char *path_dup = git__strdup(path); -+ char *workdir_dup = git__strdup(workdir); -+ char *url_dup = git__strdup(repo->url); + + struct lfs_attrs la = { path_dup, full_path_dup, workdir_dup, + lfs_oid.ptr, lfs_size.ptr, url_dup, @@ -972,6 +980,9 @@ index 000000000..bb2d49082 + const git_filter_source *src, + const char **attr_values) +{ ++ GIT_UNUSED(self); ++ GIT_UNUSED(payload); ++ GIT_UNUSED(attr_values); + const char *value; + + git_repository *repo = git_filter_source_repo(src); @@ -988,8 +999,6 @@ index 000000000..bb2d49082 + return GIT_PASSTHROUGH; + } + -+ GIT_UNUSED(self); -+ + return 0; +} + @@ -1026,9 +1035,9 @@ index 000000000..bb2d49082 + double recv_len = (double)received_size; + uint64_t elapsed = (uint64_t)elapsed_time; + double rate; -+ rate = elapsed ? recv_len / elapsed : received_size; -+ + size_t rate_unit_idx = 0; ++ ++ rate = elapsed ? recv_len / elapsed : received_size; + while (rate > 1000 && sizeUnits[rate_unit_idx + 1]) { + rate /= 1000.0; + rate_unit_idx++; @@ -1037,37 +1046,40 @@ index 000000000..bb2d49082 + printf(" [%.2f %s/s] ", rate, sizeUnits[rate_unit_idx]); +} + -+void print_progress( ++static void print_progress( + size_t count, + size_t max, + bool first_run, + size_t elapsed_time) +{ ++ float progress; ++ int i, bar_length, bar_width; ++ size_t totalSizeUnitId; ++ double totalSize; + if (max == 0) { -+ // Print received bytes + rate without percentage bar ++ /* Print received bytes + rate without percentage bar */ + printf("\rProgress: [unknown size] "); + print_download_speed_info(count, elapsed_time); + fflush(stdout); + return; + } + -+ float progress = (float)count / max; ++ progress = (float)count / max; + if (!first_run && progress < 0.01 && count > 0) + return; + -+ const int bar_width = 50; -+ int bar_length = progress * bar_width; ++ bar_width = 50; ++ bar_length = progress * bar_width; + + printf("\rProgress: ["); -+ int i; + for (i = 0; i < bar_length; ++i) { + printf("#"); + } + for (i = bar_length; i < bar_width; ++i) { + printf(" "); + } -+ size_t totalSizeUnitId = 0; -+ double totalSize = max; ++ totalSizeUnitId = 0; ++ totalSize = max; + while (totalSize > 1000 && sizeUnits[totalSizeUnitId + 1]) { + totalSize /= 1000.0; + totalSizeUnitId++; @@ -1080,7 +1092,7 @@ index 000000000..bb2d49082 + fflush(stdout); +} + -+int progress_callback( ++static int progress_callback( + void *clientp, + curl_off_t dltotal, + curl_off_t dlnow, @@ -1122,6 +1134,7 @@ index 000000000..bb2d49082 +static size_t file_write_callback(void *buffer, size_t size, size_t nmemb, void *stream) +{ + struct FtpFile *out = (struct FtpFile *)stream; ++ size_t written_items; + if (!out->stream) { + /* open file for writing */ + out->stream = fopen(out->filename, "wb"); @@ -1132,8 +1145,8 @@ index 000000000..bb2d49082 + } + } + -+ size_t written_items = fwrite(buffer, size, nmemb, out->stream); -+ return written_items * size; // return BYTES written ++ written_items = fwrite(buffer, size, nmemb, out->stream); ++ return written_items * size; /* return BYTES written */ +} + +static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp) @@ -1172,45 +1185,7 @@ index 000000000..bb2d49082 + status = setopt; \ + } + -+ -+/* Repeatedly attempts resume with a fixed wait interval. -+ - max_retries: total number of resume attempts to try (e.g., 3) -+ - interval_seconds: wait time between attempts (e.g., 2 seconds) -+ Returns the final CURLcode of the last attempt. */ -+static CURLcode download_with_resume( -+ CURL *dl_curl, -+ struct FtpFile *ftpfile, -+ int max_retries, -+ unsigned int interval_seconds) -+{ -+ CURLcode res = CURLE_OK; -+ -+ for (int attempt = 1; attempt <= max_retries; ++attempt) { -+ res = curl_resume_url_execute(dl_curl, ftpfile); -+ -+ if (res == CURLE_OK) { -+ /* Success */ -+ if (attempt > 1) -+ printf("[INFO] Resume attempt %d succeeded\n", -+ attempt); -+ return CURLE_OK; -+ } -+ -+ fprintf(stderr, "[WARN] Resume attempt %d/%d failed: %s\n", -+ attempt, max_retries, curl_easy_strerror(res)); -+ -+ if (attempt < max_retries) { -+ printf("[INFO] Waiting %u seconds before next resume attempt...\n", -+ interval_seconds); -+ fflush(stdout); -+ sleep_seconds(interval_seconds); -+ } -+ } -+ -+ return res; /* last result (failure) */ -+} -+ -+int curl_resume_url_execute(CURL *dl_curl, struct FtpFile *ftpfile) ++static int curl_resume_url_execute(CURL *dl_curl, struct FtpFile *ftpfile) +{ + /* This helper check does not work but the resume mechanism still works + curl_off_t resume_from = 0; @@ -1218,12 +1193,14 @@ index 000000000..bb2d49082 + dl_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &resume_from); + + if (resume_from == -1) { -+ fprintf(stderr, -+ "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); -+ } else { ++ fprintf(stderr, ++ "\n[ERROR] curl_easy_perform() failed with transferred a ++ partial file error and server does not support range/resume.\n"); } else ++ { + */ -+ printf("\n[INFO] curl_easy_perform() trying to resume file download\n"); ++ CURLcode res; + curl_off_t offset = 0; ++ printf("\n[INFO] curl_easy_perform() trying to resume file download\n"); + { + FILE *f = fopen(ftpfile->filename, "ab+"); + if (f) { @@ -1241,7 +1218,7 @@ index 000000000..bb2d49082 + /* Perform the request, res gets the return code */ + + /* Perform the request, res gets the return code */ -+ CURLcode res = curl_easy_perform(dl_curl); ++ res = curl_easy_perform(dl_curl); + + /* Validate that server honored Range (206) when offset > 0 */ + if (res == CURLE_OK && offset > 0) { @@ -1259,6 +1236,43 @@ index 000000000..bb2d49082 + return res; +} + ++/* Repeatedly attempts resume with a fixed wait interval. ++ - max_retries: total number of resume attempts to try (e.g., 3) ++ - interval_seconds: wait time between attempts (e.g., 2 seconds) ++ Returns the final CURLcode of the last attempt. */ ++static CURLcode download_with_resume( ++ CURL *dl_curl, ++ struct FtpFile *ftpfile, ++ int max_retries, ++ unsigned int interval_seconds) ++{ ++ CURLcode res = CURLE_OK; ++ ++ for (int attempt = 1; attempt <= max_retries; ++attempt) { ++ res = curl_resume_url_execute(dl_curl, ftpfile); ++ ++ if (res == CURLE_OK) { ++ /* Success */ ++ if (attempt > 1) ++ printf("[INFO] Resume attempt %d succeeded\n", ++ attempt); ++ return CURLE_OK; ++ } ++ ++ fprintf(stderr, "[WARN] Resume attempt %d/%d failed: %s\n", ++ attempt, max_retries, curl_easy_strerror(res)); ++ ++ if (attempt < max_retries) { ++ printf("[INFO] Waiting %u seconds before next resume attempt...\n", ++ interval_seconds); ++ fflush(stdout); ++ sleep_seconds(interval_seconds); ++ } ++ } ++ ++ return res; /* last result (failure) */ ++} ++ +/** + * lfs_download - Downloads a file using the LFS (Large File Storage) mechanism. + * @@ -1277,14 +1291,21 @@ index 000000000..bb2d49082 + * @param self: Unused parameter, reserved for future use. + * @param payload: Pointer to the lfs_attrs structure containing download parameters. + */ -+ static void lfs_download(git_filter *self, void *payload) ++static void lfs_download(git_filter *self, void *payload) +{ + GIT_UNUSED(self); ++ struct lfs_attrs *la; ++ char *tmp_out_file; ++ CURL *info_curl, *dl_curl; ++ CURLcode res = CURLE_OK; ++ CURLcode status = CURLE_OK; ++ git_str res_str = GIT_STR_INIT; ++ bool resumingFileByBlobFilter = false; + if (!payload) { + fprintf(stderr, "\n[ERROR] lfs payload not initialized\n"); + return; + } -+ struct lfs_attrs *la = (struct lfs_attrs *)payload; ++ la = (struct lfs_attrs *)payload; + + /* Currently only download is supoprted, no lfs file upload */ + if (!la->is_download) { @@ -1292,16 +1313,12 @@ index 000000000..bb2d49082 + return; + } + -+ char *tmp_out_file = append_cstr_to_buffer(la->full_path, "lfs_part"); ++ tmp_out_file = append_cstr_to_buffer(la->full_path, "lfs_part"); + if (tmp_out_file == NULL) { + fprintf(stderr, "\n[ERROR] lfs create temp filename failed\n"); + goto on_error2; + } + -+ CURL *info_curl,*dl_curl; -+ CURLcode res = CURLE_OK; -+ CURLcode status = CURLE_OK; -+ git_str res_str = GIT_STR_INIT; + /* get a curl handle */ + info_curl = curl_easy_init(); + if (info_curl) { @@ -1407,7 +1424,6 @@ index 000000000..bb2d49082 + } + + /* get a curl handle */ -+ bool resumingFileByBlobFilter = false; + dl_curl = curl_easy_init(); + if (dl_curl) { + struct FtpFile ftpfile = { tmp_out_file, NULL }; @@ -1548,13 +1564,12 @@ index 000000000..bb2d49082 + return; +} + -+void git_lfs_filter_free(git_filter *filter) ++static void git_lfs_filter_free(git_filter *filter) +{ + curl_global_cleanup(); + git__free(filter); +} + -+/*----public - ish trigger(call this in git_lfs_filter_new)-- --*/ +static void lfs_resume_env_init_once(void) +{ +#ifdef _WIN32 From c46337a1408388620fbcd8bffc88afe414e6cebb Mon Sep 17 00:00:00 2001 From: rasapala Date: Mon, 16 Mar 2026 17:19:54 +0100 Subject: [PATCH 36/49] Fix compile 2 --- third_party/libgit2/lfs.patch | 154 +++++++++++++++++++++------------- 1 file changed, 94 insertions(+), 60 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 27f04caffb..01c2c0971b 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -444,10 +444,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..84796d4b0 +index 000000000..bcbae8bc5 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,1151 @@ +@@ -0,0 +1,1185 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -503,6 +503,10 @@ index 000000000..84796d4b0 + int max_value, + int *out_value) +{ ++ char *end; ++ unsigned long long val; ++ unsigned int ival; ++ end = NULL; + const char *s = getenv(env_name); + if (!s || !*s) { + *out_value = default_value; @@ -514,8 +518,7 @@ index 000000000..84796d4b0 + s++; + + errno = 0; -+ char *end = NULL; -+ unsigned long long val = strtoull(s, &end, 10); ++ val = strtoull(s, &end, 10); + + if (errno == ERANGE || end == s) { + fprintf(stderr, "[WARN] %s: invalid number, using default=%d\n", @@ -540,7 +543,7 @@ index 000000000..84796d4b0 + val = (unsigned long long)max_value; + } + -+ int ival = (int)val; ++ ival = (int)val; + if (ival < min_value) + ival = min_value; + if (ival > max_value) @@ -561,9 +564,8 @@ index 000000000..84796d4b0 + char *end; + unsigned long long val; + unsigned int uval; -+ const char *s; + end = NULL; -+ s = getenv(env_name); ++ const char *s = getenv(env_name); + if (!s || !*s) { + *out_value = default_value; + return 0; @@ -667,11 +669,62 @@ index 000000000..84796d4b0 + bool is_download; +} lfs_attrs; + ++ ++/* ++ * Frees heap-allocated strings referenced by the struct fields. ++ * Ownership assumption: ++ * - Only call free() on fields that point to heap memory ++ * (malloc/calloc/realloc/strdup). ++ * - If any field is borrowed (e.g., string literal or external buffer), set ++ * it to NULL before calling this function to avoid invalid free(). ++ */ ++static inline void lfs_attrs_free(lfs_attrs *a) ++{ ++ if (!a) ++ return; ++ if (a->path) { ++ free((void *)a->path); ++ a->path = NULL; ++ } ++ if (a->full_path) { ++ free((void *)a->full_path); ++ a->full_path = NULL; ++ } ++ if (a->workdir) { ++ free((void *)a->workdir); ++ a->workdir = NULL; ++ } ++ if (a->lfs_oid) { ++ free((void *)a->lfs_oid); ++ a->lfs_oid = NULL; ++ } ++ if (a->lfs_size) { ++ free((void *)a->lfs_size); ++ a->lfs_size = NULL; ++ } ++ if (a->url) { ++ free((void *)a->url); ++ a->url = NULL; ++ } ++ ++ a->is_download = false; ++} ++ ++/* Use this if the struct itself was heap-allocated. */ ++static inline void lfs_attrs_delete(lfs_attrs *a) ++{ ++ if (!a) ++ return; ++ lfs_attrs_free(a); ++ free(a); ++ a = NULL; ++} ++ +static size_t get_digit(const char *buffer) +{ + char *endptr; -+ errno = 0; + size_t number; ++ errno = 0; + if (buffer == NULL) { + fprintf(stderr, "\n[ERROR] get_digit on NULL\n"); + return 0; @@ -703,17 +756,21 @@ index 000000000..84796d4b0 + */ +static char *append_cstr_to_buffer(const char *existingBuffer, const char *suffix) +{ ++ size_t existingLength; ++ size_t suffixLength; ++ size_t newSize; ++ char *newBuffer; + if (existingBuffer == NULL || suffix == NULL) { + return NULL; + } + -+ size_t existingLength = strlen(existingBuffer); -+ size_t suffixLength = strlen(suffix); ++ existingLength = strlen(existingBuffer); ++ suffixLength = strlen(suffix); + + /* +1 for the null terminator */ -+ size_t newSize = existingLength + suffixLength + 1; ++ newSize = existingLength + suffixLength + 1; + -+ char *newBuffer = (char *)malloc(newSize); ++ newBuffer = (char *)malloc(newSize); + if (newBuffer == NULL) { + return NULL; + } @@ -755,24 +812,12 @@ index 000000000..84796d4b0 + return -1; +} + -+static void print_src_oid(const git_filter_source *src) -+{ -+ const git_oid *oid = git_filter_source_id(src); -+ -+ if (oid) { -+ printf("\nsrc->git_oid %s\n", git_oid_tostr_s(oid)); -+ } else { -+ printf("\nsrc has no OID (e.g., not a blob-backed source or unavailable)\n"); -+ } -+} -+ +static int git_oid_sha256_from_git_str_blob( + git_oid *out, + const struct git_str *input, + char *pointer_line, + size_t pointer_line_cap) +{ -+ int error = -1; + git_hash_ctx ctx; + size_t CHUNK; + unsigned char *p; @@ -795,7 +840,7 @@ index 000000000..84796d4b0 + + /* 2) Stream the payload in chunks — hash *only* the file bytes. */ + CHUNK = 4 * 1024 * 1024; /* 4 MiB */ -+ *p = (const unsigned char *)input->ptr; ++ p = (const unsigned char *)input->ptr; + remaining = input->size; + + while (remaining > 0) { @@ -842,6 +887,7 @@ index 000000000..84796d4b0 + void **payload) +{ + int error = 0; ++ char line[80]; /* 75+ is enough */ + git_oid lfs_oid; + /* Init the lfs attrs to indicate git lfs clean, currently only diff support no upload of lfs file supported */ + struct lfs_attrs la = { NULL, NULL, NULL, NULL, NULL, NULL, false }; @@ -859,7 +905,6 @@ index 000000000..84796d4b0 + + /* Use lib git oid to get lfs sha256 */ + lfs_oid.type = GIT_OID_SHA256; -+ char line[80]; /* 75+ is enough */ + if (git_oid_sha256_from_git_str_blob(&lfs_oid, from, line, sizeof(line)) < 0) { + fprintf(stderr, + "\n[ERROR] failure, cannot calculate sha256\n"); @@ -914,6 +959,8 @@ index 000000000..84796d4b0 + char *path_dup = git__strdup(path); + char *workdir_dup = git__strdup(workdir); + char *url_dup = git__strdup(repo->url); ++ struct lfs_attrs la = { path_dup, NULL, workdir_dup, NULL, ++ NULL, url_dup, true }; + + lfs_oid.size = from->size; + lfs_oid.asize = from->asize; @@ -927,6 +974,7 @@ index 000000000..84796d4b0 + lfs_oid.ptr); + return -1; + } ++ la.lfs_oid = lfs_oid.ptr; + + if (get_lfs_info_match(&lfs_size, size_regexp) < 0) { + fprintf(stderr, @@ -934,6 +982,7 @@ index 000000000..84796d4b0 + lfs_size.ptr); + return -1; + } ++ la.lfs_size = lfs_size.ptr; + + if (git_repository_workdir_path(&full_path, repo, path) < 0) { + fprintf(stderr, @@ -943,10 +992,7 @@ index 000000000..84796d4b0 + } + full_path_dup = git__strdup(full_path.ptr); + git_str_dispose(&full_path); -+ -+ struct lfs_attrs la = { path_dup, full_path_dup, workdir_dup, -+ lfs_oid.ptr, lfs_size.ptr, url_dup, -+ true }; ++ la.full_path = full_path_dup; + + *payload = git__malloc(sizeof(la)); + GIT_ERROR_CHECK_ALLOC(*payload); @@ -980,14 +1026,13 @@ index 000000000..84796d4b0 + const git_filter_source *src, + const char **attr_values) +{ -+ GIT_UNUSED(self); -+ GIT_UNUSED(payload); -+ GIT_UNUSED(attr_values); + const char *value; -+ + git_repository *repo = git_filter_source_repo(src); + const char *path = git_filter_source_path(src); + ++ GIT_UNUSED(self); ++ GIT_UNUSED(payload); ++ GIT_UNUSED(attr_values); + git_attr_get( + &value, repo, GIT_ATTR_CHECK_NO_SYSTEM, path, "filter"); + @@ -1100,12 +1145,15 @@ index 000000000..84796d4b0 + curl_off_t ulnow) +{ + struct progress_data *pcs = (struct progress_data *)clientp; ++ time_t currentTime = time(NULL); ++ bool shouldPrintDueToTime = false; ++ GIT_UNUSED(ulnow); + if (dlnow == 0) { + pcs->started_download = time(NULL); + pcs->last_print_time = time(NULL); + } -+ time_t currentTime = time(NULL); -+ bool shouldPrintDueToTime = (currentTime - pcs->last_print_time >= 1); ++ ++ shouldPrintDueToTime = (currentTime - pcs->last_print_time >= 1); + if ((dltotal == dlnow) && dltotal < 10000) { + /* Usually with first messages we don't get the full size and we + don't want to print progress bar so we assume that until @@ -1293,7 +1341,6 @@ index 000000000..84796d4b0 + */ +static void lfs_download(git_filter *self, void *payload) +{ -+ GIT_UNUSED(self); + struct lfs_attrs *la; + char *tmp_out_file; + CURL *info_curl, *dl_curl; @@ -1301,6 +1348,9 @@ index 000000000..84796d4b0 + CURLcode status = CURLE_OK; + git_str res_str = GIT_STR_INIT; + bool resumingFileByBlobFilter = false; ++ struct progress_data progress_d = { time(NULL), time(NULL), false }; ++ struct memory response = { 0 }; ++ GIT_UNUSED(self); + if (!payload) { + fprintf(stderr, "\n[ERROR] lfs payload not initialized\n"); + return; @@ -1324,6 +1374,7 @@ index 000000000..84796d4b0 + if (info_curl) { + struct curl_slist *chunk = NULL; + git_str lfs_info_url = GIT_STR_INIT; ++ git_str lfs_info_data = GIT_STR_INIT; + if (git_str_join( + &lfs_info_url, '.', + la->url, @@ -1331,6 +1382,7 @@ index 000000000..84796d4b0 + fprintf(stderr, "\n[ERROR] failed to create url '%s'\n", + la->full_path); + git_str_dispose(&lfs_info_url); ++ git_str_dispose(&lfs_info_data); + goto on_error; + } + @@ -1358,7 +1410,6 @@ index 000000000..84796d4b0 + fprintf(stderr, "\n[ERROR] curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); + goto info_cleaup; + } -+ git_str lfs_info_data = GIT_STR_INIT; + + /* "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":\"9556d0a12310629e217450ac4198c49f5457f1a69e22ce7c9f8e81fab4d530a7\",\"size\":499723}]}" */ + if (git_str_join_n( @@ -1375,6 +1426,7 @@ index 000000000..84796d4b0 + /* free the custom headers */ + curl_slist_free_all(chunk); + git_str_dispose(&lfs_info_data); ++ git_str_dispose(&lfs_info_url); + goto on_error; + } + @@ -1383,7 +1435,6 @@ index 000000000..84796d4b0 + CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA)); + CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_FOLLOWLOCATION, 1L)); + -+ struct memory response = { 0 }; + CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_WRITEFUNCTION, write_callback)); + CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_USERAGENT,"git-lfs/3.5.0")); + CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_WRITEDATA, (void *)&response)); @@ -1447,7 +1498,6 @@ index 000000000..84796d4b0 + CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_WRITEDATA, (void *)&ftpfile)); + + /* progress bar options */ -+ struct progress_data progress_d = { time(NULL), time(NULL) , false }; + CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_NOPROGRESS, 0L)); + CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_XFERINFOFUNCTION, progress_callback)); + CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_XFERINFODATA, &progress_d)); @@ -1533,16 +1583,8 @@ index 000000000..84796d4b0 + goto on_error; + } + free(tmp_out_file); -+ if (la) { -+ free((char *)la->path); -+ free((char *)la->full_path); -+ free((char *)la->workdir); -+ free((char *)la->lfs_oid); -+ free((char *)la->lfs_size); -+ free((char *)la->url); -+ } -+ git__free(payload); + git_str_dispose(&res_str); ++ lfs_attrs_delete(la); + return; + + on_error: @@ -1552,15 +1594,7 @@ index 000000000..84796d4b0 + la->full_path); + fflush(stderr); + git_str_dispose(&res_str); -+ if (la) { -+ free((char *)la->path); -+ free((char *)la->full_path); -+ free((char *)la->workdir); -+ free((char *)la->lfs_oid); -+ free((char *)la->lfs_size); -+ free((char *)la->url); -+ } -+ git__free(payload); ++ lfs_attrs_delete(la); + return; +} + @@ -1581,12 +1615,12 @@ index 000000000..84796d4b0 + +git_filter *git_lfs_filter_new(void) +{ -+ /* In Windows, this inits the Winsock stuff */ -+ curl_global_init(CURL_GLOBAL_ALL); + git_filter *f = git__calloc(1, sizeof(git_filter)); + if (f == NULL) + return NULL; + ++ /* In Windows, this inits the Winsock stuff */ ++ curl_global_init(CURL_GLOBAL_ALL); + /* Initialize env-config exactly once per process */ + lfs_resume_env_init_once(); + From 3f3cc90f60a6b93ecc7e9ae37eafd2cce73d4f92 Mon Sep 17 00:00:00 2001 From: rasapala Date: Mon, 16 Mar 2026 17:28:55 +0100 Subject: [PATCH 37/49] Fix compile3 --- third_party/libgit2/lfs.patch | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 01c2c0971b..34812db0be 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -444,7 +444,7 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..bcbae8bc5 +index 000000000..68371e058 --- /dev/null +++ b/src/libgit2/lfs_filter.c @@ -0,0 +1,1185 @@ @@ -504,9 +504,8 @@ index 000000000..bcbae8bc5 + int *out_value) +{ + char *end; -+ unsigned long long val; -+ unsigned int ival; -+ end = NULL; ++ long long val; ++ int ival; + const char *s = getenv(env_name); + if (!s || !*s) { + *out_value = default_value; @@ -518,6 +517,7 @@ index 000000000..bcbae8bc5 + s++; + + errno = 0; ++ end = NULL; + val = strtoull(s, &end, 10); + + if (errno == ERANGE || end == s) { @@ -564,7 +564,6 @@ index 000000000..bcbae8bc5 + char *end; + unsigned long long val; + unsigned int uval; -+ end = NULL; + const char *s = getenv(env_name); + if (!s || !*s) { + *out_value = default_value; @@ -575,6 +574,7 @@ index 000000000..bcbae8bc5 + s++; + + errno = 0; ++ end = NULL; + val = strtoull(s, &end, 10); + + if (errno == ERANGE || end == s) { @@ -669,7 +669,6 @@ index 000000000..bcbae8bc5 + bool is_download; +} lfs_attrs; + -+ +/* + * Frees heap-allocated strings referenced by the struct fields. + * Ownership assumption: @@ -678,7 +677,7 @@ index 000000000..bcbae8bc5 + * - If any field is borrowed (e.g., string literal or external buffer), set + * it to NULL before calling this function to avoid invalid free(). + */ -+static inline void lfs_attrs_free(lfs_attrs *a) ++static void lfs_attrs_free(lfs_attrs *a) +{ + if (!a) + return; @@ -711,7 +710,7 @@ index 000000000..bcbae8bc5 +} + +/* Use this if the struct itself was heap-allocated. */ -+static inline void lfs_attrs_delete(lfs_attrs *a) ++static void lfs_attrs_delete(lfs_attrs *a) +{ + if (!a) + return; @@ -840,7 +839,7 @@ index 000000000..bcbae8bc5 + + /* 2) Stream the payload in chunks — hash *only* the file bytes. */ + CHUNK = 4 * 1024 * 1024; /* 4 MiB */ -+ p = (const unsigned char *)input->ptr; ++ p = (unsigned char *)input->ptr; + remaining = input->size; + + while (remaining > 0) { @@ -1148,6 +1147,7 @@ index 000000000..bcbae8bc5 + time_t currentTime = time(NULL); + bool shouldPrintDueToTime = false; + GIT_UNUSED(ulnow); ++ GIT_UNUSED(ultotal); + if (dlnow == 0) { + pcs->started_download = time(NULL); + pcs->last_print_time = time(NULL); @@ -1295,8 +1295,8 @@ index 000000000..bcbae8bc5 + unsigned int interval_seconds) +{ + CURLcode res = CURLE_OK; -+ -+ for (int attempt = 1; attempt <= max_retries; ++attempt) { ++ int attempt; ++ for (attempt = 1; attempt <= max_retries; ++attempt) { + res = curl_resume_url_execute(dl_curl, ftpfile); + + if (res == CURLE_OK) { From 9d2bb708c2da7214686db4d006d7a8d2438f0efe Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Tue, 17 Mar 2026 10:28:33 +0100 Subject: [PATCH 38/49] Fix struct --- src/pull_module/hf_pull_model_module.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pull_module/hf_pull_model_module.hpp b/src/pull_module/hf_pull_model_module.hpp index 46b92b8b57..a306b7bb7a 100644 --- a/src/pull_module/hf_pull_model_module.hpp +++ b/src/pull_module/hf_pull_model_module.hpp @@ -21,7 +21,7 @@ #include "../capi_frontend/server_settings.hpp" namespace ovms { -class Libgt2InitGuard; +struct Libgt2InitGuard; class HfPullModelModule : public Module { protected: HFSettingsImpl hfSettings; From ddd90b7cc02ab3159ff9bbc64bfcf19128bdb73b Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 17 Mar 2026 14:44:32 +0100 Subject: [PATCH 39/49] Bug fix --- third_party/libgit2/lfs.patch | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 34812db0be..d834dba89c 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -444,7 +444,7 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..68371e058 +index 000000000..b0971f290 --- /dev/null +++ b/src/libgit2/lfs_filter.c @@ -0,0 +1,1185 @@ @@ -1182,7 +1182,6 @@ index 000000000..68371e058 +static size_t file_write_callback(void *buffer, size_t size, size_t nmemb, void *stream) +{ + struct FtpFile *out = (struct FtpFile *)stream; -+ size_t written_items; + if (!out->stream) { + /* open file for writing */ + out->stream = fopen(out->filename, "wb"); @@ -1193,8 +1192,7 @@ index 000000000..68371e058 + } + } + -+ written_items = fwrite(buffer, size, nmemb, out->stream); -+ return written_items * size; /* return BYTES written */ ++ return fwrite(buffer, size, nmemb, out->stream); +} + +static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp) @@ -1585,6 +1583,7 @@ index 000000000..68371e058 + free(tmp_out_file); + git_str_dispose(&res_str); + lfs_attrs_delete(la); ++ fflush(stdout); + return; + + on_error: @@ -1593,6 +1592,7 @@ index 000000000..68371e058 + fprintf(stderr, "\n[ERROR] LFS download failed for file %s\n", + la->full_path); + fflush(stderr); ++ fflush(stdout); + git_str_dispose(&res_str); + lfs_attrs_delete(la); + return; @@ -1626,7 +1626,7 @@ index 000000000..68371e058 + + f->version = GIT_FILTER_VERSION; + f->attributes = "lfs"; -+ f->shutdown = git_lfs_filter_free; /* will free cfg and filter */ ++ f->shutdown = git_lfs_filter_free; + f->stream = lfs_stream; + f->check = lfs_check; + f->cleanup = lfs_download; From ea5613c6b1c17a16ceedf92520aa5de744b463ad Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 17 Mar 2026 14:57:39 +0100 Subject: [PATCH 40/49] Log offset --- third_party/libgit2/lfs.patch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index d834dba89c..7d0ec6a05f 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -444,7 +444,7 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..b0971f290 +index 000000000..7c30438b8 --- /dev/null +++ b/src/libgit2/lfs_filter.c @@ -0,0 +1,1185 @@ @@ -1246,7 +1246,6 @@ index 000000000..b0971f290 + */ + CURLcode res; + curl_off_t offset = 0; -+ printf("\n[INFO] curl_easy_perform() trying to resume file download\n"); + { + FILE *f = fopen(ftpfile->filename, "ab+"); + if (f) { @@ -1259,6 +1258,7 @@ index 000000000..b0971f290 + } + } + ++ printf("\n[INFO] curl_easy_perform() trying to resume file download from %lld bytes\n", offset); + /* Tell libcurl to resume */ + curl_easy_setopt(dl_curl, CURLOPT_RESUME_FROM_LARGE, offset); + /* Perform the request, res gets the return code */ From ccb44cc909a943eca0e353d0fcb2bcf35eba8599 Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 17 Mar 2026 15:17:52 +0100 Subject: [PATCH 41/49] Fix close --- third_party/libgit2/lfs.patch | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 7d0ec6a05f..69cce51acd 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -444,10 +444,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..7c30438b8 +index 000000000..524f7dd7f --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,1185 @@ +@@ -0,0 +1,1193 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -1182,6 +1182,7 @@ index 000000000..7c30438b8 +static size_t file_write_callback(void *buffer, size_t size, size_t nmemb, void *stream) +{ + struct FtpFile *out = (struct FtpFile *)stream; ++ size_t written_items; + if (!out->stream) { + /* open file for writing */ + out->stream = fopen(out->filename, "wb"); @@ -1192,7 +1193,8 @@ index 000000000..7c30438b8 + } + } + -+ return fwrite(buffer, size, nmemb, out->stream); ++ written_items = fwrite(buffer, size, nmemb, out->stream); ++ return written_items * size; /* return BYTES written */ +} + +static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp) @@ -1246,19 +1248,25 @@ index 000000000..7c30438b8 + */ + CURLcode res; + curl_off_t offset = 0; -+ { -+ FILE *f = fopen(ftpfile->filename, "ab+"); -+ if (f) { -+ if (fseeko(f, 0, SEEK_END) == 0) { -+ off_t pos = ftello(f); ++ printf("\n[INFO] curl_easy_perform() trying to resume file download\n"); ++ if (ftpfile->stream) { ++ fclose(ftpfile->stream); ++ } ++ ftpfile->stream = fopen(ftpfile->filename, "ab+"); ++ if (ftpfile->stream) { ++ if (fseeko(ftpfile->stream, 0, SEEK_END) == 0) { ++ off_t pos = ftello(ftpfile->stream); + if (pos > 0) + offset = (curl_off_t)pos; -+ } -+ fclose(f); + } ++ // Do not close the file because we want to append binary to the existin file ++ // fclose(f); ++ } else { ++ fprintf(stderr, "\n[ERROR] Cannot open file %s\n", ++ ftpfile->filename); ++ return -1; + } + -+ printf("\n[INFO] curl_easy_perform() trying to resume file download from %lld bytes\n", offset); + /* Tell libcurl to resume */ + curl_easy_setopt(dl_curl, CURLOPT_RESUME_FROM_LARGE, offset); + /* Perform the request, res gets the return code */ From c42c661ab00e16bc367e34dbeb74b1bd211ffb6d Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 17 Mar 2026 15:21:25 +0100 Subject: [PATCH 42/49] Fix comment --- third_party/libgit2/lfs.patch | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 69cce51acd..fd44b7ad38 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -444,10 +444,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..524f7dd7f +index 000000000..720d86c95 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,1193 @@ +@@ -0,0 +1,1194 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -504,7 +504,7 @@ index 000000000..524f7dd7f + int *out_value) +{ + char *end; -+ long long val; ++ unsigned long long val; + int ival; + const char *s = getenv(env_name); + if (!s || !*s) { @@ -1259,8 +1259,9 @@ index 000000000..524f7dd7f + if (pos > 0) + offset = (curl_off_t)pos; + } -+ // Do not close the file because we want to append binary to the existin file -+ // fclose(f); ++ /* Do not close the file because we want to append binary to the ++ existin file ++ fclose(ftpfile->stream);*/ + } else { + fprintf(stderr, "\n[ERROR] Cannot open file %s\n", + ftpfile->filename); From faaabdc29dc6d9a9f31df6b166c55f87a10ff2d1 Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 17 Mar 2026 15:44:46 +0100 Subject: [PATCH 43/49] Add comments --- third_party/libgit2/lfs.patch | 585 ++++++++++++++++++++++++---------- 1 file changed, 425 insertions(+), 160 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index fd44b7ad38..6aea4a4903 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -444,10 +444,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..720d86c95 +index 000000000..e3e48eed8 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,1194 @@ +@@ -0,0 +1,1459 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -484,7 +484,6 @@ index 000000000..720d86c95 +#include "regexp.h" +#include "time.h" + -+ +#define LFS_RESUME_ATTEMPTS_DEFAULT 5 +#define LFS_RESUME_ATTEMPTS_MAX 100 +#define LFS_RESUME_INTERVAL_DEFAULT 10 @@ -494,8 +493,22 @@ index 000000000..720d86c95 +static int g_lfs_resume_attempts = 5; /* <-- make configurable */ +static unsigned int g_lfs_resume_interval_secs = 10; /* <-- make configurable */ + -+ -+/* Parse a non-negative integer from env, with bounds and default fallback */ ++/* ++ * parse_env_nonneg_int ++ * --------------------- ++ * Parses an environment variable as a non‑negative integer. ++ * ++ * Parameters: ++ * env_name - Name of the environment variable to read. ++ * default_value - Value to use if the env variable is missing/invalid. ++ * min_value - Minimum allowed integer value. ++ * max_value - Maximum allowed integer value. ++ * out_value - Output pointer where parsed/clamped value is stored. ++ * ++ * Returns: ++ * 0 on success (or default fallback), ++ * -1 if parsing fails but default is used. ++ */ +static int parse_env_nonneg_int( + const char *env_name, + int default_value, @@ -553,7 +566,22 @@ index 000000000..720d86c95 + return 0; +} + -+/* Convenience for unsigned interval */ ++/* ++ * parse_env_nonneg_uint ++ * ---------------------- ++ * Same as parse_env_nonneg_int but operates on unsigned integers. ++ * ++ * Parameters: ++ * env_name - Environment variable name. ++ * default_value - Default unsigned value. ++ * min_value - Minimum allowed uint. ++ * max_value - Maximum allowed uint. ++ * out_value - Output pointer for parsed value. ++ * ++ * Returns: ++ * 0 on success (or default fallback); ++ * -1 on parse failure. ++ */ +static int parse_env_nonneg_uint( + const char *env_name, + unsigned int default_value, @@ -606,7 +634,15 @@ index 000000000..720d86c95 + return 0; +} + -+/* Initialize globals from env exactly once */ ++/* ++ * lfs_resume_env_init ++ * -------------------- ++ * Initializes global resume configuration from environment: ++ * GIT_LFS_RESUME_ATTEMPTS ++ * GIT_LFS_RESUME_INTERVAL ++ * ++ * Called exactly once via pthread_once / InitOnce. ++ */ +static void lfs_resume_env_init(void) +{ + parse_env_nonneg_int( @@ -626,11 +662,23 @@ index 000000000..720d86c95 +#include +#define fseeko _fseeki64 +#define ftello _ftelli64 ++/* ++ * sleep_seconds ++ * -------------- ++ * Cross-platform helper to sleep for a given number of seconds. ++ * ++ * Parameters: ++ * seconds - Number of seconds to sleep. ++ */ +static void sleep_seconds(unsigned int seconds) +{ + Sleep(seconds * 1000); +} -+ ++/* ++ * lfs_once_cb_win / lfs_once_cb_posix ++ * ----------------------------------- ++ * One-time initialization wrapper for Windows/POSIX. ++ */ +#define LFS_ONCE_INIT INIT_ONCE_STATIC_INIT +static INIT_ONCE lfs_once = LFS_ONCE_INIT; +static BOOL CALLBACK lfs_once_cb_win(PINIT_ONCE once, PVOID param, PVOID *ctx) @@ -646,7 +694,7 @@ index 000000000..720d86c95 +#else +/* POSIX systems (Linux, macOS) */ +#include -+#include ++#include +static pthread_once_t lfs_once = PTHREAD_ONCE_INIT; +static void lfs_once_cb_posix(void) +{ @@ -658,11 +706,10 @@ index 000000000..720d86c95 +} +#endif + -+typedef struct lfs_attrs -+{ ++typedef struct lfs_attrs { + const char *path; -+ const char* full_path; -+ const char* workdir; ++ const char *full_path; ++ const char *workdir; + const char *lfs_oid; + const char *lfs_size; + const char *url; @@ -671,6 +718,8 @@ index 000000000..720d86c95 + +/* + * Frees heap-allocated strings referenced by the struct fields. ++ * Parameters: ++ * a - Pointer to lfs_attrs struct. Not freed itself. + * Ownership assumption: + * - Only call free() on fields that point to heap memory + * (malloc/calloc/realloc/strdup). @@ -709,7 +758,14 @@ index 000000000..720d86c95 + a->is_download = false; +} + -+/* Use this if the struct itself was heap-allocated. */ ++/* ++ * lfs_attrs_delete ++ * ----------------- ++ * Frees both the struct fields and the struct itself. ++ * ++ * Parameters: ++ * a - Heap-allocated lfs_attrs struct. ++ */ +static void lfs_attrs_delete(lfs_attrs *a) +{ + if (!a) @@ -719,6 +775,18 @@ index 000000000..720d86c95 + a = NULL; +} + ++/* ++ * get_digit ++ * ---------- ++ * Parses a decimal number from a C string using strtoull. ++ * ++ * Parameters: ++ * buffer - The C string containing digits. ++ * ++ * Returns: ++ * The parsed integer on success, ++ * 0 on failure. ++ */ +static size_t get_digit(const char *buffer) +{ + char *endptr; @@ -732,28 +800,35 @@ index 000000000..720d86c95 + number = strtoull(buffer, &endptr, 10); + + if (errno == ERANGE) { -+ fprintf(stderr, "\n[ERROR] Conversion error\n"); ++ fprintf(stderr, "\n[ERROR] Conversion error\n"); + } + if (endptr == buffer) { + fprintf(stderr, "\n[ERROR] No digits were found\n"); + } else if (*endptr != '\0') { -+ fprintf(stderr, "\n[ERROR] Additional characters after number: %s\n", endptr); ++ fprintf(stderr, ++ "\n[ERROR] Additional characters after number: %s\n", ++ endptr); + } + + return number; +} + -+/** -+ * Appends a C-string `suffix` to `existingBuffer` by allocating a new buffer. -+ * The original `existingBuffer` is not modified. ++/* ++ * append_cstr_to_buffer ++ * ---------------------- ++ * Allocates a new string consisting of existingBuffer + suffix. ++ * ++ * Parameters: ++ * existingBuffer - Original string. ++ * suffix - String to append. + * + * Returns: -+ * - Newly allocated buffer containing the concatenation, or -+ * - NULL on allocation failure or if inputs are invalid. ++ * Newly allocated concatenated string, or NULL on failure. + * -+ * Note: Caller is responsible for freeing the returned buffer. ++ * Caller must free result. + */ -+static char *append_cstr_to_buffer(const char *existingBuffer, const char *suffix) ++static char * ++append_cstr_to_buffer(const char *existingBuffer, const char *suffix) +{ + size_t existingLength; + size_t suffixLength; @@ -782,9 +857,20 @@ index 000000000..720d86c95 + return newBuffer; +} + -+static int get_lfs_info_match( -+ git_str *output, -+ const char *regexp) ++/* ++ * get_lfs_info_match ++ * ------------------- ++ * Applies a regex to a git_str buffer and extracts the matched substring. ++ * ++ * Parameters: ++ * output - git_str to modify in place (trimmed to match). ++ * regexp - Regular expression to apply. ++ * ++ * Returns: ++ * 0 if match found, ++ * -1 otherwise. ++ */ ++static int get_lfs_info_match(git_str *output, const char *regexp) +{ + int result; + git_regexp preg; @@ -811,6 +897,22 @@ index 000000000..720d86c95 + return -1; +} + ++/* ++ * git_oid_sha256_from_git_str_blob ++ * --------------------------------- ++ * Computes SHA‑256 of a git_str blob and optionally formats: ++ * "oid sha256:" ++ * ++ * Parameters: ++ * out - git_oid output. ++ * input - git_str containing file contents. ++ * pointer_line - Optional output buffer for formatted oid line. ++ * pointer_line_cap- Capacity of pointer_line. ++ * ++ * Returns: ++ * 0 on success, ++ * -1 on error. ++ */ +static int git_oid_sha256_from_git_str_blob( + git_oid *out, + const struct git_str *input, @@ -868,33 +970,50 @@ index 000000000..720d86c95 + "\n[ERROR] failure, git_oid_fmt failed\n"); + goto error; + } -+ ++ + hex[64] = '\0'; + snprintf(pointer_line, pointer_line_cap, "oid sha256:%s", hex); + } + + git_hash_ctx_cleanup(&ctx); + return 0; -+ error: ++error: + git_hash_ctx_cleanup(&ctx); -+ return -1; -+ } ++ return -1; ++} + -+static int lfs_remove_id( -+ git_str *to, -+ const git_str *from, -+ void **payload) ++/* ++ * lfs_remove_id ++ * -------------- ++ * Converts original file content into LFS pointer file: ++ * version ... ++ * oid sha256:... ++ * size ... ++ * ++ * Used in "clean" filter (upload or diff). ++ * ++ * Parameters: ++ * to - Output git_str pointer file. ++ * from - Input git_str original file. ++ * payload - Output payload with initialized lfs_attrs (minimal). ++ * ++ * Returns: ++ * 0 on success, negative on error. ++ */ ++static int lfs_remove_id(git_str *to, const git_str *from, void **payload) +{ + int error = 0; + char line[80]; /* 75+ is enough */ + git_oid lfs_oid; -+ /* Init the lfs attrs to indicate git lfs clean, currently only diff support no upload of lfs file supported */ ++ /* Init the lfs attrs to indicate git lfs clean, currently only diff ++ * support no upload of lfs file supported */ + struct lfs_attrs la = { NULL, NULL, NULL, NULL, NULL, NULL, false }; + *payload = git__malloc(sizeof(la)); + GIT_ERROR_CHECK_ALLOC(*payload); + memcpy(*payload, &la, sizeof(la)); + -+ if(!from) return -1; ++ if (!from) ++ return -1; + + /* lfs spec - return empty pointer when the file is empty */ + if (from->size == 0) { @@ -904,9 +1023,9 @@ index 000000000..720d86c95 + + /* Use lib git oid to get lfs sha256 */ + lfs_oid.type = GIT_OID_SHA256; -+ if (git_oid_sha256_from_git_str_blob(&lfs_oid, from, line, sizeof(line)) < 0) { -+ fprintf(stderr, -+ "\n[ERROR] failure, cannot calculate sha256\n"); ++ if (git_oid_sha256_from_git_str_blob( ++ &lfs_oid, from, line, sizeof(line)) < 0) { ++ fprintf(stderr, "\n[ERROR] failure, cannot calculate sha256\n"); + return -1; + } + @@ -918,7 +1037,6 @@ index 000000000..720d86c95 + fprintf(stderr, "\n[ERROR] git_str_puts failed\n"); + return error; + } -+ + + /* 2) the oid line passed by caller (must end with '\n') */ + if ((error = git_str_puts(to, line)) < 0) { @@ -942,8 +1060,26 @@ index 000000000..720d86c95 + return 0; +} + ++/* ++ * lfs_insert_id ++ * -------------- ++ * Parses an LFS pointer file, extracts OID and size, populates payload. ++ * Returns the original input unchanged (content of LFS pointer). ++ * ++ * Parameters: ++ * to - Output buffer. ++ * from - LFS pointer file content. ++ * src - Filter source context. ++ * payload - Output lfs_attrs struct. ++ * ++ * Returns: ++ * 0 on success, negative on error. ++ */ +static int lfs_insert_id( -+ git_str *to, const git_str *from, const git_filter_source *src, void** payload) ++ git_str *to, ++ const git_str *from, ++ const git_filter_source *src, ++ void **payload) +{ + git_str lfs_oid = GIT_STR_INIT; + git_str lfs_size = GIT_STR_INIT; @@ -958,8 +1094,8 @@ index 000000000..720d86c95 + char *path_dup = git__strdup(path); + char *workdir_dup = git__strdup(workdir); + char *url_dup = git__strdup(repo->url); -+ struct lfs_attrs la = { path_dup, NULL, workdir_dup, NULL, -+ NULL, url_dup, true }; ++ struct lfs_attrs la = { path_dup, NULL, workdir_dup, NULL, ++ NULL, url_dup, true }; + + lfs_oid.size = from->size; + lfs_oid.asize = from->asize; @@ -969,7 +1105,8 @@ index 000000000..720d86c95 + lfs_size.ptr = git__strdup(from->ptr); + + if (get_lfs_info_match(&lfs_oid, obj_regexp) < 0) { -+ fprintf(stderr,"\n[ERROR] failure, cannot find lfs oid in: %s\n", ++ fprintf(stderr, ++ "\n[ERROR] failure, cannot find lfs oid in: %s\n", + lfs_oid.ptr); + return -1; + } @@ -997,28 +1134,49 @@ index 000000000..720d86c95 + GIT_ERROR_CHECK_ALLOC(*payload); + memcpy(*payload, &la, sizeof(la)); + -+ /*Just write the oryginal lfs file contents */ ++ /*Just write the oryginal lfs file contents */ + return git_str_set(to, from->ptr, from->size); +} + ++/* ++ * lfs_apply ++ * ---------- ++ * libgit2 filter entrypoint: ++ * - In SMUDGE mode → download (lfs_insert_id). ++ * - In CLEAN mode → create pointer file (lfs_remove_id). ++ */ +static int lfs_apply( -+ git_filter *self, -+ void **payload, -+ git_str *to, -+ const git_str *from, -+ const git_filter_source *src) ++ git_filter *self, ++ void **payload, ++ git_str *to, ++ const git_str *from, ++ const git_filter_source *src) +{ -+ GIT_UNUSED(self); GIT_UNUSED(payload); ++ GIT_UNUSED(self); ++ GIT_UNUSED(payload); + + /* for download of the lfs pointer files */ + if (git_filter_source_mode(src) == GIT_FILTER_SMUDGE) + return lfs_insert_id(to, from, src, payload); + else -+ /* for upload or diff of the lfs pointer files */ ++ /* for upload or diff of the lfs pointer files */ + return lfs_remove_id(to, from, payload); + return 0; +} + ++/* ++ * lfs_check ++ * ---------- ++ * Determines whether a given file path should apply the "lfs" filter. ++ * ++ * Parameters: ++ * src - Filter source. ++ * attr_values - Unused. ++ * ++ * Returns: ++ * 0 if filter applies, ++ * GIT_PASSTHROUGH if not. ++ */ +static int lfs_check( + git_filter *self, + void **payload, /* points to NULL ptr on entry, may be set */ @@ -1032,8 +1190,7 @@ index 000000000..720d86c95 + GIT_UNUSED(self); + GIT_UNUSED(payload); + GIT_UNUSED(attr_values); -+ git_attr_get( -+ &value, repo, GIT_ATTR_CHECK_NO_SYSTEM, path, "filter"); ++ git_attr_get(&value, repo, GIT_ATTR_CHECK_NO_SYSTEM, path, "filter"); + + if (value && *value) { + if (strcmp(value, "lfs") == 0) { @@ -1046,15 +1203,20 @@ index 000000000..720d86c95 + return 0; +} + ++/* ++ * lfs_stream ++ * ----------- ++ * Creates a buffered filter stream wrapper around lfs_apply(). ++ */ +static int lfs_stream( -+ git_writestream **out, -+ git_filter *self, -+ void **payload, -+ const git_filter_source *src, -+ git_writestream *next) ++ git_writestream **out, ++ git_filter *self, ++ void **payload, ++ const git_filter_source *src, ++ git_writestream *next) +{ -+ return git_filter_buffered_stream_new(out, -+ self, lfs_apply, NULL, payload, src, next); ++ return git_filter_buffered_stream_new( ++ out, self, lfs_apply, NULL, payload, src, next); +} + +struct progress_data { @@ -1074,6 +1236,15 @@ index 000000000..720d86c95 +}; + +static const char *sizeUnits[] = { "B", "KB", "MB", "GB", "TB", NULL }; ++/* ++ * print_download_speed_info ++ * -------------------------- ++ * Prints download speed in human-readable units. ++ * ++ * Parameters: ++ * received_size - Bytes downloaded. ++ * elapsed_time - Seconds elapsed. ++ */ +static void print_download_speed_info(size_t received_size, size_t elapsed_time) +{ + double recv_len = (double)received_size; @@ -1090,11 +1261,13 @@ index 000000000..720d86c95 + printf(" [%.2f %s/s] ", rate, sizeUnits[rate_unit_idx]); +} + -+static void print_progress( -+ size_t count, -+ size_t max, -+ bool first_run, -+ size_t elapsed_time) ++/* ++ * print_progress ++ * --------------- ++ * Renders a progress bar with percentage, size, and transfer speed. ++ */ ++static void ++print_progress(size_t count, size_t max, bool first_run, size_t elapsed_time) +{ + float progress; + int i, bar_length, bar_width; @@ -1136,6 +1309,11 @@ index 000000000..720d86c95 + fflush(stdout); +} + ++/* ++ * progress_callback ++ * ------------------ ++ * cURL progress callback wrapper to throttle progress prints. ++ */ +static int progress_callback( + void *clientp, + curl_off_t dltotal, @@ -1152,7 +1330,7 @@ index 000000000..720d86c95 + pcs->started_download = time(NULL); + pcs->last_print_time = time(NULL); + } -+ ++ + shouldPrintDueToTime = (currentTime - pcs->last_print_time >= 1); + if ((dltotal == dlnow) && dltotal < 10000) { + /* Usually with first messages we don't get the full size and we @@ -1161,7 +1339,8 @@ index 000000000..720d86c95 + we would print 100% progress bar */ + return 0; + } -+ /* called multiple times, so we want to print progress bar only once reached 100% */ ++ /* called multiple times, so we want to print progress bar only once ++ * reached 100% */ + if (pcs->fullDownloadPrinted) { + return 0; + } @@ -1179,7 +1358,13 @@ index 000000000..720d86c95 + return 0; +} + -+static size_t file_write_callback(void *buffer, size_t size, size_t nmemb, void *stream) ++/* ++ * file_write_callback ++ * -------------------- ++ * cURL write callback writing received bytes to disk. ++ */ ++static size_t ++file_write_callback(void *buffer, size_t size, size_t nmemb, void *stream) +{ + struct FtpFile *out = (struct FtpFile *)stream; + size_t written_items; @@ -1187,7 +1372,8 @@ index 000000000..720d86c95 + /* open file for writing */ + out->stream = fopen(out->filename, "wb"); + if (!out->stream) { -+ fprintf(stderr, "\n[ERROR] failure, cannot open file to write: %s\n", ++ fprintf(stderr, ++ "\n[ERROR] failure, cannot open file to write: %s\n", + out->filename); + return 0; /* failure, cannot open file to write */ + } @@ -1197,6 +1383,11 @@ index 000000000..720d86c95 + return written_items * size; /* return BYTES written */ +} + ++/* ++ * write_callback ++ * --------------- ++ * cURL callback for accumulating HTTP response into memory. ++ */ +static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp) +{ + size_t realsize = size * nmemb; @@ -1215,7 +1406,12 @@ index 000000000..720d86c95 + return realsize; +} + -+static void print_download_info(const char* filename, size_t bytes) ++/* ++ * print_download_info ++ * -------------------- ++ * Prints human-readable file size before downloading. ++ */ ++static void print_download_info(const char *filename, size_t bytes) +{ + double recv_len = (double)bytes; + size_t recv_unit_idx = 0; @@ -1224,8 +1420,7 @@ index 000000000..720d86c95 + recv_unit_idx++; + } + printf("\nDownloading lfs size: %.2f %s file: %s\n", recv_len, -+ sizeUnits[recv_unit_idx], -+ filename); ++ sizeUnits[recv_unit_idx], filename); +} + +#define CURL_SETOPT(setopt) \ @@ -1233,6 +1428,18 @@ index 000000000..720d86c95 + status = setopt; \ + } + ++/* ++ * curl_resume_url_execute ++ * ------------------------ ++ * Attempts to resume an interrupted download using HTTP Range. ++ * ++ * Parameters: ++ * dl_curl - CURL handle. ++ * ftpfile - Target file stream + filename. ++ * ++ * Returns: ++ * cURL result code. ++ */ +static int curl_resume_url_execute(CURL *dl_curl, struct FtpFile *ftpfile) +{ + /* This helper check does not work but the resume mechanism still works @@ -1256,8 +1463,8 @@ index 000000000..720d86c95 + if (ftpfile->stream) { + if (fseeko(ftpfile->stream, 0, SEEK_END) == 0) { + off_t pos = ftello(ftpfile->stream); -+ if (pos > 0) -+ offset = (curl_off_t)pos; ++ if (pos > 0) ++ offset = (curl_off_t)pos; + } + /* Do not close the file because we want to append binary to the + existin file @@ -1291,10 +1498,20 @@ index 000000000..720d86c95 + return res; +} + -+/* Repeatedly attempts resume with a fixed wait interval. -+ - max_retries: total number of resume attempts to try (e.g., 3) -+ - interval_seconds: wait time between attempts (e.g., 2 seconds) -+ Returns the final CURLcode of the last attempt. */ ++/* ++ * download_with_resume ++ * --------------------- ++ * Retries resuming a download multiple times with delay between attempts. ++ * ++ * Parameters: ++ * dl_curl - CURL download handle. ++ * ftpfile - FtpFile handle. ++ * max_retries - Number of attempts. ++ * interval_seconds - Delay between attempts. ++ * ++ * Returns: ++ * Final attempt's cURL code. ++ */ +static CURLcode download_with_resume( + CURL *dl_curl, + struct FtpFile *ftpfile, @@ -1328,23 +1545,19 @@ index 000000000..720d86c95 + return res; /* last result (failure) */ +} + -+/** -+ * lfs_download - Downloads a file using the LFS (Large File Storage) mechanism. -+ * -+ * This function performs the following steps: -+ * 1. Validates the input payload and initializes necessary resources. -+ * 2. Constructs a temporary output file path for the download. -+ * 3. Initializes CURL handles for HTTP requests. -+ * 4. Performs the download and writes the data to the temporary file. -+ * 5. Cleans up resources and handles errors appropriately. -+ * -+ * Error Handling: -+ * - If the payload is NULL, the function logs an error and returns immediately. -+ * - CURL initialization and operations are checked for errors, and appropriate cleanup is performed. -+ * - File operations are validated to ensure successful creation and writing. ++/* ++ * lfs_download ++ * ------------- ++ * Full LFS download implementation. ++ * Performs: ++ * - batch API request ++ * - parsing download link ++ * - actual file download (with resume support) ++ * - renaming into place + * -+ * @param self: Unused parameter, reserved for future use. -+ * @param payload: Pointer to the lfs_attrs structure containing download parameters. ++ * Parameters: ++ * self - Filter pointer (unused). ++ * payload - Populated lfs_attrs struct with OID, size, path, etc. + */ +static void lfs_download(git_filter *self, void *payload) +{ @@ -1383,8 +1596,7 @@ index 000000000..720d86c95 + git_str lfs_info_url = GIT_STR_INIT; + git_str lfs_info_data = GIT_STR_INIT; + if (git_str_join( -+ &lfs_info_url, '.', -+ la->url, ++ &lfs_info_url, '.', la->url, + "git/info/lfs/objects/batch") < 0) { + fprintf(stderr, "\n[ERROR] failed to create url '%s'\n", + la->full_path); @@ -1394,38 +1606,46 @@ index 000000000..720d86c95 + } + + /* Remove a header curl would otherwise add by itself */ -+ chunk = curl_slist_append(chunk, "Accept: application/vnd.git-lfs+json"); ++ chunk = curl_slist_append( ++ chunk, "Accept: application/vnd.git-lfs+json"); + /* Add a custom header */ -+ chunk = curl_slist_append(chunk, "Content-Type: application/vnd.git-lfs+json"); ++ chunk = curl_slist_append( ++ chunk, "Content-Type: application/vnd.git-lfs+json"); + /* set our custom set of headers */ -+ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_HTTPHEADER, chunk)); ++ CURL_SETOPT( ++ curl_easy_setopt(info_curl, CURLOPT_HTTPHEADER, chunk)); + /* First set the URL that is about to receive our POST. This URL + can just as well be an https:// URL if that is what should + receive the data. */ -+ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_URL, lfs_info_url.ptr)); ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_URL, lfs_info_url.ptr)); + /* Add cURL resiliency */ + /* unlimited data */ -+ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_CONNECTTIMEOUT, 30L)); -+ /* timeout */ -+ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_TIMEOUT, 0L)); -+ /* low speed 1KB/s */ -+ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_LOW_SPEED_LIMIT, 1024L)); -+ /* for 30s */ -+ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_LOW_SPEED_TIME, 30L)); ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_CONNECTTIMEOUT, 30L)); ++ /* timeout */ ++ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_TIMEOUT, 0L)); ++ /* low speed 1KB/s */ ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_LOW_SPEED_LIMIT, 1024L)); ++ /* for 30s */ ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_LOW_SPEED_TIME, 30L)); + + if (status != CURLE_OK) { -+ fprintf(stderr, "\n[ERROR] curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); ++ fprintf(stderr, ++ "\n[ERROR] curl_easy_setopt() failed: %s\n", ++ curl_easy_strerror(status)); + goto info_cleaup; + } + -+ /* "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":\"9556d0a12310629e217450ac4198c49f5457f1a69e22ce7c9f8e81fab4d530a7\",\"size\":499723}]}" */ ++ /* "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":\"9556d0a12310629e217450ac4198c49f5457f1a69e22ce7c9f8e81fab4d530a7\",\"size\":499723}]}" ++ */ + if (git_str_join_n( -+ &lfs_info_data, '"',5, ++ &lfs_info_data, '"', 5, + "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":", -+ la->lfs_oid, -+ ",\"size\":", -+ la->lfs_size, -+ "}]}" ) < 0) { ++ la->lfs_oid, ",\"size\":", la->lfs_size, ++ "}]}") < 0) { + fprintf(stderr, "\n[ERROR] failed to create url '%s'\n", + la->full_path); + /* always cleanup */ @@ -1438,23 +1658,32 @@ index 000000000..720d86c95 + } + + /* Now specify the POST data */ -+ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_POSTFIELDS, lfs_info_data.ptr)); -+ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA)); -+ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_FOLLOWLOCATION, 1L)); -+ -+ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_WRITEFUNCTION, write_callback)); -+ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_USERAGENT,"git-lfs/3.5.0")); -+ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_WRITEDATA, (void *)&response)); ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_POSTFIELDS, lfs_info_data.ptr)); ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA)); ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_FOLLOWLOCATION, 1L)); ++ ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_WRITEFUNCTION, write_callback)); ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_USERAGENT, "git-lfs/3.5.0")); ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_WRITEDATA, (void *)&response)); + + if (status != CURLE_OK) { -+ fprintf(stderr, "\n[ERROR] curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); ++ fprintf(stderr, ++ "\n[ERROR] curl_easy_setopt() failed: %s\n", ++ curl_easy_strerror(status)); + goto info_cleaup; + } + /* Perform the request, res gets the return code */ + res = curl_easy_perform(info_curl); + /* Check for errors */ + if (res != CURLE_OK) { -+ fprintf(stderr, "\n[ERROR] curl_easy_perform() failed: %s\n", ++ fprintf(stderr, ++ "\n[ERROR] curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + /* always cleanup */ + curl_easy_cleanup(info_curl); @@ -1470,15 +1699,15 @@ index 000000000..720d86c95 + res_str.size = response.size; + res_str.ptr = git__strdup(response.response); + free(response.response); -+ info_cleaup: -+ /* always cleanup */ -+ curl_easy_cleanup(info_curl); -+ /* free the custom headers */ -+ curl_slist_free_all(chunk); -+ git_str_dispose(&lfs_info_url); -+ git_str_dispose(&lfs_info_data); -+ if (status != CURLE_OK) -+ goto on_error; ++ info_cleaup: ++ /* always cleanup */ ++ curl_easy_cleanup(info_curl); ++ /* free the custom headers */ ++ curl_slist_free_all(chunk); ++ git_str_dispose(&lfs_info_url); ++ git_str_dispose(&lfs_info_data); ++ if (status != CURLE_OK) ++ goto on_error; + } + + /* get a curl handle */ @@ -1486,7 +1715,8 @@ index 000000000..720d86c95 + if (dl_curl) { + struct FtpFile ftpfile = { tmp_out_file, NULL }; + -+ const char *href_regexp = "\"download\"\\s*:\\s*\\{\\s*\"href\":\"([^\"]+)\""; ++ const char *href_regexp = ++ "\"download\"\\s*:\\s*\\{\\s*\"href\":\"([^\"]+)\""; + if (get_lfs_info_match(&res_str, href_regexp) < 0) { + /* always cleanup */ + curl_easy_cleanup(dl_curl); @@ -1496,39 +1726,53 @@ index 000000000..720d86c95 + /* First set the URL that is about to receive our POST. This URL + can just as well be an https:// URL if that is what should + receive the data. */ -+ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_URL, res_str.ptr)); -+ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA)); -+ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_FOLLOWLOCATION, 1L)); -+ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_USE_SSL, CURLUSESSL_ALL)); -+ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_USERAGENT,"git-lfs/3.5.0")); -+ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_WRITEFUNCTION, file_write_callback)); -+ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_WRITEDATA, (void *)&ftpfile)); ++ CURL_SETOPT( ++ curl_easy_setopt(dl_curl, CURLOPT_URL, res_str.ptr)); ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA)); ++ CURL_SETOPT( ++ curl_easy_setopt(dl_curl, CURLOPT_FOLLOWLOCATION, 1L)); ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_USE_SSL, CURLUSESSL_ALL)); ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_USERAGENT, "git-lfs/3.5.0")); ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_WRITEFUNCTION, file_write_callback)); ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_WRITEDATA, (void *)&ftpfile)); + + /* progress bar options */ + CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_NOPROGRESS, 0L)); -+ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_XFERINFOFUNCTION, progress_callback)); -+ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_XFERINFODATA, &progress_d)); ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_XFERINFOFUNCTION, progress_callback)); ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_XFERINFODATA, &progress_d)); + + /* Add cURL resiliency */ + /* unlimited data */ -+ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_CONNECTTIMEOUT, 30L)); -+ /* timeout */ -+ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_TIMEOUT, 0L)); -+ /* low speed 1KB/s */ -+ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_LOW_SPEED_LIMIT, 1024L)); -+ /* for 30s */ -+ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_LOW_SPEED_TIME, 30L)); ++ CURL_SETOPT( ++ curl_easy_setopt(dl_curl, CURLOPT_CONNECTTIMEOUT, 30L)); ++ /* timeout */ ++ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_TIMEOUT, 0L)); ++ /* low speed 1KB/s */ ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_LOW_SPEED_LIMIT, 1024L)); ++ /* for 30s */ ++ CURL_SETOPT( ++ curl_easy_setopt(dl_curl, CURLOPT_LOW_SPEED_TIME, 30L)); + if (status != CURLE_OK) { -+ fprintf(stderr, "\n[ERROR] curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); ++ fprintf(stderr, ++ "\n[ERROR] curl_easy_setopt() failed: %s\n", ++ curl_easy_strerror(status)); + curl_easy_cleanup(dl_curl); + git_str_dispose(&res_str); + goto on_error; + } + -+ /* Check for resume if previous download failed and we have the partial file on disk */ ++ /* Check for resume if previous download failed and we have the ++ * partial file on disk */ + ftpfile.stream = fopen(ftpfile.filename, "r"); -+ if (ftpfile.stream != NULL) -+ { ++ if (ftpfile.stream != NULL) { + resumingFileByBlobFilter = true; + fclose(ftpfile.stream); + ftpfile.stream = NULL; @@ -1552,10 +1796,11 @@ index 000000000..720d86c95 + dl_curl, &ftpfile, g_lfs_resume_attempts, + g_lfs_resume_interval_secs); + } -+ ++ + /* Check for errors */ + if (res != CURLE_OK) { -+ fprintf(stderr, "\n[ERROR] curl_easy_perform() failed: %s\n", ++ fprintf(stderr, ++ "\n[ERROR] curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + if (ftpfile.stream) { + fclose(ftpfile.stream); @@ -1574,7 +1819,8 @@ index 000000000..720d86c95 + curl_easy_cleanup(dl_curl); + } + -+ /* Remove lfs file and rename downloaded file to oryginal lfs filename */ ++ /* Remove lfs file and rename downloaded file to oryginal lfs filename ++ */ + if (!resumingFileByBlobFilter) { + /* File does not exist when using blob filters */ + if (p_unlink(la->full_path) < 0) { @@ -1586,7 +1832,8 @@ index 000000000..720d86c95 + } + + if (p_rename(tmp_out_file, la->full_path) < 0) { -+ fprintf(stderr, "\n[ERROR] failed to rename file to '%s'\n", la->full_path); ++ fprintf(stderr, "\n[ERROR] failed to rename file to '%s'\n", ++ la->full_path); + goto on_error; + } + free(tmp_out_file); @@ -1595,9 +1842,9 @@ index 000000000..720d86c95 + fflush(stdout); + return; + -+ on_error: ++on_error: + free(tmp_out_file); -+ on_error2: ++on_error2: + fprintf(stderr, "\n[ERROR] LFS download failed for file %s\n", + la->full_path); + fflush(stderr); @@ -1607,12 +1854,22 @@ index 000000000..720d86c95 + return; +} + ++/* ++ * git_lfs_filter_free ++ * -------------------- ++ * Frees filter instance and performs CURL cleanup. ++ */ +static void git_lfs_filter_free(git_filter *filter) +{ + curl_global_cleanup(); + git__free(filter); +} + ++/* ++ * lfs_resume_env_init_once ++ * ------------------------- ++ * Runs environment configuration initializer exactly once per process. ++ */ +static void lfs_resume_env_init_once(void) +{ +#ifdef _WIN32 @@ -1622,6 +1879,14 @@ index 000000000..720d86c95 +#endif +} + ++/* ++ * git_lfs_filter_new ++ * ------------------- ++ * Creates and initializes the LFS filter struct used by libgit2. ++ * ++ * Returns: ++ * Pointer to new git_filter struct, or NULL on allocation error. ++ */ +git_filter *git_lfs_filter_new(void) +{ + git_filter *f = git__calloc(1, sizeof(git_filter)); From 66d6ab2e7042547de199bac426f9a466ef5f6d1c Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 18 Mar 2026 09:41:25 +0100 Subject: [PATCH 44/49] Spellings --- third_party/libgit2/lfs.patch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 6aea4a4903..ffa3c66a2b 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -444,7 +444,7 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..e3e48eed8 +index 000000000..ef3499b17 --- /dev/null +++ b/src/libgit2/lfs_filter.c @@ -0,0 +1,1459 @@ @@ -1467,7 +1467,7 @@ index 000000000..e3e48eed8 + offset = (curl_off_t)pos; + } + /* Do not close the file because we want to append binary to the -+ existin file ++ existing file + fclose(ftpfile->stream);*/ + } else { + fprintf(stderr, "\n[ERROR] Cannot open file %s\n", From d1614c895200662e23c86371706cd69a67db8fbe Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 18 Mar 2026 10:19:31 +0100 Subject: [PATCH 45/49] Stdout --- third_party/libgit2/lfs.patch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index ffa3c66a2b..669a26a4a3 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -444,7 +444,7 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..ef3499b17 +index 000000000..11616ed94 --- /dev/null +++ b/src/libgit2/lfs_filter.c @@ -0,0 +1,1459 @@ @@ -654,7 +654,7 @@ index 000000000..ef3499b17 + LFS_RESUME_INTERVAL_MAX, &g_lfs_resume_interval_secs); + + /* log resolved config */ -+ fprintf(stderr, "[INFO] LFS resume: attempts=%d interval=%u s\n", ++ fprintf(stdout, "[INFO] LFS resume: attempts=%d interval=%u s\n", + g_lfs_resume_attempts, g_lfs_resume_interval_secs); +} + From ce8b04fe04a095bb878c368f925f1f0a7c5ff986 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 18 Mar 2026 13:54:01 +0100 Subject: [PATCH 46/49] Resume ENV tests --- src/test/libgit2_test.cpp | 2 +- src/test/pull_hf_model_test.cpp | 47 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/test/libgit2_test.cpp b/src/test/libgit2_test.cpp index b221ee9c54..3774d5f034 100644 --- a/src/test/libgit2_test.cpp +++ b/src/test/libgit2_test.cpp @@ -330,7 +330,7 @@ static fs::path createTempDir() { const fs::path base = fs::temp_directory_path(); std::random_device rd; std::mt19937_64 gen(rd()); - std::uniform_int_distribution dist; + std::uniform_int_distribution dist; // Try a reasonable number of times to avoid rare collisions for (int attempt = 0; attempt < 100; ++attempt) { diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index 5092f49dbf..1c740964d9 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -759,6 +759,53 @@ class HfDownloaderHfEnvTest : public ::testing::Test { EnvGuard guard; }; +TEST(Libgt2InitGuardTest, LfsFilterCaptureDefaultResumeOptions) +{ + // Need new process beacase we use INIT_ONCE in libgit2 lfs filter for env variables and once they are set they are set for the whole process lifetime, so we need to spawn new process without them set to test default values + EXPECT_EXIT({ + // Act: capture stdout during object construction + testing::internal::CaptureStdout(); + { + auto guardOrError = ovms::createGuard(); + ASSERT_EQ(std::holds_alternative(guardOrError), false); + } + std::string output = testing::internal::GetCapturedStdout(); + + // Optional: trim trailing newline + if (!output.empty() && output.back() == '\n') { + output.pop_back(); + } + + EXPECT_THAT(output, ::testing::HasSubstr("[INFO] LFS resume: attempts=5 interval=10 s")); + exit(0); + }, ::testing::ExitedWithCode(0), ""); +} + +TEST(Libgt2InitGuardTest, LfsFilterCaptureNonDefaultResumeOptions) +{ + // Need new process beacase we use INIT_ONCE in libgit2 lfs filter for env variables and once they are set they are set for the whole process lifetime, so we need to spawn new process without them set to test default values + EXPECT_EXIT({ + EnvGuard guard; + guard.set("GIT_LFS_RESUME_ATTEMPTS", "3"); + guard.set("GIT_LFS_RESUME_INTERVAL", "20"); + // Act: capture stdout during object construction + testing::internal::CaptureStdout(); + { + auto guardOrError = ovms::createGuard(); + ASSERT_EQ(std::holds_alternative(guardOrError), false); + } + std::string output = testing::internal::GetCapturedStdout(); + + // Optional: trim trailing newline + if (!output.empty() && output.back() == '\n') { + output.pop_back(); + } + + EXPECT_THAT(output, ::testing::HasSubstr("[INFO] LFS resume: attempts=3 interval=20 s")); + exit(0); + }, ::testing::ExitedWithCode(0), ""); +} + TEST_F(HfDownloaderHfEnvTest, Methods) { std::string modelName = "model/name"; std::string downloadPath = "/path/to/Download"; From 98ed4ae48b36a98302b998c09613a0c4256a25fd Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 18 Mar 2026 13:55:35 +0100 Subject: [PATCH 47/49] Style --- src/test/pull_hf_model_test.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index 1c740964d9..ba3d2143d6 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -759,9 +759,8 @@ class HfDownloaderHfEnvTest : public ::testing::Test { EnvGuard guard; }; -TEST(Libgt2InitGuardTest, LfsFilterCaptureDefaultResumeOptions) -{ - // Need new process beacase we use INIT_ONCE in libgit2 lfs filter for env variables and once they are set they are set for the whole process lifetime, so we need to spawn new process without them set to test default values +TEST(Libgt2InitGuardTest, LfsFilterCaptureDefaultResumeOptions) { + // Need new process beacase we use INIT_ONCE in libgit2 lfs filter for env variables and once they are set they are set for the whole process lifetime EXPECT_EXIT({ // Act: capture stdout during object construction testing::internal::CaptureStdout(); @@ -778,12 +777,12 @@ TEST(Libgt2InitGuardTest, LfsFilterCaptureDefaultResumeOptions) EXPECT_THAT(output, ::testing::HasSubstr("[INFO] LFS resume: attempts=5 interval=10 s")); exit(0); - }, ::testing::ExitedWithCode(0), ""); + }, + ::testing::ExitedWithCode(0), ""); } -TEST(Libgt2InitGuardTest, LfsFilterCaptureNonDefaultResumeOptions) -{ - // Need new process beacase we use INIT_ONCE in libgit2 lfs filter for env variables and once they are set they are set for the whole process lifetime, so we need to spawn new process without them set to test default values +TEST(Libgt2InitGuardTest, LfsFilterCaptureNonDefaultResumeOptions) { + // Need new process beacase we use INIT_ONCE in libgit2 lfs filter for env variables and once they are set they are set for the whole process lifetime EXPECT_EXIT({ EnvGuard guard; guard.set("GIT_LFS_RESUME_ATTEMPTS", "3"); @@ -803,7 +802,8 @@ TEST(Libgt2InitGuardTest, LfsFilterCaptureNonDefaultResumeOptions) EXPECT_THAT(output, ::testing::HasSubstr("[INFO] LFS resume: attempts=3 interval=20 s")); exit(0); - }, ::testing::ExitedWithCode(0), ""); + }, + ::testing::ExitedWithCode(0), ""); } TEST_F(HfDownloaderHfEnvTest, Methods) { From d64f5072878910915ff9d24c312ff0ca8fc28c03 Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 18 Mar 2026 15:30:24 +0100 Subject: [PATCH 48/49] Self review --- third_party/libgit2/lfs.patch | 177 ++++++++++++++++++++++------------ 1 file changed, 116 insertions(+), 61 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 669a26a4a3..684d239e41 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -444,10 +444,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..11616ed94 +index 000000000..eb7ec215f --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,1459 @@ +@@ -0,0 +1,1514 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -640,11 +640,17 @@ index 000000000..11616ed94 + * Initializes global resume configuration from environment: + * GIT_LFS_RESUME_ATTEMPTS + * GIT_LFS_RESUME_INTERVAL -+ * ++ * Initializes libcurl and set ups curl cleanup once per process ++ * + * Called exactly once via pthread_once / InitOnce. + */ +static void lfs_resume_env_init(void) +{ ++ /* initialize curl once per process */ ++ curl_global_init(CURL_GLOBAL_ALL); ++ /* register cleanup once */ ++ atexit(curl_global_cleanup); ++ + parse_env_nonneg_int( + "GIT_LFS_RESUME_ATTEMPTS", LFS_RESUME_ATTEMPTS_DEFAULT, 0, + LFS_RESUME_ATTEMPTS_MAX, &g_lfs_resume_attempts); @@ -706,17 +712,74 @@ index 000000000..11616ed94 +} +#endif + ++ +typedef struct lfs_attrs { -+ const char *path; -+ const char *full_path; -+ const char *workdir; -+ const char *lfs_oid; -+ const char *lfs_size; -+ const char *url; ++ char *path; ++ char *full_path; ++ char *workdir; ++ char *lfs_oid; ++ char *lfs_size; ++ char *url; + bool is_download; +} lfs_attrs; + +/* ++ * Allocate a clean, fully-owned structure. ++ */ ++lfs_attrs *lfs_attrs_new(void) ++{ ++ lfs_attrs *a = git__calloc(1, sizeof(lfs_attrs)); ++ return a; /* fields are NULL by calloc */ ++} ++ ++/* Internal helper: replaces a->field with strdup(value). */ ++static int lfs_attrs_replace(char **field, const char *value) ++{ ++ char *dup = NULL; ++ ++ if (value) { ++ dup = git__strdup(value); ++ if (!dup) ++ return -1; ++ } ++ ++ /* Free old field */ ++ git__free(*field); ++ *field = dup; ++ return 0; ++} ++ ++int lfs_attrs_set_path(lfs_attrs *a, const char *path) ++{ ++ return lfs_attrs_replace(&a->path, path); ++} ++ ++int lfs_attrs_set_full_path(lfs_attrs *a, const char *fp) ++{ ++ return lfs_attrs_replace(&a->full_path, fp); ++} ++ ++int lfs_attrs_set_workdir(lfs_attrs *a, const char *wd) ++{ ++ return lfs_attrs_replace(&a->workdir, wd); ++} ++ ++int lfs_attrs_set_oid(lfs_attrs *a, const char *oid) ++{ ++ return lfs_attrs_replace(&a->lfs_oid, oid); ++} ++ ++int lfs_attrs_set_size(lfs_attrs *a, const char *size) ++{ ++ return lfs_attrs_replace(&a->lfs_size, size); ++} ++ ++int lfs_attrs_set_url(lfs_attrs *a, const char *url) ++{ ++ return lfs_attrs_replace(&a->url, url); ++} ++ ++/* + * Frees heap-allocated strings referenced by the struct fields. + * Parameters: + * a - Pointer to lfs_attrs struct. Not freed itself. @@ -726,34 +789,24 @@ index 000000000..11616ed94 + * - If any field is borrowed (e.g., string literal or external buffer), set + * it to NULL before calling this function to avoid invalid free(). + */ -+static void lfs_attrs_free(lfs_attrs *a) ++void lfs_attrs_free(lfs_attrs *a) +{ + if (!a) + return; -+ if (a->path) { -+ free((void *)a->path); -+ a->path = NULL; -+ } -+ if (a->full_path) { -+ free((void *)a->full_path); -+ a->full_path = NULL; -+ } -+ if (a->workdir) { -+ free((void *)a->workdir); -+ a->workdir = NULL; -+ } -+ if (a->lfs_oid) { -+ free((void *)a->lfs_oid); -+ a->lfs_oid = NULL; -+ } -+ if (a->lfs_size) { -+ free((void *)a->lfs_size); -+ a->lfs_size = NULL; -+ } -+ if (a->url) { -+ free((void *)a->url); -+ a->url = NULL; -+ } ++ ++ git__free(a->path); ++ git__free(a->full_path); ++ git__free(a->workdir); ++ git__free(a->lfs_oid); ++ git__free(a->lfs_size); ++ git__free(a->url); ++ ++ a->path = NULL; ++ a->full_path = NULL; ++ a->workdir = NULL; ++ a->lfs_oid = NULL; ++ a->lfs_size = NULL; ++ a->url = NULL; + + a->is_download = false; +} @@ -766,13 +819,12 @@ index 000000000..11616ed94 + * Parameters: + * a - Heap-allocated lfs_attrs struct. + */ -+static void lfs_attrs_delete(lfs_attrs *a) ++void lfs_attrs_delete(lfs_attrs *a) +{ + if (!a) + return; + lfs_attrs_free(a); -+ free(a); -+ a = NULL; ++ git__free(a); +} + +/* @@ -787,10 +839,10 @@ index 000000000..11616ed94 + * The parsed integer on success, + * 0 on failure. + */ -+static size_t get_digit(const char *buffer) ++static unsigned long long get_digit(const char *buffer) +{ + char *endptr; -+ size_t number; ++ unsigned long long number; + errno = 0; + if (buffer == NULL) { + fprintf(stderr, "\n[ERROR] get_digit on NULL\n"); @@ -1007,8 +1059,12 @@ index 000000000..11616ed94 + git_oid lfs_oid; + /* Init the lfs attrs to indicate git lfs clean, currently only diff + * support no upload of lfs file supported */ -+ struct lfs_attrs la = { NULL, NULL, NULL, NULL, NULL, NULL, false }; -+ *payload = git__malloc(sizeof(la)); ++ lfs_attrs *la = lfs_attrs_new(); ++ if (!la) ++ return -1; ++ la->is_download = false; ++ ++ *payload = la; + GIT_ERROR_CHECK_ALLOC(*payload); + memcpy(*payload, &la, sizeof(la)); + @@ -1089,13 +1145,15 @@ index 000000000..11616ed94 + git_repository *repo = git_filter_source_repo(src); + const char *path = git_filter_source_path(src); + const char *workdir = git_repository_workdir(repo); -+ /* Setup memory for payload struct ownership */ -+ char *full_path_dup; -+ char *path_dup = git__strdup(path); -+ char *workdir_dup = git__strdup(workdir); -+ char *url_dup = git__strdup(repo->url); -+ struct lfs_attrs la = { path_dup, NULL, workdir_dup, NULL, -+ NULL, url_dup, true }; ++ /* Setup memory for payload struct ownership */ ++ lfs_attrs *la = lfs_attrs_new(); ++ if (!la) ++ return -1; ++ ++ lfs_attrs_set_path(la, path); ++ lfs_attrs_set_workdir(la, workdir); ++ lfs_attrs_set_url(la, repo->url); ++ la->is_download = true; + + lfs_oid.size = from->size; + lfs_oid.asize = from->asize; @@ -1110,31 +1168,28 @@ index 000000000..11616ed94 + lfs_oid.ptr); + return -1; + } -+ la.lfs_oid = lfs_oid.ptr; -+ ++ lfs_attrs_set_oid(la, lfs_oid.ptr); ++ git_str_dispose(&lfs_oid); + if (get_lfs_info_match(&lfs_size, size_regexp) < 0) { + fprintf(stderr, + "\n[ERROR] failure, cannot find lfs size in: %s\n", + lfs_size.ptr); + return -1; + } -+ la.lfs_size = lfs_size.ptr; -+ ++ lfs_attrs_set_size(la, lfs_size.ptr); ++ git_str_dispose(&lfs_size); + if (git_repository_workdir_path(&full_path, repo, path) < 0) { + fprintf(stderr, + "\n[ERROR] failure, cannot get repository path: %s\n", + path); + return -1; + } -+ full_path_dup = git__strdup(full_path.ptr); ++ lfs_attrs_set_full_path(la, full_path.ptr); + git_str_dispose(&full_path); -+ la.full_path = full_path_dup; + -+ *payload = git__malloc(sizeof(la)); -+ GIT_ERROR_CHECK_ALLOC(*payload); -+ memcpy(*payload, &la, sizeof(la)); ++ *payload = la; + -+ /*Just write the oryginal lfs file contents */ ++ /*Just write the original lfs file contents */ + return git_str_set(to, from->ptr, from->size); +} + @@ -1269,7 +1324,7 @@ index 000000000..11616ed94 +static void +print_progress(size_t count, size_t max, bool first_run, size_t elapsed_time) +{ -+ float progress; ++ double progress; + int i, bar_length, bar_width; + size_t totalSizeUnitId; + double totalSize; @@ -1281,7 +1336,7 @@ index 000000000..11616ed94 + return; + } + -+ progress = (float)count / max; ++ progress = (double)count / max; + if (!first_run && progress < 0.01 && count > 0) + return; + @@ -1819,7 +1874,7 @@ index 000000000..11616ed94 + curl_easy_cleanup(dl_curl); + } + -+ /* Remove lfs file and rename downloaded file to oryginal lfs filename ++ /* Remove lfs file and rename downloaded file to original lfs filename + */ + if (!resumingFileByBlobFilter) { + /* File does not exist when using blob filters */ From dc8994f55b1014b5687497fbc17f19d332194e60 Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 18 Mar 2026 18:18:53 +0100 Subject: [PATCH 49/49] Big memory refactor --- third_party/libgit2/lfs.patch | 487 +++++++++++++++++----------------- 1 file changed, 243 insertions(+), 244 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 684d239e41..ffa4cb7f43 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -444,10 +444,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..eb7ec215f +index 000000000..3d77e9493 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,1514 @@ +@@ -0,0 +1,1513 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -1065,9 +1065,6 @@ index 000000000..eb7ec215f + la->is_download = false; + + *payload = la; -+ GIT_ERROR_CHECK_ALLOC(*payload); -+ memcpy(*payload, &la, sizeof(la)); -+ + if (!from) + return -1; + @@ -1616,297 +1613,299 @@ index 000000000..eb7ec215f + */ +static void lfs_download(git_filter *self, void *payload) +{ -+ struct lfs_attrs *la; -+ char *tmp_out_file; -+ CURL *info_curl, *dl_curl; ++ struct lfs_attrs *la = (struct lfs_attrs *)payload; ++ char *tmp_out_file = NULL; ++ CURL *info_curl = NULL; ++ CURL *dl_curl = NULL; + CURLcode res = CURLE_OK; + CURLcode status = CURLE_OK; + git_str res_str = GIT_STR_INIT; ++ git_str lfs_info_url = GIT_STR_INIT; ++ git_str lfs_info_data = GIT_STR_INIT; + bool resumingFileByBlobFilter = false; -+ struct progress_data progress_d = { time(NULL), time(NULL), false }; ++ struct progress_data progress_d = { 0 }; + struct memory response = { 0 }; ++ struct curl_slist *chunk = NULL; ++ struct FtpFile ftpfile = { 0 }; ++ const char *href_regexp = ++ "\"download\"\\s*:\\s*\\{\\s*\"href\":\"([^\"]+)\""; + GIT_UNUSED(self); -+ if (!payload) { -+ fprintf(stderr, "\n[ERROR] lfs payload not initialized\n"); -+ return; ++ if (!la) { ++ goto cleanup; + } -+ la = (struct lfs_attrs *)payload; + + /* Currently only download is supoprted, no lfs file upload */ + if (!la->is_download) { -+ git__free(payload); -+ return; ++ goto done; + } + + tmp_out_file = append_cstr_to_buffer(la->full_path, "lfs_part"); + if (tmp_out_file == NULL) { + fprintf(stderr, "\n[ERROR] lfs create temp filename failed\n"); -+ goto on_error2; ++ goto cleanup; + } ++ ftpfile.filename = tmp_out_file; + + /* get a curl handle */ + info_curl = curl_easy_init(); -+ if (info_curl) { -+ struct curl_slist *chunk = NULL; -+ git_str lfs_info_url = GIT_STR_INIT; -+ git_str lfs_info_data = GIT_STR_INIT; -+ if (git_str_join( -+ &lfs_info_url, '.', la->url, -+ "git/info/lfs/objects/batch") < 0) { -+ fprintf(stderr, "\n[ERROR] failed to create url '%s'\n", -+ la->full_path); -+ git_str_dispose(&lfs_info_url); -+ git_str_dispose(&lfs_info_data); -+ goto on_error; -+ } -+ -+ /* Remove a header curl would otherwise add by itself */ -+ chunk = curl_slist_append( -+ chunk, "Accept: application/vnd.git-lfs+json"); -+ /* Add a custom header */ -+ chunk = curl_slist_append( -+ chunk, "Content-Type: application/vnd.git-lfs+json"); -+ /* set our custom set of headers */ -+ CURL_SETOPT( -+ curl_easy_setopt(info_curl, CURLOPT_HTTPHEADER, chunk)); -+ /* First set the URL that is about to receive our POST. This URL -+ can just as well be an https:// URL if that is what should -+ receive the data. */ -+ CURL_SETOPT(curl_easy_setopt( -+ info_curl, CURLOPT_URL, lfs_info_url.ptr)); -+ /* Add cURL resiliency */ -+ /* unlimited data */ -+ CURL_SETOPT(curl_easy_setopt( -+ info_curl, CURLOPT_CONNECTTIMEOUT, 30L)); -+ /* timeout */ -+ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_TIMEOUT, 0L)); -+ /* low speed 1KB/s */ -+ CURL_SETOPT(curl_easy_setopt( -+ info_curl, CURLOPT_LOW_SPEED_LIMIT, 1024L)); -+ /* for 30s */ -+ CURL_SETOPT(curl_easy_setopt( -+ info_curl, CURLOPT_LOW_SPEED_TIME, 30L)); -+ -+ if (status != CURLE_OK) { -+ fprintf(stderr, -+ "\n[ERROR] curl_easy_setopt() failed: %s\n", -+ curl_easy_strerror(status)); -+ goto info_cleaup; -+ } ++ if (!info_curl) { ++ fprintf(stderr, "[ERROR] curl_easy_init(info_curl) failed\n"); ++ goto cleanup; ++ } + -+ /* "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":\"9556d0a12310629e217450ac4198c49f5457f1a69e22ce7c9f8e81fab4d530a7\",\"size\":499723}]}" -+ */ -+ if (git_str_join_n( -+ &lfs_info_data, '"', 5, -+ "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":", -+ la->lfs_oid, ",\"size\":", la->lfs_size, -+ "}]}") < 0) { -+ fprintf(stderr, "\n[ERROR] failed to create url '%s'\n", -+ la->full_path); -+ /* always cleanup */ -+ curl_easy_cleanup(info_curl); -+ /* free the custom headers */ -+ curl_slist_free_all(chunk); -+ git_str_dispose(&lfs_info_data); -+ git_str_dispose(&lfs_info_url); -+ goto on_error; -+ } ++ if (git_str_join( ++ &lfs_info_url, '.', la->url, ++ "git/info/lfs/objects/batch") < 0) { ++ fprintf(stderr, "\n[ERROR] failed to create url '%s'\n", ++ la->full_path); ++ goto cleanup; ++ } + -+ /* Now specify the POST data */ -+ CURL_SETOPT(curl_easy_setopt( -+ info_curl, CURLOPT_POSTFIELDS, lfs_info_data.ptr)); -+ CURL_SETOPT(curl_easy_setopt( -+ info_curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA)); -+ CURL_SETOPT(curl_easy_setopt( -+ info_curl, CURLOPT_FOLLOWLOCATION, 1L)); -+ -+ CURL_SETOPT(curl_easy_setopt( -+ info_curl, CURLOPT_WRITEFUNCTION, write_callback)); -+ CURL_SETOPT(curl_easy_setopt( -+ info_curl, CURLOPT_USERAGENT, "git-lfs/3.5.0")); -+ CURL_SETOPT(curl_easy_setopt( -+ info_curl, CURLOPT_WRITEDATA, (void *)&response)); -+ -+ if (status != CURLE_OK) { -+ fprintf(stderr, -+ "\n[ERROR] curl_easy_setopt() failed: %s\n", -+ curl_easy_strerror(status)); -+ goto info_cleaup; -+ } -+ /* Perform the request, res gets the return code */ -+ res = curl_easy_perform(info_curl); -+ /* Check for errors */ -+ if (res != CURLE_OK) { -+ fprintf(stderr, -+ "\n[ERROR] curl_easy_perform() failed: %s\n", -+ curl_easy_strerror(res)); -+ /* always cleanup */ -+ curl_easy_cleanup(info_curl); -+ /* free the custom headers */ -+ curl_slist_free_all(chunk); -+ git_str_dispose(&lfs_info_url); -+ git_str_dispose(&lfs_info_data); -+ goto on_error; -+ } ++ /* Remove a header curl would otherwise add by itself */ ++ chunk = curl_slist_append( ++ chunk, "Accept: application/vnd.git-lfs+json"); ++ /* Add a custom header */ ++ chunk = curl_slist_append( ++ chunk, "Content-Type: application/vnd.git-lfs+json"); ++ /* set our custom set of headers */ ++ CURL_SETOPT( ++ curl_easy_setopt(info_curl, CURLOPT_HTTPHEADER, chunk)); ++ /* First set the URL that is about to receive our POST. This URL ++ can just as well be an https:// URL if that is what should ++ receive the data. */ ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_URL, lfs_info_url.ptr)); ++ /* Add cURL resiliency */ ++ /* unlimited data */ ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_CONNECTTIMEOUT, 30L)); ++ /* timeout */ ++ CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_TIMEOUT, 0L)); ++ /* low speed 1KB/s */ ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_LOW_SPEED_LIMIT, 1024L)); ++ /* for 30s */ ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_LOW_SPEED_TIME, 30L)); ++ ++ if (status != CURLE_OK) { ++ fprintf(stderr, ++ "\n[ERROR] curl_easy_setopt() failed: %s\n", ++ curl_easy_strerror(status)); ++ goto cleanup; ++ } + -+ /* Get response data */ -+ res_str.asize = response.size; -+ res_str.size = response.size; -+ res_str.ptr = git__strdup(response.response); -+ free(response.response); -+ info_cleaup: -+ /* always cleanup */ -+ curl_easy_cleanup(info_curl); -+ /* free the custom headers */ -+ curl_slist_free_all(chunk); -+ git_str_dispose(&lfs_info_url); -+ git_str_dispose(&lfs_info_data); -+ if (status != CURLE_OK) -+ goto on_error; ++ /* "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":\"9556d0a12310629e217450ac4198c49f5457f1a69e22ce7c9f8e81fab4d530a7\",\"size\":499723}]}" ++ */ ++ if (git_str_join_n( ++ &lfs_info_data, '"', 5, ++ "{\"operation\":\"download\",\"transfer\":[\"basic\"],\"objects\":[{\"oid\":", ++ la->lfs_oid, ",\"size\":", la->lfs_size, ++ "}]}") < 0) { ++ fprintf(stderr, "\n[ERROR] failed to create url '%s'\n", ++ la->full_path); ++ goto cleanup; + } + -+ /* get a curl handle */ -+ dl_curl = curl_easy_init(); -+ if (dl_curl) { -+ struct FtpFile ftpfile = { tmp_out_file, NULL }; -+ -+ const char *href_regexp = -+ "\"download\"\\s*:\\s*\\{\\s*\"href\":\"([^\"]+)\""; -+ if (get_lfs_info_match(&res_str, href_regexp) < 0) { -+ /* always cleanup */ -+ curl_easy_cleanup(dl_curl); -+ git_str_dispose(&res_str); -+ goto on_error; -+ } -+ /* First set the URL that is about to receive our POST. This URL -+ can just as well be an https:// URL if that is what should -+ receive the data. */ -+ CURL_SETOPT( -+ curl_easy_setopt(dl_curl, CURLOPT_URL, res_str.ptr)); -+ CURL_SETOPT(curl_easy_setopt( -+ dl_curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA)); -+ CURL_SETOPT( -+ curl_easy_setopt(dl_curl, CURLOPT_FOLLOWLOCATION, 1L)); -+ CURL_SETOPT(curl_easy_setopt( -+ dl_curl, CURLOPT_USE_SSL, CURLUSESSL_ALL)); -+ CURL_SETOPT(curl_easy_setopt( -+ dl_curl, CURLOPT_USERAGENT, "git-lfs/3.5.0")); -+ CURL_SETOPT(curl_easy_setopt( -+ dl_curl, CURLOPT_WRITEFUNCTION, file_write_callback)); -+ CURL_SETOPT(curl_easy_setopt( -+ dl_curl, CURLOPT_WRITEDATA, (void *)&ftpfile)); -+ -+ /* progress bar options */ -+ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_NOPROGRESS, 0L)); -+ CURL_SETOPT(curl_easy_setopt( -+ dl_curl, CURLOPT_XFERINFOFUNCTION, progress_callback)); -+ CURL_SETOPT(curl_easy_setopt( -+ dl_curl, CURLOPT_XFERINFODATA, &progress_d)); -+ -+ /* Add cURL resiliency */ -+ /* unlimited data */ -+ CURL_SETOPT( -+ curl_easy_setopt(dl_curl, CURLOPT_CONNECTTIMEOUT, 30L)); -+ /* timeout */ -+ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_TIMEOUT, 0L)); -+ /* low speed 1KB/s */ -+ CURL_SETOPT(curl_easy_setopt( -+ dl_curl, CURLOPT_LOW_SPEED_LIMIT, 1024L)); -+ /* for 30s */ -+ CURL_SETOPT( -+ curl_easy_setopt(dl_curl, CURLOPT_LOW_SPEED_TIME, 30L)); -+ if (status != CURLE_OK) { -+ fprintf(stderr, -+ "\n[ERROR] curl_easy_setopt() failed: %s\n", -+ curl_easy_strerror(status)); -+ curl_easy_cleanup(dl_curl); -+ git_str_dispose(&res_str); -+ goto on_error; -+ } ++ /* Now specify the POST data */ ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_POSTFIELDS, lfs_info_data.ptr)); ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA)); ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_FOLLOWLOCATION, 1L)); ++ ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_WRITEFUNCTION, write_callback)); ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_USERAGENT, "git-lfs/3.5.0")); ++ CURL_SETOPT(curl_easy_setopt( ++ info_curl, CURLOPT_WRITEDATA, (void *)&response)); ++ ++ if (status != CURLE_OK) { ++ fprintf(stderr, ++ "\n[ERROR] curl_easy_setopt() failed: %s\n", ++ curl_easy_strerror(status)); ++ goto cleanup; ++ } ++ /* Perform the request, res gets the return code */ ++ res = curl_easy_perform(info_curl); ++ /* Check for errors */ ++ if (res != CURLE_OK) { ++ fprintf(stderr, ++ "\n[ERROR] curl_easy_perform() failed: %s\n", ++ curl_easy_strerror(res)); ++ goto cleanup; ++ } + -+ /* Check for resume if previous download failed and we have the -+ * partial file on disk */ -+ ftpfile.stream = fopen(ftpfile.filename, "r"); -+ if (ftpfile.stream != NULL) { -+ resumingFileByBlobFilter = true; -+ fclose(ftpfile.stream); -+ ftpfile.stream = NULL; ++ /* Copy response JSON */ ++ if (response.response) { ++ git_str_set(&res_str, response.response, response.size); ++ } + -+ /* First try a resume sequence */ -+ res = download_with_resume( -+ dl_curl, &ftpfile, g_lfs_resume_attempts, -+ g_lfs_resume_interval_secs); -+ } else { -+ print_download_info( -+ la->full_path, get_digit(la->lfs_size)); -+ /* Perform the request, res gets the return code */ -+ res = curl_easy_perform(dl_curl); -+ } ++ /* get a curl handle */ ++ dl_curl = curl_easy_init(); ++ if (!dl_curl) { ++ fprintf(stderr, "[ERROR] curl_easy_init(dl_curl) failed\n"); ++ goto cleanup; ++ } ++ ++ if (get_lfs_info_match(&res_str, href_regexp) < 0) { ++ fprintf(stderr, "[ERROR] Cannot extract LFS download URL\n"); ++ goto cleanup; ++ } ++ /* Progress info */ ++ progress_d.started_download = time(NULL); ++ progress_d.last_print_time = time(NULL); ++ /* First set the URL that is about to receive our POST. This URL ++ can just as well be an https:// URL if that is what should ++ receive the data. */ ++ CURL_SETOPT( ++ curl_easy_setopt(dl_curl, CURLOPT_URL, res_str.ptr)); ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA)); ++ CURL_SETOPT( ++ curl_easy_setopt(dl_curl, CURLOPT_FOLLOWLOCATION, 1L)); ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_USE_SSL, CURLUSESSL_ALL)); ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_USERAGENT, "git-lfs/3.5.0")); ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_WRITEFUNCTION, file_write_callback)); ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_WRITEDATA, (void *)&ftpfile)); ++ ++ /* progress bar options */ ++ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_NOPROGRESS, 0L)); ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_XFERINFOFUNCTION, progress_callback)); ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_XFERINFODATA, &progress_d)); ++ ++ /* Add cURL resiliency */ ++ /* unlimited data */ ++ CURL_SETOPT( ++ curl_easy_setopt(dl_curl, CURLOPT_CONNECTTIMEOUT, 30L)); ++ /* timeout */ ++ CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_TIMEOUT, 0L)); ++ /* low speed 1KB/s */ ++ CURL_SETOPT(curl_easy_setopt( ++ dl_curl, CURLOPT_LOW_SPEED_LIMIT, 1024L)); ++ /* for 30s */ ++ CURL_SETOPT( ++ curl_easy_setopt(dl_curl, CURLOPT_LOW_SPEED_TIME, 30L)); ++ if (status != CURLE_OK) { ++ fprintf(stderr, ++ "\n[ERROR] curl_easy_setopt() failed: %s\n", ++ curl_easy_strerror(status)); ++ goto cleanup; ++ } + -+ /* Check for resume of partial download error */ -+ if (res == CURLE_PARTIAL_FILE) { -+ fprintf(stderr, -+ "[WARN] Got CURLE_PARTIAL_FILE, attempting resume sequence\n"); -+ res = download_with_resume( -+ dl_curl, &ftpfile, g_lfs_resume_attempts, -+ g_lfs_resume_interval_secs); -+ } ++ /* Check for resume if previous download failed and we have the ++ * partial file on disk */ ++ ftpfile.stream = fopen(ftpfile.filename, "r"); ++ if (ftpfile.stream != NULL) { ++ resumingFileByBlobFilter = true; ++ fclose(ftpfile.stream); ++ ftpfile.stream = NULL; ++ ++ /* First try a resume sequence */ ++ res = download_with_resume( ++ dl_curl, &ftpfile, g_lfs_resume_attempts, ++ g_lfs_resume_interval_secs); ++ } else { ++ print_download_info( ++ la->full_path, get_digit(la->lfs_size)); ++ /* Perform the request, res gets the return code */ ++ res = curl_easy_perform(dl_curl); ++ } + -+ /* Check for errors */ -+ if (res != CURLE_OK) { -+ fprintf(stderr, -+ "\n[ERROR] curl_easy_perform() failed: %s\n", -+ curl_easy_strerror(res)); -+ if (ftpfile.stream) { -+ fclose(ftpfile.stream); -+ ftpfile.stream = NULL; -+ } -+ /* always cleanup */ -+ curl_easy_cleanup(dl_curl); -+ goto on_error; -+ } ++ /* Check for resume of partial download error */ ++ if (res == CURLE_PARTIAL_FILE) { ++ fprintf(stderr, ++ "[WARN] Got CURLE_PARTIAL_FILE, attempting resume sequence\n"); ++ res = download_with_resume( ++ dl_curl, &ftpfile, g_lfs_resume_attempts, ++ g_lfs_resume_interval_secs); ++ } + ++ /* Check for errors */ ++ if (res != CURLE_OK) { ++ fprintf(stderr, ++ "\n[ERROR] curl_easy_perform() failed: %s\n", ++ curl_easy_strerror(res)); ++ /* Very important to close the file to write any bytes downloaded */ + if (ftpfile.stream) { + fclose(ftpfile.stream); + ftpfile.stream = NULL; + } -+ /* always cleanup */ -+ curl_easy_cleanup(dl_curl); ++ goto cleanup; + } + -+ /* Remove lfs file and rename downloaded file to original lfs filename -+ */ ++ /* Very important to close the file to write any bytes downloaded */ ++ if (ftpfile.stream) { ++ fclose(ftpfile.stream); ++ ftpfile.stream = NULL; ++ } ++ ++ /* Remove lfs file and rename downloaded file to original lfs filename */ + if (!resumingFileByBlobFilter) { + /* File does not exist when using blob filters */ + if (p_unlink(la->full_path) < 0) { + fprintf(stderr, + "\n[ERROR] failed to delete file '%s'\n", + la->full_path); -+ goto on_error; ++ /* Ignore error here, react on next error */ + } + } + + if (p_rename(tmp_out_file, la->full_path) < 0) { + fprintf(stderr, "\n[ERROR] failed to rename file to '%s'\n", + la->full_path); -+ goto on_error; ++ goto cleanup; ++ } ++ ++ /* ++ * SUCCESS ++ */ ++ goto done; ++ /* ++ * ---------------------------------------------------- ++ * Cleanup block — ALWAYS EXECUTED ++ * ---------------------------------------------------- ++ */ ++cleanup: ++ fprintf(stderr, "[ERROR] LFS download failed for %s\n", ++ la ? la->full_path : "(null)"); ++done: ++ /* Close stream if open */ ++ if (ftpfile.stream) { ++ fclose(ftpfile.stream); ++ ftpfile.stream = NULL; + } ++ ++ /* Free temporary file name */ + free(tmp_out_file); ++ /* Libgit2 strings */ ++ git_str_dispose(&lfs_info_url); ++ git_str_dispose(&lfs_info_data); + git_str_dispose(&res_str); -+ lfs_attrs_delete(la); -+ fflush(stdout); -+ return; ++ /* Free memory buffer for batch response */ ++ free(response.response); ++ /* cURL cleanup */ ++ if (info_curl) ++ curl_easy_cleanup(info_curl); ++ if (dl_curl) ++ curl_easy_cleanup(dl_curl); ++ if (chunk) ++ curl_slist_free_all(chunk); ++ ++ /* Free payload */ ++ if (la) ++ lfs_attrs_delete(la); + -+on_error: -+ free(tmp_out_file); -+on_error2: -+ fprintf(stderr, "\n[ERROR] LFS download failed for file %s\n", -+ la->full_path); -+ fflush(stderr); + fflush(stdout); -+ git_str_dispose(&res_str); -+ lfs_attrs_delete(la); -+ return; ++ fflush(stderr); +} + +/*