Commit 6c7a87f7e for llama.cpp
commit 6c7a87f7e5e5cd75b8a641c3471f2dee84a6ed17
Author: Adrien Gallouët <angt@huggingface.co>
Date: Mon Sep 28 16:25:24 2026 +0200
common : fix HF cache paths on Windows (#29475)
Supersedes #29158
Signed-off-by: Adrien Gallouët <angt@huggingface.co>
diff --git a/common/common.cpp b/common/common.cpp
index ae44f054a..6099f2ecc 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -969,6 +969,17 @@ void common_set_env(const std::string & name, const std::string & value) {
#endif
}
+std::filesystem::path common_get_path_from_env(const std::string & name) {
+#if defined(_WIN32)
+ const std::wstring wname = utf8_to_wstring(name);
+ const wchar_t * wvalue = _wgetenv(wname.c_str());
+ return wvalue ? std::filesystem::path(wvalue) : std::filesystem::path();
+#else
+ const char * value = std::getenv(name.c_str());
+ return value ? std::filesystem::path(value) : std::filesystem::path();
+#endif
+}
+
std::string fs_get_cache_directory() {
std::string cache_directory = "";
auto ensure_trailing_slash = [](std::string p) {
diff --git a/common/common.h b/common/common.h
index 527f4be7d..329469410 100644
--- a/common/common.h
+++ b/common/common.h
@@ -904,6 +904,9 @@ std::string fs_path_to_utf8(const std::filesystem::path & path);
std::string common_get_env(const std::string & name);
void common_set_env(const std::string & name, const std::string & value);
+// reads a path from the environment, an unset variable gives an empty path
+std::filesystem::path common_get_path_from_env(const std::string & name);
+
//
// Filesystem utils
//
diff --git a/common/download.cpp b/common/download.cpp
index 4b28a708c..2edd2a11c 100644
--- a/common/download.cpp
+++ b/common/download.cpp
@@ -286,7 +286,7 @@ static int common_download_file_single_online(const std::string & url,
static const int max_attempts = 3;
static const int retry_delay_seconds = 2;
- const bool file_exists = std::filesystem::exists(path);
+ const bool file_exists = std::filesystem::exists(std::filesystem::u8path(path));
if (file_exists && skip_etag) {
LOG_DBG("%s: using cached file: %s\n", __func__, path.c_str());
@@ -477,7 +477,7 @@ int common_download_file_single(const std::string & url,
return common_download_file_single_online(url, path, online_opts, skip_etag);
}
- if (!std::filesystem::exists(path)) {
+ if (!std::filesystem::exists(std::filesystem::u8path(path))) {
LOG_ERR("%s: required file is not available in cache (offline mode): %s\n", __func__, path.c_str());
return -1;
}
diff --git a/common/hf-cache.cpp b/common/hf-cache.cpp
index 65c9722f3..5f37084d7 100644
--- a/common/hf-cache.cpp
+++ b/common/hf-cache.cpp
@@ -44,8 +44,7 @@ static fs::path get_cache_directory() {
{HOME_DIR, fs::path(".cache") / "huggingface" / "hub"}
};
for (const auto & entry : entries) {
- if (auto * p = std::getenv(entry.var); p && *p) {
- fs::path base(p);
+ if (fs::path base = common_get_path_from_env(entry.var); !base.empty()) {
return entry.path.empty() ? base : base / entry.path;
}
}
@@ -174,7 +173,8 @@ static bool is_valid_subpath(const fs::path & path, const fs::path & subpath) {
}
static void safe_write_file(const fs::path & path, const std::string & data) {
- fs::path path_tmp = path.string() + ".tmp";
+ fs::path path_tmp = path;
+ path_tmp += ".tmp";
if (path.has_parent_path()) {
fs::create_directories(path.parent_path());
@@ -191,7 +191,7 @@ static void safe_write_file(const fs::path & path, const std::string & data) {
}
if (file.fail() || ec) {
fs::remove(path_tmp, ec);
- throw std::runtime_error("failed to write file: " + path.string());
+ throw std::runtime_error("failed to write file: " + fs_path_to_utf8(path));
}
}
@@ -241,6 +241,7 @@ static std::string get_repo_commit(const std::string & repo_id,
fs::path refs_path = get_repo_path(repo_id) / "refs";
std::string name;
std::string commit;
+ fs::path name_path;
for (const auto & branch : json["branches"]) {
if (!branch.is_object() ||
@@ -251,24 +252,28 @@ static std::string get_repo_commit(const std::string & repo_id,
std::string _name = branch["name"].get<std::string>();
std::string _commit = branch["targetCommit"].get<std::string>();
- if (!is_valid_subpath(refs_path, _name)) {
- LOG_WRN("%s: skip invalid branch: %s\n", __func__, _name.c_str());
- continue;
- }
if (!is_valid_commit(_commit)) {
LOG_WRN("%s: skip invalid commit: %s\n", __func__, _commit.c_str());
continue;
}
+ const fs::path candidate = fs::u8path(_name);
+
+ if (!is_valid_subpath(refs_path, candidate)) {
+ LOG_WRN("%s: skip invalid branch: %s\n", __func__, _name.c_str());
+ continue;
+ }
if (_name == "main") {
name = _name;
commit = _commit;
+ name_path = candidate;
break;
}
if (name.empty() || commit.empty()) {
name = _name;
commit = _commit;
+ name_path = candidate;
}
}
@@ -277,7 +282,7 @@ static std::string get_repo_commit(const std::string & repo_id,
return {};
}
- safe_write_file(refs_path / name, commit);
+ safe_write_file(refs_path / name_path, commit);
return commit;
} catch (const common_json_error & e) {
@@ -326,7 +331,9 @@ hf_files get_repo_files(const std::string & repo_id,
file.repo_id = repo_id;
file.path = item["path"].get<std::string>();
- if (!is_valid_subpath(commit_path, file.path)) {
+ const fs::path subpath = fs::u8path(file.path);
+
+ if (!is_valid_subpath(commit_path, subpath)) {
LOG_WRN("%s: skip invalid path: %s\n", __func__, file.path.c_str());
continue;
}
@@ -346,12 +353,12 @@ hf_files get_repo_files(const std::string & repo_id,
file.url = endpoint + repo_id + "/resolve/" + commit + "/" + file.path;
- fs::path final_path = commit_path / file.path;
- file.final_path = final_path.string();
+ fs::path final_path = commit_path / subpath;
+ file.final_path = fs_path_to_utf8(final_path);
if (!file.oid.empty() && !fs::exists(final_path)) {
fs::path local_path = blobs_path / file.oid;
- file.local_path = local_path.string();
+ file.local_path = fs_path_to_utf8(local_path);
} else {
file.local_path = file.final_path;
}
@@ -418,7 +425,7 @@ hf_files get_cached_files(const std::string & repo_id) {
if (!fs::exists(snapshots_path)) {
continue;
}
- std::string _repo_id = folder_name_to_repo(repo.path().filename().string());
+ std::string _repo_id = folder_name_to_repo(fs_path_to_utf8(repo.path().filename()));
if (!is_valid_repo_id(_repo_id)) {
continue;
@@ -441,8 +448,9 @@ hf_files get_cached_files(const std::string & repo_id) {
if (!path.empty()) {
hf_file file;
file.repo_id = _repo_id;
- file.path = path.generic_string();
- file.local_path = entry.path().string();
+ const auto generic_path = path.generic_u8string();
+ file.path = std::string(generic_path.begin(), generic_path.end());
+ file.local_path = fs_path_to_utf8(entry.path());
file.final_path = file.local_path;
files.push_back(std::move(file));
}
@@ -456,8 +464,8 @@ std::string finalize_file(const hf_file & file) {
static std::atomic<bool> symlinks_disabled{false};
std::error_code ec;
- fs::path local_path(file.local_path);
- fs::path final_path(file.final_path);
+ fs::path local_path = fs::u8path(file.local_path);
+ fs::path final_path = fs::u8path(file.final_path);
if (local_path == final_path || fs::exists(final_path, ec)) {
return file.final_path;
@@ -504,7 +512,7 @@ bool remove_cached_repo(const std::string & repo_id) {
std::error_code ec;
auto removed = fs::remove_all(repo_path, ec);
if (ec) {
- LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str());
+ LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, fs_path_to_utf8(repo_path).c_str(), ec.message().c_str());
return false;
}
return removed > 0;