Commit 8c442fb for stable-diffusion.cpp
commit 8c442fb0ad43cf901663d9fb6783e2cd4929a278
Author: therealshulgin <moonpapers@protonmail.com>
Date: Fri Oct 16 18:50:09 2026 +0200
fix: open model files with UTF-8 paths on Windows (#2144)
diff --git a/src/convert.cpp b/src/convert.cpp
index 7fb0825..b8a6212 100644
--- a/src/convert.cpp
+++ b/src/convert.cpp
@@ -3,6 +3,7 @@
#include <cstdint>
#include <cstring>
#include <exception>
+#include <filesystem>
#include <fstream>
#include <map>
#include <memory>
@@ -108,7 +109,7 @@ static bool preallocate_output_file(const std::string& output_path, uint64_t fil
return true;
}
- std::fstream file(output_path, std::ios::binary | std::ios::in | std::ios::out);
+ std::fstream file(std::filesystem::u8path(output_path), std::ios::binary | std::ios::in | std::ios::out);
if (!file.is_open()) {
if (error != nullptr) {
*error = "failed to open output file '" + output_path + "' for preallocation";
@@ -251,7 +252,7 @@ static bool stream_tensor_data(ModelLoader& model_loader,
for (int worker = 0; worker < n_threads; worker++) {
workers.emplace_back([&]() {
- std::fstream output_file(output_path, std::ios::binary | std::ios::in | std::ios::out);
+ std::fstream output_file(std::filesystem::u8path(output_path), std::ios::binary | std::ios::in | std::ios::out);
if (!output_file.is_open()) {
fail("failed to open output file '" + output_path + "' for tensor writing");
return;
diff --git a/src/core/ggml_tensor_utils.cpp b/src/core/ggml_tensor_utils.cpp
index 1747678..5639c7e 100644
--- a/src/core/ggml_tensor_utils.cpp
+++ b/src/core/ggml_tensor_utils.cpp
@@ -3,6 +3,7 @@
#include <algorithm>
#include <cstdlib>
#include <cstring>
+#include <filesystem>
#include <fstream>
#include "core/ggml_extend_backend.h"
@@ -100,7 +101,7 @@ void ggml_ext_tensor_diff(
}
ggml_tensor* load_tensor_from_file(ggml_context* ctx, const std::string& file_path) {
- std::ifstream file(file_path, std::ios::binary);
+ std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary);
if (!file.is_open()) {
LOG_ERROR("failed to open '%s'", file_path.c_str());
return nullptr;
diff --git a/src/core/tensor_ggml.hpp b/src/core/tensor_ggml.hpp
index 66ea222..5082ce2 100644
--- a/src/core/tensor_ggml.hpp
+++ b/src/core/tensor_ggml.hpp
@@ -3,6 +3,7 @@
#include <array>
#include <cstring>
+#include <filesystem>
#include <fstream>
#include <stdexcept>
#include <string>
@@ -105,7 +106,7 @@ namespace sd {
template <typename T>
inline Tensor<T> load_tensor_from_file_as_tensor(const std::string& file_path) {
- std::ifstream file(file_path, std::ios::binary);
+ std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary);
if (!file.is_open()) {
throw std::runtime_error("failed to open tensor file: " + file_path);
}
diff --git a/src/core/util.cpp b/src/core/util.cpp
index 5292c38..c1d463e 100644
--- a/src/core/util.cpp
+++ b/src/core/util.cpp
@@ -7,6 +7,7 @@
#include <cstdlib>
#include <cstring>
#include <exception>
+#include <filesystem>
#include <fstream>
#include <locale>
#include <regex>
@@ -123,12 +124,12 @@ int round_up_to(int value, int base) {
#include <windows.h>
bool file_exists(const std::string& filename) {
- DWORD attributes = GetFileAttributesA(filename.c_str());
+ DWORD attributes = GetFileAttributesW(std::filesystem::u8path(filename).c_str());
return (attributes != INVALID_FILE_ATTRIBUTES && !(attributes & FILE_ATTRIBUTE_DIRECTORY));
}
bool is_directory(const std::string& path) {
- DWORD attributes = GetFileAttributesA(path.c_str());
+ DWORD attributes = GetFileAttributesW(std::filesystem::u8path(path).c_str());
return (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY));
}
@@ -152,8 +153,8 @@ std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename, bo
void* mapped_data = nullptr;
size_t file_size = 0;
- HANDLE file_handle = CreateFileA(
- filename.c_str(),
+ HANDLE file_handle = CreateFileW(
+ std::filesystem::u8path(filename).c_str(),
GENERIC_READ,
FILE_SHARE_READ,
nullptr,
diff --git a/src/model_io/gguf_io.cpp b/src/model_io/gguf_io.cpp
index 3e57448..8120131 100644
--- a/src/model_io/gguf_io.cpp
+++ b/src/model_io/gguf_io.cpp
@@ -3,6 +3,7 @@
#include <algorithm>
#include <cstdint>
#include <cstdio>
+#include <filesystem>
#include <fstream>
#include <ostream>
#include <string>
@@ -19,7 +20,7 @@ static void set_error(std::string* error, const std::string& message) {
}
bool is_gguf_file(const std::string& file_path) {
- std::ifstream file(file_path, std::ios::binary);
+ std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary);
if (!file.is_open()) {
return false;
}
@@ -178,7 +179,7 @@ bool GGUFStreamingWriter::write_metadata(const std::string& file_path,
}
LOG_INFO("trying to save tensors to %s", file_path.c_str());
- FILE* file = fopen(file_path.c_str(), "wb+");
+ FILE* file = ggml_fopen(file_path.c_str(), "wb+");
if (file == nullptr) {
set_error(error, "failed to open output file '" + file_path + "'");
close();
diff --git a/src/model_io/gguf_reader_ext.h b/src/model_io/gguf_reader_ext.h
index b55d49f..f02fe3c 100644
--- a/src/model_io/gguf_reader_ext.h
+++ b/src/model_io/gguf_reader_ext.h
@@ -2,6 +2,7 @@
#define __SD_MODEL_IO_GGUF_READER_EXT_H__
#include <cstdint>
+#include <filesystem>
#include <fstream>
#include <string>
#include <vector>
@@ -184,7 +185,7 @@ private:
public:
bool load(const std::string& file_path) {
- std::ifstream fin(file_path, std::ios::binary | std::ios::ate);
+ std::ifstream fin(std::filesystem::u8path(file_path), std::ios::binary | std::ios::ate);
if (!fin) {
LOG_ERROR("failed to open '%s'", file_path.c_str());
return false;
diff --git a/src/model_io/safetensors_io.cpp b/src/model_io/safetensors_io.cpp
index 3e69742..cf424af 100644
--- a/src/model_io/safetensors_io.cpp
+++ b/src/model_io/safetensors_io.cpp
@@ -26,15 +26,15 @@ static void set_error(std::string* error, const std::string& message) {
}
static std::string resolve_index_shard_path(const std::string& index_path, const std::string& shard_path) {
- fs::path shard_fs_path(shard_path);
+ fs::path shard_fs_path = fs::u8path(shard_path);
if (shard_fs_path.is_absolute()) {
- return shard_fs_path.lexically_normal().string();
+ return shard_fs_path.lexically_normal().u8string();
}
- return (fs::path(index_path).parent_path() / shard_fs_path).lexically_normal().string();
+ return (fs::u8path(index_path).parent_path() / shard_fs_path).lexically_normal().u8string();
}
bool is_safetensors_file(const std::string& file_path) {
- std::ifstream file(file_path, std::ios::binary);
+ std::ifstream file(fs::u8path(file_path), std::ios::binary);
if (!file.is_open()) {
return false;
}
@@ -153,7 +153,7 @@ bool read_safetensors_file(const std::string& file_path,
std::string* error,
std::map<std::string, std::string>* metadata,
std::set<std::string>* tensor_names) {
- std::ifstream file(file_path, std::ios::binary);
+ std::ifstream file(fs::u8path(file_path), std::ios::binary);
if (!file.is_open()) {
set_error(error, "failed to open '" + file_path + "'");
return false;
@@ -418,7 +418,7 @@ bool read_safetensors_index_file(const std::string& file_path,
std::string* error) {
shard_paths.clear();
- std::ifstream file(file_path);
+ std::ifstream file(fs::u8path(file_path));
if (!file.is_open()) {
set_error(error, "failed to open '" + file_path + "'");
return false;
@@ -522,7 +522,7 @@ bool write_safetensors_file(const std::string& file_path,
const std::string header_str = header.dump();
- std::ofstream file(file_path, std::ios::binary);
+ std::ofstream file(fs::u8path(file_path), std::ios::binary);
if (!file.is_open()) {
set_error(error, "failed to open '" + file_path + "' for writing");
return false;
@@ -596,7 +596,7 @@ bool SafetensorsStreamingWriter::write_metadata(const std::string& file_path,
data_start_ = ST_HEADER_SIZE_LEN + header_str.size();
LOG_INFO("trying to save tensors to %s", file_path.c_str());
- std::ofstream file(file_path, std::ios::binary | std::ios::trunc);
+ std::ofstream file(fs::u8path(file_path), std::ios::binary | std::ios::trunc);
if (!file.is_open()) {
set_error(error, "failed to open '" + file_path + "' for writing");
return false;
diff --git a/src/model_io/torch_legacy_io.cpp b/src/model_io/torch_legacy_io.cpp
index 3170fe8..163e664 100644
--- a/src/model_io/torch_legacy_io.cpp
+++ b/src/model_io/torch_legacy_io.cpp
@@ -2,6 +2,7 @@
#include <algorithm>
#include <cstdint>
+#include <filesystem>
#include <fstream>
#include <string>
#include <unordered_map>
@@ -98,7 +99,7 @@ static std::string torch_legacy_diagnostics(const std::string& file_path, const
bool read_torch_legacy_file(const std::string& file_path,
std::vector<TensorStorage>& tensor_storages,
std::string* error) {
- std::ifstream file(file_path, std::ios::binary);
+ std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary);
if (!file.is_open()) {
set_error(error, "failed to open '" + file_path + "'");
return false;
diff --git a/src/model_loader.cpp b/src/model_loader.cpp
index c24351f..d881c7e 100644
--- a/src/model_loader.cpp
+++ b/src/model_loader.cpp
@@ -4,6 +4,7 @@
#include <cinttypes>
#include <cstdarg>
#include <cstdlib>
+#include <filesystem>
#include <fstream>
#include <functional>
#include <mutex>
@@ -333,7 +334,7 @@ bool ModelLoader::init_from_safetensors_index_file(const std::string& file_path,
}
}
- std::ifstream index_file(file_path);
+ std::ifstream index_file(std::filesystem::u8path(file_path));
const auto index = nlohmann::json::parse(index_file);
for (const auto& entry : index.at("weight_map").items()) {
const auto expected = (std::filesystem::u8path(file_path).parent_path() /
@@ -1150,7 +1151,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
return;
}
} else if (!mmapped) {
- file.open(file_path, std::ios::binary);
+ file.open(std::filesystem::u8path(file_path), std::ios::binary);
if (!file.is_open()) {
LOG_ERROR("failed to open '%s'", file_path.c_str());
failed = true;
diff --git a/src/runtime/imatrix.cpp b/src/runtime/imatrix.cpp
index a6f3bca..616059d 100644
--- a/src/runtime/imatrix.cpp
+++ b/src/runtime/imatrix.cpp
@@ -10,6 +10,7 @@
#include <cmath>
#include <cstdlib>
#include <cstring>
+#include <filesystem>
static IMatrixCollector imatrix_collector;
@@ -227,7 +228,7 @@ void IMatrixCollector::save_imatrix(std::string fname, int ncall) const {
LOG_WARN("storing only %zu out of %zu entries\n", to_store.size(), stats_.size());
}
- std::ofstream out(fname, std::ios::binary);
+ std::ofstream out(std::filesystem::u8path(fname), std::ios::binary);
out.write((const char*)&n_entries, sizeof(n_entries));
for (const auto& name : to_store) {
const auto& stat = stats_.at(name);
@@ -251,7 +252,7 @@ void IMatrixCollector::save_imatrix(std::string fname, int ncall) const {
}
bool IMatrixCollector::load_imatrix(const char* fname) {
- std::ifstream in(fname, std::ios::binary);
+ std::ifstream in(std::filesystem::u8path(fname), std::ios::binary);
if (!in) {
LOG_ERROR("failed to open %s\n", fname);
return false;
diff --git a/src/tokenizers/hf_tokenizer.cpp b/src/tokenizers/hf_tokenizer.cpp
index f8daf6d..6e64018 100644
--- a/src/tokenizers/hf_tokenizer.cpp
+++ b/src/tokenizers/hf_tokenizer.cpp
@@ -4,6 +4,7 @@
#include <array>
#include <climits>
#include <cstdlib>
+#include <filesystem>
#include <fstream>
#include <queue>
#include <sstream>
@@ -495,7 +496,7 @@ struct HFTokenizer::Impl {
HFTokenizer::HFTokenizer(const std::string& path)
: impl_(new Impl) {
- std::ifstream stream(path, std::ios::binary);
+ std::ifstream stream(std::filesystem::u8path(path), std::ios::binary);
tokenizer_require(stream.good(), "cannot open " + path);
TokenizerJSON config;
stream >> config;