使用python版本ncnn，加载模型和参数文件时，路径存在中文，抛出异常

[pangxiaobin]

error log | 日志或报错信息 | ログ

fopen error

context | 编译/运行环境 | バックグラウンド

how to reproduce | 复现步骤 | 再現方法

1. 使用windows 环境，安装环境， pip install ncnn
2. 加载模型文件和参数，模型文件和参数路径包含中文

more | 其他 | その他

目前的解决方案，使用load_model_mem 和laod_param_mem

param_path = "D:/我的模型/中文路径/param.param"
model_path = "D:/我的模型/中文路径/model.bin"

net = ncnn.Net()

with open(param_path, 'rb') as f:
    param_data = f.read()

with open(model_path, 'rb') as f:
    model_data = f.read()

# 加载 .param
ret_param = net.load_param_mem(param_data)
if ret_param != 0:
    raise RuntimeError(f"从内存加载 .param 失败! 错误代码: {ret_param}")

# 加载 .bin
ret_model = net.load_model_mem(model_data)
if ret_model != None:
    raise RuntimeError(f"从内存加载 .bin 失败! 错误代码: {ret_model}")

[dosubot]

这是 ncnn Python 版在 Windows 下的已知限制：当模型或参数文件路径包含中文（或其他非 ASCII 字符）时，会出现 fopen error，因为底层 C++ 实现直接用 fopen 加载文件，而 fopen 只支持窄字符路径，无法处理中文路径（除非用宽字符 API，但 ncnn 目前没有这样做）代码参考。

你现在的做法（用 Python 的 open() 读取文件内容，再用 load_param_mem 和 load_model_mem 加载）是正确且推荐的，因为 Python 的文件操作支持 Unicode 路径，可以绕过这个底层限制代码参考。

目前没有发现更直接的官方解决方案或替代方法。如果后续 ncnn 支持 Windows 下的宽字符路径（如 _wfopen），这个问题才会根本解决。现在你的 workaround 就是最佳实践。

To reply, just mention @dosu.

---

^{How did I do? Good | Irrelevant | Incorrect | Verbose | Hallucination | Report 🐛 | Other}   

[whyb]

Hello @nihui @pangxiaobin ，要解决python ncnn加载模型中文编码问题，想到有两种实现方式。

• ncnn现有的方法

int Net::load_param(const char* protopath);
int Net::load_model(const char* modelpath);

• 为了让支持中文编码，可能需要增加的函数申明

// 需要C++ 98
int Net::load_param(const wchar_t* protopath);
int Net::load_model(const wchar_t* modelpath);

// 需要C++ 17
int Net::load_param(const std::filesystem::path protopath);
int Net::load_model(const std::filesystem::path modelpath);

• 以上四个函数的具体完整实现

int Net::load_param(const wchar_t* protopath)
{
    FILE* fp = _wfopen(protopath, L"rb");
    if (!fp)
    {
        NCNN_LOGE("fopen failed for wide character path");
        return -1;
    }
    int ret = load_param(fp);
    fclose(fp);
    return ret;
}
int Net::load_model(const wchar_t* modelpath)
{
    FILE* fp = _wfopen(modelpath, L"rb"); 
    if (!fp)
    {
        NCNN_LOGE("fopen failed for wide character model path");
        return -1;
    }
    int ret = load_model(fp);
    fclose(fp);
    return ret;
}

int Net::load_param(const std::filesystem::path protopath)
{
    if (!std::filesystem::exists(protopath))
    {
        NCNN_LOGE("Param file not found");
        return -1;
    }
    size_t file_size;
    try {
        file_size = std::filesystem::file_size(protopath);
    } catch (const std::filesystem::filesystem_error& e) {
        NCNN_LOGE("Failed to get param file size");
        return -1;
    }
    if (file_size == 0)
    {
        NCNN_LOGE("Param file is empty");
        return -1;
    }
    std::ifstream ifs(protopath, std::ios::binary);
    if (!ifs.is_open())
    {
        NCNN_LOGE("Failed to open param file");
        return -1;
    }
    std::vector<unsigned char> memory_buffer(file_size);
    ifs.read(reinterpret_cast<char*>(memory_buffer.data()), file_size);
    if (ifs.fail())
    {
        NCNN_LOGE("Failed to read all bytes from param file");
        return -1;
    }
    int ret = load_param(memory_buffer.data());
    return ret;
}

int Net::load_model(const std::filesystem::path modelpath)
{
    if (!std::filesystem::exists(modelpath))
    {
        NCNN_LOGE("Model file not found");
        return -1;
    }
    size_t file_size;
    try {
        file_size = std::filesystem::file_size(modelpath);
    } catch (const std::filesystem::filesystem_error& e) {
        NCNN_LOGE("Failed to get model file size");
        return -1;
    }
    if (file_size == 0)
    {
        NCNN_LOGE("Model file is empty");
        return -1;
    }
    std::ifstream ifs(modelpath, std::ios::binary);
    if (!ifs.is_open())
    {
        NCNN_LOGE("Failed to open model file");
        return -1;
    }
    std::vector<unsigned char> memory_buffer(file_size);
    ifs.read(reinterpret_cast<char*>(memory_buffer.data()), file_size);
    if (ifs.fail())
    {
        NCNN_LOGE("Failed to read all bytes from model file");
        return -1;
    }
    int ret = load_model(memory_buffer.data());
    return ret;
}

[nihui]

Hello @nihui @pangxiaobin ，要解决python ncnn加载模型中文编码问题，想到有两种实现方式。

* ncnn现有的方法

int Net::load_param(const char* protopath);
int Net::load_model(const char* modelpath);

* 为了让支持中文编码，可能需要增加的函数申明

// 需要C++ 98
int Net::load_param(const wchar_t* protopath);
int Net::load_model(const wchar_t* modelpath);

// 需要C++ 17
int Net::load_param(const std::filesystem::path protopath);
int Net::load_model(const std::filesystem::path modelpath);

* 以上四个函数的具体完整实现

int Net::load_param(const wchar_t* protopath)
{
FILE* fp = _wfopen(protopath, L"rb");
if (!fp)
{
NCNN_LOGE("fopen failed for wide character path");
return -1;
}
int ret = load_param(fp);
fclose(fp);
return ret;
}
int Net::load_model(const wchar_t* modelpath)
{
FILE* fp = _wfopen(modelpath, L"rb");
if (!fp)
{
NCNN_LOGE("fopen failed for wide character model path");
return -1;
}
int ret = load_model(fp);
fclose(fp);
return ret;
}

int Net::load_param(const std::filesystem::path protopath)
{
if (!std::filesystem::exists(protopath))
{
NCNN_LOGE("Param file not found");
return -1;
}
size_t file_size;
try {
file_size = std::filesystem::file_size(protopath);
} catch (const std::filesystem::filesystem_error& e) {
NCNN_LOGE("Failed to get param file size");
return -1;
}
if (file_size == 0)
{
NCNN_LOGE("Param file is empty");
return -1;
}
std::ifstream ifs(protopath, std::ios::binary);
if (!ifs.is_open())
{
NCNN_LOGE("Failed to open param file");
return -1;
}
std::vector memory_buffer(file_size);
ifs.read(reinterpret_cast<char*>(memory_buffer.data()), file_size);
if (ifs.fail())
{
NCNN_LOGE("Failed to read all bytes from param file");
return -1;
}
int ret = load_param(memory_buffer.data());
return ret;
}

int Net::load_model(const std::filesystem::path modelpath)
{
if (!std::filesystem::exists(modelpath))
{
NCNN_LOGE("Model file not found");
return -1;
}
size_t file_size;
try {
file_size = std::filesystem::file_size(modelpath);
} catch (const std::filesystem::filesystem_error& e) {
NCNN_LOGE("Failed to get model file size");
return -1;
}
if (file_size == 0)
{
NCNN_LOGE("Model file is empty");
return -1;
}
std::ifstream ifs(modelpath, std::ios::binary);
if (!ifs.is_open())
{
NCNN_LOGE("Failed to open model file");
return -1;
}
std::vector memory_buffer(file_size);
ifs.read(reinterpret_cast<char*>(memory_buffer.data()), file_size);
if (ifs.fail())
{
NCNN_LOGE("Failed to read all bytes from model file");
return -1;
}
int ret = load_model(memory_buffer.data());
return ret;
}

ifstream 的实现是有问题的 memory_buffer 被释放了

[whyb]

Hello @nihui @pangxiaobin ，要解决python ncnn加载模型中文编码问题，想到有两种实现方式。

* ncnn现有的方法

int Net::load_param(const char* protopath);
int Net::load_model(const char* modelpath);

* 为了让支持中文编码，可能需要增加的函数申明

// 需要C++ 98
int Net::load_param(const wchar_t* protopath);
int Net::load_model(const wchar_t* modelpath);
// 需要C++ 17
int Net::load_param(const std::filesystem::path protopath);
int Net::load_model(const std::filesystem::path modelpath);

* 以上四个函数的具体完整实现

int Net::load_param(const wchar_t* protopath)
{
FILE* fp = _wfopen(protopath, L"rb");
if (!fp)
{
NCNN_LOGE("fopen failed for wide character path");
return -1;
}
int ret = load_param(fp);
fclose(fp);
return ret;
}
int Net::load_model(const wchar_t* modelpath)
{
FILE* fp = _wfopen(modelpath, L"rb");
if (!fp)
{
NCNN_LOGE("fopen failed for wide character model path");
return -1;
}
int ret = load_model(fp);
fclose(fp);
return ret;
}
int Net::load_param(const std::filesystem::path protopath)
{
if (!std::filesystem::exists(protopath))
{
NCNN_LOGE("Param file not found");
return -1;
}
size_t file_size;
try {
file_size = std::filesystem::file_size(protopath);
} catch (const std::filesystem::filesystem_error& e) {
NCNN_LOGE("Failed to get param file size");
return -1;
}
if (file_size == 0)
{
NCNN_LOGE("Param file is empty");
return -1;
}
std::ifstream ifs(protopath, std::ios::binary);
if (!ifs.is_open())
{
NCNN_LOGE("Failed to open param file");
return -1;
}
std::vector memory_buffer(file_size);
ifs.read(reinterpret_cast<char*>(memory_buffer.data()), file_size);
if (ifs.fail())
{
NCNN_LOGE("Failed to read all bytes from param file");
return -1;
}
int ret = load_param(memory_buffer.data());
return ret;
}
int Net::load_model(const std::filesystem::path modelpath)
{
if (!std::filesystem::exists(modelpath))
{
NCNN_LOGE("Model file not found");
return -1;
}
size_t file_size;
try {
file_size = std::filesystem::file_size(modelpath);
} catch (const std::filesystem::filesystem_error& e) {
NCNN_LOGE("Failed to get model file size");
return -1;
}
if (file_size == 0)
{
NCNN_LOGE("Model file is empty");
return -1;
}
std::ifstream ifs(modelpath, std::ios::binary);
if (!ifs.is_open())
{
NCNN_LOGE("Failed to open model file");
return -1;
}
std::vector memory_buffer(file_size);
ifs.read(reinterpret_cast<char*>(memory_buffer.data()), file_size);
if (ifs.fail())
{
NCNN_LOGE("Failed to read all bytes from model file");
return -1;
}
int ret = load_model(memory_buffer.data());
return ret;
}

ifstream 的实现是有问题的 memory_buffer 被释放了

明白了，ncnn的这俩函数load_param/load_model的内存加载版本是不会在内部存储一份模型数据的，只是存储的指针，所以我重新
实现了filesystem的两个函数，用转换成DataReaderFromMemory的版本去加载：

#include <filesystem>
#include <fstream>
#include <vector>
#include <algorithm>

static bool read_file_to_vector(const std::filesystem::path& path, std::vector<unsigned char>& buffer)
{
    if (!std::filesystem::exists(path))
    {
        NCNN_LOGE("File not found: %s", path.string().c_str());
        return false;
    }
    size_t file_size;
    try {
        file_size = std::filesystem::file_size(path);
    } catch (const std::filesystem::filesystem_error& e) {
        NCNN_LOGE("Failed to get file size for %s: %s", path.string().c_str(), e.what());
        return false;
    }
    if (file_size == 0)
    {
        NCNN_LOGE("File is empty: %s", path.string().c_str());
        return false;
    }
    std::ifstream ifs(path, std::ios::binary);
    if (!ifs.is_open())
    {
        NCNN_LOGE("Failed to open file: %s", path.string().c_str());
        return false;
    }
    buffer.resize(file_size);
    ifs.read(reinterpret_cast<char*>(buffer.data()), file_size);

    if (ifs.fail() && ifs.gcount() != file_size)
    {
        NCNN_LOGE("Failed to read entire file content for %s", path.string().c_str());
        return false;
    }
    return true;
}

int Net::load_param(const std::filesystem::path protopath)
{
    std::vector<unsigned char> param_buffer;
    if (!read_file_to_vector(protopath, param_buffer))
    {
        return -1;
    }
    const unsigned char* mem_ptr = param_buffer.data();
    ncnn::DataReaderFromMemory dr_mem(mem_ptr);
    return load_param(dr_mem);
}

int Net::load_model(const std::filesystem::path modelpath)
{
    std::vector<unsigned char> model_buffer;
    if (!read_file_to_vector(modelpath, model_buffer))
    {
        return -1;
    }
    const unsigned char* mem_ptr = model_buffer.data();
    ncnn::DataReaderFromMemory dr_mem(mem_ptr);
    return load_model(dr_mem);
}

[whyb]

@nihui @pangxiaobin 改成了用FILE*指针的方式：

#include <cstdio>
#include <filesystem>

int Net::load_param(const std::filesystem::path protopath)
{
    FILE* fp = nullptr;
#ifdef _WIN32
    fp = _wfopen(protopath.c_str(), L"rb");
#else
    fp = std::fopen(protopath.c_str(), "rb");
#endif
    if (!fp)
    {
        NCNN_LOGE("fopen %s failed", protopath.string().c_str());
        return -1;
    }
    int ret = load_param(fp);
    std::fclose(fp);
    return ret;
}

int Net::load_model(const std::filesystem::path modelpath)
{
    FILE* fp = nullptr;
#ifdef _WIN32
    fp = _wfopen(modelpath.c_str(), L"rb");
#else
    fp = std::fopen(modelpath.c_str(), "rb");
#endif
    if (!fp)
    {
        NCNN_LOGE("fopen %s failed", modelpath.string().c_str());
        return -1;
    }
    int ret = load_model(fp);
    std::fclose(fp);
    return ret;
}