Skip to content

Commit

Permalink
Do not duplicate textures files when importing blender files
Browse files Browse the repository at this point in the history
blender imports will always start with `.godot/imported` because we first convert the blend to gltf, store it in `.godot/imported` and run the import from there, so resources linked from blend files end up with duplicate textures.

Introduce a new state where the resource exists on the disk but has not been imported yet.GLTF: Don't duplicate textures when importing blend files

blender imports will always start with `.godot/imported` because we first convert the blend to gltf, store it in `.godot/imported` and run the import from there, so resources linked from blend files end up with duplicate textures after #96778

Introduce a new state where the resource exists on the disk but has not been imported yet.
  • Loading branch information
demolke committed Nov 7, 2024
1 parent 87318a2 commit 99b42ce
Show file tree
Hide file tree
Showing 5 changed files with 270 additions and 92 deletions.
76 changes: 46 additions & 30 deletions modules/gltf/gltf_document.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3930,7 +3930,7 @@ Ref<Image> GLTFDocument::_parse_image_bytes_into_image(Ref<GLTFState> p_state, c
return r_image;
}

void GLTFDocument::_parse_image_save_image(Ref<GLTFState> p_state, const Vector<uint8_t> &p_bytes, const String &p_file_extension, int p_index, Ref<Image> p_image) {
void GLTFDocument::_parse_image_save_image(Ref<GLTFState> p_state, const Vector<uint8_t> &p_bytes, const String &p_resource_uri, const String &p_file_extension, int p_index, Ref<Image> p_image) {
GLTFState::GLTFHandleBinary handling = GLTFState::GLTFHandleBinary(p_state->handle_binary_image);
if (p_image->is_empty() || handling == GLTFState::GLTFHandleBinary::HANDLE_BINARY_DISCARD_TEXTURES) {
p_state->images.push_back(Ref<Texture2D>());
Expand All @@ -3948,33 +3948,46 @@ void GLTFDocument::_parse_image_save_image(Ref<GLTFState> p_state, const Vector<
WARN_PRINT(vformat("glTF: Image index '%d' did not have a name. It will be automatically given a name based on its index.", p_index));
p_image->set_name(itos(p_index));
}
bool must_import = true;
bool must_write = true; // If the resource does not exist on the disk within res:// directory write it.
bool must_import = true; // Trigger import.
Vector<uint8_t> img_data = p_image->get_data();
Dictionary generator_parameters;
String file_path = p_state->get_extract_path().path_join(p_state->get_extract_prefix() + "_" + p_image->get_name());
file_path += p_file_extension.is_empty() ? ".png" : p_file_extension;
if (FileAccess::exists(file_path + ".import")) {
Ref<ConfigFile> config;
config.instantiate();
config->load(file_path + ".import");
if (config->has_section_key("remap", "generator_parameters")) {
generator_parameters = (Dictionary)config->get_value("remap", "generator_parameters");
}
if (!generator_parameters.has("md5")) {
must_import = false; // Didn't come from a gltf document; don't overwrite.
String file_path;
if (!p_resource_uri.is_empty()) {
// The file already exists in the res:// folder, but failed load, needs to be imported.
file_path = p_resource_uri;
must_import = true;
must_write = false; // No need to overwrite it.
} else {
// Texture data has to be written to the res:// folder and imported.
file_path = p_state->get_extract_path().path_join(p_state->get_extract_prefix() + "_" + p_image->get_name());
file_path += p_file_extension.is_empty() ? ".png" : p_file_extension;
if (FileAccess::exists(file_path + ".import")) {
Ref<ConfigFile> config;
config.instantiate();
config->load(file_path + ".import");
if (config->has_section_key("remap", "generator_parameters")) {
generator_parameters = (Dictionary)config->get_value("remap", "generator_parameters");
}
if (!generator_parameters.has("md5")) {
must_write = false; // Didn't come from a gltf document; don't overwrite.
must_import = false; // And don't import.
}
}
}
if (must_import) {

if (must_write) {
String existing_md5 = generator_parameters["md5"];
unsigned char md5_hash[16];
CryptoCore::md5(img_data.ptr(), img_data.size(), md5_hash);
String new_md5 = String::hex_encode_buffer(md5_hash, 16);
generator_parameters["md5"] = new_md5;
if (new_md5 == existing_md5) {
must_write = false;
must_import = false;
}
}
if (must_import) {
if (must_write) {
Error err = OK;
if (p_file_extension.is_empty()) {
// If a file extension was not specified, save the image data to a PNG file.
Expand All @@ -3987,6 +4000,8 @@ void GLTFDocument::_parse_image_save_image(Ref<GLTFState> p_state, const Vector<
file->store_buffer(p_bytes);
file->close();
}
}
if (must_import) {
// ResourceLoader::import will crash if not is_editor_hint(), so this case is protected above and will fall through to uncompressed.
HashMap<StringName, Variant> custom_options;
custom_options[SNAME("mipmaps/generate")] = true;
Expand All @@ -4000,7 +4015,7 @@ void GLTFDocument::_parse_image_save_image(Ref<GLTFState> p_state, const Vector<
p_state->source_images.push_back(saved_image->get_image());
return;
} else {
WARN_PRINT(vformat("glTF: Image index '%d' with the name '%s' couldn't be imported. It will be loaded directly instead, uncompressed.", p_index, p_image->get_name()));
WARN_PRINT(vformat("glTF: Image index '%d' with the name '%s' resolved to %s couldn't be imported. It will be loaded directly instead, uncompressed.", p_index, p_image->get_name(), file_path));
}
}
}
Expand Down Expand Up @@ -4068,6 +4083,9 @@ Error GLTFDocument::_parse_images(Ref<GLTFState> p_state, const String &p_base_p
while (used_names.has(image_name)) {
image_name += "_" + itos(i);
}

String resource_uri;

used_names.insert(image_name);
// Load the image data. If we get a byte array, store here for later.
Vector<uint8_t> data;
Expand All @@ -4085,19 +4103,17 @@ Error GLTFDocument::_parse_images(Ref<GLTFState> p_state, const String &p_base_p
ERR_FAIL_COND_V(p_base_path.is_empty(), ERR_INVALID_PARAMETER);
uri = uri.uri_decode();
uri = p_base_path.path_join(uri).replace("\\", "/"); // Fix for Windows.
// If the image is in the .godot/imported directory, we can't use ResourceLoader.
if (!p_base_path.begins_with("res://.godot/imported")) {
// ResourceLoader will rely on the file extension to use the relevant loader.
// The spec says that if mimeType is defined, it should take precedence (e.g.
// there could be a `.png` image which is actually JPEG), but there's no easy
// API for that in Godot, so we'd have to load as a buffer (i.e. embedded in
// the material), so we only do that only as fallback.
Ref<Texture2D> texture = ResourceLoader::load(uri, "Texture2D");
if (texture.is_valid()) {
p_state->images.push_back(texture);
p_state->source_images.push_back(texture->get_image());
continue;
}
resource_uri = uri.simplify_path();
// ResourceLoader will rely on the file extension to use the relevant loader.
// The spec says that if mimeType is defined, it should take precedence (e.g.
// there could be a `.png` image which is actually JPEG), but there's no easy
// API for that in Godot, so we'd have to load as a buffer (i.e. embedded in
// the material), so we only do that only as fallback.
Ref<Texture2D> texture = ResourceLoader::load(uri, "Texture2D");
if (texture.is_valid()) {
p_state->images.push_back(texture);
p_state->source_images.push_back(texture->get_image());
continue;
}
// mimeType is optional, but if we have it in the file extension, let's use it.
// If the mimeType does not match with the file extension, either it should be
Expand Down Expand Up @@ -4139,7 +4155,7 @@ Error GLTFDocument::_parse_images(Ref<GLTFState> p_state, const String &p_base_p
String file_extension;
Ref<Image> img = _parse_image_bytes_into_image(p_state, data, mime_type, i, file_extension);
img->set_name(image_name);
_parse_image_save_image(p_state, data, file_extension, i, img);
_parse_image_save_image(p_state, data, resource_uri, file_extension, i, img);
}

print_verbose("glTF: Total images: " + itos(p_state->images.size()));
Expand Down
2 changes: 1 addition & 1 deletion modules/gltf/gltf_document.h
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ class GLTFDocument : public Resource {
Error _serialize_images(Ref<GLTFState> p_state);
Error _serialize_lights(Ref<GLTFState> p_state);
Ref<Image> _parse_image_bytes_into_image(Ref<GLTFState> p_state, const Vector<uint8_t> &p_bytes, const String &p_mime_type, int p_index, String &r_file_extension);
void _parse_image_save_image(Ref<GLTFState> p_state, const Vector<uint8_t> &p_bytes, const String &p_file_extension, int p_index, Ref<Image> p_image);
void _parse_image_save_image(Ref<GLTFState> p_state, const Vector<uint8_t> &p_bytes, const String &p_resource_uri, const String &p_file_extension, int p_index, Ref<Image> p_image);
Error _parse_images(Ref<GLTFState> p_state, const String &p_base_path);
Error _parse_textures(Ref<GLTFState> p_state);
Error _parse_texture_samplers(Ref<GLTFState> p_state);
Expand Down
107 changes: 107 additions & 0 deletions modules/gltf/tests/test_gltf.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
/**************************************************************************/
/* test_gltf_extras.h */
/**************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/**************************************************************************/
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/

#ifndef TEST_GLTF_H
#define TEST_GLTF_H

#include "tests/test_macros.h"

#ifdef TOOLS_ENABLED

#include "core/os/os.h"
#include "editor/import/3d/resource_importer_scene.h"
#include "modules/gltf/editor/editor_scene_importer_gltf.h"
#include "modules/gltf/gltf_document.h"
#include "modules/gltf/gltf_state.h"
#include "scene/3d/mesh_instance_3d.h"
#include "scene/3d/skeleton_3d.h"
#include "scene/main/window.h"
#include "scene/resources/3d/primitive_meshes.h"
#include "scene/resources/material.h"
#include "scene/resources/packed_scene.h"

namespace TestGltf {

static Node *gltf_export_then_import(Node *p_root, String &p_tempfilebase) {
Ref<GLTFDocument> doc;
doc.instantiate();
Ref<GLTFState> state;
state.instantiate();
Error err = doc->append_from_scene(p_root, state, EditorSceneFormatImporter::IMPORT_USE_NAMED_SKIN_BINDS);
CHECK_MESSAGE(err == OK, "GLTF state generation failed.");
err = doc->write_to_filesystem(state, p_tempfilebase + ".gltf");
CHECK_MESSAGE(err == OK, "Writing GLTF to cache dir failed.");

// Setting up importers.
Ref<ResourceImporterScene> import_scene = memnew(ResourceImporterScene("PackedScene", true));
ResourceFormatImporter::get_singleton()->add_importer(import_scene);
Ref<EditorSceneFormatImporterGLTF> import_gltf;
import_gltf.instantiate();
ResourceImporterScene::add_scene_importer(import_gltf);

// GTLF importer behaves differently outside of editor, it's too late to modify Engine::get_editor_hint
// as the registration of runtime extensions already happened, so remove them. See modules/gltf/register_types.cpp
GLTFDocument::unregister_all_gltf_document_extensions();

HashMap<StringName, Variant> options(20);
options["nodes/root_type"] = "";
options["nodes/root_name"] = "";
options["nodes/apply_root_scale"] = true;
options["nodes/root_scale"] = 1.0;
options["meshes/ensure_tangents"] = true;
options["meshes/generate_lods"] = false;
options["meshes/create_shadow_meshes"] = true;
options["meshes/light_baking"] = 1;
options["meshes/lightmap_texel_size"] = 0.2;
options["meshes/force_disable_compression"] = false;
options["skins/use_named_skins"] = true;
options["animation/import"] = true;
options["animation/fps"] = 30;
options["animation/trimming"] = false;
options["animation/remove_immutable_tracks"] = true;
options["import_script/path"] = "";
options["_subresources"] = Dictionary();
options["gltf/naming_version"] = 1;

// Process gltf file, note that this generates `.scn` resource from the 2nd argument.
err = import_scene->import(p_tempfilebase + ".gltf", p_tempfilebase, options, nullptr, nullptr, nullptr);
CHECK_MESSAGE(err == OK, "GLTF import failed.");
ResourceImporterScene::remove_scene_importer(import_gltf);

Ref<PackedScene> packed_scene = ResourceLoader::load(p_tempfilebase + ".scn", "", ResourceFormatLoader::CACHE_MODE_REPLACE, &err);
CHECK_MESSAGE(err == OK, "Loading scene failed.");
Node *p_scene = packed_scene->instantiate();
return p_scene;
}
} //namespace TestGltf

#endif // TOOLS_ENABLED

#endif // TEST_GLTF_EXTRAS_H
66 changes: 5 additions & 61 deletions modules/gltf/tests/test_gltf_extras.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,75 +31,19 @@
#ifndef TEST_GLTF_EXTRAS_H
#define TEST_GLTF_EXTRAS_H

#include "test_gltf.h"
#include "tests/test_macros.h"

#ifdef TOOLS_ENABLED

#include "core/os/os.h"
#include "editor/import/3d/resource_importer_scene.h"
#include "modules/gltf/editor/editor_scene_importer_gltf.h"
#include "modules/gltf/gltf_document.h"
#include "modules/gltf/gltf_state.h"
#include "scene/3d/mesh_instance_3d.h"
#include "scene/3d/skeleton_3d.h"
#include "scene/main/window.h"
#include "scene/resources/3d/primitive_meshes.h"
#include "scene/resources/material.h"
#include "scene/resources/packed_scene.h"

namespace TestGltfExtras {

static Node *_gltf_export_then_import(Node *p_root, String &p_tempfilebase) {
Ref<GLTFDocument> doc;
doc.instantiate();
Ref<GLTFState> state;
state.instantiate();
Error err = doc->append_from_scene(p_root, state, EditorSceneFormatImporter::IMPORT_USE_NAMED_SKIN_BINDS);
CHECK_MESSAGE(err == OK, "GLTF state generation failed.");
err = doc->write_to_filesystem(state, p_tempfilebase + ".gltf");
CHECK_MESSAGE(err == OK, "Writing GLTF to cache dir failed.");

// Setting up importers.
Ref<ResourceImporterScene> import_scene = memnew(ResourceImporterScene("PackedScene", true));
ResourceFormatImporter::get_singleton()->add_importer(import_scene);
Ref<EditorSceneFormatImporterGLTF> import_gltf;
import_gltf.instantiate();
ResourceImporterScene::add_scene_importer(import_gltf);

// GTLF importer behaves differently outside of editor, it's too late to modify Engine::get_editor_hint
// as the registration of runtime extensions already happened, so remove them. See modules/gltf/register_types.cpp
GLTFDocument::unregister_all_gltf_document_extensions();

HashMap<StringName, Variant> options(20);
options["nodes/root_type"] = "";
options["nodes/root_name"] = "";
options["nodes/apply_root_scale"] = true;
options["nodes/root_scale"] = 1.0;
options["meshes/ensure_tangents"] = true;
options["meshes/generate_lods"] = false;
options["meshes/create_shadow_meshes"] = true;
options["meshes/light_baking"] = 1;
options["meshes/lightmap_texel_size"] = 0.2;
options["meshes/force_disable_compression"] = false;
options["skins/use_named_skins"] = true;
options["animation/import"] = true;
options["animation/fps"] = 30;
options["animation/trimming"] = false;
options["animation/remove_immutable_tracks"] = true;
options["import_script/path"] = "";
options["_subresources"] = Dictionary();
options["gltf/naming_version"] = 1;

// Process gltf file, note that this generates `.scn` resource from the 2nd argument.
err = import_scene->import(p_tempfilebase + ".gltf", p_tempfilebase, options, nullptr, nullptr, nullptr);
CHECK_MESSAGE(err == OK, "GLTF import failed.");
ResourceImporterScene::remove_scene_importer(import_gltf);

Ref<PackedScene> packed_scene = ResourceLoader::load(p_tempfilebase + ".scn", "", ResourceFormatLoader::CACHE_MODE_REPLACE, &err);
CHECK_MESSAGE(err == OK, "Loading scene failed.");
Node *p_scene = packed_scene->instantiate();
return p_scene;
}
namespace TestGltf {

TEST_CASE("[SceneTree][Node] GLTF test mesh and material meta export and import") {
// Setup scene.
Expand Down Expand Up @@ -135,7 +79,7 @@ TEST_CASE("[SceneTree][Node] GLTF test mesh and material meta export and import"

// Convert to GLFT and back.
String tempfile = OS::get_singleton()->get_cache_path().path_join("gltf_extras");
Node *loaded = _gltf_export_then_import(original, tempfile);
Node *loaded = gltf_export_then_import(original, tempfile);

// Compare the results.
CHECK(loaded->get_name() == "node3d");
Expand Down Expand Up @@ -200,7 +144,7 @@ TEST_CASE("[SceneTree][Node] GLTF test skeleton and bone export and import") {

// Convert to GLFT and back.
String tempfile = OS::get_singleton()->get_cache_path().path_join("gltf_bone_extras");
Node *loaded = _gltf_export_then_import(scene, tempfile);
Node *loaded = gltf_export_then_import(scene, tempfile);

// Compare the results.
CHECK(loaded->get_name() == "node3d");
Expand All @@ -215,7 +159,7 @@ TEST_CASE("[SceneTree][Node] GLTF test skeleton and bone export and import") {
memdelete(scene);
memdelete(loaded);
}
} // namespace TestGltfExtras
} //namespace TestGltf

#endif // TOOLS_ENABLED

Expand Down
Loading

0 comments on commit 99b42ce

Please sign in to comment.