Do not duplicate textures files when importing blender files

blender imports will always start with `.godot/imported` because we first convert the blend to gltf, store it in `.godot/imported` and run the import from there, so resources linked from blend files end up with duplicate textures.

Introduce a new state where the resource exists on the disk but has not been imported yet.GLTF: Don't duplicate textures when importing blend files

blender imports will always start with `.godot/imported` because we first convert the blend to gltf, store it in `.godot/imported` and run the import from there, so resources linked from blend files end up with duplicate textures after #96778

Introduce a new state where the resource exists on the disk but has not been imported yet.

core/config/engine.h

@@ -85,6 +85,7 @@ class Engine {
 	HashMap<StringName, Object *> singleton_ptrs;
 
 	bool editor_hint = false;
+	bool tests_hint = false;
 	bool project_manager_hint = false;
 	bool extension_reloading = false;
 
@@ -157,6 +158,9 @@ class Engine {
 	_FORCE_INLINE_ void set_editor_hint(bool p_enabled) { editor_hint = p_enabled; }
 	_FORCE_INLINE_ bool is_editor_hint() const { return editor_hint; }
 
+	_FORCE_INLINE_ void set_tests_hint(bool p_enabled) { tests_hint = p_enabled; }
+	_FORCE_INLINE_ bool is_tests_hint() const { return tests_hint; }
+
 	_FORCE_INLINE_ void set_project_manager_hint(bool p_enabled) { project_manager_hint = p_enabled; }
 	_FORCE_INLINE_ bool is_project_manager_hint() const { return project_manager_hint; }
 
@@ -166,6 +170,9 @@ class Engine {
 	_FORCE_INLINE_ void set_editor_hint(bool p_enabled) {}
 	_FORCE_INLINE_ bool is_editor_hint() const { return false; }
 
+	_FORCE_INLINE_ void set_tests_hint(bool p_enabled) {}
+	_FORCE_INLINE_ bool is_tests_hint() const { return false; }
+
 	_FORCE_INLINE_ void set_project_manager_hint(bool p_enabled) {}
 	_FORCE_INLINE_ bool is_project_manager_hint() const { return false; }
 

editor/editor_file_system.cpp

@@ -2710,7 +2710,7 @@ Error EditorFileSystem::_reimport_file(const String &p_file, const HashMap<Strin
 		importer = ResourceFormatImporter::get_singleton()->get_importer_by_extension(p_file.get_extension());
 		load_default = true;
 		if (importer.is_null()) {
-			ERR_FAIL_V_MSG(ERR_FILE_CANT_OPEN, "BUG: File queued for import, but can't be imported, importer for type '" + importer_name + "' not found.");
+			ERR_FAIL_V_MSG(ERR_FILE_CANT_OPEN, "BUG: File '" + p_file + "' queued for import, but can't be imported, importer for type '" + importer_name + "' not found.");
 		}
 	}
 
@@ -3193,7 +3193,7 @@ void EditorFileSystem::reimport_files(const Vector<String> &p_files) {
 }
 
 Error EditorFileSystem::reimport_append(const String &p_file, const HashMap<StringName, Variant> &p_custom_options, const String &p_custom_importer, Variant p_generator_parameters) {
-	ERR_FAIL_COND_V_MSG(!importing, ERR_INVALID_PARAMETER, "Can only append files to import during a current reimport process.");
+	ERR_FAIL_COND_V_MSG(!Engine::get_singleton()->is_tests_hint() && !importing, ERR_INVALID_PARAMETER, "Can only append files to import during a current reimport process.");
 	Vector<String> reloads;
 	reloads.append(p_file);
 
@@ -3544,5 +3544,9 @@ EditorFileSystem::EditorFileSystem() {
 }
 
 EditorFileSystem::~EditorFileSystem() {
+	if (filesystem) {
+		memdelete(filesystem);
+	}
+	filesystem = nullptr;
 	ResourceSaver::set_get_resource_id_for_path(nullptr);
 }

editor/editor_paths.cpp

@@ -105,6 +105,7 @@ void EditorPaths::create() {
 void EditorPaths::free() {
 	ERR_FAIL_NULL(singleton);
 	memdelete(singleton);
+	singleton = nullptr;
 }
 
 void EditorPaths::_bind_methods() {

modules/gltf/editor/editor_scene_importer_blend.cpp

@@ -143,14 +143,18 @@ Node *EditorSceneFormatImporterBlend::import_scene(const String &p_path, uint32_
 	const String sink = ProjectSettings::get_singleton()->get_imported_files_path().path_join(
 			vformat("%s-%s.gltf", blend_basename, p_path.md5_text()));
 	const String sink_global = ProjectSettings::get_singleton()->globalize_path(sink);
+	// If true, unpack the original images to the Godot file system and use them. Allows changing image import settings like VRAM compression.
+	// If false, allow Blender to convert the original images, such as re-packing roughness and metallic into one roughness+metallic texture.
+	// In most cases this is desired, but if the .blend file's images are not in the correct format, this must be disabled for correct behavior.
+	const bool unpack_original_images = p_options.has(SNAME("blender/materials/unpack_enabled")) && p_options[SNAME("blender/materials/unpack_enabled")];
 
 	// Handle configuration options.
 
 	Dictionary request_options;
 	Dictionary parameters_map;
 
 	parameters_map["filepath"] = sink_global;
-	parameters_map["export_keep_originals"] = true;
+	parameters_map["export_keep_originals"] = unpack_original_images;
 	parameters_map["export_format"] = "GLTF_SEPARATE";
 	parameters_map["export_yup"] = true;
 
@@ -285,12 +289,7 @@ Node *EditorSceneFormatImporterBlend::import_scene(const String &p_path, uint32_
 		parameters_map["export_apply"] = false;
 	}
 
-	if (p_options.has(SNAME("blender/materials/unpack_enabled")) && p_options[SNAME("blender/materials/unpack_enabled")]) {
-		request_options["unpack_all"] = true;
-	} else {
-		request_options["unpack_all"] = false;
-	}
-
+	request_options["unpack_all"] = unpack_original_images;
 	request_options["path"] = source_global;
 	request_options["gltf_options"] = parameters_map;
 
@@ -311,17 +310,13 @@ Node *EditorSceneFormatImporterBlend::import_scene(const String &p_path, uint32_
 	Ref<GLTFState> state;
 	state.instantiate();
 
-	String base_dir;
-	if (p_options.has(SNAME("blender/materials/unpack_enabled")) && p_options[SNAME("blender/materials/unpack_enabled")]) {
-		base_dir = sink.get_base_dir();
-	}
 	if (p_options.has(SNAME("nodes/import_as_skeleton_bones")) ? (bool)p_options[SNAME("nodes/import_as_skeleton_bones")] : false) {
 		state->set_import_as_skeleton_bones(true);
 	}
 	state->set_scene_name(blend_basename);
 	state->set_extract_path(p_path.get_base_dir());
 	state->set_extract_prefix(blend_basename);
-	err = gltf->append_from_file(sink.get_basename() + ".gltf", state, p_flags, base_dir);
+	err = gltf->append_from_file(sink.get_basename() + ".gltf", state, p_flags, sink.get_base_dir());
 	if (err != OK) {
 		if (r_err) {
 			*r_err = FAILED;

modules/gltf/gltf_document.cpp

@@ -3930,7 +3930,7 @@ Ref<Image> GLTFDocument::_parse_image_bytes_into_image(Ref<GLTFState> p_state, c
 	return r_image;
 }
 
-void GLTFDocument::_parse_image_save_image(Ref<GLTFState> p_state, const Vector<uint8_t> &p_bytes, const String &p_file_extension, int p_index, Ref<Image> p_image) {
+void GLTFDocument::_parse_image_save_image(Ref<GLTFState> p_state, const Vector<uint8_t> &p_bytes, const String &p_resource_uri, const String &p_file_extension, int p_index, Ref<Image> p_image) {
 	GLTFState::GLTFHandleBinary handling = GLTFState::GLTFHandleBinary(p_state->handle_binary_image);
 	if (p_image->is_empty() || handling == GLTFState::GLTFHandleBinary::HANDLE_BINARY_DISCARD_TEXTURES) {
 		p_state->images.push_back(Ref<Texture2D>());
@@ -3948,33 +3948,45 @@ void GLTFDocument::_parse_image_save_image(Ref<GLTFState> p_state, const Vector<
 				WARN_PRINT(vformat("glTF: Image index '%d' did not have a name. It will be automatically given a name based on its index.", p_index));
 				p_image->set_name(itos(p_index));
 			}
-			bool must_import = true;
+			bool must_write = true; // If the resource does not exist on the disk within res:// directory write it.
+			bool must_import = true; // Trigger import.
 			Vector<uint8_t> img_data = p_image->get_data();
 			Dictionary generator_parameters;
-			String file_path = p_state->get_extract_path().path_join(p_state->get_extract_prefix() + "_" + p_image->get_name());
-			file_path += p_file_extension.is_empty() ? ".png" : p_file_extension;
-			if (FileAccess::exists(file_path + ".import")) {
-				Ref<ConfigFile> config;
-				config.instantiate();
-				config->load(file_path + ".import");
-				if (config->has_section_key("remap", "generator_parameters")) {
-					generator_parameters = (Dictionary)config->get_value("remap", "generator_parameters");
-				}
-				if (!generator_parameters.has("md5")) {
-					must_import = false; // Didn't come from a gltf document; don't overwrite.
+			String file_path;
+			if (!p_resource_uri.is_empty()) {
+				file_path = p_resource_uri;
+				must_import = true;
+				must_write = !FileAccess::exists(file_path);
+			} else {
+				// Texture data has to be written to the res:// folder and imported.
+				file_path = p_state->get_extract_path().path_join(p_state->get_extract_prefix() + "_" + p_image->get_name());
+				file_path += p_file_extension.is_empty() ? ".png" : p_file_extension;
+				if (FileAccess::exists(file_path + ".import")) {
+					Ref<ConfigFile> config;
+					config.instantiate();
+					config->load(file_path + ".import");
+					if (config->has_section_key("remap", "generator_parameters")) {
+						generator_parameters = (Dictionary)config->get_value("remap", "generator_parameters");
+					}
+					if (!generator_parameters.has("md5")) {
+						must_write = false; // Didn't come from a gltf document; don't overwrite.
+						must_import = false; // And don't import.
+					}
 				}
 			}
-			if (must_import) {
+
+			if (must_write) {
 				String existing_md5 = generator_parameters["md5"];
 				unsigned char md5_hash[16];
 				CryptoCore::md5(img_data.ptr(), img_data.size(), md5_hash);
 				String new_md5 = String::hex_encode_buffer(md5_hash, 16);
 				generator_parameters["md5"] = new_md5;
 				if (new_md5 == existing_md5) {
+					must_write = false;
 					must_import = false;
 				}
 			}
-			if (must_import) {
+			if (must_write) {
 				Error err = OK;
 				if (p_file_extension.is_empty()) {
 					// If a file extension was not specified, save the image data to a PNG file.
@@ -3987,10 +3999,13 @@ void GLTFDocument::_parse_image_save_image(Ref<GLTFState> p_state, const Vector<
 					file->store_buffer(p_bytes);
 					file->close();
 				}
+			}
+			if (must_import) {
 				// ResourceLoader::import will crash if not is_editor_hint(), so this case is protected above and will fall through to uncompressed.
 				HashMap<StringName, Variant> custom_options;
 				custom_options[SNAME("mipmaps/generate")] = true;
 				// Will only use project settings defaults if custom_importer is empty.
+
 				EditorFileSystem::get_singleton()->update_file(file_path);
 				EditorFileSystem::get_singleton()->reimport_append(file_path, custom_options, String(), generator_parameters);
 			}
@@ -4000,7 +4015,7 @@ void GLTFDocument::_parse_image_save_image(Ref<GLTFState> p_state, const Vector<
 				p_state->source_images.push_back(saved_image->get_image());
 				return;
 			} else {
-				WARN_PRINT(vformat("glTF: Image index '%d' with the name '%s' couldn't be imported. It will be loaded directly instead, uncompressed.", p_index, p_image->get_name()));
+				WARN_PRINT(vformat("glTF: Image index '%d' with the name '%s' resolved to %s couldn't be imported. It will be loaded directly instead, uncompressed.", p_index, p_image->get_name(), file_path));
 			}
 		}
 	}
@@ -4068,6 +4083,9 @@ Error GLTFDocument::_parse_images(Ref<GLTFState> p_state, const String &p_base_p
 		while (used_names.has(image_name)) {
 			image_name += "_" + itos(i);
 		}
+
+		String resource_uri;
+
 		used_names.insert(image_name);
 		// Load the image data. If we get a byte array, store here for later.
 		Vector<uint8_t> data;
@@ -4085,14 +4103,14 @@ Error GLTFDocument::_parse_images(Ref<GLTFState> p_state, const String &p_base_p
 				ERR_FAIL_COND_V(p_base_path.is_empty(), ERR_INVALID_PARAMETER);
 				uri = uri.uri_decode();
 				uri = p_base_path.path_join(uri).replace("\\", "/"); // Fix for Windows.
-				// If the image is in the .godot/imported directory, we can't use ResourceLoader.
-				if (!p_base_path.begins_with("res://.godot/imported")) {
-					// ResourceLoader will rely on the file extension to use the relevant loader.
-					// The spec says that if mimeType is defined, it should take precedence (e.g.
-					// there could be a `.png` image which is actually JPEG), but there's no easy
-					// API for that in Godot, so we'd have to load as a buffer (i.e. embedded in
-					// the material), so we only do that only as fallback.
-					Ref<Texture2D> texture = ResourceLoader::load(uri, "Texture2D");
+				resource_uri = uri.simplify_path();
+				// ResourceLoader will rely on the file extension to use the relevant loader.
+				// The spec says that if mimeType is defined, it should take precedence (e.g.
+				// there could be a `.png` image which is actually JPEG), but there's no easy
+				// API for that in Godot, so we'd have to load as a buffer (i.e. embedded in
+				// the material), so we only do that only as fallback.
+				if (ResourceLoader::exists(resource_uri)) {
+					Ref<Texture2D> texture = ResourceLoader::load(resource_uri, "Texture2D");
 					if (texture.is_valid()) {
 						p_state->images.push_back(texture);
 						p_state->source_images.push_back(texture->get_image());
@@ -4103,13 +4121,13 @@ Error GLTFDocument::_parse_images(Ref<GLTFState> p_state, const String &p_base_p
 				// If the mimeType does not match with the file extension, either it should be
 				// specified in the file, or the GLTFDocumentExtension should handle it.
 				if (mime_type.is_empty()) {
-					mime_type = "image/" + uri.get_extension();
+					mime_type = "image/" + resource_uri.get_extension();
 				}
 				// Fallback to loading as byte array. This enables us to support the
 				// spec's requirement that we honor mimetype regardless of file URI.
-				data = FileAccess::get_file_as_bytes(uri);
+				data = FileAccess::get_file_as_bytes(resource_uri);
 				if (data.size() == 0) {
-					WARN_PRINT(vformat("glTF: Image index '%d' couldn't be loaded as a buffer of MIME type '%s' from URI: %s because there was no data to load. Skipping it.", i, mime_type, uri));
+					WARN_PRINT(vformat("glTF: Image index '%d' couldn't be loaded as a buffer of MIME type '%s' from URI: %s because there was no data to load. Skipping it.", i, mime_type, resource_uri));
 					p_state->images.push_back(Ref<Texture2D>()); // Placeholder to keep count.
 					p_state->source_images.push_back(Ref<Image>());
 					continue;
@@ -4139,7 +4157,7 @@ Error GLTFDocument::_parse_images(Ref<GLTFState> p_state, const String &p_base_p
 		String file_extension;
 		Ref<Image> img = _parse_image_bytes_into_image(p_state, data, mime_type, i, file_extension);
 		img->set_name(image_name);
-		_parse_image_save_image(p_state, data, file_extension, i, img);
+		_parse_image_save_image(p_state, data, resource_uri, file_extension, i, img);
 	}
 
 	print_verbose("glTF: Total images: " + itos(p_state->images.size()));

modules/gltf/gltf_document.h

@@ -190,7 +190,7 @@ class GLTFDocument : public Resource {
 	Error _serialize_images(Ref<GLTFState> p_state);
 	Error _serialize_lights(Ref<GLTFState> p_state);
 	Ref<Image> _parse_image_bytes_into_image(Ref<GLTFState> p_state, const Vector<uint8_t> &p_bytes, const String &p_mime_type, int p_index, String &r_file_extension);
-	void _parse_image_save_image(Ref<GLTFState> p_state, const Vector<uint8_t> &p_bytes, const String &p_file_extension, int p_index, Ref<Image> p_image);
+	void _parse_image_save_image(Ref<GLTFState> p_state, const Vector<uint8_t> &p_bytes, const String &p_resource_uri, const String &p_file_extension, int p_index, Ref<Image> p_image);
 	Error _parse_images(Ref<GLTFState> p_state, const String &p_base_path);
 	Error _parse_textures(Ref<GLTFState> p_state);
 	Error _parse_texture_samplers(Ref<GLTFState> p_state);