[FL-3097] fbt, faploader: minimal app module implementation (#2420)

* fbt, faploader: minimal app module implementation
* faploader, libs: moved API hashtable core to flipper_application
* example: compound api
* lib: flipper_application: naming fixes, doxygen comments
* fbt: changed `requires` manifest field behavior for app extensions
* examples: refactored plugin apps; faploader: changed new API naming; fbt: changed PLUGIN app type meaning
* loader: dropped support for debug apps & plugin menus
* moved applications/plugins -> applications/external
* Restored x bit on chiplist_convert.py
* git: fixed free-dap submodule path
* pvs: updated submodule paths
* examples: example_advanced_plugins.c: removed potential memory leak on errors
* examples: example_plugins: refined requires
* fbt: not deploying app modules for debug/sample apps; extra validation for .PLUGIN-type apps
* apps: removed cdefines for external apps
* fbt: moved ext app path definition
* fbt: reworked fap_dist handling; f18: synced api_symbols.csv
* fbt: removed resources_paths for extapps
* scripts: reworked storage
* scripts: reworked runfap.py & selfupdate.py to use new api
* wip: fal runner
* fbt: moved file packaging into separate module
* scripts: storage: fixes
* scripts: storage: minor fixes for new api
* fbt: changed internal artifact storage details for external apps
* scripts: storage: additional fixes and better error reporting; examples: using APP_DATA_PATH()
* fbt, scripts: reworked launch_app to deploy plugins; moved old runfap.py to distfap.py
* fbt: extra check for plugins descriptors
* fbt: additional checks in emitter
* fbt: better info message on SDK rebuild
* scripts: removed requirements.txt
* loader: removed remnants of plugins & debug menus
* post-review fixes

.github/CODEOWNERS

@@ -22,8 +22,8 @@
 /applications/main/subghz/ @skotopes @DrZlo13 @hedger @Skorpionm
 /applications/main/u2f/ @skotopes @DrZlo13 @hedger @nminaylov
 
-/applications/plugins/bt_hid_app/ @skotopes @DrZlo13 @hedger @gornekich
-/applications/plugins/picopass/ @skotopes @DrZlo13 @hedger @gornekich
+/applications/external/bt_hid_app/ @skotopes @DrZlo13 @hedger @gornekich
+/applications/external/picopass/ @skotopes @DrZlo13 @hedger @gornekich
 
 /applications/services/bt/ @skotopes @DrZlo13 @hedger @gornekich
 /applications/services/cli/ @skotopes @DrZlo13 @hedger @nminaylov

.gitmodules

@@ -28,6 +28,6 @@
 [submodule "lib/cxxheaderparser"]
 	path = lib/cxxheaderparser
 	url = https://github.com/robotpy/cxxheaderparser.git
-[submodule "applications/plugins/dap_link/lib/free-dap"]
-	path = applications/plugins/dap_link/lib/free-dap
+[submodule "applications/external/dap_link/lib/free-dap"]
+	path = applications/external/dap_link/lib/free-dap
 	url = https://github.com/ataradov/free-dap.git

.pvsoptions

@@ -1 +1 @@
---ignore-ccache -C gccarm --rules-config .pvsconfig -e lib/fatfs -e lib/fnv1a-hash -e lib/FreeRTOS-Kernel -e lib/heatshrink -e lib/libusb_stm32 -e lib/littlefs -e lib/mbedtls -e lib/micro-ecc -e lib/microtar -e lib/mlib -e lib/qrcode -e lib/ST25RFAL002 -e lib/STM32CubeWB -e lib/u8g2 -e lib/nanopb -e */arm-none-eabi/* -e applications/plugins/dap_link/lib/free-dap
+--ignore-ccache -C gccarm --rules-config .pvsconfig -e lib/fatfs -e lib/fnv1a-hash -e lib/FreeRTOS-Kernel -e lib/heatshrink -e lib/libusb_stm32 -e lib/littlefs -e lib/mbedtls -e lib/micro-ecc -e lib/microtar -e lib/mlib -e lib/qrcode -e lib/ST25RFAL002 -e lib/STM32CubeWB -e lib/u8g2 -e lib/nanopb -e */arm-none-eabi/* -e applications/external/dap_link/lib/free-dap

SConstruct

@@ -139,34 +139,33 @@ if GetOption("fullenv") or any(
 basic_dist = distenv.DistCommand("fw_dist", distenv["DIST_DEPENDS"])
 distenv.Default(basic_dist)
 
-dist_dir = distenv.GetProjetDirName()
+dist_dir_name = distenv.GetProjetDirName()
+dist_dir = distenv.Dir(f"#/dist/{dist_dir_name}")
+external_apps_artifacts = firmware_env["FW_EXTAPPS"]
+external_app_list = external_apps_artifacts.application_map.values()
+
 fap_dist = [
     distenv.Install(
-        distenv.Dir(f"#/dist/{dist_dir}/apps/debug_elf"),
-        list(
-            app_artifact.debug
-            for app_artifact in firmware_env["FW_EXTAPPS"].applications.values()
-        ),
+        dist_dir.Dir("debug_elf"),
+        list(app_artifact.debug for app_artifact in external_app_list),
     ),
     *(
         distenv.Install(
-            f"#/dist/{dist_dir}/apps/{app_artifact.app.fap_category}",
-            app_artifact.compact[0],
+            dist_dir.File(dist_entry[1]).dir,
+            app_artifact.compact,
         )
-        for app_artifact in firmware_env["FW_EXTAPPS"].applications.values()
+        for app_artifact in external_app_list
+        for dist_entry in app_artifact.dist_entries
     ),
 ]
 Depends(
     fap_dist,
-    list(
-        app_artifact.validator
-        for app_artifact in firmware_env["FW_EXTAPPS"].applications.values()
-    ),
+    list(app_artifact.validator for app_artifact in external_app_list),
 )
 Alias("fap_dist", fap_dist)
 # distenv.Default(fap_dist)
 
-distenv.Depends(firmware_env["FW_RESOURCES"], firmware_env["FW_EXTAPPS"].resources_dist)
+distenv.Depends(firmware_env["FW_RESOURCES"], external_apps_artifacts.resources_dist)
 
 # Copy all faps to device
 

applications/examples/application.fam

@@ -1,3 +1,4 @@
+# Placeholder
 App(
     appid="example_apps",
     name="Example apps bundle",

applications/examples/example_plugins/application.fam

@@ -0,0 +1,31 @@
+App(
+    appid="example_plugins",
+    name="Example: App w/plugin",
+    apptype=FlipperAppType.EXTERNAL,
+    entry_point="example_plugins_app",
+    stack_size=2 * 1024,
+    fap_category="Examples",
+)
+
+App(
+    appid="example_plugins_multi",
+    name="Example: App w/plugins",
+    apptype=FlipperAppType.EXTERNAL,
+    entry_point="example_plugins_multi_app",
+    stack_size=2 * 1024,
+    fap_category="Examples",
+)
+
+App(
+    appid="example_plugin1",
+    apptype=FlipperAppType.PLUGIN,
+    entry_point="example_plugin1_ep",
+    requires=["example_plugins", "example_plugins_multi"],
+)
+
+App(
+    appid="example_plugin2",
+    apptype=FlipperAppType.PLUGIN,
+    entry_point="example_plugin2_ep",
+    requires=["example_plugins_multi"],
+)

applications/examples/example_plugins/example_plugins.c

@@ -0,0 +1,70 @@
+/* 
+ * An example of a plugin host application.
+ * Loads a single plugin and calls its methods.
+ */
+
+#include "plugin_interface.h"
+
+#include <furi.h>
+
+#include <flipper_application/flipper_application.h>
+#include <loader/firmware_api/firmware_api.h>
+#include <storage/storage.h>
+
+#define TAG "example_plugins"
+
+int32_t example_plugins_app(void* p) {
+    UNUSED(p);
+
+    FURI_LOG_I(TAG, "Starting");
+
+    Storage* storage = furi_record_open(RECORD_STORAGE);
+
+    FlipperApplication* app = flipper_application_alloc(storage, firmware_api_interface);
+
+    do {
+        FlipperApplicationPreloadStatus preload_res =
+            flipper_application_preload(app, APP_DATA_PATH("plugins/example_plugin1.fal"));
+
+        if(preload_res != FlipperApplicationPreloadStatusSuccess) {
+            FURI_LOG_E(TAG, "Failed to preload plugin");
+            break;
+        }
+
+        if(!flipper_application_is_plugin(app)) {
+            FURI_LOG_E(TAG, "Plugin file is not a library");
+            break;
+        }
+
+        FlipperApplicationLoadStatus load_status = flipper_application_map_to_memory(app);
+        if(load_status != FlipperApplicationLoadStatusSuccess) {
+            FURI_LOG_E(TAG, "Failed to load plugin file");
+            break;
+        }
+
+        const FlipperAppPluginDescriptor* app_descriptor =
+            flipper_application_plugin_get_descriptor(app);
+
+        FURI_LOG_I(
+            TAG,
+            "Loaded plugin for appid '%s', API %lu",
+            app_descriptor->appid,
+            app_descriptor->ep_api_version);
+
+        furi_check(app_descriptor->ep_api_version == PLUGIN_API_VERSION);
+        furi_check(strcmp(app_descriptor->appid, PLUGIN_APP_ID) == 0);
+
+        const ExamplePlugin* plugin = app_descriptor->entry_point;
+
+        FURI_LOG_I(TAG, "Plugin name: %s", plugin->name);
+        FURI_LOG_I(TAG, "Plugin method1: %d", plugin->method1());
+        FURI_LOG_I(TAG, "Plugin method2(7,8): %d", plugin->method2(7, 8));
+        FURI_LOG_I(TAG, "Plugin method2(1337,228): %d", plugin->method2(1337, 228));
+    } while(false);
+    flipper_application_free(app);
+
+    furi_record_close(RECORD_STORAGE);
+    FURI_LOG_I(TAG, "Goodbye!");
+
+    return 0;
+}

applications/examples/example_plugins/example_plugins_multi.c

@@ -0,0 +1,43 @@
+/*
+ * An example of an advanced plugin host application.
+ * It uses PluginManager to load all plugins from a directory
+ */
+
+#include "plugin_interface.h"
+
+#include <flipper_application/flipper_application.h>
+#include <flipper_application/plugins/plugin_manager.h>
+#include <loader/firmware_api/firmware_api.h>
+
+#include <furi.h>
+
+#define TAG "example_plugins"
+
+int32_t example_plugins_multi_app(void* p) {
+    UNUSED(p);
+
+    FURI_LOG_I(TAG, "Starting");
+
+    PluginManager* manager =
+        plugin_manager_alloc(PLUGIN_APP_ID, PLUGIN_API_VERSION, firmware_api_interface);
+
+    if(plugin_manager_load_all(manager, APP_DATA_PATH("plugins")) != PluginManagerErrorNone) {
+        FURI_LOG_E(TAG, "Failed to load all libs");
+        return 0;
+    }
+
+    uint32_t plugin_count = plugin_manager_get_count(manager);
+    FURI_LOG_I(TAG, "Loaded %lu plugin(s)", plugin_count);
+
+    for(uint32_t i = 0; i < plugin_count; i++) {
+        const ExamplePlugin* plugin = plugin_manager_get_ep(manager, i);
+        FURI_LOG_I(TAG, "plugin name: %s", plugin->name);
+        FURI_LOG_I(TAG, "plugin method1: %d", plugin->method1());
+        FURI_LOG_I(TAG, "plugin method2(7,8): %d", plugin->method2(7, 8));
+    }
+
+    plugin_manager_free(manager);
+    FURI_LOG_I(TAG, "Goodbye!");
+
+    return 0;
+}

applications/examples/example_plugins/plugin1.c

@@ -0,0 +1,32 @@
+/* A simple plugin implementing example_plugins application's plugin interface */
+
+#include "plugin_interface.h"
+
+#include <flipper_application/flipper_application.h>
+
+static int example_plugin1_method1() {
+    return 42;
+}
+
+static int example_plugin1_method2(int arg1, int arg2) {
+    return arg1 + arg2;
+}
+
+/* Actual implementation of app<>plugin interface */
+static const ExamplePlugin example_plugin1 = {
+    .name = "Demo App Plugin 1",
+    .method1 = &example_plugin1_method1,
+    .method2 = &example_plugin1_method2,
+};
+
+/* Plugin descriptor to comply with basic plugin specification */
+static const FlipperAppPluginDescriptor example_plugin1_descriptor = {
+    .appid = PLUGIN_APP_ID,
+    .ep_api_version = PLUGIN_API_VERSION,
+    .entry_point = &example_plugin1,
+};
+
+/* Plugin entry point - must return a pointer to const descriptor  */
+const FlipperAppPluginDescriptor* example_plugin1_ep() {
+    return &example_plugin1_descriptor;
+}

applications/examples/example_plugins/plugin2.c

@@ -0,0 +1,32 @@
+/* Second plugin implementing example_plugins application's plugin interface */
+
+#include "plugin_interface.h"
+
+#include <flipper_application/flipper_application.h>
+
+static int example_plugin2_method1() {
+    return 1337;
+}
+
+static int example_plugin2_method2(int arg1, int arg2) {
+    return arg1 - arg2;
+}
+
+/* Actual implementation of app<>plugin interface */
+static const ExamplePlugin example_plugin2 = {
+    .name = "Demo App Plugin 2",
+    .method1 = &example_plugin2_method1,
+    .method2 = &example_plugin2_method2,
+};
+
+/* Plugin descriptor to comply with basic plugin specification */
+static const FlipperAppPluginDescriptor example_plugin2_descriptor = {
+    .appid = PLUGIN_APP_ID,
+    .ep_api_version = PLUGIN_API_VERSION,
+    .entry_point = &example_plugin2,
+};
+
+/* Plugin entry point - must return a pointer to const descriptor */
+const FlipperAppPluginDescriptor* example_plugin2_ep() {
+    return &example_plugin2_descriptor;
+}

applications/examples/example_plugins/plugin_interface.h

@@ -0,0 +1,12 @@
+#pragma once
+
+/* Common interface between a plugin and host applicaion */
+
+#define PLUGIN_APP_ID "example_plugins"
+#define PLUGIN_API_VERSION 1
+
+typedef struct {
+    const char* name;
+    int (*method1)();
+    int (*method2)(int, int);
+} ExamplePlugin;

applications/examples/example_plugins_advanced/app_api.c

@@ -0,0 +1,25 @@
+#include "app_api.h"
+
+/* Actual implementation of app's API and its private state */
+
+static uint32_t accumulator = 0;
+
+void app_api_accumulator_set(uint32_t value) {
+    accumulator = value;
+}
+
+uint32_t app_api_accumulator_get() {
+    return accumulator;
+}
+
+void app_api_accumulator_add(uint32_t value) {
+    accumulator += value;
+}
+
+void app_api_accumulator_sub(uint32_t value) {
+    accumulator -= value;
+}
+
+void app_api_accumulator_mul(uint32_t value) {
+    accumulator *= value;
+}

applications/examples/example_plugins_advanced/app_api.h

@@ -0,0 +1,25 @@
+#pragma once
+
+/* 
+ * This file contains an API that is internally implemented by the application
+ * It is also exposed to plugins to allow them to use the application's API.
+ */
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void app_api_accumulator_set(uint32_t value);
+
+uint32_t app_api_accumulator_get();
+
+void app_api_accumulator_add(uint32_t value);
+
+void app_api_accumulator_sub(uint32_t value);
+
+void app_api_accumulator_mul(uint32_t value);
+
+#ifdef __cplusplus
+}
+#endif

applications/examples/example_plugins_advanced/app_api_interface.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <flipper_application/api_hashtable/api_hashtable.h>
+
+/* 
+ * Resolver interface with private application's symbols. 
+ * Implementation is contained in app_api_table.c
+ */
+extern const ElfApiInterface* const application_api_interface;

applications/examples/example_plugins_advanced/app_api_table.cpp

@@ -0,0 +1,27 @@
+#include <flipper_application/api_hashtable/api_hashtable.h>
+#include <flipper_application/api_hashtable/compilesort.hpp>
+
+/* 
+ * This file contains an implementation of a symbol table 
+ * with private app's symbols. It is used by composite API resolver
+ * to load plugins that use internal application's APIs.
+ */
+#include "app_api_table_i.h"
+
+static_assert(!has_hash_collisions(app_api_table), "Detected API method hash collision!");
+
+constexpr HashtableApiInterface applicaton_hashtable_api_interface{
+    {
+        .api_version_major = 0,
+        .api_version_minor = 0,
+        /* generic resolver using pre-sorted array */
+        .resolver_callback = &elf_resolve_from_hashtable,
+    },
+    /* pointers to application's API table boundaries */
+    .table_cbegin = app_api_table.cbegin(),
+    .table_cend = app_api_table.cend(),
+};
+
+/* Casting to generic resolver to use in Composite API resolver */
+extern "C" const ElfApiInterface* const application_api_interface =
+    &applicaton_hashtable_api_interface;