add the ability to invoke QML functions (#56)

Author: prototypicalpro

.github/workflows/build.yml

@@ -13,8 +13,9 @@ jobs:
       matrix:
         os: [ubuntu-latest, ubuntu-20.04, windows-latest, macos-latest, macos-10.15]
         build_type: ['Release', 'Debug']
-        shared_libs: ['ON', 'OFF']
         qt_version: [[5, 12, 12], [5, 15, 2], [6, 2, 3]]
+        shared_libs: ['ON', 'OFF']
+        tests: ['ON', 'OFF']
         include:
           - os: ubuntu-latest
             triplet: 'x64-linux'
@@ -35,7 +36,12 @@ jobs:
             triplet: 'x64-osx'
             cmake_flags: ''
         exclude:
-        # Disabled until https://github.com/sgieseking/anyrpc/pull/43 is in place
+          # tests won't build with shared libs due to private symbols
+          - shared_libs: 'ON'
+            tests: 'ON'
+          - shared_libs: 'OFF'
+            tests: 'OFF'
+          # Disabled until https://github.com/sgieseking/anyrpc/pull/43 is in place
           - os: windows-latest
             shared_libs: 'ON'
     steps:
@@ -65,7 +71,7 @@ jobs:
       - name: "Configure"
         run: |
           mkdir build
-          cmake -B build -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DSPIX_BUILD_TESTS=ON -DSPIX_BUILD_EXAMPLES=ON ${{ matrix.cmake_flags}} -DBUILD_SHARED_LIBS=${{ matrix.shared_libs }} -DSPIX_QT_MAJOR=${{ matrix.qt_version[0] }} .
+          cmake -B build -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DSPIX_BUILD_TESTS=${{ matrix.tests }} -DSPIX_BUILD_EXAMPLES=ON ${{ matrix.cmake_flags }} -DBUILD_SHARED_LIBS=${{ matrix.shared_libs }} -DSPIX_QT_MAJOR=${{ matrix.qt_version[0] }} .
       
       - name: "Print cmake compile commands"
         if: ${{ !contains(matrix.os, 'windows') }}

CMakeLists.txt

@@ -6,7 +6,7 @@ option(SPIX_BUILD_TESTS "Build Spix unit tests." OFF)
 set(SPIX_QT_MAJOR "6" CACHE STRING "Major Qt version to build Spix against")
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_LIST_DIR}/cmake/modules")
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
 
 # Hide symbols unless explicitly flagged with SPIX_EXPORT
 set(CMAKE_CXX_VISIBILITY_PRESET hidden)

README.md

@@ -54,6 +54,7 @@ generate and update screenshots for your documentation.
 * Enter text
 * Check existence and visibility of items
 * Get property values of items (text, position, color, ...)
+* Invoke a method on an object
 * Take and save a screenshot
 * Quit the app
 * Remote control, also of embedded devices / iOS
@@ -146,7 +147,7 @@ resultText = s.getStringProperty("root/results", "text")
 You can also use the XMLRPC client to list the available methods. The complete list of methods are also available in the [source](lib/src/AnyRpcServer.cpp).
 ```python
 print(s.system.listMethods())
-# ['command', 'enterKey', 'existsAndVisible', 'getBoundingBox', 'getErrors', 'getStringProperty', 'inputText', 'mouseBeginDrag', 'mouseClick', 'mouseDropUrls', 'mouseEndDrag', 'quit', 'setStringProperty', 'system.listMethods', 'system.methodHelp', 'takeScreenshot', 'wait']
+# ['command', 'enterKey', 'existsAndVisible', 'getBoundingBox', 'getErrors', 'getStringProperty', 'inputText', 'invokeMethod', 'mouseBeginDrag', 'mouseClick', 'mouseDropUrls', 'mouseEndDrag', 'quit', 'setStringProperty', 'system.listMethods', 'system.methodHelp', 'takeScreenshot', 'wait']
 print(s.system.methodHelp('mouseClick'))
 # Click on the object at the given path
 ```
@@ -168,6 +169,67 @@ More specifically, Spix's matching processes works as follows:
 * `<root>` matches a top-level [`QQuickWindow`](https://doc-snapshots.qt.io/qt6-dev/qquickwindow.html) whose `objectName` (or `id` if `objectName` is empty) matches the specified string. Top-level windows are enumerated by [`QGuiApplication::topLevelWindows`](https://doc.qt.io/qt-6/qguiapplication.html#topLevelWindows).
 * `<child>` matches the first child object whose `objectName` (or `id` if `objectName` is empty) matches the specified string using a recursive search of all children and subchildren of the root. This process repeats for every subsequent child path entry.
 
+### Invoking QML methods
+
+Spix can directly invoke both internal and custom methods in QML objects: this can be a handy way to automate interactions that Spix doesn't support normally. For example, we can control the cursor in a `TextArea` by calling [`TextArea.select`](https://doc-snapshots.qt.io/qt6-6.2/qml-qtquick-textedit.html#select-method):
+```qml
+TextArea {
+    id: textArea
+}
+```
+```python
+# select characters 100-200
+s.invokeMethod("root/textArea", "select", [100, 200])
+```
+
+In addition, you can use custom functions in the QML to implement more complicated interactions, and have Spix interact with the function:
+```qml
+TextArea {
+    id: textArea
+    function customFunction(arg1, arg2) {
+        // insert QML interactions here
+        return {'key1': true, 'key2': false}
+    }
+}
+```
+```python
+# invoke the custom function
+result = s.invokeMethod("root/textArea", "customFunction", ['a string', 34])
+# prints {'key1': True, 'key2': False}
+print(result)
+```
+
+Spix supports the following types as arguments/return values:
+| Python Type       | XMLRPC Type          | QML Type(s)     | JavaScript Type(s)| Notes                                            |
+|-------------------|----------------------|-----------------|-------------------|--------------------------------------------------|
+| int               | \<int\>              | int             | number            | Values over/under int max are upcasted to double |
+| bool              | \<boolean\>          | bool            | boolean           |                                                  |
+| str               | \<string\>           | string          | string            |                                                  |
+| float             | \<double\>           | double, real    | number            | Defaults to double                               |
+| datetime.datetime | \<dateTime.iso8601\> | date            | Date              | No timezone support (always uses local timezone) |
+| dict              | \<struct\>           | var             | object            | String keys only                                 |
+| list              | \<array\>            | var             | Array             |                                                  |
+| None              | no type              | null, undefined | object, undefined | Defaults to null                                 |                              |
+
+In general Spix will attempt to coerce the arguments and return value to the correct types to match the method being invoked. Valid conversion are listed under the [`QVariant` docs](https://doc.qt.io/qt-5/qvariant.html#canConvert). If Spix cannot find a valid conversion it will generate an error.
+```qml
+Item {
+    id: item
+    function test(arg1: bool) {
+        ...
+    }
+}
+```
+```python
+# ok
+s.invokeMethod("root/item", "test", [False])
+
+# argument will implicitly be converted to a boolean (True) to match the declaration type
+s.invokeMethod("root/item", "test", [34])
+
+# no conversion from object to boolean, so an error is thrown
+s.invokeMethod("root/item", "test", [{}])
+```
 
 ## Two modes of operation
 In general, Spix can be used in two ways, which are different in how events are generated and sent
@@ -176,7 +238,7 @@ to your application:
 ### Generate Qt events directly
 You can use Spix to directly create Qt events, either from C++ as a unit test, or from
 an external script via the network through RPC. Since the Qt events are generated directly inside the
-app, and do not come from the system, the mouse coursor will not actually move and interaction
+app, and do not come from the system, the mouse cursor will not actually move and interaction
 with other applications is limited. On the plus side, this mechanism is independent from
 the system your app is running on and can easily be used to control software on an embedded
 device via the network (RPC).

examples/Basic/CMakeLists.txt

@@ -1,7 +1,7 @@
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 set(CMAKE_AUTOMOC ON)
 set(CMAKE_AUTORCC ON)
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 find_package(Qt${SPIX_QT_MAJOR} COMPONENTS Core Quick REQUIRED)

examples/BasicStandalone/CMakeLists.txt

@@ -8,7 +8,7 @@ set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/../../cmake/modules")
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 set(CMAKE_AUTOMOC ON)
 set(CMAKE_AUTORCC ON)
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 #

examples/GTest/CMakeLists.txt

@@ -1,7 +1,7 @@
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 set(CMAKE_AUTOMOC ON)
 set(CMAKE_AUTORCC ON)
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 set(SPIX_QT_MAJOR "6" CACHE STRING "Major Qt version to build Spix against")

examples/ListGridView/CMakeLists.txt

@@ -1,7 +1,7 @@
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 set(CMAKE_AUTOMOC ON)
 set(CMAKE_AUTORCC ON)
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 set(SPIX_QT_MAJOR "6" CACHE STRING "Major Qt version to build Spix against")

examples/RemoteCtrl/CMakeLists.txt

@@ -1,7 +1,7 @@
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 set(CMAKE_AUTOMOC ON)
 set(CMAKE_AUTORCC ON)
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 set(SPIX_QT_MAJOR "6" CACHE STRING "Major Qt version to build Spix against")

examples/RepeaterLoader/CMakeLists.txt

@@ -1,7 +1,7 @@
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 set(CMAKE_AUTOMOC ON)
 set(CMAKE_AUTORCC ON)
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 set(SPIX_QT_MAJOR "6" CACHE STRING "Major Qt version to build Spix against")

lib/CMakeLists.txt

@@ -54,6 +54,8 @@ set(SOURCES
     src/Commands/GetTestStatus.h
     src/Commands/InputText.cpp
     src/Commands/InputText.h
+    src/Commands/InvokeMethod.cpp
+    src/Commands/InvokeMethod.h
     src/Commands/Quit.cpp
     src/Commands/Quit.h
     src/Commands/Screenshot.cpp
@@ -92,8 +94,10 @@ set(SOURCES
     src/Scene/Qt/QtScene.cpp
     src/Scene/Qt/QtScene.h
     src/Scene/Scene.h
-
+    
+    src/Utils/AnyRpcUtils.cpp
     src/Utils/AnyRpcUtils.h
+    src/Utils/AnyRpcFunction.h
     src/Utils/DebugDump.cpp
     src/Utils/DebugDump.h
     src/Utils/QtEventRecorder.cpp

lib/include/Spix/Data/Variant.h

@@ -0,0 +1,61 @@
+/***
+ * Copyright (C) Noah Koontz. All rights reserved.
+ * Licensed under the MIT license.
+ * See LICENSE.txt file in the project root for full license information.
+ ****/
+
+#pragma once
+
+#include <chrono>
+#include <map>
+#include <string>
+#include <variant>
+#include <vector>
+
+#include <Spix/spix_export.h>
+
+namespace spix {
+
+struct Variant;
+
+namespace {
+using VariantBaseType = std::variant<std::nullptr_t, bool, long long, unsigned long long, double, std::string,
+    std::chrono::time_point<std::chrono::system_clock>, std::vector<Variant>, std::map<std::string, Variant>>;
+}
+
+/**
+ * Utility union type that contains a number of RPC-able types, including a list of itself and a map of {std::string:
+ * itself}. Inherits from std::variant. This variant is used to abstract between RPC union types (ex anyrpc::Value) the
+ * scene union types (ex. QVariant).
+ *
+ * NOTE: std::visit is broken for this variant for GCC <= 11.2 and clang <= 14.0, see
+ * http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2162r0.html for more info. Instead, type switches must be
+ * created manually using the index() method and the TypeIndex enum.
+ */
+struct SPIX_EXPORT Variant : VariantBaseType {
+    using ListType = std::vector<Variant>;
+    using MapType = std::map<std::string, Variant>;
+    using VariantType = VariantBaseType;
+    using VariantBaseType::variant;
+    VariantBaseType const& base() const { return *this; }
+    VariantBaseType& base() { return *this; }
+
+    enum TypeIndex
+    {
+        Nullptr = 0,
+        Bool,
+        Int,
+        Uint,
+        Double,
+        String,
+        Time,
+        List,
+        Map,
+        TypeIndexCount
+    };
+};
+
+static_assert(
+    Variant::TypeIndexCount == std::variant_size_v<VariantBaseType>, "Variant enum does not cover all Variant types");
+
+} // namespace spix

lib/include/Spix/TestServer.h

@@ -13,6 +13,7 @@
 
 #include <Spix/Data/Geometry.h>
 #include <Spix/Data/ItemPath.h>
+#include <Spix/Data/Variant.h>
 #include <Spix/Events/Identifiers.h>
 
 #include <Spix/spix_export.h>
@@ -57,6 +58,7 @@ class SPIX_EXPORT TestServer {
 
     std::string getStringProperty(ItemPath path, std::string propertyName);
     void setStringProperty(ItemPath path, std::string propertyName, std::string propertyValue);
+    Variant invokeMethod(ItemPath path, std::string method, std::vector<Variant> args);
     Rect getBoundingBox(ItemPath path);
     bool existsAndVisible(ItemPath path);
     std::vector<std::string> getErrors();

lib/src/AnyRpcServer.cpp

@@ -5,8 +5,8 @@
  ****/
 
 #include <Spix/AnyRpcServer.h>
-#include <Utils/AnyRpcUtils.h>
-#include <anyrpc/anyrpc.h>
+#include <Spix/Data/Variant.h>
+#include <Utils/AnyRpcFunction.h>
 #include <atomic>
 
 namespace spix {
@@ -70,6 +70,12 @@ AnyRpcServer::AnyRpcServer(int anyrpcPort)
             setStringProperty(std::move(path), std::move(property), std::move(value));
         });
 
+    utils::AddFunctionToAnyRpc<Variant(std::string, std::string, std::vector<Variant>)>(methodManager, "invokeMethod",
+        "Invoke a method on a QML object | invokeMethod(string path, string method, any[] args)",
+        [this](std::string path, std::string method, std::vector<Variant> args) {
+            return invokeMethod(std::move(path), std::move(method), std::move(args));
+        });
+
     utils::AddFunctionToAnyRpc<std::vector<double>(std::string)>(methodManager, "getBoundingBox",
         "Return the bounding box of an item in screen coordinates | getBoundingBox(string path) : (doubles) "
         "[topLeft.x, topLeft.y , width, height]",

lib/src/Commands/InvokeMethod.cpp

@@ -0,0 +1,39 @@
+/***
+ * Copyright (C) Falko Axmann. All rights reserved.
+ * Licensed under the MIT license.
+ * See LICENSE.txt file in the project root for full license information.
+ ****/
+
+#include "InvokeMethod.h"
+
+#include <Scene/Scene.h>
+
+namespace spix {
+namespace cmd {
+
+InvokeMethod::InvokeMethod(ItemPath path, std::string method, std::vector<Variant> args, std::promise<Variant> promise)
+: m_path(std::move(path))
+, m_method(std::move(method))
+, m_args(std::move(args))
+, m_promise(std::move(promise))
+{
+}
+
+void InvokeMethod::execute(CommandEnvironment& env)
+{
+    auto item = env.scene().itemAtPath(m_path);
+
+    if (item) {
+        Variant ret;
+        bool success = item->invokeMethod(m_method, m_args, ret);
+        if (!success)
+            env.state().reportError("InvokeMethod: Failed to invoke method: " + m_method);
+        m_promise.set_value(ret);
+    } else {
+        env.state().reportError("InvokeMethod: Item not found: " + m_path.string());
+        m_promise.set_value(Variant(nullptr));
+    }
+}
+
+} // namespace cmd
+} // namespace spix

lib/src/Commands/InvokeMethod.h

@@ -0,0 +1,34 @@
+/***
+ * Copyright (C) Falko Axmann. All rights reserved.
+ * Licensed under the MIT license.
+ * See LICENSE.txt file in the project root for full license information.
+ ****/
+
+#pragma once
+
+#include <Spix/spix_export.h>
+
+#include "Command.h"
+#include <Spix/Data/ItemPath.h>
+#include <Spix/Data/Variant.h>
+
+#include <future>
+
+namespace spix {
+namespace cmd {
+
+class SPIX_EXPORT InvokeMethod : public Command {
+public:
+    InvokeMethod(ItemPath path, std::string method, std::vector<Variant> args, std::promise<Variant> promise);
+
+    void execute(CommandEnvironment& env) override;
+
+private:
+    ItemPath m_path;
+    std::string m_method;
+    std::vector<Variant> m_args;
+    std::promise<Variant> m_promise;
+};
+
+} // namespace cmd
+} // namespace spix