Update feature/py3 with latest master

.github/workflows/main.yml

@@ -54,11 +54,13 @@ jobs:
         run: pip install pandas pytype toml
 
       - name: Install common dependencies for Python ${{matrix.python-version}}
-        run: pip install mock pytest-coverage pytest-mock
+        run: pip install future mock pytest-coverage pytest-mock
 
-      - name: Run Pytest tests for Python ${{matrix.python-version}}
+      - name: Run Pytest and get code coverage for Codecov
         run: >
-          pytest --cov scripts scripts/ -vv -rA
+          pytest
+          --cov=scripts --cov=ocaml/xcp-rrdd
+          scripts/ ocaml/xcp-rrdd -vv -rA
           --junitxml=.git/pytest${{matrix.python-version}}.xml
           --cov-report term-missing
           --cov-report xml:.git/coverage${{matrix.python-version}}.xml
@@ -89,6 +91,8 @@ jobs:
         run: ./pytype_reporter.py
         env:
           PR_NUMBER: ${{ github.event.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PYTYPE_REPORTER_DEBUG: True
 
       # Try to add pytype_report.py's summary file as a comment to the PR:
       # Documentation: https://github.com/marketplace/actions/add-pr-comment
@@ -134,7 +138,7 @@ jobs:
           sudo chown "$(id -u):$(id -g)" "${TMPDIR}" "${XDG_CACHE_HOME}"
           echo "TMPDIR=${TMPDIR}" >>"$GITHUB_ENV"
           echo "XDG_CACHE_HOME=${XDG_CACHE_HOME}" >>"$GITHUB_ENV"
-                        
+
       - name: Use ocaml
         uses: ocaml/setup-ocaml@v2
         with:
@@ -201,9 +205,6 @@ jobs:
       - name: quality-gate
         run: make quality-gate
 
-      - uses: reviewdog/action-actionlint@v1
-        name: GitHub Action linter from https://github.com/reviewdog/action-actionlint
-
       - name: pyflakes
         uses: reviewdog/action-pyflakes@master
         with:

.pre-commit-config.yaml

@@ -41,6 +41,7 @@ repos:
     -   id: pytype
         name: pytype
         entry: python3 pytype_reporter.py
+        pass_filenames: false
         types: [python]
         stages: [push]
         verbose: true

Makefile

@@ -62,7 +62,9 @@ test:
 	 trap "kill $${PSTREE_SLEEP_PID}" SIGINT SIGTERM EXIT; \
 	 timeout --foreground $(TEST_TIMEOUT2) \
 		 dune runtest --profile=$(PROFILE) --error-reporting=twice -j $(JOBS)
+ifneq ($(PY_TEST), NO)
 	dune build @runtest-python --profile=$(PROFILE)
+endif
 
 stresstest:
 	dune build @stresstest --profile=$(PROFILE) --no-buffer -j $(JOBS)
@@ -115,6 +117,10 @@ sdksanity: sdk
 	sed -i 's/FriendlyErrorNames.ResourceManager/null/g' ./_build/install/default/xapi/sdk/csharp/src/Failure.cs
 	cd _build/install/default/xapi/sdk/csharp/src && dotnet add package Newtonsoft.Json && dotnet build -f netstandard2.0
 
+.PHONY: sdk-build-java
+
+sdk-build-java: sdk
+	cd _build/install/default/xapi/sdk/java && mvn -f xen-api/pom.xml -B clean package install -Drevision=0.0
 
 python:
 	$(MAKE) -C scripts/examples/python build

doc/content/toolstack/high-level/daemons.md

@@ -34,6 +34,9 @@ xapi-storage-script
 message-switch
 : exchanges messages between the daemons on a host
 
+xapi-guard
+: forwards uefi and vtpm persistence calls from domains to xapi
+
 v6d
 : controls which features are enabled.
 
@@ -52,4 +55,4 @@ mpathalert
   if paths fail and need repair
 
 wsproxy
-: handles access to VM consoles
+: handles access to VM consoles

doc/content/xapi-guard/_index.md

@@ -0,0 +1,28 @@
++++
+title = "Xapi-guard"
+weight = 50
++++
+
+The `xapi-guard` daemon is the component in the xapi toolstack that is responsible for handling persistence requests from VMs (domains).
+Currently these are UEFI vars and vTPM updates.
+
+The code is in `ocaml/xapi-guard`.
+When the daemon managed only with UEFI updates it was called `varstored-guard`.
+Some files and package names still use the previous name.
+
+Principles
+----------
+1. Calls from domains must be limited in privilege to do certain API calls, and
+   to read and write from their corresponding VM in xapi's database only.
+2. Xenopsd is able to control xapi-guard through message switch, this access is
+   not limited.
+3. Listening to domain socket is restored whenever the daemon restarts to minimize disruption of running domains.
+
+
+Overview
+--------
+
+Xapi-guard forwards calls from domains to xapi to persist UEFI variables, and update vTPMs.
+To do this, it listens to 1 socket per service (varstored, or swtpm) per domain.
+To create these sockets before the domains are running, it listens to a message-switch socket.
+This socket listens to calls from xenopsd, which orchestrates the domain creation.

ocaml/forkexecd/lib/forkhelpers.ml

@@ -184,7 +184,7 @@ let safe_close_and_exec ?env stdin stdout stderr
         List.fold_left maybe_add_id_to_fd_map dest_named_fds predefined_fds
       in
 
-      let env = match env with Some e -> e | None -> default_path_env_pair in
+      let env = Option.value ~default:default_path_env_pair env in
       let syslog_stdout =
         match syslog_stdout with
         | NoSyslogging ->

ocaml/idl/datamodel_cluster_host.ml

@@ -31,7 +31,10 @@ let create =
     ()
 
 let destroy =
-  call ~name:"destroy" ~doc:"Remove a host from an existing cluster."
+  call ~name:"destroy"
+    ~doc:
+      "Remove the host from an existing cluster. This operation is allowed \
+       even if a cluster host is not enabled."
     ~params:
       [
         ( Ref _cluster_host
@@ -117,10 +120,8 @@ let t =
            ~default_value:(Some (VBool true))
            "Whether the cluster host has joined the cluster. Contrary to \
             enabled, a host that is not joined is not considered a member of \
-            the cluster, and hence no operations (e.g. enable/disable) can be \
-            performed on this host. This field can be altered by calling leave \
-            or destroy on a cluster host. It can also be set automatically if \
-            cluster stack believes that this node is not part of the cluster. "
+            the cluster, and hence enable and disable operations cannot be \
+            performed on this host."
        ; field ~qualifier:DynamicRO ~lifecycle:[] ~ty:Bool "live"
            ~default_value:(Some (VBool false))
            "Whether the underlying cluster stack thinks we are live. This \

ocaml/idl/datamodel_types.ml

@@ -334,6 +334,13 @@ let release_order_full =
     ; branding= "Citrix Hypervisor 8.2 Hotfix 2"
     ; release_date= Some "November 2020"
     }
+  ; {
+      code_name= Some "nile-preview"
+    ; version_major= 2
+    ; version_minor= 20
+    ; branding= "XenServer 8 Preview"
+    ; release_date= Some "August 2023"
+    }
   ]
 (* When you add a new release, use the version number of the latest release, "Unreleased"
    for the branding, and Some "" for the release date, until the actual values are finalised. *)

ocaml/idl/schematest.ml

@@ -3,7 +3,7 @@ let hash x = Digest.string x |> Digest.to_hex
 (* BEWARE: if this changes, check that schema has been bumped accordingly in
    ocaml/idl/datamodel_common.ml, usually schema_minor_vsn *)
 
-let last_known_schema_hash = "1e43ef93af9de55620fda75281e8a992"
+let last_known_schema_hash = "a99342e7a24557948df221c8da46ae71"
 
 let current_schema_hash : string =
   let open Datamodel_types in

ocaml/networkd/bin/network_server.ml

@@ -103,7 +103,7 @@ let set_dns_interface _dbg name =
  * constitutes adding a VLAN0 Linux device to strip those headers again.
  *)
 let need_enic_workaround () =
-  !backend_kind = Bridge && List.mem "enic" (Sysfs.list_drivers ())
+  !backend_kind = Bridge && List.mem "enic" (Sysfs.list_pci_drivers ())
 
 module Sriov = struct
   open S.Sriov

ocaml/networkd/lib/network_utils.ml

@@ -143,10 +143,10 @@ let fork_script ?on_error ?log script args =
   check_n_run ?on_error ?log fork_script_internal script args
 
 module Sysfs = struct
-  let list_drivers () =
+  let list_pci_drivers () =
     try Array.to_list (Sys.readdir "/sys/bus/pci/drivers")
     with _ ->
-      warn "Failed to obtain list of drivers from sysfs" ;
+      warn "Failed to obtain list of PCI drivers from sysfs" ;
       []
 
   let getpath dev attr = Printf.sprintf "/sys/class/net/%s/%s" dev attr
@@ -1278,25 +1278,29 @@ module Ovs = struct
 
     let get_bond_link_status name =
       try
+        (* Note: bond links are called "members" by the OVS. In old OVS
+           versions, the term "slaves" was used, which is also used by the
+           Linux kernel and in xapi. The terms bond link/slave/member are
+           used interchangably. *)
         let raw = appctl ~log:false ["bond/show"; name] in
         let lines = Astring.String.cuts ~empty:false ~sep:"\n" raw in
         List.fold_left
-          (fun (slaves, active_slave) line ->
-            let slaves =
+          (fun (members, active_member) line ->
+            let members =
               try
-                Scanf.sscanf line "slave %s@: %s" (fun slave state ->
-                    (slave, state = "enabled") :: slaves
+                Scanf.sscanf line "member %s@: %s" (fun member state ->
+                    (member, state = "enabled") :: members
                 )
-              with _ -> slaves
+              with _ -> members
             in
-            let active_slave =
+            let active_member =
               try
-                Scanf.sscanf line "active slave %s@(%s@)" (fun _ slave ->
-                    Some slave
+                Scanf.sscanf line "active member %s@(%s@)" (fun _ member ->
+                    Some member
                 )
-              with _ -> active_slave
+              with _ -> active_member
             in
-            (slaves, active_slave)
+            (members, active_member)
           )
           ([], None) lines
       with _ -> ([], None)

ocaml/xapi-consts/api_messages.ml

@@ -299,11 +299,18 @@ let pool_cpu_features_down = addMessage "POOL_CPU_FEATURES_DOWN" 5L
 let pool_cpu_features_up = addMessage "POOL_CPU_FEATURES_UP" 5L
 
 (* Cluster messages *)
+let cluster_quorum_approaching_lost =
+  addMessage "CLUSTER_QUORUM_APPROACHING_LOST" 2L
+
 let cluster_host_enable_failed = addMessage "CLUSTER_HOST_ENABLE_FAILED" 3L
 
 (* raised by external script in clustering daemon, do not delete this: it is not dead code *)
 let cluster_host_fencing = addMessage "CLUSTER_HOST_FENCING" 2L
 
+let cluster_host_leaving = addMessage "CLUSTER_HOST_LEAVING" 3L
+
+let cluster_host_joining = addMessage "CLUSTER_HOST_JOINING" 4L
+
 (* Certificate expiration messages *)
 let host_server_certificate_expiring = "HOST_SERVER_CERTIFICATE_EXPIRING"
 

ocaml/xapi-consts/constants.ml

@@ -398,3 +398,19 @@ let good_ciphersuites =
     ["ECDHE-RSA-AES256-GCM-SHA384"; "ECDHE-RSA-AES128-GCM-SHA256"]
 
 let verify_certificates_path = "/var/xapi/verify-certificates"
+
+let observer_component_xapi = "xapi"
+
+let observer_component_xenopsd = "xenopsd"
+
+let observer_component_xapi_clusterd = "xapi-clusterd"
+
+let observer_component_smapi = "smapi"
+
+let observer_components_all =
+  [
+    observer_component_xapi
+  ; observer_component_xenopsd
+  ; observer_component_xapi_clusterd
+  ; observer_component_smapi
+  ]

ocaml/xapi-guard/lib/dorpc.ml

@@ -13,7 +13,7 @@
  *)
 open Idl
 
-module D = Debug.Make (struct let name = "varstored-guard rpc" end)
+module D = Debug.Make (struct let name = "xapi-guard rpc" end)
 
 let wrap_rpc error f =
   let on_error e =

ocaml/xapi-guard/lib/dune

@@ -20,7 +20,7 @@
 )
 (library
  (name xapi_guard)
- (modules dorpc)
+ (modules dorpc types)
  (libraries
   rpclib.core
   lwt