build, vtysh: extract vtysh commands from .xref

Rather than running selected source files through the preprocessor and a
bunch of perl regex'ing to get the list of all DEFUNs, use the data
collected in frr.xref.

This not only eliminates issues we've been having with preprocessor
failures due to nonexistent header files, but is also much faster.
Where extract.pl would take 5s, this now finishes in 0.2s.  And since
this is a non-parallelizable build step towards the end of the build
(dependent on a lot of other things being done already), the speedup is
actually noticeable.

Also files containing CLI no longer need to be listed in `vtysh_scan`
since the .xref data covers everything.  `#ifndef VTYSH_EXTRACT_PL`
checks are equally obsolete.

Signed-off-by: David Lamparter <equinox@opensourcerouting.org>

Makefile.am

@@ -144,7 +144,6 @@ pkginclude_HEADERS =
 nodist_pkginclude_HEADERS =
 dist_yangmodels_DATA =
 man_MANS =
-vtysh_scan =
 vtysh_daemons =
 clippy_scan =
 
@@ -226,6 +225,7 @@ EXTRA_DIST += \
 	python/makefile.py \
 	python/tiabwarfo.py \
 	python/xrelfo.py \
+	python/xref2vtysh.py \
 	python/test_xrelfo.py \
 	python/runtests.py \
 	\

babeld/subdir.am

@@ -4,11 +4,6 @@
 
 if BABELD
 sbin_PROGRAMS += babeld/babeld
-vtysh_scan += \
-	babeld/babel_interface.c \
-	babeld/babel_zebra.c \
-	babeld/babeld.c \
-	# end
 vtysh_daemons += babeld
 endif
 

bfdd/bfdd_cli.c

@@ -26,9 +26,7 @@
 #include "lib/log.h"
 #include "lib/northbound_cli.h"
 
-#ifndef VTYSH_EXTRACT_PL
 #include "bfdd/bfdd_cli_clippy.c"
-#endif /* VTYSH_EXTRACT_PL */
 
 #include "bfd.h"
 #include "bfdd_nb.h"

bfdd/bfdd_vty.c

@@ -28,9 +28,7 @@
 
 #include "bfd.h"
 
-#ifndef VTYSH_EXTRACT_PL
 #include "bfdd/bfdd_vty_clippy.c"
-#endif
 
 /*
  * Commands help string definitions.

bfdd/subdir.am

@@ -5,8 +5,6 @@
 if BFDD
 noinst_LIBRARIES += bfdd/libbfd.a
 sbin_PROGRAMS += bfdd/bfdd
-vtysh_scan += bfdd/bfdd_vty.c
-vtysh_scan += bfdd/bfdd_cli.c
 vtysh_daemons += bfdd
 man8 += $(MANBUILD)/frr-bfdd.8
 endif

bgpd/bgp_bmp.c

@@ -2033,9 +2033,7 @@ static const struct cmd_variable_handler bmp_targets_var_handlers[] = {
 
 #define BMP_STR "BGP Monitoring Protocol\n"
 
-#ifndef VTYSH_EXTRACT_PL
 #include "bgpd/bgp_bmp_clippy.c"
-#endif
 
 DEFPY_NOSH(bmp_targets_main,
       bmp_targets_cmd,

bgpd/bgp_debug.c

@@ -1410,9 +1410,7 @@ DEFUN (no_debug_bgp_update_direct_peer,
 	return CMD_SUCCESS;
 }
 
-#ifndef VTYSH_EXTRACT_PL
 #include "bgpd/bgp_debug_clippy.c"
-#endif
 
 DEFPY (debug_bgp_update_prefix_afi_safi,
        debug_bgp_update_prefix_afi_safi_cmd,

bgpd/bgp_evpn_vty.c

@@ -3336,9 +3336,7 @@ static void write_vni_config(struct vty *vty, struct bgpevpn *vpn)
 	}
 }
 
-#ifndef VTYSH_EXTRACT_PL
 #include "bgpd/bgp_evpn_vty_clippy.c"
-#endif
 
 DEFPY(bgp_evpn_flood_control,
       bgp_evpn_flood_control_cmd,

bgpd/bgp_labelpool.c

@@ -39,9 +39,7 @@
 
 #define BGP_LABELPOOL_ENABLE_TESTS 0
 
-#ifndef VTYSH_EXTRACT_PL
 #include "bgpd/bgp_labelpool_clippy.c"
-#endif
 
 
 /*

bgpd/bgp_route.c

@@ -90,9 +90,7 @@
 #include "bgpd/bgp_flowspec_util.h"
 #include "bgpd/bgp_pbr.h"
 
-#ifndef VTYSH_EXTRACT_PL
 #include "bgpd/bgp_route_clippy.c"
-#endif
 
 DEFINE_HOOK(bgp_snmp_update_stats,
 	    (struct bgp_node *rn, struct bgp_path_info *pi, bool added),

bgpd/bgp_routemap.c

@@ -74,9 +74,7 @@
 #include "bgpd/rfapi/bgp_rfapi_cfg.h"
 #endif
 
-#ifndef VTYSH_EXTRACT_PL
 #include "bgpd/bgp_routemap_clippy.c"
-#endif
 
 /* Memo of route-map commands.
 

bgpd/bgp_rpki.c

@@ -52,16 +52,12 @@
 
 #include "lib/network.h"
 #include "lib/thread.h"
-#ifndef VTYSH_EXTRACT_PL
 #include "rtrlib/rtrlib.h"
-#endif
 #include "hook.h"
 #include "libfrr.h"
 #include "lib/version.h"
 
-#ifndef VTYSH_EXTRACT_PL
 #include "bgpd/bgp_rpki_clippy.c"
-#endif
 
 DEFINE_MTYPE_STATIC(BGPD, BGP_RPKI_CACHE, "BGP RPKI Cache server");
 DEFINE_MTYPE_STATIC(BGPD, BGP_RPKI_CACHE_GROUP, "BGP RPKI Cache server group");

bgpd/bgp_vty.c

@@ -1316,9 +1316,7 @@ void bgp_clear_soft_in(struct bgp *bgp, afi_t afi, safi_t safi)
 	bgp_clear(NULL, bgp, afi, safi, clear_all, BGP_CLEAR_SOFT_IN, NULL);
 }
 
-#ifndef VTYSH_EXTRACT_PL
 #include "bgpd/bgp_vty_clippy.c"
-#endif
 
 DEFUN_HIDDEN (bgp_local_mac,
               bgp_local_mac_cmd,

bgpd/subdir.am

@@ -6,36 +6,9 @@ if BGPD
 noinst_LIBRARIES += bgpd/libbgp.a
 sbin_PROGRAMS += bgpd/bgpd
 noinst_PROGRAMS += bgpd/bgp_btoa
-vtysh_scan += \
-	bgpd/bgp_bfd.c \
-	bgpd/bgp_debug.c \
-	bgpd/bgp_dump.c \
-	bgpd/bgp_evpn_mh.c \
-	bgpd/bgp_evpn_vty.c \
-	bgpd/bgp_filter.c \
-	bgpd/bgp_labelpool.c \
-	bgpd/bgp_mplsvpn.c \
-	bgpd/bgp_nexthop.c \
-	bgpd/bgp_route.c \
-	bgpd/bgp_routemap.c \
-	bgpd/bgp_vty.c \
-	bgpd/bgp_flowspec_vty.c \
-	# end
-
-# can be loaded as DSO - always include for vtysh
-vtysh_scan += bgpd/bgp_rpki.c
-vtysh_scan += bgpd/bgp_bmp.c
 
 vtysh_daemons += bgpd
 
-if ENABLE_BGP_VNC
-vtysh_scan += \
-	bgpd/rfapi/bgp_rfapi_cfg.c \
-	bgpd/rfapi/rfapi.c \
-	bgpd/rfapi/rfapi_vty.c \
-	bgpd/rfapi/vnc_debug.c \
-	# end
-endif
 if SNMP
 module_LTLIBRARIES += bgpd/bgpd_snmp.la
 endif

configure.ac

@@ -2747,7 +2747,6 @@ AC_CONFIG_FILES([
 	  pkgsrc/ripd.sh pkgsrc/ripngd.sh pkgsrc/zebra.sh
 	  pkgsrc/eigrpd.sh])
 
-AC_CONFIG_FILES([vtysh/extract.pl], [chmod +x vtysh/extract.pl])
 AC_CONFIG_FILES([tools/frr], [chmod +x tools/frr])
 AC_CONFIG_FILES([tools/watchfrr.sh], [chmod +x tools/watchfrr.sh])
 AC_CONFIG_FILES([tools/frrinit.sh], [chmod +x tools/frrinit.sh])

doc/developer/cli.rst

@@ -453,9 +453,7 @@ all DEFPY statements**:
    /* GPL header */
    #include ...
    ...
-   #ifndef VTYSH_EXTRACT_PL
    #include "daemon/filename_clippy.c"
-   #endif
 
    DEFPY(...)
    DEFPY(...)

doc/developer/vtysh.rst

@@ -43,9 +43,14 @@ simplifying the output. This is discussed in :ref:`vtysh-configuration`.
 Command Extraction
 ------------------
 
-When VTYSH is built, a Perl script named :file:`extract.pl` searches the FRR
-codebase looking for ``DEFUN``'s. It extracts these ``DEFUN``'s, transforms
-them into ``DEFSH``'s and appends them to ``vtysh_cmd.c``. Each ``DEFSH``
+To build ``vtysh``, the :file:`python/xref2vtysh.py` script scans through the
+:file:`frr.xref` file created earlier in the build process.  This file contains
+a list of all ``DEFUN`` and ``install_element`` sites in the code, generated
+directly from the binaries (and therefore matching exactly what is really
+available.)
+
+This list is collated and transformed into ``DEFSH`` (and ``install_element``)
+statements, output to ``vtysh_cmd.c``. Each ``DEFSH``
 contains the name of the command plus ``_vtysh``, as well as a flag that
 indicates which daemons the command was found in. When the command is executed
 in VTYSH, this flag is inspected to determine which daemons to send the command
@@ -55,6 +60,12 @@ avoiding spurious errors from daemons that don't have the command defined.
 The extraction script contains lots of hardcoded knowledge about what sources
 to look at and what flags to use for certain commands.
 
+.. note::
+
+   The ``vtysh_scan`` Makefile variable and ``#ifndef VTYSH_EXTRACT_PL``
+   checks in source files are no longer used.  Remove them when rebasing older
+   changes.
+
 .. _vtysh-special-defuns:
 
 Special DEFUNs
@@ -69,7 +80,7 @@ several VTYSH-specific ``DEFUN`` variants that each serve different purposes.
    simply forwarded to the daemons indicated in the daemon flag.
 
 ``DEFUN_NOSH``
-   Used by daemons. Has the same expansion as a ``DEFUN``, but ``extract.pl``
+   Used by daemons. Has the same expansion as a ``DEFUN``, but ``xref2vtysh.py``
    will skip these definitions when extracting commands. This is typically used
    when VTYSH must take some special action upon receiving the command, and the
    programmer therefore needs to write VTYSH's copy of the command manually

eigrpd/eigrp_cli.c

@@ -31,9 +31,7 @@
 #include "eigrp_zebra.h"
 #include "eigrp_cli.h"
 
-#ifndef VTYSH_EXTRACT_PL
 #include "eigrpd/eigrp_cli_clippy.c"
-#endif /* VTYSH_EXTRACT_PL */
 
 /*
  * XPath: /frr-eigrpd:eigrpd/instance

eigrpd/eigrp_vty.c

@@ -55,9 +55,7 @@
 #include "eigrpd/eigrp_dump.h"
 #include "eigrpd/eigrp_const.h"
 
-#ifndef VTYSH_EXTRACT_PL
 #include "eigrpd/eigrp_vty_clippy.c"
-#endif
 
 static void eigrp_vty_display_prefix_entry(struct vty *vty, struct eigrp *eigrp,
 					   struct eigrp_prefix_descriptor *pe,

eigrpd/subdir.am

@@ -4,12 +4,6 @@
 
 if EIGRPD
 sbin_PROGRAMS += eigrpd/eigrpd
-vtysh_scan += \
-	eigrpd/eigrp_cli.c \
-	eigrpd/eigrp_dump.c \
-	eigrpd/eigrp_vty.c \
-	# end
-#	eigrpd/eigrp_routemap.c
 vtysh_daemons += eigrpd
 man8 += $(MANBUILD)/frr-eigrpd.8
 endif

isisd/isis_cli.c

@@ -37,9 +37,7 @@
 #include "isisd/isis_circuit.h"
 #include "isisd/isis_csm.h"
 
-#ifndef VTYSH_EXTRACT_PL
 #include "isisd/isis_cli_clippy.c"
-#endif
 
 #ifndef FABRICD
 

isisd/subdir.am

@@ -5,16 +5,6 @@
 if ISISD
 noinst_LIBRARIES += isisd/libisis.a
 sbin_PROGRAMS += isisd/isisd
-vtysh_scan += \
-	isisd/isis_cli.c \
-	isisd/isis_ldp_sync.c \
-	isisd/isis_redist.c \
-	isisd/isis_spf.c \
-	isisd/isis_te.c \
-	isisd/isis_sr.c \
-	isisd/isis_vty_fabricd.c \
-	isisd/isisd.c \
-	# end
 vtysh_daemons += isisd
 if SNMP
 module_LTLIBRARIES += isisd/isisd_snmp.la
@@ -25,18 +15,6 @@ endif
 if FABRICD
 noinst_LIBRARIES += isisd/libfabric.a
 sbin_PROGRAMS += isisd/fabricd
-if !ISISD
-vtysh_scan += \
-	isisd/isis_cli.c \
-	isisd/isis_ldp_sync.c \
-	isisd/isis_redist.c \
-	isisd/isis_spf.c \
-	isisd/isis_te.c \
-	isisd/isis_sr.c \
-	isisd/isis_vty_fabricd.c \
-	isisd/isisd.c \
-	# end
-endif
 vtysh_daemons += fabricd
 endif
 

ldpd/ldp_vty_cmds.c

@@ -25,9 +25,7 @@
 
 #include "ldpd/ldpd.h"
 #include "ldpd/ldp_vty.h"
-#ifndef VTYSH_EXTRACT_PL
 #include "ldpd/ldp_vty_cmds_clippy.c"
-#endif
 
 DEFPY_NOSH(ldp_mpls_ldp,
 	ldp_mpls_ldp_cmd,

ldpd/subdir.am

@@ -5,7 +5,6 @@
 if LDPD
 noinst_LIBRARIES += ldpd/libldp.a
 sbin_PROGRAMS += ldpd/ldpd
-vtysh_scan += ldpd/ldp_vty_cmds.c
 vtysh_daemons += ldpd
 man8 += $(MANBUILD)/frr-ldpd.8
 endif

lib/command.h

@@ -251,9 +251,6 @@ struct cmd_node {
 /* Argc max counts. */
 #define CMD_ARGC_MAX   256
 
-/* Turn off these macros when using cpp with extract.pl */
-#ifndef VTYSH_EXTRACT_PL
-
 /* helper defines for end-user DEFUN* macros */
 #define DEFUN_CMD_ELEMENT(funcname, cmdname, cmdstr, helpstr, attrs, dnum)     \
 	static const struct cmd_element cmdname = {                            \
@@ -370,8 +367,6 @@ struct cmd_node {
 #define ALIAS_YANG(funcname, cmdname, cmdstr, helpstr)                         \
 	ALIAS_ATTR(funcname, cmdname, cmdstr, helpstr, CMD_ATTR_YANG)
 
-#endif /* VTYSH_EXTRACT_PL */
-
 /* Some macroes */
 
 /*
@@ -511,7 +506,6 @@ struct xref_install_element {
 	enum node_type node_type;
 };
 
-#ifndef VTYSH_EXTRACT_PL
 #define install_element(node_type_, cmd_element_) do {                         \
 		static const struct xref_install_element _xref                 \
 				__attribute__((used)) = {                      \
@@ -523,7 +517,6 @@ struct xref_install_element {
 		XREF_LINK(_xref.xref);                                         \
 		_install_element(node_type_, cmd_element_);                    \
 	} while (0)
-#endif
 
 extern void _install_element(enum node_type, const struct cmd_element *);
 

lib/filter_cli.c

@@ -31,9 +31,7 @@
 #include "lib/plist_int.h"
 #include "lib/printfrr.h"
 
-#ifndef VTYSH_EXTRACT_PL
 #include "lib/filter_cli_clippy.c"
-#endif /* VTYSH_EXTRACT_PL */
 
 #define ACCESS_LIST_STR "Access list entry\n"
 #define ACCESS_LIST_ZEBRA_STR "Access list name\n"

lib/if.c

@@ -35,9 +35,7 @@
 #include "buffer.h"
 #include "log.h"
 #include "northbound_cli.h"
-#ifndef VTYSH_EXTRACT_PL
 #include "lib/if_clippy.c"
-#endif
 
 DEFINE_MTYPE_STATIC(LIB, IF, "Interface");
 DEFINE_MTYPE_STATIC(LIB, CONNECTED, "Connected");