doc(api-and-naming): add rules for xfuncs and generators

bash_completion

@@ -371,11 +371,11 @@ _comp_split()
     done
     shift "$((OPTIND - 1))"
     if (($# != 2)); then
-        printf '%s\n' "bash_completion: $FUNCNAME: unexpected number of arguments." >&2
+        printf '%s\n' "bash_completion: $FUNCNAME: unexpected number of arguments" >&2
         printf '%s\n' "usage: $FUNCNAME [-a] [-F SEP] ARRAY_NAME TEXT" >&2
         return 2
     elif [[ $1 == @(*[^_a-zA-Z0-9]*|[0-9]*|''|_*|IFS|OPTIND|OPTARG|OPTERR) ]]; then
-        printf '%s\n' "bash_completion: $FUNCNAME: invalid array name '$1'." >&2
+        printf '%s\n' "bash_completion: $FUNCNAME: invalid array name '$1'" >&2
         return 2
     fi
 
@@ -422,26 +422,27 @@ _comp_compgen__error_fallback()
 #           The array name should not start with an underscores "_", which is
 #           internally used.  The array name should not be either "IFS" or
 #           "OPT{IND,ARG,ERR}".
-#   -F sep  Set a set of separator characters (used as IFS in evaluating
-#           `compgen').  The default separator is $' \t\n'.  Note that this is
-#           not the set of separators to delimit output of `compgen', but the
-#           separators in evaluating the expansions of `-W '...'`, etc.  The
-#           delimiter of the output of `compgen` is always a newline.
-#   -l      The same as -F $'\n'.  Use lines as words in evaluating compgen.
 #   -c cur  Set a word used as a prefix to filter the completions.  The default
 #           is ${cur-}.
 #   -R      The same as -c ''.  Use raw outputs without filtering.
 #   -C dir  Evaluate compgen/generator in the specified directory.
 # @var[in,opt] cur  Used as the default value of a prefix to filter the
 #   completions.
 #
-# Usage #1: _comp_compgen [-alR|-F sep|-v arr|-c cur] -- options...
+# Usage #1: _comp_compgen [-alR|-F sep|-v arr|-c cur|-C dir] -- options...
 # Call `compgen` with the specified arguments and store the results in the
 # specified array.  This function essentially performs arr=($(compgen args...))
 # but properly handles shell options, IFS, etc. using _comp_split.  This
 # function is equivalent to `_comp_split [-a] -l arr "$(IFS=sep; compgen
 # args... -- cur)"`, but this pattern is frequent in the codebase and is good
 # to separate out as a function for the possible future implementation change.
+# OPTIONS
+#   -F sep  Set a set of separator characters (used as IFS in evaluating
+#           `compgen').  The default separator is $' \t\n'.  Note that this is
+#           not the set of separators to delimit output of `compgen', but the
+#           separators in evaluating the expansions of `-W '...'`, etc.  The
+#           delimiter of the output of `compgen` is always a newline.
+#   -l      The same as -F $'\n'.  Use lines as words in evaluating compgen.
 # @param $1... options  Arguments that are passed to compgen (if $1 starts with
 #   a hyphen `-`).
 #
@@ -453,12 +454,17 @@ _comp_compgen__error_fallback()
 #
 #   Note: The array option `-V arr` in bash >= 5.3 should be instead specified
 #   as `-v arr` as a part of the `_comp_compgen` options.
+# @return  True (0) if at least one completion is generated, False (1) if no
+#   completion is generated, or 2 with an incorrect usage.
 #
-# Usage #2: _comp_compgen [-alR|-v arr|-c cur] name args...
+# Usage #2: _comp_compgen [-aR|-v arr|-c cur|-C dir|-i cmd|-x cmd] name args...
 # Call the generator `_comp_compgen_NAME ARGS...` with the specified options.
 # This provides a common interface to call the functions `_comp_compgen_NAME`,
 # which produce completion candidates, with custom options [-alR|-v arr|-c
 # cur].  The option `-F sep` is not used with this usage.
+# OPTIONS
+#   -x cmd  Call exported generator `_comp_xfunc_CMD_compgen_NAME`
+#   -i cmd  Call internal generator `_comp_cmd_CMD__compgen_NAME`
 # @param $1... name args  Calls the function _comp_compgen_NAME with the
 #   specified ARGS (if $1 does not start with a hyphen `-`).  The options
 #   [-alR|-v arr|-c cur] are inherited by the child calls of `_comp_compgen`
@@ -470,6 +476,7 @@ _comp_compgen__error_fallback()
 #   These variables are internally used to pass the effect of the options
 #   [-alR|-v arr|-c cur] to the child calls of `_comp_compgen` in
 #   `_comp_compgen_NAME`.
+# @return   Exit status of the generator.
 #
 # @remarks When no options are supplied to _comp_compgen, `_comp_compgen NAME
 # args` is equivalent to the direct call `_comp_compgen_NAME args`.  As the
@@ -502,35 +509,51 @@ _comp_compgen()
     local _append=${_comp_compgen__append-}
     local _var=${_comp_compgen__var-COMPREPLY}
     local _cur=${_comp_compgen__cur-${cur-}}
-    local _ifs=$' \t\n' _dir=""
+    local _dir=""
+    local _ifs=$' \t\n' _has_ifs=""
+    local _icmd="" _xcmd=""
 
     local _old_nocasematch=""
     if shopt -q nocasematch; then
         _old_nocasematch=set
         shopt -u nocasematch
     fi
     local OPTIND=1 OPTARG="" OPTERR=0 _opt
-    while getopts ':alF:v:Rc:C:' _opt "$@"; do
+    while getopts ':av:Rc:C:lF:i:x:' _opt "$@"; do
         case $_opt in
             a) _append=set ;;
             v)
                 if [[ $OPTARG == @(*[^_a-zA-Z0-9]*|[0-9]*|''|_*|IFS|OPTIND|OPTARG|OPTERR) ]]; then
-                    printf 'bash_completion: %s: -v: invalid array name `%s'\''.\n' "$FUNCNAME" "$OPTARG" >&2
+                    printf 'bash_completion: %s: -v: invalid array name `%s'\''\n' "$FUNCNAME" "$OPTARG" >&2
                     return 2
                 fi
                 _var=$OPTARG
                 ;;
-            l) _ifs=$'\n' ;;
-            F) _ifs=$OPTARG ;;
             c) _cur=$OPTARG ;;
             R) _cur="" ;;
             C)
                 if [[ ! $OPTARG ]]; then
-                    printf 'bash_completion: %s: -C: invalid directory name `%s'\''.\n' "$FUNCNAME" "$OPTARG" >&2
+                    printf 'bash_completion: %s: -C: invalid directory name `%s'\''\n' "$FUNCNAME" "$OPTARG" >&2
                     return 2
                 fi
                 _dir=$OPTARG
                 ;;
+            l) _has_ifs=set _ifs=$'\n' ;;
+            F) _has_ifs=set _ifs=$OPTARG ;;
+            [ix])
+                if [[ ! $OPTARG ]]; then
+                    printf 'bash_completion: %s: -%s: invalid command name `%s'\''\n' "$FUNCNAME" "$_opt" "$OPTARG" >&2
+                    return 2
+                elif [[ $_icmd ]]; then
+                    printf 'bash_completion: %s: -%s: `-i %s'\'' is already specified\n' "$FUNCNAME" "$_opt" "$_icmd" >&2
+                    return 2
+                elif [[ $_xcmd ]]; then
+                    printf 'bash_completion: %s: -%s: `-x %s'\'' is already specified\n' "$FUNCNAME" "$_opt" "$_xcmd" >&2
+                    return 2
+                fi
+                ;;&
+            i) _icmd=$OPTARG ;;
+            x) _xcmd=$OPTARG ;;
             *)
                 printf 'bash_completion: %s: usage error\n' "$FUNCNAME" >&2
                 return 2
@@ -540,15 +563,28 @@ _comp_compgen()
     [[ $_old_nocasematch ]] && shopt -s nocasematch
     shift "$((OPTIND - 1))"
     if (($# == 0)); then
-        printf 'bash_completion: %s: unexpected number of arguments.\n' "$FUNCNAME" >&2
+        printf 'bash_completion: %s: unexpected number of arguments\n' "$FUNCNAME" >&2
         printf 'usage: %s [-alR|-F SEP|-v ARR|-c CUR] -- ARGS...' "$FUNCNAME" >&2
         return 2
     fi
 
     if [[ $1 != -* ]]; then
         # usage: _comp_compgen [options] NAME args
-        if ! declare -F "_comp_compgen_$1" &>/dev/null; then
-            printf 'bash_completion: %s: unrecognized category `%s'\'' (function _comp_compgen_%s not found).\n' "$FUNCNAME" "$1" "$1" >&2
+        if [[ $_has_ifs ]]; then
+            printf 'bash_completion: %s: `-l'\'' and `-F sep'\'' are not supported for generators\n' "$FUNCNAME" >&2
+            return 2
+        fi
+
+        local -a _generator
+        if [[ $_icmd ]]; then
+            _generator=("_comp_cmd_${_icmd//[^a-zA-Z0-9_]/_}__compgen_$1")
+        elif [[ $_xcmd ]]; then
+            _generator=(_comp_xfunc "$_xcmd" "compgen_$1")
+        else
+            _generator=("_comp_compgen_$1")
+        fi
+        if ! declare -F "${_generator[0]}" &>/dev/null; then
+            printf 'bash_completion: %s: unrecognized generator `%s'\'' (function %s not found)\n' "$FUNCNAME" "$1" "${_generator[0]}" >&2
             return 2
         fi
 
@@ -570,7 +606,7 @@ _comp_compgen()
         # Note: we use $1 as a part of a function name, and we use $2... as
         # arguments to the function if any.
         # shellcheck disable=SC2145
-        _comp_compgen_"$@"
+        "${_generator[@]}" "${@:2}"
         local _status=$?
 
         # Go back to the original directory.
@@ -584,6 +620,10 @@ _comp_compgen()
     fi
 
     # usage: _comp_compgen [options] -- [compgen_options]
+    if [[ $_icmd || $_xcmd ]]; then
+        printf 'bash_completion: %s: generator name is unspecified for `%s'\''\n' "$FUNCNAME" "${_icmd:+-i $_icmd}${_xcmd:+x $_xcmd}" >&2
+        return 2
+    fi
 
     # Note: $* in the below checks would be affected by uncontrolled IFS in
     # bash >= 5.0, so we need to set IFS to the normal value.  The behavior in
@@ -596,7 +636,7 @@ _comp_compgen()
     # "${*:2:_nopt}" becomes longer, so we test \$[0-9] and \$\{[0-9]
     # separately.
     if [[ $* == *\$[0-9]* || $* == *\$\{[0-9]* ]]; then
-        printf 'bash_completion: %s: positional parameter $1, $2, ... do not work inside this function.\n' "$FUNCNAME" >&2
+        printf 'bash_completion: %s: positional parameter $1, $2, ... do not work inside this function\n' "$FUNCNAME" >&2
         return 2
     fi
 

doc/api-and-naming.md

@@ -41,15 +41,17 @@ deprecated in.
 Due to its nature, bash-completion adds a number of functions and variables in
 the shell's environment.
 
-|                                     | `bash_completion`   | `completions/*`                                                            |
-|:------------------------------------|:--------------------|:---------------------------------------------------------------------------|
-| public configuration variables      | `BASH_COMPLETION_*` | `BASH_COMPLETION_CMD_${Command^^}_${Config^^}`                             |
-| private non-local variables         | `_comp__*`          | `_comp_cmd_${Command}__${Data}`                                            |
-| private non-local mutable variables | `_comp__*_mut_*`    | `_comp_cmd_${Command}__mut_${Data}`                                        |
-| exporter function local variables   | `_*` (not `_comp*`) | `_*` (not `_comp*`)                                                        |
-| public/exported functions           | `_comp_*`           | `_comp_cmd_${Command}` (functions for `complete -F`)                       |
-|                                     |                     | `_comp_xfunc_${Command}_${Utility}` (functions for use with `_comp_xfunc`) |
-| private/internal functions          | `_comp__*`          | `_comp_cmd_${Command}__${Utility}` (utility functions)                     |
+|                                     | `bash_completion`       | `completions/*`                                                                       |
+|:------------------------------------|:------------------------|:--------------------------------------------------------------------------------------|
+| public configuration variables      | `BASH_COMPLETION_*`     | `BASH_COMPLETION_CMD_${Command^^}_${Config^^}`                                        |
+| private non-local variables         | `_comp__*`              | `_comp_cmd_${Command}__${Data}`                                                       |
+| private non-local mutable variables | `_comp__*_mut_*`        | `_comp_cmd_${Command}__mut_${Data}`                                                   |
+| exporter function local variables   | `_*` (not `_comp*`)     | `_*` (not `_comp*`)                                                                   |
+| public/exported functions           | `_comp_*`               | `_comp_cmd_${Command}` (functions for `complete -F`)                                  |
+|                                     |                         | `_comp_xfunc_${Command}_${Utility}` (functions for use with `_comp_xfunc`)            |
+|                                     | `_comp_compgen_${Name}` | `_comp_xfunc_${Command}_compgen_${Name}` (generators for use with `_comp_compgen -x`) |
+| private/internal functions          | `_comp__*`              | `_comp_cmd_${Command}__${Utility}` (utility functions)                                |
+|                                     |                         | `_comp_cmd_${Command}__compgen_${Name}` (generators for use with `_comp_compgen -i`)  |
 
 `${Command}` refers to a command name (with characters not allowed in POSIX
 function or variable names replaced by an underscore), `${Config}` the name of
@@ -110,3 +112,68 @@ distinctive and clash free enough.
 It is known that a lot of functions and variables in the tree do not follow
 these naming rules yet. Things introduced after version 2.11 should, and we are
 evaluating our options for handling older ones.
+
+## Exported functions (xfunc)
+
+Exported functions (xfunc) are the functions defined in completion files for
+specific commands but exposed to other completion files.  The xfuncs have the
+name `_comp_xfunc_CMD_NAME` where `CMD` is the name of the associated command,
+and `NAME` is the name of the utility.  The other functions defined in specific
+completion files are considered private and should not be called outside the
+file.
+
+The xfuncs can be called by `_comp_xfunc CMD NAME ARGS` from external files.
+The xfuncs are supposed to be directly called as `_comp_xfunc_CMD_NAME ARGS`
+from the same file where they are defined, or if they wrap a `_comp_cmd_NAME__*`
+function, that one should be called directly instead.
+
+Note: The name `xfunc` was initially the name of a utility function, `_xfunc`,
+to call "eXternal FUNCtions" that are defined in other completion files.  The
+concept is later extended to also mean "eXported".
+
+## Generator functions
+
+The generator functions, which have names of the form `_comp_compgen_NAME`, are
+used to generate completion candidates.  A generator function is supposed to be
+called by `_comp_compgen [OPTS] NAME ARGS` where `OPTS = -aRl|-v var|-c cur|-C
+dir|-F sep` are the options to modify the behavior (see the code comment of
+`_comp_compgen` for details).  When there are no `opts`, the generator function
+is supposed to be directly called as `_comp_compgen_NAME ARGS`.  The result is
+stored in the target variable (which is `COMPREPLY` by default but can be
+specified by `-v var` in `OPTS`).
+
+### Implementing a generator function
+
+To implement a generator function, one should generate completion candidates by
+calling `_comp_compgen` or other generators.  To avoid conflicts with the
+options specified to `_comp_compgen`, one should not directly modify or
+reference the target variable.  When post-filtering is needed, store them in
+local arrays, filter them, and finally append them by `_comp_compgen -- -W
+"${arr[@]}"`.
+
+A generator function should replace the existing content of the variable by
+default.  When the appending behavior is favored, the caller should specify it
+through `_comp_compgen -a NAME`.  The generator function do not need to process
+it because internal `_comp_compgen` calls automatically reflects the option
+`-a` specified to the outer calls of `_comp_compgen`.
+
+The exit status is implementation-defined.
+
+- The `_comp_compgen -- COMPGEN_ARGS` returns whether there is at least one
+  completion.  This is useful when one wants to reuse the array content with
+  `"${tmp[@]}"` avoiding `nounset` error.
+- Some use other rules for the exit status. E.g., `help` and `usage` return
+  whether there were options *before* filtering by cur. This is used for
+  `_comp_compgen_help || _comp_compgen_usage`.
+
+Whether to clear the target variable on runtime error (when `-a` is not
+specified in `OPTS`) is implementation-defined.  On the other hand, the
+generator function should not leave any side effects in the target variable on
+usage error.  Note that the target variable might be cleared by the internal
+calls of `_comp_compgen`.  To explicitly clear the target variable,
+`_comp_compgen_set` can be called without arguments.
+
+Exported generators are defined with the names `_comp_xfunc_CMD_compgen_NAME`
+and called by `_comp_compgen [opts] -x CMD NAME args`.  Internal generators are
+defined with the names `_comp_cmd_CMD__compgen_NAME` and called by
+`_comp_compgen [opts] -i CMD NAME args`.

test/fixtures/_comp_compgen/completions/compgen-cmd1

@@ -0,0 +1,19 @@
+# Dummy completion file for _comp_compgen tests            -*- shell-script -*-
+
+_comp_xfunc_compgen_cmd1_compgen_generator1() {
+    _comp_compgen -- -W '5foo 6bar 7baz'
+}
+
+_comp_cmd_compgen_cmd1__compgen_generator2() {
+    _comp_compgen -- -W '5abc 6def 7ghi'
+}
+
+_comp_cmd_compgen_cmd1() {
+    local cur prev words cword comp_args
+    _comp_initialize -- "$@" || return
+    _comp_compgen -- -W '012 123 234'
+    _comp_compgen -ai compgen-cmd1 generator2
+} &&
+    complete -F _comp_cmd_compgen_cmd1 compgen-cmd1
+
+# ex: filetype=sh

test/fixtures/_comp_compgen/completions/compgen-cmd2

@@ -0,0 +1,11 @@
+# Dummy completion file for _comp_compgen tests            -*- shell-script -*-
+
+_comp_cmd_compgen_cmd2() {
+    local cur prev words cword comp_args
+    _comp_initialize -- "$@" || return
+    _comp_compgen -- -W '012 123 234'
+    _comp_compgen -ax compgen-cmd1 generator1
+} &&
+    complete -F _comp_cmd_compgen_cmd2 compgen-cmd2
+
+# ex: filetype=sh

test/t/unit/test_unit_compgen.py

@@ -128,3 +128,21 @@ def test_6_option_C_4(self, functions, completion):
         # Note: we are not in the original directory that "b" exists, so Bash
         # will not suffix a slash to the directory name.
         assert completion == "b"
+
+    def test_7_icmd(self, bash, functions):
+        with bash_env_saved(bash) as bash_env:
+            bash_env.write_variable(
+                "BASH_COMPLETION_USER_DIR", "$PWD/_comp_compgen", quote=False
+            )
+
+            completions = assert_complete(bash, "compgen-cmd1 '")
+            assert completions == ["012", "123", "234", "5abc", "6def", "7ghi"]
+
+    def test_7_xcmd(self, bash, functions):
+        with bash_env_saved(bash) as bash_env:
+            bash_env.write_variable(
+                "BASH_COMPLETION_USER_DIR", "$PWD/_comp_compgen", quote=False
+            )
+
+            completions = assert_complete(bash, "compgen-cmd2 '")
+            assert completions == ["012", "123", "234", "5foo", "6bar", "7baz"]