afpd: a stock config gets the documented 64K dircache

NEWS.md

@@ -4,6 +4,9 @@ Netatalk Changelog
 Changes in 4.6.0
 ----------------
 
+* FIX: afpd: a stock configuration now gets the documented 64K-entry
+  directory cache; a stale compiled default silently capped it at 8K
+  entries. Explicit `dircache size` settings are unaffected
 * UPD: afpd: `ea` is now a (G)/(V) option; a value in [Global] applies to all
   volumes and can be overridden per volume. A `[Global] ea =` setting that
   previous releases silently ignored now takes effect

contrib/webmin_module/edit_global_section.cgi

@@ -537,7 +537,7 @@ print &ui_table_row(
 @values = get_parameter_of_section($afpconfRef, $sectionRef, 'dircache size', \%in);
 print &ui_table_row(
                     $text{'edit_global_section_dircachesize'},
-                    "<input name='p_dircache size' type='number' min='0' max='2097152' value='"
+                    "<input name='p_dircache size' type='number' min='0' max='1048576' value='"
                     . $values[0] . "'>" . " "
                     . ($values[2] ? html_escape($values[2]) . ": " . html_escape($values[1]) : '') . "\n"
 );

contrib/webmin_module/lang/en

@@ -230,7 +230,7 @@ edit_global_section_cnid_mysql_db=CNID MySQL database name
 edit_global_section_cnid_scheme=CNID scheme
 edit_global_section_cnid_server=CNID server IP address and port
 edit_global_section_dbus_daemon=D-Bus daemon directory
-edit_global_section_dircachesize=Directory cache size (bytes)
+edit_global_section_dircachesize=Directory cache size (entries)
 edit_global_section_dircache_mode=Directory cache replacement algorithm
 edit_global_section_dircache_rfork_budget=Resource fork cache memory budget (KB, 0=disabled)
 edit_global_section_dircache_rfork_maxsize=Max resource fork size to cache per entry (KB)

doc/developer/dircache.md

@@ -168,13 +168,13 @@ Every recursive lookup also results in many more stat calls.
 So even opening a small folder directly,
 still requires stat'ing every level of the whole path to be pushed into the page cache.
 
-If the dircache max size is small (by default just 8192 entries), as you move around your file share,
+If the dircache is smaller than your working set, as you move around your file share,
 old entries are pushed off (evicted) as new ones are added.
 This high entry rotation is known as "scan eviction" and means by the time you want to go back to a previous directory
 and read a cached entry, it has likely already been evicted which can cause a cascade effect of recursive lookups
 and stats calls to restore the broken cached paths if parent entries are evicted.
-So unless your whole file server has less than 8192 file and directories,
-it is recommended to increase the `dircache size` value in [afp.conf](https://netatalk.io/manual/en/afp.conf.5).
+It is recommended to increase the `dircache size` value (default 65536 entries) in
+[afp.conf](https://netatalk.io/manual/en/afp.conf.5) to match your common working set.
 
 Future releases will increase the maximum size of the dircache once existing performance issues are addressed.
 We are also considering retaining directories over files during eviction
@@ -399,7 +399,7 @@ These metrics are reported for both cache modes:
 
 - **entries**: Current number of cached entries at shutdown
 - **max_entries**: Peak entries reached during the session (high-water mark)
-- **config_max**: Maximum cache size from afp.conf configuration
+- **config_max_entries**: Maximum cache size (in entries) from afp.conf configuration
 - **lookups**: Total cache lookup operations performed
 - **validations**: Approximate filesystem validations performed (based on `dircache validation freq`)
 - **added**: Entries added to cache
@@ -423,7 +423,7 @@ LRU mode provides straightforward hit/miss metrics:
 
 ```txt
 dircache statistics (LRU): (user: jdoe) entries: 98234, max_entries: 131072,
-config_max: 131072, lookups: 2458716, hits: 1806407 (73.5%), misses: 652309 (26.5%),
+config_max_entries: 131072, lookups: 2458716, hits: 1806407 (73.5%), misses: 652309 (26.5%),
 validations: ~24587 (1.0%), added: 152341, removed: 54107, expunged: 8921,
 invalid_on_use: 234, evicted: 53873, covered_cancelled: 67, validation_freq: 100
 ```
@@ -472,7 +472,7 @@ but performance is improved across virtually all workload patterns.
 
 ```txt
 dircache statistics (ARC): (user: jdoe) entries: 98234, ghost_entries: 32838,
-max_entries: 131072, config_max: 131072, lookups: 2458716, hits: 1954327 (79.5%),
+max_entries: 131072, config_max_entries: 131072, lookups: 2458716, hits: 1954327 (79.5%),
 ghost_hits: 322351 (13.1%), total_hits: (92.6%), misses: 182038 (7.4%),
 validations: 21365 (0.9%), added: 152341, removed: 54107, expunged: 7234,
 invalid_on_use: 187, evicted: 53873, covered_cancelled: 67, validation_freq: 100

etc/afpd/dircache.c

@@ -95,10 +95,12 @@
  * Indexes
  * =======
  *
- * The maximum dircache size is:
- * max(DEFAULT_DIRCACHE_SIZE, min(size, MAX_DIRCACHE_SIZE)).
- * It is a hashtable which we use to store "struct dir"s in. If the cache get full, oldest
- * entries are evicted in chunks of DIRCACHE_FREE.
+ * The maximum dircache size resolves from the requested size: unset or
+ * below MIN_DIRCACHE_SIZE uses DEFAULT_DIRCACHE_SIZE, in-range values
+ * round up to the next power of two, larger requests clamp to
+ * MAX_DIRCACHE_SIZE. It is a hashtable which we use to store "struct
+ * dir"s in. If the cache gets full, oldest entries are evicted in
+ * chunks of DIRCACHE_FREE_QUANTUM.
  *
  * We have/need two indexes:
  * - a DID/name index on the main dircache, another hashtable
@@ -2032,58 +2034,72 @@ void dircache_promote(struct dir *dir)
 }
 
 /*!
- * @brief Initialize the dircache and indexes
+ * @brief Resolve a requested dircache size to the effective maximum
  *
- * This is called in child afpd initialization. The maximum cache size will be
- * max(DEFAULT_DIRCACHE_SIZE, min(size, MAX_DIRCACHE_SIZE)).
- * It initializes a hashtable which we use to store a directory cache in.
- * It also initializes two indexes:
- * - a DID/name index on the main dircache
- * - a queue index on the dircache (LRU mode) or four queues (ARC mode)
+ * Unset or below-minimum sizes use DEFAULT_DIRCACHE_SIZE, in-range
+ * sizes round up to the next power of two, oversize requests clamp
+ * to MAX_DIRCACHE_SIZE.
  *
  * @param[in] reqsize   requested maximum size from afp.conf
  *
- * @returns 0 on success, -1 on error
+ * @returns the effective maximum cache size
  */
-int dircache_init(int reqsize)
+unsigned int dircache_resolve_size(int reqsize)
 {
-    extern AFPObj *AFPobj;
-    int use_arc = 0;
-    unsigned int hash_size;
+    unsigned int size;
 
-    /* Check if ARC mode dircache via configuration first */
-    if (AFPobj && AFPobj->options.dircache_mode == 1) {
-        use_arc = 1;
-    }
-
-    /* Initialize the main dircache with requested size
-     * Bounds: MIN_DIRCACHE_SIZE (1K) to MAX_DIRCACHE_SIZE (1M)
-     * Default: DEFAULT_DIRCACHE_SIZE (64K) if reqsize <= 0 or out of bounds */
     if (reqsize > 0 && reqsize >= MIN_DIRCACHE_SIZE
             && reqsize <= MAX_DIRCACHE_SIZE) {
-        /* Use requested size, rounding up to next power of 2 if needed */
-        dircache_maxsize = MIN_DIRCACHE_SIZE;
+        size = MIN_DIRCACHE_SIZE;
 
-        while (dircache_maxsize < reqsize && dircache_maxsize < MAX_DIRCACHE_SIZE) {
-            dircache_maxsize *= 2;
+        while (size < reqsize && size < MAX_DIRCACHE_SIZE) {
+            size *= 2;
         }
     } else if (reqsize > MAX_DIRCACHE_SIZE) {
-        /* Requested size too large, use maximum */
-        dircache_maxsize = MAX_DIRCACHE_SIZE;
+        size = MAX_DIRCACHE_SIZE;
         LOG(log_warning, logtype_afpd,
-            "dircache_init: requested size %d exceeds maximum %d, using maximum",
+            "dircache_resolve_size: requested size %d exceeds maximum %d, using maximum",
             reqsize, MAX_DIRCACHE_SIZE);
     } else {
-        /* Use default (reqsize <= 0 or < MIN_DIRCACHE_SIZE) */
-        dircache_maxsize = DEFAULT_DIRCACHE_SIZE;
+        size = DEFAULT_DIRCACHE_SIZE;
 
         if (reqsize > 0 && reqsize < MIN_DIRCACHE_SIZE) {
             LOG(log_warning, logtype_afpd,
-                "dircache_init: requested size %d below minimum %d, using default %d",
+                "dircache_resolve_size: requested size %d below minimum %d, using default %d",
                 reqsize, MIN_DIRCACHE_SIZE, DEFAULT_DIRCACHE_SIZE);
         }
     }
 
+    return size;
+}
+
+/*!
+ * @brief Initialize the dircache and indexes
+ *
+ * This is called in child afpd initialization. Unset or below-minimum
+ * sizes use DEFAULT_DIRCACHE_SIZE, in-range sizes round up to the next
+ * power of two, oversize requests clamp to MAX_DIRCACHE_SIZE.
+ * It initializes a hashtable which we use to store a directory cache in.
+ * It also initializes two indexes:
+ * - a DID/name index on the main dircache
+ * - a queue index on the dircache (LRU mode) or four queues (ARC mode)
+ *
+ * @param[in] reqsize   requested maximum size from afp.conf
+ *
+ * @returns 0 on success, -1 on error
+ */
+int dircache_init(int reqsize)
+{
+    extern AFPObj *AFPobj;
+    int use_arc = 0;
+    unsigned int hash_size;
+
+    /* Check if ARC mode dircache via configuration first */
+    if (AFPobj && AFPobj->options.dircache_mode == 1) {
+        use_arc = 1;
+    }
+
+    dircache_maxsize = dircache_resolve_size(reqsize);
     /* Determine hash table size based on mode:
      * LRU: hash_size = c (only cached entries)
      * ARC: hash_size = 2c (c cached + c ghosts per ARC paper) */
@@ -2242,7 +2258,7 @@ void log_dircache_stat(void)
                                    (double)dircache_stat.lookups) * 100.0 : 0.0;
         LOG(log_info, logtype_afpd,
             "dircache statistics (ARC): (user: %s) "
-            "entries: %zu, ghost_entries: %zu, max_entries: %lu (%lu KB), config_max: %zu, "
+            "entries: %zu, ghost_entries: %zu, max_entries: %lu (%lu KB), config_max_entries: %zu, "
             "lookups: %llu, hits: %llu (%.1f%%), ghost_hits: %llu (%.1f%%), total_hits: (%.1f%%), misses: %llu (%.1f%%), "
             "validations: %llu (%.1f%%), "
             "added: %llu, removed: %llu, expunged: %llu, invalid_on_use: %llu, "
@@ -2315,7 +2331,7 @@ void log_dircache_stat(void)
                             ((double)dircache_stat.misses / (double)dircache_stat.lookups) * 100.0 : 0.0;
         LOG(log_info, logtype_afpd,
             "dircache statistics (LRU): (user: %s) "
-            "entries: %lu, max_entries: %lu (%lu KB), config_max: %u, lookups: %llu, hits: %llu (%.1f%%), misses: %llu (%.1f%%), "
+            "entries: %lu, max_entries: %lu (%lu KB), config_max_entries: %u, lookups: %llu, hits: %llu (%.1f%%), misses: %llu (%.1f%%), "
             "validations: %llu (%.1f%%), "
             "added: %llu, removed: %llu, expunged: %llu, invalid_on_use: %llu, evicted: %llu, "
             "covered_cancelled: %llu, validation_freq: %u",

etc/afpd/dircache.h

@@ -22,9 +22,9 @@
 #include <atalk/globals.h>
 #include <atalk/volume.h>
 
-/* Dircache size bounds */
+/* Dircache size bounds; the default is DEFAULT_DIRCACHE_SIZE in
+ * atalk/globals.h (included above) */
 #define MIN_DIRCACHE_SIZE 1024             /* 1K minimum (testing/constrained systems) */
-#define DEFAULT_DIRCACHE_SIZE 65536        /* 64K default (production) */
 #define MAX_DIRCACHE_SIZE 1048576          /* 1M maximum (high-memory servers) */
 #define DIRCACHE_FREE_QUANTUM 256
 
@@ -48,6 +48,7 @@ extern struct dir *dircache_search_by_name(const struct vol *,
 
 extern void       dircache_dump(void);
 extern void       log_dircache_stat(void);
+extern unsigned int dircache_resolve_size(int reqsize);
 extern int        dircache_set_validation_params(unsigned int freq);
 extern void       dircache_reset_validation_counter(void);
 extern void       dircache_report_invalid_entry(struct dir *dir);

include/atalk/globals.h

@@ -50,7 +50,7 @@
 #define CNID_PATH_OVERHEAD 12  /* CNID protocol header size for path resolution */
 #define CNID_MAX_PATH_LEN (CNID_PATH_OVERHEAD + MAXPATHLEN + 1)  /* Maximum path length for CNID operations */
 
-#define DEFAULT_MAX_DIRCACHE_SIZE 8192
+#define DEFAULT_DIRCACHE_SIZE 65536    /*!< default 'dircache size' (entries) */
 
 /* Directory cache validation settings */
 #define DEFAULT_DIRCACHE_VALIDATION_FREQ    1     /*!< Validate every Nth access (default 1 for backward compatibility) */

libatalk/util/netatalk_conf.c

@@ -3273,7 +3273,7 @@ int afp_config_parse(AFPObj *AFPObj, char *processname)
                                               NULL, -1);
 
         if (options->dircachesize == -1) {
-            options->dircachesize = DEFAULT_MAX_DIRCACHE_SIZE;
+            options->dircachesize = DEFAULT_DIRCACHE_SIZE;
         } else {
             LOG(log_warning, logtype_afpd,
                 "Using deprecated 'dircachesize' option, please update to 'dircache size'");

test/afpd/subtests_conf.c

@@ -40,6 +40,7 @@
 #include <atalk/util.h>
 #include <atalk/volume.h>
 
+#include "dircache.h"
 #include "peer_lock.h"
 #include "subtests_conf.h"
 #include "test.h"
@@ -1282,3 +1283,33 @@ int utest_conf_load_afp_conf_vols_locked(void)
     free(voldir);
     return failed;
 }
+
+/* utest_conf_dircache_resolve_size: bounds behaviour of the pure size
+ * resolution helper; touches no dircache state (the binary's live
+ * dircache from test.c stays intact). */
+int utest_conf_dircache_resolve_size(void)
+{
+    static const struct {
+        int reqsize;
+        unsigned int expect;
+    } cases[] = {
+        {-1, 65536},        /* unset: default */
+        {512, 65536},       /* below minimum: default, with warning */
+        {1024, 1024},       /* minimum accepted verbatim */
+        {100000, 131072},   /* in range: next power of two */
+        {2000000, 1048576}, /* above maximum: clamp, with warning */
+    };
+
+    for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
+        unsigned int got = dircache_resolve_size(cases[i].reqsize);
+
+        if (got != cases[i].expect) {
+            fprintf(test_stream(),
+                    "# utest_conf_dircache_resolve_size: %d -> %u, want %u\n",
+                    cases[i].reqsize, got, cases[i].expect);
+            return -1;
+        }
+    }
+
+    return 0;
+}

test/afpd/subtests_conf.h

@@ -19,5 +19,6 @@ extern int utest_conf_samba_ea_failure_keeps_vid(void);
 extern int utest_conf_samba_defaults_not_leaked_on_failed_volume(void);
 extern int utest_conf_samba_future_defaults(void);
 extern int utest_conf_load_afp_conf_vols_locked(void);
+extern int utest_conf_dircache_resolve_size(void);
 
 #endif /* SUBTESTS_CONF_H */

test/afpd/test.c

@@ -490,6 +490,8 @@ int main(int argc, char *argv[])
                      "ea = samba reverts performance-oriented compiled defaults (freq 100, rfork on)");
     TEST_int_or_skip(utest_conf_load_afp_conf_vols_locked(), 0,
                      "config loader fails closed under lock contention, state-neutral");
+    TEST_int_or_skip(utest_conf_dircache_resolve_size(), 0,
+                     "dircache_resolve_size: default, minimum, round-up, clamp");
     TEST(afp_options_parse_cmdline(&obj, 3, &args[0]),
          "parse afpd command-line options");
     TEST_int(afp_config_parse(&obj, NULL), 0,