FOUR-20929: Implement ETag Caching

ProcessMaker/Http/Kernel.php

@@ -43,7 +43,7 @@ class Kernel extends HttpKernel
             \Illuminate\Routing\Middleware\SubstituteBindings::class,
             Middleware\GenerateMenus::class,
             \Laravel\Passport\Http\Middleware\CreateFreshApiToken::class,
-            \ProcessMaker\Http\Middleware\IgnoreMapFiles::class,
+            Middleware\IgnoreMapFiles::class,
         ],
         'api' => [
             // API Middleware is defined with routeMiddleware below.
@@ -85,6 +85,7 @@ class Kernel extends HttpKernel
         'session_kill' => Middleware\SessionControlKill::class,
         'no-cache' => Middleware\NoCache::class,
         'admin' => Middleware\IsAdmin::class,
+        'etag' => Middleware\Etag\HandleEtag::class,
     ];
 
     /**

ProcessMaker/Http/Middleware/Etag/HandleEtag.php

@@ -0,0 +1,169 @@
+<?php
+
+namespace ProcessMaker\Http\Middleware\Etag;
+
+use Closure;
+use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Log;
+use ProcessMaker\Http\Resources\Caching\EtagManager;
+use Symfony\Component\HttpFoundation\Response;
+
+class HandleEtag
+{
+    public string $middleware = 'etag';
+
+    /**
+     * Handle an incoming request.
+     */
+    public function handle(Request $request, Closure $next): Response
+    {
+        if (!config('etag.enabled')) {
+            return $next($request);
+        }
+
+        // Process only GET and HEAD methods.
+        if (!$request->isMethod('GET') && !$request->isMethod('HEAD')) {
+            return $next($request);
+        }
+
+        // Check if specific tables are defined for the route and calculate ETag.
+        $etag = $this->generateEtagFromTablesIfNeeded($request);
+
+        // If the client has a matching ETag, return a 304 response.
+        // Otherwise, continue with the controller execution.
+        $response = $etag && $this->etagMatchesRequest($etag, $request)
+            ? $this->buildNotModifiedResponse($etag)
+            : $next($request);
+
+        // Add the pre-calculated ETag to the response if available.
+        if ($etag) {
+            $response = $this->setEtagOnResponse($response, $etag);
+        }
+
+        // If no ETag was calculated from tables, generate it based on the response.
+        if (!$etag && $this->isCacheableResponse($response)) {
+            $etag = EtagManager::getEtag($request, $response);
+            if ($etag) {
+                $response = $this->setEtagOnResponse($response, $etag);
+
+                // If the client has a matching ETag, set the response to 304.
+                if ($this->etagMatchesRequest($etag, $request)) {
+                    $response = $this->buildNotModifiedResponse($etag);
+                }
+            }
+        }
+
+        // Detect if the ETag changes frequently for dynamic responses.
+        $this->logEtagChanges($request, $etag);
+
+        return $response;
+    }
+
+    /**
+     * Determine if a response is cacheable.
+     */
+    private function isCacheableResponse(Response $response): bool
+    {
+        $cacheableStatusCodes = [200, 203, 204, 206, 304];
+        $cacheControl = $response->headers->get('Cache-Control', '');
+
+        // Verify if the status code is cacheable and does not contain "no-store".
+        return in_array($response->getStatusCode(), $cacheableStatusCodes)
+            && !str_contains($cacheControl, 'no-store');
+    }
+
+    /**
+     * Generate an ETag based on the tables defined in the route, if applicable.
+     */
+    private function generateEtagFromTablesIfNeeded(Request $request): ?string
+    {
+        $tables = $request->route()->defaults['etag_tables'] ?? null;
+
+        return $tables ? EtagManager::generateEtagFromTables(explode(',', $tables)) : null;
+    }
+
+    /**
+     * Check if the ETag matches the request.
+     */
+    private function etagMatchesRequest(string $etag, Request $request): bool
+    {
+        $noneMatch = array_map([$this, 'stripWeakTags'], $request->getETags());
+
+        return in_array($etag, $noneMatch);
+    }
+
+    /**
+     * Build a 304 Not Modified response with the given ETag.
+     */
+    private function buildNotModifiedResponse(string $etag): Response
+    {
+        $response = new Response();
+        $response->setNotModified();
+        $response->setEtag($etag);
+
+        return $response;
+    }
+
+    /**
+     * Set the ETag on a given response.
+     */
+    private function setEtagOnResponse(Response $response, string $etag): Response
+    {
+        $response->setEtag($etag);
+
+        return $response;
+    }
+
+    /**
+     * Remove the weak indicator (W/) from an ETag.
+     */
+    private function stripWeakTags(string $etag): string
+    {
+        return str_replace('W/', '', $etag);
+    }
+
+    /**
+     * Log ETag changes to detect highly dynamic responses.
+     */
+    private function logEtagChanges(Request $request, ?string $etag): void
+    {
+        if (!config('etag.enabled') || !config('etag.log_dynamic_endpoints')) {
+            return;
+        }
+
+        if (!$etag) {
+            return;
+        }
+
+        // Retrieve the history of ETags for this endpoint.
+        $url = $request->fullUrl();
+        $cacheKey = 'etag_history:' . md5($url);
+        $etagHistory = Cache::get($cacheKey, []);
+
+        // If the ETag is already in the history, it is not considered dynamic.
+        if (in_array($etag, $etagHistory, true)) {
+            return;
+        }
+
+        // Add the new ETag to the history.
+        $etagHistory[] = $etag;
+
+        // Keep the history limited to the last n ETags.
+        $etagHistoryLimit = config('etag.history_limit', 10);
+        if (count($etagHistory) > $etagHistoryLimit) {
+            array_shift($etagHistory); // Remove the oldest ETag.
+        }
+
+        // Save the updated history in the cache, valid for 30 minutes.
+        $cacheExpirationMinute = config('etag.history_cache_expiration');
+        Cache::put($cacheKey, $etagHistory, now()->addMinutes($cacheExpirationMinute));
+
+        // If the history is full and all ETags are unique, log this as a highly dynamic endpoint.
+        if (count(array_unique($etagHistory)) === $etagHistoryLimit) {
+            Log::info('ETag Dynamic endpoint detected', [
+                'url' => $url,
+            ]);
+        }
+    }
+}

ProcessMaker/Http/Resources/Caching/EtagManager.php

@@ -0,0 +1,73 @@
+<?php
+
+namespace ProcessMaker\Http\Resources\Caching;
+
+use Closure;
+use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Str;
+use Symfony\Component\HttpFoundation\Response;
+
+class EtagManager
+{
+    /**
+     * The callback used to generate the ETag.
+     */
+    protected static ?Closure $etagGenerateCallback = null;
+
+    /**
+     * Set a callback that should be used when generating the ETag.
+     */
+    public static function etagGenerateUsing(?Closure $callback): void
+    {
+        static::$etagGenerateCallback = $callback;
+    }
+
+    /**
+     * Get the ETag value for this request and response.
+     */
+    public static function getEtag(Request $request, Response $response): string
+    {
+        $etag = static::$etagGenerateCallback
+            ? call_user_func(static::$etagGenerateCallback, $request, $response)
+            : static::defaultGetEtag($response);
+
+        return (string) Str::of($etag)->start('"')->finish('"');
+    }
+
+    /**
+     * Generate a default ETag, including user-specific data by default.
+     */
+    private static function defaultGetEtag(Response $response): string
+    {
+        return md5(auth()->id() . $response->getContent());
+    }
+
+    /**
+     * Generate an ETag based on the latest update timestamps from multiple tables.
+     */
+    public static function generateEtagFromTables(array $tables, string $source = 'updated_at'): string
+    {
+        // Fetch the latest update timestamp from each table.
+        // If the source is 'etag_version', use a cached version key as the source of truth.
+        $lastUpdated = collect($tables)->map(function ($table) use ($source) {
+            if ($source === 'etag_version') {
+                /**
+                 * This is not currently implemented but serves as a placeholder for future flexibility.
+                 * The idea is to use a cached version key (e.g., "etag_version_table_name") as the source of truth.
+                 * This would allow us to version the ETag dynamically and invalidate it using model observers or other mechanisms.
+                 * If implemented, observers can increment this version key whenever the corresponding table is updated.
+                 */
+                return Cache::get("etag_version_{$table}", 0);
+            }
+
+            // Default to the updated_at column in the database.
+            return DB::table($table)->max('updated_at');
+        })->max();
+
+        $etag = md5(auth()->id() . $lastUpdated);
+
+        return (string) Str::of($etag)->start('"')->finish('"');
+    }
+}

ProcessMaker/Http/Resources/V1_1/TaskResource.php

@@ -82,7 +82,6 @@ class TaskResource extends ApiResource
             'is_administrator',
             'expires_at',
             'loggedin_at',
-            'active_at',
             'created_at',
             'updated_at',
             'delegation_user_id',

ProcessMaker/Providers/ProcessMakerServiceProvider.php

@@ -12,6 +12,7 @@
 use Illuminate\Support\Facades;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Route;
 use Illuminate\Support\Facades\URL;
 use Laravel\Dusk\DuskServiceProvider;
 use Laravel\Horizon\Horizon;
@@ -21,6 +22,7 @@
 use ProcessMaker\Events\ActivityAssigned;
 use ProcessMaker\Events\ScreenBuilderStarting;
 use ProcessMaker\Helpers\PmHash;
+use ProcessMaker\Http\Middleware\Etag\HandleEtag;
 use ProcessMaker\ImportExport\Extension;
 use ProcessMaker\ImportExport\SignalHelper;
 use ProcessMaker\Jobs\SmartInbox;
@@ -39,10 +41,13 @@ class ProcessMakerServiceProvider extends ServiceProvider
 {
     // Track the start time for service providers boot
     private static $bootStart;
+
     // Track the boot time for service providers
     private static $bootTime;
+
     // Track the boot time for each package
     private static $packageBootTiming = [];
+
     // Track the query time for each request
     private static $queryTime = 0;
 
@@ -67,6 +72,7 @@ public function boot(): void
 
         parent::boot();
 
+        Route::pushMiddlewareToGroup('api', HandleEtag::class);
         // Hook after service providers boot
         self::$bootTime = (microtime(true) - self::$bootStart) * 1000; // Convert to milliseconds
     }
@@ -221,14 +227,14 @@ protected static function registerEvents(): void
             $notifiable = get_class($event->notifiable);
             $notification = get_class($event->notification);
 
-            Facades\Log::debug("Sent Notification to {$notifiable} #{$id}: {$notification}");
+            Log::debug("Sent Notification to {$notifiable} #{$id}: {$notification}");
         });
 
         // Log Broadcasts (messages sent to laravel-echo-server and redis)
         Facades\Event::listen(BroadcastNotificationCreated::class, function ($event) {
             $channels = implode(', ', $event->broadcastOn());
 
-            Facades\Log::debug('Broadcasting Notification ' . $event->broadcastType() . 'on channel(s) ' . $channels);
+            Log::debug('Broadcasting Notification ' . $event->broadcastType() . 'on channel(s) ' . $channels);
         });
 
         // Fire job when task is assigned to a user

ProcessMaker/Providers/RouteServiceProvider.php

@@ -89,7 +89,7 @@ protected function mapApiRoutes()
     {
         Route::middleware('api')
             ->group(base_path('routes/api.php'));
-        Route::middleware('auth:api')
+        Route::middleware(['auth:api', 'etag'])
             ->group(base_path('routes/v1_1/api.php'));
     }
 

ProcessMaker/Traits/TaskResourceIncludes.php

@@ -41,7 +41,12 @@ private function includeUser()
 
     private function includeRequestor()
     {
-        return ['requestor' => new Users($this->processRequest->user)];
+        $user = $this->processRequest->user;
+
+        // Exclude 'active_at' to prevent ETag inconsistencies.
+        $user->makeHidden(['active_at']);
+
+        return ['requestor' => new Users($user)];
     }
 
     private function includeProcessRequest()

config/etag.php

@@ -0,0 +1,47 @@
+<?php
+
+return [
+    /*
+    |--------------------------------------------------------------------------
+    | Enable or Disable ETag Logging
+    |--------------------------------------------------------------------------
+    |
+    | This option allows you to enable or disable the ETag change logging
+    | feature.
+    |
+    */
+    'enabled' => env('ETAG_ENABLED', true),
+
+    /*
+    |--------------------------------------------------------------------------
+    | Log Dynamic Endpoints
+    |--------------------------------------------------------------------------
+    |
+    | Enable or disable logging when an endpoint is detected as dynamic.
+    | If set to false, no logs will be recorded for dynamic endpoints.
+    |
+    */
+    'log_dynamic_endpoints' => env('ETAG_LOG_DYNAMIC_ENDPOINTS', false),
+
+    /*
+    |--------------------------------------------------------------------------
+    | ETag History Limit
+    |--------------------------------------------------------------------------
+    |
+    | The maximum number of ETags to track per endpoint. If the number of
+    | unique ETags exceeds this limit, the oldest ETag will be removed.
+    |
+    */
+    'history_limit' => env('ETAG_HISTORY_LIMIT', 10),
+
+    /*
+    |--------------------------------------------------------------------------
+    | History Cache Expiration Time
+    |--------------------------------------------------------------------------
+    |
+    | The duration (in minutes) for which the ETag history should be stored
+    | in the cache. Adjust this based on your caching strategy.
+    |
+    */
+    'history_cache_expiration' => env('ETAG_HISTORY_CACHE_EXPIRATION_MINUTES', 30),
+];