[JENKINS-76184] Enable cache for webhook requests to avoid rate limit for large organisations

Add cache of requests for native webhook implementations

Author: nfalco79

.gitignore

@@ -22,3 +22,4 @@ META-INF/
 /.apt_generated/
 /.apt_generated_tests/
 /bin/
+/.asciidoctorconfig.adoc

docs/USER_GUIDE.adoc

@@ -94,8 +94,8 @@ You can setup a custom Jenkins URL to be used as callback URL by the webhooks.
 For Bitbucket Data Center only, it is possible chose which webhooks implementation server side to use:
 
 - Native implementation will configure the webhooks provided by default with the Server, so it will always be available.
-
-- Plugin implementation (*deprecated*) relies on the configuration available via specific APIs provided by the link:https://marketplace.atlassian.com/apps/1215474/post-webhooks-for-bitbucket?tab=overview&hosting=datacenter[Post Webhooks for Bitbucket] plugin itself. To get it worked plugin must be already pre-installed on the server instance. This provider allows custom settings managed by the _ignore committers_ trait. _Note: This specific implementation will be moved to an individual repository as soon as link:https://issues.jenkins.io/browse/JENKINS-74913[JENKINS-74913] is implemented._
+- Plugin implementation (*deprecated*) replaced by https://github.com/jenkinsci/bitbucket-webhooks-plugin[this plugin]
+- Any other extension point implementation installed in your Jenkins instance 
 
 image::images/screenshot-14.png[]
 
@@ -120,6 +120,35 @@ Any incoming webhook payloads from that given endpoint will be validated against
 
 image::images/screenshot-20.png[]
 
+=== Enable webhooks cache
+
+If your organisation has a large number of repositories (over 500), you may easily reach the API rate limit.
+Any requests made to Bitbucket beyond 1,000 per hour will be rejected. In this case, enable caching to allow webhooks from repositories beyond the 500th to be processed within the next hour (when the rate is unlocked).
+
+See https://issues.jenkins.io/browse/JENKINS-76184[JENKINS-76184] for the use case.
+
+image::images/screenshot-23.png[]
+
+=== Manual registration
+
+If your organisation does not allow credentials to handle repository webhooks than you can provide to register webhook manually. You can follow one of these official Atlassian guides: for https://support.atlassian.com/bitbucket-cloud/docs/manage-webhooks[Cloud] or for https://confluence.atlassian.com/bitbucketserver/manage-webhooks-938025878.html[Data Center].
+
+Go to the Bitbucket _Repository_ » _Repository settings_ » _Webhooks than _Add Webhook_
+
+Provide a title of your choice, if you generate a secret for payload verification than confiure signature verification in Bitbucket endpoint as in the previous chapter.
+
+Select events as shown in the image; any other types selected will be ignored.
+
+The callback URL must be configured as follow:
+* Cloud: <Jenkins root URL>/bitbucket-scmsource-hook/notify
+* Server: <Jenkins root URL>/bitbucket-scmsource-hook/notify?server_url=<Bitbucket Data Center URL>
+
+The <Jenkins root URL> must match the Jenkins public host; if Jenkins is behind a reverse proxy, ensure the URL provided matches the one in Manage Jenkins » System » Jenkins Location.
+In other cases, you can provide the Jenkins root URL to use, in the webhook configuration page on the Bitbucket endpoint.
+The <Bitbucket Data Center URL> must match the server address configured in the Bitbucket endpoint. Otherwise, incoming webhooks will be discarded.
+
+image::images/screenshot-24.png[]
+
 [id=bitbucket-creds-config]
 == Credentials configuration
 

docs/images/screenshot-23.png

@@ -113,7 +113,7 @@ public class BitbucketCloudApiClient extends AbstractBitbucketApi implements Bit
     private static final Cache<String, BitbucketTeam> cachedTeam = new Cache<>(6, HOURS);
     private static final Cache<String, List<BitbucketCloudRepository>> cachedRepositories = new Cache<>(3, HOURS);
     private static final Cache<String, BitbucketCloudCommit> cachedCommits = new Cache<>(24, HOURS);
-    private transient BitbucketRepository cachedRepository;
+    private transient BitbucketRepository localCachedRepository;
     private transient String cachedDefaultBranch;
 
     public static List<String> stats() {
@@ -265,14 +265,14 @@ public BitbucketRepository getRepository() throws IOException {
         if (repositoryName == null) {
             throw new UnsupportedOperationException("Cannot get a repository from an API instance that is not associated with a repository");
         }
-        if (!enableCache || cachedRepository == null) {
+        if (!enableCache || localCachedRepository == null) {
             String url = UriTemplate.fromTemplate(REPO_URL_TEMPLATE)
                     .set("owner", owner)
                     .set("repo", repositoryName)
                     .expand();
-            cachedRepository = getRequestAs(url, BitbucketCloudRepository.class);
+            localCachedRepository = getRequestAs(url, BitbucketCloudRepository.class);
         }
-        return cachedRepository;
+        return localCachedRepository;
     }
 
     /**

docs/images/screenshot-24.png

@@ -48,8 +48,8 @@ protected BitbucketApi create(@Nullable String serverUrl, @Nullable BitbucketAut
                 .lookupEndpoint(BitbucketCloudEndpoint.SERVER_URL, BitbucketCloudEndpoint.class)
                 .orElse(null);
         boolean enableCache = false;
-        int teamCacheDuration = 0;
-        int repositoriesCacheDuration = 0;
+        int teamCacheDuration = 360;
+        int repositoriesCacheDuration = 180;
         if (endpoint != null) {
             enableCache = endpoint.isEnableCache();
             teamCacheDuration = endpoint.getTeamCacheDuration();

src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java

@@ -39,6 +39,7 @@
 import hudson.util.ListBoxModel;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.List;
 import jenkins.model.Jenkins;
 import org.apache.commons.lang3.StringUtils;
 import org.jenkinsci.plugins.plaincredentials.StringCredentials;
@@ -84,6 +85,13 @@ public abstract class AbstractBitbucketWebhookConfiguration implements Bitbucket
      */
     private String endpointJenkinsRootURL;
 
+    private boolean enableCache = false;
+
+    /**
+     * How long, in minutes, to cache the webhook response.
+     */
+    private Integer webhooksCacheDuration;
+
     protected AbstractBitbucketWebhookConfiguration(boolean manageHooks, @CheckForNull String credentialsId,
                                        boolean enableHookSignature, @CheckForNull String hookSignatureCredentialsId) {
         this.manageHooks = manageHooks && StringUtils.isNotBlank(credentialsId);
@@ -144,7 +152,47 @@ public String getDisplayName() {
         return Messages.ServerWebhookImplementation_displayName();
     }
 
+    public boolean isEnableCache() {
+        return enableCache;
+    }
+
+    @DataBoundSetter
+    public void setEnableCache(boolean enableCache) {
+        this.enableCache = enableCache;
+    }
+
+    public Integer getWebhooksCacheDuration() {
+        return webhooksCacheDuration;
+    }
+
+    @DataBoundSetter
+    public void setWebhooksCacheDuration(Integer webhooksCacheDuration) {
+        this.webhooksCacheDuration = webhooksCacheDuration == null || webhooksCacheDuration < 0 ? Integer.valueOf(180) : webhooksCacheDuration;
+    }
+
     public abstract static class AbstractBitbucketWebhookDescriptorImpl extends BitbucketWebhookDescriptor {
+        protected abstract void clearCaches();
+        protected abstract List<String> getStats();
+
+        @RequirePOST
+        public FormValidation doShowStats() {
+            Jenkins.get().checkPermission(Jenkins.MANAGE);
+
+            List<String> stats = getStats();
+            StringBuilder builder = new StringBuilder();
+            for (String stat : stats) {
+                builder.append(stat).append("<br>");
+            }
+            return FormValidation.okWithMarkup(builder.toString());
+        }
+
+        @RequirePOST
+        public FormValidation doClear() {
+            Jenkins.get().checkPermission(Jenkins.MANAGE);
+
+            clearCaches();
+            return FormValidation.ok("Caches cleared");
+        }
 
         @RequirePOST
         public static FormValidation doCheckEndpointJenkinsRootURL(@QueryParameter String value) {

src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiFactory.java

@@ -32,6 +32,7 @@
 import edu.umd.cs.findbugs.annotations.NonNull;
 import hudson.Extension;
 import hudson.util.ListBoxModel;
+import java.util.List;
 import jenkins.model.Jenkins;
 import org.jenkinsci.Symbol;
 import org.kohsuke.stapler.DataBoundConstructor;
@@ -69,6 +70,16 @@ public Class<? extends BitbucketWebhookManager> getManager() {
     @Extension
     public static class DescriptorImpl extends AbstractBitbucketWebhookDescriptorImpl {
 
+        @Override
+        protected void clearCaches() {
+            CloudWebhookManager.clearCaches();
+        }
+
+        @Override
+        protected List<String> getStats() {
+            return CloudWebhookManager.stats();
+        }
+
         @Override
         public String getDisplayName() {
             return "Native Cloud";

src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/AbstractBitbucketWebhookConfiguration.java

@@ -24,14 +24,18 @@
 package com.cloudbees.jenkins.plugins.bitbucket.impl.webhook.cloud;
 
 import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketAuthenticatedClient;
+import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketRequestException;
 import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketWebHook;
 import com.cloudbees.jenkins.plugins.bitbucket.api.endpoint.BitbucketEndpoint;
 import com.cloudbees.jenkins.plugins.bitbucket.api.webhook.BitbucketWebhookConfiguration;
 import com.cloudbees.jenkins.plugins.bitbucket.api.webhook.BitbucketWebhookManager;
 import com.cloudbees.jenkins.plugins.bitbucket.client.BitbucketCloudPage;
+import com.cloudbees.jenkins.plugins.bitbucket.client.Cache;
 import com.cloudbees.jenkins.plugins.bitbucket.client.repository.BitbucketCloudWebhook;
 import com.cloudbees.jenkins.plugins.bitbucket.hooks.HookEventType;
+import com.cloudbees.jenkins.plugins.bitbucket.impl.client.ICheckedCallable;
 import com.cloudbees.jenkins.plugins.bitbucket.impl.endpoint.BitbucketCloudEndpoint;
+import com.cloudbees.jenkins.plugins.bitbucket.impl.util.BitbucketApiUtils;
 import com.cloudbees.jenkins.plugins.bitbucket.impl.util.JsonParser;
 import com.cloudbees.jenkins.plugins.bitbucket.util.BitbucketCredentialsUtils;
 import com.damnhandy.uri.template.UriTemplate;
@@ -48,6 +52,7 @@
 import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.concurrent.ExecutionException;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import jenkins.model.Jenkins;
@@ -57,10 +62,25 @@
 import org.apache.commons.lang3.Strings;
 import org.jenkinsci.plugins.plaincredentials.StringCredentials;
 
+import static java.util.concurrent.TimeUnit.HOURS;
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.apache.commons.lang3.StringUtils.upperCase;
+
 @Extension
 public class CloudWebhookManager implements BitbucketWebhookManager {
     private static final String WEBHOOK_URL = "/2.0/repositories{/owner,repo}/hooks{/hook}{?page,pagelen}";
     private static final Logger logger = Logger.getLogger(CloudWebhookManager.class.getName());
+    private static final Cache<String, List<BitbucketWebHook>> cachedRepositoryWebhooks = new Cache<>(3, HOURS);
+
+    public static void clearCaches() {
+        cachedRepositoryWebhooks.evictAll();
+    }
+
+    public static List<String> stats() {
+        List<String> stats = new ArrayList<>();
+        stats.add("Repositories webhooks: " + cachedRepositoryWebhooks.stats().toString());
+        return stats;
+    }
 
     // The list of events available in Bitbucket Cloud.
     private static final List<String> CLOUD_EVENTS = Collections.unmodifiableList(Arrays.asList(
@@ -87,6 +107,9 @@ public void apply(SCMSourceTrait configurationTrait) {
     @Override
     public void apply(BitbucketWebhookConfiguration configuration) {
         this.configuration = (CloudWebhookConfiguration) configuration;
+        if (this.configuration.isEnableCache()) {
+            cachedRepositoryWebhooks.setExpireDuration(this.configuration.getWebhooksCacheDuration(), MINUTES);
+        }
     }
 
     @Override
@@ -105,21 +128,38 @@ public Collection<BitbucketWebHook> read(@NonNull BitbucketAuthenticatedClient c
                 .set("pagelen", 100)
                 .expand();
 
-        List<BitbucketWebHook> resources = new ArrayList<>();
+        ICheckedCallable<List<BitbucketWebHook>, IOException> request = () -> {
+            List<BitbucketWebHook> resources = new ArrayList<>();
 
-        TypeReference<BitbucketCloudPage<BitbucketCloudWebhook>> type = new TypeReference<BitbucketCloudPage<BitbucketCloudWebhook>>(){};
-        BitbucketCloudPage<BitbucketCloudWebhook> page = JsonParser.toJava(client.get(url), type);
-        resources.addAll(page.getValues().stream()
-                .filter(hook -> hook.getUrl().startsWith(endpointJenkinsRootURL))
-                .toList());
-        while (!page.isLastPage()){
-            String response = client.get(page.getNext());
-            page = JsonParser.toJava(response, type);
+            TypeReference<BitbucketCloudPage<BitbucketCloudWebhook>> type = new TypeReference<BitbucketCloudPage<BitbucketCloudWebhook>>(){};
+            BitbucketCloudPage<BitbucketCloudWebhook> page = JsonParser.toJava(client.get(url), type);
             resources.addAll(page.getValues().stream()
                     .filter(hook -> hook.getUrl().startsWith(endpointJenkinsRootURL))
                     .toList());
+            while (!page.isLastPage()){
+                String response = client.get(page.getNext());
+                page = JsonParser.toJava(response, type);
+                resources.addAll(page.getValues().stream()
+                        .filter(hook -> hook.getUrl().startsWith(endpointJenkinsRootURL))
+                        .toList());
+            }
+            return resources;
+        };
+        if (configuration.isEnableCache()) {
+            try {
+                String cacheKey = upperCase(client.getRepositoryOwner()) + "::" + ObjectUtils.firstNonNull(client.getRepositoryName(), "<anonymous>");
+                return cachedRepositoryWebhooks.get(cacheKey, request);
+            } catch (ExecutionException e) {
+                BitbucketRequestException bre = BitbucketApiUtils.unwrap(e);
+                if (bre != null) {
+                    throw bre;
+                } else {
+                    throw new IOException(e);
+                }
+            }
+        } else {
+            return request.call();
         }
-        return resources;
     }
 
     @NonNull

src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/webhook/cloud/CloudWebhookConfiguration.java

@@ -61,6 +61,8 @@ public boolean canHandle(Map<String, String> headers, MultiValuedMap<String, Str
 
     @Override
     public void process(@NonNull String eventType, @NonNull String payload, @NonNull Map<String, Object> context, @NonNull BitbucketEndpoint endpoint) {
+        logger.warning("Plugin webhook is deprecated, it has been replaced by the bitbucket-webhooks-plugin, documentation available at https://github.com/jenkinsci/bitbucket-webhooks-plugin.");
+
         BitbucketPushEvent push = BitbucketServerWebhookPayload.pushEventFromPayload(payload);
         if (push != null) {
             if (push.getChanges().isEmpty()) {