feat: extends stamps capacity

README.md

@@ -30,6 +30,7 @@ newer Bee versions is not recommended and may not work. Stay up to date by joini
     - [2. Hardcoded postage stamp](#2-hardcoded-postage-stamp)
     - [3. Autobuy postage stamps](#3-autobuy-postage-stamps)
     - [4. Extends stamps TTL](#4-extends-stamps-ttl)
+    - [5. Extends stamps capacity](#5-extends-stamps-capacity)
     - [Enable authentication](#enable-authentication)
   - [Environment variables](#environment-variables)
   - [Curl](#curl)
@@ -56,6 +57,9 @@ The proxy can manage postage stamps for you in 4 modes of operation:
 4. It can extend the TTL of a stamp that is about to expire. To enable this, set `POSTAGE_EXTENDSTTL=true`,
    provide `POSTAGE_AMOUNT`, `POSTAGE_DEPTH` with the desired amount to use and `POSTAGE_TTL_MIN` above with
    a number above or equal to 60.
+5. It can extends the postage stamp capacity to those that are about to be fulfill. To enable this, set
+   `POSTAGE_EXTENDS_CAPACITY=true`. You can also set the env variable `POSTAGE_USAGE_THRESHOLD=0.7` to determine
+   the maximum usage level to check if the stamp needs to be extended
 
 In modes 1, 2 and 3, the proxy can be configured to require authentication secret to forward the requests. Use the
 `AUTH_SECRET` environment variable to enable it.
@@ -112,6 +116,14 @@ export BEE_DEBUG_API_URL=http://localhost:1635
 npm run start
 ```
 
+#### 5. Extends stamps capacity
+
+```sh
+export POSTAGE_EXTENDS_CAPACITY=true
+
+npm run start
+```
+
 #### Reupload pinned content
 
 ```sh
@@ -130,26 +142,28 @@ npm run start
 
 ### Environment variables
 
-| Name                    | Default Value               | Description                                                                                                |
-| ----------------------- | --------------------------- | ---------------------------------------------------------------------------------------------------------- |
-| BEE_API_URL             | http://localhost:1633       | URL of the Bee node API                                                                                    |
-| BEE_DEBUG_API_URL       | http://localhost:1635       | URL of the Bee node Debug API. Required for postage stamps autobuy and hashed identity header.             |
-| AUTH_SECRET             | undefined                   | Authentication secret, disabled if not set (this secret is checked in the request header `authorization`). |
-| HOSTNAME                | localhost                   | Hostname of the proxy. Required for Bzz.link support.                                                      |
-| PORT                    | 3000                        | Port of the proxy.                                                                                         |
-| POSTAGE_STAMP           | undefined                   | Postage stamp that should be used for all upload requests. If provided, the autobuy feature is disabled.   |
-| POSTAGE_DEPTH           | undefined                   | Postage stamp depth to be used when buying new stamps or selecting existing stamps.                        |
-| POSTAGE_AMOUNT          | undefined                   | Postage stamp amount to be used when buying new stamps or selecting existing stamps.                       |
-| POSTAGE_USAGE_THRESHOLD | 0.7                         | Usage percentage at which new postage stamp will be bought (value between 0 and 1).                        |
-| POSTAGE_USAGE_MAX       | 0.9                         | Usage percentage at which existing postage stamp should not be considered viable ( values 0 to 1).         |
-| POSTAGE_TTL_MIN         | `autobuy`: 5 \* POSTAGE_REFRESH_PERIOD. `extends TTL`: undefined | In `autobuy`, Minimal time to live for the postage stamps to still be considered for upload (in seconds). In `extends TTL` is mandatory and required to be above 60 seconds  |
-| POSTAGE_REFRESH_PERIOD  | 60                          | How frequently are the postage stamps checked in seconds.                                                  |
-| CID_SUBDOMAINS          | false                       | Enables Bzz.link subdomain translation functionality for CIDs.                                             |
-| ENS_SUBDOMAINS          | false                       | Enables Bzz.link subdomain translation functionality for ENS.                                              |
-| REMOVE_PIN_HEADER       | true                        | Removes swarm-pin header on all proxy requests.                                                            |
-| `LOG_LEVEL`             | info                        | Log level that is outputted (values: `critical`, `error`, `warn`, `info`, `verbose`, `debug`)              |
-| POSTAGE_EXTENDSTTL      | false                       | Enables extends TTL feature. Works along with POSTAGE_AMOUNT                                               |
-| EXPOSE_HASHED_IDENTITY  | false                       | Exposes `x-bee-node` header, which is the hashed identity of the Bee node for identification purposes      |
+| Name                     | Default Value               | Description                                                                                                |
+| -----------------------  | --------------------------- | ---------------------------------------------------------------------------------------------------------- |
+| BEE_API_URL              | http://localhost:1633       | URL of the Bee node API                                                                                    |
+| BEE_DEBUG_API_URL        | http://localhost:1635       | URL of the Bee node Debug API. Required for postage stamps autobuy and hashed identity header.             |
+| AUTH_SECRET              | undefined                   | Authentication secret, disabled if not set (this secret is checked in the request header `authorization`). |
+| HOSTNAME                 | localhost                   | Hostname of the proxy. Required for Bzz.link support.                                                      |
+| PORT                     | 3000                        | Port of the proxy.                                                                                         |
+| POSTAGE_STAMP            | undefined                   | Postage stamp that should be used for all upload requests. If provided, the autobuy feature is disabled.   |
+| POSTAGE_DEPTH            | undefined                   | Postage stamp depth to be used when buying new stamps or selecting existing stamps.                        |
+| POSTAGE_AMOUNT           | undefined                   | Postage stamp amount to be used when buying new stamps or selecting existing stamps.                       |
+| POSTAGE_USAGE_THRESHOLD  | 0.7                         | In `autobuy` Usage percentage at which new postage stamp will be bought. In `extends capacity` feature is use to identify which stamps have been use more than the set value. Value most be between 0 and 1 |
+| POSTAGE_USAGE_MAX        | 0.9                         | Usage percentage at which existing postage stamp should not be considered viable ( values 0 to 1).         |
+| POSTAGE_TTL_MIN          | `autoplay`: 5 \* POSTAGE_REFRESH_PERIOD. `extends TTL` undefined | In `autobuy`, Minimal time to live for the postage stamps to still be considered for upload (in seconds). In `extends TTL` is mandatory and required to be above 60 seconds  |
+| POSTAGE_REFRESH_PERIOD   | 60                          | How frequently are the postage stamps checked in seconds.                                                  |
+| CID_SUBDOMAINS           | false                       | Enables Bzz.link subdomain translation functionality for CIDs.                                             |
+| ENS_SUBDOMAINS           | false                       | Enables Bzz.link subdomain translation functionality for ENS.                                              |
+| REMOVE_PIN_HEADER        | true                        | Removes swarm-pin header on all proxy requests.                                                            |
+| `LOG_LEVEL`              | info                        | Log level that is outputted (values: `critical`, `error`, `warn`, `info`, `verbose`, `debug`)              |
+| POSTAGE_EXTENDSTTL       | false                       | Enables extending TTL feature. Works along with POSTAGE_AMOUNT                                               |
+| POSTAGE_EXTENDS_CAPACITY | false                       | Enables extending stamp capacity
+feature.                                                |
+| EXPOSE_HASHED_IDENTITY   | false                       | Exposes `x-bee-node` header, which is the hashed identity of the Bee node for identification purposes      |
 | REUPLOAD_PERIOD         | undefined                   | How frequently are the pinned content checked to be reuploaded.                                            |
 
 ### Curl

src/config.ts

@@ -18,11 +18,13 @@ interface StampsConfigHardcoded {
   mode: 'hardcoded'
   stamp: string
 }
+
 export interface StampsConfigExtends {
-  mode: 'extendsTTL'
+  mode: 'extends'
   ttlMin: number
   depth: number
   amount: string
+  usageThreshold: number
   refreshPeriod: number
   beeDebugApiUrl: string
 }
@@ -80,6 +82,7 @@ export type EnvironmentVariables = Partial<{
   POSTAGE_REFRESH_PERIOD: string
   POSTAGE_EXTENDSTTL: string
   REUPLOAD_PERIOD: string
+  POSTAGE_EXTENDS_CAPACITY: string
 }>
 
 export const SUPPORTED_LEVELS = ['critical', 'error', 'warn', 'info', 'verbose', 'debug'] as const
@@ -138,6 +141,7 @@ export function getStampsConfig({
   POSTAGE_TTL_MIN,
   POSTAGE_REFRESH_PERIOD,
   POSTAGE_EXTENDSTTL,
+  POSTAGE_EXTENDS_CAPACITY,
 }: EnvironmentVariables = {}): StampsConfig | undefined {
   const refreshPeriod = Number(POSTAGE_REFRESH_PERIOD || DEFAULT_POSTAGE_REFRESH_PERIOD)
   const beeDebugApiUrl = BEE_DEBUG_API_URL || DEFAULT_BEE_DEBUG_API_URL
@@ -146,30 +150,31 @@ export function getStampsConfig({
   if (POSTAGE_STAMP) return { mode: 'hardcoded', stamp: POSTAGE_STAMP }
   // Start autobuy
   else if (!POSTAGE_EXTENDSTTL && POSTAGE_DEPTH && POSTAGE_AMOUNT && BEE_DEBUG_API_URL) {
-    return {
-      mode: 'autobuy',
-      depth: Number(POSTAGE_DEPTH),
-      amount: POSTAGE_AMOUNT,
-      usageThreshold: Number(POSTAGE_USAGE_THRESHOLD || DEFAULT_POSTAGE_USAGE_THRESHOLD),
-      usageMax: Number(POSTAGE_USAGE_MAX || DEFAULT_POSTAGE_USAGE_MAX),
-      ttlMin: Number(POSTAGE_TTL_MIN || (refreshPeriod / 1000) * 5),
+    return createAutoBuyStampsConfig(
+      POSTAGE_DEPTH,
+      POSTAGE_AMOUNT,
+      POSTAGE_USAGE_THRESHOLD,
+      POSTAGE_USAGE_MAX,
+      POSTAGE_TTL_MIN,
       refreshPeriod,
       beeDebugApiUrl,
-    }
+    )
   } else if (
-    POSTAGE_EXTENDSTTL === 'true' &&
-    POSTAGE_AMOUNT &&
-    POSTAGE_DEPTH &&
-    Number(POSTAGE_TTL_MIN) >= MINIMAL_EXTENDS_TTL_VALUE
+    (POSTAGE_EXTENDSTTL === 'true' &&
+      POSTAGE_AMOUNT &&
+      POSTAGE_DEPTH &&
+      Number(POSTAGE_TTL_MIN) >= MINIMAL_EXTENDS_TTL_VALUE) ||
+    POSTAGE_EXTENDS_CAPACITY === 'true'
   ) {
-    return {
-      mode: 'extendsTTL',
-      depth: Number(POSTAGE_DEPTH),
-      ttlMin: Number(POSTAGE_TTL_MIN),
-      amount: POSTAGE_AMOUNT,
+    return createExtendsConfig(
+      POSTAGE_DEPTH,
+      POSTAGE_TTL_MIN,
+      POSTAGE_AMOUNT,
+      POSTAGE_EXTENDS_CAPACITY,
+      POSTAGE_USAGE_THRESHOLD,
       refreshPeriod,
       beeDebugApiUrl,
-    }
+    )
   }
   // Missing one of the variables needed for the autobuy or extends TTL
   else if (POSTAGE_DEPTH || POSTAGE_AMOUNT || POSTAGE_TTL_MIN || BEE_DEBUG_API_URL) {
@@ -184,6 +189,49 @@ export function getStampsConfig({
   return undefined
 }
 
+export function createAutoBuyStampsConfig(
+  POSTAGE_DEPTH: string,
+  POSTAGE_AMOUNT: string,
+  POSTAGE_USAGE_THRESHOLD: string | undefined,
+  POSTAGE_USAGE_MAX: string | undefined,
+  POSTAGE_TTL_MIN: string | undefined,
+  refreshPeriod: number,
+  beeDebugApiUrl: string,
+): StampsConfigAutobuy {
+  return {
+    mode: 'autobuy',
+    depth: Number(POSTAGE_DEPTH),
+    amount: POSTAGE_AMOUNT,
+    usageThreshold: Number(POSTAGE_USAGE_THRESHOLD || DEFAULT_POSTAGE_USAGE_THRESHOLD),
+    usageMax: Number(POSTAGE_USAGE_MAX || DEFAULT_POSTAGE_USAGE_MAX),
+    ttlMin: Number(POSTAGE_TTL_MIN || (refreshPeriod / 1000) * 5),
+    refreshPeriod,
+    beeDebugApiUrl,
+  }
+}
+
+export function createExtendsConfig(
+  POSTAGE_DEPTH: string | undefined,
+  POSTAGE_TTL_MIN: string | undefined,
+  POSTAGE_AMOUNT: string | undefined,
+  POSTAGE_EXTENDS_CAPACITY: string | undefined,
+  POSTAGE_USAGE_THRESHOLD: string | undefined,
+  refreshPeriod: number,
+  beeDebugApiUrl: string,
+): StampsConfigExtends {
+  return {
+    mode: 'extends',
+    depth: Number(POSTAGE_DEPTH),
+    ttlMin: Number(POSTAGE_TTL_MIN),
+    amount: POSTAGE_AMOUNT ?? '0',
+    usageThreshold: Number(
+      POSTAGE_USAGE_THRESHOLD || (POSTAGE_EXTENDS_CAPACITY === 'true' ? DEFAULT_POSTAGE_USAGE_THRESHOLD : '0'),
+    ),
+    refreshPeriod,
+    beeDebugApiUrl,
+  }
+}
+
 export function getContentConfig({ BEE_API_URL, REUPLOAD_PERIOD }: EnvironmentVariables = {}): ContentConfig | false {
   if (!REUPLOAD_PERIOD) {
     return false

src/content.ts

@@ -14,48 +14,44 @@ export class ContentManager {
   private interval?: ReturnType<typeof setInterval>
   private isReuploading = false
 
-  public async attemptRefreshContentReupload(beeApi: Bee): Promise<void> {
-    try {
-      await this.refreshContentReupload(beeApi)
-    } catch (error) {
-      logger.error('content reupload job failed', error)
-    }
-  }
-
   public async refreshContentReupload(beeApi: Bee): Promise<void> {
-    const pins = await beeApi.getAllPins()
+    try {
+      const pins = await beeApi.getAllPins()
 
-    if (!pins.length) {
-      logger.info(`no pins found`)
+      if (!pins.length) {
+        logger.info(`no pins found`)
 
-      return
-    }
+        return
+      }
 
-    logger.info(`checking pinned content (${pins.length} pins)`)
-    for (const pin of pins) {
-      const isRetrievable = await beeApi.isReferenceRetrievable(pin)
-      logger.debug(`pin ${pin} is ${isRetrievable ? 'retrievable' : 'not retrievable'}`)
+      logger.info(`checking pinned content (${pins.length} pins)`)
+      for (const pin of pins) {
+        const isRetrievable = await beeApi.isReferenceRetrievable(pin)
+        logger.debug(`pin ${pin} is ${isRetrievable ? 'retrievable' : 'not retrievable'}`)
 
-      if (!isRetrievable && !this.isReuploading) {
-        this.isReuploading = true
-        try {
-          logger.debug(`reuploading pinned content: ${pin}`)
-          await beeApi.reuploadPinnedData(pin)
-          contentReuploadCounter.inc()
-          logger.info(`pinned content reuploaded: ${pin}`)
-        } catch (error) {
-          logger.error('failed to reupload pinned content', error)
+        if (!isRetrievable && !this.isReuploading) {
+          this.isReuploading = true
+          try {
+            logger.debug(`reuploading pinned content: ${pin}`)
+            await beeApi.reuploadPinnedData(pin)
+            contentReuploadCounter.inc()
+            logger.info(`pinned content reuploaded: ${pin}`)
+          } catch (error) {
+            logger.error('failed to reupload pinned content', error)
+          }
+          this.isReuploading = false
         }
-        this.isReuploading = false
       }
+    } catch (error) {
+      logger.error('content reupload job failed', error)
     }
   }
 
   /**
    * Start the manager that checks for pinned content availability and reuploads the data if needed.
    */
   start(config: ContentConfig): void {
-    const refreshContent = async () => this.attemptRefreshContentReupload(new Bee(config.beeApiUrl))
+    const refreshContent = async () => this.refreshContentReupload(new Bee(config.beeApiUrl))
     this.stop()
     refreshContent()