Feature/extends capacity refactor

README.md

@@ -30,6 +30,7 @@ newer Bee versions is not recommended and may not work. Stay up to date by joini
     - [2. Hardcoded postage stamp](#2-hardcoded-postage-stamp)
     - [3. Autobuy postage stamps](#3-autobuy-postage-stamps)
     - [4. Extends stamps TTL](#4-extends-stamps-ttl)
+    - [5. Extends stamps capacity](#5-extends-stamps-capacity)
     - [Enable authentication](#enable-authentication)
   - [Environment variables](#environment-variables)
   - [Curl](#curl)
@@ -56,6 +57,9 @@ The proxy can manage postage stamps for you in 4 modes of operation:
 4. It can extend the TTL of a stamp that is about to expire. To enable this, set `POSTAGE_EXTENDSTTL=true`,
    provide `POSTAGE_AMOUNT`, `POSTAGE_DEPTH` with the desired amount to use and `POSTAGE_TTL_MIN` above with
    a number above or equal to 60.
+5. It can extends the postage stamp capacity to those that are about to be fulfill. To enable this, set
+   `POSTAGE_EXTENDS_CAPACITY=true`. You can also set the env variable `POSTAGE_USAGE_THRESHOLD=0.7` to determine
+   the maximum usage level to check if the stamp needs to be extended
 
 In modes 1, 2 and 3, the proxy can be configured to require authentication secret to forward the requests. Use the
 `AUTH_SECRET` environment variable to enable it.
@@ -95,7 +99,7 @@ npm run start
 ```sh
 export POSTAGE_DEPTH=20
 export POSTAGE_AMOUNT=1000000
-export BEE_DEBUG_API_URL=http://localhost:1635
+export BEE_DEBUG_API_URL=http://localhost:1635 (optional)
 
 npm run start
 ```
@@ -107,7 +111,15 @@ export POSTAGE_EXTENDSTTL=true
 export POSTAGE_TTL_MIN=60
 export POSTAGE_DEPTH=20
 export POSTAGE_AMOUNT=1000000
-export BEE_DEBUG_API_URL=http://localhost:1635
+export BEE_DEBUG_API_URL=http://localhost:1635 (optional)
+
+npm run start
+```
+
+#### 5. Extends stamps capacity
+
+```sh
+export POSTAGE_EXTENDS_CAPACITY=true
 
 npm run start
 ```
@@ -149,6 +161,8 @@ npm run start
 | REMOVE_PIN_HEADER       | true                        | Removes swarm-pin header on all proxy requests.                                                            |
 | `LOG_LEVEL`             | info                        | Log level that is outputted (values: `critical`, `error`, `warn`, `info`, `verbose`, `debug`)              |
 | POSTAGE_EXTENDSTTL      | false                       | Enables extends TTL feature. Works along with POSTAGE_AMOUNT                                               |
+| POSTAGE_EXTENDS_CAPACITY | false                       | Enables extending stamp capacity
+feature.                                                |
 | EXPOSE_HASHED_IDENTITY  | false                       | Exposes `x-bee-node` header, which is the hashed identity of the Bee node for identification purposes      |
 | REUPLOAD_PERIOD         | undefined                   | How frequently are the pinned content checked to be reuploaded.                                            |
 

src/config.ts

@@ -1,3 +1,5 @@
+import { isInteger, assertInteger, assertDecimal } from './utils'
+
 export interface AppConfig {
   beeApiUrl: string
   beeDebugApiUrl: string
@@ -14,32 +16,36 @@ export interface ServerConfig {
   port: number
 }
 
-interface StampsConfigHardcoded {
+export interface StampsConfigHardcoded {
   mode: 'hardcoded'
   stamp: string
 }
-export interface StampsConfigExtends {
-  mode: 'extendsTTL'
+
+export interface StampsConfigAutobuy {
+  mode: 'autobuy'
   ttlMin: number
   depth: number
   amount: string
+  usageThreshold: number
   refreshPeriod: number
+  usageMax: number
   beeDebugApiUrl: string
 }
 
-export interface ContentConfigReupload {
-  beeApiUrl: string
-  refreshPeriod: number
-}
-
-export interface StampsConfigAutobuy {
-  mode: 'autobuy'
+export interface StampsConfigExtends {
+  mode: 'extends'
+  enableTtl: boolean
+  enableCapacity: boolean
+  ttlMin: number
   depth: number
   amount: string
-  beeDebugApiUrl: string
   usageThreshold: number
-  usageMax: number
-  ttlMin: number
+  refreshPeriod: number
+  beeDebugApiUrl: string
+}
+
+export interface ContentConfigReupload {
+  beeApiUrl: string
   refreshPeriod: number
 }
 
@@ -80,6 +86,7 @@ export type EnvironmentVariables = Partial<{
   POSTAGE_REFRESH_PERIOD: string
   POSTAGE_EXTENDSTTL: string
   REUPLOAD_PERIOD: string
+  POSTAGE_EXTENDS_CAPACITY: string
 }>
 
 export const SUPPORTED_LEVELS = ['critical', 'error', 'warn', 'info', 'verbose', 'debug'] as const
@@ -138,52 +145,129 @@ export function getStampsConfig({
   POSTAGE_TTL_MIN,
   POSTAGE_REFRESH_PERIOD,
   POSTAGE_EXTENDSTTL,
+  POSTAGE_EXTENDS_CAPACITY,
 }: EnvironmentVariables = {}): StampsConfig | undefined {
+  if (POSTAGE_REFRESH_PERIOD) {
+    assertInteger(POSTAGE_REFRESH_PERIOD)
+  }
+
   const refreshPeriod = Number(POSTAGE_REFRESH_PERIOD || DEFAULT_POSTAGE_REFRESH_PERIOD)
   const beeDebugApiUrl = BEE_DEBUG_API_URL || DEFAULT_BEE_DEBUG_API_URL
 
   // Start in hardcoded mode
   if (POSTAGE_STAMP) return { mode: 'hardcoded', stamp: POSTAGE_STAMP }
   // Start autobuy
-  else if (!POSTAGE_EXTENDSTTL && POSTAGE_DEPTH && POSTAGE_AMOUNT && BEE_DEBUG_API_URL) {
-    return {
-      mode: 'autobuy',
-      depth: Number(POSTAGE_DEPTH),
-      amount: POSTAGE_AMOUNT,
-      usageThreshold: Number(POSTAGE_USAGE_THRESHOLD || DEFAULT_POSTAGE_USAGE_THRESHOLD),
-      usageMax: Number(POSTAGE_USAGE_MAX || DEFAULT_POSTAGE_USAGE_MAX),
-      ttlMin: Number(POSTAGE_TTL_MIN || (refreshPeriod / 1000) * 5),
+  else if (POSTAGE_EXTENDSTTL === 'true' || POSTAGE_EXTENDS_CAPACITY === 'true') {
+    return createExtendsStampsConfig(
+      POSTAGE_EXTENDSTTL,
+      POSTAGE_EXTENDS_CAPACITY,
+      POSTAGE_AMOUNT,
+      POSTAGE_USAGE_THRESHOLD,
+      POSTAGE_TTL_MIN,
+      POSTAGE_DEPTH,
       refreshPeriod,
       beeDebugApiUrl,
-    }
-  } else if (
-    POSTAGE_EXTENDSTTL === 'true' &&
-    POSTAGE_AMOUNT &&
-    POSTAGE_DEPTH &&
-    Number(POSTAGE_TTL_MIN) >= MINIMAL_EXTENDS_TTL_VALUE
-  ) {
-    return {
-      mode: 'extendsTTL',
-      depth: Number(POSTAGE_DEPTH),
-      ttlMin: Number(POSTAGE_TTL_MIN),
-      amount: POSTAGE_AMOUNT,
+    )
+  } else if (POSTAGE_DEPTH && POSTAGE_AMOUNT) {
+    return createAutobuyStampsConfig(
+      POSTAGE_DEPTH,
+      POSTAGE_AMOUNT,
+      POSTAGE_USAGE_THRESHOLD,
+      POSTAGE_USAGE_MAX,
+      POSTAGE_TTL_MIN,
       refreshPeriod,
       beeDebugApiUrl,
-    }
+    )
   }
   // Missing one of the variables needed for the autobuy or extends TTL
-  else if (POSTAGE_DEPTH || POSTAGE_AMOUNT || POSTAGE_TTL_MIN || BEE_DEBUG_API_URL) {
+  else if (POSTAGE_DEPTH || POSTAGE_AMOUNT || POSTAGE_TTL_MIN) {
     throw new Error(
-      `config: please provide POSTAGE_DEPTH=${POSTAGE_DEPTH}, POSTAGE_AMOUNT=${POSTAGE_AMOUNT}, POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN} ${
-        POSTAGE_EXTENDSTTL === 'true' ? 'at least 60 seconds ' : ''
-      }or BEE_DEBUG_API_URL=${BEE_DEBUG_API_URL} for the feature to work`,
+      `config: please provide POSTAGE_DEPTH=${POSTAGE_DEPTH}, POSTAGE_AMOUNT=${POSTAGE_AMOUNT} or POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN} for the feature to work`,
     )
   }
 
   // Stamps rewrite is disabled
   return undefined
 }
 
+export function createAutobuyStampsConfig(
+  POSTAGE_DEPTH: string,
+  POSTAGE_AMOUNT: string,
+  POSTAGE_USAGE_THRESHOLD: string | undefined,
+  POSTAGE_USAGE_MAX: string | undefined,
+  POSTAGE_TTL_MIN: string | undefined,
+  refreshPeriod: number,
+  beeDebugApiUrl: string,
+): StampsConfigAutobuy {
+  // Missing one of the variables needed for the autobuy
+  if (!isInteger(POSTAGE_DEPTH) || !isInteger(POSTAGE_AMOUNT)) {
+    throw new Error(
+      `config: please provide valid values for POSTAGE_DEPTH, POSTAGE_AMOUNT for the autobuy feature to work.
+      Current state are POSTAGE_DEPTH=${POSTAGE_DEPTH} and POSTAGE_AMOUNT=${POSTAGE_AMOUNT}`,
+    )
+  }
+
+  return {
+    mode: 'autobuy',
+    depth: Number(POSTAGE_DEPTH),
+    amount: POSTAGE_AMOUNT,
+    usageThreshold: Number(POSTAGE_USAGE_THRESHOLD || DEFAULT_POSTAGE_USAGE_THRESHOLD),
+    usageMax: Number(POSTAGE_USAGE_MAX || DEFAULT_POSTAGE_USAGE_MAX),
+    ttlMin: Number(POSTAGE_TTL_MIN || (refreshPeriod / 1000) * 5),
+    refreshPeriod,
+    beeDebugApiUrl,
+  }
+}
+
+export function createExtendsStampsConfig(
+  POSTAGE_EXTENDSTTL: string | undefined,
+  POSTAGE_EXTENDS_CAPACITY: string | undefined,
+  POSTAGE_AMOUNT: string | undefined,
+  POSTAGE_USAGE_THRESHOLD: string | undefined,
+  POSTAGE_TTL_MIN: string | undefined,
+  POSTAGE_DEPTH: string | undefined,
+  refreshPeriod: number,
+  beeDebugApiUrl: string,
+): StampsConfigExtends {
+  if (
+    POSTAGE_EXTENDSTTL === 'true' &&
+    (Number(POSTAGE_TTL_MIN) < MINIMAL_EXTENDS_TTL_VALUE ||
+      !isInteger(POSTAGE_AMOUNT) ||
+      (POSTAGE_TTL_MIN && !isInteger(POSTAGE_TTL_MIN)) ||
+      (POSTAGE_DEPTH && !isInteger(POSTAGE_DEPTH)))
+  ) {
+    throw new Error(
+      `config: to extends stamps TTL please provide POSTAGE_TTL_MIN bigger than ${MINIMAL_EXTENDS_TTL_VALUE}, valid values for
+      POSTAGE_AMOUNT, POSTAGE_TTL_MIN, POSTAGE_DEPTH. Current states are POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN},
+      POSTAGE_AMOUNT=${POSTAGE_AMOUNT} and POSTAGE_DEPTH=${POSTAGE_DEPTH}`,
+    )
+  }
+
+  if (POSTAGE_EXTENDS_CAPACITY === 'true' && POSTAGE_USAGE_THRESHOLD && !assertDecimal(POSTAGE_USAGE_THRESHOLD)) {
+    throw new Error(
+      `config: to extends capacity please provide valid number for POSTAGE_USAGE_THRESHOLD. Current states is
+      POSTAGE_USAGE_THRESHOLD=${POSTAGE_USAGE_THRESHOLD}`,
+    )
+  }
+
+  const amount = POSTAGE_AMOUNT || '0'
+  const usageThreshold = Number(POSTAGE_USAGE_THRESHOLD || DEFAULT_POSTAGE_USAGE_THRESHOLD)
+  const ttlMin = Number(POSTAGE_TTL_MIN)
+  const depth = Number(POSTAGE_DEPTH)
+
+  return {
+    mode: 'extends',
+    enableTtl: POSTAGE_EXTENDSTTL === 'true',
+    enableCapacity: POSTAGE_EXTENDS_CAPACITY === 'true',
+    depth,
+    ttlMin,
+    amount,
+    usageThreshold,
+    refreshPeriod,
+    beeDebugApiUrl,
+  }
+}
+
 export function getContentConfig({ BEE_API_URL, REUPLOAD_PERIOD }: EnvironmentVariables = {}): ContentConfig | false {
   if (!REUPLOAD_PERIOD) {
     return false

src/content.ts

@@ -14,48 +14,45 @@ export class ContentManager {
   private interval?: ReturnType<typeof setInterval>
   private isReuploading = false
 
-  public async attemptRefreshContentReupload(beeApi: Bee): Promise<void> {
-    try {
-      await this.refreshContentReupload(beeApi)
-    } catch (error) {
-      logger.error('content reupload job failed', error)
-    }
-  }
-
   public async refreshContentReupload(beeApi: Bee): Promise<void> {
-    const pins = await beeApi.getAllPins()
+    try {
+      const pins = await beeApi.getAllPins()
 
-    if (!pins.length) {
-      logger.info(`no pins found`)
+      if (!pins.length) {
+        logger.info(`no pins found`)
 
-      return
-    }
+        return
+      }
 
-    logger.info(`checking pinned content (${pins.length} pins)`)
-    for (const pin of pins) {
-      const isRetrievable = await beeApi.isReferenceRetrievable(pin)
-      logger.debug(`pin ${pin} is ${isRetrievable ? 'retrievable' : 'not retrievable'}`)
+      logger.info(`checking pinned content (${pins.length} pins)`)
+      for (const pin of pins) {
+        const isRetrievable = await beeApi.isReferenceRetrievable(pin)
+        logger.debug(`pin ${pin} is ${isRetrievable ? 'retrievable' : 'not retrievable'}`)
 
-      if (!isRetrievable && !this.isReuploading) {
-        this.isReuploading = true
-        try {
-          logger.debug(`reuploading pinned content: ${pin}`)
-          await beeApi.reuploadPinnedData(pin)
-          contentReuploadCounter.inc()
-          logger.info(`pinned content reuploaded: ${pin}`)
-        } catch (error) {
-          logger.error('failed to reupload pinned content', error)
+        if (!isRetrievable && !this.isReuploading) {
+          this.isReuploading = true
+          try {
+            logger.debug(`reuploading pinned content: ${pin}`)
+            await beeApi.reuploadPinnedData(pin)
+            contentReuploadCounter.inc()
+            logger.info(`pinned content reuploaded: ${pin}`)
+          } catch (error) {
+            logger.error('failed to reupload pinned content', error)
+          }
+          this.isReuploading = false
         }
-        this.isReuploading = false
       }
+    } catch (error) {
+      logger.error('content reupload job failed', error)
     }
   }
 
   /**
    * Start the manager that checks for pinned content availability and reuploads the data if needed.
    */
   start(config: ContentConfig): void {
-    const refreshContent = async () => this.attemptRefreshContentReupload(new Bee(config.beeApiUrl))
+    const bee = new Bee(config.beeApiUrl)
+    const refreshContent = async () => this.refreshContentReupload(bee)
     this.stop()
     refreshContent()
 

src/index.ts

@@ -2,10 +2,13 @@
 import { Application } from 'express'
 
 import { createApp } from './server'
-import { StampsManager } from './stamps'
+import { AutoBuyStampsManager, ExtendsStampManager } from './stamps'
 import { getAppConfig, getServerConfig, getStampsConfig, EnvironmentVariables, getContentConfig } from './config'
 import { logger, subscribeLogServerRequests } from './logger'
 import { ContentManager } from './content'
+import type { StampsManager } from './stamps'
+import { BeeDebug } from '@ethersphere/bee-js'
+import { BaseStampManager } from './stamps/base'
 
 async function main() {
   // Configuration
@@ -28,9 +31,22 @@ async function main() {
 
   if (stampsConfig) {
     logger.debug('stamps config', stampsConfig)
-    const stampManager = new StampsManager()
-    logger.info('starting postage stamp manager')
-    stampManager.start(stampsConfig)
+    let stampManager: StampsManager
+    const { mode } = stampsConfig
+
+    if (mode === 'hardcoded') {
+      stampManager = new BaseStampManager()
+      stampManager.start(stampsConfig)
+      logger.info('starting hardcoded postage stamp manager')
+    } else if (mode === 'autobuy') {
+      logger.info('starting autobuy postage stamp manager')
+      stampManager = new AutoBuyStampsManager(new BeeDebug(stampsConfig.beeDebugApiUrl))
+      stampManager.start(stampsConfig, async () => (stampManager as AutoBuyStampsManager).refreshStamps(stampsConfig))
+    } else {
+      logger.info('starting extends postage stamp manager')
+      stampManager = new ExtendsStampManager(new BeeDebug(stampsConfig.beeDebugApiUrl))
+      stampManager.start(stampsConfig, async () => (stampManager as ExtendsStampManager).refreshStamps(stampsConfig))
+    }
     logger.info('starting the proxy')
     app = createApp(appConfig, stampManager)
   } else {