feat(ec2_sd_configs): introduce optional pagination for AWS API calls

.github/workflows/build-docker-image.yml

@@ -0,0 +1,45 @@
+name: Build and Copy 
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - cluster
+      - master
+    paths:
+      - '**.go'
+      - '**/Dockerfile*'
+      - '**/Makefile'
+
+jobs:
+  build-and-copy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image
+        run: |
+            PKG_TAG=1.119.0-ec2-pagination make package-vmagent-amd64
+
+      - name: Get image ID
+        id: image_id
+        run: |
+          IMAGE_ID=$(docker images --format "{{.ID}}" | head -n 1)
+          echo "image_id=$IMAGE_ID" >> $GITHUB_OUTPUT
+
+      - name: Create container
+        run: docker create --name temp_vmagent ${{ steps.image_id.outputs.image_id }}
+
+      - name: Copy /vmagent-prod from container
+        run: docker cp temp_vmagent:/vmagent-prod ./vmagent-prod
+
+      - name: Remove container
+        run: docker rm temp_vmagent
+
+      - name: List copied file
+        run: ls -l ./vmagent-prod

lib/promscrape/discovery/ec2/api.go

@@ -8,10 +8,11 @@ import (
 )
 
 type apiConfig struct {
-	awsConfig       *awsapi.Config
-	instanceFilters []awsapi.Filter
-	azFilters       []awsapi.Filter
-	port            int
+	awsConfig         *awsapi.Config
+	instanceFilters   []awsapi.Filter
+	azFilters         []awsapi.Filter
+	port              int
+	maxResultsPerPage int
 
 	// A map from AZ name to AZ id.
 	azMap     map[string]string
@@ -42,10 +43,11 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
 		return nil, err
 	}
 	cfg := &apiConfig{
-		awsConfig:       awsCfg,
-		instanceFilters: sdc.InstanceFilters,
-		azFilters:       sdc.AZFilters,
-		port:            port,
+		awsConfig:         awsCfg,
+		instanceFilters:   sdc.InstanceFilters,
+		azFilters:         sdc.AZFilters,
+		port:              port,
+		maxResultsPerPage: sdc.MaxResultsPerPage,
 	}
 	return cfg, nil
 }

lib/promscrape/discovery/ec2/ec2.go

@@ -29,9 +29,10 @@ type SDConfig struct {
 	RoleARN string `yaml:"role_arn,omitempty"`
 	// RefreshInterval time.Duration `yaml:"refresh_interval"`
 	// refresh_interval is obtained from `-promscrape.ec2SDCheckInterval` command-line option.
-	Port            *int            `yaml:"port,omitempty"`
-	InstanceFilters []awsapi.Filter `yaml:"filters,omitempty"`
-	AZFilters       []awsapi.Filter `yaml:"az_filters,omitempty"`
+	Port              *int            `yaml:"port,omitempty"`
+	InstanceFilters   []awsapi.Filter `yaml:"filters,omitempty"`
+	AZFilters         []awsapi.Filter `yaml:"az_filters,omitempty"`
+	MaxResultsPerPage int             `yaml:"max_results_per_page,omitempty"`
 }
 
 // GetLabels returns ec2 labels according to sdc.

lib/promscrape/discovery/ec2/instance.go

@@ -32,6 +32,15 @@ func getReservations(cfg *apiConfig) ([]Reservation, error) {
 	var rs []Reservation
 	pageToken := ""
 	instanceFilters := awsapi.GetFiltersQueryString(cfg.instanceFilters)
+
+	if cfg.maxResultsPerPage > 0 {
+		if len(instanceFilters) == 0 {
+			instanceFilters = fmt.Sprintf("MaxResults=%d", cfg.maxResultsPerPage)
+		} else {
+			instanceFilters += fmt.Sprintf("&MaxResults=%d", cfg.maxResultsPerPage)
+		}
+	}
+
 	for {
 		data, err := cfg.awsConfig.GetEC2APIResponse("DescribeInstances", instanceFilters, pageToken)
 		if err != nil {