feat: Add proactive rate limiting implementation

examples/provider/provider_with_rate_limiting.tf

@@ -0,0 +1,36 @@
+# Configure the Auth0 Provider with rate limiting
+provider "auth0" {
+  domain        = var.auth0_domain
+  client_id     = var.auth0_client_id
+  client_secret = var.auth0_client_secret
+
+  # Set the maximum API capacity percentage to use
+  # This prevents hitting Auth0 rate limits by proactively throttling requests
+  # when the provider reaches 70% of the available rate limit capacity
+  max_api_capacity = 70
+
+  # Enable debug mode to see rate limiting logs
+  debug = true
+}
+
+# Example resources that will benefit from rate limiting
+resource "auth0_client" "my_client" {
+  name            = "My Application"
+  description     = "My Application Description"
+  app_type        = "spa"
+  callbacks       = ["https://example.com/callback"]
+  allowed_origins = ["https://example.com"]
+}
+
+resource "auth0_user" "users" {
+  count = 100 # Creating many users will benefit from rate limiting
+
+  connection_name = "Username-Password-Authentication"
+  email          = "user${count.index}@example.com"
+  password       = "passpass$WORD1"
+
+  depends_on = [auth0_client.my_client]
+}
+
+# Environment variable alternative:
+# export AUTH0_MAX_API_CAPACITY=70

internal/config/client_retry_test.go

@@ -33,7 +33,7 @@ func TestCustomClientWithRetries(t *testing.T) {
 			writer.WriteHeader(200)
 		}))
 
-		client := customClientWithRetries()
+		client := customClientWithRetries(100, false)
 
 		request, err := http.NewRequest(http.MethodGet, testServer.URL, nil)
 		require.NoError(t, err)
@@ -67,7 +67,7 @@ func TestCustomClientWithRetries(t *testing.T) {
 			writer.WriteHeader(200)
 		}))
 
-		client := customClientWithRetries()
+		client := customClientWithRetries(100, false)
 
 		request, err := http.NewRequest(http.MethodGet, testServer.URL, nil)
 		require.NoError(t, err)
@@ -93,7 +93,7 @@ func TestCustomClientWithRetries(t *testing.T) {
 			writer.WriteHeader(500)
 		}))
 
-		client := customClientWithRetries()
+		client := customClientWithRetries(100, false)
 
 		request, err := http.NewRequest(http.MethodGet, testServer.URL, nil)
 		require.NoError(t, err)

internal/config/config.go

@@ -24,6 +24,9 @@ import (
 	"github.com/zalando/go-keyring"
 
 	"github.com/auth0/terraform-provider-auth0/internal/mutex"
+	"github.com/auth0/terraform-provider-auth0/internal/ratelimit"
+	"github.com/auth0/terraform-provider-auth0/internal/transport"
+	"github.com/hashicorp/go-hclog"
 )
 
 const providerName = "Terraform-Provider-Auth0"    // #nosec G101
@@ -68,6 +71,7 @@ type authConfig struct {
 	clientAssertionPrivateKey string
 	clientAssertionSigningAlg string
 	customDomainHeader        string
+	maxAPICapacity            int
 }
 
 // ConfigureProvider will configure the *schema.Provider so that
@@ -83,6 +87,7 @@ func ConfigureProvider(terraformVersion *string) schema.ConfigureContextFunc {
 			clientAssertionPrivateKey: data.Get("client_assertion_private_key").(string),
 			clientAssertionSigningAlg: data.Get("client_assertion_signing_alg").(string),
 			customDomainHeader:        data.Get("custom_domain_header").(string),
+			maxAPICapacity:            data.Get("max_api_capacity").(int),
 		}
 
 		domain := data.Get("domain").(string)
@@ -160,7 +165,7 @@ func ConfigureProvider(terraformVersion *string) schema.ConfigureContextFunc {
 			management.WithUserAgent(userAgent(terraformVersion)),
 			management.WithAuth0ClientEnvEntry(providerName, version),
 			management.WithNoRetries(),
-			management.WithClient(customClientWithRetries()),
+			management.WithClient(customClientWithRetries(config.maxAPICapacity, debug)),
 			management.WithCustomDomainHeader(config.customDomainHeader))
 
 		if err != nil {
@@ -305,13 +310,32 @@ func authenticationOption(cfg authConfig) management.Option {
 	}
 }
 
-func customClientWithRetries() *http.Client {
+func customClientWithRetries(maxAPICapacity int, debug bool) *http.Client {
+	baseTransport := retryableErrorTransport(http.DefaultTransport)
+
+	// Apply rate limiting if maxAPICapacity is less than 100%
+	if maxAPICapacity > 0 && maxAPICapacity < 100 {
+		// Create a simple logger for the transport
+		logger := hclog.New(&hclog.LoggerOptions{
+			Name:  "auth0-rate-limiter",
+			Level: hclog.Info,
+		})
+		if debug {
+			logger.SetLevel(hclog.Debug)
+		}
+
+		rateLimitManager, err := ratelimit.NewRateLimitManager(maxAPICapacity)
+		if err != nil {
+			// If we can't create the rate limit manager, fall back to basic rate limiting
+			logger.Error("Failed to create rate limit manager, falling back to basic rate limiting", "error", err)
+		} else {
+			logger.Info(fmt.Sprintf("Auth0 provider running with max_api_capacity configuration at %d%%", maxAPICapacity))
+			baseTransport = transport.NewGovernedTransport(baseTransport, rateLimitManager, logger)
+		}
+	}
+
 	client := &http.Client{
-		Transport: rateLimitTransport(
-			retryableErrorTransport(
-				http.DefaultTransport,
-			),
-		),
+		Transport: rateLimitTransport(baseTransport),
 	}
 
 	return client

internal/provider/provider.go

@@ -1,6 +1,7 @@
 package provider
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/auth0/terraform-provider-auth0/internal/auth0/networkacl"
@@ -146,6 +147,23 @@ func New() *schema.Provider {
 				Description: "When specified, this header is added to requests targeting a set of pre-defined whitelisted URLs " +
 					"Global setting overrides all resource specific `custom_domain_header` value",
 			},
+			"max_api_capacity": {
+				Type:        schema.TypeInt,
+				Optional:    true,
+				DefaultFunc: schema.EnvDefaultFunc("AUTH0_MAX_API_CAPACITY", 100),
+				ValidateFunc: func(val interface{}, key string) (warns []string, errs []error) {
+					v := val.(int)
+					if v < 1 || v > 100 {
+						errs = append(errs, fmt.Errorf("%q must be between 1 and 100, got: %d", key, v))
+					}
+					return
+				},
+				Description: "Sets what percentage of capacity the provider can use of the total rate limit " +
+					"capacity while making calls to the Auth0 management API endpoints. Auth0 API operates with " +
+					"rate limits per endpoint. See Auth0 Rate Limit Policy: " +
+					"https://auth0.com/docs/troubleshoot/product-lifecycle/rate-limit-policy. " +
+					"It can also be sourced from the `AUTH0_MAX_API_CAPACITY` environment variable.",
+			},
 		},
 		ResourcesMap: map[string]*schema.Resource{
 			"auth0_action":                           action.NewResource(),