diff --git a/README.md b/README.md index 9991168..accfff3 100644 --- a/README.md +++ b/README.md @@ -20,117 +20,7 @@ This HTTP client is intended to be used with targetted SDK's and terraform provi - **API Handler Interface**: Provides a flexible and extensible way to interact with different APIs, including encoding and decoding requests and responses, managing authentication endpoints, and handling API-specific logic. - **Configuration via JSON or Environment Variables**: The Go API HTTP Client supports configuration via JSON files or environment variables, providing flexibility in defining authentication credentials, API endpoints, logging settings, and other parameters. -- **Cookie Jar Support**: Incorporates an optional cookie jar to manage cookies effortlessly across requests, enhancing session management and statefulness with APIs that require cookie-based authentication or tracking. This feature allows for automatic storage and sending of cookies with subsequent requests, mirroring browser-like interaction with web services. It can be enabled or disabled based on configuration, providing flexibility in how stateful interactions are handled with the target API. - -## API Handler - -The `APIHandler` interface abstracts the functionality needed to interact with various APIs, making the HTTP client adaptable to different API implementations. It includes methods for constructing resource and authentication endpoints, marshaling requests, handling responses, and managing API-specific headers. - -### Implementations - -Currently, the HTTP client supports the following API handlers: - -- **Jamf Pro**: Tailored for interacting with Jamf Pro's API, providing specialized methods for device management and configuration. -- **Microsoft Graph**: Designed for Microsoft Graph API, enabling access to various Microsoft 365 services. - -## Getting Started - -## HTTP Client Build Flow - -The HTTP client build flow can be initiated using a number of methods. The primary methods include: - -Using the SDK `BuildClientWithConfigFile` function, which reads the configuration from a JSON file and constructs the client accordingly. The configuration file specifies the authentication details, API environment settings, and client options, such as logging level, retry attempts, and concurrency limits. - -Or using the SDK `BuildClientWithEnvironmentVariables` function, which reads the configuration from environment variables and constructs the client accordingly. This method allows for more flexible configuration management, particularly in containerized environments or when using orchestration tools. - -There is also the option to the build the client manually by creating a new `Client` struct and setting the required fields directly. This method provides the most granular control over the client configuration and can be useful for advanced use cases or when integrating with existing configuration management systems. This is the approached used in related terraform providers. - -![HTTP Client Build Flow](docs/media/BuildClient.png) - -### Installation - -To use this HTTP client in your project, add the package to your Go module dependencies: - -```bash -go get github.com/yourusername/go-api-http-client -``` - -### Usage - -Example usage with a configuration file using the jamfpro SDK client builder function: - -```go -package main - -import ( - "encoding/xml" - "fmt" - "log" - - "github.com/deploymenttheory/go-api-sdk-jamfpro/sdk/jamfpro" -) - -func main() { - // Define the path to the JSON configuration file - configFilePath := "/path/to/your/clientconfig.json" - - // Initialize the Jamf Pro client with the HTTP client configuration - client, err := jamfpro.BuildClientWithConfigFile(configFilePath) - if err != nil { - log.Fatalf("Failed to initialize Jamf Pro client: %v", err) - } -} - -``` - -Example configuration file (clientconfig.json): - -```json -{ - "Auth": { - "ClientID": "client-id", // set this for oauth2 based authentication - "ClientSecret": "client-secret", // set this for oauth2 based authentication - "Username": "username", // set this for basic auth - "Password": "password" // set this for basic auth - }, - "Environment": { - "APIType": "", // define the api integration e.g "jamfpro" / "msgraph" - "InstanceName": "yourinstance", // used for "jamfpro" - "OverrideBaseDomain": "", // used for "jamfpro" - "TenantID": "tenant-id", // used for "msgraph"h - "TenantName ": "resource", // used for "msgraph" - }, - "ClientOptions": { - "Logging": { - "LogLevel": "LogLevelDebug", // "LogLevelDebug" / "LogLevelInfo" / "LogLevelWarn" / "LogLevelError" / "LogLevelFatal" / "LogLevelPanic" - "LogOutputFormat": "console", // "console" / "json" - "LogConsoleSeparator": " ", // " " / "\t" / "," / etc. - "LogExportPath": "/your/log/path/folder", - "HideSensitiveData": true // redacts sensitive data from logs - }, - "Cookies": { - "EnableCookieJar": true, // enable cookie jar support - "CustomCookies": { // set custom cookies as an alternative to cookie jar - "sessionId": "abc123", - "authToken": "xyz789" - } - }, - "Retry": { - "MaxRetryAttempts": 5, // set number of retry attempts - "EnableDynamicRateLimiting": true // enable dynamic rate limiting - }, - "Concurrency": { - "MaxConcurrentRequests": 3 // set number of concurrent requests - }, - "Redirect": { - "FollowRedirects": true, // follow redirects - "MaxRedirects": 5 // set number of redirects to follow - } - } -} -``` - - +TBC ## Reporting Issues and Feedback diff --git a/apiintegrations/apihandler/apihandler.go b/apiintegrations/apihandler/apihandler.go deleted file mode 100644 index 2572585..0000000 --- a/apiintegrations/apihandler/apihandler.go +++ /dev/null @@ -1,55 +0,0 @@ -// apiintegrations/apihandler/apihandler.go -package apihandler - -import ( - "github.com/deploymenttheory/go-api-http-client/apiintegrations/jamfpro" - "github.com/deploymenttheory/go-api-http-client/apiintegrations/msgraph" - "github.com/deploymenttheory/go-api-http-client/logger" - "go.uber.org/zap" -) - -// APIHandler is an interface for encoding, decoding, and implenting contexual api functions for different API implementations. -// It encapsulates behavior for encoding and decoding requests and responses. -type APIHandler interface { - ConstructAPIResourceEndpoint(endpointPath string, log logger.Logger) string - ConstructAPIAuthEndpoint(endpointPath string, log logger.Logger) string - MarshalRequest(body interface{}, method string, endpoint string, log logger.Logger) ([]byte, error) - MarshalMultipartRequest(formFields map[string]string, fileContents map[string][]byte, log logger.Logger) ([]byte, string, string, error) - GetContentTypeHeader(method string, log logger.Logger) string - GetAcceptHeader() string - GetDefaultBaseDomain() string - GetOAuthTokenEndpoint() string - GetOAuthTokenScope() string - GetBearerTokenEndpoint() string - GetTokenRefreshEndpoint() string - GetTokenInvalidateEndpoint() string - GetAPIBearerTokenAuthenticationSupportStatus() bool - GetAPIOAuthAuthenticationSupportStatus() bool - GetAPIOAuthWithCertAuthenticationSupportStatus() bool - GetAPIRequestHeaders(endpoint string) map[string]string // Provides standard headers required for making API requests. -} - -// LoadAPIHandler loads the appropriate API handler based on the API type. -func LoadAPIHandler(apiType, instanceName, tenantID, tenantName string, log logger.Logger) (APIHandler, error) { - var apiHandler APIHandler - switch apiType { - case "jamfpro": - apiHandler = &jamfpro.JamfAPIHandler{ - Logger: log, - InstanceName: instanceName, // Used for constructing both jamf pro resource and auth endpoints - } - log.Info("Jamf Pro API handler loaded successfully", zap.String("APIType", apiType), zap.String("InstanceName", instanceName)) - - case "msgraph": - apiHandler = &msgraph.GraphAPIHandler{ - Logger: log, - TenantID: tenantID, // Used for constructing the graph auth endpoint - } - log.Info("Microsoft Graph API handler loaded successfully", zap.String("APIType", apiType), zap.String("TenantID", tenantID), zap.String("TenantName", tenantName)) - - default: - return nil, log.Error("Unsupported API type", zap.String("APIType", apiType)) - } - - return apiHandler, nil -} diff --git a/apiintegrations/apihandler/apihandler_test.go.TODO b/apiintegrations/apihandler/apihandler_test.go.TODO deleted file mode 100644 index 172aef2..0000000 --- a/apiintegrations/apihandler/apihandler_test.go.TODO +++ /dev/null @@ -1,66 +0,0 @@ -// apiintegrations/apihandler/apihandler_test.go -package apihandler - -import ( - "testing" - - "github.com/deploymenttheory/go-api-http-client/apiintegrations/jamfpro" - "github.com/deploymenttheory/go-api-http-client/apiintegrations/msgraph" - "github.com/deploymenttheory/go-api-http-client/mocklogger" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -func TestLoadAPIHandler(t *testing.T) { - // Create a mock logger for testing purposes. - mockLog := mocklogger.NewMockLogger() - - // Define your test cases. - tests := []struct { - name string - apiType string - wantType interface{} - wantErr bool - }{ - { - name: "Load JamfPro Handler", - apiType: "jamfpro", - wantType: &jamfpro.JamfAPIHandler{}, - wantErr: false, - }, - { - name: "Load Graph Handler", - apiType: "msgraph", - wantType: &msgraph.GraphAPIHandler{}, - wantErr: false, - }, - { - name: "Unsupported API Type", - apiType: "unknown", - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Setup expectations for the mock logger based on whether an error is expected. - if tt.wantErr { - mockLog.On("Error", mock.Anything, mock.Anything, mock.Anything).Return().Once() - } - - // Attempt to load the API handler. - got, err := LoadAPIHandler(tt.apiType, mockLog) - - // Assert error handling. - if tt.wantErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - assert.IsType(t, tt.wantType, got, "Got %T, want %T", got, tt.wantType) - } - - // Assert that the mock logger's expectations were met. - mockLog.AssertExpectations(t) - }) - } -} diff --git a/apiintegrations/jamfpro/README.MD b/apiintegrations/jamfpro/README.MD deleted file mode 100644 index 1b4154f..0000000 --- a/apiintegrations/jamfpro/README.MD +++ /dev/null @@ -1,38 +0,0 @@ -# Jamf Pro API Handler - -The Jamf Pro API Handler is an integral component of the Go API HTTP Client, designed specifically for seamless integration with the Jamf Pro API. This handler facilitates the encoding and decoding of requests and responses, manages API-specific headers, and constructs endpoints for efficient API communication. - -## Features - -- **Endpoint Construction**: Dynamically constructs API resource and authentication endpoints based on the instance name and predefined URL patterns. -- **Content-Type Handling**: Determines the appropriate `Content-Type` header for requests, with specialized handling for both the Classic API (XML) and the JamfPro API (JSON). -- **Accept Header Management**: Generates a weighted `Accept` header to indicate the client's capability to process various MIME types, prioritizing XML for compatibility with the Classic API. -- **Standard Headers**: Provides a set of standard headers required for API requests, including `Accept`, `Content-Type`, and `Authorization`. -- **Request Marshaling**: Encodes request bodies into the appropriate format (XML or JSON) based on the target API endpoint, with support for multipart/form-data encoding for file uploads. - -The logic of this api handler is defined as follows: -Classic API: - -For requests (GET, POST, PUT, DELETE): - -- Encoding (Marshalling): Use XML format. -For responses (GET, POST, PUT): -- Decoding (Unmarshalling): Use XML format. -For responses (DELETE): -- Handle response codes as response body lacks anything useful. -Headers -- Sets accept headers based on weighting. XML out weighs JSON to ensure XML is returned -- Sets content header as application/xml with edge case exceptions based on need. - -JamfPro API: - -For requests (GET, POST, PUT, DELETE): - -- Encoding (Marshalling): Use JSON format. -For responses (GET, POST, PUT): -- Decoding (Unmarshalling): Use JSON format. -For responses (DELETE): -- Handle response codes as response body lacks anything useful. -Headers -- Sets accept headers based on weighting. Jamf Pro API doesn't support XML, so MIME type is skipped and returns JSON -- Set content header as application/json with edge case exceptions based on need. \ No newline at end of file diff --git a/apiintegrations/jamfpro/jamfpro_api_exceptions.go b/apiintegrations/jamfpro/jamfpro_api_exceptions.go deleted file mode 100644 index 95ff906..0000000 --- a/apiintegrations/jamfpro/jamfpro_api_exceptions.go +++ /dev/null @@ -1,45 +0,0 @@ -package jamfpro - -import ( - _ "embed" - - "encoding/json" - "log" -) - -// EndpointConfig is a struct that holds configuration details for a specific API endpoint. -// It includes what type of content it can accept and what content type it should send. -type EndpointConfig struct { - Accept string `json:"accept"` // Accept specifies the MIME type the endpoint can handle in responses. - ContentType *string `json:"content_type"` // ContentType, if not nil, specifies the MIME type to set for requests sent to the endpoint. A pointer is used to distinguish between a missing field and an empty string. -} - -// ConfigMap is a map that associates endpoint URL patterns with their corresponding configurations. -// The map's keys are strings that identify the endpoint, and the values are EndpointConfig structs -// that hold the configuration for that endpoint. -type ConfigMap map[string]EndpointConfig - -// Variables -var configMap ConfigMap - -// Embedded Resources -// -//go:embed jamfpro_api_exceptions_configuration.json -var jamfpro_api_exceptions_configuration []byte - -// init is invoked automatically on package initialization and is responsible for -// setting up the default state of the package by loading the api exceptions configuration. -func init() { - // Load the default configuration from an embedded resource. - err := loadAPIExceptionsConfiguration() - if err != nil { - log.Fatalf("Error loading Jamf Pro API exceptions configuration: %s", err) - } -} - -// loadAPIExceptionsConfiguration reads and unmarshals the jamfpro_api_exceptions_configuration JSON data from an embedded file -// into the configMap variable, which holds the exceptions configuration for endpoint-specific headers. -func loadAPIExceptionsConfiguration() error { - // Unmarshal the embedded default configuration into the global configMap. - return json.Unmarshal(jamfpro_api_exceptions_configuration, &configMap) -} diff --git a/apiintegrations/jamfpro/jamfpro_api_exceptions_configuration.json b/apiintegrations/jamfpro/jamfpro_api_exceptions_configuration.json deleted file mode 100644 index ace253b..0000000 --- a/apiintegrations/jamfpro/jamfpro_api_exceptions_configuration.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "/api/v1/icon/download/": { - "accept": "image/*", - "content_type": null - }, - "/api/v1/branding-images/download/": { - "accept": "image/*", - "content_type": null - }, - "/api/v2/inventory-preload/csv-template": { - "accept": "text/csv", - "content_type": null - }, - "/api/v1/pki/certificate-authority/active/der": { - "accept": "application/pkix-cert", - "content_type": null - } -} diff --git a/apiintegrations/jamfpro/jamfpro_api_handler.go b/apiintegrations/jamfpro/jamfpro_api_handler.go deleted file mode 100644 index cbd699f..0000000 --- a/apiintegrations/jamfpro/jamfpro_api_handler.go +++ /dev/null @@ -1,12 +0,0 @@ -// jamfpro_api_handler.go - -package jamfpro - -import "github.com/deploymenttheory/go-api-http-client/logger" - -// JamfAPIHandler implements the APIHandler interface for the Jamf Pro API. -type JamfAPIHandler struct { - OverrideBaseDomain string // OverrideBaseDomain is used to override the base domain for URL construction. - InstanceName string // InstanceName is the name of the Jamf instance. - Logger logger.Logger // Logger is the structured logger used for logging. -} diff --git a/apiintegrations/jamfpro/jamfpro_api_handler_constants.go b/apiintegrations/jamfpro/jamfpro_api_handler_constants.go deleted file mode 100644 index 40ce47d..0000000 --- a/apiintegrations/jamfpro/jamfpro_api_handler_constants.go +++ /dev/null @@ -1,60 +0,0 @@ -package jamfpro - -// Endpoint constants represent the URL suffixes used for Jamf API token interactions. -const ( - APIName = "jamf pro" // APIName: represents the name of the API. - DefaultBaseDomain = ".jamfcloud.com" // DefaultBaseDomain: represents the base domain for the jamf instance. - OAuthTokenEndpoint = "/api/oauth/token" // OAuthTokenEndpoint: The endpoint to obtain an OAuth token. - OAuthTokenScope = "" // OAuthTokenScope: Not used for Jamf. - BearerTokenEndpoint = "/api/v1/auth/token" // BearerTokenEndpoint: The endpoint to obtain a bearer token. - TokenRefreshEndpoint = "/api/v1/auth/keep-alive" // TokenRefreshEndpoint: The endpoint to refresh an existing token. - TokenInvalidateEndpoint = "/api/v1/auth/invalidate-token" // TokenInvalidateEndpoint: The endpoint to invalidate an active token. - BearerTokenAuthenticationSupport = true // BearerTokenAuthSuppport: A boolean to indicate if the API supports bearer token authentication. - OAuthAuthenticationSupport = true // OAuthAuthSuppport: A boolean to indicate if the API supports OAuth authentication. - OAuthWithCertAuthenticationSupport = false // OAuthWithCertAuthSuppport: A boolean to indicate if the API supports OAuth with client certificate authentication. -) - -// GetDefaultBaseDomain returns the default base domain used for constructing API URLs to the http client. -func (j *JamfAPIHandler) GetDefaultBaseDomain() string { - return DefaultBaseDomain -} - -// GetOAuthTokenEndpoint returns the endpoint for obtaining an OAuth token. Used for constructing API URLs for the http client. -func (j *JamfAPIHandler) GetOAuthTokenEndpoint() string { - return OAuthTokenEndpoint -} - -// GetOAuthTokenScope returns the scope for the OAuth token scope -func (j *JamfAPIHandler) GetOAuthTokenScope() string { - return OAuthTokenScope -} - -// GetBearerTokenEndpoint returns the endpoint for obtaining a bearer token. Used for constructing API URLs for the http client. -func (j *JamfAPIHandler) GetBearerTokenEndpoint() string { - return BearerTokenEndpoint -} - -// GetTokenRefreshEndpoint returns the endpoint for refreshing an existing token. Used for constructing API URLs for the http client. -func (j *JamfAPIHandler) GetTokenRefreshEndpoint() string { - return TokenRefreshEndpoint -} - -// GetTokenInvalidateEndpoint returns the endpoint for invalidating an active token. Used for constructing API URLs for the http client. -func (j *JamfAPIHandler) GetTokenInvalidateEndpoint() string { - return TokenInvalidateEndpoint -} - -// GetAPIBearerTokenAuthenticationSupportStatus returns a boolean indicating if bearer token authentication is supported in the api handler. -func (j *JamfAPIHandler) GetAPIBearerTokenAuthenticationSupportStatus() bool { - return BearerTokenAuthenticationSupport -} - -// GetAPIOAuthAuthenticationSupportStatus returns a boolean indicating if OAuth authentication is supported in the api handler. -func (j *JamfAPIHandler) GetAPIOAuthAuthenticationSupportStatus() bool { - return OAuthAuthenticationSupport -} - -// GetAPIOAuthWithCertAuthenticationSupportStatus returns a boolean indicating if OAuth with client certificate authentication is supported in the api handler. -func (j *JamfAPIHandler) GetAPIOAuthWithCertAuthenticationSupportStatus() bool { - return OAuthWithCertAuthenticationSupport -} diff --git a/apiintegrations/jamfpro/jamfpro_api_headers.go b/apiintegrations/jamfpro/jamfpro_api_headers.go deleted file mode 100644 index 688ff55..0000000 --- a/apiintegrations/jamfpro/jamfpro_api_headers.go +++ /dev/null @@ -1,88 +0,0 @@ -// jamfpro_api_headers.go -package jamfpro - -import ( - "strings" - - "github.com/deploymenttheory/go-api-http-client/logger" - "go.uber.org/zap" -) - -// GetContentTypeHeader determines the appropriate Content-Type header for a given API endpoint. -// It attempts to find a content type that matches the endpoint prefix in the global configMap. -// If a match is found and the content type is defined (not nil), it returns the specified content type. -// If the content type is nil or no match is found in configMap, it falls back to default behaviors: -// - For url endpoints starting with "/JSSResource", it defaults to "application/xml" for the Classic API. -// - For url endpoints starting with "/api", it defaults to "application/json" for the JamfPro API. -// If the endpoint does not match any of the predefined patterns, "application/json" is used as a fallback. -// This method logs the decision process at various stages for debugging purposes. -func (j *JamfAPIHandler) GetContentTypeHeader(endpoint string, log logger.Logger) string { - // Dynamic lookup from configuration should be the first priority - for key, config := range configMap { - if strings.HasPrefix(endpoint, key) { - if config.ContentType != nil { - j.Logger.Debug("Content-Type for endpoint found in configMap", zap.String("endpoint", endpoint), zap.String("content_type", *config.ContentType)) - return *config.ContentType - } - j.Logger.Debug("Content-Type for endpoint is nil in configMap, handling as special case", zap.String("endpoint", endpoint)) - // If a nil ContentType is an expected case, do not set Content-Type header. - return "" // Return empty to indicate no Content-Type should be set. - } - } - - // Special case for package upload endpoint - if strings.HasPrefix(endpoint, "/api/v1/packages") && strings.HasSuffix(endpoint, "/upload") { - j.Logger.Debug("Skipping Content-Type setting for package upload endpoint. Multipart request will handling setting directly with boundary", zap.String("endpoint", endpoint)) - return "" // Skip setting Content-Type here - } - - // If no specific configuration is found, then check for standard URL patterns. - if strings.Contains(endpoint, "/JSSResource") { - j.Logger.Debug("Content-Type for endpoint defaulting to XML for Classic API", zap.String("endpoint", endpoint)) - return "application/xml" // Classic API uses XML - } else if strings.Contains(endpoint, "/api") { - j.Logger.Debug("Content-Type for endpoint defaulting to JSON for JamfPro API", zap.String("endpoint", endpoint)) - return "application/json" // JamfPro API uses JSON - } - - // Fallback to JSON if no other match is found. - j.Logger.Debug("Content-Type for endpoint not found in configMap or standard patterns, using default JSON", zap.String("endpoint", endpoint)) - return "application/json" -} - -// GetAcceptHeader constructs and returns a weighted Accept header string for HTTP requests. -// The Accept header indicates the MIME types that the client can process and prioritizes them -// based on the quality factor (q) parameter. Higher q-values signal greater preference. -// This function specifies a range of MIME types with their respective weights, ensuring that -// the server is informed of the client's versatile content handling capabilities while -// indicating a preference for XML. The specified MIME types cover common content formats like -// images, JSON, XML, HTML, plain text, and certificates, with a fallback option for all other types. -func (j *JamfAPIHandler) GetAcceptHeader() string { - weightedAcceptHeader := "application/x-x509-ca-cert;q=0.95," + - "application/pkix-cert;q=0.94," + - "application/pem-certificate-chain;q=0.93," + - "application/octet-stream;q=0.8," + // For general binary files - "image/png;q=0.75," + - "image/jpeg;q=0.74," + - "image/*;q=0.7," + - "application/xml;q=0.65," + - "text/xml;q=0.64," + - "text/xml;charset=UTF-8;q=0.63," + - "application/json;q=0.5," + - "text/html;q=0.5," + - "text/plain;q=0.4," + - "*/*;q=0.05" // Fallback for any other types - - return weightedAcceptHeader -} - -// GetAPIRequestHeaders returns a map of standard headers required for making API requests. -func (j *JamfAPIHandler) GetAPIRequestHeaders(endpoint string) map[string]string { - headers := map[string]string{ - "Accept": j.GetAcceptHeader(), // Dynamically set based on API requirements. - "Content-Type": j.GetContentTypeHeader(endpoint, j.Logger), // Dynamically set based on the endpoint. - "Authorization": "", // To be set by the client with the actual token. - "User-Agent": "go-api-http-client-jamfpro-handler", // To be set by the client, usually with application info. - } - return headers -} diff --git a/apiintegrations/jamfpro/jamfpro_api_request.go b/apiintegrations/jamfpro/jamfpro_api_request.go deleted file mode 100644 index 4c1bdf4..0000000 --- a/apiintegrations/jamfpro/jamfpro_api_request.go +++ /dev/null @@ -1,102 +0,0 @@ -// jamfpro_api_request.go -package jamfpro - -import ( - "bytes" - "encoding/json" - "encoding/xml" - "mime/multipart" - "strings" - - "github.com/deploymenttheory/go-api-http-client/logger" - "go.uber.org/zap" -) - -// MarshalRequest encodes the request body according to the endpoint for the API. -func (j *JamfAPIHandler) MarshalRequest(body interface{}, method string, endpoint string, log logger.Logger) ([]byte, error) { - var ( - data []byte - err error - ) - - // Determine the format based on the endpoint - format := "json" - if strings.Contains(endpoint, "/JSSResource") { - format = "xml" - } else if strings.Contains(endpoint, "/api") { - format = "json" - } - - switch format { - case "xml": - data, err = xml.Marshal(body) - if err != nil { - return nil, err - } - - if method == "POST" || method == "PUT" { - j.Logger.Debug("XML Request Body", zap.String("Body", string(data))) - } - - case "json": - data, err = json.Marshal(body) - if err != nil { - j.Logger.Error("Failed marshaling JSON request", zap.Error(err)) - return nil, err - } - - if method == "POST" || method == "PUT" || method == "PATCH" { - j.Logger.Debug("JSON Request Body", zap.String("Body", string(data))) - } - } - - return data, nil -} - -// MarshalMultipartRequest handles multipart form data encoding with secure file handling and returns the encoded body and content type. -func (j *JamfAPIHandler) MarshalMultipartRequest(formFields map[string]string, fileContents map[string][]byte, log logger.Logger) ([]byte, string, string, error) { - const snippetLength = 20 - var b bytes.Buffer - writer := multipart.NewWriter(&b) - - // Log form fields - for key, val := range formFields { - err := writer.WriteField(key, val) - if err != nil { - log.Error("Failed to add form field to multipart request", zap.String("key", key), zap.Error(err)) - return nil, "", "", err - } - log.Debug("Added form field", zap.String("key", key), zap.String("value", val)) - } - - // Log file contents snippets - for key, val := range fileContents { - contentSnippet := string(val) - if len(contentSnippet) > snippetLength { - contentSnippet = contentSnippet[:snippetLength] + "..." - } - log.Debug("File content snippet", zap.String("key", key), zap.String("snippet", contentSnippet)) - - part, err := writer.CreateFormFile(key, key) - if err != nil { - log.Error("Failed to create form file in multipart request", zap.String("key", key), zap.Error(err)) - return nil, "", "", err - } - _, err = part.Write(val) - if err != nil { - log.Error("Failed to write file to multipart request", zap.String("key", key), zap.Error(err)) - return nil, "", "", err - } - } - - // Close the writer - err := writer.Close() - if err != nil { - log.Error("Failed to close multipart writer", zap.Error(err)) - return nil, "", "", err - } - - log.Debug("Multipart request constructed", zap.Any("formFields", formFields)) - - return b.Bytes(), writer.FormDataContentType(), b.String()[:snippetLength], nil -} diff --git a/apiintegrations/jamfpro/jamfpro_api_url.go b/apiintegrations/jamfpro/jamfpro_api_url.go deleted file mode 100644 index 7554f6a..0000000 --- a/apiintegrations/jamfpro/jamfpro_api_url.go +++ /dev/null @@ -1,36 +0,0 @@ -// jamfpro_api_url.go -package jamfpro - -import ( - "fmt" - - "github.com/deploymenttheory/go-api-http-client/logger" - "go.uber.org/zap" -) - -// SetBaseDomain returns the appropriate base domain for URL construction. -// It uses j.OverrideBaseDomain if set, otherwise falls back to DefaultBaseDomain. -func (j *JamfAPIHandler) SetBaseDomain() string { - if j.OverrideBaseDomain != "" { - return j.OverrideBaseDomain - } - return DefaultBaseDomain -} - -// ConstructAPIResourceEndpoint constructs the full URL for a Jamf API resource endpoint path and logs the URL. -// It uses the instance name to construct the full URL. -func (j *JamfAPIHandler) ConstructAPIResourceEndpoint(endpointPath string, log logger.Logger) string { - urlBaseDomain := j.SetBaseDomain() - url := fmt.Sprintf("https://%s%s%s", j.InstanceName, urlBaseDomain, endpointPath) - j.Logger.Debug(fmt.Sprintf("Constructed %s API resource endpoint URL", APIName), zap.String("URL", url)) - return url -} - -// ConstructAPIAuthEndpoint constructs the full URL for a Jamf API auth endpoint path and logs the URL. -// It uses the instance name to construct the full URL. -func (j *JamfAPIHandler) ConstructAPIAuthEndpoint(endpointPath string, log logger.Logger) string { - urlBaseDomain := j.SetBaseDomain() - url := fmt.Sprintf("https://%s%s%s", j.InstanceName, urlBaseDomain, endpointPath) - j.Logger.Debug(fmt.Sprintf("Constructed %s API authentication URL", APIName), zap.String("URL", url)) - return url -} diff --git a/apiintegrations/msgraph/msgraph_api_helpers.go b/apiintegrations/msgraph/msgraph_api_helpers.go new file mode 100644 index 0000000..a45ca7f --- /dev/null +++ b/apiintegrations/msgraph/msgraph_api_helpers.go @@ -0,0 +1,36 @@ +package msgraph + +import ( + "fmt" + "os" + "path/filepath" + "time" +) + +// TODO this duplicated across both integrations. Improve it another time. + +// ParseISO8601Date attempts to parse a string date in ISO 8601 format. +func ParseISO8601_Date(dateStr string) (time.Time, error) { + return time.Parse(time.RFC3339, dateStr) +} + +// SafeOpenFile opens a file safely after validating and resolving its path. +func SafeOpenFile(filePath string) (*os.File, error) { + // Clean the file path to remove any ".." or similar components that can lead to directory traversal + cleanPath := filepath.Clean(filePath) + + // Resolve the clean path to an absolute path and ensure it resolves any symbolic links + absPath, err := filepath.EvalSymlinks(cleanPath) + if err != nil { + return nil, fmt.Errorf("unable to resolve the absolute path: %s, error: %w", filePath, err) + } + + // Optionally, check if the absolute path is within a permitted directory (omitted here for brevity) + // Example: allowedPathPrefix := "/safe/directory/" + // if !strings.HasPrefix(absPath, allowedPathPrefix) { + // return nil, fmt.Errorf("access to the file path is not allowed: %s", absPath) + // } + + // Open the file if the path is deemed safe + return os.Open(absPath) +} diff --git a/apiintegrations/msgraph/msgraph_api_request_test.go b/apiintegrations/msgraph/msgraph_api_request_test.go deleted file mode 100644 index b9999af..0000000 --- a/apiintegrations/msgraph/msgraph_api_request_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// apiintegrations/msgraph/msgraph_api_request_test.go -package msgraph - -import ( - "encoding/json" - "testing" - - "github.com/deploymenttheory/go-api-http-client/mocklogger" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "go.uber.org/zap" -) - -// TestMarshalRequest tests the MarshalRequest function. -func TestMarshalRequest(t *testing.T) { - body := map[string]interface{}{ - "name": "John Doe", - "age": 30, - } - method := "POST" - endpoint := "/users" - mockLog := mocklogger.NewMockLogger() - handler := GraphAPIHandler{Logger: mockLog} - - expectedData, _ := json.Marshal(body) - - // Correct the way we setup the logger mock - mockLog.On("Debug", "JSON Request Body", mock.MatchedBy(func(fields []zap.Field) bool { - if len(fields) != 1 { - return false - } - return fields[0].Key == "Body" && fields[0].String == string(expectedData) - })).Once() - - data, err := handler.MarshalRequest(body, method, endpoint, mockLog) - - assert.NoError(t, err) - assert.Equal(t, expectedData, data) - mockLog.AssertExpectations(t) -} - -// func TestMarshalMultipartRequest(t *testing.T) { -// // Prepare the logger mock -// mockLog := mocklogger.NewMockLogger() - -// // Setting up a temporary file to simulate a file upload -// tempDir := t.TempDir() // Create a temporary directory for test files -// tempFile, err := os.CreateTemp(tempDir, "upload-*.txt") -// assert.NoError(t, err) -// defer os.Remove(tempFile.Name()) // Ensure the file is removed after the test - -// _, err = tempFile.WriteString("Test file content") -// assert.NoError(t, err) -// tempFile.Close() - -// handler := GraphAPIHandler{Logger: mockLog} - -// fields := map[string]string{"field1": "value1"} -// files := map[string]string{"fileField": tempFile.Name()} - -// // Execute the function -// body, contentType, err := handler.MarshalMultipartRequest(fields, files, mockLog) -// assert.NoError(t, err) -// assert.Contains(t, contentType, "multipart/form-data; boundary=") - -// // Check if the multipart form data contains the correct fields and file data -// reader := multipart.NewReader(bytes.NewReader(body), strings.TrimPrefix(contentType, "multipart/form-data; boundary=")) -// var foundField, foundFile bool - -// for { -// part, err := reader.NextPart() -// if err == io.EOF { -// break -// } -// assert.NoError(t, err) - -// if part.FormName() == "field1" { -// buf := new(bytes.Buffer) -// _, err = buf.ReadFrom(part) -// assert.NoError(t, err) -// assert.Equal(t, "value1", buf.String()) -// foundField = true -// } else if part.FileName() == filepath.Base(tempFile.Name()) { -// buf := new(bytes.Buffer) -// _, err = buf.ReadFrom(part) -// assert.NoError(t, err) -// assert.Equal(t, "Test file content", buf.String()) -// foundFile = true -// } -// } - -// // Ensure all expected parts were found -// assert.True(t, foundField, "Text field not found in the multipart form data") -// assert.True(t, foundFile, "File not found in the multipart form data") -// } diff --git a/authenticationhandler/authenticationhandler.go b/authenticationhandler/authenticationhandler.go deleted file mode 100644 index 1e6374c..0000000 --- a/authenticationhandler/authenticationhandler.go +++ /dev/null @@ -1,47 +0,0 @@ -// authenticationhandler/authenticationhandler.go - -package authenticationhandler - -import ( - "sync" - "time" - - "github.com/deploymenttheory/go-api-http-client/logger" -) - -// AuthTokenHandler manages authentication tokens. -type AuthTokenHandler struct { - Credentials ClientCredentials // Credentials holds the authentication credentials. - Token string // Token holds the current authentication token. - Expires time.Time // Expires indicates the expiry time of the current authentication token. - Logger logger.Logger // Logger provides structured logging capabilities for logging information, warnings, and errors. - AuthMethod string // AuthMethod specifies the method of authentication, e.g., "bearer" or "oauth". - InstanceName string // InstanceName represents the name of the instance or environment the client is interacting with. - tokenLock sync.Mutex // tokenLock ensures thread-safe access to the token and its expiry to prevent concurrent write/read issues. - HideSensitiveData bool -} - -// ClientCredentials holds the credentials necessary for authentication. -type ClientCredentials struct { - Username string - Password string - ClientID string - ClientSecret string -} - -// TokenResponse represents the structure of a token response from the API. -type TokenResponse struct { - Token string `json:"token"` - Expires time.Time `json:"expires"` -} - -// NewAuthTokenHandler creates a new instance of AuthTokenHandler. -func NewAuthTokenHandler(logger logger.Logger, authMethod string, credentials ClientCredentials, instanceName string, hideSensitiveData bool) *AuthTokenHandler { - return &AuthTokenHandler{ - Logger: logger, - AuthMethod: authMethod, - Credentials: credentials, - InstanceName: instanceName, - HideSensitiveData: hideSensitiveData, - } -} diff --git a/authenticationhandler/basicauthentication.go b/authenticationhandler/basicauthentication.go deleted file mode 100644 index c3fb33e..0000000 --- a/authenticationhandler/basicauthentication.go +++ /dev/null @@ -1,107 +0,0 @@ -// authenticationhandler/basicauthentication.go -/* The http_client_auth package focuses on authentication mechanisms for an HTTP client. -It provides structures and methods for handling both basic and bearer token based authentication */ - -package authenticationhandler - -import ( - "encoding/json" - "fmt" - "net/http" - "time" - - "github.com/deploymenttheory/go-api-http-client/apiintegrations/apihandler" - "go.uber.org/zap" -) - -// BasicAuthTokenAcquisition fetches and sets an authentication token using the stored basic authentication credentials. -func (h *AuthTokenHandler) BasicAuthTokenAcquisition(apiHandler apihandler.APIHandler, httpClient *http.Client, username string, password string) error { - - // Use the APIHandler's method to get the bearer token endpoint - bearerTokenEndpoint := apiHandler.GetBearerTokenEndpoint() - - // Construct the full authentication endpoint URL - authenticationEndpoint := apiHandler.ConstructAPIAuthEndpoint(bearerTokenEndpoint, h.Logger) - - h.Logger.Debug("Attempting to obtain token for user", zap.String("Username", username)) - - req, err := http.NewRequest("POST", authenticationEndpoint, nil) - if err != nil { - h.Logger.LogError("authentication_request_creation_error", "POST", authenticationEndpoint, 0, "", err, "Failed to create new request for token") - return err - } - req.SetBasicAuth(username, password) - - resp, err := httpClient.Do(req) - if err != nil { - h.Logger.LogError("authentication_request_error", "POST", authenticationEndpoint, 0, "", err, "Failed to make request for token") - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - h.Logger.LogError("token_authentication_failed", "POST", authenticationEndpoint, resp.StatusCode, resp.Status, fmt.Errorf("authentication failed with status code: %d", resp.StatusCode), "Token acquisition attempt resulted in a non-OK response") - return fmt.Errorf("received non-OK response status: %d", resp.StatusCode) - } - - tokenResp := &TokenResponse{} - err = json.NewDecoder(resp.Body).Decode(tokenResp) - if err != nil { - h.Logger.Error("Failed to decode token response", zap.Error(err)) - return err - } - - h.Token = tokenResp.Token - h.Expires = tokenResp.Expires - tokenDuration := time.Until(h.Expires) - - h.Logger.Info("Token obtained successfully", zap.Time("Expiry", h.Expires), zap.Duration("Duration", tokenDuration)) - - return nil -} - -// RefreshBearerToken refreshes the current authentication token. -func (h *AuthTokenHandler) RefreshBearerToken(apiHandler apihandler.APIHandler, httpClient *http.Client) error { - h.tokenLock.Lock() - defer h.tokenLock.Unlock() - - // Use the APIHandler's method to get the token refresh endpoint - apiTokenRefreshEndpoint := apiHandler.GetTokenRefreshEndpoint() - - // Construct the full authentication endpoint URL - tokenRefreshEndpoint := apiHandler.ConstructAPIAuthEndpoint(apiTokenRefreshEndpoint, h.Logger) - - h.Logger.Debug("Attempting to refresh token", zap.String("URL", tokenRefreshEndpoint)) - - req, err := http.NewRequest("POST", tokenRefreshEndpoint, nil) - if err != nil { - h.Logger.Error("Failed to create new request for token refresh", zap.Error(err)) - return err - } - req.Header.Add("Authorization", "Bearer "+h.Token) - - resp, err := httpClient.Do(req) - if err != nil { - h.Logger.Error("Failed to make request for token refresh", zap.Error(err)) - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - h.Logger.Warn("Token refresh response status is not OK", zap.Int("StatusCode", resp.StatusCode)) - return fmt.Errorf("token refresh failed with status code: %d", resp.StatusCode) - } - - tokenResp := &TokenResponse{} - err = json.NewDecoder(resp.Body).Decode(tokenResp) - if err != nil { - h.Logger.Error("Failed to decode token response", zap.Error(err)) - return err - } - - h.Token = tokenResp.Token - h.Expires = tokenResp.Expires - h.Logger.Info("Token refreshed successfully", zap.Time("Expiry", tokenResp.Expires)) - - return nil -} diff --git a/authenticationhandler/oauth2.go b/authenticationhandler/oauth2.go deleted file mode 100644 index 9cc16a2..0000000 --- a/authenticationhandler/oauth2.go +++ /dev/null @@ -1,103 +0,0 @@ -// authenticationhandler/oauth2.go - -/* The http_client_auth package focuses on authentication mechanisms for an HTTP client. -It provides structures and methods for handling OAuth-based authentication */ - -package authenticationhandler - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" - - "github.com/deploymenttheory/go-api-http-client/apiintegrations/apihandler" - "github.com/deploymenttheory/go-api-http-client/headers/redact" - "go.uber.org/zap" -) - -// OAuthResponse represents the response structure when obtaining an OAuth access token. -type OAuthResponse struct { - AccessToken string `json:"access_token"` // AccessToken is the token that can be used in subsequent requests for authentication. - ExpiresIn int64 `json:"expires_in"` // ExpiresIn specifies the duration in seconds after which the access token expires. - TokenType string `json:"token_type"` // TokenType indicates the type of token, typically "Bearer". - RefreshToken string `json:"refresh_token,omitempty"` // RefreshToken is used to obtain a new access token when the current one expires. - Error string `json:"error,omitempty"` // Error contains details if an error occurs during the token acquisition process. -} - -// OAuth2TokenAcquisition fetches an OAuth access token using the provided client ID and client secret. -// It updates the AuthTokenHandler's Token and Expires fields with the obtained values. -func (h *AuthTokenHandler) OAuth2TokenAcquisition(apiHandler apihandler.APIHandler, httpClient *http.Client, clientID, clientSecret string) error { - // Get the OAuth token endpoint from the APIHandler - oauthTokenEndpoint := apiHandler.GetOAuthTokenEndpoint() - - // Construct the full authentication endpoint URL - authenticationEndpoint := apiHandler.ConstructAPIAuthEndpoint(oauthTokenEndpoint, h.Logger) - - // Get the OAuth token scope from the APIHandler - oauthTokenScope := apiHandler.GetOAuthTokenScope() - - data := url.Values{} - data.Set("client_id", clientID) - data.Set("client_secret", clientSecret) - data.Set("scope", oauthTokenScope) - data.Set("grant_type", "client_credentials") - - h.Logger.Debug("Attempting to obtain OAuth token", zap.String("ClientID", clientID), zap.String("Scope", oauthTokenScope)) - - req, err := http.NewRequest("POST", authenticationEndpoint, strings.NewReader(data.Encode())) - if err != nil { - h.Logger.Error("Failed to create request for OAuth token", zap.Error(err)) - return err - } - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - - resp, err := httpClient.Do(req) - if err != nil { - h.Logger.Error("Failed to execute request for OAuth token", zap.Error(err)) - return err - } - defer resp.Body.Close() - - bodyBytes, err := io.ReadAll(resp.Body) - if err != nil { - h.Logger.Error("Failed to read response body", zap.Error(err)) - return err - } - - // Reset the response body to its original state - resp.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) - - oauthResp := &OAuthResponse{} - err = json.Unmarshal(bodyBytes, oauthResp) - if err != nil { - h.Logger.Error("Failed to decode OAuth response", zap.Error(err)) - return fmt.Errorf("failed to decode OAuth response: %w", err) - } - - if oauthResp.Error != "" { - h.Logger.Error("Error obtaining OAuth token", zap.String("Error", oauthResp.Error)) - return fmt.Errorf("error obtaining OAuth token: %s", oauthResp.Error) - } - - if oauthResp.AccessToken == "" { - h.Logger.Error("Empty access token received") - return fmt.Errorf("empty access token received") - } - - expiresIn := time.Duration(oauthResp.ExpiresIn) * time.Second - expirationTime := time.Now().Add(expiresIn) - - // Modified log call using the helper function - redactedAccessToken := redact.RedactSensitiveHeaderData(h.HideSensitiveData, "AccessToken", oauthResp.AccessToken) - h.Logger.Info("OAuth token obtained successfully", zap.String("AccessToken", redactedAccessToken), zap.Duration("ExpiresIn", expiresIn), zap.Time("ExpirationTime", expirationTime)) - - h.Token = oauthResp.AccessToken - h.Expires = expirationTime - - return nil -} diff --git a/authenticationhandler/tokenmanager.go b/authenticationhandler/tokenmanager.go deleted file mode 100644 index 04bb859..0000000 --- a/authenticationhandler/tokenmanager.go +++ /dev/null @@ -1,110 +0,0 @@ -// authenticationhandler/tokenmanager.go -package authenticationhandler - -import ( - "fmt" - "net/http" - "time" - - "github.com/deploymenttheory/go-api-http-client/apiintegrations/apihandler" - "go.uber.org/zap" -) - -// CheckAndRefreshAuthToken checks the token's validity and refreshes it if necessary. -// It returns true if the token is valid post any required operations and false with an error otherwise. -func (h *AuthTokenHandler) CheckAndRefreshAuthToken(apiHandler apihandler.APIHandler, httpClient *http.Client, clientCredentials ClientCredentials, tokenRefreshBufferPeriod time.Duration) (bool, error) { - const maxConsecutiveRefreshAttempts = 10 - refreshAttempts := 0 - - if h.isTokenValid(tokenRefreshBufferPeriod) { - h.Logger.Info("Authentication token is valid", zap.Bool("IsTokenValid", true)) - return true, nil - } - - for !h.isTokenValid(tokenRefreshBufferPeriod) { - h.Logger.Debug("Token found to be invalid or close to expiry, handling token acquisition or refresh.") - if err := h.obtainNewToken(apiHandler, httpClient, clientCredentials); err != nil { - h.Logger.Error("Failed to obtain new token", zap.Error(err)) - return false, err - } - - refreshAttempts++ - if refreshAttempts >= maxConsecutiveRefreshAttempts { - return false, fmt.Errorf( - "exceeded maximum consecutive token refresh attempts (%d): access token lifetime (%s) is likely too short compared to the buffer period (%s) configured for token refresh", - maxConsecutiveRefreshAttempts, - h.Expires.Sub(time.Now()).String(), // Access token lifetime - tokenRefreshBufferPeriod.String(), // Configured buffer period - ) - } - } - - isValid := h.isTokenValid(tokenRefreshBufferPeriod) - h.Logger.Info("Authentication token status check completed", zap.Bool("IsTokenValid", isValid)) - return isValid, nil -} - -// isTokenValid checks if the current token is non-empty and not about to expire. -// It considers a token valid if it exists and the time until its expiration is greater than the provided buffer period. -func (h *AuthTokenHandler) isTokenValid(tokenRefreshBufferPeriod time.Duration) bool { - isValid := h.Token != "" && time.Until(h.Expires) >= tokenRefreshBufferPeriod - h.Logger.Debug("Checking token validity", zap.Bool("IsValid", isValid), zap.Duration("TimeUntilExpiry", time.Until(h.Expires))) - return isValid -} - -// obtainNewToken acquires a new token using the credentials provided. -// It handles different authentication methods based on the AuthMethod setting. -func (h *AuthTokenHandler) obtainNewToken(apiHandler apihandler.APIHandler, httpClient *http.Client, clientCredentials ClientCredentials) error { - var err error - backoff := time.Millisecond * 100 - - for attempts := 0; attempts < 5; attempts++ { - if h.AuthMethod == "basicauth" { - err = h.BasicAuthTokenAcquisition(apiHandler, httpClient, clientCredentials.Username, clientCredentials.Password) - } else if h.AuthMethod == "oauth2" { - err = h.OAuth2TokenAcquisition(apiHandler, httpClient, clientCredentials.ClientID, clientCredentials.ClientSecret) - } else { - err = fmt.Errorf("no valid credentials provided. Unable to obtain a token") - h.Logger.Error("Authentication method not supported", zap.String("AuthMethod", h.AuthMethod)) - return err // Return the error immediately - } - - if err == nil { - break - } - - h.Logger.Error("Failed to obtain new token, retrying...", zap.Error(err), zap.Int("attempt", attempts+1)) - time.Sleep(backoff) - backoff *= 2 - } - - if err != nil { - h.Logger.Error("Failed to obtain new token after all attempts", zap.Error(err)) - return err - } - - return nil -} - -// refreshTokenIfNeeded refreshes the token if it's close to expiration. -// This function decides on the method based on the credentials type available. -func (h *AuthTokenHandler) refreshTokenIfNeeded(apiHandler apihandler.APIHandler, httpClient *http.Client, clientCredentials ClientCredentials, tokenRefreshBufferPeriod time.Duration) error { - if time.Until(h.Expires) < tokenRefreshBufferPeriod { - h.Logger.Info("Token is close to expiry and will be refreshed", zap.Duration("TimeUntilExpiry", time.Until(h.Expires))) - var err error - if clientCredentials.Username != "" && clientCredentials.Password != "" { - err = h.RefreshBearerToken(apiHandler, httpClient) - } else if clientCredentials.ClientID != "" && clientCredentials.ClientSecret != "" { - err = h.OAuth2TokenAcquisition(apiHandler, httpClient, clientCredentials.ClientID, clientCredentials.ClientSecret) - } else { - err = fmt.Errorf("unknown auth method") - h.Logger.Error("Failed to determine authentication method for token refresh", zap.String("AuthMethod", h.AuthMethod)) - } - - if err != nil { - h.Logger.Error("Failed to refresh token", zap.Error(err)) - return err - } - } - return nil -} diff --git a/authenticationhandler/validation.go b/authenticationhandler/validation.go deleted file mode 100644 index 4e45c45..0000000 --- a/authenticationhandler/validation.go +++ /dev/null @@ -1,62 +0,0 @@ -// authenticationhandler/validation.go - -package authenticationhandler - -import ( - "regexp" -) - -// IsValidClientID checks if the provided client ID is a valid UUID. -// Returns true if valid, along with an empty error message; otherwise, returns false with an error message. -func IsValidClientID(clientID string) (bool, string) { - uuidRegex := `^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$` - if regexp.MustCompile(uuidRegex).MatchString(clientID) { - return true, "" - } - return false, "Client ID is not a valid UUID format." -} - -// IsValidClientSecret checks if the provided client secret meets your application's validation criteria. -// Returns true if valid, along with an empty error message; otherwise, returns false with an error message. -func IsValidClientSecret(clientSecret string) (bool, string) { - if len(clientSecret) < 16 { - return false, "Client secret must be at least 16 characters long." - } - - // Check for at least one lowercase letter - if matched, _ := regexp.MatchString(`[a-z]`, clientSecret); !matched { - return false, "Client secret must contain at least one lowercase letter." - } - - // Check for at least one uppercase letter - if matched, _ := regexp.MatchString(`[A-Z]`, clientSecret); !matched { - return false, "Client secret must contain at least one uppercase letter." - } - - // Check for at least one digit - if matched, _ := regexp.MatchString(`\d`, clientSecret); !matched { - return false, "Client secret must contain at least one digit." - } - - return true, "" -} - -// IsValidUsername checks if the provided username meets password safe validation criteria. -// Returns true if valid, along with an empty error message; otherwise, returns false with an error message. -func IsValidUsername(username string) (bool, string) { - // Extended regex to include a common set of password safe special characters - usernameRegex := `^[a-zA-Z0-9!@#$%^&*()_\-\+=\[\]{\}\\|;:'",<.>/?]+$` - if regexp.MustCompile(usernameRegex).MatchString(username) { - return true, "" - } - return false, "Username must contain only alphanumeric characters and password safe special characters (!@#$%^&*()_-+=[{]}\\|;:'\",<.>/?)." -} - -// IsValidPassword checks if the provided password meets your application's validation criteria. -// Returns true if valid, along with an empty error message; otherwise, returns false with an error message. -func IsValidPassword(password string) (bool, string) { - if len(password) >= 8 { - return true, "" - } - return false, "Password must be at least 8 characters long." -} diff --git a/authenticationhandler/validation_test.go b/authenticationhandler/validation_test.go deleted file mode 100644 index 2307a2d..0000000 --- a/authenticationhandler/validation_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// authenticationhandler/auth_validation_test.go - -package authenticationhandler - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -// TestIsValidClientID tests the IsValidClientID function with various client ID inputs. -// It verifies that valid UUIDs are correctly identified as such, and invalid formats -// are appropriately flagged with an error message. Additionally, it checks that empty -// client IDs are considered valid according to the updated logic. -// TestIsValidClientID tests the IsValidClientID function for both valid and invalid UUIDs. -func TestIsValidClientID(t *testing.T) { - tests := []struct { - name string - clientID string - want bool - errMsg string - }{ - {"Valid UUID", "123e4567-e89b-12d3-a456-426614174000", true, ""}, - {"Invalid UUID - Wrong Length", "123e4567", false, "Client ID is not a valid UUID format."}, - {"Invalid UUID - Invalid Characters", "G23e4567-e89b-12d3-a456-426614174000", false, "Client ID is not a valid UUID format."}, - // Add more cases as needed... - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - valid, errMsg := IsValidClientID(tt.clientID) - assert.Equal(t, tt.want, valid) - if !tt.want { - assert.Equal(t, tt.errMsg, errMsg) - } - }) - } -} - -// TestIsValidClientSecret tests the IsValidClientSecret function with various client secret inputs. -// It ensures that client secrets that meet the minimum length requirement and contain the necessary -// character types are validated correctly. It also checks that short or invalid client secrets are -// flagged appropriately, and that empty client secrets are considered valid as per the updated logic. -func TestIsValidClientSecret(t *testing.T) { - tests := []struct { - name string - clientSecret string - want bool - errMsg string - }{ - {"Valid Secret", "Aa1!Aa1!Aa1!Aa1!", true, ""}, - {"Too Short", "Aa1!", false, "Client secret must be at least 16 characters long."}, - {"No Lowercase", "AAAAAAAAAAAAAA1!", false, "Client secret must contain at least one lowercase letter."}, - {"No Uppercase", "aaaaaaaaaaaaaa1!", false, "Client secret must contain at least one uppercase letter."}, - {"No Digit", "Aa!Aa!Aa!Aa!Aa!Aa!", false, "Client secret must contain at least one digit."}, - // Add more cases as needed... - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - valid, errMsg := IsValidClientSecret(tt.clientSecret) - assert.Equal(t, tt.want, valid) - if !tt.want { - assert.Equal(t, tt.errMsg, errMsg) - } - }) - } -} - -// TestIsValidUsername tests the IsValidUsername function to ensure it enforces the defined criteria for usernames. -func TestIsValidUsername(t *testing.T) { - tests := []struct { - name string - username string - want bool - errMsg string - }{ - {"Valid Username", "User123!", true, ""}, - {"Valid Special Characters", "User_@#1$", true, ""}, - {"Invalid Characters", " InvalidUsername", false, "Username must contain only alphanumeric characters and password safe special characters (!@#$%^&*()_-+=[{]}\\|;:'\",<.>/?)."}, - {"Empty Username", "", false, "Username must contain only alphanumeric characters and password safe special characters (!@#$%^&*()_-+=[{]}\\|;:'\",<.>/?)."}, - // You can add more cases here to test additional scenarios, such as extremely long usernames or usernames with only special characters. - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - valid, errMsg := IsValidUsername(tt.username) - assert.Equal(t, tt.want, valid) - if !tt.want { - assert.Equal(t, tt.errMsg, errMsg) - } - }) - } -} diff --git a/cookiejar/cookiejar.go b/cookiejar/cookiejar.go deleted file mode 100644 index 477c6d5..0000000 --- a/cookiejar/cookiejar.go +++ /dev/null @@ -1,161 +0,0 @@ -// cookiejar/cookiejar.go - -/* When both the cookie jar is enabled and specific cookies are set for an HTTP client in -your scenario, here’s what generally happens during the request processing: - -Cookie Jar Initialization: If the cookie jar is enabled through your SetupCookieJar function, -an instance of http.cookiejar.Jar is created and associated with your HTTP client. This -cookie jar will automatically handle incoming and outgoing cookies for all requests made -using this client. It manages storing cookies and automatically sending them with subsequent -requests to the domains for which they're valid. - -Setting Specific Cookies: The ApplyCustomCookies function checks for any user-defined specific -cookies (from the CustomCookies map). If found, these cookies are explicitly added to the -outgoing HTTP request headers via the SetSpecificCookies function. - -Interaction between Cookie Jar and Specific Cookies: -Cookie Precedence: When a specific cookie (added via SetSpecificCookies) shares the same name -as a cookie already handled by the cookie jar for a given domain, the behavior depends on the -implementation of the HTTP client's cookie handling and the server's cookie management rules. -Generally, the explicitly set cookie in the HTTP request header (from SetSpecificCookies) -should override any similar cookie managed by the cookie jar for that single request. - -Subsequent Requests: For subsequent requests, if the specific cookies are not added again -via ApplyCustomCookies, the cookies in the jar that were stored from previous responses -will take precedence again, unless overwritten by subsequent responses or explicit setting -again. - -Practical Usage: - -This setup allows flexibility: - -Use Cookie Jar: For general session management where cookies are automatically managed across -requests. -Use Specific Cookies: For overriding or adding specific cookies for particular requests where -customized control is necessary (such as testing scenarios, special authentication cookies, -etc.). -Logging and Debugging: Your setup also includes logging functionalities which can be very -useful to debug and verify which cookies are being sent and managed. This is crucial for -maintaining visibility into how cookies are influencing the behavior of your HTTP client -interactions.*/ - -package cookiejar - -import ( - "fmt" - "net/http" - "net/http/cookiejar" - "strings" - - "github.com/deploymenttheory/go-api-http-client/logger" - "go.uber.org/zap" -) - -// SetupCookieJar initializes the HTTP client with a cookie jar if enabled in the configuration. -func SetupCookieJar(client *http.Client, enableCookieJar bool, log logger.Logger) error { - if enableCookieJar { - jar, err := cookiejar.New(nil) // nil options use default options - if err != nil { - log.Error("Failed to create cookie jar", zap.Error(err)) - return fmt.Errorf("setupCookieJar failed: %w", err) // Wrap and return the error - } - client.Jar = jar - } - return nil -} - -// ApplyCustomCookies checks and applies custom cookies to the HTTP request if any are configured. -// It logs the names of the custom cookies being applied without exposing their values. -func ApplyCustomCookies(req *http.Request, cookies map[string]string, log logger.Logger) { - if len(cookies) > 0 { - cookieNames := make([]string, 0, len(cookies)) - for name := range cookies { - cookieNames = append(cookieNames, name) - } - log.Debug("Applying custom cookies", zap.Strings("Cookies", cookieNames)) - SetSpecificCookies(req, cookies) - } -} - -// SetSpecificCookies sets specific cookies provided in the configuration on the HTTP request. -func SetSpecificCookies(req *http.Request, cookies map[string]string) { - for name, value := range cookies { - cookie := &http.Cookie{ - Name: name, - Value: value, - } - req.AddCookie(cookie) - } -} - -// GetCookies is a middleware that extracts cookies from incoming requests and serializes them. -func GetCookies(next http.Handler, log logger.Logger) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - - // Extract cookies from the request - cookies := r.Cookies() - - // Serialize the cookies - serializedCookies := SerializeCookies(cookies) - - // Log the serialized cookies - log.Info("Serialized Cookies", zap.String("Cookies", serializedCookies)) - - // Call the next handler in the chain - next.ServeHTTP(w, r) - }) -} - -// SerializeCookies serializes a slice of *http.Cookie into a string format. -func SerializeCookies(cookies []*http.Cookie) string { - var cookieStrings []string - - for _, cookie := range cookies { - cookieStrings = append(cookieStrings, cookie.String()) - } - - return strings.Join(cookieStrings, "; ") -} - -// RedactSensitiveCookies redacts sensitive information from cookies. -// It takes a slice of *http.Cookie and returns a redacted slice of *http.Cookie. -func RedactSensitiveCookies(cookies []*http.Cookie) []*http.Cookie { - // Define sensitive cookie names that should be redacted. - sensitiveCookieNames := map[string]bool{ - "SessionID": true, // Example sensitive cookie name - // More sensitive cookie names will be added as needed. - } - - // Iterate over the cookies and redact sensitive ones. - for _, cookie := range cookies { - if _, found := sensitiveCookieNames[cookie.Name]; found { - cookie.Value = "REDACTED" - } - } - - return cookies -} - -// Utility function to convert cookies from http.Header to []*http.Cookie. -// This can be useful if cookies are stored in http.Header (e.g., from a response). -func CookiesFromHeader(header http.Header) []*http.Cookie { - cookies := []*http.Cookie{} - for _, cookieHeader := range header["Set-Cookie"] { - if cookie := ParseCookieHeader(cookieHeader); cookie != nil { - cookies = append(cookies, cookie) - } - } - return cookies -} - -// ParseCookieHeader parses a single Set-Cookie header and returns an *http.Cookie. -func ParseCookieHeader(header string) *http.Cookie { - headerParts := strings.Split(header, ";") - if len(headerParts) > 0 { - cookieParts := strings.SplitN(headerParts[0], "=", 2) - if len(cookieParts) == 2 { - return &http.Cookie{Name: cookieParts[0], Value: cookieParts[1]} - } - } - return nil -} diff --git a/cookiejar/cookiejar_test.go b/cookiejar/cookiejar_test.go deleted file mode 100644 index cf58c1a..0000000 --- a/cookiejar/cookiejar_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// cookiejar/cookiejar_test.go -package cookiejar - -import ( - "net/http" - "testing" - - "github.com/stretchr/testify/assert" -) - -// TestRedactSensitiveCookies tests the RedactSensitiveCookies function to ensure it correctly redacts sensitive cookies. -func TestRedactSensitiveCookies(t *testing.T) { - cookies := []*http.Cookie{ - {Name: "SessionID", Value: "sensitive-value-1"}, - {Name: "NonSensitiveCookie", Value: "non-sensitive-value"}, - {Name: "AnotherSensitiveCookie", Value: "sensitive-value-2"}, - } - - redactedCookies := RedactSensitiveCookies(cookies) - - // Define expected outcomes for each cookie. - expectedValues := map[string]string{ - "SessionID": "REDACTED", - "NonSensitiveCookie": "non-sensitive-value", - "AnotherSensitiveCookie": "sensitive-value-2", // Assuming this is not in the sensitive list. - } - - for _, cookie := range redactedCookies { - assert.Equal(t, expectedValues[cookie.Name], cookie.Value, "Cookie value should match expected redaction outcome") - } -} - -// TestCookiesFromHeader tests the CookiesFromHeader function to ensure it can correctly parse cookies from HTTP headers. -func TestCookiesFromHeader(t *testing.T) { - header := http.Header{ - "Set-Cookie": []string{ - "SessionID=sensitive-value; Path=/; HttpOnly", - "NonSensitiveCookie=non-sensitive-value; Path=/", - }, - } - - cookies := CookiesFromHeader(header) - - // Define expected outcomes for each cookie. - expectedCookies := []*http.Cookie{ - {Name: "SessionID", Value: "sensitive-value"}, - {Name: "NonSensitiveCookie", Value: "non-sensitive-value"}, - } - - assert.Equal(t, len(expectedCookies), len(cookies), "Number of parsed cookies should match expected") - - for i, expectedCookie := range expectedCookies { - assert.Equal(t, expectedCookie.Name, cookies[i].Name, "Cookie names should match") - assert.Equal(t, expectedCookie.Value, cookies[i].Value, "Cookie values should match") - } -} diff --git a/go.mod b/go.mod index 2f0526b..580f7fa 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/deploymenttheory/go-api-http-client -go 1.22.2 +go 1.22.4 require ( github.com/antchfx/xmlquery v1.4.0 @@ -16,8 +16,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.5.2 // indirect - go.uber.org/multierr v1.10.0 // indirect - golang.org/x/sys v0.20.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/text v0.15.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index faa12d5..f095ae9 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,8 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -38,8 +38,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= diff --git a/headers/headers.go b/headers/headers.go deleted file mode 100644 index 1623db7..0000000 --- a/headers/headers.go +++ /dev/null @@ -1,168 +0,0 @@ -// headers/headers.go -package headers - -import ( - "fmt" - "net/http" - "strings" - - "github.com/deploymenttheory/go-api-http-client/apiintegrations/apihandler" - "github.com/deploymenttheory/go-api-http-client/authenticationhandler" - "github.com/deploymenttheory/go-api-http-client/headers/redact" - "github.com/deploymenttheory/go-api-http-client/version" - - "github.com/deploymenttheory/go-api-http-client/logger" - "go.uber.org/zap" -) - -// HeaderHandler is responsible for managing and setting headers on HTTP requests. -type HeaderHandler struct { - req *http.Request // The http.Request for which headers are being managed - log logger.Logger // The logger to use for logging headers - apiHandler apihandler.APIHandler // The APIHandler to use for retrieving standard headers - authTokenHandler *authenticationhandler.AuthTokenHandler // The token to use for setting the Authorization header -} - -// NewHeaderHandler creates a new instance of HeaderHandler for a given http.Request, logger, and APIHandler. -func NewHeaderHandler(req *http.Request, log logger.Logger, apiHandler apihandler.APIHandler, authTokenHandler *authenticationhandler.AuthTokenHandler) *HeaderHandler { - return &HeaderHandler{ - req: req, - log: log, - apiHandler: apiHandler, - authTokenHandler: authTokenHandler, - } -} - -// SetAuthorization sets the Authorization header for the request. -func (h *HeaderHandler) SetAuthorization() { - token := h.authTokenHandler.Token - if !strings.HasPrefix(token, "Bearer ") { - token = "Bearer " + token - } - h.req.Header.Set("Authorization", token) -} - -// SetContentType sets the Content-Type header for the request. -func (h *HeaderHandler) SetContentType(contentType string) { - h.req.Header.Set("Content-Type", contentType) -} - -// SetAccept sets the Accept header for the request. -func (h *HeaderHandler) SetAccept(acceptHeader string) { - h.req.Header.Set("Accept", acceptHeader) -} - -// SetUserAgent sets the User-Agent header for the request. -func (h *HeaderHandler) SetUserAgent(userAgent string) { - h.req.Header.Set("User-Agent", userAgent) -} - -// SetCacheControlHeader sets the Cache-Control header for an HTTP request. -// This header specifies directives for caching mechanisms in requests and responses. -func SetCacheControlHeader(req *http.Request, cacheControlValue string) { - req.Header.Set("Cache-Control", cacheControlValue) -} - -// SetConditionalHeaders sets the If-Modified-Since and If-None-Match headers for an HTTP request. -// These headers make a request conditional to ask the server to return content only if it has changed. -func SetConditionalHeaders(req *http.Request, ifModifiedSince, ifNoneMatch string) { - if ifModifiedSince != "" { - req.Header.Set("If-Modified-Since", ifModifiedSince) - } - if ifNoneMatch != "" { - req.Header.Set("If-None-Match", ifNoneMatch) - } -} - -// SetAcceptEncodingHeader sets the Accept-Encoding header for an HTTP request. -// This header indicates the type of encoding (e.g., gzip) the client can handle. -func SetAcceptEncodingHeader(req *http.Request, acceptEncodingValue string) { - req.Header.Set("Accept-Encoding", acceptEncodingValue) -} - -// SetRefererHeader sets the Referer header for an HTTP request. -// This header indicates the address of the previous web page from which a link was followed. -func SetRefererHeader(req *http.Request, refererValue string) { - req.Header.Set("Referer", refererValue) -} - -// SetXForwardedForHeader sets the X-Forwarded-For header for an HTTP request. -// This header is used to identify the originating IP address of a client connecting through a proxy. -func SetXForwardedForHeader(req *http.Request, xForwardedForValue string) { - req.Header.Set("X-Forwarded-For", xForwardedForValue) -} - -// SetCustomHeader sets a custom header for an HTTP request. -// This function allows setting arbitrary headers for specialized API requirements. -func SetCustomHeader(req *http.Request, headerName, headerValue string) { - req.Header.Set(headerName, headerValue) -} - -// SetUserAgentHeader sets the User-Agent header for an HTTP request. -func SetUserAgentHeader() string { - return fmt.Sprintf("%s/%s", version.UserAgentBase, version.SDKVersion) -} - -// SetRequestHeaders sets the necessary HTTP headers for a given request using the APIHandler to determine the required headers. -func (h *HeaderHandler) SetRequestHeaders(endpoint string) { - // Retrieve the standard headers required for the request - standardHeaders := h.apiHandler.GetAPIRequestHeaders(endpoint) - - // Loop through the standard headers and set them on the request - for header, value := range standardHeaders { - if header == "Authorization" { - // Set the Authorization header using the token - h.SetAuthorization() // Ensure the token is correctly prefixed with "Bearer " - } else if value != "" { - h.req.Header.Set(header, value) - } - } -} - -// LogHeaders prints all the current headers in the http.Request using the zap logger. -// It uses the RedactSensitiveHeaderData function to redact sensitive data based on the hideSensitiveData flag. -func (h *HeaderHandler) LogHeaders(hideSensitiveData bool) { - if h.log.GetLogLevel() <= logger.LogLevelDebug { - // Initialize a new Header to hold the potentially redacted headers - redactedHeaders := http.Header{} - - for name, values := range h.req.Header { - // Redact sensitive values - if len(values) > 0 { - // Use the first value for simplicity; adjust if multiple values per header are expected - redactedValue := redact.RedactSensitiveHeaderData(hideSensitiveData, name, values[0]) - redactedHeaders.Set(name, redactedValue) - } - } - - // Convert the redacted headers to a string for logging - headersStr := HeadersToString(redactedHeaders) - - // Log the redacted headers - h.log.Debug("HTTP Request Headers", zap.String("Headers", headersStr)) - } -} - -// HeadersToString converts a http.Header to a string for logging, -// with each header on a new line for readability. -func HeadersToString(headers http.Header) string { - var headerStrings []string - for name, values := range headers { - // Join all values for the header with a comma, as per HTTP standard - valueStr := strings.Join(values, ", ") - headerStrings = append(headerStrings, fmt.Sprintf("%s: %s", name, valueStr)) - } - return strings.Join(headerStrings, "\n") // "\n" as seperator. -} - -// CheckDeprecationHeader checks the response headers for the Deprecation header and logs a warning if present. -func CheckDeprecationHeader(resp *http.Response, log logger.Logger) { - deprecationHeader := resp.Header.Get("Deprecation") - if deprecationHeader != "" { - - log.Warn("API endpoint is deprecated", - zap.String("Date", deprecationHeader), - zap.String("Endpoint", resp.Request.URL.String()), - ) - } -} diff --git a/headers/headers_test.go b/headers/headers_test.go.bk similarity index 100% rename from headers/headers_test.go rename to headers/headers_test.go.bk diff --git a/helpers/helpers.go b/helpers/helpers.go deleted file mode 100644 index 112a021..0000000 --- a/helpers/helpers.go +++ /dev/null @@ -1,80 +0,0 @@ -// helpers/helpers.go -package helpers - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "time" -) - -// ParseISO8601Date attempts to parse a string date in ISO 8601 format. -func ParseISO8601Date(dateStr string) (time.Time, error) { - return time.Parse(time.RFC3339, dateStr) -} - -// SafeOpenFile opens a file safely after validating and resolving its path. -func SafeOpenFile(filePath string) (*os.File, error) { - // Clean the file path to remove any ".." or similar components that can lead to directory traversal - cleanPath := filepath.Clean(filePath) - - // Resolve the clean path to an absolute path and ensure it resolves any symbolic links - absPath, err := filepath.EvalSymlinks(cleanPath) - if err != nil { - return nil, fmt.Errorf("unable to resolve the absolute path: %s, error: %w", filePath, err) - } - - // Optionally, check if the absolute path is within a permitted directory (omitted here for brevity) - // Example: allowedPathPrefix := "/safe/directory/" - // if !strings.HasPrefix(absPath, allowedPathPrefix) { - // return nil, fmt.Errorf("access to the file path is not allowed: %s", absPath) - // } - - // Open the file if the path is deemed safe - return os.Open(absPath) -} - -// UnmarshalJSON parses the duration from JSON string. -func (d *JSONDuration) UnmarshalJSON(b []byte) error { - var s string - if err := json.Unmarshal(b, &s); err != nil { - return err - } - duration, err := time.ParseDuration(s) - if err != nil { - return err - } - *d = JSONDuration(duration) - return nil -} - -// Duration returns the time.Duration value. -func (d JSONDuration) Duration() time.Duration { - return time.Duration(d) -} - -// MarshalJSON returns the JSON representation of the duration. -func (d JSONDuration) String() string { - return time.Duration(d).String() -} - -// JSONDuration wraps time.Duration for custom JSON unmarshalling. -type JSONDuration time.Duration - -// GetEnvOrDefault returns the value of an environment variable or a default value. -func GetEnvOrDefault(envKey string, defaultValue string) string { - if value, exists := os.LookupEnv(envKey); exists { - return value - } - return defaultValue -} - -// ParseJSONDuration attempts to parse a string value as a duration and returns the result or a default value. -func ParseJSONDuration(value string, defaultVal JSONDuration) JSONDuration { - result, err := time.ParseDuration(value) - if err != nil { - return defaultVal - } - return JSONDuration(result) -} diff --git a/helpers/helpers_test.go b/helpers/helpers_test.go.bk similarity index 100% rename from helpers/helpers_test.go rename to helpers/helpers_test.go.bk diff --git a/httpclient/auth_method.go b/httpclient/auth_method.go deleted file mode 100644 index c03be27..0000000 --- a/httpclient/auth_method.go +++ /dev/null @@ -1,64 +0,0 @@ -// authenticationhandler/httpclient_auth_method.go - -/* The authenticationhandler package is dedicated to managing authentication -for HTTP clients, with support for multiple authentication strategies, -including OAuth and Bearer Token mechanisms. It encapsulates the logic for -determining the appropriate authentication method based on provided credentials, -validating those credentials, and managing authentication tokens. This package -aims to provide a flexible and extendable framework for handling authentication -in a secure and efficient manner, ensuring that HTTP clients can seamlessly -authenticate against various services with minimal configuration. */ - -package httpclient - -import ( - "errors" - - "github.com/deploymenttheory/go-api-http-client/authenticationhandler" -) - -// DetermineAuthMethod determines the authentication method based on the provided credentials. -// It prefers strong authentication methods (e.g., OAuth) over weaker ones (e.g., bearer tokens). -// It logs an error and returns "unknown" if no valid credentials are provided. -func DetermineAuthMethod(authConfig AuthConfig) (string, error) { - // Initialize validation flags as true - validClientID, validClientSecret, validUsername, validPassword := true, true, true, true - clientIDErrMsg, clientSecretErrMsg, usernameErrMsg, passwordErrMsg := "", "", "", "" - - // Validate ClientID and ClientSecret for OAuth if provided - if authConfig.ClientID != "" || authConfig.ClientSecret != "" { - validClientID, clientIDErrMsg = authenticationhandler.IsValidClientID(authConfig.ClientID) - validClientSecret, clientSecretErrMsg = authenticationhandler.IsValidClientSecret(authConfig.ClientSecret) - // If both ClientID and ClientSecret are valid, use OAuth - if validClientID && validClientSecret { - return "oauth2", nil - } - } - - // Validate Username and Password for Bearer if OAuth is not valid or not provided - if authConfig.Username != "" || authConfig.Password != "" { - validUsername, usernameErrMsg = authenticationhandler.IsValidUsername(authConfig.Username) - validPassword, passwordErrMsg = authenticationhandler.IsValidPassword(authConfig.Password) - // If both Username and Password are valid, use Bearer - if validUsername && validPassword { - return "basicauth", nil - } - } - - // Construct an error message if any of the provided fields are invalid - errorMsg := "No valid credentials provided." - if !validClientID && authConfig.ClientID != "" { - errorMsg += " " + clientIDErrMsg - } - if !validClientSecret && authConfig.ClientSecret != "" { - errorMsg += " " + clientSecretErrMsg - } - if !validUsername && authConfig.Username != "" { - errorMsg += " " + usernameErrMsg - } - if !validPassword && authConfig.Password != "" { - errorMsg += " " + passwordErrMsg - } - - return "unknown", errors.New(errorMsg) -} diff --git a/httpclient/auth_method_test.go b/httpclient/auth_method_test.go deleted file mode 100644 index 43afb12..0000000 --- a/httpclient/auth_method_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package httpclient - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestDetermineAuthMethod(t *testing.T) { - tests := []struct { - name string - authConfig AuthConfig - expectedAuth string - expectError bool - }{ - { - name: "Valid OAuth credentials", - authConfig: AuthConfig{ - ClientID: "123e4567-e89b-12d3-a456-426614174000", // Valid UUID format - ClientSecret: "validSecretWith16Chars", // Ensure it's at least 16 characters - }, - expectedAuth: "oauth", - expectError: false, - }, - { - name: "Valid Bearer credentials", - authConfig: AuthConfig{ - Username: "validUsername", - Password: "validPassword", - }, - expectedAuth: "bearer", - expectError: false, - }, - { - name: "Invalid OAuth credentials", - authConfig: AuthConfig{ - ClientID: "invalidClientID", - ClientSecret: "invalidClientSecret", - }, - expectedAuth: "unknown", - expectError: true, - }, - { - name: "Invalid Bearer credentials", - authConfig: AuthConfig{ - Username: "invalidUser", - Password: "short", - }, - expectedAuth: "unknown", - expectError: true, - }, - { - name: "Missing credentials", - authConfig: AuthConfig{}, - expectedAuth: "unknown", - expectError: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - authMethod, err := DetermineAuthMethod(tt.authConfig) - - if tt.expectError { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - assert.Equal(t, tt.expectedAuth, authMethod) - }) - } -} diff --git a/httpclient/client.go b/httpclient/client.go index 0b733ba..3e23d65 100644 --- a/httpclient/client.go +++ b/httpclient/client.go @@ -8,249 +8,103 @@ like the baseURL, authentication details, and an embedded standard HTTP client. package httpclient import ( + "fmt" "net/http" "time" - "github.com/deploymenttheory/go-api-http-client/apiintegrations/apihandler" - "github.com/deploymenttheory/go-api-http-client/authenticationhandler" "github.com/deploymenttheory/go-api-http-client/concurrency" - "github.com/deploymenttheory/go-api-http-client/helpers" "github.com/deploymenttheory/go-api-http-client/logger" "github.com/deploymenttheory/go-api-http-client/redirecthandler" "go.uber.org/zap" ) -// Client represents an HTTP client to interact with a specific API. +// TODO all struct comments + +// Master struct/object type Client struct { - AuthMethod string // Specifies the authentication method: "bearer" or "oauth" - Token string // Authentication Token - Expiry time.Time // Expiry time set for the auth token - httpClient *http.Client // Internal HTTP client - clientConfig ClientConfig // HTTP Client configuration - Logger logger.Logger // Logger for logging messages - ConcurrencyHandler *concurrency.ConcurrencyHandler // ConcurrencyHandler for managing concurrent requests - APIHandler apihandler.APIHandler // APIHandler interface used to define which API handler to use - AuthTokenHandler *authenticationhandler.AuthTokenHandler // AuthTokenHandler for managing authentication + config ClientConfig + http *http.Client + + AuthToken string + AuthTokenExpiry time.Time + Logger logger.Logger + Concurrency *concurrency.ConcurrencyHandler + Integration *APIIntegration } -// Config holds configuration options for the HTTP Client. +// Options/Variables for Client type ClientConfig struct { - Auth AuthConfig // User can either supply these values manually or pass from LoadAuthConfig/Env vars - Environment EnvironmentConfig // User can either supply these values manually or pass from LoadAuthConfig/Env vars - ClientOptions ClientOptions // Optional configuration options for the HTTP Client -} - -// AuthConfig represents the structure to read authentication details from a JSON configuration file. -type AuthConfig struct { - Username string `json:"Username,omitempty"` - Password string `json:"Password,omitempty"` - ClientID string `json:"ClientID,omitempty"` - ClientSecret string `json:"ClientSecret,omitempty"` -} - -// EnvironmentConfig represents the structure to read authentication details from a JSON configuration file. -type EnvironmentConfig struct { - APIType string `json:"APIType,omitempty"` // APIType specifies the type of API integration to use - InstanceName string `json:"InstanceName,omitempty"` // Website Instance name without the root domain - OverrideBaseDomain string `json:"OverrideBaseDomain,omitempty"` // Base domain override used when the default in the api handler isn't suitable - TenantID string `json:"TenantID,omitempty"` // TenantID is the unique identifier for the tenant - TenantName string `json:"TenantName,omitempty"` // TenantName is the name of the tenant -} - -// ClientOptions holds optional configuration options for the HTTP Client. -type ClientOptions struct { - Logging LoggingConfig // Configuration related to logging - Cookies CookieConfig // Cookie handling settings - Retry RetryConfig // Retry behavior configuration - Concurrency ConcurrencyConfig // Concurrency configuration - Timeout TimeoutConfig // Custom timeout settings - Redirect RedirectConfig // Redirect handling settings -} - -// LoggingConfig holds configuration options related to logging. -type LoggingConfig struct { - LogLevel string // Tiered logging level. - LogOutputFormat string // Output format of the logs. Use "JSON" for JSON format, "console" for human-readable format - LogConsoleSeparator string // Separator in console output format. - LogExportPath string // Path to output logs to. - HideSensitiveData bool // Whether sensitive fields should be hidden in logs. -} - -// CookieConfig holds configuration related to cookie handling. -type CookieConfig struct { - EnableCookieJar bool // Enable or disable cookie jar - CustomCookies map[string]string `json:"CustomCookies,omitempty"` // Key-value pairs for setting specific cookies -} - -// RetryConfig holds configuration related to retry behavior. -type RetryConfig struct { - MaxRetryAttempts int // Maximum number of retry request attempts for retryable HTTP methods. - EnableDynamicRateLimiting bool // Whether dynamic rate limiting should be enabled. -} - -// ConcurrencyConfig holds configuration related to concurrency management. -type ConcurrencyConfig struct { - MaxConcurrentRequests int // Maximum number of concurrent requests allowed. -} - -// TimeoutConfig holds custom timeout settings. -// type TimeoutConfig struct { -// CustomTimeout time.Duration // Custom timeout for the HTTP client -// TokenRefreshBufferPeriod time.Duration // Buffer period before token expiry to attempt token refresh -// TotalRetryDuration time.Duration // Total duration to attempt retries -// } - -type TimeoutConfig struct { - CustomTimeout helpers.JSONDuration // Custom timeout for the HTTP client - TokenRefreshBufferPeriod helpers.JSONDuration // Buffer period before token expiry to attempt token refresh - TotalRetryDuration helpers.JSONDuration // Total duration to attempt retries -} - -// RedirectConfig holds configuration related to redirect handling. -type RedirectConfig struct { - FollowRedirects bool // Enable or disable following redirects - MaxRedirects int // Maximum number of redirects to follow + Integration APIIntegration + HideSensitiveData bool + CustomCookies []*http.Cookie + MaxRetryAttempts int + MaxConcurrentRequests int + EnableDynamicRateLimiting bool + CustomTimeout time.Duration + TokenRefreshBufferPeriod time.Duration + TotalRetryDuration time.Duration // TODO do we need this now it's in the integration? + FollowRedirects bool + MaxRedirects int + ConcurrencyManagementEnabled bool } // BuildClient creates a new HTTP client with the provided configuration. -func BuildClient(config ClientConfig) (*Client, error) { - - // Parse the log level string to logger.LogLevel - parsedLogLevel := logger.ParseLogLevelFromString(config.ClientOptions.Logging.LogLevel) - - // Initialize the logger with parsed config values - log := logger.BuildLogger(parsedLogLevel, config.ClientOptions.Logging.LogOutputFormat, config.ClientOptions.Logging.LogConsoleSeparator, config.ClientOptions.Logging.LogExportPath) - - // Set the logger's level (optional if BuildLogger already sets the level based on the input) - log.SetLevel(parsedLogLevel) - - // Use the APIType from the config to determine which API handler to load - apiHandler, err := apihandler.LoadAPIHandler(config.Environment.APIType, config.Environment.InstanceName, config.Environment.TenantID, config.Environment.TenantName, log) +func BuildClient(config ClientConfig, populateDefaultValues bool, log logger.Logger) (*Client, error) { + err := validateClientConfig(config, populateDefaultValues) if err != nil { - log.Error("Failed to load API handler", zap.String("APIType", config.Environment.APIType), zap.Error(err)) - return nil, err + return nil, fmt.Errorf("invalid configuration: %v", err) } - // Determine the authentication method using the helper function - authMethod, err := DetermineAuthMethod(config.Auth) - if err != nil { - log.Error("Failed to determine authentication method", zap.Error(err)) - return nil, err - } - - // Initialize AuthTokenHandler - clientCredentials := authenticationhandler.ClientCredentials{ - Username: config.Auth.Username, - Password: config.Auth.Password, - ClientID: config.Auth.ClientID, - ClientSecret: config.Auth.ClientSecret, - } - - authTokenHandler := authenticationhandler.NewAuthTokenHandler( - log, - authMethod, - clientCredentials, - config.Environment.InstanceName, - config.ClientOptions.Logging.HideSensitiveData, - ) - - log.Info("Initializing new HTTP client with the provided configuration") + log.Info(fmt.Sprintf("initializing new http client, auth: %s", config.Integration.Domain())) - // Initialize the internal HTTP client httpClient := &http.Client{ - Timeout: config.ClientOptions.Timeout.CustomTimeout.Duration(), + Timeout: config.CustomTimeout, } - // Conditionally setup cookie jar - // if err := SetupCookieJar(httpClient, config, log); err != nil { - // log.Error("Error setting up cookie jar", zap.Error(err)) - // return nil, err - // } - - // Conditionally setup redirect handling - if err := redirecthandler.SetupRedirectHandler(httpClient, config.ClientOptions.Redirect.FollowRedirects, config.ClientOptions.Redirect.MaxRedirects, log); err != nil { + // TODO refactor redirects + if err := redirecthandler.SetupRedirectHandler(httpClient, config.FollowRedirects, config.MaxRedirects, log); err != nil { log.Error("Failed to set up redirect handler", zap.Error(err)) return nil, err } - // Initialize ConcurrencyMetrics specifically for ConcurrencyHandler - concurrencyMetrics := &concurrency.ConcurrencyMetrics{} - - // Initialize the ConcurrencyHandler with the newly created ConcurrencyMetrics - concurrencyHandler := concurrency.NewConcurrencyHandler( - config.ClientOptions.Concurrency.MaxConcurrentRequests, - log, - concurrencyMetrics, - ) + var concurrencyHandler *concurrency.ConcurrencyHandler + if config.ConcurrencyManagementEnabled { + concurrencyMetrics := &concurrency.ConcurrencyMetrics{} + concurrencyHandler = concurrency.NewConcurrencyHandler( + config.MaxConcurrentRequests, + log, + concurrencyMetrics, + ) + } else { + concurrencyHandler = nil + } - // Create a new HTTP client with the provided configuration. client := &Client{ - APIHandler: apiHandler, - AuthMethod: authMethod, - httpClient: httpClient, - clientConfig: config, - Logger: log, - ConcurrencyHandler: concurrencyHandler, - AuthTokenHandler: authTokenHandler, + Integration: &config.Integration, + http: httpClient, + config: config, + Logger: log, + Concurrency: concurrencyHandler, + } + + if len(client.config.CustomCookies) > 0 { + client.loadCustomCookies(config.CustomCookies) } - // Log the client's configuration. - log.Info("New API client initialized", - zap.String("API Type", config.Environment.APIType), - zap.String("Instance Name", config.Environment.InstanceName), - zap.String("Override Base Domain", config.Environment.OverrideBaseDomain), - zap.String("Tenant ID", config.Environment.TenantID), - zap.String("Tenant Name", config.Environment.TenantName), - zap.String("Authentication Method", authMethod), - zap.String("Logging Level", config.ClientOptions.Logging.LogLevel), - zap.String("Log Encoding Format", config.ClientOptions.Logging.LogOutputFormat), - zap.String("Log Separator", config.ClientOptions.Logging.LogConsoleSeparator), - zap.Bool("Hide Sensitive Data In Logs", config.ClientOptions.Logging.HideSensitiveData), - zap.Bool("Cookie Jar Enabled", config.ClientOptions.Cookies.EnableCookieJar), - zap.Int("Max Retry Attempts", config.ClientOptions.Retry.MaxRetryAttempts), - zap.Bool("Enable Dynamic Rate Limiting", config.ClientOptions.Retry.EnableDynamicRateLimiting), - zap.Int("Max Concurrent Requests", config.ClientOptions.Concurrency.MaxConcurrentRequests), - zap.Bool("Follow Redirects", config.ClientOptions.Redirect.FollowRedirects), - zap.Int("Max Redirects", config.ClientOptions.Redirect.MaxRedirects), - zap.Duration("Token Refresh Buffer Period", config.ClientOptions.Timeout.TokenRefreshBufferPeriod.Duration()), - zap.Duration("Total Retry Duration", config.ClientOptions.Timeout.TotalRetryDuration.Duration()), - zap.Duration("Custom Timeout", config.ClientOptions.Timeout.CustomTimeout.Duration()), + log.Debug("New API client initialized", + zap.String("Authentication Method", (*client.Integration).GetAuthMethodDescriptor()), + zap.Bool("Hide Sensitive Data In Logs", config.HideSensitiveData), + zap.Int("Max Retry Attempts", config.MaxRetryAttempts), + zap.Bool("Enable Dynamic Rate Limiting", config.EnableDynamicRateLimiting), + zap.Int("Max Concurrent Requests", config.MaxConcurrentRequests), + zap.Bool("Follow Redirects", config.FollowRedirects), + zap.Int("Max Redirects", config.MaxRedirects), + zap.Duration("Token Refresh Buffer Period", config.TokenRefreshBufferPeriod), + zap.Duration("Total Retry Duration", config.TotalRetryDuration), + zap.Duration("Custom Timeout", config.CustomTimeout), ) return client, nil } - -// // SetupCookieJar sets up the cookie jar for the HTTP client if enabled in the configuration. -// func SetupCookieJar(client *http.Client, clientConfig ClientConfig, log logger.Logger) error { -// if clientConfig.ClientOptions.Cookies.EnableCookieJar { -// jar, err := cookiejar.New(nil) // nil options use default options -// if err != nil { -// log.Error("Failed to create cookie jar", zap.Error(err)) -// return fmt.Errorf("setupCookieJar failed: %w", err) // Wrap and return the error -// } - -// if clientConfig.ClientOptions.Cookies.CustomCookies != nil { -// var CookieList []*http.Cookie -// CookieList = make([]*http.Cookie, 0) -// for k, v := range clientConfig.ClientOptions.Cookies.CustomCookies { -// newCookie := &http.Cookie{ -// Name: k, -// Value: v, -// } -// CookieList = append(CookieList, newCookie) -// } - -// cookieUrl, err := url.Parse(fmt.Sprintf("http://%s.jamfcloud.com", clientConfig.Environment.InstanceName)) -// if err != nil { -// return err -// } - -// jar.SetCookies(cookieUrl, CookieList) -// } - -// client.Jar = jar -// } -// return nil -// } diff --git a/httpclient/client_configuration.go b/httpclient/client_configuration.go deleted file mode 100644 index fadc668..0000000 --- a/httpclient/client_configuration.go +++ /dev/null @@ -1,368 +0,0 @@ -// httpclient/client_configuration.go -// Description: This file contains functions to load and validate configuration values from a JSON file or environment variables. -package httpclient - -import ( - "encoding/json" - "fmt" - "log" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/deploymenttheory/go-api-http-client/helpers" - "github.com/deploymenttheory/go-api-http-client/logger" -) - -const ( - DefaultLogLevel = logger.LogLevelInfo - DefaultMaxRetryAttempts = 3 - DefaultEnableDynamicRateLimiting = true - DefaultMaxConcurrentRequests = 5 - DefaultTokenBufferPeriod = helpers.JSONDuration(5 * time.Minute) - DefaultTotalRetryDuration = helpers.JSONDuration(5 * time.Minute) - DefaultTimeout = helpers.JSONDuration(10 * time.Second) - FollowRedirects = true - MaxRedirects = 10 - ConfigFileExtension = ".json" -) - -// LoadConfigFromFile loads configuration values from a JSON file into the ClientConfig struct. -// This function opens the specified configuration file, reads its content, and unmarshals the JSON data -// into the ClientConfig struct. It's designed to initialize the client configuration with values -// from a file, complementing or overriding defaults and environment variable settings. -func LoadConfigFromFile(filePath string) (*ClientConfig, error) { - // Clean up the file path to prevent directory traversal - cleanPath := filepath.Clean(filePath) - - // Resolve the cleanPath to an absolute path to ensure it resolves any symbolic links - absPath, err := filepath.EvalSymlinks(cleanPath) - if err != nil { - return nil, fmt.Errorf("unable to resolve the absolute path of the configuration file: %s, error: %w", filePath, err) - } - - // Check for suspicious patterns in the resolved path - if strings.Contains(absPath, "..") { - return nil, fmt.Errorf("invalid path, path traversal patterns detected: %s", filePath) - } - - // Ensure the file has the correct extension - if filepath.Ext(absPath) != ConfigFileExtension { - return nil, fmt.Errorf("invalid file extension for configuration file: %s, expected .json", filePath) - } - - // Read the entire file - fileBytes, err := os.ReadFile(absPath) - if err != nil { - return nil, fmt.Errorf("failed to read the configuration file: %s, error: %w", filePath, err) - } - - // Initialize an instance of ClientConfig - var config ClientConfig - - // Unmarshal the file content into the ClientConfig struct - if err := json.Unmarshal(fileBytes, &config); err != nil { - return nil, fmt.Errorf("failed to unmarshal the configuration file: %s, error: %w", filePath, err) - } - - log.Printf("Configuration successfully loaded from file: %s", filePath) - - // Set default values if necessary and validate the configuration - setLoggerDefaultValues(&config) - setClientDefaultValues(&config) - if err := validateMandatoryConfiguration(&config); err != nil { - return nil, fmt.Errorf("configuration validation failed: %w", err) - } - - return &config, nil -} - -// LoadConfigFromEnv populates the ClientConfig structure with values from environment variables. -// It updates the configuration for authentication, environment specifics, and client options -// based on the presence of environment variables. For each configuration option, if an environment -// variable is set, its value is used; otherwise, the existing value in the ClientConfig structure -// is retained. It also sets default values if necessary and validates the final configuration, -// returning an error if the configuration is incomplete. -func LoadConfigFromEnv(config *ClientConfig) (*ClientConfig, error) { - if config == nil { - config = &ClientConfig{} // Initialize config if nil - } - - // AuthConfig - config.Auth.Username = getEnvOrDefault("USERNAME", config.Auth.Username) - log.Printf("Username env value found and set to: %s", config.Auth.Username) - - config.Auth.Password = getEnvOrDefault("PASSWORD", config.Auth.Password) - log.Printf("Password env value found and set") - - config.Auth.ClientID = getEnvOrDefault("CLIENT_ID", config.Auth.ClientID) - log.Printf("ClientID env value found and set to: %s", config.Auth.ClientID) - - config.Auth.ClientSecret = getEnvOrDefault("CLIENT_SECRET", config.Auth.ClientSecret) - log.Printf("ClientSecret env value found and set") - - // EnvironmentConfig - config.Environment.APIType = getEnvOrDefault("API_TYPE", config.Environment.APIType) - log.Printf("APIType env value found and set to: %s", config.Environment.APIType) - - config.Environment.InstanceName = getEnvOrDefault("INSTANCE_NAME", config.Environment.InstanceName) - log.Printf("InstanceName env value found and set to: %s", config.Environment.InstanceName) - - config.Environment.OverrideBaseDomain = getEnvOrDefault("OVERRIDE_BASE_DOMAIN", config.Environment.OverrideBaseDomain) - log.Printf("OverrideBaseDomain env value found and set to: %s", config.Environment.OverrideBaseDomain) - - config.Environment.TenantID = getEnvOrDefault("TENANT_ID", config.Environment.TenantID) - log.Printf("TenantID env value found and set to: %s", config.Environment.TenantID) - - config.Environment.TenantName = getEnvOrDefault("TENANT_NAME", config.Environment.TenantName) - log.Printf("TenantName env value found and set to: %s", config.Environment.TenantName) - - // ClientOptions - - // Logging - config.ClientOptions.Logging.LogLevel = getEnvOrDefault("LOG_LEVEL", config.ClientOptions.Logging.LogLevel) - log.Printf("LogLevel env value found and set to: %s", config.ClientOptions.Logging.LogLevel) - - config.ClientOptions.Logging.LogOutputFormat = getEnvOrDefault("LOG_OUTPUT_FORMAT", config.ClientOptions.Logging.LogOutputFormat) - log.Printf("LogOutputFormat env value found and set to: %s", config.ClientOptions.Logging.LogOutputFormat) - - config.ClientOptions.Logging.LogConsoleSeparator = getEnvOrDefault("LOG_CONSOLE_SEPARATOR", config.ClientOptions.Logging.LogConsoleSeparator) - log.Printf("LogConsoleSeparator env value found and set to: %s", config.ClientOptions.Logging.LogConsoleSeparator) - - config.ClientOptions.Logging.LogExportPath = getEnvOrDefault("LOG_EXPORT_PATH", config.ClientOptions.Logging.LogExportPath) - log.Printf("LogExportPath env value found and set to: %s", config.ClientOptions.Logging.LogExportPath) - - config.ClientOptions.Logging.HideSensitiveData = parseBool(getEnvOrDefault("HIDE_SENSITIVE_DATA", strconv.FormatBool(config.ClientOptions.Logging.HideSensitiveData))) - log.Printf("HideSensitiveData env value found and set to: %t", config.ClientOptions.Logging.HideSensitiveData) - - // Cookies - config.ClientOptions.Cookies.EnableCookieJar = parseBool(getEnvOrDefault("ENABLE_COOKIE_JAR", strconv.FormatBool(config.ClientOptions.Cookies.EnableCookieJar))) - log.Printf("EnableCookieJar env value found and set to: %t", config.ClientOptions.Cookies.EnableCookieJar) - - // Load specific cookies from environment variable - cookieStr := getEnvOrDefault("CUSTOM_COOKIES", "") - if cookieStr != "" { - config.ClientOptions.Cookies.CustomCookies = parseCookiesFromString(cookieStr) - log.Printf("CustomCookies env value found and set") - } - - // Retry - config.ClientOptions.Retry.MaxRetryAttempts = parseInt(getEnvOrDefault("MAX_RETRY_ATTEMPTS", strconv.Itoa(config.ClientOptions.Retry.MaxRetryAttempts)), DefaultMaxRetryAttempts) - log.Printf("MaxRetryAttempts env value found and set to: %d", config.ClientOptions.Retry.MaxRetryAttempts) - - config.ClientOptions.Retry.EnableDynamicRateLimiting = parseBool(getEnvOrDefault("ENABLE_DYNAMIC_RATE_LIMITING", strconv.FormatBool(config.ClientOptions.Retry.EnableDynamicRateLimiting))) - log.Printf("EnableDynamicRateLimiting env value found and set to: %t", config.ClientOptions.Retry.EnableDynamicRateLimiting) - - // Concurrency - config.ClientOptions.Concurrency.MaxConcurrentRequests = parseInt(getEnvOrDefault("MAX_CONCURRENT_REQUESTS", strconv.Itoa(config.ClientOptions.Concurrency.MaxConcurrentRequests)), DefaultMaxConcurrentRequests) - log.Printf("MaxConcurrentRequests env value found and set to: %d", config.ClientOptions.Concurrency.MaxConcurrentRequests) - - // timeouts - config.ClientOptions.Timeout.TokenRefreshBufferPeriod = helpers.ParseJSONDuration(getEnvOrDefault("TOKEN_REFRESH_BUFFER_PERIOD", config.ClientOptions.Timeout.TokenRefreshBufferPeriod.String()), DefaultTokenBufferPeriod) - log.Printf("TokenRefreshBufferPeriod env value found and set to: %s", config.ClientOptions.Timeout.TokenRefreshBufferPeriod) - - config.ClientOptions.Timeout.TotalRetryDuration = helpers.ParseJSONDuration(getEnvOrDefault("TOTAL_RETRY_DURATION", config.ClientOptions.Timeout.TotalRetryDuration.String()), DefaultTotalRetryDuration) - log.Printf("TotalRetryDuration env value found and set to: %s", config.ClientOptions.Timeout.TotalRetryDuration) - - config.ClientOptions.Timeout.CustomTimeout = helpers.ParseJSONDuration(getEnvOrDefault("CUSTOM_TIMEOUT", config.ClientOptions.Timeout.CustomTimeout.String()), DefaultTimeout) - log.Printf("CustomTimeout env value found and set to: %s", config.ClientOptions.Timeout.CustomTimeout) - - // Redirects - config.ClientOptions.Redirect.FollowRedirects = parseBool(getEnvOrDefault("FOLLOW_REDIRECTS", strconv.FormatBool(config.ClientOptions.Redirect.FollowRedirects))) - log.Printf("FollowRedirects env value set to: %t", config.ClientOptions.Redirect.FollowRedirects) - - config.ClientOptions.Redirect.MaxRedirects = parseInt(getEnvOrDefault("MAX_REDIRECTS", strconv.Itoa(config.ClientOptions.Redirect.MaxRedirects)), MaxRedirects) - log.Printf("MaxRedirects env value set to: %d", config.ClientOptions.Redirect.MaxRedirects) - - // Set default values if necessary - setLoggerDefaultValues(config) - setClientDefaultValues(config) - - // Validate final configuration - if err := validateMandatoryConfiguration(config); err != nil { - return nil, err // Return the error if the configuration is incomplete - } - - return config, nil -} - -// validateMandatoryConfiguration checks if any essential configuration fields are missing, -// and returns an error with details about the missing configurations. -// This ensures the caller can understand what specific configurations need attention. -func validateMandatoryConfiguration(config *ClientConfig) error { - var missingFields []string - - // Check for mandatory fields related to the environment - if config.Environment.InstanceName == "" { - missingFields = append(missingFields, "Environment.InstanceName") - } - if config.Environment.APIType == "" { - missingFields = append(missingFields, "Environment.APIType") - } - - // Check for mandatory fields related to the client options - if config.ClientOptions.Logging.LogLevel == "" { - missingFields = append(missingFields, "ClientOptions.Logging.LogLevel") - } - if config.ClientOptions.Logging.LogOutputFormat == "" { - missingFields = append(missingFields, "ClientOptions.Logging.LogOutputFormat") - } - if config.ClientOptions.Logging.LogConsoleSeparator == "" { - missingFields = append(missingFields, "ClientOptions.Logging.LogConsoleSeparator") - } - - // Check for either OAuth credentials pair or Username and Password pair - usingOAuth := config.Auth.ClientID != "" && config.Auth.ClientSecret != "" - usingBasicAuth := config.Auth.Username != "" && config.Auth.Password != "" - - if !(usingOAuth || usingBasicAuth) { - if config.Auth.ClientID == "" { - missingFields = append(missingFields, "Auth.ClientID") - } - if config.Auth.ClientSecret == "" { - missingFields = append(missingFields, "Auth.ClientSecret") - } - if config.Auth.Username == "" { - missingFields = append(missingFields, "Auth.Username") - } - if config.Auth.Password == "" { - missingFields = append(missingFields, "Auth.Password") - } - } - - // Default setting for MaxRedirects - if config.ClientOptions.Redirect.MaxRedirects <= 0 { - config.ClientOptions.Redirect.MaxRedirects = MaxRedirects - log.Printf("MaxRedirects not set or invalid, set to default value: %d", MaxRedirects) - } - - // If there are missing fields, construct and return an error message detailing what is missing - if len(missingFields) > 0 { - errorMessage := fmt.Sprintf("Mandatory configuration missing: %s. Ensure that either OAuth credentials (ClientID and ClientSecret) or Basic Auth credentials (Username and Password) are fully provided.", strings.Join(missingFields, ", ")) - return fmt.Errorf(errorMessage) - } - - // If no fields are missing, return nil indicating the configuration is complete - return nil -} - -// setClientDefaultValues sets default values for the client configuration options if none are provided. -// It checks each configuration option and sets it to the default value if it is either negative, zero, -// or not set. This function ensures that the configuration adheres to expected minimums or defaults, -// enhancing robustness and fault tolerance. It uses the standard log package for logging, ensuring that -// default value settings are transparent before the zap logger is initialized. -func setClientDefaultValues(config *ClientConfig) { - if config.ClientOptions.Retry.MaxRetryAttempts < 0 { - config.ClientOptions.Retry.MaxRetryAttempts = DefaultMaxRetryAttempts - log.Printf("MaxRetryAttempts was negative, set to default value: %d", DefaultMaxRetryAttempts) - } - - if config.ClientOptions.Concurrency.MaxConcurrentRequests <= 0 { - config.ClientOptions.Concurrency.MaxConcurrentRequests = DefaultMaxConcurrentRequests - log.Printf("MaxConcurrentRequests was negative or zero, set to default value: %d", DefaultMaxConcurrentRequests) - } - - if config.ClientOptions.Timeout.TokenRefreshBufferPeriod < 0 { - config.ClientOptions.Timeout.TokenRefreshBufferPeriod = DefaultTokenBufferPeriod - log.Printf("TokenRefreshBufferPeriod was negative, set to default value: %s", DefaultTokenBufferPeriod) - } - - if config.ClientOptions.Timeout.TotalRetryDuration <= 0 { - config.ClientOptions.Timeout.TotalRetryDuration = DefaultTotalRetryDuration - log.Printf("TotalRetryDuration was negative or zero, set to default value: %s", DefaultTotalRetryDuration) - } - - if config.ClientOptions.Timeout.TokenRefreshBufferPeriod == 0 { - config.ClientOptions.Timeout.TokenRefreshBufferPeriod = DefaultTokenBufferPeriod - log.Printf("TokenRefreshBufferPeriod not set, set to default value: %s", DefaultTokenBufferPeriod) - } - - if config.ClientOptions.Timeout.TotalRetryDuration == 0 { - config.ClientOptions.Timeout.TotalRetryDuration = DefaultTotalRetryDuration - log.Printf("TotalRetryDuration not set, set to default value: %s", DefaultTotalRetryDuration) - } - - if config.ClientOptions.Timeout.CustomTimeout == 0 { - config.ClientOptions.Timeout.CustomTimeout = DefaultTimeout - log.Printf("CustomTimeout not set, set to default value: %s", DefaultTimeout) - } - - if !config.ClientOptions.Redirect.FollowRedirects { - config.ClientOptions.Redirect.FollowRedirects = FollowRedirects - log.Printf("FollowRedirects not set, set to default value: %t", FollowRedirects) - } - - if config.ClientOptions.Redirect.MaxRedirects <= 0 { - config.ClientOptions.Redirect.MaxRedirects = MaxRedirects - log.Printf("MaxRedirects not set or invalid, set to default value: %d", MaxRedirects) - } - - // Log completion of setting default values - log.Println("Default values set for client configuration") -} - -// Helper function to get environment variable or default value -func getEnvOrDefault(envKey string, defaultValue string) string { - if value, exists := os.LookupEnv(envKey); exists { - return value - } - return defaultValue -} - -// Helper function to parse boolean from environment variable -func parseBool(value string) bool { - result, err := strconv.ParseBool(value) - if err != nil { - return false - } - return result -} - -// Helper function to parse int from environment variable -func parseInt(value string, defaultVal int) int { - result, err := strconv.Atoi(value) - if err != nil { - return defaultVal - } - return result -} - -// Helper function to parse duration from environment variable -func parseDuration(value string, defaultVal time.Duration) time.Duration { - result, err := time.ParseDuration(value) - if err != nil { - return defaultVal - } - return result -} - -// setLoggerDefaultValues sets default values for the client logger configuration options if none are provided. -// It checks each configuration option and sets it to the default value if it is either negative, zero, -// or not set. It also logs each default value being set. -func setLoggerDefaultValues(config *ClientConfig) { - // Set default value if none is provided - if config.ClientOptions.Logging.LogConsoleSeparator == "" { - config.ClientOptions.Logging.LogConsoleSeparator = "," - log.Println("LogConsoleSeparator not set, set to default value: ,") - } - - // Log completion of setting default values - log.Println("Default values set for logger configuration") -} - -// parseCookiesFromString parses a semi-colon separated string of key=value pairs into a map. -func parseCookiesFromString(cookieStr string) map[string]string { - cookies := make(map[string]string) - pairs := strings.Split(cookieStr, ";") - for _, pair := range pairs { - kv := strings.SplitN(pair, "=", 2) - if len(kv) == 2 { - key := strings.TrimSpace(kv[0]) - value := strings.TrimSpace(kv[1]) - cookies[key] = value - } - } - return cookies -} diff --git a/httpclient/client_configuration_test.go b/httpclient/client_configuration_test.go deleted file mode 100644 index 49b7a6a..0000000 --- a/httpclient/client_configuration_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package httpclient - -import ( - "os" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestLoadConfigFromFile(t *testing.T) { - // Create a temporary file - tmpFile, err := os.CreateTemp("", "config-*.json") - assert.NoError(t, err) - defer os.Remove(tmpFile.Name()) // Clean up the file after test - - // Write updated JSON configuration to the temp file - configJSON := `{ - "Auth": { - "ClientID": "787xxxxd-98bb-xxxx-8d17-xxx0f8cbfb7b", - "ClientSecret": "xxxxxxxxxxxxx" - }, - "Environment": { - "InstanceName": "lbgsandbox", - "OverrideBaseDomain": "", - "APIType": "jamfpro" - }, - "ClientOptions": { - "LogLevel": "LogLevelDebug", - "LogOutputFormat": "console", - "LogConsoleSeparator": " ", - "HideSensitiveData": true, - "EnableDynamicRateLimiting": true, - "MaxRetryAttempts": 5, - "MaxConcurrentRequests": 3, - "EnableCookieJar": true, - "FollowRedirects": true, - "MaxRedirects": 5 - } - }` - _, err = tmpFile.WriteString(configJSON) - assert.NoError(t, err) - assert.NoError(t, tmpFile.Close()) - - // Test loading from the temp file - config, err := LoadConfigFromFile(tmpFile.Name()) - assert.NoError(t, err) - assert.Equal(t, "787xxxxd-98bb-xxxx-8d17-xxx0f8cbfb7b", config.Auth.ClientID) - assert.Equal(t, "xxxxxxxxxxxxx", config.Auth.ClientSecret) - assert.Equal(t, "lbgsandbox", config.Environment.InstanceName) - assert.Equal(t, "jamfpro", config.Environment.APIType) - assert.Equal(t, "LogLevelDebug", config.ClientOptions.Logging.LogLevel) - assert.Equal(t, "console", config.ClientOptions.Logging.LogOutputFormat) - assert.Equal(t, " ", config.ClientOptions.Logging.LogConsoleSeparator) - assert.True(t, config.ClientOptions.Logging.HideSensitiveData) - assert.True(t, config.ClientOptions.Retry.EnableDynamicRateLimiting) - assert.Equal(t, 5, config.ClientOptions.Retry.MaxRetryAttempts) - assert.Equal(t, 3, config.ClientOptions.Concurrency.MaxConcurrentRequests) - assert.True(t, config.ClientOptions.Cookies.EnableCookieJar) - assert.True(t, config.ClientOptions.Redirect.FollowRedirects) - assert.Equal(t, 5, config.ClientOptions.Redirect.MaxRedirects) -} - -func TestGetEnvOrDefault(t *testing.T) { - const envKey = "TEST_ENV_VAR" - defer os.Unsetenv(envKey) - - // Scenario 1: Environment variable is set - expectedValue := "test_value" - os.Setenv(envKey, expectedValue) - assert.Equal(t, expectedValue, getEnvOrDefault(envKey, "default_value")) - - // Scenario 2: Environment variable is not set - assert.Equal(t, "default_value", getEnvOrDefault("NON_EXISTENT_ENV_VAR", "default_value")) -} - -func TestParseBool(t *testing.T) { - assert.True(t, parseBool("true")) - assert.False(t, parseBool("false")) - assert.False(t, parseBool("invalid_value")) -} - -func TestParseInt(t *testing.T) { - assert.Equal(t, 42, parseInt("42", 10)) - assert.Equal(t, 10, parseInt("invalid_value", 10)) -} - -func TestParseDuration(t *testing.T) { - assert.Equal(t, 5*time.Minute, parseDuration("5m", 1*time.Minute)) - assert.Equal(t, 1*time.Minute, parseDuration("invalid_value", 1*time.Minute)) -} - -func TestSetLoggerDefaultValues(t *testing.T) { - config := &ClientConfig{ClientOptions: ClientOptions{}} - setLoggerDefaultValues(config) - assert.Equal(t, ",", config.ClientOptions.Logging.LogConsoleSeparator) -} diff --git a/httpclient/config_validation.go b/httpclient/config_validation.go new file mode 100644 index 0000000..4b879b2 --- /dev/null +++ b/httpclient/config_validation.go @@ -0,0 +1,118 @@ +// httpclient/client_configuration.go +// Description: This file contains functions to load and validate configuration values from a JSON file or environment variables. +package httpclient + +import ( + "errors" + "time" +) + +const ( + DefaultLogLevelString = "LogLevelInfo" + DefaultLogOutputFormatString = "pretty" + DefaultLogConsoleSeparator = " " + DefaultLogExportPath = "/defaultlogs" + DefaultMaxRetryAttempts = 3 + DefaultMaxConcurrentRequests = 1 + DefaultExportLogs = false + DefaultHideSensitiveData = false + DefaultEnableDynamicRateLimiting = false + DefaultCustomTimeout = 5 * time.Second + DefaultTokenRefreshBufferPeriod = 2 * time.Minute + DefaultTotalRetryDuration = 5 * time.Minute + DefaultFollowRedirects = false + DefaultMaxRedirects = 5 +) + +// TODO migrate all the loose strings + +// TODO LoadConfigFromFile Func +func LoadConfigFromFile(filepath string) (*ClientConfig, error) { + return nil, nil +} + +// TODO LoadConfigFromEnv Func +func LoadConfigFromEnv() (*ClientConfig, error) { + return nil, nil +} + +// TODO Review validateClientConfig +func validateClientConfig(config ClientConfig, populateDefaults bool) error { + + if populateDefaults { + SetDefaultValuesClientConfig(&config) + } + + // TODO adjust these strings to have links to documentation & centralise them + if config.Integration == nil { + return errors.New("no api integration supplied, please see documentation") + } + + if config.MaxRetryAttempts < 0 { + return errors.New("max retry cannot be less than 0") + } + + if config.MaxConcurrentRequests < 1 { + return errors.New("maximum concurrent requests cannot be less than 1") + } + + if config.CustomTimeout.Seconds() < 0 { + return errors.New("timeout cannot be less than 0 seconds") + } + + if config.TokenRefreshBufferPeriod.Seconds() < 0 { + return errors.New("refresh buffer period cannot be less than 0 seconds") + } + + if config.TotalRetryDuration.Seconds() < 0 { + return errors.New("total retry duration cannot be less than 0 seconds") + } + + if config.FollowRedirects { + if DefaultMaxRedirects < 1 { + return errors.New("max redirects cannot be less than 1") + } + } + + return nil +} + +func SetDefaultValuesClientConfig(config *ClientConfig) { + + if !config.HideSensitiveData { + config.HideSensitiveData = DefaultHideSensitiveData + } + + if config.MaxRetryAttempts == 0 { + config.MaxRetryAttempts = DefaultMaxRetryAttempts + } + + if config.MaxConcurrentRequests == 0 { + config.MaxRetryAttempts = DefaultMaxConcurrentRequests + } + + if !config.EnableDynamicRateLimiting { + config.EnableDynamicRateLimiting = DefaultEnableDynamicRateLimiting + } + + if config.CustomTimeout == 0 { + config.CustomTimeout = DefaultCustomTimeout + } + + if config.TokenRefreshBufferPeriod == 0 { + config.TokenRefreshBufferPeriod = DefaultTokenRefreshBufferPeriod + } + + if config.TotalRetryDuration == 0 { + config.TotalRetryDuration = DefaultTotalRetryDuration + } + + if !config.FollowRedirects { + config.FollowRedirects = DefaultFollowRedirects + } + + if config.MaxRedirects == 0 { + config.MaxRedirects = DefaultMaxRedirects + } + +} diff --git a/httpclient/cookies.go b/httpclient/cookies.go new file mode 100644 index 0000000..ff9c5e2 --- /dev/null +++ b/httpclient/cookies.go @@ -0,0 +1,27 @@ +package httpclient + +import ( + "fmt" + "net/http" + "net/http/cookiejar" + "net/url" +) + +func (c *Client) loadCustomCookies(cookiesList []*http.Cookie) error { + cookieJar, err := cookiejar.New(nil) + if err != nil { + return err + } + + cookieUrl, err := url.Parse((*c.Integration).Domain()) + + if err != nil { + return err + } + + c.http.Jar = cookieJar + c.http.Jar.SetCookies(cookieUrl, cookiesList) + c.Logger.Debug(fmt.Sprintf("%+v", c.http.Jar)) + + return nil +} diff --git a/httpclient/downloadrequest.go b/httpclient/downloadrequest.go deleted file mode 100644 index 7ac8670..0000000 --- a/httpclient/downloadrequest.go +++ /dev/null @@ -1,89 +0,0 @@ -// httpclient/download.go -package httpclient - -import ( - "io" - "net/http" - - "github.com/deploymenttheory/go-api-http-client/authenticationhandler" - "github.com/deploymenttheory/go-api-http-client/headers" - "github.com/deploymenttheory/go-api-http-client/response" - "go.uber.org/zap" -) - -// DoDownloadRequest performs a download from a given URL. It follows the same authentication, -// header setting, and URL construction as the DoMultipartRequest function. The downloaded data -// is written to the provided writer. -// -// Parameters: -// - method: The HTTP method to use (e.g., GET). -// - endpoint: The API endpoint from which the file will be downloaded. -// - out: A writer where the downloaded data will be written. -// -// Returns: -// - A pointer to the http.Response received from the server. -// - An error if the request could not be sent or the response could not be processed. -// -// The function first validates the authentication token, constructs the full URL for -// the request, sets the required headers (including Authorization), and sends the request. -// -// If debug mode is enabled, the function logs all the request headers before sending the request. -// After the request is sent, the function checks the response status code. If the response is -// not within the success range (200-299), it logs an error and returns the response and an error. -// If the response is successful, the function writes the response body to the provided writer. -// -// Note: -// The caller should handle closing the response body when successful. -func (c *Client) DoDownloadRequest(method, endpoint string, out io.Writer) (*http.Response, error) { - log := c.Logger - - // Auth Token validation check - clientCredentials := authenticationhandler.ClientCredentials{ - Username: c.clientConfig.Auth.Username, - Password: c.clientConfig.Auth.Password, - ClientID: c.clientConfig.Auth.ClientID, - ClientSecret: c.clientConfig.Auth.ClientSecret, - } - - valid, err := c.AuthTokenHandler.CheckAndRefreshAuthToken(c.APIHandler, c.httpClient, clientCredentials, c.clientConfig.ClientOptions.Timeout.TokenRefreshBufferPeriod.Duration()) - if err != nil || !valid { - return nil, err - } - - // Construct URL using the ConstructAPIResourceEndpoint function - url := c.APIHandler.ConstructAPIResourceEndpoint(endpoint, log) - - // Create the request - req, err := http.NewRequest(method, url, nil) - if err != nil { - return nil, err - } - - // Initialize HeaderManager - headerHandler := headers.NewHeaderHandler(req, c.Logger, c.APIHandler, c.AuthTokenHandler) - - // Use HeaderManager to set headers - headerHandler.SetRequestHeaders(endpoint) - headerHandler.LogHeaders(c.clientConfig.ClientOptions.Logging.HideSensitiveData) - - // Execute the request - resp, err := c.httpClient.Do(req) - if err != nil { - log.Error("Failed to send download request", zap.String("method", method), zap.String("endpoint", endpoint), zap.Error(err)) - return nil, err - } - - // Check for successful status code - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - // Handle error responses - return nil, response.HandleAPIErrorResponse(resp, log) - } - - // Write the response body to the provided writer - defer resp.Body.Close() - if _, err := io.Copy(out, resp.Body); err != nil { - return nil, err - } - - return resp, nil -} diff --git a/httpclient/headers.go b/httpclient/headers.go new file mode 100644 index 0000000..442b777 --- /dev/null +++ b/httpclient/headers.go @@ -0,0 +1,88 @@ +// headers/headers.go +package httpclient + +import ( + "net/http" + + "github.com/deploymenttheory/go-api-http-client/logger" + "go.uber.org/zap" +) + +// CheckDeprecationHeader checks the response headers for the Deprecation header and logs a warning if present. +func CheckDeprecationHeader(resp *http.Response, log logger.Logger) { + deprecationHeader := resp.Header.Get("Deprecation") + if deprecationHeader != "" { + + log.Warn("API endpoint is deprecated", + zap.String("Date", deprecationHeader), + zap.String("Endpoint", resp.Request.URL.String()), + ) + } +} + +// TODO review the need for headers below. Do they need to be in the Integration? + +// SetCacheControlHeader sets the Cache-Control header for an HTTP request. +// This header specifies directives for caching mechanisms in requests and responses. +// func SetCacheControlHeader(req *http.Request, cacheControlValue string) { +// req.Header.Set("Cache-Control", cacheControlValue) +// } + +// SetConditionalHeaders sets the If-Modified-Since and If-None-Match headers for an HTTP request. +// These headers make a request conditional to ask the server to return content only if it has changed. +// func SetConditionalHeaders(req *http.Request, ifModifiedSince, ifNoneMatch string) { +// if ifModifiedSince != "" { +// req.Header.Set("If-Modified-Since", ifModifiedSince) +// } +// if ifNoneMatch != "" { +// req.Header.Set("If-None-Match", ifNoneMatch) +// } +// } + +// SetAcceptEncodingHeader sets the Accept-Encoding header for an HTTP request. +// This header indicates the type of encoding (e.g., gzip) the client can handle. +// func SetAcceptEncodingHeader(req *http.Request, acceptEncodingValue string) { +// req.Header.Set("Accept-Encoding", acceptEncodingValue) +// } + +// SetRefererHeader sets the Referer header for an HTTP request. +// This header indicates the address of the previous web page from which a link was followed. +// func SetRefererHeader(req *http.Request, refererValue string) { +// req.Header.Set("Referer", refererValue) +// } + +// SetXForwardedForHeader sets the X-Forwarded-For header for an HTTP request. +// This header is used to identify the originating IP address of a client connecting through a proxy. +// func SetXForwardedForHeader(req *http.Request, xForwardedForValue string) { +// req.Header.Set("X-Forwarded-For", xForwardedForValue) +// } + +// LogHeaders prints all the current headers in the http.Request using the zap logger. +// It uses the RedactSensitiveHeaderData function to redact sensitive data based on the hideSensitiveData flag. +// func (c *Client) LogHeaders(req *http.Request, hideSensitiveData bool) { +// if c.Logger.GetLogLevel() <= logger.LogLevelDebug { +// redactedHeaders := http.Header{} + +// for name, values := range req.Header { +// if len(values) > 0 { +// redactedValue := redact.RedactSensitiveHeaderData(hideSensitiveData, name, values[0]) +// redactedHeaders.Set(name, redactedValue) +// } +// } + +// headersStr := HeadersToString(redactedHeaders) + +// c.Logger.Debug("HTTP Request Headers", zap.String("Headers", headersStr)) +// } +// } + +// HeadersToString converts a http.Header to a string for logging, +// with each header on a new line for readability. +// func HeadersToString(headers http.Header) string { +// var headerStrings []string +// for name, values := range headers { +// valueStr := strings.Join(values, ", ") +// headerStrings = append(headerStrings, fmt.Sprintf("%s: %s", name, valueStr)) +// } +// return strings.Join(headerStrings, "\n") // "\n" as seperator. +// } diff --git a/httpmethod/httpmethod.go b/httpclient/httpmethod.go similarity index 62% rename from httpmethod/httpmethod.go rename to httpclient/httpmethod.go index 84d520e..75034bb 100644 --- a/httpmethod/httpmethod.go +++ b/httpclient/httpmethod.go @@ -1,5 +1,5 @@ // httpmethod/httpmethod.go -package httpmethod +package httpclient /* Ref: https://www.rfc-editor.org/rfc/rfc7231#section-8.1.3 @@ -21,26 +21,17 @@ import "net/http" // IsIdempotentHTTPMethod checks if the given HTTP method is idempotent. func IsIdempotentHTTPMethod(method string) bool { - idempotentHTTPMethods := map[string]bool{ + methods := map[string]bool{ http.MethodGet: true, http.MethodPut: true, http.MethodDelete: true, http.MethodHead: true, http.MethodOptions: true, http.MethodTrace: true, + http.MethodPost: false, + http.MethodPatch: false, + http.MethodConnect: false, } - return idempotentHTTPMethods[method] -} - -// IsNonIdempotentHTTPMethod checks if the given HTTP method is non-idempotent. -// PATCH can be idempotent but often isn't used as such. -func IsNonIdempotentHTTPMethod(method string) bool { - nonIdempotentHTTPMethods := map[string]bool{ - http.MethodPost: true, - http.MethodPatch: true, - http.MethodConnect: true, - } - - return nonIdempotentHTTPMethods[method] + return methods[method] } diff --git a/httpclient/integration.go b/httpclient/integration.go new file mode 100644 index 0000000..cb5c248 --- /dev/null +++ b/httpclient/integration.go @@ -0,0 +1,17 @@ +// apiintegrations/apihandler/apihandler.go +package httpclient + +import ( + "net/http" +) + +// TODO comment +type APIIntegration interface { + Domain() string + GetAuthMethodDescriptor() string + CheckRefreshToken() error + PrepRequestParamsAndAuth(req *http.Request) error + PrepRequestBody(body interface{}, method string, endpoint string) ([]byte, error) + MarshalMultipartRequest(fields map[string]string, files map[string]string) ([]byte, string, error) + GetSessionCookies() ([]*http.Cookie, error) +} diff --git a/httpclient/multipartrequest.go b/httpclient/multipartrequest.go index 4c736b1..9a949fd 100644 --- a/httpclient/multipartrequest.go +++ b/httpclient/multipartrequest.go @@ -13,9 +13,6 @@ import ( "sync" "time" - "github.com/deploymenttheory/go-api-http-client/authenticationhandler" - "github.com/deploymenttheory/go-api-http-client/cookiejar" - "github.com/deploymenttheory/go-api-http-client/headers" "github.com/deploymenttheory/go-api-http-client/logger" "github.com/deploymenttheory/go-api-http-client/response" "go.uber.org/zap" @@ -77,24 +74,12 @@ func (c *Client) DoMultiPartRequest(method, endpoint string, files map[string][] return nil, fmt.Errorf("unsupported HTTP method: %s", method) } - clientCredentials := authenticationhandler.ClientCredentials{ - Username: c.clientConfig.Auth.Username, - Password: c.clientConfig.Auth.Password, - ClientID: c.clientConfig.Auth.ClientID, - ClientSecret: c.clientConfig.Auth.ClientSecret, - } - - valid, err := c.AuthTokenHandler.CheckAndRefreshAuthToken(c.APIHandler, c.httpClient, clientCredentials, c.clientConfig.ClientOptions.Timeout.TokenRefreshBufferPeriod.Duration()) - if err != nil || !valid { - return nil, err - } - log.Info("Executing multipart file upload request", zap.String("method", method), zap.String("endpoint", endpoint)) - url := c.APIHandler.ConstructAPIResourceEndpoint(endpoint, log) + url := (*c.Integration).Domain() + endpoint // Create a context with timeout based on the custom timeout duration - ctx, cancel := context.WithTimeout(context.Background(), c.clientConfig.ClientOptions.Timeout.CustomTimeout.Duration()) + ctx, cancel := context.WithTimeout(context.Background(), c.config.CustomTimeout) defer cancel() var body io.Reader @@ -118,13 +103,9 @@ func (c *Client) DoMultiPartRequest(method, endpoint string, files map[string][] return nil, err } - cookiejar.ApplyCustomCookies(req, c.clientConfig.ClientOptions.Cookies.CustomCookies, c.Logger) - - req.Header.Set("Content-Type", contentType) + // TODO cookies - headerHandler := headers.NewHeaderHandler(req, c.Logger, c.APIHandler, c.AuthTokenHandler) - headerHandler.SetRequestHeaders(endpoint) - headerHandler.LogHeaders(c.clientConfig.ClientOptions.Logging.HideSensitiveData) + (*c.Integration).PrepRequestParamsAndAuth(req) var resp *http.Response var requestErr error @@ -148,7 +129,7 @@ func (c *Client) DoMultiPartRequest(method, endpoint string, files map[string][] req.Header.Set("Content-Type", contentType) } - resp, requestErr = c.httpClient.Do(req) + resp, requestErr = c.http.Do(req) duration := time.Since(startTime) if requestErr != nil { diff --git a/httpclient/multipartrequest.go.back b/httpclient/multipartrequest.go.back deleted file mode 100644 index 1ba30e8..0000000 --- a/httpclient/multipartrequest.go.back +++ /dev/null @@ -1,452 +0,0 @@ -package httpclient - -import ( - "context" - "encoding/base64" - "fmt" - "io" - "mime/multipart" - "net/http" - "net/textproto" - "os" - "path/filepath" - "sync" - "time" - - "github.com/deploymenttheory/go-api-http-client/authenticationhandler" - "github.com/deploymenttheory/go-api-http-client/cookiejar" - "github.com/deploymenttheory/go-api-http-client/headers" - "github.com/deploymenttheory/go-api-http-client/logger" - "github.com/deploymenttheory/go-api-http-client/response" - "go.uber.org/zap" -) - -// UploadState represents the state of an upload operation, including the last uploaded byte. -// This struct is used to track the progress of file uploads for resumable uploads and to resume uploads from the last uploaded byte. -type UploadState struct { - LastUploadedByte int64 - sync.Mutex -} - -// DoMultiPartRequest creates and executes a multipart/form-data HTTP request for file uploads and form fields. -// This function handles constructing the multipart request body, setting the necessary headers, and executing the request. -// It supports custom content types and headers for each part of the multipart request, and handles authentication and -// logging throughout the process. - -// Parameters: -// - method: A string representing the HTTP method to be used for the request. This method should be either POST or PUT -// as these are the only methods that support multipart/form-data requests. -// - endpoint: The target API endpoint for the request. This should be a relative path that will be appended to the base URL -// configured for the HTTP client. -// - files: A map where the key is the field name and the value is a slice of file paths to be included in the request. -// - formDataFields: A map of additional form fields to be included in the multipart request, where the key is the field name -// and the value is the field value. -// - fileContentTypes: A map specifying the content type for each file part. The key is the field name and the value is the -// content type (e.g., "image/jpeg"). -// - formDataPartHeaders: A map specifying custom headers for each part of the multipart form data. The key is the field name -// and the value is an http.Header containing the headers for that part. -// - out: A pointer to an output variable where the response will be deserialized. This should be a pointer to a struct that -// matches the expected response schema. - -// Returns: -// - *http.Response: The HTTP response received from the server. In case of successful execution, this response contains -// the status code, headers, and body of the response. In case of errors, this response may contain the last received -// HTTP response that led to the failure. -// - error: An error object indicating failure during request execution. This could be due to network issues, server errors, -// or a failure in request serialization/deserialization. - -// Usage: -// This function is suitable for executing multipart/form-data HTTP requests, particularly for file uploads along with -// additional form fields. It ensures proper authentication, sets necessary headers, and logs the process for debugging -// and monitoring purposes. - -// Example: -// var result MyResponseType -// resp, err := client.DoMultiPartRequest("POST", "/api/upload", files, formDataFields, fileContentTypes, formDataPartHeaders, &result) -// -// if err != nil { -// // Handle error -// } -// -// // Use `result` or `resp` as needed -func (c *Client) DoMultiPartRequest(method, endpoint string, files map[string][]string, formDataFields map[string]string, fileContentTypes map[string]string, formDataPartHeaders map[string]http.Header, out interface{}) (*http.Response, error) { - log := c.Logger - - if method != http.MethodPost && method != http.MethodPut { - log.Error("HTTP method not supported for multipart request", zap.String("method", method)) - return nil, fmt.Errorf("unsupported HTTP method: %s", method) - } - - clientCredentials := authenticationhandler.ClientCredentials{ - Username: c.clientConfig.Auth.Username, - Password: c.clientConfig.Auth.Password, - ClientID: c.clientConfig.Auth.ClientID, - ClientSecret: c.clientConfig.Auth.ClientSecret, - } - - valid, err := c.AuthTokenHandler.CheckAndRefreshAuthToken(c.APIHandler, c.httpClient, clientCredentials, c.clientConfig.ClientOptions.Timeout.TokenRefreshBufferPeriod.Duration()) - if err != nil || !valid { - return nil, err - } - - log.Info("Executing multipart file upload request", zap.String("method", method), zap.String("endpoint", endpoint)) - - url := c.APIHandler.ConstructAPIResourceEndpoint(endpoint, log) - - // Create a context with timeout based on the custom timeout duration - ctx, cancel := context.WithTimeout(context.Background(), c.clientConfig.ClientOptions.Timeout.CustomTimeout.Duration()) - defer cancel() - - body, contentType, err := createStreamingMultipartRequestBody(files, formDataFields, fileContentTypes, formDataPartHeaders, log) - if err != nil { - log.Error("Failed to create streaming multipart request body", zap.Error(err)) - return nil, err - } - - req, err := http.NewRequestWithContext(ctx, method, url, body) - if err != nil { - log.Error("Failed to create HTTP request", zap.Error(err)) - return nil, err - } - - cookiejar.ApplyCustomCookies(req, c.clientConfig.ClientOptions.Cookies.CustomCookies, c.Logger) - - req.Header.Set("Content-Type", contentType) - - headerHandler := headers.NewHeaderHandler(req, c.Logger, c.APIHandler, c.AuthTokenHandler) - headerHandler.SetRequestHeaders(endpoint) - headerHandler.LogHeaders(c.clientConfig.ClientOptions.Logging.HideSensitiveData) - - var resp *http.Response - var requestErr error - - // Retry logic - maxRetries := 3 - for attempt := 1; attempt <= maxRetries; attempt++ { - startTime := time.Now() - - resp, requestErr = c.httpClient.Do(req) - duration := time.Since(startTime) - - if requestErr != nil { - log.Error("Failed to send request", zap.String("method", method), zap.String("endpoint", endpoint), zap.Error(requestErr)) - if attempt < maxRetries { - log.Info("Retrying request", zap.Int("attempt", attempt)) - time.Sleep(2 * time.Second) - continue - } - return nil, requestErr - } - - log.Debug("Request sent successfully", zap.String("method", method), zap.String("endpoint", endpoint), zap.Int("status_code", resp.StatusCode), zap.Duration("duration", duration)) - - if resp.StatusCode >= 200 && resp.StatusCode < 300 { - return resp, response.HandleAPISuccessResponse(resp, out, log) - } - - // If status code indicates a server error, retry - if resp.StatusCode >= 500 && attempt < maxRetries { - log.Info("Retrying request due to server error", zap.Int("status_code", resp.StatusCode), zap.Int("attempt", attempt)) - time.Sleep(2 * time.Second) - continue - } - - return resp, response.HandleAPIErrorResponse(resp, log) - } - - return resp, requestErr -} - -// createStreamingMultipartRequestBody creates a streaming multipart request body with the provided files and form fields. -// This function constructs the body of a multipart/form-data request using an io.Pipe, allowing the request to be sent in chunks. -// It supports custom content types and headers for each part of the multipart request, and logs the process for debugging -// and monitoring purposes. - -// Parameters: -// - files: A map where the key is the field name and the value is a slice of file paths to be included in the request. -// Each file path corresponds to a file that will be included in the multipart request. -// - formDataFields: A map of additional form fields to be included in the multipart request, where the key is the field name -// and the value is the field value. These are regular form fields that accompany the file uploads. -// - fileContentTypes: A map specifying the content type for each file part. The key is the field name and the value is the -// content type (e.g., "image/jpeg"). -// - formDataPartHeaders: A map specifying custom headers for each part of the multipart form data. The key is the field name -// and the value is an http.Header containing the headers for that part. -// - log: An instance of a logger implementing the logger.Logger interface, used to log informational messages, warnings, -// and errors encountered during the construction of the multipart request body. - -// Returns: -// - io.Reader: The constructed multipart request body reader. This reader streams the multipart form data payload ready to be sent. -// - string: The content type of the multipart request body. This includes the boundary string used by the multipart writer. -// - error: An error object indicating failure during the construction of the multipart request body. This could be due to issues -// such as file reading errors or multipart writer errors. -func createStreamingMultipartRequestBody(files map[string][]string, formDataFields map[string]string, fileContentTypes map[string]string, formDataPartHeaders map[string]http.Header, log logger.Logger) (io.Reader, string, error) { - pr, pw := io.Pipe() - writer := multipart.NewWriter(pw) - - go func() { - defer func() { - if err := writer.Close(); err != nil { - log.Error("Failed to close multipart writer", zap.Error(err)) - } - if err := pw.Close(); err != nil { - log.Error("Failed to close pipe writer", zap.Error(err)) - } - }() - - for fieldName, filePaths := range files { - for _, filePath := range filePaths { - log.Debug("Adding file part", zap.String("field_name", fieldName), zap.String("file_path", filePath)) - if err := addFilePart(writer, fieldName, filePath, fileContentTypes, formDataPartHeaders, log); err != nil { - log.Error("Failed to add file part", zap.Error(err)) - pw.CloseWithError(err) - return - } - } - } - - for key, val := range formDataFields { - log.Debug("Adding form field", zap.String("field_name", key), zap.String("field_value", val)) - if err := addFormField(writer, key, val, log); err != nil { - log.Error("Failed to add form field", zap.Error(err)) - pw.CloseWithError(err) - return - } - } - }() - - return pr, writer.FormDataContentType(), nil -} - -// addFilePart adds a base64 encoded file part to the multipart writer with the provided field name and file path. -// This function opens the specified file, sets the appropriate content type and headers, and adds it to the multipart writer. - -// Parameters: -// - writer: The multipart writer used to construct the multipart request body. -// - fieldName: The field name for the file part. -// - filePath: The path to the file to be included in the request. -// - fileContentTypes: A map specifying the content type for each file part. The key is the field name and the value is the -// content type (e.g., "image/jpeg"). -// - formDataPartHeaders: A map specifying custom headers for each part of the multipart form data. The key is the field name -// and the value is an http.Header containing the headers for that part. -// - log: An instance of a logger implementing the logger.Logger interface, used to log informational messages, warnings, -// and errors encountered during the addition of the file part. - -// Returns: -// - error: An error object indicating failure during the addition of the file part. This could be due to issues such as -// file reading errors or multipart writer errors. -func addFilePart(writer *multipart.Writer, fieldName, filePath string, fileContentTypes map[string]string, formDataPartHeaders map[string]http.Header, log logger.Logger) error { - file, err := os.Open(filePath) - if err != nil { - log.Error("Failed to open file", zap.String("filePath", filePath), zap.Error(err)) - return err - } - defer file.Close() - - // Default fileContentType - contentType := "application/octet-stream" - if ct, ok := fileContentTypes[fieldName]; ok { - contentType = ct - } - - header := setFormDataPartHeader(fieldName, filepath.Base(filePath), contentType, formDataPartHeaders[fieldName]) - - part, err := writer.CreatePart(header) - if err != nil { - log.Error("Failed to create form file part", zap.String("fieldName", fieldName), zap.Error(err)) - return err - } - - encoder := base64.NewEncoder(base64.StdEncoding, part) - defer encoder.Close() - - fileSize, err := file.Stat() - if err != nil { - log.Error("Failed to get file info", zap.String("filePath", filePath), zap.Error(err)) - return err - } - - progressLogger := logUploadProgress(file, fileSize.Size(), log) - uploadState := &UploadState{} - if err := chunkFileUpload(file, encoder, log, progressLogger, uploadState); err != nil { - log.Error("Failed to copy file content", zap.String("filePath", filePath), zap.Error(err)) - return err - } - - return nil -} - -// addFormField adds a form field to the multipart writer with the provided key and value. -// This function adds a regular form field (non-file) to the multipart request body. - -// Parameters: -// - writer: The multipart writer used to construct the multipart request body. -// - key: The field name for the form field. -// - val: The value of the form field. -// - log: An instance of a logger implementing the logger.Logger interface, used to log informational messages, warnings, -// and errors encountered during the addition of the form field. - -// Returns: -// - error: An error object indicating failure during the addition of the form field. This could be due to issues such as -// multipart writer errors. -func addFormField(writer *multipart.Writer, key, val string, log logger.Logger) error { - fieldWriter, err := writer.CreateFormField(key) - if err != nil { - log.Error("Failed to create form field", zap.String("key", key), zap.Error(err)) - return err - } - if _, err := fieldWriter.Write([]byte(val)); err != nil { - log.Error("Failed to write form field", zap.String("key", key), zap.Error(err)) - return err - } - return nil -} - -// setFormDataPartHeader creates a textproto.MIMEHeader for a form data field with the provided field name, file name, content type, and custom headers. -// This function constructs the MIME headers for a multipart form data part, including the content disposition, content type, -// and any custom headers specified. - -// Parameters: -// - fieldname: The name of the form field. -// - filename: The name of the file being uploaded (if applicable). -// - contentType: The content type of the form data part (e.g., "image/jpeg"). -// - customHeaders: A map of custom headers to be added to the form data part. The key is the header name and the value is the -// header value. - -// Returns: -// - textproto.MIMEHeader: The constructed MIME header for the form data part. -func setFormDataPartHeader(fieldname, filename, contentType string, customHeaders http.Header) textproto.MIMEHeader { - header := textproto.MIMEHeader{} - header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, fieldname, filename)) - header.Set("Content-Type", contentType) - header.Set("Content-Transfer-Encoding", "base64") - for key, values := range customHeaders { - for _, value := range values { - header.Add(key, value) - } - } - return header -} - -// chunkFileUpload reads the file upload into chunks and writes it to the writer. -// This function reads the file in chunks and writes it to the provided writer, allowing for progress logging during the upload. -// The chunk size is set to 8192 KB (8 MB) by default. This is a common chunk size used for file uploads to cloud storage services. - -// Azure Blob Storage has a minimum chunk size of 4 MB and a maximum of 100 MB for block blobs. -// GCP Cloud Storage has a minimum chunk size of 256 KB and a maximum of 5 GB. -// AWS S3 has a minimum chunk size of 5 MB and a maximum of 5 GB. - -// The function also calculates the total number of chunks and logs the chunk number during the upload process. - -// Parameters: -// - file: The file to be uploaded. -// - writer: The writer to which the file content will be written. -// - log: An instance of a logger implementing the logger.Logger interface, used to log informational messages, warnings, -// and errors encountered during the file upload. -// - updateProgress: A function to update the upload progress, typically used for logging purposes. -// - uploadState: A pointer to an UploadState struct used to track the progress of the file upload for resumable uploads. - -// Returns: -// - error: An error object indicating failure during the file upload. This could be due to issues such as file reading errors -// or writer errors. -func chunkFileUpload(file *os.File, writer io.Writer, log logger.Logger, updateProgress func(int64), uploadState *UploadState) error { - const chunkSize = 8 * 1024 * 1024 // 8 MB - buffer := make([]byte, chunkSize) - totalWritten := int64(0) - chunkWritten := int64(0) - fileName := filepath.Base(file.Name()) - - // Seek to the last uploaded byte - file.Seek(uploadState.LastUploadedByte, io.SeekStart) - - // Calculate the total number of chunks - fileInfo, err := file.Stat() - if err != nil { - return fmt.Errorf("failed to get file info: %v", err) - } - totalChunks := (fileInfo.Size() + chunkSize - 1) / chunkSize - currentChunk := uploadState.LastUploadedByte / chunkSize - - for { - n, err := file.Read(buffer) - if err != nil && err != io.EOF { - return err - } - if n == 0 { - break - } - - written, err := writer.Write(buffer[:n]) - if err != nil { - // Save the state before returning the error - uploadState.Lock() - uploadState.LastUploadedByte += totalWritten - uploadState.Unlock() - return err - } - - totalWritten += int64(written) - chunkWritten += int64(written) - updateProgress(int64(written)) - - if chunkWritten >= chunkSize { - currentChunk++ - log.Debug("File Upload Chunk Sent", - zap.String("file_name", fileName), - zap.Int64("chunk_number", currentChunk), - zap.Int64("total_chunks", totalChunks), - zap.Int64("kb_sent", chunkWritten/1024), - zap.Int64("total_kb_sent", totalWritten/1024)) - chunkWritten = 0 - } - } - - // Log any remaining bytes that were written but didn't reach the log threshold - if chunkWritten > 0 { - currentChunk++ - log.Debug("Final Upload Chunk Sent", - zap.String("file_name", fileName), - zap.Int64("chunk_number", currentChunk), - zap.Int64("total_chunks", totalChunks), - zap.Int64("kb_sent", chunkWritten/1024), - zap.Int64("total_kb_sent", totalWritten/1024)) - } - - return nil -} - -// logUploadProgress logs the upload progress based on the percentage of the total file size. -// This function returns a closure that logs the upload progress each time it is called, updating the percentage completed. - -// Parameters: -// - file: The file being uploaded. used for logging the file name. -// - fileSize: The total size of the file being uploaded. -// - log: An instance of a logger implementing the logger.Logger interface, used to log informational messages, warnings, -// and errors encountered during the upload. - -// Returns: -// - func(int64): A function that takes the number of bytes written as an argument and logs the upload progress. -// logUploadProgress logs the upload progress based on the percentage of the total file size. -func logUploadProgress(file *os.File, fileSize int64, log logger.Logger) func(int64) { - var uploaded int64 = 0 - const logInterval = 5 // Log every 5% increment - lastLoggedPercentage := int64(0) - startTime := time.Now() - fileName := filepath.Base(file.Name()) - - return func(bytesWritten int64) { - uploaded += bytesWritten - percentage := (uploaded * 100) / fileSize - - if percentage >= lastLoggedPercentage+logInterval { - elapsedTime := time.Since(startTime) - - log.Info("Upload progress", - zap.String("file_name", fileName), - zap.Float64("uploaded_MB's", float64(uploaded)/1048576), // Log in MB (1024 * 1024) - zap.Float64("total_filesize_in_MB", float64(fileSize)/1048576), - zap.String("total_uploaded_percentage", fmt.Sprintf("%d%%", percentage)), - zap.Duration("elapsed_time", elapsedTime)) - lastLoggedPercentage = percentage - } - } -} diff --git a/httpclient/ping.go b/httpclient/ping.go deleted file mode 100644 index 66612a1..0000000 --- a/httpclient/ping.go +++ /dev/null @@ -1,184 +0,0 @@ -// httpclient/ping.go -package httpclient - -import ( - "fmt" - "net" - "net/http" - "os" - "time" - - "github.com/deploymenttheory/go-api-http-client/ratehandler" - - "go.uber.org/zap" - "golang.org/x/net/icmp" - "golang.org/x/net/ipv4" -) - -// DoPole performs an HTTP "ping" to the specified endpoint using the given HTTP method, body, -// and output variable. It attempts the request until a 200 OK response is received or the -// maximum number of retry attempts is reached. The function uses a backoff strategy for retries -// to manage load on the server and network. This function is useful for checking the availability -// or health of an endpoint, particularly in environments where network reliability might be an issue. - -// Parameters: -// - method: The HTTP method to be used for the request. This should typically be "GET" for a ping operation, but the function is designed to accommodate any HTTP method. -// - endpoint: The target API endpoint for the ping request. This should be a relative path that will be appended to the base URL configured for the HTTP client. -// - body: The payload for the request, if any. For a typical ping operation, this would be nil, but the function is designed to accommodate requests that might require a body. -// - out: A pointer to an output variable where the response will be deserialized. This is included to maintain consistency with the signature of other request functions, but for a ping operation, it might not be used. - -// Returns: -// - *http.Response: The HTTP response from the server. In case of a successful ping (200 OK), -// this response contains the status code, headers, and body of the response. In case of errors -// or if the maximum number of retries is reached without a successful response, this will be the -// last received HTTP response. -// -// - error: An error object indicating failure during the execution of the ping operation. This -// could be due to network issues, server errors, or reaching the maximum number of retry attempts -// without receiving a 200 OK response. - -// Usage: -// This function is intended for use in scenarios where it's necessary to confirm the availability -// or health of an endpoint, with built-in retry logic to handle transient network or server issues. -// The caller is responsible for handling the response and error according to their needs, including -// closing the response body when applicable to avoid resource leaks. - -// Example: -// var result MyResponseType -// resp, err := client.DoPing("GET", "/api/health", nil, &result) -// -// if err != nil { -// // Handle error -// } -// -// // Process response -func (c *Client) DoPole(method, endpoint string, body, out interface{}) (*http.Response, error) { - log := c.Logger - log.Debug("Starting HTTP Ping", zap.String("method", method), zap.String("endpoint", endpoint)) - - // Initialize retry count and define maximum retries - var retryCount int - maxRetries := c.clientConfig.ClientOptions.Retry.MaxRetryAttempts - - // Loop until a successful response is received or maximum retries are reached - for retryCount <= maxRetries { - // Use the existing 'do' function for sending the request - resp, err := c.executeRequestWithRetries(method, endpoint, body, out) - - // If request is successful and returns 200 status code, return the response - if err == nil && resp.StatusCode == http.StatusOK { - log.Debug("Ping successful", zap.String("method", method), zap.String("endpoint", endpoint)) - return resp, nil - } - - // Increment retry count and log the attempt - retryCount++ - log.Warn("Ping failed, retrying...", zap.String("method", method), zap.String("endpoint", endpoint), zap.Int("retryCount", retryCount)) - - // Calculate backoff duration and wait before retrying - backoffDuration := ratehandler.CalculateBackoff(retryCount) - time.Sleep(backoffDuration) - } - - // If maximum retries are reached without a successful response, return an error - log.Error("Ping failed after maximum retries", zap.String("method", method), zap.String("endpoint", endpoint)) - return nil, fmt.Errorf("ping failed after %d retries", maxRetries) -} - -// DoPing performs an ICMP "ping" to the specified host. It sends ICMP echo requests and waits for echo replies. -// This function is useful for checking the availability or health of a host, particularly in environments where -// network reliability might be an issue. - -// Parameters: -// - host: The target host for the ping request. -// - timeout: The timeout for waiting for a ping response. - -// Returns: -// - error: An error object indicating failure during the execution of the ping operation or nil if the ping was successful. - -// Usage: -// This function is intended for use in scenarios where it's necessary to confirm the availability or health of a host. -// The caller is responsible for handling the error according to their needs. - -// Example: -// err := client.DoPing("www.example.com", 3*time.Second) -// if err != nil { -// // Handle error -// } - -func (c *Client) DoPing(host string, timeout time.Duration) error { - log := c.Logger - - // Listen for ICMP replies - conn, err := icmp.ListenPacket("ip4:icmp", "0.0.0.0") - if err != nil { - log.Error("Failed to listen for ICMP packets", zap.Error(err)) - return fmt.Errorf("failed to listen for ICMP packets: %w", err) - } - defer conn.Close() - - // Resolve the IP address of the host - dst, err := net.ResolveIPAddr("ip4", host) - if err != nil { - log.Error("Failed to resolve IP address", zap.String("host", host), zap.Error(err)) - return fmt.Errorf("failed to resolve IP address for host %s: %w", host, err) - } - - // Create an ICMP Echo Request message - msg := icmp.Message{ - Type: ipv4.ICMPTypeEcho, Code: 0, - Body: &icmp.Echo{ - ID: os.Getpid() & 0xffff, Seq: 1, // Use PID as ICMP ID - Data: []byte("HELLO"), // Data payload - }, - } - - // Marshal the message into bytes - msgBytes, err := msg.Marshal(nil) - if err != nil { - log.Error("Failed to marshal ICMP message", zap.Error(err)) - return fmt.Errorf("failed to marshal ICMP message: %w", err) - } - - // Send the ICMP Echo Request message - if _, err := conn.WriteTo(msgBytes, dst); err != nil { - log.Error("Failed to send ICMP message", zap.Error(err)) - return fmt.Errorf("failed to send ICMP message: %w", err) - } - - // Set read timeout - if err := conn.SetReadDeadline(time.Now().Add(timeout)); err != nil { - log.Error("Failed to set read deadline", zap.Error(err)) - return fmt.Errorf("failed to set read deadline: %w", err) - } - - // Wait for an ICMP Echo Reply message - reply := make([]byte, 1500) - n, _, err := conn.ReadFrom(reply) - if err != nil { - log.Error("Failed to receive ICMP reply", zap.Error(err)) - return fmt.Errorf("failed to receive ICMP reply: %w", err) - } - - // Parse the ICMP message from the reply - parsedMsg, err := icmp.ParseMessage(1, reply[:n]) - if err != nil { - log.Error("Failed to parse ICMP message", zap.Error(err)) - return fmt.Errorf("failed to parse ICMP message: %w", err) - } - - // Check if the message is an ICMP Echo Reply - if echoReply, ok := parsedMsg.Type.(*ipv4.ICMPType); ok { - if *echoReply != ipv4.ICMPTypeEchoReply { - log.Error("Did not receive ICMP Echo Reply", zap.String("received_type", echoReply.String())) - return fmt.Errorf("did not receive ICMP Echo Reply, received type: %s", echoReply.String()) - } - } else { - // Handle the case where the type assertion fails - log.Error("Failed to assert ICMP message type") - return fmt.Errorf("failed to assert ICMP message type") - } - - log.Info("Ping successful", zap.String("host", host)) - return nil -} diff --git a/httpclient/request.go b/httpclient/request.go index ca82db0..29211cf 100644 --- a/httpclient/request.go +++ b/httpclient/request.go @@ -7,16 +7,15 @@ import ( "net/http" "time" - "github.com/deploymenttheory/go-api-http-client/authenticationhandler" - "github.com/deploymenttheory/go-api-http-client/cookiejar" - "github.com/deploymenttheory/go-api-http-client/headers" - "github.com/deploymenttheory/go-api-http-client/httpmethod" "github.com/deploymenttheory/go-api-http-client/ratehandler" "github.com/deploymenttheory/go-api-http-client/response" "github.com/deploymenttheory/go-api-http-client/status" "go.uber.org/zap" ) +// TODO remove collapsable comment + +// region comment // DoRequest constructs and executes an HTTP request based on the provided method, endpoint, request body, and output variable. // This function serves as a dispatcher, deciding whether to execute the request with or without retry logic based on the // idempotency of the HTTP method. Idempotent methods (GET, PUT, DELETE) are executed with retries to handle transient errors @@ -63,19 +62,21 @@ import ( // within the client's concurrency model. // - The decision to retry requests is based on the idempotency of the HTTP method and the client's retry configuration, // including maximum retry attempts and total retry duration. +// endregion func (c *Client) DoRequest(method, endpoint string, body, out interface{}) (*http.Response, error) { log := c.Logger - if httpmethod.IsIdempotentHTTPMethod(method) { + if IsIdempotentHTTPMethod(method) { return c.executeRequestWithRetries(method, endpoint, body, out) - } else if httpmethod.IsNonIdempotentHTTPMethod(method) { + } else if !IsIdempotentHTTPMethod(method) { return c.executeRequest(method, endpoint, body, out) } else { return nil, log.Error("HTTP method not supported", zap.String("method", method)) } } +// region comment // executeRequestWithRetries executes an HTTP request using the specified method, endpoint, request body, and output variable. // It is designed for idempotent HTTP methods (GET, PUT, DELETE), where the request can be safely retried in case of // transient errors or rate limiting. The function implements a retry mechanism that respects the client's configuration @@ -108,10 +109,12 @@ func (c *Client) DoRequest(method, endpoint string, body, out interface{}) (*htt // - The function respects the client's concurrency token, acquiring and releasing it as needed to ensure safe concurrent // operations. // - The retry mechanism employs exponential backoff with jitter to mitigate the impact of retries on the server. +// endregion func (c *Client) executeRequestWithRetries(method, endpoint string, body, out interface{}) (*http.Response, error) { + // TODO review refactor executeRequestWithRetries log := c.Logger ctx := context.Background() - totalRetryDeadline := time.Now().Add(c.clientConfig.ClientOptions.Timeout.TotalRetryDuration.Duration()) + totalRetryDeadline := time.Now().Add(c.config.TotalRetryDuration) var resp *http.Response var err error @@ -151,7 +154,7 @@ func (c *Client) executeRequestWithRetries(method, endpoint string, body, out in if status.IsTransientError(resp) { retryCount++ - if retryCount > c.clientConfig.ClientOptions.Retry.MaxRetryAttempts { + if retryCount > c.config.MaxRetryAttempts { log.Warn("Max retry attempts reached", zap.String("method", method), zap.String("endpoint", endpoint)) break } @@ -177,6 +180,7 @@ func (c *Client) executeRequestWithRetries(method, endpoint string, body, out in return resp, response.HandleAPIErrorResponse(resp, log) } +// region comment // executeRequest executes an HTTP request using the specified method, endpoint, and request body without implementing // retry logic. It is primarily designed for non-idempotent HTTP methods like POST and PATCH, where the request should // not be automatically retried within this function due to the potential side effects of re-submitting the same data. @@ -206,7 +210,10 @@ func (c *Client) executeRequestWithRetries(method, endpoint string, body, out in // execution. // - The function logs detailed information about the request execution, including the method, endpoint, status code, and // any errors encountered. +// +// endregion func (c *Client) executeRequest(method, endpoint string, body, out interface{}) (*http.Response, error) { + // TODO review refactor execute Request log := c.Logger ctx := context.Background() @@ -227,87 +234,66 @@ func (c *Client) executeRequest(method, endpoint string, body, out interface{}) return nil, response.HandleAPIErrorResponse(res, log) } -// doRequest contains the shared logic for making the HTTP request, including authentication, -// setting headers, managing concurrency, and logging. This function performs the following steps: -// 1. Authenticates the client using the provided credentials and refreshes the auth token if necessary. -// 2. Acquires a concurrency permit to control the number of concurrent requests. -// 3. Increments the total request counter within the ConcurrencyHandler's metrics. -// 4. Marshals the request data based on the provided body, method, and endpoint. -// 5. Constructs the full URL for the API endpoint. -// 6. Creates the HTTP request and applies custom cookies and headers. -// 7. Executes the HTTP request and logs relevant information. -// 8. Adjusts concurrency settings based on the response and logs the response details. func (c *Client) doRequest(ctx context.Context, method, endpoint string, body interface{}) (*http.Response, error) { - log := c.Logger - - // Authenticate the client using the provided credentials and refresh the auth token if necessary. - clientCredentials := authenticationhandler.ClientCredentials{ - Username: c.clientConfig.Auth.Username, - Password: c.clientConfig.Auth.Password, - ClientID: c.clientConfig.Auth.ClientID, - ClientSecret: c.clientConfig.Auth.ClientSecret, - } - valid, err := c.AuthTokenHandler.CheckAndRefreshAuthToken(c.APIHandler, c.httpClient, clientCredentials, c.clientConfig.ClientOptions.Timeout.TokenRefreshBufferPeriod.Duration()) - if err != nil || !valid { - return nil, err - } + // TODO Concurrency - Refactor or remove this + // if c.config.ConcurrencyManagementEnabled { + // _, requestID, err := c.Concurrency.AcquireConcurrencyPermit(ctx) + // if err != nil { + // return nil, c.Logger.Error("Failed to acquire concurrency permit", zap.Error(err)) - // Acquire a concurrency permit to control the number of concurrent requests. - ctx, requestID, err := c.ConcurrencyHandler.AcquireConcurrencyPermit(ctx) - if err != nil { - return nil, c.Logger.Error("Failed to acquire concurrency permit", zap.Error(err)) - } + // } - // Ensure the concurrency permit is released after the function exits. - defer func() { - c.ConcurrencyHandler.ReleaseConcurrencyPermit(requestID) - }() + // defer func() { + // c.Concurrency.ReleaseConcurrencyPermit(requestID) + // }() - // Increment the total request counter within the ConcurrencyHandler's metrics. - c.ConcurrencyHandler.Metrics.Lock.Lock() - c.ConcurrencyHandler.Metrics.TotalRequests++ - c.ConcurrencyHandler.Metrics.Lock.Unlock() + // c.Concurrency.Metrics.Lock.Lock() + // c.Concurrency.Metrics.TotalRequests++ + // c.Concurrency.Metrics.Lock.Unlock() + // } - // Marshal the request data based on the provided api handler - requestData, err := c.APIHandler.MarshalRequest(body, method, endpoint, log) + requestData, err := (*c.Integration).PrepRequestBody(body, method, endpoint) if err != nil { return nil, err } + requestDataBytes := bytes.NewBuffer(requestData) - // Construct the full URL for the API endpoint. - url := c.APIHandler.ConstructAPIResourceEndpoint(endpoint, log) + url := (*c.Integration).Domain() + endpoint - // Create the HTTP request and apply custom cookies and headers. - req, err := http.NewRequest(method, url, bytes.NewBuffer(requestData)) + req, err := http.NewRequest(method, url, requestDataBytes) if err != nil { return nil, err } - cookiejar.ApplyCustomCookies(req, c.clientConfig.ClientOptions.Cookies.CustomCookies, log) - - headerHandler := headers.NewHeaderHandler(req, c.Logger, c.APIHandler, c.AuthTokenHandler) - headerHandler.SetRequestHeaders(endpoint) - headerHandler.LogHeaders(c.clientConfig.ClientOptions.Logging.HideSensitiveData) + err = (*c.Integration).PrepRequestParamsAndAuth(req) + if err != nil { + return nil, err + } req = req.WithContext(ctx) - log.LogCookies("outgoing", req, method, endpoint) - // Execute the HTTP request and log relevant information. - startTime := time.Now() - resp, err := c.httpClient.Do(req) + // TODO Concurrency - Refactor or remove this + // startTime := time.Now() + + resp, err := c.http.Do(req) if err != nil { - log.Error("Failed to send request", zap.String("method", method), zap.String("endpoint", endpoint), zap.Error(err)) + c.Logger.Error("Failed to send request", zap.String("method", method), zap.String("endpoint", endpoint), zap.Error(err)) return nil, err } - // Adjust concurrency settings based on the response and log the response details. - duration := time.Since(startTime) - c.ConcurrencyHandler.EvaluateAndAdjustConcurrency(resp, duration) - log.LogCookies("incoming", req, method, endpoint) - headers.CheckDeprecationHeader(resp, log) + // TODO Concurrency - Refactor or remove this + // if c.config.ConcurrencyManagementEnabled { + // duration := time.Since(startTime) + // c.Concurrency.EvaluateAndAdjustConcurrency(resp, duration) + // } + + // TODO review LogCookies + c.Logger.LogCookies("incoming", req, method, endpoint) + + CheckDeprecationHeader(resp, c.Logger) - log.Debug("Request sent successfully", zap.String("method", method), zap.String("endpoint", endpoint), zap.Int("status_code", resp.StatusCode)) + c.Logger.Debug("Request sent successfully", zap.String("method", method), zap.String("endpoint", endpoint), zap.Int("status_code", resp.StatusCode)) return resp, nil } diff --git a/httpclient/request.go.back b/httpclient/request.go.back deleted file mode 100644 index 7ebb2fd..0000000 --- a/httpclient/request.go.back +++ /dev/null @@ -1,404 +0,0 @@ -// request.go -package request - -import ( - "bytes" - "context" - "net/http" - "time" - - "github.com/deploymenttheory/go-api-http-client/authenticationhandler" - "github.com/deploymenttheory/go-api-http-client/cookiejar" - "github.com/deploymenttheory/go-api-http-client/headers" - "github.com/deploymenttheory/go-api-http-client/httpmethod" - "github.com/deploymenttheory/go-api-http-client/logger" - - "github.com/deploymenttheory/go-api-http-client/ratehandler" - "github.com/deploymenttheory/go-api-http-client/response" - "github.com/deploymenttheory/go-api-http-client/status" - "go.uber.org/zap" -) - -// DoRequest constructs and executes an HTTP request based on the provided method, endpoint, request body, and output variable. -// This function serves as a dispatcher, deciding whether to execute the request with or without retry logic based on the -// idempotency of the HTTP method. Idempotent methods (GET, PUT, DELETE) are executed with retries to handle transient errors -// and rate limits, while non-idempotent methods (POST, PATCH) are executed without retries to avoid potential side effects -// of duplicating non-idempotent operations. The function uses an instance of a logger implementing the logger.Logger interface, -// used to log informational messages, warnings, and errors encountered during the execution of the request. -// It also applies redirect handling to the client if configured, allowing the client to follow redirects up to a maximum -// number of times. - -// Parameters: -// - method: A string representing the HTTP method to be used for the request. This method determines the execution path -// and whether the request will be retried in case of failures. -// - endpoint: The target API endpoint for the request. This should be a relative path that will be appended to the base URL -// configured for the HTTP client. -// - body: The payload for the request, which will be serialized into the request body. The serialization format (e.g., JSON, XML) -// is determined by the content-type header and the specific implementation of the API handler used by the client. -// - out: A pointer to an output variable where the response will be deserialized. The function expects this to be a pointer to -// a struct that matches the expected response schema. - -// Returns: -// - *http.Response: The HTTP response received from the server. In case of successful execution, this response contains -// the status code, headers, and body of the response. In case of errors, particularly after exhausting retries for -// idempotent methods, this response may contain the last received HTTP response that led to the failure. -// - error: An error object indicating failure during request execution. This could be due to network issues, server errors, -// or a failure in request serialization/deserialization. For idempotent methods, an error is returned if all retries are -// exhausted without success. - -// Usage: -// This function is the primary entry point for executing HTTP requests using the client. It abstracts away the details of -// request retries, serialization, and response handling, providing a simplified interface for making HTTP requests. It is -// suitable for a wide range of HTTP operations, from fetching data with GET requests to submitting data with POST requests. - -// Example: -// var result MyResponseType -// resp, err := client.DoRequest("GET", "/api/resource", nil, &result, logger) -// if err != nil { -// // Handle error -// } -// // Use `result` or `resp` as needed - -// Note: -// - The caller is responsible for closing the response body when not nil to avoid resource leaks. -// - The function ensures concurrency control by managing concurrency tokens internally, providing safe concurrent operations -// within the client's concurrency model. -// - The decision to retry requests is based on the idempotency of the HTTP method and the client's retry configuration, -// including maximum retry attempts and total retry duration. - -func (c *Client) DoRequest(method, endpoint string, body, out interface{}) (*http.Response, error) { - log := c.Logger - - if httpmethod.IsIdempotentHTTPMethod(method) { - return c.executeRequestWithRetries(method, endpoint, body, out) - } else if httpmethod.IsNonIdempotentHTTPMethod(method) { - return c.executeRequest(method, endpoint, body, out) - } else { - return nil, log.Error("HTTP method not supported", zap.String("method", method)) - } -} - -// executeRequestWithRetries executes an HTTP request using the specified method, endpoint, request body, and output variable. -// It is designed for idempotent HTTP methods (GET, PUT, DELETE), where the request can be safely retried in case of -// transient errors or rate limiting. The function implements a retry mechanism that respects the client's configuration -// for maximum retry attempts and total retry duration. Each retry attempt uses exponential backoff with jitter to avoid -// thundering herd problems. An instance of a logger (conforming to the logger.Logger interface) is used for logging the -// request, retry attempts, and any errors encountered. -// -// Parameters: -// - method: The HTTP method to be used for the request (e.g., "GET", "PUT", "DELETE"). -// - endpoint: The API endpoint to which the request will be sent. This should be a relative path that will be appended -// to the base URL of the HTTP client. -// - body: The request payload, which will be marshaled into the request body based on the content type. Can be nil for -// methods that do not send a payload. -// - out: A pointer to the variable where the unmarshaled response will be stored. The function expects this to be a -// pointer to a struct that matches the expected response schema. -// - log: -// -// Returns: -// - *http.Response: The HTTP response from the server, which may be the response from a successful request or the last -// failed attempt if all retries are exhausted. -// - error: An error object if an error occurred during the request execution or if all retry attempts failed. The error -// may be a structured API error parsed from the response or a generic error indicating the failure reason. -// -// Usage: -// This function should be used for operations that are safe to retry and where the client can tolerate the additional -// latency introduced by the retry mechanism. It is particularly useful for handling transient errors and rate limiting -// responses from the server. -// -// Note: -// - The caller is responsible for closing the response body to prevent resource leaks. -// - The function respects the client's concurrency token, acquiring and releasing it as needed to ensure safe concurrent -// operations. -// - The retry mechanism employs exponential backoff with jitter to mitigate the impact of retries on the server. -func (c *Client) executeRequestWithRetries(method, endpoint string, body, out interface{}) (*http.Response, error) { - log := c.Logger - - // Include the core logic for handling non-idempotent requests with retries here. - log.Debug("Executing request with retries", zap.String("method", method), zap.String("endpoint", endpoint)) - - // Auth Token validation check - clientCredentials := authenticationhandler.ClientCredentials{ - Username: c.clientConfig.Auth.Username, - Password: c.clientConfig.Auth.Password, - ClientID: c.clientConfig.Auth.ClientID, - ClientSecret: c.clientConfig.Auth.ClientSecret, - } - - valid, err := c.AuthTokenHandler.CheckAndRefreshAuthToken(c.APIHandler, c.httpClient, clientCredentials, c.clientConfig.ClientOptions.Timeout.TokenRefreshBufferPeriod.Duration()) - if err != nil || !valid { - return nil, err - } - - // Acquire a concurrency permit along with a unique request ID - ctx, requestID, err := c.ConcurrencyHandler.AcquireConcurrencyPermit(context.Background()) - if err != nil { - return nil, c.Logger.Error("Failed to acquire concurrency permit", zap.Error(err)) - } - - // Ensure the permit is released after the function exits - defer func() { - c.ConcurrencyHandler.ReleaseConcurrencyPermit(requestID) - }() - - // Marshal Request with correct encoding defined in api handler - requestData, err := c.APIHandler.MarshalRequest(body, method, endpoint, log) - if err != nil { - return nil, err - } - - // Construct URL with correct structure defined in api handler - url := c.APIHandler.ConstructAPIResourceEndpoint(endpoint, log) - - // Increment total request counter within ConcurrencyHandler's metrics - c.ConcurrencyHandler.Metrics.Lock.Lock() - c.ConcurrencyHandler.Metrics.TotalRequests++ - c.ConcurrencyHandler.Metrics.Lock.Unlock() - - // Create a new HTTP request with the provided method, URL, and body - req, err := http.NewRequest(method, url, bytes.NewBuffer(requestData)) - if err != nil { - return nil, err - } - - // Apply custom cookies if configured - cookiejar.ApplyCustomCookies(req, c.clientConfig.ClientOptions.Cookies.CustomCookies, log) - - // Set request headers - headerHandler := headers.NewHeaderHandler(req, c.Logger, c.APIHandler, c.AuthTokenHandler) - headerHandler.SetRequestHeaders(endpoint) - headerHandler.LogHeaders(c.clientConfig.ClientOptions.Logging.HideSensitiveData) - - // Define a retry deadline based on the client's total retry duration configuration - totalRetryDeadline := time.Now().Add(c.clientConfig.ClientOptions.Timeout.TotalRetryDuration.Duration()) - - var resp *http.Response - var retryCount int - for time.Now().Before(totalRetryDeadline) { // Check if the current time is before the total retry deadline - req = req.WithContext(ctx) - - // Log outgoing cookies - log.LogCookies("outgoing", req, method, endpoint) - - // Execute the HTTP request - resp, err = c.do(req, log, method, endpoint) - - // Log outgoing cookies - log.LogCookies("incoming", req, method, endpoint) - - // Check for successful status code - if err == nil && resp.StatusCode >= 200 && resp.StatusCode < 400 { - if resp.StatusCode >= 300 { - log.Warn("Redirect response received", zap.Int("status_code", resp.StatusCode), zap.String("location", resp.Header.Get("Location"))) - } - // Handle the response as successful. - return resp, response.HandleAPISuccessResponse(resp, out, log) - } - - // Leverage TranslateStatusCode for more descriptive error logging - statusMessage := status.TranslateStatusCode(resp) - - // Check for non-retryable errors - if resp != nil && status.IsNonRetryableStatusCode(resp) { - log.Warn("Non-retryable error received", zap.Int("status_code", resp.StatusCode), zap.String("status_message", statusMessage)) - return resp, response.HandleAPIErrorResponse(resp, log) - } - - // Parsing rate limit headers if a rate-limit error is detected - if status.IsRateLimitError(resp) { - waitDuration := ratehandler.ParseRateLimitHeaders(resp, log) - if waitDuration > 0 { - log.Warn("Rate limit encountered, waiting before retrying", zap.Duration("waitDuration", waitDuration)) - time.Sleep(waitDuration) - continue // Continue to next iteration after waiting - } - } - - // Handling retryable errors with exponential backoff - if status.IsTransientError(resp) { - retryCount++ - if retryCount > c.clientConfig.ClientOptions.Retry.MaxRetryAttempts { - log.Warn("Max retry attempts reached", zap.String("method", method), zap.String("endpoint", endpoint)) - break // Stop retrying if max attempts are reached - } - waitDuration := ratehandler.CalculateBackoff(retryCount) - log.Warn("Retrying request due to transient error", zap.String("method", method), zap.String("endpoint", endpoint), zap.Int("retryCount", retryCount), zap.Duration("waitDuration", waitDuration), zap.Error(err)) - time.Sleep(waitDuration) // Wait before retrying - continue // Continue to next iteration after waiting - } - - // Handle error responses - if err != nil || !status.IsRetryableStatusCode(resp.StatusCode) { - if apiErr := response.HandleAPIErrorResponse(resp, log); apiErr != nil { - err = apiErr - } - log.LogError("request_error", method, endpoint, resp.StatusCode, resp.Status, err, status.TranslateStatusCode(resp)) - break - } - } - - // Handles final non-API error. - if err != nil { - return nil, err - } - - return resp, response.HandleAPIErrorResponse(resp, log) -} - -// executeRequest executes an HTTP request using the specified method, endpoint, and request body without implementing -// retry logic. It is primarily designed for non idempotent HTTP methods like POST and PATCH, where the request should -// not be automatically retried within this function due to the potential side effects of re-submitting the same data. -// -// Parameters: -// - method: The HTTP method to be used for the request, typically "POST" or "PATCH". -// - endpoint: The API endpoint to which the request will be sent. This should be a relative path that will be appended -// to the base URL of the HTTP client. -// - body: The request payload, which will be marshaled into the request body based on the content type. This can be any -// data structure that can be marshaled into the expected request format (e.g., JSON, XML). -// - out: A pointer to the variable where the unmarshaled response will be stored. This should be a pointer to a struct -// -// that matches the expected response schema. -// - log: An instance of a logger (conforming to the logger.Logger interface) used for logging the request and any errors -// encountered. -// -// Returns: -// - *http.Response: The HTTP response from the server. This includes the status code, headers, and body of the response. -// - error: An error object if an error occurred during the request execution. This could be due to network issues, -// server errors, or issues with marshaling/unmarshaling the request/response. -// -// Usage: -// This function is suitable for operations where the request should not be retried automatically, such as data submission -// operations where retrying could result in duplicate data processing. It ensures that the request is executed exactly -// once and provides detailed logging for debugging purposes. -// -// Note: -// - The caller is responsible for closing the response body to prevent resource leaks. -// - The function ensures concurrency control by acquiring and releasing a concurrency token before and after the request -// execution. -// - The function logs detailed information about the request execution, including the method, endpoint, status code, and -// any errors encountered. -func (c *Client) executeRequest(method, endpoint string, body, out interface{}) (*http.Response, error) { - log := c.Logger - - // Include the core logic for handling idempotent requests here. - log.Debug("Executing request without retries", zap.String("method", method), zap.String("endpoint", endpoint)) - - // Auth Token validation check - clientCredentials := authenticationhandler.ClientCredentials{ - Username: c.clientConfig.Auth.Username, - Password: c.clientConfig.Auth.Password, - ClientID: c.clientConfig.Auth.ClientID, - ClientSecret: c.clientConfig.Auth.ClientSecret, - } - - valid, err := c.AuthTokenHandler.CheckAndRefreshAuthToken(c.APIHandler, c.httpClient, clientCredentials, c.clientConfig.ClientOptions.Timeout.TokenRefreshBufferPeriod.Duration()) - if err != nil || !valid { - return nil, err - } - - // Acquire a concurrency permit along with a unique request ID - ctx, requestID, err := c.ConcurrencyHandler.AcquireConcurrencyPermit(context.Background()) - if err != nil { - return nil, c.Logger.Error("Failed to acquire concurrency permit", zap.Error(err)) - } - - // Ensure the permit is released after the function exits - defer func() { - c.ConcurrencyHandler.ReleaseConcurrencyPermit(requestID) - }() - - // Marshal Request with correct encoding defined in api handler - requestData, err := c.APIHandler.MarshalRequest(body, method, endpoint, log) - if err != nil { - return nil, err - } - - // Construct URL using the ConstructAPIResourceEndpoint function - url := c.APIHandler.ConstructAPIResourceEndpoint(endpoint, log) - - // Create a new HTTP request with the provided method, URL, and body - req, err := http.NewRequest(method, url, bytes.NewBuffer(requestData)) - if err != nil { - return nil, err - } - - // Apply custom cookies if configured - cookiejar.ApplyCustomCookies(req, c.clientConfig.ClientOptions.Cookies.CustomCookies, log) - - // Set request headers - headerHandler := headers.NewHeaderHandler(req, c.Logger, c.APIHandler, c.AuthTokenHandler) - headerHandler.SetRequestHeaders(endpoint) - headerHandler.LogHeaders(c.clientConfig.ClientOptions.Logging.HideSensitiveData) - - req = req.WithContext(ctx) - - // Log outgoing cookies - log.LogCookies("outgoing", req, method, endpoint) - - // Measure the time taken to execute the request and receive the response - startTime := time.Now() - - // Execute the HTTP request - resp, err := c.do(req, log, method, endpoint) - if err != nil { - return nil, err - } - - // Calculate the duration between sending the request and receiving the response - duration := time.Since(startTime) - - // Evaluate and adjust concurrency based on the request's feedback - c.ConcurrencyHandler.EvaluateAndAdjustConcurrency(resp, duration) - - // Log outgoing cookies - log.LogCookies("incoming", req, method, endpoint) - - // Checks for the presence of a deprecation header in the HTTP response and logs if found. - headers.CheckDeprecationHeader(resp, log) - - // Check for successful status code, including redirects - if resp.StatusCode >= 200 && resp.StatusCode < 400 { - // Warn on redirects but proceed as successful - if resp.StatusCode >= 300 { - log.Warn("Redirect response received", zap.Int("status_code", resp.StatusCode), zap.String("location", resp.Header.Get("Location"))) - } - return resp, response.HandleAPISuccessResponse(resp, out, log) - - } - - // Handle error responses for status codes outside the successful range - return nil, response.HandleAPIErrorResponse(resp, log) -} - -// do sends an HTTP request using the client's HTTP client. It logs the request and error details, if any, -// using structured logging with zap fields. -// -// Parameters: -// - req: The *http.Request object that contains all the details of the HTTP request to be sent. -// - log: An instance of a logger (conforming to the logger.Logger interface) used for logging the request details and any -// errors. -// - method: The HTTP method used for the request, used for logging. -// - endpoint: The API endpoint the request is being sent to, used for logging. -// -// Returns: -// - *http.Response: The HTTP response from the server. -// - error: An error object if an error occurred while sending the request or nil if no error occurred. -// -// Usage: -// This function should be used whenever the client needs to send an HTTP request. It abstracts away the common logic of -// request execution and error handling, providing detailed logs for debugging and monitoring. -func (c *Client) do(req *http.Request, log logger.Logger, method, endpoint string) (*http.Response, error) { - - resp, err := c.httpClient.Do(req) - - if err != nil { - // Log the error with structured logging, including method, endpoint, and the error itself - log.Error("Failed to send request", zap.String("method", method), zap.String("endpoint", endpoint), zap.Error(err)) - return nil, err - } - - // Log the response status code for successful requests - log.Debug("Request sent successfully", zap.String("method", method), zap.String("endpoint", endpoint), zap.Int("status_code", resp.StatusCode)) - - return resp, nil -} diff --git a/httpclient/utility.go b/httpclient/utility.go new file mode 100644 index 0000000..9abf70f --- /dev/null +++ b/httpclient/utility.go @@ -0,0 +1,76 @@ +package httpclient + +import ( + "errors" + "fmt" + "path/filepath" + "regexp" + "strings" +) + +// TODO all func comments in here + +const ConfigFileExtension = ".json" + +func validateFilePath(path string) (string, error) { + cleanPath := filepath.Clean(path) + + absPath, err := filepath.EvalSymlinks(cleanPath) + if err != nil { + return "", fmt.Errorf("unable to resolve the absolute path of the configuration file: %s, error: %w", path, err) + } + + if strings.Contains(absPath, "..") { + return "", fmt.Errorf("invalid path, path traversal patterns detected: %s", path) + } + + if filepath.Ext(absPath) != ConfigFileExtension { + return "", fmt.Errorf("invalid file extension for configuration file: %s, expected .json", path) + } + + return path, nil + +} + +func validateValidClientID(clientID string) error { + uuidRegex := `^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$` + if regexp.MustCompile(uuidRegex).MatchString(clientID) { + return nil + } + return errors.New("clientID failed regex check") +} + +func validateClientSecret(clientSecret string) error { + if len(clientSecret) < 16 { + return errors.New("client secret must be at least 16 characters long") + } + + if matched, _ := regexp.MatchString(`[a-z]`, clientSecret); !matched { + return errors.New("client secret must contain at least one lowercase letter") + } + + if matched, _ := regexp.MatchString(`[A-Z]`, clientSecret); !matched { + return errors.New("client secret must contain at least one uppercase letter") + } + + if matched, _ := regexp.MatchString(`\d`, clientSecret); !matched { + return errors.New("client secret must contain at least one digit") + } + + return nil +} + +func validateUsername(username string) error { + usernameRegex := `^[a-zA-Z0-9!@#$%^&*()_\-\+=\[\]{\}\\|;:'",<.>/?]+$` + if !regexp.MustCompile(usernameRegex).MatchString(username) { + return errors.New("username failed regex test") + } + return nil +} + +func validatePassword(password string) error { + if len(password) < 8 { + return errors.New("password not long enough") + } + return nil +} diff --git a/httpmethod/httpmethod_test.go b/httpmethod/httpmethod_test.go deleted file mode 100644 index 19e3361..0000000 --- a/httpmethod/httpmethod_test.go +++ /dev/null @@ -1,62 +0,0 @@ -// httpmethod/httpmethod_test.go -package httpmethod - -import ( - "net/http" - "testing" - - "github.com/stretchr/testify/assert" -) - -// TestIsIdempotentHTTPMethod tests the IsIdempotentHTTPMethod function with various HTTP methods -func TestIsIdempotentHTTPMethod(t *testing.T) { - tests := []struct { - method string - expected bool - }{ - {http.MethodGet, true}, - {http.MethodPut, true}, - {http.MethodDelete, true}, - {http.MethodHead, true}, - {http.MethodOptions, true}, - {http.MethodTrace, true}, - // Non-idempotent methods - {http.MethodPost, false}, - {http.MethodPatch, false}, - {http.MethodConnect, false}, - } - - for _, tt := range tests { - t.Run(tt.method, func(t *testing.T) { - result := IsIdempotentHTTPMethod(tt.method) - assert.Equal(t, tt.expected, result, "Idempotency status should match expected for method "+tt.method) - }) - } -} - -// TestIsNonIdempotentHTTPMethod tests the IsNonIdempotentHTTPMethod function with various HTTP methods -func TestIsNonIdempotentHTTPMethod(t *testing.T) { - tests := []struct { - method string - expected bool - }{ - // Non-idempotent methods - {http.MethodPost, true}, - {http.MethodPatch, true}, - {http.MethodConnect, true}, - // Idempotent methods - {http.MethodGet, false}, - {http.MethodPut, false}, - {http.MethodDelete, false}, - {http.MethodHead, false}, - {http.MethodOptions, false}, - {http.MethodTrace, false}, - } - - for _, tt := range tests { - t.Run(tt.method, func(t *testing.T) { - result := IsNonIdempotentHTTPMethod(tt.method) - assert.Equal(t, tt.expected, result, "Non-idempotency status should match expected for method "+tt.method) - }) - } -} diff --git a/logger/zaplogger_config.go b/logger/zaplogger_config.go index 50869a0..0e561a6 100644 --- a/logger/zaplogger_config.go +++ b/logger/zaplogger_config.go @@ -11,19 +11,19 @@ import ( // BuildLogger creates and returns a new zap logger instance. // It configures the logger with JSON formatting and a custom encoder to ensure the 'pid', 'application', and 'timestamp' fields // appear at the end of each log message. The function panics if the logger cannot be initialized. -func BuildLogger(logLevel LogLevel, encoding string, logConsoleSeparator string, logExportPath string) Logger { - // Set default encoding to console if not provided +func BuildLogger(logLevel LogLevel, encoding string, logConsoleSeparator string, logFilepath string, exportLogs bool) Logger { + if encoding == "" { encoding = "console" + } else if encoding == "pretty" { + encoding = "console" } - // Ensure the log path is correct and get the final log file path - logPath, err := EnsureLogFilePath(logExportPath) + logFilepath, err := GetLogFilepath(logFilepath) if err != nil { panic(err) } - // Set up custom encoder configuration encoderCfg := zap.NewProductionEncoderConfig() // Time settings @@ -74,8 +74,10 @@ func BuildLogger(logLevel LogLevel, encoding string, logConsoleSeparator string, } // Conditionally set the OutputPaths to include the log export path if provided - if logExportPath != "" { - config.OutputPaths = append(config.OutputPaths, logPath) + if exportLogs { + if logFilepath != "" { + config.OutputPaths = append(config.OutputPaths, logFilepath) + } } // Build the logger from the configuration diff --git a/logger/zaplogger_log_levels.go b/logger/zaplogger_log_levels.go index deeecd4..ffa3794 100644 --- a/logger/zaplogger_log_levels.go +++ b/logger/zaplogger_log_levels.go @@ -5,25 +5,17 @@ import ( "go.uber.org/zap" ) -// LogLevel represents the level of logging. Higher values denote more severe log messages. type LogLevel int const ( - // LogLevelDebug is for messages that are useful during software debugging. - LogLevelDebug LogLevel = -1 // Zap's DEBUG level - // LogLevelInfo is for informational messages, indicating normal operation. - LogLevelInfo LogLevel = 0 // Zap's INFO level - // LogLevelWarn is for messages that highlight potential issues in the system. - LogLevelWarn LogLevel = 1 // Zap's WARN level - // LogLevelError is for messages that highlight errors in the application's execution. - LogLevelError LogLevel = 2 // Zap's ERROR level - // LogLevelDPanic is for severe error conditions that are actionable in development. - LogLevelDPanic LogLevel = 3 // Zap's DPANIC level - // LogLevelPanic is for severe error conditions that should cause the program to panic. - LogLevelPanic LogLevel = 4 // Zap's PANIC level - // LogLevelFatal is for errors that require immediate program termination. - LogLevelFatal LogLevel = 5 // Zap's FATAL level - LogLevelNone + LogLevelDebug LogLevel = -1 + LogLevelInfo LogLevel = 0 + LogLevelWarn LogLevel = 1 + LogLevelError LogLevel = 2 + LogLevelDPanic LogLevel = 3 + LogLevelPanic LogLevel = 4 + LogLevelFatal LogLevel = 5 + LogLevelNone = 0 ) // ParseLogLevelFromString takes a string representation of the log level and returns the corresponding LogLevel. @@ -52,6 +44,8 @@ func ParseLogLevelFromString(levelStr string) LogLevel { // ToZapFields converts a variadic list of key-value pairs into a slice of Zap fields. // This allows for structured logging with strongly-typed values. The function assumes // that keys are strings and values can be of any type, leveraging zap.Any for type detection. + +// QUERY What does this do? Why is it in steps of two? Surely we can ammend this to accept a [{k, v}, {k, v}] approach? func ToZapFields(keysAndValues ...interface{}) []zap.Field { var fields []zap.Field for i := 0; i < len(keysAndValues)-1; i += 2 { diff --git a/logger/zaplogger_logpath.go b/logger/zaplogger_logpath.go index dfc1e21..8911b9c 100644 --- a/logger/zaplogger_logpath.go +++ b/logger/zaplogger_logpath.go @@ -12,7 +12,8 @@ import ( // If the path is a directory, it appends a timestamp-based filename. // If the path includes a filename, it checks for the existence of the file. // If no path is provided, it defaults to creating a log file in the current directory with a timestamp-based name. -func EnsureLogFilePath(logPath string) (string, error) { +// TODO refactor this it's very confusing. +func GetLogFilepath(logPath string) (string, error) { if logPath == "" { // Default to the current directory with a timestamp-based filename if no path is provided logPath = filepath.Join(".", "log_"+time.Now().Format("20060102_150405")+".log") @@ -31,6 +32,8 @@ func EnsureLogFilePath(logPath string) (string, error) { // Ensure the directory exists dir := filepath.Dir(logPath) + + // TODO does this work for every OS? if err := os.MkdirAll(dir, 0750); err != nil { return "", err } diff --git a/redirecthandler/redirecthandler.go b/redirecthandler/redirecthandler.go index 4daf0e3..8dd96fa 100644 --- a/redirecthandler/redirecthandler.go +++ b/redirecthandler/redirecthandler.go @@ -221,7 +221,7 @@ func (r *RedirectHandler) GetRedirectHistory(req *http.Request) []*url.URL { // SetupRedirectHandler configures the HTTP client for redirect handling based on the client configuration. func SetupRedirectHandler(client *http.Client, followRedirects bool, maxRedirects int, log logger.Logger) error { if followRedirects { - if maxRedirects < 0 { + if maxRedirects < 1 { log.Error("Invalid maxRedirects value", zap.Int("maxRedirects", maxRedirects)) return fmt.Errorf("invalid maxRedirects value: %d", maxRedirects) }