From f2054d0bce1b3200a1b0644d73520f2e3c949ed3 Mon Sep 17 00:00:00 2001 From: Denis Rechkunov Date: Mon, 26 Apr 2021 14:19:43 +0200 Subject: [PATCH] Add `BaseAPIClient` (#113) Can be used to easily perform HTTP requests to various JSON APIs. Also, it's going to be used in the code generator. --- pkg/http/clients/base_api_client.go | 220 ++++++++++++++++ pkg/http/clients/base_api_client_test.go | 312 +++++++++++++++++++++++ 2 files changed, 532 insertions(+) create mode 100644 pkg/http/clients/base_api_client.go create mode 100644 pkg/http/clients/base_api_client_test.go diff --git a/pkg/http/clients/base_api_client.go b/pkg/http/clients/base_api_client.go new file mode 100644 index 00000000..e465c6d2 --- /dev/null +++ b/pkg/http/clients/base_api_client.go @@ -0,0 +1,220 @@ +package clients + +import ( + "context" + "encoding/json" + "io" + "io/ioutil" + "net/http" + "net/url" + "strings" + + cerrors "github.com/contiamo/go-base/v3/pkg/errors" + "github.com/contiamo/go-base/v3/pkg/tokens" + "github.com/contiamo/go-base/v3/pkg/tracing" + "github.com/opentracing/opentracing-go" + otext "github.com/opentracing/opentracing-go/ext" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// APIError describes an error during unsuccessful HTTP request to an API +type APIError struct { + // Status is the HTTP status of the API response + Status int + // Header is the set of response headers + Header http.Header + // Response is the response body + Response []byte +} + +// Error implements the error interface +func (e APIError) Error() string { + return http.StatusText(e.Status) +} + +// TokenProvider is a function that gets the token string for each request +type TokenProvider func() (token string, err error) + +// TokenProviderFromCreator creates a token provider out of token creator. +func TokenProviderFromCreator(tc tokens.Creator, reference string, opts tokens.Options) TokenProvider { + return func() (token string, err error) { + return tc.Create(reference, opts) + } +} + +var ( + // NoopTokenProvider is a token provider that returns an empty string which is ignored by `DoRequest`. + NoopTokenProvider = TokenProvider(func() (token string, err error) { return "", nil }) +) + +// BaseAPIClient describes all basic HTTP client operations required to work with a JSON API +type BaseAPIClient interface { + // GetBaseURL returns the base URL of the service which can be used in HTTP request tasks. + GetBaseURL() string + // DoRequest performs the HTTP request with the given parameters, marshals the payload and + // unmarshals the response into the given output if the status code is successful + DoRequest(ctx context.Context, method, path string, query url.Values, payload, out interface{}) error +} + +// NewBaseAPIClient creates a new instance of the base API client implementation. +// Never use `debug=true` in production environments, it will leak sensitive data +func NewBaseAPIClient(basePath, tokenHeaderName string, tokenProvider TokenProvider, client *http.Client, debug bool) BaseAPIClient { + return &baseAPIClient{ + Tracer: tracing.NewTracer("clients", "BaseAPIClient"), + basePath: basePath, + tokenHeaderName: tokenHeaderName, + tokenProvider: tokenProvider, + client: client, + debug: debug, + } +} + +type baseAPIClient struct { + tracing.Tracer + + basePath string + tokenHeaderName string + tokenProvider TokenProvider + client *http.Client + debug bool +} + +func (t baseAPIClient) GetBaseURL() string { + return t.basePath +} + +func (t baseAPIClient) DoRequest(ctx context.Context, method, path string, query url.Values, payload, out interface{}) (err error) { + span, ctx := t.StartSpan(ctx, "DoRequest") + defer func() { + t.FinishSpan(span, err) + }() + span.SetTag("method", method) + span.SetTag("path", path) + + queryString := query.Encode() + span.SetTag("query", queryString) + + url := t.GetBaseURL() + path + if queryString != "" { + url += "?" + queryString + } + + logrus := logrus. + WithField("method", method). + WithField("url", url) + + logrus.Debug("creating the request token...") + token, err := t.tokenProvider() + if err != nil { + return errors.Wrap(err, "failed to create request token") + } + logrus.Debug("token created.") + + var payloadReader io.Reader + if payload != nil { + // streaming the payload + r, w := io.Pipe() + payloadReader = r + encoder := json.NewEncoder(w) + go func() { + mErr := encoder.Encode(payload) + if mErr != nil { + _ = w.CloseWithError(mErr) + } else { + _ = w.Close() + } + }() + } + + logrus.Debug("creating the HTTP request...") + req, err := http.NewRequest(method, url, payloadReader) + if err != nil { + return errors.Wrap(err, "failed to create a new request") + } + + // so, the HTTP request can be cancelled + req = req.WithContext(ctx) + + req.Header.Add("Content-Type", "application/json") + if token != "" { + req.Header.Add(t.tokenHeaderName, token) + } else { + span.LogKV("token", "token value is empty, header was not set") + } + + // set tracing headers so we can connect spans in different services + err = opentracing.GlobalTracer().Inject( + span.Context(), + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(req.Header), + ) + if err != nil { + // this error should not crash the request, we log it and skip it + otext.Error.Set(span, true) + span.SetTag("tracing.inject.err", err.Error()) + logrus.Error(errors.Wrap(err, "cannot set tracing headers")) + err = nil + } + logrus.Debug("HTTP request created.") + + logrus.Debug("doing request...") + resp, err := t.client.Do(req) + if err != nil { + return errors.Wrap(err, "failed to do request") + } + logrus.Debug("request is done.") + + span.SetTag("response.status", resp.StatusCode) + + logrus.Debug("reading the response...") + defer logrus.Debug("reading the response finished.") + + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + contentType := resp.Header.Get("content-type") + contentType = strings.ToLower(contentType) + span.SetTag("resp.contentType", contentType) + + if strings.Contains(contentType, "json") { + decoder := json.NewDecoder(resp.Body) + return errors.Wrap(decoder.Decode(out), "failed to decode JSON response") + } + + return nil + } + + // these are the cases we can clearly map validation errors, + // should effectively be server errors because they indicate some kind of bug in our implementation, + // the Hub http layer validation should be strong enough to capture user fixable errors + switch resp.StatusCode { + case http.StatusUnauthorized: + return cerrors.ErrAuthorization + case http.StatusForbidden: + return cerrors.ErrPermission + case http.StatusNotFound: + return cerrors.ErrNotFound + case http.StatusNotImplemented: + return cerrors.ErrNotImplemented + default: + if t.debug { + // ignore the error on purpose here + requestBody, _ := json.Marshal(payload) + span.LogKV("request.body", string(requestBody)) + } + + // general error processing + response, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "failed to read response body") + } + span.LogKV("response.body", string(response)) + err = APIError{ + Status: resp.StatusCode, + Header: resp.Header.Clone(), + Response: response, + } + logrus.Error(errors.Wrap(err, "request failed")) + logrus.Error(string(response)) + return err + } +} diff --git a/pkg/http/clients/base_api_client_test.go b/pkg/http/clients/base_api_client_test.go new file mode 100644 index 00000000..cb39356b --- /dev/null +++ b/pkg/http/clients/base_api_client_test.go @@ -0,0 +1,312 @@ +package clients + +import ( + "bytes" + "context" + "encoding/json" + + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "os" + "testing" + "time" + + cerrors "github.com/contiamo/go-base/v3/pkg/errors" + ctesting "github.com/contiamo/go-base/v3/pkg/testing" + "github.com/contiamo/go-base/v3/pkg/tokens" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +type invalidPayload struct { +} + +func (p invalidPayload) MarshalJSON() ([]byte, error) { + return nil, errors.New("invalid payload") +} + +func TestBaseAPIClientDoRequest(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + logrus.SetOutput(ioutil.Discard) + defer logrus.SetOutput(os.Stdout) + + type payload struct { + String string `json:"string"` + Integer int `json:"integer"` + } + + type response struct { + Answer string `json:"answer"` + } + + var ( + out response + resp = response{ + Answer: "everything", + } + tokenError = errors.New("token error") + ) + + cases := []struct { + name string + + method string + path string + query url.Values + payload interface{} + out interface{} + + serverStatus int + serverResponse []byte + + token string + tokenErr error + + expResponse interface{} + expError error + expErrorStr string + }{ + { + name: "Posts payload, gets response back with 200", + method: http.MethodPost, + path: "/some/path", + query: url.Values{ + "q1": []string{"v1"}, + "q2": []string{"v2", "v3"}, + }, + payload: payload{ + String: "some test", + Integer: 42, + }, + out: &out, + + token: "tokenSample", + + serverStatus: http.StatusOK, + serverResponse: ctesting.ToJSONBytes(t, resp), + + expResponse: &response{ + Answer: resp.Answer, + }, + }, + { + name: "Posts nothing, gets response back with 200", + method: http.MethodPost, + path: "/some/path", + out: &out, + + token: "tokenSample", + + serverStatus: http.StatusOK, + serverResponse: ctesting.ToJSONBytes(t, resp), + + expResponse: &response{ + Answer: resp.Answer, + }, + }, + { + name: "Posts nothing, gets nothing with 204", + method: http.MethodPost, + path: "/some/path", + out: &out, + + token: "tokenSample", + + serverStatus: http.StatusNoContent, + + expResponse: &response{}, + }, + { + name: "Gets response with 200", + method: http.MethodGet, + path: "/some/path", + out: &out, + + token: "tokenSample", + + serverStatus: http.StatusOK, + serverResponse: ctesting.ToJSONBytes(t, resp), + + expResponse: &response{ + Answer: resp.Answer, + }, + }, + { + name: "Gets response with 200 without token", + method: http.MethodGet, + path: "/some/path", + out: &out, + + serverStatus: http.StatusOK, + serverResponse: ctesting.ToJSONBytes(t, resp), + + expResponse: &response{ + Answer: resp.Answer, + }, + }, + { + name: "Returns ErrAuthorization on 401", + method: http.MethodGet, + path: "/some/path", + out: &out, + + token: "tokenSample", + serverStatus: http.StatusUnauthorized, + + expError: cerrors.ErrAuthorization, + }, + { + name: "Returns ErrPermission on 403", + method: http.MethodGet, + path: "/some/path", + out: &out, + + token: "tokenSample", + serverStatus: http.StatusForbidden, + + expError: cerrors.ErrPermission, + }, + { + name: "Returns ErrNotFound on 404", + method: http.MethodGet, + path: "/some/path", + out: &out, + + token: "tokenSample", + serverStatus: http.StatusNotFound, + + expError: cerrors.ErrNotFound, + }, + { + name: "Returns ErrNotImplemented on 501", + method: http.MethodGet, + path: "/some/path", + out: &out, + + token: "tokenSample", + serverStatus: http.StatusNotImplemented, + + expError: cerrors.ErrNotImplemented, + }, + { + name: "Returns error response from the server on 500", + method: http.MethodGet, + path: "/some/path", + out: &out, + + token: "tokenSample", + + serverStatus: http.StatusInternalServerError, + serverResponse: []byte("some crazy internal stuff"), + + expError: APIError{ + Status: http.StatusInternalServerError, + Header: http.Header{ + "Content-Length": []string{"25"}, + "Content-Type": []string{"application/json"}, + "Date": []string{"fixed value"}, + }, + Response: []byte("some crazy internal stuff"), + }, + }, + { + name: "Propogates the token creator error", + method: http.MethodGet, + path: "/some/path", + out: &out, + + serverStatus: http.StatusOK, + token: "tokenSample", + tokenErr: tokenError, + + expError: tokenError, + }, + { + name: "Posts payload with invalid JSON and propagates the error", + method: http.MethodPost, + path: "/some/path", + payload: invalidPayload{}, + out: &out, + + token: "tokenSample", + serverStatus: http.StatusOK, + serverResponse: ctesting.ToJSONBytes(t, resp), + + expErrorStr: "json: error calling MarshalJSON for type clients.invalidPayload: invalid payload", + }, + { + name: "Gets invalid JSON and propagates the error", + method: http.MethodPost, + path: "/some/path", + out: &out, + + token: "tokenSample", + serverStatus: http.StatusOK, + serverResponse: []byte("invalid"), + + expErrorStr: "failed to decode JSON response: invalid character 'i' looking for beginning of value", + }, + } + + basePath := "/base" + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + out = response{} // reset the value, so make sure the new one is received by the current test case + + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, basePath+tc.path, r.URL.Path) + require.Equal(t, tc.query.Encode(), r.URL.Query().Encode()) + require.Equal(t, tc.token, r.Header.Get("X-Request-Token")) + require.Equal(t, "application/json", r.Header.Get("Content-Type")) + + // ignore all the payload errors, we just need to compare bytes + payload, _ := ioutil.ReadAll(r.Body) + payload = bytes.TrimSpace(payload) + if tc.payload != nil { + // ignore the serialization error, we compare bytes anyway + expBytes, _ := json.Marshal(tc.payload) + require.Equal(t, string(expBytes), string(payload)) + } else { + require.Empty(t, payload) + } + + if len(tc.serverResponse) > 0 { + w.Header().Add("content-type", "application/json") + } + w.Header().Add("date", "fixed value") + w.WriteHeader(tc.serverStatus) + _, _ = w.Write(tc.serverResponse) + })) + defer s.Close() + + tm := &tokens.CreatorMock{ + Err: tc.tokenErr, + Token: tc.token, + } + tp := TokenProviderFromCreator(tm, "test", tokens.Options{}) + + c := NewBaseAPIClient(s.URL+basePath, "X-Request-Token", tp, http.DefaultClient, true) + err := c.DoRequest(ctx, tc.method, tc.path, tc.query, tc.payload, tc.out) + if tc.expError != nil { + require.Error(t, err) + require.Equal(t, tc.expError, errors.Cause(err)) + return + } + if tc.expErrorStr != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tc.expErrorStr) + return + } + + require.NoError(t, err) + require.EqualValues(t, tc.expResponse, tc.out) + }) + } + +}