feat: Improved steps (part of target upload work)

behaviour/README.md

@@ -6,6 +6,8 @@ in the `features/steps` directory.
 
 We use Python's [behave] package as an executor of the tests.
 
+>   If you're not familiar with Python's behaviour testing read its [tutorial].
+
 To run the tests (from a suitable environment), run `behave` from the `bdd` directory: -
 
     behave
@@ -20,7 +22,22 @@ There is a `.behaverc` that is used to alter its default behaviour.
 
 >   To see a test summary you could add `--summary`.
 
+## Step definition design
+Feature steps are located in the standard `features/steps` directory. Direct
+implementations of steps can be foun din corresponding **given**, **then**,
+and **when** files (e.g. `steps_when.py`). Common logic is located in other
+(`_utils.py`) files.
+
+You will find: -
+
+-   AWX logic used to deploy stack components in `awx_utis.py`.
+-   Fragalysis Front-end logic in `browser_utils.py`. This module
+    wraps-up API calls and provides stack login mechanics.
+-   General Fragalysis API support in `api_utils.py`.
+-   AWS S3-like storage support in `s3_utils.py`.
+
 ---
 
 [behave]: https://behave.readthedocs.io/en/latest/
 [gherkin]: https://cucumber.io/docs/gherkin/reference/
+[tutorial]: https://behave.readthedocs.io/en/stable/tutorial.html

behaviour/features/api-errors-for-anonymous-users.feature

@@ -12,15 +12,16 @@ Feature: Key GET methods need authentication
     The stack should also be functional, by responding correctly on the landing page.
 
     Given an empty behaviour stack tagged latest
-    Then the stack landing page should return http 200
+    Then the landing page response should be OK
 
   Scenario Template: Some REST GET methods should return 'Not Authorized'
 
     Here we do not login to the stack and therefore, for an un-authenticated user,
     the chosen methods are expected to return 'Not Authorized'.
 
-    When I call <method> on the behaviour stack
-    Then I should get http 403
+    Given I do not login to the behaviour stack
+    When I call <method>
+    Then the response should be FORBIDDEN
 
     Examples:
       | method                            |
@@ -33,8 +34,9 @@ Feature: Key GET methods need authentication
     Here we do not login to the stack and therefore, for an un-authenticated user,
     the chosen methods are expected to return 'Not Allowed'.
 
-    When I call <method> on the behaviour stack
-    Then I should get http 405
+    Given I do not login to the behaviour stack
+    When I call <method>
+    Then the response should be METHOD_NOT_ALLOWED
 
     Examples:
       | method                            |

behaviour/features/api-for-anonymous-users.feature

@@ -13,7 +13,7 @@ Feature: Empty stack public API operations
     The stack should also be functional, by responding correctly on the landing page.
 
     Given an empty behaviour stack tagged latest
-    Then the stack landing page should return http 200
+    Then the landing page response should be OK
 
   Scenario Template: Check the main public API methods
 
@@ -22,8 +22,9 @@ Feature: Empty stack public API operations
     An empty stack has no data so everything tested here should return
     successfully and with an empty list of objects.
 
-    When I call <method> on the behaviour stack
-    Then I should get http 200
+    Given I do not login to the behaviour stack
+    When I call <method>
+    Then the response should be OK
     And the length of the returned list should be 0
 
     Examples:
@@ -78,6 +79,7 @@ Feature: Empty stack public API operations
     A small number of methods return objects, even on an empty stack.
     Here we check that an unauthenticated user sees this 'public' data.
 
-    When I call /api/tag_category on the behaviour stack
-    Then I should get http 200
+    Given I do not login to the behaviour stack
+    When I call /api/tag_category
+    Then the response should be OK
     And the length of the returned list should be 9

behaviour/features/steps/api_utils.py

@@ -6,44 +6,27 @@
 from requests import Response
 from requests_toolbelt import MultipartEncoder
 
-LANDING_PAGE_METHOD: str = "/viewer/react/landing/"
+# Trailing slashes are important!
+LANDING_PAGE_ENDPOINT: str = "/viewer/react/landing/"
+UPLOAD_ENDPOINT = "/api/upload_target_experiments/"
 
 # this needs to be kept more or less up to date
 USER_AGENT: str = (
     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
 )
 
 
-def call_api(*, base_url: str, method: str, session_id: Optional[str]) -> Response:
-    """Calls thge GET REST endpoint for an API method using an optional session ID.
+def api_get_request(
+    *, base_url: str, endpoint: str, session_id: Optional[str]
+) -> Response:
+    """Calls the GET REST endpoint using an optional session ID.
     The base url is the root of the apu, i.e. https://example.com. The method is the
     API method to call, i.e. /api/job_config and the session ID is the session ID to
     use for the call."""
 
     with requests.Session() as session:
-
-        session.headers.update(
-            {
-                "User-Agent": USER_AGENT,
-                "Referer": urljoin(base_url, LANDING_PAGE_METHOD),
-                "Referrer-policy": "same-origin",
-            }
-        )
-        session.get(base_url)  # A GET sets any csrftoken
-        if csrftoken := session.cookies.get("csrftoken", None):
-            session.headers.update(
-                {
-                    "X-CSRFToken": csrftoken,
-                    "User-Agent": USER_AGENT,
-                }
-            )
-        if session_id:
-            session.cookies.update(
-                {
-                    "sessionid": session_id,
-                }
-            )
-        return session.get(urljoin(base_url, method))
+        _prepare_session(session, base_url=base_url, session_id=session_id)
+        return session.get(urljoin(base_url, endpoint))
 
 
 def upload_target_experiment(
@@ -55,28 +38,11 @@ def upload_target_experiment(
     file_name: str,
 ) -> Response:
     """Uploads target data to the stack using the given TAS and file path."""
+    assert session_id
+
     with requests.Session() as session:
 
-        session.headers.update(
-            {
-                "User-Agent": USER_AGENT,
-                "Referer": urljoin(base_url, LANDING_PAGE_METHOD),
-                "Referrer-policy": "same-origin",
-            }
-        )
-        session.get(base_url)  # A GET sets any csrftoken
-        if csrftoken := session.cookies.get("csrftoken", None):
-            session.headers.update(
-                {
-                    "X-CSRFToken": csrftoken,
-                    "User-Agent": USER_AGENT,
-                }
-            )
-        session.cookies.update(
-            {
-                "sessionid": session_id,
-            }
-        )
+        _prepare_session(session, base_url=base_url, session_id=session_id)
 
         encoder = MultipartEncoder(
             fields={
@@ -89,9 +55,35 @@ def upload_target_experiment(
             }
         )
 
-        url = urljoin(base_url, "/api/upload_target_experiments")
-        print(f"Uploading to {url}...")
-        response = session.post(url, data=encoder)
-        print(f"Uploaded ({response.status_code})")
+        content_type = encoder.content_type
+        session.headers.update({"Content-Type": content_type})
+        url = urljoin(base_url, UPLOAD_ENDPOINT)
+        return session.post(url, data=encoder, stream=True)
+
+
+# Local functions
 
-        return response
+
+def _prepare_session(session, *, base_url: str, session_id: str) -> None:
+    """Prepares a session for use with the stack."""
+    session.headers.update(
+        {
+            "User-Agent": USER_AGENT,
+            "Referer": urljoin(base_url, LANDING_PAGE_ENDPOINT),
+            "Referrer-policy": "same-origin",
+        }
+    )
+    session.get(base_url)  # A GET sets any csrftoken
+    if csrftoken := session.cookies.get("csrftoken", None):
+        session.headers.update(
+            {
+                "X-CSRFToken": csrftoken,
+                "User-Agent": USER_AGENT,
+            }
+        )
+    if session_id:
+        session.cookies.update(
+            {
+                "sessionid": session_id,
+            }
+        )

behaviour/features/steps/browser_utils.py

@@ -69,7 +69,7 @@ def _run_login_logic_for_cas(spw: sync_playwright, *, host_url, user, password)
     resp = page.goto(f"{host_url}/api/token")
     raw_text = unescape(resp.text())
     session_id_value: str = _RE_SESSION_ID.search(raw_text).group(1)
-    print("Got Session ID")
+    print(f"Got Session ID ({session_id_value})")
 
     browser.close()
 

behaviour/features/steps/steps_given.py

@@ -1,3 +1,4 @@
+import json
 from typing import Dict
 
 from awx_utils import get_stack_url, get_stack_username, launch_awx_job_template
@@ -16,6 +17,9 @@
 @given("an empty {stack_name} stack tagged {image_tag}")  # pylint: disable=not-callable
 def step_impl(context, stack_name, image_tag) -> None:
     """Wipe any existing stack content and create a new (empty) one.
+    The user can pass in a JSON-encoded set of extra variables
+    via the context.text attribute. This appears as a string.
+
     If successful it sets the following context members: -
     stack_name [e.g. 'behaviour']
     stack_url [e.g. https://example.com]
@@ -42,6 +46,13 @@ def step_impl(context, stack_name, image_tag) -> None:
         "stack_oidc_rp_client_secret": get_stack_client_id_secret(),
     }
 
+    # If the user has passed in extra variables, merge them in.
+    if context.text:
+        print(context.text)
+        step_vars = json.loads(context.text)
+        extra_vars |= step_vars
+        print(f"Using step text as extra variables: {step_vars}")
+
     wipe_jt = _AWX_STACK_WIPE_JOB_TEMPLATE % {
         "username": get_stack_username().capitalize()
     }
@@ -65,11 +76,20 @@ def step_impl(context, stack_name) -> None:  # pylint: disable=function-redefine
     status_code"""
     assert context.failed is False
 
-    context.stack_name = stack_name
+    context.stack_name = stack_name.lower()
     context.session_id = login(get_stack_url(context.stack_name))
     assert context.session_id
 
 
+@given("I do not login to the {stack_name} stack")  # pylint: disable=not-callable
+def step_impl(context, stack_name) -> None:  # pylint: disable=function-redefined
+    """Relies on context members: -
+    status_code"""
+    assert context.failed is False
+
+    context.stack_name = stack_name.lower()
+
+
 @given("I can access the {bucket_name} bucket")  # pylint: disable=not-callable
 def step_impl(context, bucket_name) -> None:  # pylint: disable=function-redefined
     """Just make suer we can access the bucket"""