diff --git a/packages/aws-durable-execution-sdk-python-extras/README.md b/packages/aws-durable-execution-sdk-python-extras/README.md index c30f30ce5..273cb3f33 100644 --- a/packages/aws-durable-execution-sdk-python-extras/README.md +++ b/packages/aws-durable-execution-sdk-python-extras/README.md @@ -124,7 +124,7 @@ A step retry costs one durable operation and one more Lambda invocation. Most fa 1. **Inside one step attempt.** The boto3 client retries throttling and server errors of `RunMicrovm`, `CreateMicrovmAuthToken`, and `TerminateMicrovm`. The request step retries the POST itself: connection errors, timeouts, 429, and 5xx, with a delay from 250 milliseconds to 4 seconds. It retries one 401 or 403 with a new auth token. The endpoint answers this way until the MicroVM's `run` hook returns. This tier ends after `retry_window`, or 10 seconds before the invocation times out, whichever comes first. 2. **The step retry strategy.** When the first tier gives up, the step retries after a longer delay, outside the invocation. `default_microvm_retry_strategy` is the default, unless the config sets `retry_strategy`. It retries throttling, server errors, connection failures, timeouts, and the end of the first tier, for at most 5 attempts, with a delay of 2, 4, 8, and 16 seconds. Every other error, such as `ValidationException`, `AccessDeniedException`, or `InsufficientCapacityException`, fails the step at once. -A route that answers another 4xx status fails the operation without a retry, with `MicrovmDeliveryError`. So does a redirect, 3xx. The operation does not follow redirects, and the message names the `Location`. Set the path that the route serves, for example with its trailing slash. The JavaScript operation follows redirects. +A route that answers another 4xx status fails the operation without a retry, with `MicrovmDeliveryError`. So does a redirect, 3xx. The operation does not follow redirects, and the message names the `Location`. Set the path that the route serves, for example with its trailing slash. The JavaScript operation also fails every 3xx at once, with `MicrovmDeliveryError`. The POST ignores the proxy settings in the environment, as the JavaScript SDK's `fetch` does. boto3 reads `HTTPS_PROXY`. So in a network that reaches the MicroVM endpoint only through a proxy, the launch succeeds and every POST fails. Pass an `opener` in the config for that network, for example `urllib.request.build_opener(urllib.request.ProxyHandler({"https": "http://proxy.internal:3128"}))`. When an opener follows a redirect, the delivery still fails at once with `MicrovmDeliveryError`. The followed request carries no auth token, because the operation adds its headers as unredirected headers. diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/request.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/request.py index c2b98a197..c7f905c81 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/request.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/request.py @@ -273,14 +273,13 @@ def send_job(options: SendJobOptions) -> int: 3. Any other 4xx, or a 3xx: raise :class:`MicrovmRequestRejectedError` at once. - The function does not follow a redirect. The JavaScript SDK's ``fetch`` - follows one, and it differs in two ways. A 301, 302, or 303 turns the - POST into a GET without the job, and the GET's 2xx counts as accepted. - So the job never starts, and the operation waits until its timeout. A - 307 or 308 to another host receives the auth token. So a redirect fails - the delivery at once, with the ``Location`` in the message. The headers - are added as unredirected headers, so an opener that follows a redirect - does not send the auth token on either. + The function does not follow a redirect, and neither does the JavaScript + operation. A followed 301, 302, or 303 would turn the POST into a GET + without the job. A followed 307 or 308 to another host would receive the + auth token. So a redirect fails the delivery at once, with the + ``Location`` in the message. The headers are added as unredirected + headers, so an opener that follows a redirect does not send the auth + token on either. The delay starts at 250 milliseconds and doubles up to 4 seconds. The tier ends at the retry window, or 10 seconds before the invocation times diff --git a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py index 0fa3f5c0e..4d3c83a8e 100644 --- a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py +++ b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py @@ -1197,6 +1197,49 @@ def handler(event: Any, context: DurableContext) -> Any: return handler, invocations +class _InvocationLog(list[Any]): + """A request list for the fake endpoint. It also records, for each + request, the number of the invocation that sent it.""" + + def __init__(self, invocations: list[int]) -> None: + super().__init__() + self._invocations = invocations + self.sent_in: list[int] = [] + + def append(self, request: Any) -> None: + # The request step blocks its invocation until the POST gets an + # answer or times out. The fake endpoint answers at once. The local + # runner runs one invocation at a time. So the invocation that sent + # the request is the last one that started. + self.sent_in.append(len(self._invocations)) + super().append(request) + + +def record_invocation_of_each_request( + endpoint: Any, invocations: list[int] +) -> list[int]: + """Return a list that gets the invocation number of each request that + ``endpoint`` receives, in order. ``invocations`` is the counter of + :func:`handler_across_deploy`.""" + log = _InvocationLog(invocations) + endpoint.requests = log + return log.sent_in + + +def two_jobs_with_a_wait_between( + vm: MicrovmSession, ctx: DurableContext, event: Any +) -> Any: + """Two jobs with a durable wait between them. + + In the local runner, a durable wait ends the invocation. So the second + job is delivered in a later invocation than the first, after the deploy + of :func:`handler_across_deploy`.""" + build = vm.invoke(ctx, "build", job_input=event, options=job("/build")) + ctx.wait(Duration.from_seconds(1), name="pause") + test = vm.invoke(ctx, "test", job_input=build, options=job()) + return {"build": build, "test": test, "microvm": vm.microvm_id} + + def checkpoint_launch_as(monkeypatch: pytest.MonkeyPatch, **fields: Any) -> None: """Make the launch step checkpoint the MicroVM fields of a real launch, plus ``fields``, and no resume mode of its own. So a test controls the @@ -1257,14 +1300,22 @@ def test_still_resumes_before_each_job_when_a_deploy_turns_suspending_off( client = fake_client( endpoint=endpoint.url, states=["RUNNING", "SUSPENDED", "RUNNING"] ) - handler, invocations = handler_across_deploy(client, first, {}, two_jobs) + handler, invocations = handler_across_deploy( + client, first, {}, two_jobs_with_a_wait_between + ) + sent_in = record_invocation_of_each_request(endpoint, invocations) result = run(handler, None, [("build", succeed_with(1)), ("test", succeed_with(2))]) assert outcome(result)["result"]["test"] == 2 - assert len(invocations) > 1 - # The second job runs after the deploy. It checks the state, and resumes - # the MicroVM before delivery. + # The first job runs in the first invocation, with the launch's config. + # The second job runs in a later invocation, after the deploy. + assert len(sent_in) == 2 + assert sent_in[0] == 1 + assert sent_in[1] > sent_in[0] + # The first job checks the state once: RUNNING. The second job checks it + # after the deploy: SUSPENDED. It resumes the MicroVM, and checks the + # state again before delivery: RUNNING. assert len(client.requests("get_microvm")) == 3 assert client.requests("resume_microvm") == [{"microvmIdentifier": "mvm-1"}] assert len(endpoint.requests) == 2 @@ -1280,15 +1331,21 @@ def test_makes_no_state_check_when_a_deploy_turns_suspending_on( endpoint = fake_endpoint() client = fake_client(endpoint=endpoint.url) handler, invocations = handler_across_deploy( - client, {}, {"auto_suspend_on_idle": True}, two_jobs + client, {}, {"auto_suspend_on_idle": True}, two_jobs_with_a_wait_between ) + sent_in = record_invocation_of_each_request(endpoint, invocations) result = run(handler, None, [("build", succeed_with(1)), ("test", succeed_with(2))]) assert outcome(result)["result"]["test"] == 2 - assert len(invocations) > 1 + # The first job runs in the first invocation, with the launch's config. + # The second job runs in a later invocation, after the deploy. + assert len(sent_in) == 2 + assert sent_in[0] == 1 + assert sent_in[1] > sent_in[0] [launch] = client.requests("run_microvm") assert "autoSuspendIdleSeconds" not in json.loads(launch["runHookPayload"]) + # The second job runs after the deploy. It still checks no state. assert client.requests("get_microvm") == [] assert len(endpoint.requests) == 2 assert launch_checkpoint(result)["resumeBeforeDelivery"] is False