From fb2cc3d3a2e2ce0c929d28ec8db5262215dd50ba Mon Sep 17 00:00:00 2001 From: skullcmd Date: Tue, 28 Apr 2026 21:59:13 +0000 Subject: [PATCH] fix(ec2-worker): post-launch EIP allocate+associate on multi-ENI launches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #79's ANYSCAN_EC2_ASSOCIATE_PUBLIC_IP=true on a multi-ENI launch is hard-rejected by AWS: InvalidParameterCombination — The associatePublicIPAddress parameter cannot be specified when launching with multiple network interfaces. AWS only honors AssociatePublicIpAddress on NetworkInterfaces[] when exactly one entry is supplied, even if the field appears only on the primary entry of a multi-NIC payload. The entire RunInstances call fails. Reported in PR #65 issuecomment-4339242358 (anygpt-52). Fix: when len(NetworkInterfaces) > 1, suppress the field inline and allocate-address + associate-address on the primary ENI post-launch. The recreate path now also releases the previously-recorded EIP before terminating the old instance so we don't leak Elastic IPs on every recreate. - build_network_interfaces only emits AssociatePublicIpAddress when the resulting payload is single-NIC (target_count == 1). - Ec2WorkerManager._associate_public_ip_post_launch allocates an EIP (Domain=vpc), associates it with the primary ENI (DeviceIndex=0), records AllocationId/AssociationId in self.state. Allocate or associate failures are surfaced in eni_attach.public_ip but do not abort the recreate — the worker is still usable on private IPs. - Ec2WorkerManager._release_recorded_eip disassociates and releases any previously-recorded EIP at the start of recreate_instance. Tests: - New: launch payload free of AssociatePublicIpAddress on multi-ENI; allocate_address + associate_address called post-launch with the primary ENI's NetworkInterfaceId; allocation_id persisted. - New: AllocateAddress failure does not abort recreate. - New: AssociateAddress failure still records AllocationId so the next recreate can release it. - New: previously-recorded EIP is disassociated + released before terminating the old instance on the next recreate. - Updated: prior tests that asserted the broken inline-flag behavior on multi-NIC now assert the field is suppressed everywhere. Co-Authored-By: Claude Opus 4.7 (1M context) --- tools/ec2_worker_manager.py | 165 +++++++++++++++++-- tools/test_ec2_worker_manager.py | 264 ++++++++++++++++++++++++++----- 2 files changed, 378 insertions(+), 51 deletions(-) diff --git a/tools/ec2_worker_manager.py b/tools/ec2_worker_manager.py index bf6b742..94eb3e9 100644 --- a/tools/ec2_worker_manager.py +++ b/tools/ec2_worker_manager.py @@ -214,18 +214,32 @@ def build_network_interfaces( available ENI slots. `associate_public_ip` (off by default) sets AssociatePublicIpAddress - on the primary ENI only — the entry with DeviceIndex=0 and (when - multi-card) NetworkCardIndex=0. AWS rejects the field on secondaries. + on the primary ENI — but ONLY when the resulting payload has exactly + one NetworkInterface entry. AWS hard-rejects RunInstances with + `InvalidParameterCombination — The associatePublicIPAddress + parameter cannot be specified when launching with multiple network + interfaces` whenever len(NetworkInterfaces) > 1, even if the field + appears only on the primary entry. Multi-ENI launches that need a + public IP must allocate+associate an Elastic IP post-launch — see + `Ec2WorkerManager._associate_public_ip_post_launch`. + Needed because RunInstances ignores the subnet's MapPublicIpOnLaunch when an explicit NetworkInterfaces[] is passed; without this opt-in the launch comes up with no public IP. See PR #65 - issuecomment-4338158487 (anygpt-48) for the failure mode. + issuecomment-4338158487 (anygpt-48) for the original failure mode + and PR #65 issuecomment-4339242358 (anygpt-52) for the multi-ENI + rejection that motivated the post-launch path. """ if target_count < 1: raise ValueError("target_count must be >= 1") if not subnet_ids: raise ValueError("subnet_ids must contain at least one subnet") placements = distribute_enis_across_cards(target_count, network_cards) + # AWS rejects AssociatePublicIpAddress whenever more than one + # NetworkInterface entry is supplied. Only emit it inline when the + # resulting payload is single-NIC; multi-ENI launches must rely on + # post-launch allocate-address + associate-address. + inline_associate_public_ip = associate_public_ip and len(placements) == 1 interfaces: list[dict[str, Any]] = [] primary_assigned = False for sequence_index, (card_index, device_index) in enumerate(placements): @@ -241,13 +255,7 @@ def build_network_interfaces( spec["NetworkCardIndex"] = card_index if security_group_ids: spec["Groups"] = list(security_group_ids) - # AssociatePublicIpAddress is only valid on the primary ENI. - # Set it on exactly one entry: DeviceIndex=0 plus (when we're - # emitting NetworkCardIndex) NetworkCardIndex=0. The placement - # helper guarantees the primary ENI lands first, so we only - # need to flip the flag the first time we see a DeviceIndex=0 - # entry on card 0. - if associate_public_ip and not primary_assigned and device_index == 0: + if inline_associate_public_ip and not primary_assigned and device_index == 0: on_primary_card = (not network_cards) or card_index == 0 if on_primary_card: spec["AssociatePublicIpAddress"] = True @@ -918,6 +926,12 @@ def _resolve_eni_subnet_pool(self) -> list[str]: def recreate_instance(self) -> dict[str, Any]: current_id = self.current_instance_id() terminated = None + # If a previous run allocated an Elastic IP for the multi-ENI + # post-launch path, release it before terminating so the + # AllocationId doesn't leak. Terminating the instance auto- + # disassociates the EIP but leaves it allocated to the account + # (charged hourly until released). + released_eip = self._release_recorded_eip() if current_id: terminated = self.ec2.terminate_instances(InstanceIds=[current_id]) ssh_rule = self.ensure_ssh_access() @@ -1004,13 +1018,142 @@ def recreate_instance(self) -> dict[str, Any]: self.state["launched_at_epoch"] = launch_time.timestamp() except Exception: pass + + # Multi-ENI launches cannot carry AssociatePublicIpAddress on the + # NetworkInterfaces[] payload (AWS hard-rejects with + # InvalidParameterCombination), so we have to allocate an Elastic + # IP and associate it with the primary ENI post-launch when the + # operator opted into ANYSCAN_EC2_ASSOCIATE_PUBLIC_IP. Any failure + # is recorded in eni_attach but does not abort the recreate: + # operators can retry the EIP step manually and the worker is + # still usable on private IPs in the meantime. See PR #65 + # issuecomment-4339242358 (anygpt-52) for the failure mode. + if target_count > 1 and self.config.associate_public_ip: + eni_attach_info["public_ip"] = self._associate_public_ip_post_launch( + instance + ) + self._save_state() - return { + result: dict[str, Any] = { "terminated": terminated, "launched": instance, "ssh_rule": ssh_rule, "eni_attach": eni_attach_info, } + if released_eip is not None: + result["released_eip"] = released_eip + return result + + def _release_recorded_eip(self) -> dict[str, Any] | None: + """Release the EIP recorded by the previous post-launch association. + + Returns None when no EIP was recorded; otherwise returns a status + dict suitable for embedding in recreate_instance's result. Errors + are logged in the dict but do not raise: a stuck release should + not block the worker recreate. + """ + allocation_id = self.state.get("eip_allocation_id") + if not allocation_id: + return None + association_id = self.state.get("eip_association_id") + disassoc_error: dict[str, str] | None = None + if association_id: + try: + self.ec2.disassociate_address(AssociationId=association_id) + except botocore.exceptions.ClientError as exc: + disassoc_error = { + "code": exc.response["Error"].get("Code") or "", + "message": exc.response["Error"].get("Message") or "", + } + try: + self.ec2.release_address(AllocationId=allocation_id) + except botocore.exceptions.ClientError as exc: + return { + "status": "release_address_failed", + "allocation_id": allocation_id, + "code": exc.response["Error"].get("Code"), + "error": exc.response["Error"].get("Message"), + "disassociate_error": disassoc_error, + } + self.state.pop("eip_allocation_id", None) + self.state.pop("eip_association_id", None) + return { + "status": "released", + "allocation_id": allocation_id, + "disassociate_error": disassoc_error, + } + + def _associate_public_ip_post_launch( + self, instance: dict[str, Any] + ) -> dict[str, Any]: + """Allocate an Elastic IP and associate it with the primary ENI. + + Used after a multi-ENI RunInstances call where the + AssociatePublicIpAddress field on NetworkInterfaces[] would have + been rejected by AWS. Failures are returned as a status dict + rather than raised so the worker still comes up on private IPs. + + The AllocationId/AssociationId are recorded in self.state so the + next recreate_instance can release the EIP before launching a + replacement (otherwise we leak EIPs on every recreate). + """ + primary_eni_id: str | None = None + for nic in instance.get("NetworkInterfaces", []) or []: + attachment = nic.get("Attachment") or {} + if attachment.get("DeviceIndex") == 0: + primary_eni_id = nic.get("NetworkInterfaceId") + break + if not primary_eni_id: + return { + "status": "no_primary_eni", + "error": "no DeviceIndex=0 ENI found in RunInstances response", + } + + try: + alloc = self.ec2.allocate_address(Domain="vpc") + except botocore.exceptions.ClientError as exc: + return { + "status": "allocate_address_failed", + "code": exc.response["Error"].get("Code"), + "error": exc.response["Error"].get("Message"), + } + allocation_id = alloc.get("AllocationId") + public_ip = alloc.get("PublicIp") + if not allocation_id: + return { + "status": "allocate_address_no_id", + "error": "AllocateAddress did not return AllocationId", + } + + try: + assoc = self.ec2.associate_address( + AllocationId=allocation_id, + NetworkInterfaceId=primary_eni_id, + AllowReassociation=True, + ) + except botocore.exceptions.ClientError as exc: + # Allocation succeeded but association failed; record the + # AllocationId so the next recreate can release it, otherwise + # we leak an unattached EIP on every retry. + self.state["eip_allocation_id"] = allocation_id + return { + "status": "associate_address_failed", + "code": exc.response["Error"].get("Code"), + "error": exc.response["Error"].get("Message"), + "allocation_id": allocation_id, + "public_ip": public_ip, + } + association_id = assoc.get("AssociationId") + self.state["eip_allocation_id"] = allocation_id + if association_id: + self.state["eip_association_id"] = association_id + return { + "status": "associated", + "allocation_id": allocation_id, + "association_id": association_id, + "network_interface_id": primary_eni_id, + "public_ip": public_ip, + } def run_once(self) -> dict[str, Any]: snapshot = self.health_snapshot() diff --git a/tools/test_ec2_worker_manager.py b/tools/test_ec2_worker_manager.py index 41d914f..3ba4f8b 100644 --- a/tools/test_ec2_worker_manager.py +++ b/tools/test_ec2_worker_manager.py @@ -338,8 +338,16 @@ def test_associate_public_ip_default_omits_field(self): for spec in ifs: self.assertNotIn("AssociatePublicIpAddress", spec) - def test_associate_public_ip_true_sets_on_primary_only(self): - """Multi-card: only DeviceIndex=0 + NetworkCardIndex=0 gets the flag.""" + def test_associate_public_ip_suppressed_when_multi_nic_multi_card(self): + """AWS rejects AssociatePublicIpAddress whenever len(NetworkInterfaces) > 1. + + Even though the field would only logically belong on the + primary entry, the AWS API surfaces this as + InvalidParameterCombination on the entire RunInstances call. So + when the helper produces more than one NIC, the field must not + appear anywhere in the payload — multi-ENI launches that need a + public IP must allocate+associate an Elastic IP post-launch. + """ cards = _c6in_metal_describe()["InstanceTypes"][0]["NetworkInfo"][ "NetworkCards" ] @@ -350,39 +358,32 @@ def test_associate_public_ip_true_sets_on_primary_only(self): network_cards=cards, associate_public_ip=True, ) - public_ip_entries = [ - spec for spec in ifs if spec.get("AssociatePublicIpAddress") is True - ] - self.assertEqual(len(public_ip_entries), 1) - primary = public_ip_entries[0] - self.assertEqual(primary["DeviceIndex"], 0) - self.assertEqual(primary["NetworkCardIndex"], 0) - # Every other entry is silent on the field — AWS rejects it on - # secondaries, so we must not emit a `False` either. - secondaries = [ - spec for spec in ifs if spec is not primary - ] - for spec in secondaries: + for spec in ifs: self.assertNotIn("AssociatePublicIpAddress", spec) - def test_associate_public_ip_true_single_card_path(self): - """Single-card (network_cards=None): primary entry is just DeviceIndex=0.""" + def test_associate_public_ip_suppressed_when_multi_nic_single_card(self): + """Single-card target_count>1 is also rejected by AWS.""" ifs = m.build_network_interfaces( target_count=3, subnet_ids=["subnet-aaa"], security_group_ids=["sg-1"], associate_public_ip=True, ) - public_ip_entries = [ - spec for spec in ifs if spec.get("AssociatePublicIpAddress") is True - ] - self.assertEqual(len(public_ip_entries), 1) - self.assertEqual(public_ip_entries[0]["DeviceIndex"], 0) - # NetworkCardIndex was not emitted on this path. - self.assertNotIn("NetworkCardIndex", public_ip_entries[0]) - for spec in ifs[1:]: + for spec in ifs: self.assertNotIn("AssociatePublicIpAddress", spec) + def test_associate_public_ip_emitted_on_single_nic_payload(self): + """target_count=1: AWS accepts AssociatePublicIpAddress on the only NIC.""" + ifs = m.build_network_interfaces( + target_count=1, + subnet_ids=["subnet-aaa"], + security_group_ids=["sg-1"], + associate_public_ip=True, + ) + self.assertEqual(len(ifs), 1) + self.assertIs(ifs[0].get("AssociatePublicIpAddress"), True) + self.assertEqual(ifs[0]["DeviceIndex"], 0) + def _make_config(**overrides) -> Any: base = dict( @@ -421,10 +422,26 @@ def _make_config(**overrides) -> Any: class _FakeEc2Client: """Minimal in-memory EC2 stub recording RunInstances calls.""" - def __init__(self, describe_payload: dict[str, Any] | None = None) -> None: + def __init__( + self, + describe_payload: dict[str, Any] | None = None, + *, + allocate_address_error: dict[str, str] | None = None, + associate_address_error: dict[str, str] | None = None, + release_address_error: dict[str, str] | None = None, + ) -> None: self.describe_payload = describe_payload self.run_instances_calls: list[dict[str, Any]] = [] self.terminate_calls: list[list[str]] = [] + self.allocate_address_calls: list[dict[str, Any]] = [] + self.associate_address_calls: list[dict[str, Any]] = [] + self.disassociate_address_calls: list[dict[str, Any]] = [] + self.release_address_calls: list[dict[str, Any]] = [] + self.allocate_address_error = allocate_address_error + self.associate_address_error = associate_address_error + self.release_address_error = release_address_error + self._next_alloc_id = 1 + self._next_assoc_id = 1 def describe_instance_types(self, *, InstanceTypes): if self.describe_payload is None: @@ -435,12 +452,57 @@ def describe_instance_types(self, *, InstanceTypes): def run_instances(self, **kwargs): self.run_instances_calls.append(kwargs) - return {"Instances": [{"InstanceId": "i-fake-123"}]} + # Reflect the requested NetworkInterfaces back as the launched + # instance's NetworkInterfaces with synthetic IDs/Attachments so + # the post-launch EIP path can find a primary ENI. + nics_in = kwargs.get("NetworkInterfaces") or [] + nics_out = [] + for idx, nic in enumerate(nics_in): + nics_out.append( + { + "NetworkInterfaceId": f"eni-fake-{idx}", + "Attachment": {"DeviceIndex": nic.get("DeviceIndex", idx)}, + } + ) + return { + "Instances": [ + { + "InstanceId": "i-fake-123", + "NetworkInterfaces": nics_out, + } + ] + } def terminate_instances(self, *, InstanceIds): self.terminate_calls.append(list(InstanceIds)) return {"TerminatingInstances": [{"InstanceId": InstanceIds[0]}]} + def allocate_address(self, **kwargs): + self.allocate_address_calls.append(kwargs) + if self.allocate_address_error is not None: + raise _StubClientError({"Error": self.allocate_address_error}) + alloc_id = f"eipalloc-fake-{self._next_alloc_id}" + self._next_alloc_id += 1 + return {"AllocationId": alloc_id, "PublicIp": "203.0.113.42"} + + def associate_address(self, **kwargs): + self.associate_address_calls.append(kwargs) + if self.associate_address_error is not None: + raise _StubClientError({"Error": self.associate_address_error}) + assoc_id = f"eipassoc-fake-{self._next_assoc_id}" + self._next_assoc_id += 1 + return {"AssociationId": assoc_id} + + def disassociate_address(self, **kwargs): + self.disassociate_address_calls.append(kwargs) + return {} + + def release_address(self, **kwargs): + self.release_address_calls.append(kwargs) + if self.release_address_error is not None: + raise _StubClientError({"Error": self.release_address_error}) + return {} + # describe_instances + status calls happen in current_instance_id / # _describe_instance; the tests below stub current_instance_id() to # return None so those paths never fire. @@ -574,8 +636,12 @@ def test_eni_subnet_ids_pool_round_robins(self): subnet_sequence, ["subnet-X", "subnet-Y", "subnet-X", "subnet-Y"] ) - def test_associate_public_ip_knob_propagates_to_primary_eni(self): - """ANYSCAN_EC2_ASSOCIATE_PUBLIC_IP=1 lands on the primary NIC only.""" + def test_associate_public_ip_knob_triggers_post_launch_path_on_multi_eni(self): + """ANYSCAN_EC2_ASSOCIATE_PUBLIC_IP=1 + multi-ENI: NetworkInterfaces[] + carries no AssociatePublicIpAddress (AWS rejects it), and the + manager allocates+associates an Elastic IP post-launch on the + primary ENI instead. + """ config = _make_config( max_enis=15, instance_type="c6in.metal", @@ -587,23 +653,39 @@ def test_associate_public_ip_knob_propagates_to_primary_eni(self): with mock.patch.object(manager, "current_instance_id", return_value=None), \ mock.patch.object(manager, "ensure_ssh_access", return_value=None), \ mock.patch.object(manager, "build_user_data", return_value="#!fake"): - manager.recreate_instance() + result = manager.recreate_instance() + # Launch payload must be free of AssociatePublicIpAddress on + # every NIC — that's what AWS hard-rejects with + # InvalidParameterCombination on multi-ENI launches. call = ec2.run_instances_calls[0] nics = call["NetworkInterfaces"] - primary_candidates = [ - nic for nic in nics - if nic.get("DeviceIndex") == 0 and nic.get("NetworkCardIndex") == 0 - ] - self.assertEqual(len(primary_candidates), 1) - self.assertIs(primary_candidates[0].get("AssociatePublicIpAddress"), True) - secondaries = [nic for nic in nics if nic is not primary_candidates[0]] - self.assertGreater(len(secondaries), 0) - for nic in secondaries: + self.assertGreater(len(nics), 1) + for nic in nics: self.assertNotIn("AssociatePublicIpAddress", nic) + # Post-launch EIP path was taken. + self.assertEqual(len(ec2.allocate_address_calls), 1) + self.assertEqual(ec2.allocate_address_calls[0], {"Domain": "vpc"}) + self.assertEqual(len(ec2.associate_address_calls), 1) + assoc_call = ec2.associate_address_calls[0] + self.assertTrue(assoc_call["NetworkInterfaceId"].startswith("eni-fake-")) + self.assertTrue(assoc_call["AllocationId"].startswith("eipalloc-fake-")) + self.assertIs(assoc_call.get("AllowReassociation"), True) + + # Status surfaces in eni_attach so the daemon log shows it. + public_ip_status = result["eni_attach"]["public_ip"] + self.assertEqual(public_ip_status["status"], "associated") + self.assertEqual(public_ip_status["public_ip"], "203.0.113.42") + + # AllocationId/AssociationId persist in state for cleanup on the + # next recreate_instance. + self.assertIn("eip_allocation_id", manager.state) + self.assertIn("eip_association_id", manager.state) + def test_associate_public_ip_default_off_emits_no_field(self): - """Default config (knob off) → no NIC carries AssociatePublicIpAddress.""" + """Default config (knob off) → no NIC carries AssociatePublicIpAddress + and no post-launch EIP path runs.""" config = _make_config(max_enis=15, instance_type="c6in.metal") ec2 = _FakeEc2Client(describe_payload=_c6in_metal_describe()) manager = _make_manager(config, ec2) @@ -616,6 +698,108 @@ def test_associate_public_ip_default_off_emits_no_field(self): call = ec2.run_instances_calls[0] for nic in call["NetworkInterfaces"]: self.assertNotIn("AssociatePublicIpAddress", nic) + self.assertEqual(ec2.allocate_address_calls, []) + self.assertEqual(ec2.associate_address_calls, []) + + def test_associate_public_ip_allocate_failure_does_not_abort_recreate(self): + """An AllocateAddress error surfaces in eni_attach but the recreate + result still reports the launched instance — the worker is usable + on private IPs, operators can retry the EIP step manually.""" + config = _make_config( + max_enis=15, + instance_type="c6in.metal", + associate_public_ip=True, + ) + ec2 = _FakeEc2Client( + describe_payload=_c6in_metal_describe(), + allocate_address_error={ + "Code": "AddressLimitExceeded", + "Message": "EIP cap reached", + }, + ) + manager = _make_manager(config, ec2) + + with mock.patch.object(manager, "current_instance_id", return_value=None), \ + mock.patch.object(manager, "ensure_ssh_access", return_value=None), \ + mock.patch.object(manager, "build_user_data", return_value="#!fake"): + result = manager.recreate_instance() + + self.assertEqual(result["launched"]["InstanceId"], "i-fake-123") + public_ip_status = result["eni_attach"]["public_ip"] + self.assertEqual(public_ip_status["status"], "allocate_address_failed") + self.assertEqual(public_ip_status["code"], "AddressLimitExceeded") + self.assertEqual(ec2.associate_address_calls, []) + self.assertNotIn("eip_allocation_id", manager.state) + + def test_associate_public_ip_associate_failure_records_alloc_for_cleanup(self): + """If allocate succeeds but associate fails, the AllocationId is + still recorded so the next recreate releases it (no EIP leak).""" + config = _make_config( + max_enis=15, + instance_type="c6in.metal", + associate_public_ip=True, + ) + ec2 = _FakeEc2Client( + describe_payload=_c6in_metal_describe(), + associate_address_error={ + "Code": "InvalidNetworkInterfaceID.NotFound", + "Message": "ENI vanished", + }, + ) + manager = _make_manager(config, ec2) + + with mock.patch.object(manager, "current_instance_id", return_value=None), \ + mock.patch.object(manager, "ensure_ssh_access", return_value=None), \ + mock.patch.object(manager, "build_user_data", return_value="#!fake"): + result = manager.recreate_instance() + + public_ip_status = result["eni_attach"]["public_ip"] + self.assertEqual(public_ip_status["status"], "associate_address_failed") + self.assertIn("allocation_id", public_ip_status) + self.assertEqual( + manager.state["eip_allocation_id"], public_ip_status["allocation_id"] + ) + self.assertNotIn("eip_association_id", manager.state) + + def test_release_recorded_eip_runs_before_terminate_on_recreate(self): + """A previous post-launch EIP is released before terminating the + old instance, so we don't leak EIPs across recreates.""" + config = _make_config( + max_enis=15, + instance_type="c6in.metal", + associate_public_ip=True, + ) + ec2 = _FakeEc2Client(describe_payload=_c6in_metal_describe()) + manager = _make_manager(config, ec2) + # Simulate a prior recreate having allocated/associated. + manager.state["eip_allocation_id"] = "eipalloc-old-9" + manager.state["eip_association_id"] = "eipassoc-old-9" + + with mock.patch.object( + manager, "current_instance_id", return_value="i-old-prev" + ), mock.patch.object( + manager, "ensure_ssh_access", return_value=None + ), mock.patch.object( + manager, "build_user_data", return_value="#!fake" + ): + result = manager.recreate_instance() + + # Old EIP disassociated + released, new one allocated for the + # fresh instance. + self.assertEqual( + ec2.disassociate_address_calls, + [{"AssociationId": "eipassoc-old-9"}], + ) + self.assertEqual( + ec2.release_address_calls, [{"AllocationId": "eipalloc-old-9"}] + ) + self.assertEqual(result["released_eip"]["status"], "released") + self.assertEqual( + result["released_eip"]["allocation_id"], "eipalloc-old-9" + ) + self.assertEqual(len(ec2.allocate_address_calls), 1) + # State reflects the new EIP, not the released one. + self.assertNotEqual(manager.state["eip_allocation_id"], "eipalloc-old-9") class FromEnvIntegrationTests(unittest.TestCase):