From 09eeaea809cb657935ece40b6d948a0a349ef3fd Mon Sep 17 00:00:00 2001 From: Ethan Ann Date: Wed, 22 Jul 2026 15:00:28 -0400 Subject: [PATCH 1/3] Modernize Python version support and drop six dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prep work for replacing the deprecated uamqp library with a vendored pyamqp. These changes are safe standalone — the actual uamqp → pyamqp swap follows in a subsequent commit. - setup.py: bump python_requires to >=3.8, drop Py2/3.5/3.6/3.7 classifiers, add 3.11/3.12, drop the io.open shim - setup.cfg: remove universal wheel flag (no longer Py2 compatible) - sastoken.py, iothub_amqp_client.py: replace six.moves.urllib with stdlib urllib.parse --- setup.cfg | 7 ------- setup.py | 11 +++-------- src/azure/iot/hub/iothub_amqp_client.py | 4 ++-- src/azure/iot/hub/sastoken.py | 6 +++--- 4 files changed, 8 insertions(+), 20 deletions(-) diff --git a/setup.cfg b/setup.cfg index ac9d3bb..9b3c251 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,10 +3,3 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. #-------------------------------------------------------------------------- - -[bdist_wheel] -# This flag says to generate wheels that support both Python 2 and Python -# 3. If your code will not run unchanged on both Python 2 and 3, you will -# need to generate separate wheels for each Python version that you -# support. -universal=1 diff --git a/setup.py b/setup.py index cd0559c..648cf53 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,6 @@ # -------------------------------------------------------------------------- from setuptools import setup, find_packages -from io import open # io.open needed for Python 2 compat import re # azure v0.x is not compatible with this package @@ -64,23 +63,19 @@ "Topic :: Software Development :: Libraries :: Python Modules", "License :: OSI Approved :: MIT License", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ], install_requires=[ "msrest>=0.6.21,<1.0.0", - # NOTE: Python 2.7, 3.5 support dropped >= 1.4.0 "uamqp>=1.2.14,<2.0.0", "azure-core>=1.10.0,<2.0.0", ], - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4", + python_requires=">=3.8", packages=find_packages( where="src", exclude=[ diff --git a/src/azure/iot/hub/iothub_amqp_client.py b/src/azure/iot/hub/iothub_amqp_client.py index ab995d9..b04b41b 100644 --- a/src/azure/iot/hub/iothub_amqp_client.py +++ b/src/azure/iot/hub/iothub_amqp_client.py @@ -9,7 +9,7 @@ import hashlib import hmac from uuid import uuid4 -import six.moves.urllib as urllib +from urllib import parse as urllib_parse from azure.core.credentials import AccessToken import uamqp @@ -74,7 +74,7 @@ def get_token(): sas = base64.b64decode(shared_access_key) string_to_sign = (hostname + "\n" + str(expiry)).encode("utf-8") signed_hmac_sha256 = hmac.HMAC(sas, string_to_sign, hashlib.sha256) - signature = urllib.parse.quote(base64.b64encode(signed_hmac_sha256.digest())) + signature = urllib_parse.quote(base64.b64encode(signed_hmac_sha256.digest())) return AccessToken( "SharedAccessSignature sr={}&sig={}&se={}&skn={}".format( hostname, signature, expiry, shared_access_key_name diff --git a/src/azure/iot/hub/sastoken.py b/src/azure/iot/hub/sastoken.py index a674c64..39d5c69 100644 --- a/src/azure/iot/hub/sastoken.py +++ b/src/azure/iot/hub/sastoken.py @@ -9,7 +9,7 @@ import hmac import hashlib import time -import six.moves.urllib as urllib +from urllib import parse as urllib_parse class SasTokenError(Exception): @@ -47,7 +47,7 @@ class SasToken(object): _device_token_format = "SharedAccessSignature sr={}&sig={}&se={}" def __init__(self, uri, key, key_name=None, ttl=3600): - self._uri = urllib.parse.quote_plus(uri) + self._uri = urllib_parse.quote_plus(uri) self._key = key self._key_name = key_name self.ttl = ttl @@ -73,7 +73,7 @@ def _build_token(self): message = (self._uri + "\n" + str(self.expiry_time)).encode(self._encoding_type) signing_key = base64.b64decode(self._key.encode(self._encoding_type)) signed_hmac = hmac.HMAC(signing_key, message, hashlib.sha256) - signature = urllib.parse.quote(base64.b64encode(signed_hmac.digest())) + signature = urllib_parse.quote(base64.b64encode(signed_hmac.digest())) except (TypeError, base64.binascii.Error) as e: raise SasTokenError("Unable to build SasToken from given values", e) if self._key_name: From f87e348104c3b2894cad4b6e04f170934ce1f0b0 Mon Sep 17 00:00:00 2001 From: Ethan Ann Date: Wed, 22 Jul 2026 15:24:53 -0400 Subject: [PATCH 2/3] Replace deprecated uamqp with vendored pyamqp Vendors azure-servicebus._pyamqp (sync subset) into src/azure/iot/hub/_pyamqp/ and rewrites the C2D AMQP client against it. Public API (IoTHubRegistryManager, send_c2d_message, TransportType) is unchanged; TransportType is now re-exported from azure.iot.hub. Vendored from Azure/azure-sdk-for-python@42f59593caec801dabfe3d222ddb8561b46b5e72 (see src/azure/iot/hub/_pyamqp/VENDOR.md). --- ..._manager_c2d_amqp_over_websocket_sample.py | 3 +- setup.py | 5 +- src/azure/iot/hub/__init__.py | 2 + src/azure/iot/hub/_pyamqp/VENDOR.md | 54 + src/azure/iot/hub/_pyamqp/__init__.py | 21 + src/azure/iot/hub/_pyamqp/_connection.py | 886 +++++++++++++ src/azure/iot/hub/_pyamqp/_decode.py | 525 ++++++++ src/azure/iot/hub/_pyamqp/_encode.py | 1052 ++++++++++++++++ src/azure/iot/hub/_pyamqp/_platform.py | 99 ++ src/azure/iot/hub/_pyamqp/_transport.py | 772 ++++++++++++ src/azure/iot/hub/_pyamqp/authentication.py | 184 +++ src/azure/iot/hub/_pyamqp/cbs.py | 283 +++++ src/azure/iot/hub/_pyamqp/client.py | 1103 +++++++++++++++++ src/azure/iot/hub/_pyamqp/constants.py | 348 ++++++ src/azure/iot/hub/_pyamqp/described.py | 31 + src/azure/iot/hub/_pyamqp/endpoints.py | 277 +++++ src/azure/iot/hub/_pyamqp/error.py | 352 ++++++ src/azure/iot/hub/_pyamqp/link.py | 276 +++++ src/azure/iot/hub/_pyamqp/management_link.py | 260 ++++ .../iot/hub/_pyamqp/management_operation.py | 125 ++ src/azure/iot/hub/_pyamqp/message.py | 287 +++++ src/azure/iot/hub/_pyamqp/outcomes.py | 162 +++ src/azure/iot/hub/_pyamqp/performatives.py | 640 ++++++++++ src/azure/iot/hub/_pyamqp/receiver.py | 143 +++ src/azure/iot/hub/_pyamqp/sasl.py | 140 +++ src/azure/iot/hub/_pyamqp/sender.py | 197 +++ src/azure/iot/hub/_pyamqp/session.py | 446 +++++++ src/azure/iot/hub/_pyamqp/types.py | 92 ++ src/azure/iot/hub/_pyamqp/utils.py | 139 +++ src/azure/iot/hub/iothub_amqp_client.py | 80 +- src/azure/iot/hub/iothub_registry_manager.py | 8 +- tests/test_iothub_amqp_client.py | 239 ++-- tests/test_iothub_registry_manager.py | 20 +- 33 files changed, 9086 insertions(+), 165 deletions(-) create mode 100644 src/azure/iot/hub/_pyamqp/VENDOR.md create mode 100644 src/azure/iot/hub/_pyamqp/__init__.py create mode 100644 src/azure/iot/hub/_pyamqp/_connection.py create mode 100644 src/azure/iot/hub/_pyamqp/_decode.py create mode 100644 src/azure/iot/hub/_pyamqp/_encode.py create mode 100644 src/azure/iot/hub/_pyamqp/_platform.py create mode 100644 src/azure/iot/hub/_pyamqp/_transport.py create mode 100644 src/azure/iot/hub/_pyamqp/authentication.py create mode 100644 src/azure/iot/hub/_pyamqp/cbs.py create mode 100644 src/azure/iot/hub/_pyamqp/client.py create mode 100644 src/azure/iot/hub/_pyamqp/constants.py create mode 100644 src/azure/iot/hub/_pyamqp/described.py create mode 100644 src/azure/iot/hub/_pyamqp/endpoints.py create mode 100644 src/azure/iot/hub/_pyamqp/error.py create mode 100644 src/azure/iot/hub/_pyamqp/link.py create mode 100644 src/azure/iot/hub/_pyamqp/management_link.py create mode 100644 src/azure/iot/hub/_pyamqp/management_operation.py create mode 100644 src/azure/iot/hub/_pyamqp/message.py create mode 100644 src/azure/iot/hub/_pyamqp/outcomes.py create mode 100644 src/azure/iot/hub/_pyamqp/performatives.py create mode 100644 src/azure/iot/hub/_pyamqp/receiver.py create mode 100644 src/azure/iot/hub/_pyamqp/sasl.py create mode 100644 src/azure/iot/hub/_pyamqp/sender.py create mode 100644 src/azure/iot/hub/_pyamqp/session.py create mode 100644 src/azure/iot/hub/_pyamqp/types.py create mode 100644 src/azure/iot/hub/_pyamqp/utils.py diff --git a/samples/iothub_registry_manager_c2d_amqp_over_websocket_sample.py b/samples/iothub_registry_manager_c2d_amqp_over_websocket_sample.py index 785f8b9..b5a110a 100644 --- a/samples/iothub_registry_manager_c2d_amqp_over_websocket_sample.py +++ b/samples/iothub_registry_manager_c2d_amqp_over_websocket_sample.py @@ -6,8 +6,7 @@ import os import msrest -from azure.iot.hub import IoTHubRegistryManager -from uamqp import TransportType +from azure.iot.hub import IoTHubRegistryManager, TransportType connection_str = os.getenv("IOTHUB_CONNECTION_STRING") device_id = os.getenv("IOTHUB_DEVICE_ID") diff --git a/setup.py b/setup.py index 648cf53..1990047 100644 --- a/setup.py +++ b/setup.py @@ -72,8 +72,11 @@ ], install_requires=[ "msrest>=0.6.21,<1.0.0", - "uamqp>=1.2.14,<2.0.0", "azure-core>=1.10.0,<2.0.0", + # Runtime dependencies of the vendored pyamqp (see src/azure/iot/hub/_pyamqp/VENDOR.md). + "certifi>=2017.4.17", + "typing_extensions>=4.0.1", + "websocket-client>=1.0.0", ], python_requires=">=3.8", packages=find_packages( diff --git a/src/azure/iot/hub/__init__.py b/src/azure/iot/hub/__init__.py index c744817..2814f30 100644 --- a/src/azure/iot/hub/__init__.py +++ b/src/azure/iot/hub/__init__.py @@ -8,6 +8,7 @@ from .iothub_job_manager import IoTHubJobManager from .iothub_http_runtime_manager import IoTHubHttpRuntimeManager from .digital_twin_client import DigitalTwinClient +from ._pyamqp.constants import TransportType __all__ = [ "IoTHubRegistryManager", @@ -15,4 +16,5 @@ "IoTHubJobManager", "IoTHubHttpRuntimeManager", "DigitalTwinClient", + "TransportType", ] diff --git a/src/azure/iot/hub/_pyamqp/VENDOR.md b/src/azure/iot/hub/_pyamqp/VENDOR.md new file mode 100644 index 0000000..ce917fa --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/VENDOR.md @@ -0,0 +1,54 @@ +# Vendored `pyamqp` + +This directory contains a vendored copy of the pure-Python AMQP implementation +that ships privately as `azure.servicebus._pyamqp` in the +[`Azure/azure-sdk-for-python`](https://github.com/Azure/azure-sdk-for-python) +repository. + +`pyamqp` is not published as a standalone PyPI package; vendoring is the +supported way to consume it from outside the SDKs that ship it. + +## Provenance + +- **Upstream repo:** https://github.com/Azure/azure-sdk-for-python +- **Upstream path:** `sdk/servicebus/azure-servicebus/azure/servicebus/_pyamqp/` +- **Pinned commit:** `42f59593caec801dabfe3d222ddb8561b46b5e72` (tip of `main` at time of fetch) +- **Date copied:** 2026-07-22 + +## Local modifications + +The vendored tree is otherwise byte-identical to upstream at the pinned commit, +with the following exceptions: + +1. **`aio/` subtree removed.** This package only performs synchronous C2D + message sends; the async subtree carried a large maintenance surface and + required `asyncio` machinery unused by this SDK. +2. **`_message_backcompat.py` removed.** It was unreferenced by any code path + we import, and it was the only file with a cross-package reference + (`from ..amqp._amqp_message import ...`, guarded by `TYPE_CHECKING`) — a + dangling import that would surface under static analysis. + +No source edits were made to the remaining files. Internal imports were already +relative (`from .foo import bar`), so no import rewrites were required. + +## Why we vendor rather than depend + +`pyamqp` lives at a private module path inside `azure-servicebus` +(`azure.servicebus._pyamqp`). Depending on a private module of another SDK is +brittle: upstream is free to reshape or rename it without notice. Vendoring +gives this package a stable, reviewable snapshot that upgrades happen on our +schedule. + +## Refresh procedure + +To refresh against a newer upstream commit: + +1. Choose a target commit SHA on `Azure/azure-sdk-for-python` `main`. +2. Sparse-fetch `sdk/servicebus/azure-servicebus/azure/servicebus/_pyamqp/` + at that SHA. +3. Replace this directory's contents with the fetched tree. +4. Re-apply the two removals listed under **Local modifications** (`aio/` and + `_message_backcompat.py`). +5. Update the **Pinned commit** and **Date copied** fields above. +6. Run the test suite and the AMQP-over-WebSocket sample against a real + IoT Hub to confirm the refresh is safe. diff --git a/src/azure/iot/hub/_pyamqp/__init__.py b/src/azure/iot/hub/_pyamqp/__init__.py new file mode 100644 index 0000000..fc95444 --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/__init__.py @@ -0,0 +1,21 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +__version__ = "2.0.0a1" + + +from ._connection import Connection +from ._transport import SSLTransport + +from .client import AMQPClient, ReceiveClient, SendClient + +__all__ = [ + "Connection", + "SSLTransport", + "AMQPClient", + "ReceiveClient", + "SendClient", +] diff --git a/src/azure/iot/hub/_pyamqp/_connection.py b/src/azure/iot/hub/_pyamqp/_connection.py new file mode 100644 index 0000000..1dfd1d8 --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/_connection.py @@ -0,0 +1,886 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import uuid +import logging +import time +from urllib.parse import urlparse +import socket +from ssl import SSLError +from typing import Any, Dict, List, Tuple, Optional, NamedTuple, Type, Union, cast + +from ._transport import Transport +from .sasl import SASLTransport, SASLWithWebSocket +from .session import Session +from .performatives import OpenFrame, CloseFrame +from .constants import ( + PORT, + SECURE_PORT, + SOCKET_TIMEOUT, + WEBSOCKET_PORT, + MAX_CHANNELS, + MAX_FRAME_SIZE_BYTES, + HEADER_FRAME, + WS_TIMEOUT_INTERVAL, + ConnectionState, + EMPTY_FRAME, + TransportType, +) + +from .error import ErrorCondition, AMQPConnectionError, AMQPError + +_LOGGER = logging.getLogger(__name__) +_CLOSING_STATES = ( + ConnectionState.OC_PIPE, + ConnectionState.CLOSE_PIPE, + ConnectionState.DISCARDING, + ConnectionState.CLOSE_SENT, + ConnectionState.END, +) + + +def get_local_timeout(now: float, idle_timeout: float, last_frame_received_time: float) -> bool: + """Check whether the local timeout has been reached since a new incoming frame was received. + + :param float now: The current time to check against. + :param float idle_timeout: The idle timeout value. + :param float last_frame_received_time: The time the last frame was received. + :returns: Whether to shutdown the connection due to timeout. + :rtype: bool + """ + if idle_timeout and last_frame_received_time: + time_since_last_received = now - last_frame_received_time + return time_since_last_received > idle_timeout + return False + + +class Connection: # pylint:disable=too-many-instance-attributes + """An AMQP Connection. + + :ivar str state: The connection state. + :param str endpoint: The endpoint to connect to. Must be fully qualified with scheme and port number. + :keyword str container_id: The ID of the source container. If not set a GUID will be generated. + :keyword int max_frame_size: Proposed maximum frame size in bytes. Default value is 64kb. + :keyword int channel_max: The maximum channel number that may be used on the Connection. Default value is 65535. + :keyword float idle_timeout: Connection idle time-out in seconds. + :keyword list(str) outgoing_locales: Locales available for outgoing text. + :keyword list(str) incoming_locales: Desired locales for incoming text in decreasing level of preference. + :keyword list(str) offered_capabilities: The extension capabilities the sender supports. + :keyword list(str) desired_capabilities: The extension capabilities the sender may use if the receiver supports + :keyword dict properties: Connection properties. + :keyword bool allow_pipelined_open: Allow frames to be sent on the connection before a response Open frame + has been received. Default value is `True`. + :keyword float idle_timeout_empty_frame_send_ratio: Portion of the idle timeout time to wait before sending an + empty frame. The default portion is 50% of the idle timeout value (i.e. `0.5`). + :keyword float idle_wait_time: The time in seconds to sleep while waiting for a response from the endpoint. + Default value is `0.1`. + :keyword bool network_trace: Whether to log the network traffic. Default value is `False`. If enabled, frames + will be logged at the logging.INFO level. + :keyword str transport_type: Determines if the transport type is Amqp or AmqpOverWebSocket. + Defaults to TransportType.Amqp. It will be AmqpOverWebSocket if using http_proxy. + :keyword Dict http_proxy: HTTP proxy settings. This must be a dictionary with the following + keys: `'proxy_hostname'` (str value) and `'proxy_port'` (int value). When using these settings, + the transport_type would be AmqpOverWebSocket. + Additionally the following keys may also be present: `'username', 'password'`. + :keyword float socket_timeout: The maximum time in seconds that the underlying socket in the transport should + wait when reading or writing data before timing out. The default value is 0.2 (for transport type Amqp), + and 1 for transport type AmqpOverWebsocket. + """ + + def __init__( # pylint:disable=too-many-locals + self, + endpoint: str, + *, + container_id: Optional[str] = None, + max_frame_size: int = MAX_FRAME_SIZE_BYTES, + channel_max: int = MAX_CHANNELS, + idle_timeout: Optional[float] = None, + outgoing_locales: Optional[List[str]] = None, + incoming_locales: Optional[List[str]] = None, + offered_capabilities: Optional[List[str]] = None, + desired_capabilities: Optional[List[str]] = None, + properties: Optional[Dict[str, Any]] = None, + allow_pipelined_open: bool = True, + idle_timeout_empty_frame_send_ratio: float = 0.5, + idle_wait_time: float = 0.1, + network_trace: bool = False, + transport_type: TransportType = TransportType.Amqp, + http_proxy: Optional[Dict[str, Any]] = None, + socket_timeout: Optional[float] = None, + **kwargs: Any, + ): # pylint:disable=too-many-statements + parsed_url = urlparse(endpoint) + + if parsed_url.hostname is None: + raise ValueError(f"Invalid endpoint: {endpoint}") + self._hostname = parsed_url.hostname + kwargs["http_proxy"] = http_proxy + endpoint = self._hostname + if parsed_url.port and not kwargs.get("use_tls", True): + self._port = parsed_url.port + elif parsed_url.scheme == "amqps": + self._port = SECURE_PORT + else: + self._port = PORT + self.state: Optional[ConnectionState] = None + + # Set the port for AmqpOverWebsocket + if transport_type.value == TransportType.AmqpOverWebsocket.value: + self._port = WEBSOCKET_PORT + + # Custom Endpoint + custom_endpoint_address = kwargs.get("custom_endpoint_address") + custom_endpoint = None + custom_port = None + if custom_endpoint_address: + custom_parsed_url = urlparse(custom_endpoint_address) + if transport_type.value == TransportType.Amqp.value: + custom_port = custom_parsed_url.port or SECURE_PORT + custom_endpoint = f"{custom_parsed_url.hostname}" + else: + custom_port = custom_parsed_url.port or WEBSOCKET_PORT + custom_endpoint = f"{custom_parsed_url.hostname}:{custom_port}{custom_parsed_url.path}" + self._container_id = container_id or str(uuid.uuid4()) + self._network_trace = network_trace + self._network_trace_params = {"amqpConnection": self._container_id, "amqpSession": "", "amqpLink": ""} + + transport = kwargs.get("transport") + self._transport_type = transport_type + self._socket_timeout = socket_timeout + + if self._transport_type.value == TransportType.Amqp.value and self._socket_timeout is None: + self._socket_timeout = SOCKET_TIMEOUT + elif self._transport_type.value == TransportType.AmqpOverWebsocket.value and self._socket_timeout is None: + self._socket_timeout = WS_TIMEOUT_INTERVAL + + if transport: + self._transport = transport + elif "sasl_credential" in kwargs: + sasl_transport: Union[Type[SASLTransport], Type[SASLWithWebSocket]] = SASLTransport + if self._transport_type.name == "AmqpOverWebsocket" or kwargs.get("http_proxy"): + sasl_transport = SASLWithWebSocket + endpoint = parsed_url.hostname + parsed_url.path + self._transport = sasl_transport( + host=endpoint, + credential=kwargs["sasl_credential"], + port=self._port, + custom_endpoint=custom_endpoint, + custom_port=custom_port, + socket_timeout=self._socket_timeout, + network_trace_params=self._network_trace_params, + **kwargs, + ) + else: + self._transport = Transport( + parsed_url.netloc, + transport_type=self._transport_type, + socket_timeout=self._socket_timeout, + network_trace_params=self._network_trace_params, + **kwargs, + ) + self._max_frame_size: int = max_frame_size + self._remote_max_frame_size: Optional[int] = None + self._channel_max: int = channel_max + self._idle_timeout: Optional[float] = idle_timeout + self._outgoing_locales: Optional[List[str]] = outgoing_locales + self._incoming_locales: Optional[List[str]] = incoming_locales + self._offered_capabilities: Optional[List[str]] = offered_capabilities + self._desired_capabilities: Optional[List[str]] = desired_capabilities + self._properties: Optional[Dict[str, Any]] = properties + self._remote_properties: Optional[Dict[str, str]] = None + + self._allow_pipelined_open: bool = allow_pipelined_open + self._remote_idle_timeout: Optional[float] = None + self._remote_idle_timeout_send_frame: Optional[float] = None + self._idle_timeout_empty_frame_send_ratio: float = idle_timeout_empty_frame_send_ratio + self._last_frame_received_time: Optional[float] = None + self._last_frame_sent_time: Optional[float] = None + self._idle_wait_time: float = idle_wait_time + self._error: Optional[AMQPConnectionError] = None + self._outgoing_endpoints: Dict[int, Session] = {} + self._incoming_endpoints: Dict[int, Session] = {} + + def __enter__(self) -> "Connection": + self.open() + return self + + def __exit__(self, *args: Any) -> None: + self.close() + + def _set_state(self, new_state: ConnectionState) -> None: + """Update the connection state. + :param ~pyamqp.constants.ConnectionState new_state: The new state of the connection. + """ + if new_state is None: + return + previous_state = self.state + self.state = new_state + _LOGGER.info("Connection state changed: %r -> %r", previous_state, new_state, extra=self._network_trace_params) + for session in self._outgoing_endpoints.values(): + session._on_connection_state_change() # pylint:disable=protected-access + + def _connect(self) -> None: + """Initiate the connection. + + If `allow_pipelined_open` is enabled, the incoming response header will be processed immediately + and the state on exiting will be HDR_EXCH. Otherwise, the function will return before waiting for + the response header and the final state will be HDR_SENT. + + :raises ValueError: If a reciprocating protocol header is not received during negotiation. + """ + try: + if not self.state: + self._transport.connect() + self._set_state(ConnectionState.START) + self._transport.negotiate() + self._outgoing_header() + self._set_state(ConnectionState.HDR_SENT) + if not self._allow_pipelined_open: + # TODO: List/tuple expected as variable args + self._read_frame(wait=True) + if self.state != ConnectionState.HDR_EXCH: + self._disconnect() + raise ValueError("Did not receive reciprocal protocol header. Disconnecting.") + else: + self._set_state(ConnectionState.HDR_SENT) + except (OSError, IOError, SSLError, socket.error) as exc: + # FileNotFoundError is being raised for exception parity with uamqp when invalid + # `connection_verify` file path is passed in. Remove later when resolving issue #27128. + if isinstance(exc, FileNotFoundError) and exc.filename and "ca_certs" in exc.filename: + raise + raise AMQPConnectionError( + ErrorCondition.SocketError, + description="Failed to initiate the connection due to exception: " + str(exc), + error=exc, + ) from exc + + def _disconnect(self) -> None: + """Disconnect the transport and set state to END.""" + if self.state == ConnectionState.END: + return + self._set_state(ConnectionState.END) + self._transport.close() + + def _can_read(self) -> bool: + """Whether the connection is in a state where it is legal to read for incoming frames. + :return: True if the connection is in a state where it is legal to read for incoming frames. + :rtype: bool + """ + return self.state not in (ConnectionState.CLOSE_RCVD, ConnectionState.END) + + def _read_frame(self, wait: Union[bool, float] = True, **kwargs: Any) -> bool: + """Read an incoming frame from the transport. + + :param Union[bool, float] wait: Whether to block on the socket while waiting for an incoming frame. + The default value is `False`, where the frame will block for the configured timeout only (0.1 seconds). + If set to `True`, socket will block indefinitely. If set to a timeout value in seconds, the socket will + block for at most that value. + :rtype: Tuple[int, Optional[Tuple[int, NamedTuple]]] + :returns: A tuple with the incoming channel number, and the frame in the form or a tuple of performative + descriptor and field values. + """ + # Since we use `sock.settimeout()` in the transport for reading/writing, that acts as a + # "block with timeout" when we pass in a timeout value. If `wait` is float value, then + # timeout was set during socket init. + if wait is not True: # wait is float/int/False + new_frame = self._transport.receive_frame(**kwargs) + else: + with self._transport.block(): + new_frame = self._transport.receive_frame(**kwargs) + return self._process_incoming_frame(*new_frame) + + def _can_write(self) -> bool: + """Whether the connection is in a state where it is legal to write outgoing frames. + :return: Whether the connection is in a state where it is legal to write outgoing frames. + :rtype: bool + """ + return self.state not in _CLOSING_STATES + + def _send_frame(self, channel: int, frame: NamedTuple, **kwargs: Any) -> None: + """Send a frame over the connection. + + :param int channel: The outgoing channel number. + :param NamedTuple frame: The outgoing frame. + :rtype: None + """ + if self._error: + raise self._error + + if self._can_write(): + try: + self._last_frame_sent_time = time.time() + # Since we use `sock.settimeout()` in the transport for reading/writing, + # that acts as a "block with timeout" when we pass in a timeout value. + self._transport.send_frame(channel, frame, **kwargs) + except (OSError, IOError, SSLError, socket.error) as exc: + self._error = AMQPConnectionError( + ErrorCondition.SocketError, + description="Can not send frame out due to exception: " + str(exc), + error=exc, + ) + except Exception: # pylint:disable=try-except-raise + raise + else: + _LOGGER.info("Cannot write frame in current state: %r", self.state, extra=self._network_trace_params) + + def _get_next_outgoing_channel(self) -> int: + """Get the next available outgoing channel number within the max channel limit. + + :raises ValueError: If maximum channels has been reached. + :returns: The next available outgoing channel number. + :rtype: int + """ + if (len(self._incoming_endpoints) + len(self._outgoing_endpoints)) >= self._channel_max: + raise ValueError("Maximum number of channels ({}) has been reached.".format(self._channel_max)) + next_channel = next(i for i in range(1, self._channel_max) if i not in self._outgoing_endpoints) + return next_channel + + def _outgoing_empty(self) -> None: + """Send an empty frame to prevent the connection from reaching an idle timeout.""" + if self._error: + raise self._error + + try: + if self._can_write(): + if self._network_trace: + _LOGGER.debug("-> EmptyFrame()", extra=self._network_trace_params) + self._transport.write(EMPTY_FRAME) + self._last_frame_sent_time = time.time() + except (OSError, IOError, SSLError, socket.error) as exc: + self._error = AMQPConnectionError( + ErrorCondition.SocketError, + description="Can not send empty frame due to exception: " + str(exc), + error=exc, + ) + except Exception: # pylint:disable=try-except-raise + raise + + def _outgoing_header(self) -> None: + """Send the AMQP protocol header to initiate the connection.""" + self._last_frame_sent_time = time.time() + if self._network_trace: + _LOGGER.debug("-> Header(%r)", HEADER_FRAME, extra=self._network_trace_params) + self._transport.write(HEADER_FRAME) + + def _incoming_header(self, _, frame: bytes) -> None: + """Process an incoming AMQP protocol header and update the connection state. + + :param int _: Ignored. + :param bytes frame: The incoming frame. + """ + if self._network_trace: + _LOGGER.debug("<- Header(%r)", frame, extra=self._network_trace_params) + if self.state == ConnectionState.START: + self._set_state(ConnectionState.HDR_RCVD) + elif self.state == ConnectionState.HDR_SENT: + self._set_state(ConnectionState.HDR_EXCH) + elif self.state == ConnectionState.OPEN_PIPE: + self._set_state(ConnectionState.OPEN_SENT) + + def _outgoing_open(self) -> None: + """Send an Open frame to negotiate the AMQP connection functionality.""" + open_frame = OpenFrame( + container_id=self._container_id, + hostname=self._hostname, + max_frame_size=self._max_frame_size, + channel_max=self._channel_max, + idle_timeout=self._idle_timeout * 1000 if self._idle_timeout else None, # Convert to milliseconds + outgoing_locales=self._outgoing_locales, + incoming_locales=self._incoming_locales, + offered_capabilities=self._offered_capabilities if self.state == ConnectionState.OPEN_RCVD else None, + desired_capabilities=self._desired_capabilities if self.state == ConnectionState.HDR_EXCH else None, + properties=self._properties, + ) + if self._network_trace: + _LOGGER.debug("-> %r", open_frame, extra=self._network_trace_params) + self._send_frame(0, open_frame) + + def _incoming_open(self, channel: int, frame) -> None: + """Process incoming Open frame to finish the connection negotiation. + + The incoming frame format is:: + + - frame[0]: container_id (str) + - frame[1]: hostname (str) + - frame[2]: max_frame_size (int) + - frame[3]: channel_max (int) + - frame[4]: idle_timeout (Optional[int]) + - frame[5]: outgoing_locales (Optional[List[bytes]]) + - frame[6]: incoming_locales (Optional[List[bytes]]) + - frame[7]: offered_capabilities (Optional[List[bytes]]) + - frame[8]: desired_capabilities (Optional[List[bytes]]) + - frame[9]: properties (Optional[Dict[bytes, bytes]]) + + :param int channel: The incoming channel number. + :param frame: The incoming Open frame. + :type frame: Tuple[Any, ...] + :rtype: None + """ + # TODO: Add type hints for full frame tuple contents. + if self._network_trace: + _LOGGER.debug("<- %r", OpenFrame(*frame), extra=self._network_trace_params) + if channel != 0: + _LOGGER.error("OPEN frame received on a channel that is not 0.", extra=self._network_trace_params) + self.close( + error=AMQPError( + condition=ErrorCondition.NotAllowed, description="OPEN frame received on a channel that is not 0." + ) + ) + self._set_state(ConnectionState.END) + if self.state == ConnectionState.OPENED: + _LOGGER.error("OPEN frame received in the OPENED state.", extra=self._network_trace_params) + self.close() + if frame[4]: + self._remote_idle_timeout = cast(float, frame[4] / 1000) # Convert to seconds + self._remote_idle_timeout_send_frame = self._idle_timeout_empty_frame_send_ratio * self._remote_idle_timeout + + if frame[2] < 512: + # Max frame size is less than supported minimum. + # If any of the values in the received open frame are invalid then the connection shall be closed. + # The error amqp:invalid-field shall be set in the error.condition field of the CLOSE frame. + self.close( + error=AMQPError( + condition=ErrorCondition.InvalidField, + description="Failed parsing OPEN frame: Max frame size is less than supported minimum.", + ) + ) + _LOGGER.error( + "Failed parsing OPEN frame: Max frame size is less than supported minimum.", + extra=self._network_trace_params, + ) + return + self._remote_max_frame_size = frame[2] + self._remote_properties = frame[9] + if self.state == ConnectionState.OPEN_SENT: + self._set_state(ConnectionState.OPENED) + elif self.state == ConnectionState.HDR_EXCH: + self._set_state(ConnectionState.OPEN_RCVD) + self._outgoing_open() + self._set_state(ConnectionState.OPENED) + else: + self.close( + error=AMQPError( + condition=ErrorCondition.IllegalState, + description=f"connection is an illegal state: {self.state}", + ) + ) + _LOGGER.error("Connection is an illegal state: %r", self.state, extra=self._network_trace_params) + + def _outgoing_close(self, error: Optional[AMQPError] = None) -> None: + """Send a Close frame to shutdown connection with optional error information. + :param ~pyamqp.error.AMQPError or None error: Optional error information. + """ + close_frame = CloseFrame(error=error) + if self._network_trace: + _LOGGER.debug("-> %r", close_frame, extra=self._network_trace_params) + self._send_frame(0, close_frame) + + def _incoming_close(self, channel: int, frame: Tuple[Any, ...]) -> None: + """Process incoming Open frame to finish the connection negotiation. + + The incoming frame format is:: + + - frame[0]: error (Optional[AMQPError]) + + :param int channel: The incoming channel number. + :param tuple frame: The incoming Close frame. + """ + if self._network_trace: + _LOGGER.debug("<- %r", CloseFrame(*frame), extra=self._network_trace_params) + disconnect_states = [ + ConnectionState.HDR_RCVD, + ConnectionState.HDR_EXCH, + ConnectionState.OPEN_RCVD, + ConnectionState.CLOSE_SENT, + ConnectionState.DISCARDING, + ] + if self.state in disconnect_states: + self._disconnect() + return + + close_error = None + if channel > self._channel_max: + _LOGGER.error( + "CLOSE frame received on a channel greated than support max.", extra=self._network_trace_params + ) + close_error = AMQPError(condition=ErrorCondition.InvalidField, description="Invalid channel", info=None) + + self._set_state(ConnectionState.CLOSE_RCVD) + self._outgoing_close(error=close_error) + self._disconnect() + + if frame[0]: + self._error = AMQPConnectionError(condition=frame[0][0], description=frame[0][1], info=frame[0][2]) + _LOGGER.warning("Connection closed with error: %r", frame[0], extra=self._network_trace_params) + + def _incoming_begin(self, channel: int, frame: Tuple[Any, ...]) -> None: + """Process incoming Begin frame to finish negotiating a new session. + + The incoming frame format is:: + + - frame[0]: remote_channel (int) + - frame[1]: next_outgoing_id (int) + - frame[2]: incoming_window (int) + - frame[3]: outgoing_window (int) + - frame[4]: handle_max (int) + - frame[5]: offered_capabilities (Optional[List[bytes]]) + - frame[6]: desired_capabilities (Optional[List[bytes]]) + - frame[7]: properties (Optional[Dict[bytes, bytes]]) + + :param int channel: The incoming channel number. + :param frame: The incoming Begin frame. + :type frame: Tuple[Any, ...] + :rtype: None + """ + try: + existing_session = self._outgoing_endpoints[frame[0]] + self._incoming_endpoints[channel] = existing_session + self._incoming_endpoints[channel]._incoming_begin(frame) # pylint:disable=protected-access + except KeyError: + new_session = Session.from_incoming_frame(self, channel) + self._incoming_endpoints[channel] = new_session + new_session._incoming_begin(frame) # pylint:disable=protected-access + + def _incoming_end(self, channel: int, frame: Tuple[Any, ...]) -> None: + """Process incoming End frame to close a session. + + The incoming frame format is:: + + - frame[0]: error (Optional[AMQPError]) + + :param int channel: The incoming channel number. + :param frame: The incoming End frame. + :type frame: Tuple[Any, ...] + :rtype: None + """ + try: + self._incoming_endpoints[channel]._incoming_end(frame) # pylint:disable=protected-access + outgoing_channel = self._incoming_endpoints[channel].channel + self._incoming_endpoints.pop(channel) + self._outgoing_endpoints.pop(outgoing_channel) + except KeyError: + # close the connection + self.close( + error=AMQPError( + condition=ErrorCondition.ConnectionCloseForced, description="Invalid channel number received" + ) + ) + _LOGGER.error( + "END frame received on invalid channel. Closing connection.", extra=self._network_trace_params + ) + + def _process_incoming_frame( # pylint:disable=too-many-return-statements + self, channel: int, frame: Optional[Union[bytes, Tuple[Any, ...]]] + ) -> bool: + """Process an incoming frame, either directly or by passing to the necessary Session. + + :param int channel: The channel the frame arrived on. + :param frame: A tuple containing the performative descriptor and the field values of the frame. + This parameter can be None in the case of an empty frame or a socket timeout. + :type frame: Optional[Tuple[int, NamedTuple]] + :rtype: bool + :returns: A boolean to indicate whether more frames in a batch can be processed or whether the + incoming frame has altered the state. If `True` is returned, the state has changed and the batch + should be interrupted. + """ + try: + performative, fields = cast(Union[bytes, Tuple], frame) + except TypeError: + return True # Empty Frame or socket timeout + fields = cast(Tuple[Any, ...], fields) + try: + self._last_frame_received_time = time.time() + if performative == 20: + self._incoming_endpoints[channel]._incoming_transfer(fields) # pylint:disable=protected-access + return False + if performative == 21: + self._incoming_endpoints[channel]._incoming_disposition(fields) # pylint:disable=protected-access + return False + if performative == 19: + self._incoming_endpoints[channel]._incoming_flow(fields) # pylint:disable=protected-access + return False + if performative == 18: + self._incoming_endpoints[channel]._incoming_attach(fields) # pylint:disable=protected-access + return False + if performative == 22: + self._incoming_endpoints[channel]._incoming_detach(fields) # pylint:disable=protected-access + return True + if performative == 17: + self._incoming_begin(channel, fields) + return True + if performative == 23: + self._incoming_end(channel, fields) + return True + if performative == 16: + self._incoming_open(channel, fields) + return True + if performative == 24: + self._incoming_close(channel, fields) + return True + if performative == 0: + self._incoming_header(channel, cast(bytes, fields)) + return True + if performative == 1: + return False + _LOGGER.error("Unrecognized incoming frame: %r", frame, extra=self._network_trace_params) + return True + except KeyError: + return True # TODO: channel error + + def _process_outgoing_frame(self, channel: int, frame) -> None: + """Send an outgoing frame if the connection is in a legal state. + + :param int channel: The channel to send the frame on. + :param NamedTuple frame: The frame to send. + :return: None + :rtype: None + :raises ValueError: If the connection is not open or not in a valid state. + """ + if not self._allow_pipelined_open and self.state in [ + ConnectionState.OPEN_PIPE, + ConnectionState.OPEN_SENT, + ]: + raise ValueError("Connection not configured to allow pipeline send.") + if self.state not in [ + ConnectionState.OPEN_PIPE, + ConnectionState.OPEN_SENT, + ConnectionState.OPENED, + ]: + raise AMQPConnectionError(ErrorCondition.SocketError, description="Connection not open.") + now = time.time() + if get_local_timeout( + now, + cast(float, self._idle_timeout), + cast(float, self._last_frame_received_time), + ) or self._get_remote_timeout(now): + _LOGGER.info( + "No frame received for the idle timeout. Closing connection.", extra=self._network_trace_params + ) + self.close( + error=AMQPError( + condition=ErrorCondition.ConnectionCloseForced, + description="No frame received for the idle timeout.", + ), + wait=False, + ) + return + self._send_frame(channel, frame) + + def _get_remote_timeout(self, now: float) -> bool: + """Check whether the local connection has reached the remote endpoints idle timeout since + the last outgoing frame was sent. + + If the time since the last since frame is greater than the allowed idle interval, an Empty + frame will be sent to maintain the connection. + + :param float now: The current time to check against. + :rtype: bool + :returns: Whether the local connection should be shutdown due to timeout. + """ + if self._remote_idle_timeout and self._last_frame_sent_time: + time_since_last_sent = now - self._last_frame_sent_time + if time_since_last_sent > cast(int, self._remote_idle_timeout_send_frame): + self._outgoing_empty() + return False + + def _wait_for_response(self, wait: Union[bool, float], end_state: ConnectionState) -> None: + """Wait for an incoming frame to be processed that will result in a desired state change. + + :param wait: Whether to wait for an incoming frame to be processed. Can be set to `True` to wait + indefinitely, or an int to wait for a specified amount of time (in seconds). To not wait, set to `False`. + :type wait: bool or float + :param ConnectionState end_state: The desired end state to wait until. + :rtype: None + """ + if wait is True: + self.listen(wait=False) + while self.state != end_state: + time.sleep(self._idle_wait_time) + self.listen(wait=False) + elif wait: + self.listen(wait=False) + timeout = time.time() + wait + while self.state != end_state: + if time.time() >= timeout: + break + time.sleep(self._idle_wait_time) + self.listen(wait=False) + + def listen(self, wait: Union[float, bool] = False, batch: int = 1, **kwargs: Any) -> None: + """Listen on the socket for incoming frames and process them. + + :param wait: Whether to block on the socket until a frame arrives. If set to `True`, socket will + block indefinitely. Alternatively, if set to a time in seconds, the socket will block for at most + the specified timeout. Default value is `False`, where the socket will block for its configured read + timeout (by default 0.2 seconds). + :type wait: float or bool + :param int batch: The number of frames to attempt to read and process before returning. The default value + is 1, i.e. process frames one-at-a-time. A higher value should only be used when a receiver is established + and is processing incoming Transfer frames. + :rtype: None + """ + if self._error: + raise self._error + + try: + if self.state not in _CLOSING_STATES: + now = time.time() + if get_local_timeout( + now, + cast(float, self._idle_timeout), + cast(float, self._last_frame_received_time), + ) or self._get_remote_timeout(now): + _LOGGER.info( + "No frame received for the idle timeout. Closing connection.", extra=self._network_trace_params + ) + self.close( + error=AMQPError( + condition=ErrorCondition.ConnectionCloseForced, + description="No frame received for the idle timeout.", + ), + wait=False, + ) + return + if self.state == ConnectionState.END: + self._error = AMQPConnectionError( + condition=ErrorCondition.ConnectionCloseForced, description="Connection was already closed." + ) + return + for _ in range(batch): + if self._can_read(): + if self._read_frame(wait=wait, **kwargs): + break + else: + _LOGGER.info( + "Connection cannot read frames in this state: %r", self.state, extra=self._network_trace_params + ) + break + except (OSError, IOError, SSLError, socket.error) as exc: + self._error = AMQPConnectionError( + ErrorCondition.SocketError, + description="Can not read frame due to exception: " + str(exc), + error=exc, + ) + except Exception: # pylint:disable=try-except-raise + raise + + def create_session( + self, + *, + name: Optional[str] = None, + next_outgoing_id: int = 0, + incoming_window: int = 1, + outgoing_window: int = 1, + handle_max: int = 4294967295, + offered_capabilities: Optional[List[str]] = None, + desired_capabilities: Optional[List[str]] = None, + properties: Optional[Dict[str, Any]] = None, + allow_pipelined_open: Optional[bool] = None, + idle_wait_time: Optional[float] = None, + network_trace: Optional[bool] = None, + **kwargs: Any, + ) -> Session: + """Create a new session within this connection. + + :keyword str name: The name of the connection. If not set a GUID will be generated. + :keyword int next_outgoing_id: The transfer-id of the first transfer id the sender will send. + Default value is 0. + :keyword int incoming_window: The initial incoming-window of the Session. Default value is 1. + :keyword int outgoing_window: The initial outgoing-window of the Session. Default value is 1. + :keyword int handle_max: The maximum handle value that may be used on the session. Default value is 4294967295. + :keyword list(str) offered_capabilities: The extension capabilities the session supports. + :keyword list(str) desired_capabilities: The extension capabilities the session may use if + the endpoint supports it. + :keyword dict properties: Session properties. + :keyword bool allow_pipelined_open: Allow frames to be sent on the connection before a response Open frame + has been received. Default value is that configured for the connection. + :keyword float idle_wait_time: The time in seconds to sleep while waiting for a response from the endpoint. + Default value is that configured for the connection. + :keyword bool network_trace: Whether to log the network traffic of this session. If enabled, frames + will be logged at the logging.INFO level. Default value is that configured for the connection. + + :return: A new Session. + :rtype: ~pyamqp._session.Session + """ + assigned_channel = self._get_next_outgoing_channel() + kwargs["offered_capabilities"] = offered_capabilities + session = Session( + self, + assigned_channel, + name=name, + handle_max=handle_max, + properties=properties, + next_outgoing_id=next_outgoing_id, + incoming_window=incoming_window, + outgoing_window=outgoing_window, + desired_capabilities=desired_capabilities, + allow_pipelined_open=allow_pipelined_open or self._allow_pipelined_open, + idle_wait_time=idle_wait_time or self._idle_wait_time, + network_trace=network_trace or self._network_trace, + network_trace_params=dict(self._network_trace_params), + **kwargs, + ) + self._outgoing_endpoints[assigned_channel] = session + return session + + def open(self, wait: bool = False) -> None: + """Send an Open frame to start the connection. + + Alternatively, this will be called on entering a Connection context manager. + + :param bool wait: Whether to wait to receive an Open response from the endpoint. Default is `False`. + :raises ValueError: If `wait` is set to `False` and `allow_pipelined_open` is disabled. + :rtype: None + """ + self._connect() + self._outgoing_open() + if self.state == ConnectionState.HDR_EXCH: + self._set_state(ConnectionState.OPEN_SENT) + elif self.state == ConnectionState.HDR_SENT: + self._set_state(ConnectionState.OPEN_PIPE) + if wait: + self._wait_for_response(wait, ConnectionState.OPENED) + elif not self._allow_pipelined_open: + raise ValueError("Connection has been configured to not allow piplined-open. Please set 'wait' parameter.") + + def close(self, error: Optional[AMQPError] = None, wait: bool = False) -> None: + """Close the connection and disconnect the transport. + + Alternatively this method will be called on exiting a Connection context manager. + + :param ~uamqp.AMQPError error: Optional error information to include in the close request. + :param bool wait: Whether to wait for a service Close response. Default is `False`. + :rtype: None + """ + try: + if self.state in [ + ConnectionState.END, + ConnectionState.CLOSE_SENT, + ConnectionState.DISCARDING, + ]: + return + self._outgoing_close(error=error) + if error: + self._error = AMQPConnectionError( + condition=error.condition, + description=error.description, + info=error.info, + ) + if self.state == ConnectionState.OPEN_PIPE: + self._set_state(ConnectionState.OC_PIPE) + elif self.state == ConnectionState.OPEN_SENT: + self._set_state(ConnectionState.CLOSE_PIPE) + elif error: + self._set_state(ConnectionState.DISCARDING) + else: + self._set_state(ConnectionState.CLOSE_SENT) + self._wait_for_response(wait, ConnectionState.END) + except Exception as exc: # pylint:disable=broad-except + # If error happened during closing, ignore the error and set state to END + _LOGGER.info("An error occurred when closing the connection: %r", exc, extra=self._network_trace_params) + self._set_state(ConnectionState.END) + finally: + self._disconnect() diff --git a/src/azure/iot/hub/_pyamqp/_decode.py b/src/azure/iot/hub/_pyamqp/_decode.py new file mode 100644 index 0000000..32ef0dd --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/_decode.py @@ -0,0 +1,525 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import struct +import uuid +import logging +import decimal +from typing import ( + Callable, + List, + Optional, + Tuple, + Dict, + Any, + Union, + cast, + TYPE_CHECKING, +) + +from typing_extensions import Literal + + +from . import described +from .message import Message, Header, Properties + +if TYPE_CHECKING: + from .message import MessageDict + +_LOGGER = logging.getLogger(__name__) +_HEADER_PREFIX = memoryview(b"AMQP") + +# Maximum number of elements permitted in any AMQP compound type (list, array, map). +# +# The AMQP 1.0 wire format encodes compound element counts as either a 1-byte +# field (the *_small variants, naturally bounded at 255) or a 4-byte field +# (the *_large variants, wire-level maximum 0xFFFFFFFF). The large variants +# allocate a Python list/dict sized directly from this count, so without an +# upper bound a small frame can demand a multi-gigabyte allocation. This cap +# is applied at every large-variant decode site to keep allocation sizes +# proportional to the bytes actually delivered. +# +# The value mirrors MAX_AMQPVALUE_ITEM_COUNT (65536) in the C reference +# implementation, azure-uamqp-c, and the equivalent MAX_COMPOUND_COUNT bound +# applied across list/array/map decode sites in the Java AMQP codec. Real +# AMQP traffic does not approach this many elements in a single compound +# value, so the bound functions as a hard ceiling rather than a practical +# constraint on legitimate workloads. +_MAX_COMPOUND_COUNT = 65536 +_COMPOSITES = { + 35: "received", + 36: "accepted", + 37: "rejected", + 38: "released", + 39: "modified", +} + +c_unsigned_char = struct.Struct(">B") +c_signed_char = struct.Struct(">b") +c_unsigned_short = struct.Struct(">H") +c_signed_short = struct.Struct(">h") +c_unsigned_int = struct.Struct(">I") +c_signed_int = struct.Struct(">i") +c_unsigned_long = struct.Struct(">L") +c_unsigned_long_long = struct.Struct(">Q") +c_signed_long_long = struct.Struct(">q") +c_float = struct.Struct(">f") +c_double = struct.Struct(">d") + +DECIMAL128_EXPONENT_MAX = 6144 +DECIMAL128_EXPONENT_MIN = -6143 +DECIMAL128_MAX_DIGITS = 34 +DECIMAL128_BIAS = 6176 + + + +def _decode_null(buffer: memoryview) -> Tuple[memoryview, None]: + return buffer, None + + +def _decode_true(buffer: memoryview) -> Tuple[memoryview, Literal[True]]: + return buffer, True + + +def _decode_false(buffer: memoryview) -> Tuple[memoryview, Literal[False]]: + return buffer, False + + +def _decode_zero(buffer: memoryview) -> Tuple[memoryview, Literal[0]]: + return buffer, 0 + + +def _decode_empty(buffer: memoryview) -> Tuple[memoryview, List[Any]]: + return buffer, [] + + +def _decode_boolean(buffer: memoryview) -> Tuple[memoryview, bool]: + return buffer[1:], buffer[:1] == b"\x01" + + +def _decode_ubyte(buffer: memoryview) -> Tuple[memoryview, int]: + return buffer[1:], buffer[0] + + +def _decode_ushort(buffer: memoryview) -> Tuple[memoryview, int]: + return buffer[2:], c_unsigned_short.unpack(buffer[:2])[0] + + +def _decode_uint_small(buffer: memoryview) -> Tuple[memoryview, int]: + return buffer[1:], buffer[0] + + +def _decode_uint_large(buffer: memoryview) -> Tuple[memoryview, int]: + return buffer[4:], c_unsigned_int.unpack(buffer[:4])[0] + + +def _decode_ulong_small(buffer: memoryview) -> Tuple[memoryview, int]: + return buffer[1:], buffer[0] + + +def _decode_ulong_large(buffer: memoryview) -> Tuple[memoryview, int]: + return buffer[8:], c_unsigned_long_long.unpack(buffer[:8])[0] + + +def _decode_byte(buffer: memoryview) -> Tuple[memoryview, int]: + return buffer[1:], c_signed_char.unpack(buffer[:1])[0] + + +def _decode_short(buffer: memoryview) -> Tuple[memoryview, int]: + return buffer[2:], c_signed_short.unpack(buffer[:2])[0] + + +def _decode_int_small(buffer: memoryview) -> Tuple[memoryview, int]: + return buffer[1:], c_signed_char.unpack(buffer[:1])[0] + + +def _decode_int_large(buffer: memoryview) -> Tuple[memoryview, int]: + return buffer[4:], c_signed_int.unpack(buffer[:4])[0] + + +def _decode_long_small(buffer: memoryview) -> Tuple[memoryview, int]: + return buffer[1:], c_signed_char.unpack(buffer[:1])[0] + + +def _decode_long_large(buffer: memoryview) -> Tuple[memoryview, int]: + return buffer[8:], c_signed_long_long.unpack(buffer[:8])[0] + + +def _decode_float(buffer: memoryview) -> Tuple[memoryview, float]: + return buffer[4:], c_float.unpack(buffer[:4])[0] + + +def _decode_double(buffer: memoryview) -> Tuple[memoryview, float]: + return buffer[8:], c_double.unpack(buffer[:8])[0] + + +def _decode_timestamp(buffer: memoryview) -> Tuple[memoryview, int]: + return buffer[8:], c_signed_long_long.unpack(buffer[:8])[0] + + +def _decode_uuid(buffer: memoryview) -> Tuple[memoryview, uuid.UUID]: + return buffer[16:], uuid.UUID(bytes=buffer[:16].tobytes()) + + +def _decode_binary_small(buffer: memoryview) -> Tuple[memoryview, bytes]: + length_index = buffer[0] + 1 + return buffer[length_index:], buffer[1:length_index].tobytes() + + +def _decode_binary_large(buffer: memoryview) -> Tuple[memoryview, bytes]: + length_index = c_unsigned_long.unpack(buffer[:4])[0] + 4 + return buffer[length_index:], buffer[4:length_index].tobytes() + + +def _decode_decimal128(buffer: memoryview) -> Tuple[memoryview, decimal.Decimal]: + """ + Decode a Decimal128 value from the buffer. + + The Decimal128 encoding is a 16-byte encoding that represents a 34-digit decimal number. + The encoding uses the IEEE 754-2008 decimal128 format. + See: https://docs.oasis-open.org/amqp/core/v1.0/os/amqp-core-types-v1.0-os.html#type-decimal128 + + :param buffer: The buffer containing the Decimal128 encoded value. + :type buffer: memoryview + :return: A tuple containing the remaining buffer and the decoded decimal.Decimal value. + :rtype: Tuple[memoryview, decimal.Decimal] + """ + + # per the spec decimal128 has a width of 16 bytes + dec_value = bytearray(buffer[:16]) + + # Determine the sign of the decimal number + sign = 1 if dec_value[0] & 0x80 else 0 + biased_exponent = 0 + + # Check if the exponent is not a special value + if dec_value[0] & 0x60 != 0x60: + # Extract the biased exponent from the first two bytes + biased_exponent = ((dec_value[0] & 0x7F) << 9) | ((dec_value[1] & 0xFE) >> 1) + dec_value[0] = 0 + dec_value[1] &= 0x01 + elif dec_value[0] & 0x78 != 0: + # Handle special values (NaN and Infinity) + if (dec_value[0] & 0x78) == 0x78: + return buffer[16:], decimal.Decimal('NaN') + return buffer[16:], decimal.Decimal('-Infinity') + else: + # If the exponent is zero, return zero + return buffer[16:], decimal.Decimal(0) + + # Calculate the actual exponent by subtracting the bias + exponent = biased_exponent - DECIMAL128_BIAS + + # Extract the significant digits from the remaining 14 bytes + hi = c_unsigned_int.unpack(dec_value[4:8])[0] + middle = c_unsigned_int.unpack(dec_value[8:12])[0] + lo = c_unsigned_int.unpack(dec_value[12:16])[0] + digits = tuple(int(digit) for digit in f"{hi:08}{middle:08}{lo:08}".lstrip('0')) + + # Create a decimal context with the appropriate precision and exponent range + decimal_ctx = decimal.Context( + prec = DECIMAL128_MAX_DIGITS, + Emin = DECIMAL128_EXPONENT_MIN, + Emax = DECIMAL128_EXPONENT_MAX, + capitals = 1, + clamp = 1, + ) + + # Create the decimal value using the context + with decimal.localcontext(decimal_ctx) as ctx: + return buffer[16:], ctx.create_decimal((sign, digits, exponent)) + +def _decode_list_small(buffer: memoryview) -> Tuple[memoryview, List[Any]]: + count = buffer[1] + buffer = buffer[2:] + values = [None] * count + for i in range(count): + buffer, values[i] = _DECODE_BY_CONSTRUCTOR[buffer[0]](buffer[1:]) + return buffer, values + + +def _decode_list_large(buffer: memoryview) -> Tuple[memoryview, List[Any]]: + count = c_unsigned_long.unpack(buffer[4:8])[0] + # Validate the wire-supplied count before allocating `[None] * count`, + # which would otherwise scale linearly with an untrusted 32-bit value. + if count > _MAX_COMPOUND_COUNT: + raise ValueError( + f"AMQP list element count {count} exceeds maximum {_MAX_COMPOUND_COUNT}" + ) + buffer = buffer[8:] + values = [None] * count + for i in range(count): + buffer, values[i] = _DECODE_BY_CONSTRUCTOR[buffer[0]](buffer[1:]) + return buffer, values + + +def _decode_map_small(buffer: memoryview) -> Tuple[memoryview, Dict[Any, Any]]: + raw_count = buffer[1] + if raw_count % 2 != 0: + raise ValueError( + f"AMQP map element count {raw_count} must be even (key/value pairs)" + ) + count = raw_count // 2 + buffer = buffer[2:] + values = {} + for _ in range(count): + buffer, key = _DECODE_BY_CONSTRUCTOR[buffer[0]](buffer[1:]) + buffer, value = _DECODE_BY_CONSTRUCTOR[buffer[0]](buffer[1:]) + values[key] = value + return buffer, values + + +def _decode_map_large(buffer: memoryview) -> Tuple[memoryview, Dict[Any, Any]]: + # Validate the raw on-wire count *before* halving it (the AMQP encoding + # stores total entries; pairs = entries / 2). Checking pre-halve keeps + # the comparison aligned with the bound used by _decode_list_large / + # _decode_array_large. Odd counts are rejected explicitly: silently + # flooring to (raw_count - 1) // 2 would leave a trailing key with no + # value, leaking bytes into the next decoder. + raw_count = c_unsigned_long.unpack(buffer[4:8])[0] + if raw_count > _MAX_COMPOUND_COUNT: + raise ValueError( + f"AMQP map element count {raw_count} exceeds maximum {_MAX_COMPOUND_COUNT}" + ) + if raw_count % 2 != 0: + raise ValueError( + f"AMQP map element count {raw_count} must be even (key/value pairs)" + ) + count = raw_count // 2 + buffer = buffer[8:] + values = {} + for _ in range(count): + buffer, key = _DECODE_BY_CONSTRUCTOR[buffer[0]](buffer[1:]) + buffer, value = _DECODE_BY_CONSTRUCTOR[buffer[0]](buffer[1:]) + values[key] = value + return buffer, values + + +def _decode_array_small(buffer: memoryview) -> Tuple[memoryview, List[Any]]: + count = buffer[1] # Ignore first byte (size) and just rely on count + if count: + values = [None] * count + subconstructor = buffer[2] + + if subconstructor == 0: + composite_type = buffer[3] + buffer, descriptor = _DECODE_BY_CONSTRUCTOR[composite_type](buffer[4:]) + subconstructor = buffer[0] + buffer = buffer[1:] + for i in range(count): + buffer, values[i] = _decode_described_array(buffer, subconstructor, descriptor) + else: + buffer = buffer[3:] + for i in range(count): + buffer, values[i] = _DECODE_BY_CONSTRUCTOR[subconstructor](buffer) + return buffer, values + return buffer[2:], [] + + +def _decode_array_large(buffer: memoryview) -> Tuple[memoryview, List[Any]]: + count = c_unsigned_long.unpack(buffer[4:8])[0] + # Validate the wire-supplied count before allocating `[None] * count`. + # An Array32 frame's COUNT is read directly from the network and would + # otherwise drive a Python list allocation of arbitrary size. + if count > _MAX_COMPOUND_COUNT: + raise ValueError( + f"AMQP array element count {count} exceeds maximum {_MAX_COMPOUND_COUNT}" + ) + if count: + values = [None] * count + subconstructor = buffer[8] + + if subconstructor == 0: + composite_type = buffer[9] + buffer, descriptor = _DECODE_BY_CONSTRUCTOR[composite_type](buffer[10:]) + subconstructor = buffer[0] + buffer = buffer[1:] + for i in range(count): + buffer, values[i] = _decode_described_array(buffer, subconstructor, descriptor) + else: + buffer = buffer[9:] + for i in range(count): + buffer, values[i] = _DECODE_BY_CONSTRUCTOR[subconstructor](buffer) + return buffer, values + return buffer[8:], [] + + +def _decode_described(buffer: memoryview) -> Tuple[memoryview, object]: + # TODO: to move the cursor of the buffer to the described value based on size of the + # descriptor without decoding descriptor value + composite_type = buffer[0] + buffer, descriptor = _DECODE_BY_CONSTRUCTOR[composite_type](buffer[1:]) + tp = buffer[0] + buffer, value = _DECODE_BY_CONSTRUCTOR[tp](buffer[1:]) + try: + value = _DESCR_BY_CONSTRUCTOR[tp](value, descriptor=descriptor) + except KeyError: + pass + try: + composite_type = cast(int, _COMPOSITES[descriptor]) + return buffer, {composite_type: value} + except KeyError: + return buffer, value + + +def _decode_described_array(buffer: memoryview, tp: int, descriptor) -> Tuple[memoryview, Any]: + buffer, value = _DECODE_BY_CONSTRUCTOR[tp](buffer) + try: + value = _DESCR_BY_CONSTRUCTOR[tp](value, descriptor=descriptor) + except KeyError: + pass + try: + composite_type = cast(int, _COMPOSITES[descriptor]) + return buffer, {composite_type: value} + except KeyError: + return buffer, value + + +def decode_payload(buffer: memoryview) -> Message: + message: Dict[str, Union[Properties, Header, Dict, bytes, List]] = {} + while buffer: + # Ignore the first two bytes, they will always be the constructors for + # described type then ulong. + descriptor = buffer[2] + buffer, value = _DECODE_BY_CONSTRUCTOR[buffer[3]](buffer[4:]) + + if descriptor == 112: + message["header"] = Header(*value) + elif descriptor == 113: + message["delivery_annotations"] = value + elif descriptor == 114: + message["message_annotations"] = value + elif descriptor == 115: + message["properties"] = Properties(*value) + elif descriptor == 116: + message["application_properties"] = value + elif descriptor == 117: + try: + cast(List, message["data"]).append(value) + except KeyError: + message["data"] = [value] + elif descriptor == 118: + try: + cast(List, message["sequence"]).append(value) + except KeyError: + message["sequence"] = [value] + elif descriptor == 119: + message["value"] = value + elif descriptor == 120: + message["footer"] = value + # TODO: we can possibly swap out the Message construct with a TypedDict + # for both input and output so we get the best of both. + # casting to TypedDict with named fields to allow for unpacking with ** + message_properties = cast("MessageDict", message) + return Message(**message_properties) + + +def decode_frame(data: memoryview) -> Tuple[int, List[Any]]: + # Ignore the first two bytes, they will always be the constructors for + # described type then ulong. + frame_type = data[2] + compound_list_type = data[3] + if compound_list_type == 0xD0: + # list32 0xd0: data[4:8] is size, data[8:12] is count. The AMQP 1.0 + # wire format defines COUNT as an unsigned 32-bit field; decoding it + # as a signed int and skipping the cap would let a malicious peer + # request a multi-gigabyte field-list allocation below. + count = c_unsigned_long.unpack(data[8:12])[0] + if count > _MAX_COMPOUND_COUNT: + raise ValueError( + f"AMQP frame field count {count} exceeds maximum {_MAX_COMPOUND_COUNT}" + ) + buffer = data[12:] + else: + # list8 0xc0: data[4] is size, data[5] is count (1 byte, bounded at 255). + count = data[5] + buffer = data[6:] + fields: List[Optional[memoryview]] = [None] * count + for i in range(count): + buffer, fields[i] = _DECODE_BY_CONSTRUCTOR[buffer[0]](buffer[1:]) + if frame_type == 20: + fields.append(buffer) + return frame_type, fields + + +def decode_empty_frame(header: memoryview) -> Tuple[int, bytes]: + if header[0:4] == _HEADER_PREFIX: + return 0, header.tobytes() + if header[5] == 0: + return 1, b"EMPTY" + raise ValueError("Received unrecognized empty frame") + + +_DECODE_BY_CONSTRUCTOR: List[Callable] = cast(List[Callable], [None] * 256) +_DECODE_BY_CONSTRUCTOR[0] = _decode_described +_DECODE_BY_CONSTRUCTOR[64] = _decode_null +_DECODE_BY_CONSTRUCTOR[65] = _decode_true +_DECODE_BY_CONSTRUCTOR[66] = _decode_false +_DECODE_BY_CONSTRUCTOR[67] = _decode_zero +_DECODE_BY_CONSTRUCTOR[68] = _decode_zero +_DECODE_BY_CONSTRUCTOR[69] = _decode_empty +_DECODE_BY_CONSTRUCTOR[80] = _decode_ubyte +_DECODE_BY_CONSTRUCTOR[81] = _decode_byte +_DECODE_BY_CONSTRUCTOR[82] = _decode_uint_small +_DECODE_BY_CONSTRUCTOR[83] = _decode_ulong_small +_DECODE_BY_CONSTRUCTOR[84] = _decode_int_small +_DECODE_BY_CONSTRUCTOR[85] = _decode_long_small +_DECODE_BY_CONSTRUCTOR[86] = _decode_boolean +_DECODE_BY_CONSTRUCTOR[96] = _decode_ushort +_DECODE_BY_CONSTRUCTOR[97] = _decode_short +_DECODE_BY_CONSTRUCTOR[112] = _decode_uint_large +_DECODE_BY_CONSTRUCTOR[113] = _decode_int_large +_DECODE_BY_CONSTRUCTOR[114] = _decode_float +_DECODE_BY_CONSTRUCTOR[128] = _decode_ulong_large +_DECODE_BY_CONSTRUCTOR[129] = _decode_long_large +_DECODE_BY_CONSTRUCTOR[130] = _decode_double +_DECODE_BY_CONSTRUCTOR[131] = _decode_timestamp +_DECODE_BY_CONSTRUCTOR[148] = _decode_decimal128 +_DECODE_BY_CONSTRUCTOR[152] = _decode_uuid +_DECODE_BY_CONSTRUCTOR[160] = _decode_binary_small +_DECODE_BY_CONSTRUCTOR[161] = _decode_binary_small +_DECODE_BY_CONSTRUCTOR[163] = _decode_binary_small +_DECODE_BY_CONSTRUCTOR[176] = _decode_binary_large +_DECODE_BY_CONSTRUCTOR[177] = _decode_binary_large +_DECODE_BY_CONSTRUCTOR[179] = _decode_binary_large +_DECODE_BY_CONSTRUCTOR[192] = _decode_list_small +_DECODE_BY_CONSTRUCTOR[193] = _decode_map_small +_DECODE_BY_CONSTRUCTOR[208] = _decode_list_large +_DECODE_BY_CONSTRUCTOR[209] = _decode_map_large +_DECODE_BY_CONSTRUCTOR[224] = _decode_array_small +_DECODE_BY_CONSTRUCTOR[240] = _decode_array_large + +_DESCR_BY_CONSTRUCTOR: Dict[int, Any] = { + 67: described.DescribedInt, + 68: described.DescribedInt, + 69: described.DescribedList, + 80: described.DescribedInt, + 81: described.DescribedInt, + 82: described.DescribedInt, + 83: described.DescribedInt, + 84: described.DescribedInt, + 85: described.DescribedInt, + 96: described.DescribedInt, + 97: described.DescribedInt, + 112: described.DescribedInt, + 113: described.DescribedInt, + 114: described.DescribedFloat, + 128: described.DescribedInt, + 129: described.DescribedInt, + 130: described.DescribedFloat, + 131: described.DescribedInt, + 160: described.DescribedBytes, + 161: described.DescribedBytes, + 163: described.DescribedBytes, + 176: described.DescribedBytes, + 177: described.DescribedBytes, + 179: described.DescribedBytes, + 192: described.DescribedList, + 193: described.DescribedDict, + 208: described.DescribedList, + 209: described.DescribedDict, + 224: described.DescribedList, + 240: described.DescribedList, +} diff --git a/src/azure/iot/hub/_pyamqp/_encode.py b/src/azure/iot/hub/_pyamqp/_encode.py new file mode 100644 index 0000000..573e0da --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/_encode.py @@ -0,0 +1,1052 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +# pylint: disable=too-many-lines +# pylint: disable=unused-argument +# TODO: fix mypy errors for _code/_definition/__defaults__ (issue #26500) +import calendar +import struct +import uuid +from datetime import datetime +from typing import ( + Iterable, + NamedTuple, + Union, + Tuple, + Dict, + Any, + cast, + Sized, + Optional, + List, + Callable, + TYPE_CHECKING, + Sequence, + Collection, +) + +try: + from typing import TypeAlias # type: ignore +except ImportError: + from typing_extensions import TypeAlias + +from decimal import Decimal +from typing_extensions import Buffer + + + +from .types import ( + TYPE, + VALUE, + AMQPTypes, + FieldDefinition, + ObjDefinition, + ConstructorBytes, +) +from .message import Message +from . import performatives + +if TYPE_CHECKING: + from .message import Header, Properties + + Performative: TypeAlias = Union[ + performatives.OpenFrame, + performatives.BeginFrame, + performatives.AttachFrame, + performatives.FlowFrame, + performatives.TransferFrame, + performatives.DispositionFrame, + performatives.DetachFrame, + performatives.EndFrame, + performatives.CloseFrame, + performatives.SASLMechanism, + performatives.SASLInit, + performatives.SASLChallenge, + performatives.SASLResponse, + performatives.SASLOutcome, + Message, + Header, + Properties, + ] + +_FRAME_OFFSET = b"\x02" +_FRAME_TYPE = b"\x00" +AQMPSimpleType = Union[bool, float, str, bytes, uuid.UUID, None] + +_DECIMAl128bias = 6176 + + +def _construct(byte: bytes, construct: bool) -> bytes: + return byte if construct else b"" + + +def encode_null(output: bytearray, *args: Any, **kwargs: Any) -> None: + """ + encoding code="0x40" category="fixed" width="0" label="the null value" + + :param bytearray output: The output buffer to write to. + :param any args: Ignored. + """ + output.extend(ConstructorBytes.null) + + +def encode_boolean(output: bytearray, value: bool, with_constructor: bool = True, **kwargs: Any) -> None: + """ + + + + + :param bytearray output: The output buffer to write to. + :param bool value: The boolean to encode. + :param bool with_constructor: Whether to include the constructor byte. + """ + value = bool(value) + if with_constructor: + output.extend(_construct(ConstructorBytes.bool, with_constructor)) + output.extend(b"\x01" if value else b"\x00") + return + + output.extend(ConstructorBytes.bool_true if value else ConstructorBytes.bool_false) + + +def encode_ubyte(output: bytearray, value: Union[int, bytes], with_constructor: bool = True, **kwargs: Any) -> None: + """ + + + :param bytearray output: The output buffer to write to. + :param int or bytes value: The ubyte to encode. + :param bool with_constructor: Whether to include the constructor byte. + """ + try: + value = int(value) + except ValueError: + value = cast(bytes, value) + value = ord(value) + try: + output.extend(_construct(ConstructorBytes.ubyte, with_constructor)) + output.extend(struct.pack(">B", abs(value))) + except struct.error as exc: + raise ValueError("Unsigned byte value must be 0-255") from exc + + +def encode_ushort(output: bytearray, value: int, with_constructor: bool = True, **kwargs: Any) -> None: + """ + + + :param bytearray output: The output buffer to write to. + :param int value: The ushort to encode. + :param bool with_constructor: Whether to include the constructor byte. + """ + value = int(value) + try: + output.extend(_construct(ConstructorBytes.ushort, with_constructor)) + output.extend(struct.pack(">H", abs(value))) + except struct.error as exc: + raise ValueError("Unsigned byte value must be 0-65535") from exc + + +def encode_uint(output: bytearray, value: int, with_constructor: bool = True, use_smallest: bool = True) -> None: + """ + + + + + :param bytearray output: The output buffer to write to. + :param int value: The uint to encode. + :param bool with_constructor: Whether to include the constructor byte. + :param bool use_smallest: Whether to use the smallest possible encoding. + """ + value = int(value) + if value == 0: + output.extend(ConstructorBytes.uint_0) + return + try: + if use_smallest and value <= 255: + output.extend(_construct(ConstructorBytes.uint_small, with_constructor)) + output.extend(struct.pack(">B", abs(value))) + return + output.extend(_construct(ConstructorBytes.uint_large, with_constructor)) + output.extend(struct.pack(">I", abs(value))) + except struct.error as exc: + raise ValueError("Value supplied for unsigned int invalid: {}".format(value)) from exc + + +def encode_ulong(output: bytearray, value: int, with_constructor: bool = True, use_smallest: bool = True) -> None: + """ + + + + + :param bytearray output: The output buffer to write to. + :param int value: The ulong to encode. + :param bool with_constructor: Whether to include the constructor byte. + :param bool use_smallest: Whether to use the smallest possible encoding. + """ + value = int(value) + if value == 0: + output.extend(ConstructorBytes.ulong_0) + return + try: + if use_smallest and value <= 255: + output.extend(_construct(ConstructorBytes.ulong_small, with_constructor)) + output.extend(struct.pack(">B", abs(value))) + return + output.extend(_construct(ConstructorBytes.ulong_large, with_constructor)) + output.extend(struct.pack(">Q", abs(value))) + except struct.error as exc: + raise ValueError("Value supplied for unsigned long invalid: {}".format(value)) from exc + + +def encode_byte(output: bytearray, value: int, with_constructor: bool = True, **kwargs: Any) -> None: + """ + + + :param bytearray output: The output buffer to write to. + :param byte value: The byte to encode. + :param bool with_constructor: Whether to include the constructor byte. + """ + value = int(value) + try: + output.extend(_construct(ConstructorBytes.byte, with_constructor)) + output.extend(struct.pack(">b", value)) + except struct.error as exc: + raise ValueError("Byte value must be -128-127") from exc + + +def encode_short(output: bytearray, value: int, with_constructor: bool = True, **kwargs: Any) -> None: + """ + + + :param bytearray output: The output buffer to write to. + :param int value: The short to encode. + :param bool with_constructor: Whether to include the constructor byte. + """ + value = int(value) + try: + output.extend(_construct(ConstructorBytes.short, with_constructor)) + output.extend(struct.pack(">h", value)) + except struct.error as exc: + raise ValueError("Short value must be -32768-32767") from exc + + +def encode_int(output: bytearray, value: int, with_constructor: bool = True, use_smallest: bool = True) -> None: + """ + + + + :param bytearray output: The output buffer to write to. + :param int value: The int to encode. + :param bool with_constructor: Whether to include the constructor byte. + :param bool use_smallest: Whether to use the smallest possible encoding. + """ + value = int(value) + try: + if use_smallest and (-128 <= value <= 127): + output.extend(_construct(ConstructorBytes.int_small, with_constructor)) + output.extend(struct.pack(">b", value)) + return + output.extend(_construct(ConstructorBytes.int_large, with_constructor)) + output.extend(struct.pack(">i", value)) + except struct.error as exc: + raise ValueError("Value supplied for int invalid: {}".format(value)) from exc + + +def encode_long( + output: bytearray, value: Union[int, datetime], with_constructor: bool = True, use_smallest: bool = True +) -> None: + """ + + + + :param bytearray output: The output buffer to write to. + :param int or datetime value: The UUID to encode. + :param bool with_constructor: Whether to include the constructor byte. + :param bool use_smallest: Whether to use the smallest possible encoding. + """ + if isinstance(value, datetime): + value = int((calendar.timegm(value.utctimetuple()) * 1000) + (value.microsecond / 1000)) + try: + if use_smallest and (-128 <= value <= 127): + output.extend(_construct(ConstructorBytes.long_small, with_constructor)) + output.extend(struct.pack(">b", value)) + return + output.extend(_construct(ConstructorBytes.long_large, with_constructor)) + output.extend(struct.pack(">q", value)) + except struct.error as exc: + raise ValueError("Value supplied for long invalid: {}".format(value)) from exc + + +def encode_float(output: bytearray, value: float, with_constructor: bool = True, **kwargs: Any) -> None: + """ + + + :param bytearray output: The output buffer to write to. + :param float value: The value to encode. + :param bool with_constructor: Whether to include the constructor byte. + """ + value = float(value) + output.extend(_construct(ConstructorBytes.float, with_constructor)) + output.extend(struct.pack(">f", value)) + + +def encode_double(output: bytearray, value: float, with_constructor: bool = True, **kwargs: Any) -> None: + """ + + + :param bytearray output: The output buffer to write to. + :param float value: The double to encode. + :param bool with_constructor: Whether to include the constructor byte. + """ + value = float(value) + output.extend(_construct(ConstructorBytes.double, with_constructor)) + output.extend(struct.pack(">d", value)) + + +def encode_decimal128(output: bytearray, value: Decimal, with_constructor: bool = True, **kwargs: Any) -> None: + """ + + + :param bytearray output: The output buffer to write to. + :param decimal value: The decimal to encode. + :param bool with_constructor: Whether to include the constructor byte. + """ + # only dealing with decimal128 for now as those are the only ones supported for our current use case + # they can be added as needed + output.extend(_construct(ConstructorBytes.decimal128, with_constructor)) + sign, digits, exponent = value.as_tuple() + significand = int("".join(map(str, digits))) + + # Split significand into high and low parts + high = significand >> 64 + low = significand & ((1 << 64) - 1) + + if isinstance(exponent, int): + biased_exponent = exponent + _DECIMAl128bias + else: + raise ValueError(f"Invalid exponent type: {type(exponent)}") + + # Adjust high part based on biased exponent + if high >> 49 == 1: + high = (high & 0x7FFFFFFFFFFF) | ((biased_exponent & 0x3FFF) << 47) + else: + high |= biased_exponent << 49 + + # Add sign bit to high part + high |= sign << 63 + + output.extend(struct.pack(">Q", high)) + output.extend(struct.pack(">Q", low)) + + +def encode_timestamp( + output: bytearray, value: Union[int, datetime], with_constructor: bool = True, **kwargs: Any +) -> None: + """ + + + :param bytearray output: The output buffer to write to. + :param int or datetime value: The timestamp to encode. + :param bool with_constructor: Whether to include the constructor byte. + """ + if isinstance(value, datetime): + value = int((calendar.timegm(value.utctimetuple()) * 1000) + (value.microsecond / 1000)) + + value = int(value) + output.extend(_construct(ConstructorBytes.timestamp, with_constructor)) + output.extend(struct.pack(">q", value)) + + +def encode_uuid( + output: bytearray, value: Union[uuid.UUID, str, bytes], with_constructor: bool = True, **kwargs: Any +) -> None: + """ + + + :param bytearray output: The output buffer to write to. + :param uuid.UUID or str or bytes value: The UUID to encode. + :param bool with_constructor: Whether to include the constructor byte. + """ + if isinstance(value, str): + value = uuid.UUID(value).bytes + elif isinstance(value, uuid.UUID): + value = value.bytes + elif isinstance(value, bytes): + value = uuid.UUID(bytes=value).bytes + else: + raise TypeError("Invalid UUID type: {}".format(type(value))) + output.extend(_construct(ConstructorBytes.uuid, with_constructor)) + output.extend(value) + + +def encode_binary( + output: bytearray, value: Union[bytes, bytearray], with_constructor: bool = True, use_smallest: bool = True +) -> None: + """ + + + + :param bytearray output: The output buffer to write to. + :param bytes or bytearray value: The value to encode. + :param bool with_constructor: Whether to include the constructor in the output. + :param bool use_smallest: Whether to use the smallest possible encoding. + """ + length = len(value) + if use_smallest and length <= 255: + output.extend(_construct(ConstructorBytes.binary_small, with_constructor)) + output.extend(struct.pack(">B", length)) + output.extend(value) + return + try: + output.extend(_construct(ConstructorBytes.binary_large, with_constructor)) + output.extend(struct.pack(">L", length)) + output.extend(value) + except struct.error as exc: + raise ValueError("Binary data to long to encode") from exc + + +def encode_string( + output: bytearray, value: Union[bytes, str], with_constructor: bool = True, use_smallest: bool = True +) -> None: + """ + + + + :param bytearray output: The output buffer to write to. + :param str value: The string to encode. + :param bool with_constructor: Whether to include the constructor byte. + :param bool use_smallest: Whether to use the smallest possible encoding. + """ + if isinstance(value, str): + value = value.encode("utf-8") + length = len(value) + if use_smallest and length <= 255: + output.extend(_construct(ConstructorBytes.string_small, with_constructor)) + output.extend(struct.pack(">B", length)) + output.extend(value) + return + try: + output.extend(_construct(ConstructorBytes.string_large, with_constructor)) + output.extend(struct.pack(">L", length)) + output.extend(value) + except struct.error as exc: + raise ValueError("String value too long to encode.") from exc + + +def encode_symbol( + output: bytearray, value: Union[bytes, str], with_constructor: bool = True, use_smallest: bool = True +) -> None: + """ + + + + :param bytearray output: The output buffer to write to. + :param bytes or str value: The value to encode. + :param bool with_constructor: Whether to include the constructor byte. + :param bool use_smallest: Whether to use the smallest possible encoding. + """ + if isinstance(value, str): + value = value.encode("utf-8") + length = len(value) + if use_smallest and length <= 255: + output.extend(_construct(ConstructorBytes.symbol_small, with_constructor)) + output.extend(struct.pack(">B", length)) + output.extend(value) + return + try: + output.extend(_construct(ConstructorBytes.symbol_large, with_constructor)) + output.extend(struct.pack(">L", length)) + output.extend(value) + except struct.error as exc: + raise ValueError("Symbol value too long to encode.") from exc + + +def encode_list( + output: bytearray, value: Sequence[Any], with_constructor: bool = True, use_smallest: bool = True +) -> None: + """ + + + + + :param bytearray output: The output buffer to write to. + :param sequence value: The list to encode. + :param bool with_constructor: Whether to include the constructor in the output. + :param bool use_smallest: Whether to use the smallest possible encoding. + """ + count = len(cast(Sized, value)) + if use_smallest and count == 0: + output.extend(ConstructorBytes.list_0) + return + encoded_size = 0 + encoded_values = bytearray() + for item in value: + encode_value(encoded_values, item, with_constructor=True) + encoded_size += len(encoded_values) + if use_smallest and count <= 255 and encoded_size < 255: + output.extend(_construct(ConstructorBytes.list_small, with_constructor)) + output.extend(struct.pack(">B", encoded_size + 1)) + output.extend(struct.pack(">B", count)) + else: + try: + output.extend(_construct(ConstructorBytes.list_large, with_constructor)) + output.extend(struct.pack(">L", encoded_size + 4)) + output.extend(struct.pack(">L", count)) + except struct.error as exc: + raise ValueError("List is too large or too long to be encoded.") from exc + output.extend(encoded_values) + + +def encode_map( + output: bytearray, + value: Union[Dict[Any, Any], Iterable[Tuple[Any, Any]]], + with_constructor: bool = True, + use_smallest: bool = True, +) -> None: + """ + + + + :param bytearray output: The output buffer to write to. + :param dict value: The value to encode. + :param bool with_constructor: Whether to include the constructor in the output. + :param bool use_smallest: Whether to use the smallest possible encoding. + """ + count = len(cast(Sized, value)) * 2 + encoded_size = 0 + encoded_values = bytearray() + items: Iterable[Any] + if isinstance(value, dict): + items = value.items() + elif isinstance(value, Iterable): + items = value + + for key, data in items: + encode_value(encoded_values, key, with_constructor=True) + encode_value(encoded_values, data, with_constructor=True) + encoded_size = len(encoded_values) + if use_smallest and count <= 255 and encoded_size < 255: + output.extend(_construct(ConstructorBytes.map_small, with_constructor)) + output.extend(struct.pack(">B", encoded_size + 1)) + output.extend(struct.pack(">B", count)) + else: + try: + output.extend(_construct(ConstructorBytes.map_large, with_constructor)) + output.extend(struct.pack(">L", encoded_size + 4)) + output.extend(struct.pack(">L", count)) + except struct.error as exc: + raise ValueError("Map is too large or too long to be encoded.") from exc + output.extend(encoded_values) + + +def _check_element_type(item: Dict[str, Any], element_type: Any) -> Any: + if not element_type: + try: + return item["TYPE"] + except (KeyError, TypeError): + return type(item) + try: + if item["TYPE"] != element_type: + raise TypeError("All elements in an array must be the same type.") + except (KeyError, TypeError) as exc: + if not isinstance(item, element_type): + raise TypeError("All elements in an array must be the same type.") from exc + return element_type + + +def encode_array( + output: bytearray, value: Sequence[Any], with_constructor: bool = True, use_smallest: bool = True +) -> None: + """ + + + + :param bytearray output: The output buffer to write to. + :param sequence value: The array to encode. + :param bool with_constructor: Whether to include the constructor in the output. + :param bool use_smallest: Whether to use the smallest possible encoding. + """ + count = len(cast(Sized, value)) + encoded_size = 0 + encoded_values = bytearray() + first_item = True + element_type = None + for item in value: + element_type = _check_element_type(item, element_type) + encode_value(encoded_values, item, with_constructor=first_item, use_smallest=False) + first_item = False + if item is None: + encoded_size -= 1 + break + encoded_size += len(encoded_values) + if use_smallest and count <= 255 and encoded_size < 255: + output.extend(_construct(ConstructorBytes.array_small, with_constructor)) + output.extend(struct.pack(">B", encoded_size + 1)) + output.extend(struct.pack(">B", count)) + else: + try: + output.extend(_construct(ConstructorBytes.array_large, with_constructor)) + output.extend(struct.pack(">L", encoded_size + 4)) + output.extend(struct.pack(">L", count)) + except struct.error as exc: + raise ValueError("Array is too large or too long to be encoded.") from exc + output.extend(encoded_values) + + +def encode_described(output: bytearray, value: Tuple[Any, Any], _: bool = None, **kwargs: Any) -> None: # type: ignore + output.extend(ConstructorBytes.descriptor) + encode_value(output, value[0], **kwargs) + encode_value(output, value[1], **kwargs) + + +def encode_fields(value: Optional[Dict[bytes, Any]]) -> Dict[str, Any]: + """A mapping from field name to value. + + The fields type is a map where the keys are restricted to be of type symbol (this excludes the possibility + of a null key). There is no further restriction implied by the fields type on the allowed values for the + entries or the set of allowed keys. + + + + :param dict or None value: The field values to encode. + :return: The encoded field values. + :rtype: dict + """ + if not value: + return {TYPE: AMQPTypes.null, VALUE: None} + fields = {TYPE: AMQPTypes.map, VALUE: []} + for key, data in value.items(): + if isinstance(key, str): + key = key.encode("utf-8") # type: ignore + cast(List, fields[VALUE]).append(({TYPE: AMQPTypes.symbol, VALUE: key}, data)) + return fields + + +def encode_annotations(value: Optional[Dict[Union[str, bytes], Any]]) -> Dict[str, Any]: + """The annotations type is a map where the keys are restricted to be of type symbol or of type ulong. + + All ulong keys, and all symbolic keys except those beginning with "x-" are reserved. + On receiving an annotations map containing keys or values which it does not recognize, and for which the + key does not begin with the string 'x-opt-' an AMQP container MUST detach the link with the not-implemented + amqp-error. + + + + :param dict or None value: The annotations to encode. + :return: The encoded annotations. + :rtype: dict + """ + if not value: + return {TYPE: AMQPTypes.null, VALUE: None} + fields = {TYPE: AMQPTypes.map, VALUE: []} + for key, data in value.items(): + if isinstance(key, int): + field_key = {TYPE: AMQPTypes.ulong, VALUE: key} + else: + field_key = {TYPE: AMQPTypes.symbol, VALUE: key} + try: + cast(List, fields[VALUE]).append((field_key, {TYPE: data[TYPE], VALUE: data[VALUE]})) + except (KeyError, TypeError): + cast(List, fields[VALUE]).append((field_key, {TYPE: None, VALUE: data})) + return fields + + +def encode_application_properties( + value: Optional[Dict[Union[str, bytes], AQMPSimpleType]] +) -> Dict[Union[str, bytes], Any]: + """The application-properties section is a part of the bare message used for structured application data. + + + + + + Intermediaries may use the data within this structure for the purposes of filtering or routing. + The keys of this map are restricted to be of type string (which excludes the possibility of a null key) + and the values are restricted to be of simple types only, that is (excluding map, list, and array types). + + :param dict value: The application properties to encode. + :return: The encoded application properties. + :rtype: dict + """ + if not value: + return {TYPE: AMQPTypes.null, VALUE: None} + fields: Dict[Union[str, bytes], Any] = {TYPE: AMQPTypes.map, VALUE: cast(List, [])} + for key, data in value.items(): + cast(List, fields[VALUE]).append(({TYPE: AMQPTypes.string, VALUE: key}, data)) + return fields + + +def encode_message_id(value: Union[int, uuid.UUID, bytes, str]) -> Dict[str, Union[int, uuid.UUID, bytes, str]]: + """ + + + + + + :param any value: The message ID to encode. + :return: The encoded message ID. + :rtype: dict + """ + if isinstance(value, int): + return {TYPE: AMQPTypes.ulong, VALUE: value} + if isinstance(value, uuid.UUID): + return {TYPE: AMQPTypes.uuid, VALUE: value} + if isinstance(value, bytes): + return {TYPE: AMQPTypes.binary, VALUE: value} + if isinstance(value, str): + return {TYPE: AMQPTypes.string, VALUE: value} + raise TypeError("Unsupported Message ID type.") + + +def encode_node_properties(value: Optional[Dict[str, Any]]) -> Dict[str, Any]: + """Properties of a node. + + + + A symbol-keyed map containing properties of a node used when requesting creation or reporting + the creation of a dynamic node. The following common properties are defined:: + + - `lifetime-policy`: The lifetime of a dynamically generated node. Definitionally, the lifetime will + never be less than the lifetime of the link which caused its creation, however it is possible to extend + the lifetime of dynamically created node using a lifetime policy. The value of this entry MUST be of a type + which provides the lifetime-policy archetype. The following standard lifetime-policies are defined below: + delete-on-close, delete-on-no-links, delete-on-no-messages or delete-on-no-links-or-messages. + + - `supported-dist-modes`: The distribution modes that the node supports. The value of this entry MUST be one or + more symbols which are valid distribution-modes. That is, the value MUST be of the same type as would be valid + in a field defined with the following attributes: + type="symbol" multiple="true" requires="distribution-mode" + + :param dict value: The node properties. + :return: The encoded node properties. + :rtype: dict + """ + if not value: + return {TYPE: AMQPTypes.null, VALUE: None} + # TODO + fields = {TYPE: AMQPTypes.map, VALUE: []} + # fields[{TYPE: AMQPTypes.symbol, VALUE: b'lifetime-policy'}] = { + # TYPE: AMQPTypes.described, + # VALUE: ( + # {TYPE: AMQPTypes.ulong, VALUE: value['lifetime_policy']}, + # {TYPE: AMQPTypes.list, VALUE: []} + # ) + # } + # fields[{TYPE: AMQPTypes.symbol, VALUE: b'supported-dist-modes'}] = {} + return fields + + +def encode_filter_set(value: Optional[Dict[str, Any]]) -> Dict[str, Any]: + """A set of predicates to filter the Messages admitted onto the Link. + + + + A set of named filters. Every key in the map MUST be of type symbol, every value MUST be either null or of a + described type which provides the archetype filter. A filter acts as a function on a message which returns a + boolean result indicating whether the message can pass through that filter or not. A message will pass + through a filter-set if and only if it passes through each of the named filters. If the value for a given key is + null, this acts as if there were no such key present (i.e., all messages pass through the null filter). + + Filter types are a defined extension point. The filter types that a given source supports will be indicated + by the capabilities of the source. + + :param dict value: A set of named filters. + :return: A set of encoded named filters. + :rtype: dict + """ + if not value: + return {TYPE: AMQPTypes.null, VALUE: None} + fields = {TYPE: AMQPTypes.map, VALUE: cast(List, [])} + for name, data in value.items(): + described_filter: Dict[str, Union[Tuple[Dict[str, Any], Any], Optional[str]]] + if data is None: + described_filter = {TYPE: AMQPTypes.null, VALUE: None} + else: + if isinstance(name, str): + name = name.encode("utf-8") # type: ignore + if isinstance(data, (str, bytes)): + described_filter = data # type: ignore + # handle the situation when data is a tuple or list of length 2 + else: + try: + descriptor, filter_value = data + described_filter = { + TYPE: AMQPTypes.described, + VALUE: ({TYPE: AMQPTypes.symbol, VALUE: descriptor}, filter_value), + } + # if its not a type that is known, raise the error from the server + except (ValueError, TypeError): + described_filter = data + + cast(List, fields[VALUE]).append(({TYPE: AMQPTypes.symbol, VALUE: name}, described_filter)) + return fields + + +def encode_unknown(output: bytearray, value: Optional[object], **kwargs: Any) -> None: + """ + Dynamic encoding according to the type of `value`. + :param bytearray output: The output buffer. + :param any value: The value to encode. + """ + if value is None: + encode_null(output, **kwargs) + elif isinstance(value, bool): + encode_boolean(output, value, **kwargs) + elif isinstance(value, str): + encode_string(output, value, **kwargs) + elif isinstance(value, uuid.UUID): + encode_uuid(output, value, **kwargs) + elif isinstance(value, (bytearray, bytes)): + encode_binary(output, value, **kwargs) + elif isinstance(value, float): + encode_double(output, value, **kwargs) + elif isinstance(value, int): + encode_int(output, value, **kwargs) + elif isinstance(value, datetime): + encode_timestamp(output, value, **kwargs) + elif isinstance(value, list): + encode_list(output, value, **kwargs) + elif isinstance(value, tuple): + encode_described(output, cast(Tuple[Any, Any], value), **kwargs) + elif isinstance(value, dict): + encode_map(output, value, **kwargs) + elif isinstance(value, Decimal): + encode_decimal128(output, value, **kwargs) + else: + raise TypeError("Unable to encode unknown value: {}".format(value)) + + +_FIELD_DEFINITIONS: Dict[FieldDefinition, Callable[[Any], Any]] = { + FieldDefinition.fields: encode_fields, + FieldDefinition.annotations: encode_annotations, + FieldDefinition.message_id: encode_message_id, + FieldDefinition.app_properties: encode_application_properties, + FieldDefinition.node_properties: encode_node_properties, + FieldDefinition.filter_set: encode_filter_set, +} + +_ENCODE_MAP = { + None: encode_unknown, + AMQPTypes.null: encode_null, + AMQPTypes.boolean: encode_boolean, + AMQPTypes.ubyte: encode_ubyte, + AMQPTypes.byte: encode_byte, + AMQPTypes.ushort: encode_ushort, + AMQPTypes.short: encode_short, + AMQPTypes.uint: encode_uint, + AMQPTypes.int: encode_int, + AMQPTypes.ulong: encode_ulong, + AMQPTypes.long: encode_long, + AMQPTypes.float: encode_float, + AMQPTypes.double: encode_double, + AMQPTypes.timestamp: encode_timestamp, + AMQPTypes.uuid: encode_uuid, + AMQPTypes.binary: encode_binary, + AMQPTypes.string: encode_string, + AMQPTypes.symbol: encode_symbol, + AMQPTypes.list: encode_list, + AMQPTypes.map: encode_map, + AMQPTypes.array: encode_array, + AMQPTypes.described: encode_described, + AMQPTypes.decimal128: encode_decimal128, +} + + +def encode_value(output: bytearray, value: Any, **kwargs: Any) -> None: + try: + cast(Callable, _ENCODE_MAP[value[TYPE]])(output, value[VALUE], **kwargs) + except (KeyError, TypeError): + encode_unknown(output, value, **kwargs) + + +def describe_performative(performative: NamedTuple) -> Dict[str, Sequence[Collection[str]]]: + body: List[Dict[str, Any]] = [] + for index, value in enumerate(performative): + # TODO: fix mypy + field = performative._definition[index] # type: ignore # pylint: disable=protected-access + if value is None: + body.append({TYPE: AMQPTypes.null, VALUE: None}) + elif field is None: + continue + elif isinstance(field.type, FieldDefinition): + if field.multiple: + body.append( + { + TYPE: AMQPTypes.array, + VALUE: [_FIELD_DEFINITIONS[field.type](v) for v in value], # type: ignore + } + ) + else: + body.append(_FIELD_DEFINITIONS[field.type](value)) # type: ignore + elif isinstance(field.type, ObjDefinition): + body.append(describe_performative(value)) + else: + if field.multiple: + body.append( + { + TYPE: AMQPTypes.array, + VALUE: [{TYPE: field.type, VALUE: v} for v in value], + } + ) + else: + body.append({TYPE: field.type, VALUE: value}) + + return { + TYPE: AMQPTypes.described, + VALUE: ( + {TYPE: AMQPTypes.ulong, VALUE: performative._code}, # type: ignore # pylint: disable=protected-access + {TYPE: AMQPTypes.list, VALUE: body}, + ), + } + + +def encode_payload(output: bytearray, payload: Message) -> bytes: + + if payload[0]: # header + # TODO: Header and Properties encoding can be optimized to + # 1. not encoding trailing None fields + # Possible fix 1: + # header = payload[0] + # header = header[0:max(i for i, v in enumerate(header) if v is not None) + 1] + # Possible fix 2: + # itertools.dropwhile(lambda x: x is None, header[::-1]))[::-1] + # 2. encoding bool without constructor + # Possible fix 3: + # header = list(payload[0]) + # while header[-1] is None: + # del header[-1] + encode_value(output, describe_performative(payload[0])) + + if payload[2]: # message annotations + encode_value( + output, + { + TYPE: AMQPTypes.described, + VALUE: ( + {TYPE: AMQPTypes.ulong, VALUE: 0x00000072}, + encode_annotations(payload[2]), + ), + }, + ) + + if payload[3]: # properties + # TODO: Header and Properties encoding can be optimized to + # 1. not encoding trailing None fields + # 2. encoding bool without constructor + encode_value(output, describe_performative(payload[3])) + + if payload[4]: # application properties + encode_value( + output, + { + TYPE: AMQPTypes.described, + VALUE: ( + {TYPE: AMQPTypes.ulong, VALUE: 0x00000074}, + encode_application_properties(payload[4]), + ), + }, + ) + + if payload[5]: # data + for item_value in payload[5]: + encode_value( + output, + { + TYPE: AMQPTypes.described, + VALUE: ( + {TYPE: AMQPTypes.ulong, VALUE: 0x00000075}, + {TYPE: AMQPTypes.binary, VALUE: item_value}, + ), + }, + ) + + if payload[6]: # sequence + for item_value in payload[6]: + encode_value( + output, + { + TYPE: AMQPTypes.described, + VALUE: ( + {TYPE: AMQPTypes.ulong, VALUE: 0x00000076}, + {TYPE: None, VALUE: item_value}, + ), + }, + ) + + if payload[7]: # value + encode_value( + output, + { + TYPE: AMQPTypes.described, + VALUE: ( + {TYPE: AMQPTypes.ulong, VALUE: 0x00000077}, + {TYPE: None, VALUE: payload[7]}, + ), + }, + ) + + if payload[8]: # footer + encode_value( + output, + { + TYPE: AMQPTypes.described, + VALUE: ( + {TYPE: AMQPTypes.ulong, VALUE: 0x00000078}, + encode_annotations(payload[8]), + ), + }, + ) + + # TODO: + # currently the delivery annotations must be finally encoded instead of being encoded at the 2nd position + # otherwise the event hubs service would ignore the delivery annotations + # -- received message doesn't have it populated + # check with service team? + if payload[1]: # delivery annotations + encode_value( + output, + { + TYPE: AMQPTypes.described, + VALUE: ( + {TYPE: AMQPTypes.ulong, VALUE: 0x00000071}, + encode_annotations(payload[1]), + ), + }, + ) + + return output + + +def encode_frame(frame: Optional[NamedTuple], frame_type: bytes = _FRAME_TYPE) -> Tuple[bytes, Optional[bytes]]: + # TODO: allow passing type specific bytes manually, e.g. Empty Frame needs padding + if frame is None: + size = 8 + header = size.to_bytes(4, "big") + _FRAME_OFFSET + frame_type + return header, None + + frame_description = describe_performative(frame) + frame_data = bytearray() + encode_value(frame_data, frame_description) + if isinstance(frame, performatives.TransferFrame): + # casting from Optional[Buffer] since payload will not be None at this point + frame_data += cast(Buffer, frame.payload) + + size = len(frame_data) + 8 + header = size.to_bytes(4, "big") + _FRAME_OFFSET + frame_type + return header, frame_data diff --git a/src/azure/iot/hub/_pyamqp/_platform.py b/src/azure/iot/hub/_pyamqp/_platform.py new file mode 100644 index 0000000..9289fb1 --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/_platform.py @@ -0,0 +1,99 @@ +"""Platform compatibility.""" + +# pylint: skip-file + +from typing import Tuple, cast +import platform +import re +import struct +import sys + +# Jython does not have this attribute +try: + from socket import SOL_TCP +except ImportError: # pragma: no cover + from socket import IPPROTO_TCP as SOL_TCP # noqa + + +RE_NUM = re.compile(r"(\d+).+") + + +def _linux_version_to_tuple(s): + # type: (str) -> Tuple[int, int, int] + return cast(Tuple[int, int, int], tuple(map(_versionatom, s.split(".")[:3]))) + + +def _versionatom(s): + # type: (str) -> int + if s.isdigit(): + return int(s) + match = RE_NUM.match(s) + return int(match.groups()[0]) if match else 0 + + +# available socket options for TCP level +KNOWN_TCP_OPTS = { + "TCP_CORK", + "TCP_DEFER_ACCEPT", + "TCP_KEEPCNT", + "TCP_KEEPIDLE", + "TCP_KEEPINTVL", + "TCP_LINGER2", + "TCP_MAXSEG", + "TCP_NODELAY", + "TCP_QUICKACK", + "TCP_SYNCNT", + "TCP_USER_TIMEOUT", + "TCP_WINDOW_CLAMP", +} + +LINUX_VERSION = None +if sys.platform.startswith("linux"): + LINUX_VERSION = _linux_version_to_tuple(platform.release()) + if LINUX_VERSION < (2, 6, 37): + KNOWN_TCP_OPTS.remove("TCP_USER_TIMEOUT") + + # Windows Subsystem for Linux is an edge-case: the Python socket library + # returns most TCP_* enums, but they aren't actually supported + if platform.release().endswith("Microsoft"): + KNOWN_TCP_OPTS = {"TCP_NODELAY", "TCP_KEEPIDLE", "TCP_KEEPINTVL", "TCP_KEEPCNT"} + +elif sys.platform.startswith("darwin"): + KNOWN_TCP_OPTS.remove("TCP_USER_TIMEOUT") + +elif "bsd" in sys.platform: + KNOWN_TCP_OPTS.remove("TCP_USER_TIMEOUT") + +# According to MSDN Windows platforms support getsockopt(TCP_MAXSSEG) but not +# setsockopt(TCP_MAXSEG) on IPPROTO_TCP sockets. +elif sys.platform.startswith("win"): + KNOWN_TCP_OPTS = {"TCP_NODELAY"} + +elif sys.platform.startswith("cygwin"): + KNOWN_TCP_OPTS = {"TCP_NODELAY"} + +# illumos does not allow to set the TCP_MAXSEG socket option, +# even if the Oracle documentation says otherwise. +elif sys.platform.startswith("sunos"): + KNOWN_TCP_OPTS.remove("TCP_MAXSEG") + +# aix does not allow to set the TCP_MAXSEG +# or the TCP_USER_TIMEOUT socket options. +elif sys.platform.startswith("aix"): + KNOWN_TCP_OPTS.remove("TCP_MAXSEG") + KNOWN_TCP_OPTS.remove("TCP_USER_TIMEOUT") + +pack = struct.pack +pack_into = struct.pack_into +unpack = struct.unpack +unpack_from = struct.unpack_from + +__all__ = [ + "LINUX_VERSION", + "SOL_TCP", + "KNOWN_TCP_OPTS", + "pack", + "pack_into", + "unpack", + "unpack_from", +] diff --git a/src/azure/iot/hub/_pyamqp/_transport.py b/src/azure/iot/hub/_pyamqp/_transport.py new file mode 100644 index 0000000..bfb1f95 --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/_transport.py @@ -0,0 +1,772 @@ +# ------------------------------------------------------------------------- # pylint: disable=file-needs-copyright-header,useless-suppression +# This is a fork of the transport.py which was originally written by Barry Pederson and +# maintained by the Celery project: https://github.com/celery/py-amqp. +# +# Copyright (C) 2009 Barry Pederson +# +# The license text can also be found here: +# http://www.opensource.org/licenses/BSD-3-Clause +# +# License +# ======= +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS +# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ------------------------------------------------------------------------- + +import errno +import re +import socket +import ssl +import struct +from ssl import SSLError +from contextlib import contextmanager +from io import BytesIO +import logging +from threading import Lock + +from ._platform import KNOWN_TCP_OPTS, SOL_TCP +from ._encode import encode_frame +from ._decode import decode_frame, decode_empty_frame +from .constants import ( + TLS_HEADER_FRAME, + WEBSOCKET_PORT, + TransportType, + AMQP_WS_SUBPROTOCOL, + WS_TIMEOUT_INTERVAL, + SOCKET_TIMEOUT, + CONNECT_TIMEOUT, +) +from .error import AuthenticationException, ErrorCondition + + +try: + import fcntl +except ImportError: # pragma: no cover + fcntl = None # type: ignore # noqa + + +def set_cloexec(fd, cloexec): # noqa # pylint: disable=inconsistent-return-statements + """Set flag to close fd after exec. + :param int fd: File descriptor. + :param bool cloexec: Close on exec. + :return: Returns 0 on success, raises an OSError exception on failure. + :rtype: int + """ + if fcntl is None: + return + try: + FD_CLOEXEC = fcntl.FD_CLOEXEC + except AttributeError: + raise NotImplementedError( + "close-on-exec flag not supported on this platform", + ) from None + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + if cloexec: + flags |= FD_CLOEXEC + else: + flags &= ~FD_CLOEXEC + return fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + +_LOGGER = logging.getLogger(__name__) +_UNAVAIL = {errno.EAGAIN, errno.EINTR, errno.ENOENT, errno.EWOULDBLOCK} + +AMQP_PORT = 5672 +AMQPS_PORT = 5671 +AMQP_FRAME = memoryview(b"AMQP") +EMPTY_BUFFER = b"" +SIGNED_INT_MAX = 0x7FFFFFFF + +# Match things like: [fe80::1]:5432, from RFC 2732 +IPV6_LITERAL = re.compile(r"\[([\.0-9a-f:]+)\](?::(\d+))?") + +DEFAULT_SOCKET_SETTINGS = { + "TCP_NODELAY": 1, + "TCP_USER_TIMEOUT": 1000, + "TCP_KEEPIDLE": 60, + "TCP_KEEPINTVL": 10, + "TCP_KEEPCNT": 9, +} + + +def get_errno(exc): + """Get exception errno (if set). + + Notes: + :exc:`socket.error` and :exc:`IOError` first got + the ``.errno`` attribute in Py2.7. + + :param Exception exc: Exception instance. + :return: Exception errno. + :rtype: int + """ + try: + return exc.errno + except AttributeError: + try: + # e.args = (errno, reason) + if isinstance(exc.args, tuple) and len(exc.args) == 2: + return exc.args[0] + except AttributeError: + pass + return 0 + + +# TODO: fails when host = hostname:port/path. fix +def to_host_port(host, port=AMQP_PORT): + """Convert hostname:port string to host, port tuple. + :param str host: Hostname. + :param int port: Port number. + :return: Tuple of (host, port). + :rtype: tuple + """ + m = IPV6_LITERAL.match(host) + if m: + host = m.group(1) + if m.group(2): + port = int(m.group(2)) + else: + if ":" in host: + host, port = host.rsplit(":", 1) + port = int(port) + return host, port + + +class UnexpectedFrame(Exception): + pass + + +class _AbstractTransport(object): # pylint: disable=too-many-instance-attributes + """Common superclass for TCP and SSL transports.""" + + def __init__( + self, + host, + *, + port=AMQP_PORT, + connect_timeout=CONNECT_TIMEOUT, + socket_timeout=SOCKET_TIMEOUT, + socket_settings=None, + raise_on_initial_eintr=True, + use_tls: bool = True, + **kwargs, + ): + self._quick_recv = None + self.connected = False + self.sock = None + self.raise_on_initial_eintr = raise_on_initial_eintr + self._read_buffer = BytesIO() + self.host, self.port = to_host_port(host, port) + self.network_trace_params = kwargs.get("network_trace_params") + + self.connect_timeout = connect_timeout + self.socket_timeout = socket_timeout + self.socket_settings = socket_settings + self.socket_lock = Lock() + + self._use_tls = use_tls + + def connect(self): + try: + # are we already connected? + if self.connected: + return + self.sock = socket.create_connection((self.host, self.port), self.connect_timeout) + try: + set_cloexec(self.sock, True) + except NotImplementedError: + pass + self._init_socket( + self.socket_settings, + self.socket_timeout, + ) + # we've sent the banner; signal connect + # EINTR, EAGAIN, EWOULDBLOCK would signal that the banner + # has _not_ been sent + self.connected = True + except (OSError, IOError, SSLError) as e: + _LOGGER.info("Transport connection failed: %r", e, extra=self.network_trace_params) + # if not fully connected, close socket, and reraise error + if self.sock and not self.connected: + self.sock.close() + self.sock = None + raise + + @contextmanager + def block(self): + bocking_timeout = None + sock = self.sock + prev = sock.gettimeout() + if prev != bocking_timeout: + sock.settimeout(bocking_timeout) + try: + yield self.sock + except SSLError as exc: + if "timed out" in str(exc): + # http://bugs.python.org/issue10272 + raise socket.timeout() + if "The operation did not complete" in str(exc): + # Non-blocking SSL sockets can throw SSLError + raise socket.timeout() + raise + except socket.error as exc: + if get_errno(exc) == errno.EWOULDBLOCK: + raise socket.timeout() + raise + finally: + if bocking_timeout != prev: + sock.settimeout(prev) + + @contextmanager + def non_blocking(self): + non_bocking_timeout = 0.0 + sock = self.sock + prev = sock.gettimeout() + if prev != non_bocking_timeout: + sock.settimeout(non_bocking_timeout) + try: + yield self.sock + except SSLError as exc: + if "timed out" in str(exc): + # http://bugs.python.org/issue10272 + raise socket.timeout() + if "The operation did not complete" in str(exc): + # Non-blocking SSL sockets can throw SSLError + raise socket.timeout() + raise + except socket.error as exc: + if get_errno(exc) == errno.EWOULDBLOCK: + raise socket.timeout() + raise + finally: + if non_bocking_timeout != prev: + sock.settimeout(prev) + + + def _init_socket(self, socket_settings, socket_timeout): + self.sock.settimeout(None) # set socket back to blocking mode + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + self._set_socket_options(socket_settings) + + # set socket timeouts + # for timeout, interval in ((socket.SO_SNDTIMEO, write_timeout), + # (socket.SO_RCVTIMEO, read_timeout)): + # if interval is not None: + # sec = int(interval) + # usec = int((interval - sec) * 1000000) + # self.sock.setsockopt( + # socket.SOL_SOCKET, timeout, + # pack('ll', sec, usec), + # ) + self._setup_transport() + self.sock.settimeout(socket_timeout) # set socket to blocking with timeout + + def _get_tcp_socket_defaults(self, sock): + tcp_opts = {} + for opt in KNOWN_TCP_OPTS: + enum = None + if opt == "TCP_USER_TIMEOUT": + try: + from socket import TCP_USER_TIMEOUT as enum + except ImportError: + # should be in Python 3.6+ on Linux. + enum = 18 + elif hasattr(socket, opt): + enum = getattr(socket, opt) + + if enum: + if opt in DEFAULT_SOCKET_SETTINGS: + tcp_opts[enum] = DEFAULT_SOCKET_SETTINGS[opt] + elif hasattr(socket, opt): + tcp_opts[enum] = sock.getsockopt(SOL_TCP, getattr(socket, opt)) + return tcp_opts + + def _set_socket_options(self, socket_settings): + tcp_opts = self._get_tcp_socket_defaults(self.sock) + if socket_settings: + tcp_opts.update(socket_settings) + for opt, val in tcp_opts.items(): + self.sock.setsockopt(SOL_TCP, opt, val) + + def _read(self, n, initial=False, buffer=None, _errnos=None): + """Read exactly n bytes from the peer. + :param int n: The number of bytes to read. + :param bool initial: Whether this is the initial read of a frame. + :param bytearray or None buffer: A buffer to read into. + :param list or None _errnos: A list of socket errors to catch. + :return: None + :rtype: None + """ + raise NotImplementedError("Must be overriden in subclass") + + def _setup_transport(self): + """Do any additional initialization of the class.""" + + def _shutdown_transport(self): + """Do any preliminary work in shutting down the connection.""" + + def _write(self, s): + """Completely write a string to the peer. + :param str s: The string to write. + :return: None + :rtype: None + """ + raise NotImplementedError("Must be overriden in subclass") + + def close(self): + with self.socket_lock: + if self.sock is not None: + self._shutdown_transport() + # Call shutdown first to make sure that pending messages + # reach the AMQP broker if the program exits after + # calling this method. + try: + self.sock.shutdown(socket.SHUT_RDWR) + except Exception as exc: # pylint: disable=broad-except + # TODO: shutdown could raise OSError, Transport endpoint is not connected if the endpoint is already + # disconnected. can we safely ignore the errors since the close operation is initiated by us. + _LOGGER.debug( + "Transport endpoint is already disconnected: %r", exc, extra=self.network_trace_params + ) + self.sock.close() + self.sock = None + self.connected = False + + def read(self, verify_frame_type=0): + with self.socket_lock: + read = self._read + read_frame_buffer = BytesIO() + try: + frame_header = memoryview(bytearray(8)) + read_frame_buffer.write(read(8, buffer=frame_header, initial=True)) + + channel = struct.unpack(">H", frame_header[6:])[0] + size = frame_header[0:4] + if size == AMQP_FRAME: # Empty frame or AMQP header negotiation TODO + return frame_header, channel, None + size = struct.unpack(">I", size)[0] + offset = frame_header[4] + frame_type = frame_header[5] + if verify_frame_type is not None and frame_type != verify_frame_type: + _LOGGER.debug( + "Received invalid frame type: %r, expected: %r", + frame_type, + verify_frame_type, + extra=self.network_trace_params, + ) + raise ValueError(f"Received invalid frame type: {frame_type}, expected: {verify_frame_type}") + + # >I is an unsigned int, but the argument to sock.recv is signed, + # so we know the size can be at most 2 * SIGNED_INT_MAX + payload_size = size - len(frame_header) + payload = memoryview(bytearray(payload_size)) + if size > SIGNED_INT_MAX: + read_frame_buffer.write(read(SIGNED_INT_MAX, buffer=payload)) + read_frame_buffer.write(read(size - SIGNED_INT_MAX, buffer=payload[SIGNED_INT_MAX:])) + else: + read_frame_buffer.write(read(payload_size, buffer=payload)) + except (socket.timeout, TimeoutError): + read_frame_buffer.write(self._read_buffer.getvalue()) + self._read_buffer = read_frame_buffer + self._read_buffer.seek(0) + raise + except (OSError, IOError, SSLError, socket.error) as exc: + # Don't disconnect for ssl read time outs + # http://bugs.python.org/issue10272 + if isinstance(exc, SSLError) and "timed out" in str(exc): + raise socket.timeout() + if get_errno(exc) not in _UNAVAIL: + self.connected = False + _LOGGER.debug("Transport read failed: %r", exc, extra=self.network_trace_params) + raise + offset -= 2 + return frame_header, channel, payload[offset:] + + def write(self, s): + with self.socket_lock: + try: + self._write(s) + except socket.timeout: + raise + except (OSError, IOError, socket.error) as exc: + _LOGGER.debug("Transport write failed: %r", exc, extra=self.network_trace_params) + if get_errno(exc) not in _UNAVAIL: + self.connected = False + raise + + def receive_frame(self, **kwargs): + try: + header, channel, payload = self.read(**kwargs) + if not payload: + decoded = decode_empty_frame(header) + else: + decoded = decode_frame(payload) + return channel, decoded + except (socket.timeout, TimeoutError): + return None, None + + def send_frame(self, channel, frame, **kwargs): + header, performative = encode_frame(frame, **kwargs) + if performative is None: + data = header + else: + encoded_channel = struct.pack(">H", channel) + data = header + encoded_channel + performative + self.write(data) + + def negotiate(self): + pass + + +class SSLTransport(_AbstractTransport): + """Transport that works over SSL.""" + + def __init__(self, host, *, port=AMQPS_PORT, socket_timeout=None, ssl_opts=None, **kwargs): + self.sslopts = ssl_opts if isinstance(ssl_opts, dict) else {} + self._custom_endpoint = kwargs.get("custom_endpoint") + self._custom_port = kwargs.get("custom_port") + self.sslopts["server_hostname"] = self._custom_endpoint or host + self._read_buffer = BytesIO() + super(SSLTransport, self).__init__( + self._custom_endpoint or host, port=self._custom_port or port, socket_timeout=socket_timeout, **kwargs + ) + + def _setup_transport(self): + """Wrap the socket in an SSL object.""" + if self._use_tls: + self.sock = self._wrap_socket(self.sock, **self.sslopts) + self._quick_recv = self.sock.recv + + def _wrap_socket(self, sock, **sslopts): + if "context" in sslopts: + context = sslopts.pop("context") + return context.wrap_socket(sock, **sslopts) + return self._wrap_socket_sni(sock, **sslopts) + + def _wrap_socket_sni( + self, + sock, + keyfile=None, + certfile=None, + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=None, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, + server_hostname=None, + ciphers=None, + ssl_version=None, + ): + """Socket wrap with SNI headers. + + Default `ssl.wrap_socket` method augmented with support for + setting the server_hostname field required for SNI hostname header + + :param socket.socket sock: socket to wrap + :param str or None keyfile: key file path + :param str or None certfile: cert file path + :param int cert_reqs: cert requirements + :param str or None ca_certs: ca certs file path + :param bool do_handshake_on_connect: do handshake on connect + :param bool suppress_ragged_eofs: suppress ragged EOFs + :param str or None server_hostname: server hostname + :param str or None ciphers: ciphers to use + :param int or None ssl_version: ssl version to use + :return: wrapped socket + :rtype: SSLSocket + """ + # Setup the right SSL version; default to optimal versions across + # ssl implementations + if ssl_version is None: + ssl_version = ssl.PROTOCOL_TLS_CLIENT + purpose = ssl.Purpose.SERVER_AUTH + + opts = { + "sock": sock, + "do_handshake_on_connect": do_handshake_on_connect, + "suppress_ragged_eofs": suppress_ragged_eofs, + "server_hostname": server_hostname, + } + + context = ssl.SSLContext(ssl_version) + + if ca_certs is not None: + try: + context.load_verify_locations(ca_certs) + except FileNotFoundError as exc: + exc.filename = {"ca_certs": ca_certs} + raise exc from None + elif context.verify_mode != ssl.CERT_NONE: + # load the default system root CA certs. + context.load_default_certs(purpose=purpose) + + if certfile is not None: + context.load_cert_chain(certfile, keyfile) + + if ciphers is not None: + context.set_ciphers(ciphers) + + if cert_reqs == ssl.CERT_NONE and server_hostname is None: + context.check_hostname = False + context.verify_mode = cert_reqs + + sock = context.wrap_socket(**opts) + return sock + + def _shutdown_transport(self): + """Unwrap a SSL socket, so we can call shutdown().""" + if self.sock is not None: + try: + if self._use_tls: + self.sock = self.sock.unwrap() + except OSError: + pass + + def _read( + self, + n, + initial=False, + buffer=None, + _errnos=(errno.ENOENT, errno.EAGAIN, errno.EINTR), + ): + # According to SSL_read(3), it can at most return 16kb of data. + # Thus, we use an internal read buffer like TCPTransport._read + # to get the exact number of bytes wanted. + length = 0 + view = buffer or memoryview(bytearray(n)) + nbytes = self._read_buffer.readinto(view) + toread = n - nbytes + length += nbytes + try: + while toread: + try: + nbytes = self.sock.recv_into(view[length:]) + except socket.error as exc: + # ssl.sock.read may cause a SSLerror without errno + # http://bugs.python.org/issue10272 + if isinstance(exc, SSLError) and "timed out" in str(exc): + raise socket.timeout() + # ssl.sock.read may cause ENOENT if the + # operation couldn't be performed (Issue celery#1414). + if exc.errno in _errnos: + if initial and self.raise_on_initial_eintr: + raise socket.timeout() + continue + raise + if not nbytes: + raise IOError("Server unexpectedly closed connection") + + length += nbytes + toread -= nbytes + except: # noqa + self._read_buffer = BytesIO(view[:length]) + raise + return view + + def _write(self, s): + """Write a string out to the SSL socket fully. + :param str s: The string to write. + """ + try: + write = self.sock.send + except AttributeError: + raise IOError("Socket has already been closed.") from None + + while s: + try: + n = write(s) + except ValueError: + # AG: sock._sslobj might become null in the meantime if the + # remote connection has hung up. + # In python 3.4, a ValueError is raised is self._sslobj is + # None. + n = 0 + if not n: + raise IOError("Socket closed.") + s = s[n:] + + def negotiate(self): + with self.block(): + self.write(TLS_HEADER_FRAME) + _, returned_header = self.receive_frame(verify_frame_type=None) + if returned_header[1] == TLS_HEADER_FRAME: + raise ValueError( + f"""Mismatching TLS header protocol. Expected: {TLS_HEADER_FRAME!r},""" + """received: {returned_header[1]!r}""" + ) + + +def Transport(host, transport_type, socket_timeout=None, ssl_opts=True, **kwargs): + """Create transport. + + Given a few parameters from the Connection constructor, + select and create a subclass of _AbstractTransport. + + :param str host: The host to connect to. + :param ~pyamqp.TransportType transport_type: The transport type. + :param int or None socket_timeout: The socket timeout. + :param bool ssl_opts: SSL options. + :return: Transport instance. + :rtype: ~pyamqp.transport._AbstractTransport + """ + if transport_type == TransportType.AmqpOverWebsocket: + transport = WebSocketTransport + else: + transport = SSLTransport + return transport(host, socket_timeout=socket_timeout, ssl_opts=ssl_opts, **kwargs) + + +class WebSocketTransport(_AbstractTransport): + def __init__( + self, + host, + *, + port=WEBSOCKET_PORT, + socket_timeout=None, + ssl_opts=None, + **kwargs, + ): + self.sslopts = ssl_opts if isinstance(ssl_opts, dict) else {} + self.socket_timeout = socket_timeout or WS_TIMEOUT_INTERVAL + self._host = host + self._custom_endpoint = kwargs.get("custom_endpoint") + super().__init__(host, port=port, socket_timeout=socket_timeout, **kwargs) + self.sock = None + self._http_proxy = kwargs.get("http_proxy", None) + + def connect(self): + http_proxy_host, http_proxy_port, http_proxy_auth = None, None, None + if self._http_proxy: + http_proxy_host = self._http_proxy["proxy_hostname"] + http_proxy_port = self._http_proxy["proxy_port"] + username = self._http_proxy.get("username", None) + password = self._http_proxy.get("password", None) + if username or password: + http_proxy_auth = (username, password) + try: + from websocket import ( + create_connection, + WebSocketAddressException, + WebSocketTimeoutException, + WebSocketConnectionClosedException, + ) + except ImportError: + raise ImportError("Please install websocket-client library to use sync websocket transport.") from None + try: + self.sock = create_connection( + url=( + "wss://{}".format(self._custom_endpoint or self._host) + if self._use_tls + else "ws://{}".format(self._custom_endpoint or self._host) + ), + subprotocols=[AMQP_WS_SUBPROTOCOL], + timeout=self.socket_timeout, # timeout for read/write operations + skip_utf8_validation=True, + sslopt=self.sslopts if self._use_tls else None, + http_proxy_host=http_proxy_host, + http_proxy_port=http_proxy_port, + http_proxy_auth=http_proxy_auth, + ) + except WebSocketAddressException as exc: + raise AuthenticationException( + ErrorCondition.ClientError, + description="Failed to authenticate the connection due to exception: " + str(exc), + error=exc, + ) from exc + # TODO: resolve pylance error when type: ignore is removed below, issue #22051 + except (WebSocketTimeoutException, SSLError, WebSocketConnectionClosedException) as exc: # type: ignore + self.close() + raise ConnectionError("Websocket failed to establish connection: %r" % exc) from exc + except (OSError, IOError, SSLError) as e: + _LOGGER.info("Websocket connection failed: %r", e, extra=self.network_trace_params) + self.close() + raise + + def _read(self, n, initial=False, buffer=None, _errnos=None): + """Read exactly n bytes from the peer. + + :param int n: The number of bytes to read. + :param bool initial: Whether this is the first read in a sequence. + :param bytearray or None buffer: The buffer to read into. + :param list or None _errnos: A list of errno values to catch and retry on. + :return: The data read. + :rtype: bytearray + """ + from websocket import WebSocketTimeoutException, WebSocketConnectionClosedException + + try: + length = 0 + view = buffer or memoryview(bytearray(n)) + nbytes = self._read_buffer.readinto(view) + length += nbytes + n -= nbytes + try: + while n: + data = self.sock.recv() + if len(data) <= n: + view[length : length + len(data)] = data + n -= len(data) + length += len(data) + else: + view[length : length + n] = data[0:n] + self._read_buffer = BytesIO(data[n:]) + n = 0 + return view + except AttributeError: + raise IOError("Websocket connection has already been closed.") from None + except WebSocketTimeoutException as wte: + raise TimeoutError("Websocket receive timed out (%s)" % wte) from wte + except (WebSocketConnectionClosedException, SSLError) as e: + raise ConnectionError("Websocket disconnected: %r" % e) from e + except: + self._read_buffer = BytesIO(view[:length]) + raise + + def close(self): + with self.socket_lock: + if self.sock: + self._shutdown_transport() + self.sock = None + + def _shutdown_transport(self): + # TODO Sync and Async close functions named differently + """Do any preliminary work in shutting down the connection.""" + if self.sock: + self.sock.close() + + def _write(self, s): + """Completely write a string to the peer. + ABNF, OPCODE_BINARY = 0x2 + See http://tools.ietf.org/html/rfc5234 + http://tools.ietf.org/html/rfc6455#section-5.2 + + :param str s: The string to write. + """ + from websocket import WebSocketConnectionClosedException, WebSocketTimeoutException + + try: + self.sock.send_binary(s) + except AttributeError: + raise IOError("Websocket connection has already been closed.") from None + except WebSocketTimeoutException as e: + raise socket.timeout("Websocket send timed out (%s)" % e) from e + except (WebSocketConnectionClosedException, SSLError) as e: + raise ConnectionError("Websocket disconnected: %r" % e) from e diff --git a/src/azure/iot/hub/_pyamqp/authentication.py b/src/azure/iot/hub/_pyamqp/authentication.py new file mode 100644 index 0000000..e846681 --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/authentication.py @@ -0,0 +1,184 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +import time +from functools import partial + +from typing import Any, Callable, NamedTuple, Optional, Tuple, Union +from .sasl import SASLAnonymousCredential, SASLPlainCredential +from .utils import generate_sas_token + +from .constants import ( + AUTH_DEFAULT_EXPIRATION_SECONDS, + TOKEN_TYPE_JWT, + TOKEN_TYPE_SASTOKEN, + AUTH_TYPE_CBS, + AUTH_TYPE_SASL_PLAIN, +) + + +class AccessToken(NamedTuple): + token: Union[str, bytes] + expires_on: int + + +def _generate_sas_access_token( + auth_uri: str, sas_name: str, sas_key: str, expiry_in: float = AUTH_DEFAULT_EXPIRATION_SECONDS +) -> AccessToken: + expires_on = int(time.time() + expiry_in) + token = generate_sas_token(auth_uri, sas_name, sas_key, expires_on) + return AccessToken(token, expires_on) + + +class SASLPlainAuth: + # TODO: + # 1. naming decision, suffix with Auth vs Credential + auth_type = AUTH_TYPE_SASL_PLAIN + + def __init__(self, authcid: str, passwd: str, authzid: Optional[str] = None): + self.sasl = SASLPlainCredential(authcid, passwd, authzid) + + +class _CBSAuth: + # TODO: + # 1. naming decision, suffix with Auth vs Credential + auth_type = AUTH_TYPE_CBS + + def __init__( # pylint: disable=unused-argument + self, + uri: str, + audience: str, + token_type: Union[str, bytes], + get_token: Callable[[], AccessToken], + *, + expires_in: Optional[float] = AUTH_DEFAULT_EXPIRATION_SECONDS, + expires_on: Optional[float] = None, + **kwargs: Any + ): + """ + CBS authentication using JWT tokens. + + :param uri: The AMQP endpoint URI. This must be provided as + a decoded string. + :type uri: str + :param audience: The token audience field. For SAS tokens + this is usually the URI. + :type audience: str + :param get_token: The callback function used for getting and refreshing + tokens. It should return a valid jwt token each time it is called. + :type get_token: callable object + :param token_type: The type field of the token request. + Default value is `"jwt"`. + :type token_type: str + + """ + self.sasl = SASLAnonymousCredential() + self.uri = uri + self.audience = audience + self.token_type = token_type + self.get_token = get_token + self.expires_in = expires_in + self.expires_on = expires_on + + @staticmethod + def _set_expiry(expires_in: Optional[float] = None, expires_on: Optional[float] = None) -> Tuple[float, float]: + if not expires_on and not expires_in: + raise ValueError("Must specify either 'expires_on' or 'expires_in'.") + + expires_in_interval: float = 0 if expires_in is None else expires_in + expires_on_time: float = 0 if expires_on is None else expires_on + + if not expires_on and expires_in: + expires_on_time = time.time() + expires_in + elif expires_on and not expires_in: + expires_in_interval = expires_on - time.time() + if expires_in_interval < 1: + raise ValueError("Token has already expired.") + + return expires_in_interval, expires_on_time + + +class JWTTokenAuth(_CBSAuth): + # TODO: + # 1. naming decision, suffix with Auth vs Credential + def __init__( + self, + uri: str, + audience: str, + get_token: Callable[..., AccessToken], + *, + token_type: Union[str, bytes] = TOKEN_TYPE_JWT, + **kwargs: Any + ): + """ + CBS authentication using JWT tokens. + + :param uri: The AMQP endpoint URI. This must be provided as + a decoded string. + :type uri: str + :param audience: The token audience field. For SAS tokens + this is usually the URI. + :type audience: str + :param get_token: The callback function used for getting and refreshing + tokens. It should return a valid jwt token each time it is called. + :type get_token: callable object + :param token_type: The type field of the token request. + Default value is `"jwt"`. + :type token_type: str + + """ + super(JWTTokenAuth, self).__init__(uri, audience, token_type, get_token) + self.get_token = get_token + + +class SASTokenAuth(_CBSAuth): + # TODO: + # 1. naming decision, suffix with Auth vs Credential + def __init__( + self, + uri: str, + audience: str, + username: str, + password: str, + *, + expires_in: Optional[float] = AUTH_DEFAULT_EXPIRATION_SECONDS, + expires_on: Optional[float] = None, + token_type: Union[str, bytes] = TOKEN_TYPE_SASTOKEN, + **kwargs: Any + ): + """ + CBS authentication using SAS tokens. + + :param uri: The AMQP endpoint URI. This must be provided as + a decoded string. + :type uri: str + :param audience: The token audience field. For SAS tokens + this is usually the URI. + :type audience: str + :param username: The SAS token username, also referred to as the key + name or policy name. This can optionally be encoded into the URI. + :type username: str + :param password: The SAS token password, also referred to as the key. + This can optionally be encoded into the URI. + :type password: str + :param expires_in: The total remaining seconds until the token + expires. + :type expires_in: int + :param expires_on: The timestamp at which the SAS token will expire + formatted as seconds since epoch. + :type expires_on: float + :param token_type: The type field of the token request. + Default value is `"servicebus.windows.net:sastoken"`. + :type token_type: str + + """ + self.username = username + self.password = password + expires_in, expires_on = self._set_expiry(expires_in, expires_on) + self.get_token = partial(_generate_sas_access_token, uri, username, password, expires_in) + super(SASTokenAuth, self).__init__( + uri, audience, token_type, self.get_token, expires_in=expires_in, expires_on=expires_on + ) diff --git a/src/azure/iot/hub/_pyamqp/cbs.py b/src/azure/iot/hub/_pyamqp/cbs.py new file mode 100644 index 0000000..4b8c038 --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/cbs.py @@ -0,0 +1,283 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +import logging +from uuid import uuid4 +from datetime import datetime +from typing import Any, Optional, Tuple, Union + +from .utils import utc_now, utc_from_timestamp +from .management_link import ManagementLink +from .message import Message, Properties +from .error import ( + AuthenticationException, + ErrorCondition, + TokenAuthFailure, + TokenExpired, +) +from .constants import ( + CbsState, + CbsAuthState, + CBS_PUT_TOKEN, + CBS_EXPIRATION, + CBS_NAME, + CBS_TYPE, + CBS_OPERATION, + ManagementExecuteOperationResult, + ManagementOpenResult, +) + +from .session import Session +from .authentication import JWTTokenAuth, SASTokenAuth + +_LOGGER = logging.getLogger(__name__) + + +def check_expiration_and_refresh_status(expires_on: int, refresh_window: int) -> Tuple[bool, bool]: + seconds_since_epoc = int(utc_now().timestamp()) + is_expired = seconds_since_epoc >= expires_on + is_refresh_required = (expires_on - seconds_since_epoc) <= refresh_window + return is_expired, is_refresh_required + + +def check_put_timeout_status(auth_timeout: float, token_put_time: int) -> bool: + if auth_timeout > 0: + return (int(utc_now().timestamp()) - token_put_time) >= auth_timeout + return False + + +class CBSAuthenticator: # pylint:disable=too-many-instance-attributes, disable=unused-argument + def __init__( + self, session: Session, auth: Union[JWTTokenAuth, SASTokenAuth], *, auth_timeout: float, **kwargs: Any + ) -> None: + self._session = session + self._connection = self._session._connection + self._mgmt_link: ManagementLink = self._session.create_request_response_link_pair( + endpoint="$cbs", + on_amqp_management_open_complete=self._on_amqp_management_open_complete, + on_amqp_management_error=self._on_amqp_management_error, + status_code_field=b"status-code", + status_description_field=b"status-description", + ) + + # FIXME: probably can remove the None check as it should fail callable too + if not auth.get_token or not callable(auth.get_token): # type: ignore + raise ValueError("get_token must be a callable object.") + + self._auth = auth + self._encoding = "UTF-8" + self._auth_timeout: float = auth_timeout + self._token_put_time: Optional[int] = None + self._expires_on: Optional[int] = None + self._token: Optional[str] = None + self._refresh_window: Optional[int] = None + self._network_trace_params = { + "amqpConnection": self._session._connection._container_id, + "amqpSession": self._session.name, + "amqpLink": "", + } + + self._token_status_code: Optional[int] = None + self._token_status_description: Optional[str] = None + + self.state = CbsState.CLOSED + self.auth_state = CbsAuthState.IDLE + + def _put_token(self, token: str, token_type: str, audience: str, expires_on: Optional[datetime] = None) -> None: + message = Message( # type: ignore # TODO: missing positional args header, etc. + value=token, + properties=Properties(message_id=uuid4()), # type: ignore + application_properties={ + CBS_NAME: audience, + CBS_OPERATION: CBS_PUT_TOKEN, + CBS_TYPE: token_type, + CBS_EXPIRATION: expires_on, + }, + ) + self._mgmt_link.execute_operation( + message, + self._on_execute_operation_complete, + timeout=self._auth_timeout, + operation=CBS_PUT_TOKEN, + type=token_type, + ) + + def _on_amqp_management_open_complete(self, management_open_result: ManagementOpenResult) -> None: + if self.state in (CbsState.CLOSED, CbsState.ERROR): + _LOGGER.debug( + "CSB with status: %r encounters unexpected AMQP management open complete.", + self.state, + extra=self._network_trace_params, + ) + elif self.state == CbsState.OPEN: + self.state = CbsState.ERROR + _LOGGER.info( + "Unexpected AMQP management open complete in OPEN, CBS error occurred.", + extra=self._network_trace_params, + ) + elif self.state == CbsState.OPENING: + self.state = CbsState.OPEN if management_open_result == ManagementOpenResult.OK else CbsState.CLOSED + _LOGGER.debug( + "CBS completed opening with status: %r", management_open_result, extra=self._network_trace_params + ) + + def _on_amqp_management_error(self) -> None: + if self.state == CbsState.CLOSED: + _LOGGER.info("Unexpected AMQP error in CLOSED state.", extra=self._network_trace_params) + elif self.state == CbsState.OPENING: + self.state = CbsState.ERROR + self._mgmt_link.close() + _LOGGER.info( + "CBS failed to open with status: %r", ManagementOpenResult.ERROR, extra=self._network_trace_params + ) + elif self.state == CbsState.OPEN: + self.state = CbsState.ERROR + _LOGGER.info("CBS error occurred.", extra=self._network_trace_params) + + def _on_execute_operation_complete( + self, + execute_operation_result: ManagementExecuteOperationResult, + status_code: int, + status_description: str, + _, + error_condition: Optional[str] = None, + ) -> None: + if error_condition: + _LOGGER.info("CBS Put token error: %r", error_condition, extra=self._network_trace_params) + self.auth_state = CbsAuthState.ERROR + return + _LOGGER.debug( + "CBS Put token result (%r), status code: %s, status_description: %s.", + execute_operation_result, + status_code, + status_description, + extra=self._network_trace_params, + ) + self._token_status_code = status_code + self._token_status_description = status_description + + if execute_operation_result == ManagementExecuteOperationResult.OK: + self.auth_state = CbsAuthState.OK + elif execute_operation_result == ManagementExecuteOperationResult.ERROR: + self.auth_state = CbsAuthState.ERROR + # put-token-message sending failure, rejected + self._token_status_code = 0 + self._token_status_description = "Auth message has been rejected." + elif execute_operation_result == ManagementExecuteOperationResult.FAILED_BAD_STATUS: + self.auth_state = CbsAuthState.ERROR + + def _update_status(self) -> None: + if self.auth_state in (CbsAuthState.OK, CbsAuthState.REFRESH_REQUIRED): + is_expired, is_refresh_required = check_expiration_and_refresh_status( + self._expires_on, self._refresh_window # type: ignore + ) + _LOGGER.debug( + "CBS status check: state == %r, expired == %r, refresh required == %r", + self.auth_state, + is_expired, + is_refresh_required, + extra=self._network_trace_params, + ) + if is_expired: + self.auth_state = CbsAuthState.EXPIRED + elif is_refresh_required: + self.auth_state = CbsAuthState.REFRESH_REQUIRED + elif self.auth_state == CbsAuthState.IN_PROGRESS: + _LOGGER.debug( + "CBS update in progress. Token put time: %r", self._token_put_time, extra=self._network_trace_params + ) + if self._token_put_time is not None: + put_timeout = check_put_timeout_status(self._auth_timeout, self._token_put_time) + if put_timeout: + self.auth_state = CbsAuthState.TIMEOUT + + def _cbs_link_ready(self) -> Optional[bool]: + if self.state == CbsState.OPEN: + return True + if self.state != CbsState.OPEN: + return False + if self.state in (CbsState.CLOSED, CbsState.ERROR): + raise TokenAuthFailure( + status_code=ErrorCondition.ClientError, + status_description="CBS authentication link is in broken status, please recreate the cbs link.", + ) + return None + + def open(self) -> None: + self.state = CbsState.OPENING + self._mgmt_link.open() + + def close(self) -> None: + self._mgmt_link.close() + self.state = CbsState.CLOSED + + def update_token(self) -> None: + self.auth_state = CbsAuthState.IN_PROGRESS + access_token = self._auth.get_token() + if not access_token: + _LOGGER.info("Token refresh function received an empty token object.", extra=self._network_trace_params) + elif not access_token.token: + _LOGGER.info("Token refresh function received an empty token.", extra=self._network_trace_params) + self._expires_on = access_token.expires_on + expires_in = self._expires_on - int(utc_now().timestamp()) + self._refresh_window = int(float(expires_in) * 0.1) + token_type: Optional[str] = None + + if isinstance(access_token.token, bytes): + self._token = access_token.token.decode() + elif isinstance(access_token.token, str): + self._token = access_token.token + else: + raise ValueError("Token must be a string or bytes.") + if isinstance(self._auth.token_type, bytes): + token_type = self._auth.token_type.decode() + elif isinstance(self._auth.token_type, str): + token_type = self._auth.token_type + else: + raise ValueError("Token type must be a string or bytes.") + + self._token_put_time = int(utc_now().timestamp()) + if self._token and token_type: + self._put_token( + self._token, + token_type, + self._auth.audience, # type: ignore + utc_from_timestamp(self._expires_on), + ) + + def handle_token(self) -> bool: # pylint: disable=inconsistent-return-statements + if not self._cbs_link_ready(): + return False + self._update_status() + if self.auth_state == CbsAuthState.IDLE: + self.update_token() + return False + if self.auth_state == CbsAuthState.IN_PROGRESS: + return False + if self.auth_state == CbsAuthState.OK: + return True + if self.auth_state == CbsAuthState.REFRESH_REQUIRED: + _LOGGER.info("Token will expire soon - attempting to refresh.", extra=self._network_trace_params) + self.update_token() + return False + if self.auth_state == CbsAuthState.FAILURE: + raise AuthenticationException( + condition=ErrorCondition.InternalError, + description="Failed to open CBS authentication link.", + ) + if self.auth_state == CbsAuthState.ERROR: + raise TokenAuthFailure( + self._token_status_code, + self._token_status_description, + encoding=self._encoding, # TODO: drop off all the encodings + ) + if self.auth_state == CbsAuthState.TIMEOUT: + raise TimeoutError("Authentication attempt timed-out.") + if self.auth_state == CbsAuthState.EXPIRED: + raise TokenExpired( + condition=ErrorCondition.InternalError, + description="CBS Authentication Expired.", + ) diff --git a/src/azure/iot/hub/_pyamqp/client.py b/src/azure/iot/hub/_pyamqp/client.py new file mode 100644 index 0000000..657c9bd --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/client.py @@ -0,0 +1,1103 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +# pylint: disable=client-accepts-api-version-keyword +# pylint: disable=missing-client-constructor-parameter-credential +# pylint: disable=client-method-missing-type-annotations +# pylint: disable=too-many-lines +# TODO: Check types of kwargs (issue exists for this) +import logging +import threading +import queue +import time +import uuid +from functools import partial +from typing import Any, Callable, Dict, List, Optional, Tuple, Union, overload, cast +import certifi +from typing_extensions import Literal + +from ._connection import Connection +from .message import _MessageDelivery, Message +from .error import ( + AMQPException, + ErrorCondition, + MessageException, + MessageSendFailed, + RetryPolicy, + AMQPError, +) +from .outcomes import Received, Rejected, Released, Accepted, Modified + +from .constants import ( + MAX_CHANNELS, + MessageDeliveryState, + SenderSettleMode, + ReceiverSettleMode, + LinkDeliverySettleReason, + TransportType, + SEND_DISPOSITION_ACCEPT, + SEND_DISPOSITION_REJECT, + AUTH_TYPE_CBS, + MAX_FRAME_SIZE_BYTES, + INCOMING_WINDOW, + OUTGOING_WINDOW, + DEFAULT_AUTH_TIMEOUT, + MESSAGE_DELIVERY_DONE_STATES, + LINK_MAX_MESSAGE_SIZE +) + +from .management_operation import ManagementOperation +from .cbs import CBSAuthenticator + +Outcomes = Union[Received, Rejected, Released, Accepted, Modified] + + +_logger = logging.getLogger(__name__) + + +class AMQPClient(object): # pylint: disable=too-many-instance-attributes + """An AMQP client. + :param hostname: The AMQP endpoint to connect to. + :type hostname: str + :keyword auth: Authentication for the connection. This should be one of the following: + - pyamqp.authentication.SASLAnonymous + - pyamqp.authentication.SASLPlain + - pyamqp.authentication.SASTokenAuth + - pyamqp.authentication.JWTTokenAuth + If no authentication is supplied, SASLAnnoymous will be used by default. + :paramtype auth: ~pyamqp.authentication + :keyword client_name: The name for the client, also known as the Container ID. + If no name is provided, a random GUID will be used. + :paramtype client_name: str or bytes + :keyword network_trace: Whether to turn on network trace logs. If `True`, trace logs + will be logged at INFO level. Default is `False`. + :paramtype network_trace: bool + :keyword retry_policy: A policy for parsing errors on link, connection and message + disposition to determine whether the error should be retryable. + :paramtype retry_policy: ~pyamqp.error.RetryPolicy + :keyword keep_alive_interval: If set, a thread will be started to keep the connection + alive during periods of user inactivity. The value will determine how long the + thread will sleep (in seconds) between pinging the connection. If 0 or None, no + thread will be started. + :paramtype keep_alive_interval: int + :keyword max_frame_size: Maximum AMQP frame size. Default is 63488 bytes. + :paramtype max_frame_size: int + :keyword channel_max: Maximum number of Session channels in the Connection. + :paramtype channel_max: int + :keyword idle_timeout: Timeout in seconds after which the Connection will close + if there is no further activity. + :paramtype idle_timeout: int + :keyword auth_timeout: Timeout in seconds for CBS authentication. Otherwise this value will be ignored. + Default value is 60s. + :paramtype auth_timeout: int + :keyword properties: Connection properties. + :paramtype properties: dict[str, any] + :keyword remote_idle_timeout_empty_frame_send_ratio: Portion of the idle timeout time to wait before sending an + empty frame. The default portion is 50% of the idle timeout value (i.e. `0.5`). + :paramtype remote_idle_timeout_empty_frame_send_ratio: float + :keyword incoming_window: The size of the allowed window for incoming messages. + :paramtype incoming_window: int + :keyword outgoing_window: The size of the allowed window for outgoing messages. + :paramtype outgoing_window: int + :keyword handle_max: The maximum number of concurrent link handles. + :paramtype handle_max: int + :keyword on_attach: A callback function to be run on receipt of an ATTACH frame. + The function must take 4 arguments: source, target, properties and error. + :paramtype on_attach: func[ + ~pyamqp.endpoint.Source, ~pyamqp.endpoint.Target, dict, ~pyamqp.error.AMQPConnectionError] + :keyword send_settle_mode: The mode by which to settle message send + operations. If set to `Unsettled`, the client will wait for a confirmation + from the service that the message was successfully sent. If set to 'Settled', + the client will not wait for confirmation and assume success. + :paramtype send_settle_mode: ~pyamqp.constants.SenderSettleMode + :keyword receive_settle_mode: The mode by which to settle message receive + operations. If set to `PeekLock`, the receiver will lock a message once received until + the client accepts or rejects the message. If set to `ReceiveAndDelete`, the service + will assume successful receipt of the message and clear it from the queue. The + default is `PeekLock`. + :paramtype receive_settle_mode: ~pyamqp.constants.ReceiverSettleMode + :keyword desired_capabilities: The extension capabilities desired from the peer endpoint. + :paramtype desired_capabilities: list[bytes] + :keyword max_message_size: The maximum allowed message size negotiated for the Link. + :paramtype max_message_size: int + :keyword link_properties: Metadata to be sent in the Link ATTACH frame. + :paramtype link_properties: dict[str, any] + :keyword link_credit: The Link credit that determines how many + messages the Link will attempt to handle per connection iteration. + The default is 300. + :paramtype link_credit: int + :keyword transport_type: The type of transport protocol that will be used for communicating with + the service. Default is `TransportType.Amqp` in which case port 5671 is used. + If the port 5671 is unavailable/blocked in the network environment, `TransportType.AmqpOverWebsocket` could + be used instead which uses port 443 for communication. + :paramtype transport_type: ~pyamqp.constants.TransportType + :keyword http_proxy: HTTP proxy settings. This must be a dictionary with the following + keys: `'proxy_hostname'` (str value) and `'proxy_port'` (int value). + Additionally the following keys may also be present: `'username', 'password'`. + :paramtype http_proxy: dict[str, str] + :keyword custom_endpoint_address: The custom endpoint address to use for establishing a connection to + the service, allowing network requests to be routed through any application gateways or + other paths needed for the host environment. Default is None. + Unless specified otherwise, default transport type is TransportType.AmqpOverWebsockets. + If port is not specified in the `custom_endpoint_address`, by default port 443 will be used. + :paramtype custom_endpoint_address: str + :keyword connection_verify: Path to the custom CA_BUNDLE file of the SSL certificate which is used to + authenticate the identity of the connection endpoint. Ignored if ssl_context passed in. Default is None + in which case `certifi.where()` will be used. + :paramtype connection_verify: str + :keyword ssl_context: An instance of ssl.SSLContext to be used. If this is specified, connection_verify is ignored. + :paramtype ssl_context: ssl.SSLContext or None + :keyword float socket_timeout: The maximum time in seconds that the underlying socket in the transport should + wait when reading or writing data before timing out. The default value is 0.2 (for transport type Amqp), + and 1 for transport type AmqpOverWebsocket. + """ + + def __init__(self, hostname, **kwargs): + # I think these are just strings not instances of target or source + self._hostname = hostname + self._auth = kwargs.pop("auth", None) + self._name = kwargs.pop("client_name", str(uuid.uuid4())) + self._shutdown = False + self._connection = None + self._session = None + self._link = None + self._external_connection = False + self._cbs_authenticator = None + self._auth_timeout = kwargs.pop("auth_timeout", DEFAULT_AUTH_TIMEOUT) + self._mgmt_links = {} + self._mgmt_link_lock = threading.Lock() + self._retry_policy = kwargs.pop("retry_policy", RetryPolicy()) + self._keep_alive_interval = kwargs.get("keep_alive_interval", 0) + self._keep_alive_interval = int(self._keep_alive_interval) if self._keep_alive_interval is not None else 0 + self._keep_alive_thread = None + + # Connection settings + self._max_frame_size = kwargs.pop("max_frame_size", MAX_FRAME_SIZE_BYTES) + self._channel_max = kwargs.pop("channel_max", MAX_CHANNELS) + self._idle_timeout = kwargs.pop("idle_timeout", None) + self._properties = kwargs.pop("properties", None) + self._remote_idle_timeout_empty_frame_send_ratio = kwargs.pop( + "remote_idle_timeout_empty_frame_send_ratio", None + ) + self._network_trace = kwargs.pop("network_trace", False) + self._network_trace_params = {"amqpConnection": "", "amqpSession": "", "amqpLink": ""} + + # Session settings + self._outgoing_window = kwargs.pop("outgoing_window", OUTGOING_WINDOW) + self._incoming_window = kwargs.pop("incoming_window", INCOMING_WINDOW) + self._handle_max = kwargs.pop("handle_max", None) + + # Link settings + self._send_settle_mode = kwargs.pop("send_settle_mode", SenderSettleMode.Unsettled) + self._receive_settle_mode = kwargs.pop("receive_settle_mode", ReceiverSettleMode.Second) + self._desired_capabilities = kwargs.pop("desired_capabilities", None) + self._on_attach = kwargs.pop("on_attach", None) + + # transport + if kwargs.get("transport_type") is TransportType.Amqp and kwargs.get("http_proxy") is not None: + raise ValueError("Http proxy settings can't be passed if transport_type is explicitly set to Amqp") + self._transport_type = kwargs.pop("transport_type", TransportType.Amqp) + self._socket_timeout = kwargs.pop("socket_timeout", None) + self._http_proxy = kwargs.pop("http_proxy", None) + + # Custom Endpoint + self._custom_endpoint_address = kwargs.get("custom_endpoint_address") + connection_verify = kwargs.get("connection_verify") + ssl_context = kwargs.get("ssl_context") + self._ssl_opts = {} + if ssl_context: + self._ssl_opts["context"] = ssl_context + else: # str or None + self._ssl_opts["ca_certs"] = connection_verify or certifi.where() + + # Emulator + self._use_tls: bool = kwargs.get("use_tls", True) + + def __enter__(self): + """Run Client in a context manager. + + :return: The Client object. + :rtype: ~pyamqp.AMQPClient + """ + self.open() + return self + + def __exit__(self, *args): + """Close and destroy Client on exiting a context manager. + :param any args: Ignored. + """ + self.close() + + def _keep_alive(self): + start_time = time.time() + try: + while self._connection and not self._shutdown: + current_time = time.time() + elapsed_time = current_time - start_time + if elapsed_time >= self._keep_alive_interval: + self._connection.listen(wait=self._socket_timeout, batch=self._link.current_link_credit) + start_time = current_time + time.sleep(1) + except Exception as e: # pylint: disable=broad-except + _logger.debug("Connection keep-alive for %r failed: %r.", self.__class__.__name__, e) + + def _client_ready(self): + """Determine whether the client is ready to start sending and/or + receiving messages. To be ready, the connection must be open and + authentication complete. + + :returns: True if ready, False otherwise. + :rtype: bool + """ + return True + + def _client_run(self, **kwargs): + """Perform a single Connection iteration.""" + self._connection.listen(wait=self._socket_timeout, **kwargs) + + def _close_link(self): + if self._link and not self._link._is_closed: # pylint: disable=protected-access + self._link.detach(close=True) + self._link = None + + def _do_retryable_operation(self, operation, *args, **kwargs): + retry_settings = self._retry_policy.configure_retries() + retry_active = True + absolute_timeout = kwargs.pop("timeout", 0) or 0 + start_time = time.time() + while retry_active: + try: + if absolute_timeout < 0: + raise TimeoutError("Operation timed out.") + return operation(*args, timeout=absolute_timeout, **kwargs) + except AMQPException as exc: + if not self._retry_policy.is_retryable(exc): + raise + if absolute_timeout >= 0: + retry_active = self._retry_policy.increment(retry_settings, exc) + if not retry_active: + break + time.sleep(self._retry_policy.get_backoff_time(retry_settings, exc)) + if exc.condition == ErrorCondition.LinkDetachForced: + self._close_link() # if link level error, close and open a new link + if exc.condition in ( + ErrorCondition.ConnectionCloseForced, + ErrorCondition.SocketError, + ): + # if connection detach or socket error, close and open a new connection + self.close() + finally: + end_time = time.time() + if absolute_timeout > 0: + absolute_timeout -= end_time - start_time + raise retry_settings["history"][-1] + + def open(self, connection=None): + """Open the client. The client can create a new Connection + or an existing Connection can be passed in. This existing Connection + may have an existing CBS authentication Session, which will be + used for this client as well. Otherwise a new Session will be + created. + + :param connection: An existing Connection that may be shared between + multiple clients. + :type connection: ~pyamqp.Connection + """ + # pylint: disable=protected-access + if self._session: + return # already open. + if connection: + self._connection = connection + self._external_connection = True + elif not self._connection: + self._connection = Connection( + "amqps://" + self._hostname if self._use_tls else "amqp://" + self._hostname, + sasl_credential=self._auth.sasl, + ssl_opts=self._ssl_opts, + container_id=self._name, + max_frame_size=self._max_frame_size, + channel_max=self._channel_max, + idle_timeout=self._idle_timeout, + properties=self._properties, + network_trace=self._network_trace, + transport_type=self._transport_type, + http_proxy=self._http_proxy, + custom_endpoint_address=self._custom_endpoint_address, + socket_timeout=self._socket_timeout, + use_tls=self._use_tls, + ) + self._connection.open() + if not self._session: + self._session = self._connection.create_session( + incoming_window=self._incoming_window, + outgoing_window=self._outgoing_window, + ) + self._session.begin() + if self._keep_alive_interval: + self._keep_alive_thread = threading.Thread(target=self._keep_alive) + self._keep_alive_thread.daemon = True + self._keep_alive_thread.start() + if self._auth.auth_type == AUTH_TYPE_CBS: + self._cbs_authenticator = CBSAuthenticator( + session=self._session, auth=self._auth, auth_timeout=self._auth_timeout + ) + self._cbs_authenticator.open() + self._network_trace_params["amqpConnection"] = self._connection._container_id + self._network_trace_params["amqpSession"] = self._session.name + self._shutdown = False + + def close(self): + """Close the client. This includes closing the Session + and CBS authentication layer as well as the Connection. + If the client was opened using an external Connection, + this will be left intact. + + No further messages can be sent or received and the client + cannot be re-opened. + + All pending, unsent messages will remain uncleared to allow + them to be inspected and queued to a new client. + """ + self._shutdown = True + if not self._session: + return # already closed. + self._close_link() + if self._cbs_authenticator: + self._cbs_authenticator.close() + self._cbs_authenticator = None + self._session.end() + self._session = None + if not self._external_connection: + self._connection.close() + self._connection = None + if self._keep_alive_thread: + try: + self._keep_alive_thread.join() + except RuntimeError: # Probably thread failed to start in .open() + logging.debug("Keep alive thread failed to join.", exc_info=True) + self._keep_alive_thread = None + self._network_trace_params["amqpConnection"] = "" + self._network_trace_params["amqpSession"] = "" + + def auth_complete(self): + """Whether the authentication handshake is complete during + connection initialization. + + :return: Whether the authentication handshake is complete. + :rtype: bool + """ + if self._cbs_authenticator and not self._cbs_authenticator.handle_token(): + self._connection.listen(wait=self._socket_timeout) + return False + return True + + def client_ready(self): + """ + Whether the handler has completed all start up processes such as + establishing the connection, session, link and authentication, and + is not ready to process messages. + + :return: Whether the handler is ready to process messages. + :rtype: bool + """ + if not self.auth_complete(): + return False + if not self._client_ready(): + try: + self._connection.listen(wait=self._socket_timeout) + except ValueError: + return True + return False + return True + + def do_work(self, **kwargs): + """Run a single connection iteration. + This will return `True` if the connection is still open + and ready to be used for further work, or `False` if it needs + to be shut down. + + :return: Whether the connection is still open and ready to be used. + :rtype: bool + :raises: TimeoutError if CBS authentication timeout reached. + """ + if self._shutdown: + return False + if not self.client_ready(): + return True + return self._client_run(**kwargs) + + def mgmt_request( + self, + message, + *, + operation: Optional[Union[str, bytes]] = None, + operation_type: Optional[Union[str, bytes]] = None, + node: str = "$management", + timeout: float = 0, + **kwargs + ): + """ + :param message: The message to send in the management request. + :type message: ~pyamqp.message.Message + :keyword str operation: The type of operation to be performed. This value will + be service-specific, but common values include READ, CREATE and UPDATE. + This value will be added as an application property on the message. + :keyword str operation_type: The type on which to carry out the operation. This will + be specific to the entities of the service. This value will be added as + an application property on the message. + :keyword str node: The target node. Default node is `$management`. + :keyword float timeout: Provide an optional timeout in seconds within which a response + to the management request must be received. + :returns: The response to the management request. + :rtype: ~pyamqp.message.Message + """ + + # The method also takes "status_code_field" and "status_description_field" + # keyword arguments as alternate names for the status code and description + # in the response body. Those two keyword arguments are used in Azure services only. + with self._mgmt_link_lock: + try: + mgmt_link = self._mgmt_links[node] + except KeyError: + mgmt_link = ManagementOperation(self._session, endpoint=node, **kwargs) + self._mgmt_links[node] = mgmt_link + mgmt_link.open() + + while not self.client_ready(): + time.sleep(0.05) + + while not mgmt_link.ready(): + self._connection.listen(wait=False) + operation_type = operation_type or b"empty" + status, description, response = mgmt_link.execute( + message, operation=operation, operation_type=operation_type, timeout=timeout + ) + return status, description, response + + +class SendClient(AMQPClient): + """ + An AMQP client for sending messages. + :param target: The target AMQP service endpoint. This can either be the URI as + a string or a ~pyamqp.endpoint.Target object. + :type target: str, bytes or ~pyamqp.endpoint.Target + :keyword auth: Authentication for the connection. This should be one of the following: + - pyamqp.authentication.SASLAnonymous + - pyamqp.authentication.SASLPlain + - pyamqp.authentication.SASTokenAuth + - pyamqp.authentication.JWTTokenAuth + If no authentication is supplied, SASLAnnoymous will be used by default. + :paramtype auth: ~pyamqp.authentication + :keyword client_name: The name for the client, also known as the Container ID. + If no name is provided, a random GUID will be used. + :paramtype client_name: str or bytes + :keyword network_trace: Whether to turn on network trace logs. If `True`, trace logs + will be logged at INFO level. Default is `False`. + :paramtype network_trace: bool + :keyword retry_policy: A policy for parsing errors on link, connection and message + disposition to determine whether the error should be retryable. + :paramtype retry_policy: ~pyamqp.error.RetryPolicy + :keyword keep_alive_interval: If set, a thread will be started to keep the connection + alive during periods of user inactivity. The value will determine how long the + thread will sleep (in seconds) between pinging the connection. If 0 or None, no + thread will be started. + :paramtype keep_alive_interval: int + :keyword max_frame_size: Maximum AMQP frame size. Default is 63488 bytes. + :paramtype max_frame_size: int + :keyword channel_max: Maximum number of Session channels in the Connection. + :paramtype channel_max: int + :keyword idle_timeout: Timeout in seconds after which the Connection will close + if there is no further activity. + :paramtype idle_timeout: int + :keyword auth_timeout: Timeout in seconds for CBS authentication. Otherwise this value will be ignored. + Default value is 60s. + :paramtype auth_timeout: int + :keyword properties: Connection properties. + :paramtype properties: dict[str, any] + :keyword remote_idle_timeout_empty_frame_send_ratio: Portion of the idle timeout time to wait before sending an + empty frame. The default portion is 50% of the idle timeout value (i.e. `0.5`). + :paramtype remote_idle_timeout_empty_frame_send_ratio: float + :keyword incoming_window: The size of the allowed window for incoming messages. + :paramtype incoming_window: int + :keyword outgoing_window: The size of the allowed window for outgoing messages. + :paramtype outgoing_window: int + :keyword handle_max: The maximum number of concurrent link handles. + :paramtype handle_max: int + :keyword on_attach: A callback function to be run on receipt of an ATTACH frame. + The function must take 4 arguments: source, target, properties and error. + :paramtype on_attach: func[ + ~pyamqp.endpoint.Source, ~pyamqp.endpoint.Target, dict, ~pyamqp.error.AMQPConnectionError] + :keyword send_settle_mode: The mode by which to settle message send + operations. If set to `Unsettled`, the client will wait for a confirmation + from the service that the message was successfully sent. If set to 'Settled', + the client will not wait for confirmation and assume success. + :paramtype send_settle_mode: ~pyamqp.constants.SenderSettleMode + :keyword receive_settle_mode: The mode by which to settle message receive + operations. If set to `PeekLock`, the receiver will lock a message once received until + the client accepts or rejects the message. If set to `ReceiveAndDelete`, the service + will assume successful receipt of the message and clear it from the queue. The + default is `PeekLock`. + :paramtype receive_settle_mode: ~pyamqp.constants.ReceiverSettleMode + :keyword desired_capabilities: The extension capabilities desired from the peer endpoint. + :paramtype desired_capabilities: list[bytes] + :keyword max_message_size: The maximum allowed message size negotiated for the Link. + :paramtype max_message_size: int + :keyword link_properties: Metadata to be sent in the Link ATTACH frame. + :paramtype link_properties: dict[str, any] + :keyword link_credit: The Link credit that determines how many + messages the Link will attempt to handle per connection iteration. + The default is 300. + :paramtype link_credit: int + :keyword transport_type: The type of transport protocol that will be used for communicating with + the service. Default is `TransportType.Amqp` in which case port 5671 is used. + If the port 5671 is unavailable/blocked in the network environment, `TransportType.AmqpOverWebsocket` could + be used instead which uses port 443 for communication. + :paramtype transport_type: ~pyamqp.constants.TransportType + :keyword http_proxy: HTTP proxy settings. This must be a dictionary with the following + keys: `'proxy_hostname'` (str value) and `'proxy_port'` (int value). + Additionally the following keys may also be present: `'username', 'password'`. + :paramtype http_proxy: dict[str, str] + :keyword custom_endpoint_address: The custom endpoint address to use for establishing a connection to + the service, allowing network requests to be routed through any application gateways or + other paths needed for the host environment. Default is None. + Unless specified otherwise, default transport type is TransportType.AmqpOverWebsockets. + If port is not specified in the `custom_endpoint_address`, by default port 443 will be used. + :paramtype custom_endpoint_address: str + :keyword connection_verify: Path to the custom CA_BUNDLE file of the SSL certificate which is used to + authenticate the identity of the connection endpoint. Ignored if ssl_context passed in. Default is None + in which case `certifi.where()` will be used. + :paramtype connection_verify: str + :keyword ssl_context: An instance of ssl.SSLContext to be used. If this is specified, connection_verify is ignored. + :paramtype ssl_context: ssl.SSLContext or None + """ + + def __init__(self, hostname, target, **kwargs): + self.target = target + # Sender and Link settings + self._max_message_size = kwargs.pop("max_message_size", LINK_MAX_MESSAGE_SIZE) + self._link_properties = kwargs.pop("link_properties", None) + self._link_credit = kwargs.pop("link_credit", None) + super(SendClient, self).__init__(hostname, **kwargs) + + def _client_ready(self): + """Determine whether the client is ready to start receiving messages. + To be ready, the connection must be open and authentication complete, + The Session, Link and MessageReceiver must be open and in non-errored + states. + + :return: Whether the client is ready to start receiving messages. + :rtype: bool + """ + if not self._link: + self._link = self._session.create_sender_link( + target_address=self.target, + link_credit=self._link_credit, + send_settle_mode=self._send_settle_mode, + rcv_settle_mode=self._receive_settle_mode, + max_message_size=self._max_message_size, + properties=self._link_properties, + ) + self._link.attach() + return False + if self._link.get_state().value != 3: # ATTACHED + return False + return True + + def _client_run(self, **kwargs): + """MessageSender Link is now open - perform message send + on all pending messages. + Will return True if operation successful and client can remain open for + further work. + + :return: Whether the client can remain open for further work. + :rtype: bool + """ + self._link.update_pending_deliveries() + self._connection.listen(wait=self._socket_timeout, **kwargs) + return True + + def _transfer_message(self, message_delivery, timeout=0): + message_delivery.state = MessageDeliveryState.WaitingForSendAck + on_send_complete = partial(self._on_send_complete, message_delivery) + delivery = self._link.send_transfer( + message_delivery.message, + on_send_complete=on_send_complete, + timeout=timeout, + send_async=True, + ) + return delivery + + @staticmethod + def _process_send_error(message_delivery, condition, description=None, info=None): + try: + amqp_condition = ErrorCondition(condition) + except ValueError: + error = MessageException(condition, description=description, info=info) + else: + error = MessageSendFailed(amqp_condition, description=description, info=info) + message_delivery.state = MessageDeliveryState.Error + message_delivery.error = error + + def _on_send_complete(self, message_delivery, reason, state): + message_delivery.reason = reason + if reason == LinkDeliverySettleReason.DISPOSITION_RECEIVED: + if state and SEND_DISPOSITION_ACCEPT in state: + message_delivery.state = MessageDeliveryState.Ok + else: + try: + error_info = state[SEND_DISPOSITION_REJECT] + self._process_send_error( + message_delivery, + condition=error_info[0][0], + description=error_info[0][1], + info=error_info[0][2], + ) + except TypeError: + self._process_send_error(message_delivery, condition=ErrorCondition.UnknownError) + elif reason == LinkDeliverySettleReason.SETTLED: + message_delivery.state = MessageDeliveryState.Ok + elif reason == LinkDeliverySettleReason.TIMEOUT: + message_delivery.state = MessageDeliveryState.Timeout + message_delivery.error = TimeoutError("Sending message timed out.") + else: + # NotDelivered and other unknown errors + self._process_send_error(message_delivery, condition=ErrorCondition.UnknownError) + + def _send_message_impl(self, message, *, timeout: float = 0): + expire_time = (time.time() + timeout) if timeout else None + self.open() + message_delivery = _MessageDelivery(message, MessageDeliveryState.WaitingToBeSent, expire_time) + while not self.client_ready(): + time.sleep(0.05) + + self._transfer_message(message_delivery, timeout) + running = True + while running and message_delivery.state not in MESSAGE_DELIVERY_DONE_STATES: + running = self.do_work() + if message_delivery.state not in MESSAGE_DELIVERY_DONE_STATES: + raise MessageException( + condition=ErrorCondition.ClientError, description="Send failed - connection not running." + ) + + if message_delivery.state in ( + MessageDeliveryState.Error, + MessageDeliveryState.Cancelled, + MessageDeliveryState.Timeout, + ): + try: + raise message_delivery.error # pylint: disable=raising-bad-type + except TypeError: + # This is a default handler + raise MessageException(condition=ErrorCondition.UnknownError, description="Send failed.") from None + + def send_message(self, message, *, timeout: float = 0, **kwargs): + """ + :param ~pyamqp.message.Message message: + :keyword float timeout: timeout in seconds. If set to + 0, the client will continue to wait until the message is sent or error happens. The + default is 0. + """ + self._do_retryable_operation(self._send_message_impl, message=message, timeout=timeout, **kwargs) + + +class ReceiveClient(AMQPClient): # pylint:disable=too-many-instance-attributes + """ + An AMQP client for receiving messages. + :param source: The source AMQP service endpoint. This can either be the URI as + a string or a ~pyamqp.endpoint.Source object. + :type source: str, bytes or ~pyamqp.endpoint.Source + :keyword auth: Authentication for the connection. This should be one of the following: + - pyamqp.authentication.SASLAnonymous + - pyamqp.authentication.SASLPlain + - pyamqp.authentication.SASTokenAuth + - pyamqp.authentication.JWTTokenAuth + If no authentication is supplied, SASLAnnoymous will be used by default. + :paramtype auth: ~pyamqp.authentication + :keyword client_name: The name for the client, also known as the Container ID. + If no name is provided, a random GUID will be used. + :paramtype client_name: str or bytes + :keyword network_trace: Whether to turn on network trace logs. If `True`, trace logs + will be logged at INFO level. Default is `False`. + :paramtype network_trace: bool + :keyword retry_policy: A policy for parsing errors on link, connection and message + disposition to determine whether the error should be retryable. + :paramtype retry_policy: ~pyamqp.error.RetryPolicy + :keyword keep_alive_interval: If set, a thread will be started to keep the connection + alive during periods of user inactivity. The value will determine how long the + thread will sleep (in seconds) between pinging the connection. If 0 or None, no + thread will be started. + :paramtype keep_alive_interval: int + :keyword max_frame_size: Maximum AMQP frame size. Default is 63488 bytes. + :paramtype max_frame_size: int + :keyword channel_max: Maximum number of Session channels in the Connection. + :paramtype channel_max: int + :keyword idle_timeout: Timeout in seconds after which the Connection will close + if there is no further activity. + :paramtype idle_timeout: int + :keyword auth_timeout: Timeout in seconds for CBS authentication. Otherwise this value will be ignored. + Default value is 60s. + :paramtype auth_timeout: int + :keyword properties: Connection properties. + :paramtype properties: dict[str, any] + :keyword remote_idle_timeout_empty_frame_send_ratio: Portion of the idle timeout time to wait before sending an + empty frame. The default portion is 50% of the idle timeout value (i.e. `0.5`). + :paramtype remote_idle_timeout_empty_frame_send_ratio: float + :keyword incoming_window: The size of the allowed window for incoming messages. + :paramtype incoming_window: int + :keyword outgoing_window: The size of the allowed window for outgoing messages. + :paramtype outgoing_window: int + :keyword handle_max: The maximum number of concurrent link handles. + :paramtype handle_max: int + :keyword on_attach: A callback function to be run on receipt of an ATTACH frame. + The function must take 4 arguments: source, target, properties and error. + :paramtype on_attach: func[ + ~pyamqp.endpoint.Source, ~pyamqp.endpoint.Target, dict, ~pyamqp.error.AMQPConnectionError] + :keyword send_settle_mode: The mode by which to settle message send + operations. If set to `Unsettled`, the client will wait for a confirmation + from the service that the message was successfully sent. If set to 'Settled', + the client will not wait for confirmation and assume success. + :paramtype send_settle_mode: ~pyamqp.constants.SenderSettleMode + :keyword receive_settle_mode: The mode by which to settle message receive + operations. If set to `PeekLock`, the receiver will lock a message once received until + the client accepts or rejects the message. If set to `ReceiveAndDelete`, the service + will assume successful receipt of the message and clear it from the queue. The + default is `PeekLock`. + :paramtype receive_settle_mode: ~pyamqp.constants.ReceiverSettleMode + :keyword desired_capabilities: The extension capabilities desired from the peer endpoint. + :paramtype desired_capabilities: list[bytes] + :keyword max_message_size: The maximum allowed message size negotiated for the Link. + :paramtype max_message_size: int + :keyword link_properties: Metadata to be sent in the Link ATTACH frame. + :paramtype link_properties: dict[str, any] + :keyword link_credit: The Link credit that determines how many + messages the Link will attempt to handle per connection iteration. + The default is 300. + :paramtype link_credit: int + :keyword transport_type: The type of transport protocol that will be used for communicating with + the service. Default is `TransportType.Amqp` in which case port 5671 is used. + If the port 5671 is unavailable/blocked in the network environment, `TransportType.AmqpOverWebsocket` could + be used instead which uses port 443 for communication. + :paramtype transport_type: ~pyamqp.constants.TransportType + :keyword http_proxy: HTTP proxy settings. This must be a dictionary with the following + keys: `'proxy_hostname'` (str value) and `'proxy_port'` (int value). + Additionally the following keys may also be present: `'username', 'password'`. + :paramtype http_proxy: dict[str, str] + :keyword custom_endpoint_address: The custom endpoint address to use for establishing a connection to + the service, allowing network requests to be routed through any application gateways or + other paths needed for the host environment. Default is None. + Unless specified otherwise, default transport type is TransportType.AmqpOverWebsockets. + If port is not specified in the `custom_endpoint_address`, by default port 443 will be used. + :paramtype custom_endpoint_address: str + :keyword connection_verify: Path to the custom CA_BUNDLE file of the SSL certificate which is used to + authenticate the identity of the connection endpoint. Ignored if ssl_context passed in. Default is None + in which case `certifi.where()` will be used. + :paramtype connection_verify: str + :keyword ssl_context: An instance of ssl.SSLContext to be used. If this is specified, connection_verify is ignored. + :paramtype ssl_context: ssl.SSLContext or None + """ + + def __init__(self, hostname, source, **kwargs): + self.source = source + self._streaming_receive = kwargs.pop("streaming_receive", False) + self._received_messages = queue.Queue() + self._message_received_callback = kwargs.pop("message_received_callback", None) + + # Sender and Link settings + self._max_message_size = kwargs.pop("max_message_size", MAX_FRAME_SIZE_BYTES) + self._link_properties = kwargs.pop("link_properties", None) + self._link_credit = kwargs.pop("link_credit", 300) + + # Iterator + self._timeout = kwargs.pop("timeout", 0) + self._timeout_reached = False + self._last_activity_timestamp = time.time() + + super(ReceiveClient, self).__init__(hostname, **kwargs) + + def _client_ready(self): + """Determine whether the client is ready to start receiving messages. + To be ready, the connection must be open and authentication complete, + The Session, Link and MessageReceiver must be open and in non-errored + states. + + :return: True if the client is ready to start receiving messages. + :rtype: bool + """ + if not self._link: + self._link = self._session.create_receiver_link( + source_address=self.source, + link_credit=0, # link_credit=0 on flow frame sent before client is ready + send_settle_mode=self._send_settle_mode, + rcv_settle_mode=self._receive_settle_mode, + max_message_size=self._max_message_size, + on_transfer=self._message_received, + properties=self._link_properties, + desired_capabilities=self._desired_capabilities, + on_attach=self._on_attach, + ) + self._link.attach() + return False + if self._link.get_state().value != 3: # ATTACHED + return False + return True + + def _client_run(self, **kwargs): + """MessageReceiver Link is now open - start receiving messages. + Will return True if operation successful and client can remain open for + further work. + + :return: Whether the client can remain open for further work. + :rtype: bool + """ + try: + if self._link.current_link_credit <= 0: + self._link.flow(link_credit=self._link_credit) + self._connection.listen(wait=self._socket_timeout, **kwargs) + except ValueError: + _logger.info("Timeout reached, closing receiver.", extra=self._network_trace_params) + self._shutdown = True + return False + return True + + def _message_received(self, frame, message): + """Callback run on receipt of every message. If there is + a user-defined callback, this will be called. + Additionally if the client is retrieving messages for a batch + or iterator, the message will be added to an internal queue. + + :param message: Received message. + :type message: ~pyamqp.message.Message + :param frame: Received frame. + :type frame: tuple + """ + self._last_activity_timestamp = time.time() + if self._message_received_callback: + self._message_received_callback(message) + if not self._streaming_receive: + self._received_messages.put((frame, message)) + + def _receive_message_batch_impl( + self, + max_batch_size: Optional[int] = None, + on_message_received: Optional[Callable] = None, + timeout: float = 0, + ): + self._message_received_callback = on_message_received + max_batch_size = max_batch_size or self._link_credit + timeout = time.time() + timeout if timeout else 0 + receiving = True + batch: List[Message] = [] + self.open() + while len(batch) < max_batch_size: + try: + # TODO: This drops the transfer frame data + _, message = self._received_messages.get_nowait() + batch.append(message) + self._received_messages.task_done() + except queue.Empty: + break + else: + return batch + + to_receive_size = max_batch_size - len(batch) + before_queue_size = self._received_messages.qsize() + + while receiving and to_receive_size > 0: + if timeout and time.time() > timeout: + break + + receiving = self.do_work(batch=to_receive_size) + cur_queue_size = self._received_messages.qsize() + # after do_work, check how many new messages have been received since previous iteration + received = cur_queue_size - before_queue_size + if to_receive_size < max_batch_size and received == 0: + # there are already messages in the batch, and no message is received in the current cycle + # return what we have + break + + to_receive_size -= received + before_queue_size = cur_queue_size + + while len(batch) < max_batch_size: + try: + _, message = self._received_messages.get_nowait() + batch.append(message) + self._received_messages.task_done() + except queue.Empty: + break + return batch + + def close(self): + self._received_messages = queue.Queue() + super(ReceiveClient, self).close() + + def receive_message_batch( # pylint: disable=unused-argument + self, + *, + max_batch_size: Optional[int] = None, + on_message_received: Optional[Callable] = None, + timeout: float = 0, + **kwargs + ): + """Receive a batch of messages. Messages returned in the batch have already been + accepted - if you wish to add logic to accept or reject messages based on custom + criteria, pass in a callback. This method will return as soon as some messages are + available rather than waiting to achieve a specific batch size, and therefore the + number of messages returned per call will vary up to the maximum allowed. + + :keyword max_batch_size: The maximum number of messages that can be returned in + one call. This value cannot be larger than the prefetch value, and if not specified, + the prefetch value will be used. + :paramtype max_batch_size: int + :keyword on_message_received: A callback to process messages as they arrive from the + service. It takes a single argument, a ~pyamqp.message.Message object. + :paramtype on_message_received: callable[~pyamqp.message.Message] + :keyword timeout: The timeout in milliseconds for which to wait to receive any messages. + If no messages are received in this time, an empty list will be returned. If set to + 0, the client will continue to wait until at least one message is received. The + default is 0. + :paramtype timeout: float + :return: A list of messages. + :rtype: list[~pyamqp.message.Message] + """ + return self._do_retryable_operation( + self._receive_message_batch_impl, + max_batch_size=max_batch_size, + on_message_received=on_message_received, + timeout=timeout, + ) + + def receive_messages_iter(self, timeout=None, on_message_received=None): + """Receive messages by generator. Messages returned in the generator have already been + accepted - if you wish to add logic to accept or reject messages based on custom + criteria, pass in a callback. + + :param int or None timeout: The timeout in milliseconds for which to wait to receive any messages. + :param on_message_received: A callback to process messages as they arrive from the + service. It takes a single argument, a ~pyamqp.message.Message object. + :type on_message_received: callable[~pyamqp.message.Message] + :return: A generator of messages. + :rtype: generator[~pyamqp.message.Message] + """ + self._message_received_callback = on_message_received + return self._message_generator(timeout=timeout) + + def _message_generator(self, timeout=None): + """Iterate over processed messages in the receive queue. + + :param int or None timeout: The timeout in milliseconds for which to wait to receive any messages. + :return: A generator of messages. + :rtype: generator[~pyamqp.message.Message] + """ + self.open() + self._timeout_reached = False + receiving = True + message = None + self._last_activity_timestamp = time.time() + self._timeout = timeout if timeout else self._timeout + try: + while receiving and not self._timeout_reached: + if self._timeout > 0: + if time.time() - self._last_activity_timestamp >= self._timeout: + self._timeout_reached = True + + if not self._timeout_reached: + receiving = self.do_work() + + while not self._received_messages.empty(): + message = self._received_messages.get() + self._last_activity_timestamp = time.time() + self._received_messages.task_done() + yield message + + finally: + if self._shutdown: + self.close() + + @overload + def settle_messages( + self, + delivery_id: Union[int, Tuple[int, int]], + delivery_tag: bytes, + outcome: Literal["accepted"], + *, + batchable: Optional[bool] = None + ): ... + + @overload + def settle_messages( + self, + delivery_id: Union[int, Tuple[int, int]], + delivery_tag: bytes, + outcome: Literal["released"], + *, + batchable: Optional[bool] = None + ): ... + + @overload + def settle_messages( + self, + delivery_id: Union[int, Tuple[int, int]], + delivery_tag: bytes, + outcome: Literal["rejected"], + *, + error: Optional[AMQPError] = None, + batchable: Optional[bool] = None + ): ... + + @overload + def settle_messages( + self, + delivery_id: Union[int, Tuple[int, int]], + delivery_tag: bytes, + outcome: Literal["modified"], + *, + delivery_failed: Optional[bool] = None, + undeliverable_here: Optional[bool] = None, + message_annotations: Optional[Dict[Union[str, bytes], Any]] = None, + batchable: Optional[bool] = None + ): ... + + @overload + def settle_messages( + self, + delivery_id: Union[int, Tuple[int, int]], + delivery_tag: bytes, + outcome: Literal["received"], + *, + section_number: int, + section_offset: int, + batchable: Optional[bool] = None + ): ... + + def settle_messages(self, delivery_id: Union[int, Tuple[int, int]], delivery_tag: bytes, outcome: str, **kwargs): + batchable = kwargs.pop("batchable", None) + if outcome.lower() == "accepted": + state: Outcomes = Accepted() + elif outcome.lower() == "released": + state = Released() + elif outcome.lower() == "rejected": + state = Rejected(**kwargs) + elif outcome.lower() == "modified": + state = Modified(**kwargs) + elif outcome.lower() == "received": + state = Received(**kwargs) + else: + raise ValueError("Unrecognized message output: {}".format(outcome)) + try: + first, last = cast(Tuple, delivery_id) + except TypeError: + first = delivery_id + last = None + self._link.send_disposition( + first_delivery_id=first, + last_delivery_id=last, + delivery_tag=delivery_tag, + settled=True, + delivery_state=state, + batchable=batchable, + wait=True, + ) diff --git a/src/azure/iot/hub/_pyamqp/constants.py b/src/azure/iot/hub/_pyamqp/constants.py new file mode 100644 index 0000000..ea6e84c --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/constants.py @@ -0,0 +1,348 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import cast +from collections import namedtuple +from enum import Enum +import struct + +_AS_BYTES = struct.Struct(">B") + +#: The IANA assigned port number for AMQP.The standard AMQP port number that has been assigned by IANA +#: for TCP, UDP, and SCTP.There are currently no UDP or SCTP mappings defined for AMQP. +#: The port number is reserved for future transport mappings to these protocols. +PORT = 5672 + +# default port for AMQP over Websocket +WEBSOCKET_PORT = 443 + +# subprotocol for AMQP over Websocket +AMQP_WS_SUBPROTOCOL = "AMQPWSB10" + +#: The IANA assigned port number for secure AMQP (amqps).The standard AMQP port number that has been assigned +#: by IANA for secure TCP using TLS. Implementations listening on this port should NOT expect a protocol +#: handshake before TLS is negotiated. +SECURE_PORT = 5671 + + +# default port for AMQP over Websocket +WEBSOCKET_PORT = 443 + + +# subprotocol for AMQP over Websocket +AMQP_WS_SUBPROTOCOL = "AMQPWSB10" + + +MAJOR = 1 #: Major protocol version. +MINOR = 0 #: Minor protocol version. +REV = 0 #: Protocol revision. +HEADER_FRAME = b"AMQP\x00" + _AS_BYTES.pack(MAJOR) + _AS_BYTES.pack(MINOR) + _AS_BYTES.pack(REV) + + +TLS_MAJOR = 1 #: Major protocol version. +TLS_MINOR = 0 #: Minor protocol version. +TLS_REV = 0 #: Protocol revision. +TLS_HEADER_FRAME = b"AMQP\x02" + _AS_BYTES.pack(TLS_MAJOR) + _AS_BYTES.pack(TLS_MINOR) + _AS_BYTES.pack(TLS_REV) + +SASL_MAJOR = 1 #: Major protocol version. +SASL_MINOR = 0 #: Minor protocol version. +SASL_REV = 0 #: Protocol revision. +SASL_HEADER_FRAME = b"AMQP\x03" + _AS_BYTES.pack(SASL_MAJOR) + _AS_BYTES.pack(SASL_MINOR) + _AS_BYTES.pack(SASL_REV) + +EMPTY_FRAME = b"\x00\x00\x00\x08\x02\x00\x00\x00" + +#: The lower bound for the agreed maximum frame size (in bytes). During the initial Connection negotiation, the +#: two peers must agree upon a maximum frame size. This constant defines the minimum value to which the maximum +#: frame size can be set. By defining this value, the peers can guarantee that they can send frames of up to this +#: size until they have agreed a definitive maximum frame size for that Connection. +MIN_MAX_FRAME_SIZE = 512 +MAX_FRAME_SIZE_BYTES = 1024 * 1024 +LINK_MAX_MESSAGE_SIZE = 0 +MAX_CHANNELS = 65535 +INCOMING_WINDOW = 64 * 1024 +OUTGOING_WINDOW = 64 * 1024 + +DEFAULT_LINK_CREDIT = 10000 + +FIELD = namedtuple("FIELD", "name, type, mandatory, default, multiple") + +STRING_FILTER = b"apache.org:selector-filter:string" + +DEFAULT_AUTH_TIMEOUT = 60 +AUTH_DEFAULT_EXPIRATION_SECONDS = 3600 +TOKEN_TYPE_JWT = "jwt" +TOKEN_TYPE_SASTOKEN = "servicebus.windows.net:sastoken" +CBS_PUT_TOKEN = "put-token" +CBS_NAME = "name" +CBS_OPERATION = "operation" +CBS_TYPE = "type" +CBS_EXPIRATION = "expiration" + +SEND_DISPOSITION_ACCEPT = "accepted" +SEND_DISPOSITION_REJECT = "rejected" + +AUTH_TYPE_SASL_PLAIN = "AUTH_SASL_PLAIN" +AUTH_TYPE_CBS = "AUTH_CBS" + +DEFAULT_WEBSOCKET_HEARTBEAT_SECONDS = 10 +CONNECT_TIMEOUT = 1 +SOCKET_TIMEOUT = 0.2 +WS_TIMEOUT_INTERVAL = 1 + + +class ConnectionState(Enum): + #: In this state a Connection exists, but nothing has been sent or received. This is the state an + #: implementation would be in immediately after performing a socket connect or socket accept. + START = 0 + #: In this state the Connection header has been received from our peer, but we have not yet sent anything. + HDR_RCVD = 1 + #: In this state the Connection header has been sent to our peer, but we have not yet received anything. + HDR_SENT = 2 + #: In this state we have sent and received the Connection header, but we have not yet sent or + #: received an open frame. + HDR_EXCH = 3 + #: In this state we have sent both the Connection header and the open frame, but + #: we have not yet received anything. + OPEN_PIPE = 4 + #: In this state we have sent the Connection header, the open frame, any pipelined Connection traffic, + #: and the close frame, but we have not yet received anything. + OC_PIPE = 5 + #: In this state we have sent and received the Connection header, and received an open frame from + #: our peer, but have not yet sent an open frame. + OPEN_RCVD = 6 + #: In this state we have sent and received the Connection header, and sent an open frame to our peer, + #: but have not yet received an open frame. + OPEN_SENT = 7 + #: In this state we have send and received the Connection header, sent an open frame, any pipelined + #: Connection traffic, and the close frame, but we have not yet received an open frame. + CLOSE_PIPE = 8 + #: In this state the Connection header and the open frame have both been sent and received. + OPENED = 9 + #: In this state we have received a close frame indicating that our partner has initiated a close. + #: This means we will never have to read anything more from this Connection, however we can + #: continue to write frames onto the Connection. If desired, an implementation could do a TCP half-close + #: at this point to shutdown the read side of the Connection. + CLOSE_RCVD = 10 + #: In this state we have sent a close frame to our partner. It is illegal to write anything more onto + #: the Connection, however there may still be incoming frames. If desired, an implementation could do + #: a TCP half-close at this point to shutdown the write side of the Connection. + CLOSE_SENT = 11 + #: The DISCARDING state is a variant of the CLOSE_SENT state where the close is triggered by an error. + #: In this case any incoming frames on the connection MUST be silently discarded until the peer's close + #: frame is received. + DISCARDING = 12 + #: In this state it is illegal for either endpoint to write anything more onto the Connection. The + #: Connection may be safely closed and discarded. + END = 13 + + +class SessionState(Enum): + #: In the UNMAPPED state, the Session endpoint is not mapped to any incoming or outgoing channels on the + #: Connection endpoint. In this state an endpoint cannot send or receive frames. + UNMAPPED = 0 + #: In the BEGIN_SENT state, the Session endpoint is assigned an outgoing channel number, but there is no entry + #: in the incoming channel map. In this state the endpoint may send frames but cannot receive them. + BEGIN_SENT = 1 + #: In the BEGIN_RCVD state, the Session endpoint has an entry in the incoming channel map, but has not yet + #: been assigned an outgoing channel number. The endpoint may receive frames, but cannot send them. + BEGIN_RCVD = 2 + #: In the MAPPED state, the Session endpoint has both an outgoing channel number and an entry in the incoming + #: channel map. The endpoint may both send and receive frames. + MAPPED = 3 + #: In the END_SENT state, the Session endpoint has an entry in the incoming channel map, but is no longer + #: assigned an outgoing channel number. The endpoint may receive frames, but cannot send them. + END_SENT = 4 + #: In the END_RCVD state, the Session endpoint is assigned an outgoing channel number, but there is no entry in + #: the incoming channel map. The endpoint may send frames, but cannot receive them. + END_RCVD = 5 + #: The DISCARDING state is a variant of the END_SENT state where the end is triggered by an error. In this + #: case any incoming frames on the session MUST be silently discarded until the peer's end frame is received. + DISCARDING = 6 + + +class SessionTransferState(Enum): + + OKAY = 0 + ERROR = 1 + BUSY = 2 + + +class LinkDeliverySettleReason(Enum): + + DISPOSITION_RECEIVED = 0 + SETTLED = 1 + NOT_DELIVERED = 2 + TIMEOUT = 3 + CANCELLED = 4 + + +class LinkState(Enum): + + DETACHED = 0 + ATTACH_SENT = 1 + ATTACH_RCVD = 2 + ATTACHED = 3 + DETACH_SENT = 4 + DETACH_RCVD = 5 + ERROR = 6 + + +class ManagementLinkState(Enum): + + IDLE = 0 + OPENING = 1 + CLOSING = 2 + OPEN = 3 + ERROR = 4 + + +class ManagementOpenResult(Enum): + + OPENING = 0 + OK = 1 + ERROR = 2 + CANCELLED = 3 + + +class ManagementExecuteOperationResult(Enum): + + OK = 0 + ERROR = 1 + FAILED_BAD_STATUS = 2 + LINK_CLOSED = 3 + + +class CbsState(Enum): + CLOSED = 0 + OPENING = 1 + OPEN = 2 + ERROR = 3 + + +class CbsAuthState(Enum): + OK = 0 + IDLE = 1 + IN_PROGRESS = 2 + TIMEOUT = 3 + REFRESH_REQUIRED = 4 + EXPIRED = 5 + ERROR = 6 # Put token rejected or complete but fail authentication + FAILURE = 7 # Fail to open cbs links + + +class Role(object): + """Link endpoint role. + + Valid Values: + - False: Sender + - True: Receiver + + + + + + """ + + Sender = False + Receiver = True + + +class SenderSettleMode(object): + """Settlement policy for a Sender. + + Valid Values: + - 0: The Sender will send all deliveries initially unsettled to the Receiver. + - 1: The Sender will send all deliveries settled to the Receiver. + - 2: The Sender may send a mixture of settled and unsettled deliveries to the Receiver. + + + + + + + """ + + Unsettled = 0 + Settled = 1 + Mixed = 2 + + +class ReceiverSettleMode(object): + """Settlement policy for a Receiver. + + Valid Values: + - 0: The Receiver will spontaneously settle all incoming transfers. + - 1: The Receiver will only settle after sending the disposition to the Sender and + receiving a disposition indicating settlement of the delivery from the sender. + + + + + + """ + + First = 0 + Second = 1 + + +class SASLCode(object): + """Codes to indicate the outcome of the sasl dialog. + + + + + + + + + """ + + #: Connection authentication succeeded. + Ok = 0 + #: Connection authentication failed due to an unspecified problem with the supplied credentials. + Auth = 1 + #: Connection authentication failed due to a system error. + Sys = 2 + #: Connection authentication failed due to a system error that is unlikely to be corrected without intervention. + SysPerm = 3 + #: Connection authentication failed due to a transient system error. + SysTemp = 4 + + +class MessageDeliveryState(object): + + WaitingToBeSent = 0 + WaitingForSendAck = 1 + Ok = 2 + Error = 3 + Timeout = 4 + Cancelled = 5 + + +MESSAGE_DELIVERY_DONE_STATES = ( + MessageDeliveryState.Ok, + MessageDeliveryState.Error, + MessageDeliveryState.Timeout, + MessageDeliveryState.Cancelled, +) + + +class TransportType(Enum): + """Transport type + The underlying transport protocol type: + Amqp: AMQP over the default TCP transport protocol, it uses port 5671. + AmqpOverWebsocket: Amqp over the Web Sockets transport protocol, it uses + port 443. + """ + + Amqp = 1 + AmqpOverWebsocket = 2 + + def __eq__(self, __o: object) -> bool: + try: + __o = cast(Enum, __o) + return self.value == __o.value + except AttributeError: + return super().__eq__(__o) diff --git a/src/azure/iot/hub/_pyamqp/described.py b/src/azure/iot/hub/_pyamqp/described.py new file mode 100644 index 0000000..a86a15f --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/described.py @@ -0,0 +1,31 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +class Described: + def __new__(cls, value, descriptor=None): + obj = super().__new__(cls, value) + obj.descriptor = descriptor + return obj + + +class DescribedInt(Described, int): + pass + + +class DescribedFloat(Described, float): + pass + + +class DescribedBytes(Described, bytes): + pass + + +class DescribedList(Described, list): + pass + + +class DescribedDict(Described, dict): + pass diff --git a/src/azure/iot/hub/_pyamqp/endpoints.py b/src/azure/iot/hub/_pyamqp/endpoints.py new file mode 100644 index 0000000..5149597 --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/endpoints.py @@ -0,0 +1,277 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +# The messaging layer defines two concrete types (source and target) to be used as the source and target of a +# link. These types are supplied in the source and target fields of the attach frame when establishing or +# resuming link. The source is comprised of an address (which the container of the outgoing Link Endpoint will +# resolve to a Node within that container) coupled with properties which determine: +# +# - which messages from the sending Node will be sent on the Link +# - how sending the message affects the state of that message at the sending Node +# - the behavior of Messages which have been transferred on the Link, but have not yet reached a +# terminal state at the receiver, when the source is destroyed. + +# TODO: fix mypy errors for _code/_definition/__defaults__ (issue #26500) +from collections import namedtuple + +from .types import AMQPTypes, FieldDefinition, ObjDefinition +from .constants import FIELD +from .performatives import _CAN_ADD_DOCSTRING + + +class TerminusDurability(object): + """Durability policy for a terminus. + + + + + + + + Determines which state of the terminus is held durably. + """ + + #: No Terminus state is retained durably + NoDurability = 0 + #: Only the existence and configuration of the Terminus is retained durably. + Configuration = 1 + #: In addition to the existence and configuration of the Terminus, the unsettled state for durable + #: messages is retained durably. + UnsettledState = 2 + + +class ExpiryPolicy(object): + """Expiry policy for a terminus. + + + + + + + + + Determines when the expiry timer of a terminus starts counting down from the timeout + value. If the link is subsequently re-attached before the terminus is expired, then the + count down is aborted. If the conditions for the terminus-expiry-policy are subsequently + re-met, the expiry timer restarts from its originally configured timeout value. + """ + + #: The expiry timer starts when Terminus is detached. + LinkDetach = b"link-detach" + #: The expiry timer starts when the most recently associated session is ended. + SessionEnd = b"session-end" + #: The expiry timer starts when most recently associated connection is closed. + ConnectionClose = b"connection-close" + #: The Terminus never expires. + Never = b"never" + + +class DistributionMode(object): + """Link distribution policy. + + + + + + + Policies for distributing messages when multiple links are connected to the same node. + """ + + #: Once successfully transferred over the link, the message will no longer be available + #: to other links from the same node. + Move = b"move" + #: Once successfully transferred over the link, the message is still available for other + #: links from the same node. + Copy = b"copy" + + +class LifeTimePolicy(object): + #: Lifetime of dynamic node scoped to lifetime of link which caused creation. + #: A node dynamically created with this lifetime policy will be deleted at the point that the link + #: which caused its creation ceases to exist. + DeleteOnClose = 0x0000002B + #: Lifetime of dynamic node scoped to existence of links to the node. + #: A node dynamically created with this lifetime policy will be deleted at the point that there remain + #: no links for which the node is either the source or target. + DeleteOnNoLinks = 0x0000002C + #: Lifetime of dynamic node scoped to existence of messages on the node. + #: A node dynamically created with this lifetime policy will be deleted at the point that the link which + #: caused its creation no longer exists and there remain no messages at the node. + DeleteOnNoMessages = 0x0000002D + #: Lifetime of node scoped to existence of messages on or links to the node. + #: A node dynamically created with this lifetime policy will be deleted at the point that the there are no + #: links which have this node as their source or target, and there remain no messages at the node. + DeleteOnNoLinksOrMessages = 0x0000002E + + +class SupportedOutcomes(object): + #: Indicates successful processing at the receiver. + accepted = b"amqp:accepted:list" + #: Indicates an invalid and unprocessable message. + rejected = b"amqp:rejected:list" + #: Indicates that the message was not (and will not be) processed. + released = b"amqp:released:list" + #: Indicates that the message was modified, but not processed. + modified = b"amqp:modified:list" + + +class ApacheFilters(object): + #: Exact match on subject - analogous to legacy AMQP direct exchange bindings. + legacy_amqp_direct_binding = b"apache.org:legacy-amqp-direct-binding:string" + #: Pattern match on subject - analogous to legacy AMQP topic exchange bindings. + legacy_amqp_topic_binding = b"apache.org:legacy-amqp-topic-binding:string" + #: Matching on message headers - analogous to legacy AMQP headers exchange bindings. + legacy_amqp_headers_binding = b"apache.org:legacy-amqp-headers-binding:map" + #: Filter out messages sent from the same connection as the link is currently associated with. + no_local_filter = b"apache.org:no-local-filter:list" + #: SQL-based filtering syntax. + selector_filter = b"apache.org:selector-filter:string" + + +Source = namedtuple( + "Source", + [ + "address", + "durable", + "expiry_policy", + "timeout", + "dynamic", + "dynamic_node_properties", + "distribution_mode", + "filters", + "default_outcome", + "outcomes", + "capabilities", + ], + defaults=(None,) * 11, # type: ignore +) +Source._code = 0x00000028 # type: ignore # pylint: disable=protected-access +Source._definition = ( # type: ignore # pylint: disable=protected-access + FIELD("address", AMQPTypes.string, False, None, False), + FIELD("durable", AMQPTypes.uint, False, "none", False), + FIELD("expiry_policy", AMQPTypes.symbol, False, ExpiryPolicy.SessionEnd, False), + FIELD("timeout", AMQPTypes.uint, False, 0, False), + FIELD("dynamic", AMQPTypes.boolean, False, False, False), + FIELD("dynamic_node_properties", FieldDefinition.node_properties, False, None, False), + FIELD("distribution_mode", AMQPTypes.symbol, False, None, False), + FIELD("filters", FieldDefinition.filter_set, False, None, False), + FIELD("default_outcome", ObjDefinition.delivery_state, False, None, False), + FIELD("outcomes", AMQPTypes.symbol, False, None, True), + FIELD("capabilities", AMQPTypes.symbol, False, None, True), +) +if _CAN_ADD_DOCSTRING: + Source.__doc__ = """ + For containers which do not implement address resolution (and do not admit spontaneous link + attachment from their partners) but are instead only used as producers of messages, it is unnecessary to provide + spurious detail on the source. For this purpose it is possible to use a "minimal" source in which all the + fields are left unset. + + :param str address: The address of the source. + The address of the source MUST NOT be set when sent on a attach frame sent by the receiving Link Endpoint + where the dynamic fiag is set to true (that is where the receiver is requesting the sender to create an + addressable node). The address of the source MUST be set when sent on a attach frame sent by the sending + Link Endpoint where the dynamic fiag is set to true (that is where the sender has created an addressable + node at the request of the receiver and is now communicating the address of that created node). + The generated name of the address SHOULD include the link name and the container-id of the remote container + to allow for ease of identification. + :param ~uamqp.endpoints.TerminusDurability durable: Indicates the durability of the terminus. + Indicates what state of the terminus will be retained durably: the state of durable messages, only + existence and configuration of the terminus, or no state at all. + :param ~uamqp.endpoints.ExpiryPolicy expiry_policy: The expiry policy of the Source. + Determines when the expiry timer of a Terminus starts counting down from the timeout value. If the link + is subsequently re-attached before the Terminus is expired, then the count down is aborted. If the + conditions for the terminus-expiry-policy are subsequently re-met, the expiry timer restarts from its + originally configured timeout value. + :param int timeout: Duration that an expiring Source will be retained in seconds. + The Source starts expiring as indicated by the expiry-policy. + :param bool dynamic: Request dynamic creation of a remote Node. + When set to true by the receiving Link endpoint, this field constitutes a request for the sending peer + to dynamically create a Node at the source. In this case the address field MUST NOT be set. When set to + true by the sending Link Endpoint this field indicates creation of a dynamically created Node. In this case + the address field will contain the address of the created Node. The generated address SHOULD include the + Link name and Session-name or client-id in some recognizable form for ease of traceability. + :param dict dynamic_node_properties: Properties of the dynamically created Node. + If the dynamic field is not set to true this field must be left unset. When set by the receiving Link + endpoint, this field contains the desired properties of the Node the receiver wishes to be created. When + set by the sending Link endpoint this field contains the actual properties of the dynamically created node. + :param uamqp.endpoints.DistributionMode distribution_mode: The distribution mode of the Link. + This field MUST be set by the sending end of the Link if the endpoint supports more than one + distribution-mode. This field MAY be set by the receiving end of the Link to indicate a preference when a + Node supports multiple distribution modes. + :param dict filters: A set of predicates to filter the Messages admitted onto the Link. + The receiving endpoint sets its desired filter, the sending endpoint sets the filter actually in place + (including any filters defaulted at the node). The receiving endpoint MUST check that the filter in place + meets its needs and take responsibility for detaching if it does not. + Common filter types, along with the capabilities they are associated with are registered + here: http://www.amqp.org/specification/1.0/filters. + :param ~uamqp.outcomes.DeliveryState default_outcome: Default outcome for unsettled transfers. + Indicates the outcome to be used for transfers that have not reached a terminal state at the receiver + when the transfer is settled, including when the Source is destroyed. The value MUST be a valid + outcome (e.g. Released or Rejected). + :param list(bytes) outcomes: Descriptors for the outcomes that can be chosen on this link. + The values in this field are the symbolic descriptors of the outcomes that can be chosen on this link. + This field MAY be empty, indicating that the default-outcome will be assumed for all message transfers + (if the default-outcome is not set, and no outcomes are provided, then the accepted outcome must be + supported by the source). When present, the values MUST be a symbolic descriptor of a valid outcome, + e.g. "amqp:accepted:list". + :param list(bytes) capabilities: The extension capabilities the sender supports/desires. + See http://www.amqp.org/specification/1.0/source-capabilities. + """ + + +Target = namedtuple( + "Target", + ["address", "durable", "expiry_policy", "timeout", "dynamic", "dynamic_node_properties", "capabilities"], + defaults=(None,) * 7, # type: ignore +) +Target._code = 0x00000029 # type: ignore # pylint: disable=protected-access +Target._definition = ( # type: ignore # pylint: disable=protected-access + FIELD("address", AMQPTypes.string, False, None, False), + FIELD("durable", AMQPTypes.uint, False, "none", False), + FIELD("expiry_policy", AMQPTypes.symbol, False, ExpiryPolicy.SessionEnd, False), + FIELD("timeout", AMQPTypes.uint, False, 0, False), + FIELD("dynamic", AMQPTypes.boolean, False, False, False), + FIELD("dynamic_node_properties", FieldDefinition.node_properties, False, None, False), + FIELD("capabilities", AMQPTypes.symbol, False, None, True), +) +if _CAN_ADD_DOCSTRING: + Target.__doc__ = """ + For containers which do not implement address resolution (and do not admit spontaneous link attachment + from their partners) but are instead only used as consumers of messages, it is unnecessary to provide spurious + detail on the source. For this purpose it is possible to use a 'minimal' target in which all the + fields are left unset. + + :param str address: The address of the source. + The address of the source MUST NOT be set when sent on a attach frame sent by the receiving Link Endpoint + where the dynamic fiag is set to true (that is where the receiver is requesting the sender to create an + addressable node). The address of the source MUST be set when sent on a attach frame sent by the sending + Link Endpoint where the dynamic fiag is set to true (that is where the sender has created an addressable + node at the request of the receiver and is now communicating the address of that created node). + The generated name of the address SHOULD include the link name and the container-id of the remote container + to allow for ease of identification. + :param ~uamqp.endpoints.TerminusDurability durable: Indicates the durability of the terminus. + Indicates what state of the terminus will be retained durably: the state of durable messages, only + existence and configuration of the terminus, or no state at all. + :param ~uamqp.endpoints.ExpiryPolicy expiry_policy: The expiry policy of the Source. + Determines when the expiry timer of a Terminus starts counting down from the timeout value. If the link + is subsequently re-attached before the Terminus is expired, then the count down is aborted. If the + conditions for the terminus-expiry-policy are subsequently re-met, the expiry timer restarts from its + originally configured timeout value. + :param int timeout: Duration that an expiring Source will be retained in seconds. + The Source starts expiring as indicated by the expiry-policy. + :param bool dynamic: Request dynamic creation of a remote Node. + When set to true by the receiving Link endpoint, this field constitutes a request for the sending peer + to dynamically create a Node at the source. In this case the address field MUST NOT be set. When set to + true by the sending Link Endpoint this field indicates creation of a dynamically created Node. In this case + the address field will contain the address of the created Node. The generated address SHOULD include the + Link name and Session-name or client-id in some recognizable form for ease of traceability. + :param dict dynamic_node_properties: Properties of the dynamically created Node. + If the dynamic field is not set to true this field must be left unset. When set by the receiving Link + endpoint, this field contains the desired properties of the Node the receiver wishes to be created. When + set by the sending Link endpoint this field contains the actual properties of the dynamically created node. + :param list(bytes) capabilities: The extension capabilities the sender supports/desires. + See http://www.amqp.org/specification/1.0/source-capabilities. + """ diff --git a/src/azure/iot/hub/_pyamqp/error.py b/src/azure/iot/hub/_pyamqp/error.py new file mode 100644 index 0000000..b4a2aaa --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/error.py @@ -0,0 +1,352 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +# TODO: fix mypy errors for _code/_definition/__defaults__ (issue #26500) +from typing import Any, Union, Optional +from collections import namedtuple + +from enum import Enum +from .constants import SECURE_PORT, FIELD +from .types import AMQPTypes, FieldDefinition + + +class ErrorCondition(bytes, Enum): + # Shared error conditions: + + #: An internal error occurred. Operator intervention may be required to resume normaloperation. + InternalError = b"amqp:internal-error" + #: A peer attempted to work with a remote entity that does not exist. + NotFound = b"amqp:not-found" + #: A peer attempted to work with a remote entity to which it has no access due tosecurity settings. + UnauthorizedAccess = b"amqp:unauthorized-access" + #: Data could not be decoded. + DecodeError = b"amqp:decode-error" + #: A peer exceeded its resource allocation. + ResourceLimitExceeded = b"amqp:resource-limit-exceeded" + #: The peer tried to use a frame in a manner that is inconsistent with the semantics defined in the specification. + NotAllowed = b"amqp:not-allowed" + #: An invalid field was passed in a frame body, and the operation could not proceed. + InvalidField = b"amqp:invalid-field" + #: The peer tried to use functionality that is not implemented in its partner. + NotImplemented = b"amqp:not-implemented" + #: The client attempted to work with a server entity to which it has no access + #: because another client is working with it. + ResourceLocked = b"amqp:resource-locked" + #: The client made a request that was not allowed because some precondition failed. + PreconditionFailed = b"amqp:precondition-failed" + #: A server entity the client is working with has been deleted. + ResourceDeleted = b"amqp:resource-deleted" + #: The peer sent a frame that is not permitted in the current state of the Session. + IllegalState = b"amqp:illegal-state" + #: The peer cannot send a frame because the smallest encoding of the performative with the currently + #: valid values would be too large to fit within a frame of the agreed maximum frame size. + FrameSizeTooSmall = b"amqp:frame-size-too-small" + + # Symbols used to indicate connection error conditions: + + #: An operator intervened to close the Connection for some reason. The client may retry at some later date. + ConnectionCloseForced = b"amqp:connection:forced" + #: A valid frame header cannot be formed from the incoming byte stream. + ConnectionFramingError = b"amqp:connection:framing-error" + #: The container is no longer available on the current connection. The peer should attempt reconnection + #: to the container using the details provided in the info map. + ConnectionRedirect = b"amqp:connection:redirect" + + # Symbols used to indicate session error conditions: + + #: The peer violated incoming window for the session. + SessionWindowViolation = b"amqp:session:window-violation" + #: Input was received for a link that was detached with an error. + SessionErrantLink = b"amqp:session:errant-link" + #: An attach was received using a handle that is already in use for an attached Link. + SessionHandleInUse = b"amqp:session:handle-in-use" + #: A frame (other than attach) was received referencing a handle which + #: is not currently in use of an attached Link. + SessionUnattachedHandle = b"amqp:session:unattached-handle" + + # Symbols used to indicate link error conditions: + + #: An operator intervened to detach for some reason. + LinkDetachForced = b"amqp:link:detach-forced" + #: The peer sent more Message transfers than currently allowed on the link. + LinkTransferLimitExceeded = b"amqp:link:transfer-limit-exceeded" + #: The peer sent a larger message than is supported on the link. + LinkMessageSizeExceeded = b"amqp:link:message-size-exceeded" + #: The address provided cannot be resolved to a terminus at the current container. + LinkRedirect = b"amqp:link:redirect" + #: The link has been attached elsewhere, causing the existing attachment to be forcibly closed. + LinkStolen = b"amqp:link:stolen" + + # Customized symbols used to indicate client error conditions. + # TODO: check whether Client/Unknown/Vendor Error are exposed in EH/SB as users might be depending + # on the code for error handling + ClientError = b"amqp:client-error" + UnknownError = b"amqp:unknown-error" + VendorError = b"amqp:vendor-error" + SocketError = b"amqp:socket-error" + + +class RetryMode(str, Enum): # pylint: disable=enum-must-inherit-case-insensitive-enum-meta + EXPONENTIAL = "exponential" + FIXED = "fixed" + + +class RetryPolicy: + + no_retry = [ + ErrorCondition.DecodeError, + ErrorCondition.LinkMessageSizeExceeded, + ErrorCondition.NotFound, + ErrorCondition.NotImplemented, + ErrorCondition.LinkRedirect, + ErrorCondition.NotAllowed, + ErrorCondition.UnauthorizedAccess, + ErrorCondition.LinkStolen, + ErrorCondition.ResourceLimitExceeded, + ErrorCondition.ConnectionRedirect, + ErrorCondition.PreconditionFailed, + ErrorCondition.InvalidField, + ErrorCondition.ResourceDeleted, + ErrorCondition.IllegalState, + ErrorCondition.FrameSizeTooSmall, + ErrorCondition.ConnectionFramingError, + ErrorCondition.SessionUnattachedHandle, + ErrorCondition.SessionHandleInUse, + ErrorCondition.SessionErrantLink, + ErrorCondition.SessionWindowViolation, + ] + + def __init__(self, **kwargs): + """ + keyword int retry_total: + keyword float retry_backoff_factor: + keyword float retry_backoff_max: + keyword RetryMode retry_mode: + keyword list no_retry: + keyword dict custom_retry_policy: + """ + self.total_retries = kwargs.pop("retry_total", 3) + # TODO: A. consider letting retry_backoff_factor be either a float or a callback obj which returns a float + # to give more extensibility on customization of retry backoff time, the callback could take the exception + # as input. + self.backoff_factor = kwargs.pop("retry_backoff_factor", 0.8) + self.backoff_max = kwargs.pop("retry_backoff_max", 120) + self.retry_mode = kwargs.pop("retry_mode", RetryMode.EXPONENTIAL) + self.no_retry.extend(kwargs.get("no_retry", [])) + self.custom_condition_backoff = kwargs.pop("custom_condition_backoff", None) + # TODO: B. As an alternative of option A, we could have a new kwarg serve the goal + + def configure_retries(self, **kwargs): + return { + "total": kwargs.pop("retry_total", self.total_retries), + "backoff": kwargs.pop("retry_backoff_factor", self.backoff_factor), + "max_backoff": kwargs.pop("retry_backoff_max", self.backoff_max), + "retry_mode": kwargs.pop("retry_mode", self.retry_mode), + "history": [], + } + + def increment(self, settings, error): + settings["total"] -= 1 + settings["history"].append(error) + if settings["total"] < 0: + return False + return True + + def is_retryable(self, error): + try: + if error.condition in self.no_retry: + return False + except TypeError: + pass + return True + + def get_backoff_time(self, settings, error): + try: + return self.custom_condition_backoff[error.condition] + except (KeyError, TypeError): + pass + + consecutive_errors_len = len(settings["history"]) + if consecutive_errors_len <= 1: + return 0 + + if self.retry_mode == RetryMode.FIXED: + backoff_value = settings["backoff"] + else: + backoff_value = settings["backoff"] * (2 ** (consecutive_errors_len - 1)) + return min(settings["max_backoff"], backoff_value) + + +AMQPError = namedtuple("AMQPError", ["condition", "description", "info"], defaults=[None, None]) +AMQPError.__new__.__defaults__ = (None,) * len(AMQPError._fields) # type: ignore +AMQPError._code = 0x0000001D # type: ignore # pylint: disable=protected-access +AMQPError._definition = ( # type: ignore # pylint: disable=protected-access + FIELD("condition", AMQPTypes.symbol, True, None, False), + FIELD("description", AMQPTypes.string, False, None, False), + FIELD("info", FieldDefinition.fields, False, None, False), +) + + +class AMQPException(Exception): + """Base exception for all errors. + + :param bytes condition: The error code. + :keyword str description: A description of the error. + :keyword dict info: A dictionary of additional data associated with the error. + """ + + def __init__(self, condition: bytes, **kwargs: Any): + self.condition: Union[bytes, ErrorCondition] = condition or ErrorCondition.UnknownError + self.description: Optional[Union[str, bytes]] = kwargs.pop("description", None) + self.info: Optional[str] = kwargs.get("info", None) + self.message: Optional[str] = kwargs.get("message", None) + self.inner_error: Optional[str] = kwargs.get("error", None) + message = self.message or "Error condition: {}".format( + str(condition) if isinstance(condition, ErrorCondition) else condition.decode() + ) + if self.description: + if isinstance(self.description, bytes): + message += "\n Error Description: {}".format(self.description.decode()) + else: + message += "\n Error Description: {}".format(self.description) + super(AMQPException, self).__init__(message) + + +class AMQPDecodeError(AMQPException): + """An error occurred while decoding an incoming frame.""" + + +class AMQPConnectionError(AMQPException): + """Details of a Connection-level error.""" + + +class AMQPConnectionRedirect(AMQPConnectionError): + """Details of a Connection-level redirect response. + + The container is no longer available on the current connection. + The peer should attempt reconnection to the container using the details provided. + + :param bytes condition: The error code. + :keyword str description: A description of the error. + :keyword dict info: A dictionary of additional data associated with the error. + """ + + def __init__(self, condition, description=None, info=None): + self.hostname = info.get(b"hostname", b"").decode("utf-8") + self.network_host = info.get(b"network-host", b"").decode("utf-8") + self.port = int(info.get(b"port", SECURE_PORT)) + super(AMQPConnectionRedirect, self).__init__(condition, description=description, info=info) + + +class AMQPSessionError(AMQPException): + """Details of a Session-level error. + + :param bytes condition: The error code. + :keyword str description: A description of the error. + :keyword dict info: A dictionary of additional data associated with the error. + """ + + +class AMQPLinkError(AMQPException): + """Details of a Link-level error. + + :param bytes condition: The error code. + :keyword str description: A description of the error. + :keyword dict info: A dictionary of additional data associated with the error. + """ + + +class AMQPLinkRedirect(AMQPLinkError): + """Details of a Link-level redirect response. + + The address provided cannot be resolved to a terminus at the current container. + The supplied information may allow the client to locate and attach to the terminus. + + :param bytes condition: The error code. + :keyword str description: A description of the error. + :keyword dict info: A dictionary of additional data associated with the error. + """ + + def __init__(self, condition, description=None, info=None): + self.hostname = info.get(b"hostname", b"").decode("utf-8") + self.network_host = info.get(b"network-host", b"").decode("utf-8") + self.port = int(info.get(b"port", SECURE_PORT)) + self.address = info.get(b"address", b"").decode("utf-8") + super().__init__(condition, description=description, info=info) + + +class AuthenticationException(AMQPException): + """Details of a Authentication error. + + :param bytes condition: The error code. + :keyword str description: A description of the error. + :keyword dict info: A dictionary of additional data associated with the error. + """ + + +class TokenExpired(AuthenticationException): + """Details of a Token expiration error. + + :param bytes condition: The error code. + :keyword str description: A description of the error. + :keyword dict info: A dictionary of additional data associated with the error. + """ + + +class TokenAuthFailure(AuthenticationException): + """Failure to authenticate with token.""" + + def __init__(self, status_code, status_description, **kwargs): + encoding = kwargs.get("encoding", "utf-8") + self.status_code = status_code + self.status_description = status_description + message = "CBS Token authentication failed.\nStatus code: {}".format(self.status_code) + if self.status_description: + try: + message += "\nDescription: {}".format(self.status_description.decode(encoding)) + except (TypeError, AttributeError): + message += "\nDescription: {}".format(self.status_description) + super(TokenAuthFailure, self).__init__(condition=ErrorCondition.ClientError, message=message) + + +class MessageException(AMQPException): + """Details of a Message error. + + :param bytes condition: The error code. + :keyword str description: A description of the error. + :keyword dict info: A dictionary of additional data associated with the error. + + """ + + +class MessageSendFailed(MessageException): + """Details of a Message send failed error. + + :param bytes condition: The error code. + :keyword str description: A description of the error. + :keyword dict info: A dictionary of additional data associated with the error. + """ + + +class ErrorResponse(object): + """AMQP error object.""" + + def __init__(self, **kwargs): + self.condition = kwargs.get("condition") + self.description = kwargs.get("description") + + info = kwargs.get("info") + error_info = kwargs.get("error_info") + if isinstance(error_info, list) and len(error_info) >= 1: + if isinstance(error_info[0], list) and len(error_info[0]) >= 1: + self.condition = error_info[0][0] + if len(error_info[0]) >= 2: + self.description = error_info[0][1] + if len(error_info[0]) >= 3: + info = error_info[0][2] + + self.info = info + self.error = error_info diff --git a/src/azure/iot/hub/_pyamqp/link.py b/src/azure/iot/hub/_pyamqp/link.py new file mode 100644 index 0000000..c76b0e6 --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/link.py @@ -0,0 +1,276 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from typing import Any, Optional, TYPE_CHECKING +import uuid +import logging + +from .error import AMQPError, ErrorCondition, AMQPLinkError, AMQPLinkRedirect, AMQPConnectionError +from .endpoints import Source, Target +from .constants import DEFAULT_LINK_CREDIT, SessionState, LinkState, Role, SenderSettleMode, ReceiverSettleMode +from .performatives import AttachFrame, DetachFrame + +if TYPE_CHECKING: + from .session import Session + +_LOGGER = logging.getLogger(__name__) + + +class Link: # pylint: disable=too-many-instance-attributes + """An AMQP Link. + + This object should not be used directly - instead use one of directional + derivatives: Sender or Receiver. + """ + + def __init__( + self, session: "Session", handle: int, name: Optional[str] = None, role: bool = Role.Receiver, **kwargs: Any + ) -> None: + self.state = LinkState.DETACHED + self.name = name or str(uuid.uuid4()) + self.handle = handle + self.remote_handle = None + self.role = role + source_address = kwargs["source_address"] + target_address = kwargs["target_address"] + self.source = ( + source_address + if isinstance(source_address, Source) + else Source( + address=kwargs["source_address"], + durable=kwargs.get("source_durable"), + expiry_policy=kwargs.get("source_expiry_policy"), + timeout=kwargs.get("source_timeout"), + dynamic=kwargs.get("source_dynamic"), + dynamic_node_properties=kwargs.get("source_dynamic_node_properties"), + distribution_mode=kwargs.get("source_distribution_mode"), + filters=kwargs.get("source_filters"), + default_outcome=kwargs.get("source_default_outcome"), + outcomes=kwargs.get("source_outcomes"), + capabilities=kwargs.get("source_capabilities"), + ) + ) + self.target = ( + target_address + if isinstance(target_address, Target) + else Target( + address=kwargs["target_address"], + durable=kwargs.get("target_durable"), + expiry_policy=kwargs.get("target_expiry_policy"), + timeout=kwargs.get("target_timeout"), + dynamic=kwargs.get("target_dynamic"), + dynamic_node_properties=kwargs.get("target_dynamic_node_properties"), + capabilities=kwargs.get("target_capabilities"), + ) + ) + link_credit = kwargs.get("link_credit") + self.link_credit = link_credit if link_credit is not None else DEFAULT_LINK_CREDIT + self.current_link_credit = self.link_credit + self.send_settle_mode = kwargs.pop("send_settle_mode", SenderSettleMode.Mixed) + self.rcv_settle_mode = kwargs.pop("rcv_settle_mode", ReceiverSettleMode.First) + self.unsettled = kwargs.pop("unsettled", None) + self.incomplete_unsettled = kwargs.pop("incomplete_unsettled", None) + self.initial_delivery_count = kwargs.pop("initial_delivery_count", 0) + self.delivery_count = self.initial_delivery_count + self.received_delivery_id = None + self.max_message_size = kwargs.pop("max_message_size", None) + self.remote_max_message_size = None + self.available = kwargs.pop("available", None) + self.properties = kwargs.pop("properties", None) + self.remote_properties = None + self.offered_capabilities = None + self.desired_capabilities = kwargs.pop("desired_capabilities", None) + + self.network_trace = kwargs["network_trace"] + self.network_trace_params = kwargs["network_trace_params"] + self.network_trace_params["amqpLink"] = self.name + self._session = session + self._is_closed = False + self._on_link_state_change = kwargs.get("on_link_state_change") + self._on_attach = kwargs.get("on_attach") + self._error: Optional[AMQPLinkError] = None + + def __enter__(self) -> "Link": + self.attach() + return self + + def __exit__(self, *args) -> None: + self.detach(close=True) + + @classmethod + def from_incoming_frame(cls, session, handle, frame): + # TODO: Assuming we establish all links for now... + # check link_create_from_endpoint in C lib + raise NotImplementedError("Pending") + + def get_state(self) -> LinkState: + if self._error: + raise self._error + + return self.state + + def _check_if_closed(self) -> None: + if self._is_closed: + if self._error: + raise self._error + raise AMQPConnectionError( + condition=ErrorCondition.InternalError, description="Link already closed." + ) from None + + def _set_state(self, new_state: LinkState) -> None: + """Update the link state. + :param ~pyamqp.constants.LinkState new_state: The new state. + """ + if new_state is None: + return + previous_state = self.state + self.state = new_state + _LOGGER.info("Link state changed: %r -> %r", previous_state, new_state, extra=self.network_trace_params) + try: + if self._on_link_state_change is not None: + self._on_link_state_change(previous_state, new_state) + except Exception as e: # pylint: disable=broad-except + _LOGGER.error("Link state change callback failed: '%r'", e, extra=self.network_trace_params) + + def _on_session_state_change(self) -> None: + if self._session.state == SessionState.MAPPED: + if not self._is_closed and self.state == LinkState.DETACHED: + self._outgoing_attach() + self._set_state(LinkState.ATTACH_SENT) + elif self._session.state == SessionState.DISCARDING: + self._set_state(LinkState.DETACHED) + + def _outgoing_attach(self) -> None: + self.delivery_count = self.initial_delivery_count + attach_frame = AttachFrame( + name=self.name, + handle=self.handle, + role=self.role, + send_settle_mode=self.send_settle_mode, + rcv_settle_mode=self.rcv_settle_mode, + source=self.source, + target=self.target, + unsettled=self.unsettled, + incomplete_unsettled=self.incomplete_unsettled, + initial_delivery_count=self.initial_delivery_count if self.role == Role.Sender else None, + max_message_size=self.max_message_size, + offered_capabilities=self.offered_capabilities if self.state == LinkState.ATTACH_RCVD else None, + desired_capabilities=self.desired_capabilities if self.state == LinkState.DETACHED else None, + properties=self.properties, + ) + if self.network_trace: + _LOGGER.debug("-> %r", attach_frame, extra=self.network_trace_params) + self._session._outgoing_attach(attach_frame) # pylint: disable=protected-access + + def _incoming_attach(self, frame) -> None: + if self.network_trace: + _LOGGER.debug("<- %r", AttachFrame(*frame), extra=self.network_trace_params) + if self._is_closed: + raise ValueError("Invalid link") + if not frame[5] or not frame[6]: + _LOGGER.info("Cannot get source or target. Detaching link", extra=self.network_trace_params) + self._set_state(LinkState.DETACHED) + raise ValueError("Invalid link") + self.remote_handle = frame[1] # handle + self.remote_max_message_size = frame[10] # max_message_size + self.offered_capabilities = frame[11] # offered_capabilities + self.remote_properties = frame[13] # incoming map of properties about the link + if self.state == LinkState.DETACHED: + self._set_state(LinkState.ATTACH_RCVD) + elif self.state == LinkState.ATTACH_SENT: + self._set_state(LinkState.ATTACHED) + if self._on_attach: + try: + if frame[5]: + frame[5] = Source(*frame[5]) + if frame[6]: + frame[6] = Target(*frame[6]) + self._on_attach(AttachFrame(*frame)) + except Exception as e: # pylint: disable=broad-except + _LOGGER.warning("Callback for link attach raised error: %r", e, extra=self.network_trace_params) + + def _outgoing_flow(self, **kwargs: Any) -> None: + flow_frame = { + "handle": self.handle, + "delivery_count": self.delivery_count, + "link_credit": self.current_link_credit, + "available": kwargs.get("available"), + "drain": kwargs.get("drain"), + "echo": kwargs.get("echo"), + "properties": kwargs.get("properties"), + } + self._session._outgoing_flow(flow_frame) # pylint: disable=protected-access + + def _incoming_flow(self, frame): + pass + + def _incoming_disposition(self, frame): + pass + + def _outgoing_detach(self, close: bool = False, error: Optional[AMQPError] = None) -> None: + detach_frame = DetachFrame(handle=self.handle, closed=close, error=error) + if self.network_trace: + _LOGGER.debug("-> %r", detach_frame, extra=self.network_trace_params) + self._session._outgoing_detach(detach_frame) # pylint: disable=protected-access + if close: + self._is_closed = True + + def _incoming_detach(self, frame) -> None: + if self.network_trace: + _LOGGER.debug("<- %r", DetachFrame(*frame), extra=self.network_trace_params) + if self.state == LinkState.ATTACHED: + self._outgoing_detach(close=frame[1]) # closed + elif frame[1] and not self._is_closed and self.state in [LinkState.ATTACH_SENT, LinkState.ATTACH_RCVD]: + # Received a closing detach after we sent a non-closing detach. + # In this case, we MUST signal that we closed by reattaching and then sending a closing detach. + self._outgoing_attach() + self._outgoing_detach(close=True) + # TODO: on_detach_hook + if frame[2]: # error + # frame[2][0] is condition, frame[2][1] is description, frame[2][2] is info + condition = frame[2][0] + error_cls = AMQPLinkRedirect if condition == ErrorCondition.LinkRedirect else AMQPLinkError + + # description and info are optional fields, from the AMQP spec. + # https://docs.oasis-open.org/amqp/core/v1.0/os/amqp-core-transport-v1.0-os.html#type-error + description = None if len(frame[2]) < 2 else frame[2][1] + info = None if len(frame[2]) < 3 else frame[2][2] + + self._error = error_cls(condition=condition, description=description, info=info) + self._set_state(LinkState.ERROR) + else: + if self.state != LinkState.DETACH_SENT: + # Handle the case of when the remote side detaches without sending an error. + # We should detach as per the spec but then retry connecting + self._error = AMQPLinkError( + condition=ErrorCondition.UnknownError, description="Link detached unexpectedly.", retryable=True + ) + self._set_state(LinkState.DETACHED) + + def attach(self) -> None: + if self._is_closed: + raise ValueError("Link already closed.") + self._outgoing_attach() + self._set_state(LinkState.ATTACH_SENT) + + def detach(self, close: bool = False, error: Optional[AMQPError] = None) -> None: + if self.state in (LinkState.DETACHED, LinkState.DETACH_SENT, LinkState.ERROR): + return + try: + self._check_if_closed() + if self.state in [LinkState.ATTACH_SENT, LinkState.ATTACH_RCVD]: + self._outgoing_detach(close=close, error=error) + self._set_state(LinkState.DETACHED) + elif self.state == LinkState.ATTACHED: + self._outgoing_detach(close=close, error=error) + self._set_state(LinkState.DETACH_SENT) + except Exception as exc: # pylint: disable=broad-except + _LOGGER.info("An error occurred when detaching the link: %r", exc, extra=self.network_trace_params) + self._set_state(LinkState.DETACHED) + + def flow(self, *, link_credit: Optional[int] = None, **kwargs: Any) -> None: + self.current_link_credit = link_credit if link_credit is not None else self.link_credit + self._outgoing_flow(**kwargs) diff --git a/src/azure/iot/hub/_pyamqp/management_link.py b/src/azure/iot/hub/_pyamqp/management_link.py new file mode 100644 index 0000000..5a680f8 --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/management_link.py @@ -0,0 +1,260 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import time +import logging +from uuid import uuid4 +from functools import partial +from collections import namedtuple +from typing import Optional, Union +from threading import Lock + +from .sender import SenderLink +from .receiver import ReceiverLink +from .constants import ( + ManagementLinkState, + LinkState, + SenderSettleMode, + ReceiverSettleMode, + ManagementExecuteOperationResult, + ManagementOpenResult, + SEND_DISPOSITION_REJECT, + MessageDeliveryState, + LinkDeliverySettleReason, +) +from .error import AMQPException, ErrorCondition +from .message import Properties, _MessageDelivery + +_LOGGER = logging.getLogger(__name__) + +PendingManagementOperation = namedtuple("PendingManagementOperation", ["message", "on_execute_operation_complete"]) + + +class ManagementLink(object): # pylint:disable=too-many-instance-attributes + """ + # TODO: Fill in docstring + """ + + def __init__(self, session, endpoint, **kwargs): + self.state = ManagementLinkState.IDLE + self._pending_operations = [] + self.lock = Lock() + self._session = session + self._network_trace_params = kwargs.get("network_trace_params") + self._request_link: SenderLink = session.create_sender_link( + endpoint, + source_address=endpoint, + on_link_state_change=self._on_sender_state_change, + send_settle_mode=SenderSettleMode.Unsettled, + rcv_settle_mode=ReceiverSettleMode.First, + network_trace=kwargs.get("network_trace", False), + ) + self._response_link: ReceiverLink = session.create_receiver_link( + endpoint, + target_address=endpoint, + on_link_state_change=self._on_receiver_state_change, + on_transfer=self._on_message_received, + send_settle_mode=SenderSettleMode.Unsettled, + rcv_settle_mode=ReceiverSettleMode.First, + network_trace=kwargs.get("network_trace", False), + ) + self._on_amqp_management_error = kwargs.get("on_amqp_management_error") + self._on_amqp_management_open_complete = kwargs.get("on_amqp_management_open_complete") + + self._status_code_field = kwargs.get("status_code_field", b"statusCode") + self._status_description_field = kwargs.get("status_description_field", b"statusDescription") + + self._sender_connected = False + self._receiver_connected = False + + def __enter__(self): + self.open() + return self + + def __exit__(self, *args): + self.close() + + def _on_sender_state_change(self, previous_state, new_state): + _LOGGER.info( + "Management link sender state changed: %r -> %r", + previous_state, + new_state, + extra=self._network_trace_params, + ) + if new_state == previous_state: + return + if self.state == ManagementLinkState.OPENING: + if new_state == LinkState.ATTACHED: + self._sender_connected = True + if self._receiver_connected: + self.state = ManagementLinkState.OPEN + self._on_amqp_management_open_complete(ManagementOpenResult.OK) + elif new_state in [LinkState.DETACHED, LinkState.DETACH_SENT, LinkState.DETACH_RCVD, LinkState.ERROR]: + self.state = ManagementLinkState.IDLE + self._on_amqp_management_open_complete(ManagementOpenResult.ERROR) + elif self.state == ManagementLinkState.OPEN: + if new_state is not LinkState.ATTACHED: + self.state = ManagementLinkState.ERROR + self._on_amqp_management_error() + elif self.state == ManagementLinkState.CLOSING: + if new_state not in [LinkState.DETACHED, LinkState.DETACH_SENT, LinkState.DETACH_RCVD]: + self.state = ManagementLinkState.ERROR + self._on_amqp_management_error() + elif self.state == ManagementLinkState.ERROR: + # All state transitions shall be ignored. + return + + def _on_receiver_state_change(self, previous_state, new_state): + _LOGGER.info( + "Management link receiver state changed: %r -> %r", + previous_state, + new_state, + extra=self._network_trace_params, + ) + if new_state == previous_state: + return + if self.state == ManagementLinkState.OPENING: + if new_state == LinkState.ATTACHED: + self._receiver_connected = True + if self._sender_connected: + self.state = ManagementLinkState.OPEN + self._on_amqp_management_open_complete(ManagementOpenResult.OK) + elif new_state in [LinkState.DETACHED, LinkState.DETACH_SENT, LinkState.DETACH_RCVD, LinkState.ERROR]: + self.state = ManagementLinkState.IDLE + self._on_amqp_management_open_complete(ManagementOpenResult.ERROR) + elif self.state == ManagementLinkState.OPEN: + if new_state is not LinkState.ATTACHED: + self.state = ManagementLinkState.ERROR + self._on_amqp_management_error() + elif self.state == ManagementLinkState.CLOSING: + if new_state not in [LinkState.DETACHED, LinkState.DETACH_SENT, LinkState.DETACH_RCVD]: + self.state = ManagementLinkState.ERROR + self._on_amqp_management_error() + elif self.state == ManagementLinkState.ERROR: + # All state transitions shall be ignored. + return + + def _on_message_received(self, _, message): + message_properties = message.properties + correlation_id = message_properties[5] + response_detail = message.application_properties + + status_code = response_detail.get(self._status_code_field) + status_description = response_detail.get(self._status_description_field) + + to_remove_operation = None + for operation in self._pending_operations: + if operation.message.properties.message_id == correlation_id: + to_remove_operation = operation + break + if to_remove_operation: + mgmt_result = ( + ManagementExecuteOperationResult.OK + if 200 <= status_code <= 299 + else ManagementExecuteOperationResult.FAILED_BAD_STATUS + ) + to_remove_operation.on_execute_operation_complete( + mgmt_result, status_code, status_description, message, response_detail.get(b"error-condition") + ) + with self.lock: + self._pending_operations.remove(to_remove_operation) + + def _on_send_complete(self, message_delivery, reason, state): # todo: reason is never used, should check spec + if reason == LinkDeliverySettleReason.DISPOSITION_RECEIVED and SEND_DISPOSITION_REJECT in state: + # sample reject state: {'rejected': [[b'amqp:not-allowed', b"Invalid command 'RE1AD'.", None]]} + to_remove_operation = None + for operation in self._pending_operations: + if message_delivery.message == operation.message: + to_remove_operation = operation + break + self._pending_operations.remove(to_remove_operation) + # TODO: better error handling + # AMQPException is too general? to be more specific: MessageReject(Error) or AMQPManagementError? + # or should there an error mapping which maps the condition to the error type + to_remove_operation.on_execute_operation_complete( # The callback is defined in management_operation.py + ManagementExecuteOperationResult.ERROR, + None, + None, + message_delivery.message, + error=AMQPException( + condition=state[SEND_DISPOSITION_REJECT][0][0], # 0 is error condition + description=state[SEND_DISPOSITION_REJECT][0][1], # 1 is error description + info=state[SEND_DISPOSITION_REJECT][0][2], # 2 is error info + ), + ) + + def open(self): + if self.state != ManagementLinkState.IDLE: + raise ValueError("Management links are already open or opening.") + self.state = ManagementLinkState.OPENING + self._response_link.attach() + self._request_link.attach() + + def execute_operation( + self, + message, + on_execute_operation_complete, + *, + operation: Optional[Union[bytes, str]] = None, + type: Optional[Union[bytes, str]] = None, + locales: Optional[str] = None, + timeout: Optional[float] = None + ): + """Execute a request and wait on a response. + + :param message: The message to send in the management request. + :type message: ~uamqp.message.Message + :param on_execute_operation_complete: Callback to be called when the operation is complete. + The following value will be passed to the callback: operation_id, operation_result, status_code, + status_description, raw_message and error. + :type on_execute_operation_complete: Callable[[str, str, int, str, ~uamqp.message.Message, Exception], None] + :keyword operation: The type of operation to be performed. This value will + be service-specific, but common values include READ, CREATE and UPDATE. + This value will be added as an application property on the message. + :paramtype operation: bytes or str + :keyword type: The type on which to carry out the operation. This will + be specific to the entities of the service. This value will be added as + an application property on the message. + :paramtype type: bytes or str + :keyword str locales: A list of locales that the sending peer permits for incoming + informational text in response messages. + :keyword float timeout: Provide an optional timeout in seconds within which a response + to the management request must be received. + :rtype: None + """ + message.application_properties["operation"] = operation + message.application_properties["type"] = type + if locales: + message.application_properties["locales"] = locales + try: + # TODO: namedtuple is immutable, which may push us to re-think about the namedtuple approach for Message + new_properties = message.properties._replace(message_id=uuid4()) + except AttributeError: + new_properties = Properties(message_id=uuid4()) + message = message._replace(properties=new_properties) + expire_time = (time.time() + timeout) if timeout else None + message_delivery = _MessageDelivery(message, MessageDeliveryState.WaitingToBeSent, expire_time) + + on_send_complete = partial(self._on_send_complete, message_delivery) + + self._request_link.send_transfer(message, on_send_complete=on_send_complete, timeout=timeout) + self._pending_operations.append(PendingManagementOperation(message, on_execute_operation_complete)) + + def close(self): + if self.state != ManagementLinkState.IDLE: + self.state = ManagementLinkState.CLOSING + self._response_link.detach(close=True) + self._request_link.detach(close=True) + for pending_operation in self._pending_operations: + pending_operation.on_execute_operation_complete( + ManagementExecuteOperationResult.LINK_CLOSED, + None, + None, + pending_operation.message, + AMQPException(condition=ErrorCondition.ClientError, description="Management link already closed."), + ) + self._pending_operations = [] + self.state = ManagementLinkState.IDLE diff --git a/src/azure/iot/hub/_pyamqp/management_operation.py b/src/azure/iot/hub/_pyamqp/management_operation.py new file mode 100644 index 0000000..59abd1b --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/management_operation.py @@ -0,0 +1,125 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import logging +import uuid +import time +from functools import partial + +from .management_link import ManagementLink +from .error import AMQPLinkError, ErrorCondition + +from .constants import ManagementOpenResult, ManagementExecuteOperationResult + +_LOGGER = logging.getLogger(__name__) + + +class ManagementOperation(object): + def __init__(self, session, endpoint="$management", **kwargs): + self._mgmt_link_open_status = None + + self._session = session + self._connection = self._session._connection + self._network_trace_params = { + "amqpConnection": self._session._connection._container_id, + "amqpSession": self._session.name, + "amqpLink": "", + } + self._mgmt_link: ManagementLink = self._session.create_request_response_link_pair( + endpoint=endpoint, + on_amqp_management_open_complete=self._on_amqp_management_open_complete, + on_amqp_management_error=self._on_amqp_management_error, + **kwargs + ) + self._responses = {} + self._mgmt_error = None + + def _on_amqp_management_open_complete(self, result): + """Callback run when the send/receive links are open and ready + to process messages. + + :param result: Whether the link opening was successful. + :type result: int + """ + self._mgmt_link_open_status = result + + def _on_amqp_management_error(self): + """Callback run if an error occurs in the send/receive links.""" + # TODO: This probably shouldn't be ValueError + self._mgmt_error = ValueError("Management Operation error occurred.") + + def _on_execute_operation_complete( + self, operation_id, operation_result, status_code, status_description, raw_message, error=None + ): + _LOGGER.debug( + "Management operation completed, id: %r; result: %r; code: %r; description: %r, error: %r", + operation_id, + operation_result, + status_code, + status_description, + error, + extra=self._network_trace_params, + ) + + if operation_result in (ManagementExecuteOperationResult.ERROR, ManagementExecuteOperationResult.LINK_CLOSED): + self._mgmt_error = error + _LOGGER.error( + "Failed to complete management operation due to error: %r.", error, extra=self._network_trace_params + ) + else: + self._responses[operation_id] = (status_code, status_description, raw_message) + + def execute(self, message, operation=None, operation_type=None, timeout=0): + start_time = time.time() + operation_id = str(uuid.uuid4()) + self._responses[operation_id] = None + self._mgmt_error = None + + self._mgmt_link.execute_operation( + message, + partial(self._on_execute_operation_complete, operation_id), + timeout=timeout, + operation=operation, + type=operation_type, + ) + + while not self._responses[operation_id] and not self._mgmt_error: + if timeout and timeout > 0: + now = time.time() + if (now - start_time) >= timeout: + raise TimeoutError("Failed to receive mgmt response in {}ms".format(timeout)) + self._connection.listen() + + if self._mgmt_error: + self._responses.pop(operation_id) + raise self._mgmt_error + + response = self._responses.pop(operation_id) + return response + + def open(self): + self._mgmt_link_open_status = ManagementOpenResult.OPENING + self._mgmt_link.open() + + def ready(self): + try: + raise self._mgmt_error + except TypeError: + pass + + if self._mgmt_link_open_status == ManagementOpenResult.OPENING: + return False + if self._mgmt_link_open_status == ManagementOpenResult.OK: + return True + # ManagementOpenResult.ERROR or CANCELLED + # TODO: update below with correct status code + info + raise AMQPLinkError( + condition=ErrorCondition.ClientError, + description="Failed to open mgmt link, management link status: {}".format(self._mgmt_link_open_status), + info=None, + ) + + def close(self): + self._mgmt_link.close() diff --git a/src/azure/iot/hub/_pyamqp/message.py b/src/azure/iot/hub/_pyamqp/message.py new file mode 100644 index 0000000..800a8cf --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/message.py @@ -0,0 +1,287 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +# TODO: fix mypy errors for _code/_definition/__defaults__ (issue #26500) +from typing import NamedTuple, Optional, Union, TYPE_CHECKING, Dict, Any, List, Iterable +from typing_extensions import TypedDict + +from .types import AMQPTypes, FieldDefinition +from .constants import FIELD, MessageDeliveryState +from .performatives import _CAN_ADD_DOCSTRING + +if TYPE_CHECKING: + from uuid import UUID + + class MessageDict(TypedDict): # needed for use with spread operator + """ + Typing for Message, used with the spread operator. + """ + + header: Optional["Header"] + delivery_annotations: Optional[Dict[Union[str, bytes], Any]] + message_annotations: Optional[Dict[Union[str, bytes], Any]] + properties: Optional["Properties"] + application_properties: Optional[Dict[Union[str, bytes], Any]] + data: Optional[Union[bytes, Iterable[bytes]]] + sequence: Optional[List[Any]] + value: Optional[Any] + footer: Optional[Dict[Any, Any]] + + +class Header(NamedTuple): + durable: Optional[bool] = None + priority: Optional[int] = None + ttl: Optional[int] = None + first_acquirer: Optional[bool] = None + delivery_count: Optional[int] = None + + +Header._code = 0x00000070 # type: ignore # pylint:disable=protected-access +Header._definition = ( # type: ignore # pylint:disable=protected-access + FIELD("durable", AMQPTypes.boolean, False, None, False), + FIELD("priority", AMQPTypes.ubyte, False, None, False), + FIELD("ttl", AMQPTypes.uint, False, None, False), + FIELD("first_acquirer", AMQPTypes.boolean, False, None, False), + FIELD("delivery_count", AMQPTypes.uint, False, None, False), +) + +if _CAN_ADD_DOCSTRING: + Header.__doc__ = """ + Transport headers for a Message. + + The header section carries standard delivery details about the transfer of a Message through the AMQP + network. If the header section is omitted the receiver MUST assume the appropriate default values for + the fields within the header unless other target or node specific defaults have otherwise been set. + + :param bool durable: Specify durability requirements. + Durable Messages MUST NOT be lost even if an intermediary is unexpectedly terminated and restarted. + A target which is not capable of fulfilling this guarantee MUST NOT accept messages where the durable + header is set to true: if the source allows the rejected outcome then the message should be rejected + with the precondition-failed error, otherwise the link must be detached by the receiver with the same error. + :param int priority: Relative Message priority. + This field contains the relative Message priority. Higher numbers indicate higher priority Messages. + Messages with higher priorities MAY be delivered before those with lower priorities. An AMQP intermediary + implementing distinct priority levels MUST do so in the following manner: + + - If n distince priorities are implemented and n is less than 10 - priorities 0 to (5 - ceiling(n/2)) + MUST be treated equivalently and MUST be the lowest effective priority. The priorities (4 + fioor(n/2)) + and above MUST be treated equivalently and MUST be the highest effective priority. The priorities + (5 ceiling(n/2)) to (4 + fioor(n/2)) inclusive MUST be treated as distinct priorities. + - If n distinct priorities are implemented and n is 10 or greater - priorities 0 to (n - 1) MUST be + distinct, and priorities n and above MUST be equivalent to priority (n - 1). Thus, for example, if 2 + distinct priorities are implemented, then levels 0 to 4 are equivalent, and levels 5 to 9 are equivalent + and levels 4 and 5 are distinct. If 3 distinct priorities are implements the 0 to 3 are equivalent, + 5 to 9 are equivalent and 3, 4 and 5 are distinct. This scheme ensures that if two priorities are distinct + for a server which implements m separate priority levels they are also distinct for a server which + implements n different priority levels where n > m. + + :param int ttl: Time to live in ms. + Duration in milliseconds for which the Message should be considered 'live'. If this is set then a message + expiration time will be computed based on the time of arrival at an intermediary. Messages that live longer + than their expiration time will be discarded (or dead lettered). When a message is transmitted by an + intermediary that was received with a ttl, the transmitted message's header should contain a ttl that is + computed as the difference between the current time and the formerly computed message expiration + time, i.e. the reduced ttl, so that messages will eventually die if they end up in a delivery loop. + :param bool first_acquirer: If this value is true, then this message has not been acquired by any other Link. + If this value is false, then this message may have previously been acquired by another Link or Links. + :param int delivery_count: The number of prior unsuccessful delivery attempts. + The number of unsuccessful previous attempts to deliver this message. If this value is non-zero it may + be taken as an indication that the delivery may be a duplicate. On first delivery, the value is zero. + It is incremented upon an outcome being settled at the sender, according to rules defined for each outcome. + """ + + +class Properties(NamedTuple): + message_id: Optional[Union[str, bytes, "UUID"]] = None + user_id: Optional[Union[str, bytes]] = None + to: Optional[Union[str, bytes]] = None + subject: Optional[Union[str, bytes]] = None + reply_to: Optional[Union[str, bytes]] = None + correlation_id: Optional[Union[str, bytes]] = None + content_type: Optional[Union[str, bytes]] = None + content_encoding: Optional[Union[str, bytes]] = None + absolute_expiry_time: Optional[int] = None + creation_time: Optional[int] = None + group_id: Optional[Union[str, bytes]] = None + group_sequence: Optional[int] = None + reply_to_group_id: Optional[Union[str, bytes]] = None + + +Properties._code = 0x00000073 # type: ignore # pylint:disable=protected-access +Properties._definition = ( # type: ignore # pylint:disable=protected-access + FIELD("message_id", FieldDefinition.message_id, False, None, False), + FIELD("user_id", AMQPTypes.binary, False, None, False), + FIELD("to", AMQPTypes.string, False, None, False), + FIELD("subject", AMQPTypes.string, False, None, False), + FIELD("reply_to", AMQPTypes.string, False, None, False), + FIELD("correlation_id", FieldDefinition.message_id, False, None, False), + FIELD("content_type", AMQPTypes.symbol, False, None, False), + FIELD("content_encoding", AMQPTypes.symbol, False, None, False), + FIELD("absolute_expiry_time", AMQPTypes.timestamp, False, None, False), + FIELD("creation_time", AMQPTypes.timestamp, False, None, False), + FIELD("group_id", AMQPTypes.string, False, None, False), + FIELD("group_sequence", AMQPTypes.uint, False, None, False), + FIELD("reply_to_group_id", AMQPTypes.string, False, None, False), +) + +if _CAN_ADD_DOCSTRING: + Properties.__doc__ = """ + Immutable properties of the Message. + + The properties section is used for a defined set of standard properties of the message. The properties + section is part of the bare message and thus must, if retransmitted by an intermediary, remain completely + unaltered. + + :param message_id: Application Message identifier. + Message-id is an optional property which uniquely identifies a Message within the Message system. + The Message producer is usually responsible for setting the message-id in such a way that it is assured + to be globally unique. A broker MAY discard a Message as a duplicate if the value of the message-id + matches that of a previously received Message sent to the same Node. + :param bytes user_id: Creating user id. + The identity of the user responsible for producing the Message. The client sets this value, and it MAY + be authenticated by intermediaries. + :param to: The address of the Node the Message is destined for. + The to field identifies the Node that is the intended destination of the Message. On any given transfer + this may not be the Node at the receiving end of the Link. + :param str subject: The subject of the message. + A common field for summary information about the Message content and purpose. + :param reply_to: The Node to send replies to. + The address of the Node to send replies to. + :param correlation_id: Application correlation identifier. + This is a client-specific id that may be used to mark or identify Messages between clients. + :param bytes content_type: MIME content type. + The RFC-2046 MIME type for the Message's application-data section (body). As per RFC-2046 this may contain + a charset parameter defining the character encoding used: e.g. 'text/plain; charset="utf-8"'. + For clarity, the correct MIME type for a truly opaque binary section is application/octet-stream. + When using an application-data section with a section code other than data, contenttype, if set, SHOULD + be set to a MIME type of message/x-amqp+?, where '?' is either data, map or list. + :param bytes content_encoding: MIME content type. + The Content-Encoding property is used as a modifier to the content-type. When present, its value indicates + what additional content encodings have been applied to the application-data, and thus what decoding + mechanisms must be applied in order to obtain the media-type referenced by the content-type header field. + Content-Encoding is primarily used to allow a document to be compressed without losing the identity of + its underlying content type. Content Encodings are to be interpreted as per Section 3.5 of RFC 2616. + Valid Content Encodings are registered at IANA as "Hypertext Transfer Protocol (HTTP) Parameters" + (http://www.iana.org/assignments/http-parameters/httpparameters.xml). Content-Encoding MUST not be set when + the application-data section is other than data. Implementations MUST NOT use the identity encoding. + Instead, implementations should not set this property. Implementations SHOULD NOT use the compress + encoding, except as to remain compatible with messages originally sent with other protocols, + e.g. HTTP or SMTP. Implementations SHOULD NOT specify multiple content encoding values except as to be + compatible with messages originally sent with other protocols, e.g. HTTP or SMTP. + :param datetime absolute_expiry_time: The time when this message is considered expired. + An absolute time when this message is considered to be expired. + :param datetime creation_time: The time when this message was created. + An absolute time when this message was created. + :param str group_id: The group this message belongs to. + Identifies the group the message belongs to. + :param int group_sequence: The sequence-no of this message within its group. + The relative position of this message within its group. + :param str reply_to_group_id: The group the reply message belongs to. + This is a client-specific id that is used so that client can send replies to this message to a specific group. + """ + + +# TODO: should be a class, namedtuple or dataclass, immutability vs performance, need to collect performance data +class Message(NamedTuple): + header: Optional[Header] = None + delivery_annotations: Optional[Dict[Union[str, bytes], Any]] = None + message_annotations: Optional[Dict[Union[str, bytes], Any]] = None + properties: Optional[Properties] = None + application_properties: Optional[Dict[Union[str, bytes], Any]] = None + data: Optional[Union[bytes, Iterable[bytes]]] = None + sequence: Optional[List[Any]] = None + value: Optional[Any] = None + footer: Optional[Dict[Any, Any]] = None + + +Message._code = 0 # type: ignore # pylint:disable=protected-access +Message._definition = ( # type: ignore # pylint:disable=protected-access + (0x00000070, FIELD("header", Header, False, None, False)), + (0x00000071, FIELD("delivery_annotations", FieldDefinition.annotations, False, None, False)), + (0x00000072, FIELD("message_annotations", FieldDefinition.annotations, False, None, False)), + (0x00000073, FIELD("properties", Properties, False, None, False)), + (0x00000074, FIELD("application_properties", AMQPTypes.map, False, None, False)), + (0x00000075, FIELD("data", AMQPTypes.binary, False, None, True)), + (0x00000076, FIELD("sequence", AMQPTypes.list, False, None, False)), + (0x00000077, FIELD("value", None, False, None, False)), + (0x00000078, FIELD("footer", FieldDefinition.annotations, False, None, False)), +) +if _CAN_ADD_DOCSTRING: + Message.__doc__ = """ + An annotated message consists of the bare message plus sections for annotation at the head and tail + of the bare message. + + There are two classes of annotations: annotations that travel with the message indefinitely, and + annotations that are consumed by the next node. + The exact structure of a message, together with its encoding, is defined by the message format. This document + defines the structure and semantics of message format 0 (MESSAGE-FORMAT). Altogether a message consists of the + following sections: + + - Zero or one header. + - Zero or one delivery-annotations. + - Zero or one message-annotations. + - Zero or one properties. + - Zero or one application-properties. + - The body consists of either: one or more data sections, one or more amqp-sequence sections, + or a single amqp-value section. + - Zero or one footer. + + :param ~uamqp.message.Header header: Transport headers for a Message. + The header section carries standard delivery details about the transfer of a Message through the AMQP + network. If the header section is omitted the receiver MUST assume the appropriate default values for + the fields within the header unless other target or node specific defaults have otherwise been set. + :param dict delivery_annotations: The delivery-annotations section is used for delivery-specific non-standard + properties at the head of the message. Delivery annotations convey information from the sending peer to + the receiving peer. If the recipient does not understand the annotation it cannot be acted upon and its + effects (such as any implied propagation) cannot be acted upon. Annotations may be specific to one + implementation, or common to multiple implementations. The capabilities negotiated on link attach and on + the source and target should be used to establish which annotations a peer supports. A registry of defined + annotations and their meanings can be found here: http://www.amqp.org/specification/1.0/delivery-annotations. + If the delivery-annotations section is omitted, it is equivalent to a delivery-annotations section + containing an empty map of annotations. + :param dict message_annotations: The message-annotations section is used for properties of the message which + are aimed at the infrastructure and should be propagated across every delivery step. Message annotations + convey information about the message. Intermediaries MUST propagate the annotations unless the annotations + are explicitly augmented or modified (e.g. by the use of the modified outcome). + The capabilities negotiated on link attach and on the source and target may be used to establish which + annotations a peer understands, however it a network of AMQP intermediaries it may not be possible to know + if every intermediary will understand the annotation. Note that for some annotation it may not be necessary + for the intermediary to understand their purpose - they may be being used purely as an attribute which can be + filtered on. A registry of defined annotations and their meanings can be found here: + http://www.amqp.org/specification/1.0/message-annotations. If the message-annotations section is omitted, + it is equivalent to a message-annotations section containing an empty map of annotations. + :param ~uamqp.message.Properties: Immutable properties of the Message. + The properties section is used for a defined set of standard properties of the message. The properties + section is part of the bare message and thus must, if retransmitted by an intermediary, remain completely + unaltered. + :param dict application_properties: The application-properties section is a part of the bare message used + for structured application data. Intermediaries may use the data within this structure for the purposes + of filtering or routing. The keys of this map are restricted to be of type string (which excludes the + possibility of a null key) and the values are restricted to be of simple types only (that is excluding + map, list, and array types). + :param list(bytes) data_body: A data section contains opaque binary data. + :param list sequence_body: A sequence section contains an arbitrary number of structured data elements. + :param value_body: An amqp-value section contains a single AMQP value. + :param dict footer: Transport footers for a Message. + The footer section is used for details about the message or delivery which can only be calculated or + evaluated once the whole bare message has been constructed or seen (for example message hashes, HMACs, + signatures and encryption details). A registry of defined footers and their meanings can be found + here: http://www.amqp.org/specification/1.0/footer. + """ + + +class BatchMessage(Message): + _code = 0x80013700 + + +class _MessageDelivery: + def __init__(self, message, state=MessageDeliveryState.WaitingToBeSent, expiry=None): + self.message = message + self.state = state + self.expiry = expiry + self.reason = None + self.delivery = None + self.error = None diff --git a/src/azure/iot/hub/_pyamqp/outcomes.py b/src/azure/iot/hub/_pyamqp/outcomes.py new file mode 100644 index 0000000..477791a --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/outcomes.py @@ -0,0 +1,162 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +# The Messaging layer defines a concrete set of delivery states which can be used (via the disposition frame) +# to indicate the state of the message at the receiver. + +# Delivery states may be either terminal or non-terminal. Once a delivery reaches a terminal delivery-state, +# the state for that delivery will no longer change. A terminal delivery-state is referred to as an outcome. + +# The following outcomes are formally defined by the messaging layer to indicate the result of processing at the +# receiver: + +# - accepted: indicates successful processing at the receiver +# - rejected: indicates an invalid and unprocessable message +# - released: indicates that the message was not (and will not be) processed +# - modified: indicates that the message was modified, but not processed + +# The following non-terminal delivery-state is formally defined by the messaging layer for use during link +# recovery to allow the sender to resume the transfer of a large message without retransmitting all the +# message data: + +# - received: indicates partial message data seen by the receiver as well as the starting point for a +# resumed transfer + +# TODO: fix mypy errors for _code/_definition/__defaults__ (issue #26500) +from collections import namedtuple + +from .types import AMQPTypes, FieldDefinition, ObjDefinition +from .constants import FIELD +from .performatives import _CAN_ADD_DOCSTRING + + +Received = namedtuple("Received", ["section_number", "section_offset"]) +Received._code = 0x00000023 # type: ignore # pylint:disable=protected-access +Received._definition = ( # type: ignore # pylint:disable=protected-access + FIELD("section_number", AMQPTypes.uint, True, None, False), + FIELD("section_offset", AMQPTypes.ulong, True, None, False), +) +if _CAN_ADD_DOCSTRING: + Received.__doc__ = """ + At the target the received state indicates the furthest point in the payload of the message + which the target will not need to have resent if the link is resumed. At the source the received state represents + the earliest point in the payload which the Sender is able to resume transferring at in the case of link + resumption. When resuming a delivery, if this state is set on the first transfer performative it indicates + the offset in the payload at which the first resumed delivery is starting. The Sender MUST NOT send the + received state on transfer or disposition performatives except on the first transfer performative on a + resumed delivery. + + :param int section_number: + When sent by the Sender this indicates the first section of the message (with sectionnumber 0 being the + first section) for which data can be resent. Data from sections prior to the given section cannot be + retransmitted for this delivery. When sent by the Receiver this indicates the first section of the message + for which all data may not yet have been received. + :param int section_offset: + When sent by the Sender this indicates the first byte of the encoded section data of the section given by + section-number for which data can be resent (with section-offset 0 being the first byte). Bytes from the + same section prior to the given offset section cannot be retransmitted for this delivery. When sent by the + Receiver this indicates the first byte of the given section which has not yet been received. Note that if + a receiver has received all of section number X (which contains N bytes of data), but none of section + number X + 1, then it may indicate this by sending either Received(section-number=X, section-offset=N) or + Received(section-number=X+1, section-offset=0). The state Received(sectionnumber=0, section-offset=0) + indicates that no message data at all has been transferred. + """ + + +Accepted = namedtuple("Accepted", []) +Accepted._code = 0x00000024 # type: ignore # pylint:disable=protected-access +Accepted._definition = () # type: ignore # pylint:disable=protected-access +if _CAN_ADD_DOCSTRING: + Accepted.__doc__ = """ + The accepted outcome. + + At the source the accepted state means that the message has been retired from the node, and transfer of + payload data will not be able to be resumed if the link becomes suspended. A delivery may become accepted at + the source even before all transfer frames have been sent, this does not imply that the remaining transfers + for the delivery will not be sent - only the aborted fiag on the transfer performative can be used to indicate + a premature termination of the transfer. At the target, the accepted outcome is used to indicate that an + incoming Message has been successfully processed, and that the receiver of the Message is expecting the sender + to transition the delivery to the accepted state at the source. The accepted outcome does not increment the + delivery-count in the header of the accepted Message. + """ + + +Rejected = namedtuple("Rejected", ["error"]) +Rejected.__new__.__defaults__ = (None,) * len(Rejected._fields) # type: ignore +Rejected._code = 0x00000025 # type: ignore # pylint:disable=protected-access +Rejected._definition = (FIELD("error", ObjDefinition.error, False, None, False),) # type: ignore # pylint:disable=protected-access +if _CAN_ADD_DOCSTRING: + Rejected.__doc__ = """ + The rejected outcome. + + At the target, the rejected outcome is used to indicate that an incoming Message is invalid and therefore + unprocessable. The rejected outcome when applied to a Message will cause the delivery-count to be incremented + in the header of the rejected Message. At the source, the rejected outcome means that the target has informed + the source that the message was rejected, and the source has taken the required action. The delivery SHOULD + NOT ever spontaneously attain the rejected state at the source. + + :param ~uamqp.error.AMQPError error: The error that caused the message to be rejected. + The value supplied in this field will be placed in the delivery-annotations of the rejected Message + associated with the symbolic key "rejected". + """ + + +Released = namedtuple("Released", []) +Released._code = 0x00000026 # type: ignore # pylint:disable=protected-access +Released._definition = () # type: ignore # pylint:disable=protected-access +if _CAN_ADD_DOCSTRING: + Released.__doc__ = """ + The released outcome. + + At the source the released outcome means that the message is no longer acquired by the receiver, and has been + made available for (re-)delivery to the same or other targets receiving from the node. The message is unchanged + at the node (i.e. the delivery-count of the header of the released Message MUST NOT be incremented). + As released is a terminal outcome, transfer of payload data will not be able to be resumed if the link becomes + suspended. A delivery may become released at the source even before all transfer frames have been sent, this + does not imply that the remaining transfers for the delivery will not be sent. The source MAY spontaneously + attain the released outcome for a Message (for example the source may implement some sort of time bound + acquisition lock, after which the acquisition of a message at a node is revoked to allow for delivery to an + alternative consumer). + + At the target, the released outcome is used to indicate that a given transfer was not and will not be acted upon. + """ + + +Modified = namedtuple("Modified", ["delivery_failed", "undeliverable_here", "message_annotations"]) +Modified.__new__.__defaults__ = (None,) * len(Modified._fields) # type: ignore +Modified._code = 0x00000027 # type: ignore # pylint:disable=protected-access +Modified._definition = ( # type: ignore # pylint:disable=protected-access + FIELD("delivery_failed", AMQPTypes.boolean, False, None, False), + FIELD("undeliverable_here", AMQPTypes.boolean, False, None, False), + FIELD("message_annotations", FieldDefinition.fields, False, None, False), +) +if _CAN_ADD_DOCSTRING: + Modified.__doc__ = """ + The modified outcome. + + At the source the modified outcome means that the message is no longer acquired by the receiver, and has been + made available for (re-)delivery to the same or other targets receiving from the node. The message has been + changed at the node in the ways indicated by the fields of the outcome. As modified is a terminal outcome, + transfer of payload data will not be able to be resumed if the link becomes suspended. A delivery may become + modified at the source even before all transfer frames have been sent, this does not imply that the remaining + transfers for the delivery will not be sent. The source MAY spontaneously attain the modified outcome for a + Message (for example the source may implement some sort of time bound acquisition lock, after which the + acquisition of a message at a node is revoked to allow for delivery to an alternative consumer with the + message modified in some way to denote the previous failed, e.g. with delivery-failed set to true). + At the target, the modified outcome is used to indicate that a given transfer was not and will not be acted + upon, and that the message should be modified in the specified ways at the node. + + :param bool delivery_failed: Count the transfer as an unsuccessful delivery attempt. + If the delivery-failed fiag is set, any Messages modified MUST have their deliverycount incremented. + :param bool undeliverable_here: Prevent redelivery. + If the undeliverable-here is set, then any Messages released MUST NOT be redelivered to the modifying + Link Endpoint. + :param dict message_annotations: Message attributes. + Map containing attributes to combine with the existing message-annotations held in the Message's header + section. Where the existing message-annotations of the Message contain an entry with the same key as an + entry in this field, the value in this field associated with that key replaces the one in the existing + headers; where the existing message-annotations has no such value, the value in this map is added. + """ diff --git a/src/azure/iot/hub/_pyamqp/performatives.py b/src/azure/iot/hub/_pyamqp/performatives.py new file mode 100644 index 0000000..5dc70b2 --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/performatives.py @@ -0,0 +1,640 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +# TODO: fix mypy errors for _code/_definition/__defaults__ (issue #26500) +from collections import namedtuple +import sys +from typing import NamedTuple, Optional +from typing_extensions import Buffer + +from .types import AMQPTypes, FieldDefinition, ObjDefinition +from .constants import FIELD + +_CAN_ADD_DOCSTRING = sys.version_info.major >= 3 + + +OpenFrame = namedtuple( + "OpenFrame", + [ + "container_id", + "hostname", + "max_frame_size", + "channel_max", + "idle_timeout", + "outgoing_locales", + "incoming_locales", + "offered_capabilities", + "desired_capabilities", + "properties", + ], +) +OpenFrame._code = 0x00000010 # type: ignore # pylint:disable=protected-access +OpenFrame._definition = ( # type: ignore # pylint:disable=protected-access + FIELD("container_id", AMQPTypes.string, True, None, False), + FIELD("hostname", AMQPTypes.string, False, None, False), + FIELD("max_frame_size", AMQPTypes.uint, False, 4294967295, False), + FIELD("channel_max", AMQPTypes.ushort, False, 65535, False), + FIELD("idle_timeout", AMQPTypes.uint, False, None, False), + FIELD("outgoing_locales", AMQPTypes.symbol, False, None, True), + FIELD("incoming_locales", AMQPTypes.symbol, False, None, True), + FIELD("offered_capabilities", AMQPTypes.symbol, False, None, True), + FIELD("desired_capabilities", AMQPTypes.symbol, False, None, True), + FIELD("properties", FieldDefinition.fields, False, None, False), +) +if _CAN_ADD_DOCSTRING: + OpenFrame.__doc__ = """ + OPEN performative. Negotiate Connection parameters. + + The first frame sent on a connection in either direction MUST contain an Open body. + (Note that theConnection header which is sent first on the Connection is *not* a frame.) + The fields indicate thecapabilities and limitations of the sending peer. + + :param str container_id: The ID of the source container. + :param str hostname: The name of the target host. + The dns name of the host (either fully qualified or relative) to which the sendingpeer is connecting. + It is not mandatory to provide the hostname. If no hostname isprovided the receiving peer should select + a default based on its own configuration.This field can be used by AMQP proxies to determine the correct + back-end service toconnect the client to.This field may already have been specified by the sasl-init frame, + if a SASL layer is used, or, the server name indication extension as described in RFC-4366, if a TLSlayer + is used, in which case this field SHOULD be null or contain the same value. It is undefined what a different + value to those already specific means. + :param int max_frame_size: Proposed maximum frame size in bytes. + The largest frame size that the sending peer is able to accept on this Connection. + If this field is not set it means that the peer does not impose any specific limit. A peer MUST NOT send + frames larger than its partner can handle. A peer that receives an oversized frame MUST close the Connection + with the framing-error error-code. Both peers MUST accept frames of up to 512 (MIN-MAX-FRAME-SIZE) + octets large. + :param int channel_max: The maximum channel number that may be used on the Connection. + The channel-max value is the highest channel number that may be used on the Connection. This value plus one + is the maximum number of Sessions that can be simultaneously active on the Connection. A peer MUST not use + channel numbers outside the range that its partner can handle. A peer that receives a channel number + outside the supported range MUST close the Connection with the framing-error error-code. + :param int idle_timeout: Idle time-out in milliseconds. + The idle time-out required by the sender. A value of zero is the same as if it was not set (null). If the + receiver is unable or unwilling to support the idle time-out then it should close the connection with + an error explaining why (eg, because it is too small). If the value is not set, then the sender does not + have an idle time-out. However, senders doing this should be aware that implementations MAY choose to use + an internal default to efficiently manage a peer's resources. + :param list(str) outgoing_locales: Locales available for outgoing text. + A list of the locales that the peer supports for sending informational text. This includes Connection, + Session and Link error descriptions. A peer MUST support at least the en-US locale. Since this value + is always supported, it need not be supplied in the outgoing-locales. A null value or an empty list implies + that only en-US is supported. + :param list(str) incoming_locales: Desired locales for incoming text in decreasing level of preference. + A list of locales that the sending peer permits for incoming informational text. This list is ordered in + decreasing level of preference. The receiving partner will chose the first (most preferred) incoming locale + from those which it supports. If none of the requested locales are supported, en-US will be chosen. Note + that en-US need not be supplied in this list as it is always the fallback. A peer may determine which of the + permitted incoming locales is chosen by examining the partner's supported locales asspecified in the + outgoing_locales field. A null value or an empty list implies that only en-US is supported. + :param list(str) offered_capabilities: The extension capabilities the sender supports. + If the receiver of the offered-capabilities requires an extension capability which is not present in the + offered-capability list then it MUST close the connection. A list of commonly defined connection capabilities + and their meanings can be found here: http://www.amqp.org/specification/1.0/connection-capabilities. + :param list(str) required_capabilities: The extension capabilities the sender may use if the receiver supports + them. The desired-capability list defines which extension capabilities the sender MAY use if the receiver + offers them (i.e. they are in the offered-capabilities list received by the sender of the + desired-capabilities). If the receiver of the desired-capabilities offers extension capabilities which are + not present in the desired-capability list it received, then it can be sure those (undesired) capabilities + will not be used on the Connection. + :param dict properties: Connection properties. + The properties map contains a set of fields intended to indicate information about the connection and its + container. A list of commonly defined connection properties and their meanings can be found + here: http://www.amqp.org/specification/1.0/connection-properties. + """ + + +BeginFrame = namedtuple( + "BeginFrame", + [ + "remote_channel", + "next_outgoing_id", + "incoming_window", + "outgoing_window", + "handle_max", + "offered_capabilities", + "desired_capabilities", + "properties", + ], +) +BeginFrame._code = 0x00000011 # type: ignore # pylint:disable=protected-access +BeginFrame._definition = ( # type: ignore # pylint:disable=protected-access + FIELD("remote_channel", AMQPTypes.ushort, False, None, False), + FIELD("next_outgoing_id", AMQPTypes.uint, True, None, False), + FIELD("incoming_window", AMQPTypes.uint, True, None, False), + FIELD("outgoing_window", AMQPTypes.uint, True, None, False), + FIELD("handle_max", AMQPTypes.uint, False, 4294967295, False), + FIELD("offered_capabilities", AMQPTypes.symbol, False, None, True), + FIELD("desired_capabilities", AMQPTypes.symbol, False, None, True), + FIELD("properties", FieldDefinition.fields, False, None, False), +) +if _CAN_ADD_DOCSTRING: + BeginFrame.__doc__ = """ + BEGIN performative. Begin a Session on a channel. + + Indicate that a Session has begun on the channel. + + :param int remote_channel: The remote channel for this Session. + If a Session is locally initiated, the remote-channel MUST NOT be set. When an endpoint responds to a + remotely initiated Session, the remote-channel MUST be set to the channel on which the remote Session + sent the begin. + :param int next_outgoing_id: The transfer-id of the first transfer id the sender will send. + The next-outgoing-id is used to assign a unique transfer-id to all outgoing transfer frames on a given + session. The next-outgoing-id may be initialized to an arbitrary value and is incremented after each + successive transfer according to RFC-1982 serial number arithmetic. + :param int incoming_window: The initial incoming-window of the sender. + The incoming-window defines the maximum number of incoming transfer frames that the endpoint can currently + receive. This identifies a current maximum incoming transfer-id that can be computed by subtracting one + from the sum of incoming-window and next-incoming-id. + :param int outgoing_window: The initial outgoing-window of the sender. + The outgoing-window defines the maximum number of outgoing transfer frames that the endpoint can currently + send. This identifies a current maximum outgoing transfer-id that can be computed by subtracting one from + the sum of outgoing-window and next-outgoing-id. + :param int handle_max: The maximum handle value that may be used on the Session. + The handle-max value is the highest handle value that may be used on the Session. A peer MUST NOT attempt + to attach a Link using a handle value outside the range that its partner can handle. A peer that receives + a handle outside the supported range MUST close the Connection with the framing-error error-code. + :param list(str) offered_capabilities: The extension capabilities the sender supports. + A list of commonly defined session capabilities and their meanings can be found + here: http://www.amqp.org/specification/1.0/session-capabilities. + :param list(str) desired_capabilities: The extension capabilities the sender may use if the receiver + supports them. + :param dict properties: Session properties. + The properties map contains a set of fields intended to indicate information about the session and its + container. A list of commonly defined session properties and their meanings can be found + here: http://www.amqp.org/specification/1.0/session-properties. + """ + + +AttachFrame = namedtuple( + "AttachFrame", + [ + "name", + "handle", + "role", + "send_settle_mode", + "rcv_settle_mode", + "source", + "target", + "unsettled", + "incomplete_unsettled", + "initial_delivery_count", + "max_message_size", + "offered_capabilities", + "desired_capabilities", + "properties", + ], +) +AttachFrame._code = 0x00000012 # type: ignore # pylint:disable=protected-access +AttachFrame._definition = ( # type: ignore # pylint:disable=protected-access + FIELD("name", AMQPTypes.string, True, None, False), + FIELD("handle", AMQPTypes.uint, True, None, False), + FIELD("role", AMQPTypes.boolean, True, None, False), + FIELD("send_settle_mode", AMQPTypes.ubyte, False, 2, False), + FIELD("rcv_settle_mode", AMQPTypes.ubyte, False, 0, False), + FIELD("source", ObjDefinition.source, False, None, False), + FIELD("target", ObjDefinition.target, False, None, False), + FIELD("unsettled", AMQPTypes.map, False, None, False), + FIELD("incomplete_unsettled", AMQPTypes.boolean, False, False, False), + FIELD("initial_delivery_count", AMQPTypes.uint, False, None, False), + FIELD("max_message_size", AMQPTypes.ulong, False, None, False), + FIELD("offered_capabilities", AMQPTypes.symbol, False, None, True), + FIELD("desired_capabilities", AMQPTypes.symbol, False, None, True), + FIELD("properties", FieldDefinition.fields, False, None, False), +) +if _CAN_ADD_DOCSTRING: + AttachFrame.__doc__ = """ + ATTACH performative. Attach a Link to a Session. + + The attach frame indicates that a Link Endpoint has been attached to the Session. The opening flag + is used to indicate that the Link Endpoint is newly created. + + :param str name: The name of the link. + This name uniquely identifies the link from the container of the source to the container of the target + node, e.g. if the container of the source node is A, and the container of the target node is B, the link + may be globally identified by the (ordered) tuple(A,B,). + :param int handle: The handle of the link. + The handle MUST NOT be used for other open Links. An attempt to attach using a handle which is already + associated with a Link MUST be responded to with an immediate close carrying a Handle-in-usesession-error. + To make it easier to monitor AMQP link attach frames, it is recommended that implementations always assign + the lowest available handle to this field. + :param bool role: The role of the link endpoint. Either Role.Sender (False) or Role.Receiver (True). + :param str send_settle_mode: The settlement mode for the Sender. + Determines the settlement policy for deliveries sent at the Sender. When set at the Receiver this indicates + the desired value for the settlement mode at the Sender. When set at the Sender this indicates the actual + settlement mode in use. + :param str rcv_settle_mode: The settlement mode of the Receiver. + Determines the settlement policy for unsettled deliveries received at the Receiver. When set at the Sender + this indicates the desired value for the settlement mode at the Receiver. When set at the Receiver this + indicates the actual settlement mode in use. + :param ~uamqp.messaging.Source source: The source for Messages. + If no source is specified on an outgoing Link, then there is no source currently attached to the Link. + A Link with no source will never produce outgoing Messages. + :param ~uamqp.messaging.Target target: The target for Messages. + If no target is specified on an incoming Link, then there is no target currently attached to the Link. + A Link with no target will never permit incoming Messages. + :param dict unsettled: Unsettled delivery state. + This is used to indicate any unsettled delivery states when a suspended link is resumed. The map is keyed + by delivery-tag with values indicating the delivery state. The local and remote delivery states for a given + delivery-tag MUST be compared to resolve any in-doubt deliveries. If necessary, deliveries MAY be resent, + or resumed based on the outcome of this comparison. If the local unsettled map is too large to be encoded + within a frame of the agreed maximum frame size then the session may be ended with the + frame-size-too-smallerror. The endpoint SHOULD make use of the ability to send an incomplete unsettled map + to avoid sending an error. The unsettled map MUST NOT contain null valued keys. When reattaching + (as opposed to resuming), the unsettled map MUST be null. + :param bool incomplete_unsettled: + If set to true this field indicates that the unsettled map provided is not complete. When the map is + incomplete the recipient of the map cannot take the absence of a delivery tag from the map as evidence of + settlement. On receipt of an incomplete unsettled map a sending endpoint MUST NOT send any new deliveries + (i.e. deliveries where resume is not set to true) to its partner (and a receiving endpoint which sent an + incomplete unsettled map MUST detach with an error on receiving a transfer which does not have the resume + flag set to true). + :param int initial_delivery_count: This MUST NOT be null if role is sender, + and it is ignored if the role is receiver. + :param int max_message_size: The maximum message size supported by the link endpoint. + This field indicates the maximum message size supported by the link endpoint. Any attempt to deliver a + message larger than this results in a message-size-exceeded link-error. If this field is zero or unset, + there is no maximum size imposed by the link endpoint. + :param list(str) offered_capabilities: The extension capabilities the sender supports. + A list of commonly defined session capabilities and their meanings can be found + here: http://www.amqp.org/specification/1.0/link-capabilities. + :param list(str) desired_capabilities: The extension capabilities the sender may use if the receiver + supports them. + :param dict properties: Link properties. + The properties map contains a set of fields intended to indicate information about the link and its + container. A list of commonly defined link properties and their meanings can be found + here: http://www.amqp.org/specification/1.0/link-properties. + """ + + +FlowFrame = namedtuple( + "FlowFrame", + [ + "next_incoming_id", + "incoming_window", + "next_outgoing_id", + "outgoing_window", + "handle", + "delivery_count", + "link_credit", + "available", + "drain", + "echo", + "properties", + ], +) +FlowFrame.__new__.__defaults__ = (None, None, None, None, None, None, None) # type: ignore +FlowFrame._code = 0x00000013 # type: ignore # pylint:disable=protected-access +FlowFrame._definition = ( # type: ignore # pylint:disable=protected-access + FIELD("next_incoming_id", AMQPTypes.uint, False, None, False), + FIELD("incoming_window", AMQPTypes.uint, True, None, False), + FIELD("next_outgoing_id", AMQPTypes.uint, True, None, False), + FIELD("outgoing_window", AMQPTypes.uint, True, None, False), + FIELD("handle", AMQPTypes.uint, False, None, False), + FIELD("delivery_count", AMQPTypes.uint, False, None, False), + FIELD("link_credit", AMQPTypes.uint, False, None, False), + FIELD("available", AMQPTypes.uint, False, None, False), + FIELD("drain", AMQPTypes.boolean, False, False, False), + FIELD("echo", AMQPTypes.boolean, False, False, False), + FIELD("properties", FieldDefinition.fields, False, None, False), +) +if _CAN_ADD_DOCSTRING: + FlowFrame.__doc__ = """ + FLOW performative. Update link state. + + Updates the flow state for the specified Link. + + :param int next_incoming_id: Identifies the expected transfer-id of the next incoming transfer frame. + This value is not set if and only if the sender has not yet received the begin frame for the session. + :param int incoming_window: Defines the maximum number of incoming transfer frames that the endpoint + concurrently receive. + :param int next_outgoing_id: The transfer-id that will be assigned to the next outgoing transfer frame. + :param int outgoing_window: Defines the maximum number of outgoing transfer frames that the endpoint could + potentially currently send, if it was not constrained by restrictions imposed by its peer's incoming-window. + :param int handle: If set, indicates that the flow frame carries flow state information for the local Link + Endpoint associated with the given handle. If not set, the flow frame is carrying only information + pertaining to the Session Endpoint. If set to a handle that is not currently associated with an attached + Link, the recipient MUST respond by ending the session with an unattached-handle session error. + :param int delivery_count: The endpoint's delivery-count. + When the handle field is not set, this field MUST NOT be set. When the handle identifies that the flow + state is being sent from the Sender Link Endpoint to Receiver Link Endpoint this field MUST be set to the + current delivery-count of the Link Endpoint. When the flow state is being sent from the Receiver Endpoint + to the Sender Endpoint this field MUST be set to the last known value of the corresponding Sending Endpoint. + In the event that the Receiving Link Endpoint has not yet seen the initial attach frame from the Sender + this field MUST NOT be set. + :param int link_credit: The current maximum number of Messages that can be received. + The current maximum number of Messages that can be handled at the Receiver Endpoint of the Link. Only the + receiver endpoint can independently set this value. The sender endpoint sets this to the last known + value seen from the receiver. When the handle field is not set, this field MUST NOT be set. + :param int available: The number of available Messages. + The number of Messages awaiting credit at the link sender endpoint. Only the sender can independently set + this value. The receiver sets this to the last known value seen from the sender. When the handle field is + not set, this field MUST NOT be set. + :param bool drain: Indicates drain mode. + When flow state is sent from the sender to the receiver, this field contains the actual drain mode of the + sender. When flow state is sent from the receiver to the sender, this field contains the desired drain + mode of the receiver. When the handle field is not set, this field MUST NOT be set. + :param bool echo: Request link state from other endpoint. + :param dict properties: Link state properties. + A list of commonly defined link state properties and their meanings can be found + here: http://www.amqp.org/specification/1.0/link-state-properties. + """ + + +class TransferFrame(NamedTuple): + handle: Optional[int] = None + delivery_id: Optional[int] = None + delivery_tag: Optional[bytes] = None + message_format: Optional[int] = None + settled: Optional[bool] = None + more: Optional[bool] = None + rcv_settle_mode: Optional[str] = None + state: Optional[bytes] = None + resume: Optional[bool] = None + aborted: Optional[bool] = None + batchable: Optional[bool] = None + payload: Optional[Buffer] = None + + +TransferFrame._code = 0x00000014 # type: ignore # pylint:disable=protected-access +TransferFrame._definition = ( # type: ignore # pylint:disable=protected-access + FIELD("handle", AMQPTypes.uint, True, None, False), + FIELD("delivery_id", AMQPTypes.uint, False, None, False), + FIELD("delivery_tag", AMQPTypes.binary, False, None, False), + FIELD("message_format", AMQPTypes.uint, False, 0, False), + FIELD("settled", AMQPTypes.boolean, False, None, False), + FIELD("more", AMQPTypes.boolean, False, False, False), + FIELD("rcv_settle_mode", AMQPTypes.ubyte, False, None, False), + FIELD("state", ObjDefinition.delivery_state, False, None, False), + FIELD("resume", AMQPTypes.boolean, False, False, False), + FIELD("aborted", AMQPTypes.boolean, False, False, False), + FIELD("batchable", AMQPTypes.boolean, False, False, False), + None, +) + +if _CAN_ADD_DOCSTRING: + TransferFrame.__doc__ = """ + TRANSFER performative. Transfer a Message. + + The transfer frame is used to send Messages across a Link. Messages may be carried by a single transfer up + to the maximum negotiated frame size for the Connection. Larger Messages may be split across several + transfer frames. + + :param int handle: Specifies the Link on which the Message is transferred. + :param int delivery_id: Alias for delivery-tag. + The delivery-id MUST be supplied on the first transfer of a multi-transfer delivery. On continuation + transfers the delivery-id MAY be omitted. It is an error if the delivery-id on a continuation transfer + differs from the delivery-id on the first transfer of a delivery. + :param bytes delivery_tag: Uniquely identifies the delivery attempt for a given Message on this Link. + This field MUST be specified for the first transfer of a multi transfer message and may only be + omitted for continuation transfers. + :param int message_format: Indicates the message format. + This field MUST be specified for the first transfer of a multi transfer message and may only be omitted + for continuation transfers. + :param bool settled: If not set on the first (or only) transfer for a delivery, then the settled flag MUST + be interpreted as being false. For subsequent transfers if the settled flag is left unset then it MUST be + interpreted as true if and only if the value of the settled flag on any of the preceding transfers was + true; if no preceding transfer was sent with settled being true then the value when unset MUST be taken + as false. If the negotiated value for snd-settle-mode at attachment is settled, then this field MUST be + true on at least one transfer frame for a delivery (i.e. the delivery must be settled at the Sender at + the point the delivery has been completely transferred). If the negotiated value for snd-settle-mode at + attachment is unsettled, then this field MUST be false (or unset) on every transfer frame for a delivery + (unless the delivery is aborted). + :param bool more: Indicates that the Message has more content. + Note that if both the more and aborted fields are set to true, the aborted flag takes precedence. That is + a receiver should ignore the value of the more field if the transfer is marked as aborted. A sender + SHOULD NOT set the more flag to true if it also sets the aborted flag to true. + :param str rcv_settle_mode: If first, this indicates that the Receiver MUST settle the delivery once it has + arrived without waiting for the Sender to settle first. If second, this indicates that the Receiver MUST + NOT settle until sending its disposition to the Sender and receiving a settled disposition from the sender. + If not set, this value is defaulted to the value negotiated on link attach. If the negotiated link value is + first, then it is illegal to set this field to second. If the message is being sent settled by the Sender, + the value of this field is ignored. The (implicit or explicit) value of this field does not form part of the + transfer state, and is not retained if a link is suspended and subsequently resumed. + :param bytes state: The state of the delivery at the sender. + When set this informs the receiver of the state of the delivery at the sender. This is particularly useful + when transfers of unsettled deliveries are resumed after a resuming a link. Setting the state on the + transfer can be thought of as being equivalent to sending a disposition immediately before the transfer + performative, i.e. it is the state of the delivery (not the transfer) that existed at the point the frame + was sent. Note that if the transfer performative (or an earlier disposition performative referring to the + delivery) indicates that the delivery has attained a terminal state, then no future transfer or disposition + sent by the sender can alter that terminal state. + :param bool resume: Indicates a resumed delivery. + If true, the resume flag indicates that the transfer is being used to reassociate an unsettled delivery + from a dissociated link endpoint. The receiver MUST ignore resumed deliveries that are not in its local + unsettled map. The sender MUST NOT send resumed transfers for deliveries not in its local unsettledmap. + If a resumed delivery spans more than one transfer performative, then the resume flag MUST be set to true + on the first transfer of the resumed delivery. For subsequent transfers for the same delivery the resume + flag may be set to true, or may be omitted. In the case where the exchange of unsettled maps makes clear + that all message data has been successfully transferred to the receiver, and that only the final state + (and potentially settlement) at the sender needs to be conveyed, then a resumed delivery may carry no + payload and instead act solely as a vehicle for carrying the terminal state of the delivery at the sender. + :param bool aborted: Indicates that the Message is aborted. + Aborted Messages should be discarded by the recipient (any payload within the frame carrying the performative + MUST be ignored). An aborted Message is implicitly settled. + :param bool batchable: Batchable hint. + If true, then the issuer is hinting that there is no need for the peer to urgently communicate updated + delivery state. This hint may be used to artificially increase the amount of batching an implementation + uses when communicating delivery states, and thereby save bandwidth. If the message being delivered is too + large to fit within a single frame, then the setting of batchable to true on any of the transfer + performatives for the delivery is equivalent to setting batchable to true for all the transfer performatives + for the delivery. The batchable value does not form part of the transfer state, and is not retained if a + link is suspended and subsequently resumed. + """ + + +DispositionFrame = namedtuple("DispositionFrame", ["role", "first", "last", "settled", "state", "batchable"]) +DispositionFrame._code = 0x00000015 # type: ignore # pylint:disable=protected-access +DispositionFrame._definition = ( # type: ignore # pylint:disable=protected-access + FIELD("role", AMQPTypes.boolean, True, None, False), + FIELD("first", AMQPTypes.uint, True, None, False), + FIELD("last", AMQPTypes.uint, False, None, False), + FIELD("settled", AMQPTypes.boolean, False, False, False), + FIELD("state", ObjDefinition.delivery_state, False, None, False), + FIELD("batchable", AMQPTypes.boolean, False, False, False), +) +if _CAN_ADD_DOCSTRING: + DispositionFrame.__doc__ = """ + DISPOSITION performative. Inform remote peer of delivery state changes. + + The disposition frame is used to inform the remote peer of local changes in the state of deliveries. + The disposition frame may reference deliveries from many different links associated with a session, + although all links MUST have the directionality indicated by the specified role. Note that it is possible + for a disposition sent from sender to receiver to refer to a delivery which has not yet completed + (i.e. a delivery which is spread over multiple frames and not all frames have yet been sent). The use of such + interleaving is discouraged in favor of carrying the modified state on the next transfer performative for + the delivery. The disposition performative may refer to deliveries on links that are no longer attached. + As long as the links have not been closed or detached with an error then the deliveries are still "live" and + the updated state MUST be applied. + + :param str role: Directionality of disposition. + The role identifies whether the disposition frame contains information about sending link endpoints + or receiving link endpoints. + :param int first: Lower bound of deliveries. + Identifies the lower bound of delivery-ids for the deliveries in this set. + :param int last: Upper bound of deliveries. + Identifies the upper bound of delivery-ids for the deliveries in this set. If not set, + this is taken to be the same as first. + :param bool settled: Indicates deliveries are settled. + If true, indicates that the referenced deliveries are considered settled by the issuing endpoint. + :param bytes state: Indicates state of deliveries. + Communicates the state of all the deliveries referenced by this disposition. + :param bool batchable: Batchable hint. + If true, then the issuer is hinting that there is no need for the peer to urgently communicate the impact + of the updated delivery states. This hint may be used to artificially increase the amount of batching an + implementation uses when communicating delivery states, and thereby save bandwidth. + """ + +DetachFrame = namedtuple("DetachFrame", ["handle", "closed", "error"]) +DetachFrame._code = 0x00000016 # type: ignore # pylint:disable=protected-access +DetachFrame._definition = ( # type: ignore # pylint:disable=protected-access + FIELD("handle", AMQPTypes.uint, True, None, False), + FIELD("closed", AMQPTypes.boolean, False, False, False), + FIELD("error", ObjDefinition.error, False, None, False), +) +if _CAN_ADD_DOCSTRING: + DetachFrame.__doc__ = """ + DETACH performative. Detach the Link Endpoint from the Session. + + Detach the Link Endpoint from the Session. This un-maps the handle and makes it available for + use by other Links + + :param int handle: The local handle of the link to be detached. + :param bool handle: If true then the sender has closed the link. + :param ~uamqp.error.AMQPError error: Error causing the detach. + If set, this field indicates that the Link is being detached due to an error condition. + The value of the field should contain details on the cause of the error. + """ + + +EndFrame = namedtuple("EndFrame", ["error"]) +EndFrame._code = 0x00000017 # type: ignore # pylint:disable=protected-access +EndFrame._definition = (FIELD("error", ObjDefinition.error, False, None, False),) # type: ignore # pylint:disable=protected-access +if _CAN_ADD_DOCSTRING: + EndFrame.__doc__ = """ + END performative. End the Session. + + Indicates that the Session has ended. + + :param ~uamqp.error.AMQPError error: Error causing the end. + If set, this field indicates that the Session is being ended due to an error condition. + The value of the field should contain details on the cause of the error. + """ + + +CloseFrame = namedtuple("CloseFrame", ["error"]) +CloseFrame._code = 0x00000018 # type: ignore # pylint:disable=protected-access +CloseFrame._definition = (FIELD("error", ObjDefinition.error, False, None, False),) # type: ignore # pylint:disable=protected-access +if _CAN_ADD_DOCSTRING: + CloseFrame.__doc__ = """ + CLOSE performative. Signal a Connection close. + + Sending a close signals that the sender will not be sending any more frames (or bytes of any other kind) on + the Connection. Orderly shutdown requires that this frame MUST be written by the sender. It is illegal to + send any more frames (or bytes of any other kind) after sending a close frame. + + :param ~uamqp.error.AMQPError error: Error causing the close. + If set, this field indicates that the Connection is being closed due to an error condition. + The value of the field should contain details on the cause of the error. + """ + + +SASLMechanism = namedtuple("SASLMechanism", ["sasl_server_mechanisms"]) +SASLMechanism._code = 0x00000040 # type: ignore # pylint:disable=protected-access +SASLMechanism._definition = (FIELD("sasl_server_mechanisms", AMQPTypes.symbol, True, None, True),) # type: ignore # pylint:disable=protected-access +if _CAN_ADD_DOCSTRING: + SASLMechanism.__doc__ = """ + Advertise available sasl mechanisms. + + dvertises the available SASL mechanisms that may be used for authentication. + + :param list(bytes) sasl_server_mechanisms: Supported sasl mechanisms. + A list of the sasl security mechanisms supported by the sending peer. + It is invalid for this list to be null or empty. If the sending peer does not require its partner to + authenticate with it, then it should send a list of one element with its value as the SASL mechanism + ANONYMOUS. The server mechanisms are ordered in decreasing level of preference. + """ + + +SASLInit = namedtuple("SASLInit", ["mechanism", "initial_response", "hostname"]) +SASLInit._code = 0x00000041 # type: ignore # pylint:disable=protected-access +SASLInit._definition = ( # type: ignore # pylint:disable=protected-access + FIELD("mechanism", AMQPTypes.symbol, True, None, False), + FIELD("initial_response", AMQPTypes.binary, False, None, False), + FIELD("hostname", AMQPTypes.string, False, None, False), +) +if _CAN_ADD_DOCSTRING: + SASLInit.__doc__ = """ + Initiate sasl exchange. + + Selects the sasl mechanism and provides the initial response if needed. + + :param bytes mechanism: Selected security mechanism. + The name of the SASL mechanism used for the SASL exchange. If the selected mechanism is not supported by + the receiving peer, it MUST close the Connection with the authentication-failure close-code. Each peer + MUST authenticate using the highest-level security profile it can handle from the list provided by the + partner. + :param bytes initial_response: Security response data. + A block of opaque data passed to the security mechanism. The contents of this data are defined by the + SASL security mechanism. + :param str hostname: The name of the target host. + The DNS name of the host (either fully qualified or relative) to which the sending peer is connecting. It + is not mandatory to provide the hostname. If no hostname is provided the receiving peer should select a + default based on its own configuration. This field can be used by AMQP proxies to determine the correct + back-end service to connect the client to, and to determine the domain to validate the client's credentials + against. This field may already have been specified by the server name indication extension as described + in RFC-4366, if a TLS layer is used, in which case this field SHOULD benull or contain the same value. + It is undefined what a different value to those already specific means. + """ + + +SASLChallenge = namedtuple("SASLChallenge", ["challenge"]) +SASLChallenge._code = 0x00000042 # type: ignore # pylint:disable=protected-access +SASLChallenge._definition = (FIELD("challenge", AMQPTypes.binary, True, None, False),) # type: ignore # pylint:disable=protected-access +if _CAN_ADD_DOCSTRING: + SASLChallenge.__doc__ = """ + Security mechanism challenge. + + Send the SASL challenge data as defined by the SASL specification. + + :param bytes challenge: Security challenge data. + Challenge information, a block of opaque binary data passed to the security mechanism. + """ + + +SASLResponse = namedtuple("SASLResponse", ["response"]) +SASLResponse._code = 0x00000043 # type: ignore # pylint:disable=protected-access +SASLResponse._definition = (FIELD("response", AMQPTypes.binary, True, None, False),) # type: ignore # pylint:disable=protected-access +if _CAN_ADD_DOCSTRING: + SASLResponse.__doc__ = """ + Security mechanism response. + + Send the SASL response data as defined by the SASL specification. + + :param bytes response: Security response data. + """ + + +SASLOutcome = namedtuple("SASLOutcome", ["code", "additional_data"]) +SASLOutcome._code = 0x00000044 # type: ignore # pylint:disable=protected-access +SASLOutcome._definition = ( # type: ignore # pylint:disable=protected-access + FIELD("code", AMQPTypes.ubyte, True, None, False), + FIELD("additional_data", AMQPTypes.binary, False, None, False), +) +if _CAN_ADD_DOCSTRING: + SASLOutcome.__doc__ = """ + Indicates the outcome of the sasl dialog. + + This frame indicates the outcome of the SASL dialog. Upon successful completion of the SASL dialog the + Security Layer has been established, and the peers must exchange protocol headers to either starta nested + Security Layer, or to establish the AMQP Connection. + + :param int code: Indicates the outcome of the sasl dialog. + A reply-code indicating the outcome of the SASL dialog. + :param bytes additional_data: Additional data as specified in RFC-4422. + The additional-data field carries additional data on successful authentication outcomeas specified by + the SASL specification (RFC-4422). If the authentication is unsuccessful, this field is not set. + """ diff --git a/src/azure/iot/hub/_pyamqp/receiver.py b/src/azure/iot/hub/_pyamqp/receiver.py new file mode 100644 index 0000000..78db44f --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/receiver.py @@ -0,0 +1,143 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import uuid +import logging +from typing import Optional, Union + +from ._decode import decode_payload +from .link import Link +from .constants import LinkState, Role +from .performatives import TransferFrame, DispositionFrame +from .outcomes import Received, Accepted, Rejected, Released, Modified +from .error import AMQPException, ErrorCondition + + +_LOGGER = logging.getLogger(__name__) + + +class ReceiverLink(Link): + def __init__(self, session, handle, source_address, **kwargs): + name = kwargs.pop("name", None) or str(uuid.uuid4()) + role = Role.Receiver + if "target_address" not in kwargs: + kwargs["target_address"] = "receiver-link-{}".format(name) + super(ReceiverLink, self).__init__(session, handle, name, role, source_address=source_address, **kwargs) + self._on_transfer = kwargs.pop("on_transfer") + self._received_payload = bytearray() + self._first_frame = None + self._received_delivery_tags = set() + + @classmethod + def from_incoming_frame(cls, session, handle, frame): + # TODO: Assuming we establish all links for now... + # check link_create_from_endpoint in C lib + raise NotImplementedError("Pending") + + def _process_incoming_message(self, frame, message): + try: + return self._on_transfer(frame, message) + except Exception as e: # pylint: disable=broad-except + _LOGGER.error("Transfer callback function failed with error: %r", e, extra=self.network_trace_params) + return None + + def _incoming_attach(self, frame): + super(ReceiverLink, self)._incoming_attach(frame) + if frame[9] is None: # initial_delivery_count + _LOGGER.info("Cannot get initial-delivery-count. Detaching link", extra=self.network_trace_params) + self._set_state(LinkState.DETACHED) # TODO: Send detach now? + self.delivery_count = frame[9] + self.current_link_credit = self.link_credit + self._outgoing_flow() + + def _incoming_transfer(self, frame): + if self.network_trace: + _LOGGER.debug("<- %r", TransferFrame(payload=b"***", *frame[:-1]), extra=self.network_trace_params) + + # If more is false --> this is the last frame of the message + if not frame[5]: + self.current_link_credit -= 1 + self.delivery_count += 1 + self.received_delivery_id = frame[1] # delivery_id + if self.received_delivery_id is not None: + self._first_frame = frame + if not self.received_delivery_id and not self._received_payload: + pass # TODO: delivery error + if self._received_payload or frame[5]: # more + self._received_payload.extend(frame[11]) + if not frame[5]: + self._received_delivery_tags.add(self._first_frame[2]) + if self._received_payload: + message = decode_payload(memoryview(self._received_payload)) + self._received_payload = bytearray() + else: + message = decode_payload(frame[11]) + delivery_state = self._process_incoming_message(self._first_frame, message) + + if not frame[4] and delivery_state: # settled + self._outgoing_disposition( + first=self._first_frame[1], + last=self._first_frame[1], + delivery_tag=self._first_frame[2], + settled=True, + state=delivery_state, + batchable=None, + ) + + def _wait_for_response(self, wait: Union[bool, float]) -> None: + if wait is True: + self._session._connection.listen(wait=False) # pylint: disable=protected-access + if self.state == LinkState.ERROR: + if self._error: + raise self._error + elif wait: + self._session._connection.listen(wait=wait) # pylint: disable=protected-access + if self.state == LinkState.ERROR: + if self._error: + raise self._error + + def _outgoing_disposition( + self, + first: int, + last: Optional[int], + delivery_tag: bytes, + settled: Optional[bool], + state: Optional[Union[Received, Accepted, Rejected, Released, Modified]], + batchable: Optional[bool], + ): + if delivery_tag not in self._received_delivery_tags: + raise AMQPException(condition=ErrorCondition.IllegalState, description="Delivery tag not found.") + + disposition_frame = DispositionFrame( + role=self.role, first=first, last=last, settled=settled, state=state, batchable=batchable + ) + if self.network_trace: + _LOGGER.debug("-> %r", DispositionFrame(*disposition_frame), extra=self.network_trace_params) + self._session._outgoing_disposition(disposition_frame) # pylint: disable=protected-access + self._received_delivery_tags.remove(delivery_tag) + + def attach(self): + super().attach() + self._received_payload = bytearray() + + def send_disposition( + self, + *, + wait: Union[bool, float] = False, + first_delivery_id: int, + last_delivery_id: Optional[int] = None, + delivery_tag: bytes, + settled: Optional[bool] = None, + delivery_state: Optional[Union[Received, Accepted, Rejected, Released, Modified]] = None, + batchable: Optional[bool] = None + ): + if self._is_closed: + raise ValueError("Link already closed.") + self._outgoing_disposition( + first_delivery_id, last_delivery_id, delivery_tag, settled, delivery_state, batchable + ) + if not settled: + self._wait_for_response(wait) diff --git a/src/azure/iot/hub/_pyamqp/sasl.py b/src/azure/iot/hub/_pyamqp/sasl.py new file mode 100644 index 0000000..83b065b --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/sasl.py @@ -0,0 +1,140 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from ._transport import SSLTransport, WebSocketTransport, AMQPS_PORT +from .constants import SASLCode, SASL_HEADER_FRAME, WEBSOCKET_PORT +from .performatives import SASLInit + + +_SASL_FRAME_TYPE = b"\x01" + + +class SASLPlainCredential(object): + """PLAIN SASL authentication mechanism. + See https://tools.ietf.org/html/rfc4616 for details + """ + + mechanism = b"PLAIN" + + def __init__(self, authcid, passwd, authzid=None): + self.authcid = authcid + self.passwd = passwd + self.authzid = authzid + + def start(self): + if self.authzid: + login_response = self.authzid.encode("utf-8") + else: + login_response = b"" + login_response += b"\0" + login_response += self.authcid.encode("utf-8") + login_response += b"\0" + login_response += self.passwd.encode("utf-8") + return login_response + + +class SASLAnonymousCredential(object): + """ANONYMOUS SASL authentication mechanism. + See https://tools.ietf.org/html/rfc4505 for details + """ + + mechanism = b"ANONYMOUS" + + def start(self): + return b"" + + +class SASLExternalCredential(object): + """EXTERNAL SASL mechanism. + Enables external authentication, i.e. not handled through this protocol. + Only passes 'EXTERNAL' as authentication mechanism, but no further + authentication data. + """ + + mechanism = b"EXTERNAL" + + def start(self): + return b"" + + +class SASLTransportMixin: + def _negotiate(self): + self.write(SASL_HEADER_FRAME) + _, returned_header = self.receive_frame() + if returned_header[1] != SASL_HEADER_FRAME: + raise ValueError( + f"""Mismatching AMQP header protocol. Expected: {SASL_HEADER_FRAME!r},""" + """received: {returned_header[1]!r}""" + ) + + _, supported_mechanisms = self.receive_frame(verify_frame_type=1) + if self.credential.mechanism not in supported_mechanisms[1][0]: # sasl_server_mechanisms + raise ValueError("Unsupported SASL credential type: {}".format(self.credential.mechanism)) + sasl_init = SASLInit( + mechanism=self.credential.mechanism, + initial_response=self.credential.start(), + hostname=self.host, + ) + self.send_frame(0, sasl_init, frame_type=_SASL_FRAME_TYPE) + + _, next_frame = self.receive_frame(verify_frame_type=1) + frame_type, fields = next_frame + if frame_type != 0x00000044: # SASLOutcome + raise NotImplementedError("Unsupported SASL challenge") + if fields[0] == SASLCode.Ok: # code + return + raise ValueError("SASL negotiation failed.\nOutcome: {}\nDetails: {}".format(*fields)) + + +class SASLTransport(SSLTransport, SASLTransportMixin): + def __init__( + self, + host, + credential, + *, + port=AMQPS_PORT, + socket_timeout=None, + ssl_opts=None, + **kwargs, + ): + self.credential = credential + ssl_opts = ssl_opts or True + super(SASLTransport, self).__init__( + host, + port=port, + socket_timeout=socket_timeout, + ssl_opts=ssl_opts, + **kwargs, + ) + + def negotiate(self): + with self.block(): + self._negotiate() + + +class SASLWithWebSocket(WebSocketTransport, SASLTransportMixin): + def __init__( + self, + host, + credential, + *, + port=WEBSOCKET_PORT, + socket_timeout=None, + ssl_opts=None, + **kwargs, + ): + self.credential = credential + ssl_opts = ssl_opts or True + super().__init__( + host, + port=port, + socket_timeout=socket_timeout, + ssl_opts=ssl_opts, + **kwargs, + ) + + def negotiate(self): + self._negotiate() diff --git a/src/azure/iot/hub/_pyamqp/sender.py b/src/azure/iot/hub/_pyamqp/sender.py new file mode 100644 index 0000000..ea6f202 --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/sender.py @@ -0,0 +1,197 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import struct +import uuid +import logging +import time +from threading import Lock + +from ._encode import encode_payload +from .link import Link +from .constants import SessionTransferState, LinkDeliverySettleReason, LinkState, Role, SenderSettleMode, SessionState +from .error import AMQPLinkError, ErrorCondition, MessageException + +_LOGGER = logging.getLogger(__name__) + + +class PendingDelivery(object): + def __init__(self, **kwargs): + self.message = kwargs.get("message") + self.sent = False + self.frame = None + self.on_delivery_settled = kwargs.get("on_delivery_settled") + self.start = time.time() + self.transfer_state = None + self.timeout = kwargs.get("timeout") + self.settled = kwargs.get("settled", False) + self._network_trace_params = kwargs.get("network_trace_params") + + def on_settled(self, reason, state): + if self.on_delivery_settled and not self.settled: + try: + self.on_delivery_settled(reason, state) + except Exception as e: # pylint:disable=broad-except + _LOGGER.warning("Message 'on_send_complete' callback failed: %r", e, extra=self._network_trace_params) + self.settled = True + + +class SenderLink(Link): + def __init__(self, session, handle, target_address, **kwargs): + name = kwargs.pop("name", None) or str(uuid.uuid4()) + role = Role.Sender + if "source_address" not in kwargs: + kwargs["source_address"] = "sender-link-{}".format(name) + super(SenderLink, self).__init__(session, handle, name, role, target_address=target_address, **kwargs) + self._pending_deliveries = [] + self.lock = Lock() + + @classmethod + def from_incoming_frame(cls, session, handle, frame): + # TODO: Assuming we establish all links for now... + # check link_create_from_endpoint in C lib + raise NotImplementedError("Pending") + + # In theory we should not need to purge pending deliveries on attach/dettach - as a link should + # be resume-able, however this is not yet supported. + def _incoming_attach(self, frame): + try: + super(SenderLink, self)._incoming_attach(frame) + except AMQPLinkError: + self._remove_pending_deliveries() + raise + self.current_link_credit = self.link_credit + self._outgoing_flow() + self.update_pending_deliveries() + + def _incoming_detach(self, frame): + super(SenderLink, self)._incoming_detach(frame) + self._remove_pending_deliveries() + + def _incoming_flow(self, frame): + rcv_link_credit = frame[6] # link_credit + rcv_delivery_count = frame[5] # delivery_count + if frame[4] is not None: # handle + if rcv_link_credit is None or rcv_delivery_count is None: + _LOGGER.info( + "Unable to get link-credit or delivery-count from incoming ATTACH. Detaching link.", + extra=self.network_trace_params, + ) + self._remove_pending_deliveries() + self._set_state(LinkState.DETACHED) # TODO: Send detach now? + else: + self.current_link_credit = rcv_delivery_count + rcv_link_credit - self.delivery_count + self.update_pending_deliveries() + + def _outgoing_transfer(self, delivery): + output = bytearray() + encode_payload(output, delivery.message) + delivery_count = self.delivery_count + 1 + delivery.frame = { + "handle": self.handle, + "delivery_tag": struct.pack(">I", abs(delivery_count)), + "message_format": delivery.message._code, # pylint:disable=protected-access + "settled": delivery.settled, + "more": False, + "rcv_settle_mode": None, + "state": None, + "resume": None, + "aborted": None, + "batchable": None, + "payload": output, + } + self._session._outgoing_transfer( # pylint:disable=protected-access + delivery, self.network_trace_params if self.network_trace else None + ) + sent_and_settled = False + if delivery.transfer_state == SessionTransferState.OKAY: + self.delivery_count = delivery_count + self.current_link_credit -= 1 + delivery.sent = True + if delivery.settled: + delivery.on_settled(LinkDeliverySettleReason.SETTLED, None) + sent_and_settled = True + # elif delivery.transfer_state == SessionTransferState.ERROR: + # TODO: Session wasn't mapped yet - re-adding to the outgoing delivery queue? + return sent_and_settled + + def _incoming_disposition(self, frame): + if not frame[3]: # settled + return + range_end = (frame[2] or frame[1]) + 1 # first or last + settled_ids = list(range(frame[1], range_end)) + unsettled = [] + for delivery in self._pending_deliveries: + if delivery.sent and delivery.frame["delivery_id"] in settled_ids: + delivery.on_settled(LinkDeliverySettleReason.DISPOSITION_RECEIVED, frame[4]) # state + continue + unsettled.append(delivery) + self._pending_deliveries = unsettled + + def _remove_pending_deliveries(self): + for delivery in self._pending_deliveries: + delivery.on_settled(LinkDeliverySettleReason.NOT_DELIVERED, None) + self._pending_deliveries = [] + + def _on_session_state_change(self): + if self._session.state == SessionState.DISCARDING: + self._remove_pending_deliveries() + super()._on_session_state_change() + + def update_pending_deliveries(self): + # TODO: Temporary fix until connection.listen removed from keep alive thread. + with self.lock: + if self.current_link_credit <= 0: + self.current_link_credit = self.link_credit + self._outgoing_flow() + now = time.time() + pending = [] + + for delivery in self._pending_deliveries: + if delivery.timeout and (now - delivery.start) >= delivery.timeout: + delivery.on_settled(LinkDeliverySettleReason.TIMEOUT, None) + continue + if not delivery.sent: + sent_and_settled = self._outgoing_transfer(delivery) + if sent_and_settled: + continue + pending.append(delivery) + self._pending_deliveries = pending + + def send_transfer(self, message, *, send_async=False, **kwargs): + self._check_if_closed() + if self.state != LinkState.ATTACHED: + raise AMQPLinkError(condition=ErrorCondition.ClientError, description="Link is not attached.") + settled = self.send_settle_mode == SenderSettleMode.Settled + if self.send_settle_mode == SenderSettleMode.Mixed: + settled = kwargs.pop("settled", True) + delivery = PendingDelivery( + on_delivery_settled=kwargs.get("on_send_complete"), + timeout=kwargs.get("timeout"), + message=message, + settled=settled, + network_trace_params=self.network_trace_params, + ) + if self.current_link_credit == 0 or send_async: + self._pending_deliveries.append(delivery) + else: + sent_and_settled = self._outgoing_transfer(delivery) + if not sent_and_settled: + self._pending_deliveries.append(delivery) + return delivery + + def cancel_transfer(self, delivery): + try: + index = self._pending_deliveries.index(delivery) + except ValueError: + raise ValueError("Found no matching pending transfer.") from None + delivery = self._pending_deliveries[index] + if delivery.sent: + raise MessageException( + ErrorCondition.ClientError, + message="Transfer cannot be cancelled. Message has already been sent and awaiting disposition.", + ) + delivery.on_settled(LinkDeliverySettleReason.CANCELLED, None) + self._pending_deliveries.pop(index) diff --git a/src/azure/iot/hub/_pyamqp/session.py b/src/azure/iot/hub/_pyamqp/session.py new file mode 100644 index 0000000..a5d44f0 --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/session.py @@ -0,0 +1,446 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from __future__ import annotations +import uuid +import logging +import time +from typing import Union, Optional + +from .constants import ConnectionState, SessionState, SessionTransferState, Role +from .sender import SenderLink +from .receiver import ReceiverLink +from .management_link import ManagementLink +from .performatives import ( + BeginFrame, + EndFrame, + FlowFrame, + TransferFrame, + DispositionFrame, +) +from .error import AMQPError, ErrorCondition +from ._encode import encode_frame + +_LOGGER = logging.getLogger(__name__) + + +class Session(object): # pylint: disable=too-many-instance-attributes + """ + :param int remote_channel: The remote channel for this Session. + :param int next_outgoing_id: The transfer-id of the first transfer id the sender will send. + :param int incoming_window: The initial incoming-window of the sender. + :param int outgoing_window: The initial outgoing-window of the sender. + :param int handle_max: The maximum handle value that may be used on the Session. + :param list(str) offered_capabilities: The extension capabilities the sender supports. + :param list(str) desired_capabilities: The extension capabilities the sender may use if the receiver supports + :param dict properties: Session properties. + """ + + def __init__(self, connection, channel, **kwargs): + self.name = kwargs.pop("name", None) or str(uuid.uuid4()) + self.state = SessionState.UNMAPPED + self.handle_max = kwargs.get("handle_max", 4294967295) + self.properties = kwargs.pop("properties", None) + self.remote_properties = None + self.channel = channel + self.remote_channel = None + self.next_outgoing_id = kwargs.pop("next_outgoing_id", 0) + self.next_incoming_id = None + self.incoming_window = kwargs.pop("incoming_window", 1) + self.outgoing_window = kwargs.pop("outgoing_window", 1) + self.target_incoming_window = self.incoming_window + self.remote_incoming_window = 0 + self.remote_outgoing_window = 0 + self.offered_capabilities = None + self.desired_capabilities = kwargs.pop("desired_capabilities", None) + + self.allow_pipelined_open = kwargs.pop("allow_pipelined_open", True) + self.idle_wait_time = kwargs.get("idle_wait_time", 0.1) + self.network_trace = kwargs["network_trace"] + self.network_trace_params = kwargs["network_trace_params"] + self.network_trace_params["amqpSession"] = self.name + + self.links = {} + self._connection = connection + self._output_handles = {} + self._input_handles = {} + + def __enter__(self): + self.begin() + return self + + def __exit__(self, *args): + self.end() + + @classmethod + def from_incoming_frame(cls, connection, channel): + # TODO: check session_create_from_endpoint in C lib + new_session = cls(connection, channel) + return new_session + + def _set_state(self, new_state: SessionState) -> None: + """Update the session state. + :param ~pyamqp.constants.SessionState new_state: The new state to set. + """ + if new_state is None: + return + previous_state = self.state + self.state = new_state + _LOGGER.info( + "Session state changed: %r -> %r", + previous_state, + new_state, + extra=self.network_trace_params, + ) + for link in self.links.values(): + link._on_session_state_change() # pylint: disable=protected-access + + def _on_connection_state_change(self): + if self._connection.state in [ConnectionState.CLOSE_RCVD, ConnectionState.END]: + if self.state not in [SessionState.DISCARDING, SessionState.UNMAPPED]: + self._set_state(SessionState.DISCARDING) + + def _get_next_output_handle(self) -> int: + """Get the next available outgoing handle number within the max handle limit. + + :raises ValueError: If maximum handle has been reached. + :returns: The next available outgoing handle number. + :rtype: int + """ + if len(self._output_handles) >= self.handle_max: + raise ValueError("Maximum number of handles ({}) has been reached.".format(self.handle_max)) + next_handle = next(i for i in range(1, self.handle_max) if i not in self._output_handles) + return next_handle + + def _outgoing_begin(self): + begin_frame = BeginFrame( + remote_channel=self.remote_channel if self.state == SessionState.BEGIN_RCVD else None, + next_outgoing_id=self.next_outgoing_id, + outgoing_window=self.outgoing_window, + incoming_window=self.incoming_window, + handle_max=self.handle_max, + offered_capabilities=self.offered_capabilities if self.state == SessionState.BEGIN_RCVD else None, + desired_capabilities=self.desired_capabilities if self.state == SessionState.UNMAPPED else None, + properties=self.properties, + ) + if self.network_trace: + _LOGGER.debug("-> %r", begin_frame, extra=self.network_trace_params) + self._connection._process_outgoing_frame(self.channel, begin_frame) # pylint: disable=protected-access + + def _incoming_begin(self, frame): + if self.network_trace: + _LOGGER.debug("<- %r", BeginFrame(*frame), extra=self.network_trace_params) + self.handle_max = frame[4] # handle_max + self.next_incoming_id = frame[1] # next_outgoing_id + self.remote_incoming_window = frame[2] # incoming_window + self.remote_outgoing_window = frame[3] # outgoing_window + self.remote_properties = frame[7] # incoming map of properties about the session + if self.state == SessionState.BEGIN_SENT: + self.remote_channel = frame[0] # remote_channel + self._set_state(SessionState.MAPPED) + elif self.state == SessionState.UNMAPPED: + self._set_state(SessionState.BEGIN_RCVD) + self._outgoing_begin() + self._set_state(SessionState.MAPPED) + + def _outgoing_end(self, error=None): + end_frame = EndFrame(error=error) + if self.network_trace: + _LOGGER.debug("-> %r", end_frame, extra=self.network_trace_params) + self._connection._process_outgoing_frame(self.channel, end_frame) # pylint: disable=protected-access + + def _incoming_end(self, frame): + if self.network_trace: + _LOGGER.debug("<- %r", EndFrame(*frame), extra=self.network_trace_params) + if self.state not in [ + SessionState.END_RCVD, + SessionState.END_SENT, + SessionState.DISCARDING, + ]: + self._set_state(SessionState.END_RCVD) + for _, link in self.links.items(): + link.detach() + # TODO: handling error + self._outgoing_end() + self._set_state(SessionState.UNMAPPED) + + def _outgoing_attach(self, frame): + self._connection._process_outgoing_frame(self.channel, frame) # pylint: disable=protected-access + + def _incoming_attach(self, frame): + try: + self._input_handles[frame[1]] = self.links[frame[0].decode("utf-8")] # name and handle + self._input_handles[frame[1]]._incoming_attach(frame) # pylint: disable=protected-access + except KeyError: + try: + outgoing_handle = self._get_next_output_handle() + except ValueError: + _LOGGER.error( + "Unable to attach new link - cannot allocate more handles.", extra=self.network_trace_params + ) + # detach the link that would have been set. + self.links[frame[0].decode("utf-8")].detach( + error=AMQPError( + condition=ErrorCondition.LinkDetachForced, + description="""Cannot allocate more handles, """ + """the max number of handles is {}. Detaching link""".format(self.handle_max), + info=None, + ) + ) + return + if frame[2] == Role.Sender: # role + new_link = ReceiverLink.from_incoming_frame(self, outgoing_handle, frame) + else: + new_link = SenderLink.from_incoming_frame(self, outgoing_handle, frame) + new_link._incoming_attach(frame) # pylint: disable=protected-access + self.links[frame[0]] = new_link + self._output_handles[outgoing_handle] = new_link + self._input_handles[frame[1]] = new_link + except ValueError as e: + # Reject Link + _LOGGER.debug("Unable to attach new link: %r", e, extra=self.network_trace_params) + self._input_handles[frame[1]].detach() + + def _outgoing_flow(self, frame=None): + link_flow = frame or {} + link_flow.update( + { + "next_incoming_id": self.next_incoming_id, + "incoming_window": self.incoming_window, + "next_outgoing_id": self.next_outgoing_id, + "outgoing_window": self.outgoing_window, + } + ) + flow_frame = FlowFrame(**link_flow) + if self.network_trace: + _LOGGER.debug("-> %r", flow_frame, extra=self.network_trace_params) + self._connection._process_outgoing_frame(self.channel, flow_frame) # pylint: disable=protected-access + + def _incoming_flow(self, frame): + if self.network_trace: + _LOGGER.debug("<- %r", FlowFrame(*frame), extra=self.network_trace_params) + self.next_incoming_id = frame[2] # next_outgoing_id + remote_incoming_id = frame[0] or self.next_outgoing_id # next_incoming_id TODO "initial-outgoing-id" + self.remote_incoming_window = remote_incoming_id + frame[1] - self.next_outgoing_id # incoming_window + self.remote_outgoing_window = frame[3] # outgoing_window + if frame[4] is not None: # handle + self._input_handles[frame[4]]._incoming_flow(frame) # pylint: disable=protected-access + else: + for link in self._output_handles.values(): + if self.remote_incoming_window > 0 and not link._is_closed: # pylint: disable=protected-access + link._incoming_flow(frame) # pylint: disable=protected-access + + def _outgoing_transfer(self, delivery, network_trace_params): + if self.state != SessionState.MAPPED: + delivery.transfer_state = SessionTransferState.ERROR + if self.remote_incoming_window <= 0: + delivery.transfer_state = SessionTransferState.BUSY + else: + payload = delivery.frame["payload"] + payload_size = len(payload) + + delivery.frame["delivery_id"] = self.next_outgoing_id + # calculate the transfer frame encoding size excluding the payload + delivery.frame["payload"] = b"" + # TODO: encoding a frame would be expensive, we might want to improve depending on the perf test results + encoded_frame = encode_frame(TransferFrame(**delivery.frame))[1] + transfer_overhead_size = len(encoded_frame) + + # available size for payload per frame is calculated as following: + # remote max frame size - transfer overhead (calculated) - header (8 bytes) + available_frame_size = ( + self._connection._remote_max_frame_size - transfer_overhead_size - 8 # pylint: disable=protected-access + ) + + start_idx = 0 + remaining_payload_cnt = payload_size + # encode n-1 frames if payload_size > available_frame_size + while remaining_payload_cnt > available_frame_size: + tmp_delivery_frame = { + "handle": delivery.frame["handle"], + "delivery_tag": delivery.frame["delivery_tag"], + "message_format": delivery.frame["message_format"], + "settled": delivery.frame["settled"], + "more": True, + "rcv_settle_mode": delivery.frame["rcv_settle_mode"], + "state": delivery.frame["state"], + "resume": delivery.frame["resume"], + "aborted": delivery.frame["aborted"], + "batchable": delivery.frame["batchable"], + "delivery_id": self.next_outgoing_id, + } + if network_trace_params: + # We determine the logging for the outgoing Transfer frames based on the source + # Link configuration rather than the Session, because it's only at the Session + # level that we can determine how many outgoing frames are needed and their + # delivery IDs. + # TODO: Obscuring the payload for now to investigate the potential for leaks. + _LOGGER.debug( + "-> %r", TransferFrame(payload=b"***", **tmp_delivery_frame), extra=network_trace_params + ) + self._connection._process_outgoing_frame( # pylint: disable=protected-access + self.channel, + TransferFrame(payload=payload[start_idx : start_idx + available_frame_size], **tmp_delivery_frame), + ) + start_idx += available_frame_size + remaining_payload_cnt -= available_frame_size + + # encode the last frame + tmp_delivery_frame = { + "handle": delivery.frame["handle"], + "delivery_tag": delivery.frame["delivery_tag"], + "message_format": delivery.frame["message_format"], + "settled": delivery.frame["settled"], + "more": False, + "rcv_settle_mode": delivery.frame["rcv_settle_mode"], + "state": delivery.frame["state"], + "resume": delivery.frame["resume"], + "aborted": delivery.frame["aborted"], + "batchable": delivery.frame["batchable"], + "delivery_id": self.next_outgoing_id, + } + if network_trace_params: + # We determine the logging for the outgoing Transfer frames based on the source + # Link configuration rather than the Session, because it's only at the Session + # level that we can determine how many outgoing frames are needed and their + # delivery IDs. + # TODO: Obscuring the payload for now to investigate the potential for leaks. + _LOGGER.debug("-> %r", TransferFrame(payload=b"***", **tmp_delivery_frame), extra=network_trace_params) + self._connection._process_outgoing_frame( # pylint: disable=protected-access + self.channel, TransferFrame(payload=payload[start_idx:], **tmp_delivery_frame) + ) + self.next_outgoing_id += 1 + self.remote_incoming_window -= 1 + self.outgoing_window -= 1 + # TODO: We should probably handle an error at the connection and update state accordingly + delivery.transfer_state = SessionTransferState.OKAY + + def _incoming_transfer(self, frame): + # TODO: should this be only if more=False? + self.next_incoming_id += 1 + self.remote_outgoing_window -= 1 + self.incoming_window -= 1 + try: + self._input_handles[frame[0]]._incoming_transfer(frame) # pylint: disable=protected-access + except KeyError: + _LOGGER.error( + "Received Transfer frame on unattached link. Ending session.", extra=self.network_trace_params + ) + self._set_state(SessionState.DISCARDING) + self.end( + error=AMQPError( + condition=ErrorCondition.SessionUnattachedHandle, + description="""Invalid handle reference in received frame: """ + """Handle is not currently associated with an attached link""", + ) + ) + return + if self.incoming_window == 0: + self.incoming_window = self.target_incoming_window + self._outgoing_flow() + + def _outgoing_disposition(self, frame): + self._connection._process_outgoing_frame(self.channel, frame) # pylint: disable=protected-access + + def _incoming_disposition(self, frame): + if self.network_trace: + _LOGGER.debug("<- %r", DispositionFrame(*frame), extra=self.network_trace_params) + for link in self._input_handles.values(): + link._incoming_disposition(frame) # pylint: disable=protected-access + + def _outgoing_detach(self, frame): + self._connection._process_outgoing_frame(self.channel, frame) # pylint: disable=protected-access + + def _incoming_detach(self, frame): + try: + link = self._input_handles[frame[0]] # handle + link._incoming_detach(frame) # pylint: disable=protected-access + # if link._is_closed: TODO + # self.links.pop(link.name, None) + # self._input_handles.pop(link.remote_handle, None) + # self._output_handles.pop(link.handle, None) + except KeyError: + self._set_state(SessionState.DISCARDING) + self._connection.close( + error=AMQPError( + condition=ErrorCondition.SessionUnattachedHandle, + description="""Invalid handle reference in received frame: """ + """Handle is not currently associated with an attached link""", + ) + ) + + def _wait_for_response(self, wait: Union[bool, float], end_state: SessionState) -> None: + if wait is True: + self._connection.listen(wait=False) + while self.state != end_state: + time.sleep(self.idle_wait_time) + self._connection.listen(wait=False) + elif wait: + self._connection.listen(wait=False) + timeout = time.time() + wait + while self.state != end_state: + if time.time() >= timeout: + break + time.sleep(self.idle_wait_time) + self._connection.listen(wait=False) + + def begin(self, wait=False): + self._outgoing_begin() + self._set_state(SessionState.BEGIN_SENT) + if wait: + self._wait_for_response(wait, SessionState.BEGIN_SENT) + elif not self.allow_pipelined_open: + raise ValueError("Connection has been configured to not allow piplined-open. Please set 'wait' parameter.") + + def end(self, error: Optional[AMQPError] = None, wait: bool = False) -> None: + try: + if self.state not in [SessionState.UNMAPPED, SessionState.DISCARDING]: + self._outgoing_end(error=error) + for _, link in self.links.items(): + link.detach() + new_state = SessionState.DISCARDING if error else SessionState.END_SENT + self._set_state(new_state) + self._wait_for_response(wait, SessionState.UNMAPPED) + except Exception as exc: # pylint: disable=broad-except + _LOGGER.info("An error occurred when ending the session: %r", exc, extra=self.network_trace_params) + self._set_state(SessionState.UNMAPPED) + + def create_receiver_link(self, source_address, **kwargs): + assigned_handle = self._get_next_output_handle() + link = ReceiverLink( + self, + handle=assigned_handle, + source_address=source_address, + network_trace=kwargs.pop("network_trace", self.network_trace), + network_trace_params=dict(self.network_trace_params), + **kwargs, + ) + self.links[link.name] = link + self._output_handles[assigned_handle] = link + return link + + def create_sender_link(self, target_address, **kwargs): + assigned_handle = self._get_next_output_handle() + link = SenderLink( + self, + handle=assigned_handle, + target_address=target_address, + network_trace=kwargs.pop("network_trace", self.network_trace), + network_trace_params=dict(self.network_trace_params), + **kwargs, + ) + self._output_handles[assigned_handle] = link + self.links[link.name] = link + return link + + def create_request_response_link_pair(self, endpoint, **kwargs): + return ManagementLink( + self, + endpoint, + network_trace=kwargs.pop("network_trace", self.network_trace), + network_trace_params=dict(self.network_trace_params), + **kwargs, + ) diff --git a/src/azure/iot/hub/_pyamqp/types.py b/src/azure/iot/hub/_pyamqp/types.py new file mode 100644 index 0000000..03b1501 --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/types.py @@ -0,0 +1,92 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from enum import Enum + + +TYPE = "TYPE" +VALUE = "VALUE" + + +class AMQPTypes: + null = "NULL" + boolean = "BOOL" + ubyte = "UBYTE" + byte = "BYTE" + ushort = "USHORT" + short = "SHORT" + uint = "UINT" + int = "INT" + ulong = "ULONG" + long = "LONG" + float = "FLOAT" + double = "DOUBLE" + timestamp = "TIMESTAMP" + uuid = "UUID" + binary = "BINARY" + string = "STRING" + symbol = "SYMBOL" + list = "LIST" + map = "MAP" + array = "ARRAY" + described = "DESCRIBED" + decimal128 = "DECIMAL128" + + +class FieldDefinition(Enum): + fields = "fields" + annotations = "annotations" + message_id = "message-id" + app_properties = "application-properties" + node_properties = "node-properties" + filter_set = "filter-set" + + +class ObjDefinition(Enum): + source = "source" + target = "target" + delivery_state = "delivery-state" + error = "error" + + +class ConstructorBytes(object): + null = b"\x40" + bool = b"\x56" + bool_true = b"\x41" + bool_false = b"\x42" + ubyte = b"\x50" + byte = b"\x51" + ushort = b"\x60" + short = b"\x61" + uint_0 = b"\x43" + uint_small = b"\x52" + int_small = b"\x54" + uint_large = b"\x70" + int_large = b"\x71" + ulong_0 = b"\x44" + ulong_small = b"\x53" + long_small = b"\x55" + ulong_large = b"\x80" + long_large = b"\x81" + float = b"\x72" + double = b"\x82" + timestamp = b"\x83" + uuid = b"\x98" + binary_small = b"\xA0" + binary_large = b"\xB0" + string_small = b"\xA1" + string_large = b"\xB1" + symbol_small = b"\xA3" + symbol_large = b"\xB3" + list_0 = b"\x45" + list_small = b"\xC0" + list_large = b"\xD0" + map_small = b"\xC1" + map_large = b"\xD1" + array_small = b"\xE0" + array_large = b"\xF0" + descriptor = b"\x00" + decimal128 = b"\x94" diff --git a/src/azure/iot/hub/_pyamqp/utils.py b/src/azure/iot/hub/_pyamqp/utils.py new file mode 100644 index 0000000..eec4e34 --- /dev/null +++ b/src/azure/iot/hub/_pyamqp/utils.py @@ -0,0 +1,139 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import datetime +from typing import TYPE_CHECKING +from base64 import b64encode +from hashlib import sha256 +from hmac import HMAC +from urllib.parse import urlencode, quote_plus +import time +from datetime import timezone +from .types import TYPE, VALUE, AMQPTypes +from ._encode import encode_payload +from .message import Properties + +if TYPE_CHECKING: + from .message import Message + +TZ_UTC: timezone = timezone.utc +# Number of seconds between the Unix epoch (1/1/1970) and year 1 CE. +# This is the lowest value that can be represented by an AMQP timestamp. +CE_ZERO_SECONDS: int = -62_135_596_800 + +def utc_from_timestamp(timestamp: float) -> datetime.datetime: + """ + :param float timestamp: Timestamp in seconds to be converted to datetime. + :rtype: datetime.datetime + :returns: A datetime object representing the timestamp in UTC. + """ + # The AMQP timestamp is the number of seconds since the Unix epoch. + # AMQP brokers represent the lowest value as -62_135_596_800 (the + # number of seconds between the Unix epoch (1/1/1970) and year 1 CE) as + # a sentinel for a time which is not set. + if timestamp == CE_ZERO_SECONDS: + return datetime.datetime.min.replace(tzinfo=TZ_UTC) + return datetime.datetime.fromtimestamp(timestamp, tz=TZ_UTC) + + +def utc_now(): + return datetime.datetime.now(tz=TZ_UTC) + + +def encode(value, encoding="UTF-8"): + return value.encode(encoding) if isinstance(value, str) else value + + +def generate_sas_token(audience, policy, key, expiry=None): + """ + Generate a sas token according to the given audience, policy, key and expiry + + :param str audience: + :param str policy: + :param str key: + :param int expiry: abs expiry time + :return: A sas token + :rtype: str + """ + if not expiry: + expiry = int(time.time()) + 3600 # Default to 1 hour. + + encoded_uri = quote_plus(audience) + encoded_policy = quote_plus(policy).encode("utf-8") + encoded_key = key.encode("utf-8") + + ttl = int(expiry) + sign_key = "%s\n%d" % (encoded_uri, ttl) + signature = b64encode(HMAC(encoded_key, sign_key.encode("utf-8"), sha256).digest()) + result = {"sr": audience, "sig": signature, "se": str(ttl)} + if policy: + result["skn"] = encoded_policy + return "SharedAccessSignature " + urlencode(result) + + +def add_batch(batch, message): + # Add a message to a batch + output = bytearray() + encode_payload(output, message) + batch[5].append(output) + +def set_message_properties(message, properties: list): + if not message[3]: + message[3] = Properties(*properties) + +def set_message_annotations(message, annotations: dict): + if not message[2]: + message[2] = annotations + +def encode_str(data, encoding="utf-8"): + try: + return data.encode(encoding) + except AttributeError: + return data + + +def normalized_data_body(data, **kwargs): + # A helper method to normalize input into AMQP Data Body format + encoding = kwargs.get("encoding", "utf-8") + if isinstance(data, list): + return [encode_str(item, encoding) for item in data] + return [encode_str(data, encoding)] + + +def normalized_sequence_body(sequence): # pylint:disable=inconsistent-return-statements + # A helper method to normalize input into AMQP Sequence Body format + if isinstance(sequence, list) and all((isinstance(b, list) for b in sequence)): + return sequence + if isinstance(sequence, list): + return [sequence] + + +def get_message_encoded_size(message): + output = bytearray() + encode_payload(output, message) + return len(output) + + +def amqp_long_value(value): + # A helper method to wrap a Python int as AMQP long + # TODO: wrapping one line in a function is expensive, find if there's a better way to do it + return {TYPE: AMQPTypes.long, VALUE: value} + + +def amqp_uint_value(value): + # A helper method to wrap a Python int as AMQP uint + return {TYPE: AMQPTypes.uint, VALUE: value} + + +def amqp_string_value(value): + return {TYPE: AMQPTypes.string, VALUE: value} + + +def amqp_symbol_value(value): + return {TYPE: AMQPTypes.symbol, VALUE: value} + + +def amqp_array_value(value): + return {TYPE: AMQPTypes.array, VALUE: value} diff --git a/src/azure/iot/hub/iothub_amqp_client.py b/src/azure/iot/hub/iothub_amqp_client.py index b04b41b..9bd1d02 100644 --- a/src/azure/iot/hub/iothub_amqp_client.py +++ b/src/azure/iot/hub/iothub_amqp_client.py @@ -11,7 +11,12 @@ from uuid import uuid4 from urllib import parse as urllib_parse from azure.core.credentials import AccessToken -import uamqp + +from ._pyamqp import SendClient +from ._pyamqp.authentication import JWTTokenAuth +from ._pyamqp.constants import TransportType +from ._pyamqp.error import AMQPException +from ._pyamqp.message import Message, Properties default_sas_expiry = 3600 @@ -35,40 +40,41 @@ def send_message_to_device(self, device_id, message, app_props): :raises: Exception if the Send command is not able to send the message """ - msg_content = message - msg_props = uamqp.message.MessageProperties() - msg_props.message_id = str(uuid4()) - msg_props.to = "/devices/{}/messages/devicebound".format(device_id) - + properties_kwargs = { + "message_id": str(uuid4()), + "to": "/devices/{}/messages/devicebound".format(device_id), + } app_properties = {} - # loop through all properties and pull out the custom - # properties for prop_key, prop_value in app_props.items(): if prop_key == "contentType": - msg_props.content_type = prop_value + properties_kwargs["content_type"] = prop_value elif prop_key == "contentEncoding": - msg_props.content_encoding = prop_value + properties_kwargs["content_encoding"] = prop_value elif prop_key == "correlationId": - msg_props.correlation_id = prop_value + properties_kwargs["correlation_id"] = prop_value elif prop_key == "expiryTimeUtc": - msg_props.absolute_expiry_time = prop_value + properties_kwargs["absolute_expiry_time"] = prop_value elif prop_key == "messageId": - msg_props.message_id = prop_value + properties_kwargs["message_id"] = prop_value else: app_properties[prop_key] = prop_value - message = uamqp.Message( - msg_content, properties=msg_props, application_properties=app_properties + msg_body = message.encode("utf-8") if isinstance(message, str) else message + amqp_message = Message( + properties=Properties(**properties_kwargs), + application_properties=app_properties, + data=[msg_body], ) - self.amqp_client.queue_message(message) - results = self.amqp_client.send_all_messages(close_on_done=False) - if uamqp.constants.MessageState.SendFailed in results: + + try: + self.amqp_client.send_message(amqp_message) + except AMQPException: raise Exception("C2D message send failure") class IoTHubAmqpClientSharedAccessKeyAuth(IoTHubAmqpClientBase): - def __init__(self, hostname, shared_access_key_name, shared_access_key, transport_type=uamqp.TransportType.Amqp): + def __init__(self, hostname, shared_access_key_name, shared_access_key, transport_type=TransportType.Amqp): def get_token(): expiry = int(time.time() + default_sas_expiry) sas = base64.b64decode(shared_access_key) @@ -82,16 +88,15 @@ def get_token(): expiry, ) - auth = uamqp.authentication.JWTTokenAuth( - audience="https://" + hostname, - uri="https://" + hostname, - get_token=get_token, + auth = JWTTokenAuth( + "https://" + hostname, + "https://" + hostname, + get_token, token_type=b"servicebus.windows.net:sastoken", - transport_type=transport_type, ) - auth.update_token() - self.amqp_client = uamqp.SendClient( - target="amqps://" + hostname + "/messages/devicebound", + self.amqp_client = SendClient( + hostname, + "amqps://" + hostname + "/messages/devicebound", auth=auth, keep_alive_interval=120, transport_type=transport_type, @@ -100,19 +105,22 @@ def get_token(): class IoTHubAmqpClientTokenAuth(IoTHubAmqpClientBase): def __init__( - self, hostname, token_credential, token_scope="https://iothubs.azure.net/.default", transport_type=uamqp.TransportType.Amqp + self, hostname, token_credential, token_scope="https://iothubs.azure.net/.default", transport_type=TransportType.Amqp ): def get_token(): result = token_credential.get_token(token_scope) return AccessToken("Bearer " + result.token, result.expires_on) - auth = uamqp.authentication.JWTTokenAuth( - audience=token_scope, - uri="https://" + hostname, - get_token=get_token, + auth = JWTTokenAuth( + "https://" + hostname, + token_scope, + get_token, token_type=b"bearer", - transport_type=transport_type ) - auth.update_token() - target = "amqps://" + hostname + "/messages/devicebound" - self.amqp_client = uamqp.SendClient(target=target, auth=auth, keep_alive_interval=120, transport_type=transport_type) + self.amqp_client = SendClient( + hostname, + "amqps://" + hostname + "/messages/devicebound", + auth=auth, + keep_alive_interval=120, + transport_type=transport_type, + ) diff --git a/src/azure/iot/hub/iothub_registry_manager.py b/src/azure/iot/hub/iothub_registry_manager.py index 6bb59b5..b674444 100644 --- a/src/azure/iot/hub/iothub_registry_manager.py +++ b/src/azure/iot/hub/iothub_registry_manager.py @@ -14,7 +14,7 @@ AuthenticationMechanism, DeviceCapabilities, ) -from uamqp import TransportType +from ._pyamqp.constants import TransportType def _ensure_quoted(etag): @@ -68,7 +68,7 @@ def __init__(self, connection_string=None, host=None, token_credential=None, tra Default value: None :param transport_type: The underlying transport protocol type: Amqp: AMQP over the default TCP transport protocol, it uses port 5671. AmqpOverWebsocket: Amqp over the Web Sockets transport protocol, it uses port 443. Default value: Amqp - :type transport_type: :class:`uamqp.TransportType` + :type transport_type: :class:`azure.iot.hub.TransportType` :returns: Instance of the IoTHubRegistryManager object. :rtype: :class:`azure.iot.hub.IoTHubRegistryManager` @@ -105,7 +105,7 @@ def from_connection_string(cls, connection_string, transport_type=TransportType. with IoTHub. :param transport_type: The underlying transport protocol type: Amqp: AMQP over the default TCP transport protocol, it uses port 5671. AmqpOverWebsocket: Amqp over the Web Sockets transport protocol, it uses port 443. Default value: Amqp - :type transport_type: :class:`uamqp.TransportType` + :type transport_type: :class:`azure.iot.hub.TransportType` :rtype: :class:`azure.iot.hub.IoTHubRegistryManager` """ @@ -124,7 +124,7 @@ def from_token_credential(cls, url, token_credential, transport_type=TransportTy :type token_credential: :class:`azure.core.TokenCredential` :param transport_type: The underlying transport protocol type: Amqp: AMQP over the default TCP transport protocol, it uses port 5671. AmqpOverWebsocket: Amqp over the Web Sockets transport protocol, it uses port 443. Default value: Amqp - :type transport_type: :class:`uamqp.TransportType` + :type transport_type: :class:`azure.iot.hub.TransportType` :rtype: :class:`azure.iot.hub.IoTHubRegistryManager` """ diff --git a/tests/test_iothub_amqp_client.py b/tests/test_iothub_amqp_client.py index 4df9938..f96fcf9 100644 --- a/tests/test_iothub_amqp_client.py +++ b/tests/test_iothub_amqp_client.py @@ -7,9 +7,10 @@ import base64 import hashlib import pytest -import uamqp import hmac from azure.core.credentials import AccessToken +from azure.iot.hub._pyamqp.constants import TransportType +from azure.iot.hub._pyamqp.error import AMQPException, ErrorCondition from azure.iot.hub.iothub_amqp_client import ( IoTHubAmqpClientSharedAccessKeyAuth, IoTHubAmqpClientTokenAuth, @@ -33,78 +34,62 @@ fake_token = "fasdjkhg" fake_token_expiry = 1584727659 +expected_target = "amqps://" + fake_hostname + "/messages/devicebound" +expected_devicebound_to = "/devices/" + fake_device_id + "/messages/devicebound" + @pytest.fixture(autouse=True) -def mock_uamqp_SendClient(mocker): - mock_uamqp_SendClient = mocker.patch.object(uamqp, "SendClient") - return mock_uamqp_SendClient +def mock_SendClient(mocker): + return mocker.patch("azure.iot.hub.iothub_amqp_client.SendClient") class SharedIotHubAmqpClientSendMessageToDeviceTests(object): - @pytest.mark.it("Sends messages with application properties using the uamqp SendClient") - def test_send_message_to_device_app_prop(self, client, mocker, mock_uamqp_SendClient): + @pytest.mark.it("Sends messages with application properties using the pyamqp SendClient") + def test_send_message_to_device_app_prop(self, client, mocker, mock_SendClient): client.send_message_to_device(fake_device_id, fake_message, fake_app_prop) - amqp_client_obj = mock_uamqp_SendClient.return_value - - # Message was queued - assert amqp_client_obj.queue_message.call_count == 1 - msg_obj = amqp_client_obj.queue_message.call_args[0][0] - # Message was configured with properties and destination - assert str(msg_obj) == fake_message - assert msg_obj.properties.to == bytes( - str("/devices/" + fake_device_id + "/messages/devicebound").encode("utf-8") - ) + amqp_client_obj = mock_SendClient.return_value + + assert amqp_client_obj.send_message.call_count == 1 + msg_obj = amqp_client_obj.send_message.call_args[0][0] + + assert msg_obj.data == [fake_message.encode("utf-8")] + assert msg_obj.properties.to == expected_devicebound_to assert msg_obj.application_properties == fake_app_prop - # Message was sent - assert amqp_client_obj.send_all_messages.call_count == 1 - assert amqp_client_obj.send_all_messages.call_args == mocker.call(close_on_done=False) - @pytest.mark.it("Sends messages with system properties using the uamqp SendClient") - def test_send_message_to_device_sys_prop(self, client, mocker, mock_uamqp_SendClient): + @pytest.mark.it("Sends messages with system properties using the pyamqp SendClient") + def test_send_message_to_device_sys_prop(self, client, mocker, mock_SendClient): client.send_message_to_device(fake_device_id, fake_message, fake_sys_prop) - amqp_client_obj = mock_uamqp_SendClient.return_value - - # Message was queued - assert amqp_client_obj.queue_message.call_count == 1 - msg_obj = amqp_client_obj.queue_message.call_args[0][0] - # Message was configured with properties and destination - assert str(msg_obj) == fake_message - assert msg_obj.properties.to == bytes( - str("/devices/" + fake_device_id + "/messages/devicebound").encode("utf-8") - ) + amqp_client_obj = mock_SendClient.return_value + + assert amqp_client_obj.send_message.call_count == 1 + msg_obj = amqp_client_obj.send_message.call_args[0][0] + + assert msg_obj.data == [fake_message.encode("utf-8")] + assert msg_obj.properties.to == expected_devicebound_to assert msg_obj.application_properties == {} - assert msg_obj.properties.content_type == bytes( - str(fake_sys_prop["contentType"]).encode("utf-8") - ) - assert msg_obj.properties.content_encoding == bytes( - str(fake_sys_prop["contentEncoding"]).encode("utf-8") - ) - assert msg_obj.properties.correlation_id == bytes( - str(fake_sys_prop["correlationId"]).encode("utf-8") - ) + assert msg_obj.properties.content_type == fake_sys_prop["contentType"] + assert msg_obj.properties.content_encoding == fake_sys_prop["contentEncoding"] + assert msg_obj.properties.correlation_id == fake_sys_prop["correlationId"] assert msg_obj.properties.absolute_expiry_time == fake_sys_prop["expiryTimeUtc"] assert msg_obj.properties.message_id == fake_sys_prop["messageId"] - # Message was sent - assert amqp_client_obj.send_all_messages.call_count == 1 - assert amqp_client_obj.send_all_messages.call_args == mocker.call(close_on_done=False) - - @pytest.mark.it("Raises an Exception if send_all_messages fails") - def test_raise_exception_on_send_fail(self, client, mocker, mock_uamqp_SendClient): - amqp_client_obj = mock_uamqp_SendClient.return_value - mocker.patch.object( - amqp_client_obj, "send_all_messages", {uamqp.constants.MessageState.SendFailed} + + @pytest.mark.it("Raises an Exception if pyamqp send_message raises an AMQPException") + def test_raise_exception_on_send_fail(self, client, mocker, mock_SendClient): + amqp_client_obj = mock_SendClient.return_value + amqp_client_obj.send_message.side_effect = AMQPException( + condition=ErrorCondition.UnknownError, description="fake failure" ) - with pytest.raises(Exception): + with pytest.raises(Exception, match="C2D message send failure"): client.send_message_to_device(fake_device_id, fake_message, fake_app_prop) class SharedIotHubAmqpClientDisconnectSyncTests(object): @pytest.mark.it( - "Calls close() on the uamqp SendClient and removes it when disconnect_sync() is called" + "Calls close() on the pyamqp SendClient and removes it when disconnect_sync() is called" ) - def test_disconnect_sync(self, client, mock_uamqp_SendClient): + def test_disconnect_sync(self, client, mock_SendClient): assert client.amqp_client is not None - amqp_client_obj = mock_uamqp_SendClient.return_value + amqp_client_obj = mock_SendClient.return_value client.disconnect_sync() @@ -134,8 +119,8 @@ class TestIoTHubAmqpClientSharedAccessKeyAuthInstantiation( @pytest.mark.it( "Creates a JWTTokenAuth instance with the correct parameters and uses it to create an AMQP SendClient" ) - def test_create_JWTTokenAuth_with_sas_token(self, mocker, mock_uamqp_SendClient): - amqp_token_init_mock = mocker.patch.object(uamqp.authentication, "JWTTokenAuth") + def test_create_JWTTokenAuth_with_sas_token(self, mocker, mock_SendClient): + amqp_token_init_mock = mocker.patch("azure.iot.hub.iothub_amqp_client.JWTTokenAuth") amqp_token_mock = amqp_token_init_mock.return_value IoTHubAmqpClientSharedAccessKeyAuth( @@ -144,43 +129,52 @@ def test_create_JWTTokenAuth_with_sas_token(self, mocker, mock_uamqp_SendClient) # JWTTokenAuth creation assert amqp_token_init_mock.call_count == 1 - assert amqp_token_init_mock.call_args[1]["uri"] == "https://" + fake_hostname - assert amqp_token_init_mock.call_args[1]["audience"] == "https://" + fake_hostname - assert amqp_token_init_mock.call_args[1]["token_type"] == b"servicebus.windows.net:sastoken" - assert amqp_token_init_mock.call_args[1]["transport_type"] == uamqp.TransportType.Amqp - assert amqp_token_mock.update_token.call_count == 1 + args, kwargs = amqp_token_init_mock.call_args + assert args[0] == "https://" + fake_hostname + assert args[1] == "https://" + fake_hostname + assert callable(args[2]) + assert kwargs["token_type"] == b"servicebus.windows.net:sastoken" # AMQP SendClient is created - assert mock_uamqp_SendClient.call_count == 1 - expected_target = "amqps://" + fake_hostname + "/messages/devicebound" - assert mock_uamqp_SendClient.call_args == mocker.call( - target=expected_target, auth=amqp_token_mock, keep_alive_interval=120, transport_type=uamqp.TransportType.Amqp + assert mock_SendClient.call_count == 1 + assert mock_SendClient.call_args == mocker.call( + fake_hostname, + expected_target, + auth=amqp_token_mock, + keep_alive_interval=120, + transport_type=TransportType.Amqp, ) @pytest.mark.it( "Creates a JWTTokenAuth instance with the correct parameters and uses it to create an AMQP over Websocket SendClient" ) - def test_create_JWTTokenAuth_with_sas_token_amqp_over_websocket(self, mocker, mock_uamqp_SendClient): - amqp_token_init_mock = mocker.patch.object(uamqp.authentication, "JWTTokenAuth") + def test_create_JWTTokenAuth_with_sas_token_amqp_over_websocket(self, mocker, mock_SendClient): + amqp_token_init_mock = mocker.patch("azure.iot.hub.iothub_amqp_client.JWTTokenAuth") amqp_token_mock = amqp_token_init_mock.return_value IoTHubAmqpClientSharedAccessKeyAuth( - fake_hostname, fake_shared_access_key_name, fake_shared_access_key, transport_type=uamqp.TransportType.AmqpOverWebsocket + fake_hostname, + fake_shared_access_key_name, + fake_shared_access_key, + transport_type=TransportType.AmqpOverWebsocket, ) # JWTTokenAuth creation assert amqp_token_init_mock.call_count == 1 - assert amqp_token_init_mock.call_args[1]["uri"] == "https://" + fake_hostname - assert amqp_token_init_mock.call_args[1]["audience"] == "https://" + fake_hostname - assert amqp_token_init_mock.call_args[1]["token_type"] == b"servicebus.windows.net:sastoken" - assert amqp_token_init_mock.call_args[1]["transport_type"] == uamqp.TransportType.AmqpOverWebsocket - assert amqp_token_mock.update_token.call_count == 1 + args, kwargs = amqp_token_init_mock.call_args + assert args[0] == "https://" + fake_hostname + assert args[1] == "https://" + fake_hostname + assert callable(args[2]) + assert kwargs["token_type"] == b"servicebus.windows.net:sastoken" # AMQP over Websocket SendClient is created - assert mock_uamqp_SendClient.call_count == 1 - expected_target = "amqps://" + fake_hostname + "/messages/devicebound" - assert mock_uamqp_SendClient.call_args == mocker.call( - target=expected_target, auth=amqp_token_mock, keep_alive_interval=120, transport_type=uamqp.TransportType.AmqpOverWebsocket + assert mock_SendClient.call_count == 1 + assert mock_SendClient.call_args == mocker.call( + fake_hostname, + expected_target, + auth=amqp_token_mock, + keep_alive_interval=120, + transport_type=TransportType.AmqpOverWebsocket, ) @pytest.mark.it("Creates an HMAC to generate a shared access signature") @@ -189,15 +183,19 @@ def test_creates_hmac(self, mocker): hmac_digest_mock = hmac_mock.return_value.digest hmac_digest_mock.return_value = b"\xd2\x06\xf7\x12\xf1\xe9\x95$\x90\xfd\x12\x9a\xb1\xbe\xb4\xf8\xf3\xc4\x1ap\x8a\xab'\x8a.D\xfb\x84\x96\xca\xf3z" + # Also patch JWTTokenAuth so its get_token callback is invoked and exercises the HMAC path. + amqp_token_init_mock = mocker.patch("azure.iot.hub.iothub_amqp_client.JWTTokenAuth") + IoTHubAmqpClientSharedAccessKeyAuth( fake_hostname, fake_shared_access_key_name, fake_shared_access_key ) + get_token = amqp_token_init_mock.call_args[0][2] + get_token() assert hmac_mock.call_count == 1 assert hmac_mock.call_args == mocker.call( base64.b64decode(fake_shared_access_key + "="), mocker.ANY, hashlib.sha256 ) - assert hmac_digest_mock.call_count == 1 @@ -242,9 +240,9 @@ class TestIotHubAmqpClientTokenAuthInstantiation(IoTHubAmqpClientTokenAuthTestCo "Creates a JWTTokenAuth instance with the correct parameters and uses it to create an AMQP SendClient when a token scope is specified" ) def test_create_JWTTokenAuth_with_bearer_token_custom_scope( - self, mocker, mock_azure_identity_TokenCredential, mock_uamqp_SendClient + self, mocker, mock_azure_identity_TokenCredential, mock_SendClient ): - amqp_token_init_mock = mocker.patch.object(uamqp.authentication, "JWTTokenAuth") + amqp_token_init_mock = mocker.patch("azure.iot.hub.iothub_amqp_client.JWTTokenAuth") amqp_token_mock = amqp_token_init_mock.return_value IoTHubAmqpClientTokenAuth( @@ -253,78 +251,95 @@ def test_create_JWTTokenAuth_with_bearer_token_custom_scope( # JWTTokenAuth Creation assert amqp_token_init_mock.call_count == 1 - assert amqp_token_init_mock.call_args[1]["uri"] == "https://" + fake_hostname - assert amqp_token_init_mock.call_args[1]["audience"] == fake_token_scope - assert amqp_token_init_mock.call_args[1]["token_type"] == b"bearer" - assert amqp_token_init_mock.call_args[1]["transport_type"] == uamqp.TransportType.Amqp - assert amqp_token_mock.update_token.call_count == 1 + args, kwargs = amqp_token_init_mock.call_args + assert args[0] == "https://" + fake_hostname + assert args[1] == fake_token_scope + assert callable(args[2]) + assert kwargs["token_type"] == b"bearer" # AMQP SendClient is created - assert mock_uamqp_SendClient.call_count == 1 - expected_target = "amqps://" + fake_hostname + "/messages/devicebound" - assert mock_uamqp_SendClient.call_args == mocker.call( - target=expected_target, auth=amqp_token_mock, keep_alive_interval=120, transport_type=uamqp.TransportType.Amqp + assert mock_SendClient.call_count == 1 + assert mock_SendClient.call_args == mocker.call( + fake_hostname, + expected_target, + auth=amqp_token_mock, + keep_alive_interval=120, + transport_type=TransportType.Amqp, ) @pytest.mark.it( "Creates a JWTTokenAuth instance with the correct parameters and uses it to create an AMQP SendClient if no token scope is provided (uses the default scope of 'https://iothubs.azure.net/.default')" ) def test_create_JWTTokenAuth_with_bearer_token_default_scope( - self, mocker, mock_azure_identity_TokenCredential, mock_uamqp_SendClient + self, mocker, mock_azure_identity_TokenCredential, mock_SendClient ): - amqp_token_init_mock = mocker.patch.object(uamqp.authentication, "JWTTokenAuth") + amqp_token_init_mock = mocker.patch("azure.iot.hub.iothub_amqp_client.JWTTokenAuth") amqp_token_mock = amqp_token_init_mock.return_value IoTHubAmqpClientTokenAuth(fake_hostname, mock_azure_identity_TokenCredential) # JWTTokenAuth Creation assert amqp_token_init_mock.call_count == 1 - assert amqp_token_init_mock.call_args[1]["uri"] == "https://" + fake_hostname - assert amqp_token_init_mock.call_args[1]["audience"] == "https://iothubs.azure.net/.default" - assert amqp_token_init_mock.call_args[1]["token_type"] == b"bearer" - assert amqp_token_init_mock.call_args[1]["transport_type"] == uamqp.TransportType.Amqp - assert amqp_token_mock.update_token.call_count == 1 + args, kwargs = amqp_token_init_mock.call_args + assert args[0] == "https://" + fake_hostname + assert args[1] == "https://iothubs.azure.net/.default" + assert callable(args[2]) + assert kwargs["token_type"] == b"bearer" # AMQP SendClient is created - assert mock_uamqp_SendClient.call_count == 1 - expected_target = "amqps://" + fake_hostname + "/messages/devicebound" - assert mock_uamqp_SendClient.call_args == mocker.call( - target=expected_target, auth=amqp_token_mock, keep_alive_interval=120, transport_type=uamqp.TransportType.Amqp + assert mock_SendClient.call_count == 1 + assert mock_SendClient.call_args == mocker.call( + fake_hostname, + expected_target, + auth=amqp_token_mock, + keep_alive_interval=120, + transport_type=TransportType.Amqp, ) @pytest.mark.it( "Creates a JWTTokenAuth instance with the correct parameters and uses it to create an AMQP over WebSocket SendClient if no token scope is provided (uses the default scope of 'https://iothubs.azure.net/.default')" ) def test_create_JWTTokenAuth_with_bearer_token_default_scope_AmqpOverWebsocket( - self, mocker, mock_azure_identity_TokenCredential, mock_uamqp_SendClient + self, mocker, mock_azure_identity_TokenCredential, mock_SendClient ): - amqp_token_init_mock = mocker.patch.object(uamqp.authentication, "JWTTokenAuth") + amqp_token_init_mock = mocker.patch("azure.iot.hub.iothub_amqp_client.JWTTokenAuth") amqp_token_mock = amqp_token_init_mock.return_value - IoTHubAmqpClientTokenAuth(fake_hostname, mock_azure_identity_TokenCredential, transport_type=uamqp.TransportType.AmqpOverWebsocket) + IoTHubAmqpClientTokenAuth( + fake_hostname, + mock_azure_identity_TokenCredential, + transport_type=TransportType.AmqpOverWebsocket, + ) # JWTTokenAuth Creation assert amqp_token_init_mock.call_count == 1 - assert amqp_token_init_mock.call_args[1]["uri"] == "https://" + fake_hostname - assert amqp_token_init_mock.call_args[1]["audience"] == "https://iothubs.azure.net/.default" - assert amqp_token_init_mock.call_args[1]["token_type"] == b"bearer" - assert amqp_token_init_mock.call_args[1]["transport_type"] == uamqp.TransportType.AmqpOverWebsocket - assert amqp_token_mock.update_token.call_count == 1 + args, kwargs = amqp_token_init_mock.call_args + assert args[0] == "https://" + fake_hostname + assert args[1] == "https://iothubs.azure.net/.default" + assert callable(args[2]) + assert kwargs["token_type"] == b"bearer" # AMQP over Websocket SendClient is created - assert mock_uamqp_SendClient.call_count == 1 - expected_target = "amqps://" + fake_hostname + "/messages/devicebound" - assert mock_uamqp_SendClient.call_args == mocker.call( - target=expected_target, auth=amqp_token_mock, keep_alive_interval=120, transport_type=uamqp.TransportType.AmqpOverWebsocket + assert mock_SendClient.call_count == 1 + assert mock_SendClient.call_args == mocker.call( + fake_hostname, + expected_target, + auth=amqp_token_mock, + keep_alive_interval=120, + transport_type=TransportType.AmqpOverWebsocket, ) @pytest.mark.it( "Retrieves the token from the azure-identity TokenCredential using the specified token scope" ) - def test_retrieve_token_from_azure_identity(self, mock_azure_identity_TokenCredential): + def test_retrieve_token_from_azure_identity(self, mocker, mock_azure_identity_TokenCredential): + amqp_token_init_mock = mocker.patch("azure.iot.hub.iothub_amqp_client.JWTTokenAuth") + IoTHubAmqpClientTokenAuth( fake_hostname, mock_azure_identity_TokenCredential, fake_token_scope ) + get_token = amqp_token_init_mock.call_args[0][2] + get_token() mock_azure_identity_TokenCredential.get_token.assert_called_once_with(fake_token_scope) diff --git a/tests/test_iothub_registry_manager.py b/tests/test_iothub_registry_manager.py index 3d91cb1..2104a0a 100644 --- a/tests/test_iothub_registry_manager.py +++ b/tests/test_iothub_registry_manager.py @@ -9,7 +9,7 @@ from azure.iot.hub.iothub_registry_manager import IoTHubRegistryManager from azure.iot.hub import iothub_amqp_client from azure.iot.hub.protocol.iot_hub_gateway_service_ap_is import IotHubGatewayServiceAPIs -from uamqp import TransportType +from azure.iot.hub import TransportType """---Constants---""" @@ -129,12 +129,12 @@ def mock_module_constructor(mocker): @pytest.fixture(scope="function") -def mock_uamqp_send_message_to_device(mocker): - mock_uamqp_send = mocker.patch.object( +def mock_send_message_to_device(mocker): + mock_send = mocker.patch.object( iothub_amqp_client.IoTHubAmqpClientSharedAccessKeyAuth, "send_message_to_device", ) - return mock_uamqp_send + return mock_send @pytest.mark.describe("IoTHubRegistryManager - .from_connection_string()") @@ -1433,13 +1433,13 @@ def test_invoke_device_module_method_payload_none( class TestSendC2dMessage(object): @pytest.mark.it("Test send c2d message") def test_send_c2d_message( - self, mocker, mock_uamqp_send_message_to_device, iothub_registry_manager + self, mocker, mock_send_message_to_device, iothub_registry_manager ): iothub_registry_manager.send_c2d_message(fake_device_id, fake_message_to_send) - assert mock_uamqp_send_message_to_device.call_count == 1 - assert mock_uamqp_send_message_to_device.call_args == mocker.call( + assert mock_send_message_to_device.call_count == 1 + assert mock_send_message_to_device.call_args == mocker.call( fake_device_id, fake_message_to_send, {} ) @@ -1448,14 +1448,14 @@ def test_send_c2d_message( class TestSendC2dMessageWithProperties(object): @pytest.mark.it("Test send c2d message with properties") def test_send_c2d_message_with_properties( - self, mocker, mock_uamqp_send_message_to_device, iothub_registry_manager + self, mocker, mock_send_message_to_device, iothub_registry_manager ): iothub_registry_manager.send_c2d_message( fake_device_id, fake_message_to_send, fake_properties ) - assert mock_uamqp_send_message_to_device.call_count == 1 - assert mock_uamqp_send_message_to_device.call_args == mocker.call( + assert mock_send_message_to_device.call_count == 1 + assert mock_send_message_to_device.call_args == mocker.call( fake_device_id, fake_message_to_send, fake_properties ) From 825976834941fa929055b34cd7791f3fb5536180 Mon Sep 17 00:00:00 2001 From: Ethan Ann Date: Thu, 23 Jul 2026 17:45:30 -0400 Subject: [PATCH 3/3] Address PR review feedback - Add Python 3.13 and 3.14 classifiers. - Use explicit keyword arguments when constructing JWTTokenAuth and SendClient for resilience against upstream signature changes. - Introduce C2DMessageSendError and tighten the send-failure test to assert on the specific type instead of bare Exception. - Rename and reword the HMAC test to clarify that pyamqp resolves tokens lazily, so the captured get_token callback must be invoked to drive the SAS-signing path. --- setup.py | 2 + src/azure/iot/hub/iothub_amqp_client.py | 26 ++++---- tests/test_iothub_amqp_client.py | 79 +++++++++++++------------ 3 files changed, 59 insertions(+), 48 deletions(-) diff --git a/setup.py b/setup.py index 1990047..2bbb26f 100644 --- a/setup.py +++ b/setup.py @@ -69,6 +69,8 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ], install_requires=[ "msrest>=0.6.21,<1.0.0", diff --git a/src/azure/iot/hub/iothub_amqp_client.py b/src/azure/iot/hub/iothub_amqp_client.py index 9bd1d02..816967b 100644 --- a/src/azure/iot/hub/iothub_amqp_client.py +++ b/src/azure/iot/hub/iothub_amqp_client.py @@ -22,6 +22,10 @@ default_sas_expiry = 3600 +class C2DMessageSendError(Exception): + """Raised when a cloud-to-device message fails to send.""" + + class IoTHubAmqpClientBase: def disconnect_sync(self): """ @@ -70,7 +74,7 @@ def send_message_to_device(self, device_id, message, app_props): try: self.amqp_client.send_message(amqp_message) except AMQPException: - raise Exception("C2D message send failure") + raise C2DMessageSendError("C2D message send failure") class IoTHubAmqpClientSharedAccessKeyAuth(IoTHubAmqpClientBase): @@ -89,14 +93,14 @@ def get_token(): ) auth = JWTTokenAuth( - "https://" + hostname, - "https://" + hostname, - get_token, + uri="https://" + hostname, + audience="https://" + hostname, + get_token=get_token, token_type=b"servicebus.windows.net:sastoken", ) self.amqp_client = SendClient( - hostname, - "amqps://" + hostname + "/messages/devicebound", + hostname=hostname, + target="amqps://" + hostname + "/messages/devicebound", auth=auth, keep_alive_interval=120, transport_type=transport_type, @@ -112,14 +116,14 @@ def get_token(): return AccessToken("Bearer " + result.token, result.expires_on) auth = JWTTokenAuth( - "https://" + hostname, - token_scope, - get_token, + uri="https://" + hostname, + audience=token_scope, + get_token=get_token, token_type=b"bearer", ) self.amqp_client = SendClient( - hostname, - "amqps://" + hostname + "/messages/devicebound", + hostname=hostname, + target="amqps://" + hostname + "/messages/devicebound", auth=auth, keep_alive_interval=120, transport_type=transport_type, diff --git a/tests/test_iothub_amqp_client.py b/tests/test_iothub_amqp_client.py index f96fcf9..54445db 100644 --- a/tests/test_iothub_amqp_client.py +++ b/tests/test_iothub_amqp_client.py @@ -12,6 +12,7 @@ from azure.iot.hub._pyamqp.constants import TransportType from azure.iot.hub._pyamqp.error import AMQPException, ErrorCondition from azure.iot.hub.iothub_amqp_client import ( + C2DMessageSendError, IoTHubAmqpClientSharedAccessKeyAuth, IoTHubAmqpClientTokenAuth, ) @@ -73,13 +74,13 @@ def test_send_message_to_device_sys_prop(self, client, mocker, mock_SendClient): assert msg_obj.properties.absolute_expiry_time == fake_sys_prop["expiryTimeUtc"] assert msg_obj.properties.message_id == fake_sys_prop["messageId"] - @pytest.mark.it("Raises an Exception if pyamqp send_message raises an AMQPException") + @pytest.mark.it("Raises a C2DMessageSendError if pyamqp send_message raises an AMQPException") def test_raise_exception_on_send_fail(self, client, mocker, mock_SendClient): amqp_client_obj = mock_SendClient.return_value amqp_client_obj.send_message.side_effect = AMQPException( condition=ErrorCondition.UnknownError, description="fake failure" ) - with pytest.raises(Exception, match="C2D message send failure"): + with pytest.raises(C2DMessageSendError, match="C2D message send failure"): client.send_message_to_device(fake_device_id, fake_message, fake_app_prop) @@ -129,17 +130,17 @@ def test_create_JWTTokenAuth_with_sas_token(self, mocker, mock_SendClient): # JWTTokenAuth creation assert amqp_token_init_mock.call_count == 1 - args, kwargs = amqp_token_init_mock.call_args - assert args[0] == "https://" + fake_hostname - assert args[1] == "https://" + fake_hostname - assert callable(args[2]) + _, kwargs = amqp_token_init_mock.call_args + assert kwargs["uri"] == "https://" + fake_hostname + assert kwargs["audience"] == "https://" + fake_hostname + assert callable(kwargs["get_token"]) assert kwargs["token_type"] == b"servicebus.windows.net:sastoken" # AMQP SendClient is created assert mock_SendClient.call_count == 1 assert mock_SendClient.call_args == mocker.call( - fake_hostname, - expected_target, + hostname=fake_hostname, + target=expected_target, auth=amqp_token_mock, keep_alive_interval=120, transport_type=TransportType.Amqp, @@ -161,35 +162,39 @@ def test_create_JWTTokenAuth_with_sas_token_amqp_over_websocket(self, mocker, mo # JWTTokenAuth creation assert amqp_token_init_mock.call_count == 1 - args, kwargs = amqp_token_init_mock.call_args - assert args[0] == "https://" + fake_hostname - assert args[1] == "https://" + fake_hostname - assert callable(args[2]) + _, kwargs = amqp_token_init_mock.call_args + assert kwargs["uri"] == "https://" + fake_hostname + assert kwargs["audience"] == "https://" + fake_hostname + assert callable(kwargs["get_token"]) assert kwargs["token_type"] == b"servicebus.windows.net:sastoken" # AMQP over Websocket SendClient is created assert mock_SendClient.call_count == 1 assert mock_SendClient.call_args == mocker.call( - fake_hostname, - expected_target, + hostname=fake_hostname, + target=expected_target, auth=amqp_token_mock, keep_alive_interval=120, transport_type=TransportType.AmqpOverWebsocket, ) - @pytest.mark.it("Creates an HMAC to generate a shared access signature") - def test_creates_hmac(self, mocker): + @pytest.mark.it( + "Signs the SAS token with HMAC-SHA256 when the JWTTokenAuth get_token callback is invoked" + ) + def test_get_token_callback_signs_sas_with_hmac(self, mocker): hmac_mock = mocker.patch.object(hmac, "HMAC") hmac_digest_mock = hmac_mock.return_value.digest hmac_digest_mock.return_value = b"\xd2\x06\xf7\x12\xf1\xe9\x95$\x90\xfd\x12\x9a\xb1\xbe\xb4\xf8\xf3\xc4\x1ap\x8a\xab'\x8a.D\xfb\x84\x96\xca\xf3z" - # Also patch JWTTokenAuth so its get_token callback is invoked and exercises the HMAC path. amqp_token_init_mock = mocker.patch("azure.iot.hub.iothub_amqp_client.JWTTokenAuth") IoTHubAmqpClientSharedAccessKeyAuth( fake_hostname, fake_shared_access_key_name, fake_shared_access_key ) - get_token = amqp_token_init_mock.call_args[0][2] + + # pyamqp's JWTTokenAuth resolves tokens lazily via the get_token callback rather + # than at construction, so invoke the captured callback to drive the SAS-signing path. + get_token = amqp_token_init_mock.call_args.kwargs["get_token"] get_token() assert hmac_mock.call_count == 1 @@ -251,17 +256,17 @@ def test_create_JWTTokenAuth_with_bearer_token_custom_scope( # JWTTokenAuth Creation assert amqp_token_init_mock.call_count == 1 - args, kwargs = amqp_token_init_mock.call_args - assert args[0] == "https://" + fake_hostname - assert args[1] == fake_token_scope - assert callable(args[2]) + _, kwargs = amqp_token_init_mock.call_args + assert kwargs["uri"] == "https://" + fake_hostname + assert kwargs["audience"] == fake_token_scope + assert callable(kwargs["get_token"]) assert kwargs["token_type"] == b"bearer" # AMQP SendClient is created assert mock_SendClient.call_count == 1 assert mock_SendClient.call_args == mocker.call( - fake_hostname, - expected_target, + hostname=fake_hostname, + target=expected_target, auth=amqp_token_mock, keep_alive_interval=120, transport_type=TransportType.Amqp, @@ -280,17 +285,17 @@ def test_create_JWTTokenAuth_with_bearer_token_default_scope( # JWTTokenAuth Creation assert amqp_token_init_mock.call_count == 1 - args, kwargs = amqp_token_init_mock.call_args - assert args[0] == "https://" + fake_hostname - assert args[1] == "https://iothubs.azure.net/.default" - assert callable(args[2]) + _, kwargs = amqp_token_init_mock.call_args + assert kwargs["uri"] == "https://" + fake_hostname + assert kwargs["audience"] == "https://iothubs.azure.net/.default" + assert callable(kwargs["get_token"]) assert kwargs["token_type"] == b"bearer" # AMQP SendClient is created assert mock_SendClient.call_count == 1 assert mock_SendClient.call_args == mocker.call( - fake_hostname, - expected_target, + hostname=fake_hostname, + target=expected_target, auth=amqp_token_mock, keep_alive_interval=120, transport_type=TransportType.Amqp, @@ -313,17 +318,17 @@ def test_create_JWTTokenAuth_with_bearer_token_default_scope_AmqpOverWebsocket( # JWTTokenAuth Creation assert amqp_token_init_mock.call_count == 1 - args, kwargs = amqp_token_init_mock.call_args - assert args[0] == "https://" + fake_hostname - assert args[1] == "https://iothubs.azure.net/.default" - assert callable(args[2]) + _, kwargs = amqp_token_init_mock.call_args + assert kwargs["uri"] == "https://" + fake_hostname + assert kwargs["audience"] == "https://iothubs.azure.net/.default" + assert callable(kwargs["get_token"]) assert kwargs["token_type"] == b"bearer" # AMQP over Websocket SendClient is created assert mock_SendClient.call_count == 1 assert mock_SendClient.call_args == mocker.call( - fake_hostname, - expected_target, + hostname=fake_hostname, + target=expected_target, auth=amqp_token_mock, keep_alive_interval=120, transport_type=TransportType.AmqpOverWebsocket, @@ -338,7 +343,7 @@ def test_retrieve_token_from_azure_identity(self, mocker, mock_azure_identity_To IoTHubAmqpClientTokenAuth( fake_hostname, mock_azure_identity_TokenCredential, fake_token_scope ) - get_token = amqp_token_init_mock.call_args[0][2] + get_token = amqp_token_init_mock.call_args.kwargs["get_token"] get_token() mock_azure_identity_TokenCredential.get_token.assert_called_once_with(fake_token_scope)