diff --git a/SUPPORT.md b/SUPPORT.md index c4fcc78..344f745 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -4,7 +4,7 @@ This article explains where to get help with this aMQTT project. Please read through the following guidelines. > 👉 **Note**: before participating in our community, please read our -> [code of conduct][coc]. +> [code of conduct](code_of_conduct.md). > By interacting with this repository, organization, or community you agree to > abide by its terms. @@ -32,4 +32,4 @@ Here are some tips: ## Contributions -See [`contributing.md`][contributing] on how to contribute. +See [contributing](contributing.md) on how to contribute. diff --git a/amqtt/broker.py b/amqtt/broker.py index 7dbe408..8a94a69 100644 --- a/amqtt/broker.py +++ b/amqtt/broker.py @@ -1,4 +1,5 @@ import asyncio +import copy from asyncio import CancelledError, futures from collections import deque from collections.abc import Generator @@ -7,6 +8,7 @@ from functools import partial import logging import re import ssl +from pathlib import Path from typing import Any, ClassVar from transitions import Machine, MachineError @@ -24,17 +26,16 @@ from amqtt.adapters import ( from amqtt.errors import AMQTTError, BrokerError, MQTTError, NoDataError from amqtt.mqtt.protocol.broker_handler import BrokerProtocolHandler from amqtt.session import ApplicationMessage, OutgoingApplicationMessage, Session -from amqtt.utils import format_client_message, gen_client_id +from amqtt.utils import format_client_message, gen_client_id, read_yaml_config from .plugins.manager import BaseContext, PluginManager -type CONFIG_LISTENER = dict[str, int | bool | dict[str, Any]] +type CONFIG_LISTENER = dict[str, Any] type _BROADCAST = dict[str, Session | str | bytes | int | None] -_defaults: CONFIG_LISTENER = { - "timeout-disconnect-delay": 2, - "auth": {"allow-anonymous": True, "password-file": None}, -} + +_defaults = read_yaml_config(Path(__file__).parent / "scripts/default_broker.yaml") + # Default port numbers DEFAULT_PORTS = {"tcp": 1883, "ws": 8883} @@ -140,7 +141,7 @@ class Broker: """MQTT 3.1.1 compliant broker implementation. Args: - config: dictionary of configuration options (see config yaml format) + config: dictionary of configuration options (see [broker configuration](broker_config.md)). loop: asyncio loop. defaults to `asyncio.get_event_loop()`. plugin_namespace: plugin namespace to use when loading plugin entry_points. defaults to `amqtt.broker.plugins`. @@ -164,7 +165,7 @@ class Broker: ) -> None: """Initialize the broker.""" self.logger = logging.getLogger(__name__) - self.config = _defaults.copy() + self.config = copy.deepcopy(_defaults or {}) if config is not None: self.config.update(config) self._build_listeners_config(self.config) diff --git a/amqtt/client.py b/amqtt/client.py index 05ff302..e04a67e 100644 --- a/amqtt/client.py +++ b/amqtt/client.py @@ -6,6 +6,7 @@ import copy from functools import wraps import logging import ssl +from pathlib import Path from typing import TYPE_CHECKING, Any, cast from urllib.parse import urlparse, urlunparse @@ -24,21 +25,12 @@ from amqtt.mqtt.constants import QOS_0, QOS_1, QOS_2 from amqtt.mqtt.protocol.client_handler import ClientProtocolHandler from amqtt.plugins.manager import BaseContext, PluginManager from amqtt.session import ApplicationMessage, OutgoingApplicationMessage, Session -from amqtt.utils import gen_client_id +from amqtt.utils import gen_client_id, read_yaml_config if TYPE_CHECKING: from websockets.asyncio.client import ClientConnection -_defaults: dict[str, Any] = { - "keep_alive": 10, - "ping_delay": 1, - "default_qos": 0, - "default_retain": False, - "auto_reconnect": True, - "reconnect_max_interval": 10, - "reconnect_retries": 2, -} - +_defaults: dict[str, Any] | None = read_yaml_config(Path(__file__).parent / "scripts/default_broker.yaml") class ClientContext(BaseContext): """ClientContext is used as the context passed to plugins interacting with the client. @@ -91,45 +83,13 @@ class MQTTClient: Args: client_id: MQTT client ID to use when connecting to the broker. If none, it will be generated randomly by `amqtt.utils.gen_client_id` - config: Client configuration with the following keys: - - `keep_alive`: keep alive (in seconds) to send when connecting to the broker (defaults to `10` seconds). `MQTTClient` will _auto-ping_ the broker if no message is sent within the keep-alive interval. This avoids disconnection from the broker. - - `ping_delay`: _auto-ping_ delay before keep-alive times out (defaults to `1` seconds). - - `default_qos`: Default QoS (`0`) used by `publish()` if `qos` argument is not given. - - `default_retain`: Default retain (`False`) used by `publish()` if `qos` argument is not given. - - `auto_reconnect`: enable or disable auto-reconnect feature (defaults to `True`). - - `reconnect_max_interval`: maximum interval (in seconds) to wait before two connection retries (defaults to `10`). - - `reconnect_retries`: maximum number of connect retries (defaults to `2`). Negative value will cause client to reconnect infinitely. - - Example: - - ```python - config = { - 'keep_alive': 10, - 'ping_delay': 1, - 'default_qos': 0, - 'default_retain': False, - 'auto_reconnect': True, - 'reconnect_max_interval': 5, - 'reconnect_retries': 10, - 'topics': { - 'test': { 'qos': 1 }, - 'some_topic': { 'qos': 2, 'retain': True } - } - } - ``` + config: dictionary of configuration options (see [client configuration](client_config.md)). """ def __init__(self, client_id: str | None = None, config: dict[str, Any] | None = None) -> None: self.logger = logging.getLogger(__name__) - self.config = copy.deepcopy(_defaults) + self.config = copy.deepcopy(_defaults or {}) if config is not None: self.config.update(config) self.client_id = client_id if client_id is not None else gen_client_id() diff --git a/amqtt/scripts/broker_script.py b/amqtt/scripts/broker_script.py index d0df947..0b06be3 100644 --- a/amqtt/scripts/broker_script.py +++ b/amqtt/scripts/broker_script.py @@ -22,22 +22,6 @@ import amqtt from amqtt.broker import Broker from amqtt.utils import read_yaml_config -default_config = { - "listeners": { - "default": { - "type": "tcp", - "bind": "0.0.0.0:1883", - }, - }, - "sys_interval": 10, - "auth": { - "allow-anonymous": True, - "password-file": Path(__file__).parent / "passwd", - "plugins": ["auth_file", "auth_anonymous"], - }, - "topic-check": {"enabled": False}, -} - logger = logging.getLogger(__name__) diff --git a/amqtt/scripts/default_broker.yaml b/amqtt/scripts/default_broker.yaml index 7659b1b..1ec1016 100644 --- a/amqtt/scripts/default_broker.yaml +++ b/amqtt/scripts/default_broker.yaml @@ -5,9 +5,8 @@ listeners: bind: 0.0.0.0:1883 sys_interval: 20 auth: + plugins: + - auth_anonymous allow-anonymous: true -plugins: - - auth_file - - auth_anonymous topic-check: enabled: False diff --git a/amqtt/scripts/default_client.yaml b/amqtt/scripts/default_client.yaml index 0ebdf86..b422baf 100644 --- a/amqtt/scripts/default_client.yaml +++ b/amqtt/scripts/default_client.yaml @@ -3,6 +3,6 @@ keep_alive: 10 ping_delay: 1 default_qos: 0 default_retain: false -auto_reconnect: false +auto_reconnect: true reconnect_max_interval: 10 reconnect_retries: 2 diff --git a/docs/assets/extra.css b/docs/assets/extra.css index 4a4cc05..be8f419 100644 --- a/docs/assets/extra.css +++ b/docs/assets/extra.css @@ -20,3 +20,13 @@ h2.doc-heading-parameter { display: block; } +.md-nav__item--section>.md-nav__link[for], +.md-nav--lifted>.md-nav__list>.md-nav__item>[for], +.md-nav__title { + font-size: 14px; + color: #840b2d !important; +} + +.md-nav__link--active { + color: #f15581 !important; +} diff --git a/docs/packaged_plugins.md b/docs/packaged_plugins.md index 569a603..21ac005 100644 --- a/docs/packaged_plugins.md +++ b/docs/packaged_plugins.md @@ -20,10 +20,14 @@ plugin_alias = "module.submodule.file:ClassName" auth: plugins: - auth_anonymous - allow-anonymous: true # or false + allow-anonymous: true # if false, providing a username will allow access ``` +!!! danger + even if `allow-anonymous` is set to `false`, the plugin will still allow access if a username is provided by the client + + ## auth_file (Auth Plugin) `amqtt.plugins.authentication:FileAuthPlugin` @@ -58,6 +62,9 @@ print(sha512_crypt.hash(passwd)) ## Taboo (Topic Plugin) +`amqtt.plugins.topic_checking:TopicTabooPlugin` + + Prevents using topics named: `prohibited`, `top-secret`, and `data/classified` **Configuration** @@ -71,6 +78,8 @@ topic-check: ## ACL (Topic Plugin) +`amqtt.plugins.topic_checking:TopicAccessControlListPlugin` + **Configuration** ```yaml @@ -78,9 +87,11 @@ topic-check: enabled: true plugins: - topic_acl - publish-acl: True # or False + publish-acl: + - username: ["list", "of", "allowed", "topics", "for", "publishing"] + - . acl: - - username: ["list", "of", "allowed", "topics"] + - username: ["list", "of", "allowed", "topics", "for", "subscribing"] - . ``` diff --git a/docs/references/amqtt.md b/docs/references/amqtt.md index b69383a..72837a3 100644 --- a/docs/references/amqtt.md +++ b/docs/references/amqtt.md @@ -23,154 +23,7 @@ amqtt [-c ] [-d] Without the `-c` argument, the broker will run with the following, default configuration: ```yaml -listeners: - default: - type: tcp - bind: 0.0.0.0:1883 -sys_interval: 20 -auth: - allow-anonymous: true -plugins: - - auth_file - - auth_anonymous +--8<-- "../amqtt/amqtt/scripts/default_broker.yaml" ``` -Using the `-c` argument allows for configuration with a YAML structured file. The following sections contain the available configuration elements: - -## Field Descriptions - -### `listeners` - -Defines network listeners for the MQTT server (list). - -#### `` - -`default` for parameters used across all interfaces _or_ name for the specific interface (mapping). - -Each entry supports these parameters: - -- `bind` (string, _required_) - Address and port to bind to, in the form `host:port` (e.g., `0.0.0.0:1883`). - -- `type` (string, _required_) - Protocol type. Typically `"tcp"` or `"ws"`. - -- `max-connections` (integer, _required_) - Maximum number of clients that can connect to this interface - -- `ssl` (string, _optional, default: `off`_) - Disable (`off`) SSL/TLS or enable (`on`) with one of `cafile`, `capath`, `cadata` or `certfile`/`keyfile`. - -- `cafile` (string, _optional_) - Path to a file of concatenated CA certificates in PEM format. See [Certificates](https://docs.python.org/3/library/ssl.html#ssl-certificates) for more info. - -- `capath` (string, _optional_) - Path to a directory containing several CA certificates in PEM format, following an [OpenSSL specific layout](https://docs.openssl.org/master/man3/SSL_CTX_load_verify_locations/). - -- `cadata` (string, _optional_) - Either an ASCII string of one or more PEM-encoded certificates or a bytes-like object of DER-encoded certificates - -- `certfile` (string, _optional_) - Path to a single file in PEM format containing the certificate as well as any number of CA certificates needed to establish the certificate's authenticity - -- `keyfile` (string, _optional_) - A file containing the private key. Otherwise the private key will be taken from certfile as well - -### timeout-disconnect-delay - -Client disconnect timeout without a keep-alive (integer, _optional_) - -### plugins - -Entry points for optional functionality (_list of strings_); included plugins are: - -- `auth_file` – Enables file-based authentication -- `auth_anonymous` – Enables anonymous access -- `event_logger_plugin` -- `packet_logger_plugin` -- `topic_taboo` -- `topic_acl` -- `broker_sys` - -### auth - -Authentication and authorization settings (mapping). - -- `allow-anonymous` (boolean, _optional for `auth_anonymous` plugin_) - Allow (`true`) or prevent (`false`) anonymous client to connections. - -- `password-file` (string, _required for `auth_file` plugin_) - Path to file which includes `username:password` pair, one per line. The password should be encoded using sha-512 with `mkpasswd -m sha-512` or: - -```python -import sys -from getpass import getpass -from passlib.hash import sha512_crypt - -passwd = input() if not sys.stdin.isatty() else getpass() -print(sha512_crypt.hash(passwd)) -``` - -### sys-interval - -Interval in seconds to publish system statistics to `$SYS` topics (integer, _optional for `broker_sys` plugin, defaults to TBD_). - -## Configuration example - -```yaml -listeners: - default: - max-connections: 500 - type: tcp - my-tcp-1: - bind: 127.0.0.1:1883 - my-tcp-2: - bind: 1.2.3.4:1883 - max-connections: 1000 - my-tcp-tls-1: - bind: 127.0.0.1:8883 - ssl: on - cafile: /some/cafile - my-ws-1: - bind: 0.0.0.0:9001 - type: ws - my-wss-1: - bind: 0.0.0.0:9003 - type: ws - ssl: on - certfile: /some/certfile - keyfile: /some/key -plugins: - - auth_file - - broker_sys - -timeout-disconnect-delay: 2 -auth: - password-file: /some/passwd_file -``` - -The `listeners` section defines 5 bindings: - -- `my-tcp-1`: an unsecured TCP listener on port 1883 allowing `500` clients connections simultaneously -- `my-tcp-2`: an unsecured TCP listener on port 1884 allowing `1000` client connections -- `my-tcp-ssl-1`: a secured TCP listener on port 8883 allowing `500` clients connections simultaneously -- `my-ws-1`: an unsecured websocket listener on port 9001 allowing `500` clients connections simultaneously -- `my-wss-1`: a secured websocket listener on port 9003 allowing `500` - -The plugins section enables: - -- `auth_file` plugin, requiring `password-file` to be defined in the `auth` section -- `broker_sys` plugin, requiring `sys_interval` to be defined - -Authentication allows anonymous logins and password file based authentication. Password files are required to be text files containing user name and password in the form of: - -``` -username:password -``` - -where `password` should be the encrypted password. Use the `mkpasswd -m sha-512` command to build encoded passphrase. Password file example: - -``` -# Test user with 'test' password encrypted with sha-512 -test:$6$l4zQEHEcowc1Pnv4$HHrh8xnsZoLItQ8BmpFHM4r6q5UqK3DnXp2GaTm5zp5buQ7NheY3Xt9f6godVKbEtA.hOC7IEDwnok3pbAOip. -``` +Using the `-c` argument allows for configuration with a YAML structured file; see [broker configuration](broker_config.md). diff --git a/docs/references/amqtt_pub.md b/docs/references/amqtt_pub.md index a951d3c..1b84a20 100644 --- a/docs/references/amqtt_pub.md +++ b/docs/references/amqtt_pub.md @@ -46,15 +46,14 @@ Note that for simplicity, `amqtt_pub` uses mostly the same argument syntax as [m ## Configuration -If `-c` argument is given, `amqtt_pub` will read specific MQTT settings for the given configuration file. This file must be a valid [YAML](http://yaml.org/) file which may contain the following configuration elements: +Without the `-c` argument, the broker will run with the following, default configuration: + +```yaml +--8<-- "../amqtt/amqtt/scripts/default_client.yaml" +``` + +Using the `-c` argument allows for configuration with a YAML structured file; see [client configuration](client_config.md). -- `keep_alive`: Keep-alive timeout sent to the broker. Defaults to `10` seconds. -- `ping_delay`: Auto-ping delay before keep-alive timeout. Defaults to 1. Setting to `0` will disable to 0 and may lead to broker disconnection. -- `default_qos`: Default QoS for messages published. Defaults to 0. -- `default_retain`: Default retain value to messages published. Defaults to `false`. -- `auto_reconnect`: Enable or disable auto-reconnect if connection with the broker is interrupted. Defaults to `false`. -- `reconnect_retries`: Maximum reconnection retries. Defaults to `2`. Negative value will cause client to reconnect infinitely. -- `reconnect_max_interval`: Maximum interval between 2 connection retry. Defaults to `10`. ## Examples diff --git a/docs/references/broker.md b/docs/references/broker.md index 9ef1d30..f49a69f 100644 --- a/docs/references/broker.md +++ b/docs/references/broker.md @@ -7,27 +7,10 @@ The `amqtt.broker.Broker` class provides a complete MQTT 3.1.1 broker implementa The following example shows how to start a broker using the default configuration: ```python -import logging -import asyncio -import os -from amqtt.broker import Broker - - -async def broker_coro(): - broker = Broker() - await broker.start() - - -if __name__ == '__main__': - formatter = "[%(asctime)s] :: %(levelname)s :: %(name)s :: %(message)s" - logging.basicConfig(level=logging.INFO, format=formatter) - asyncio.get_event_loop().run_until_complete(broker_coro()) - asyncio.get_event_loop().run_forever() +--8<-- "../amqtt/samples/broker_simple.py" ``` -When executed, this script gets the default event loop and asks it to run the `broker_coro` until it completes. -`broker_coro` creates `amqtt.broker.Broker` instance and then starts the broker for serving using the `start()` method. -Once completed, the loop is ran forever, making this script never stop... +This will start the broker and let it run until it is shutdown by `^c`. ## Reference @@ -40,66 +23,11 @@ The `amqtt.broker` module provides the following key methods in the `Broker` cla ### Broker configuration -The `Broker` class's `__init__` method accepts a `config` parameter which allows setup of behavior and default settings. This argument must be a Python dict object. For convenience, it is presented below as a YAML file[^1]: +The `Broker` class's `__init__` method accepts a `config` parameter which allows setup of default and custom behaviors. -```yaml -listeners: - default: - max-connections: 50000 - type: tcp - my-tcp-1: - bind: 127.0.0.1:1883 - my-tcp-2: - bind: 1.2.3.4:1884 - max-connections: 1000 - my-tcp-ssl-1: - bind: 127.0.0.1:8885 - ssl: on - cafile: /some/cafile - capath: /some/folder - capath: certificate data - certfile: /some/certfile - keyfile: /some/key - my-ws-1: - bind: 0.0.0.0:8080 - type: ws -timeout-disconnect-delay: 2 -auth: - plugins: ['auth.anonymous'] #List of plugins to activate for authentication among all registered plugins - allow-anonymous: true / false - password-file: /some/passwd_file -topic-check: - enabled: true / false # Set to False if topic filtering is not needed - plugins: ['topic_acl'] #List of plugins to activate for topic filtering among all registered plugins - acl: - # username: [list of allowed topics] - username1: ['repositories/+/master', 'calendar/#', 'data/memes'] # List of topics on which client1 can publish and subscribe - username2: ... - anonymous: [] # List of topics on which an anonymous client can publish and subscribe -``` - -The `listeners` section allows defining network listeners which must be started by the `Broker`. Several listeners can be setup. `default` subsection defines common attributes for all listeners. Each listener can have the following settings: - -- `bind`: IP address and port binding. -- `max-connections`: Set maximum number of active connection for the listener. `0` means no limit. -- `type`: transport protocol type; can be `tcp` for classic TCP listener or `ws` for MQTT over websocket. -- `ssl`: enables (`on`) or disable secured connection over the transport protocol. -- `cafile`, `cadata`, `certfile` and `keyfile`: mandatory parameters for SSL secured connections. - -The `auth` section setup authentication behaviour: - -- `plugins`: defines the list of activated plugins. Note the plugins must be defined in the `amqtt.broker.plugins` [entry point](https://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins). -- `allow-anonymous`: used by the internal `amqtt.plugins.authentication.AnonymousAuthPlugin` plugin. This parameter enables (`on`) or disable anonymous connection, i.e. connection without username. -- `password-file`: used by the internal `amqtt.plugins.authentication.FileAuthPlugin` plugin. This parameter gives to path of the password file to load for authenticating users. - -The `topic-check` section setup access control policies for publishing and subscribing to topics: - -- `enabled`: set to true if you want to impose an access control policy. Otherwise, set it to false. -- `plugins`: defines the list of activated plugins. Note the plugins must be defined in the `amqtt.broker.plugins` [entry point](https://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins). -- additional parameters: depending on the plugin used for access control, additional parameters should be added. - - In case of `topic_acl` plugin, the Access Control List (ACL) must be defined in the parameter `acl`. - - For each username, a list with the allowed topics must be defined. - - If the client logs in anonymously, the `anonymous` entry within the ACL is used in order to grant/deny subscriptions. +Details on the `config` parameter structure is a dictionary whose structure is identical to yaml formatted file[^1] +used by the included broker script: [broker configuration](broker_config.md) + ::: amqtt.broker.Broker diff --git a/docs/references/client.md b/docs/references/client.md index 2bfc5d4..923e899 100644 --- a/docs/references/client.md +++ b/docs/references/client.md @@ -129,4 +129,21 @@ amqtt/LYRf52W[56SOjW04 <-in-- PubcompPacket(ts=2015-11-11 21:54:48.713107, fixed Both coroutines have the same results except that `test_coro2()` manages messages flow in parallel which may be more efficient. + +### Client configuration + +The `MQTTClient` class's `__init__` method accepts a `config` parameter which allows setup of default and custom behaviors. + +Details on the `config` parameter structure is a dictionary whose structure is identical to yaml formatted file[^1] +used by the included broker script: [client configuration](client_config.md) + + + +::: amqtt.broker.Broker + +[^1]: See [PyYAML](http://pyyaml.org/wiki/PyYAMLDocumentation) for loading YAML files as Python dict. + + + + ::: amqtt.client.MQTTClient diff --git a/mkdocs.rtd.yml b/mkdocs.rtd.yml index b69afb8..979f635 100644 --- a/mkdocs.rtd.yml +++ b/mkdocs.rtd.yml @@ -6,7 +6,17 @@ site_url: http://github.com repo_url: https://github.com/Yakifo/amqtt repo_name: Yakifo/amqtt site_dir: "site" -watch: [mkdocs.rtd.yml, README.md, CONTRIBUTING.md, SUPPORT.md, SECURITY.md, CODE_OF_CONDUCT.md, CONTRIBUTING.md, docs, amqtt] +watch: + - mkdocs.rtd.yml + - README.md + - CONTRIBUTING.md + - SUPPORT.md + - SECURITY.md + - CODE_OF_CONDUCT.md + - CONTRIBUTING.md + - docs + - amqtt + - samples copyright: TBD edit_uri: edit/main/docs/ @@ -30,6 +40,9 @@ nav: - Plugins: - Packaged: packaged_plugins.md - Custom: custom_plugins.md + - Configuration: + - Broker: references/broker_config.md + - Client: references/client_config.md - Reference: - Support: support.md - Contributing: contributing.md diff --git a/samples/broker_start.py b/samples/broker_start.py index f68df53..1055ff1 100644 --- a/samples/broker_start.py +++ b/samples/broker_start.py @@ -45,3 +45,4 @@ if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format=formatter) asyncio.get_event_loop().run_until_complete(test_coro()) asyncio.get_event_loop().run_forever() +