consolidating configuration documentation for both broker and client. including making sure there's consistency in default configuration

SUPPORT.md

@@ -4,7 +4,7 @@ This article explains where to get help with this aMQTT project.
 Please read through the following guidelines.
 
 > 👉 **Note**: before participating in our community, please read our
-> [code of conduct][coc].
+> [code of conduct](code_of_conduct.md).
 > By interacting with this repository, organization, or community you agree to
 > abide by its terms.
 
@@ -32,4 +32,4 @@ Here are some tips:
 
 ## Contributions
 
-See [`contributing.md`][contributing] on how to contribute.
+See [contributing](contributing.md) on how to contribute.

amqtt/broker.py

@@ -1,4 +1,5 @@
 import asyncio
+import copy
 from asyncio import CancelledError, futures
 from collections import deque
 from collections.abc import Generator
@@ -7,6 +8,7 @@ from functools import partial
 import logging
 import re
 import ssl
+from pathlib import Path
 from typing import Any, ClassVar
 
 from transitions import Machine, MachineError
@@ -24,17 +26,16 @@ from amqtt.adapters import (
 from amqtt.errors import AMQTTError, BrokerError, MQTTError, NoDataError
 from amqtt.mqtt.protocol.broker_handler import BrokerProtocolHandler
 from amqtt.session import ApplicationMessage, OutgoingApplicationMessage, Session
-from amqtt.utils import format_client_message, gen_client_id
+from amqtt.utils import format_client_message, gen_client_id, read_yaml_config
 
 from .plugins.manager import BaseContext, PluginManager
 
-type CONFIG_LISTENER = dict[str, int | bool | dict[str, Any]]
+type CONFIG_LISTENER = dict[str, Any]
 type _BROADCAST = dict[str, Session | str | bytes | int | None]
 
-_defaults: CONFIG_LISTENER = {
+
-    "timeout-disconnect-delay": 2,
+_defaults = read_yaml_config(Path(__file__).parent / "scripts/default_broker.yaml")
-    "auth": {"allow-anonymous": True, "password-file": None},
+
-}
 
 # Default port numbers
 DEFAULT_PORTS = {"tcp": 1883, "ws": 8883}
@@ -140,7 +141,7 @@ class Broker:
     """MQTT 3.1.1 compliant broker implementation.
 
     Args:
-        config: dictionary of configuration options (see config yaml format)
+        config: dictionary of configuration options (see [broker configuration](broker_config.md)).
         loop: asyncio loop. defaults to `asyncio.get_event_loop()`.
         plugin_namespace: plugin namespace to use when loading plugin entry_points. defaults to `amqtt.broker.plugins`.
 
@@ -164,7 +165,7 @@ class Broker:
     ) -> None:
         """Initialize the broker."""
         self.logger = logging.getLogger(__name__)
-        self.config = _defaults.copy()
+        self.config = copy.deepcopy(_defaults or {})
         if config is not None:
             self.config.update(config)
         self._build_listeners_config(self.config)

amqtt/client.py

@@ -6,6 +6,7 @@ import copy
 from functools import wraps
 import logging
 import ssl
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, cast
 from urllib.parse import urlparse, urlunparse
 
@@ -24,21 +25,12 @@ from amqtt.mqtt.constants import QOS_0, QOS_1, QOS_2
 from amqtt.mqtt.protocol.client_handler import ClientProtocolHandler
 from amqtt.plugins.manager import BaseContext, PluginManager
 from amqtt.session import ApplicationMessage, OutgoingApplicationMessage, Session
-from amqtt.utils import gen_client_id
+from amqtt.utils import gen_client_id, read_yaml_config
 
 if TYPE_CHECKING:
     from websockets.asyncio.client import ClientConnection
 
-_defaults: dict[str, Any] = {
+_defaults: dict[str, Any] | None = read_yaml_config(Path(__file__).parent / "scripts/default_broker.yaml")
-    "keep_alive": 10,
-    "ping_delay": 1,
-    "default_qos": 0,
-    "default_retain": False,
-    "auto_reconnect": True,
-    "reconnect_max_interval": 10,
-    "reconnect_retries": 2,
-}
-
 
 class ClientContext(BaseContext):
     """ClientContext is used as the context passed to plugins interacting with the client.
@@ -91,45 +83,13 @@ class MQTTClient:
 
     Args:
         client_id: MQTT client ID to use when connecting to the broker. If none, it will be generated randomly by `amqtt.utils.gen_client_id`
-        config: Client configuration with the following keys:
+        config: dictionary of configuration options (see [client configuration](client_config.md)).
-
-            `keep_alive`: keep alive (in seconds) to send when connecting to the broker (defaults to `10` seconds). `MQTTClient` will _auto-ping_ the broker if no message is sent within the keep-alive interval. This avoids disconnection from the broker.
-
-            `ping_delay`: _auto-ping_ delay before keep-alive times out (defaults to `1` seconds).
-
-            `default_qos`: Default QoS (`0`) used by `publish()` if `qos` argument is not given.
-
-            `default_retain`: Default retain (`False`) used by `publish()` if `qos` argument is not given.
-
-            `auto_reconnect`: enable or disable auto-reconnect feature (defaults to `True`).
-
-            `reconnect_max_interval`: maximum interval (in seconds) to wait before two connection retries (defaults to `10`).
-
-            `reconnect_retries`: maximum number of connect retries (defaults to `2`). Negative value will cause client to reconnect infinitely.
-
-            Example:
-
-            ```python
-            config = {
-                'keep_alive': 10,
-                'ping_delay': 1,
-                'default_qos': 0,
-                'default_retain': False,
-                'auto_reconnect': True,
-                'reconnect_max_interval': 5,
-                'reconnect_retries': 10,
-                'topics': {
-                    'test': { 'qos': 1 },
-                    'some_topic': { 'qos': 2, 'retain': True }
-                }
-            }
-            ```
 
     """
 
     def __init__(self, client_id: str | None = None, config: dict[str, Any] | None = None) -> None:
         self.logger = logging.getLogger(__name__)
-        self.config = copy.deepcopy(_defaults)
+        self.config = copy.deepcopy(_defaults or {})
         if config is not None:
             self.config.update(config)
         self.client_id = client_id if client_id is not None else gen_client_id()

amqtt/scripts/broker_script.py

@@ -22,22 +22,6 @@ import amqtt
 from amqtt.broker import Broker
 from amqtt.utils import read_yaml_config
 
-default_config = {
-    "listeners": {
-        "default": {
-            "type": "tcp",
-            "bind": "0.0.0.0:1883",
-        },
-    },
-    "sys_interval": 10,
-    "auth": {
-        "allow-anonymous": True,
-        "password-file": Path(__file__).parent / "passwd",
-        "plugins": ["auth_file", "auth_anonymous"],
-    },
-    "topic-check": {"enabled": False},
-}
-
 logger = logging.getLogger(__name__)
 
 

amqtt/scripts/default_broker.yaml

@@ -5,9 +5,8 @@ listeners:
     bind: 0.0.0.0:1883
 sys_interval: 20
 auth:
+  plugins:
+    - auth_anonymous
   allow-anonymous: true
-plugins:
-  - auth_file
-  - auth_anonymous
 topic-check:
   enabled: False

amqtt/scripts/default_client.yaml

@@ -3,6 +3,6 @@ keep_alive: 10
 ping_delay: 1
 default_qos: 0
 default_retain: false
-auto_reconnect: false
+auto_reconnect: true
 reconnect_max_interval: 10
 reconnect_retries: 2

docs/assets/extra.css

@@ -20,3 +20,13 @@ h2.doc-heading-parameter {
   display: block;
 }
 
+.md-nav__item--section>.md-nav__link[for],
+.md-nav--lifted>.md-nav__list>.md-nav__item>[for],
+.md-nav__title {
+  font-size: 14px;
+  color: #840b2d !important;
+}
+
+.md-nav__link--active {
+  color: #f15581 !important;
+}

docs/packaged_plugins.md

@@ -20,10 +20,14 @@ plugin_alias = "module.submodule.file:ClassName"
 auth:
   plugins:
     - auth_anonymous
-  allow-anonymous: true # or false
+  allow-anonymous: true # if false, providing a username will allow access
 
 ```
 
+!!! danger
+    even if `allow-anonymous` is set to `false`, the plugin will still allow access if a username is provided by the client
+
+
 ## auth_file (Auth Plugin)
 
 `amqtt.plugins.authentication:FileAuthPlugin`
@@ -58,6 +62,9 @@ print(sha512_crypt.hash(passwd))
 
 ## Taboo (Topic Plugin)
 
+`amqtt.plugins.topic_checking:TopicTabooPlugin`
+
+
 Prevents using topics named: `prohibited`, `top-secret`, and `data/classified`
 
 **Configuration**
@@ -71,6 +78,8 @@ topic-check:
 
 ## ACL (Topic Plugin)
 
+`amqtt.plugins.topic_checking:TopicAccessControlListPlugin`
+
 **Configuration**
 
 ```yaml
@@ -78,9 +87,11 @@ topic-check:
   enabled: true
   plugins:
     - topic_acl
-  publish-acl: True # or False
+  publish-acl:
+    - username: ["list", "of", "allowed", "topics", "for", "publishing"]
+    - .
   acl:
-    - username: ["list", "of", "allowed", "topics"]
+    - username: ["list", "of", "allowed", "topics", "for", "subscribing"]
     - .
 ```
 

docs/references/amqtt.md

@@ -23,154 +23,7 @@ amqtt [-c <config_file> ] [-d]
 Without the `-c` argument, the broker will run with the following, default configuration:
 
 ```yaml
-listeners:
+--8<-- "../amqtt/amqtt/scripts/default_broker.yaml"
-  default:
-    type: tcp
-    bind: 0.0.0.0:1883
-sys_interval: 20
-auth:
-  allow-anonymous: true
-plugins:
-  - auth_file
-  - auth_anonymous
 ```
 
-Using the `-c` argument allows for configuration with a YAML structured file. The following sections contain the available configuration elements:
+Using the `-c` argument allows for configuration with a YAML structured file; see [broker configuration](broker_config.md). 
-
-## Field Descriptions
-
-### `listeners`
-
-Defines network listeners for the MQTT server (list).
-
-#### `<interface name>`
-
-`default` for parameters used across all interfaces _or_ name for the specific interface (mapping).
-
-Each entry supports these parameters:
-
-- `bind` (string, _required_)  
-  Address and port to bind to, in the form `host:port` (e.g., `0.0.0.0:1883`).
-
-- `type` (string, _required_)  
-  Protocol type. Typically `"tcp"` or `"ws"`.
-
-- `max-connections` (integer, _required_)  
-  Maximum number of clients that can connect to this interface
-
-- `ssl` (string, _optional, default: `off`_)  
-  Disable (`off`) SSL/TLS or enable (`on`) with one of `cafile`, `capath`, `cadata` or `certfile`/`keyfile`.
-
-- `cafile` (string, _optional_)  
-  Path to a file of concatenated CA certificates in PEM format. See [Certificates](https://docs.python.org/3/library/ssl.html#ssl-certificates) for more info.
-
-- `capath` (string, _optional_)  
-  Path to a directory containing several CA certificates in PEM format, following an [OpenSSL specific layout](https://docs.openssl.org/master/man3/SSL_CTX_load_verify_locations/).
-
-- `cadata` (string, _optional_)  
-  Either an ASCII string of one or more PEM-encoded certificates or a bytes-like object of DER-encoded certificates
-
-- `certfile` (string, _optional_)  
-  Path to a single file in PEM format containing the certificate as well as any number of CA certificates needed to establish the certificate's authenticity
-
-- `keyfile` (string, _optional_)  
-  A file containing the private key. Otherwise the private key will be taken from certfile as well
-
-### timeout-disconnect-delay
-
-Client disconnect timeout without a keep-alive (integer, _optional_)
-
-### plugins
-
-Entry points for optional functionality (_list of strings_); included plugins are:
-
-- `auth_file` – Enables file-based authentication
-- `auth_anonymous` – Enables anonymous access
-- `event_logger_plugin`
-- `packet_logger_plugin`
-- `topic_taboo`
-- `topic_acl`
-- `broker_sys`
-
-### auth
-
-Authentication and authorization settings (mapping).
-
-- `allow-anonymous` (boolean, _optional for `auth_anonymous` plugin_)  
-  Allow (`true`) or prevent (`false`) anonymous client to connections.
-
-- `password-file` (string, _required for `auth_file` plugin_)  
-  Path to file which includes `username:password` pair, one per line. The password should be encoded using sha-512 with `mkpasswd -m sha-512` or:
-
-```python
-import sys
-from getpass import getpass
-from passlib.hash import sha512_crypt
-
-passwd = input() if not sys.stdin.isatty() else getpass()
-print(sha512_crypt.hash(passwd))
-```
-
-### sys-interval
-
-Interval in seconds to publish system statistics to `$SYS` topics (integer, _optional for `broker_sys` plugin, defaults to TBD_).
-
-## Configuration example
-
-```yaml
-listeners:
-  default:
-    max-connections: 500
-    type: tcp
-  my-tcp-1:
-    bind: 127.0.0.1:1883
-  my-tcp-2:
-    bind: 1.2.3.4:1883
-    max-connections: 1000
-  my-tcp-tls-1:
-    bind: 127.0.0.1:8883
-    ssl: on
-    cafile: /some/cafile
-  my-ws-1:
-    bind: 0.0.0.0:9001
-    type: ws
-  my-wss-1:
-    bind: 0.0.0.0:9003
-    type: ws
-    ssl: on
-    certfile: /some/certfile
-    keyfile: /some/key
-plugins:
-  - auth_file
-  - broker_sys
-
-timeout-disconnect-delay: 2
-auth:
-  password-file: /some/passwd_file
-```
-
-The `listeners` section defines 5 bindings:
-
-- `my-tcp-1`: an unsecured TCP listener on port 1883 allowing `500` clients connections simultaneously
-- `my-tcp-2`: an unsecured TCP listener on port 1884 allowing `1000` client connections
-- `my-tcp-ssl-1`: a secured TCP listener on port 8883 allowing `500` clients connections simultaneously
-- `my-ws-1`: an unsecured websocket listener on port 9001 allowing `500` clients connections simultaneously
-- `my-wss-1`: a secured websocket listener on port 9003 allowing `500`
-
-The plugins section enables:
-
-- `auth_file` plugin, requiring `password-file` to be defined in the `auth` section
-- `broker_sys` plugin, requiring `sys_interval` to be defined
-
-Authentication allows anonymous logins and password file based authentication. Password files are required to be text files containing user name and password in the form of:
-
-```
-username:password
-```
-
-where `password` should be the encrypted password. Use the `mkpasswd -m sha-512` command to build encoded passphrase. Password file example:
-
-```
-# Test user with 'test' password encrypted with sha-512
-test:$6$l4zQEHEcowc1Pnv4$HHrh8xnsZoLItQ8BmpFHM4r6q5UqK3DnXp2GaTm5zp5buQ7NheY3Xt9f6godVKbEtA.hOC7IEDwnok3pbAOip.
-```

docs/references/amqtt_pub.md

@@ -46,15 +46,14 @@ Note that for simplicity, `amqtt_pub` uses mostly the same argument syntax as [m
 
 ## Configuration
 
-If `-c` argument is given, `amqtt_pub` will read specific MQTT settings for the given configuration file. This file must be a valid [YAML](http://yaml.org/) file which may contain the following configuration elements:
+Without the `-c` argument, the broker will run with the following, default configuration:
+
+```yaml
+--8<-- "../amqtt/amqtt/scripts/default_client.yaml"
+```
+
+Using the `-c` argument allows for configuration with a YAML structured file; see [client configuration](client_config.md). 
 
-- `keep_alive`: Keep-alive timeout sent to the broker. Defaults to `10` seconds.
-- `ping_delay`: Auto-ping delay before keep-alive timeout. Defaults to 1. Setting to `0` will disable to 0 and may lead to broker disconnection.
-- `default_qos`: Default QoS for messages published. Defaults to 0.
-- `default_retain`: Default retain value to messages published. Defaults to `false`.
-- `auto_reconnect`: Enable or disable auto-reconnect if connection with the broker is interrupted. Defaults to `false`.
-- `reconnect_retries`: Maximum reconnection retries. Defaults to `2`. Negative value will cause client to reconnect infinitely.
-- `reconnect_max_interval`: Maximum interval between 2 connection retry. Defaults to `10`.
 
 ## Examples
 

docs/references/broker.md

@@ -7,27 +7,10 @@ The `amqtt.broker.Broker` class provides a complete MQTT 3.1.1 broker implementa
 The following example shows how to start a broker using the default configuration:
 
 ```python
-import logging
+--8<-- "../amqtt/samples/broker_simple.py"
-import asyncio
-import os
-from amqtt.broker import Broker
-
-
-async def broker_coro():
-    broker = Broker()
-    await broker.start()
-
-
-if __name__ == '__main__':
-    formatter = "[%(asctime)s] :: %(levelname)s :: %(name)s :: %(message)s"
-    logging.basicConfig(level=logging.INFO, format=formatter)
-    asyncio.get_event_loop().run_until_complete(broker_coro())
-    asyncio.get_event_loop().run_forever()
 ```
 
-When executed, this script gets the default event loop and asks it to run the `broker_coro` until it completes.
+This will start the broker and let it run until it is shutdown by `^c`.
-`broker_coro` creates `amqtt.broker.Broker` instance and then starts the broker for serving using the `start()` method.
-Once completed, the loop is ran forever, making this script never stop...
 
 ## Reference
 
@@ -40,66 +23,11 @@ The `amqtt.broker` module provides the following key methods in the `Broker` cla
 
 ### Broker configuration
 
-The `Broker` class's `__init__` method accepts a `config` parameter which allows setup of behavior and default settings. This argument must be a Python dict object. For convenience, it is presented below as a YAML file[^1]:
+The `Broker` class's `__init__` method accepts a `config` parameter which allows setup of default and custom behaviors.
 
-```yaml
+Details on the `config` parameter structure is a dictionary whose structure is identical to yaml formatted file[^1]
-listeners:
+used by the included broker script: [broker configuration](broker_config.md)
-    default:
+  
-        max-connections: 50000
-        type: tcp
-    my-tcp-1:
-        bind: 127.0.0.1:1883
-    my-tcp-2:
-        bind: 1.2.3.4:1884
-        max-connections: 1000
-    my-tcp-ssl-1:
-        bind: 127.0.0.1:8885
-        ssl: on
-        cafile: /some/cafile
-        capath: /some/folder
-        capath: certificate data
-        certfile: /some/certfile
-        keyfile: /some/key
-    my-ws-1:
-        bind: 0.0.0.0:8080
-        type: ws
-timeout-disconnect-delay: 2
-auth:
-    plugins: ['auth.anonymous'] #List of plugins to activate for authentication among all registered plugins
-    allow-anonymous: true / false
-    password-file: /some/passwd_file
-topic-check:
-    enabled: true / false  # Set to False if topic filtering is not needed
-    plugins: ['topic_acl'] #List of plugins to activate for topic filtering among all registered plugins
-    acl:
-        # username: [list of allowed topics]
-        username1: ['repositories/+/master', 'calendar/#', 'data/memes']  # List of topics on which client1 can publish and subscribe
-        username2: ...
-        anonymous: []  # List of topics on which an anonymous client can publish and subscribe
-```
-
-The `listeners` section allows defining network listeners which must be started by the `Broker`. Several listeners can be setup. `default` subsection defines common attributes for all listeners. Each listener can have the following settings:
-
-- `bind`: IP address and port binding.
-- `max-connections`: Set maximum number of active connection for the listener. `0` means no limit.
-- `type`: transport protocol type; can be `tcp` for classic TCP listener or `ws` for MQTT over websocket.
-- `ssl`: enables (`on`) or disable secured connection over the transport protocol.
-- `cafile`, `cadata`, `certfile` and `keyfile`: mandatory parameters for SSL secured connections.
-
-The `auth` section setup authentication behaviour:
-
-- `plugins`: defines the list of activated plugins. Note the plugins must be defined in the `amqtt.broker.plugins` [entry point](https://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins).
-- `allow-anonymous`: used by the internal `amqtt.plugins.authentication.AnonymousAuthPlugin` plugin. This parameter enables (`on`) or disable anonymous connection, i.e. connection without username.
-- `password-file`: used by the internal `amqtt.plugins.authentication.FileAuthPlugin` plugin. This parameter gives to path of the password file to load for authenticating users.
-
-The `topic-check` section setup access control policies for publishing and subscribing to topics:
-
-- `enabled`: set to true if you want to impose an access control policy. Otherwise, set it to false.
-- `plugins`: defines the list of activated plugins. Note the plugins must be defined in the `amqtt.broker.plugins` [entry point](https://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins).
-- additional parameters: depending on the plugin used for access control, additional parameters should be added.
-  - In case of `topic_acl` plugin, the Access Control List (ACL) must be defined in the parameter `acl`.
-    - For each username, a list with the allowed topics must be defined.
-    - If the client logs in anonymously, the `anonymous` entry within the ACL is used in order to grant/deny subscriptions.
 
 
 ::: amqtt.broker.Broker

docs/references/client.md

@@ -129,4 +129,21 @@ amqtt/LYRf52W[56SOjW04 <-in-- PubcompPacket(ts=2015-11-11 21:54:48.713107, fixed
 
 Both coroutines have the same results except that `test_coro2()` manages messages flow in parallel which may be more efficient.
 
+
+### Client configuration
+
+The `MQTTClient` class's `__init__` method accepts a `config` parameter which allows setup of default and custom behaviors.
+
+Details on the `config` parameter structure is a dictionary whose structure is identical to yaml formatted file[^1]
+used by the included broker script: [client configuration](client_config.md)
+  
+
+
+::: amqtt.broker.Broker
+
+[^1]: See [PyYAML](http://pyyaml.org/wiki/PyYAMLDocumentation) for loading YAML files as Python dict.
+
+
+
+
 ::: amqtt.client.MQTTClient

mkdocs.rtd.yml

@@ -6,7 +6,17 @@ site_url: http://github.com
 repo_url: https://github.com/Yakifo/amqtt
 repo_name: Yakifo/amqtt
 site_dir: "site"
-watch: [mkdocs.rtd.yml, README.md, CONTRIBUTING.md, SUPPORT.md, SECURITY.md, CODE_OF_CONDUCT.md, CONTRIBUTING.md, docs, amqtt]
+watch:
+  - mkdocs.rtd.yml
+  - README.md
+  - CONTRIBUTING.md
+  - SUPPORT.md
+  - SECURITY.md
+  - CODE_OF_CONDUCT.md
+  - CONTRIBUTING.md
+  - docs
+  - amqtt
+  - samples
 copyright: TBD
 edit_uri: edit/main/docs/
 
@@ -30,6 +40,9 @@ nav:
       - Plugins:
           - Packaged: packaged_plugins.md
           - Custom: custom_plugins.md
+      - Configuration:
+          - Broker: references/broker_config.md
+          - Client: references/client_config.md
       - Reference:
           - Support: support.md
           - Contributing: contributing.md

samples/broker_start.py

@@ -45,3 +45,4 @@ if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO, format=formatter)
     asyncio.get_event_loop().run_until_complete(test_coro())
     asyncio.get_event_loop().run_forever()
+