subproperties in config

src/configs/v2config.py

@@ -3,6 +3,7 @@
 import argparse, yaml
 from dataclasses import dataclass, field
 from typing import List
+from feeders.feeder import Feeder
 from step import Step
 from utils import Util
 from enrichers import Enricher
@@ -13,15 +14,16 @@ from collections import defaultdict
 class ConfigV2:
     # TODO: should Config inherit from Step so it can have it's own configurations?
     configurable_parents = [
+        Feeder,
         Enricher,
-        Util
+        # Util
     ]
-    feeder : Step #TODO:= BaseFeeder
+    feeder: Step  # TODO:= BaseFeeder
-    archivers: List[Step] = field(default_factory=[]) #TODO: fix type
+    archivers: List[Step] = field(default_factory=[])  # TODO: fix type
     enrichers: List[Enricher] = field(default_factory=[])
-    formatters: List[Step] = field(default_factory=[]) #TODO: fix type
+    formatters: List[Step] = field(default_factory=[])  # TODO: fix type
-    storages: List[Step] = field(default_factory=[]) #TODO: fix type
+    storages: List[Step] = field(default_factory=[])  # TODO: fix type
-    databases: List[Step] = field(default_factory=[]) #TODO: fix type
+    databases: List[Step] = field(default_factory=[])  # TODO: fix type
 
     def __init__(self) -> None:
         self.defaults = {}
@@ -39,13 +41,27 @@ class ConfigV2:
 
         for configurable in self.configurable_parents:
             child: Step
+            # print(f"{configurable=}")
             for child in configurable.__subclasses__():
+                # print(f"{child=} {child.configs()=}")
+
                 for config, details in child.configs().items():
+                    print(config, details)
                     assert "." not in child.name, f"class prop name cannot contain dots('.'): {child.name}"
                     assert "." not in config, f"config property cannot contain dots('.'): {config}"
+                    if (is_nested := type(details["default"]) == dict):
+                        for subconfig, subdefault in details["default"].items():
+                            assert "." not in subconfig, f"config subproperty cannot contain dots('.'): {subconfig}"
+                            config_path = f"{child.name}.{config}.{subconfig}"
+                            parser.add_argument(f'--{config_path}', action='store', dest=config_path, help=details['help'] + f"({subconfig})")
+                            self.defaults[config_path] = subdefault
+
                     config_path = f"{child.name}.{config}"
-                    parser.add_argument(f'--{config_path}', action='store', dest=config_path, help=details['help'])
+                    print(config_path)
                     self.defaults[config_path] = details["default"]
+                    if not is_nested:
+                        # nested cannot be directly set on the CLI
+                        parser.add_argument(f'--{config_path}', action='store', dest=config_path, help=details['help'])
 
         args = parser.parse_args()
 
@@ -53,28 +69,48 @@ class ConfigV2:
         with open(args.config, "r", encoding="utf-8") as inf:
             self.yaml_config = yaml.safe_load(inf)
 
+        # print(f"{self.yaml_config.get('configurations', {})=}")
         # 3. CONFIGS: decide value with priority: CLI >> config.yaml >> default
         self.config = defaultdict(dict)
         for config_path, default in self.defaults.items():
-            child, config = tuple(config_path.split("."))
+            config_steps = config_path.split(".")
-            val = getattr(args, config_path)
+            if len(config_steps) == 2:  # not nested
-            if val is None:
+                child, config = tuple(config_steps)
-                val = self.yaml_config.get("configurations", {}).get(child, {}).get(config, default)
+                val = getattr(args, config_path, None)
-            self.config[child][config] = val
+                if val is None:
+                    val = self.yaml_config.get("configurations", {}).get(child, {}).get(config, default)
+                # self.config[child][config] = val
+
+            elif len(config_steps) == 3:  # nested
+                child, config, subconfig = tuple(config_steps)
+                val = getattr(args, config_path)
+                if config not in self.config[child]:
+                    self.config[child][config] = {}
+                if val is None:
+                    val = self.yaml_config.get("configurations", {}).get(child, {}).get(config, {}).get(subconfig, default)
+                print(child, config, subconfig, val)
+                self.config[child][config][subconfig] = val
+
+            # child, config = tuple(config_path.split("."))
+            # # print(config_path)
+            # val = getattr(args, config_path)
+            # # print(child, config, val)
+            # if val is None:
+            #     val = self.yaml_config.get("configurations", {}).get(child, {}).get(config, default)
+            # self.config[child][config] = val
         self.config = dict(self.config)
 
         # 4. STEPS: read steps and validate they exist
         steps = self.yaml_config.get("steps", {})
         assert "archivers" in steps, "your configuration steps are missing the archivers property"
         assert "storages" in steps, "your configuration steps are missing the storages property"
-        
+
-        print(self.config)
+        print("config.py", self.config)
-        
+
-        # self.feeder = Feeder.init
+        self.feeder = Feeder.init(steps.get("feeder", "cli_feeder"), self.config)
-        self.enrichers = [Enricher.init(steps.get("enrichers", [])[0], self.config)]
+        self.enrichers = [Enricher.init(e, self.config) for e in steps.get("enrichers", [])]
-        
+
-        
+        print("enrichers", [e for e in self.enrichers])
-        print(self.enrichers)
 
     def validate(self):
         pass