config management update

AGENTS.md

@@ -194,5 +194,17 @@ After creating the issue, update the TODO line in the `AGENTS.md` file with the
 
 ### Work in Progress
 
-* TODO Add configuration support to allow users to override module and template spec with their own (e.g. defaults -> compose -> spec -> general ...)
+* FIXME We need proper validation to ensure all variable names are unique across all sections
+* FIXME Insufficient Error Messages for Template Loading
+* FIXME Excessive Generic Exception Catching
+* FIXME No Rollback on Config Write Failures: If writing config fails partway through, the config file can be left in a corrupted state. There's no atomic write operation.
+* FIXME Inconsistent Logging Levels: Some important operations use `DEBUG` when they should use `INFO`, and vice versa.
+* TODO Memory Inefficiency in Template File Collection: The template loads all file paths into memory immediately, even when only metadata is needed (like for `list` command). This is wasteful when listing many templates.
+* TODO Missing Input Validation in ConfigManager
 * TODO Add compose deploy command to deploy a generated compose project to a local or remote docker environment
+* TODO No Caching for Module Specs: Each template loads module specs independently. If listing 50 compose templates, the compose module spec is imported 50 times.
+* TODO Missing Type Hints in Some Functions: While most code has type hints, some functions are missing them, reducing IDE support and static analysis capability.
+* TODO No Dry-Run Mode for Generate Command: A dry-run mode would allow users to see what files would be generated without actually writing them to disk.
+* TODO Template Validation Command: A command to validate the structure and variable definitions of a template without generating it.
+* TODO Interactive Variable Prompt Improvements: The interactive prompt could be improved with better navigation, help text, and validation feedback.
+* TODO Better Error Recovery in Jinja2 Rendering

cli/core/config.py

@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+
+import yaml
+from rich.console import Console
+
+from .variables import Variable, VariableSection, VariableCollection
+
+logger = logging.getLogger(__name__)
+console = Console()
+
+class ConfigManager:
+    """Manages configuration for the CLI application."""
+    
+    def __init__(self, config_path: Optional[Union[str, Path]] = None) -> None:
+        """Initialize the configuration manager.
+        
+        Args:
+            config_path: Path to the configuration file. If None, uses default location.
+        """
+        if config_path is None:
+            # Default to ~/.config/boilerplates/config.yaml
+            config_dir = Path.home() / ".config" / "boilerplates"
+            config_dir.mkdir(parents=True, exist_ok=True)
+            self.config_path = config_dir / "config.yaml"
+        else:
+            self.config_path = Path(config_path)
+        
+        # Create default config if it doesn't exist
+        if not self.config_path.exists():
+            self._create_default_config()
+    
+    def _create_default_config(self) -> None:
+        """Create a default configuration file."""
+        default_config = {
+            "defaults": {},
+            "preferences": {
+                "editor": "vim",
+                "output_dir": None,
+                "library_paths": []
+            }
+        }
+        self._write_config(default_config)
+        logger.info(f"Created default configuration at {self.config_path}")
+    
+    def _read_config(self) -> Dict[str, Any]:
+        """Read configuration from file.
+        
+        Returns:
+            Dictionary containing the configuration.
+            
+        Raises:
+            yaml.YAMLError: If YAML parsing fails.
+        """
+        try:
+            with open(self.config_path, 'r') as f:
+                config = yaml.safe_load(f) or {}
+            return config
+        except yaml.YAMLError as e:
+            logger.error(f"Failed to parse YAML configuration: {e}")
+            raise
+        except Exception as e:
+            logger.error(f"Failed to read configuration file: {e}")
+            raise
+    
+    def _write_config(self, config: Dict[str, Any]) -> None:
+        """Write configuration to file.
+        
+        Args:
+            config: Dictionary containing the configuration to write.
+        """
+        try:
+            with open(self.config_path, 'w') as f:
+                yaml.dump(config, f, default_flow_style=False)
+            logger.debug(f"Configuration written to {self.config_path}")
+        except Exception as e:
+            logger.error(f"Failed to write configuration file: {e}")
+            raise
+    
+    
+    def get_config_path(self) -> Path:
+        """Get the path to the configuration file.
+        
+        Returns:
+            Path to the configuration file.
+        """
+        return self.config_path
+    
+    # -------------------------
+    # SECTION: Defaults Management
+    # -------------------------
+    
+    def get_defaults(self, module_name: str) -> Dict[str, Any]:
+        """Get default variable values for a module.
+        
+        Returns defaults in a flat format:
+        {
+            "var_name": "value",
+            "var2_name": "value2"
+        }
+        
+        Args:
+            module_name: Name of the module
+            
+        Returns:
+            Dictionary of default values (flat key-value pairs)
+        """
+        config = self._read_config()
+        defaults = config.get("defaults", {})
+        return defaults.get(module_name, {})
+    
+    def set_defaults(self, module_name: str, defaults: Dict[str, Any]) -> None:
+        """Set default variable values for a module.
+        
+        Args:
+            module_name: Name of the module
+            defaults: Dictionary of defaults (flat key-value pairs):
+                      {"var_name": "value", "var2_name": "value2"}
+        """
+        config = self._read_config()
+        
+        if "defaults" not in config:
+            config["defaults"] = {}
+        
+        config["defaults"][module_name] = defaults
+        self._write_config(config)
+        logger.info(f"Updated defaults for module '{module_name}'")
+    
+    def set_default_value(self, module_name: str, var_name: str, value: Any) -> None:
+        """Set a single default variable value.
+        
+        Args:
+            module_name: Name of the module
+            var_name: Name of the variable
+            value: Default value to set
+        """
+        defaults = self.get_defaults(module_name)
+        defaults[var_name] = value
+        self.set_defaults(module_name, defaults)
+        logger.info(f"Set default for '{module_name}.{var_name}' = '{value}'")
+    
+    def get_default_value(self, module_name: str, var_name: str) -> Optional[Any]:
+        """Get a single default variable value.
+        
+        Args:
+            module_name: Name of the module
+            var_name: Name of the variable
+            
+        Returns:
+            Default value or None if not set
+        """
+        defaults = self.get_defaults(module_name)
+        return defaults.get(var_name)
+    
+    def clear_defaults(self, module_name: str) -> None:
+        """Clear all defaults for a module.
+        
+        Args:
+            module_name: Name of the module
+        """
+        config = self._read_config()
+        
+        if "defaults" in config and module_name in config["defaults"]:
+            del config["defaults"][module_name]
+            self._write_config(config)
+            logger.info(f"Cleared defaults for module '{module_name}'")
+    
+    # !SECTION
+    
+    # -------------------------
+    # SECTION: Preferences Management
+    # -------------------------
+    
+    def get_preference(self, key: str) -> Optional[Any]:
+        """Get a user preference value.
+        
+        Args:
+            key: Preference key (e.g., 'editor', 'output_dir', 'library_paths')
+            
+        Returns:
+            Preference value or None if not set
+        """
+        config = self._read_config()
+        preferences = config.get("preferences", {})
+        return preferences.get(key)
+    
+    def set_preference(self, key: str, value: Any) -> None:
+        """Set a user preference value.
+        
+        Args:
+            key: Preference key
+            value: Preference value
+        """
+        config = self._read_config()
+        
+        if "preferences" not in config:
+            config["preferences"] = {}
+        
+        config["preferences"][key] = value
+        self._write_config(config)
+        logger.info(f"Set preference '{key}' = '{value}'")
+    
+    def get_all_preferences(self) -> Dict[str, Any]:
+        """Get all user preferences.
+        
+        Returns:
+            Dictionary of all preferences
+        """
+        config = self._read_config()
+        return config.get("preferences", {})
+    
+    # !SECTION

cli/core/display.py

@@ -149,11 +149,8 @@ class DisplayManager:
                 variables_table.add_row("", "", "", "", "", style="dim")
             first_section = False
 
-            is_dimmed = False
-            if section.toggle:
-                toggle_var = section.variables.get(section.toggle)
-                if toggle_var and not toggle_var.get_typed_value():
-                    is_dimmed = True
+            # Use section's native is_enabled() method
+            is_dimmed = not section.is_enabled()
 
             disabled_text = " (disabled)" if is_dimmed else ""
             required_text = " [yellow](required)[/yellow]" if section.required else ""
@@ -162,11 +159,8 @@ class DisplayManager:
 
             for var_name, variable in section.variables.items():
                 row_style = "dim" if is_dimmed else None
-                default_val = str(variable.value) if variable.value is not None else ""
-                if variable.sensitive:
-                    default_val = "********"
-                elif len(default_val) > 30:
-                    default_val = default_val[:27] + "..."
+                # Use variable's native get_display_value() method
+                default_val = variable.get_display_value(mask_sensitive=True, max_length=30)
 
                 variables_table.add_row(
                     f"  {var_name}",
@@ -178,3 +172,69 @@ class DisplayManager:
                 )
 
         console.print(variables_table)
+
+    def display_config_tree(self, spec: dict, module_name: str, show_all: bool = False) -> None:
+        """Display configuration spec as a tree view.
+        
+        Args:
+            spec: The configuration spec dictionary
+            module_name: Name of the module
+            show_all: If True, show all details including descriptions
+        """
+        if not spec:
+            console.print(f"[yellow]No configuration found for module '{module_name}'[/yellow]")
+            return
+
+        # Create root tree node
+        tree = Tree(f"[bold blue]\ue5fc {str.capitalize(module_name)} Configuration[/bold blue]")
+
+        for section_name, section_data in spec.items():
+            if not isinstance(section_data, dict):
+                continue
+
+            # Determine if this is a section with variables
+            # Guard against None from empty YAML sections
+            section_vars = section_data.get("vars") or {}
+            section_desc = section_data.get("description", "")
+            section_required = section_data.get("required", False)
+            section_toggle = section_data.get("toggle", None)
+
+            # Build section label
+            section_label = f"[cyan]{section_name}[/cyan]"
+            if section_required:
+                section_label += " [yellow](required)[/yellow]"
+            if section_toggle:
+                section_label += f" [dim](toggle: {section_toggle})[/dim]"
+            
+            if show_all and section_desc:
+                section_label += f"\n  [dim]{section_desc}[/dim]"
+
+            section_node = tree.add(section_label)
+
+            # Add variables
+            if section_vars:
+                for var_name, var_data in section_vars.items():
+                    if isinstance(var_data, dict):
+                        var_type = var_data.get("type", "string")
+                        var_default = var_data.get("default", "")
+                        var_desc = var_data.get("description", "")
+                        var_sensitive = var_data.get("sensitive", False)
+
+                        # Build variable label
+                        var_label = f"[green]{var_name}[/green] [dim]({var_type})[/dim]"
+                        
+                        if var_default is not None and var_default != "":
+                            display_val = "********" if var_sensitive else str(var_default)
+                            if not var_sensitive and len(display_val) > 30:
+                                display_val = display_val[:27] + "..."
+                            var_label += f" = [yellow]{display_val}[/yellow]"
+                        
+                        if show_all and var_desc:
+                            var_label += f"\n    [dim]{var_desc}[/dim]"
+                        
+                        section_node.add(var_label)
+                    else:
+                        # Simple key-value pair
+                        section_node.add(f"[green]{var_name}[/green] = [yellow]{var_data}[/yellow]")
+
+        console.print(tree)

cli/core/module.py

@@ -163,6 +163,20 @@ class Module(ABC):
       console.print(f"[red]Template '{id}' not found in module '{self.name}'[/red]")
       return
     
+    # Apply config defaults (same as in generate)
+    # This ensures the display shows the actual defaults that will be used
+    if template.variables:
+      from .config import ConfigManager
+      config = ConfigManager()
+      config_defaults = config.get_defaults(self.name)
+      
+      if config_defaults:
+        logger.debug(f"Loading config defaults for module '{self.name}'")
+        # Apply config defaults (this respects the variable types and validation)
+        successful = template.variables.apply_defaults(config_defaults, "config")
+        if successful:
+          logger.debug(f"Applied config defaults for: {', '.join(successful)}")
+    
     self._display_template_details(template, id)
 
   def generate(
@@ -173,17 +187,39 @@ class Module(ABC):
     var: Optional[list[str]] = Option(None, "--var", "-v", help="Variable override (repeatable). Use KEY=VALUE or --var KEY VALUE"),
     ctx: Context = None,
   ) -> None:
-    """Generate from template."""
+    """Generate from template.
+    
+    Variable precedence chain (lowest to highest):
+    1. Module spec (defined in cli/modules/*.py)
+    2. Template spec (from template.yaml)
+    3. Config defaults (from ~/.config/boilerplates/config.yaml)
+    4. CLI overrides (--var flags)
+    """
 
     logger.info(f"Starting generation for template '{id}' from module '{self.name}'")
     template = self._load_template_by_id(id)
 
+    # Apply config defaults (precedence: config > template > module)
+    # Config only sets VALUES, not the spec structure
+    if template.variables:
+      from .config import ConfigManager
+      config = ConfigManager()
+      config_defaults = config.get_defaults(self.name)
+      
+      if config_defaults:
+        logger.info(f"Loading config defaults for module '{self.name}'")
+        # Apply config defaults (this respects the variable types and validation)
+        successful = template.variables.apply_defaults(config_defaults, "config")
+        if successful:
+          logger.debug(f"Applied config defaults for: {', '.join(successful)}")
+    
+    # Apply CLI overrides (highest precedence)
     extra_args = list(ctx.args) if ctx and hasattr(ctx, "args") else []
     cli_overrides = parse_var_inputs(var or [], extra_args)
     if cli_overrides:
       logger.info(f"Received {len(cli_overrides)} variable overrides from CLI")
       if template.variables:
-        successful_overrides = template.variables.apply_overrides(cli_overrides, " -> cli")
+        successful_overrides = template.variables.apply_defaults(cli_overrides, "cli")
         if successful_overrides:
           logger.debug(f"Applied CLI overrides for: {', '.join(successful_overrides)}")
 
@@ -245,6 +281,139 @@ class Module(ABC):
       # Stop execution without letting Typer/Click print the exception again.
       raise Exit(code=1)
 
+  # --------------------------
+  # SECTION: Config Commands
+  # --------------------------
+
+  def config_get(
+    self,
+    var_name: Optional[str] = Argument(None, help="Variable name to get (omit to show all defaults)"),
+  ) -> None:
+    """Get config default value(s) for this module.
+    
+    Examples:
+        # Get all defaults for module
+        cli compose config get
+        
+        # Get specific variable default
+        cli compose config get service_name
+    """
+    from .config import ConfigManager
+    config = ConfigManager()
+    
+    if var_name:
+      # Get specific variable default
+      value = config.get_default_value(self.name, var_name)
+      if value is not None:
+        console.print(f"[green]{var_name}[/green] = [yellow]{value}[/yellow]")
+      else:
+        console.print(f"[red]No default set for variable '{var_name}' in module '{self.name}'[/red]")
+    else:
+      # Show all defaults (flat list)
+      defaults = config.get_defaults(self.name)
+      if defaults:
+        console.print(f"[bold]Config defaults for module '{self.name}':[/bold]\n")
+        for var_name, var_value in defaults.items():
+          console.print(f"  [green]{var_name}[/green] = [yellow]{var_value}[/yellow]")
+      else:
+        console.print(f"[yellow]No defaults configured for module '{self.name}'[/yellow]")
+
+  def config_set(
+    self,
+    var_name: str = Argument(..., help="Variable name to set default for"),
+    value: str = Argument(..., help="Default value"),
+  ) -> None:
+    """Set a default value for a variable in config.
+    
+    This only sets the DEFAULT VALUE, not the variable spec.
+    The variable must be defined in the module or template spec.
+    
+    Examples:
+        # Set default value
+        cli compose config set service_name my-awesome-app
+        
+        # Set author for all compose templates
+        cli compose config set author "Christian Lempa"
+    """
+    from .config import ConfigManager
+    config = ConfigManager()
+    
+    # Set the default value
+    config.set_default_value(self.name, var_name, value)
+    console.print(f"[green]✓ Set default:[/green] [cyan]{var_name}[/cyan] = [yellow]{value}[/yellow]")
+    console.print(f"\n[dim]This will be used as the default value when generating templates with this module.[/dim]")
+
+  def config_remove(
+    self,
+    var_name: str = Argument(..., help="Variable name to remove"),
+  ) -> None:
+    """Remove a specific default variable value.
+    
+    Examples:
+        # Remove a default value
+        cli compose config remove service_name
+    """
+    from .config import ConfigManager
+    config = ConfigManager()
+    defaults = config.get_defaults(self.name)
+    
+    if not defaults:
+      console.print(f"[yellow]No defaults configured for module '{self.name}'[/yellow]")
+      return
+    
+    if var_name in defaults:
+      del defaults[var_name]
+      config.set_defaults(self.name, defaults)
+      console.print(f"[green]✓ Removed default for '{var_name}'[/green]")
+    else:
+      console.print(f"[red]No default found for variable '{var_name}'[/red]")
+
+  def config_clear(
+    self,
+    var_name: Optional[str] = Argument(None, help="Variable name to clear (omit to clear all defaults)"),
+    force: bool = Option(False, "--force", "-f", help="Skip confirmation prompt"),
+  ) -> None:
+    """Clear config default value(s) for this module.
+    
+    Examples:
+        # Clear specific variable default
+        cli compose config clear service_name
+        
+        # Clear all defaults for module
+        cli compose config clear --force
+    """
+    from .config import ConfigManager
+    config = ConfigManager()
+    defaults = config.get_defaults(self.name)
+    
+    if not defaults:
+      console.print(f"[yellow]No defaults configured for module '{self.name}'[/yellow]")
+      return
+    
+    if var_name:
+      # Clear specific variable
+      if var_name in defaults:
+        del defaults[var_name]
+        config.set_defaults(self.name, defaults)
+        console.print(f"[green]✓ Cleared default for '{var_name}'[/green]")
+      else:
+        console.print(f"[red]No default found for variable '{var_name}'[/red]")
+    else:
+      # Clear all defaults
+      if not force:
+        console.print(f"[bold yellow]⚠️  Warning:[/bold yellow] This will clear ALL defaults for module '[cyan]{self.name}[/cyan]'")
+        console.print()
+        # Show what will be cleared
+        for var_name, var_value in defaults.items():
+          console.print(f"  [green]{var_name}[/green] = [yellow]{var_value}[/yellow]")
+        console.print()
+        if not Confirm.ask(f"[bold red]Are you sure?[/bold red]", default=False):
+          console.print("[green]Operation cancelled.[/green]")
+          return
+      
+      config.clear_defaults(self.name)
+      console.print(f"[green]✓ Cleared all defaults for module '{self.name}'[/green]")
+
   # !SECTION
 
   # ------------------------------
@@ -269,6 +438,14 @@ class Module(ABC):
       context_settings={"allow_extra_args": True, "ignore_unknown_options": True}
     )(module_instance.generate)
     
+    # Add config commands (simplified - only manage default values)
+    config_app = Typer(help="Manage default values for template variables")
+    config_app.command("get", help="Get default value(s)")(module_instance.config_get)
+    config_app.command("set", help="Set a default value")(module_instance.config_set)
+    config_app.command("remove", help="Remove a specific default value")(module_instance.config_remove)
+    config_app.command("clear", help="Clear default value(s)")(module_instance.config_clear)
+    module_app.add_typer(config_app, name="config")
+    
     app.add_typer(module_app, name=cls.name, help=cls.description)
     logger.info(f"Module '{cls.name}' CLI commands registered")
 
@@ -394,4 +571,4 @@ class Module(ABC):
     """Display template information panel and variables table."""
     self.display.display_template_details(template, template_id)
 
-# !SECTION
+# !SECTION

cli/core/prompt.py

@@ -65,8 +65,8 @@ class PromptHandler:
             collected[toggle_var.name] = new_value
             toggle_var.value = new_value
           
-          # Skip remaining variables in section if disabled
-          if not new_value:
+          # Use section's native is_enabled() method
+          if not section.is_enabled():
             continue
 
       # Collect variables in this section
@@ -95,15 +95,10 @@ class PromptHandler:
   def _prompt_variable(self, variable: Variable, required: bool = False) -> Any:
     """Prompt for a single variable value based on its type."""
     logger.debug(f"Prompting for variable '{variable.name}' (type: {variable.type})")
-    prompt_text = variable.prompt or variable.description or variable.name
-
-    # Normalize default value once and reuse. This centralizes handling for
-    # enums, bools, ints and strings and avoids duplicated fallback logic.
-    default_value = self._normalize_default(variable)
-
-    # Friendly hint for common semantic types — only show if a default exists
-    if default_value is not None and variable.type in ["hostname", "email", "url"]:
-      prompt_text += f" ({variable.type})"
+    
+    # Use variable's native methods for prompt text and default value
+    prompt_text = variable.get_prompt_text()
+    default_value = variable.get_normalized_default()
 
     # If variable is required and there's no default, mark it in the prompt
     if required and default_value is None:
@@ -111,11 +106,10 @@ class PromptHandler:
 
     handler = self._get_prompt_handler(variable)
 
-    # Attach the optional 'extra' explanation inline (dimmed) so it appears
-    # after the main question rather than before it.
-    if getattr(variable, 'extra', None):
-      # Put the extra hint inline (same line) instead of on the next line.
-      prompt_text = f"{prompt_text} [dim]{variable.extra}[/dim]"
+    # Add validation hint (includes both extra text and enum options)
+    hint = variable.get_validation_hint()
+    if hint:
+      prompt_text = f"{prompt_text} [dim]{hint}[/dim]"
 
     while True:
       try:
@@ -138,46 +132,6 @@ class PromptHandler:
         default_value = variable.value
         handler = self._get_prompt_handler(variable)
 
-  def _normalize_default(self, variable: Variable) -> Any:
-    """Return a normalized default suitable for prompt handlers.
-
-    Tries to use the typed value if available, otherwise falls back to the raw
-    stored value. For enums, ensures the default is one of the options.
-    """
-    try:
-      typed = variable.get_typed_value()
-    except Exception:
-      typed = variable.value
-
-    # Special-case enums: ensure default is valid
-    if variable.type == "enum":
-      options = variable.options or []
-      if not options:
-        return typed
-      # If typed is falsy or not in options, pick first option as fallback
-      if typed is None or str(typed) not in options:
-        return options[0]
-      return str(typed)
-
-    # For booleans and ints return as-is (handlers will accept these types)
-    if variable.type == "bool":
-      if isinstance(typed, bool):
-        return typed
-      if typed is None:
-        return None
-      return bool(typed)
-
-    if variable.type == "int":
-      try:
-        return int(typed) if typed is not None and typed != "" else None
-      except Exception:
-        return None
-
-    # Default: return string or None
-    if typed is None:
-      return None
-    return str(typed)
-
   def _get_prompt_handler(self, variable: Variable) -> Callable:
     """Return the prompt function for a variable type."""
     handlers = {
@@ -221,28 +175,21 @@ class PromptHandler:
     return IntPrompt.ask(prompt_text, default=default_int)
 
   def _prompt_enum(self, prompt_text: str, options: list[str], default: Any = None, extra: str | None = None) -> str:
-    """Prompt for enum selection with validation. """
+    """Prompt for enum selection with validation.
+    
+    Note: prompt_text should already include hint from variable.get_validation_hint()
+    but we keep this for backward compatibility and fallback.
+    """
     if not options:
       return self._prompt_string(prompt_text, default)
 
-    # Build a single inline hint that contains both the options and any extra
-    # explanation, rendered dimmed and appended to the prompt on one line.
-    hint_parts: list[str] = []
-    hint_parts.append(f"Options: {', '.join(options)}")
-    if extra:
-      hint_parts.append(extra)
-
-    # Show options and extra inline (same line) in a single dimmed block.
-    options_text = f" [dim]{' — '.join(hint_parts)}[/dim]"
-    prompt_text_with_options = prompt_text + options_text
-
     # Validate default is in options
     if default and str(default) not in options:
       default = options[0]
 
     while True:
       value = Prompt.ask(
-        prompt_text_with_options,
+        prompt_text,
         default=str(default) if default else options[0],
         show_default=True,
       )

cli/core/template.py

@@ -6,10 +6,10 @@ from typing import Any, Dict, List, Set, Optional, Literal
 from dataclasses import dataclass, field
 import logging
 import os
+import yaml
 from jinja2 import Environment, FileSystemLoader, meta
 from jinja2 import nodes
 from jinja2.visitor import NodeVisitor
-import frontmatter
 
 logger = logging.getLogger(__name__)
 
@@ -44,13 +44,18 @@ class TemplateMetadata:
   # files: List[str] = field(default_factory=list) # No longer needed, as TemplateFile handles this
   library: str = "unknown"
 
-  def __init__(self, post: frontmatter.Post, library_name: str | None = None) -> None:
-    """Initialize TemplateMetadata from frontmatter post."""
+  def __init__(self, template_data: dict, library_name: str | None = None) -> None:
+    """Initialize TemplateMetadata from parsed YAML template data.
+    
+    Args:
+        template_data: Parsed YAML data from template.yaml
+        library_name: Name of the library this template belongs to
+    """
     # Validate metadata format first
-    self._validate_metadata(post)
+    self._validate_metadata(template_data)
     
     # Extract metadata section
-    metadata_section = post.metadata.get("metadata", {})
+    metadata_section = template_data.get("metadata", {})
     
     self.name = metadata_section.get("name", "")
     # YAML block scalar (|) preserves a trailing newline. Remove only trailing newlines
@@ -70,9 +75,16 @@ class TemplateMetadata:
     self.library = library_name or "unknown"
 
   @staticmethod
-  def _validate_metadata(post: frontmatter.Post) -> None:
-    """Validate that template has required 'metadata' section with all required fields."""
-    metadata_section = post.metadata.get("metadata")
+  def _validate_metadata(template_data: dict) -> None:
+    """Validate that template has required 'metadata' section with all required fields.
+    
+    Args:
+        template_data: Parsed YAML data from template.yaml
+        
+    Raises:
+        ValueError: If metadata section is missing or incomplete
+    """
+    metadata_section = template_data.get("metadata")
     if metadata_section is None:
       raise ValueError("Template format error: missing 'metadata' section")
     
@@ -112,14 +124,30 @@ class Template:
       # Find and parse the main template file (template.yaml or template.yml)
       main_template_path = self._find_main_template_file()
       with open(main_template_path, "r", encoding="utf-8") as f:
-        self._post = frontmatter.load(f) # Store post for later access to spec
+        # Load all YAML documents (handles templates with empty lines before ---)
+        documents = list(yaml.safe_load_all(f))
+        
+        # Filter out None/empty documents and get the first non-empty one
+        valid_docs = [doc for doc in documents if doc is not None]
+        
+        if not valid_docs:
+          raise ValueError("Template file contains no valid YAML data")
+        
+        if len(valid_docs) > 1:
+          logger.warning(f"Template file contains multiple YAML documents, using the first one")
+        
+        self._template_data = valid_docs[0]
+      
+      # Validate template data
+      if not isinstance(self._template_data, dict):
+        raise ValueError("Template file must contain a valid YAML dictionary")
 
       # Load metadata (always needed)
-      self.metadata = TemplateMetadata(self._post, library_name)
+      self.metadata = TemplateMetadata(self._template_data, library_name)
       logger.debug(f"Loaded metadata: {self.metadata}")
 
       # Validate 'kind' field (always needed)
-      self._validate_kind(self._post)
+      self._validate_kind(self._template_data)
 
       # Collect file paths (relatively lightweight, needed for various lazy loads)
       # This will now populate self.template_files
@@ -154,45 +182,32 @@ class Template:
       raise ValueError(f"Error loading module specifications for kind '{kind}': {e}")
 
   def _merge_specs(self, module_specs: dict, template_specs: dict) -> dict:
-    """Deep merge template specs with module specs."""
-    merged_specs = {}
-    for section_key in module_specs.keys():
-      module_section = module_specs.get(section_key, {})
-      template_section = template_specs.get(section_key, {})
-      merged_section = {**module_section}
-      for key in ['title', 'prompt', 'description', 'toggle', 'required']:
-        if key in template_section:
-          merged_section[key] = template_section[key]
-      module_vars = module_section.get('vars') if isinstance(module_section.get('vars'), dict) else {}
-      template_vars = template_section.get('vars') if isinstance(template_section.get('vars'), dict) else {}
-
-      # Deep-merge variables while preserving the ordering from the module spec.
-      # Template-only variables are appended at the end in template order.
-      from collections import OrderedDict
-
-      merged_vars: OrderedDict = OrderedDict()
-
-      # First, keep module variables in their original order, merging any
-      # template-provided keys for the same variable.
-      for var_name, mod_var in module_vars.items():
-        mod_var = mod_var or {}
-        tmpl_var = template_vars.get(var_name, {}) or {}
-        merged_vars[var_name] = {**mod_var, **tmpl_var}
-
-      # Then, append any template-only variables in the template's order.
-      for var_name, tmpl_var in template_vars.items():
-        if var_name in merged_vars:
-          continue
-        merged_vars[var_name] = {**(tmpl_var or {})}
-
-      merged_section['vars'] = merged_vars
-      merged_specs[section_key] = merged_section
+    """Deep merge template specs with module specs using VariableCollection.
     
-    for section_key in template_specs.keys():
-      if section_key not in module_specs:
-        merged_specs[section_key] = {**template_specs[section_key]}
-        
-    return merged_specs
+    Uses VariableCollection's native merge() method for consistent merging logic.
+    Module specs are base, template specs override with origin tracking.
+    """
+    # Create VariableCollection from module specs (base)
+    module_collection = VariableCollection(module_specs) if module_specs else VariableCollection({})
+    
+    # Set origin for module variables
+    for section in module_collection.get_sections().values():
+      for variable in section.variables.values():
+        if not variable.origin:
+          variable.origin = "module"
+    
+    # Merge template specs into module specs (template overrides)
+    if template_specs:
+      merged_collection = module_collection.merge(template_specs, origin="template")
+    else:
+      merged_collection = module_collection
+    
+    # Convert back to dict format
+    merged_spec = {}
+    for section_key, section in merged_collection.get_sections().items():
+      merged_spec[section_key] = section.to_dict()
+    
+    return merged_spec
 
   def _collect_template_files(self) -> None:
     """Collects all TemplateFile objects in the template directory."""
@@ -282,33 +297,22 @@ class Template:
     return visitor.found
 
   def _filter_specs_to_used(self, used_variables: set, merged_specs: dict, module_specs: dict, template_specs: dict) -> dict:
-    """Filter specs to only include variables used in the templates."""
+    """Filter specs to only include variables used in templates using VariableCollection.
+    
+    Uses VariableCollection's native filter_to_used() method.
+    Keeps sensitive variables even if not used.
+    """
+    # Create VariableCollection from merged specs
+    merged_collection = VariableCollection(merged_specs)
+    
+    # Filter to only used variables (and sensitive ones)
+    filtered_collection = merged_collection.filter_to_used(used_variables, keep_sensitive=True)
+    
+    # Convert back to dict format
     filtered_specs = {}
-    for section_key, section_data in merged_specs.items():
-      if "vars" in section_data and isinstance(section_data["vars"], dict):
-        filtered_vars = {}
-        for var_name, var_data in section_data["vars"].items():
-          is_used = var_name in used_variables
-          is_sensitive = var_data.get("sensitive", False)
-          
-          # Include variables that are either used in templates OR marked as sensitive
-          if is_used or is_sensitive:
-            module_has_var = var_name in module_specs.get(section_key, {}).get("vars", {})
-            template_has_var = var_name in template_specs.get(section_key, {}).get("vars", {})
-            
-            if module_has_var and template_has_var:
-              origin = "module -> template"
-            elif template_has_var:
-              origin = "template"
-            else:
-              origin = "module"
-            
-            var_data_with_origin = {**var_data, "origin": origin}
-            
-            filtered_vars[var_name] = var_data_with_origin
-        
-        if filtered_vars:
-          filtered_specs[section_key] = {**section_data, "vars": filtered_vars}
+    for section_key, section in filtered_collection.get_sections().items():
+      filtered_specs[section_key] = section.to_dict()
+    
     return filtered_specs
 
   # ---------------------------
@@ -316,9 +320,16 @@ class Template:
   # ---------------------------
 
   @staticmethod
-  def _validate_kind(post: frontmatter.Post) -> None:
-    """Validate that template has required 'kind' field."""
-    if not post.metadata.get("kind"):
+  def _validate_kind(template_data: dict) -> None:
+    """Validate that template has required 'kind' field.
+    
+    Args:
+        template_data: Parsed YAML data from template.yaml
+        
+    Raises:
+        ValueError: If 'kind' field is missing
+    """
+    if not template_data.get("kind"):
       raise ValueError("Template format error: missing 'kind' field")
 
   def _validate_variable_definitions(self, used_variables: set[str], merged_specs: dict[str, Any]) -> None:
@@ -396,13 +407,18 @@ class Template:
     return rendered_files
 
   def mask_sensitive_values(self, rendered_files: Dict[str, str], variables: VariableCollection) -> Dict[str, str]:
-    """Mask sensitive values in rendered files."""
+    """Mask sensitive values in rendered files using Variable's native masking."""
     masked_files = {}
-    sensitive_vars = variables.get_sensitive_variables()
     
+    # Get all variables (not just sensitive ones) to use their native get_display_value()
     for file_path, content in rendered_files.items():
-      for var_name, var_value in sensitive_vars.items():
-        content = content.replace(str(var_value), "********")
+      # Iterate through all sections and variables
+      for section in variables.get_sections().values():
+        for variable in section.variables.values():
+          if variable.sensitive and variable.value:
+            # Use variable's native masking - always returns "********" for sensitive vars
+            masked_value = variable.get_display_value(mask_sensitive=True)
+            content = content.replace(str(variable.value), masked_value)
       masked_files[file_path] = content
       
     return masked_files
@@ -421,12 +437,14 @@ class Template:
 
   @property
   def template_specs(self) -> dict:
-      return self._post.metadata.get("spec", {})
+      """Get the spec section from template YAML data."""
+      return self._template_data.get("spec", {})
 
   @property
   def module_specs(self) -> dict:
+      """Get the spec from the module definition."""
       if self.__module_specs is None:
-          kind = self._post.metadata.get("kind")
+          kind = self._template_data.get("kind")
           self.__module_specs = self._load_module_specs(kind)
       return self.__module_specs
 
@@ -461,7 +479,8 @@ class Template:
           try:
             jinja_defaults = self._extract_jinja_default_values()
             for section_key, section_data in filtered_specs.items():
-              vars_dict = section_data.get('vars', {})
+              # Guard against None from empty YAML sections
+              vars_dict = section_data.get('vars') or {}
               for var_name, var_data in vars_dict.items():
                 if 'default' not in var_data or var_data.get('default') in (None, ''):
                   if var_name in jinja_defaults:

cli/core/variables.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from collections import OrderedDict
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Set
+from typing import Any, Dict, List, Optional, Set, Union
 from urllib.parse import urlparse
 import logging
 import re
@@ -192,7 +192,175 @@ class Variable:
   def get_typed_value(self) -> Any:
     """Return the stored value converted to the appropriate Python type."""
     return self.convert(self.value)
-
+  
+  def to_dict(self) -> Dict[str, Any]:
+    """Serialize Variable to a dictionary for storage.
+    
+    Returns:
+        Dictionary representation of the variable with only relevant fields.
+    """
+    var_dict = {}
+    
+    if self.type:
+      var_dict["type"] = self.type
+    
+    if self.value is not None:
+      var_dict["default"] = self.value
+    
+    if self.description:
+      var_dict["description"] = self.description
+    
+    if self.prompt:
+      var_dict["prompt"] = self.prompt
+    
+    if self.sensitive:
+      var_dict["sensitive"] = self.sensitive
+    
+    if self.extra:
+      var_dict["extra"] = self.extra
+    
+    if self.options:
+      var_dict["options"] = self.options
+    
+    if self.origin:
+      var_dict["origin"] = self.origin
+    
+    return var_dict
+  
+  # -------------------------
+  # SECTION: Display Methods
+  # -------------------------
+  
+  def get_display_value(self, mask_sensitive: bool = True, max_length: int = 30) -> str:
+    """Get formatted display value with optional masking and truncation.
+    
+    Args:
+        mask_sensitive: If True, mask sensitive values with asterisks
+        max_length: Maximum length before truncation (0 = no limit)
+        
+    Returns:
+        Formatted string representation of the value
+    """
+    if self.value is None:
+      return ""
+    
+    # Mask sensitive values
+    if self.sensitive and mask_sensitive:
+      return "********"
+    
+    # Convert to string
+    display = str(self.value)
+    
+    # Truncate if needed
+    if max_length > 0 and len(display) > max_length:
+      return display[:max_length - 3] + "..."
+    
+    return display
+  
+  def get_normalized_default(self) -> Any:
+    """Get normalized default value suitable for prompts and display.
+    
+    Handles type conversion and provides sensible defaults for different types.
+    Especially useful for enum, bool, and int types in interactive prompts.
+    
+    Returns:
+        Normalized default value appropriate for the variable type
+    """
+    try:
+      typed = self.get_typed_value()
+    except Exception:
+      typed = self.value
+    
+    # Enum: ensure default is valid option
+    if self.type == "enum":
+      if not self.options:
+        return typed
+      # If typed is invalid or missing, use first option
+      if typed is None or str(typed) not in self.options:
+        return self.options[0]
+      return str(typed)
+    
+    # Boolean: return as bool type
+    if self.type == "bool":
+      if isinstance(typed, bool):
+        return typed
+      return None if typed is None else bool(typed)
+    
+    # Integer: return as int type
+    if self.type == "int":
+      try:
+        return int(typed) if typed is not None and typed != "" else None
+      except Exception:
+        return None
+    
+    # Default: return string or None
+    return None if typed is None else str(typed)
+  
+  def get_prompt_text(self) -> str:
+    """Get formatted prompt text for interactive input.
+    
+    Returns:
+        Prompt text with optional type hints and descriptions
+    """
+    prompt_text = self.prompt or self.description or self.name
+    
+    # Add type hint for semantic types if there's a default
+    if self.value is not None and self.type in ["hostname", "email", "url"]:
+      prompt_text += f" ({self.type})"
+    
+    return prompt_text
+  
+  def get_validation_hint(self) -> Optional[str]:
+    """Get validation hint for prompts (e.g., enum options).
+    
+    Returns:
+        Formatted hint string or None if no hint needed
+    """
+    hints = []
+    
+    # Add enum options
+    if self.type == "enum" and self.options:
+      hints.append(f"Options: {', '.join(self.options)}")
+    
+    # Add extra help text
+    if self.extra:
+      hints.append(self.extra)
+    
+    return " — ".join(hints) if hints else None
+  
+  def clone(self, update: Optional[Dict[str, Any]] = None) -> 'Variable':
+    """Create a deep copy of the variable with optional field updates.
+    
+    This is more efficient than converting to dict and back when copying variables.
+    
+    Args:
+        update: Optional dictionary of field updates to apply to the clone
+        
+    Returns:
+        New Variable instance with copied data
+        
+    Example:
+        var2 = var1.clone(update={'origin': 'template'})
+    """
+    data = {
+      'name': self.name,
+      'type': self.type,
+      'value': self.value,
+      'description': self.description,
+      'prompt': self.prompt,
+      'options': self.options.copy() if self.options else None,
+      'section': self.section,
+      'origin': self.origin,
+      'sensitive': self.sensitive,
+      'extra': self.extra,
+    }
+    
+    # Apply updates if provided
+    if update:
+      data.update(update)
+    
+    return Variable(data)
+  
   # !SECTION
 
 # !SECTION
@@ -230,6 +398,111 @@ class VariableSection:
 
   def variable_names(self) -> list[str]:
     return list(self.variables.keys())
+  
+  def to_dict(self) -> Dict[str, Any]:
+    """Serialize VariableSection to a dictionary for storage.
+    
+    Returns:
+        Dictionary representation of the section with all metadata and variables.
+    """
+    section_dict = {}
+    
+    if self.title:
+      section_dict["title"] = self.title
+    
+    if self.description:
+      section_dict["description"] = self.description
+    
+    if self.prompt:
+      section_dict["prompt"] = self.prompt
+    
+    if self.toggle:
+      section_dict["toggle"] = self.toggle
+    
+    # Always store required flag
+    section_dict["required"] = self.required
+    
+    # Serialize all variables using their own to_dict method
+    section_dict["vars"] = {}
+    for var_name, variable in self.variables.items():
+      section_dict["vars"][var_name] = variable.to_dict()
+    
+    return section_dict
+  
+  # -------------------------
+  # SECTION: State Methods
+  # -------------------------
+  
+  def is_enabled(self) -> bool:
+    """Check if section is currently enabled based on toggle variable.
+    
+    Returns:
+        True if section is enabled (no toggle or toggle is True), False otherwise
+    """
+    if not self.toggle:
+      return True
+    
+    toggle_var = self.variables.get(self.toggle)
+    if not toggle_var:
+      return True
+    
+    try:
+      return bool(toggle_var.get_typed_value())
+    except Exception:
+      return False
+  
+  def get_toggle_value(self) -> Optional[bool]:
+    """Get the current value of the toggle variable.
+    
+    Returns:
+        Boolean value of toggle variable, or None if no toggle exists
+    """
+    if not self.toggle:
+      return None
+    
+    toggle_var = self.variables.get(self.toggle)
+    if not toggle_var:
+      return None
+    
+    try:
+      return bool(toggle_var.get_typed_value())
+    except Exception:
+      return None
+  
+  def clone(self, origin_update: Optional[str] = None) -> 'VariableSection':
+    """Create a deep copy of the section with all variables.
+    
+    This is more efficient than converting to dict and back when copying sections.
+    
+    Args:
+        origin_update: Optional origin string to apply to all cloned variables
+        
+    Returns:
+        New VariableSection instance with deep-copied variables
+        
+    Example:
+        section2 = section1.clone(origin_update='template')
+    """
+    # Create new section with same metadata
+    cloned = VariableSection({
+      'key': self.key,
+      'title': self.title,
+      'prompt': self.prompt,
+      'description': self.description,
+      'toggle': self.toggle,
+      'required': self.required,
+    })
+    
+    # Deep copy all variables
+    for var_name, variable in self.variables.items():
+      if origin_update:
+        cloned.variables[var_name] = variable.clone(update={'origin': origin_update})
+      else:
+        cloned.variables[var_name] = variable.clone()
+    
+    return cloned
+  
+  # !SECTION
 
 # !SECTION
 
@@ -280,7 +553,9 @@ class VariableCollection:
         continue
       
       section = self._create_section(section_key, section_data)
-      self._initialize_variables(section, section_data.get("vars", {}))
+      # Guard against None from empty YAML sections (vars: with no content)
+      vars_data = section_data.get("vars") or {}
+      self._initialize_variables(section, vars_data)
       self._sections[section_key] = section
 
   def _create_section(self, key: str, data: dict[str, Any]) -> VariableSection:
@@ -297,6 +572,10 @@ class VariableCollection:
 
   def _initialize_variables(self, section: VariableSection, vars_data: dict[str, Any]) -> None:
     """Initialize variables for a section."""
+    # Guard against None from empty YAML sections
+    if vars_data is None:
+      vars_data = {}
+    
     for var_name, var_data in vars_data.items():
       var_init_data = {"name": var_name, **var_data}
       variable = Variable(var_init_data)
@@ -342,15 +621,23 @@ class VariableCollection:
   # NOTE: These helper methods reduce code duplication across module.py and prompt.py
   # by centralizing common variable collection operations
 
-  def apply_overrides(self, overrides: dict[str, Any], origin_suffix: str = " -> cli") -> list[str]:
-    """Apply multiple variable overrides at once."""
+  def apply_defaults(self, defaults: dict[str, Any], origin: str = "cli") -> list[str]:
+    """Apply default values to variables, updating their origin.
+    
+    Args:
+        defaults: Dictionary mapping variable names to their default values
+        origin: Source of these defaults (e.g., 'config', 'cli')
+        
+    Returns:
+        List of variable names that were successfully updated
+    """
     # NOTE: This method uses the _variable_map for a significant performance gain,
     # as it allows direct O(1) lookup of variables instead of iterating
     # through all sections to find a match.
-    successful_overrides = []
+    successful = []
     errors = []
     
-    for var_name, value in overrides.items():
+    for var_name, value in defaults.items():
       try:
         variable = self._variable_map.get(var_name)
         if not variable:
@@ -361,21 +648,20 @@ class VariableCollection:
         converted_value = variable.convert(value)
         variable.value = converted_value
         
-        # Update origin to show override
-        if variable.origin:
-          variable.origin = variable.origin + origin_suffix
-        else:
-          variable.origin = origin_suffix.lstrip(" -> ")
+        # Set origin to the current source (not a chain)
+        variable.origin = origin
         
-        successful_overrides.append(var_name)
+        successful.append(var_name)
           
       except ValueError as e:
-        error_msg = f"Invalid override value for '{var_name}': {value} - {e}"
+        error_msg = f"Invalid value for '{var_name}': {value} - {e}"
         errors.append(error_msg)
         logger.error(error_msg)
     
     if errors:
-      logger.warning(f"Some CLI overrides failed: {'; '.join(errors)}")
+      logger.warning(f"Some defaults failed to apply: {'; '.join(errors)}")
+    
+    return successful
   
   def validate_all(self) -> None:
     """Validate all variables in the collection, skipping disabled sections."""
@@ -412,6 +698,197 @@ class VariableCollection:
       logger.error(error_msg)
       raise ValueError(error_msg)
 
+  def merge(self, other_spec: Union[Dict[str, Any], 'VariableCollection'], origin: str = "override") -> 'VariableCollection':
+    """Merge another spec or VariableCollection into this one with precedence tracking.
+    
+    OPTIMIZED: Works directly on objects without dict conversions for better performance.
+    
+    The other spec/collection has higher precedence and will override values in self.
+    Creates a new VariableCollection with merged data.
+    
+    Args:
+        other_spec: Either a spec dictionary or another VariableCollection to merge
+        origin: Origin label for variables from other_spec (e.g., 'template', 'config')
+        
+    Returns:
+        New VariableCollection with merged data
+        
+    Example:
+        module_vars = VariableCollection(module_spec)
+        template_vars = module_vars.merge(template_spec, origin='template')
+        # Variables from template_spec override module_spec
+        # Origins tracked: 'module' or 'module -> template'
+    """
+    # Convert dict to VariableCollection if needed (only once)
+    if isinstance(other_spec, dict):
+      other = VariableCollection(other_spec)
+    else:
+      other = other_spec
+    
+    # Create new collection without calling __init__ (optimization)
+    merged = VariableCollection.__new__(VariableCollection)
+    merged._sections = {}
+    merged._variable_map = {}
+    
+    # First pass: clone sections from self
+    for section_key, self_section in self._sections.items():
+      if section_key in other._sections:
+        # Section exists in both - will merge
+        merged._sections[section_key] = self._merge_sections(
+          self_section, 
+          other._sections[section_key], 
+          origin
+        )
+      else:
+        # Section only in self - clone it
+        merged._sections[section_key] = self_section.clone()
+    
+    # Second pass: add sections that only exist in other
+    for section_key, other_section in other._sections.items():
+      if section_key not in merged._sections:
+        # New section from other - clone with origin update
+        merged._sections[section_key] = other_section.clone(origin_update=origin)
+    
+    # Rebuild variable map for O(1) lookups
+    for section in merged._sections.values():
+      for var_name, variable in section.variables.items():
+        merged._variable_map[var_name] = variable
+    
+    return merged
+  
+  def _infer_origin_from_context(self) -> str:
+    """Infer origin from existing variables (fallback)."""
+    for section in self._sections.values():
+      for variable in section.variables.values():
+        if variable.origin:
+          return variable.origin
+    return "template"
+  
+  def _merge_sections(self, self_section: VariableSection, other_section: VariableSection, origin: str) -> VariableSection:
+    """Merge two sections, with other_section taking precedence.
+    
+    Args:
+        self_section: Base section
+        other_section: Section to merge in (takes precedence)
+        origin: Origin label for merged variables
+        
+    Returns:
+        New merged VariableSection
+    """
+    # Start with a clone of self_section
+    merged_section = self_section.clone()
+    
+    # Update section metadata from other (other takes precedence)
+    if other_section.title:
+      merged_section.title = other_section.title
+    if other_section.prompt:
+      merged_section.prompt = other_section.prompt
+    if other_section.description:
+      merged_section.description = other_section.description
+    if other_section.toggle:
+      merged_section.toggle = other_section.toggle
+    # Required flag always updated
+    merged_section.required = other_section.required
+    
+    # Merge variables
+    for var_name, other_var in other_section.variables.items():
+      if var_name in merged_section.variables:
+        # Variable exists in both - merge with other taking precedence
+        self_var = merged_section.variables[var_name]
+        
+        # Build update dict with other's values taking precedence
+        update = {}
+        if other_var.type:
+          update['type'] = other_var.type
+        if other_var.value is not None:
+          update['value'] = other_var.value
+        if other_var.description:
+          update['description'] = other_var.description
+        if other_var.prompt:
+          update['prompt'] = other_var.prompt
+        if other_var.options:
+          update['options'] = other_var.options
+        if other_var.sensitive:
+          update['sensitive'] = other_var.sensitive
+        if other_var.extra:
+          update['extra'] = other_var.extra
+        
+        # Update origin tracking (only keep the current source, not the chain)
+        update['origin'] = origin
+        
+        # Clone with updates
+        merged_section.variables[var_name] = self_var.clone(update=update)
+      else:
+        # New variable from other - clone with origin
+        merged_section.variables[var_name] = other_var.clone(update={'origin': origin})
+    
+    return merged_section
+  
+  def filter_to_used(self, used_variables: Set[str], keep_sensitive: bool = True) -> 'VariableCollection':
+    """Filter collection to only variables that are used (or sensitive).
+    
+    OPTIMIZED: Works directly on objects without dict conversions for better performance.
+    
+    Creates a new VariableCollection containing only the variables in used_variables.
+    Sections with no remaining variables are removed.
+    
+    Args:
+        used_variables: Set of variable names that are actually used
+        keep_sensitive: If True, also keep sensitive variables even if not in used set
+        
+    Returns:
+        New VariableCollection with filtered variables
+        
+    Example:
+        all_vars = VariableCollection(spec)
+        used_vars = all_vars.filter_to_used({'var1', 'var2', 'var3'})
+        # Only var1, var2, var3 (and any sensitive vars) remain
+    """
+    # Create new collection without calling __init__ (optimization)
+    filtered = VariableCollection.__new__(VariableCollection)
+    filtered._sections = {}
+    filtered._variable_map = {}
+    
+    # Filter each section
+    for section_key, section in self._sections.items():
+      # Create a new section with same metadata
+      filtered_section = VariableSection({
+        'key': section.key,
+        'title': section.title,
+        'prompt': section.prompt,
+        'description': section.description,
+        'toggle': section.toggle,
+        'required': section.required,
+      })
+      
+      # Clone only the variables that should be included
+      for var_name, variable in section.variables.items():
+        # Include if used OR if sensitive (and keep_sensitive is True)
+        should_include = (
+          var_name in used_variables or 
+          (keep_sensitive and variable.sensitive)
+        )
+        
+        if should_include:
+          filtered_section.variables[var_name] = variable.clone()
+      
+      # Only add section if it has variables
+      if filtered_section.variables:
+        filtered._sections[section_key] = filtered_section
+        # Add variables to map
+        for var_name, variable in filtered_section.variables.items():
+          filtered._variable_map[var_name] = variable
+    
+    return filtered
+  
+  def get_all_variable_names(self) -> Set[str]:
+    """Get set of all variable names across all sections.
+    
+    Returns:
+        Set of all variable names
+    """
+    return set(self._variable_map.keys())
+
   # !SECTION
 
 # !SECTION

cli/modules/compose.py

@@ -138,12 +138,6 @@ spec = OrderedDict(
             "type": "bool",
             "default": False,
           },
-          "database_type": {
-            "description": "Database type",
-            "type": "enum",
-            "options": ["postgres", "mysql", "mariadb", "sqlite"],
-            "default": "postgres",
-          },
           "database_host": {
             "description": "Database host",
             "type": "str",
@@ -151,8 +145,7 @@ spec = OrderedDict(
           },
           "database_port": {
             "description": "Database port",
-            "type": "int",
-            "default": 5432,
+            "type": "int"
           },
           "database_name": {
             "description": "Database name",

library/compose/ansiblesemaphore/compose.yaml.j2

@@ -1,6 +1,3 @@
-volumes:
-  semaphore-mysql:
-    driver: local
 services:
   mysql:
     image: docker.io/library/mysql:8.4
@@ -40,3 +37,7 @@ services:
     restart: unless-stopped
     depends_on:
       - mysql
+
+volumes:
+  semaphore-mysql:
+    driver: local

library/compose/ansiblesemaphore/template.yaml

@@ -7,15 +7,6 @@ metadata:
   author: Christian Lempa
   date: '2025-09-28'
   tags:
-  - volumes
-  - docker
-  - compose
-spec:
-  general:
-    vars:
-      volumes_version:
-        type: string
-        description: Volumes version
-        default: latest
-
----
+    - ansible
+    - terraform
+spec: {}