validation_runner.py 4.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153
  1. """Run template validation across dependency matrix cases."""
  2. from __future__ import annotations
  3. from collections.abc import Callable
  4. from dataclasses import dataclass, field
  5. from typing import TYPE_CHECKING
  6. from ..exceptions import TemplateRenderError, TemplateSyntaxError, TemplateValidationError
  7. from ..validators import get_validator_registry
  8. if TYPE_CHECKING:
  9. from cli.core.template import Template
  10. from .dependency_matrix import ValidationCase
  11. @dataclass(frozen=True)
  12. class ValidationFailure:
  13. """A failure from template, semantic, or kind-specific validation."""
  14. case_name: str
  15. stage: str
  16. message: str
  17. file_path: str = ""
  18. validator: str = ""
  19. @dataclass(frozen=True)
  20. class KindValidationFailure:
  21. """A kind-specific validation failure."""
  22. file_path: str
  23. message: str
  24. validator: str
  25. @dataclass
  26. class KindValidationResult:
  27. """Kind-specific validation result."""
  28. validator: str
  29. available: bool = True
  30. skipped: bool = False
  31. failures: list[KindValidationFailure] = field(default_factory=list)
  32. warnings: list[str] = field(default_factory=list)
  33. details: list[str] = field(default_factory=list)
  34. @property
  35. def ok(self) -> bool:
  36. return self.available and not self.failures
  37. KindValidator = Callable[[dict[str, str], str], KindValidationResult]
  38. @dataclass
  39. class MatrixValidationSummary:
  40. """Aggregated validation results for a matrix run."""
  41. total_cases: int = 0
  42. failures: list[ValidationFailure] = field(default_factory=list)
  43. kind_available: bool = True
  44. kind_skipped_cases: set[str] = field(default_factory=set)
  45. @property
  46. def ok(self) -> bool:
  47. return not self.failures
  48. class ValidationRunner:
  49. """Render validation cases and run semantic and optional kind validation."""
  50. def __init__(
  51. self,
  52. template: Template,
  53. cases: list[ValidationCase],
  54. *,
  55. semantic: bool = True,
  56. kind_validator: KindValidator | None = None,
  57. ) -> None:
  58. self.template = template
  59. self.cases = cases
  60. self.semantic = semantic
  61. self.kind_validator = kind_validator
  62. def run(self) -> MatrixValidationSummary:
  63. summary = MatrixValidationSummary(total_cases=len(self.cases))
  64. for case in self.cases:
  65. try:
  66. rendered_files, _ = self.template.render(case.variables)
  67. except (TemplateRenderError, TemplateSyntaxError, TemplateValidationError, ValueError) as exc:
  68. summary.failures.append(ValidationFailure(case_name=case.name, stage="tpl", message=str(exc)))
  69. continue
  70. if self.semantic:
  71. self._run_semantic(case.name, rendered_files, summary)
  72. if self.kind_validator is not None:
  73. self._run_kind(case.name, rendered_files, summary)
  74. return summary
  75. def _run_semantic(
  76. self,
  77. case_name: str,
  78. rendered_files: dict[str, str],
  79. summary: MatrixValidationSummary,
  80. ) -> None:
  81. registry = get_validator_registry()
  82. for file_path, content in rendered_files.items():
  83. result = registry.validate_file(content, file_path)
  84. for error in result.errors:
  85. validator = registry.get_validator(file_path)
  86. summary.failures.append(
  87. ValidationFailure(
  88. case_name=case_name,
  89. stage="sem",
  90. file_path=file_path,
  91. validator=validator.__class__.__name__ if validator else "semantic",
  92. message=error,
  93. )
  94. )
  95. def _run_kind(
  96. self,
  97. case_name: str,
  98. rendered_files: dict[str, str],
  99. summary: MatrixValidationSummary,
  100. ) -> None:
  101. result = self.kind_validator(rendered_files, case_name)
  102. summary.kind_available = summary.kind_available and result.available
  103. if not result.available:
  104. summary.kind_skipped_cases.add(case_name)
  105. return
  106. if result.skipped:
  107. summary.kind_skipped_cases.add(case_name)
  108. return
  109. for failure in result.failures:
  110. summary.failures.append(
  111. ValidationFailure(
  112. case_name=case_name,
  113. stage="kind",
  114. file_path=failure.file_path,
  115. validator=failure.validator,
  116. message=failure.message,
  117. )
  118. )