From cb4560db5c84428bf8f68832a0e95aab9ee47562 Mon Sep 17 00:00:00 2001 From: Azamat Date: Fri, 20 Mar 2026 21:38:28 +0300 Subject: [PATCH] [feat] add observability config --- adapter/config/loader.py | 225 +++++++++++++++++++++++++++++++------- adapter/config/model.py | 15 +++ adapter/otel/bootstrap.py | 91 +++++++++------ config/app.yaml | 8 ++ 4 files changed, 268 insertions(+), 71 deletions(-) diff --git a/adapter/config/loader.py b/adapter/config/loader.py index 550482c..3953923 100644 --- a/adapter/config/loader.py +++ b/adapter/config/loader.py @@ -10,8 +10,10 @@ from .model import ( AppSectionConfig, HttpConfig, LoggingConfig, + MetricsConfig, OtelConfig, SecurityConfig, + TracingConfig, ) ROOT_DIR = Path(__file__).resolve().parents[2] @@ -34,9 +36,44 @@ def load_config( app_section = _section(yaml_data, 'app') http_section = _section(yaml_data, 'http') logging_section = _section(yaml_data, 'logging') - otel_section = _section(yaml_data, 'otel') + metrics_section = _section(yaml_data, 'metrics') + tracing_section = _section(yaml_data, 'tracing') security_section = _section(yaml_data, 'security') + logging_output = _yaml_or_env_choice( + logging_section, + 'output', + 'logging.output', + env_values, + {'stdout', 'file', 'otel'}, + 'APP_LOGGING_OUTPUT', + ) + + logging_format = _yaml_or_env_choice( + logging_section, + 'format', + 'logging.format', + env_values, + {'text', 'json'}, + 'APP_LOGGING_FORMAT', + ) + + metrics_enabled = _yaml_or_env_bool( + metrics_section, + 'enabled', + 'metrics.enabled', + env_values, + 'APP_METRICS_ENABLED', + ) + + tracing_enabled = _yaml_or_env_bool( + tracing_section, + 'enabled', + 'tracing.enabled', + env_values, + 'APP_TRACING_ENABLED', + ) + return AppConfig( app=AppSectionConfig( name=_yaml_or_env_str( @@ -68,43 +105,28 @@ def load_config( env_values, 'APP_LOGGING_LEVEL', ), + output=logging_output, + format=logging_format, + file_path=( + None + if logging_output != 'file' + else _yaml_or_env_str( + logging_section, + 'file_path', + 'logging.file_path', + env_values, + 'APP_LOGGING_FILE_PATH', + ) + ), ), - otel=OtelConfig( - service_name=_yaml_or_env_str( - otel_section, - 'service_name', - 'otel.service_name', - env_values, - 'APP_OTEL_SERVICE_NAME', - ), - logs_endpoint=_yaml_or_env_str( - otel_section, - 'logs_endpoint', - 'otel.logs_endpoint', - env_values, - 'APP_OTEL_LOGS_ENDPOINT', - ), - metrics_endpoint=_yaml_or_env_str( - otel_section, - 'metrics_endpoint', - 'otel.metrics_endpoint', - env_values, - 'APP_OTEL_METRICS_ENDPOINT', - ), - traces_endpoint=_yaml_or_env_str( - otel_section, - 'traces_endpoint', - 'otel.traces_endpoint', - env_values, - 'APP_OTEL_TRACES_ENDPOINT', - ), - metric_export_interval=_yaml_or_env_int( - otel_section, - 'metric_export_interval', - 'otel.metric_export_interval', - env_values, - 'OTEL_METRIC_EXPORT_INTERVAL', - ), + metrics=MetricsConfig(enabled=metrics_enabled), + tracing=TracingConfig(enabled=tracing_enabled), + otel=_load_otel_config( + yaml_data, + env_values, + enable_logs=logging_output == 'otel', + enable_metrics=metrics_enabled, + enable_tracing=tracing_enabled, ), security=SecurityConfig( token_header=_yaml_or_env_str( @@ -184,6 +206,94 @@ def _section(data: Mapping[str, object], name: str) -> dict[str, object]: return section +def _optional_section(data: Mapping[str, object], name: str) -> dict[str, object]: + value = data.get(name) + if value is None: + return {} + if not isinstance(value, dict): + raise ConfigError(f'invalid {name}') + + section: dict[str, object] = {} + for key, item in value.items(): + if not isinstance(key, str): + raise ConfigError(f'invalid {name}') + section[key] = item + return section + + +def _load_otel_config( + data: Mapping[str, object], + env: Mapping[str, str], + *, + enable_logs: bool, + enable_metrics: bool, + enable_tracing: bool, +) -> OtelConfig: + if not any((enable_logs, enable_metrics, enable_tracing)): + return OtelConfig( + service_name='', + logs_endpoint='', + metrics_endpoint='', + traces_endpoint='', + metric_export_interval=0, + ) + + otel_section = _optional_section(data, 'otel') + service_name = _yaml_or_env_str( + otel_section, + 'service_name', + 'otel.service_name', + env, + 'APP_OTEL_SERVICE_NAME', + ) + logs_endpoint = '' + metrics_endpoint = '' + traces_endpoint = '' + metric_export_interval = 0 + + if enable_logs: + logs_endpoint = _yaml_or_env_str( + otel_section, + 'logs_endpoint', + 'otel.logs_endpoint', + env, + 'APP_OTEL_LOGS_ENDPOINT', + ) + + if enable_metrics: + metrics_endpoint = _yaml_or_env_str( + otel_section, + 'metrics_endpoint', + 'otel.metrics_endpoint', + env, + 'APP_OTEL_METRICS_ENDPOINT', + ) + metric_export_interval = _yaml_or_env_int( + otel_section, + 'metric_export_interval', + 'otel.metric_export_interval', + env, + 'OTEL_METRIC_EXPORT_INTERVAL', + ) + + if enable_tracing: + traces_endpoint = _yaml_or_env_str( + otel_section, + 'traces_endpoint', + 'otel.traces_endpoint', + env, + 'APP_OTEL_TRACES_ENDPOINT', + ) + + return OtelConfig( + service_name=service_name, + logs_endpoint=logs_endpoint, + metrics_endpoint=metrics_endpoint, + traces_endpoint=traces_endpoint, + metric_export_interval=metric_export_interval, + ) + + def _yaml_or_env_str( section: Mapping[str, object], field_name: str, @@ -212,6 +322,35 @@ def _yaml_or_env_int( return _as_int(section[field_name], label) +def _yaml_or_env_bool( + section: Mapping[str, object], + field_name: str, + label: str, + env: Mapping[str, str], + env_name: str | None = None, +) -> bool: + if env_name is not None and env_name in env: + return _as_bool(env[env_name], env_name) + if field_name not in section: + raise ConfigError(f'missing {label}') + return _as_bool(section[field_name], label) + + +def _yaml_or_env_choice( + section: Mapping[str, object], + field_name: str, + label: str, + env: Mapping[str, str], + choices: set[str], + env_name: str | None = None, +) -> str: + value = _yaml_or_env_str(section, field_name, label, env, env_name) + normalized = value.lower() + if normalized not in choices: + raise ConfigError(f'invalid {label}') + return normalized + + def _env_str(env: Mapping[str, str], name: str) -> str: if name not in env: raise ConfigError(f'missing {name}') @@ -244,6 +383,18 @@ def _as_int(value: object, label: str) -> int: raise ConfigError(f'invalid {label}') +def _as_bool(value: object, label: str) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + text = value.strip().lower() + if text == 'true': + return True + if text == 'false': + return False + raise ConfigError(f'invalid {label}') + + def _display_path(path: Path) -> str: if path.is_relative_to(ROOT_DIR): return str(path.relative_to(ROOT_DIR)) diff --git a/adapter/config/model.py b/adapter/config/model.py index c540dee..2e7d74e 100644 --- a/adapter/config/model.py +++ b/adapter/config/model.py @@ -16,6 +16,19 @@ class HttpConfig: @dataclass(frozen=True, slots=True) class LoggingConfig: level: str + output: str + format: str + file_path: str | None + + +@dataclass(frozen=True, slots=True) +class MetricsConfig: + enabled: bool + + +@dataclass(frozen=True, slots=True) +class TracingConfig: + enabled: bool @dataclass(frozen=True, slots=True) @@ -39,5 +52,7 @@ class AppConfig: app: AppSectionConfig http: HttpConfig logging: LoggingConfig + metrics: MetricsConfig + tracing: TracingConfig otel: OtelConfig security: SecurityConfig diff --git a/adapter/otel/bootstrap.py b/adapter/otel/bootstrap.py index 1b5082a..923087f 100644 --- a/adapter/otel/bootstrap.py +++ b/adapter/otel/bootstrap.py @@ -20,21 +20,33 @@ from .tracing import OtelTracer @dataclass(frozen=True, slots=True) class OtelRuntime: - logger_provider: LoggerProvider - meter_provider: MeterProvider - tracer_provider: TracerProvider - logger: OtelLogger - metrics: OtelMetrics - tracer: OtelTracer + logger_provider: LoggerProvider | None = None + meter_provider: MeterProvider | None = None + tracer_provider: TracerProvider | None = None + logger: OtelLogger | None = None + metrics: OtelMetrics | None = None + tracer: OtelTracer | None = None def shutdown(self) -> None: - self.meter_provider.shutdown() - self.tracer_provider.shutdown() - self.logger_provider.shutdown() + if self.meter_provider is not None: + self.meter_provider.shutdown() + if self.tracer_provider is not None: + self.tracer_provider.shutdown() + if self.logger_provider is not None: + self.logger_provider.shutdown() -def setup_otel(config: AppConfig) -> OtelRuntime: - if config.otel.metric_export_interval <= 0: +def setup_otel( + config: AppConfig, + *, + enable_logs: bool, + enable_metrics: bool, + enable_tracing: bool, +) -> OtelRuntime: + if not any((enable_logs, enable_metrics, enable_tracing)): + raise ValueError('empty otel runtime') + + if enable_metrics and config.otel.metric_export_interval <= 0: raise ValueError('invalid metric export interval') resource = Resource.create( @@ -44,30 +56,41 @@ def setup_otel(config: AppConfig) -> OtelRuntime: } ) - logger_provider = LoggerProvider(resource=resource, shutdown_on_exit=False) - logger_provider.add_log_record_processor( - BatchLogRecordProcessor(OTLPLogExporter(endpoint=config.otel.logs_endpoint)) - ) - - metric_reader = PeriodicExportingMetricReader( - OTLPMetricExporter(endpoint=config.otel.metrics_endpoint), - export_interval_millis=config.otel.metric_export_interval, - ) - meter_provider = MeterProvider( - metric_readers=[metric_reader], - resource=resource, - shutdown_on_exit=False, - ) - - tracer_provider = TracerProvider(resource=resource, shutdown_on_exit=False) - tracer_provider.add_span_processor( - BatchSpanProcessor(OTLPSpanExporter(endpoint=config.otel.traces_endpoint)) - ) - scope_name = config.app.name - logger = OtelLogger(logger_provider.get_logger(scope_name), config.logging.level) - metrics_adapter = OtelMetrics(meter_provider.get_meter(scope_name)) - tracer = OtelTracer(tracer_provider.get_tracer(scope_name)) + logger_provider = None + meter_provider = None + tracer_provider = None + logger = None + metrics_adapter = None + tracer = None + + if enable_logs: + logger_provider = LoggerProvider(resource=resource, shutdown_on_exit=False) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter(endpoint=config.otel.logs_endpoint)) + ) + logger = OtelLogger( + logger_provider.get_logger(scope_name), config.logging.level + ) + + if enable_metrics: + metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=config.otel.metrics_endpoint), + export_interval_millis=config.otel.metric_export_interval, + ) + meter_provider = MeterProvider( + metric_readers=[metric_reader], + resource=resource, + shutdown_on_exit=False, + ) + metrics_adapter = OtelMetrics(meter_provider.get_meter(scope_name)) + + if enable_tracing: + tracer_provider = TracerProvider(resource=resource, shutdown_on_exit=False) + tracer_provider.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=config.otel.traces_endpoint)) + ) + tracer = OtelTracer(tracer_provider.get_tracer(scope_name)) return OtelRuntime( logger_provider=logger_provider, diff --git a/config/app.yaml b/config/app.yaml index b5c7241..280019b 100644 --- a/config/app.yaml +++ b/config/app.yaml @@ -8,6 +8,14 @@ http: logging: level: INFO + output: stdout + format: text + +metrics: + enabled: true + +tracing: + enabled: true otel: service_name: web-python-skelet