[feat] add observability config
This commit is contained in:
Azamat
parent
a930185754
commit
cb4560db5c
4 changed files with 268 additions and 71 deletions
|
|
@ -10,8 +10,10 @@ from .model import (
|
|||
AppSectionConfig,
|
||||
HttpConfig,
|
||||
LoggingConfig,
|
||||
MetricsConfig,
|
||||
OtelConfig,
|
||||
SecurityConfig,
|
||||
TracingConfig,
|
||||
)
|
||||
|
||||
ROOT_DIR = Path(__file__).resolve().parents[2]
|
||||
|
|
@ -34,9 +36,44 @@ def load_config(
|
|||
app_section = _section(yaml_data, 'app')
|
||||
http_section = _section(yaml_data, 'http')
|
||||
logging_section = _section(yaml_data, 'logging')
|
||||
otel_section = _section(yaml_data, 'otel')
|
||||
metrics_section = _section(yaml_data, 'metrics')
|
||||
tracing_section = _section(yaml_data, 'tracing')
|
||||
security_section = _section(yaml_data, 'security')
|
||||
|
||||
logging_output = _yaml_or_env_choice(
|
||||
logging_section,
|
||||
'output',
|
||||
'logging.output',
|
||||
env_values,
|
||||
{'stdout', 'file', 'otel'},
|
||||
'APP_LOGGING_OUTPUT',
|
||||
)
|
||||
|
||||
logging_format = _yaml_or_env_choice(
|
||||
logging_section,
|
||||
'format',
|
||||
'logging.format',
|
||||
env_values,
|
||||
{'text', 'json'},
|
||||
'APP_LOGGING_FORMAT',
|
||||
)
|
||||
|
||||
metrics_enabled = _yaml_or_env_bool(
|
||||
metrics_section,
|
||||
'enabled',
|
||||
'metrics.enabled',
|
||||
env_values,
|
||||
'APP_METRICS_ENABLED',
|
||||
)
|
||||
|
||||
tracing_enabled = _yaml_or_env_bool(
|
||||
tracing_section,
|
||||
'enabled',
|
||||
'tracing.enabled',
|
||||
env_values,
|
||||
'APP_TRACING_ENABLED',
|
||||
)
|
||||
|
||||
return AppConfig(
|
||||
app=AppSectionConfig(
|
||||
name=_yaml_or_env_str(
|
||||
|
|
@ -68,43 +105,28 @@ def load_config(
|
|||
env_values,
|
||||
'APP_LOGGING_LEVEL',
|
||||
),
|
||||
output=logging_output,
|
||||
format=logging_format,
|
||||
file_path=(
|
||||
None
|
||||
if logging_output != 'file'
|
||||
else _yaml_or_env_str(
|
||||
logging_section,
|
||||
'file_path',
|
||||
'logging.file_path',
|
||||
env_values,
|
||||
'APP_LOGGING_FILE_PATH',
|
||||
)
|
||||
),
|
||||
),
|
||||
otel=OtelConfig(
|
||||
service_name=_yaml_or_env_str(
|
||||
otel_section,
|
||||
'service_name',
|
||||
'otel.service_name',
|
||||
env_values,
|
||||
'APP_OTEL_SERVICE_NAME',
|
||||
),
|
||||
logs_endpoint=_yaml_or_env_str(
|
||||
otel_section,
|
||||
'logs_endpoint',
|
||||
'otel.logs_endpoint',
|
||||
env_values,
|
||||
'APP_OTEL_LOGS_ENDPOINT',
|
||||
),
|
||||
metrics_endpoint=_yaml_or_env_str(
|
||||
otel_section,
|
||||
'metrics_endpoint',
|
||||
'otel.metrics_endpoint',
|
||||
env_values,
|
||||
'APP_OTEL_METRICS_ENDPOINT',
|
||||
),
|
||||
traces_endpoint=_yaml_or_env_str(
|
||||
otel_section,
|
||||
'traces_endpoint',
|
||||
'otel.traces_endpoint',
|
||||
env_values,
|
||||
'APP_OTEL_TRACES_ENDPOINT',
|
||||
),
|
||||
metric_export_interval=_yaml_or_env_int(
|
||||
otel_section,
|
||||
'metric_export_interval',
|
||||
'otel.metric_export_interval',
|
||||
env_values,
|
||||
'OTEL_METRIC_EXPORT_INTERVAL',
|
||||
),
|
||||
metrics=MetricsConfig(enabled=metrics_enabled),
|
||||
tracing=TracingConfig(enabled=tracing_enabled),
|
||||
otel=_load_otel_config(
|
||||
yaml_data,
|
||||
env_values,
|
||||
enable_logs=logging_output == 'otel',
|
||||
enable_metrics=metrics_enabled,
|
||||
enable_tracing=tracing_enabled,
|
||||
),
|
||||
security=SecurityConfig(
|
||||
token_header=_yaml_or_env_str(
|
||||
|
|
@ -184,6 +206,94 @@ def _section(data: Mapping[str, object], name: str) -> dict[str, object]:
|
|||
return section
|
||||
|
||||
|
||||
def _optional_section(data: Mapping[str, object], name: str) -> dict[str, object]:
|
||||
value = data.get(name)
|
||||
if value is None:
|
||||
return {}
|
||||
if not isinstance(value, dict):
|
||||
raise ConfigError(f'invalid {name}')
|
||||
|
||||
section: dict[str, object] = {}
|
||||
for key, item in value.items():
|
||||
if not isinstance(key, str):
|
||||
raise ConfigError(f'invalid {name}')
|
||||
section[key] = item
|
||||
return section
|
||||
|
||||
|
||||
def _load_otel_config(
|
||||
data: Mapping[str, object],
|
||||
env: Mapping[str, str],
|
||||
*,
|
||||
enable_logs: bool,
|
||||
enable_metrics: bool,
|
||||
enable_tracing: bool,
|
||||
) -> OtelConfig:
|
||||
if not any((enable_logs, enable_metrics, enable_tracing)):
|
||||
return OtelConfig(
|
||||
service_name='',
|
||||
logs_endpoint='',
|
||||
metrics_endpoint='',
|
||||
traces_endpoint='',
|
||||
metric_export_interval=0,
|
||||
)
|
||||
|
||||
otel_section = _optional_section(data, 'otel')
|
||||
service_name = _yaml_or_env_str(
|
||||
otel_section,
|
||||
'service_name',
|
||||
'otel.service_name',
|
||||
env,
|
||||
'APP_OTEL_SERVICE_NAME',
|
||||
)
|
||||
logs_endpoint = ''
|
||||
metrics_endpoint = ''
|
||||
traces_endpoint = ''
|
||||
metric_export_interval = 0
|
||||
|
||||
if enable_logs:
|
||||
logs_endpoint = _yaml_or_env_str(
|
||||
otel_section,
|
||||
'logs_endpoint',
|
||||
'otel.logs_endpoint',
|
||||
env,
|
||||
'APP_OTEL_LOGS_ENDPOINT',
|
||||
)
|
||||
|
||||
if enable_metrics:
|
||||
metrics_endpoint = _yaml_or_env_str(
|
||||
otel_section,
|
||||
'metrics_endpoint',
|
||||
'otel.metrics_endpoint',
|
||||
env,
|
||||
'APP_OTEL_METRICS_ENDPOINT',
|
||||
)
|
||||
metric_export_interval = _yaml_or_env_int(
|
||||
otel_section,
|
||||
'metric_export_interval',
|
||||
'otel.metric_export_interval',
|
||||
env,
|
||||
'OTEL_METRIC_EXPORT_INTERVAL',
|
||||
)
|
||||
|
||||
if enable_tracing:
|
||||
traces_endpoint = _yaml_or_env_str(
|
||||
otel_section,
|
||||
'traces_endpoint',
|
||||
'otel.traces_endpoint',
|
||||
env,
|
||||
'APP_OTEL_TRACES_ENDPOINT',
|
||||
)
|
||||
|
||||
return OtelConfig(
|
||||
service_name=service_name,
|
||||
logs_endpoint=logs_endpoint,
|
||||
metrics_endpoint=metrics_endpoint,
|
||||
traces_endpoint=traces_endpoint,
|
||||
metric_export_interval=metric_export_interval,
|
||||
)
|
||||
|
||||
|
||||
def _yaml_or_env_str(
|
||||
section: Mapping[str, object],
|
||||
field_name: str,
|
||||
|
|
@ -212,6 +322,35 @@ def _yaml_or_env_int(
|
|||
return _as_int(section[field_name], label)
|
||||
|
||||
|
||||
def _yaml_or_env_bool(
|
||||
section: Mapping[str, object],
|
||||
field_name: str,
|
||||
label: str,
|
||||
env: Mapping[str, str],
|
||||
env_name: str | None = None,
|
||||
) -> bool:
|
||||
if env_name is not None and env_name in env:
|
||||
return _as_bool(env[env_name], env_name)
|
||||
if field_name not in section:
|
||||
raise ConfigError(f'missing {label}')
|
||||
return _as_bool(section[field_name], label)
|
||||
|
||||
|
||||
def _yaml_or_env_choice(
|
||||
section: Mapping[str, object],
|
||||
field_name: str,
|
||||
label: str,
|
||||
env: Mapping[str, str],
|
||||
choices: set[str],
|
||||
env_name: str | None = None,
|
||||
) -> str:
|
||||
value = _yaml_or_env_str(section, field_name, label, env, env_name)
|
||||
normalized = value.lower()
|
||||
if normalized not in choices:
|
||||
raise ConfigError(f'invalid {label}')
|
||||
return normalized
|
||||
|
||||
|
||||
def _env_str(env: Mapping[str, str], name: str) -> str:
|
||||
if name not in env:
|
||||
raise ConfigError(f'missing {name}')
|
||||
|
|
@ -244,6 +383,18 @@ def _as_int(value: object, label: str) -> int:
|
|||
raise ConfigError(f'invalid {label}')
|
||||
|
||||
|
||||
def _as_bool(value: object, label: str) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
text = value.strip().lower()
|
||||
if text == 'true':
|
||||
return True
|
||||
if text == 'false':
|
||||
return False
|
||||
raise ConfigError(f'invalid {label}')
|
||||
|
||||
|
||||
def _display_path(path: Path) -> str:
|
||||
if path.is_relative_to(ROOT_DIR):
|
||||
return str(path.relative_to(ROOT_DIR))
|
||||
|
|
|
|||
|
|
@ -16,6 +16,19 @@ class HttpConfig:
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class LoggingConfig:
|
||||
level: str
|
||||
output: str
|
||||
format: str
|
||||
file_path: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MetricsConfig:
|
||||
enabled: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TracingConfig:
|
||||
enabled: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -39,5 +52,7 @@ class AppConfig:
|
|||
app: AppSectionConfig
|
||||
http: HttpConfig
|
||||
logging: LoggingConfig
|
||||
metrics: MetricsConfig
|
||||
tracing: TracingConfig
|
||||
otel: OtelConfig
|
||||
security: SecurityConfig
|
||||
|
|
|
|||
|
|
@ -20,21 +20,33 @@ from .tracing import OtelTracer
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OtelRuntime:
|
||||
logger_provider: LoggerProvider
|
||||
meter_provider: MeterProvider
|
||||
tracer_provider: TracerProvider
|
||||
logger: OtelLogger
|
||||
metrics: OtelMetrics
|
||||
tracer: OtelTracer
|
||||
logger_provider: LoggerProvider | None = None
|
||||
meter_provider: MeterProvider | None = None
|
||||
tracer_provider: TracerProvider | None = None
|
||||
logger: OtelLogger | None = None
|
||||
metrics: OtelMetrics | None = None
|
||||
tracer: OtelTracer | None = None
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.meter_provider.shutdown()
|
||||
self.tracer_provider.shutdown()
|
||||
self.logger_provider.shutdown()
|
||||
if self.meter_provider is not None:
|
||||
self.meter_provider.shutdown()
|
||||
if self.tracer_provider is not None:
|
||||
self.tracer_provider.shutdown()
|
||||
if self.logger_provider is not None:
|
||||
self.logger_provider.shutdown()
|
||||
|
||||
|
||||
def setup_otel(config: AppConfig) -> OtelRuntime:
|
||||
if config.otel.metric_export_interval <= 0:
|
||||
def setup_otel(
|
||||
config: AppConfig,
|
||||
*,
|
||||
enable_logs: bool,
|
||||
enable_metrics: bool,
|
||||
enable_tracing: bool,
|
||||
) -> OtelRuntime:
|
||||
if not any((enable_logs, enable_metrics, enable_tracing)):
|
||||
raise ValueError('empty otel runtime')
|
||||
|
||||
if enable_metrics and config.otel.metric_export_interval <= 0:
|
||||
raise ValueError('invalid metric export interval')
|
||||
|
||||
resource = Resource.create(
|
||||
|
|
@ -44,30 +56,41 @@ def setup_otel(config: AppConfig) -> OtelRuntime:
|
|||
}
|
||||
)
|
||||
|
||||
logger_provider = LoggerProvider(resource=resource, shutdown_on_exit=False)
|
||||
logger_provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(OTLPLogExporter(endpoint=config.otel.logs_endpoint))
|
||||
)
|
||||
|
||||
metric_reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=config.otel.metrics_endpoint),
|
||||
export_interval_millis=config.otel.metric_export_interval,
|
||||
)
|
||||
meter_provider = MeterProvider(
|
||||
metric_readers=[metric_reader],
|
||||
resource=resource,
|
||||
shutdown_on_exit=False,
|
||||
)
|
||||
|
||||
tracer_provider = TracerProvider(resource=resource, shutdown_on_exit=False)
|
||||
tracer_provider.add_span_processor(
|
||||
BatchSpanProcessor(OTLPSpanExporter(endpoint=config.otel.traces_endpoint))
|
||||
)
|
||||
|
||||
scope_name = config.app.name
|
||||
logger = OtelLogger(logger_provider.get_logger(scope_name), config.logging.level)
|
||||
metrics_adapter = OtelMetrics(meter_provider.get_meter(scope_name))
|
||||
tracer = OtelTracer(tracer_provider.get_tracer(scope_name))
|
||||
logger_provider = None
|
||||
meter_provider = None
|
||||
tracer_provider = None
|
||||
logger = None
|
||||
metrics_adapter = None
|
||||
tracer = None
|
||||
|
||||
if enable_logs:
|
||||
logger_provider = LoggerProvider(resource=resource, shutdown_on_exit=False)
|
||||
logger_provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(OTLPLogExporter(endpoint=config.otel.logs_endpoint))
|
||||
)
|
||||
logger = OtelLogger(
|
||||
logger_provider.get_logger(scope_name), config.logging.level
|
||||
)
|
||||
|
||||
if enable_metrics:
|
||||
metric_reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=config.otel.metrics_endpoint),
|
||||
export_interval_millis=config.otel.metric_export_interval,
|
||||
)
|
||||
meter_provider = MeterProvider(
|
||||
metric_readers=[metric_reader],
|
||||
resource=resource,
|
||||
shutdown_on_exit=False,
|
||||
)
|
||||
metrics_adapter = OtelMetrics(meter_provider.get_meter(scope_name))
|
||||
|
||||
if enable_tracing:
|
||||
tracer_provider = TracerProvider(resource=resource, shutdown_on_exit=False)
|
||||
tracer_provider.add_span_processor(
|
||||
BatchSpanProcessor(OTLPSpanExporter(endpoint=config.otel.traces_endpoint))
|
||||
)
|
||||
tracer = OtelTracer(tracer_provider.get_tracer(scope_name))
|
||||
|
||||
return OtelRuntime(
|
||||
logger_provider=logger_provider,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,14 @@ http:
|
|||
|
||||
logging:
|
||||
level: INFO
|
||||
output: stdout
|
||||
format: text
|
||||
|
||||
metrics:
|
||||
enabled: true
|
||||
|
||||
tracing:
|
||||
enabled: true
|
||||
|
||||
otel:
|
||||
service_name: web-python-skelet
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue