Добавлены метрики Prometheus и дашборды Grafana (этап 2.5)

Все три Go-сервиса (ingest, device-control, rule-engine) отдают
/metrics в формате Prometheus: счётчики MQTT/RabbitMQ/ClickHouse
операций, гистограммы длительности батч-флашей и диспатча правил,
переходы устройств online/offline. Добавлены сервисы prometheus и
grafana в docker-compose с провижининг конфигом (datasource +
два готовых дашборда: "Ingest & Telemetry" и "Automation & Devices").
This commit is contained in:
2026-08-11 21:58:59 +05:00
parent 3ade5c2512
commit 81040eec62
31 changed files with 712 additions and 53 deletions
@@ -5,6 +5,7 @@ package config
import (
"fmt"
"os"
"strconv"
"time"
)
@@ -16,6 +17,8 @@ type Config struct {
RabbitMQURL string
RuleCacheRefreshInterval time.Duration
MetricsPort int
}
func Load() (Config, error) {
@@ -47,6 +50,10 @@ func Load() (Config, error) {
}
cfg.RuleCacheRefreshInterval = interval
if cfg.MetricsPort, err = getEnvInt("RULE_ENGINE_METRICS_PORT", 9102); err != nil {
return Config{}, err
}
return cfg, nil
}
@@ -57,6 +64,18 @@ func getEnv(key, fallback string) string {
return fallback
}
func getEnvInt(key string, fallback int) (int, error) {
v := os.Getenv(key)
if v == "" {
return fallback, nil
}
n, err := strconv.Atoi(v)
if err != nil {
return 0, fmt.Errorf("%s: %w", key, err)
}
return n, nil
}
func getEnvDuration(key string, fallback time.Duration) (time.Duration, error) {
v := os.Getenv(key)
if v == "" {
@@ -11,6 +11,7 @@ import (
"time"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/devicecontrolclient"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/metrics"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/rules"
)
@@ -57,6 +58,8 @@ func New(ruleSource RuleSource, dispatcher Dispatcher, publisher TriggerPublishe
// the whole reading for. A rule the device itself rejected (bad device_id,
// unsupported action) is terminal and does not cause a retry.
func (e *Engine) HandleReading(ctx context.Context, r Reading) error {
metrics.ReadingsConsumed.Inc()
matched := e.rules.MatchingRules(r.DeviceID, r.SensorType)
var firstTransportErr error
@@ -69,8 +72,11 @@ func (e *Engine) HandleReading(ctx context.Context, r Reading) error {
if !matchesCondition {
continue
}
metrics.RulesMatched.Inc()
start := time.Now()
result, dispatchErr := e.dispatcher.Dispatch(ctx, rule.TargetDeviceID, rule.ActionType, rule.ActionParams)
metrics.DispatchDuration.WithLabelValues(rule.ActionType).Observe(time.Since(start).Seconds())
success := dispatchErr == nil && result.Success
errMsg := result.Error
@@ -78,13 +84,16 @@ func (e *Engine) HandleReading(ctx context.Context, r Reading) error {
case dispatchErr != nil:
errMsg = dispatchErr.Error()
e.logger.Error("dispatch action failed", "rule_id", rule.ID, "target_device_id", rule.TargetDeviceID, "error", dispatchErr)
metrics.RulesTriggered.WithLabelValues(rule.ActionType, "transport_error").Inc()
if firstTransportErr == nil {
firstTransportErr = dispatchErr
}
case !result.Success:
e.logger.Warn("device rejected command", "rule_id", rule.ID, "target_device_id", rule.TargetDeviceID, "error", result.Error)
metrics.RulesTriggered.WithLabelValues(rule.ActionType, "rejected").Inc()
default:
e.logger.Info("rule triggered", "rule_id", rule.ID, "target_device_id", rule.TargetDeviceID, "action_type", rule.ActionType)
metrics.RulesTriggered.WithLabelValues(rule.ActionType, "success").Inc()
}
if pubErr := e.publisher.PublishRuleTriggered(ctx, rule.ID, rule.ZoneID, rule.TargetDeviceID, rule.ActionType, success, errMsg); pubErr != nil {
@@ -7,7 +7,10 @@ import (
"log/slog"
"testing"
"github.com/prometheus/client_golang/prometheus/testutil"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/devicecontrolclient"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/metrics"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/rules"
)
@@ -154,3 +157,30 @@ func TestHandleReading_InvalidOperatorSkipsRuleButContinues(t *testing.T) {
t.Fatalf("got dispatch calls %+v, want only the valid rule dispatched", dispatcher.calls)
}
}
func TestHandleReading_RecordsMetrics(t *testing.T) {
source := &fakeRuleSource{rules: []rules.Rule{
{ID: 1, ConditionOperator: ">", ConditionValue: 28, TargetDeviceID: "fan-1", ActionType: "turn_on"},
}}
dispatcher := &fakeDispatcher{result: devicecontrolclient.Result{Success: true}}
e := New(source, dispatcher, &fakePublisher{}, discardLogger())
triggeredCounter := metrics.RulesTriggered.WithLabelValues("turn_on", "success")
beforeConsumed := testutil.ToFloat64(metrics.ReadingsConsumed)
beforeMatched := testutil.ToFloat64(metrics.RulesMatched)
beforeTriggered := testutil.ToFloat64(triggeredCounter)
if err := e.HandleReading(context.Background(), Reading{Value: 30}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := testutil.ToFloat64(metrics.ReadingsConsumed); got != beforeConsumed+1 {
t.Fatalf("got readings_consumed %v, want %v", got, beforeConsumed+1)
}
if got := testutil.ToFloat64(metrics.RulesMatched); got != beforeMatched+1 {
t.Fatalf("got rules_matched %v, want %v", got, beforeMatched+1)
}
if got := testutil.ToFloat64(triggeredCounter); got != beforeTriggered+1 {
t.Fatalf("got rules_triggered{turn_on,success} %v, want %v", got, beforeTriggered+1)
}
}
@@ -0,0 +1,37 @@
// Package metrics defines rule-engine-service's Prometheus metrics.
// Registered automatically (via promauto) into the default registry on
// import; served by main.go's dedicated metrics HTTP server.
package metrics
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
ReadingsConsumed = promauto.NewCounter(prometheus.CounterOpts{
Name: "rule_engine_readings_consumed_total",
Help: "Total number of telemetry.new_reading events consumed from RabbitMQ.",
})
RulesMatched = promauto.NewCounter(prometheus.CounterOpts{
Name: "rule_engine_rules_matched_total",
Help: "Total number of rule conditions that evaluated true and were dispatched.",
})
RulesTriggered = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "rule_engine_rules_triggered_total",
Help: "Total number of rule dispatch attempts, by action type and outcome.",
}, []string{"action_type", "outcome"})
DispatchDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
Name: "rule_engine_dispatch_duration_seconds",
Help: "Duration of the gRPC/HTTP call to device-control-service, by action type.",
Buckets: prometheus.DefBuckets,
}, []string{"action_type"})
CacheRefreshes = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "rule_engine_cache_refresh_total",
Help: "Total number of automation_rules cache refreshes from PostgreSQL, by outcome.",
}, []string{"outcome"})
)
@@ -10,6 +10,7 @@ import (
"sync"
"time"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/metrics"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/rules"
)
@@ -88,12 +89,14 @@ func (c *Cache) loop() {
func (c *Cache) refresh(ctx context.Context) error {
dbRows, err := c.fetcher.FetchActiveRules(ctx)
if err != nil {
metrics.CacheRefreshes.WithLabelValues("error").Inc()
return err
}
index := buildIndex(dbRows, c.logger)
c.mu.Lock()
c.index = index
c.mu.Unlock()
metrics.CacheRefreshes.WithLabelValues("success").Inc()
return nil
}