cacto/home_automatization
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Добавлены метрики Prometheus и дашборды Grafana (этап 2.5)

Browse Source 
Все три Go-сервиса (ingest, device-control, rule-engine) отдают
/metrics в формате Prometheus: счётчики MQTT/RabbitMQ/ClickHouse
операций, гистограммы длительности батч-флашей и диспатча правил,
переходы устройств online/offline. Добавлены сервисы prometheus и
grafana в docker-compose с провижининг конфигом (datasource +
два готовых дашборда: "Ingest & Telemetry" и "Automation & Devices").
This commit is contained in:
cacto
2026-08-11 21:58:59 +05:00
parent 3ade5c2512
commit 81040eec62
31 changed files with 712 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/rule-engine-service/README.md
+6
View File
@@ -39,6 +39,12 @@
- **Кэш правил — только для чтения активных правил**, никакой записи назад в
Postgres. CRUD правил — это зона ответственности Laravel (этап 2).
## Метрики
`GET /metrics` (порт `RULE_ENGINE_METRICS_PORT`, по умолчанию 9102) —
счётчики обработанных показаний, сработавших правил (по action_type и
исходу), latency вызова device-control-service, обновлений кэша правил.
## Запуск
```bash
services/rule-engine-service/cmd/rule-engine-service/main.go
+22
View File
@@ -5,12 +5,16 @@ package main
import (
"context"
"fmt"
"log/slog"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/prometheus/client_golang/prometheus/promhttp"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/config"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/devicecontrolclient"
@@ -63,6 +67,17 @@ func run(logger *slog.Logger) error {
eng := engine.New(cache, dcClient, mq, logger)
metricsServer := &http.Server{
Addr: fmt.Sprintf(":%d", cfg.MetricsPort),
Handler: promhttp.Handler(),
}
go func() {
logger.Info("metrics server started", "metrics_port", cfg.MetricsPort)
if err := metricsServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
logger.Error("metrics server stopped", "error", err)
}
}()
logger.Info("rule-engine-service started")
err = mq.ConsumeReadings(ctx, func(ctx context.Context, event rabbitmq.ReadingEvent) rabbitmq.HandleResult {
@@ -84,5 +99,12 @@ func run(logger *slog.Logger) error {
}
logger.Info("shutting down")
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := metricsServer.Shutdown(shutdownCtx); err != nil {
logger.Error("metrics server shutdown failed", "error", err)
}
return nil
}
services/rule-engine-service/go.mod
+12 -4
View File
@@ -5,18 +5,26 @@ go 1.25.0
require (
git.cactoz.su/cacto/home_automatization/proto v0.0.0-00010101000000-000000000000
github.com/jackc/pgx/v5 v5.10.0
github.com/prometheus/client_golang v1.24.1
github.com/rabbitmq/amqp091-go v1.13.0
google.golang.org/grpc v1.82.1
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
golang.org/x/net v0.53.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.43.0 // indirect
golang.org/x/text v0.36.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect
golang.org/x/net v0.57.0 // indirect
golang.org/x/sync v0.22.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.40.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect
google.golang.org/protobuf v1.36.11 // indirect
)
services/rule-engine-service/go.sum
+26 -8
View File
@@ -1,3 +1,5 @@
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -21,8 +23,22 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
github.com/rabbitmq/amqp091-go v1.13.0 h1:L8NA1WtF76C6KA3LAoufjfLgbist/If1UQYcsOjtxXA=
github.com/rabbitmq/amqp091-go v1.13.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -44,14 +60,16 @@ go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw=
services/rule-engine-service/internal/config/config.go
+19
View File
@@ -5,6 +5,7 @@ package config
import (
"fmt"
"os"
"strconv"
"time"
)
@@ -16,6 +17,8 @@ type Config struct {
RabbitMQURL string
RuleCacheRefreshInterval time.Duration
MetricsPort int
}
func Load() (Config, error) {
@@ -47,6 +50,10 @@ func Load() (Config, error) {
}
cfg.RuleCacheRefreshInterval = interval
if cfg.MetricsPort, err = getEnvInt("RULE_ENGINE_METRICS_PORT", 9102); err != nil {
return Config{}, err
}
return cfg, nil
}
@@ -57,6 +64,18 @@ func getEnv(key, fallback string) string {
return fallback
}
func getEnvInt(key string, fallback int) (int, error) {
v := os.Getenv(key)
if v == "" {
return fallback, nil
}
n, err := strconv.Atoi(v)
if err != nil {
return 0, fmt.Errorf("%s: %w", key, err)
}
return n, nil
}
func getEnvDuration(key string, fallback time.Duration) (time.Duration, error) {
v := os.Getenv(key)
if v == "" {
services/rule-engine-service/internal/engine/engine.go
+9
View File
@@ -11,6 +11,7 @@ import (
"time"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/devicecontrolclient"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/metrics"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/rules"
)
@@ -57,6 +58,8 @@ func New(ruleSource RuleSource, dispatcher Dispatcher, publisher TriggerPublishe
// the whole reading for. A rule the device itself rejected (bad device_id,
// unsupported action) is terminal and does not cause a retry.
func (e *Engine) HandleReading(ctx context.Context, r Reading) error {
metrics.ReadingsConsumed.Inc()
matched := e.rules.MatchingRules(r.DeviceID, r.SensorType)
var firstTransportErr error
@@ -69,8 +72,11 @@ func (e *Engine) HandleReading(ctx context.Context, r Reading) error {
if !matchesCondition {
continue
}
metrics.RulesMatched.Inc()
start := time.Now()
result, dispatchErr := e.dispatcher.Dispatch(ctx, rule.TargetDeviceID, rule.ActionType, rule.ActionParams)
metrics.DispatchDuration.WithLabelValues(rule.ActionType).Observe(time.Since(start).Seconds())
success := dispatchErr == nil && result.Success
errMsg := result.Error
@@ -78,13 +84,16 @@ func (e *Engine) HandleReading(ctx context.Context, r Reading) error {
case dispatchErr != nil:
errMsg = dispatchErr.Error()
e.logger.Error("dispatch action failed", "rule_id", rule.ID, "target_device_id", rule.TargetDeviceID, "error", dispatchErr)
metrics.RulesTriggered.WithLabelValues(rule.ActionType, "transport_error").Inc()
if firstTransportErr == nil {
firstTransportErr = dispatchErr
}
case !result.Success:
e.logger.Warn("device rejected command", "rule_id", rule.ID, "target_device_id", rule.TargetDeviceID, "error", result.Error)
metrics.RulesTriggered.WithLabelValues(rule.ActionType, "rejected").Inc()
default:
e.logger.Info("rule triggered", "rule_id", rule.ID, "target_device_id", rule.TargetDeviceID, "action_type", rule.ActionType)
metrics.RulesTriggered.WithLabelValues(rule.ActionType, "success").Inc()
}
if pubErr := e.publisher.PublishRuleTriggered(ctx, rule.ID, rule.ZoneID, rule.TargetDeviceID, rule.ActionType, success, errMsg); pubErr != nil {
services/rule-engine-service/internal/engine/engine_test.go
+30
View File
@@ -7,7 +7,10 @@ import (
"log/slog"
"testing"
"github.com/prometheus/client_golang/prometheus/testutil"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/devicecontrolclient"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/metrics"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/rules"
)
@@ -154,3 +157,30 @@ func TestHandleReading_InvalidOperatorSkipsRuleButContinues(t *testing.T) {
t.Fatalf("got dispatch calls %+v, want only the valid rule dispatched", dispatcher.calls)
}
}
func TestHandleReading_RecordsMetrics(t *testing.T) {
source := &fakeRuleSource{rules: []rules.Rule{
{ID: 1, ConditionOperator: ">", ConditionValue: 28, TargetDeviceID: "fan-1", ActionType: "turn_on"},
}}
dispatcher := &fakeDispatcher{result: devicecontrolclient.Result{Success: true}}
e := New(source, dispatcher, &fakePublisher{}, discardLogger())
triggeredCounter := metrics.RulesTriggered.WithLabelValues("turn_on", "success")
beforeConsumed := testutil.ToFloat64(metrics.ReadingsConsumed)
beforeMatched := testutil.ToFloat64(metrics.RulesMatched)
beforeTriggered := testutil.ToFloat64(triggeredCounter)
if err := e.HandleReading(context.Background(), Reading{Value: 30}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := testutil.ToFloat64(metrics.ReadingsConsumed); got != beforeConsumed+1 {
t.Fatalf("got readings_consumed %v, want %v", got, beforeConsumed+1)
}
if got := testutil.ToFloat64(metrics.RulesMatched); got != beforeMatched+1 {
t.Fatalf("got rules_matched %v, want %v", got, beforeMatched+1)
}
if got := testutil.ToFloat64(triggeredCounter); got != beforeTriggered+1 {
t.Fatalf("got rules_triggered{turn_on,success} %v, want %v", got, beforeTriggered+1)
}
}
services/rule-engine-service/internal/metrics/metrics.go
+37
View File
@@ -0,0 +1,37 @@
// Package metrics defines rule-engine-service's Prometheus metrics.
// Registered automatically (via promauto) into the default registry on
// import; served by main.go's dedicated metrics HTTP server.
package metrics
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
ReadingsConsumed = promauto.NewCounter(prometheus.CounterOpts{
Name: "rule_engine_readings_consumed_total",
Help: "Total number of telemetry.new_reading events consumed from RabbitMQ.",
})
RulesMatched = promauto.NewCounter(prometheus.CounterOpts{
Name: "rule_engine_rules_matched_total",
Help: "Total number of rule conditions that evaluated true and were dispatched.",
})
RulesTriggered = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "rule_engine_rules_triggered_total",
Help: "Total number of rule dispatch attempts, by action type and outcome.",
}, []string{"action_type", "outcome"})
DispatchDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
Name: "rule_engine_dispatch_duration_seconds",
Help: "Duration of the gRPC/HTTP call to device-control-service, by action type.",
Buckets: prometheus.DefBuckets,
}, []string{"action_type"})
CacheRefreshes = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "rule_engine_cache_refresh_total",
Help: "Total number of automation_rules cache refreshes from PostgreSQL, by outcome.",
}, []string{"outcome"})
)
services/rule-engine-service/internal/rulecache/cache.go
+3
View File
@@ -10,6 +10,7 @@ import (
"sync"
"time"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/metrics"
"git.cactoz.su/cacto/home_automatization/services/rule-engine-service/internal/rules"
)
@@ -88,12 +89,14 @@ func (c *Cache) loop() {
func (c *Cache) refresh(ctx context.Context) error {
dbRows, err := c.fetcher.FetchActiveRules(ctx)
if err != nil {
metrics.CacheRefreshes.WithLabelValues("error").Inc()
return err
}
index := buildIndex(dbRows, c.logger)
c.mu.Lock()
c.index = index
c.mu.Unlock()
metrics.CacheRefreshes.WithLabelValues("success").Inc()
return nil
}