cacto/home_automatization
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Добавлены метрики Prometheus и дашборды Grafana (этап 2.5)

Browse Source 
Все три Go-сервиса (ingest, device-control, rule-engine) отдают
/metrics в формате Prometheus: счётчики MQTT/RabbitMQ/ClickHouse
операций, гистограммы длительности батч-флашей и диспатча правил,
переходы устройств online/offline. Добавлены сервисы prometheus и
grafana в docker-compose с провижининг конфигом (datasource +
два готовых дашборда: "Ingest & Telemetry" и "Automation & Devices").
This commit is contained in:
cacto
2026-08-11 21:58:59 +05:00
parent 3ade5c2512
commit 81040eec62
31 changed files with 712 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/ingest-service/README.md
+7
View File
@@ -20,6 +20,13 @@ MQTT/ClickHouse/RabbitMQ, как и на диаграмме архитектур
понадобится lookup через реестр устройств (с кэшированием, как rule-engine
кэширует правила) — это естественное развитие, пока не реализовано.
## Метрики
`GET /metrics` (порт `INGEST_METRICS_PORT`, по умолчанию 9101) — формат
Prometheus. Счётчики принятых/невалидных/отброшенных (backlog переполнен)
MQTT-сообщений, флашей батча в ClickHouse (с latency и размером батча),
публикаций в RabbitMQ.
## Запуск
```bash
services/ingest-service/cmd/ingest-service/main.go
+49 -1
View File
@@ -5,16 +5,21 @@ package main
import (
"context"
"fmt"
"log/slog"
"net/http"
"os"
"os/signal"
"sync"
"syscall"
"time"
"github.com/prometheus/client_golang/prometheus/promhttp"
"git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/batch"
chstore "git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/clickhouse"
"git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/config"
"git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/metrics"
"git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/mqttclient"
"git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/rabbitmq"
"git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/telemetry"
@@ -59,7 +64,7 @@ func run(logger *slog.Logger) error {
}
defer publisher.Close()
batcher := batch.New(cfg.BatchMaxSize, cfg.BatchFlushInterval, cfg.BatchFlushTimeout, store.InsertBatch, logger)
batcher := batch.New(cfg.BatchMaxSize, cfg.BatchFlushInterval, cfg.BatchFlushTimeout, instrumentedInsert(store), logger)
batcher.Start()
defer batcher.Stop()
@@ -77,6 +82,9 @@ func run(logger *slog.Logger) error {
cancel()
if err != nil {
logger.Error("publish reading event failed", "device_id", reading.DeviceID, "error", err)
metrics.RabbitMQPublishes.WithLabelValues("error").Inc()
} else {
metrics.RabbitMQPublishes.WithLabelValues("success").Inc()
}
}
}()
@@ -87,9 +95,12 @@ func run(logger *slog.Logger) error {
Topic: cfg.MQTTTopic,
QoS: 1,
}, func(payload []byte, receivedAt time.Time) {
metrics.MQTTMessagesReceived.Inc()
reading, err := telemetry.ParseReading(payload, receivedAt)
if err != nil {
logger.Warn("dropping invalid telemetry payload", "error", err)
metrics.MQTTMessagesInvalid.Inc()
return
}
@@ -97,12 +108,24 @@ func run(logger *slog.Logger) error {
case incoming <- reading:
default:
logger.Error("dropping reading: worker backlog full", "device_id", reading.DeviceID)
metrics.MQTTMessagesDropped.Inc()
}
}, logger)
if err != nil {
return err
}
metricsServer := &http.Server{
Addr: fmt.Sprintf(":%d", cfg.MetricsPort),
Handler: promhttp.Handler(),
}
go func() {
logger.Info("metrics server started", "metrics_port", cfg.MetricsPort)
if err := metricsServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
logger.Error("metrics server stopped", "error", err)
}
}()
logger.Info("ingest-service started", "mqtt_topic", cfg.MQTTTopic)
stop := make(chan os.Signal, 1)
@@ -114,5 +137,30 @@ func run(logger *slog.Logger) error {
close(incoming)
workerWG.Wait()
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := metricsServer.Shutdown(shutdownCtx); err != nil {
logger.Error("metrics server shutdown failed", "error", err)
}
return nil
}
// instrumentedInsert wraps store.InsertBatch with duration/size/outcome
// metrics, without teaching the batch or clickhouse packages about Prometheus.
func instrumentedInsert(store *chstore.Store) batch.FlushFunc {
return func(ctx context.Context, readings []telemetry.Reading) error {
start := time.Now()
err := store.InsertBatch(ctx, readings)
metrics.BatchFlushDuration.Observe(time.Since(start).Seconds())
metrics.BatchSize.Observe(float64(len(readings)))
if err != nil {
metrics.BatchFlushes.WithLabelValues("error").Inc()
} else {
metrics.BatchFlushes.WithLabelValues("success").Inc()
}
return err
}
}
services/ingest-service/go.mod
+11 -4
View File
@@ -5,26 +5,33 @@ go 1.25.0
require (
github.com/ClickHouse/clickhouse-go/v2 v2.47.0
github.com/eclipse/paho.mqtt.golang v1.5.1
github.com/prometheus/client_golang v1.24.1
github.com/rabbitmq/amqp091-go v1.13.0
)
require (
github.com/ClickHouse/ch-go v0.73.0 // indirect
github.com/andybalholm/brotli v1.2.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/go-faster/city v1.0.1 // indirect
github.com/go-faster/errors v0.7.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/klauspost/compress v1.18.6 // indirect
github.com/klauspost/compress v1.19.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/paulmach/orb v0.13.0 // indirect
github.com/pierrec/lz4/v4 v4.1.27 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect
github.com/segmentio/asm v1.2.1 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
go.opentelemetry.io/otel v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/net v0.56.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.46.0 // indirect
golang.org/x/net v0.57.0 // indirect
golang.org/x/sync v0.21.0 // indirect
golang.org/x/sys v0.47.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
)
services/ingest-service/go.sum
+26 -8
View File
@@ -4,6 +4,8 @@ github.com/ClickHouse/clickhouse-go/v2 v2.47.0 h1:ZDAzrnKSOPTIsm4tdUNfrii2yc8dk4
github.com/ClickHouse/clickhouse-go/v2 v2.47.0/go.mod h1:sPj7C7UYQ2MWHcfX+4eGN6nwnCqwUKfgO6PcwKpd6K8=
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -20,14 +22,26 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/paulmach/orb v0.13.0 h1:r7n7mQGGF+cj/CbcivEj9J3HGK+XR+yXnvzRdq9saIw=
github.com/paulmach/orb v0.13.0/go.mod h1:6scRWINywA2Jf05dcjOfLfxrUIMECvTSG2MVbRLxu/k=
github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk=
github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
github.com/rabbitmq/amqp091-go v1.13.0 h1:L8NA1WtF76C6KA3LAoufjfLgbist/If1UQYcsOjtxXA=
github.com/rabbitmq/amqp091-go v1.13.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0=
@@ -44,14 +58,18 @@ go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
services/ingest-service/internal/config/config.go
+5
View File
@@ -24,6 +24,8 @@ type Config struct {
BatchMaxSize int
BatchFlushInterval time.Duration
BatchFlushTimeout time.Duration
MetricsPort int
}
func Load() (Config, error) {
@@ -55,6 +57,9 @@ func Load() (Config, error) {
if cfg.BatchFlushTimeout, err = getEnvDuration("INGEST_BATCH_FLUSH_TIMEOUT", 10*time.Second); err != nil {
return Config{}, err
}
if cfg.MetricsPort, err = getEnvInt("INGEST_METRICS_PORT", 9101); err != nil {
return Config{}, err
}
return cfg, nil
}
services/ingest-service/internal/metrics/metrics.go
+48
View File
@@ -0,0 +1,48 @@
// Package metrics defines ingest-service's Prometheus metrics. Registered
// automatically (via promauto) into the default registry on import; served
// by main.go's dedicated metrics HTTP server.
package metrics
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
MQTTMessagesReceived = promauto.NewCounter(prometheus.CounterOpts{
Name: "ingest_mqtt_messages_received_total",
Help: "Total number of telemetry messages received from MQTT.",
})
MQTTMessagesInvalid = promauto.NewCounter(prometheus.CounterOpts{
Name: "ingest_mqtt_messages_invalid_total",
Help: "Total number of telemetry messages dropped for failing validation.",
})
MQTTMessagesDropped = promauto.NewCounter(prometheus.CounterOpts{
Name: "ingest_mqtt_messages_dropped_total",
Help: "Total number of readings dropped because the worker backlog was full.",
})
BatchFlushes = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "ingest_batch_flush_total",
Help: "Total number of batch flushes to ClickHouse, by outcome.",
}, []string{"outcome"})
BatchFlushDuration = promauto.NewHistogram(prometheus.HistogramOpts{
Name: "ingest_batch_flush_duration_seconds",
Help: "Duration of ClickHouse batch insert calls.",
Buckets: prometheus.DefBuckets,
})
BatchSize = promauto.NewHistogram(prometheus.HistogramOpts{
Name: "ingest_batch_size",
Help: "Number of readings per flushed batch.",
Buckets: []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000},
})
RabbitMQPublishes = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "ingest_rabbitmq_publish_total",
Help: "Total number of new-reading events published to RabbitMQ, by outcome.",
}, []string{"outcome"})
)