Implement ingest-service: MQTT -> ClickHouse (batched) + RabbitMQ event

Subscribes to devices/+/telemetry, validates payloads, buffers writes into
ClickHouse (flush on size or interval), and emits a per-reading event on
RabbitMQ for rule-engine-service to consume later. Adds the initial Postgres
and ClickHouse schema migrations, a Dockerfile, and wires the service into
docker-compose. Verified end-to-end via docker compose + mosquitto_pub: row
lands in ClickHouse and the event reaches the telemetry.new_reading queue.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-26 16:52:45 +05:00
co-authored by Claude Sonnet 5
parent 6ce487735f
commit 938700cf86
19 changed files with 990 additions and 2 deletions
@@ -0,0 +1,76 @@
// Package clickhouse writes batches of telemetry readings to ClickHouse.
package clickhouse
import (
"context"
"crypto/tls"
"fmt"
ch "github.com/ClickHouse/clickhouse-go/v2"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
"git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/telemetry"
)
type Config struct {
Addr string
Database string
Username string
Password string
UseTLS bool
}
type Store struct {
conn driver.Conn
}
func Connect(ctx context.Context, cfg Config) (*Store, error) {
opts := &ch.Options{
Addr: []string{cfg.Addr},
Auth: ch.Auth{
Database: cfg.Database,
Username: cfg.Username,
Password: cfg.Password,
},
}
if cfg.UseTLS {
opts.TLS = &tls.Config{}
}
conn, err := ch.Open(opts)
if err != nil {
return nil, fmt.Errorf("open clickhouse connection: %w", err)
}
if err := conn.Ping(ctx); err != nil {
return nil, fmt.Errorf("ping clickhouse: %w", err)
}
return &Store{conn: conn}, nil
}
func (s *Store) Close() error {
return s.conn.Close()
}
// InsertBatch writes readings to the telemetry table in a single INSERT.
func (s *Store) InsertBatch(ctx context.Context, readings []telemetry.Reading) error {
if len(readings) == 0 {
return nil
}
batch, err := s.conn.PrepareBatch(ctx, "INSERT INTO telemetry (device_id, zone_id, sensor_type, value, recorded_at, received_at)")
if err != nil {
return fmt.Errorf("prepare batch: %w", err)
}
defer batch.Close()
for _, r := range readings {
if err := batch.Append(r.DeviceID, r.ZoneID, r.SensorType, r.Value, r.RecordedAt, r.ReceivedAt); err != nil {
return fmt.Errorf("append reading for device %s: %w", r.DeviceID, err)
}
}
if err := batch.Send(); err != nil {
return fmt.Errorf("send batch: %w", err)
}
return nil
}