Implement ingest-service: MQTT -> ClickHouse (batched) + RabbitMQ event

Subscribes to devices/+/telemetry, validates payloads, buffers writes into
ClickHouse (flush on size or interval), and emits a per-reading event on
RabbitMQ for rule-engine-service to consume later. Adds the initial Postgres
and ClickHouse schema migrations, a Dockerfile, and wires the service into
docker-compose. Verified end-to-end via docker compose + mosquitto_pub: row
lands in ClickHouse and the event reaches the telemetry.new_reading queue.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

services/ingest-service/internal/batch/batcher.go

@@ -0,0 +1,105 @@
+// Package batch accumulates telemetry readings and flushes them in bulk,
+// either once a size threshold is hit or on a fixed interval — so ClickHouse
+// gets bulk inserts instead of one round-trip per reading.
+package batch
+
+import (
+	"context"
+	"log/slog"
+	"sync"
+	"time"
+
+	"git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/telemetry"
+)
+
+// FlushFunc persists one batch. It is called with a context bounded by
+// FlushTimeout, both for interval-triggered and shutdown-triggered flushes.
+type FlushFunc func(ctx context.Context, readings []telemetry.Reading) error
+
+// Batcher buffers readings in memory and flushes them via FlushFunc when
+// MaxSize is reached or Interval elapses, whichever comes first.
+type Batcher struct {
+	maxSize      int
+	interval     time.Duration
+	flushTimeout time.Duration
+	flush        FlushFunc
+	logger       *slog.Logger
+
+	mu  sync.Mutex
+	buf []telemetry.Reading
+
+	stop chan struct{}
+	done chan struct{}
+}
+
+func New(maxSize int, interval, flushTimeout time.Duration, flush FlushFunc, logger *slog.Logger) *Batcher {
+	return &Batcher{
+		maxSize:      maxSize,
+		interval:     interval,
+		flushTimeout: flushTimeout,
+		flush:        flush,
+		logger:       logger,
+	}
+}
+
+// Start begins the interval-flush loop. Call once before the first Add.
+func (b *Batcher) Start() {
+	b.stop = make(chan struct{})
+	b.done = make(chan struct{})
+	go b.loop()
+}
+
+func (b *Batcher) loop() {
+	defer close(b.done)
+	ticker := time.NewTicker(b.interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			b.flushNow("interval")
+		case <-b.stop:
+			b.flushNow("shutdown")
+			return
+		}
+	}
+}
+
+// Add appends a reading, flushing immediately if the buffer just reached
+// maxSize rather than waiting for the next tick.
+func (b *Batcher) Add(r telemetry.Reading) {
+	b.mu.Lock()
+	b.buf = append(b.buf, r)
+	full := len(b.buf) >= b.maxSize
+	b.mu.Unlock()
+
+	if full {
+		b.flushNow("size")
+	}
+}
+
+// Stop flushes any remaining buffered readings and waits for the flush loop
+// to exit. Safe to call once, after Start.
+func (b *Batcher) Stop() {
+	close(b.stop)
+	<-b.done
+}
+
+func (b *Batcher) flushNow(reason string) {
+	b.mu.Lock()
+	if len(b.buf) == 0 {
+		b.mu.Unlock()
+		return
+	}
+	readings := b.buf
+	b.buf = nil
+	b.mu.Unlock()
+
+	ctx, cancel := context.WithTimeout(context.Background(), b.flushTimeout)
+	defer cancel()
+
+	if err := b.flush(ctx, readings); err != nil {
+		b.logger.Error("batch flush failed", "reason", reason, "size", len(readings), "error", err)
+		return
+	}
+	b.logger.Debug("batch flushed", "reason", reason, "size", len(readings))
+}

services/ingest-service/internal/batch/batcher_test.go

@@ -0,0 +1,84 @@
+package batch
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"testing"
+	"time"
+
+	"git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/telemetry"
+)
+
+func discardLogger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(io.Discard, nil))
+}
+
+func reading(deviceID string) telemetry.Reading {
+	return telemetry.Reading{DeviceID: deviceID, ZoneID: "z1", SensorType: "temperature", Value: 1}
+}
+
+func awaitBatch(t *testing.T, flushed chan []telemetry.Reading, timeout time.Duration) []telemetry.Reading {
+	t.Helper()
+	select {
+	case batch := <-flushed:
+		return batch
+	case <-time.After(timeout):
+		t.Fatal("timed out waiting for flush")
+		return nil
+	}
+}
+
+func TestBatcher_FlushesOnSize(t *testing.T) {
+	flushed := make(chan []telemetry.Reading, 1)
+	b := New(3, time.Hour, time.Second, func(_ context.Context, r []telemetry.Reading) error {
+		flushed <- r
+		return nil
+	}, discardLogger())
+	b.Start()
+	defer b.Stop()
+
+	b.Add(reading("a"))
+	b.Add(reading("b"))
+	b.Add(reading("c")) // hits maxSize, should trigger an immediate flush
+
+	batch := awaitBatch(t, flushed, time.Second)
+	if len(batch) != 3 {
+		t.Fatalf("got %d readings, want 3", len(batch))
+	}
+}
+
+func TestBatcher_FlushesOnInterval(t *testing.T) {
+	flushed := make(chan []telemetry.Reading, 1)
+	b := New(100, 20*time.Millisecond, time.Second, func(_ context.Context, r []telemetry.Reading) error {
+		flushed <- r
+		return nil
+	}, discardLogger())
+	b.Start()
+	defer b.Stop()
+
+	b.Add(reading("a"))
+
+	batch := awaitBatch(t, flushed, time.Second)
+	if len(batch) != 1 {
+		t.Fatalf("got %d readings, want 1", len(batch))
+	}
+}
+
+func TestBatcher_StopFlushesRemainder(t *testing.T) {
+	flushed := make(chan []telemetry.Reading, 1)
+	b := New(100, time.Hour, time.Second, func(_ context.Context, r []telemetry.Reading) error {
+		flushed <- r
+		return nil
+	}, discardLogger())
+	b.Start()
+
+	b.Add(reading("a"))
+	b.Add(reading("b"))
+	b.Stop()
+
+	batch := awaitBatch(t, flushed, time.Second)
+	if len(batch) != 2 {
+		t.Fatalf("got %d readings, want 2", len(batch))
+	}
+}