Implement ingest-service: MQTT -> ClickHouse (batched) + RabbitMQ event

Subscribes to devices/+/telemetry, validates payloads, buffers writes into ClickHouse (flush on size or interval), and emits a per-reading event on RabbitMQ for rule-engine-service to consume later. Adds the initial Postgres and ClickHouse schema migrations, a Dockerfile, and wires the service into docker-compose. Verified end-to-end via docker compose + mosquitto_pub: row lands in ClickHouse and the event reaches the telemetry.new_reading queue. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-26 16:52:45 +05:00
parent 6ce487735f
commit 938700cf86
19 changed files with 990 additions and 2 deletions
@@ -0,0 +1,118 @@
+// Command ingest-service subscribes to device telemetry over MQTT, batches
+// it into ClickHouse, and publishes a "new reading" event per message for
+// rule-engine-service to consume.
+package main
+
+import (
+	"context"
+	"log/slog"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/batch"
+	chstore "git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/clickhouse"
+	"git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/config"
+	"git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/mqttclient"
+	"git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/rabbitmq"
+	"git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/telemetry"
+)
+
+// incomingBufferSize bounds how many parsed readings can wait for the worker
+// goroutine before the MQTT callback starts dropping messages. It exists so
+// a slow ClickHouse/RabbitMQ round-trip can't stall MQTT message delivery.
+const incomingBufferSize = 1000
+
+func main() {
+	logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
+
+	if err := run(logger); err != nil {
+		logger.Error("ingest-service exited with error", "error", err)
+		os.Exit(1)
+	}
+}
+
+func run(logger *slog.Logger) error {
+	cfg, err := config.Load()
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	store, err := chstore.Connect(ctx, chstore.Config{
+		Addr:     cfg.ClickHouseAddr,
+		Database: cfg.ClickHouseDatabase,
+		Username: cfg.ClickHouseUsername,
+		Password: cfg.ClickHousePassword,
+	})
+	cancel()
+	if err != nil {
+		return err
+	}
+	defer store.Close()
+
+	publisher, err := rabbitmq.Connect(cfg.RabbitMQURL)
+	if err != nil {
+		return err
+	}
+	defer publisher.Close()
+
+	batcher := batch.New(cfg.BatchMaxSize, cfg.BatchFlushInterval, cfg.BatchFlushTimeout, store.InsertBatch, logger)
+	batcher.Start()
+	defer batcher.Stop()
+
+	incoming := make(chan telemetry.Reading, incomingBufferSize)
+
+	var workerWG sync.WaitGroup
+	workerWG.Add(1)
+	go func() {
+		defer workerWG.Done()
+		for reading := range incoming {
+			batcher.Add(reading)
+
+			pubCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			err := publisher.PublishReading(pubCtx, reading)
+			cancel()
+			if err != nil {
+				logger.Error("publish reading event failed", "device_id", reading.DeviceID, "error", err)
+			}
+		}
+	}()
+
+	subscriber, err := mqttclient.Connect(mqttclient.Config{
+		BrokerURL: cfg.MQTTBrokerURL,
+		ClientID:  cfg.MQTTClientID,
+		Topic:     cfg.MQTTTopic,
+		QoS:       1,
+	}, func(payload []byte, receivedAt time.Time) {
+		reading, err := telemetry.ParseReading(payload, receivedAt)
+		if err != nil {
+			logger.Warn("dropping invalid telemetry payload", "error", err)
+			return
+		}
+
+		select {
+		case incoming <- reading:
+		default:
+			logger.Error("dropping reading: worker backlog full", "device_id", reading.DeviceID)
+		}
+	}, logger)
+	if err != nil {
+		return err
+	}
+
+	logger.Info("ingest-service started", "mqtt_topic", cfg.MQTTTopic)
+
+	stop := make(chan os.Signal, 1)
+	signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
+	<-stop
+
+	logger.Info("shutting down")
+	subscriber.Close()
+	close(incoming)
+	workerWG.Wait()
+
+	return nil
+}