Implement ingest-service: MQTT -> ClickHouse (batched) + RabbitMQ event

Subscribes to devices/+/telemetry, validates payloads, buffers writes into
ClickHouse (flush on size or interval), and emits a per-reading event on
RabbitMQ for rule-engine-service to consume later. Adds the initial Postgres
and ClickHouse schema migrations, a Dockerfile, and wires the service into
docker-compose. Verified end-to-end via docker compose + mosquitto_pub: row
lands in ClickHouse and the event reaches the telemetry.new_reading queue.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-26 16:52:45 +05:00
co-authored by Claude Sonnet 5
parent 6ce487735f
commit 938700cf86
19 changed files with 990 additions and 2 deletions
@@ -0,0 +1,69 @@
// Package telemetry parses and validates raw device telemetry payloads.
package telemetry
import (
"encoding/json"
"fmt"
"time"
)
// Reading is a single validated sensor reading, ready to be stored in
// ClickHouse and published as an event for rule-engine-service.
type Reading struct {
DeviceID string
ZoneID string
SensorType string
Value float64
RecordedAt time.Time
ReceivedAt time.Time
}
// payload mirrors the wire format devices publish to devices/{device_id}/telemetry.
//
// ZoneID travels with the payload (rather than being looked up from
// PostgreSQL) so ingest-service stays decoupled from the device registry,
// matching the ingest→ClickHouse/RabbitMQ-only data flow in the platform
// architecture — the publishing client is responsible for knowing its own zone.
type payload struct {
DeviceID string `json:"device_id"`
ZoneID string `json:"zone_id"`
SensorType string `json:"sensor_type"`
Value float64 `json:"value"`
Timestamp string `json:"timestamp"`
}
// ParseReading decodes and validates a raw MQTT message body. receivedAt is
// the time the server received the message, stamped independently of
// whatever the device claims, so ingest/processing delay can be measured later.
func ParseReading(raw []byte, receivedAt time.Time) (Reading, error) {
var p payload
if err := json.Unmarshal(raw, &p); err != nil {
return Reading{}, fmt.Errorf("invalid json: %w", err)
}
if p.DeviceID == "" {
return Reading{}, fmt.Errorf("device_id is required")
}
if p.ZoneID == "" {
return Reading{}, fmt.Errorf("zone_id is required")
}
if p.SensorType == "" {
return Reading{}, fmt.Errorf("sensor_type is required")
}
if p.Timestamp == "" {
return Reading{}, fmt.Errorf("timestamp is required")
}
recordedAt, err := time.Parse(time.RFC3339, p.Timestamp)
if err != nil {
return Reading{}, fmt.Errorf("invalid timestamp %q: %w", p.Timestamp, err)
}
return Reading{
DeviceID: p.DeviceID,
ZoneID: p.ZoneID,
SensorType: p.SensorType,
Value: p.Value,
RecordedAt: recordedAt,
ReceivedAt: receivedAt,
}, nil
}
@@ -0,0 +1,74 @@
package telemetry
import (
"testing"
"time"
)
func TestParseReading_Valid(t *testing.T) {
receivedAt := time.Date(2026, 7, 26, 12, 0, 5, 0, time.UTC)
raw := []byte(`{
"device_id": "esp32-abc123",
"zone_id": "growbox-1",
"sensor_type": "temperature",
"value": 24.5,
"timestamp": "2026-07-26T12:00:00Z"
}`)
got, err := ParseReading(raw, receivedAt)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
want := Reading{
DeviceID: "esp32-abc123",
ZoneID: "growbox-1",
SensorType: "temperature",
Value: 24.5,
RecordedAt: time.Date(2026, 7, 26, 12, 0, 0, 0, time.UTC),
ReceivedAt: receivedAt,
}
if got != want {
t.Fatalf("got %+v, want %+v", got, want)
}
}
func TestParseReading_Invalid(t *testing.T) {
receivedAt := time.Now()
cases := map[string]string{
"malformed json": `{not json`,
"missing device_id": `{
"zone_id": "growbox-1", "sensor_type": "temperature",
"value": 1, "timestamp": "2026-07-26T12:00:00Z"
}`,
"missing zone_id": `{
"device_id": "esp32-1", "sensor_type": "temperature",
"value": 1, "timestamp": "2026-07-26T12:00:00Z"
}`,
"missing sensor_type": `{
"device_id": "esp32-1", "zone_id": "growbox-1",
"value": 1, "timestamp": "2026-07-26T12:00:00Z"
}`,
"missing timestamp": `{
"device_id": "esp32-1", "zone_id": "growbox-1",
"sensor_type": "temperature", "value": 1
}`,
"invalid timestamp format": `{
"device_id": "esp32-1", "zone_id": "growbox-1",
"sensor_type": "temperature", "value": 1, "timestamp": "not-a-date"
}`,
"value wrong type": `{
"device_id": "esp32-1", "zone_id": "growbox-1",
"sensor_type": "temperature", "value": "not-a-number", "timestamp": "2026-07-26T12:00:00Z"
}`,
}
for name, raw := range cases {
t.Run(name, func(t *testing.T) {
if _, err := ParseReading([]byte(raw), receivedAt); err == nil {
t.Fatalf("expected error, got nil")
}
})
}
}