Implement ingest-service: MQTT -> ClickHouse (batched) + RabbitMQ event
Subscribes to devices/+/telemetry, validates payloads, buffers writes into ClickHouse (flush on size or interval), and emits a per-reading event on RabbitMQ for rule-engine-service to consume later. Adds the initial Postgres and ClickHouse schema migrations, a Dockerfile, and wires the service into docker-compose. Verified end-to-end via docker compose + mosquitto_pub: row lands in ClickHouse and the event reaches the telemetry.new_reading queue. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
// Package telemetry parses and validates raw device telemetry payloads.
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Reading is a single validated sensor reading, ready to be stored in
|
||||
// ClickHouse and published as an event for rule-engine-service.
|
||||
type Reading struct {
|
||||
DeviceID string
|
||||
ZoneID string
|
||||
SensorType string
|
||||
Value float64
|
||||
RecordedAt time.Time
|
||||
ReceivedAt time.Time
|
||||
}
|
||||
|
||||
// payload mirrors the wire format devices publish to devices/{device_id}/telemetry.
|
||||
//
|
||||
// ZoneID travels with the payload (rather than being looked up from
|
||||
// PostgreSQL) so ingest-service stays decoupled from the device registry,
|
||||
// matching the ingest→ClickHouse/RabbitMQ-only data flow in the platform
|
||||
// architecture — the publishing client is responsible for knowing its own zone.
|
||||
type payload struct {
|
||||
DeviceID string `json:"device_id"`
|
||||
ZoneID string `json:"zone_id"`
|
||||
SensorType string `json:"sensor_type"`
|
||||
Value float64 `json:"value"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
}
|
||||
|
||||
// ParseReading decodes and validates a raw MQTT message body. receivedAt is
|
||||
// the time the server received the message, stamped independently of
|
||||
// whatever the device claims, so ingest/processing delay can be measured later.
|
||||
func ParseReading(raw []byte, receivedAt time.Time) (Reading, error) {
|
||||
var p payload
|
||||
if err := json.Unmarshal(raw, &p); err != nil {
|
||||
return Reading{}, fmt.Errorf("invalid json: %w", err)
|
||||
}
|
||||
|
||||
if p.DeviceID == "" {
|
||||
return Reading{}, fmt.Errorf("device_id is required")
|
||||
}
|
||||
if p.ZoneID == "" {
|
||||
return Reading{}, fmt.Errorf("zone_id is required")
|
||||
}
|
||||
if p.SensorType == "" {
|
||||
return Reading{}, fmt.Errorf("sensor_type is required")
|
||||
}
|
||||
if p.Timestamp == "" {
|
||||
return Reading{}, fmt.Errorf("timestamp is required")
|
||||
}
|
||||
recordedAt, err := time.Parse(time.RFC3339, p.Timestamp)
|
||||
if err != nil {
|
||||
return Reading{}, fmt.Errorf("invalid timestamp %q: %w", p.Timestamp, err)
|
||||
}
|
||||
|
||||
return Reading{
|
||||
DeviceID: p.DeviceID,
|
||||
ZoneID: p.ZoneID,
|
||||
SensorType: p.SensorType,
|
||||
Value: p.Value,
|
||||
RecordedAt: recordedAt,
|
||||
ReceivedAt: receivedAt,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestParseReading_Valid(t *testing.T) {
|
||||
receivedAt := time.Date(2026, 7, 26, 12, 0, 5, 0, time.UTC)
|
||||
raw := []byte(`{
|
||||
"device_id": "esp32-abc123",
|
||||
"zone_id": "growbox-1",
|
||||
"sensor_type": "temperature",
|
||||
"value": 24.5,
|
||||
"timestamp": "2026-07-26T12:00:00Z"
|
||||
}`)
|
||||
|
||||
got, err := ParseReading(raw, receivedAt)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
want := Reading{
|
||||
DeviceID: "esp32-abc123",
|
||||
ZoneID: "growbox-1",
|
||||
SensorType: "temperature",
|
||||
Value: 24.5,
|
||||
RecordedAt: time.Date(2026, 7, 26, 12, 0, 0, 0, time.UTC),
|
||||
ReceivedAt: receivedAt,
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("got %+v, want %+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReading_Invalid(t *testing.T) {
|
||||
receivedAt := time.Now()
|
||||
|
||||
cases := map[string]string{
|
||||
"malformed json": `{not json`,
|
||||
"missing device_id": `{
|
||||
"zone_id": "growbox-1", "sensor_type": "temperature",
|
||||
"value": 1, "timestamp": "2026-07-26T12:00:00Z"
|
||||
}`,
|
||||
"missing zone_id": `{
|
||||
"device_id": "esp32-1", "sensor_type": "temperature",
|
||||
"value": 1, "timestamp": "2026-07-26T12:00:00Z"
|
||||
}`,
|
||||
"missing sensor_type": `{
|
||||
"device_id": "esp32-1", "zone_id": "growbox-1",
|
||||
"value": 1, "timestamp": "2026-07-26T12:00:00Z"
|
||||
}`,
|
||||
"missing timestamp": `{
|
||||
"device_id": "esp32-1", "zone_id": "growbox-1",
|
||||
"sensor_type": "temperature", "value": 1
|
||||
}`,
|
||||
"invalid timestamp format": `{
|
||||
"device_id": "esp32-1", "zone_id": "growbox-1",
|
||||
"sensor_type": "temperature", "value": 1, "timestamp": "not-a-date"
|
||||
}`,
|
||||
"value wrong type": `{
|
||||
"device_id": "esp32-1", "zone_id": "growbox-1",
|
||||
"sensor_type": "temperature", "value": "not-a-number", "timestamp": "2026-07-26T12:00:00Z"
|
||||
}`,
|
||||
}
|
||||
|
||||
for name, raw := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := ParseReading([]byte(raw), receivedAt); err == nil {
|
||||
t.Fatalf("expected error, got nil")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user