Implement ingest-service: MQTT -> ClickHouse (batched) + RabbitMQ event
Subscribes to devices/+/telemetry, validates payloads, buffers writes into ClickHouse (flush on size or interval), and emits a per-reading event on RabbitMQ for rule-engine-service to consume later. Adds the initial Postgres and ClickHouse schema migrations, a Dockerfile, and wires the service into docker-compose. Verified end-to-end via docker compose + mosquitto_pub: row lands in ClickHouse and the event reaches the telemetry.new_reading queue. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,76 @@
|
||||
// Package clickhouse writes batches of telemetry readings to ClickHouse.
|
||||
package clickhouse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
|
||||
ch "github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
|
||||
"git.cactoz.su/cacto/home_automatization/services/ingest-service/internal/telemetry"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Addr string
|
||||
Database string
|
||||
Username string
|
||||
Password string
|
||||
UseTLS bool
|
||||
}
|
||||
|
||||
type Store struct {
|
||||
conn driver.Conn
|
||||
}
|
||||
|
||||
func Connect(ctx context.Context, cfg Config) (*Store, error) {
|
||||
opts := &ch.Options{
|
||||
Addr: []string{cfg.Addr},
|
||||
Auth: ch.Auth{
|
||||
Database: cfg.Database,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
},
|
||||
}
|
||||
if cfg.UseTLS {
|
||||
opts.TLS = &tls.Config{}
|
||||
}
|
||||
|
||||
conn, err := ch.Open(opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open clickhouse connection: %w", err)
|
||||
}
|
||||
if err := conn.Ping(ctx); err != nil {
|
||||
return nil, fmt.Errorf("ping clickhouse: %w", err)
|
||||
}
|
||||
return &Store{conn: conn}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Close() error {
|
||||
return s.conn.Close()
|
||||
}
|
||||
|
||||
// InsertBatch writes readings to the telemetry table in a single INSERT.
|
||||
func (s *Store) InsertBatch(ctx context.Context, readings []telemetry.Reading) error {
|
||||
if len(readings) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
batch, err := s.conn.PrepareBatch(ctx, "INSERT INTO telemetry (device_id, zone_id, sensor_type, value, recorded_at, received_at)")
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare batch: %w", err)
|
||||
}
|
||||
defer batch.Close()
|
||||
|
||||
for _, r := range readings {
|
||||
if err := batch.Append(r.DeviceID, r.ZoneID, r.SensorType, r.Value, r.RecordedAt, r.ReceivedAt); err != nil {
|
||||
return fmt.Errorf("append reading for device %s: %w", r.DeviceID, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user