Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,8 @@ By default, readings are published at `/mbmd/<unique id>/<reading>`. Rate limiti

There is also the option to directly insert the data into an influxdb database by using the command-line options available. InfluxDB 1.8 and 2.0 are currently supported. to enable this, add the `--influx-database` and the `--influx-url` commandline parameter. More advanced configuration is available, to learn more checkout the [mbmd_run.md](docs/mbmd_run.md) documentation

If InfluxDB is unavailable, mbmd keeps unsent data in memory and writes it once the database is back. The default is 1 hour, configurable with `--influx-buffer` (`0` disables it). Buffered data is lost if mbmd restarts.

# Supported Devices

`mbmd` supports a range of DIN rail meters and grid inverters.
Expand Down
1 change: 1 addition & 0 deletions cmd/confighandler.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ type InfluxConfig struct {
Token string
User string
Password string
Buffer time.Duration
}

// AdapterConfig describes device communication parameters
Expand Down
8 changes: 7 additions & 1 deletion cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,11 @@ any type is considered valid.
"",
"InfluxDB password (optional)",
)
runCmd.PersistentFlags().Duration(
"influx-buffer",
time.Hour,
"Keep unsent data in memory for this long while InfluxDB is unavailable. Set 0 to disable.",
)

pflags := runCmd.PersistentFlags()

Expand All @@ -166,7 +171,7 @@ any type is considered valid.
bindPFlagsWithPrefix(pflags, "mqtt", "broker", "topic", "user", "password", "clientid", "qos", "retain")

// influx
bindPFlagsWithPrefix(pflags, "influx", "url", "database", "measurement", "organization", "token", "user", "password")
bindPFlagsWithPrefix(pflags, "influx", "url", "database", "measurement", "organization", "token", "user", "password", "buffer")
}

// checkVersion validates if updates are available
Expand Down Expand Up @@ -329,6 +334,7 @@ func run(cmd *cobra.Command, args []string) {
viper.GetString("influx.token"),
viper.GetString("influx.user"),
viper.GetString("influx.password"),
viper.GetDuration("influx.buffer"),
)

tee.AttachRunner(server.NewSnipRunner(influx.Run))
Expand Down
1 change: 1 addition & 0 deletions docs/mbmd_run.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ mbmd run [flags]
If the adapter is a TCP connection (identified by :port), the device type (SUNS) is ignored and
any type is considered valid.
Example: -d SDM:1@/dev/USB11 -d SMA:126@localhost:502
--influx-buffer duration Keep unsent data in memory for this long while InfluxDB is unavailable. Set 0 to disable. (default 1h0m0s)
--influx-database string InfluxDB database
--influx-measurement string InfluxDB measurement (default "data")
--influx-organization string InfluxDB organization
Expand Down
2 changes: 2 additions & 0 deletions mbmd.dist.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ influx:
measurement: mbmd
user:
password:
buffer: 1h # keep unsent data in memory while influx is down, 0 to disable

# influxdb_v2 config
influx:
Expand All @@ -26,6 +27,7 @@ influx:
measurement: mbmd
organization:
token:
buffer: 1h # keep unsent data in memory while influx is down, 0 to disable

# adapters are referenced by device
adapters:
Expand Down
46 changes: 45 additions & 1 deletion server/influx.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,17 @@ import (
api "github.com/influxdata/influxdb-client-go/v2/api"
)

const (
// influxFlushInterval is how often points are sent to the database.
// Each flush creates one batch in the client's retry queue.
influxFlushInterval = time.Second
// influxBatchSize is the maximum number of points per write request
influxBatchSize = 5_000
// influxMaxRetryInterval caps the backoff between retries, so data
// is written soon after the database becomes available again.
influxMaxRetryInterval = 30 * time.Second
)

// Influx is an InfluxDB v2 publisher
type Influx struct {
client influxdb.Client
Expand All @@ -25,13 +36,14 @@ func NewInfluxClient(
token string,
user string,
password string,
buffer time.Duration,
) *Influx {
// InfluxDB v1 compatibility
if token == "" && user != "" {
token = fmt.Sprintf("%s:%s", user, password)
}

client := influxdb.NewClient(url, token)
client := influxdb.NewClientWithOptions(url, token, influxOptions(buffer))

if database == "" {
log.Fatal("influx: missing database")
Expand All @@ -47,6 +59,38 @@ func NewInfluxClient(
}
}

// influxOptions configures the client's in-memory retry buffer so that
// points are kept for the given duration while the database is unreachable.
//
// The client's defaults drop data almost immediately: the retry buffer
// limit is divided by the batch size (50,000 / 5,000 = 10 batches), and with
// one batch per flush interval that only holds ~10s of data. In addition,
// batches expire after 3 minutes and are discarded after 5 retries.
//
// A buffer of 0 disables retrying, so failed writes are dropped.
func influxOptions(buffer time.Duration) *influxdb.Options {
opts := influxdb.DefaultOptions().
SetFlushInterval(uint(influxFlushInterval.Milliseconds())).
SetBatchSize(influxBatchSize)

if buffer <= 0 {
return opts.SetMaxRetries(0)
}

// one batch is created per flush interval
batches := uint(buffer/influxFlushInterval) + 1

return opts.
// the limit is given in points but enforced in batches (limit / batch size)
SetRetryBufferLimit(batches * influxBatchSize).
// batches are dropped once older than this
SetMaxRetryTime(uint(buffer.Milliseconds())).
SetMaxRetryInterval(uint(influxMaxRetryInterval.Milliseconds())).
// retries are bounded by MaxRetryTime, not by count. Retry intervals
// are at least 5s, so this can not be reached before batches expire.
SetMaxRetries(uint(buffer / time.Second))
}

// Run Influx publisher
func (m *Influx) Run(in <-chan QuerySnip) {
// log errors
Expand Down
111 changes: 111 additions & 0 deletions server/influx_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package server

import (
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"

"github.com/volkszaehler/mbmd/meters"
)

func TestInfluxOptions(t *testing.T) {
opts := influxOptions(time.Hour)

if batches := opts.WriteOptions().RetryBufferLimit() / opts.WriteOptions().BatchSize(); batches < 3600 {
t.Errorf("retry buffer holds %d batches, want >= 3600", batches)
}
if got := opts.WriteOptions().MaxRetryTime(); got != uint(time.Hour.Milliseconds()) {
t.Errorf("max retry time %d, want %d", got, time.Hour.Milliseconds())
}
if opts.WriteOptions().MaxRetries() == 0 {
t.Error("retries disabled")
}

if got := influxOptions(0).WriteOptions().MaxRetries(); got != 0 {
t.Errorf("buffer 0: max retries %d, want 0", got)
}
}

// fakeInflux counts written lines and fails all writes while down is set
type fakeInflux struct {
down atomic.Bool
mu sync.Mutex
lines int
}

func (f *fakeInflux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if f.down.Load() {
w.WriteHeader(http.StatusServiceUnavailable)
return
}
body, _ := io.ReadAll(r.Body)
f.mu.Lock()
f.lines += len(strings.Split(strings.TrimSpace(string(body)), "\n"))
f.mu.Unlock()
w.WriteHeader(http.StatusNoContent)
}

func (f *fakeInflux) count() int {
f.mu.Lock()
defer f.mu.Unlock()
return f.lines
}

// TestInfluxOutage verifies that points written while the database is down
// are delivered once it is back.
func TestInfluxOutage(t *testing.T) {
if testing.Short() {
t.Skip("slow")
}

fake := &fakeInflux{}
srv := httptest.NewServer(fake)
defer srv.Close()

influx := NewInfluxClient(srv.URL, "db", "mbmd", "org", "", "", "", time.Hour)

in := make(chan QuerySnip)
done := make(chan struct{})
go func() {
influx.Run(in)
close(done)
}()

// outage of 20s, longer than the ~10s the client's default buffer covers
fake.down.Store(true)

const points = 100
for i := range points {
in <- QuerySnip{
Device: "test",
MeasurementResult: meters.MeasurementResult{Measurement: meters.Power, Value: float64(i)},
}
time.Sleep(200 * time.Millisecond)
}

fake.down.Store(false)

// keep data flowing, retries are triggered by new writes
deadline := time.Now().Add(time.Minute)
sent := points
for fake.count() < points && time.Now().Before(deadline) {
in <- QuerySnip{
Device: "test",
MeasurementResult: meters.MeasurementResult{Measurement: meters.Power, Value: -1},
}
sent++
time.Sleep(time.Second)
}

close(in)
<-done

if got := fake.count(); got != sent {
t.Errorf("received %d of %d points", got, sent)
}
}
Loading