OrgGo
/
Mainflux.mainflux
mirror of https://github.com/mainflux/mainflux.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

NOISSUE - Remove twins-service mqtt dependency and publish notifs to nats (#1042) 

* Add Publish func to nats

Signed-off-by: Darko Draskovic <darko.draskovic@gmail.com>

* Remove mqtt client

Signed-off-by: Darko Draskovic <darko.draskovic@gmail.com>

* Add nats publisher

Signed-off-by: Darko Draskovic <darko.draskovic@gmail.com>

* Separate nats publisher from subscriber

Signed-off-by: Darko Draskovic <darko.draskovic@gmail.com>

* Update tests creation methods

Signed-off-by: Darko Draskovic <darko.draskovic@gmail.com>

* Add logger to NATS Publisher

Signed-off-by: Darko Draskovic <darko.draskovic@gmail.com>
This commit is contained in:
Darko Draskovic 2020-02-20 00:10:10 +01:00 committed by GitHub
parent a30a3b9063
commit 3b5d51276f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 167 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
cmd/twins/main.go
View File

@ -15,8 +15,6 @@ import (
"syscall"
"time"
"github.com/mainflux/mainflux/twins/mqtt"
kitprometheus "github.com/go-kit/kit/metrics/prometheus"
"github.com/mainflux/mainflux"
authapi "github.com/mainflux/mainflux/authn/api/grpc"
@ -26,7 +24,8 @@ import (
"github.com/mainflux/mainflux/twins/api"
twapi "github.com/mainflux/mainflux/twins/api/http"
twmongodb "github.com/mainflux/mainflux/twins/mongodb"
twnats "github.com/mainflux/mainflux/twins/nats"
natspub "github.com/mainflux/mainflux/twins/nats/publisher"
natssub "github.com/mainflux/mainflux/twins/nats/subscriber"
"github.com/mainflux/mainflux/twins/uuid"
nats "github.com/nats-io/go-nats"
opentracing "github.com/opentracing/opentracing-go"
@ -51,7 +50,6 @@ const (
defSingleUserToken = ""
defClientTLS = "false"
defCACerts = ""
defMqttURL = "tcp://localhost:1883"
defThingID = ""
defThingKey = ""
defChannelID = ""
@ -72,7 +70,6 @@ const (
envSingleUserToken = "MF_TWINS_SINGLE_USER_TOKEN"
envClientTLS = "MF_TWINS_CLIENT_TLS"
envCACerts = "MF_TWINS_CA_CERTS"
envMqttURL = "MF_TWINS_MQTT_URL"
envThingID = "MF_TWINS_THING_ID"
envThingKey = "MF_TWINS_THING_KEY"
envChannelID = "MF_TWINS_CHANNEL_ID"
@ -93,7 +90,6 @@ type config struct {
singleUserToken string
clientTLS bool
caCerts string
mqttURL string
thingID string
thingKey string
channelID string
@ -125,12 +121,6 @@ func main() {
dbTracer, dbCloser := initJaeger("twins_db", cfg.jaegerURL, logger)
defer dbCloser.Close()
pc := mqtt.Connect(cfg.mqttURL, cfg.thingID, cfg.thingKey, logger)
mc := mqtt.New(pc, cfg.channelID)
mcTracer, mcCloser := initJaeger("twins_mqtt", cfg.jaegerURL, logger)
defer mcCloser.Close()
nc, err := nats.Connect(cfg.NatsURL)
if err != nil {
logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err))
@ -144,7 +134,7 @@ func main() {
tracer, closer := initJaeger("twins", cfg.jaegerURL, logger)
defer closer.Close()
svc := newService(nc, ncTracer, mc, mcTracer, auth, dbTracer, db, logger)
svc := newService(nc, ncTracer, cfg.channelID, auth, dbTracer, db, logger)
errs := make(chan error, 2)
@ -188,7 +178,6 @@ func loadConfig() config {
singleUserToken: mainflux.Env(envSingleUserToken, defSingleUserToken),
clientTLS: tls,
caCerts: mainflux.Env(envCACerts, defCACerts),
mqttURL: mainflux.Env(envMqttURL, defMqttURL),
thingID: mainflux.Env(envThingID, defThingID),
channelID: mainflux.Env(envChannelID, defChannelID),
thingKey: mainflux.Env(envThingKey, defThingKey),
@ -256,13 +245,15 @@ func connectToAuth(cfg config, logger logger.Logger) *grpc.ClientConn {
return conn
}
func newService(nc *nats.Conn, ncTracer opentracing.Tracer, mc mqtt.Mqtt, mcTracer opentracing.Tracer, users mainflux.AuthNServiceClient, dbTracer opentracing.Tracer, db *mongo.Database, logger logger.Logger) twins.Service {
func newService(nc *nats.Conn, ncTracer opentracing.Tracer, chanID string, users mainflux.AuthNServiceClient, dbTracer opentracing.Tracer, db *mongo.Database, logger logger.Logger) twins.Service {
twinRepo := twmongodb.NewTwinRepository(db)
stateRepo := twmongodb.NewStateRepository(db)
idp := uuid.New()
svc := twins.New(nc, mc, users, twinRepo, stateRepo, idp)
np := natspub.NewPublisher(nc, chanID, logger)
svc := twins.New(users, twinRepo, stateRepo, idp, np)
svc = api.LoggingMiddleware(svc, logger)
svc = api.MetricsMiddleware(
svc,
@ -280,7 +271,7 @@ func newService(nc *nats.Conn, ncTracer opentracing.Tracer, mc mqtt.Mqtt, mcTrac
}, []string{"method"}),
)
twnats.Subscribe(nc, mc, svc, logger)
natssub.NewSubscriber(nc, chanID, svc, logger)
return svc
}

13
twins/api/http/endpoint_test.go
View File

@ -16,12 +16,10 @@ import (
"testing"
"time"
mqtt "github.com/eclipse/paho.mqtt.golang"
"github.com/mainflux/mainflux/twins"
httpapi "github.com/mainflux/mainflux/twins/api/http"
"github.com/mainflux/mainflux/twins/mocks"
twmqtt "github.com/mainflux/mainflux/twins/mqtt"
nats "github.com/nats-io/go-nats"
nats "github.com/mainflux/mainflux/twins/nats/publisher"
"github.com/opentracing/opentracing-go/mocktracer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -71,14 +69,7 @@ func newService(tokens map[string]string) twins.Service {
statesRepo := mocks.NewStateRepository()
idp := mocks.NewIdentityProvider()
nc, _ := nats.Connect(natsURL)
opts := mqtt.NewClientOptions()
pc := mqtt.NewClient(opts)
mc := twmqtt.New(pc, topic)
return twins.New(nc, mc, auth, twinsRepo, statesRepo, idp)
return twins.New(auth, twinsRepo, statesRepo, idp, &nats.Publisher{})
}
func newServer(svc twins.Service) *httptest.Server {

84
twins/mqtt/publisher.go
View File

@ -1,84 +0,0 @@
// Copyright (c) Mainflux
// SPDX-License-Identifier: Apache-2.0
package mqtt
import (
"fmt"
"os"
mqtt "github.com/eclipse/paho.mqtt.golang"
"github.com/mainflux/mainflux/logger"
)
// Mqtt stores mqtt client and topic
type Mqtt struct {
client mqtt.Client
channelID string
}
// New instantiates the mqtt service.
func New(mc mqtt.Client, channelID string) Mqtt {
return Mqtt{
client: mc,
channelID: channelID,
}
}
// Connect to MQTT broker
func Connect(mqttURL, id, key string, logger logger.Logger) mqtt.Client {
opts := mqtt.NewClientOptions()
opts.AddBroker(mqttURL)
opts.SetClientID("twins")
opts.SetUsername(id)
opts.SetPassword(key)
opts.SetCleanSession(true)
opts.SetAutoReconnect(true)
opts.SetOnConnectHandler(func(c mqtt.Client) {
logger.Info("Connected to MQTT broker")
})
opts.SetConnectionLostHandler(func(c mqtt.Client, err error) {
logger.Error(fmt.Sprintf("MQTT connection lost: %s", err.Error()))
os.Exit(1)
})
client := mqtt.NewClient(opts)
if token := client.Connect(); token.Wait() && token.Error() != nil {
logger.Error(fmt.Sprintf("Failed to connect to MQTT broker: %s", token.Error()))
os.Exit(1)
}
return client
}
func (m *Mqtt) Channel() string {
return m.channelID
}
func (m *Mqtt) publish(twinID, crudOp string, payload *[]byte) error {
topic := fmt.Sprintf("channels/%s/messages/%s/%s", m.channelID, twinID, crudOp)
if len(twinID) < 1 {
topic = fmt.Sprintf("channels/%s/messages/%s", m.channelID, crudOp)
}
token := m.client.Publish(topic, 0, false, *payload)
token.Wait()
return token.Error()
}
// Publish sends mqtt message to a predefined topic
func (m *Mqtt) Publish(twinID *string, err *error, succOp, failOp string, payload *[]byte) error {
op := succOp
if *err != nil {
op = failOp
esb := []byte((*err).Error())
payload = &esb
}
if err := m.publish(*twinID, op, payload); err != nil {
return err
}
return nil
}

66
twins/nats/publisher/publisher.go Normal file
View File

@ -0,0 +1,66 @@
// Copyright (c) Mainflux
// SPDX-License-Identifier: Apache-2.0
package publisher
import (
"fmt"
log "github.com/mainflux/mainflux/logger"
"github.com/gogo/protobuf/proto"
"github.com/mainflux/mainflux"
"github.com/nats-io/go-nats"
)
const (
prefix = "channel"
publisher = "twins"
)
// Publisher is used to publish twins related notifications
type Publisher struct {
natsClient *nats.Conn
channelID string
logger log.Logger
}
// NewPublisher instances Pubsub strucure
func NewPublisher(nc *nats.Conn, chID string, logger log.Logger) *Publisher {
return &Publisher{
natsClient: nc,
channelID: chID,
logger: logger,
}
}
// Publish sends twins CRUD and state saving related operations
func (p *Publisher) Publish(twinID *string, err *error, succOp, failOp string, payload *[]byte) {
if p.channelID == "" {
return
}
op := succOp
if *err != nil {
op = failOp
esb := []byte((*err).Error())
payload = &esb
}
pl := *payload
if pl == nil {
pl = []byte(fmt.Sprintf("{\"deleted\":\"%s\"}", *twinID))
}
subject := fmt.Sprintf("%s.%s.%s", prefix, p.channelID, op)
mc := mainflux.Message{
Channel: p.channelID,
Subtopic: op,
Payload: pl,
Publisher: publisher,
}
b, _ := proto.Marshal(&mc)
if err := p.natsClient.Publish(subject, []byte(b)); err != nil {
p.logger.Warn(fmt.Sprintf("Failed to publish notification on NATS: %s", err))
}
}

61
twins/nats/subscriber.go
View File

@ -1,61 +0,0 @@
// Copyright (c) Mainflux
// SPDX-License-Identifier: Apache-2.0
package nats
import (
"fmt"
"github.com/gogo/protobuf/proto"
"github.com/mainflux/mainflux"
log "github.com/mainflux/mainflux/logger"
"github.com/mainflux/mainflux/twins"
"github.com/mainflux/mainflux/twins/mqtt"
"github.com/nats-io/go-nats"
)
const (
queue = "twins"
input = "channel.>"
)
var crudOp = map[string]string{
"stateSucc": "state/success",
"stateFail": "state/failure",
}
type pubsub struct {
natsClient *nats.Conn
mqttClient mqtt.Mqtt
logger log.Logger
svc twins.Service
}
// Subscribe to appropriate NATS topic
func Subscribe(nc *nats.Conn, mc mqtt.Mqtt, svc twins.Service, logger log.Logger) {
ps := pubsub{
natsClient: nc,
mqttClient: mc,
logger: logger,
svc: svc,
}
ps.natsClient.QueueSubscribe(input, queue, ps.handleMsg)
}
func (ps *pubsub) handleMsg(m *nats.Msg) {
var msg mainflux.Message
if err := proto.Unmarshal(m.Data, &msg); err != nil {
ps.logger.Warn(fmt.Sprintf("Unmarshalling failed: %s", err))
return
}
if msg.Channel == ps.mqttClient.Channel() {
return
}
if err := ps.svc.SaveStates(&msg); err != nil {
ps.logger.Error(fmt.Sprintf("State save failed: %s", err))
return
}
}

62
twins/nats/subscriber/subscriber.go Normal file
View File

@ -0,0 +1,62 @@
// Copyright (c) Mainflux
// SPDX-License-Identifier: Apache-2.0
package subscriber
import (
"fmt"
"os"
"github.com/gogo/protobuf/proto"
"github.com/mainflux/mainflux"
log "github.com/mainflux/mainflux/logger"
"github.com/mainflux/mainflux/twins"
"github.com/nats-io/go-nats"
)
const (
queue = "twins"
input = "channel.>"
)
// Subscriber is used to intercept messages and save corresponding twin states
type Subscriber struct {
natsClient *nats.Conn
logger log.Logger
svc twins.Service
channelID string
}
// NewSubscriber instances Subscriber strucure and subscribes to appropriate NATS topic
func NewSubscriber(nc *nats.Conn, chID string, svc twins.Service, logger log.Logger) *Subscriber {
s := Subscriber{
natsClient: nc,
logger: logger,
svc: svc,
channelID: chID,
}
if _, err := s.natsClient.QueueSubscribe(input, queue, s.handleMsg); err != nil {
logger.Error(fmt.Sprintf("Failed to subscribe to NATS: %s", err))
os.Exit(1)
}
return &s
}
func (s *Subscriber) handleMsg(m *nats.Msg) {
var msg mainflux.Message
if err := proto.Unmarshal(m.Data, &msg); err != nil {
s.logger.Warn(fmt.Sprintf("Unmarshalling failed: %s", err))
return
}
if msg.Channel == s.channelID {
return
}
if err := s.svc.SaveStates(&msg); err != nil {
s.logger.Error(fmt.Sprintf("State save failed: %s", err))
return
}
}

55
twins/service.go
View File

@ -11,9 +11,8 @@ import (
"time"
"github.com/mainflux/mainflux"
"github.com/mainflux/mainflux/twins/mqtt"
nats "github.com/mainflux/mainflux/twins/nats/publisher"
"github.com/mainflux/senml"
"github.com/nats-io/go-nats"
)
var (
@ -68,43 +67,43 @@ type Service interface {
}
var crudOp = map[string]string{
"createSucc": "create/success",
"createFail": "create/failure",
"updateSucc": "update/success",
"updateFail": "update/failure",
"getSucc": "get/success",
"getFail": "get/failure",
"removeSucc": "remove/success",
"removeFail": "remove/failure",
"createSucc": "create.success",
"createFail": "create.failure",
"updateSucc": "update.success",
"updateFail": "update.failure",
"getSucc": "get.success",
"getFail": "get.failure",
"removeSucc": "remove.success",
"removeFail": "remove.failure",
"stateSucc": "save.success",
"stateFail": "save.failure",
}
type twinsService struct {
natsClient *nats.Conn
mqttClient mqtt.Mqtt
auth mainflux.AuthNServiceClient
twins TwinRepository
states StateRepository
idp IdentityProvider
auth mainflux.AuthNServiceClient
twins TwinRepository
states StateRepository
idp IdentityProvider
nats *nats.Publisher
}
var _ Service = (*twinsService)(nil)
// New instantiates the twins service implementation.
func New(nc *nats.Conn, mc mqtt.Mqtt, auth mainflux.AuthNServiceClient, twins TwinRepository, sr StateRepository, idp IdentityProvider) Service {
func New(auth mainflux.AuthNServiceClient, twins TwinRepository, sr StateRepository, idp IdentityProvider, n *nats.Publisher) Service {
return &twinsService{
natsClient: nc,
mqttClient: mc,
auth: auth,
twins: twins,
states: sr,
idp: idp,
auth: auth,
twins: twins,
states: sr,
idp: idp,
nats: n,
}
}
func (ts *twinsService) AddTwin(ctx context.Context, token string, twin Twin, def Definition) (tw Twin, err error) {
var id string
var b []byte
defer ts.mqttClient.Publish(&id, &err, crudOp["createSucc"], crudOp["createFail"], &b)
defer ts.nats.Publish(&id, &err, crudOp["createSucc"], crudOp["createFail"], &b)
res, err := ts.auth.Identify(ctx, &mainflux.Token{Value: token})
if err != nil {
@ -143,7 +142,7 @@ func (ts *twinsService) AddTwin(ctx context.Context, token string, twin Twin, de
func (ts *twinsService) UpdateTwin(ctx context.Context, token string, twin Twin, def Definition) (err error) {
var b []byte
var id string
defer ts.mqttClient.Publish(&id, &err, crudOp["updateSucc"], crudOp["updateFail"], &b)
defer ts.nats.Publish(&id, &err, crudOp["updateSucc"], crudOp["updateFail"], &b)
_, err = ts.auth.Identify(ctx, &mainflux.Token{Value: token})
if err != nil {
@ -198,7 +197,7 @@ func (ts *twinsService) UpdateTwin(ctx context.Context, token string, twin Twin,
func (ts *twinsService) ViewTwin(ctx context.Context, token, id string) (tw Twin, err error) {
var b []byte
defer ts.mqttClient.Publish(&id, &err, crudOp["getSucc"], crudOp["getFail"], &b)
defer ts.nats.Publish(&id, &err, crudOp["getSucc"], crudOp["getFail"], &b)
_, err = ts.auth.Identify(ctx, &mainflux.Token{Value: token})
if err != nil {
@ -226,7 +225,7 @@ func (ts *twinsService) ViewTwinByThing(ctx context.Context, token, thingid stri
func (ts *twinsService) RemoveTwin(ctx context.Context, token, id string) (err error) {
var b []byte
defer ts.mqttClient.Publish(&id, &err, crudOp["removeSucc"], crudOp["removeFail"], &b)
defer ts.nats.Publish(&id, &err, crudOp["removeSucc"], crudOp["removeFail"], &b)
_, err = ts.auth.Identify(ctx, &mainflux.Token{Value: token})
if err != nil {
@ -276,7 +275,7 @@ func (ts *twinsService) SaveStates(msg *mainflux.Message) error {
func (ts *twinsService) saveState(msg *mainflux.Message, id string) error {
var b []byte
var err error
defer ts.mqttClient.Publish(&id, &err, crudOp["stateSucc"], crudOp["stateFail"], &b)
defer ts.nats.Publish(&id, &err, crudOp["stateSucc"], crudOp["stateFail"], &b)
tw, err := ts.twins.RetrieveByID(context.TODO(), id)
if err != nil {

17
twins/service_test.go
View File

@ -8,14 +8,11 @@ import (
"fmt"
"testing"
mqtt "github.com/eclipse/paho.mqtt.golang"
"github.com/mainflux/mainflux/twins"
"github.com/mainflux/mainflux/twins/mocks"
nats "github.com/nats-io/go-nats"
nats "github.com/mainflux/mainflux/twins/nats/publisher"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
twmqtt "github.com/mainflux/mainflux/twins/mqtt"
)
const (
@ -25,8 +22,6 @@ const (
wrongToken = "wrong-token"
email = "user@example.com"
natsURL = "nats://localhost:4222"
mqttURL = "tcp://localhost:1883"
topic = "topic"
)
func newService(tokens map[string]string) twins.Service {
@ -34,15 +29,7 @@ func newService(tokens map[string]string) twins.Service {
twinsRepo := mocks.NewTwinRepository()
statesRepo := mocks.NewStateRepository()
idp := mocks.NewIdentityProvider()
nc, _ := nats.Connect(natsURL)
opts := mqtt.NewClientOptions()
pc := mqtt.NewClient(opts)
mc := twmqtt.New(pc, topic)
return twins.New(nc, mc, auth, twinsRepo, statesRepo, idp)
return twins.New(auth, twinsRepo, statesRepo, idp, &nats.Publisher{})
}
func TestAddTwin(t *testing.T) {