Single endpoint for web+ui (Issue #4). Migrated to modern protobuf+grpc.

Makefile

@@ -23,8 +23,9 @@ daemon-unittests:
 	go tool cover -html=reports/unittests.out -o reports/unittests.html
 
 grpc:
-	protoc -I.:/usr/share/gocode/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis/ --go_out=plugins=grpc:gen/grpc/ OliveTin.proto 
-	protoc -I.:/usr/share/gocode/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis/ --grpc-gateway_out=gen/grpc --grpc-gateway_opt paths=source_relative OliveTin.proto
+	protoc -I.:$(GOPATH)/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis/ --go_out=. --go-grpc_out=. --grpc-gateway_out=. OliveTin.proto 
+#	protoc --go-grpc_out=grpc:gen/grpc/ OliveTin.proto 
+#	protoc -I.:$(GOPATH)/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis/ --grpc-gateway_out=gen/grpc --grpc-gateway_opt paths=source_relative OliveTin.proto
 
 podman-image:
 	buildah bud -t olivetin
@@ -46,6 +47,7 @@ webui-codestyle:
 	cd webui && stylelint style.css
 
 release-common: 
+	rm -rf webui/node_modules/
 	rm -rf releases/
 	mkdir -p releases/common/
 	cp -r webui releases/common/

OliveTin.proto

@@ -1,5 +1,7 @@
 syntax = "proto3";
 
+option go_package = "gen/grpc";
+
 import "google/api/annotations.proto";
 
 message ActionButton {
@@ -28,13 +30,13 @@ message StartActionResponse {
 service OliveTinApi {
 	rpc GetButtons(GetButtonsRequest) returns (GetButtonsResponse) {
 		option (google.api.http) = {
-			get: "/GetButtons"
+			get: "/api/GetButtons"
 		};
 	}
 
 	rpc StartAction(StartActionRequest) returns (StartActionResponse) {
 		option (google.api.http) = {
-			get: "/StartAction"
+			get: "/api/StartAction"
 		};
 	}
 

cmd/OliveTin/main.go

@@ -4,8 +4,8 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	grpcapi "github.com/jamesread/OliveTin/internal/grpcapi"
-	restapi "github.com/jamesread/OliveTin/internal/restapi"
-	webuiServer "github.com/jamesread/OliveTin/internal/webuiServer"
+
+	"github.com/jamesread/OliveTin/internal/httpservers"
 
 	config "github.com/jamesread/OliveTin/internal/config"
 	"github.com/spf13/viper"
@@ -39,22 +39,19 @@ func init() {
 		os.Exit(1)
 	}
 
-	log.SetLevel(cfg.GetLogLevel())
+	if logLevel, err := log.ParseLevel(cfg.LogLevel); err == nil {
+		log.SetLevel(logLevel)
+	}
 
 	viper.WatchConfig()
 }
 
 func main() {
-	log.WithFields(log.Fields{
-		"listenAddressGrpcActions": cfg.ListenAddressGrpcActions,
-		"listenAddressRestActions": cfg.ListenAddressRestActions,
-		"listenAddressWebUI":       cfg.ListenAddressWebUI,
-	}).Info("OliveTin started")
+	log.Info("OliveTin started")
 
 	log.Debugf("%+v", cfg)
 
-	go grpcapi.Start(cfg.ListenAddressGrpcActions, cfg)
-	go restapi.Start(cfg.ListenAddressRestActions, cfg.ListenAddressGrpcActions, cfg)
+	go grpcapi.Start(cfg)
 
-	webuiServer.Start(cfg.ListenAddressWebUI, cfg.ListenAddressRestActions)
+	httpservers.StartServers(cfg)
 }

go.mod

@@ -3,18 +3,14 @@ module github.com/jamesread/OliveTin
 go 1.15
 
 require (
-	github.com/golang/protobuf v1.5.2
-	github.com/grpc-ecosystem/grpc-gateway v1.16.0
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.3.0
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.4.0
 	github.com/sirupsen/logrus v1.8.1
-	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.7.1
-	github.com/stretchr/testify v1.7.0 // indirect
-	golang.org/x/text v0.3.5 // indirect
-	golang.org/x/tools v0.1.1 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/genproto v0.0.0-20210224155714-063164c882e6
+	github.com/stretchr/testify v1.7.0
+	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
+	google.golang.org/genproto v0.0.0-20210520160233-290a1ae68a05
 	google.golang.org/grpc v1.37.0
-	gopkg.in/yaml.v2 v2.3.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.0
+	google.golang.org/protobuf v1.26.0
 )

internal/config/config.go

@@ -1,10 +1,8 @@
 package config
 
-import (
-	log "github.com/sirupsen/logrus"
-	"strings"
-)
+import ()
 
+// ActionButton represents a button that is shown in the webui.
 type ActionButton struct {
 	Title   string
 	Icon    string
@@ -13,6 +11,8 @@ type ActionButton struct {
 	Timeout int
 }
 
+// Entity represents a "thing" that can have multiple actions associated with it.
+// for example, a media player with a start and stop action.
 type Entity struct {
 	Title         string
 	Icon          string
@@ -20,34 +20,28 @@ type Entity struct {
 	CSS           map[string]string
 }
 
+// Config is the global config used through the whole app.
 type Config struct {
-	ListenAddressWebUI       string
-	ListenAddressRestActions string
-	ListenAddressGrpcActions string
-	LogLevel                 string
-	ActionButtons            []ActionButton `mapstructure:"actions"`
-	Entities                 []Entity       `mapstructure:"omitempty"`
+	UseSingleHTTPFrontend           bool
+	ListenAddressSingleHTTPFrontend string
+	ListenAddressWebUI              string
+	ListenAddressRestActions        string
+	ListenAddressGrpcActions        string
+	ExternalRestAddress             string
+	LogLevel                        string
+	ActionButtons                   []ActionButton `mapstructure:"actions"`
+	Entities                        []Entity       `mapstructure:"omitempty"`
 }
 
+// DefaultConfig gets a new Config structure with sensible default values.
 func DefaultConfig() *Config {
 	config := Config{}
-	config.ListenAddressWebUI = "0.0.0.0:1337"
-	config.ListenAddressRestActions = "0.0.0.0:1338"
-	config.ListenAddressGrpcActions = "0.0.0.0:1339"
+	config.UseSingleHTTPFrontend = true
+	config.ListenAddressSingleHTTPFrontend = "0.0.0.0:1337"
+	config.ListenAddressRestActions = "localhost:1338"
+	config.ListenAddressGrpcActions = "localhost:1339"
+	config.ListenAddressWebUI = "localhost:1340"
 	config.LogLevel = "INFO"
 
 	return &config
 }
-
-func (cfg *Config) GetLogLevel() log.Level {
-	switch strings.ToUpper(cfg.LogLevel) {
-	case "INFO":
-		return log.InfoLevel
-	case "WARN":
-		return log.WarnLevel
-	case "DEBUG":
-		return log.DebugLevel
-	default:
-		return log.InfoLevel
-	}
-}

internal/config/config_test.go

@@ -1,27 +1,10 @@
 package config
 
 import (
-	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"testing"
 )
 
-func TestGetLog(t *testing.T) {
-	c := Config{}
-	c.LogLevel = ""
-
-	assert.Equal(t, c.GetLogLevel(), log.InfoLevel, "Info log level should be default")
-
-	c.LogLevel = "INFO"
-	assert.Equal(t, c.GetLogLevel(), log.InfoLevel, "set info log level")
-
-	c.LogLevel = "WARN"
-	assert.Equal(t, c.GetLogLevel(), log.WarnLevel, "set warn log level")
-
-	c.LogLevel = "DEBUG"
-	assert.Equal(t, c.GetLogLevel(), log.DebugLevel, "set debug log level")
-}
-
 func TestCreateDefaultConfig(t *testing.T) {
 	c := DefaultConfig()
 

internal/executor/executor.go

@@ -11,6 +11,7 @@ import (
 	"time"
 )
 
+// ExecAction executes an action.
 func ExecAction(cfg *config.Config, action string) *pb.StartActionResponse {
 	res := &pb.StartActionResponse{}
 	res.TimedOut = false

internal/grpcapi/grpcApi.go

@@ -16,6 +16,7 @@ var (
 )
 
 type oliveTinAPI struct {
+	pb.UnimplementedOliveTinApiServer
 }
 
 func (api *oliveTinAPI) StartAction(ctx ctx.Context, req *pb.StartActionRequest) (*pb.StartActionResponse, error) {
@@ -39,10 +40,11 @@ func (api *oliveTinAPI) GetButtons(ctx ctx.Context, req *pb.GetButtonsRequest) (
 	return res, nil
 }
 
-func Start(listenAddress string, globalConfig *config.Config) {
+// Start will start the GRPC API.
+func Start(globalConfig *config.Config) {
 	cfg = globalConfig
 
-	lis, err := net.Listen("tcp", listenAddress)
+	lis, err := net.Listen("tcp", cfg.ListenAddressGrpcActions)
 
 	if err != nil {
 		log.Fatalf("Failed to listen - %v", err)

internal/grpcapi/grpcApi_test.go

@@ -3,17 +3,17 @@ package grpcapi
 // Thank you: https://stackoverflow.com/questions/42102496/testing-a-grpc-service
 
 import (
-	"net"
 	"context"
 	"github.com/stretchr/testify/assert"
-	"testing"
-	"google.golang.org/grpc/test/bufconn"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/test/bufconn"
+	"net"
+	"testing"
 
 	log "github.com/sirupsen/logrus"
 
-	config "github.com/jamesread/OliveTin/internal/config"
 	pb "github.com/jamesread/OliveTin/gen/grpc"
+	config "github.com/jamesread/OliveTin/internal/config"
 )
 
 const bufSize = 1024 * 1024
@@ -21,47 +21,47 @@ const bufSize = 1024 * 1024
 var lis *bufconn.Listener
 
 func init() {
-    lis = bufconn.Listen(bufSize)
-    s := grpc.NewServer()
-    pb.RegisterOliveTinApiServer(s, newServer())
-
-    go func() {
-        if err := s.Serve(lis); err != nil {
-            log.Fatalf("Server exited with error: %v", err)
-        }
-    }()
+	lis = bufconn.Listen(bufSize)
+	s := grpc.NewServer()
+	pb.RegisterOliveTinApiServer(s, newServer())
+
+	go func() {
+		if err := s.Serve(lis); err != nil {
+			log.Fatalf("Server exited with error: %v", err)
+		}
+	}()
 }
 
 func bufDialer(context.Context, string) (net.Conn, error) {
-    return lis.Dial()
+	return lis.Dial()
 }
 
 func getNewTestServerAndClient(t *testing.T, injectedConfig *config.Config) (*grpc.ClientConn, pb.OliveTinApiClient) {
 	cfg = injectedConfig
 
-    ctx := context.Background()
+	ctx := context.Background()
 
-    conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(bufDialer), grpc.WithInsecure())
+	conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(bufDialer), grpc.WithInsecure())
 
-    if err != nil {
-        t.Fatalf("Failed to dial bufnet: %v", err)
-    }
+	if err != nil {
+		t.Fatalf("Failed to dial bufnet: %v", err)
+	}
 
-    client := pb.NewOliveTinApiClient(conn)
+	client := pb.NewOliveTinApiClient(conn)
 
 	return conn, client
 }
 
 func TestGetButtonsAndStart(t *testing.T) {
-	cfg = config.DefaultConfig();
+	cfg = config.DefaultConfig()
 	btn1 := config.ActionButton{}
 	btn1.Title = "blat"
 	btn1.Shell = "echo 'test'"
-	cfg.ActionButtons = append(cfg.ActionButtons, btn1);
+	cfg.ActionButtons = append(cfg.ActionButtons, btn1)
 
 	conn, client := getNewTestServerAndClient(t, cfg)
 
-    respGb, err := client.GetButtons(context.Background(), &pb.GetButtonsRequest{})
+	respGb, err := client.GetButtons(context.Background(), &pb.GetButtonsRequest{})
 
 	if err != nil {
 		t.Errorf("GetButtons: %v", err)
@@ -71,12 +71,12 @@ func TestGetButtonsAndStart(t *testing.T) {
 
 	assert.Equal(t, 1, len(respGb.Actions), "Got 1 action button back")
 
-    log.Printf("Response: %+v", respGb)
+	log.Printf("Response: %+v", respGb)
 
 	respSa, err := client.StartAction(context.Background(), &pb.StartActionRequest{ActionName: "blat"})
 
 	assert.Nil(t, err, "Empty err after start action")
 	assert.NotNil(t, respSa, "Empty err after start action")
 
-    defer conn.Close()
+	defer conn.Close()
 }

internal/httpservers/httpServer.go

@@ -0,0 +1,16 @@
+package httpservers
+
+import (
+	config "github.com/jamesread/OliveTin/internal/config"
+)
+
+// StartServers will start 3 HTTP servers. The WebUI, the Rest API, and a proxy
+// for both of them.
+func StartServers(cfg *config.Config) {
+	go startWebUIServer(cfg)
+	go startRestAPIServer(cfg)
+
+	if cfg.UseSingleHTTPFrontend {
+		StartSingleHTTPFrontend(cfg)
+	}
+}

internal/httpservers/restapi.go

@@ -0,0 +1,54 @@
+package httpservers
+
+import (
+	"context"
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
+	log "github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
+	"google.golang.org/protobuf/encoding/protojson"
+	"net/http"
+
+	gw "github.com/jamesread/OliveTin/gen/grpc"
+
+	cors "github.com/jamesread/OliveTin/internal/cors"
+
+	config "github.com/jamesread/OliveTin/internal/config"
+)
+
+var (
+	cfg *config.Config
+)
+
+func startRestAPIServer(globalConfig *config.Config) error {
+	cfg = globalConfig
+
+	log.WithFields(log.Fields{
+		"address": cfg.ListenAddressGrpcActions,
+	}).Info("Starting REST API")
+
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	// The JSONPb.EmitDefaults is necssary, so "empty" fields are returned in JSON.
+	//mux := runtime.NewServeMux(runtime.WithMarshalerOption(runtime.MIMEWildcard, &runtime.JSONPb{OrigName: true, EmitDefaults: true}))
+	mux := runtime.NewServeMux(
+		runtime.WithMarshalerOption(runtime.MIMEWildcard, &runtime.HTTPBodyMarshaler{
+			Marshaler: &runtime.JSONPb{
+				MarshalOptions: protojson.MarshalOptions{
+					UseProtoNames:   true,
+					EmitUnpopulated: true,
+				},
+			},
+		}),
+	)
+	opts := []grpc.DialOption{grpc.WithInsecure()}
+
+	err := gw.RegisterOliveTinApiHandlerFromEndpoint(ctx, mux, cfg.ListenAddressGrpcActions, opts)
+
+	if err != nil {
+		log.Fatalf("gw error %v", err)
+	}
+
+	return http.ListenAndServe(cfg.ListenAddressRestActions, cors.AllowCors(mux))
+}

internal/httpservers/singleFrontend.go

@@ -0,0 +1,50 @@
+package httpservers
+
+/*
+This file implements a very simple, lightweight reverse proxy so that REST and
+the webui can be accessed from a single endpoint.
+
+This makes external reverse proxies (treafik, haproxy, etc) easier, CORS goes
+away, and several other issues.
+*/
+
+import (
+	config "github.com/jamesread/OliveTin/internal/config"
+	log "github.com/sirupsen/logrus"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+)
+
+// StartSingleHTTPFrontend will create a reverse proxy that proxies the API
+// and webui internally.
+func StartSingleHTTPFrontend(cfg *config.Config) {
+	log.WithFields(log.Fields{
+		"address": cfg.ListenAddressSingleHTTPFrontend,
+	}).Info("Starting single HTTP frontend")
+
+	apiURL, _ := url.Parse("http://" + cfg.ListenAddressRestActions)
+	apiProxy := httputil.NewSingleHostReverseProxy(apiURL)
+
+	webuiURL, _ := url.Parse("http://" + cfg.ListenAddressWebUI)
+	webuiProxy := httputil.NewSingleHostReverseProxy(webuiURL)
+
+	mux := http.NewServeMux()
+
+	mux.HandleFunc("/api/", func(w http.ResponseWriter, r *http.Request) {
+		log.Debugf("api req: %v", r.URL)
+		apiProxy.ServeHTTP(w, r)
+	})
+
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		log.Debugf("ui req: %v", r.URL)
+		webuiProxy.ServeHTTP(w, r)
+	})
+
+	srv := &http.Server{
+		Addr:    cfg.ListenAddressSingleHTTPFrontend,
+		Handler: mux,
+	}
+
+	log.Fatal(srv.ListenAndServe())
+}

internal/httpservers/webuiServer.go

@@ -0,0 +1,63 @@
+package httpservers
+
+import (
+	"encoding/json"
+	//	cors "github.com/jamesread/OliveTin/internal/cors"
+	log "github.com/sirupsen/logrus"
+	"net/http"
+	"os"
+
+	config "github.com/jamesread/OliveTin/internal/config"
+)
+
+type webUISettings struct {
+	Rest string
+}
+
+func findWebuiDir() string {
+	directoriesToSearch := []string{
+		"./webui",
+		"/var/www/olivetin/",
+	}
+
+	for _, dir := range directoriesToSearch {
+		if _, err := os.Stat(dir); !os.IsNotExist(err) {
+			log.Infof("Found the webui directory here: %v", dir)
+
+			return dir
+		}
+	}
+
+	log.Warnf("Did not find the webui directory, you will probably get 404 errors.")
+
+	return "./webui" // Should not exist
+}
+
+func startWebUIServer(cfg *config.Config) {
+	log.WithFields(log.Fields{
+		"address": cfg.ListenAddressWebUI,
+	}).Info("Starting WebUI server")
+
+	mux := http.NewServeMux()
+	mux.Handle("/", http.FileServer(http.Dir(findWebuiDir())))
+	mux.HandleFunc("/webUiSettings.json", func(w http.ResponseWriter, r *http.Request) {
+		restAddress := ""
+
+		if !cfg.UseSingleHTTPFrontend {
+			restAddress = cfg.ExternalRestAddress
+		}
+
+		jsonRet, _ := json.Marshal(webUISettings{
+			Rest: restAddress + "/api/",
+		})
+
+		w.Write([]byte(jsonRet))
+	})
+
+	srv := &http.Server{
+		Addr:    cfg.ListenAddressWebUI,
+		Handler: mux,
+	}
+
+	log.Fatal(srv.ListenAndServe())
+}

internal/webuiServer/webuiServer_test.go → internal/httpservers/webuiServer_test.go

@@ -1,4 +1,4 @@
-package webuiServer
+package httpservers
 
 import (
 	"github.com/stretchr/testify/assert"

internal/restapi/restapi.go

@@ -1,39 +0,0 @@
-package restapi
-
-import (
-	"context"
-	"github.com/grpc-ecosystem/grpc-gateway/runtime"
-	log "github.com/sirupsen/logrus"
-	"google.golang.org/grpc"
-	"net/http"
-
-	gw "github.com/jamesread/OliveTin/gen/grpc"
-
-	cors "github.com/jamesread/OliveTin/internal/cors"
-
-	config "github.com/jamesread/OliveTin/internal/config"
-)
-
-var (
-	cfg *config.Config
-)
-
-func Start(listenAddressRest string, listenAddressGrpc string, globalConfig *config.Config) error {
-	cfg = globalConfig
-
-	ctx := context.Background()
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	// The JSONPb.EmitDefaults is necssary, so "empty" fields are returned in JSON.
-	mux := runtime.NewServeMux(runtime.WithMarshalerOption(runtime.MIMEWildcard, &runtime.JSONPb{OrigName: true, EmitDefaults: true}))
-	opts := []grpc.DialOption{grpc.WithInsecure()}
-
-	err := gw.RegisterOliveTinApiHandlerFromEndpoint(ctx, mux, listenAddressGrpc, opts)
-
-	if err != nil {
-		log.Fatalf("gw error %v", err)
-	}
-
-	return http.ListenAndServe(listenAddressRest, cors.AllowCors(mux))
-}

internal/webuiServer/webuiServer.go

@@ -1,48 +0,0 @@
-package webuiServer
-
-import (
-	"encoding/json"
-	cors "github.com/jamesread/OliveTin/internal/cors"
-	log "github.com/sirupsen/logrus"
-	"net/http"
-	"os"
-)
-
-type WebUISettings struct {
-	Rest string
-}
-
-func findWebuiDir() string {
-	directoriesToSearch := []string{
-		"./webui",
-		"/var/www/olivetin/",
-	}
-
-	for _, dir := range directoriesToSearch {
-		if _, err := os.Stat(dir); !os.IsNotExist(err) {
-			log.Infof("Found the webui directory here: %v", dir)
-
-			return dir
-		}
-	}
-
-	log.Warnf("Did not find the webui directory, you will probably get 404 errors.")
-
-	return "./webui" // Should not exist
-}
-
-func Start(listenAddress string, listenAddressRest string) {
-	http.Handle("/", cors.AllowCors(http.FileServer(http.Dir(findWebuiDir()))))
-
-	http.HandleFunc("/webUiSettings.json", func(w http.ResponseWriter, r *http.Request) {
-		ret := WebUISettings{
-			Rest: "http://" + listenAddressRest + "/",
-		}
-
-		jsonRet, _ := json.Marshal(ret)
-
-		w.Write([]byte(jsonRet))
-	})
-
-	log.Fatal(http.ListenAndServe(listenAddress, nil))
-}