Read from input + use go 1.19 + add prometheus (#36)

Author: sbs2001

.github/workflows/build-binary-package.yml

@@ -10,10 +10,10 @@ jobs:
     name: Build and upload binary package
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Go 1.13
+    - name: Set up Go 1.19
       uses: actions/setup-go@v1
       with:
-        go-version: 1.13
+        go-version: 1.19
       id: go
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2

.github/workflows/go.yml

@@ -23,8 +23,3 @@ jobs:
       run: make build
     - name: Test
       run: go test -v
-
-      
-
-      
-  

config.go

@@ -13,19 +13,33 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+type PrometheusConfig struct {
+	Enabled       bool   `yaml:"enabled"`
+	ListenAddress string `yaml:"listen_addr"`
+	ListenPort    string `yaml:"listen_port"`
+}
+
 type bouncerConfig struct {
-	BinPath                string        `yaml:"bin_path"` // path to binary
-	PidDir                 string        `yaml:"piddir"`
-	UpdateFrequency        string        `yaml:"update_frequency"`
-	Daemon                 bool          `yaml:"daemonize"`
-	LogMode                string        `yaml:"log_mode"`
-	LogDir                 string        `yaml:"log_dir"`
-	LogLevel               log.Level     `yaml:"log_level"`
-	CompressLogs           *bool         `yaml:"compress_logs,omitempty"`
-	LogMaxSize             int           `yaml:"log_max_size,omitempty"`
-	LogMaxFiles            int           `yaml:"log_max_files,omitempty"`
-	LogMaxAge              int           `yaml:"log_max_age,omitempty"`
-	CacheRetentionDuration time.Duration `yaml:"cache_retention_duration"`
+	BinPath                    string           `yaml:"bin_path"` // path to binary
+	PidDir                     string           `yaml:"piddir"`
+	UpdateFrequency            string           `yaml:"update_frequency"`
+	IncludeScenariosContaining []string         `yaml:"include_scenarios_containing"`
+	ExcludeScenariosContaining []string         `yaml:"exclude_scenarios_containing"`
+	OnlyIncludeDecisionsFrom   []string         `yaml:"only_include_decisions_from"`
+	Daemon                     bool             `yaml:"daemonize"`
+	LogMode                    string           `yaml:"log_mode"`
+	LogDir                     string           `yaml:"log_dir"`
+	LogLevel                   log.Level        `yaml:"log_level"`
+	LogMaxSize                 int              `yaml:"log_max_size,omitempty"`
+	LogMaxFiles                int              `yaml:"log_max_files,omitempty"`
+	LogMaxAge                  int              `yaml:"log_max_age,omitempty"`
+	CompressLogs               *bool            `yaml:"compress_logs,omitempty"`
+	APIUrl                     string           `yaml:"api_url"`
+	APIKey                     string           `yaml:"api_key"`
+	CacheRetentionDuration     time.Duration    `yaml:"cache_retention_duration"`
+	FeedViaStdin               bool             `yaml:"feed_via_stdin"`
+	TotalRetries               int              `yaml:"total_retries"`
+	PrometheusConfig           PrometheusConfig `yaml:"prometheus"`
 }
 
 func NewConfig(configPath string) (*bouncerConfig, error) {
@@ -56,7 +70,7 @@ func NewConfig(configPath string) (*bouncerConfig, error) {
 	}
 
 	/*Configure logging*/
-	if err = types.SetDefaultLoggerConfig(config.LogMode, config.LogDir, config.LogLevel, config.LogMaxSize, config.LogMaxFiles, config.LogMaxAge, config.CompressLogs, false); err != nil {
+	if err := types.SetDefaultLoggerConfig(config.LogMode, config.LogDir, config.LogLevel, config.LogMaxSize, config.LogMaxFiles, config.LogMaxAge, config.CompressLogs, false); err != nil {
 		log.Fatal(err.Error())
 	}
 	if config.LogMode == "file" {

config/crowdsec-custom-bouncer.yaml

@@ -1,10 +1,24 @@
 bin_path: ${BINARY_PATH}
+feed_via_stdin: false # Invokes binary once and feeds incoming decisions to it's stdin.
+total_retries: 0 # number of times to restart binary. relevant if feed_via_stdin=true . Set to -1 for infinite retries.
+include_scenarios_containing: [] # ignore IPs banned for triggering scenarios not containing either of provided word, eg ["ssh", "http"]
+exclude_scenarios_containing: [] # ignore IPs banned for triggering scenarios containing either of provided word
+only_include_decisions_from: []
 piddir: /var/run/
 update_frequency: 10s
 cache_retention_duration: 10s
 daemonize: true
 log_mode: file
 log_dir: /var/log/
 log_level: info
+log_compression: true
+log_max_size: 100
+log_max_backups: 3
+log_max_age: 30
 api_url: http://localhost:8080/
 api_key: ${API_KEY}
+
+prometheus:
+  enabled: true
+  listen_addr: 127.0.0.1
+  listen_port: 60602 

custom.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os/exec"
 	"strconv"
 	"time"
@@ -17,15 +18,23 @@ type DecisionKey struct {
 	Type  string
 }
 
+type DecisionWithAction struct {
+	models.Decision
+	Action string `json:"action,omitempty"`
+}
+
 type customBouncer struct {
 	path                    string
+	binaryStdin             io.Writer
+	feedViaStdin            bool
 	newDecisionValueSet     map[DecisionKey]struct{}
 	expiredDecisionValueSet map[DecisionKey]struct{}
 }
 
-func newCustomBouncer(path string) (*customBouncer, error) {
+func newCustomBouncer(cfg *bouncerConfig) (*customBouncer, error) {
 	return &customBouncer{
-		path: path,
+		path:         cfg.BinPath,
+		feedViaStdin: cfg.FeedViaStdin,
 	}, nil
 }
 
@@ -53,10 +62,19 @@ func (c *customBouncer) Add(decision *models.Decision) error {
 		return err
 	}
 	log.Debugf("custom [%s] : add ban on %s for %s sec (%s)", c.path, *decision.Value, strconv.Itoa(int(banDuration.Seconds())), *decision.Scenario)
-	str, err := serializeDecision(decision)
+	var str string
+	if c.feedViaStdin {
+		str, err = serializeDecision(decision, "add")
+	} else {
+		str, err = serializeDecision(decision, "")
+	}
 	if err != nil {
 		log.Warningf("serialize: %s", err)
 	}
+	if c.feedViaStdin {
+		fmt.Fprintln(c.binaryStdin, str)
+		return nil
+	}
 	cmd := exec.Command(c.path, "add", *decision.Value, strconv.Itoa(int(banDuration.Seconds())), *decision.Scenario, str)
 	if out, err := cmd.CombinedOutput(); err != nil {
 		log.Errorf("Error in 'add' command (%s): %v --> %s", cmd.String(), err, string(out))
@@ -73,7 +91,16 @@ func (c *customBouncer) Delete(decision *models.Decision) error {
 	if err != nil {
 		return err
 	}
-	str, err := serializeDecision(decision)
+	var str string
+	if c.feedViaStdin {
+		str, err = serializeDecision(decision, "del")
+	} else {
+		str, err = serializeDecision(decision, "")
+	}
+	if c.feedViaStdin {
+		fmt.Fprintln(c.binaryStdin, str)
+		return nil
+	}
 	if err != nil {
 		log.Warningf("serialize: %s", err)
 	}
@@ -90,8 +117,9 @@ func (c *customBouncer) ShutDown() error {
 	return nil
 }
 
-func serializeDecision(decision *models.Decision) (string, error) {
-	serbyte, err := json.Marshal(decision)
+func serializeDecision(decision *models.Decision, action string) (string, error) {
+	d := DecisionWithAction{Decision: *decision, Action: action}
+	serbyte, err := json.Marshal(d)
 	if err != nil {
 		return "", fmt.Errorf("serialize error : %s", err)
 	}

go.mod

@@ -5,8 +5,12 @@ go 1.19
 require (
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/crowdsecurity/crowdsec v1.4.1
-	github.com/crowdsecurity/go-cs-bouncer v0.0.0-20220808104920-19304be490bc
+	github.com/crowdsecurity/go-cs-bouncer v0.0.0-20220817075151-29237cbe9873
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/prometheus/client_golang v1.13.0
 	github.com/sirupsen/logrus v1.9.0
+	golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c // indirect
+	golang.org/x/sys v0.0.0-20220823224334-20c2bfdbfe24 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
 	gopkg.in/yaml.v2 v2.4.0
@@ -15,25 +19,30 @@ require (
 require (
 	github.com/antonmedv/expr v1.9.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/crowdsecurity/grokky v0.1.0 // indirect
 	github.com/go-openapi/analysis v0.21.4 // indirect
-	github.com/go-openapi/errors v0.20.2 // indirect
+	github.com/go-openapi/errors v0.20.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
-	github.com/go-openapi/loads v0.21.1 // indirect
-	github.com/go-openapi/spec v0.20.6 // indirect
+	github.com/go-openapi/loads v0.21.2 // indirect
+	github.com/go-openapi/spec v0.20.7 // indirect
 	github.com/go-openapi/strfmt v0.21.3 // indirect
-	github.com/go-openapi/swag v0.22.0 // indirect
 	github.com/go-openapi/validate v0.22.0 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	go.mongodb.org/mongo-driver v1.10.1 // indirect
-	golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 // indirect
-	golang.org/x/sys v0.0.0-20220804214406-8e32c043e418 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )