Multi-arch build workflow

.github/release.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import shutil
+import subprocess
+import sys
+
+
+def _goos():
+    yield 'linux'
+    yield 'freebsd'
+
+
+def _goarch(goos):
+    yield '386'
+    yield 'amd64'
+    yield 'arm'
+    yield 'arm64'
+    if goos == 'linux':
+        yield 'ppc64le'
+        yield 's390x'
+    yield 'riscv64'
+
+
+def _goarm(goarch):
+    if goarch != 'arm':
+        yield ''
+        return
+    yield '6'
+    yield '7'
+
+
+def _static():
+    yield True
+    yield False
+
+
+def _build_tarball(os):
+    if os == 'linux':
+        yield True
+    yield False
+
+
+def filename_for_entry(prog_name, entry):
+    arch = entry['goarch']
+    if entry['goarch'] == 'arm':
+        arch += 'v' + entry['goarm']
+    ret = f'{prog_name}-{entry["goos"]}-{arch}'
+    if entry['static']:
+        ret += '-static'
+    if entry['build_tarball']:
+        ret += '.tgz'
+    return ret
+
+
+def matrix(prog_name):
+    for goos in _goos():
+        for goarch in _goarch(goos):
+            for goarm in _goarm(goarch):
+                for static in _static():
+                    for build_tarball in _build_tarball(goos):
+                        yield {
+                            'goos': goos,
+                            'goarch': goarch,
+                            'goarm': goarm,
+                            'static': static,
+                            'build_tarball': build_tarball,
+                        }
+
+
+def print_matrix(prog_name):
+    j = {'include': list(matrix(prog_name))}
+
+    if os.isatty(sys.stdout.fileno()):
+        print(json.dumps(j, indent=2))
+    else:
+        print(json.dumps(j))
+
+
+default_tarball = {
+    'goos': 'linux',
+    'goarch': 'amd64',
+    'goarm': '',
+    'static': False,
+    'build_tarball': True,
+}
+
+default_binary = {
+    'goos': 'linux',
+    'goarch': 'amd64',
+    'goarm': '',
+    'static': False,
+    'build_tarball': False,
+}
+
+
+def run_build(prog_name):
+    # call the makefile for each matrix entry
+
+    default_tarball_filename = None
+    default_binary_filename = None
+
+    for entry in matrix(prog_name):
+        env = {'GOOS': entry['goos'], 'GOARCH': entry['goarch']}
+
+        if entry['goarm']:
+            env['GOARM'] = entry['goarm']
+
+        if entry['static']:
+            env['BUILD_STATIC'] = 'yes'
+
+        if entry['build_tarball']:
+            target = 'tarball'
+        else:
+            target = 'binary'
+
+        print(f"Running make {target} for {env}")
+
+        subprocess.run(['make', target], env=os.environ | env, check=True)
+
+        want_filename = filename_for_entry(prog_name, entry)
+
+        if entry['build_tarball']:
+            os.rename(f'{prog_name}.tgz', want_filename)
+        else:
+            os.rename(f'{prog_name}', want_filename)
+
+        # if this is the default tarball or binary, save the filename
+        # we'll use it later to publish a "default" package
+
+        if entry == default_tarball:
+            default_tarball_filename = want_filename
+
+        if entry == default_binary:
+            default_binary_filename = want_filename
+
+        # Remove the directory to reuse it
+        subprocess.run(['make', 'clean-release-dir'], env=os.environ | env, check=True)
+
+    # publish the default tarball and binary
+    if default_tarball_filename:
+        shutil.copy(default_tarball_filename, f'{prog_name}.tgz')
+
+    if default_binary_filename:
+        shutil.copy(default_binary_filename, f'{prog_name}')
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Build release binaries and tarballs for all supported platforms')
+    parser.add_argument('action', help='Action to perform (ex. run-build, print-matrix)')
+    parser.add_argument('prog_name', help='Name of the program (ex. crowdsec-firewall-bouncer)')
+
+    args = parser.parse_args()
+
+    if args.action == 'print-matrix':
+        print_matrix(args.prog_name)
+
+    if args.action == 'run-build':
+        run_build(args.prog_name)
+
+
+if __name__ == '__main__':
+    main()

.github/workflows/build-binary-package.yml

@@ -1,26 +1,37 @@
 # .github/workflows/build-docker-image.yml
 name: build-binary-package
 
-on: 
+on:
   release:
     types: prereleased
 
+env:
+  PROGRAM_NAME: crowdsec-custom-bouncer
+
 jobs:
-  build-binary-package:
-    name: Build and upload binary package
+  build:
+    name: Build and upload all platforms
     runs-on: ubuntu-latest
+
     steps:
+
     - name: Set up Go 1.20.1
       uses: actions/setup-go@v3
       with:
         go-version: 1.20.1
-    - name: Check out code into the Go module directory
+
+    - name: Check out repository
       uses: actions/checkout@v3
-    - name: Build the binaries
-      run: make release
-    - name: Upload to release
-      uses: JasonEtco/upload-to-release@master
       with:
-        args: crowdsec-custom-bouncer.tgz application/x-gzip
+        fetch-depth: 0
+
+    - name: Build all versions
+      run: |
+        make platform-all
+
+    - name: Upload to release
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        tag_name="${GITHUB_REF##*/}"
+        hub release edit $(find . -name "$PROGRAM_NAME*" -maxdepth 1 -printf "-a %p ") -m "" "$tag_name"

.gitignore

@@ -1,5 +1,12 @@
+# vscode
+.vscode
+
 # Binaries for programs and plugins
-crowdsec-custom-bouncer
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
 
 # Test binary, built with `go test -c`
 *.test
@@ -9,3 +16,8 @@ crowdsec-custom-bouncer
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
+
+# built binaries
+/crowdsec-custom-bouncer
+/crowdsec-custom-bouncer-*
+/crowdsec-custom-bouncer.tgz

Makefile

@@ -1,45 +1,83 @@
-# Go parameters
 GOCMD=go
 GOBUILD=$(GOCMD) build
-GOCLEAN=$(GOCMD) clean
 GOTEST=$(GOCMD) test
-GOGET=$(GOCMD) get
 
-PREFIX?="/"
-PID_DIR = $(PREFIX)"/var/run/"
 BINARY_NAME=crowdsec-custom-bouncer
+REPO_NAME=cs-custom-bouncer
+TARBALL_NAME=$(BINARY_NAME).tgz
 
-#Current versioning information from env
-BUILD_VERSION?="$(shell git describe --tags)"
-BUILD_TIMESTAMP=$(shell date +%F"_"%T)
-BUILD_TAG="$(shell git rev-parse HEAD)"
-export LD_OPTS=-ldflags "-s -w -X github.com/crowdsecurity/cs-custom-bouncer/pkg/version.Version=$(BUILD_VERSION) \
--X github.com/crowdsecurity/cs-custom-bouncer/pkg/version.BuildDate=$(BUILD_TIMESTAMP) \
--X github.com/crowdsecurity/cs-custom-bouncer/pkg/version.Tag=$(BUILD_TAG)"
+# Current versioning information from env
+BUILD_VERSION?=$(shell git describe --tags)
+BUILD_TIMESTAMP?=$(shell date +%F"_"%T)
+BUILD_TAG?=$(shell git rev-parse HEAD)
 
-RELDIR = "crowdsec-custom-bouncer-${BUILD_VERSION}"
+LD_OPTS_VARS=\
+-X 'github.com/crowdsecurity/$(REPO_NAME)/pkg/version.Version=$(BUILD_VERSION)' \
+-X 'github.com/crowdsecurity/$(REPO_NAME)/pkg/version.BuildDate=$(BUILD_TIMESTAMP)' \
+-X 'github.com/crowdsecurity/$(REPO_NAME)/pkg/version.Tag=$(BUILD_TAG)'
 
-all: clean build test
+ifdef BUILD_STATIC
+	export LD_OPTS=-ldflags "-a -s -w -extldflags '-static' $(LD_OPTS_VARS)" -tags netgo
+else
+	export LD_OPTS=-ldflags "-a -s -w $(LD_OPTS_VARS)"
+endif
 
-static: clean
-	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME) -v -a -tags netgo -ldflags '-w -extldflags "-static"'
+LD_OPTS += -trimpath
 
-build: goversion clean
-	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME) -v
+.PHONY: all
+all: build test
+
+# Remove everything including all platform binaries and tarballs
+.PHONY: clean
+clean: clean-release-dir
+	@$(RM) $(BINARY_NAME)
+	@$(RM) $(TARBALL_NAME)
+	@$(RM) -r $(BINARY_NAME)-*	# platform binary name and leftover release dir
+	@$(RM) $(BINARY_NAME)-*.tgz	# platform release file
+
+#
+# Build binaries
+#
+
+.PHONY: binary
+binary: goversion
+	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
+
+.PHONY: build
+build: goversion clean binary
+
+#
+# Unit and integration tests
+#
+
+.PHONY: lint
+lint:
+	golangci-lint run
 
 .PHONY: test
 test:
-	@$(GOTEST) ./...
+	@$(GOTEST) $(LD_OPTS) ./...
 
-clean:
-	@rm -f $(BINARY_NAME)
-	@rm -rf ${RELDIR}
-	@rm -f crowdsec-custom-bouncer.tgz || ""
+.PHONY: func-tests
+func-tests: build
+	pipenv install --dev
+	pipenv run pytest -v
 
-.PHONY: release
-release: build
-	@if [ -z ${BUILD_VERSION} ] ; then BUILD_VERSION="local" ; fi
-	@if [ -d $(RELDIR) ]; then echo "$(RELDIR) already exists, clean" ;  exit 1 ; fi
+#
+# Build release tarballs
+#
+
+RELDIR = $(BINARY_NAME)-$(BUILD_VERSION)
+
+# Called during release, to reuse the directory for other platforms
+.PHONY: clean-release-dir
+clean-release-dir:
+	@$(RM) -r $(RELDIR)
+
+.PHONY: tarball
+tarball: binary
+	@if [ -z $(BUILD_VERSION) ]; then BUILD_VERSION="local" ; fi
+	@if [ -d $(RELDIR) ]; then echo "$(RELDIR) already exists, please run 'make clean' and retry" ;  exit 1 ; fi
 	@echo Building Release to dir $(RELDIR)
 	@mkdir $(RELDIR)/
 	@cp $(BINARY_NAME) $(RELDIR)/
@@ -50,11 +88,18 @@ release: build
 	@chmod +x $(RELDIR)/install.sh
 	@chmod +x $(RELDIR)/uninstall.sh
 	@chmod +x $(RELDIR)/upgrade.sh
-	@tar cvzf crowdsec-custom-bouncer.tgz $(RELDIR)
+	@tar cvzf $(TARBALL_NAME) $(RELDIR)
 
-.PHONY: func-tests
-func-tests: build
-	pipenv install --dev
-	pipenv run pytest -v
+.PHONY: release
+release: clean tarball
+
+#
+# Build binaries and release tarballs for all platforms
+#
+
+.PHONY: platform-all
+platform-all: goversion clean
+	python3 .github/release.py run-build $(BINARY_NAME)
 
+# Check if go is the right version
 include mk/goversion.mk