build: fix rpm builds, broken by ef9a63b: support package revisions

fixup of 9d5c892023

.circleci/config.yml

@@ -283,6 +283,7 @@ jobs:
     steps:
       - attach_workspace:
           at: .
+      - run: make wrapup-and-checksum
       - store_artifacts:
           path: artifacts
 

.circleci/download_artifacts.py

@@ -0,0 +1,84 @@
+import argparse
+from pathlib import Path
+import re
+
+import requests
+
+import time
+
+import os
+import argparse
+from pathlib import Path
+
+
+circle_token = os.environ.get('CIRCLE_TOKEN')
+if not circle_token:
+    raise ValueError('CIRCLE_TOKEN environment variable not set')
+
+parser = argparse.ArgumentParser(description='Download artifacts from CircleCI')
+parser.add_argument('build_num', type=str, help='Build number')
+parser.add_argument('dst', type=Path, help='Destination directory')
+parser.add_argument('--prefix', type=str, default='', help='Filter for prefix')
+parser.add_argument('--match', type=str, default='.*', help='Only include paths matching the given regex')
+
+args = parser.parse_args()
+
+res = requests.get(
+    f"https://circleci.com/api/v1.1/project/github/zrepl/zrepl/{args.build_num}/artifacts",
+    headers={
+        "Circle-Token": circle_token,
+    },
+)
+res.raise_for_status()
+
+# https://circleci.com/docs/api/v1/index.html#artifacts-of-a-job
+# [ {
+#   "path" : "raw-test-output/go-test-report.xml",
+#   "pretty_path" : "raw-test-output/go-test-report.xml",
+#   "node_index" : 0,
+#   "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test-report.xml"
+# }, {
+#   "path" : "raw-test-output/go-test.out",
+#   "pretty_path" : "raw-test-output/go-test.out",
+#   "node_index" : 0,
+#   "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test.out"
+# } ]
+res = res.json()
+
+for artifact in res:
+    if not artifact["pretty_path"].startswith(args.prefix):
+        continue
+    if not re.match(args.match, artifact["pretty_path"]):
+        continue
+    stripped = artifact["pretty_path"][len(args.prefix):]
+    print(f"Downloading {artifact['pretty_path']} to {args.dst / stripped}")
+    artifact_rel = Path(stripped)
+    artifact_dst = args.dst / artifact_rel
+    artifact_dst.parent.mkdir(parents=True, exist_ok=True)
+
+    res = requests.get(
+        artifact["url"],
+        headers={
+            "Circle-Token": circle_token,
+        },
+        stream=True,
+    )
+    res.raise_for_status()
+
+    total_size = int(res.headers.get("Content-Length", 0))
+    block_size = 128 * 1024
+    with open(artifact_dst, "wb") as f:
+        progress = 0
+        start_time = time.time()
+        for chunk in res.iter_content(chunk_size=block_size):
+            f.write(chunk)
+            progress += len(chunk)
+            percent = progress / total_size * 100
+            elapsed_time = time.time() - start_time
+            if elapsed_time >= 5:
+                print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)", end="\r")
+                start_time = time.time()
+        print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)")
+    print("Download complete!")
+
+print("All files downloaded")

Makefile

@@ -1,5 +1,5 @@
 .PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
-.PHONY: release bins-all release-noarch
+.PHONY: release release-noarch
 .DEFAULT_GOAL := zrepl-bin
 
 ARTIFACTDIR := artifacts
@@ -14,13 +14,15 @@ ifndef _ZREPL_VERSION
     endif
 endif
 
+ZREPL_PACKAGE_RELEASE := 1
+
 GO := go
 GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
 GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
 GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
 GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
 GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
-GO_ENV_VARS := GO111MODULE=on
+GO_ENV_VARS := GO111MODULE=on CGO_ENABLED=0
 GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
 GO_MOD_READONLY := -mod=readonly
 GO_EXTRA_BUILDFLAGS :=
@@ -50,21 +52,20 @@ printvars:
 release: clean
 	# no cross-platform support for target test
 	$(MAKE) test-go
-	$(MAKE) bins-all
+	$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="vet"
+	$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="lint"
+	$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="zrepl-bin"
+	$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="test-platform-bin"
 	$(MAKE) noarch
-	$(MAKE) wrapup-and-checksum
-	$(MAKE) check-git-clean
-ifeq (SIGN, 1)
-	$(MAKE) sign
-endif
-	@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
 
 release-docker: $(ARTIFACTDIR)
 	sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile
 	docker build -t zrepl_release --pull -f artifacts/release-docker.Dockerfile .
 	docker run --rm -i -v $(CURDIR):/src -u $$(id -u):$$(id -g) \
 		zrepl_release \
-		make release GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+		make release \
+			GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
+			ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
 
 debs-docker:
 	$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb
@@ -81,9 +82,14 @@ rpm: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion
 	$(eval _ZREPL_RPM_TOPDIR_ABS := $(CURDIR)/$(ARTIFACTDIR)/rpmbuild)
 	rm -rf "$(_ZREPL_RPM_TOPDIR_ABS)"
 	mkdir "$(_ZREPL_RPM_TOPDIR_ABS)"
-	mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)"/{SPECS,RPMS,BUILD,BUILDROOT}
-	sed "s/^Version:.*/Version:          $(_ZREPL_RPM_VERSION)/g" \
-	    packaging/rpm/zrepl.spec >  $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
+	for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS; do \
+		mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)/$$d"; \
+	done
+	sed \
+		-e "s/^Version:.*/Version:          $(_ZREPL_RPM_VERSION)/g" \
+		-e "s/^Release:.*/Release:          $(ZREPL_PACKAGE_RELEASE)/g" \
+	    packaging/rpm/zrepl.spec \
+		>  $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
 
 	# see /usr/lib/rpm/platform
 ifeq ($(GOARCH),amd64)
@@ -110,13 +116,16 @@ rpm-docker:
 	docker build -t zrepl_rpm_pkg --pull -f packaging/rpm/Dockerfile .
 	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
 		zrepl_rpm_pkg \
-		make rpm GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+		make rpm \
+			GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
+			ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
+
 
 deb: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
 
 	cp packaging/deb/debian/changelog.template packaging/deb/debian/changelog
 	sed -i 's/DATE_DASH_R_OUTPUT/$(shell date -R)/' packaging/deb/debian/changelog
-	VERSION="$(subst -,.,$(_ZREPL_VERSION))"; \
+	VERSION="$(subst -,.,$(_ZREPL_VERSION))-$(ZREPL_PACKAGE_RELEASE)"; \
 		export VERSION="$${VERSION#v}"; \
 		sed -i 's/VERSION/'"$$VERSION"'/' packaging/deb/debian/changelog
 
@@ -142,9 +151,11 @@ deb-docker:
 	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
 		--ulimit nofile=1024:1024 \
 		zrepl_debian_pkg \
-		make deb GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+		make deb \
+			GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
+			ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
 
-# expects `release` target to have run before
+# expects `release`, `deb` & `rpm` targets to have run before
 NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
 wrapup-and-checksum:
 	rm -f $(NOARCH_TARBALL)
@@ -161,7 +172,7 @@ wrapup-and-checksum:
 		config/samples
 	rm -rf "$(ARTIFACTDIR)/release"
 	mkdir -p "$(ARTIFACTDIR)/release"
-	cp -l $(ARTIFACTDIR)/zrepl-* \
+	cp -l $(ARTIFACTDIR)/zrepl* \
 		$(ARTIFACTDIR)/platformtest-* \
 		"$(ARTIFACTDIR)/release"
 	cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
@@ -181,39 +192,42 @@ check-git-clean:
 
 tag-release:
 	test -n "$(ZREPL_TAG_VERSION)" || exit 1
-	git tag -u E27CA5FC -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
+	git tag -u '328A6627FA98061D!' -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
 
 sign:
-	gpg -u "89BC 5D89 C845 568B F578  B306 CDBD 8EC8 E27C A5FC" \
+	gpg -u '328A6627FA98061D!' \
 		--armor \
 		--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
 
 clean: docs-clean
 	rm -rf "$(ARTIFACTDIR)"
 
-##################### BINARIES #####################
-.PHONY: bins-all lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list
+download-circleci-release:
+	rm -rf "$(ARTIFACTDIR)"
+	mkdir -p "$(ARTIFACTDIR)/release"
+	python3 .circleci/download_artifacts.py --prefix 'artifacts/release/' "$(BUILD_NUM)" "$(ARTIFACTDIR)/release"
 
-BINS_ALL_TARGETS := zrepl-bin test-platform-bin vet lint
-GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split($$0, comps, " "); split(comps[3], v, "."); if (v[1] == "go1" && v[2] >= 13) { print "illumos"; } else { print "noillumos"; }}')
-bins-all:
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=amd64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=386
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=arm	  GOARM=7
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=arm64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=amd64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm     GOARM=7
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=386
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin    GOARCH=amd64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris   GOARCH=amd64
-ifeq ($(GO_SUPPORTS_ILLUMOS), illumos)
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=illumos   GOARCH=amd64
-else ifeq ($(GO_SUPPORTS_ILLUMOS), noillumos)
-	@echo "SKIPPING ILLUMOS BUILD BECAUSE GO VERSION DOESN'T SUPPORT IT"
-else
-	@echo "CANNOT DETERMINE WHETHER GO VERSION SUPPORTS GOOS=illumos"; exit 1
-endif
+##################### MULTI-ARCH HELPERS #####################
+
+_run_make_foreach_target_tuple:
+	if [ "$(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG)" = "" ]; then \
+		echo "RUN_MAKE_FOREACH_TARGET_TUPLE_ARG must be set"; \
+		exit 1; \
+	fi
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd   GOARCH=amd64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd   GOARCH=386
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd   GOARCH=arm	  GOARM=7
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd   GOARCH=arm64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux     GOARCH=amd64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux     GOARCH=arm64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux     GOARCH=arm     GOARM=7
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux     GOARCH=386
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=darwin    GOARCH=amd64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=solaris   GOARCH=amd64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=illumos   GOARCH=amd64
+
+##################### REGULAR TARGETS #####################
+.PHONY: lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list
 
 lint:
 	$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...

packaging/rpm/zrepl.spec

@@ -16,7 +16,7 @@ zrepl is a one-stop, integrated solution for ZFS replication.}
 %define __strip /usr/bin/true
 
 Name:             zrepl
-Release:          1
+Release:          SUBSTITUTED_BY_MAKEFILE
 Summary:          One-stop, integrated solution for ZFS replication
 License:          MIT
 URL:              https://zrepl.github.io/