Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

build: reorg Makefile #4574

Merged
merged 6 commits into from
Jan 14, 2022
Merged

build: reorg Makefile #4574

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
db84d22
build: reorg Makefile
disksing Jan 13, 2022
454494e
Merge branch 'master' into makefile2
disksing Jan 13, 2022
913c355
minor update
disksing Jan 13, 2022
b9bfbfc
minor fix
disksing Jan 14, 2022
becc590
Merge branch 'master' into makefile2
disksing Jan 14, 2022
f5e138d
add back daily coverage
disksing Jan 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
319 changes: 146 additions & 173 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,41 +1,16 @@
PD_PKG := github.com/tikv/pd

TEST_PKGS := $(shell find . -iname "*_test.go" -exec dirname {} \; | \
sort -u | sed -e "s/^\./github.com\/tikv\/pd/")
INTEGRATION_TEST_PKGS := $(shell find . -iname "*_test.go" -exec dirname {} \; | \
sort -u | sed -e "s/^\./github.com\/tikv\/pd/" | grep -E "tests")
TSO_INTEGRATION_TEST_PKGS := $(shell find . -iname "*_test.go" -exec dirname {} \; | \
sort -u | sed -e "s/^\./github.com\/tikv\/pd/" | grep -E "server/tso")
BASIC_TEST_PKGS := $(filter-out $(INTEGRATION_TEST_PKGS),$(TEST_PKGS))

PACKAGES := go list ./...
PACKAGE_DIRECTORIES := $(PACKAGES) | sed 's|$(PD_PKG)/||'
GOCHECKER := awk '{ print } END { if (NR > 0) { exit 1 } }'
OVERALLS := overalls
# Build pd-server, pd-ctl, pd-recover
default: build

TASK_COUNT=1
TASK_ID=1
# Development validation.
all: dev
dev: build check tools test

BUILD_BIN_PATH := $(shell pwd)/bin
GO_TOOLS_BIN_PATH := $(shell pwd)/.tools/bin
PATH := $(GO_TOOLS_BIN_PATH):$(PATH)
SHELL := env PATH='$(PATH)' GOBIN='$(GO_TOOLS_BIN_PATH)' $(shell which bash)
# Lightweight development validation.
dev-basic: build check basic-test

FAILPOINT_ENABLE := $$(find $$PWD/ -type d | grep -vE "\.git" | xargs failpoint-ctl enable)
FAILPOINT_DISABLE := $$(find $$PWD/ -type d | grep -vE "\.git" | xargs failpoint-ctl disable)
.PHONY: default all dev dev-basic

DEADLOCK_ENABLE := $$(\
find . -name "*.go" \
| xargs -n 1 sed -i.bak 's/sync\.RWMutex/deadlock.RWMutex/;s/sync\.Mutex/deadlock.Mutex/' && \
find . -name "*.go" | xargs grep -lE "(deadlock\.RWMutex|deadlock\.Mutex)" \
| xargs goimports -w)
DEADLOCK_DISABLE := $$(\
find . -name "*.go" \
| xargs -n 1 sed -i.bak 's/deadlock\.RWMutex/sync.RWMutex/;s/deadlock\.Mutex/sync.Mutex/' && \
find . -name "*.go" | xargs grep -lE "(sync\.RWMutex|sync\.Mutex)" \
| xargs goimports -w && \
find . -name "*.bak" | xargs rm && \
go mod tidy)
#### Build ####

BUILD_FLAGS ?=
BUILD_TAGS ?=
Expand Down Expand Up @@ -78,20 +53,7 @@ ifneq ($(DASHBOARD), 0)
LDFLAGS += -X "github.com/pingcap/tidb-dashboard/pkg/utils/version.BuildGitHash=$(shell scripts/describe-dashboard.sh git-hash)"
endif

GOVER_MAJOR := $(shell go version | sed -E -e "s/.*go([0-9]+)[.]([0-9]+).*/\1/")
GOVER_MINOR := $(shell go version | sed -E -e "s/.*go([0-9]+)[.]([0-9]+).*/\2/")
GO111 := $(shell [ $(GOVER_MAJOR) -gt 1 ] || [ $(GOVER_MAJOR) -eq 1 ] && [ $(GOVER_MINOR) -ge 11 ]; echo $$?)
ifeq ($(GO111), 1)
$(error "go below 1.11 does not support modules")
endif

default: build

all: dev

dev: build check tools test

ci: build check basic-test
BUILD_BIN_PATH := $(shell pwd)/bin

build: pd-server pd-ctl pd-recover

Expand All @@ -107,160 +69,196 @@ ifneq ($(DASHBOARD_DISTRIBUTION_DIR),)
endif
PD_SERVER_DEP += dashboard-ui

pd-server: export GO111MODULE=on
pd-server: ${PD_SERVER_DEP}
CGO_ENABLED=$(BUILD_CGO_ENABLED) go build $(BUILD_FLAGS) -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -tags "$(BUILD_TAGS)" -o $(BUILD_BIN_PATH)/pd-server cmd/pd-server/main.go

pd-server-basic: export GO111MODULE=on
pd-server-basic:
SWAGGER=0 DASHBOARD=0 $(MAKE) pd-server

# dependent
install-all-tools: export GO111MODULE=on
install-all-tools:
@mkdir -p $(GO_TOOLS_BIN_PATH)
@which golangci-lint >/dev/null 2>&1 || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GO_TOOLS_BIN_PATH) v1.43.0
@grep '_' tools.go | sed 's/"//g' | awk '{print $$2}' | xargs go install
.PHONY: build tools pd-server pd-server-basic

install-golangci-lint:
@mkdir -p $(GO_TOOLS_BIN_PATH)
@which golangci-lint >/dev/null 2>&1 || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GO_TOOLS_BIN_PATH) v1.43.0
# Tools

install-go-tools: export GO111MODULE=on
install-go-tools:
@mkdir -p $(GO_TOOLS_BIN_PATH)
@grep '_' tools.go | sed 's/"//g' | awk '{print $$2}' | xargs go install
pd-ctl:
CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-ctl tools/pd-ctl/main.go
pd-tso-bench:
CGO_ENABLED=0 go build -o $(BUILD_BIN_PATH)/pd-tso-bench tools/pd-tso-bench/main.go
pd-recover:
CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-recover tools/pd-recover/main.go
pd-analysis:
CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-analysis tools/pd-analysis/main.go
pd-heartbeat-bench:
CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-heartbeat-bench tools/pd-heartbeat-bench/main.go
simulator:
CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-simulator tools/pd-simulator/main.go
regions-dump:
CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/regions-dump tools/regions-dump/main.go
stores-dump:
CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/stores-dump tools/stores-dump/main.go

.PHONY: pd-ctl pd-tso-bench pd-recover pd-analysis pd-heartbeat-bench simulator regions-dump stores-dump

#### Build utils ###

swagger-spec: export GO111MODULE=on
swagger-spec: install-go-tools
swagger-spec: install-tools
go mod vendor
swag init --parseVendor -generalInfo server/api/router.go --exclude vendor/github.com/pingcap/tidb-dashboard --output docs/swagger
go mod tidy
rm -rf vendor

dashboard-ui: export GO111MODULE=on
dashboard-ui:
./scripts/embed-dashboard-ui.sh

dashboard-replace-distro-info:
rm -f pkg/dashboard/distro/distro_info.go
cp $(DASHBOARD_DISTRIBUTION_DIR)/distro_info.go pkg/dashboard/distro/distro_info.go

# Tools
pd-ctl: export GO111MODULE=on
pd-ctl:
CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-ctl tools/pd-ctl/main.go
pd-tso-bench: export GO111MODULE=on
pd-tso-bench:
CGO_ENABLED=0 go build -o $(BUILD_BIN_PATH)/pd-tso-bench tools/pd-tso-bench/main.go
pd-recover: export GO111MODULE=on
pd-recover:
CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-recover tools/pd-recover/main.go
pd-analysis: export GO111MODULE=on
pd-analysis:
CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-analysis tools/pd-analysis/main.go
pd-heartbeat-bench: export GO111MODULE=on
pd-heartbeat-bench:
CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-heartbeat-bench tools/pd-heartbeat-bench/main.go
.PHONY: swagger-spec dashboard-ui dashboard-replace-distro-info

#### Static tools ####

PD_PKG := github.com/tikv/pd
PACKAGES := $(shell go list ./...)

test: install-go-tools
GO_TOOLS_BIN_PATH := $(shell pwd)/.tools/bin
PATH := $(GO_TOOLS_BIN_PATH):$(PATH)
SHELL := env PATH='$(PATH)' GOBIN='$(GO_TOOLS_BIN_PATH)' $(shell which bash)

install-tools:
@mkdir -p $(GO_TOOLS_BIN_PATH)
@which golangci-lint >/dev/null 2>&1 || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GO_TOOLS_BIN_PATH) v1.43.0
@grep '_' tools.go | sed 's/"//g' | awk '{print $$2}' | xargs go install

.PHONY: install-tools

#### Static checks ####

check: install-tools static tidy check-plugin errdoc check-testing-t docker-build-test

static: install-tools
@ # Not running vet and fmt through metalinter becauase it ends up looking at vendor
@ echo "gofmt ..."
@ gofmt -s -l -d $(PACKAGE_DIRECTORIES) 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }'
@ echo "golangci-lint ..."
@ golangci-lint run $(PACKAGE_DIRECTORIES)
@ echo "revive ..."
@revive -formatter friendly -config revive.toml $(PACKAGES)

tidy:
@echo "go mod tidy"
go mod tidy
git diff --quiet go.mod go.sum

check-plugin:
@echo "checking plugin"
cd ./plugin/scheduler_example && $(MAKE) evictLeaderPlugin.so && rm evictLeaderPlugin.so

errdoc: install-tools
@echo "generator errors.toml"
./scripts/check-errdoc.sh

docker-build-test:
$(eval DOCKER_PS_EXIT_CODE=$(shell docker ps > /dev/null 2>&1 ; echo $$?))
@if [ $(DOCKER_PS_EXIT_CODE) -ne 0 ]; then \
echo "Encountered problem while invoking docker cli. Is the docker daemon running?"; \
fi
docker build --no-cache -t tikv/pd .

check-testing-t:
./scripts/check-testing-t.sh

.PHONY: check static tidy check-plugin errdoc docker-build-test check-testing-t

#### Test utils ####

FAILPOINT_ENABLE := $$(find $$PWD/ -type d | grep -vE "\.git" | xargs failpoint-ctl enable)
FAILPOINT_DISABLE := $$(find $$PWD/ -type d | grep -vE "\.git" | xargs failpoint-ctl disable)

DEADLOCK_ENABLE := $$(\
find . -name "*.go" \
| xargs -n 1 sed -i.bak 's/sync\.RWMutex/deadlock.RWMutex/;s/sync\.Mutex/deadlock.Mutex/' && \
find . -name "*.go" | xargs grep -lE "(deadlock\.RWMutex|deadlock\.Mutex)" \
| xargs goimports -w)
DEADLOCK_DISABLE := $$(\
find . -name "*.go" \
| xargs -n 1 sed -i.bak 's/deadlock\.RWMutex/sync.RWMutex/;s/deadlock\.Mutex/sync.Mutex/' && \
find . -name "*.go" | xargs grep -lE "(sync\.RWMutex|sync\.Mutex)" \
| xargs goimports -w && \
find . -name "*.bak" | xargs rm && \
go mod tidy)

deadlock-enable: install-tools
# Enabling deadlock...
@$(DEADLOCK_ENABLE)

deadlock-disable: install-tools
# Disabling deadlock...
@$(DEADLOCK_DISABLE)

failpoint-enable: install-tools
# Converting failpoints...
@$(FAILPOINT_ENABLE)

failpoint-disable: install-tools
# Restoring failpoints...
@$(FAILPOINT_DISABLE)

.PHONY: deadlock-enable deadlock-disable failpoint-enable failpoint-disable

#### Test ####

PACKAGE_DIRECTORIES := $(subst $(PD_PKG)/,,$(PACKAGES))
TEST_PKGS := $(filter $(shell find . -iname "*_test.go" -exec dirname {} \; | \
sort -u | sed -e "s/^\./github.com\/tikv\/pd/"),$(PACKAGES))
BASIC_TEST_PKGS := $(filter-out $(PD_PKG)/tests%,$(TEST_PKGS))

test: install-tools
# testing all pkgs...
@$(DEADLOCK_ENABLE)
@$(FAILPOINT_ENABLE)
CGO_ENABLED=1 GO111MODULE=on go test -tags tso_function_test -timeout 20m -race -cover $(TEST_PKGS) || { $(FAILPOINT_DISABLE); $(DEADLOCK_DISABLE); exit 1; }
CGO_ENABLED=1 go test -tags tso_function_test -timeout 20m -race -cover $(TEST_PKGS) || { $(FAILPOINT_DISABLE); $(DEADLOCK_DISABLE); exit 1; }
@$(FAILPOINT_DISABLE)
@$(DEADLOCK_DISABLE)

basic-test:
basic-test: install-tools
# testing basic pkgs...
@$(FAILPOINT_ENABLE)
GO111MODULE=on go test $(BASIC_TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; }
go test $(BASIC_TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; }
@$(FAILPOINT_DISABLE)

test-with-cover: install-go-tools dashboard-ui
test-with-cover: install-tools dashboard-ui
# testing all pkgs (expect TSO consistency test) with converage...
@$(FAILPOINT_ENABLE)
for PKG in $(TEST_PKGS); do\
set -euo pipefail;\
CGO_ENABLED=1 GO111MODULE=on go test -race -covermode=atomic -coverprofile=coverage.tmp -coverpkg=./... $$PKG 2>&1 | grep -v "no packages being tested" && tail -n +2 coverage.tmp >> covprofile || { $(FAILPOINT_DISABLE); rm coverage.tmp && exit 1;}; \
CGO_ENABLED=1 go test -race -covermode=atomic -coverprofile=coverage.tmp -coverpkg=./... $$PKG 2>&1 | grep -v "no packages being tested" && tail -n +2 coverage.tmp >> covprofile || { $(FAILPOINT_DISABLE); rm coverage.tmp && exit 1;}; \
rm coverage.tmp;\
done
@$(FAILPOINT_DISABLE)

# The command should be used in daily CI，it will split some tasks to run parallel.
# It should retain report.xml,coverage,coverage.xml and package.list to analyze.
disksing marked this conversation as resolved.
Show resolved Hide resolved
test-with-cover-parallel: install-go-tools dashboard-ui split
@$(FAILPOINT_ENABLE)
set -euo pipefail;\
CGO_ENABLED=1 GO111MODULE=on gotestsum --junitfile report.xml -- -v --race -covermode=atomic -coverprofile=coverage $(shell cat package.list) 2>&1 || { $(FAILPOINT_DISABLE); }; \
gocov convert coverage | gocov-xml >> coverage.xml;\
@$(FAILPOINT_DISABLE)
TSO_INTEGRATION_TEST_PKGS := $(PD_PKG)/tests/server/tso

test-tso-function: install-go-tools
test-tso-function: install-tools
# testing TSO function...
@$(DEADLOCK_ENABLE)
@$(FAILPOINT_ENABLE)
CGO_ENABLED=1 GO111MODULE=on go test -race -tags without_dashboard,tso_function_test $(TSO_INTEGRATION_TEST_PKGS) || { $(FAILPOINT_DISABLE); $(DEADLOCK_DISABLE); exit 1; }
CGO_ENABLED=1 go test -race -tags without_dashboard,tso_function_test $(TSO_INTEGRATION_TEST_PKGS) || { $(FAILPOINT_DISABLE); $(DEADLOCK_DISABLE); exit 1; }
@$(FAILPOINT_DISABLE)
@$(DEADLOCK_DISABLE)

test-tso-consistency: install-go-tools
test-tso-consistency: install-tools
# testing TSO consistency...
@$(DEADLOCK_ENABLE)
@$(FAILPOINT_ENABLE)
CGO_ENABLED=1 GO111MODULE=on go test -race -tags without_dashboard,tso_consistency_test $(TSO_INTEGRATION_TEST_PKGS) || { $(FAILPOINT_DISABLE); $(DEADLOCK_DISABLE); exit 1; }
CGO_ENABLED=1 go test -race -tags without_dashboard,tso_consistency_test $(TSO_INTEGRATION_TEST_PKGS) || { $(FAILPOINT_DISABLE); $(DEADLOCK_DISABLE); exit 1; }
@$(FAILPOINT_DISABLE)
@$(DEADLOCK_DISABLE)

check: install-all-tools check-all check-plugin errdoc check-testing-t docker-build-test

check-all: static lint tidy
@echo "checking"

check-plugin:
@echo "checking plugin"
cd ./plugin/scheduler_example && $(MAKE) evictLeaderPlugin.so && rm evictLeaderPlugin.so

static: export GO111MODULE=on
static: install-golangci-lint
@ # Not running vet and fmt through metalinter becauase it ends up looking at vendor
gofmt -s -l -d $$($(PACKAGE_DIRECTORIES)) 2>&1 | $(GOCHECKER)
golangci-lint run $$($(PACKAGE_DIRECTORIES))

lint: install-go-tools
@echo "linting"
revive -formatter friendly -config revive.toml $$($(PACKAGES))

tidy:
@echo "go mod tidy"
GO111MODULE=on go mod tidy
git diff --quiet go.mod go.sum

errdoc: install-go-tools
@echo "generator errors.toml"
./scripts/check-errdoc.sh

docker-build-test:
$(eval DOCKER_PS_EXIT_CODE=$(shell docker ps > /dev/null 2>&1 ; echo $$?))
@if [ $(DOCKER_PS_EXIT_CODE) -ne 0 ]; then \
echo "Encountered problem while invoking docker cli. Is the docker daemon running?"; \
fi
docker build --no-cache -t tikv/pd .

check-testing-t:
./scripts/check-testing-t.sh

simulator: export GO111MODULE=on
simulator:
CGO_ENABLED=0 go build -o $(BUILD_BIN_PATH)/pd-simulator tools/pd-simulator/main.go
.PHONY: test basic-test test-with-cover test-tso-function test-tso-consistency

regions-dump: export GO111MODULE=on
regions-dump:
CGO_ENABLED=0 go build -o $(BUILD_BIN_PATH)/regions-dump tools/regions-dump/main.go
#### Clean up ####

stores-dump: export GO111MODULE=on
stores-dump:
CGO_ENABLED=0 go build -o $(BUILD_BIN_PATH)/stores-dump tools/stores-dump/main.go
clean: failpoint-disable deadlock-disable clean-test clean-build

clean-test:
# Cleaning test tmp...
Expand All @@ -275,29 +273,4 @@ clean-build:
rm -rf $(BUILD_BIN_PATH)
rm -rf $(GO_TOOLS_BIN_PATH)

deadlock-enable: install-go-tools
# Enabling deadlock...
@$(DEADLOCK_ENABLE)

deadlock-disable: install-go-tools
# Disabling deadlock...
@$(DEADLOCK_DISABLE)

failpoint-enable: install-go-tools
# Converting failpoints...
@$(FAILPOINT_ENABLE)

failpoint-disable: install-go-tools
# Restoring failpoints...
@$(FAILPOINT_DISABLE)

split:
# todo: it will remove server/api,/tests and tso packages after daily CI integrate all verify CI.
go list ./... | grep -v -E "github.com/tikv/pd/server/api|github.com/tikv/pd/tests/client|github.com/tikv/pd/tests/server/tso" > packages.list;\
split packages.list -n r/${TASK_COUNT} packages_unit_ -a 1 --numeric-suffixes=1;\
cat packages_unit_${TASK_ID} |tr "\n" " " >package.list;\
rm packages*;

clean: failpoint-disable deadlock-disable clean-test clean-build

.PHONY: all ci vendor tidy clean-test clean-build clean
.PHONY: clean clean-test clean-build
Loading