From 8f3bcfac9385eb72da40505dc29fdcbe4b3e9f58 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Mon, 5 Sep 2022 23:09:41 +0200 Subject: [PATCH] Move to github actions (#11235) Signed-off-by: Julien Pivotto Signed-off-by: Julien Pivotto --- .circleci/config.yml | 184 +----------------- .github/actions/build/action.yml | 24 +++ .github/actions/check_proto/action.yml | 20 ++ .github/actions/publish_images/action.yml | 46 +++++ .github/actions/publish_main/action.yml | 40 ++++ .github/actions/publish_release/action.yml | 46 +++++ .../actions/publish_release_image/action.yml | 52 +++++ .github/actions/setup_environment/action.yml | 33 ++++ .github/workflows/repo_sync.yml | 12 ++ .github/workflows/test.yml | 147 ++++++++++++++ Makefile.common | 5 +- 11 files changed, 434 insertions(+), 175 deletions(-) create mode 100644 .github/actions/build/action.yml create mode 100644 .github/actions/check_proto/action.yml create mode 100644 .github/actions/publish_images/action.yml create mode 100644 .github/actions/publish_main/action.yml create mode 100644 .github/actions/publish_release/action.yml create mode 100644 .github/actions/publish_release_image/action.yml create mode 100644 .github/actions/setup_environment/action.yml create mode 100644 .github/workflows/repo_sync.yml create mode 100644 .github/workflows/test.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 648049b80..965b27394 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,197 +1,33 @@ --- +# Prometheus has switched to GitHub action. +# Circle CI is not disabled repository-wise so that previous pull requests +# continue working. +# This file does not generate any CircleCI workflow. + version: 2.1 -orbs: - prometheus: prometheus/prometheus@0.16.0 - go: circleci/go@1.7.0 - win: circleci/windows@2.3.0 - executors: - # Whenever the Go version is updated here, .promu.yml - # should also be updated. golang: docker: - - image: quay.io/prometheus/golang-builder:1.18-base - golang_oldest: - docker: - - image: quay.io/prometheus/golang-builder:1.17-base + - image: busybox jobs: - test_go: + noopjob: executor: golang steps: - - prometheus/setup_environment - - go/load-cache: - key: v1 - run: - command: make GO_ONLY=1 - environment: - # Run garbage collection more aggressively to avoid getting OOMed during the lint phase. - GOGC: "20" - # By default Go uses GOMAXPROCS but a Circle CI executor has many - # cores (> 30) while the CPU and RAM resources are throttled. If we - # don't limit this to the number of allocated cores, the job is - # likely to get OOMed and killed. - GOOPTS: "-p 2" - GOMAXPROCS: "2" - GO111MODULE: "on" - - run: go test ./tsdb/ -test.tsdb-isolation=false - - run: make -C documentation/examples/remote_storage - - run: make -C documentation/examples - - prometheus/check_proto: - version: "3.15.8" - - prometheus/store_artifact: - file: prometheus - - prometheus/store_artifact: - file: promtool - - go/save-cache: - key: v1 - - store_test_results: - path: test-results - - test_ui: - executor: golang - - steps: - - checkout - - restore_cache: - keys: - - v3-npm-deps-{{ checksum "web/ui/package-lock.json" }} - - v3-npm-deps- - - run: make assets-tarball - - run: make ui-lint - - run: make ui-test - - persist_to_workspace: - root: . - paths: - - .tarballs - - save_cache: - key: v3-npm-deps-{{ checksum "web/ui/package-lock.json" }} - paths: - - ~/.npm - - test_windows: - executor: - name: win/default - shell: powershell - working_directory: /go/src/github.com/prometheus/prometheus - steps: - - checkout - - run: - # Temporary workaround until circleci updates go. - command: | - choco upgrade -y golang - - run: - command: refreshenv - - run: - command: | - $TestTargets = go list ./... | Where-Object { $_ -NotMatch "(github.com/prometheus/prometheus/discovery.*|github.com/prometheus/prometheus/config|github.com/prometheus/prometheus/web)"} - go test $TestTargets -vet=off -v - environment: - GOGC: "20" - GOOPTS: "-p 2" - - test_golang_oldest: - executor: golang_oldest - steps: - - checkout - - run: make build - - run: go test ./tsdb/... - - run: go test ./tsdb/ -test.tsdb-isolation=false - - test_mixins: - executor: golang - steps: - - checkout - - run: go install ./cmd/promtool/. - - run: go install github.com/google/go-jsonnet/cmd/jsonnet@latest - - run: go install github.com/google/go-jsonnet/cmd/jsonnetfmt@latest - - run: go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest - - run: make -C documentation/prometheus-mixin clean - - run: make -C documentation/prometheus-mixin jb_install - - run: make -C documentation/prometheus-mixin - - run: git diff --exit-code - - repo_sync: - executor: golang - steps: - - checkout - - run: ./scripts/sync_repo_files.sh + command: "true" workflows: version: 2 prometheus: jobs: - - test_go: - filters: - tags: - only: /.*/ - - test_ui: - filters: - tags: - only: /.*/ - - test_golang_oldest: - filters: - tags: - only: /.*/ - - test_mixins: - filters: - tags: - only: /.*/ - - test_windows: - filters: - branches: - only: /^(release-.*|.*build-all.*)$/ - tags: - only: /^v2(\.[0-9]+){2}(-.+|[^-.]*)$/ - - prometheus/build: - name: build - parallelism: 3 - promu_opts: "-p linux/amd64 -p windows/amd64 -p linux/arm64 -p darwin/amd64 -p darwin/arm64 -p linux/386" - filters: - tags: - ignore: /^v2(\.[0-9]+){2}(-.+|[^-.]*)$/ - branches: - ignore: /^(main|release-.*|.*build-all.*)$/ - - prometheus/build: - name: build_all - parallelism: 12 - filters: - branches: - only: /^(main|release-.*|.*build-all.*)$/ - tags: - only: /^v2(\.[0-9]+){2}(-.+|[^-.]*)$/ - - prometheus/publish_main: - context: org-context - requires: - - test_go - - test_ui - - build_all - filters: - branches: - only: main - image: circleci/golang:1-node - - prometheus/publish_release: - context: org-context - requires: - - test_go - - test_ui - - build_all - filters: - tags: - only: /^v2(\.[0-9]+){2}(-.+|[^-.]*)$/ - branches: - ignore: /.*/ - image: circleci/golang:1-node - daily: + - noopjob triggers: - schedule: - cron: "49 19 * * *" + cron: "0 0 30 2 *" filters: branches: only: - main - jobs: - - repo_sync: - context: org-context diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml new file mode 100644 index 000000000..9c11c5f46 --- /dev/null +++ b/.github/actions/build/action.yml @@ -0,0 +1,24 @@ +name: Build +inputs: + thread: + type: integer + description: Current thread + required: true + default: 3 + parallelism: + type: integer + description: Number of builds to do in parallel + default: 3 + promu_opts: + type: string + description: Options to pass to promu +runs: + using: composite + steps: + - uses: ./.github/actions/setup_environment + - run: ~/go/bin/promu crossbuild -v --parallelism ${{ inputs.parallelism }} --parallelism-thread ${{ inputs.thread }} ${{ inputs.promu_opts }} + shell: bash + - uses: actions/upload-artifact@v3 + with: + path: | + .build diff --git a/.github/actions/check_proto/action.yml b/.github/actions/check_proto/action.yml new file mode 100644 index 000000000..9d9d773e6 --- /dev/null +++ b/.github/actions/check_proto/action.yml @@ -0,0 +1,20 @@ +name: Check proto files +inputs: + version: + type: string + description: Protoc version + default: "3.5.1" +runs: + using: composite + steps: + - run: | + env + set -x + curl -s -L https://github.com/protocolbuffers/protobuf/releases/download/v${{ inputs.version }}/protoc-${{ inputs.version }}-linux-x86_64.zip > /tmp/protoc.zip + unzip -d /tmp /tmp/protoc.zip + chmod +x /tmp/bin/protoc + export PATH=/tmp/bin:$PATH + make proto + shell: bash + - run: git diff --exit-code + shell: bash diff --git a/.github/actions/publish_images/action.yml b/.github/actions/publish_images/action.yml new file mode 100644 index 000000000..fd21fd22d --- /dev/null +++ b/.github/actions/publish_images/action.yml @@ -0,0 +1,46 @@ +name: Publish image +inputs: + registry: + type: string + description: Docker registry + organization: + type: string + description: Organization + login: + type: string + description: Username + password: + type: string + description: Password + dockerfile_path: + description: Path to Dockerfile + type: string + default: "" + dockerbuild_context: + description: Path to Dockerbuild context + type: string + default: "" + container_image_name: + description: Name of the container image + type: string + default: "" +runs: + using: composite + steps: + - name: Download all workflow run artifacts + uses: actions/download-artifact@v3 + - uses: ./.github/actions/setup_environment + - if: inputs.dockerfile_path != "" + run: echo "export DOCKERFILE_PATH=${{ inputs.dockerfile_path }}" >> /tmp/tmp-profile + - if: inputs.container_image_name != "" + run: echo "export DOCKER_IMAGE_NAME=${{ inputs.container_image_name }}" >> /tmp/tmp-profile + - if: inputs.dockerbuild_context != "" + run: echo "export DOCKERBUILD_CONTEXT=${{ inputs.dockerbuild_context }}" >> /tmp/tmp-profile + - run: | + touch /tmp/tmp-profile + . /tmp/tmp-profile + make docker DOCKER_REPO=${{ inputs.registry }}/${{ inputs.organization }} + docker images + echo ${{ inputs.password }} | docker login -u ${{ inputs.login }} --password-stdin ${{ inputs.registry }} + make docker-publish DOCKER_REPO=${{ inputs.registry }}/${{ inputs.organization }} + make docker-manifest DOCKER_REPO=${{ inputs.registry }}/${{ inputs.organization }} diff --git a/.github/actions/publish_main/action.yml b/.github/actions/publish_main/action.yml new file mode 100644 index 000000000..08d57d9a6 --- /dev/null +++ b/.github/actions/publish_main/action.yml @@ -0,0 +1,40 @@ +name: Publish image +inputs: + docker_hub_organization: + type: string + description: DockerHub organization + default: prom + docker_hub_login: + type: string + description: DockerHub username + docker_hub_password: + type: string + description: DockerHub password + quay_io_organization: + type: string + description: Quay.io organization + default: prometheus + quay_io_login: + type: string + description: Quay.io username + quay_io_password: + type: string + description: Quay.io password +runs: + using: composite + steps: + - uses: ./.github/actions/setup_environment + - uses: ./.github/actions/publish_images + if: inputs.docker_hub_organization != '' && inputs.docker_hub_login != '' + with: + registry: docker.io + organization: ${{ inputs.docker_hub_organization }} + login: ${{ inputs.docker_hub_login }} + password: ${{ inputs.docker_hub_password }} + - uses: ./.github/actions/publish_images + if: inputs.quay_io_organization != '' && inputs.quay_io_login != '' + with: + registry: quay.io + organization: ${{ inputs.quay_io_organization }} + login: ${{ inputs.quay_io_login }} + password: ${{ inputs.quay_io_password }} diff --git a/.github/actions/publish_release/action.yml b/.github/actions/publish_release/action.yml new file mode 100644 index 000000000..56f68bda6 --- /dev/null +++ b/.github/actions/publish_release/action.yml @@ -0,0 +1,46 @@ +name: Publish image +inputs: + docker_hub_organization: + type: string + description: DockerHub organization + default: prom + docker_hub_login: + type: string + description: DockerHub username + docker_hub_password: + type: string + description: DockerHub password + quay_io_organization: + type: string + description: Quay.io organization + default: prometheus + quay_io_login: + type: string + description: Quay.io username + quay_io_password: + type: string + description: Quay.io password +runs: + using: composite + steps: + - uses: ./.github/actions/setup_environment + - run: promu crossbuild tarballs + shell: bash + - run: promu checksum .tarballs + shell: bash + - run: promu release .tarballs + shell: bash + - uses: ./.github/actions/publish_release_images + if: inputs.docker_hub_organization != '' && inputs.docker_hub_login != '' + with: + registry: docker.io + organization: ${{ inputs.docker_hub_organization }} + login: ${{ inputs.docker_hub_login }} + password: ${{ inputs.docker_hub_password }} + - uses: ./.github/actions/publish_release_images + if: inputs.quay_io_organization != '' && inputs.quay_io_login != '' + with: + registry: quay.io + organization: ${{ inputs.quay_io_organization }} + login: ${{ inputs.quay_io_login }} + password: ${{ inputs.quay_io_password }} diff --git a/.github/actions/publish_release_image/action.yml b/.github/actions/publish_release_image/action.yml new file mode 100644 index 000000000..f8b8ab1ee --- /dev/null +++ b/.github/actions/publish_release_image/action.yml @@ -0,0 +1,52 @@ +name: Publish release image +inputs: + registry: + type: string + description: Docker registry + organization: + type: string + description: Organization + login: + type: string + description: Username + password: + type: string + description: Password + dockerfile_path: + description: Path to Dockerfile + type: string + default: "" + dockerbuild_context: + description: Path to Dockerbuild context + type: string + default: "" + container_image_name: + description: Name of the container image + type: string + default: "" +runs: + using: composite + steps: + - name: Download all workflow run artifacts + uses: actions/download-artifact@v3 + - uses: ./.github/actions/setup_environment + - if: inputs.dockerfile_path != "" + run: echo "export DOCKERFILE_PATH=${{ inputs.dockerfile_path }}" }} /tmp/tmp-profile + - if: inputs.container_image_name != "" + run: echo "export DOCKER_IMAGE_NAME=${{ inputs.container_image_name }}" }} /tmp/tmp-profile + - if: inputs.dockerbuild_context != "" + run: echo "export DOCKERBUILD_CONTEXT=${{ inputs.dockerbuild_context }}" }} /tmp/tmp-profile + - run: | + current_tag=${GITHUB_REF#refs/*/} + touch /tmp/tmp-profile + . /tmp/tmp-profile + make docker DOCKER_IMAGE_TAG="$current_tag" DOCKER_REPO=${{ inputs.registry }}/${{ inputs.organization }} + docker images + echo ${{ inputs.password }} | docker login -u ${{ inputs.login }} --password-stdin ${{ inputs.registry }} + make docker-publish DOCKER_IMAGE_TAG="$current_tag" DOCKER_REPO=${{ inputs.registry }}/${{ inputs.organization }} + make docker-manifest DOCKER_IMAGE_TAG="$current_tag" DOCKER_REPO=${{ inputs.registry }}/${{ inputs.organization }} + if [[ "$current_tag" =~ ^v[0-9]+(\.[0-9]+){2}$ ]]; then + make docker-tag-latest DOCKER_IMAGE_TAG="$current_tag" DOCKER_REPO=${{ input.registry }}/${{ input.organization }} + make docker-publish DOCKER_IMAGE_TAG="latest" DOCKER_REPO=${{ input.registry }}/${{ input.organization }} + make docker-manifest DOCKER_IMAGE_TAG="latest" DOCKER_REPO=${{ input.registry }}/${{ input.organization }} + fi diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml new file mode 100644 index 000000000..f1258c8be --- /dev/null +++ b/.github/actions/setup_environment/action.yml @@ -0,0 +1,33 @@ +name: Setup environment +inputs: + enable_go: + type: boolean + description: Whether to enable go specific features, such as caching. + default: true + enable_npm: + type: boolean + description: Whether to enable npm specific features, such as caching. + default: true +runs: + using: composite + steps: + - uses: actions/cache@v3 + if: ${{ inputs.enable_go }} + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - uses: actions/cache@v3 + if: ${{ inputs.enable_npm }} + with: + path: | + ~/.npm + key: ${{ runner.os }}-npm-${{ hashFiles('web/ui/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-npm- + - run: make promu + shell: bash + if: ${{ inputs.enable_go }} diff --git a/.github/workflows/repo_sync.yml b/.github/workflows/repo_sync.yml new file mode 100644 index 000000000..ca8197878 --- /dev/null +++ b/.github/workflows/repo_sync.yml @@ -0,0 +1,12 @@ +--- +on: + schedule: + - cron: '44 17 * * *' +jobs: + repo_sync: + runs-on: ubuntu-latest + container: + image: quay.io/prometheus/golang-builder + steps: + - uses: actions/checkout@v3 + - run: ./scripts/sync_repo_files.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..48b47670b --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,147 @@ +--- +name: tests +on: + pull_request: + push: +jobs: + test_go: + runs-on: ubuntu-latest + # Whenever the Go version is updated here, .promu.yml + # should also be updated. + container: + image: quay.io/prometheus/golang-builder:1.18-base + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup_environment + - run: make GO_ONLY=1 SKIP_GOLANGCI_LINT=1 + - run: go test ./tsdb/ -test.tsdb-isolation=false + - run: make -C documentation/examples/remote_storage + - run: make -C documentation/examples + - uses: ./.github/actions/check_proto + with: + version: "3.15.8" + + test_ui: + runs-on: ubuntu-latest + # Whenever the Go version is updated here, .promu.yml + # should also be updated. + container: + image: quay.io/prometheus/golang-builder:1.18-base + + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup_environment + with: + enable_go: false + enable_npm: true + - run: make assets-tarball + - run: make ui-lint + - run: make ui-test + - uses: actions/upload-artifact@v3 + with: + path: | + .tarballs + + test_windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: '<1.19' + - run: | + $TestTargets = go list ./... | Where-Object { $_ -NotMatch "(github.com/prometheus/prometheus/discovery.*|github.com/prometheus/prometheus/config|github.com/prometheus/prometheus/web)"} + go test $TestTargets -vet=off -v + shell: powershell + + test_golang_oldest: + runs-on: ubuntu-latest + # The go verson in this image should be N-1 wrt test_go. + container: + image: quay.io/prometheus/golang-builder:1.17-base + steps: + - uses: actions/checkout@v3 + - run: make build + - run: go test ./tsdb/... + - run: go test ./tsdb/ -test.tsdb-isolation=false + + test_mixins: + runs-on: ubuntu-latest + # Whenever the Go version is updated here, .promu.yml + # should also be updated. + container: + image: quay.io/prometheus/golang-builder:1.18-base + steps: + - uses: actions/checkout@v3 + - run: go install ./cmd/promtool/. + - run: go install github.com/google/go-jsonnet/cmd/jsonnet@latest + - run: go install github.com/google/go-jsonnet/cmd/jsonnetfmt@latest + - run: go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest + - run: make -C documentation/prometheus-mixin clean + - run: make -C documentation/prometheus-mixin jb_install + - run: make -C documentation/prometheus-mixin + - run: git diff --exit-code + + build: + runs-on: ubuntu-latest + if: | + !(github.event_name == 'create' && github.event.ref_type != 'tag' && !startsWith(github.event.ref, 'refs/tags/v2.')) + && + !(github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-')) + && + !(github.event_name == 'push' && github.event.ref == 'refs/heads/main') + strategy: + matrix: + thread: [ 0, 1, 2 ] + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/build + with: + promu_opts: "-p linux/amd64 -p windows/amd64 -p linux/arm64 -p darwin/amd64 -p darwin/arm64 -p linux/386" + parallelism: 3 + thread: ${{ matrix.thread }} + build_all: + runs-on: ubuntu-latest + if: | + (github.event_name == 'create' && github.event.ref_type != 'tag' && !startsWith(github.event.ref, 'refs/tags/v2.')) + || + (github.event_name == 'pull_request' && startsWith(github.event.pull_request.base.ref, 'release-')) + || + (github.event_name == 'push' && github.event.ref == 'refs/heads/main') + strategy: + matrix: + thread: [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ] + + # Whenever the Go version is updated here, .promu.yml + # should also be updated. + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/build + with: + parallelism: 12 + thread: ${{ matrix.thread }} + + publish_main: + runs-on: ubuntu-latest + needs: [test_ui, test_go, test_windows, build_all] + if: "github.event_name == 'push' && github.event.ref == 'refs/heads/main'" + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/publish_main + with: + docker_hub_login: ${{ secrets.docker_hub_login }} + docker_hub_password: ${{ secrets.docker_hub_password }} + quay_io_login: ${{ secrets.quay_io_login }} + quay_io_password: ${{ secrets.quay_io_password }} + publish_release: + runs-on: ubuntu-latest + needs: [test_ui, test_go, test_windows, build_all] + if: github.event_name == 'create' && github.event.ref_type == 'tag' && startsWith(github.event.ref, 'refs/tags/v2.') + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/publish_release + with: + docker_hub_login: ${{ secrets.docker_hub_login }} + docker_hub_password: ${{ secrets.docker_hub_password }} + quay_io_login: ${{ secrets.quay_io_login }} + quay_io_password: ${{ secrets.quay_io_password }} diff --git a/Makefile.common b/Makefile.common index 6c8e3e219..ada0008a0 100644 --- a/Makefile.common +++ b/Makefile.common @@ -58,6 +58,7 @@ endif PROMU_VERSION ?= 0.13.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz +SKIP_GOLANGCI_LINT := GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= GOLANGCI_LINT_VERSION ?= v1.45.2 @@ -67,7 +68,9 @@ ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386)) # If we're in CI and there is an Actions file, that means the linter # is being run in Actions, so we don't need to run it here. - ifeq (,$(CIRCLE_JOB)) + ifneq (,$(SKIP_GOLANGCI_LINT)) + GOLANGCI_LINT := + else ifeq (,$(CIRCLE_JOB)) GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint else ifeq (,$(wildcard .github/workflows/golangci-lint.yml)) GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint