From 839115df8e713cde01d9d2bd16d94249bd19a394 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 22 Oct 2022 14:09:44 +0200 Subject: [PATCH] separate snapshot and release deploy jobs (#5669) --- .github/python-version.txt | 1 + .github/workflows/main.yml | 97 +++++++++++++++++++++----------------- release/build.py | 13 +++-- setup.py | 2 +- 4 files changed, 61 insertions(+), 52 deletions(-) create mode 100644 .github/python-version.txt diff --git a/.github/python-version.txt b/.github/python-version.txt new file mode 100644 index 000000000..c8cfe3959 --- /dev/null +++ b/.github/python-version.txt @@ -0,0 +1 @@ +3.10 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fba11b4ab..fc76e5fe9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,58 +10,58 @@ jobs: if: github.event_name == 'pull_request' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: persist-credentials: false - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version-file: .github/python-version.txt - uses: TrueBrain/actions-flake8@c2deca24d388aa5aedd6478332aa9df4600b5eac # v2.1 # mirrored at https://github.com/mitmproxy/mitmproxy/settings/actions lint-local: if: github.event_name == 'push' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: persist-credentials: false - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version-file: .github/python-version.txt - run: pip install tox - run: tox -e flake8 filename-matching: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: persist-credentials: false - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version-file: .github/python-version.txt - run: pip install tox - run: tox -e filename_matching mypy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: persist-credentials: false - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version-file: .github/python-version.txt - run: pip install tox - run: tox -e mypy individual-coverage: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: persist-credentials: false fetch-depth: 0 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: '3.9' # there's a weird bug on 3.10 where some lines are not counted as covered. + python-version-file: .github/python-version.txt - run: pip install tox - run: tox -e individual_coverage test: @@ -80,11 +80,11 @@ jobs: runs-on: ${{ matrix.os }} steps: - run: printenv - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: persist-credentials: false fetch-depth: 0 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} - run: pip install tox @@ -112,21 +112,21 @@ jobs: platform: macos - image: windows-2019 platform: windows - - image: ubuntu-20.04 # Oldest avalible version so we get oldest glibc possible. + - image: ubuntu-20.04 # Oldest available version so we get oldest glibc possible. platform: linux runs-on: ${{ matrix.image }} env: CI_BUILD_KEY: ${{ secrets.CI_BUILD_KEY }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: persist-credentials: false fetch-depth: 0 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version-file: .github/python-version.txt - if: matrix.platform == 'windows' - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: release/installbuilder/setup key: installbuilder @@ -146,15 +146,15 @@ jobs: test-web-ui: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: persist-credentials: false - run: git rev-parse --abbrev-ref HEAD - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 with: node-version: '14' - name: Cache Node.js modules - uses: actions/cache@v2 + uses: actions/cache@v3 with: # npm cache files are stored in `~/.npm` on Linux/macOS path: ~/.npm @@ -175,12 +175,12 @@ jobs: docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: persist-credentials: false - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version-file: .github/python-version.txt - run: | wget -q https://github.com/gohugoio/hugo/releases/download/v0.92.1/hugo_extended_0.92.1_Linux-64bit.deb echo "a9440adfd3ecce40089def287dee4e42ffae252ba08c77d1ac575b880a079ce6 hugo_extended_0.92.1_Linux-64bit.deb" | sha256sum -c @@ -201,21 +201,21 @@ jobs: ) environment: deploy-docker needs: - - test - - test-web-ui - - build - - docs + - test + - test-web-ui + - build + - docs runs-on: ubuntu-latest env: DOCKER_USERNAME: mitmbot DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: persist-credentials: false - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version-file: .github/python-version.txt - uses: actions/download-artifact@v2 with: name: binaries.linux @@ -227,8 +227,8 @@ jobs: deploy: # This action has access to our AWS keys, so we are extra careful here. # In particular, we don't blindly `pip install` anything to minimize the risk of supply chain attacks. - if: github.repository == 'mitmproxy/mitmproxy' && github.event_name == 'push' - environment: deploy + if: github.repository == 'mitmproxy/mitmproxy' && (startsWith(github.ref, 'refs/heads/') || startsWith(github.ref, 'refs/tags/')) + environment: ${{ (github.ref == 'refs/heads/citest' || startsWith(github.ref, 'refs/tags/')) && 'deploy-release' || 'deploy-snapshot' }} needs: - test - test-web-ui @@ -236,7 +236,9 @@ jobs: - docs runs-on: ubuntu-latest env: - TWINE_USERNAME: mitmproxy + # PyPI and MSFT keys are only available for the deploy-release environment + # The AWS access key for snapshots is scoped to branches/* as well. + TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -246,14 +248,17 @@ jobs: MSFT_CLIENT_ID: ${{ secrets.MSFT_CLIENT_ID }} MSFT_CLIENT_SECRET: ${{ secrets.MSFT_CLIENT_SECRET }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: persist-credentials: false - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version-file: .github/python-version.txt - run: sudo apt-get update - - run: sudo apt-get install -y twine awscli + - run: sudo apt-get install -y awscli + - if: startsWith(github.ref, 'refs/tags/') + run: sudo apt-get install -y twine + - uses: actions/download-artifact@v3 with: name: docs @@ -272,10 +277,14 @@ jobs: path: release/dist - run: ls docs/public - run: ls release/dist + - run: ./release/deploy.py - - if: github.ref == 'refs/heads/citest' + + - name: Deploy to Microsoft Store (test flight) + if: github.ref == 'refs/heads/citest' run: ./release/deploy-microsoft-store.py release/dist/*.msix env: MSFT_APP_FLIGHT: 174ca570-8cae-4444-9858-c07293f1f13a - - if: startsWith(github.ref, 'refs/tags/') + - name: Deploy to Microsoft Store + if: startsWith(github.ref, 'refs/tags/') run: ./release/deploy-microsoft-store.py release/dist/*.msix diff --git a/release/build.py b/release/build.py index 7931abe84..8d9b04e64 100644 --- a/release/build.py +++ b/release/build.py @@ -53,7 +53,11 @@ def wheel(): DIST_DIR, ] ) - (whl,) = DIST_DIR.glob("mitmproxy-*-py3-none-any.whl") + if os.environ.get("GITHUB_REF", "").startswith("refs/tags/"): + ver = version() # assert for tags that the version matches the tag. + else: + ver = "*" + (whl,) = DIST_DIR.glob(f"mitmproxy-{ver}-py3-none-any.whl") print(f"Found wheel package: {whl}") subprocess.check_call(["tox", "-e", "wheeltest", "--", whl]) @@ -80,12 +84,7 @@ def archive(path: Path) -> tarfile.TarFile | ZipFile2: def version() -> str: - if ref := os.environ.get("GITHUB_REF", ""): - if ref.startswith("refs/heads/"): - return ref.replace("refs/heads/", "") - if ref.startswith("refs/tags/"): - return ref.replace("refs/tags/", "") - return os.environ.get("BUILD_VERSION", "dev") + return os.environ.get("GITHUB_REF_NAME", "").replace("/", "-") or os.environ.get("BUILD_VERSION", "dev") def operating_system() -> Literal["windows", "linux", "macos", "unknown"]: diff --git a/setup.py b/setup.py index 5c2c4efc1..cbf32566b 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ with open(os.path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() long_description_content_type = "text/markdown" -with open(os.path.join(here, "mitmproxy", "version.py")) as f: +with open(os.path.join(here, "mitmproxy/version.py")) as f: match = re.search(r'VERSION = "(.+?)"', f.read()) assert match VERSION = match.group(1)