From 4f775f56b7dd2c1111a2c286eea166a99e80f7db Mon Sep 17 00:00:00 2001 From: Sam McCall Date: Mon, 17 Feb 2020 18:41:16 +0100 Subject: [PATCH] Add automatic weekly snapshot releases with Github Actions (#278) Add automatic weekly snapshot releases with Github Actions --- .github/actions/pick/action.yaml | 15 +++ .github/actions/pick/index.js | 18 +++ .github/actions/pick/package-lock.json | 56 +++++++++ .github/actions/pick/package.json | 14 +++ .github/workflows/autobuild.yaml | 153 +++++++++++++++++++++++++ .github/workflows/periodic.yaml | 55 +++++++++ .gitignore | 2 + releases.md | 54 +++++++++ 8 files changed, 367 insertions(+) create mode 100644 .github/actions/pick/action.yaml create mode 100644 .github/actions/pick/index.js create mode 100644 .github/actions/pick/package-lock.json create mode 100644 .github/actions/pick/package.json create mode 100644 .github/workflows/autobuild.yaml create mode 100644 .github/workflows/periodic.yaml create mode 100644 .gitignore create mode 100644 releases.md diff --git a/.github/actions/pick/action.yaml b/.github/actions/pick/action.yaml new file mode 100644 index 0000000..cce59a4 --- /dev/null +++ b/.github/actions/pick/action.yaml @@ -0,0 +1,15 @@ +name: 'Pick commit' +description: 'Pick last successful commit from a repo' +inputs: + repo: + description: 'Github repository e.g. llvm/llvm-project' + required: true + token: + description: 'Github auth token' + required: true +outputs: + sha: + description: 'The SHA1 of the picked commit' +runs: + using: 'node12' + main: 'index.js' diff --git a/.github/actions/pick/index.js b/.github/actions/pick/index.js new file mode 100644 index 0000000..189d10d --- /dev/null +++ b/.github/actions/pick/index.js @@ -0,0 +1,18 @@ +const core = require('@actions/core'); +const last = require('last-successful-gh-commit').default; +try { + var parts = core.getInput('repo').split('/', 2); + last({ + owner: parts[0], + name: parts[1], + token: core.getInput('token'), + }) + .then(commit => core.setOutput('sha', commit.node.oid)) + .catch(error => { + console.error(error); + core.setFailed(error.message); + }); +} catch (error) { + console.error(error); + core.setFailed(error.message); +} diff --git a/.github/actions/pick/package-lock.json b/.github/actions/pick/package-lock.json new file mode 100644 index 0000000..3fb33c1 --- /dev/null +++ b/.github/actions/pick/package-lock.json @@ -0,0 +1,56 @@ +{ + "name": "pick", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@actions/core": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.2.tgz", + "integrity": "sha512-IbCx7oefq+Gi6FWbSs2Fnw8VkEI6Y4gvjrYprY3RV//ksq/KPMlClOerJ4jRosyal6zkUIc8R9fS/cpRMlGClg==" + }, + "axios": { + "version": "0.16.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.16.2.tgz", + "integrity": "sha1-uk+S8XFn37q0CYN4VFS5rBScPG0=", + "requires": { + "follow-redirects": "^1.2.3", + "is-buffer": "^1.1.5" + } + }, + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "follow-redirects": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.10.0.tgz", + "integrity": "sha512-4eyLK6s6lH32nOvLLwlIOnr9zrL8Sm+OvW4pVTJNoXeGzYIkHVf+pADQi+OJ0E67hiuSLezPVPyBcIZO50TmmQ==", + "requires": { + "debug": "^3.0.0" + } + }, + "is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, + "last-successful-gh-commit": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/last-successful-gh-commit/-/last-successful-gh-commit-1.0.1.tgz", + "integrity": "sha1-8Mn8HfEvb764g/5vrcy3W4p5EVo=", + "requires": { + "axios": "^0.16.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } +} diff --git a/.github/actions/pick/package.json b/.github/actions/pick/package.json new file mode 100644 index 0000000..1f68268 --- /dev/null +++ b/.github/actions/pick/package.json @@ -0,0 +1,14 @@ +{ + "name": "pick", + "version": "1.0.0", + "description": "Pick the last successful commit from a github repo", + "main": "index.js", + "scripts": {}, + "keywords": [], + "author": "sam.mccall@gmail.com", + "license": "Apache-2.0", + "dependencies": { + "@actions/core": "^1.2.2", + "last-successful-gh-commit": "^1.0.1" + } +} diff --git a/.github/workflows/autobuild.yaml b/.github/workflows/autobuild.yaml new file mode 100644 index 0000000..3825f52 --- /dev/null +++ b/.github/workflows/autobuild.yaml @@ -0,0 +1,153 @@ +# Workflow to build binaries for a release. +# Triggered by release creation, which should include `llvm-project@`. +# +# Because the build takes more than an hour, our GITHUB_TOKEN credentials may +# expire. A token `secrets.RELEASE_TOKEN` must exist with public_repo scope. +name: Build release binaries +on: + release: + types: created +jobs: + prepare: + runs-on: ubuntu-latest + steps: + - name: Find LLVM commit + run: | + grep -m 1 -o "llvm-project@[[:xdigit:]]\{40,\}" << EOF | cut -f 2 -d@ > commit + ${{ github.event.release.body }} + EOF + - name: Mark as draft + run: > + curl --fail --show-error -XPATCH + "-HAuthorization: Bearer ${{ secrets.RELEASE_TOKEN }}" + "https://api.github.com/repos/${{ github.repository }}/releases/${{ github.event.release.id }}" + "-d" '{"draft": true}' + - name: Persist release info + uses: actions/upload-artifact@v1 + with: + name: release + path: commit + # Build clangd using CMake/Ninja. + # + # This step is a template that runs on each OS, build config varies slightly. + # Uploading releases needs a per-job token that expires after an hour. + build: + name: Build ${{ matrix.config.name }} + needs: prepare + runs-on: ${{ matrix.config.os }} + strategy: + fail-fast: false + matrix: + config: + - name: windows + os: windows-latest + preinstall: choco install ninja + vcvars: C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat + # FIXME: remove ALLOW_OLD_TOOLCHAIN once VS 16.5 is available. + cmake: > + "-DCMAKE_C_COMPILER=cl" + "-DCMAKE_CXX_COMPILER=cl" + "-DCMAKE_CXX_FLAGS_RELEASE=/O2 /DNDEBUG" + "-DCMAKE_C_FLAGS_RELEASE=/O2 /DNDEBUG" + "-DLLVM_ENABLE_ZLIB=OFF" + "-DLLVM_USE_CRT_RELEASE=MT" + "-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON" + - name: mac + os: macos-latest + preinstall: brew install ninja zlib p7zip + cmake: > + "-DCMAKE_C_COMPILER=clang" + "-DCMAKE_CXX_COMPILER=clang++" + "-DCMAKE_C_FLAGS_RELEASE=-O3 -gline-tables-only -DNDEBUG" + "-DCMAKE_CXX_FLAGS_RELEASE=-O3 -gline-tables-only -DNDEBUG" + "-DLLVM_ENABLE_ZLIB=FORCE_ON" + - name: linux + os: ubuntu-latest + preinstall: sudo apt-get install ninja-build libz-dev + cmake: > + "-DCMAKE_C_COMPILER=clang" + "-DCMAKE_CXX_COMPILER=clang++" + "-DCMAKE_CXX_FLAGS_RELEASE=-O3 -gline-tables-only -DNDEBUG" + "-DCMAKE_C_FLAGS_RELEASE=-O3 -gline-tables-only -DNDEBUG" + "-DCMAKE_EXE_LINKER_FLAGS_RELEASE=-static-libgcc -Wl,--compress-debug-sections=zlib" + "-DLLVM_STATIC_LINK_CXX_STDLIB=ON" + "-DLLVM_ENABLE_ZLIB=FORCE_ON" + "-DZLIB_LIBRARY=/usr/lib/x86_64-linux-gnu/libz.a" + steps: + - name: Clone scripts + uses: actions/checkout@v2 + with: { ref: master } + - name: Install tools + run: ${{ matrix.config.preinstall }} + # Visual Studio tools require a bunch of environment variables to be set. + # Run vcvars64.bat and re-export the current environment to the workflow. + # (It'd be nice to only export the variables that *changed*, oh well). + - name: Visual Studio environment + if: matrix.config.name == 'windows' + shell: powershell + run: | + cmd /c "`"${{ matrix.config.vcvars }}`">NUL && set" | Foreach-Object { + $name, $value = $_ -split '=', 2 + if ($value) { echo "::set-env name=$($name)::$($value)" } + } + - name: Fetch target commit + uses: actions/download-artifact@v1 + with: { name: release } + - name: Set target commit + run: | + echo ::set-env name=LLVM_COMMIT::$(cat release/commit) + echo ::set-env name=CLANGD_DIR::clangd_${{ github.event.release.tag_name }} + shell: bash + - name: Clone LLVM + uses: actions/checkout@v2 + with: + repository: llvm/llvm-project + path: llvm-project + ref: ${{ env.LLVM_COMMIT }} + - name: CMake + run: > + mkdir ${{ env.CLANGD_DIR }} + + cp llvm-project/llvm/LICENSE.TXT ${{ env.CLANGD_DIR }} + + cmake -G Ninja -S llvm-project/llvm -B ${{ env.CLANGD_DIR }} + "-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra" + "-DLLVM_ENABLE_ASSERTIONS=OFF" + "-DLLVM_ENABLE_BACKTRACES=ON" + "-DLLVM_ENABLE_TERMINFO=OFF" + "-DCMAKE_BUILD_TYPE=Release" + "-DCLANG_PLUGIN_SUPPORT=OFF" + "-DLLVM_ENABLE_PLUGINS=OFF" + ${{ matrix.config.cmake }} + - name: Ninja + run: ninja -C ${{ env.CLANGD_DIR }} clangd + - name: Archive + run: > + 7z a clangd.zip + ${{ env.CLANGD_DIR }}/LICENSE.TXT + ${{ env.CLANGD_DIR }}/bin/clangd* + ${{ env.CLANGD_DIR }}/lib/clang + - name: Upload artifact + uses: actions/upload-artifact@v1 + with: + name: ${{ matrix.config.name }} + path: clangd.zip + - name: Upload asset + uses: actions/upload-release-asset@v1.0.1 + env: { GITHUB_TOKEN: "${{ secrets.RELEASE_TOKEN }}" } + with: + upload_url: ${{ github.event.release.upload_url }} + asset_name: clangd-${{ matrix.config.name }}-${{ github.event.release.tag_name }}.zip + asset_path: clangd.zip + asset_content_type: application/zip + # Create the release, and upload the artifacts to it. + finalize: + runs-on: ubuntu-latest + needs: build + steps: + - name: Publish release + run: > + curl -XPATCH + "-HAuthorization: Bearer ${{ secrets.RELEASE_TOKEN }}" + "https://api.github.com/repos/${{ github.repository }}/releases/${{ github.event.release.id }}" + "-d" '{"draft": false}' diff --git a/.github/workflows/periodic.yaml b/.github/workflows/periodic.yaml new file mode 100644 index 0000000..1eb8d72 --- /dev/null +++ b/.github/workflows/periodic.yaml @@ -0,0 +1,55 @@ +# Workflow to create an auto-buildable release periodically. +# +# Releases created with automation credentials don't trigger workflows. +# Therefore a token `secrets.RELEASE_TOKEN` must exist with public_repo scope. +name: Periodic release +on: + # Run weekly on sunday at 21:37 UTC (arbitrary) + schedule: + - cron: '37 21 * * SUN' + # Allow triggering manually: + # curl -XPOST -d '{"event_type":"periodic"}' \ + # "-HAuthorization: Bearer " \ + # https://api.github.com/repos/clangd/clangd/dispatches + repository_dispatch: { types: periodic } +jobs: + # Choose the commit to build a release from. + # + # We want to avoid unbuildable revisions: choose the last green from CI. + # FIXME: the criteria should be some consistent set of buildbots passing. + pick: + name: Create draft release + runs-on: ubuntu-latest + steps: + - name: Clone scripts + uses: actions/checkout@v2 + - name: Install NPM dependencies + run: npm install --prefix .github/actions/pick + - name: Pick last successful commit + id: pick + uses: ./.github/actions/pick + with: + repo: llvm/llvm-project + token: ${{ secrets.GITHUB_TOKEN }} + - name: Compute release info + run: | + echo ::set-env name=RELEASE_COMMIT_SHORT::$(printf "%.12s" ${{ steps.pick.outputs.sha }}) + echo ::set-env name=RELEASE_DATE::$(date -u +%Y%m%d) + - name: Create release + uses: actions/create-release@master + id: release + env: { GITHUB_TOKEN: "${{ secrets.RELEASE_TOKEN }}" } + with: + tag_name: snapshot_${{ env.RELEASE_DATE }} + release_name: ${{ env.RELEASE_DATE }} @${{ env.RELEASE_COMMIT_SHORT }} + body: | + Unstable snapshot of clangd on ${{ env.RELEASE_DATE }}. + + Built from llvm/llvm-project@${{ steps.pick.outputs.sha }}. + prerelease: true + # It would be nice to use draft releases, to hide them from users. + # But drafts don't fire events to trigger the autobuild workflow. + # Instead, that workflow marks the release as draft until it's built. + # As a result, the empty release will be briefly visible to users. + draft: false + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d570088 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +node_modules/ + diff --git a/releases.md b/releases.md new file mode 100644 index 0000000..68c11d0 --- /dev/null +++ b/releases.md @@ -0,0 +1,54 @@ +# Binary releases + +This repository has automation to build binary releases of clangd for the most +common platforms. This doesn't cover as many systems as the official releases +from http://releases.llvm.org/, and distro packages etc. The main advantages +is being able to cut releases easily whenever we want. + +The releases are just a zip archive containing the `clangd` binary, and the +clang builtin headers. They should be runnable immediately after extracting the +archive. The linux binary has `libstdc++` and other dependencies statically +linked for maximum portability. + +## Creating a release manually + +In GitHub, click the "Releases" tab and create a new release. +The tag name is significant, it's used in the directory name (`clangd_tagname`). + +Because clangd sources don't live in this repository, the release description +must contain a magic string indicating the revision to build at, e.g. + + Built at llvm/llvm-project@0399d5a9682b3cef71c653373e38890c63c4c365 + +This must be a full SHA, not a short-hash, a tag or branch name, etc. +The release must **not** be a draft release, as that won't trigger automation. + +## Building release binaries + +Creating the release will trigger the `autobuild` workflow, visible under the +"Actions" tab. This runs a sequence of steps: + +- First, the release is marked as a draft, so it's invisible to non-owners. +- Next, the sources are checked out and built on each of {mac, windows, linux}. + These run in parallel, and the results are zipped and added to the release. +- If all builds succeeded, the release is published (marked as non-draft) + +## Automatic snapshot releases + +The `periodic` workflow runs on a weekly schedule and creates a release based +on the last green revision at llvm/llvm-project. +This in turn triggers the `autobuild` workflow above to produce binaries. + +These snapshot releases are marked as "pre-release" - they don't undergo any +serious testing and so aren't particularly stable. However they're useful for +people to try the latest clangd. + +## Credentials + +Rather than the default GitHub Actions access token, these actions use a +Personal Access Token added to the repository as a secret. This is because: + +- The default access token expires after an hour. Builds take longer than that. +- Releases created using the default access token don't trigger workflows. + +If you fork the repository, you must provide the `RELEASE_TOKEN` secret.