Add automatic weekly snapshot releases with Github Actions (#278)

Add automatic weekly snapshot releases with Github Actions

.github/actions/pick/action.yaml

@@ -0,0 +1,15 @@
+name: 'Pick commit'
+description: 'Pick last successful commit from a repo'
+inputs:
+  repo:
+    description: 'Github repository e.g. llvm/llvm-project'
+    required: true
+  token:
+    description: 'Github auth token'
+    required: true
+outputs:
+  sha:
+    description: 'The SHA1 of the picked commit'
+runs:
+  using: 'node12'
+  main: 'index.js'

.github/actions/pick/index.js

@@ -0,0 +1,18 @@
+const core = require('@actions/core');
+const last = require('last-successful-gh-commit').default;
+try {
+  var parts = core.getInput('repo').split('/', 2);
+  last({
+    owner: parts[0],
+    name: parts[1],
+    token: core.getInput('token'),
+  })
+  .then(commit => core.setOutput('sha', commit.node.oid))
+  .catch(error => {
+    console.error(error);
+    core.setFailed(error.message);
+  });
+} catch (error) {
+  console.error(error);
+  core.setFailed(error.message);
+}

.github/actions/pick/package-lock.json

@@ -0,0 +1,56 @@
+{
+  "name": "pick",
+  "version": "1.0.0",
+  "lockfileVersion": 1,
+  "requires": true,
+  "dependencies": {
+    "@actions/core": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.2.tgz",
+      "integrity": "sha512-IbCx7oefq+Gi6FWbSs2Fnw8VkEI6Y4gvjrYprY3RV//ksq/KPMlClOerJ4jRosyal6zkUIc8R9fS/cpRMlGClg=="
+    },
+    "axios": {
+      "version": "0.16.2",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.16.2.tgz",
+      "integrity": "sha1-uk+S8XFn37q0CYN4VFS5rBScPG0=",
+      "requires": {
+        "follow-redirects": "^1.2.3",
+        "is-buffer": "^1.1.5"
+      }
+    },
+    "debug": {
+      "version": "3.2.6",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
+      "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
+      "requires": {
+        "ms": "^2.1.1"
+      }
+    },
+    "follow-redirects": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.10.0.tgz",
+      "integrity": "sha512-4eyLK6s6lH32nOvLLwlIOnr9zrL8Sm+OvW4pVTJNoXeGzYIkHVf+pADQi+OJ0E67hiuSLezPVPyBcIZO50TmmQ==",
+      "requires": {
+        "debug": "^3.0.0"
+      }
+    },
+    "is-buffer": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
+    },
+    "last-successful-gh-commit": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/last-successful-gh-commit/-/last-successful-gh-commit-1.0.1.tgz",
+      "integrity": "sha1-8Mn8HfEvb764g/5vrcy3W4p5EVo=",
+      "requires": {
+        "axios": "^0.16.2"
+      }
+    },
+    "ms": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+    }
+  }
+}

.github/actions/pick/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "pick",
+  "version": "1.0.0",
+  "description": "Pick the last successful commit from a github repo",
+  "main": "index.js",
+  "scripts": {},
+  "keywords": [],
+  "author": "sam.mccall@gmail.com",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@actions/core": "^1.2.2",
+    "last-successful-gh-commit": "^1.0.1"
+  }
+}

.github/workflows/autobuild.yaml

@@ -0,0 +1,153 @@
+# Workflow to build binaries for a release.
+# Triggered by release creation, which should include `llvm-project@<full-sha>`.
+#
+# Because the build takes more than an hour, our GITHUB_TOKEN credentials may
+# expire. A token `secrets.RELEASE_TOKEN` must exist with public_repo scope.
+name: Build release binaries
+on:
+  release:
+    types: created
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Find LLVM commit
+      run: |
+        grep -m 1 -o "llvm-project@[[:xdigit:]]\{40,\}" << EOF | cut -f 2 -d@ > commit
+        ${{ github.event.release.body }}
+        EOF
+    - name: Mark as draft
+      run: >
+        curl --fail --show-error -XPATCH
+        "-HAuthorization: Bearer ${{ secrets.RELEASE_TOKEN }}"
+        "https://api.github.com/repos/${{ github.repository }}/releases/${{ github.event.release.id }}"
+        "-d" '{"draft": true}'
+    - name: Persist release info
+      uses: actions/upload-artifact@v1
+      with:
+        name: release
+        path: commit
+  # Build clangd using CMake/Ninja.
+  #
+  # This step is a template that runs on each OS, build config varies slightly.
+  # Uploading releases needs a per-job token that expires after an hour.
+  build:
+    name: Build ${{ matrix.config.name }}
+    needs: prepare
+    runs-on: ${{ matrix.config.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+        - name: windows
+          os: windows-latest
+          preinstall: choco install ninja
+          vcvars: C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat
+          # FIXME: remove ALLOW_OLD_TOOLCHAIN once VS 16.5 is available.
+          cmake: >
+            "-DCMAKE_C_COMPILER=cl"
+            "-DCMAKE_CXX_COMPILER=cl"
+            "-DCMAKE_CXX_FLAGS_RELEASE=/O2 /DNDEBUG"
+            "-DCMAKE_C_FLAGS_RELEASE=/O2 /DNDEBUG"
+            "-DLLVM_ENABLE_ZLIB=OFF"
+            "-DLLVM_USE_CRT_RELEASE=MT"
+            "-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON"
+        - name: mac
+          os: macos-latest
+          preinstall: brew install ninja zlib p7zip
+          cmake: >
+            "-DCMAKE_C_COMPILER=clang"
+            "-DCMAKE_CXX_COMPILER=clang++"
+            "-DCMAKE_C_FLAGS_RELEASE=-O3 -gline-tables-only -DNDEBUG"
+            "-DCMAKE_CXX_FLAGS_RELEASE=-O3 -gline-tables-only -DNDEBUG"
+            "-DLLVM_ENABLE_ZLIB=FORCE_ON"
+        - name: linux
+          os: ubuntu-latest
+          preinstall: sudo apt-get install ninja-build libz-dev
+          cmake: >
+            "-DCMAKE_C_COMPILER=clang"
+            "-DCMAKE_CXX_COMPILER=clang++"
+            "-DCMAKE_CXX_FLAGS_RELEASE=-O3 -gline-tables-only -DNDEBUG"
+            "-DCMAKE_C_FLAGS_RELEASE=-O3 -gline-tables-only -DNDEBUG"
+            "-DCMAKE_EXE_LINKER_FLAGS_RELEASE=-static-libgcc -Wl,--compress-debug-sections=zlib"
+            "-DLLVM_STATIC_LINK_CXX_STDLIB=ON"
+            "-DLLVM_ENABLE_ZLIB=FORCE_ON"
+            "-DZLIB_LIBRARY=/usr/lib/x86_64-linux-gnu/libz.a"
+    steps:
+    - name: Clone scripts
+      uses: actions/checkout@v2
+      with: { ref: master }
+    - name: Install tools
+      run: ${{ matrix.config.preinstall }}
+    # Visual Studio tools require a bunch of environment variables to be set.
+    # Run vcvars64.bat and re-export the current environment to the workflow.
+    # (It'd be nice to only export the variables that *changed*, oh well).
+    - name: Visual Studio environment
+      if: matrix.config.name == 'windows'
+      shell: powershell
+      run: |
+        cmd /c "`"${{ matrix.config.vcvars }}`">NUL && set" | Foreach-Object {
+          $name, $value = $_ -split '=', 2
+          if ($value) { echo "::set-env name=$($name)::$($value)" }
+        }
+    - name: Fetch target commit
+      uses: actions/download-artifact@v1
+      with: { name: release }
+    - name: Set target commit
+      run: |
+        echo ::set-env name=LLVM_COMMIT::$(cat release/commit)
+        echo ::set-env name=CLANGD_DIR::clangd_${{ github.event.release.tag_name }}
+      shell: bash
+    - name: Clone LLVM
+      uses: actions/checkout@v2
+      with:
+        repository: llvm/llvm-project
+        path: llvm-project
+        ref: ${{ env.LLVM_COMMIT }}
+    - name: CMake
+      run: >
+        mkdir ${{ env.CLANGD_DIR }}
+
+        cp llvm-project/llvm/LICENSE.TXT ${{ env.CLANGD_DIR }}
+
+        cmake -G Ninja -S llvm-project/llvm -B ${{ env.CLANGD_DIR }}
+        "-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra"
+        "-DLLVM_ENABLE_ASSERTIONS=OFF"
+        "-DLLVM_ENABLE_BACKTRACES=ON"
+        "-DLLVM_ENABLE_TERMINFO=OFF"
+        "-DCMAKE_BUILD_TYPE=Release"
+        "-DCLANG_PLUGIN_SUPPORT=OFF"
+        "-DLLVM_ENABLE_PLUGINS=OFF"
+        ${{ matrix.config.cmake }}
+    - name: Ninja
+      run: ninja -C ${{ env.CLANGD_DIR }} clangd
+    - name: Archive
+      run: >
+        7z a clangd.zip
+        ${{ env.CLANGD_DIR }}/LICENSE.TXT
+        ${{ env.CLANGD_DIR }}/bin/clangd*
+        ${{ env.CLANGD_DIR }}/lib/clang
+    - name: Upload artifact
+      uses: actions/upload-artifact@v1
+      with:
+        name: ${{ matrix.config.name }}
+        path: clangd.zip
+    - name: Upload asset
+      uses: actions/upload-release-asset@v1.0.1
+      env: { GITHUB_TOKEN: "${{ secrets.RELEASE_TOKEN }}" }
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_name: clangd-${{ matrix.config.name }}-${{ github.event.release.tag_name }}.zip
+        asset_path: clangd.zip
+        asset_content_type: application/zip
+  # Create the release, and upload the artifacts to it.
+  finalize:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+    - name: Publish release
+      run: >
+        curl -XPATCH
+        "-HAuthorization: Bearer ${{ secrets.RELEASE_TOKEN }}"
+        "https://api.github.com/repos/${{ github.repository }}/releases/${{ github.event.release.id }}"
+        "-d" '{"draft": false}'

.github/workflows/periodic.yaml

@@ -0,0 +1,55 @@
+# Workflow to create an auto-buildable release periodically.
+#
+# Releases created with automation credentials don't trigger workflows.
+# Therefore a token `secrets.RELEASE_TOKEN` must exist with public_repo scope.
+name: Periodic release
+on:
+  # Run weekly on sunday at 21:37 UTC (arbitrary)
+  schedule:
+  - cron: '37 21 * * SUN'
+  # Allow triggering manually:
+  # curl -XPOST -d '{"event_type":"periodic"}' \
+  #  "-HAuthorization: Bearer <token>" \
+  #   https://api.github.com/repos/clangd/clangd/dispatches
+  repository_dispatch: { types: periodic }
+jobs:
+  # Choose the commit to build a release from.
+  #
+  # We want to avoid unbuildable revisions: choose the last green from CI.
+  # FIXME: the criteria should be some consistent set of buildbots passing.
+  pick:
+    name: Create draft release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Clone scripts
+      uses: actions/checkout@v2
+    - name: Install NPM dependencies
+      run: npm install --prefix .github/actions/pick
+    - name: Pick last successful commit
+      id: pick
+      uses: ./.github/actions/pick
+      with:
+        repo: llvm/llvm-project
+        token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Compute release info
+      run: |
+        echo ::set-env name=RELEASE_COMMIT_SHORT::$(printf "%.12s" ${{ steps.pick.outputs.sha }})
+        echo ::set-env name=RELEASE_DATE::$(date -u +%Y%m%d)
+    - name: Create release
+      uses: actions/create-release@master
+      id: release
+      env: { GITHUB_TOKEN: "${{ secrets.RELEASE_TOKEN }}" }
+      with:
+        tag_name: snapshot_${{ env.RELEASE_DATE }}
+        release_name: ${{ env.RELEASE_DATE }} @${{ env.RELEASE_COMMIT_SHORT }}
+        body: |
+          Unstable snapshot of clangd on ${{ env.RELEASE_DATE }}.
+
+          Built from llvm/llvm-project@${{ steps.pick.outputs.sha }}.
+        prerelease: true
+        # It would be nice to use draft releases, to hide them from users.
+        # But drafts don't fire events to trigger the autobuild workflow.
+        # Instead, that workflow marks the release as draft until it's built.
+        # As a result, the empty release will be briefly visible to users.
+        draft: false
+

.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+

releases.md

@@ -0,0 +1,54 @@
+# Binary releases
+
+This repository has automation to build binary releases of clangd for the most
+common platforms. This doesn't cover as many systems as the official releases
+from http://releases.llvm.org/, and distro packages etc. The main advantages
+is being able to cut releases easily whenever we want.
+
+The releases are just a zip archive containing the `clangd` binary, and the
+clang builtin headers. They should be runnable immediately after extracting the
+archive. The linux binary has `libstdc++` and other dependencies statically
+linked for maximum portability.
+
+## Creating a release manually
+
+In GitHub, click the "Releases" tab and create a new release.
+The tag name is significant, it's used in the directory name (`clangd_tagname`).
+
+Because clangd sources don't live in this repository, the release description
+must contain a magic string indicating the revision to build at, e.g.
+
+   Built at llvm/llvm-project@0399d5a9682b3cef71c653373e38890c63c4c365
+
+This must be a full SHA, not a short-hash, a tag or branch name, etc.
+The release must **not** be a draft release, as that won't trigger automation.
+
+## Building release binaries
+
+Creating the release will trigger the `autobuild` workflow, visible under the
+"Actions" tab. This runs a sequence of steps:
+
+- First, the release is marked as a draft, so it's invisible to non-owners.
+- Next, the sources are checked out and built on each of {mac, windows, linux}.
+  These run in parallel, and the results are zipped and added to the release.
+- If all builds succeeded, the release is published (marked as non-draft)
+
+## Automatic snapshot releases
+
+The `periodic` workflow runs on a weekly schedule and creates a release based
+on the last green revision at llvm/llvm-project.
+This in turn triggers the `autobuild` workflow above to produce binaries.
+
+These snapshot releases are marked as "pre-release" - they don't undergo any
+serious testing and so aren't particularly stable. However they're useful for
+people to try the latest clangd.
+
+## Credentials
+
+Rather than the default GitHub Actions access token, these actions use a
+Personal Access Token added to the repository as a secret. This is because:
+
+- The default access token expires after an hour. Builds take longer than that.
+- Releases created using the default access token don't trigger workflows.
+
+If you fork the repository, you must provide the `RELEASE_TOKEN` secret.