feat: add release CI

This commit is contained in:
Luc Georges 2024-06-26 10:49:26 +02:00
parent 17ea67a6a7
commit 98cd60ba40
No known key found for this signature in database
GPG Key ID: 8A2B84C1466CDF50
12 changed files with 712 additions and 11 deletions

View File

@ -0,0 +1,8 @@
FROM node:slim
COPY . /action
WORKDIR /action
RUN npm install --production
ENTRYPOINT ["node", "/action/main.js"]

View File

@ -0,0 +1,21 @@
# github-release
Copy-pasted from
https://github.com/rust-lang/rust-analyzer/tree/2df30e1e07eafc1de0359566423f471920693a34/.github/actions/github-release
An action used to publish GitHub releases for `wasmtime`.
As of the time of this writing there's a few actions floating around which
perform github releases but they all tend to have their set of drawbacks.
Additionally nothing handles deleting releases which we need for our rolling
`dev` release.
To handle all this, this action rolls its own implementation using the
actions/toolkit repository and packages published there. These run in a Docker
container and take various inputs to orchestrate the release from the build.
More comments can be found in `main.js`.
Testing this is really hard. If you want to try though run `npm install` and
then `node main.js`. You'll have to configure a bunch of env vars though to get
anything reasonably working.

View File

@ -0,0 +1,15 @@
name: 'wasmtime github releases'
description: 'wasmtime github releases'
inputs:
token:
description: ''
required: true
name:
description: ''
required: true
files:
description: ''
required: true
runs:
using: 'docker'
image: 'Dockerfile'

144
.github/actions/github-release/main.js vendored Normal file
View File

@ -0,0 +1,144 @@
const core = require('@actions/core');
const path = require("path");
const fs = require("fs");
const github = require('@actions/github');
const glob = require('glob');
function sleep(milliseconds) {
return new Promise(resolve => setTimeout(resolve, milliseconds));
}
async function runOnce() {
// Load all our inputs and env vars. Note that `getInput` reads from `INPUT_*`
const files = core.getInput('files');
const name = core.getInput('name');
const token = core.getInput('token');
const slug = process.env.GITHUB_REPOSITORY;
const owner = slug.split('/')[0];
const repo = slug.split('/')[1];
const sha = process.env.HEAD_SHA;
core.info(`files: ${files}`);
core.info(`name: ${name}`);
const options = {
request: {
timeout: 30000,
}
};
const octokit = github.getOctokit(token, options);
// Delete the previous release since we can't overwrite one. This may happen
// due to retrying an upload or it may happen because we're doing the dev
// release.
const releases = await octokit.paginate("GET /repos/:owner/:repo/releases", { owner, repo });
for (const release of releases) {
if (release.tag_name !== name) {
continue;
}
const release_id = release.id;
core.info(`deleting release ${release_id}`);
await octokit.rest.repos.deleteRelease({ owner, repo, release_id });
}
// We also need to update the `dev` tag while we're at it on the `dev` branch.
if (name == 'nightly') {
try {
core.info(`updating nightly tag`);
await octokit.rest.git.updateRef({
owner,
repo,
ref: 'tags/nightly',
sha,
force: true,
});
} catch (e) {
core.error(e);
core.info(`creating nightly tag`);
await octokit.rest.git.createTag({
owner,
repo,
tag: 'nightly',
message: 'nightly release',
object: sha,
type: 'commit',
});
}
}
// Creates an official GitHub release for this `tag`, and if this is `dev`
// then we know that from the previous block this should be a fresh release.
core.info(`creating a release`);
const release = await octokit.rest.repos.createRelease({
owner,
repo,
name,
tag_name: name,
target_commitish: sha,
prerelease: name === 'nightly',
});
const release_id = release.data.id;
// Upload all the relevant assets for this release as just general blobs.
for (const file of glob.sync(files)) {
const size = fs.statSync(file).size;
const name = path.basename(file);
await runWithRetry(async function() {
// We can't overwrite assets, so remove existing ones from a previous try.
let assets = await octokit.rest.repos.listReleaseAssets({
owner,
repo,
release_id
});
for (const asset of assets.data) {
if (asset.name === name) {
core.info(`delete asset ${name}`);
const asset_id = asset.id;
await octokit.rest.repos.deleteReleaseAsset({ owner, repo, asset_id });
}
}
core.info(`upload ${file}`);
const headers = { 'content-length': size, 'content-type': 'application/octet-stream' };
const data = fs.createReadStream(file);
await octokit.rest.repos.uploadReleaseAsset({
data,
headers,
name,
url: release.data.upload_url,
});
});
}
}
async function runWithRetry(f) {
const retries = 10;
const maxDelay = 4000;
let delay = 1000;
for (let i = 0; i < retries; i++) {
try {
await f();
break;
} catch (e) {
if (i === retries - 1)
throw e;
core.error(e);
const currentDelay = Math.round(Math.random() * delay);
core.info(`sleeping ${currentDelay} ms`);
await sleep(currentDelay);
delay = Math.min(delay * 2, maxDelay);
}
}
}
async function run() {
await runWithRetry(runOnce);
}
run().catch(err => {
core.error(err);
core.setFailed(err.message);
});

View File

@ -0,0 +1,10 @@
{
"name": "wasmtime-github-release",
"version": "0.0.0",
"main": "main.js",
"dependencies": {
"@actions/core": "^1.6",
"@actions/github": "^5.0",
"glob": "^7.1.5"
}
}

180
.github/workflows/release.yml vendored Normal file
View File

@ -0,0 +1,180 @@
name: release
on:
workflow_dispatch:
push:
branches:
- 'release/**'
env:
CARGO_INCREMENTAL: 0
CARGO_NET_RETRY: 10
RUSTFLAGS: "-D warnings -W unreachable-pub"
RUSTUP_MAX_RETRIES: 10
FETCH_DEPTH: 0 # pull in the tags for the version string
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_LINKER: arm-linux-gnueabihf-gcc
jobs:
dist:
strategy:
matrix:
include:
- os: windows-latest
target: x86_64-pc-windows-msvc
code-target: win32-x64
- os: windows-latest
target: i686-pc-windows-msvc
code-target: win32-ia32
- os: windows-latest
target: aarch64-pc-windows-msvc
code-target: win32-arm64
- os: ubuntu-22.04
target: x86_64-unknown-linux-gnu
code-target: linux-x64
- os: ubuntu-22.04
target: aarch64-unknown-linux-gnu
code-target: linux-arm64
- os: ubuntu-22.04
target: arm-unknown-linux-gnueabihf
code-target: linux-armhf
- os: macos-12
target: x86_64-apple-darwin
code-target: darwin-x64
- os: macos-12
target: aarch64-apple-darwin
code-target: darwin-arm64
env:
LSP_AI_TARGET: ${{ matrix.target }}
name: dist (${{ matrix.target }})
runs-on: ${{ matrix.os }}
container: ${{ matrix.container }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: ${{ env.FETCH_DEPTH }}
- name: Install Rust toolchain
run: |
rustup update --no-self-update stable
rustup target add ${{ matrix.target }}
rustup component add rust-src
- name: Update apt repositories
if: contains(matrix.os, 'ubuntu')
run: sudo apt-get update -y
- name: Install AArch64 target toolchain
if: matrix.target == 'aarch64-unknown-linux-gnu'
run: sudo apt-get install gcc-aarch64-linux-gnu libc6-dev-arm64-cross g++-aarch64-linux-gnu
- name: Install ARM target toolchain
if: matrix.target == 'arm-unknown-linux-gnueabihf'
run: sudo apt-get install gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf
- name: Dist
run: cargo xtask dist
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: dist-${{ matrix.target }}
path: ./dist
dist-x86_64-unknown-linux-musl:
name: dist (x86_64-unknown-linux-musl)
runs-on: ubuntu-latest
env:
LLM_LS_TARGET: x86_64-unknown-linux-musl
# For some reason `-crt-static` is not working for clang without lld
RUSTFLAGS: "-C link-arg=-fuse-ld=lld -C target-feature=-crt-static"
container:
image: rust:alpine
volumes:
- /usr/local/cargo/registry:/usr/local/cargo/registry
steps:
- name: Install dependencies
run: apk add --no-cache git clang lld musl-dev nodejs npm openssl-dev pkgconfig g++
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: ${{ env.FETCH_DEPTH }}
- name: Dist
run: cargo xtask dist
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: dist-x86_64-unknown-linux-musl
path: ./dist
publish:
name: publish
runs-on: ubuntu-latest
needs: ["dist", "dist-x86_64-unknown-linux-musl"]
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: ${{ env.FETCH_DEPTH }}
- run: echo "HEAD_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
- run: 'echo "HEAD_SHA: $HEAD_SHA"'
- name: Split branch name
env:
BRANCH: ${{ github.ref_name }}
id: split
run: echo "tag=${BRANCH##*/}" >> $GITHUB_OUTPUT
- uses: actions/download-artifact@v4
with:
name: dist-aarch64-apple-darwin
path: dist
- uses: actions/download-artifact@v4
with:
name: dist-x86_64-apple-darwin
path: dist
- uses: actions/download-artifact@v4
with:
name: dist-x86_64-unknown-linux-gnu
path: dist
- uses: actions/download-artifact@v4
with:
name: dist-x86_64-unknown-linux-musl
path: dist
- uses: actions/download-artifact@v4
with:
name: dist-aarch64-unknown-linux-gnu
path: dist
- uses: actions/download-artifact@v4
with:
name: dist-arm-unknown-linux-gnueabihf
path: dist
- uses: actions/download-artifact@v4
with:
name: dist-x86_64-pc-windows-msvc
path: dist
- uses: actions/download-artifact@v4
with:
name: dist-i686-pc-windows-msvc
path: dist
- uses: actions/download-artifact@v4
with:
name: dist-aarch64-pc-windows-msvc
path: dist
- run: ls -al ./dist
- name: Publish Release
uses: ./.github/actions/github-release
with:
files: "dist/*"
name: ${{ steps.split.outputs.tag }}
token: ${{ secrets.GITHUB_TOKEN }}

78
Cargo.lock generated
View File

@ -266,9 +266,9 @@ dependencies = [
[[package]]
name = "bumpalo"
version = "3.15.4"
version = "3.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa"
checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
[[package]]
name = "byteorder"
@ -1658,9 +1658,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]]
name = "memchr"
version = "2.7.1"
version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "memo-map"
@ -3218,9 +3218,9 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.34"
version = "0.3.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749"
checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885"
dependencies = [
"deranged",
"itoa",
@ -3239,9 +3239,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
[[package]]
name = "time-macros"
version = "0.2.17"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774"
checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf"
dependencies = [
"num-conv",
"time-core",
@ -4134,6 +4134,55 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "write-json"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23f6174b2566cc4a74f95e1367ec343e7fa80c93cc8087f5c4a3d6a1088b2118"
[[package]]
name = "xflags"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d9e15fbb3de55454b0106e314b28e671279009b363e6f1d8e39fdc3bf048944"
dependencies = [
"xflags-macros",
]
[[package]]
name = "xflags-macros"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "672423d4fea7ffa2f6c25ba60031ea13dc6258070556f125cc4d790007d4a155"
[[package]]
name = "xshell"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db0ab86eae739efd1b054a8d3d16041914030ac4e01cd1dca0cf252fd8b6437"
dependencies = [
"xshell-macros",
]
[[package]]
name = "xshell-macros"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d422e8e38ec76e2f06ee439ccc765e9c6a9638b9e7c9f2e8255e4d41e8bd852"
[[package]]
name = "xtask"
version = "0.1.0"
dependencies = [
"anyhow",
"flate2",
"time",
"write-json",
"xflags",
"xshell",
"zip",
]
[[package]]
name = "xxhash-rust"
version = "0.8.10"
@ -4165,3 +4214,16 @@ name = "zeroize"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d"
[[package]]
name = "zip"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261"
dependencies = [
"byteorder",
"crc32fast",
"crossbeam-utils",
"flate2",
"time",
]

View File

@ -1,7 +1,5 @@
[workspace]
members = [
"crates/*",
]
members = ["crates/*", "xtask/"]
resolver = "2"
[workspace.package]
@ -10,3 +8,13 @@ license = "MIT"
description = "LSP-AI is an open-source language server that serves as a backend for AI-powered functionality, designed to assist and empower software engineers, not replace them."
repository = "https://github.com/SilasMarvin/lsp-ai"
readme = "README.md"
authors = ["Silvas Marvin <>"]
[profile.dev.package]
# This speeds up `cargo xtask dist`.
miniz_oxide.opt-level = 3
[profile.release]
incremental = true
# Set this to 1 or 2 to get more useful backtraces in debugger.
debug = 0

19
xtask/Cargo.toml Normal file
View File

@ -0,0 +1,19 @@
[package]
name = "xtask"
version = "0.1.0"
publish = false
edition.workspace = true
license.workspace = true
authors.workspace = true
[dependencies]
anyhow = "1"
flate2 = "1"
write-json = "0.1"
xshell = "0.2"
xflags = "0.3"
time = { version = "0.3", default-features = false }
zip = { version = "0.6", default-features = false, features = [
"deflate",
"time",
] }

147
xtask/src/dist.rs Normal file
View File

@ -0,0 +1,147 @@
use std::{
env,
fs::File,
io::{self, BufWriter},
path::{Path, PathBuf},
};
use flate2::{write::GzEncoder, Compression};
use time::OffsetDateTime;
use xshell::{cmd, Shell};
use zip::{write::FileOptions, DateTime, ZipWriter};
use crate::{flags, project_root};
impl flags::Dist {
pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> {
let branch = sh.var("GITHUB_REF").unwrap_or_default();
let release = if branch.starts_with("refs/heads/release/") {
branch.replace("refs/heads/release/", "")
} else {
"0.0.0".to_owned()
};
let project_root = project_root();
let target = Target::get(&project_root);
let dist = project_root.join("dist");
sh.remove_path(&dist)?;
sh.create_dir(&dist)?;
dist_server(sh, &release, &target)?;
Ok(())
}
}
fn dist_server(sh: &Shell, release: &str, target: &Target) -> anyhow::Result<()> {
let _e = sh.push_env("CFG_RELEASE", release);
let _e = sh.push_env("CARGO_PROFILE_RELEASE_LTO", "thin");
// Uncomment to enable debug info for releases. Note that:
// * debug info is split on windows and macs, so it does nothing for those platforms,
// * on Linux, this blows up the binary size from 8MB to 43MB, which is unreasonable.
// let _e = sh.push_env("CARGO_PROFILE_RELEASE_DEBUG", "1");
if target.name.contains("-linux-") {
env::set_var("CC", "clang");
}
let target_name = &target.name;
cmd!(sh, "cargo build --manifest-path ./crates/lsp-ai/Cargo.toml --bin lsp-ai --target {target_name} --release").run()?;
let dst = Path::new("dist").join(&target.artifact_name);
gzip(&target.server_path, &dst.with_extension("gz"))?;
if target_name.contains("-windows-") {
zip(
&target.server_path,
target.symbols_path.as_ref(),
&dst.with_extension("zip"),
)?;
}
Ok(())
}
fn gzip(src_path: &Path, dest_path: &Path) -> anyhow::Result<()> {
let mut encoder = GzEncoder::new(File::create(dest_path)?, Compression::best());
let mut input = io::BufReader::new(File::open(src_path)?);
io::copy(&mut input, &mut encoder)?;
encoder.finish()?;
Ok(())
}
fn zip(src_path: &Path, symbols_path: Option<&PathBuf>, dest_path: &Path) -> anyhow::Result<()> {
let file = File::create(dest_path)?;
let mut writer = ZipWriter::new(BufWriter::new(file));
writer.start_file(
src_path.file_name().unwrap().to_str().unwrap(),
FileOptions::default()
.last_modified_time(
DateTime::try_from(OffsetDateTime::from(
std::fs::metadata(src_path)?.modified()?,
))
.unwrap(),
)
.unix_permissions(0o755)
.compression_method(zip::CompressionMethod::Deflated)
.compression_level(Some(9)),
)?;
let mut input = io::BufReader::new(File::open(src_path)?);
io::copy(&mut input, &mut writer)?;
if let Some(symbols_path) = symbols_path {
writer.start_file(
symbols_path.file_name().unwrap().to_str().unwrap(),
FileOptions::default()
.last_modified_time(
DateTime::try_from(OffsetDateTime::from(
std::fs::metadata(src_path)?.modified()?,
))
.unwrap(),
)
.compression_method(zip::CompressionMethod::Deflated)
.compression_level(Some(9)),
)?;
let mut input = io::BufReader::new(File::open(symbols_path)?);
io::copy(&mut input, &mut writer)?;
}
writer.finish()?;
Ok(())
}
struct Target {
name: String,
server_path: PathBuf,
symbols_path: Option<PathBuf>,
artifact_name: String,
}
impl Target {
fn get(project_root: &Path) -> Self {
let name = match env::var("LSP_AI_TARGET") {
Ok(target) => target,
_ => {
if cfg!(target_os = "linux") {
"x86_64-unknown-linux-gnu".to_string()
} else if cfg!(target_os = "windows") {
"x86_64-pc-windows-msvc".to_string()
} else if cfg!(target_os = "macos") {
"x86_64-apple-darwin".to_string()
} else {
panic!("Unsupported OS, maybe try setting LSP_AI_TARGET")
}
}
};
let out_path = project_root.join("target").join(&name).join("release");
let (exe_suffix, symbols_path) = if name.contains("-windows-") {
(".exe".into(), Some(out_path.join("lsp_ai.pdb")))
} else {
(String::new(), None)
};
let server_path = out_path.join(format!("lsp-ai{exe_suffix}"));
let artifact_name = format!("lsp-ai-{name}{exe_suffix}");
Self {
name,
server_path,
symbols_path,
artifact_name,
}
}
}

43
xtask/src/flags.rs Normal file
View File

@ -0,0 +1,43 @@
#![allow(unreachable_pub)]
xflags::xflags! {
src "./src/flags.rs"
/// Run custom build command.
cmd xtask {
cmd dist {}
}
}
// generated start
// The following code is generated by `xflags` macro.
// Run `env UPDATE_XFLAGS=1 cargo build` to regenerate.
#[derive(Debug)]
pub struct Xtask {
pub subcommand: XtaskCmd,
}
#[derive(Debug)]
pub enum XtaskCmd {
Dist(Dist),
}
#[derive(Debug)]
pub struct Dist;
impl Xtask {
#[allow(dead_code)]
pub fn from_env_or_exit() -> Self {
Self::from_env_or_exit_()
}
#[allow(dead_code)]
pub fn from_env() -> xflags::Result<Self> {
Self::from_env_()
}
#[allow(dead_code)]
pub fn from_vec(args: Vec<std::ffi::OsString>) -> xflags::Result<Self> {
Self::from_vec_(args)
}
}

44
xtask/src/main.rs Normal file
View File

@ -0,0 +1,44 @@
//! See <https://github.com/matklad/cargo-xtask/>.
//!
//! This binary defines various auxiliary build commands, which are not
//! expressible with just `cargo`.
//!
//! This binary is integrated into the `cargo` command line by using an alias in
//! `.cargo/config`.
#![warn(
rust_2018_idioms,
unused_lifetimes,
semicolon_in_expressions_from_macros
)]
mod flags;
mod dist;
use std::{
env,
path::{Path, PathBuf},
};
use xshell::Shell;
fn main() -> anyhow::Result<()> {
let flags = flags::Xtask::from_env_or_exit();
let sh = &Shell::new()?;
sh.change_dir(project_root());
match flags.subcommand {
flags::XtaskCmd::Dist(cmd) => cmd.run(sh),
}
}
fn project_root() -> PathBuf {
Path::new(
&env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| env!("CARGO_MANIFEST_DIR").to_owned()),
)
.ancestors()
.nth(1)
.unwrap()
.to_path_buf()
}