use crate::{    ExtensionLibraryKind, ExtensionManifest, GrammarManifestEntry, parse_wasm_extension_version,};use ::fs::Fs;use anyhow::{Context as _, Result, bail};use futures::{    FutureExt, StreamExt,    channel::oneshot::{self, Sender},    io,};use heck::ToSnakeCase;use http_client::{self, AsyncBody, HttpClient};use language::LanguageConfig;use path::PathExt;use semver::Version;use serde::Deserialize;use std::{    env, fs, mem,    num::NonZeroUsize,    ops::Not,    path::{Path, PathBuf},    sync::Arc,};use util::{ResultExt, command::Stdio};use wasm_encoder::{ComponentSectionId, Encode as _, RawSection, Section as _};use wasmparser::Parser; /// Currently, we compile with Rust's `wasm32-wasip2` target, which works with WASI `preview2` and the component model.const RUST_TARGET: &str = "wasm32-wasip2"; /// Compiling Tree-sitter parsers from C to WASM requires Clang 17, and a WASM build of libc/// and clang's runtime library. The `wasi-sdk` provides these binaries.////// Once Clang 17 and its wasm target are available via system package managers, we won't need/// to download this.const WASI_SDK_URL: &str = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/";const WASI_SDK_ASSET_NAME: Option<&str> = cfg_select! {    all(target_os = "macos", target_arch = "x86_64") => Some("wasi-sdk-25.0-x86_64-macos.tar.gz"),    all(target_os = "macos", target_arch = "aarch64") => Some("wasi-sdk-25.0-arm64-macos.tar.gz"),    all(target_os = "linux", target_arch = "x86_64") => Some("wasi-sdk-25.0-x86_64-linux.tar.gz"),    all(target_os = "linux", target_arch = "aarch64") => Some("wasi-sdk-25.0-arm64-linux.tar.gz"),    all(target_os = "freebsd", target_arch = "x86_64") => Some("wasi-sdk-25.0-x86_64-linux.tar.gz"),    all(target_os = "freebsd", target_arch = "aarch64") => Some("wasi-sdk-25.0-arm64-linux.tar.gz"),    all(target_os = "windows", target_arch = "x86_64") => Some("wasi-sdk-25.0-x86_64-windows.tar.gz"),    _ => None}; pub struct ExtensionBuilder {    cache_dir: PathBuf,    pub http: Arc<dyn HttpClient>,} pub enum CompilationConcurrency {    Unbounded,    Bounded(NonZeroUsize),} const DEFAULT_COMPILATION_CONCURRENCY: NonZeroUsize = NonZeroUsize::new(3).unwrap(); pub struct CompileExtensionOptions {    pub release: bool,    pub max_concurrency: CompilationConcurrency,} impl CompileExtensionOptions {    pub const fn dev() -> Self {        Self {            release: false,            max_concurrency: CompilationConcurrency::Bounded(DEFAULT_COMPILATION_CONCURRENCY),        }    }} #[derive(Deserialize)]struct CargoToml {    package: CargoTomlPackage,} #[derive(Deserialize)]struct CargoTomlPackage {    name: String,} impl ExtensionBuilder {    pub fn new(http_client: Arc<dyn HttpClient>, cache_dir: PathBuf) -> Self {        Self {            cache_dir,            http: http_client,        }    }     pub async fn compile_extension(        &self,        extension_dir: &Path,        extension_manifest: &mut ExtensionManifest,        options: CompileExtensionOptions,        fs: Arc<dyn Fs>,    ) -> Result<()> {        let start = std::time::Instant::now();         populate_defaults(extension_manifest, extension_dir, fs.clone()).await?;         if extension_dir.is_relative() {            bail!(                "extension dir {} is not an absolute path",                extension_dir.display()            );        }         fs.create_dir(&self.cache_dir)            .await            .context("failed to create cache dir")?;         let (tx, mut rx) = oneshot::channel();         let clang_path = extension_manifest.grammars.is_empty().not().then(|| {            std::iter::repeat_n(                async {                    self.install_wasi_sdk_if_needed()                        .await                        .log_err()                        .map(Arc::new)                }                .shared(),                extension_manifest.grammars.len(),            )        });         let rust_compilation_task =            (extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust)).then(|| {                async {                    log::info!("compiling Rust extension {}", extension_dir.display());                    self.compile_rust_extension(extension_dir, extension_manifest, tx, &options)                        .await                        .context("failed to compile Rust extension")?;                     log::info!("compiled Rust extension {}", extension_dir.display());                    Ok(())                }                .boxed()            });         let grammar_compilation_tasks = extension_manifest            .grammars            .iter()            .zip(clang_path.into_iter().flatten())            .map(|((grammar_name, grammar_metadata), clang_path_task)| {                async move {                    let snake_cased_grammar_name = grammar_name.to_snake_case();                    if grammar_name.as_ref() != snake_cased_grammar_name.as_str() {                        bail!(                            "grammar name '{grammar_name}' must be \                                written in snake_case: {snake_cased_grammar_name}"                        );                    }                     log::info!(                        "compiling grammar {grammar_name} for extension {}",                        extension_dir.display()                    );                     let clang_path = clang_path_task                        .await                        .context("Failed to resolve clang path")?;                     self.compile_grammar(                        extension_dir,                        grammar_name.as_ref(),                        grammar_metadata,                        &clang_path,                    )                    .await                    .with_context(|| format!("failed to compile grammar '{grammar_name}'"))?;                    log::info!(                        "compiled grammar {grammar_name} for extension {}",                        extension_dir.display()                    );                     Ok(())                }                .boxed()            });         let tasks = rust_compilation_task            .into_iter()            .chain(grammar_compilation_tasks)            .collect::<Vec<_>>();         match options.max_concurrency {            CompilationConcurrency::Unbounded => {                futures::future::try_join_all(tasks).await?;            }            CompilationConcurrency::Bounded(max_concurrency) => {                let mut stream = futures::stream::iter(tasks).buffered(max_concurrency.get());                 while let Some(result) = stream.next().await {                    result?;                }            }        }         if let Ok(version) = rx.try_recv() {            extension_manifest.lib.version = version;        }         log::info!(            "finished compiling extension {} in {time:.2}s",            extension_dir.display(),            time = start.elapsed().as_secs_f64(),        );        Ok(())    }     async fn compile_rust_extension(        &self,        extension_dir: &Path,        manifest: &ExtensionManifest,        wasm_extension_api_version_tx: Sender<Version>,        options: &CompileExtensionOptions,    ) -> anyhow::Result<()> {        self.install_rust_wasm_target_if_needed().await?;         let cargo_toml_content = fs::read_to_string(extension_dir.join("Cargo.toml"))?;        let cargo_toml: CargoToml = toml::from_str(&cargo_toml_content)?;         log::info!(            "compiling Rust crate for extension {}",            extension_dir.display()        );        let output = util::command::new_command("cargo")            .args(["build", "--target", RUST_TARGET])            .args(options.release.then_some("--release"))            .arg("--target-dir")            .arg(extension_dir.join("target"))            // WASI builds do not work with sccache and just stuck, so disable it.            .env("RUSTC_WRAPPER", "")            .current_dir(extension_dir)            .output()            .await            .context("failed to run `cargo`")?;        if !output.status.success() {            bail!(                "failed to build extension {}",                String::from_utf8_lossy(&output.stderr)            );        }         log::info!(            "compiled Rust crate for extension {}",            extension_dir.display()        );         let mut wasm_path = PathBuf::from(extension_dir);        wasm_path.extend([            "target",            RUST_TARGET,            if options.release { "release" } else { "debug" },            &cargo_toml                .package                .name                // The wasm32-wasip2 target normalizes `-` in package names to `_` in the resulting `.wasm` file.                .replace('-', "_"),        ]);        wasm_path.set_extension("wasm");         log::info!(            "encoding wasm component for extension {}",            extension_dir.display()        );         let component_bytes = fs::read(&wasm_path)            .with_context(|| format!("failed to read output module `{}`", wasm_path.display()))?;         let component_bytes = self            .strip_custom_sections(&component_bytes)            .context("failed to strip debug sections from wasm component")?;         let wasm_extension_api_version =            parse_wasm_extension_version(&manifest.id, &component_bytes)                .context("compiled wasm did not contain a valid zed extension api version")?;        wasm_extension_api_version_tx            .send(wasm_extension_api_version)            .map_err(|_| anyhow::anyhow!("Failed to send API version"))?;         let extension_file = extension_dir.join("extension.wasm");        fs::write(extension_file.clone(), &component_bytes)            .context("failed to write extension.wasm")?;         log::info!(            "extension {} written to {}",            extension_dir.display(),            extension_file.display()        );         Ok(())    }     async fn compile_grammar(        &self,        extension_dir: &Path,        grammar_name: &str,        grammar_metadata: &GrammarManifestEntry,        clang_path: &Path,    ) -> Result<()> {        let mut grammar_repo_dir = extension_dir.to_path_buf();        grammar_repo_dir.extend(["grammars", grammar_name]);         let mut grammar_wasm_path = grammar_repo_dir.clone();        grammar_wasm_path.set_extension("wasm");         log::info!("checking out {grammar_name} parser");        self.checkout_repo(            &grammar_repo_dir,            &grammar_metadata.repository,            &grammar_metadata.rev,        )        .await?;         let base_grammar_path = grammar_metadata            .path            .as_ref()            .map(|path| grammar_repo_dir.join(path))            .unwrap_or(grammar_repo_dir);         let src_path = base_grammar_path.join("src");        let parser_path = src_path.join("parser.c");        let scanner_path = src_path.join("scanner.c");         // Skip recompiling if the WASM object is already newer than the source files        if file_newer_than_deps(&grammar_wasm_path, &[&parser_path, &scanner_path]).unwrap_or(false)        {            log::info!(                "skipping compilation of {grammar_name} parser because the existing compiled grammar is up to date"            );        } else {            log::info!("compiling {grammar_name} parser");            let clang_output = util::command::new_command(&clang_path)                .args(["-fPIC", "-shared", "-Os"])                .arg(format!("-Wl,--export=tree_sitter_{grammar_name}"))                .arg("-o")                .arg(&grammar_wasm_path)                .arg("-I")                .arg(&src_path)                .arg(&parser_path)                .args(scanner_path.exists().then_some(scanner_path))                .output()                .await                .context("failed to run clang")?;             if !clang_output.status.success() {                bail!(                    "failed to compile {} parser with clang: {}",                    grammar_name,                    String::from_utf8_lossy(&clang_output.stderr),                );            }        }         Ok(())    }     async fn checkout_repo(&self, directory: &Path, url: &str, rev: &str) -> Result<()> {        let git_dir = directory.join(".git");         if directory.exists() {            let remotes_output = util::command::new_command("git")                .arg("--git-dir")                .arg(&git_dir)                .args(["remote", "get-url", "origin"])                .env("GIT_CONFIG_GLOBAL", "/dev/null")                .output()                .await?;            let has_remote = remotes_output.status.success()                && String::from_utf8_lossy(&remotes_output.stdout).trim() == url;            if !has_remote {                bail!(                    "grammar directory '{}' already exists, but is not a git clone of '{}'",                    directory.display(),                    url                );            }        } else {            fs::create_dir_all(directory).with_context(|| {                format!("failed to create grammar directory {}", directory.display(),)            })?;            let init_output = util::command::new_command("git")                .arg("init")                .current_dir(directory)                .output()                .await?;            if !init_output.status.success() {                bail!(                    "failed to run `git init` in directory '{}'",                    directory.display()                );            }             let remote_add_output = util::command::new_command("git")                .arg("--git-dir")                .arg(&git_dir)                .args(["remote", "add", "origin", url])                .output()                .await                .context("failed to execute `git remote add`")?;            if !remote_add_output.status.success() {                bail!(                    "failed to add remote {url} for git repository {}",                    git_dir.display()                );            }        }         let fetch_output = util::command::new_command("git")            .arg("--git-dir")            .arg(&git_dir)            .args(["fetch", "--depth", "1", "origin", rev])            .output()            .await            .context("failed to execute `git fetch`")?;         let checkout_output = util::command::new_command("git")            .arg("--git-dir")            .arg(&git_dir)            .args(["checkout", rev])            .current_dir(directory)            .output()            .await            .context("failed to execute `git checkout`")?;        if !checkout_output.status.success() {            if !fetch_output.status.success() {                bail!(                    "failed to fetch revision {} in directory '{}'",                    rev,                    directory.display()                );            }            bail!(                "failed to checkout revision {} in directory '{}': {}",                rev,                directory.display(),                String::from_utf8_lossy(&checkout_output.stderr)            );        }         Ok(())    }     async fn install_rust_wasm_target_if_needed(&self) -> Result<()> {        let rustc_output = util::command::new_command("rustc")            .args(["--print", "target-libdir", "--target", RUST_TARGET])            .output()            .await            .context("failed to run rustc")?;        if !rustc_output.status.success() {            bail!(                "failed to retrieve the `{RUST_TARGET}` target libdir: {}",                String::from_utf8_lossy(&rustc_output.stderr)            );        }         let target_libdir = PathBuf::from(String::from_utf8(rustc_output.stdout)?.trim());        if target_libdir.exists() {            return Ok(());        }         if which::which("rustup").is_err() {            bail!(                "the `{RUST_TARGET}` target is not installed, and `rustup` is not available to \                 install it. Add the target to your Rust toolchain (e.g. `targets = \                 [\"{RUST_TARGET}\"]` for Nix rust-overlay/fenix toolchains) or install it via \                 your package manager"            );        }         let output = util::command::new_command("rustup")            .args(["target", "add", RUST_TARGET])            .stderr(Stdio::piped())            .stdout(Stdio::inherit())            .output()            .await            .context("failed to run `rustup target add`")?;        if !output.status.success() {            bail!(                "failed to install the `{RUST_TARGET}` target: {}",                String::from_utf8_lossy(&output.stderr)            );        }         Ok(())    }     async fn install_wasi_sdk_if_needed(&self) -> Result<PathBuf> {        if let Some(sdk_path) = env::var_os("WASI_SDK_PATH").filter(|path| !path.is_empty()) {            let sdk_path = PathBuf::from(sdk_path);            let clang_path = wasi_sdk_clang_path(&sdk_path);            if fs::metadata(&clang_path).is_ok_and(|metadata| metadata.is_file()) {                log::info!("using wasi-sdk from WASI_SDK_PATH: {sdk_path:?}");                return Ok(clang_path);            }            log::warn!(

Lines 1–500 of 933. Load more