// Copyright 2020 The Jujutsu Authors//// Licensed under the Apache License, Version 2.0 (the "License");// you may not use this file except in compliance with the License.// You may obtain a copy of the License at//// https://www.apache.org/licenses/LICENSE-2.0//// Unless required by applicable law or agreed to in writing, software// distributed under the License is distributed on an "AS IS" BASIS,// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.// See the License for the specific language governing permissions and// limitations under the License. #![allow(missing_docs)] use std::any::Any;use std::collections::{BTreeMap, HashMap, HashSet};use std::fmt::Debug;use std::io::{ErrorKind, Write};use std::path::{Path, PathBuf};use std::time::SystemTime;use std::{fs, io}; use itertools::Itertools as _;use prost::Message;use tempfile::NamedTempFile;use thiserror::Error; use crate::backend::{CommitId, MillisSinceEpoch, Timestamp};use crate::content_hash::blake2b_hash;use crate::file_util::{persist_content_addressed_temp_file, IoResultExt as _, PathError};use crate::merge::Merge;use crate::object_id::{HexPrefix, ObjectId, PrefixResolution};use crate::op_store::{    OpStore, OpStoreError, OpStoreResult, Operation, OperationId, OperationMetadata, RefTarget,    RemoteRef, RemoteRefState, RemoteView, View, ViewId, WorkspaceId,};use crate::{dag_walk, git, op_store}; // BLAKE2b-512 hash length in bytesconst OPERATION_ID_LENGTH: usize = 64;const VIEW_ID_LENGTH: usize = 64; #[derive(Debug, Error)]#[error("Failed to read {kind} with ID {id}")]struct DecodeError {    kind: &'static str,    id: String,    #[source]    err: prost::DecodeError,} impl From<DecodeError> for OpStoreError {    fn from(err: DecodeError) -> Self {        OpStoreError::Other(err.into())    }} #[derive(Debug)]pub struct SimpleOpStore {    path: PathBuf,    empty_view_id: ViewId,    root_operation_id: OperationId,} impl SimpleOpStore {    pub fn name() -> &'static str {        "simple_op_store"    }     /// Creates an empty OpStore, panics if it already exists    pub fn init(store_path: &Path) -> Self {        fs::create_dir(store_path.join("views")).unwrap();        fs::create_dir(store_path.join("operations")).unwrap();        Self::load(store_path)    }     /// Load an existing OpStore    pub fn load(store_path: &Path) -> Self {        SimpleOpStore {            path: store_path.to_path_buf(),            empty_view_id: ViewId::from_bytes(&[0; VIEW_ID_LENGTH]),            root_operation_id: OperationId::from_bytes(&[0; OPERATION_ID_LENGTH]),        }    }     fn view_path(&self, id: &ViewId) -> PathBuf {        self.path.join("views").join(id.hex())    }     fn operation_path(&self, id: &OperationId) -> PathBuf {        self.path.join("operations").join(id.hex())    }} impl OpStore for SimpleOpStore {    fn as_any(&self) -> &dyn Any {        self    }     fn name(&self) -> &str {        Self::name()    }     fn root_operation_id(&self) -> &OperationId {        &self.root_operation_id    }     fn read_view(&self, id: &ViewId) -> OpStoreResult<View> {        if *id == self.empty_view_id {            return Ok(View::default());        }         let path = self.view_path(id);        let buf = fs::read(path).map_err(|err| io_to_read_error(err, id))?;         let proto = crate::protos::op_store::View::decode(&*buf).map_err(|err| DecodeError {            kind: "view",            id: id.hex(),            err,        })?;        Ok(view_from_proto(proto))    }     fn write_view(&self, view: &View) -> OpStoreResult<ViewId> {        let temp_file =            NamedTempFile::new_in(&self.path).map_err(|err| io_to_write_error(err, "view"))?;         let proto = view_to_proto(view);        temp_file            .as_file()            .write_all(&proto.encode_to_vec())            .map_err(|err| io_to_write_error(err, "view"))?;         let id = ViewId::new(blake2b_hash(view).to_vec());         persist_content_addressed_temp_file(temp_file, self.view_path(&id))            .map_err(|err| io_to_write_error(err, "view"))?;        Ok(id)    }     fn read_operation(&self, id: &OperationId) -> OpStoreResult<Operation> {        if *id == self.root_operation_id {            return Ok(Operation::make_root(self.empty_view_id.clone()));        }         let path = self.operation_path(id);        let buf = fs::read(path).map_err(|err| io_to_read_error(err, id))?;         let proto =            crate::protos::op_store::Operation::decode(&*buf).map_err(|err| DecodeError {                kind: "operation",                id: id.hex(),                err,            })?;        let mut operation = operation_from_proto(proto);        if operation.parents.is_empty() {            // Repos created before we had the root operation will have an operation without            // parents.            operation.parents.push(self.root_operation_id.clone());        }        Ok(operation)    }     fn write_operation(&self, operation: &Operation) -> OpStoreResult<OperationId> {        assert!(!operation.parents.is_empty());        let temp_file =            NamedTempFile::new_in(&self.path).map_err(|err| io_to_write_error(err, "operation"))?;         let proto = operation_to_proto(operation);        temp_file            .as_file()            .write_all(&proto.encode_to_vec())            .map_err(|err| io_to_write_error(err, "operation"))?;         let id = OperationId::new(blake2b_hash(operation).to_vec());         persist_content_addressed_temp_file(temp_file, self.operation_path(&id))            .map_err(|err| io_to_write_error(err, "operation"))?;        Ok(id)    }     fn resolve_operation_id_prefix(        &self,        prefix: &HexPrefix,    ) -> OpStoreResult<PrefixResolution<OperationId>> {        let op_dir = self.path.join("operations");        let find = || -> io::Result<_> {            let matches_root = prefix.matches(&self.root_operation_id);            let hex_prefix = prefix.hex();            if hex_prefix.len() == OPERATION_ID_LENGTH * 2 {                // Fast path for full-length ID                if matches_root || op_dir.join(hex_prefix).try_exists()? {                    let id = OperationId::from_bytes(prefix.as_full_bytes().unwrap());                    return Ok(PrefixResolution::SingleMatch(id));                } else {                    return Ok(PrefixResolution::NoMatch);                }            }             let mut matched = matches_root.then(|| self.root_operation_id.clone());            for entry in op_dir.read_dir()? {                let Ok(name) = entry?.file_name().into_string() else {                    continue; // Skip invalid UTF-8                };                if !name.starts_with(&hex_prefix) {                    continue;                }                let Ok(id) = OperationId::try_from_hex(&name) else {                    continue; // Skip invalid hex                };                if matched.is_some() {                    return Ok(PrefixResolution::AmbiguousMatch);                }                matched = Some(id);            }            if let Some(id) = matched {                Ok(PrefixResolution::SingleMatch(id))            } else {                Ok(PrefixResolution::NoMatch)            }        };        find()            .context(&op_dir)            .map_err(|err| OpStoreError::Other(err.into()))    }     #[tracing::instrument(skip(self))]    fn gc(&self, head_ids: &[OperationId], keep_newer: SystemTime) -> OpStoreResult<()> {        let to_op_id = |entry: &fs::DirEntry| -> Option<OperationId> {            let name = entry.file_name().into_string().ok()?;            OperationId::try_from_hex(&name).ok()        };        let to_view_id = |entry: &fs::DirEntry| -> Option<ViewId> {            let name = entry.file_name().into_string().ok()?;            ViewId::try_from_hex(&name).ok()        };        let remove_file_if_not_new = |entry: &fs::DirEntry| -> Result<(), PathError> {            let path = entry.path();            // Check timestamp, but there's still TOCTOU problem if an existing            // file is renewed.            let metadata = entry.metadata().context(&path)?;            let mtime = metadata.modified().expect("unsupported platform?");            if mtime > keep_newer {                tracing::trace!(?path, "not removing");                Ok(())            } else {                tracing::trace!(?path, "removing");                fs::remove_file(&path).context(&path)            }        };         // Reachable objects are resolved without considering the keep_newer        // parameter. We could collect ancestors of the "new" operations here,        // but more files can be added anyway after that.        let read_op = |id: &OperationId| self.read_operation(id).map(|data| (id.clone(), data));        let reachable_ops: HashMap<OperationId, Operation> = dag_walk::dfs_ok(            head_ids.iter().map(read_op),            |(id, _)| id.clone(),            |(_, data)| data.parents.iter().map(read_op).collect_vec(),        )        .try_collect()?;        let reachable_views: HashSet<&ViewId> =            reachable_ops.values().map(|data| &data.view_id).collect();        tracing::info!(            reachable_op_count = reachable_ops.len(),            reachable_view_count = reachable_views.len(),            "collected reachable objects"        );         let prune_ops = || -> Result<(), PathError> {            let op_dir = self.path.join("operations");            for entry in op_dir.read_dir().context(&op_dir)? {                let entry = entry.context(&op_dir)?;                let Some(id) = to_op_id(&entry) else {                    tracing::trace!(?entry, "skipping invalid file name");                    continue;                };                if reachable_ops.contains_key(&id) {                    continue;                }                // If the operation was added after collecting reachable_views,                // its view mtime would also be renewed. So there's no need to                // update the reachable_views set to preserve the view.                remove_file_if_not_new(&entry)?;            }            Ok(())        };        prune_ops().map_err(|err| OpStoreError::Other(err.into()))?;         let prune_views = || -> Result<(), PathError> {            let view_dir = self.path.join("views");            for entry in view_dir.read_dir().context(&view_dir)? {                let entry = entry.context(&view_dir)?;                let Some(id) = to_view_id(&entry) else {                    tracing::trace!(?entry, "skipping invalid file name");                    continue;                };                if reachable_views.contains(&id) {                    continue;                }                remove_file_if_not_new(&entry)?;            }            Ok(())        };        prune_views().map_err(|err| OpStoreError::Other(err.into()))?;         Ok(())    }} fn io_to_read_error(err: std::io::Error, id: &impl ObjectId) -> OpStoreError {    if err.kind() == ErrorKind::NotFound {        OpStoreError::ObjectNotFound {            object_type: id.object_type(),            hash: id.hex(),            source: Box::new(err),        }    } else {        OpStoreError::ReadObject {            object_type: id.object_type(),            hash: id.hex(),            source: Box::new(err),        }    }} fn io_to_write_error(err: std::io::Error, object_type: &'static str) -> OpStoreError {    OpStoreError::WriteObject {        object_type,        source: Box::new(err),    }} fn timestamp_to_proto(timestamp: &Timestamp) -> crate::protos::op_store::Timestamp {    crate::protos::op_store::Timestamp {        millis_since_epoch: timestamp.timestamp.0,        tz_offset: timestamp.tz_offset,    }} fn timestamp_from_proto(proto: crate::protos::op_store::Timestamp) -> Timestamp {    Timestamp {        timestamp: MillisSinceEpoch(proto.millis_since_epoch),        tz_offset: proto.tz_offset,    }} fn operation_metadata_to_proto(    metadata: &OperationMetadata,) -> crate::protos::op_store::OperationMetadata {    crate::protos::op_store::OperationMetadata {        start_time: Some(timestamp_to_proto(&metadata.start_time)),        end_time: Some(timestamp_to_proto(&metadata.end_time)),        description: metadata.description.clone(),        hostname: metadata.hostname.clone(),        username: metadata.username.clone(),        is_snapshot: metadata.is_snapshot,        tags: metadata.tags.clone(),    }} fn operation_metadata_from_proto(    proto: crate::protos::op_store::OperationMetadata,) -> OperationMetadata {    let start_time = timestamp_from_proto(proto.start_time.unwrap_or_default());    let end_time = timestamp_from_proto(proto.end_time.unwrap_or_default());    OperationMetadata {        start_time,        end_time,        description: proto.description,        hostname: proto.hostname,        username: proto.username,        is_snapshot: proto.is_snapshot,        tags: proto.tags,    }} fn operation_to_proto(operation: &Operation) -> crate::protos::op_store::Operation {    let mut proto = crate::protos::op_store::Operation {        view_id: operation.view_id.as_bytes().to_vec(),        metadata: Some(operation_metadata_to_proto(&operation.metadata)),        ..Default::default()    };    for parent in &operation.parents {        proto.parents.push(parent.to_bytes());    }    proto} fn operation_from_proto(proto: crate::protos::op_store::Operation) -> Operation {    let parents = proto.parents.into_iter().map(OperationId::new).collect();    let view_id = ViewId::new(proto.view_id);    let metadata = operation_metadata_from_proto(proto.metadata.unwrap_or_default());    Operation {        view_id,        parents,        metadata,    }} fn view_to_proto(view: &View) -> crate::protos::op_store::View {    let mut proto = crate::protos::op_store::View {        // New/loaded view should have been migrated to the latest format        has_git_refs_migrated_to_remote: true,        ..Default::default()    };    for (workspace_id, commit_id) in &view.wc_commit_ids {        proto            .wc_commit_ids            .insert(workspace_id.as_str().to_string(), commit_id.to_bytes());    }    for head_id in &view.head_ids {        proto.head_ids.push(head_id.to_bytes());    }     proto.branches = branch_views_to_proto_legacy(&view.local_branches, &view.remote_views);     for (name, target) in &view.tags {        proto.tags.push(crate::protos::op_store::Tag {            name: name.clone(),            target: ref_target_to_proto(target),        });    }     for (git_ref_name, target) in &view.git_refs {        proto.git_refs.push(crate::protos::op_store::GitRef {            name: git_ref_name.clone(),            target: ref_target_to_proto(target),            ..Default::default()        });    }     proto.git_head = ref_target_to_proto(&view.git_head);     proto} fn view_from_proto(proto: crate::protos::op_store::View) -> View {    let mut view = View::default();    // For compatibility with old repos before we had support for multiple working    // copies    #[allow(deprecated)]    if !proto.wc_commit_id.is_empty() {        view.wc_commit_ids            .insert(WorkspaceId::default(), CommitId::new(proto.wc_commit_id));    }    for (workspace_id, commit_id) in proto.wc_commit_ids {        view.wc_commit_ids            .insert(WorkspaceId::new(workspace_id), CommitId::new(commit_id));    }    for head_id_bytes in proto.head_ids {        view.head_ids.insert(CommitId::new(head_id_bytes));    }     let (local_branches, remote_views) = branch_views_from_proto_legacy(proto.branches);    view.local_branches = local_branches;    view.remote_views = remote_views;     for tag_proto in proto.tags {        view.tags            .insert(tag_proto.name, ref_target_from_proto(tag_proto.target));    }     for git_ref in proto.git_refs {        let target = if git_ref.target.is_some() {            ref_target_from_proto(git_ref.target)        } else {            // Legacy format            RefTarget::normal(CommitId::new(git_ref.commit_id))        };        view.git_refs.insert(git_ref.name, target);    }     #[allow(deprecated)]    if proto.git_head.is_some() {        view.git_head = ref_target_from_proto(proto.git_head);    } else if !proto.git_head_legacy.is_empty() {        view.git_head = RefTarget::normal(CommitId::new(proto.git_head_legacy));    }     if !proto.has_git_refs_migrated_to_remote {        migrate_git_refs_to_remote(&mut view);    }     view} fn branch_views_to_proto_legacy(    local_branches: &BTreeMap<String, RefTarget>,    remote_views: &BTreeMap<String, RemoteView>,) -> Vec<crate::protos::op_store::Branch> {    op_store::merge_join_branch_views(local_branches, remote_views)        .map(|(name, branch_target)| {            let local_target = ref_target_to_proto(branch_target.local_target);            let remote_branches = branch_target                .remote_refs                .iter()                .map(

Lines 1–500 of 1004. Load more