History

Start work on storing channel buffers

Max Brunsfeld committed 3 years ago

Commit ff5035e1 parent 29e433816 files · +484 −4Browse files

Unified · Split

16 files
  1. Cargo.lock +2 −0
  2. Cargo.toml +1 −0
  3. crates/collab/Cargo.toml +2 −0
  4. crates/collab/migrations.sqlite/20221109000000_test_schema.sql +26 −1
  5. crates/collab/migrations/20230819154600_add_channel_buffers.sql +25 −0
  6. crates/collab/src/db.rs +2 −0
  7. crates/collab/src/db/ids.rs +1 −0
  8. crates/collab/src/db/queries.rs +4 −0
  9. crates/collab/src/db/queries/buffer_tests.rs +41 −0
  10. crates/collab/src/db/queries/buffers.rs +271 −0
  11. crates/collab/src/db/tables.rs +3 −0
  12. crates/collab/src/db/tables/buffer.rs +32 −0
  13. crates/collab/src/db/tables/buffer_operation.rs +37 −0
  14. crates/collab/src/db/tables/buffer_snapshot.rs +30 −0
  15. crates/collab/src/db/test_db.rs +6 −2
  16. crates/rpc/Cargo.toml +1 −1
Cargo.lock+2−0
@@ -1444,6 +1444,7 @@ dependencies = [ · 1443 unmodified lines
14441444 "pretty_assertions",
14451445 "project",
14461446 "prometheus",
1447+ "prost 0.8.0",
14471448 "rand 0.8.5",
14481449 "reqwest",
14491450 "rpc",
@@ -1456,6 +1457,7 @@ dependencies = [ · 6 unmodified lines
14561457 "settings",
14571458 "sha-1 0.9.8",
14581459 "sqlx",
1460+ "text",
14591461 "theme",
14601462 "time 0.3.24",
14611463 "tokio",
Cargo.toml+1−0
@@ -96,6 +96,7 @@ log = { version = "0.4.16", features = ["kv_unstable_serde"] } · 95 unmodified lines
9696 ordered-float = { version = "2.1.1" }
9797 parking_lot = { version = "0.11.1" }
9898 postage = { version = "0.5", features = ["futures-traits"] }
99+prost = { version = "0.8" }
99100 rand = { version = "0.8.5" }
100101 refineable = { path = "./crates/refineable" }
101102 regex = { version = "1.5" }
crates/collab/Cargo.toml+2−0
@@ -16,6 +16,7 @@ required-features = ["seed-support"] · 15 unmodified lines
1616 [dependencies]
1717 collections = { path = "../collections" }
1818 live_kit_server = { path = "../live_kit_server" }
19+text = { path = "../text" }
1920 rpc = { path = "../rpc" }
2021 util = { path = "../util" }
2122
@@ -35,6 +36,7 @@ log.workspace = true · 13 unmodified lines
3536 nanoid = "0.4"
3637 parking_lot.workspace = true
3738 prometheus = "0.13"
39+prost.workspace = true
3840 rand.workspace = true
3941 reqwest = { version = "0.11", features = ["json"], optional = true }
4042 scrypt = "0.7"
crates/collab/migrations.sqlite/20221109000000_test_schema.sql+26−1
@@ -189,7 +189,8 @@ CREATE INDEX "index_followers_on_room_id" ON "followers" ("room_id"); · 188 unmodified lines
189189 CREATE TABLE "channels" (
190190 "id" INTEGER PRIMARY KEY AUTOINCREMENT,
191191 "name" VARCHAR NOT NULL,
192− "created_at" TIMESTAMP NOT NULL DEFAULT now
192+ "created_at" TIMESTAMP NOT NULL DEFAULT now,
193+ "main_buffer_id" INTEGER REFERENCES buffers (id)
193194 );
194195
195196 CREATE TABLE "channel_paths" (
@@ -208,3 +209,27 @@ CREATE TABLE "channel_members" ( · 12 unmodified lines
208209 );
209210
210211 CREATE UNIQUE INDEX "index_channel_members_on_channel_id_and_user_id" ON "channel_members" ("channel_id", "user_id");
212+
213+CREATE TABLE "buffers" (
214+ "id" INTEGER PRIMARY KEY AUTOINCREMENT,
215+ "epoch" INTEGER NOT NULL DEFAULT 0
216+);
217+
218+CREATE TABLE "buffer_operations" (
219+ "buffer_id" INTEGER NOT NULL REFERENCES buffers (id) ON DELETE CASCADE,
220+ "epoch" INTEGER NOT NULL,
221+ "replica_id" INTEGER NOT NULL,
222+ "lamport_timestamp" INTEGER NOT NULL,
223+ "local_timestamp" INTEGER NOT NULL,
224+ "version" BLOB NOT NULL,
225+ "is_undo" BOOLEAN NOT NULL,
226+ "value" BLOB NOT NULL,
227+ PRIMARY KEY(buffer_id, epoch, lamport_timestamp, replica_id)
228+);
229+
230+CREATE TABLE "buffer_snapshots" (
231+ "buffer_id" INTEGER NOT NULL REFERENCES buffers (id) ON DELETE CASCADE,
232+ "epoch" INTEGER NOT NULL,
233+ "text" TEXT NOT NULL,
234+ PRIMARY KEY(buffer_id, epoch)
235+);
crates/collab/migrations/20230819154600_add_channel_buffers.sql+25−0
@@ -0,0 +1,25 @@
1+CREATE TABLE "buffers" (
2+ "id" SERIAL PRIMARY KEY,
3+ "epoch" INTEGER NOT NULL DEFAULT 0
4+);
5+
6+CREATE TABLE "buffer_operations" (
7+ "buffer_id" INTEGER NOT NULL REFERENCES buffers (id) ON DELETE CASCADE,
8+ "epoch" INTEGER NOT NULL,
9+ "replica_id" INTEGER NOT NULL,
10+ "local_timestamp" INTEGER NOT NULL,
11+ "lamport_timestamp" INTEGER NOT NULL,
12+ "version" BYTEA NOT NULL,
13+ "is_undo" BOOLEAN NOT NULL,
14+ "value" BYTEA NOT NULL,
15+ PRIMARY KEY(buffer_id, epoch, lamport_timestamp, replica_id)
16+);
17+
18+CREATE TABLE "buffer_snapshots" (
19+ "buffer_id" INTEGER NOT NULL REFERENCES buffers (id) ON DELETE CASCADE,
20+ "epoch" INTEGER NOT NULL,
21+ "text" TEXT NOT NULL,
22+ PRIMARY KEY(buffer_id, epoch)
23+);
24+
25+ALTER TABLE "channels" ADD COLUMN "main_buffer_id" INTEGER REFERENCES buffers (id);
crates/collab/src/db.rs+2−0
@@ -52,6 +52,8 @@ pub struct Database { · 51 unmodified lines
5252 runtime: Option<tokio::runtime::Runtime>,
5353 }
5454
55+// The `Database` type has so many methods that its impl blocks are split into
56+// separate files in the `queries` folder.
5557 impl Database {
5658 pub async fn new(options: ConnectOptions, executor: Executor) -> Result<Self> {
5759 Ok(Self {
crates/collab/src/db/ids.rs+1−0
@@ -110,6 +110,7 @@ fn value_to_integer(v: Value) -> Result<i32, ValueTypeErr> { · 109 unmodified lines
110110 }
111111 }
112112
113+id_type!(BufferId);
113114 id_type!(AccessTokenId);
114115 id_type!(ChannelId);
115116 id_type!(ChannelMemberId);
crates/collab/src/db/queries.rs+4−0
@@ -1,6 +1,7 @@
11 use super::*;
22
33 pub mod access_tokens;
4+pub mod buffers;
45 pub mod channels;
56 pub mod contacts;
67 pub mod projects;
@@ -8,3 +9,6 @@ pub mod rooms; · 1 unmodified line
89 pub mod servers;
910 pub mod signups;
1011 pub mod users;
12+
13+#[cfg(test)]
14+pub mod buffer_tests;
crates/collab/src/db/queries/buffer_tests.rs+41−0
@@ -0,0 +1,41 @@
1+use super::*;
2+use crate::test_both_dbs;
3+use language::proto;
4+use text::Buffer;
5+
6+test_both_dbs!(test_buffers, test_buffers_postgres, test_buffers_sqlite);
7+
8+async fn test_buffers(db: &Arc<Database>) {
9+ let buffer_id = db.create_buffer().await.unwrap();
10+
11+ let mut buffer = Buffer::new(0, 0, "".to_string());
12+ let mut operations = Vec::new();
13+ operations.push(buffer.edit([(0..0, "hello world")]));
14+ operations.push(buffer.edit([(5..5, ", cruel")]));
15+ operations.push(buffer.edit([(0..5, "goodbye")]));
16+ operations.push(buffer.undo().unwrap().1);
17+ assert_eq!(buffer.text(), "hello, cruel world");
18+
19+ let operations = operations
20+ .into_iter()
21+ .map(|op| proto::serialize_operation(&language::Operation::Buffer(op)))
22+ .collect::<Vec<_>>();
23+
24+ db.update_buffer(buffer_id, &operations).await.unwrap();
25+
26+ let buffer_data = db.get_buffer(buffer_id).await.unwrap();
27+
28+ let mut buffer_2 = Buffer::new(0, 0, buffer_data.base_text);
29+ buffer_2
30+ .apply_ops(buffer_data.operations.into_iter().map(|operation| {
31+ let operation = proto::deserialize_operation(operation).unwrap();
32+ if let language::Operation::Buffer(operation) = operation {
33+ operation
34+ } else {
35+ unreachable!()
36+ }
37+ }))
38+ .unwrap();
39+
40+ assert_eq!(buffer_2.text(), "hello, cruel world");
41+}
crates/collab/src/db/queries/buffers.rs+271−0
@@ -0,0 +1,271 @@
1+use super::*;
2+use prost::Message;
3+
4+pub struct Buffer {
5+ pub base_text: String,
6+ pub operations: Vec<proto::Operation>,
7+}
8+
9+impl Database {
10+ pub async fn create_buffer(&self) -> Result<BufferId> {
11+ self.transaction(|tx| async move {
12+ let buffer = buffer::ActiveModel::new().insert(&*tx).await?;
13+ Ok(buffer.id)
14+ })
15+ .await
16+ }
17+
18+ pub async fn update_buffer(
19+ &self,
20+ buffer_id: BufferId,
21+ operations: &[proto::Operation],
22+ ) -> Result<()> {
23+ self.transaction(|tx| async move {
24+ let buffer = buffer::Entity::find_by_id(buffer_id)
25+ .one(&*tx)
26+ .await?
27+ .ok_or_else(|| anyhow!("no such buffer"))?;
28+ buffer_operation::Entity::insert_many(operations.iter().filter_map(|operation| {
29+ match operation.variant.as_ref()? {
30+ proto::operation::Variant::Edit(operation) => {
31+ let value =
32+ serialize_edit_operation(&operation.ranges, &operation.new_text);
33+ let version = serialize_version(&operation.version);
34+ Some(buffer_operation::ActiveModel {
35+ buffer_id: ActiveValue::Set(buffer_id),
36+ epoch: ActiveValue::Set(buffer.epoch),
37+ replica_id: ActiveValue::Set(operation.replica_id as i32),
38+ lamport_timestamp: ActiveValue::Set(operation.lamport_timestamp as i32),
39+ local_timestamp: ActiveValue::Set(operation.local_timestamp as i32),
40+ is_undo: ActiveValue::Set(false),
41+ version: ActiveValue::Set(version),
42+ value: ActiveValue::Set(value),
43+ })
44+ }
45+ proto::operation::Variant::Undo(operation) => {
46+ let value = serialize_undo_operation(&operation.counts);
47+ let version = serialize_version(&operation.version);
48+ Some(buffer_operation::ActiveModel {
49+ buffer_id: ActiveValue::Set(buffer_id),
50+ epoch: ActiveValue::Set(buffer.epoch),
51+ replica_id: ActiveValue::Set(operation.replica_id as i32),
52+ lamport_timestamp: ActiveValue::Set(operation.lamport_timestamp as i32),
53+ local_timestamp: ActiveValue::Set(operation.local_timestamp as i32),
54+ is_undo: ActiveValue::Set(true),
55+ version: ActiveValue::Set(version),
56+ value: ActiveValue::Set(value),
57+ })
58+ }
59+ proto::operation::Variant::UpdateSelections(_) => None,
60+ proto::operation::Variant::UpdateDiagnostics(_) => None,
61+ proto::operation::Variant::UpdateCompletionTriggers(_) => None,
62+ }
63+ }))
64+ .exec(&*tx)
65+ .await?;
66+
67+ Ok(())
68+ })
69+ .await
70+ }
71+
72+ pub async fn get_buffer(&self, id: BufferId) -> Result<Buffer> {
73+ self.transaction(|tx| async move {
74+ let buffer = buffer::Entity::find_by_id(id)
75+ .one(&*tx)
76+ .await?
77+ .ok_or_else(|| anyhow!("no such buffer"))?;
78+
79+ let base_text = if buffer.epoch > 0 {
80+ buffer_snapshot::Entity::find()
81+ .filter(
82+ buffer_snapshot::Column::BufferId
83+ .eq(id)
84+ .and(buffer_snapshot::Column::Epoch.eq(buffer.epoch)),
85+ )
86+ .one(&*tx)
87+ .await?
88+ .ok_or_else(|| anyhow!("no such snapshot"))?
89+ .text
90+ } else {
91+ String::new()
92+ };
93+
94+ let mut rows = buffer_operation::Entity::find()
95+ .filter(
96+ buffer_operation::Column::BufferId
97+ .eq(id)
98+ .and(buffer_operation::Column::Epoch.eq(buffer.epoch)),
99+ )
100+ .stream(&*tx)
101+ .await?;
102+ let mut operations = Vec::new();
103+ while let Some(row) = rows.next().await {
104+ let row = row?;
105+ let version = deserialize_version(&row.version)?;
106+ let operation = if row.is_undo {
107+ let counts = deserialize_undo_operation(&row.value)?;
108+ proto::operation::Variant::Undo(proto::operation::Undo {
109+ replica_id: row.replica_id as u32,
110+ local_timestamp: row.local_timestamp as u32,
111+ lamport_timestamp: row.lamport_timestamp as u32,
112+ version,
113+ counts,
114+ })
115+ } else {
116+ let (ranges, new_text) = deserialize_edit_operation(&row.value)?;
117+ proto::operation::Variant::Edit(proto::operation::Edit {
118+ replica_id: row.replica_id as u32,
119+ local_timestamp: row.local_timestamp as u32,
120+ lamport_timestamp: row.lamport_timestamp as u32,
121+ version,
122+ ranges,
123+ new_text,
124+ })
125+ };
126+ operations.push(proto::Operation {
127+ variant: Some(operation),
128+ })
129+ }
130+
131+ Ok(Buffer {
132+ base_text,
133+ operations,
134+ })
135+ })
136+ .await
137+ }
138+}
139+
140+mod storage {
141+ #![allow(non_snake_case)]
142+
143+ use prost::Message;
144+
145+ pub const VERSION: usize = 1;
146+
147+ #[derive(Message)]
148+ pub struct VectorClock {
149+ #[prost(message, repeated, tag = "1")]
150+ pub entries: Vec<VectorClockEntry>,
151+ }
152+
153+ #[derive(Message)]
154+ pub struct VectorClockEntry {
155+ #[prost(uint32, tag = "1")]
156+ pub replica_id: u32,
157+ #[prost(uint32, tag = "2")]
158+ pub timestamp: u32,
159+ }
160+
161+ #[derive(Message)]
162+ pub struct TextEdit {
163+ #[prost(message, repeated, tag = "1")]
164+ pub ranges: Vec<Range>,
165+ #[prost(string, repeated, tag = "2")]
166+ pub texts: Vec<String>,
167+ }
168+
169+ #[derive(Message)]
170+ pub struct Range {
171+ #[prost(uint64, tag = "1")]
172+ pub start: u64,
173+ #[prost(uint64, tag = "2")]
174+ pub end: u64,
175+ }
176+
177+ #[derive(Message)]
178+ pub struct Undo {
179+ #[prost(message, repeated, tag = "1")]
180+ pub entries: Vec<UndoCount>,
181+ }
182+
183+ #[derive(Message)]
184+ pub struct UndoCount {
185+ #[prost(uint32, tag = "1")]
186+ pub replica_id: u32,
187+ #[prost(uint32, tag = "2")]
188+ pub local_timestamp: u32,
189+ #[prost(uint32, tag = "3")]
190+ pub count: u32,
191+ }
192+}
193+
194+fn serialize_version(version: &Vec<proto::VectorClockEntry>) -> Vec<u8> {
195+ storage::VectorClock {
196+ entries: version
197+ .iter()
198+ .map(|entry| storage::VectorClockEntry {
199+ replica_id: entry.replica_id,
200+ timestamp: entry.timestamp,
201+ })
202+ .collect(),
203+ }
204+ .encode_to_vec()
205+}
206+
207+fn deserialize_version(bytes: &[u8]) -> Result<Vec<proto::VectorClockEntry>> {
208+ let clock = storage::VectorClock::decode(bytes).map_err(|error| anyhow!("{}", error))?;
209+ Ok(clock
210+ .entries
211+ .into_iter()
212+ .map(|entry| proto::VectorClockEntry {
213+ replica_id: entry.replica_id,
214+ timestamp: entry.timestamp,
215+ })
216+ .collect())
217+}
218+
219+fn serialize_edit_operation(ranges: &[proto::Range], texts: &[String]) -> Vec<u8> {
220+ storage::TextEdit {
221+ ranges: ranges
222+ .iter()
223+ .map(|range| storage::Range {
224+ start: range.start,
225+ end: range.end,
226+ })
227+ .collect(),
228+ texts: texts.to_vec(),
229+ }
230+ .encode_to_vec()
231+}
232+
233+fn deserialize_edit_operation(bytes: &[u8]) -> Result<(Vec<proto::Range>, Vec<String>)> {
234+ let edit = storage::TextEdit::decode(bytes).map_err(|error| anyhow!("{}", error))?;
235+ let ranges = edit
236+ .ranges
237+ .into_iter()
238+ .map(|range| proto::Range {
239+ start: range.start,
240+ end: range.end,
241+ })
242+ .collect();
243+ Ok((ranges, edit.texts))
244+}
245+
246+fn serialize_undo_operation(counts: &Vec<proto::UndoCount>) -> Vec<u8> {
247+ storage::Undo {
248+ entries: counts
249+ .iter()
250+ .map(|entry| storage::UndoCount {
251+ replica_id: entry.replica_id,
252+ local_timestamp: entry.local_timestamp,
253+ count: entry.count,
254+ })
255+ .collect(),
256+ }
257+ .encode_to_vec()
258+}
259+
260+fn deserialize_undo_operation(bytes: &[u8]) -> Result<Vec<proto::UndoCount>> {
261+ let undo = storage::Undo::decode(bytes).map_err(|error| anyhow!("{}", error))?;
262+ Ok(undo
263+ .entries
264+ .iter()
265+ .map(|entry| proto::UndoCount {
266+ replica_id: entry.replica_id,
267+ local_timestamp: entry.local_timestamp,
268+ count: entry.count,
269+ })
270+ .collect())
271+}
crates/collab/src/db/tables.rs+3−0
@@ -1,4 +1,7 @@
11 pub mod access_token;
2+pub mod buffer;
3+pub mod buffer_operation;
4+pub mod buffer_snapshot;
25 pub mod channel;
36 pub mod channel_member;
47 pub mod channel_path;
crates/collab/src/db/tables/buffer.rs+32−0
@@ -0,0 +1,32 @@
1+use crate::db::BufferId;
2+use sea_orm::entity::prelude::*;
3+
4+#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
5+#[sea_orm(table_name = "buffers")]
6+pub struct Model {
7+ #[sea_orm(primary_key)]
8+ pub id: BufferId,
9+ pub epoch: i32,
10+}
11+
12+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
13+pub enum Relation {
14+ #[sea_orm(has_many = "super::buffer_operation::Entity")]
15+ Operations,
16+ #[sea_orm(has_many = "super::buffer_snapshot::Entity")]
17+ Snapshots,
18+}
19+
20+impl Related<super::buffer_operation::Entity> for Entity {
21+ fn to() -> RelationDef {
22+ Relation::Operations.def()
23+ }
24+}
25+
26+impl Related<super::buffer_snapshot::Entity> for Entity {
27+ fn to() -> RelationDef {
28+ Relation::Snapshots.def()
29+ }
30+}
31+
32+impl ActiveModelBehavior for ActiveModel {}
crates/collab/src/db/tables/buffer_operation.rs+37−0
@@ -0,0 +1,37 @@
1+use crate::db::BufferId;
2+use sea_orm::entity::prelude::*;
3+
4+#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
5+#[sea_orm(table_name = "buffer_operations")]
6+pub struct Model {
7+ #[sea_orm(primary_key)]
8+ pub buffer_id: BufferId,
9+ #[sea_orm(primary_key)]
10+ pub epoch: i32,
11+ #[sea_orm(primary_key)]
12+ pub lamport_timestamp: i32,
13+ #[sea_orm(primary_key)]
14+ pub replica_id: i32,
15+ pub local_timestamp: i32,
16+ pub version: Vec<u8>,
17+ pub is_undo: bool,
18+ pub value: Vec<u8>,
19+}
20+
21+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
22+pub enum Relation {
23+ #[sea_orm(
24+ belongs_to = "super::buffer::Entity",
25+ from = "Column::BufferId",
26+ to = "super::buffer::Column::Id"
27+ )]
28+ Buffer,
29+}
30+
31+impl Related<super::buffer::Entity> for Entity {
32+ fn to() -> RelationDef {
33+ Relation::Buffer.def()
34+ }
35+}
36+
37+impl ActiveModelBehavior for ActiveModel {}
crates/collab/src/db/tables/buffer_snapshot.rs+30−0
@@ -0,0 +1,30 @@
1+use crate::db::BufferId;
2+use sea_orm::entity::prelude::*;
3+
4+#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
5+#[sea_orm(table_name = "buffer_snapshots")]
6+pub struct Model {
7+ #[sea_orm(primary_key)]
8+ pub buffer_id: BufferId,
9+ #[sea_orm(primary_key)]
10+ pub epoch: i32,
11+ pub text: String,
12+}
13+
14+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
15+pub enum Relation {
16+ #[sea_orm(
17+ belongs_to = "super::buffer::Entity",
18+ from = "Column::BufferId",
19+ to = "super::buffer::Column::Id"
20+ )]
21+ Buffer,
22+}
23+
24+impl Related<super::buffer::Entity> for Entity {
25+ fn to() -> RelationDef {
26+ Relation::Buffer.def()
27+ }
28+}
29+
30+impl ActiveModelBehavior for ActiveModel {}
crates/collab/src/db/test_db.rs+6−2
@@ -96,13 +96,17 @@ macro_rules! test_both_dbs { · 95 unmodified lines
9696 ($test_name:ident, $postgres_test_name:ident, $sqlite_test_name:ident) => {
9797 #[gpui::test]
9898 async fn $postgres_test_name() {
99− let test_db = TestDb::postgres(Deterministic::new(0).build_background());
99+ let test_db = crate::db::test_db::TestDb::postgres(
100+ gpui::executor::Deterministic::new(0).build_background(),
101+ );
100102 $test_name(test_db.db()).await;
101103 }
102104
103105 #[gpui::test]
104106 async fn $sqlite_test_name() {
105− let test_db = TestDb::sqlite(Deterministic::new(0).build_background());
107+ let test_db = crate::db::test_db::TestDb::sqlite(
108+ gpui::executor::Deterministic::new(0).build_background(),
109+ );
106110 $test_name(test_db.db()).await;
107111 }
108112 };
crates/rpc/Cargo.toml+1−1
@@ -23,7 +23,7 @@ async-tungstenite = "0.16" · 22 unmodified lines
2323 base64 = "0.13"
2424 futures.workspace = true
2525 parking_lot.workspace = true
26−prost = "0.8"
26+prost.workspace = true
2727 rand.workspace = true
2828 rsa = "0.4"
2929 serde.workspace = true

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t