Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat concurrent replay wal #1505

Merged
merged 10 commits into from
Mar 25, 2024
Merged

Feat concurrent replay wal #1505

Changes from 8 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
3c8ae48
feat: add sst level config for benchmark
zealchen Feb 19, 2024
e518369
Merge pull request #1 from zealchen/feat_bench_config
zealchen Feb 19, 2024
2513c68
Merge branch 'apache:main' into main
zealchen Feb 21, 2024
405f617
Merge branch 'apache:main' into main
zealchen Feb 24, 2024
7e560e9
Merge branch 'apache:main' into main
zealchen Mar 8, 2024
ca42135
Merge branch 'apache:main' into main
zealchen Mar 12, 2024
8c3bd80
add concurrent replay for tables
zealchen Mar 23, 2024
b1ff6b6
Merge branch 'main' into feat_concurrent_replay_wal
zealchen Mar 24, 2024
2ab5a75
remove Arc
Mar 25, 2024
cfa69b6
remove Arc
Mar 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 20 additions & 5 deletions src/analytic_engine/src/instance/wal_replayer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,13 +182,28 @@ impl Replay for TableBasedReplay {
debug!("Replay wal logs on table mode, context:{context}, tables:{table_datas:?}",);

let mut failed_tables = HashMap::new();
let read_ctx = ReadContext {
let read_ctx = Arc::new(ReadContext {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This Arc is not required.

-        for table_data in table_datas {
-            let table_id = table_data.id;
-            if let Err(e) = Self::recover_table_logs(context, table_data, &read_ctx).await {
+
+        let mut tasks = futures::stream::iter(
+            table_datas
+                .iter()
+                .map(|table_data| {
+                    let table_id = table_data.id;
+                    let read_ctx = &read_ctx;
+                    async move {
+                        let ret = Self::recover_table_logs(context, table_data, read_ctx).await;
+                        (table_id, ret)
+                    }
+                })
+                .collect::<Vec<_>>(),
+        )
+        .buffer_unordered(20);
+        while let Some((table_id, ret)) = tasks.next().await {
+            if let Err(e) = ret {
+                // If occur error, mark this table as failed and store the cause.
                 failed_tables.insert(table_id, e);
             }
         }

This compile success for me.

batch_size: context.wal_replay_batch_size,
..Default::default()
};
for table_data in table_datas {
let table_id = table_data.id;
if let Err(e) = Self::recover_table_logs(context, table_data, &read_ctx).await {
});

let mut tasks = futures::stream::iter(
table_datas
.iter()
.map(|table_data| {
let table_id = table_data.id;
let read_ctx = read_ctx.clone();
async move {
let ret = Self::recover_table_logs(context, table_data, &read_ctx).await;
(table_id, ret)
}
})
.collect::<Vec<_>>(),
)
.buffer_unordered(20);
while let Some((table_id, ret)) = tasks.next().await {
if let Err(e) = ret {
// If occur error, mark this table as failed and store the cause.
failed_tables.insert(table_id, e);
}
}
Expand Down
Loading