Skip to content

Commit 21e8dc9

Browse files
olemon111olemon111
committed
init from SWP-Tree-BG1 and update README
0 parents  commit 21e8dc9

File tree

185 files changed

+76235
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

185 files changed

+76235
-0
lines changed

‎.gitignore

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
## Test
2+
a.out
3+
# *.txt
4+
*.xlsx
5+
6+
*.svg
7+
8+
## CMake
9+
build/
10+
# src/combotree_config.h
11+
12+
# CMakeLists.txt.user
13+
# CMakeCache.txt
14+
# CMakeFiles
15+
# CMakeScripts
16+
# Testing
17+
# Makefile
18+
# cmake_install.cmake
19+
# install_manifest.txt
20+
# compile_commands.json
21+
# CTestTestfile.cmake
22+
# _deps
23+
24+
## VSCode
25+
.vscode
26+
27+
## TeX
28+
29+
# Core latex/pdflatex auxiliary files:
30+
*.aux
31+
*.lof
32+
*.log
33+
*.lot
34+
*.fls
35+
*.out
36+
*.toc
37+
*.fmt
38+
*.fot
39+
*.cb
40+
*.cb2
41+
.*.lb
42+
43+
# Intermediate documents:
44+
*.dvi
45+
*.xdv
46+
*-converted-to.*
47+
48+
# tcolorbox
49+
*.listing
50+
51+
# minted
52+
_minted*
53+
*.pyg
54+
55+
# Generated if empty string is given at "Please type another file name for output:"
56+
.pdf
57+
58+
# Bibliography auxiliary files (bibtex/biblatex/biber):
59+
*.bbl
60+
*.bcf
61+
*.blg
62+
*-blx.aux
63+
*-blx.bib
64+
*.run.xml
65+
66+
# Build tool auxiliary files:
67+
*.fdb_latexmk
68+
*.synctex
69+
*.synctex(busy)
70+
*.synctex.gz
71+
*.synctex.gz(busy)
72+
*.pdfsync
73+
*.dat
74+
75+
show_file.sh
76+
result/*
77+
# *.spec
78+
*.bak
79+
*.old

‎CMakeLists.txt

+247
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,247 @@
1+
project(WOLI)
2+
cmake_minimum_required(VERSION 3.0)
3+
4+
# cmake -D CMAKE_C_COMPILER=/usr/local/bin/gcc CMAKE_CXX_COMPILER=/usr/local/bin/g++ ../
5+
# SET(CMAKE_C_COMPILER /usr/local/bin/gcc)
6+
# SET(CMAKE_CXX_COMPILER /usr/local/bin/g++)
7+
set(MKL_LINK_DIRECTORY "/opt/intel/mkl/lib/intel64")
8+
set(MKL_INCLUDE_DIRECTORY "/opt/intel/mkl/include")
9+
include(FetchContent)
10+
include(ExternalProject)
11+
12+
# use `cmake -DSERVER:BOOL=ON ..` when running in server
13+
option(SERVER "Run in server" ON)
14+
option(USE_LIBPMEM "libpmem or libvmmalloc" ON)
15+
option(BUF_SORT "Sort buffer in KVBufer" OFF)
16+
option(STREAMING_LOAD "Use Non-temporal Load" OFF)
17+
option(STREAMING_STORE "Use Non-temporal Store" OFF)
18+
option(NO_LOCK "Don't use lock" OFF)
19+
option(BRANGE "Multi-thread expanding" ON)
20+
option(NO_ENTRY_BUF "BEntry without KVBuffer" ON)
21+
22+
# ComboTree Configuration
23+
# use `make clean && make CXX_DEFINES="-DNAME=VALUE"` to override during compile
24+
if(SERVER)
25+
set(PMEM_DIR \"/mnt/pmem0/\")
26+
set(CLEVEL_PMEM_FILE_SIZE "(1024*1024*1024*32UL)")
27+
set(CLEVEL_PMEM_FILE \"/mnt/pmem0/combotree-clevel-\")
28+
set(BLEVEL_PMEM_FILE \"/mnt/pmem0/combotree-blevel-\")
29+
set(ALEVEL_PMEM_FILE \"/mnt/pmem0/combotree-alevel-\")
30+
set(PGM_INDEX_PMEM_FILE \"/mnt/pmem0/combotree-pgmindex-\")
31+
set(COMMON_PMEM_FILE \"/mnt/pmem0/common-alloctor\")
32+
else()
33+
set(PMEM_DIR \"/mnt/pmem0/\")
34+
set(CLEVEL_PMEM_FILE_SIZE "(1024*1024*512UL)")
35+
set(CLEVEL_PMEM_FILE \"/mnt/pmem0/combotree-clevel-\")
36+
set(BLEVEL_PMEM_FILE \"/mnt/pmem0/combotree-blevel-\")
37+
set(ALEVEL_PMEM_FILE \"/mnt/pmem0/combotree-alevel-\")
38+
set(PGM_INDEX_PMEM_FILE \"/mnt/pmem0/combotree-pgmindex-\")
39+
set(COMMON_PMEM_FILE \"/mnt/pmem0/common-alloctor\")
40+
endif(SERVER)
41+
42+
if(BRANGE)
43+
set(EXPAND_THREADS 4)
44+
endif(BRANGE)
45+
46+
if(NO_ENTRY_BUF)
47+
set(BLEVEL_EXPAND_BUF_KEY 4)
48+
else()
49+
set(BLEVEL_EXPAND_BUF_KEY 6)
50+
endif(NO_ENTRY_BUF)
51+
52+
set(EXPANSION_FACTOR 2)
53+
set(DEFAULT_SPAN 2)
54+
set(PMEMKV_THRESHOLD 10000)
55+
set(ENTRY_SIZE_FACTOR 1.2)
56+
57+
configure_file(
58+
"${PROJECT_SOURCE_DIR}/src/combotree_config.h.in"
59+
"${PROJECT_SOURCE_DIR}/src/combotree_config.h"
60+
)
61+
62+
# -march=native used to detect CPU flags and use corrosponding clflush instruction
63+
set(CMAKE_CXX_FLAGS "-march=native -Wall -Wno-reorder -O3")
64+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mrtm -msse4.1 -mavx2")
65+
66+
# Require C++17
67+
set(CMAKE_CXX_STANDARD_REQUIRED on)
68+
set(CMAKE_CXX_STANDARD 17)
69+
70+
FIND_PACKAGE(OpenMP REQUIRED)
71+
72+
if(OPENMP_FOUND)
73+
message("OPENMP FOUND")
74+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
75+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
76+
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
77+
endif()
78+
79+
add_definitions(-DNDEBUG -DNDEBUGGING)
80+
81+
# MK for RMI
82+
link_directories(${MKL_LINK_DIRECTORY})
83+
include_directories(${MKL_INCLUDE_DIRECTORY})
84+
85+
# combotree shared library
86+
include_directories(include)
87+
88+
list(APPEND libs_to_link pmemobj pmem tbb papi tcmalloc_minimal mkl_rt pthread)
89+
90+
set(COMBO_TREE_SRC
91+
src/alevel.cc
92+
src/enviroment.cc
93+
src/pgm_index.cc
94+
src/rmi_index.cc
95+
src/learn_index.cc
96+
src/blevel.cc
97+
src/bentry.cc
98+
src/clevel.cc
99+
src/combotree.cc
100+
src/pmemkv.cc
101+
102+
src/lbtree.cc
103+
src/mempool.cc
104+
src/nvm-common.cc
105+
src/tree.cc
106+
107+
# src/lbtree_wrapper.cc
108+
109+
# src/lbtree_wrapper.hpp
110+
)
111+
112+
add_library(combotree SHARED ${COMBO_TREE_SRC})
113+
target_link_libraries(combotree pmem pmemobj mkl_rt pthread)
114+
115+
# # benchmark
116+
# add_executable(benchmark tests/benchmark.cc)
117+
# target_link_libraries(benchmark combotree)
118+
119+
# # multi_benchmark
120+
# add_executable(multi_benchmark tests/multi_benchmark.cc)
121+
# target_link_libraries(multi_benchmark combotree)
122+
123+
# # multi_benchmark
124+
# add_executable(multi_bench tests/multi_bench.cc)
125+
# target_link_libraries(multi_bench combotree)
126+
127+
# # scalability_test
128+
# add_executable(scalability_test tests/scalability_test.cc)
129+
# target_link_libraries(scalability_test combotree)
130+
131+
# Unit Test
132+
enable_testing()
133+
134+
include_directories(src)
135+
136+
# # # generate_data
137+
# add_executable(generate_data tests/generate_data.cc)
138+
139+
# # # sortbuffer_test
140+
# add_executable(sortbuffer_test tests/sortbuffer_test.cc)
141+
142+
# # # clevel_test
143+
# add_executable(clevel_test tests/clevel_test.cc src/clevel.cc)
144+
# target_link_libraries(clevel_test pmem)
145+
# add_test(clevel_test clevel_test)
146+
147+
# # # combotree_test
148+
# add_executable(combotree_test tests/combotree_test.cc)
149+
# target_link_libraries(combotree_test combotree)
150+
# add_test(combotree_test combotree_test)
151+
152+
# # # multi_combotree_test
153+
# add_executable(multi_combotree_test tests/multi_combotree_test.cc)
154+
# target_link_libraries(multi_combotree_test combotree)
155+
# add_test(multi_combotree_test multi_combotree_test)
156+
157+
# # # pgm_index_test
158+
# add_executable(pgm_index_test tests/pgm_index_test.cc)
159+
# target_link_libraries(pgm_index_test combotree)
160+
# add_test(pgm_index_test pgm_index_test)
161+
162+
# # # pgm_index_test
163+
# add_executable(xindex_test tests/xindex_test.cc)
164+
# target_link_libraries(xindex_test combotree)
165+
# add_test(xindex_test xindex_test)
166+
167+
# # # learn_index_test
168+
# add_executable(learn_index_test tests/learn_index_test.cc)
169+
# target_link_libraries(learn_index_test combotree)
170+
# add_test(learn_index_test learn_index_test)
171+
172+
# # # alex_test
173+
# add_executable(alex_test tests/alex_test.cc)
174+
# target_link_libraries(alex_test combotree)
175+
# add_test(alex_test alex_test)
176+
177+
# # bucket_test
178+
# add_executable(bucket_test tests/bucket_test.cc)
179+
# target_link_libraries(bucket_test combotree)
180+
# add_test(bucket_test bucket_test)
181+
182+
# # # nvm_test
183+
# add_executable(nvm_test tests/nvm_test.cc)
184+
# target_link_libraries(nvm_test pmem)
185+
# add_test(nvm_test nvm_test)
186+
187+
# # # microbench
188+
# add_executable(microbench tests/micro_bench.cc)
189+
# target_link_libraries(microbench combotree)
190+
# add_test(microbench microbench)
191+
192+
# # microbench_swp
193+
add_executable(microbench_swp tests/micro_bench_swp.cc)
194+
195+
# add_dependencies(microbench_swp PMDK)
196+
# target_link_libraries(microbench_swp PRIVATE ${libs_to_link})
197+
target_link_libraries(microbench_swp combotree)
198+
add_test(microbench_swp microbench_swp)
199+
200+
# # # ycsb
201+
# add_executable(ycsb tests/ycsb.cc)
202+
# target_link_libraries(ycsb combotree)
203+
# add_test(ycsb ycsb)
204+
205+
# add_executable(ycsb-mem tests/ycsb-mem.cc)
206+
# target_link_libraries(ycsb-mem combotree)
207+
# add_test(ycsb-mem ycsb-mem)
208+
add_definitions(-DPMEM)
209+
add_definitions(-DPMDK)
210+
message(STATUS "Persistence support: PMDK")
211+
212+
# # #################### PMDK ####################
213+
# set(PMDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/pmdk")
214+
# ExternalProject_Add(PMDK
215+
# GIT_REPOSITORY git@github.com:XiangpengHao/pmdk.git
216+
# GIT_TAG addr-patch
217+
# BUILD_IN_SOURCE 1
218+
# BUILD_COMMAND $(MAKE) EXTRA_CFLAGS=-Wno-error
219+
# PREFIX ${PMDK_PREFIX}
220+
# CONFIGURE_COMMAND ""
221+
# INSTALL_COMMAND ""
222+
# LOG_DOWNLOAD ON
223+
# LOG_CONFIGURE ON
224+
# LOG_BUILD ON
225+
# )
226+
227+
# include_directories(${PMDK_PREFIX}/src/PMDK/src/include)
228+
229+
# # if(${CMAKE_BUILD_TYPE} STREQUAL "Debug")
230+
# # link_directories(${PMDK_PREFIX}/src/PMDK/src/debug)
231+
# # else()
232+
# link_directories(${PMDK_PREFIX}/src/PMDK/src/nondebug)
233+
234+
# # endif()
235+
236+
# #################### Epoch Reclaim ####################
237+
FetchContent_Declare(
238+
epoch_reclaimer
239+
GIT_REPOSITORY git@github.com:XiangpengHao/VeryPM.git
240+
GIT_TAG master
241+
)
242+
243+
# GIT_REPOSITORY https://github.com/XiangpengHao/epoch-reclaimer.git
244+
if(NOT epoch_reclaimer_POPULATED)
245+
FetchContent_Populate(epoch_reclaimer)
246+
include_directories(${epoch_reclaimer_SOURCE_DIR})
247+
endif()

‎README.md

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# SWP-Tree
2+
3+
> 一种新的动态适应各种倾斜负载的基于混合介质的持久化学习索引结构
4+
>
5+
> 本项目为其背景测试
6+
7+
### 测试方案：
8+
9+
- ALEX (https://github.com/microsoft/ALEX)
10+
- LIPP (https://github.com/Jiacheng-WU/lipp)
11+
- XIndex(https://ipads.se.sjtu.edu.cn:1312/opensource/xindex.git)
12+
- PGM (https://github.com/gvinciguerra/PGM-index)
13+
14+
### 测试工具
15+
16+
- YCSB (https://github.com/basicthinker/YCSB-C)
17+
18+
19+
### 测试负载
20+
- uniform负载
21+
22+
### 测试指标
23+
24+
- 吞吐量(Kops/s)
25+
26+
- DRAM空间占用(GB of DRAM)
27+
28+
### 环境配置
29+
30+
- 2 million KVs (8B key-8B value)
31+
- 每个测试项执行10 million指令

‎build.sh

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/usr/bin/env bash
2+
mkdir -p build
3+
cd build
4+
rm -rf *
5+
cmake ..
6+
make -j16
7+
cd ..
8+
chmod +x ./tests/run_microbench_swp.sh

0 commit comments

Comments
 (0)