From 4a9bdc68c2562e4157a6542c60c46962e63b6e32 Mon Sep 17 00:00:00 2001 From: David Yeung Date: Wed, 3 May 2017 15:25:46 -0400 Subject: [PATCH] feat(libs/dsm): adds dsm as a lib --- libs/dsm/CMakeLists.txt | 48 ++++ libs/dsm/Makefile | 90 +++++++ libs/dsm/cmake/FindCapnp.cmake | 186 +++++++++++++++ libs/dsm/cmake/FindTBB.cmake | 283 +++++++++++++++++++++++ libs/dsm/src/Messenger.cc | 251 ++++++++++++++++++++ libs/dsm/src/MulticoreEbb.h | 153 ++++++++++++ libs/dsm/src/RemoteMem.cc | 203 ++++++++++++++++ libs/dsm/src/RemoteMem.h | 35 +++ libs/dsm/src/StaticEbbIds.h | 13 ++ libs/dsm/src/example/hosted/Printer.cc | 27 +++ libs/dsm/src/example/hosted/Printer.h | 28 +++ libs/dsm/src/example/hosted/sharedmem.cc | 45 ++++ libs/dsm/src/example/native/Printer.cc | 60 +++++ libs/dsm/src/example/native/Printer.h | 30 +++ libs/dsm/src/example/native/ebbrtcfg.h | 13 ++ libs/dsm/src/example/native/sharedmem.cc | 87 +++++++ libs/dsm/src/sharedmem.h | 59 +++++ 17 files changed, 1611 insertions(+) create mode 100644 libs/dsm/CMakeLists.txt create mode 100644 libs/dsm/Makefile create mode 100644 libs/dsm/cmake/FindCapnp.cmake create mode 100644 libs/dsm/cmake/FindTBB.cmake create mode 100644 libs/dsm/src/Messenger.cc create mode 100644 libs/dsm/src/MulticoreEbb.h create mode 100644 libs/dsm/src/RemoteMem.cc create mode 100644 libs/dsm/src/RemoteMem.h create mode 100644 libs/dsm/src/StaticEbbIds.h create mode 100644 libs/dsm/src/example/hosted/Printer.cc create mode 100644 libs/dsm/src/example/hosted/Printer.h create mode 100644 libs/dsm/src/example/hosted/sharedmem.cc create mode 100644 libs/dsm/src/example/native/Printer.cc create mode 100644 libs/dsm/src/example/native/Printer.h create mode 100644 libs/dsm/src/example/native/ebbrtcfg.h create mode 100644 libs/dsm/src/example/native/sharedmem.cc create mode 100644 libs/dsm/src/sharedmem.h diff --git a/libs/dsm/CMakeLists.txt b/libs/dsm/CMakeLists.txt new file mode 100644 index 00000000..c178db86 --- /dev/null +++ b/libs/dsm/CMakeLists.txt @@ -0,0 +1,48 @@ +cmake_minimum_required(VERSION 2.6 FATAL_ERROR) +project("sharedmem-ebbrt" C CXX) + +set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") +set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g3") +set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os -DNDEBUG") +set(CMAKE_CXX_FLAGS_RELEASE "-O4 -flto -DNDEBUG") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g3") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++14 -Wall -Werror") + +set(HOSTED_SOURCES + src/sharedmem.h + src/RemoteMem.h + src/RemoteMem.cc + src/hosted/sharedmem.cc + src/hosted/Printer.cc) + +set(BAREMETAL_SOURCES + src/sharedmem.h + src/RemoteMem.h + src/RemoteMem.cc + src/baremetal/sharedmem.cc + src/baremetal/Printer.cc) + +# Baremetal ======================================================== +if( ${CMAKE_SYSTEM_NAME} STREQUAL "EbbRT") + add_executable(sharedmem.elf ${BAREMETAL_SOURCES}) + add_custom_command(TARGET sharedmem.elf POST_BUILD + COMMAND objcopy -O elf32-i386 sharedmem.elf sharedmem.elf32 ) + +# Hosted =========================================================== +elseif( ${CMAKE_SYSTEM_NAME} STREQUAL "Linux" ) + find_package(EbbRT REQUIRED) + find_package(Boost 1.53.0 REQUIRED COMPONENTS + filesystem system coroutine context ) + find_package(Capnp REQUIRED) + find_package(TBB REQUIRED) + find_package(Threads REQUIRED) + + include_directories(${EBBRT_INCLUDE_DIRS}) + add_executable(sharedmem ${HOSTED_SOURCES}) + target_link_libraries(sharedmem ${EBBRT_LIBRARIES} + ${CAPNP_LIBRARIES_LITE} ${CMAKE_THREAD_LIBS_INIT} + ${Boost_LIBRARIES} ${TBB_LIBRARIES} + ) +else() + message(FATAL_ERROR "System name unsupported: ${CMAKE_SYSTEM_NAME}") +endif() diff --git a/libs/dsm/Makefile b/libs/dsm/Makefile new file mode 100644 index 00000000..9ceeb14e --- /dev/null +++ b/libs/dsm/Makefile @@ -0,0 +1,90 @@ +MYDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) + +CD ?= cd +CMAKE ?= cmake +CP ?= cp +ECHO ?= echo +MAKE ?= make +MKDIR ?= mkdir + +EBBRTSYSROOT ?= $(abspath $(EBBRT_SYSROOT)) +CMAKE_TOOLCHAIN_FILE ?= $(EBBRTSYSROOT)/usr/misc/ebbrt.cmake +BAREMETAL_PREFIX_PATH= $(EBBRTSYSROOT)/usr/ + +BUILD_PATH ?= $(MYDIR) +DEBUG_PATH ?= $(BUILD_PATH)/Debug +RELEASE_PATH ?= $(BUILD_PATH)/Release +BAREMETAL_DEBUG_DIR ?= $(DEBUG_PATH)/bm +BAREMETAL_RELEASE_DIR ?= $(RELEASE_PATH)/bm +HOSTED_DEBUG_DIR ?= $(DEBUG_PATH) +HOSTED_RELEASE_DIR ?= $(RELEASE_PATH) + +all: Debug Release +hosted: hosted-debug hosted-release +baremetal: baremetal-debug baremetal-release +Debug: baremetal-debug hosted-debug +Release: baremetal-release hosted-release + +# ENVIRONMENT VARIABLES +check-ebbrt-sysroot: +ifndef EBBRT_SYSROOT + $(error EBBRT_SYSROOT is undefined) +endif + +$(BUILD_PATH): + $(MKDIR) $@ + +$(DEBUG_PATH): | $(BUILD_PATH) + $(MKDIR) $@ + +$(RELEASE_PATH): | $(BUILD_PATH) + $(MKDIR) $@ + +ifneq ($(DEBUG_PATH), $(BAREMETAL_DEBUG_DIR)) +$(BAREMETAL_DEBUG_DIR): | $(DEBUG_PATH) + $(MKDIR) $@ +endif + +ifneq ($(RELEASE_PATH), $(BAREMETAL_RELEASE_DIR)) +$(BAREMETAL_RELEASE_DIR): | $(RELEASE_PATH) + $(MKDIR) $@ +endif + +ifneq ($(DEBUG_PATH), $(HOSTED_DEBUG_DIR)) +$(HOSTED_DEBUG_DIR): | $(DEBUG_PATH) + $(MKDIR) $@ +endif + +ifneq ($(RELEASE_PATH), $(HOSTED_RELEASE_DIR)) +$(HOSTED_RELEASE_DIR): | $(RELEASE_PATH) + $(MKDIR) $@ +endif + +baremetal-debug: | check-ebbrt-sysroot $(BAREMETAL_DEBUG_DIR) + $(CD) $(BAREMETAL_DEBUG_DIR) && \ + EBBRT_SYSROOT=$(EBBRTSYSROOT) $(CMAKE) -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_PREFIX_PATH=$(BAREMETAL_PREFIX_PATH) \ + -DCMAKE_TOOLCHAIN_FILE=$(CMAKE_TOOLCHAIN_FILE) $(MYDIR) && $(MAKE) + +baremetal-release: | check-ebbrt-sysroot $(BAREMETAL_RELEASE_DIR) + $(CD) $(BAREMETAL_RELEASE_DIR) && \ + EBBRT_SYSROOT=$(EBBRTSYSROOT) $(CMAKE) -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH=$(BAREMETAL_PREFIX_PATH) \ + -DCMAKE_TOOLCHAIN_FILE=$(CMAKE_TOOLCHAIN_FILE) $(MYDIR) && \ + $(MAKE) + +hosted-debug: | $(HOSTED_DEBUG_DIR) + $(CD) $(HOSTED_DEBUG_DIR) && $(CMAKE) -DCMAKE_BUILD_TYPE=Debug \ + $(MYDIR) && $(MAKE) + +hosted-release: | $(HOSTED_RELEASE_DIR) + $(CD) $(HOSTED_RELEASE_DIR) && $(CMAKE) -DCMAKE_BUILD_TYPE=Release \ + $(MYDIR) && $(MAKE) + +clean: + $(MAKE) clean -C $(HOSTED_DEBUG_DIR) && \ + $(MAKE) clean -C $(HOSTED_RELEASE_DIR) && \ + $(MAKE) clean -C $(BAREMETAL_DEBUG_DIR) && \ + $(MAKE) clean -C $(BAREMETAL_RELEASE_DIR) + +.PHONY: Debug Release all clean baremetal baremetal-debug baremetal-release hosted hosted-debug hosted-release diff --git a/libs/dsm/cmake/FindCapnp.cmake b/libs/dsm/cmake/FindCapnp.cmake new file mode 100644 index 00000000..b9bafb4b --- /dev/null +++ b/libs/dsm/cmake/FindCapnp.cmake @@ -0,0 +1,186 @@ +# +# Finds the Cap'n Proto libraries, and compiles schema files. +# +# Configuration variables (optional): +# CAPNPC_OUTPUT_DIR +# Directory to place compiled schema sources (default: the same directory as the schema file). +# CAPNPC_IMPORT_DIRS +# List of additional include directories for the schema compiler. +# (CMAKE_CURRENT_SOURCE_DIR and CAPNP_INCLUDE_DIRS are always included.) +# CAPNPC_SRC_PREFIX +# Schema file source prefix (default: CMAKE_CURRENT_SOURCE_DIR). +# CAPNPC_FLAGS +# Additional flags to pass to the schema compiler. +# +# Variables that are discovered: +# CAPNP_EXECUTABLE +# Path to the `capnp` tool (can be set to override). +# CAPNPC_CXX_EXECUTABLE +# Path to the `capnpc-c++` tool (can be set to override). +# CAPNP_INCLUDE_DIRS +# Include directories for the library's headers (can be set to override). +# CANP_LIBRARIES +# The Cap'n Proto library paths. +# CAPNP_LIBRARIES_LITE +# Paths to only the 'lite' libraries. +# CAPNP_DEFINITIONS +# Compiler definitions required for building with the library. +# CAPNP_FOUND +# Set if the libraries have been located. +# +# Example usage: +# +# find_package(CapnProto REQUIRED) +# include_directories(${CAPNP_INCLUDE_DIRS}) +# add_definitions(${CAPNP_DEFINITIONS}) +# +# capnp_generate_cpp(CAPNP_SRCS CAPNP_HDRS schema.capnp) +# add_executable(a a.cc ${CAPNP_SRCS} ${CAPNP_HDRS}) +# target_link_library(a ${CAPNP_LIBRARIES}) +# +# For out-of-source builds: +# +# set(CAPNPC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}) +# include_directories(${CAPNPC_OUTPUT_DIR}) +# capnp_generate_cpp(...) +# + +# CAPNP_GENERATE_CPP =========================================================== + +function(CAPNP_GENERATE_CPP SOURCES HEADERS) + if(NOT ARGN) + message(SEND_ERROR "CAPNP_GENERATE_CPP() called without any source files.") + endif() + if(NOT CAPNP_EXECUTABLE) + message(SEND_ERROR "Could not locate capnp executable (CAPNP_EXECUTABLE).") + endif() + if(NOT CAPNPC_CXX_EXECUTABLE) + message(SEND_ERROR "Could not locate capnpc-c++ executable (CAPNPC_CXX_EXECUTABLE).") + endif() + if(NOT CAPNP_INCLUDE_DIRS) + message(SEND_ERROR "Could not locate capnp header files (CAPNP_INCLUDE_DIRS).") + endif() + + # Default compiler includes + set(include_path -I ${CMAKE_CURRENT_SOURCE_DIR} -I ${CAPNP_INCLUDE_DIRS}) + + if(DEFINED CAPNPC_IMPORT_DIRS) + # Append each directory as a series of '-I' flags in ${include_path} + foreach(directory ${CAPNPC_IMPORT_DIRS}) + get_filename_component(absolute_path "${directory}" ABSOLUTE) + list(APPEND include_path -I ${absolute_path}) + endforeach() + endif() + + if(DEFINED CAPNPC_OUTPUT_DIR) + # Prepend a ':' to get the format for the '-o' flag right + set(output_dir ":${CAPNPC_OUTPUT_DIR}") + else() + set(output_dir ":.") + endif() + + if(NOT DEFINED CAPNPC_SRC_PREFIX) + set(CAPNPC_SRC_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}") + endif() + get_filename_component(CAPNPC_SRC_PREFIX "${CAPNPC_SRC_PREFIX}" ABSOLUTE) + + set(${SOURCES}) + set(${HEADERS}) + foreach(schema_file ${ARGN}) + get_filename_component(file_path "${schema_file}" ABSOLUTE) + get_filename_component(file_dir "${file_path}" PATH) + + # Figure out where the output files will go + if (NOT DEFINED CAPNPC_OUTPUT_DIR) + set(output_base "${file_path}") + else() + # Output files are placed in CAPNPC_OUTPUT_DIR, at a location as if they were + # relative to CAPNPC_SRC_PREFIX. + string(LENGTH "${CAPNPC_SRC_PREFIX}" prefix_len) + string(SUBSTRING "${file_path}" 0 ${prefix_len} output_prefix) + if(NOT "${CAPNPC_SRC_PREFIX}" STREQUAL "${output_prefix}") + message(SEND_ERROR "Could not determine output path for '${schema_file}' ('${file_path}') with source prefix '${CAPNPC_SRC_PREFIX}' into '${CAPNPC_OUTPUT_DIR}'.") + endif() + + string(SUBSTRING "${file_path}" ${prefix_len} -1 output_path) + set(output_base "${CAPNPC_OUTPUT_DIR}${output_path}") + endif() + + add_custom_command( + OUTPUT "${output_base}.c++" "${output_base}.h" + COMMAND "${CAPNP_EXECUTABLE}" + ARGS compile + -o ${CAPNPC_CXX_EXECUTABLE}${output_dir} + --src-prefix ${CAPNPC_SRC_PREFIX} + ${include_path} + ${CAPNPC_FLAGS} + ${file_path} + DEPENDS "${schema_file}" + COMMENT "Compiling Cap'n Proto schema ${schema_file}" + VERBATIM + ) + list(APPEND ${SOURCES} "${output_base}.c++") + list(APPEND ${HEADERS} "${output_base}.h") + endforeach() + + set_source_files_properties(${${SOURCES}} ${${HEADERS}} PROPERTIES GENERATED TRUE) + set(${SOURCES} ${${SOURCES}} PARENT_SCOPE) + set(${HEADERS} ${${HEADERS}} PARENT_SCOPE) +endfunction() + +# Find Libraries/Paths ========================================================= + +find_library(CAPNP_LIB_KJ kj +) +find_library(CAPNP_LIB_KJ-ASYNC kj-async +) +find_library(CAPNP_LIB_CAPNP capnp +) +find_library(CAPNP_LIB_CAPNP-RPC capnp-rpc +) +find_library(CAPNP_LIB_CAPNP-JSON capnp-json +) +mark_as_advanced(CAPNP_LIB_KJ CAPNP_LIB_KJ-ASYNC CAPNP_LIB_CAPNP CAPNP_LIB_CAPNP-RPC CAPNP_LIB_CAPNP-JSON) +set(CAPNP_LIBRARIES_LITE + ${CAPNP_LIB_CAPNP} + ${CAPNP_LIB_KJ} +) +set(CAPNP_LIBRARIES + ${CAPNP_LIB_CAPNP-JSON} + ${CAPNP_LIB_CAPNP-RPC} + ${CAPNP_LIB_CAPNP} + ${CAPNP_LIB_KJ-ASYNC} + ${CAPNP_LIB_KJ} +) + +# Was only the 'lite' library found? +if(CAPNP_LIB_CAPNP AND NOT CAPNP_LIB_CAPNP-RPC) + set(CAPNP_DEFINITIONS -DCAPNP_LITE) +else() + set(CAPNP_DEFINITIONS) +endif() + +find_path(CAPNP_INCLUDE_DIRS capnp/generated-header-support.h + HINTS "${PKGCONFIG_CAPNP_INCLUDEDIR}" ${PKGCONFIG_CAPNP_INCLUDE_DIRS} +) + +find_program(CAPNP_EXECUTABLE + NAMES capnp + DOC "Cap'n Proto Command-line Tool" + HINTS "${PKGCONFIG_CAPNP_PREFIX}/bin" +) + +find_program(CAPNPC_CXX_EXECUTABLE + NAMES capnpc-c++ + DOC "Capn'n Proto C++ Compiler" + HINTS "${PKGCONFIG_CAPNP_PREFIX}/bin" +) + +# Only *require* the include directory, libkj, and libcapnp. If compiling with +# CAPNP_LITE, nothing else will be found. +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(CAPNP DEFAULT_MSG + CAPNP_INCLUDE_DIRS + CAPNP_LIB_KJ + CAPNP_LIB_CAPNP +) diff --git a/libs/dsm/cmake/FindTBB.cmake b/libs/dsm/cmake/FindTBB.cmake new file mode 100644 index 00000000..aa72dfad --- /dev/null +++ b/libs/dsm/cmake/FindTBB.cmake @@ -0,0 +1,283 @@ +# Locate Intel Threading Building Blocks include paths and libraries +# FindTBB.cmake can be found at https://code.google.com/p/findtbb/ +# Written by Hannes Hofmann +# Improvements by Gino van den Bergen , +# Florian Uhlig , +# Jiri Marsik + +# The MIT License +# +# Copyright (c) 2011 Hannes Hofmann +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +# GvdB: This module uses the environment variable TBB_ARCH_PLATFORM which defines architecture and compiler. +# e.g. "ia32/vc8" or "em64t/cc4.1.0_libc2.4_kernel2.6.16.21" +# TBB_ARCH_PLATFORM is set by the build script tbbvars[.bat|.sh|.csh], which can be found +# in the TBB installation directory (TBB_INSTALL_DIR). +# +# GvdB: Mac OS X distribution places libraries directly in lib directory. +# +# For backwards compatibility, you may explicitely set the CMake variables TBB_ARCHITECTURE and TBB_COMPILER. +# TBB_ARCHITECTURE [ ia32 | em64t | itanium ] +# which architecture to use +# TBB_COMPILER e.g. vc9 or cc3.2.3_libc2.3.2_kernel2.4.21 or cc4.0.1_os10.4.9 +# which compiler to use (detected automatically on Windows) + +# This module respects +# TBB_INSTALL_DIR or $ENV{TBB21_INSTALL_DIR} or $ENV{TBB_INSTALL_DIR} + +# This module defines +# TBB_INCLUDE_DIRS, where to find task_scheduler_init.h, etc. +# TBB_LIBRARY_DIRS, where to find libtbb, libtbbmalloc +# TBB_DEBUG_LIBRARY_DIRS, where to find libtbb_debug, libtbbmalloc_debug +# TBB_INSTALL_DIR, the base TBB install directory +# TBB_LIBRARIES, the libraries to link against to use TBB. +# TBB_DEBUG_LIBRARIES, the libraries to link against to use TBB with debug symbols. +# TBB_FOUND, If false, don't try to use TBB. +# TBB_INTERFACE_VERSION, as defined in tbb/tbb_stddef.h + + +if (WIN32) +# has em64t/vc8 em64t/vc9 +# has ia32/vc7.1 ia32/vc8 ia32/vc9 +set(_TBB_DEFAULT_INSTALL_DIR "C:/Program Files/Intel/TBB" "C:/Program Files (x86)/Intel/TBB") +set(_TBB_LIB_NAME "tbb") +set(_TBB_LIB_MALLOC_NAME "${_TBB_LIB_NAME}malloc") +set(_TBB_LIB_DEBUG_NAME "${_TBB_LIB_NAME}_debug") +set(_TBB_LIB_MALLOC_DEBUG_NAME "${_TBB_LIB_MALLOC_NAME}_debug") +if (MSVC71) +set (_TBB_COMPILER "vc7.1") +endif(MSVC71) +if (MSVC80) +set(_TBB_COMPILER "vc8") +endif(MSVC80) +if (MSVC90) +set(_TBB_COMPILER "vc9") +endif(MSVC90) +if(MSVC10) +set(_TBB_COMPILER "vc10") +endif(MSVC10) +# Todo: add other Windows compilers such as ICL. +set(_TBB_ARCHITECTURE ${TBB_ARCHITECTURE}) +endif (WIN32) + +if (UNIX) +if (APPLE) +# MAC +set(_TBB_DEFAULT_INSTALL_DIR "/Library/Frameworks/Intel_TBB.framework/Versions") +# libs: libtbb.dylib, libtbbmalloc.dylib, *_debug +set(_TBB_LIB_NAME "tbb") +set(_TBB_LIB_MALLOC_NAME "${_TBB_LIB_NAME}malloc") +set(_TBB_LIB_DEBUG_NAME "${_TBB_LIB_NAME}_debug") +set(_TBB_LIB_MALLOC_DEBUG_NAME "${_TBB_LIB_MALLOC_NAME}_debug") +# default flavor on apple: ia32/cc4.0.1_os10.4.9 +# Jiri: There is no reason to presume there is only one flavor and +# that user's setting of variables should be ignored. +if(NOT TBB_COMPILER) +set(_TBB_COMPILER "cc4.0.1_os10.4.9") +elseif (NOT TBB_COMPILER) +set(_TBB_COMPILER ${TBB_COMPILER}) +endif(NOT TBB_COMPILER) +if(NOT TBB_ARCHITECTURE) +set(_TBB_ARCHITECTURE "ia32") +elseif(NOT TBB_ARCHITECTURE) +set(_TBB_ARCHITECTURE ${TBB_ARCHITECTURE}) +endif(NOT TBB_ARCHITECTURE) +else (APPLE) +# LINUX +set(_TBB_DEFAULT_INSTALL_DIR "/opt/intel/tbb" "/usr/local/include" "/usr/include") +set(_TBB_LIB_NAME "tbb") +set(_TBB_LIB_MALLOC_NAME "${_TBB_LIB_NAME}malloc") +set(_TBB_LIB_DEBUG_NAME "${_TBB_LIB_NAME}_debug") +set(_TBB_LIB_MALLOC_DEBUG_NAME "${_TBB_LIB_MALLOC_NAME}_debug") +# has em64t/cc3.2.3_libc2.3.2_kernel2.4.21 em64t/cc3.3.3_libc2.3.3_kernel2.6.5 em64t/cc3.4.3_libc2.3.4_kernel2.6.9 em64t/cc4.1.0_libc2.4_kernel2.6.16.21 +# has ia32/* +# has itanium/* +set(_TBB_COMPILER ${TBB_COMPILER}) +set(_TBB_ARCHITECTURE ${TBB_ARCHITECTURE}) +endif (APPLE) +endif (UNIX) + +if (CMAKE_SYSTEM MATCHES "SunOS.*") +# SUN +# not yet supported +# has em64t/cc3.4.3_kernel5.10 +# has ia32/* +endif (CMAKE_SYSTEM MATCHES "SunOS.*") + + +#-- Clear the public variables +set (TBB_FOUND "NO") + + +#-- Find TBB install dir and set ${_TBB_INSTALL_DIR} and cached ${TBB_INSTALL_DIR} +# first: use CMake variable TBB_INSTALL_DIR +if (TBB_INSTALL_DIR) +set (_TBB_INSTALL_DIR ${TBB_INSTALL_DIR}) +endif (TBB_INSTALL_DIR) +# second: use environment variable +if (NOT _TBB_INSTALL_DIR) +if (NOT "$ENV{TBB_INSTALL_DIR}" STREQUAL "") +set (_TBB_INSTALL_DIR $ENV{TBB_INSTALL_DIR}) +endif (NOT "$ENV{TBB_INSTALL_DIR}" STREQUAL "") +# Intel recommends setting TBB21_INSTALL_DIR +if (NOT "$ENV{TBB21_INSTALL_DIR}" STREQUAL "") +set (_TBB_INSTALL_DIR $ENV{TBB21_INSTALL_DIR}) +endif (NOT "$ENV{TBB21_INSTALL_DIR}" STREQUAL "") +if (NOT "$ENV{TBB22_INSTALL_DIR}" STREQUAL "") +set (_TBB_INSTALL_DIR $ENV{TBB22_INSTALL_DIR}) +endif (NOT "$ENV{TBB22_INSTALL_DIR}" STREQUAL "") +if (NOT "$ENV{TBB30_INSTALL_DIR}" STREQUAL "") +set (_TBB_INSTALL_DIR $ENV{TBB30_INSTALL_DIR}) +endif (NOT "$ENV{TBB30_INSTALL_DIR}" STREQUAL "") +endif (NOT _TBB_INSTALL_DIR) +# third: try to find path automatically +if (NOT _TBB_INSTALL_DIR) +if (_TBB_DEFAULT_INSTALL_DIR) +set (_TBB_INSTALL_DIR ${_TBB_DEFAULT_INSTALL_DIR}) +endif (_TBB_DEFAULT_INSTALL_DIR) +endif (NOT _TBB_INSTALL_DIR) +# sanity check +if (NOT _TBB_INSTALL_DIR) +message ("ERROR: Unable to find Intel TBB install directory. ${_TBB_INSTALL_DIR}") +else (NOT _TBB_INSTALL_DIR) +# finally: set the cached CMake variable TBB_INSTALL_DIR +if (NOT TBB_INSTALL_DIR) +set (TBB_INSTALL_DIR ${_TBB_INSTALL_DIR} CACHE PATH "Intel TBB install directory") +mark_as_advanced(TBB_INSTALL_DIR) +endif (NOT TBB_INSTALL_DIR) + + +#-- A macro to rewrite the paths of the library. This is necessary, because +# find_library() always found the em64t/vc9 version of the TBB libs +macro(TBB_CORRECT_LIB_DIR var_name) +# if (NOT "${_TBB_ARCHITECTURE}" STREQUAL "em64t") +string(REPLACE em64t "${_TBB_ARCHITECTURE}" ${var_name} ${${var_name}}) +# endif (NOT "${_TBB_ARCHITECTURE}" STREQUAL "em64t") +string(REPLACE ia32 "${_TBB_ARCHITECTURE}" ${var_name} ${${var_name}}) +string(REPLACE vc7.1 "${_TBB_COMPILER}" ${var_name} ${${var_name}}) +string(REPLACE vc8 "${_TBB_COMPILER}" ${var_name} ${${var_name}}) +string(REPLACE vc9 "${_TBB_COMPILER}" ${var_name} ${${var_name}}) +string(REPLACE vc10 "${_TBB_COMPILER}" ${var_name} ${${var_name}}) +endmacro(TBB_CORRECT_LIB_DIR var_content) + + +#-- Look for include directory and set ${TBB_INCLUDE_DIR} +set (TBB_INC_SEARCH_DIR ${_TBB_INSTALL_DIR}/include) +# Jiri: tbbvars now sets the CPATH environment variable to the directory +# containing the headers. +find_path(TBB_INCLUDE_DIR +tbb/task_scheduler_init.h +PATHS ${TBB_INC_SEARCH_DIR} ENV CPATH +) +mark_as_advanced(TBB_INCLUDE_DIR) + + +#-- Look for libraries +# GvdB: $ENV{TBB_ARCH_PLATFORM} is set by the build script tbbvars[.bat|.sh|.csh] +if (NOT $ENV{TBB_ARCH_PLATFORM} STREQUAL "") +set (_TBB_LIBRARY_DIR +${_TBB_INSTALL_DIR}/lib/$ENV{TBB_ARCH_PLATFORM} +${_TBB_INSTALL_DIR}/$ENV{TBB_ARCH_PLATFORM}/lib +) +endif (NOT $ENV{TBB_ARCH_PLATFORM} STREQUAL "") +# Jiri: This block isn't mutually exclusive with the previous one +# (hence no else), instead I test if the user really specified +# the variables in question. +if ((NOT ${TBB_ARCHITECTURE} STREQUAL "") AND (NOT ${TBB_COMPILER} STREQUAL "")) +# HH: deprecated +message(STATUS "[Warning] FindTBB.cmake: The use of TBB_ARCHITECTURE and TBB_COMPILER is deprecated and may not be supported in future versions. Please set \$ENV{TBB_ARCH_PLATFORM} (using tbbvars.[bat|csh|sh]).") +# Jiri: It doesn't hurt to look in more places, so I store the hints from +# ENV{TBB_ARCH_PLATFORM} and the TBB_ARCHITECTURE and TBB_COMPILER +# variables and search them both. +set (_TBB_LIBRARY_DIR "${_TBB_INSTALL_DIR}/${_TBB_ARCHITECTURE}/${_TBB_COMPILER}/lib" ${_TBB_LIBRARY_DIR}) +endif ((NOT ${TBB_ARCHITECTURE} STREQUAL "") AND (NOT ${TBB_COMPILER} STREQUAL "")) + +# GvdB: Mac OS X distribution places libraries directly in lib directory. +list(APPEND _TBB_LIBRARY_DIR ${_TBB_INSTALL_DIR}/lib) + +# Jiri: No reason not to check the default paths. From recent versions, +# tbbvars has started exporting the LIBRARY_PATH and LD_LIBRARY_PATH +# variables, which now point to the directories of the lib files. +# It all makes more sense to use the ${_TBB_LIBRARY_DIR} as a HINTS +# argument instead of the implicit PATHS as it isn't hard-coded +# but computed by system introspection. Searching the LIBRARY_PATH +# and LD_LIBRARY_PATH environment variables is now even more important +# that tbbvars doesn't export TBB_ARCH_PLATFORM and it facilitates +# the use of TBB built from sources. +find_library(TBB_LIBRARY ${_TBB_LIB_NAME} HINTS ${_TBB_LIBRARY_DIR} +PATHS ENV LIBRARY_PATH ENV LD_LIBRARY_PATH) +find_library(TBB_MALLOC_LIBRARY ${_TBB_LIB_MALLOC_NAME} HINTS ${_TBB_LIBRARY_DIR} +PATHS ENV LIBRARY_PATH ENV LD_LIBRARY_PATH) + +#Extract path from TBB_LIBRARY name +get_filename_component(TBB_LIBRARY_DIR ${TBB_LIBRARY} PATH) + +#TBB_CORRECT_LIB_DIR(TBB_LIBRARY) +#TBB_CORRECT_LIB_DIR(TBB_MALLOC_LIBRARY) +mark_as_advanced(TBB_LIBRARY TBB_MALLOC_LIBRARY) + +#-- Look for debug libraries +# Jiri: Changed the same way as for the release libraries. +find_library(TBB_LIBRARY_DEBUG ${_TBB_LIB_DEBUG_NAME} HINTS ${_TBB_LIBRARY_DIR} +PATHS ENV LIBRARY_PATH ENV LD_LIBRARY_PATH) +find_library(TBB_MALLOC_LIBRARY_DEBUG ${_TBB_LIB_MALLOC_DEBUG_NAME} HINTS ${_TBB_LIBRARY_DIR} +PATHS ENV LIBRARY_PATH ENV LD_LIBRARY_PATH) + +# Jiri: Self-built TBB stores the debug libraries in a separate directory. +# Extract path from TBB_LIBRARY_DEBUG name +get_filename_component(TBB_LIBRARY_DEBUG_DIR ${TBB_LIBRARY_DEBUG} PATH) + +#TBB_CORRECT_LIB_DIR(TBB_LIBRARY_DEBUG) +#TBB_CORRECT_LIB_DIR(TBB_MALLOC_LIBRARY_DEBUG) +mark_as_advanced(TBB_LIBRARY_DEBUG TBB_MALLOC_LIBRARY_DEBUG) + + +if (TBB_INCLUDE_DIR) +if (TBB_LIBRARY) +set (TBB_FOUND "YES") +set (TBB_LIBRARIES ${TBB_LIBRARY} ${TBB_MALLOC_LIBRARY} ${TBB_LIBRARIES}) +set (TBB_DEBUG_LIBRARIES ${TBB_LIBRARY_DEBUG} ${TBB_MALLOC_LIBRARY_DEBUG} ${TBB_DEBUG_LIBRARIES}) +set (TBB_INCLUDE_DIRS ${TBB_INCLUDE_DIR} CACHE PATH "TBB include directory" FORCE) +set (TBB_LIBRARY_DIRS ${TBB_LIBRARY_DIR} CACHE PATH "TBB library directory" FORCE) +# Jiri: Self-built TBB stores the debug libraries in a separate directory. +set (TBB_DEBUG_LIBRARY_DIRS ${TBB_LIBRARY_DEBUG_DIR} CACHE PATH "TBB debug library directory" FORCE) +mark_as_advanced(TBB_INCLUDE_DIRS TBB_LIBRARY_DIRS TBB_DEBUG_LIBRARY_DIRS TBB_LIBRARIES TBB_DEBUG_LIBRARIES) +message(STATUS "Found Intel TBB") +endif (TBB_LIBRARY) +endif (TBB_INCLUDE_DIR) + +if (NOT TBB_FOUND) +message("ERROR: Intel TBB NOT found!") +message(STATUS "Looked for Threading Building Blocks in ${_TBB_INSTALL_DIR}") +# do only throw fatal, if this pkg is REQUIRED +if (TBB_FIND_REQUIRED) +message(FATAL_ERROR "Could NOT find TBB library.") +endif (TBB_FIND_REQUIRED) +endif (NOT TBB_FOUND) + +endif (NOT _TBB_INSTALL_DIR) + +if (TBB_FOUND) +set(TBB_INTERFACE_VERSION 0) +FILE(READ "${TBB_INCLUDE_DIRS}/tbb/tbb_stddef.h" _TBB_VERSION_CONTENTS) +STRING(REGEX REPLACE ".*#define TBB_INTERFACE_VERSION ([0-9]+).*" "\\1" TBB_INTERFACE_VERSION "${_TBB_VERSION_CONTENTS}") +set(TBB_INTERFACE_VERSION "${TBB_INTERFACE_VERSION}") +endif (TBB_FOUND) diff --git a/libs/dsm/src/Messenger.cc b/libs/dsm/src/Messenger.cc new file mode 100644 index 00000000..8d3b25b1 --- /dev/null +++ b/libs/dsm/src/Messenger.cc @@ -0,0 +1,251 @@ +// Copyright Boston University SESA Group 2013 - 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +#include "Messenger.h" +#include "Cpu.h" +#include "../Message.h" +#include "../SharedIOBufRef.h" +#include "../UniqueIOBuf.h" +#include "Debug.h" + +uint16_t ebbrt::Messenger::port_; + +ebbrt::Messenger::Messenger() {} + +ebbrt::Messenger::Connection::Connection(ebbrt::NetworkManager::TcpPcb pcb) + : TcpHandler(std::move(pcb)) {} + +void ebbrt::Messenger::Connection::check_preallocate() { + // preallocate buffer if payload occupancy ratio drops below threshold + size_t capacity = 0; + for (auto& buf : *buf_) { + capacity += buf.Capacity(); + } + auto buffer_len = buf_->ComputeChainDataLength(); + auto dp = buf_->GetDataPointer(); + auto& header = dp.Get
(); + auto message_len = sizeof(Header) + header.length; + auto ratio = static_cast(buffer_len) / static_cast(capacity); + if (ratio < kOccupancyRatio) { + // allocate message buffer and coalesce chain + auto newbuf = MakeUniqueIOBuf(message_len, false); + auto dp = newbuf->GetMutDataPointer(); + for (auto& buf : *buf_) { + auto len = buf.Length(); + std::memcpy(static_cast(dp.Data()), buf.Data(), len); + dp.Advance(len); + preallocate_ += len; + } + assert(newbuf->CountChainElements() == 1); + assert(newbuf->ComputeChainDataLength() == message_len); + assert(preallocate_ == buffer_len); + buf_ = std::move(newbuf); + } +} + +void ebbrt::Messenger::Connection::preallocated(std::unique_ptr b) { + auto len = b->Length(); + auto ptr = buf_->MutData(); + ptr += preallocate_; + std::memcpy(reinterpret_cast(ptr), b->Data(), len); + + auto dp = buf_->GetMutDataPointer(); + preallocate_ += len; + auto& header = dp.Get
(); + auto message_len = sizeof(Header) + header.length; + + if (preallocate_ == message_len) { + kassert(buf_->Length() == message_len); + preallocate_ = 0; + process_message(std::move(buf_)); + } + return; +} + +void ebbrt::Messenger::Connection::process_message( + std::unique_ptr b) { + auto dp = b->GetDataPointer(); + // TODO(dschatz): get rid of datapointer + auto& header = dp.Get
(); + b->AdvanceChain(sizeof(Header)); + auto& ref = GetMessagableRef(header.id, header.type_code); + ref.ReceiveMessageInternal(NetworkId(Pcb().GetRemoteAddress()), std::move(b)); + return; +} +std::unique_ptr +ebbrt::Messenger::Connection::process_message_chain( + std::unique_ptr b) { + + auto dp = b->GetDataPointer(); + auto& header = dp.Get
(); + auto message_len = sizeof(Header) + header.length; + + // Our buffer contains the data of multiple messages. We need to + // split the buffer at the first messsage boundary and preserve + // the rest. + auto orig_len = b->ComputeChainDataLength(); + std::unique_ptr tail_chain; + std::unique_ptr split; + uint32_t length = 0; + // check if message is contained within first buffer + if (b->Length() >= message_len) { + length = b->Length(); + split = std::move(b); + tail_chain = split->Pop(); + b = nullptr; + } else { + for (auto& buf : *b) { + length += buf.Length(); + if (length >= message_len) { + kassert(b->IsChained()); + auto tmp = static_cast(b->UnlinkEnd(buf).release()); + split = std::unique_ptr(tmp); + tail_chain = split->Pop(); + break; + } + } + } + // we "divide" the buffer by clone and resize + auto left_shard_len = split->Length() - (length - message_len); + auto right_shard_len = split->Length() - left_shard_len; + auto split_c = IOBuf::Create(SharedIOBufRef::CloneView, + std::move(split)); + auto remainder = + IOBuf::Create(SharedIOBufRef::CloneView, *split_c); + split_c->TrimEnd(right_shard_len); + remainder->Advance(left_shard_len); + + // combined data + if (!b) { + b = std::move(split_c); + } else { + b->PrependChain(std::move(split_c)); + } + if (tail_chain) { + remainder->PrependChain(std::move(tail_chain)); + } + kassert(orig_len == + (b->ComputeChainDataLength() + remainder->ComputeChainDataLength())); + + process_message(std::move(b)); + return std::move(remainder); +} + +void ebbrt::Messenger::Connection::Receive(std::unique_ptr b) { + kassert(b->Length() != 0); + + // check if we've preallocated a message buffer + if (preallocate_) { + preallocated(std::move(b)); + return; + } + // otherwise, process buffer chain + if (buf_) { + buf_->PrependChain(std::move(b)); + } else { + buf_ = std::move(b); + } + + while (buf_) { + auto buffer_len = buf_->ComputeChainDataLength(); + if (buffer_len < sizeof(Header)) { + return; + } + auto dp = buf_->GetDataPointer(); + auto& header = dp.Get
(); + auto message_len = sizeof(Header) + header.length; + if (likely(buffer_len == message_len)) { + process_message(std::move(buf_)); + return; + } else if (buffer_len < message_len) { + // check if we need to preallocate + // only do this check at certain chain lengths + if (buf_->CountChainElements() % kPreallocateChainLen == 0) { + check_preallocate(); + } + return; + } else if (buffer_len > message_len) { + // process message from chain and return remaining data + buf_ = process_message_chain(std::move(buf_)); + } + } + return; +} + +void ebbrt::Messenger::Connection::Connected() { promise_.SetValue(this); } +// These need to remove themselves from the hash table +void ebbrt::Messenger::Connection::Close() { + ebbrt::kabort("UNIMPLEMENTED: Messenger Close\n"); +} +void ebbrt::Messenger::Connection::Abort() { + ebbrt::kabort("UNIMPLEMENTED: Messenger Abort\n"); +} + +ebbrt::Future +ebbrt::Messenger::Connection::GetFuture() { + return promise_.GetFuture(); +} + +void ebbrt::Messenger::StartListening(uint16_t port) { + port_ = port; + ebbrt::kprintf("STARTLISTENING GET CALLED ON PORT %d\n", port); + listening_pcb_.Bind(port, [this](NetworkManager::TcpPcb pcb) { + auto addr = pcb.GetRemoteAddress(); + std::lock_guard lock(lock_); + if (connection_map_.find(addr) != connection_map_.end()) + throw std::runtime_error("Store to promise"); + + //auto index = cpu_index.fetch_add(1) % ebbrt::Cpu::Count(); + pcb.BindCpu((size_t)0); + auto connection = new Connection(std::move(pcb)); + connection->Install(); + connection_map_.emplace(addr, + MakeReadyFuture(connection).Share()); + }); +} + +ebbrt::Messenger::NetworkId ebbrt::Messenger::LocalNetworkId() { + // ebbrt::NetworkManager::Interface iface = + // ebbrt::network_manager->FirstInterface(); + // return NetworkId(iface.IPV4Addr()); + kabort("UNIMPLEMENTED\n"); +} + +ebbrt::Future ebbrt::Messenger::Send(NetworkId to, EbbId id, + uint64_t type_code, + std::unique_ptr&& data) { + // make sure we have a pending connection + { + std::lock_guard lock(lock_); + auto it = connection_map_.find(to.ip); + if (it == connection_map_.end()) { + // we don't have a pending connection, start one + ebbrt::kprintf("Did not see a pending connection to %s, start one!\n", to.ToString().c_str()); + NetworkManager::TcpPcb pcb; + pcb.Connect(to.ip, port_); + auto connection = new Connection(std::move(pcb)); + connection->Install(); + ebbrt::kprintf("NEW CONNECTION ESTABLISHED\n"); + auto f = connection->GetFuture(); + connection_map_.emplace(to.ip, connection->GetFuture().Share()); + } + } + + // construct header + auto buf = MakeUniqueIOBuf(sizeof(Header)); + auto dp = buf->GetMutDataPointer(); + auto& h = dp.Get
(); + h.length = data->ComputeChainDataLength(); + h.type_code = type_code; + h.id = id; + // Cast to non const is ok because we then take the whole chain as const + buf->PrependChain(std::move(data)); + + return connection_map_[to.ip].Then([data = std::move(buf)]( + SharedFuture f) mutable { + ebbrt::kprintf("FUTRUE GET CALLED?\n"); + f.Get()->Send(std::move(data)); + f.Get()->Pcb().Output(); + }); +} diff --git a/libs/dsm/src/MulticoreEbb.h b/libs/dsm/src/MulticoreEbb.h new file mode 100644 index 00000000..aa6b82af --- /dev/null +++ b/libs/dsm/src/MulticoreEbb.h @@ -0,0 +1,153 @@ +// Copyright Boston University SESA Group 2013 - 2014. +// + +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +#ifndef MULTICOREEBB_H_ +#define MULTICOREEBB_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* + * MulticoreEbb class template defines a multicore Ebb class (i.e, one rep per + * core/context) with an optional shared root object. The (2) templates + * arguments + * specify the base rep and root class types. + * + * class MyEbb : MulticoreEbb + * + * A rep of the multicore Ebb contains a list of the other + * reps local to the node, as well as a pointer to shared root object. + * + * A Create() function is used to construct the initial ref and format the + * local_id_map entry. + */ +namespace ebbrt { +namespace detail { +template using RepMap = boost::container::flat_map; +} +using detail::RepMap; + +/* forward declarations of class templates */ +template class MulticoreEbbRoot; +template class MulticoreEbb; + +/* Ebb Root class templace with R-type Rep Map */ +template class MulticoreEbbRoot { +public: + MulticoreEbbRoot(EbbId id) : id_(id){}; + +protected: + static void HandleFault(EbbId id); + // Derived class can re-implement protected methods, which hides + // (doesn't conflict with) these base class implementations. + R *create_rep() { + auto rep = new R(id_); + rep->SetRoot(static_cast(this)); + return rep; + }; + R *create_initial_rep() { return create_rep(); }; + R *cache_rep_(size_t core); + SpinLock lock_; + RepMap reps_; + EbbId id_; + +private: + friend class MulticoreEbb; +}; + +/* Multicore Ebb class template with typed Root */ +template class MulticoreEbb { +// static_assert(std::is_base_of, R>::value, +// "Root type must inherit from MulticoreEbbRoot"); + +public: + MulticoreEbb() = delete; // Must ber constructed w/ root pointer + MulticoreEbb(ebbrt::EbbId id) : id_(id) {}; + void SetRoot(R * root) { root_ = root; }; + static T &HandleFault(EbbId id); + +protected: + R *root_; + EbbId id_; +private: + // By making the base root type a friend allows the rep constructor to + // initialize protected members + friend class MulticoreEbbRoot; +}; + +template R *MulticoreEbbRoot::cache_rep_(size_t core) { + auto it = reps_.find(core); + if (it != reps_.end()) { + // rep was already cached for this core, return it + return it->second; + } else { + // construct a new rep and cache the address + R *rep; + if (reps_.size() == 0) { + // construct initial rep on the node + rep = create_initial_rep(); + } else { + rep = create_rep(); + } + // cached rep + { + std::lock_guard guard(lock_); + reps_[core] = rep; + } + return rep; + } +} + +template +void MulticoreEbbRoot::HandleFault(EbbId id) { + // Default behavior is to construct and cache a local root object + LocalIdMap::Accessor accessor; + auto created = local_id_map->Insert(accessor, id); + // In this case of a race, the thread that successfully wrote to + // the local_id_map will create a new root object + if (created) { + ebbrt::kprintf("New root #%d\n", id); + accessor->second = new T(id); + } + +} + +template T &MulticoreEbb::HandleFault(EbbId id) { +retry : { + + // Check for root in LocalIdMap (read-lock) + LocalIdMap::ConstAccessor accessor; + auto found = local_id_map->Find(accessor, id); + if (found) { + // Ask root for representative (remain locked) + // Let root check for exiting rep and constructs one if needed + auto root = boost::any_cast(accessor->second); + T *rep = root->cache_rep_((size_t)Cpu::GetMine()); + EbbRef::CacheRef(id, *rep); + return *rep; + } +} + // No root was found on this node. Trigger global miss for this root type + MulticoreEbbRoot::HandleFault(id); + // retry, expecting we'll find the root on the next pass + goto retry; +} + +} // namespace ebbrt + +#endif // COMMON_SRC_INCLUDE_EBBRT_MULTICOREEBB_H_ diff --git a/libs/dsm/src/RemoteMem.cc b/libs/dsm/src/RemoteMem.cc new file mode 100644 index 00000000..587f1688 --- /dev/null +++ b/libs/dsm/src/RemoteMem.cc @@ -0,0 +1,203 @@ +#include +#include +#include +#include "RemoteMem.h" + +RemoteMemory & +RemoteMemory::HandleFault(ebbrt::EbbId id){ + { + // Trying to find if registered in the local Id map + ebbrt::LocalIdMap::ConstAccessor accessor; + auto found = ebbrt::local_id_map->Find(accessor, id); + if (found) { + // If found the cache reference and return the rep pointer + auto& r = *boost::any_cast(accessor->second); + ebbrt::EbbRef::CacheRef(id, r); + return r; + } + } + // join with home node, found in global_id_map +#ifdef __ebbrt__ + ebbrt::EventManager::EventContext context; + auto f = ebbrt::global_id_map->Get(id); + RemoteMemory* p; + f.Then([&f, &context, &p, id](ebbrt::Future inner) { + p = new RemoteMemory(); + p->addTo(ebbrt::Messenger::NetworkId(inner.Get())); + ebbrt::event_manager->ActivateContext(std::move(context)); + }); + ebbrt::event_manager->SaveContext(context); + auto inserted = ebbrt::local_id_map->Insert(std::make_pair(id, p)); + if (inserted) { + ebbrt::EbbRef::CacheRef(id, *p); + return *p; + } +#else + RemoteMemory* p; + ebbrt::global_id_map->Set(id, ebbrt::messenger->LocalNetworkId().ToBytes()); + p = new RemoteMemory(); + ebbrt::EbbRef::CacheRef(id, *p); + auto inserted = ebbrt::local_id_map->Insert(std::make_pair(id, p)); + if (inserted) { + ebbrt::EbbRef::CacheRef(id, *p); + return *p; + } +#endif + delete p; + // retry reading + ebbrt::LocalIdMap::ConstAccessor accessor; + ebbrt::local_id_map->Find(accessor, id); + auto& r = *boost::any_cast(accessor->second); + ebbrt::EbbRef::CacheRef(id, r); + return r; +} + + +ebbrt::Future RemoteMemory::QueryMaster(int i){ + //this join is a consistent join that hold untill the node itself knows its joined + //ebbrt::kprintf("joining the home node\n"); + int id; + ebbrt::Promise promise; + auto f = promise.GetFuture(); + { + std::lock_guard guard(lock_); + if (i == 0){ +#ifdef __ebbrt__ + int p = 32765; + ebbrt::kprintf("Start Listening on port %d\n", p); + ebbrt::messenger->StartListening(p); +#endif + id = 10; + }else{ + id = 8; + } + bool inserted; + std::tie(std::ignore, inserted) = + promise_map_.emplace(id, std::move(promise)); + assert(inserted); + } + auto buf = ebbrt::MakeUniqueIOBuf(sizeof(uint32_t)); + auto dp = buf->GetMutDataPointer(); + dp.Get() = i; + SendMessage(nodelist[0], std::move(buf)); + return f; +} + +void RemoteMemory::sendPage(ebbrt::Messenger::NetworkId dst){ + + auto buffer = ebbrt::MakeUniqueIOBuf(sizeof(uint32_t)+sizeof(uint64_t)+sizeof(uint64_t)+tem_buffer[0]); + ebbrt::kprintf("size of buffer = %d, len = %d, loop = %d\n", sizeof(buffer), tem_buffer[0], tem_buffer[1]); + auto dp = buffer->GetMutDataPointer(); + dp.Get() = 8; + dp.Get() = tem_buffer[0]; + dp.Get() = tem_buffer[1]; + for (uint64_t i = 0; i < tem_buffer[1]; i++){ + dp.Get() = tem_buffer[i+2]; + // ebbrt::kprintf("BM: value read into buffer 0x%llx\n", tem_buffer[i+2]); + } + ebbrt::kprintf("destination nid: %s\n", dst.ToString().c_str()); + + SendMessage(dst, std::move(buffer)); +} + +void RemoteMemory::cachePage(uint64_t len, volatile uint32_t * pptr, uint64_t iteration){ + ebbrt::kprintf("Cached the paged\n"); + + tem_buffer.push_back(len); + tem_buffer.push_back(iteration); + for (uint64_t i = 0; i < iteration; i++){ + tem_buffer.push_back(*pptr); + //ebbrt::kprintf("BM: value read into buffer 0x%llx\n", *pptr); + pptr++; + } + ebbrt::kprintf("size of tem_buffer = %d, len = %d, loop = %d\n", tem_buffer.size(), len, iteration); + cached = true; + return; +} + +void RemoteMemory::getPage(volatile uint32_t* pptr){ + if(tem_buffer.size() > 0){ + ebbrt::kprintf("iteration is %d\n", tem_buffer[1]); + for (uint64_t i = 0; i < tem_buffer[1]; i++){ + *pptr = tem_buffer[i+2]; + //ebbrt::kprintf("setting physical page value 0x%x\n", *pptr); + pptr++; + } + }else{ + ebbrt::kabort("did not get a page"); + } +} + +void RemoteMemory::ReceiveMessage(ebbrt::Messenger::NetworkId nid, + std::unique_ptr&& buffer){ + auto dp = buffer->GetDataPointer(); + auto id = dp.Get(); + ebbrt::kprintf("id is %d\n", id); + + if(id == 0){ + auto buf = ebbrt::MakeUniqueIOBuf(sizeof(uint64_t)); + auto dp = buf->GetMutDataPointer(); + addTo(nid); + if (size() == 1) { + dp.Get() = 10; + }else{ + dp.Get() = 11; + } + dp.Get() = 10; + SendMessage(nid, std::move(buf)); + return; + } + + if (id == 8){ +#ifdef __ebbrt__ + auto len = dp.Get(); + auto iteration = dp.Get(); + tem_buffer.push_back(len); + tem_buffer.push_back(iteration); + for(uint64_t i = 0; i < iteration; i++){ + tem_buffer.push_back(dp.Get()); + } + auto it = promise_map_.find(8); + assert(it != promise_map_.end()); + it->second.SetValue(1); + promise_map_.erase(it); + +#else + auto buf = ebbrt::MakeUniqueIOBuf(sizeof(uint64_t)); + auto dp = buf->GetMutDataPointer(); + dp.Get() = 9; + char dst[20]; + std::strcpy(dst, nid.ToString().c_str()); + auto ip = std::strtok(dst, "."); + while(ip != NULL){ + dp.Get() = std::atoi(ip); + ip = std::strtok(NULL, "."); + } + SendMessage(nodelist[0], std::move(buf)); +#endif + return; + } + + if (id == 9){ +#ifdef __ebbrt__ + if (cached){ + std::array ar_ip = {0, 0, 0, 0}; + for(int i = 0; i < 4; i++){ + ar_ip[i] = dp.Get(); + } + auto dst_nid = ebbrt::Messenger::NetworkId(std::string(reinterpret_cast(ar_ip.data()), 4)); + sendPage(dst_nid); + } +#endif + return; + } + + if(id >= 10 && id < 12){ + auto trans_num = dp.Get(); + auto it = promise_map_.find(trans_num); + assert(it != promise_map_.end()); + it->second.SetValue(id); + promise_map_.erase(it); + return; + } +} diff --git a/libs/dsm/src/RemoteMem.h b/libs/dsm/src/RemoteMem.h new file mode 100644 index 00000000..08cc6a78 --- /dev/null +++ b/libs/dsm/src/RemoteMem.h @@ -0,0 +1,35 @@ +#include +#include "StaticEbbIds.h" +#include +#include +#include +#include +#include +#include "StaticEbbIds.h" +#include + + +class RemoteMemory : public ebbrt::Messagable{ + private: + std::mutex lock_; + std::vector nodelist; + bool cached = false; + std::vector tem_buffer; + std::unordered_map> promise_map_; + RemoteMemory() : ebbrt::Messagable(kRemoteMEbbId), nodelist{} {} + void addTo(ebbrt::Messenger::NetworkId nid){ + if (std::find(nodelist.begin(), nodelist.end(), nid) == nodelist.end()){ + nodelist.push_back(nid); + } + } + + public: + void ReceiveMessage(ebbrt::Messenger::NetworkId nid, std::unique_ptr&& buffer); + static RemoteMemory & HandleFault(ebbrt::EbbId id); + ebbrt::Future QueryMaster(int i); + int size(){ return int(nodelist.size()); } + void sendPage(ebbrt::Messenger::NetworkId dst); + void getPage(volatile uint32_t* pptr); + void cachePage(uint64_t len, volatile uint32_t * pptr, uint64_t iteration); +}; +constexpr auto rm = ebbrt::EbbRef(kRemoteMEbbId); diff --git a/libs/dsm/src/StaticEbbIds.h b/libs/dsm/src/StaticEbbIds.h new file mode 100644 index 00000000..b6e45c69 --- /dev/null +++ b/libs/dsm/src/StaticEbbIds.h @@ -0,0 +1,13 @@ +// Copyright Boston University SESA Group 2013 - 2016. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +#ifndef APPS_SHAREDMEM_SRC_STATICEBBIDS_H_ +#define APPS_SHAREDMEM_SRC_STATICEBBIDS_H_ + +#include + +enum : ebbrt::EbbId { kPrinterEbbId = ebbrt::kFirstStaticUserId, + kRemoteMEbbId}; + +#endif // APPS_SHAREDMEM_SRC_STATICEBBIDS_H_ diff --git a/libs/dsm/src/example/hosted/Printer.cc b/libs/dsm/src/example/hosted/Printer.cc new file mode 100644 index 00000000..4823630c --- /dev/null +++ b/libs/dsm/src/example/hosted/Printer.cc @@ -0,0 +1,27 @@ +// Copyright Boston University SESA Group 2013 - 2016. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +#include "Printer.h" + +#include + +#include + +EBBRT_PUBLISH_TYPE(, Printer); + +Printer::Printer() : ebbrt::Messagable(kPrinterEbbId) {} + +ebbrt::Future Printer::Init() { + return ebbrt::global_id_map->Set( + kPrinterEbbId, ebbrt::messenger->LocalNetworkId().ToBytes()); +} + +void Printer::Print(const char* str) {} + +void Printer::ReceiveMessage(ebbrt::Messenger::NetworkId nid, + std::unique_ptr&& buffer) { + auto output = std::string(reinterpret_cast(buffer->Data()), + buffer->Length()); + std::cout << nid.ToString() << ": " << output; +} diff --git a/libs/dsm/src/example/hosted/Printer.h b/libs/dsm/src/example/hosted/Printer.h new file mode 100644 index 00000000..9b8c6083 --- /dev/null +++ b/libs/dsm/src/example/hosted/Printer.h @@ -0,0 +1,28 @@ +// Copyright Boston University SESA Group 2013 - 2016. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +#ifndef APPS_SHAREDMEM_HOSTED_SRC_PRINTER_H_ +#define APPS_SHAREDMEM_HOSTED_SRC_PRINTER_H_ + +#include + +#include +#include + +#include "../../src/StaticEbbIds.h" + +class Printer : public ebbrt::StaticSharedEbb, + public ebbrt::Messagable { + public: + Printer(); + + static ebbrt::Future Init(); + void Print(const char* string); + void ReceiveMessage(ebbrt::Messenger::NetworkId nid, + std::unique_ptr&& buffer); +}; + +constexpr auto printer = ebbrt::EbbRef(kPrinterEbbId); + +#endif // APPS_SHAREDMEM_HOSTED_SRC_PRINTER_H_ diff --git a/libs/dsm/src/example/hosted/sharedmem.cc b/libs/dsm/src/example/hosted/sharedmem.cc new file mode 100644 index 00000000..9bd70f64 --- /dev/null +++ b/libs/dsm/src/example/hosted/sharedmem.cc @@ -0,0 +1,45 @@ +// Copyright Boston University SESA Group 2013 - 2016. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../RemoteMem.h" +#include "Printer.h" + +int main(int argc, char** argv) { + auto bindir = boost::filesystem::system_complete(argv[0]).parent_path() / + "/bm/sharedmem.elf32"; + + ebbrt::Runtime runtime; + ebbrt::Context c(runtime); + boost::asio::signal_set sig(c.io_service_, SIGINT); + { + ebbrt::ContextActivation activation(c); + + // ensure clean quit on ctrl-c + sig.async_wait([&c](const boost::system::error_code& ec, + int signal_number) { c.io_service_.stop(); }); + Printer::Init().Then([bindir](ebbrt::Future f) { + f.Get(); + if (rm->size() == 0){ + ebbrt::node_allocator->AllocateNode(bindir.string()); + std::chrono::seconds sec(20); + std::this_thread::sleep_for(sec); + ebbrt::node_allocator->AllocateNode(bindir.string()); + } + }); + } + c.Run(); + + return 0; +} diff --git a/libs/dsm/src/example/native/Printer.cc b/libs/dsm/src/example/native/Printer.cc new file mode 100644 index 00000000..0df6037b --- /dev/null +++ b/libs/dsm/src/example/native/Printer.cc @@ -0,0 +1,60 @@ +// Copyright Boston University SESA Group 2013 - 2016. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +#include "Printer.h" + +#include +#include +#include + +EBBRT_PUBLISH_TYPE(, Printer); + +Printer::Printer(ebbrt::Messenger::NetworkId nid) + : Messagable(kPrinterEbbId), remote_nid_(std::move(nid)) {} + +Printer& Printer::HandleFault(ebbrt::EbbId id) { + { + ebbrt::LocalIdMap::ConstAccessor accessor; + auto found = ebbrt::local_id_map->Find(accessor, id); + if (found) { + auto& pr = *boost::any_cast(accessor->second); + ebbrt::EbbRef::CacheRef(id, pr); + return pr; + } + } + + ebbrt::EventManager::EventContext context; + auto f = ebbrt::global_id_map->Get(id); + Printer* p; + f.Then([&f, &context, &p](ebbrt::Future inner) { + p = new Printer(ebbrt::Messenger::NetworkId(inner.Get())); + ebbrt::event_manager->ActivateContext(std::move(context)); + }); + ebbrt::event_manager->SaveContext(context); + auto inserted = ebbrt::local_id_map->Insert(std::make_pair(id, p)); + if (inserted) { + ebbrt::EbbRef::CacheRef(id, *p); + return *p; + } + + delete p; + // retry reading + ebbrt::LocalIdMap::ConstAccessor accessor; + ebbrt::local_id_map->Find(accessor, id); + auto& pr = *boost::any_cast(accessor->second); + ebbrt::EbbRef::CacheRef(id, pr); + return pr; +} + +void Printer::Print(const char* str) { + auto len = strlen(str) + 1; + auto buf = ebbrt::MakeUniqueIOBuf(len); + snprintf(reinterpret_cast(buf->MutData()), len, "%s", str); + SendMessage(remote_nid_, std::move(buf)); +} + +void Printer::ReceiveMessage(ebbrt::Messenger::NetworkId nid, + std::unique_ptr&& buffer) { + throw std::runtime_error("Printer: Received message unexpectedly!"); +} diff --git a/libs/dsm/src/example/native/Printer.h b/libs/dsm/src/example/native/Printer.h new file mode 100644 index 00000000..562e0ace --- /dev/null +++ b/libs/dsm/src/example/native/Printer.h @@ -0,0 +1,30 @@ +// Copyright Boston University SESA Group 2013 - 2016. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +#ifndef APPS_SHAREDMEM_BAREMETAL_SRC_PRINTER_H_ +#define APPS_SHAREDMEM_BAREMETAL_SRC_PRINTER_H_ + +#include + +#include + +#include "../../src/StaticEbbIds.h" + +class Printer : public ebbrt::Messagable { + public: + explicit Printer(ebbrt::Messenger::NetworkId nid); + + static Printer& HandleFault(ebbrt::EbbId id); + + void Print(const char* string); + void ReceiveMessage(ebbrt::Messenger::NetworkId nid, + std::unique_ptr&& buffer); + + private: + ebbrt::Messenger::NetworkId remote_nid_; +}; + +constexpr auto printer = ebbrt::EbbRef(kPrinterEbbId); + +#endif // APPS_SHAREDMEM_BAREMETAL_SRC_PRINTER_H_ diff --git a/libs/dsm/src/example/native/ebbrtcfg.h b/libs/dsm/src/example/native/ebbrtcfg.h new file mode 100644 index 00000000..79106207 --- /dev/null +++ b/libs/dsm/src/example/native/ebbrtcfg.h @@ -0,0 +1,13 @@ +// Copyright Boston University SESA Group 2013 - 2016. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +#ifndef APPS_SHAREDMEM_BAREMETAL_SRC_EBBRTCFG_H_ +#define APPS_SHAREDMEM_BAREMETAL_SRC_EBBRTCFG_H_ + +#define __EBBRT_ENABLE_FDT__ 1 +#define __EBBRT_ENABLE_DISTRIBUTED_RUNTIME__ 1 +#define __EBBRT_ENABLE_NETWORKING__ 1 +#define __EBBRT_ENABLE_STATIC_IP__ 1 + +#endif // APPS_SHAREDMEM_BAREMETAL_SRC_EBBRTCFG_H_ diff --git a/libs/dsm/src/example/native/sharedmem.cc b/libs/dsm/src/example/native/sharedmem.cc new file mode 100644 index 00000000..e104136b --- /dev/null +++ b/libs/dsm/src/example/native/sharedmem.cc @@ -0,0 +1,87 @@ +// Copyright Boston University SESA Group 2013 - 2016. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include "Printer.h" +#include "../sharedmem.h" + +// a function to be called to invoke events in other cores +void test(uintptr_t addr); +// this function use a for loop to fill the physical page with desireed value +void fillInValue(int value, int iteration, volatile uint32_t * ptr); +//this function use a for loop to lazly invoked the page fault handler +void lazyMap(int value, int iteration, int multiplier, volatile uint32_t * ptr); + +void AppMain() { + int len = 0; // number of pages need to be allocated physical allocator + int mul = 1; // multipler for how many times more virtual pages to be allocated + auto ps = ebbrt::pmem::kPageSize; // page size by default + int iteration = ((int)ps*(1<Print("BEGIN TO ALLOCATE THE VIRTUAL PAGE.\n"); + auto pf = std::make_unique(); + pf->setPage(len, v); + auto vfn = ebbrt::vmem_allocator->Alloc(mul*(1<Print("BEGIN TO UNMAP THE VIRTUAL PAGE.\n"); + //start to unmap the virtual region by traversing the page talbes and delete the entries + TraversePageTable( + ebbrt::vmem::GetPageTableRoot(), addr, addr + mul*(1 << len)*ps, 0, 4, + [=](ebbrt::vmem::Pte& entry, uint64_t base_virt, size_t level) { + kassert(entry.Present()); + entry.Clear(); + std::atomic_thread_fence(std::memory_order_release); + asm volatile("invlpg (%[addr])" : : [addr] "r"(base_virt) : "memory"); + }, + [=](ebbrt::vmem::Pte& entry) { + ebbrt::kprintf("Asked to unmap memory that wasn't mapped!\n"); + ebbrt::kabort(); + return false; + }); + }; + + ptr = (volatile uint32_t *)addr; + ebbrt::kbugon( (int)*ptr != v, "wrong value for remap the virtual region!!\n"); + // the following is for spawning different events on other cores. + // auto f = [addr]() { + // ebbrt::kprintf("Hello from %u\n", (size_t)ebbrt::Cpu::GetMine()); + // test(addr); + // test(addr); + // }; + //ebbrt::event_manager->SpawnLocal(f); + // ebbrt::event_manager->SpawnRemote(f, 1); +} + +void test(uintptr_t addr) { + auto ptr = (volatile uint32_t *)addr; + (void)*ptr; +} + +void fillInValue(int value, int iteration, volatile uint32_t * ptr){ + for(int i = 0; i < iteration; i++){ + *ptr = value; + ptr++; + } + +} + +void lazyMap(int value, int iteration,int multiplier, volatile uint32_t *ptr){ + for(int i = 0; i < multiplier * iteration; i++){ + volatile auto val = *ptr; + ebbrt::kbugon( (int)val != value, "wrong value in the virtual memory!!\n"); + ptr++; + } + +} + diff --git a/libs/dsm/src/sharedmem.h b/libs/dsm/src/sharedmem.h new file mode 100644 index 00000000..7384d217 --- /dev/null +++ b/libs/dsm/src/sharedmem.h @@ -0,0 +1,59 @@ +#include +#include +#include +#include +#include +#include +#include "RemoteMem.h" + + +class NewPageFaultHandler : public ebbrt::VMemAllocator::PageFaultHandler { + uint64_t pageLen; + int value; + uint64_t granularity; + uint64_t iteration; + int numberOfPages; + uint64_t ps = ebbrt::pmem::kPageSize; +public: + void setPage(int l, int v) { + numberOfPages = l; + pageLen = ps*(1 << l); + value = v; + iteration = ps*(1<= 1<<21){ + ps = 1<<21; + } + granularity = pageLen/ps; + } + void HandleFault(ebbrt::idt::ExceptionFrame* ef, uintptr_t faulted_address) override { + ebbrt::kprintf("faulted address %#llx, desired page %d, value 0x%x, on core %d\n", faulted_address, numberOfPages, value, (size_t)ebbrt::Cpu::GetMine()); + auto pfn = ebbrt::page_allocator->Alloc(numberOfPages); + auto pptr = (volatile uint32_t*) pfn.ToAddr(); + auto f_c = rm->QueryMaster(0); + auto reply = f_c.Block().Get(); + if (reply == 10){ + for(uint64_t i = 0; i < iteration; i++){ + *pptr = value; + pptr++; + } + auto pptr_ = (volatile uint32_t * )pfn.ToAddr(); + rm->cachePage(pageLen, pptr_, iteration); + }else{ + auto f_get = rm->QueryMaster(8); + auto reply = f_get.Block().Get(); + if (reply == 1){ + rm->getPage(pptr); + } + } + auto vpage = ebbrt::Pfn::Down(faulted_address); + auto page = pfn; + for (uint64_t i = 0; i < granularity; i++){ + ebbrt::vmem::MapMemory(vpage, page, ps); + vpage = ebbrt::Pfn::Down(vpage.ToAddr() + ps); + page = ebbrt::Pfn::Down(page.ToAddr() + ps); + } + } +}; + + +