diff --git a/DataStructures/RangeTable.h b/DataStructures/RangeTable.h new file mode 100644 index 00000000000..1358d3d21cb --- /dev/null +++ b/DataStructures/RangeTable.h @@ -0,0 +1,231 @@ +#ifndef __RANGE_TABLE_H__ +#define __RANGE_TABLE_H__ + +#include "SharedMemoryFactory.h" +#include "SharedMemoryVectorWrapper.h" + +#include + +#include +#include +#include + +/* + * These pre-declarations are needed because parsing C++ is hard + * and otherwise the compiler gets confused. + */ + +template class RangeTable; + +template +std::ostream& operator<<(std::ostream &out, const RangeTable &table); + +template +std::istream& operator>>(std::istream &in, RangeTable &table); + +/** + * Stores adjacent ranges in a compressed format. + * + * Maximum supported length of a range is 255. + * + * Note: BLOCK_SIZE is the number of differential encodoed values. + * But each block consists of an absolute value and BLOCK_SIZE differential values. + * So the effective block size is sizeof(unsigned) + BLOCK_SIZE. + */ +template +class RangeTable +{ +public: + + typedef std::array BlockT; + typedef typename ShM::vector BlockContainerT; + typedef typename ShM::vector OffsetContainerT; + typedef decltype(boost::irange(0u,0u)) RangeT; + + friend std::ostream& operator<< <>(std::ostream &out, const RangeTable &table); + friend std::istream& operator>> <>(std::istream &in, RangeTable &table); + + RangeTable() {} + + // for loading from shared memory + explicit RangeTable(OffsetContainerT& external_offsets, BlockContainerT& external_blocks, const unsigned sum_lengths) + : sum_lengths(sum_lengths) + { + block_offsets.swap(external_offsets); + diff_blocks.swap(external_blocks); + } + + // construct table from length vector + explicit RangeTable(const std::vector& lengths) + { + const unsigned number_of_blocks = [&lengths]() { + unsigned num = (lengths.size() + 1) / (BLOCK_SIZE + 1); + if ((lengths.size() + 1) % (BLOCK_SIZE + 1) != 0) + { + num += 1; + } + return num; + }(); + + block_offsets.reserve(number_of_blocks); + diff_blocks.reserve(number_of_blocks); + + unsigned last_length = 0; + unsigned lengths_prefix_sum = 0; + unsigned block_idx = 0; + unsigned block_counter = 0; + BlockT block; + unsigned block_sum = 0; + for (const unsigned l : lengths) + { + // first entry of a block: encode absolute offset + if (block_idx == 0) + { + block_offsets.push_back(lengths_prefix_sum); + block_sum = 0; + } + else + { + block[block_idx - 1] = last_length; + block_sum += last_length; + } + + BOOST_ASSERT((block_idx == 0 && block_offsets[block_counter] == lengths_prefix_sum) + || lengths_prefix_sum == (block_offsets[block_counter]+block_sum)); + + // block is full + if (BLOCK_SIZE == block_idx) + { + diff_blocks.push_back(block); + block_counter++; + } + + // we can only store strings with length 255 + BOOST_ASSERT(l <= 255); + + lengths_prefix_sum += l; + last_length = l; + + block_idx = (block_idx + 1) % (BLOCK_SIZE + 1); + } + + // Last block can't be finished because we didn't add the sentinel + BOOST_ASSERT (block_counter == (number_of_blocks - 1)); + + // one block missing: starts with guard value + if (0 == block_idx) + { + // the last value is used as sentinel + block_offsets.push_back(lengths_prefix_sum); + block_idx = (block_idx + 1) % BLOCK_SIZE; + } + + while (0 != block_idx) + { + block[block_idx - 1] = last_length; + last_length = 0; + block_idx = (block_idx + 1) % (BLOCK_SIZE + 1); + } + diff_blocks.push_back(block); + + BOOST_ASSERT(diff_blocks.size() == number_of_blocks && block_offsets.size() == number_of_blocks); + + sum_lengths = lengths_prefix_sum; + } + + inline RangeT GetRange(const unsigned id) const + { + BOOST_ASSERT(id < block_offsets.size() + diff_blocks.size() * BLOCK_SIZE); + // internal_idx 0 is implicitly stored in block_offsets[block_idx] + const unsigned internal_idx = id % (BLOCK_SIZE + 1); + const unsigned block_idx = id / (BLOCK_SIZE + 1); + + BOOST_ASSERT(block_idx < diff_blocks.size()); + + unsigned begin_idx = 0; + unsigned end_idx = 0; + begin_idx = block_offsets[block_idx]; + const BlockT& block = diff_blocks[block_idx]; + if (internal_idx > 0) + { + begin_idx += PrefixSumAtIndex(internal_idx - 1, block); + } + + // next index inside current block + if (internal_idx < BLOCK_SIZE) + { + // note internal_idx - 1 is the *current* index for uint8_blocks + end_idx = begin_idx + block[internal_idx]; + } + else + { + BOOST_ASSERT(block_idx < block_offsets.size() - 1); + end_idx = block_offsets[block_idx + 1]; + } + + BOOST_ASSERT(begin_idx < sum_lengths && end_idx <= sum_lengths); + BOOST_ASSERT(begin_idx <= end_idx); + + return boost::irange(begin_idx, end_idx); + } +private: + + inline unsigned PrefixSumAtIndex(int index, const BlockT& block) const; + + // contains offset for each differential block + OffsetContainerT block_offsets; + // blocks of differential encoded offsets, should be aligned + BlockContainerT diff_blocks; + unsigned sum_lengths; +}; + +template +unsigned RangeTable::PrefixSumAtIndex(int index, const BlockT& block) const +{ + // this loop looks inefficent, but a modern compiler + // will emit nice SIMD here, at least for sensible block sizes. (I checked.) + unsigned sum = 0; + for (int i = 0; i <= index; ++i) + { + sum += block[i]; + } + + return sum; +} + +template +std::ostream& operator<<(std::ostream &out, const RangeTable &table) +{ + // write number of block + const unsigned number_of_blocks = table.diff_blocks.size(); + out.write((char *) &number_of_blocks, sizeof(unsigned)); + // write total length + out.write((char *) &table.sum_lengths, sizeof(unsigned)); + // write block offsets + out.write((char *) table.block_offsets.data(), sizeof(unsigned) * table.block_offsets.size()); + // write blocks + out.write((char *) table.diff_blocks.data(), BLOCK_SIZE * table.diff_blocks.size()); + + return out; +} + +template +std::istream& operator>>(std::istream &in, RangeTable &table) +{ + // read number of block + unsigned number_of_blocks; + in.read((char *) &number_of_blocks, sizeof(unsigned)); + // read total length + in.read((char *) &table.sum_lengths, sizeof(unsigned)); + + table.block_offsets.resize(number_of_blocks); + table.diff_blocks.resize(number_of_blocks); + + // read block offsets + in.read((char *) table.block_offsets.data(), sizeof(unsigned) * number_of_blocks); + // read blocks + in.read((char *) table.diff_blocks.data(), BLOCK_SIZE * number_of_blocks); + return in; +} + +#endif diff --git a/DataStructures/StaticRTree.h b/DataStructures/StaticRTree.h index c1cad7fdbc9..112b1c41478 100644 --- a/DataStructures/StaticRTree.h +++ b/DataStructures/StaticRTree.h @@ -766,7 +766,9 @@ class StaticRTree } const uint64_t seek_pos = sizeof(uint64_t) + leaf_id * sizeof(LeafNode); thread_local_rtree_stream->seekg(seek_pos); + BOOST_ASSERT_MSG(thread_local_rtree_stream->good(), "Seeking to position in leaf file failed."); thread_local_rtree_stream->read((char *)&result_node, sizeof(LeafNode)); + BOOST_ASSERT_MSG(thread_local_rtree_stream->good(), "Reading from leaf file failed."); } inline bool EdgesAreEquivalent(const FixedPointCoordinate &a, diff --git a/Extractor/ExtractionContainers.cpp b/Extractor/ExtractionContainers.cpp index 29598f03525..91981d66a00 100644 --- a/Extractor/ExtractionContainers.cpp +++ b/Extractor/ExtractionContainers.cpp @@ -30,6 +30,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../Util/OSRMException.h" #include "../Util/SimpleLogger.h" #include "../Util/TimingUtil.h" +#include "../DataStructures/RangeTable.h" #include #include @@ -64,6 +65,7 @@ void ExtractionContainers::PrepareData(const std::string &output_file_name, { unsigned number_of_used_nodes = 0; unsigned number_of_used_edges = 0; + std::cout << "[extractor] Sorting used nodes ... " << std::flush; TIMER_START(sorting_used_nodes); stxxl::sort(used_node_id_list.begin(), used_node_id_list.end(), Cmp(), stxxl_memory); @@ -395,32 +397,23 @@ void ExtractionContainers::PrepareData(const std::string &output_file_name, std::string name_file_streamName = (output_file_name + ".names"); boost::filesystem::ofstream name_file_stream(name_file_streamName, std::ios::binary); - // write number of names - const unsigned number_of_names = name_list.size() + 1; - name_file_stream.write((char *)&(number_of_names), sizeof(unsigned)); - - // compute total number of chars - unsigned total_number_of_chars = 0; - for (const std::string &temp_string : name_list) - { - total_number_of_chars += temp_string.length(); - } - // write total number of chars - name_file_stream.write((char *)&(total_number_of_chars), sizeof(unsigned)); - // write prefixe sums - unsigned name_lengths_prefix_sum = 0; + unsigned total_length = 0; + std::vector name_lengths; for (const std::string &temp_string : name_list) { - name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned)); - name_lengths_prefix_sum += temp_string.length(); + const unsigned string_length = std::min(static_cast(temp_string.length()), 255u); + name_lengths.push_back(string_length); + total_length += string_length; } - // duplicate on purpose! - name_file_stream.write((char *)&(name_lengths_prefix_sum), sizeof(unsigned)); + RangeTable<> table(name_lengths); + name_file_stream << table; + + name_file_stream.write((char*) &total_length, sizeof(unsigned)); // write all chars consecutively for (const std::string &temp_string : name_list) { - const unsigned string_length = temp_string.length(); + const unsigned string_length = std::min(static_cast(temp_string.length()), 255u); name_file_stream.write(temp_string.c_str(), string_length); } diff --git a/Server/DataStructures/InternalDataFacade.h b/Server/DataStructures/InternalDataFacade.h index 3c63bd461e7..afa82f58a79 100644 --- a/Server/DataStructures/InternalDataFacade.h +++ b/Server/DataStructures/InternalDataFacade.h @@ -38,6 +38,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../../DataStructures/SharedMemoryVectorWrapper.h" #include "../../DataStructures/StaticGraph.h" #include "../../DataStructures/StaticRTree.h" +#include "../../DataStructures/RangeTable.h" #include "../../Util/BoostFileSystemFix.h" #include "../../Util/GraphLoader.h" #include "../../Util/ProgramOptions.h" @@ -66,13 +67,13 @@ template class InternalDataFacade : public BaseDataFacade::vector m_name_ID_list; ShM::vector m_turn_instruction_list; ShM::vector m_names_char_list; - ShM::vector m_name_begin_indices; ShM::vector m_egde_is_compressed; ShM::vector m_geometry_indices; ShM::vector m_geometry_list; std::shared_ptr::vector, false>> m_static_rtree; + RangeTable<16, false> m_name_table; void LoadTimestamp(const boost::filesystem::path ×tamp_path) { @@ -203,16 +204,12 @@ template class InternalDataFacade : public BaseDataFacade> m_name_table; + unsigned number_of_chars = 0; - name_stream.read((char *)&number_of_names, sizeof(unsigned)); name_stream.read((char *)&number_of_chars, sizeof(unsigned)); - BOOST_ASSERT_MSG(0 != number_of_names, "name file broken"); BOOST_ASSERT_MSG(0 != number_of_chars, "name file broken"); - - m_name_begin_indices.resize(number_of_names); - name_stream.read((char *)&m_name_begin_indices[0], number_of_names * sizeof(unsigned)); - m_names_char_list.resize(number_of_chars + 1); //+1 gives sentinel element name_stream.read((char *)&m_names_char_list[0], number_of_chars * sizeof(char)); BOOST_ASSERT_MSG(0 != m_names_char_list.size(), "could not load any names"); @@ -384,18 +381,16 @@ template class InternalDataFacade : public BaseDataFacade class SharedDataFacade : public BaseDataFacade QueryGraph; typedef typename StaticGraph::NodeArrayEntry GraphNode; typedef typename StaticGraph::EdgeArrayEntry GraphEdge; + typedef typename RangeTable<16, true>::BlockT NameIndexBlock; typedef typename QueryGraph::InputEdge InputEdge; typedef typename super::RTreeLeaf RTreeLeaf; typedef typename StaticRTree::vector, true>::TreeNode @@ -84,43 +86,51 @@ template class SharedDataFacade : public BaseDataFacade::vector, true>> m_static_rtree; + std::shared_ptr> m_name_table; + void LoadChecksum() { - m_check_sum = data_layout->checksum; + m_check_sum = *data_layout->GetBlockPtr(shared_memory, SharedDataLayout::HSGR_CHECKSUM); SimpleLogger().Write() << "set checksum: " << m_check_sum; } + void LoadTimestamp() { - char *timestamp_ptr = shared_memory + data_layout->GetTimeStampOffset(); - m_timestamp.resize(data_layout->timestamp_length); + char *timestamp_ptr = data_layout->GetBlockPtr(shared_memory, SharedDataLayout::TIMESTAMP); + m_timestamp.resize(data_layout->GetBlockSize(SharedDataLayout::TIMESTAMP)); std::copy( - timestamp_ptr, timestamp_ptr + data_layout->timestamp_length, m_timestamp.begin()); + timestamp_ptr, + timestamp_ptr + data_layout->GetBlockSize(SharedDataLayout::TIMESTAMP), + m_timestamp.begin()); } void LoadRTree(const boost::filesystem::path &file_index_path) { BOOST_ASSERT_MSG(!m_coordinate_list->empty(), "coordinates must be loaded before r-tree"); - RTreeNode *tree_ptr = (RTreeNode *)(shared_memory + data_layout->GetRSearchTreeOffset()); + RTreeNode *tree_ptr = data_layout->GetBlockPtr(shared_memory, SharedDataLayout::R_SEARCH_TREE); m_static_rtree = std::make_shared::vector, true>>( - tree_ptr, data_layout->r_search_tree_size, file_index_path, m_coordinate_list); + tree_ptr, + data_layout->num_entries[SharedDataLayout::R_SEARCH_TREE], + file_index_path, + m_coordinate_list); } void LoadGraph() { - m_number_of_nodes = data_layout->graph_node_list_size; + m_number_of_nodes = data_layout->num_entries[SharedDataLayout::GRAPH_NODE_LIST]; GraphNode *graph_nodes_ptr = - (GraphNode *)(shared_memory + data_layout->GetGraphNodeListOffset()); + data_layout->GetBlockPtr(shared_memory, SharedDataLayout::GRAPH_NODE_LIST); GraphEdge *graph_edges_ptr = - (GraphEdge *)(shared_memory + data_layout->GetGraphEdgeListOffset()); + data_layout->GetBlockPtr(shared_memory, SharedDataLayout::GRAPH_EDGE_LIST); typename ShM::vector node_list(graph_nodes_ptr, - data_layout->graph_node_list_size); + data_layout->num_entries[SharedDataLayout::GRAPH_NODE_LIST]); typename ShM::vector edge_list(graph_edges_ptr, - data_layout->graph_edge_list_size); + data_layout->num_entries[SharedDataLayout::GRAPH_EDGE_LIST]); m_query_graph.reset(new QueryGraph(node_list, edge_list)); } @@ -128,63 +138,62 @@ template class SharedDataFacade : public BaseDataFacadeGetCoordinateListOffset()); + data_layout->GetBlockPtr(shared_memory, SharedDataLayout::COORDINATE_LIST); m_coordinate_list = std::make_shared::vector>( - coordinate_list_ptr, data_layout->coordinate_list_size); + coordinate_list_ptr, data_layout->num_entries[SharedDataLayout::COORDINATE_LIST]); TurnInstruction *turn_instruction_list_ptr = - (TurnInstruction *)(shared_memory + data_layout->GetTurnInstructionListOffset()); + data_layout->GetBlockPtr(shared_memory, SharedDataLayout::TURN_INSTRUCTION); typename ShM::vector turn_instruction_list( - turn_instruction_list_ptr, data_layout->turn_instruction_list_size); + turn_instruction_list_ptr, data_layout->num_entries[SharedDataLayout::TURN_INSTRUCTION]); m_turn_instruction_list.swap(turn_instruction_list); - unsigned *name_id_list_ptr = - (unsigned *)(shared_memory + data_layout->GetNameIDListOffset()); + unsigned *name_id_list_ptr = data_layout->GetBlockPtr(shared_memory, SharedDataLayout::NAME_ID_LIST); typename ShM::vector name_id_list(name_id_list_ptr, - data_layout->name_id_list_size); + data_layout->num_entries[SharedDataLayout::NAME_ID_LIST]); m_name_ID_list.swap(name_id_list); } void LoadViaNodeList() { - NodeID *via_node_list_ptr = (NodeID *)(shared_memory + data_layout->GetViaNodeListOffset()); + NodeID *via_node_list_ptr = data_layout->GetBlockPtr(shared_memory, SharedDataLayout::VIA_NODE_LIST); typename ShM::vector via_node_list(via_node_list_ptr, - data_layout->via_node_list_size); + data_layout->num_entries[SharedDataLayout::VIA_NODE_LIST]); m_via_node_list.swap(via_node_list); } void LoadNames() { - unsigned *street_names_index_ptr = - (unsigned *)(shared_memory + data_layout->GetNameIndexOffset()); - typename ShM::vector name_begin_indices(street_names_index_ptr, - data_layout->name_index_list_size); - m_name_begin_indices.swap(name_begin_indices); - - char *names_list_ptr = (char *)(shared_memory + data_layout->GetNameListOffset()); + unsigned *offsets_ptr = data_layout->GetBlockPtr(shared_memory, SharedDataLayout::NAME_OFFSETS); + NameIndexBlock *blocks_ptr = data_layout->GetBlockPtr(shared_memory, SharedDataLayout::NAME_BLOCKS); + typename ShM::vector name_offsets(offsets_ptr, + data_layout->num_entries[SharedDataLayout::NAME_OFFSETS]); + typename ShM::vector name_blocks(blocks_ptr, + data_layout->num_entries[SharedDataLayout::NAME_BLOCKS]); + + char *names_list_ptr = data_layout->GetBlockPtr(shared_memory, SharedDataLayout::NAME_CHAR_LIST); typename ShM::vector names_char_list(names_list_ptr, - data_layout->name_char_list_size); + data_layout->num_entries[SharedDataLayout::NAME_CHAR_LIST]); + m_name_table = std::make_shared>(name_offsets, name_blocks, names_char_list.size()); + m_names_char_list.swap(names_char_list); } void LoadGeometries() { - unsigned *geometries_compressed_ptr = - (unsigned *)(shared_memory + data_layout->GetGeometriesIndicatorOffset()); + unsigned *geometries_compressed_ptr = data_layout->GetBlockPtr(shared_memory, SharedDataLayout::GEOMETRIES_INDICATORS); typename ShM::vector egde_is_compressed(geometries_compressed_ptr, - data_layout->geometries_indicators); + data_layout->num_entries[SharedDataLayout::GEOMETRIES_INDICATORS]); m_egde_is_compressed.swap(egde_is_compressed); - unsigned *geometries_index_ptr = - (unsigned *)(shared_memory + data_layout->GetGeometriesIndexListOffset()); + unsigned *geometries_index_ptr = data_layout->GetBlockPtr(shared_memory, SharedDataLayout::GEOMETRIES_INDEX); typename ShM::vector geometry_begin_indices( - geometries_index_ptr, data_layout->geometries_index_list_size); + geometries_index_ptr, data_layout->num_entries[SharedDataLayout::GEOMETRIES_INDEX]); m_geometry_indices.swap(geometry_begin_indices); - unsigned *geometries_list_ptr = - (unsigned *)(shared_memory + data_layout->GetGeometryListOffset()); + unsigned *geometries_list_ptr = data_layout->GetBlockPtr(shared_memory, SharedDataLayout::GEOMETRIES_LIST); typename ShM::vector geometry_list(geometries_list_ptr, - data_layout->geometries_list_size); + data_layout->num_entries[SharedDataLayout::GEOMETRIES_LIST]); m_geometry_list.swap(geometry_list); } @@ -219,21 +228,28 @@ template class SharedDataFacade : public BaseDataFacadePtr()); - boost::filesystem::path ram_index_path(data_layout->ram_index_file_name); - if (!boost::filesystem::exists(ram_index_path)) - { - throw OSRMException("no leaf index file given. " - "Is any data loaded into shared memory?"); - } m_large_memory.reset(SharedMemoryFactory::Get(CURRENT_DATA)); shared_memory = (char *)(m_large_memory->Ptr()); + std::ofstream out("debug.bin"); + out.write(shared_memory, data_layout->GetSizeOfLayout()); + out.close(); + + const char* file_index_ptr = data_layout->GetBlockPtr(shared_memory, SharedDataLayout::FILE_INDEX_PATH); + boost::filesystem::path file_index_path(file_index_ptr); + if (!boost::filesystem::exists(file_index_path)) + { + SimpleLogger().Write(logDEBUG) << "Leaf file name " << file_index_path.string(); + throw OSRMException("Could not load leaf index file." + "Is any data loaded into shared memory?"); + } + LoadGraph(); LoadChecksum(); LoadNodeAndEdgeInformation(); LoadGeometries(); - LoadRTree(ram_index_path); + LoadRTree(file_index_path); LoadTimestamp(); LoadViaNodeList(); LoadNames(); @@ -339,18 +355,16 @@ template class SharedDataFacade : public BaseDataFacadeGetRange(name_id); - BOOST_ASSERT_MSG(begin_index <= end_index, "string ends before begin"); result.clear(); - result.resize(end_index - begin_index); - std::copy(m_names_char_list.begin() + begin_index, - m_names_char_list.begin() + end_index, - result.begin()); + if (range.begin() != range.end()) + { + result.resize(range.back() - range.front() + 1); + std::copy(m_names_char_list.begin() + range.front(), + m_names_char_list.begin() + range.back() + 1, + result.begin()); + } } std::string GetTimestamp() const { return m_timestamp; } diff --git a/Server/DataStructures/SharedDataType.h b/Server/DataStructures/SharedDataType.h index 6204f44fb5e..e926b5555ad 100644 --- a/Server/DataStructures/SharedDataType.h +++ b/Server/DataStructures/SharedDataType.h @@ -28,230 +28,145 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef SHARED_DATA_TYPE_H_ #define SHARED_DATA_TYPE_H_ -#include "BaseDataFacade.h" - -#include "../../DataStructures/QueryEdge.h" -#include "../../DataStructures/StaticGraph.h" -#include "../../DataStructures/StaticRTree.h" -#include "../../DataStructures/TurnInstructions.h" - -#include "../../typedefs.h" - -#include +#include "../../Util/SimpleLogger.h" #include -typedef BaseDataFacade::RTreeLeaf RTreeLeaf; -typedef StaticRTree::vector, true>::TreeNode RTreeNode; -typedef StaticGraph QueryGraph; +#include + +// Added at the start and end of each block as sanity check +constexpr char CANARY[] = "OSRM"; struct SharedDataLayout { - uint64_t name_index_list_size; - uint64_t name_char_list_size; - uint64_t name_id_list_size; - uint64_t via_node_list_size; - uint64_t graph_node_list_size; - uint64_t graph_edge_list_size; - uint64_t coordinate_list_size; - uint64_t turn_instruction_list_size; - uint64_t r_search_tree_size; - uint64_t geometries_index_list_size; - uint64_t geometries_list_size; - uint64_t geometries_indicators; - - unsigned checksum; - unsigned timestamp_length; - - char ram_index_file_name[1024]; + enum BlockID { + NAME_OFFSETS = 0, + NAME_BLOCKS, + NAME_CHAR_LIST, + NAME_ID_LIST, + VIA_NODE_LIST, + GRAPH_NODE_LIST, + GRAPH_EDGE_LIST, + COORDINATE_LIST, + TURN_INSTRUCTION, + R_SEARCH_TREE, + GEOMETRIES_INDEX, + GEOMETRIES_LIST, + GEOMETRIES_INDICATORS, + HSGR_CHECKSUM, + TIMESTAMP, + FILE_INDEX_PATH, + NUM_BLOCKS + }; + + std::array num_entries; + std::array entry_size; SharedDataLayout() - : name_index_list_size(0), name_char_list_size(0), name_id_list_size(0), - via_node_list_size(0), graph_node_list_size(0), graph_edge_list_size(0), - coordinate_list_size(0), turn_instruction_list_size(0), r_search_tree_size(0), - geometries_index_list_size(0), geometries_list_size(0), geometries_indicators(0), - checksum(0), timestamp_length(0) - + : num_entries() + , entry_size() { - ram_index_file_name[0] = '\0'; } void PrintInformation() const { SimpleLogger().Write(logDEBUG) << "-"; - SimpleLogger().Write(logDEBUG) << "name_index_list_size: " << name_index_list_size; - SimpleLogger().Write(logDEBUG) << "name_char_list_size: " << name_char_list_size; - SimpleLogger().Write(logDEBUG) << "name_id_list_size: " << name_id_list_size; - SimpleLogger().Write(logDEBUG) << "via_node_list_size: " << via_node_list_size; - SimpleLogger().Write(logDEBUG) << "graph_node_list_size: " << graph_node_list_size; - SimpleLogger().Write(logDEBUG) << "graph_edge_list_size: " << graph_edge_list_size; - SimpleLogger().Write(logDEBUG) << "timestamp_length: " << timestamp_length; - SimpleLogger().Write(logDEBUG) << "coordinate_list_size: " << coordinate_list_size; - SimpleLogger().Write(logDEBUG) - << "turn_instruction_list_size: " << turn_instruction_list_size; - SimpleLogger().Write(logDEBUG) << "r_search_tree_size: " << r_search_tree_size; - SimpleLogger().Write(logDEBUG) << "geometries_indicators: " << geometries_indicators - << "/" << ((geometries_indicators / 8) + 1); - SimpleLogger().Write(logDEBUG) - << "geometries_index_list_size: " << geometries_index_list_size; - SimpleLogger().Write(logDEBUG) << "geometries_list_size: " << geometries_list_size; - SimpleLogger().Write(logDEBUG) << "checksum: " << checksum; - SimpleLogger().Write(logDEBUG) << "sizeof(checksum): " << sizeof(checksum); - SimpleLogger().Write(logDEBUG) << "ram index file name: " << ram_index_file_name; - } - - uint64_t GetSizeOfLayout() const - { - uint64_t result = - (name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) + - (name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) + - (graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) + - (graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) + - (timestamp_length * sizeof(char)) + - (coordinate_list_size * sizeof(FixedPointCoordinate)) + - (turn_instruction_list_size * sizeof(TurnInstructionsClass)) + - (r_search_tree_size * sizeof(RTreeNode)) + - (geometries_indicators / 32 + 1) * sizeof(unsigned) + - (geometries_index_list_size * sizeof(unsigned)) + - (geometries_list_size * sizeof(unsigned)) + sizeof(checksum) + 1024 * sizeof(char); - return result; - } - - uint64_t GetNameIndexOffset() const { return 0; } - uint64_t GetNameListOffset() const - { - uint64_t result = (name_index_list_size * sizeof(unsigned)); - return result; - } - uint64_t GetNameIDListOffset() const - { - uint64_t result = - (name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)); - return result; - } - uint64_t GetViaNodeListOffset() const - { - uint64_t result = (name_index_list_size * sizeof(unsigned)) + - (name_char_list_size * sizeof(char)) + - (name_id_list_size * sizeof(unsigned)); - return result; - } - uint64_t GetGraphNodeListOffset() const - { - uint64_t result = - (name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) + - (name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)); - return result; - } - uint64_t GetGraphEdgeListOffset() const - { - uint64_t result = - (name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) + - (name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) + - (graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)); - return result; - } - uint64_t GetTimeStampOffset() const - { - uint64_t result = - (name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) + - (name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) + - (graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) + - (graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)); - return result; - } - uint64_t GetCoordinateListOffset() const - { - uint64_t result = - (name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) + - (name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) + - (graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) + - (graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) + - (timestamp_length * sizeof(char)); - return result; - } - uint64_t GetTurnInstructionListOffset() const - { - uint64_t result = - (name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) + - (name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) + - (graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) + - (graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) + - (timestamp_length * sizeof(char)) + - (coordinate_list_size * sizeof(FixedPointCoordinate)); - return result; - } - uint64_t GetRSearchTreeOffset() const - { - uint64_t result = - (name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) + - (name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) + - (graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) + - (graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) + - (timestamp_length * sizeof(char)) + - (coordinate_list_size * sizeof(FixedPointCoordinate)) + - (turn_instruction_list_size * sizeof(TurnInstructionsClass)); - return result; - } - uint64_t GetGeometriesIndicatorOffset() const - { - uint64_t result = - (name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) + - (name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) + - (graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) + - (graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) + - (timestamp_length * sizeof(char)) + - (coordinate_list_size * sizeof(FixedPointCoordinate)) + - (turn_instruction_list_size * sizeof(TurnInstructionsClass)) + - (r_search_tree_size * sizeof(RTreeNode)); - return result; - } - - uint64_t GetGeometriesIndexListOffset() const - { - uint64_t result = - (name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) + - (name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) + - (graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) + - (graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) + - (timestamp_length * sizeof(char)) + - (coordinate_list_size * sizeof(FixedPointCoordinate)) + - (turn_instruction_list_size * sizeof(TurnInstructionsClass)) + - (r_search_tree_size * sizeof(RTreeNode)) + - (geometries_indicators / 32 + 1) * sizeof(unsigned); + SimpleLogger().Write(logDEBUG) << "name_offsets_size: " << num_entries[NAME_OFFSETS]; + SimpleLogger().Write(logDEBUG) << "name_blocks_size: " << num_entries[NAME_BLOCKS]; + SimpleLogger().Write(logDEBUG) << "name_char_list_size: " << num_entries[NAME_CHAR_LIST]; + SimpleLogger().Write(logDEBUG) << "name_id_list_size: " << num_entries[NAME_ID_LIST]; + SimpleLogger().Write(logDEBUG) << "via_node_list_size: " << num_entries[VIA_NODE_LIST]; + SimpleLogger().Write(logDEBUG) << "graph_node_list_size: " << num_entries[GRAPH_NODE_LIST]; + SimpleLogger().Write(logDEBUG) << "graph_edge_list_size: " << num_entries[GRAPH_EDGE_LIST]; + SimpleLogger().Write(logDEBUG) << "timestamp_length: " << num_entries[TIMESTAMP]; + SimpleLogger().Write(logDEBUG) << "coordinate_list_size: " << num_entries[COORDINATE_LIST]; + SimpleLogger().Write(logDEBUG) << "turn_instruction_list_size: " << num_entries[TURN_INSTRUCTION]; + SimpleLogger().Write(logDEBUG) << "r_search_tree_size: " << num_entries[R_SEARCH_TREE]; + SimpleLogger().Write(logDEBUG) << "geometries_indicators: " << num_entries[GEOMETRIES_INDICATORS] + << "/" << ((num_entries[GEOMETRIES_INDICATORS] / 8) + 1); + SimpleLogger().Write(logDEBUG) << "geometries_index_list_size: " << num_entries[GEOMETRIES_INDEX]; + SimpleLogger().Write(logDEBUG) << "geometries_list_size: " << num_entries[GEOMETRIES_LIST]; + SimpleLogger().Write(logDEBUG) << "sizeof(checksum): " << entry_size[HSGR_CHECKSUM]; + + SimpleLogger().Write(logDEBUG) << "NAME_OFFSETS " << ": " << GetBlockSize(NAME_OFFSETS ); + SimpleLogger().Write(logDEBUG) << "NAME_BLOCKS " << ": " << GetBlockSize(NAME_BLOCKS ); + SimpleLogger().Write(logDEBUG) << "NAME_CHAR_LIST " << ": " << GetBlockSize(NAME_CHAR_LIST ); + SimpleLogger().Write(logDEBUG) << "NAME_ID_LIST " << ": " << GetBlockSize(NAME_ID_LIST ); + SimpleLogger().Write(logDEBUG) << "VIA_NODE_LIST " << ": " << GetBlockSize(VIA_NODE_LIST ); + SimpleLogger().Write(logDEBUG) << "GRAPH_NODE_LIST " << ": " << GetBlockSize(GRAPH_NODE_LIST ); + SimpleLogger().Write(logDEBUG) << "GRAPH_EDGE_LIST " << ": " << GetBlockSize(GRAPH_EDGE_LIST ); + SimpleLogger().Write(logDEBUG) << "COORDINATE_LIST " << ": " << GetBlockSize(COORDINATE_LIST ); + SimpleLogger().Write(logDEBUG) << "TURN_INSTRUCTION " << ": " << GetBlockSize(TURN_INSTRUCTION ); + SimpleLogger().Write(logDEBUG) << "R_SEARCH_TREE " << ": " << GetBlockSize(R_SEARCH_TREE ); + SimpleLogger().Write(logDEBUG) << "GEOMETRIES_INDEX " << ": " << GetBlockSize(GEOMETRIES_INDEX ); + SimpleLogger().Write(logDEBUG) << "GEOMETRIES_LIST " << ": " << GetBlockSize(GEOMETRIES_LIST ); + SimpleLogger().Write(logDEBUG) << "GEOMETRIES_INDICATORS" << ": " << GetBlockSize(GEOMETRIES_INDICATORS); + SimpleLogger().Write(logDEBUG) << "HSGR_CHECKSUM " << ": " << GetBlockSize(HSGR_CHECKSUM ); + SimpleLogger().Write(logDEBUG) << "TIMESTAMP " << ": " << GetBlockSize(TIMESTAMP ); + SimpleLogger().Write(logDEBUG) << "FILE_INDEX_PATH " << ": " << GetBlockSize(FILE_INDEX_PATH ); + } + + template + inline void SetBlockSize(BlockID bid, uint64_t entries) + { + num_entries[bid] = entries; + entry_size[bid] = sizeof(T); + } + + inline uint64_t GetBlockSize(BlockID bid) const + { + // special encoding + if (bid == GEOMETRIES_INDICATORS) + { + return (num_entries[GEOMETRIES_INDICATORS]/32 + 1) * entry_size[GEOMETRIES_INDICATORS]; + } + + return num_entries[bid] * entry_size[bid]; + } + + inline uint64_t GetSizeOfLayout() const + { + return GetBlockOffset(NUM_BLOCKS) + NUM_BLOCKS*2*sizeof(CANARY); + } + + inline uint64_t GetBlockOffset(BlockID bid) const + { + uint64_t result = sizeof(CANARY); + for (unsigned i = 0; i < bid; i++) + { + result += GetBlockSize((BlockID) i) + 2*sizeof(CANARY); + } return result; } - uint64_t GetGeometryListOffset() const - { - uint64_t result = - (name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) + - (name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) + - (graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) + - (graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) + - (timestamp_length * sizeof(char)) + - (coordinate_list_size * sizeof(FixedPointCoordinate)) + - (turn_instruction_list_size * sizeof(TurnInstructionsClass)) + - (r_search_tree_size * sizeof(RTreeNode)) + - (geometries_indicators / 32 + 1) * sizeof(unsigned) + - (geometries_index_list_size * sizeof(unsigned)); - return result; - } - uint64_t GetChecksumOffset() const - { - uint64_t result = - (name_index_list_size * sizeof(unsigned)) + (name_char_list_size * sizeof(char)) + - (name_id_list_size * sizeof(unsigned)) + (via_node_list_size * sizeof(NodeID)) + - (graph_node_list_size * sizeof(QueryGraph::NodeArrayEntry)) + - (graph_edge_list_size * sizeof(QueryGraph::EdgeArrayEntry)) + - (timestamp_length * sizeof(char)) + - (coordinate_list_size * sizeof(FixedPointCoordinate)) + - (turn_instruction_list_size * sizeof(TurnInstructionsClass)) + - (r_search_tree_size * sizeof(RTreeNode)) + - (geometries_indicators / 32 + 1) * sizeof(unsigned) + - (geometries_index_list_size * sizeof(unsigned)) + - (geometries_list_size * sizeof(unsigned)); - return result; + template + inline T* GetBlockPtr(char* shared_memory, BlockID bid) + { + T* ptr = (T*)(shared_memory + GetBlockOffset(bid)); + if (WRITE_CANARY) + { + char* start_canary_ptr = shared_memory + GetBlockOffset(bid) - sizeof(CANARY); + char* end_canary_ptr = shared_memory + GetBlockOffset(bid) + GetBlockSize(bid); + std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr); + std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr); + } + else + { + char* start_canary_ptr = shared_memory + GetBlockOffset(bid) - sizeof(CANARY); + char* end_canary_ptr = shared_memory + GetBlockOffset(bid) + GetBlockSize(bid); + bool start_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), start_canary_ptr); + bool end_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), end_canary_ptr); + if (!start_canary_alive) + { + throw OSRMException("Start canary of block corrupted."); + } + if (!end_canary_alive) + { + throw OSRMException("End canary of block corrupted."); + } + } + + return ptr; } }; diff --git a/datastore.cpp b/datastore.cpp index dfd6ce95c8c..3916ce56917 100644 --- a/datastore.cpp +++ b/datastore.cpp @@ -26,7 +26,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "DataStructures/OriginalEdgeData.h" +#include "DataStructures/RangeTable.h" +#include "DataStructures/QueryEdge.h" #include "DataStructures/SharedMemoryFactory.h" +#include "DataStructures/SharedMemoryVectorWrapper.h" +#include "DataStructures/StaticGraph.h" +#include "DataStructures/StaticRTree.h" +#include "DataStructures/TurnInstructions.h" +#include "Server/DataStructures/BaseDataFacade.h" #include "Server/DataStructures/SharedDataType.h" #include "Server/DataStructures/SharedBarriers.h" #include "Util/BoostFileSystemFix.h" @@ -35,6 +42,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "Util/FingerPrint.h" #include "typedefs.h" +#include + +typedef BaseDataFacade::RTreeLeaf RTreeLeaf; +typedef StaticRTree::vector, true>::TreeNode RTreeNode; +typedef StaticGraph QueryGraph; + #ifdef __linux__ #include #endif @@ -161,7 +174,7 @@ int main(const int argc, const char *argv[]) BOOST_ASSERT(!paths_iterator->second.empty()); const boost::filesystem::path index_file_path_absolute = boost::filesystem::portable_canonical(paths_iterator->second); - const std::string &file_index_file_name = index_file_path_absolute.string(); + const std::string &file_index_path = index_file_path_absolute.string(); paths_iterator = server_paths.find("nodesdata"); BOOST_ASSERT(server_paths.end() != paths_iterator); BOOST_ASSERT(!paths_iterator->second.empty()); @@ -201,37 +214,34 @@ int main(const int argc, const char *argv[]) SharedDataLayout *shared_layout_ptr = static_cast(layout_memory->Ptr()); shared_layout_ptr = new (layout_memory->Ptr()) SharedDataLayout(); - std::copy(file_index_file_name.begin(), - (file_index_file_name.length() <= 1024 ? file_index_file_name.end() - : file_index_file_name.begin() + 1023), - shared_layout_ptr->ram_index_file_name); - // add zero termination - unsigned end_of_string_index = std::min((std::size_t)1023, file_index_file_name.length()); - shared_layout_ptr->ram_index_file_name[end_of_string_index] = '\0'; + shared_layout_ptr->SetBlockSize(SharedDataLayout::FILE_INDEX_PATH, file_index_path.length() + 1); // collect number of elements to store in shared memory object SimpleLogger().Write() << "load names from: " << names_data_path; // number of entries in name index boost::filesystem::ifstream name_stream(names_data_path, std::ios::binary); - unsigned name_index_size = 0; - name_stream.read((char *)&name_index_size, sizeof(unsigned)); - shared_layout_ptr->name_index_list_size = name_index_size; - SimpleLogger().Write() << "size: " << name_index_size; - BOOST_ASSERT_MSG(0 != shared_layout_ptr->name_index_list_size, "name file broken"); + unsigned name_blocks = 0; + name_stream.read((char *)&name_blocks, sizeof(unsigned)); + shared_layout_ptr->SetBlockSize(SharedDataLayout::NAME_OFFSETS, name_blocks); + shared_layout_ptr->SetBlockSize::BlockT>(SharedDataLayout::NAME_BLOCKS, name_blocks); + SimpleLogger().Write() << "name offsets size: " << name_blocks; + BOOST_ASSERT_MSG(0 != name_blocks, "name file broken"); unsigned number_of_chars = 0; name_stream.read((char *)&number_of_chars, sizeof(unsigned)); - shared_layout_ptr->name_char_list_size = number_of_chars; + shared_layout_ptr->SetBlockSize(SharedDataLayout::NAME_CHAR_LIST, number_of_chars); // Loading information for original edges boost::filesystem::ifstream edges_input_stream(edges_data_path, std::ios::binary); unsigned number_of_original_edges = 0; edges_input_stream.read((char *)&number_of_original_edges, sizeof(unsigned)); - shared_layout_ptr->via_node_list_size = number_of_original_edges; - shared_layout_ptr->name_id_list_size = number_of_original_edges; - shared_layout_ptr->turn_instruction_list_size = number_of_original_edges; - shared_layout_ptr->geometries_indicators = number_of_original_edges; + // note: settings this all to the same size is correct, we extract them from the same struct + shared_layout_ptr->SetBlockSize(SharedDataLayout::VIA_NODE_LIST, number_of_original_edges); + shared_layout_ptr->SetBlockSize(SharedDataLayout::NAME_ID_LIST, number_of_original_edges); + shared_layout_ptr->SetBlockSize(SharedDataLayout::TURN_INSTRUCTION, number_of_original_edges); + // note: there are 32 geometry indicators in one unsigned block + shared_layout_ptr->SetBlockSize(SharedDataLayout::GEOMETRIES_INDICATORS, number_of_original_edges); boost::filesystem::ifstream hsgr_input_stream(hsgr_path, std::ios::binary); @@ -250,26 +260,26 @@ int main(const int argc, const char *argv[]) // load checksum unsigned checksum = 0; hsgr_input_stream.read((char *)&checksum, sizeof(unsigned)); - shared_layout_ptr->checksum = checksum; + shared_layout_ptr->SetBlockSize(SharedDataLayout::HSGR_CHECKSUM, 1); // load graph node size unsigned number_of_graph_nodes = 0; hsgr_input_stream.read((char *)&number_of_graph_nodes, sizeof(unsigned)); BOOST_ASSERT_MSG((0 != number_of_graph_nodes), "number of nodes is zero"); - shared_layout_ptr->graph_node_list_size = number_of_graph_nodes; + shared_layout_ptr->SetBlockSize(SharedDataLayout::GRAPH_NODE_LIST, number_of_graph_nodes); // load graph edge size unsigned number_of_graph_edges = 0; hsgr_input_stream.read((char *)&number_of_graph_edges, sizeof(unsigned)); BOOST_ASSERT_MSG(0 != number_of_graph_edges, "number of graph edges is zero"); - shared_layout_ptr->graph_edge_list_size = number_of_graph_edges; + shared_layout_ptr->SetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST, number_of_graph_edges); // load rsearch tree size boost::filesystem::ifstream tree_node_file(ram_index_path, std::ios::binary); uint32_t tree_size = 0; tree_node_file.read((char *)&tree_size, sizeof(uint32_t)); - shared_layout_ptr->r_search_tree_size = tree_size; + shared_layout_ptr->SetBlockSize(SharedDataLayout::R_SEARCH_TREE, tree_size); // load timestamp size std::string m_timestamp; @@ -295,13 +305,13 @@ int main(const int argc, const char *argv[]) { m_timestamp.resize(25); } - shared_layout_ptr->timestamp_length = m_timestamp.length(); + shared_layout_ptr->SetBlockSize(SharedDataLayout::TIMESTAMP, m_timestamp.length()); // load coordinate size boost::filesystem::ifstream nodes_input_stream(nodes_data_path, std::ios::binary); unsigned coordinate_list_size = 0; nodes_input_stream.read((char *)&coordinate_list_size, sizeof(unsigned)); - shared_layout_ptr->coordinate_list_size = coordinate_list_size; + shared_layout_ptr->SetBlockSize(SharedDataLayout::COORDINATE_LIST, coordinate_list_size); // load geometries sizes std::ifstream geometry_input_stream(geometries_data_path.string().c_str(), std::ios::binary); @@ -309,11 +319,11 @@ int main(const int argc, const char *argv[]) unsigned number_of_compressed_geometries = 0; geometry_input_stream.read((char *)&number_of_geometries_indices, sizeof(unsigned)); - shared_layout_ptr->geometries_index_list_size = number_of_geometries_indices; + shared_layout_ptr->SetBlockSize(SharedDataLayout::GEOMETRIES_INDEX, number_of_geometries_indices); boost::iostreams::seek( geometry_input_stream, number_of_geometries_indices * sizeof(unsigned), BOOST_IOS::cur); geometry_input_stream.read((char *)&number_of_compressed_geometries, sizeof(unsigned)); - shared_layout_ptr->geometries_list_size = number_of_compressed_geometries; + shared_layout_ptr->SetBlockSize(SharedDataLayout::GEOMETRIES_LIST, number_of_compressed_geometries); // allocate shared memory block SimpleLogger().Write() << "allocating shared memory of " @@ -323,35 +333,70 @@ int main(const int argc, const char *argv[]) char *shared_memory_ptr = static_cast(shared_memory->Ptr()); // read actual data into shared memory object // + + // hsgr checksum + unsigned* checksum_ptr = shared_layout_ptr->GetBlockPtr(shared_memory_ptr, + SharedDataLayout::HSGR_CHECKSUM); + *checksum_ptr = checksum; + + // ram index file name + char* file_index_path_ptr = shared_layout_ptr->GetBlockPtr(shared_memory_ptr, + SharedDataLayout::FILE_INDEX_PATH); + // make sure we have 0 ending + std::fill(file_index_path_ptr, + file_index_path_ptr + + shared_layout_ptr->GetBlockSize(SharedDataLayout::FILE_INDEX_PATH), + 0); + std::copy(file_index_path.begin(), file_index_path.end(), file_index_path_ptr); + // Loading street names - unsigned *name_index_ptr = - (unsigned *)(shared_memory_ptr + shared_layout_ptr->GetNameIndexOffset()); - if (shared_layout_ptr->name_index_list_size > 0) + unsigned *name_offsets_ptr = + shared_layout_ptr->GetBlockPtr(shared_memory_ptr, + SharedDataLayout::NAME_OFFSETS); + if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_OFFSETS) > 0) { - name_stream.read((char *)name_index_ptr, - shared_layout_ptr->name_index_list_size * sizeof(unsigned)); + name_stream.read((char *)name_offsets_ptr, + shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_OFFSETS)); } - char *name_char_ptr = shared_memory_ptr + shared_layout_ptr->GetNameListOffset(); - if (shared_layout_ptr->name_char_list_size > 0) + unsigned *name_blocks_ptr = + shared_layout_ptr->GetBlockPtr(shared_memory_ptr, + SharedDataLayout::NAME_BLOCKS); + if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_BLOCKS) > 0) { - name_stream.read(name_char_ptr, shared_layout_ptr->name_char_list_size * sizeof(char)); + name_stream.read((char *)name_blocks_ptr, + shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_BLOCKS)); } + + char *name_char_ptr = + shared_layout_ptr->GetBlockPtr(shared_memory_ptr, + SharedDataLayout::NAME_CHAR_LIST); + unsigned temp_length; + name_stream.read((char*) &temp_length, sizeof(unsigned)); + + BOOST_ASSERT_MSG(temp_length == shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST), + "Name file corrupted!"); + + if (shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST) > 0) + { + name_stream.read(name_char_ptr, shared_layout_ptr->GetBlockSize(SharedDataLayout::NAME_CHAR_LIST)); + } + name_stream.close(); // load original edge information - NodeID *via_node_ptr = - (NodeID *)(shared_memory_ptr + shared_layout_ptr->GetViaNodeListOffset()); + NodeID *via_node_ptr = shared_layout_ptr->GetBlockPtr(shared_memory_ptr, + SharedDataLayout::VIA_NODE_LIST); - unsigned *name_id_ptr = - (unsigned *)(shared_memory_ptr + shared_layout_ptr->GetNameIDListOffset()); + unsigned *name_id_ptr = shared_layout_ptr->GetBlockPtr(shared_memory_ptr, + SharedDataLayout::NAME_ID_LIST); TurnInstruction *turn_instructions_ptr = - (TurnInstruction *)(shared_memory_ptr + - shared_layout_ptr->GetTurnInstructionListOffset()); + shared_layout_ptr->GetBlockPtr(shared_memory_ptr, + SharedDataLayout::TURN_INSTRUCTION); - unsigned *geometries_indicator_ptr = - (unsigned *)(shared_memory_ptr + shared_layout_ptr->GetGeometriesIndicatorOffset()); + unsigned *geometries_indicator_ptr = shared_layout_ptr->GetBlockPtr(shared_memory_ptr, + SharedDataLayout::GEOMETRIES_INDICATORS); OriginalEdgeData current_edge_data; for (unsigned i = 0; i < number_of_original_edges; ++i) @@ -382,33 +427,32 @@ int main(const int argc, const char *argv[]) // load compressed geometry unsigned temporary_value; unsigned *geometries_index_ptr = - (unsigned *)(shared_memory_ptr + shared_layout_ptr->GetGeometriesIndexListOffset()); + shared_layout_ptr->GetBlockPtr(shared_memory_ptr, SharedDataLayout::GEOMETRIES_INDEX); geometry_input_stream.seekg(0, geometry_input_stream.beg); geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned)); - BOOST_ASSERT(temporary_value == shared_layout_ptr->geometries_index_list_size); - if (shared_layout_ptr->geometries_index_list_size > 0) + BOOST_ASSERT(temporary_value == shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_INDEX]); + + if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_INDEX) > 0) { geometry_input_stream.read((char *)geometries_index_ptr, - shared_layout_ptr->geometries_index_list_size * - sizeof(unsigned)); + shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_INDEX)); } - unsigned *geometries_list_ptr = - (unsigned *)(shared_memory_ptr + shared_layout_ptr->GetGeometryListOffset()); + shared_layout_ptr->GetBlockPtr(shared_memory_ptr, SharedDataLayout::GEOMETRIES_LIST); geometry_input_stream.read((char *)&temporary_value, sizeof(unsigned)); - BOOST_ASSERT(temporary_value == shared_layout_ptr->geometries_list_size); + BOOST_ASSERT(temporary_value == shared_layout_ptr->num_entries[SharedDataLayout::GEOMETRIES_LIST]); - if (shared_layout_ptr->geometries_list_size > 0) + if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_LIST) > 0) { geometry_input_stream.read((char *)geometries_list_ptr, - shared_layout_ptr->geometries_list_size * sizeof(unsigned)); + shared_layout_ptr->GetBlockSize(SharedDataLayout::GEOMETRIES_LIST)); } // Loading list of coordinates FixedPointCoordinate *coordinates_ptr = - (FixedPointCoordinate *)(shared_memory_ptr + - shared_layout_ptr->GetCoordinateListOffset()); + shared_layout_ptr->GetBlockPtr(shared_memory_ptr, + SharedDataLayout::COORDINATE_LIST); NodeInfo current_node; for (unsigned i = 0; i < coordinate_list_size; ++i) @@ -419,40 +463,37 @@ int main(const int argc, const char *argv[]) nodes_input_stream.close(); // store timestamp - char *timestamp_ptr = - static_cast(shared_memory_ptr + shared_layout_ptr->GetTimeStampOffset()); + char *timestamp_ptr = shared_layout_ptr->GetBlockPtr(shared_memory_ptr, SharedDataLayout::TIMESTAMP); std::copy(m_timestamp.c_str(), m_timestamp.c_str() + m_timestamp.length(), timestamp_ptr); // store search tree portion of rtree - char *rtree_ptr = - static_cast(shared_memory_ptr + shared_layout_ptr->GetRSearchTreeOffset()); + char *rtree_ptr = shared_layout_ptr->GetBlockPtr(shared_memory_ptr, SharedDataLayout::R_SEARCH_TREE); + if (tree_size > 0) { - tree_node_file.read(rtree_ptr, sizeof(RTreeNode) * tree_size); + tree_node_file.read(rtree_ptr, sizeof(RTreeNode) * tree_size); } tree_node_file.close(); // load the nodes of the search graph QueryGraph::NodeArrayEntry *graph_node_list_ptr = - (QueryGraph::NodeArrayEntry *)(shared_memory_ptr + - shared_layout_ptr->GetGraphNodeListOffset()); - if (shared_layout_ptr->graph_node_list_size > 0) + shared_layout_ptr->GetBlockPtr(shared_memory_ptr, + SharedDataLayout::GRAPH_NODE_LIST); + if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST) > 0) { hsgr_input_stream.read((char *)graph_node_list_ptr, - shared_layout_ptr->graph_node_list_size * - sizeof(QueryGraph::NodeArrayEntry)); + shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_NODE_LIST)); } // load the edges of the search graph QueryGraph::EdgeArrayEntry *graph_edge_list_ptr = - (QueryGraph::EdgeArrayEntry *)(shared_memory_ptr + - shared_layout_ptr->GetGraphEdgeListOffset()); - if (shared_layout_ptr->graph_edge_list_size > 0) + shared_layout_ptr->GetBlockPtr(shared_memory_ptr, + SharedDataLayout::GRAPH_EDGE_LIST); + if (shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST) > 0) { hsgr_input_stream.read((char *)graph_edge_list_ptr, - shared_layout_ptr->graph_edge_list_size * - sizeof(QueryGraph::EdgeArrayEntry)); + shared_layout_ptr->GetBlockSize(SharedDataLayout::GRAPH_EDGE_LIST)); } hsgr_input_stream.close();