-
Notifications
You must be signed in to change notification settings - Fork 3.5k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Differantial encoding for name offsets #1069
Merged
Merged
Changes from all commits
Commits
Show all changes
11 commits
Select commit
Hold shift + click to select a range
f90ce77
Use differential encoding for name offsets
TheMarex 7a7d0c0
Integrate RangeTable into server
TheMarex 807f1d7
Initial support for SharedDataFacade
TheMarex 50bf769
Constify some parts of RangeTable
TheMarex 1d62ed0
Fix off-by-one since back() gives last value inside [begin,end)
TheMarex ef60ae6
Fix edge cases in RangeTable
TheMarex 4c17aeb
Removed SSE code in RangeTable to rely on compiler optimazation
TheMarex aedcc2f
Add array inlcude
TheMarex e29b7a6
Fix some minor style issues
TheMarex 40e2d79
Fix VC2013 issues
TheMarex c009dce
Another VC2013 fix
TheMarex File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,231 @@ | ||
| #ifndef __RANGE_TABLE_H__ | ||
| #define __RANGE_TABLE_H__ | ||
|
|
||
| #include "SharedMemoryFactory.h" | ||
| #include "SharedMemoryVectorWrapper.h" | ||
|
|
||
| #include <boost/range/irange.hpp> | ||
|
|
||
| #include <fstream> | ||
| #include <vector> | ||
| #include <array> | ||
|
|
||
| /* | ||
| * These pre-declarations are needed because parsing C++ is hard | ||
| * and otherwise the compiler gets confused. | ||
| */ | ||
|
|
||
| template<unsigned BLOCK_SIZE=16, bool USE_SHARED_MEMORY = false> class RangeTable; | ||
|
|
||
| template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY> | ||
| std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table); | ||
|
|
||
| template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY> | ||
| std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table); | ||
|
|
||
| /** | ||
| * Stores adjacent ranges in a compressed format. | ||
| * | ||
| * Maximum supported length of a range is 255. | ||
| * | ||
| * Note: BLOCK_SIZE is the number of differential encodoed values. | ||
| * But each block consists of an absolute value and BLOCK_SIZE differential values. | ||
| * So the effective block size is sizeof(unsigned) + BLOCK_SIZE. | ||
| */ | ||
| template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY> | ||
| class RangeTable | ||
| { | ||
| public: | ||
|
|
||
| typedef std::array<unsigned char, BLOCK_SIZE> BlockT; | ||
| typedef typename ShM<BlockT, USE_SHARED_MEMORY>::vector BlockContainerT; | ||
| typedef typename ShM<unsigned, USE_SHARED_MEMORY>::vector OffsetContainerT; | ||
| typedef decltype(boost::irange(0u,0u)) RangeT; | ||
|
|
||
| friend std::ostream& operator<< <>(std::ostream &out, const RangeTable &table); | ||
| friend std::istream& operator>> <>(std::istream &in, RangeTable &table); | ||
|
|
||
| RangeTable() {} | ||
|
|
||
| // for loading from shared memory | ||
| explicit RangeTable(OffsetContainerT& external_offsets, BlockContainerT& external_blocks, const unsigned sum_lengths) | ||
| : sum_lengths(sum_lengths) | ||
| { | ||
| block_offsets.swap(external_offsets); | ||
| diff_blocks.swap(external_blocks); | ||
| } | ||
|
|
||
| // construct table from length vector | ||
| explicit RangeTable(const std::vector<unsigned>& lengths) | ||
| { | ||
| const unsigned number_of_blocks = [&lengths]() { | ||
| unsigned num = (lengths.size() + 1) / (BLOCK_SIZE + 1); | ||
| if ((lengths.size() + 1) % (BLOCK_SIZE + 1) != 0) | ||
| { | ||
| num += 1; | ||
| } | ||
| return num; | ||
| }(); | ||
|
|
||
| block_offsets.reserve(number_of_blocks); | ||
| diff_blocks.reserve(number_of_blocks); | ||
|
|
||
| unsigned last_length = 0; | ||
| unsigned lengths_prefix_sum = 0; | ||
| unsigned block_idx = 0; | ||
| unsigned block_counter = 0; | ||
| BlockT block; | ||
| unsigned block_sum = 0; | ||
| for (const unsigned l : lengths) | ||
| { | ||
| // first entry of a block: encode absolute offset | ||
| if (block_idx == 0) | ||
| { | ||
| block_offsets.push_back(lengths_prefix_sum); | ||
| block_sum = 0; | ||
| } | ||
| else | ||
| { | ||
| block[block_idx - 1] = last_length; | ||
| block_sum += last_length; | ||
| } | ||
|
|
||
| BOOST_ASSERT((block_idx == 0 && block_offsets[block_counter] == lengths_prefix_sum) | ||
| || lengths_prefix_sum == (block_offsets[block_counter]+block_sum)); | ||
|
|
||
| // block is full | ||
| if (BLOCK_SIZE == block_idx) | ||
| { | ||
| diff_blocks.push_back(block); | ||
| block_counter++; | ||
| } | ||
|
|
||
| // we can only store strings with length 255 | ||
| BOOST_ASSERT(l <= 255); | ||
|
|
||
| lengths_prefix_sum += l; | ||
| last_length = l; | ||
|
|
||
| block_idx = (block_idx + 1) % (BLOCK_SIZE + 1); | ||
| } | ||
|
|
||
| // Last block can't be finished because we didn't add the sentinel | ||
| BOOST_ASSERT (block_counter == (number_of_blocks - 1)); | ||
|
|
||
| // one block missing: starts with guard value | ||
| if (0 == block_idx) | ||
| { | ||
| // the last value is used as sentinel | ||
| block_offsets.push_back(lengths_prefix_sum); | ||
| block_idx = (block_idx + 1) % BLOCK_SIZE; | ||
| } | ||
|
|
||
| while (0 != block_idx) | ||
| { | ||
| block[block_idx - 1] = last_length; | ||
| last_length = 0; | ||
| block_idx = (block_idx + 1) % (BLOCK_SIZE + 1); | ||
| } | ||
| diff_blocks.push_back(block); | ||
|
|
||
| BOOST_ASSERT(diff_blocks.size() == number_of_blocks && block_offsets.size() == number_of_blocks); | ||
|
|
||
| sum_lengths = lengths_prefix_sum; | ||
| } | ||
|
|
||
| inline RangeT GetRange(const unsigned id) const | ||
| { | ||
| BOOST_ASSERT(id < block_offsets.size() + diff_blocks.size() * BLOCK_SIZE); | ||
| // internal_idx 0 is implicitly stored in block_offsets[block_idx] | ||
| const unsigned internal_idx = id % (BLOCK_SIZE + 1); | ||
| const unsigned block_idx = id / (BLOCK_SIZE + 1); | ||
|
|
||
| BOOST_ASSERT(block_idx < diff_blocks.size()); | ||
|
|
||
| unsigned begin_idx = 0; | ||
| unsigned end_idx = 0; | ||
| begin_idx = block_offsets[block_idx]; | ||
| const BlockT& block = diff_blocks[block_idx]; | ||
| if (internal_idx > 0) | ||
| { | ||
| begin_idx += PrefixSumAtIndex(internal_idx - 1, block); | ||
| } | ||
|
|
||
| // next index inside current block | ||
| if (internal_idx < BLOCK_SIZE) | ||
| { | ||
| // note internal_idx - 1 is the *current* index for uint8_blocks | ||
| end_idx = begin_idx + block[internal_idx]; | ||
| } | ||
| else | ||
| { | ||
| BOOST_ASSERT(block_idx < block_offsets.size() - 1); | ||
| end_idx = block_offsets[block_idx + 1]; | ||
| } | ||
|
|
||
| BOOST_ASSERT(begin_idx < sum_lengths && end_idx <= sum_lengths); | ||
| BOOST_ASSERT(begin_idx <= end_idx); | ||
|
|
||
| return boost::irange(begin_idx, end_idx); | ||
| } | ||
| private: | ||
|
|
||
| inline unsigned PrefixSumAtIndex(int index, const BlockT& block) const; | ||
|
|
||
| // contains offset for each differential block | ||
| OffsetContainerT block_offsets; | ||
| // blocks of differential encoded offsets, should be aligned | ||
| BlockContainerT diff_blocks; | ||
| unsigned sum_lengths; | ||
| }; | ||
|
|
||
| template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY> | ||
| unsigned RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY>::PrefixSumAtIndex(int index, const BlockT& block) const | ||
| { | ||
| // this loop looks inefficent, but a modern compiler | ||
| // will emit nice SIMD here, at least for sensible block sizes. (I checked.) | ||
| unsigned sum = 0; | ||
| for (int i = 0; i <= index; ++i) | ||
| { | ||
| sum += block[i]; | ||
| } | ||
|
|
||
| return sum; | ||
| } | ||
|
|
||
| template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY> | ||
| std::ostream& operator<<(std::ostream &out, const RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table) | ||
| { | ||
| // write number of block | ||
| const unsigned number_of_blocks = table.diff_blocks.size(); | ||
| out.write((char *) &number_of_blocks, sizeof(unsigned)); | ||
| // write total length | ||
| out.write((char *) &table.sum_lengths, sizeof(unsigned)); | ||
| // write block offsets | ||
| out.write((char *) table.block_offsets.data(), sizeof(unsigned) * table.block_offsets.size()); | ||
| // write blocks | ||
| out.write((char *) table.diff_blocks.data(), BLOCK_SIZE * table.diff_blocks.size()); | ||
|
|
||
| return out; | ||
| } | ||
|
|
||
| template<unsigned BLOCK_SIZE, bool USE_SHARED_MEMORY> | ||
| std::istream& operator>>(std::istream &in, RangeTable<BLOCK_SIZE, USE_SHARED_MEMORY> &table) | ||
| { | ||
| // read number of block | ||
| unsigned number_of_blocks; | ||
| in.read((char *) &number_of_blocks, sizeof(unsigned)); | ||
| // read total length | ||
| in.read((char *) &table.sum_lengths, sizeof(unsigned)); | ||
|
|
||
| table.block_offsets.resize(number_of_blocks); | ||
| table.diff_blocks.resize(number_of_blocks); | ||
|
|
||
| // read block offsets | ||
| in.read((char *) table.block_offsets.data(), sizeof(unsigned) * number_of_blocks); | ||
| // read blocks | ||
| in.read((char *) table.diff_blocks.data(), BLOCK_SIZE * number_of_blocks); | ||
| return in; | ||
| } | ||
|
|
||
| #endif | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We can probably omit this c'tor or delete it if not needed.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That one is actually needed, because otherwise you can't use the stream operators. (And I won't get the default constructor because I already defined constructors)