Add support for plain table format to sst_dump.

Summary:
This diff enables the command line tool `sst_dump` to work for sst files
under plain table format.  Changes include:
  * In tools/sst_dump.cc:
    - add support for plain table format
    - display prefix_extractor information when --show_properties is on
  * In table/format.cc
    - Now the table magic number of a Footer can be later initialized
      via ReadFooterFromFile().
  * In table/meta_bocks:
    - add function ReadTableMagicNumber() that reads the magic number of
      the specified file.

Minor fixes:
 - remove a duplicate #include in table/table_test.cc
 - fix a commentary typo in include/rocksdb/memtablerep.h
 - fix lint errors.

Test Plan:
Runs sst_dump with both block-based and plain-table format files with
different arguments, specifically those with --show-properties and --from.

* sample output:
  https://reviews.facebook.net/P261

Reviewers: kailiu, sdong, xjin

CC: leveldb

Differential Revision: https://reviews.facebook.net/D15903

Author: yhchiang

include/rocksdb/memtablerep.h

@@ -21,7 +21,7 @@
 // types built in:
 //  - SkipListRep: This is the default; it is backed by a skip list.
 //  - HashSkipListRep: The memtable rep that is best used for keys that are
-//  structured like "prefix:suffix" where iteration withing a prefix is
+//  structured like "prefix:suffix" where iteration within a prefix is
 //  common and iteration across different prefixes is rare. It is backed by
 //  a hash map where each bucket is a skip list.
 //  - VectorRep: This is backed by an unordered std::vector. On iteration, the
@@ -85,7 +85,7 @@ class MemTableRep {
     // Initialize an iterator over the specified collection.
     // The returned iterator is not valid.
     // explicit Iterator(const MemTableRep* collection);
-    virtual ~Iterator() { };
+    virtual ~Iterator() {}
 
     // Returns true iff the iterator is positioned at a valid node.
     virtual bool Valid() const = 0;
@@ -143,7 +143,7 @@ class MemTableRep {
 // new MemTableRep objects
 class MemTableRepFactory {
  public:
-  virtual ~MemTableRepFactory() { };
+  virtual ~MemTableRepFactory() {}
   virtual MemTableRep* CreateMemTableRep(MemTableRep::KeyComparator&,
                                          Arena*) = 0;
   virtual const char* Name() const = 0;
@@ -159,7 +159,8 @@ class MemTableRepFactory {
 //     bytes reserved for usage.
 class VectorRepFactory : public MemTableRepFactory {
   const size_t count_;
-public:
+
+ public:
   explicit VectorRepFactory(size_t count = 0) : count_(count) { }
   virtual MemTableRep* CreateMemTableRep(MemTableRep::KeyComparator&,
                                          Arena*) override;
@@ -170,9 +171,9 @@ class VectorRepFactory : public MemTableRepFactory {
 
 // This uses a skip list to store keys. It is the default.
 class SkipListFactory : public MemTableRepFactory {
-public:
- virtual MemTableRep* CreateMemTableRep(MemTableRep::KeyComparator&,
-                                        Arena*) override;
+ public:
+  virtual MemTableRep* CreateMemTableRep(MemTableRep::KeyComparator&,
+                                         Arena*) override;
   virtual const char* Name() const override {
     return "SkipListFactory";
   }
@@ -196,4 +197,4 @@ extern MemTableRepFactory* NewHashSkipListRepFactory(
 extern MemTableRepFactory* NewHashLinkListRepFactory(
     const SliceTransform* transform, size_t bucket_count = 50000);
 
-}
+}  // namespace rocksdb

table/format.cc

@@ -9,6 +9,8 @@
 
 #include "table/format.h"
 
+#include <string>
+
 #include "port/port.h"
 #include "rocksdb/env.h"
 #include "table/block.h"
@@ -43,22 +45,30 @@ void Footer::EncodeTo(std::string* dst) const {
   metaindex_handle_.EncodeTo(dst);
   index_handle_.EncodeTo(dst);
   dst->resize(2 * BlockHandle::kMaxEncodedLength);  // Padding
-  PutFixed32(dst, static_cast<uint32_t>(kTableMagicNumber & 0xffffffffu));
-  PutFixed32(dst, static_cast<uint32_t>(kTableMagicNumber >> 32));
+  PutFixed32(dst, static_cast<uint32_t>(table_magic_number() & 0xffffffffu));
+  PutFixed32(dst, static_cast<uint32_t>(table_magic_number() >> 32));
   assert(dst->size() == original_size + kEncodedLength);
 }
 
 Status Footer::DecodeFrom(Slice* input) {
   assert(input != nullptr);
   assert(input->size() >= kEncodedLength);
 
-  const char* magic_ptr = input->data() + kEncodedLength - 8;
+  const char* magic_ptr =
+      input->data() + kEncodedLength - kMagicNumberLengthByte;
   const uint32_t magic_lo = DecodeFixed32(magic_ptr);
   const uint32_t magic_hi = DecodeFixed32(magic_ptr + 4);
   const uint64_t magic = ((static_cast<uint64_t>(magic_hi) << 32) |
                           (static_cast<uint64_t>(magic_lo)));
-  if (magic != kTableMagicNumber) {
-    return Status::InvalidArgument("not an sstable (bad magic number)");
+  if (HasInitializedTableMagicNumber()) {
+    if (magic != table_magic_number()) {
+      char buffer[80];
+      snprintf(buffer, sizeof(buffer) - 1,
+               "not an sstable (bad magic number --- %lx)", magic);
+      return Status::InvalidArgument(buffer);
+    }
+  } else {
+    set_table_magic_number(magic);
   }
 
   Status result = metaindex_handle_.DecodeFrom(input);
@@ -221,7 +231,7 @@ Status UncompressBlockContents(const char* data, size_t n,
     default:
       return Status::Corruption("bad block type");
   }
-  result->compression_type = kNoCompression; // not compressed any more
+  result->compression_type = kNoCompression;  // not compressed any more
   return Status::OK();
 }
 

table/format.h

@@ -21,6 +21,9 @@ class Block;
 class RandomAccessFile;
 struct ReadOptions;
 
+// the length of the magic number in bytes.
+const int kMagicNumberLengthByte = 8;
+
 // BlockHandle is a pointer to the extent of a file that stores a data
 // block or a meta block.
 class BlockHandle {
@@ -63,12 +66,16 @@ class BlockHandle {
 // end of every table file.
 class Footer {
  public:
+  // Constructs a footer without specifying its table magic number.
+  // In such case, the table magic number of such footer should be
+  // initialized via @ReadFooterFromFile().
+  Footer() : Footer(kInvalidTableMagicNumber) {}
+
   // @table_magic_number serves two purposes:
   //  1. Identify different types of the tables.
   //  2. Help us to identify if a given file is a valid sst.
-  Footer(uint64_t table_magic_number) :
-      kTableMagicNumber(table_magic_number) {
-  }
+  explicit Footer(uint64_t table_magic_number)
+      : table_magic_number_(table_magic_number) {}
 
   // The block handle for the metaindex block of the table
   const BlockHandle& metaindex_handle() const { return metaindex_handle_; }
@@ -78,24 +85,52 @@ class Footer {
   const BlockHandle& index_handle() const {
     return index_handle_;
   }
+
   void set_index_handle(const BlockHandle& h) {
     index_handle_ = h;
   }
 
+  uint64_t table_magic_number() const { return table_magic_number_; }
+
   void EncodeTo(std::string* dst) const;
+
+  // Set the current footer based on the input slice.  If table_magic_number_
+  // is not set (i.e., HasInitializedTableMagicNumber() is true), then this
+  // function will also initialize table_magic_number_.  Otherwise, this
+  // function will verify whether the magic number specified in the input
+  // slice matches table_magic_number_ and update the current footer only
+  // when the test passes.
   Status DecodeFrom(Slice* input);
 
   // Encoded length of a Footer.  Note that the serialization of a
   // Footer will always occupy exactly this many bytes.  It consists
   // of two block handles and a magic number.
   enum {
-    kEncodedLength = 2*BlockHandle::kMaxEncodedLength + 8
+    kEncodedLength = 2 * BlockHandle::kMaxEncodedLength + 8
   };
 
+  const uint64_t kInvalidTableMagicNumber = 0;
+
  private:
+  // Set the table_magic_number only when it was not previously
+  // initialized.  Return true on success.
+  bool set_table_magic_number(uint64_t magic_number) {
+    if (HasInitializedTableMagicNumber()) {
+      table_magic_number_ = magic_number;
+      return true;
+    }
+    return false;
+  }
+
+  // return true if @table_magic_number_ is set to a value different
+  // from @kInvalidTableMagicNumber.
+  bool HasInitializedTableMagicNumber() const {
+    return (table_magic_number_ != kInvalidTableMagicNumber);
+  }
+
   BlockHandle metaindex_handle_;
   BlockHandle index_handle_;
-  const uint64_t kTableMagicNumber;
+  uint64_t table_magic_number_;
 };
 
 // Read the footer from file

table/meta_blocks.cc

@@ -2,12 +2,13 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-
 #include "table/meta_blocks.h"
 
 #include <map>
+#include <string>
 
 #include "rocksdb/table.h"
+#include "rocksdb/table_properties.h"
 #include "table/block.h"
 #include "table/format.h"
 #include "util/coding.h"
@@ -104,9 +105,8 @@ bool NotifyCollectTableCollectorsOnAdd(
     Status s = collector->Add(key, value);
     all_succeeded = all_succeeded && s.ok();
     if (!s.ok()) {
-      LogPropertiesCollectionError(
-          info_log, "Add", /* method */ collector->Name()
-      );
+      LogPropertiesCollectionError(info_log, "Add" /* method */,
+                                   collector->Name());
     }
   }
   return all_succeeded;
@@ -123,9 +123,8 @@ bool NotifyCollectTableCollectorsOnFinish(
 
     all_succeeded = all_succeeded && s.ok();
     if (!s.ok()) {
-      LogPropertiesCollectionError(
-          info_log, "Finish", /* method */ collector->Name()
-      );
+      LogPropertiesCollectionError(info_log, "Finish" /* method */,
+                                   collector->Name());
     } else {
       builder->Add(user_collected_properties);
     }
@@ -151,37 +150,29 @@ Status ReadProperties(
   BlockContents block_contents;
   ReadOptions read_options;
   read_options.verify_checksums = false;
-  Status s = ReadBlockContents(
-      file,
-      read_options,
-      handle,
-      &block_contents,
-      env,
-      false
-  );
+  Status s = ReadBlockContents(file, read_options, handle, &block_contents, env,
+                               false);
 
   if (!s.ok()) {
     return s;
   }
 
   Block properties_block(block_contents);
   std::unique_ptr<Iterator> iter(
-      properties_block.NewIterator(BytewiseComparator())
-  );
+      properties_block.NewIterator(BytewiseComparator()));
 
   // All pre-defined properties of type uint64_t
   std::unordered_map<std::string, uint64_t*> predefined_uint64_properties = {
-    { TablePropertiesNames::kDataSize, &table_properties->data_size },
-    { TablePropertiesNames::kIndexSize, &table_properties->index_size },
-    { TablePropertiesNames::kFilterSize, &table_properties->filter_size },
-    { TablePropertiesNames::kRawKeySize, &table_properties->raw_key_size },
-    { TablePropertiesNames::kRawValueSize, &table_properties->raw_value_size },
-    { TablePropertiesNames::kNumDataBlocks,
-      &table_properties->num_data_blocks },
-    { TablePropertiesNames::kNumEntries, &table_properties->num_entries },
-    { TablePropertiesNames::kFormatVersion, &table_properties->format_version },
-    { TablePropertiesNames::kFixedKeyLen, &table_properties->fixed_key_len },
-  };
+      {TablePropertiesNames::kDataSize, &table_properties->data_size},
+      {TablePropertiesNames::kIndexSize, &table_properties->index_size},
+      {TablePropertiesNames::kFilterSize, &table_properties->filter_size},
+      {TablePropertiesNames::kRawKeySize, &table_properties->raw_key_size},
+      {TablePropertiesNames::kRawValueSize, &table_properties->raw_value_size},
+      {TablePropertiesNames::kNumDataBlocks,
+       &table_properties->num_data_blocks},
+      {TablePropertiesNames::kNumEntries, &table_properties->num_entries},
+      {TablePropertiesNames::kFormatVersion, &table_properties->format_version},
+      {TablePropertiesNames::kFixedKeyLen, &table_properties->fixed_key_len}};
 
   std::string last_key;
   for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
@@ -192,10 +183,8 @@ Status ReadProperties(
 
     auto key = iter->key().ToString();
     // properties block is strictly sorted with no duplicate key.
-    assert(
-        last_key.empty() ||
-        BytewiseComparator()->Compare(key, last_key) > 0
-    );
+    assert(last_key.empty() ||
+           BytewiseComparator()->Compare(key, last_key) > 0);
     last_key = key;
 
     auto raw_val = iter->value();
@@ -218,8 +207,7 @@ Status ReadProperties(
     } else {
       // handle user-collected properties
       table_properties->user_collected_properties.insert(
-          std::make_pair(key, raw_val.ToString())
-      );
+          {key, raw_val.ToString()});
     }
   }
 
@@ -244,43 +232,54 @@ Status ReadTableProperties(
   BlockContents metaindex_contents;
   ReadOptions read_options;
   read_options.verify_checksums = false;
-  s = ReadBlockContents(
-      file,
-      read_options,
-      metaindex_handle,
-      &metaindex_contents,
-      env,
-      false
-  );
+  s = ReadBlockContents(file, read_options, metaindex_handle,
+                        &metaindex_contents, env, false);
   if (!s.ok()) {
     return s;
   }
   Block metaindex_block(metaindex_contents);
   std::unique_ptr<Iterator> meta_iter(
-      metaindex_block.NewIterator(BytewiseComparator())
-  );
+      metaindex_block.NewIterator(BytewiseComparator()));
 
   // -- Read property block
   meta_iter->Seek(kPropertiesBlock);
   TableProperties table_properties;
   if (meta_iter->Valid() &&
       meta_iter->key() == kPropertiesBlock &&
       meta_iter->status().ok()) {
-    s = ReadProperties(
-        meta_iter->value(),
-        file,
-        env,
-        info_log,
-        properties
-    );
+    s = ReadProperties(meta_iter->value(), file, env, info_log, properties);
   } else {
     s = Status::Corruption(
-        "Unable to read the property block from the plain table"
-    );
+        "Unable to read the property block from the plain table");
   }
 
   return s;
 }
 
+Status ReadTableMagicNumber(const std::string& file_path,
+                            const Options& options,
+                            const EnvOptions& env_options,
+                            uint64_t* table_magic_number) {
+  unique_ptr<RandomAccessFile> file;
+  Status s = options.env->NewRandomAccessFile(file_path, &file, env_options);
+  if (!s.ok()) {
+    return s;
+  }
+
+  uint64_t file_size;
+  options.env->GetFileSize(file_path, &file_size);
+  if (file_size < Footer::kEncodedLength) {
+    return Status::InvalidArgument("file is too short to be an sstable");
+  }
+
+  Footer footer;
+  s = ReadFooterFromFile(file.get(), file_size, &footer);
+  if (!s.ok()) {
+    return s;
+  }
+
+  *table_magic_number = footer.table_magic_number();
+  return Status::OK();
+}
 
 }  // namespace rocksdb