v1.1

CMakeLists.txt

@@ -18,9 +18,16 @@ if(TESTS)
     set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
     FetchContent_MakeAvailable(googletest)
     enable_testing()
+
     add_executable(binfmt_tests tests.cpp)
+    add_executable(binfmt_benchmarks benchmarks.cpp)
+
     target_compile_definitions(binfmt_tests PRIVATE -DTESTS)
+    target_compile_definitions(binfmt_benchmarks PRIVATE -DTESTS)
+
     target_link_libraries(binfmt_tests gtest_main)
+    target_link_libraries(binfmt_benchmarks gtest_main)
+
     include(GoogleTest)
     gtest_discover_tests(binfmt_tests)
 endif()

README.md

@@ -2,6 +2,27 @@
 [![CodeFactor](https://www.codefactor.io/repository/github/nbdy/binfmt/badge)](https://www.codefactor.io/repository/github/nbdy/binfmt)
 <br>
 `A header only framework for binary file formats`
+## benchmarks
+### single insert (append EntryType)
+|Number|Time (ms)|
+|------|---------|
+|1k    |1290     |
+|10k   |13260    |
+|100k  |toolong  |
+
+### vector insert (append std::vector<EntryType>)
+|Number|Time (ms)|
+|------|---------|
+|1k    |< 0 s    |
+|1M    |~ 2 s    |
+
+
+### read 
+|Number|Time (ms)|
+|------|---------|
+|1M    |< 0 s    |
+
+
 ## Minimal example
 ```c++
 #include <iostream>

benchmarks.cpp

@@ -0,0 +1,155 @@
+//
+// Created by nbdy on 02.09.21.
+//
+
+#include <chrono>
+#include <gtest/gtest.h>
+#include "binfmt.h"
+
+#define TEST_DIRECTORY "/tmp/xxxxxxxxxx__xxxxxxxxxxx__xx"
+#define TEST_BINARY_FILE "/tmp/xxxxxxx__xxxxxxxxx.bin"
+#define TEST_MAX_ENTRIES 1000000000
+
+#define TIMESTAMP std::chrono::high_resolution_clock::to_time_t(std::chrono::high_resolution_clock::now())
+
+std::string TIMEIT_NAME = "TIMEIT";
+time_t TIMEIT_START_TIMESTAMP = 0;
+time_t TIMEIT_END_TIMESTAMP = 0;
+time_t TIMEIT_DIFF = 0;
+
+#define TIMEIT_START(name) \
+TIMEIT_NAME = name;        \
+TIMEIT_START_TIMESTAMP = TIMESTAMP;
+
+#define TIMEIT_END \
+TIMEIT_END_TIMESTAMP = TIMESTAMP;
+
+#define TIMEIT_RESULT \
+TIMEIT_DIFF = TIMEIT_END_TIMESTAMP - TIMEIT_START_TIMESTAMP; \
+std::cout << TIMEIT_NAME << ": " << std::to_string(TIMEIT_DIFF) << " s" << std::endl;
+
+struct TestBinaryFileHeader : public BinaryFileHeaderBase {
+  TestBinaryFileHeader(): BinaryFileHeaderBase(0x7357, 0x8888) {}
+};
+
+struct TestBinaryEntry {
+  uint32_t uNumber;
+  int32_t  iNumber;
+  float    fNumber;
+  char     cChar;
+};
+
+typedef BinaryEntryContainer<TestBinaryEntry> TestBinaryEntryContainer;
+typedef BinaryFile<TestBinaryFileHeader, TestBinaryEntry, TestBinaryEntryContainer, TEST_MAX_ENTRIES> TestBinaryFile;
+
+
+TestBinaryFile getRandomTestFile() {
+  TestBinaryFile r(TEST_BINARY_FILE, TestBinaryFileHeader{});
+  EXPECT_TRUE(Fs::exists(r.getFilePath()));
+  return r;
+}
+
+char generateRandomChar(){
+  return 'A' + std::rand() % 24; // NOLINT(cert-msc50-cpp,cppcoreguidelines-narrowing-conversions)
+}
+
+int generateRandomInteger() {
+  return std::rand() % 10000000; // NOLINT(cert-msc50-cpp)
+}
+
+float generateRandomFloat() {
+  return static_cast<float>(std::rand()) / static_cast<float>(RAND_MAX); // NOLINT(cert-msc50-cpp)
+}
+
+TestBinaryEntryContainer generateRandomTestEntryContainer() {
+  return TestBinaryEntryContainer(TestBinaryEntry {
+      static_cast<uint32_t>(generateRandomInteger()),
+      generateRandomInteger(),
+      generateRandomFloat(),
+      generateRandomChar()
+  });
+}
+
+void cleanupTestFile(TestBinaryFile f) {
+  f.deleteFile();
+  EXPECT_FALSE(Fs::exists(f.getFilePath()));
+}
+
+TEST(BinaryFile, test1kInsert) {
+  auto t = getRandomTestFile();
+  std::vector<TestBinaryEntry> entries;
+  for(uint32_t i = 0; i < 1000; i++) {
+    entries.push_back(generateRandomTestEntryContainer().entry);
+  }
+  TIMEIT_START("1kWrite")
+  auto r = t.append(entries);
+  TIMEIT_END
+  TIMEIT_RESULT
+  EXPECT_TRUE(r.ok);
+  EXPECT_EQ(t.getFileSize(), 1000 * sizeof(TestBinaryEntryContainer) + sizeof(TestBinaryFileHeader));
+  cleanupTestFile(t);
+}
+
+TEST(BinaryFile, test10kInsert) {
+  auto t = getRandomTestFile();
+  std::vector<TestBinaryEntry> entries;
+  for(uint32_t i = 0; i < 10000; i++) {
+    entries.push_back(generateRandomTestEntryContainer().entry);
+  }
+  TIMEIT_START("10kWrite")
+  auto r = t.append(entries);
+  TIMEIT_END
+  TIMEIT_RESULT
+  EXPECT_TRUE(r.ok);
+  EXPECT_EQ(t.getFileSize(), 10000 * sizeof(TestBinaryEntryContainer) + sizeof(TestBinaryFileHeader));
+  int ec = 0;
+  std::vector<TestBinaryEntryContainer> allEntries;
+  EXPECT_TRUE(t.getAllEntries(allEntries));
+  for(const auto& entry : allEntries) {
+    EXPECT_EQ(entry.checksum, TestBinaryEntryContainer(entries[ec]).checksum);
+    ec++;
+  }
+  cleanupTestFile(t);
+}
+
+TEST(BinaryFile, test100kInsert) {
+  auto t = getRandomTestFile();
+  std::vector<TestBinaryEntry> entries;
+  for(uint32_t i = 0; i < 100000; i++) {
+    entries.push_back(generateRandomTestEntryContainer().entry);
+  }
+  TIMEIT_START("100kWrite")
+  auto r = t.append(entries);
+  TIMEIT_END
+  TIMEIT_RESULT
+  EXPECT_TRUE(r.ok);
+  EXPECT_EQ(t.getFileSize(), 100000 * sizeof(TestBinaryEntryContainer) + sizeof(TestBinaryFileHeader));
+  cleanupTestFile(t);
+}
+
+TEST(BinaryFile, test1MInsert) {
+  auto t = getRandomTestFile();
+  std::vector<TestBinaryEntry> entries;
+  uint32_t insertCount = 10000000;
+  std::cout << "Generating file of size " << std::to_string(insertCount * sizeof(TestBinaryEntryContainer) + sizeof(TestBinaryFileHeader)) << std::endl;
+  for(uint32_t i = 0; i < insertCount; i++) {
+    entries.push_back(generateRandomTestEntryContainer().entry);
+  }
+  TIMEIT_START("1MWrite")
+  auto r = t.append(entries);
+  TIMEIT_END
+  TIMEIT_RESULT
+  EXPECT_TRUE(r.ok);
+  EXPECT_EQ(t.getFileSize(), insertCount * sizeof(TestBinaryEntryContainer) + sizeof(TestBinaryFileHeader));
+  int ec = 0;
+  std::vector<TestBinaryEntryContainer> allEntries;
+  TIMEIT_START("1MRead")
+  EXPECT_TRUE(t.getAllEntries(allEntries));
+  TIMEIT_END
+  TIMEIT_RESULT
+  for(const auto& entry : allEntries) {
+    EXPECT_EQ(entry.checksum, TestBinaryEntryContainer(entries[ec]).checksum);
+    ec++;
+  }
+  //cleanupTestFile(t);
+}

binfmt.h

@@ -145,7 +145,13 @@ struct FileUtils {
     }
     if (Fs::exists(FilePath) || Create) {
       r = std::fopen(FilePath.c_str(), Create ? "wbe" : "r+be");
-      if (fseek(r, offset, SEEK_SET) != 0) {
+      auto fn = fileno(r);
+      if(lockf(fn, F_LOCK, 0) != 0) {
+        (void) CloseBinaryFile(r);
+        r = nullptr;
+      }
+      auto sr = fseek(r, offset, SEEK_SET);
+      if (sr != 0) {
         (void) CloseBinaryFile(r);
         r = nullptr;
       }
@@ -159,7 +165,9 @@ struct FileUtils {
    * @return
    */
   static bool CloseBinaryFile(FILE* fp) {
-    auto r = syncfs(fileno(fp));
+    auto fn = fileno(fp);
+    (void) lockf(fn, F_ULOCK, 0);
+    auto r = syncfs(fn);
     return fclose(fp) == 0 && r == 0;
   }
 
@@ -231,6 +239,26 @@ struct FileUtils {
     return r == 1;
   }
 
+  /*!
+   * Write data entry to file. Skips size of HeaderType in bytes.
+   * @tparam EntryType
+   * @param FilePath
+   * @param entry
+   * @return false if file does not exist, seek to offset failed or we can't write the object
+   */
+  template<typename HeaderType, typename EntryType>
+  static bool WriteDataVector(const std::string& FilePath, std::vector<EntryType> entries, uint32_t offset = 0) {
+    uint32_t o = sizeof(HeaderType) + offset * sizeof(EntryType);
+    FILE* fp = OpenBinaryFile(FilePath, false, o);
+    if (fp == nullptr) {
+      std::cout << strerror(errno) << std::endl;
+      return false;
+    }
+    auto r = fwrite(&entries[0], sizeof(EntryType), entries.size(), fp);
+    (void) CloseBinaryFile(fp);
+    return r == entries.size();
+  }
+
   /*!
    *
    * @tparam HeaderType
@@ -482,6 +510,27 @@ class BinaryFile {
     return AppendResult{rewind, ok, m_uCurrentAppendOffset};
   }
 
+  AppendResult append(std::vector<EntryType> entries) {
+    AppendResult r {
+      false, true, 0
+    };
+    bool rewind = false;
+    if(getFileSize() + sizeof(ContainerType) * entries.size() > m_uMaxFileSize) {
+      r.rewind = true;
+      auto sizeLeft = m_uMaxFileSize - getFileSize();
+      uint32_t entriesLeft = sizeLeft / sizeof(ContainerType) - 1;
+      std::vector<EntryType> tmpEntries(entries.begin(), entries.begin() + entriesLeft);
+      entries = std::vector<EntryType>(entries.begin() + entriesLeft + 1, entries.end());
+      r = append(tmpEntries);
+    }
+    std::vector<ContainerType> containers;
+    for(const auto& entry : entries) {
+      containers.push_back(ContainerType(entry));
+    }
+    r.ok = FileUtils::WriteDataVector<HeaderType, ContainerType>(m_sFilePath, containers);
+    return r;
+  }
+
   bool setEntryAt(EntryType entry, uint32_t position) {
     return FileUtils::WriteData<HeaderType, ContainerType>(m_sFilePath, ContainerType(entry), position);
   }

tests.cpp

@@ -3,11 +3,14 @@
 //
 
 #include "gtest/gtest.h"
+#include <chrono>
 #include "binfmt.h"
+#include <thread>
 
 #define TEST_DIRECTORY "/tmp/xxxxxxxxxx__xxxxxxxxxxx__xx"
 #define TEST_BINARY_FILE "/tmp/xxxxxxx__xxxxxxxxx.bin"
-#define TEST_MAX_ENTRIES 50
+#define TEST_MAX_ENTRIES 1000
+
 
 struct TestBinaryFileHeader : public BinaryFileHeaderBase {
   TestBinaryFileHeader(): BinaryFileHeaderBase(0x7357, 0x8888) {}
@@ -63,7 +66,7 @@ std::vector<TestBinaryEntryContainer> appendRandomAmountOfEntriesV(TestBinaryFil
   return r;
 }
 
-std::vector<TestBinaryEntryContainer> appendExactAmountOfEntriesV(TestBinaryFile f, int count = 42) {
+std::vector<TestBinaryEntryContainer> appendExactAmountOfEntriesV(TestBinaryFile f, uint32_t count = 42) {
   std::vector<TestBinaryEntryContainer> r;
   for(int i = 0; i < count; i++) {
     auto v = generateRandomTestEntryContainer();
@@ -286,3 +289,4 @@ TEST(BinaryFile, testRemoveEntryAt) {
   EXPECT_EQ(t0.checksum, t1.checksum);
   cleanupTestFile(t);
 }
+