[RocksDB] Add stacktrace signal handler

Summary:
This diff provides the ability to print out a stacktrace when the process receives certain signals.
Currently, we enable this for the following signals (program error related):
SIGILL SIGSEGV SIGBUS SIGABRT
Application simply #include "util/stack_trace.h" and call leveldb::InstallStackTraceHandler() during initialization, if signal handler is needed. It's not done automatically when openning db, because it's the application(process)'s responsibility to install signal handler and some applications might already have their own (like fbcode).

Sample output:
Received signal 11 (Segmentation fault)
#0  0x408ff0 ./signal_test() [0x408ff0] /home/haobo/rocksdb/util/signal_test.cc:4
facebook#1  0x40827d ./signal_test() [0x40827d] /home/haobo/rocksdb/util/signal_test.cc:24
facebook#2  0x7f8bb183172e /usr/local/fbcode/gcc-4.7.1-glibc-2.14.1/lib/libc.so.6(__libc_start_main+0x10e) [0x7f8bb183172e] ??:0
facebook#3  0x408ebc ./signal_test() [0x408ebc] /home/engshare/third-party/src/glibc/glibc-2.14.1/glibc-2.14.1/csu/../sysdeps/x86_64/elf/start.S:113
Segmentation fault (core dumped)

For each frame, we print the raw pointer, the symbol provided by backtrace_symbols (still not good enough), and the source file/line. Note that address translation is done by directly shell out to addr2line. ??:0 means addr2line fails to do the translation. Hacky, but I think it's good for now.

Test Plan: signal_test.cc

Reviewers: dhruba, MarkCallaghan

Reviewed By: dhruba

CC: leveldb

Differential Revision: https://reviews.facebook.net/D10173

Author: haoboxu

Makefile

@@ -68,7 +68,7 @@ TOOLS = \
         ldb \
 	db_repl_stress
 
-PROGRAMS = db_bench $(TESTS) $(TOOLS)
+PROGRAMS = db_bench signal_test $(TESTS) $(TOOLS)
 BENCHMARKS = db_bench_sqlite3 db_bench_tree_db
 
 LIBRARY = librocksdb.a
@@ -155,6 +155,9 @@ db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL)
 db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL)
 	$(CXX) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) -lkyotocabinet
 
+signal_test: util/signal_test.o $(LIBOBJECTS)
+	$(CXX) util/signal_test.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+
 arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
 
@@ -228,7 +231,7 @@ memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHAR
 rocksdb_shell: tools/shell/ShellContext.o tools/shell/ShellState.o tools/shell/LeveldbShell.o tools/shell/DBClientProxy.o tools/shell/ShellContext.h tools/shell/ShellState.h tools/shell/DBClientProxy.h $(LIBOBJECTS)
 	$(CXX) tools/shell/ShellContext.o tools/shell/ShellState.o tools/shell/LeveldbShell.o tools/shell/DBClientProxy.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
 
-DBClientProxy_test: tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY) 
+DBClientProxy_test: tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY)
 	$(CXX) tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY) $(EXEC_LDFLAGS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS)
 
 filelock_test: util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS)

build_detect_platform

@@ -64,14 +64,17 @@ PLATFORM_SHARED_LDFLAGS="${EXEC_LDFLAGS_SHARED} -shared -Wl,-soname -Wl,"
 PLATFORM_SHARED_CFLAGS="-fPIC"
 PLATFORM_SHARED_VERSIONED=true
 
+# generic port files (working on all platform by #ifdef) go directly in /port
+GENERIC_PORT_FILES=`find port -name '*.cc' | tr "\n" " "`
+
 # On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
 case "$TARGET_OS" in
     Darwin)
         PLATFORM=OS_MACOSX
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -DOS_MACOSX"
         PLATFORM_SHARED_EXT=dylib
         PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
-        PORT_FILE=port/port_posix.cc
+        # PORT_FILES=port/darwin/darwin_specific.cc
         ;;
     Linux)
         PLATFORM=OS_LINUX
@@ -80,43 +83,43 @@ case "$TARGET_OS" in
             COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
         fi
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
-        PORT_FILE=port/port_posix.cc
+        # PORT_FILES=port/linux/linux_specific.cc
         ;;
     SunOS)
         PLATFORM=OS_SOLARIS
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS"
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
-        PORT_FILE=port/port_posix.cc
+        # PORT_FILES=port/sunos/sunos_specific.cc
         ;;
     FreeBSD)
         PLATFORM=OS_FREEBSD
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
-        PORT_FILE=port/port_posix.cc
+        # PORT_FILES=port/freebsd/freebsd_specific.cc
         ;;
     NetBSD)
         PLATFORM=OS_NETBSD
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD"
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lgcc_s"
-        PORT_FILE=port/port_posix.cc
+        # PORT_FILES=port/netbsd/netbsd_specific.cc
         ;;
     OpenBSD)
         PLATFORM=OS_OPENBSD
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD"
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread"
-        PORT_FILE=port/port_posix.cc
+        # PORT_FILES=port/openbsd/openbsd_specific.cc
         ;;
     DragonFly)
         PLATFORM=OS_DRAGONFLYBSD
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_DRAGONFLYBSD"
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
-        PORT_FILE=port/port_posix.cc
+        # PORT_FILES=port/dragonfly/dragonfly_specific.cc
         ;;
     OS_ANDROID_CROSSCOMPILE)
         PLATFORM=OS_ANDROID
 	COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
 	PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS "  # All pthread features are in the Android C library
-        PORT_FILE=port/port_posix.cc
+        # PORT_FILES=port/android/android.cc
         CROSS_COMPILE=true
         ;;
     *)
@@ -149,7 +152,7 @@ set +f # re-enable globbing
 
 # The sources consist of the portable files, plus the platform-specific port
 # file.
-echo "SOURCES=$PORTABLE_FILES $PORT_FILE" >> $OUTPUT
+echo "SOURCES=$PORTABLE_FILES $GENERIC_PORT_FILES $PORT_FILES" >> $OUTPUT
 echo "SOURCESCPP=$PORTABLE_CPP" >> $OUTPUT
 echo "MEMENV_SOURCES=helpers/memenv/memenv.cc" >> $OUTPUT
 

db/db_bench.cc

@@ -19,6 +19,7 @@
 #include "util/histogram.h"
 #include "util/mutexlock.h"
 #include "util/random.h"
+#include "util/stack_trace.h"
 #include "util/testutil.h"
 #include "hdfs/env_hdfs.h"
 
@@ -1665,6 +1666,8 @@ unique_ptr<char []> GenerateKeyFromInt(int v)
 }  // namespace leveldb
 
 int main(int argc, char** argv) {
+  leveldb::InstallStackTraceHandler();
+
   FLAGS_write_buffer_size = leveldb::Options().write_buffer_size;
   FLAGS_max_write_buffer_number = leveldb::Options().max_write_buffer_number;
   FLAGS_open_files = leveldb::Options().max_open_files;

port/stack_trace.cc

@@ -0,0 +1,94 @@
+#include "util/stack_trace.h"
+
+#ifdef OS_LINUX
+
+#include <execinfo.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+namespace leveldb {
+
+static const char* GetExecutableName()
+{
+  static char name[1024];
+
+  char link[1024];
+  snprintf(link, sizeof(link), "/proc/%d/exe", getpid());
+  auto read = readlink(link, name, sizeof(name));
+  if (-1 == read) {
+    return nullptr;
+  } else {
+    name[read] = 0;
+    return name;
+  }
+}
+
+static void StackTraceHandler(int sig) {
+  // reset to default handler
+  signal(sig, SIG_DFL);
+
+  printf("Received signal %d (%s)\n", sig, strsignal(sig));
+
+  const int kMaxFrames = 100;
+  void *frames[kMaxFrames];
+
+  auto num_frames = backtrace(frames, kMaxFrames);
+  auto symbols = backtrace_symbols(frames, num_frames);
+
+  auto executable = GetExecutableName();
+
+  const int kSkip = 2; // skip the top two signal handler related frames
+
+  for (int i = kSkip; i < num_frames; ++i)
+  {
+    printf("#%-2d %p ", i - kSkip, frames[i]);
+    if (symbols) {
+      printf("%s ", symbols[i]);
+    }
+    if (executable) {
+      // out source to addr2line, for the address translation
+      const int kLineMax = 256;
+      char cmd[kLineMax];
+      sprintf(cmd,"addr2line %p -e %s 2>&1", frames[i] , executable);
+      auto f = popen(cmd, "r");
+      if (f) {
+        char line[kLineMax];
+        while (fgets(line, sizeof(line), f)) {
+          printf("%s", line);
+        }
+        pclose(f);
+      } else {
+        printf("\n");
+      }
+    } else {
+      printf("\n");
+    }
+  }
+
+  // re-signal to default handler (so we still get core dump if needed...)
+  raise(sig);
+}
+
+void InstallStackTraceHandler() {
+  // just use the plain old signal as it's simple and sufficient
+  // for this use case
+  signal(SIGILL, StackTraceHandler);
+  signal(SIGSEGV, StackTraceHandler);
+  signal(SIGBUS, StackTraceHandler);
+  signal(SIGABRT, StackTraceHandler);
+}
+
+}   // namespace leveldb
+
+#else // no-op for non-linux system for now
+
+namespace leveldb {
+
+void InstallStackTraceHandler() {}
+
+}
+
+#endif // OS_LINUX

util/stack_trace.h

@@ -0,0 +1,13 @@
+#ifndef STACK_TRACE_H
+#define STACK_TRACE_H
+
+namespace leveldb {
+
+// Install a signal handler to print callstack on the following signals:
+// SIGILL SIGSEGV SIGBUS SIGABRT
+// Currently supports linux only. No-op otherwise.
+void InstallStackTraceHandler();
+
+}   // namespace leveldb
+
+#endif