Update our encryption to match changes in Facebook sources

.bazelrc

@@ -0,0 +1 @@
+build --crosstool_top=@toolchain//cpp:toolchain

.gitignore

@@ -33,7 +33,6 @@ manifest_dump
 sst_dump
 blob_dump
 column_aware_encoding_exp
-util/build_version.cc
 build_tools/VALGRIND_LOGS/
 coverage/COVERAGE_REPORT
 .gdbhistory

BUILD

@@ -0,0 +1,78 @@
+config_setting(
+    name = "linux",
+    constraint_values = [
+        "@bazel_tools//platforms:linux",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "osx",
+    constraint_values = [
+        "@bazel_tools//platforms:osx",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "windows",
+    constraint_values = [
+        "@bazel_tools//platforms:windows",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "tests_enabled_debug_mode",
+    values = {
+        "compilation_mode": "dbg",
+        "define": "enable_tests=1",
+    },
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "tests_enabled_fastbuild_mode",
+    values = {
+        "compilation_mode": "fastbuild",
+        "define": "enable_tests=1",
+    },
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "rocksdb",
+    deps = [
+        "//cache",
+        "//db",
+        "//env",
+        "//include",
+        "//memtable",
+        "//monitoring",
+        "//options",
+        "//port",
+        "//table",
+        "//third_party/gtest",
+        "//third_party/lz4",
+        "//third_party/snappy",
+        "//util",
+        "//utilities",
+        "//utilities/backupable",
+        "//utilities/checkpoint",
+        "//utilities/leveldb_options",
+        "//utilities/merge_operators",
+        "//utilities/merge_operators/string_append",
+        "//utilities/options",
+        "//utilities/table_properties_collectors",
+        "//utilities/transactions",
+        "//utilities/ttl",
+        "//utilities/write_batch_with_index",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "empty_main",
+    srcs = ["empty_main.cc"],
+    visibility = ["//visibility:public"],
+)

CMakeLists.txt

@@ -174,6 +174,7 @@ else()
   endif()
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
   if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+    add_definitions(-DNDEBUG)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
     include(CheckCXXCompilerFlag)
     CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
@@ -504,7 +505,6 @@ set(SOURCES
         db/merge_helper.cc
         db/merge_operator.cc
         db/range_del_aggregator.cc
-        db/range_del_aggregator_v2.cc
         db/range_tombstone_fragmenter.cc
         db/repair.cc
         db/snapshot_impl.cc
@@ -523,6 +523,7 @@ set(SOURCES
         env/env_chroot.cc
         env/env_encryption.cc
         env/env_hdfs.cc
+        env/env_openssl.cc
         env/mock_env.cc
         memtable/alloc_tracker.cc
         memtable/hash_cuckoo_rep.cc
@@ -907,7 +908,6 @@ if(WITH_TESTS)
         db/plain_table_db_test.cc
         db/prefix_test.cc
         db/range_del_aggregator_test.cc
-        db/range_del_aggregator_v2_test.cc
         db/range_tombstone_fragmenter_test.cc
         db/repair_test.cc
         db/table_properties_collector_test.cc
@@ -919,6 +919,7 @@ if(WITH_TESTS)
         db/write_callback_test.cc
         db/write_controller_test.cc
         env/env_basic_test.cc
+        env/env_openssl_test.cc
         env/env_test.cc
         env/mock_env_test.cc
         memtable/inlineskiplist_test.cc
@@ -957,6 +958,7 @@ if(WITH_TESTS)
         util/hash_test.cc
         util/heap_test.cc
         util/rate_limiter_test.cc
+        util/library_loader_test.cc
         util/repeatable_thread_test.cc
         util/slice_transform_test.cc
         util/timer_queue_test.cc

HISTORY.md

@@ -1,11 +1,12 @@
 # Rocksdb Change Log
-## Unreleased
-### New Features
+## 5.18.3 (2/11/2019)
+### Bug Fixes
+* Fix possible LSM corruption when both range deletions and subcompactions are used. The symptom of this corruption is L1+ files overlapping in the user key space.
 
+## 5.18.2 (01/31/2019)
 ### Public API Change
-
-### Bug Fixes
-* Fix a deadlock caused by compaction and file ingestion waiting for each other in the event of write stalls.
+* Change time resolution in FileOperationInfo.
+* Deleting Blob files also go through SStFileManager.
 
 ## 5.18.0 (11/30/2018)
 ### New Features
@@ -18,6 +19,7 @@
 * Add xxhash64 checksum support
 * Introduced `MemoryAllocator`, which lets the user specify custom memory allocator for block based table.
 * Improved `DeleteRange` to prevent read performance degradation. The feature is no longer marked as experimental.
+* Enabled checkpoint on readonly db (DBImplReadOnly).
 
 ### Public API Change
 * `DBOptions::use_direct_reads` now affects reads issued by `BackupEngine` on the database's SSTs.
@@ -34,6 +36,8 @@
 * Fixed Get correctness bug in the presence of range tombstones where merge operands covered by a range tombstone always result in NotFound.
 * Start populating `NO_FILE_CLOSES` ticker statistic, which was always zero previously.
 * The default value of NewBloomFilterPolicy()'s argument use_block_based_builder is changed to false. Note that this new default may cause large temp memory usage when building very large SST files.
+* Fix a deadlock caused by compaction and file ingestion waiting for each other in the event of write stalls.
+* Make DB ignore dropped column families while committing results of atomic flush.
 
 ## 5.17.0 (10/05/2018)
 ### Public API Change

Makefile

@@ -419,8 +419,10 @@ TESTS = \
 	coding_test \
 	inlineskiplist_test \
 	env_basic_test \
+	env_openssl_test \
 	env_test \
 	hash_test \
+	library_loader_test \
 	thread_local_test \
 	rate_limiter_test \
 	perf_context_test \
@@ -543,7 +545,6 @@ TESTS = \
 	persistent_cache_test \
 	statistics_test \
 	lua_test \
-	range_del_aggregator_test \
 	lru_cache_test \
 	object_registry_test \
 	repair_test \
@@ -554,7 +555,7 @@ TESTS = \
 	trace_analyzer_test \
 	repeatable_thread_test \
 	range_tombstone_fragmenter_test \
-	range_del_aggregator_v2_test \
+	range_del_aggregator_test \
 	sst_file_reader_test \
 
 PARALLEL_TEST = \
@@ -1136,6 +1137,9 @@ coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS)
 hash_test: util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
+library_loader_test: util/library_loader_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 option_change_migration_test: utilities/option_change_migration/option_change_migration_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1307,6 +1311,9 @@ sim_cache_test: utilities/simulator_cache/sim_cache_test.o db/db_test_util.o $(L
 spatial_db_test: utilities/spatialdb/spatial_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
+env_openssl_test: env/env_openssl_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 env_mirror_test: utilities/env_mirror_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1588,9 +1595,6 @@ repeatable_thread_test: util/repeatable_thread_test.o $(LIBOBJECTS) $(TESTHARNES
 range_tombstone_fragmenter_test: db/range_tombstone_fragmenter_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-range_del_aggregator_v2_test: db/range_del_aggregator_v2_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 sst_file_reader_test: table/sst_file_reader_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 

TARGETS

@@ -67,13 +67,11 @@ is_opt_mode = build_mode.startswith("opt")
 if is_opt_mode:
     rocksdb_compiler_flags.append("-DNDEBUG")
 
-default_allocator = read_config("fbcode", "default_allocator")
-
 sanitizer = read_config("fbcode", "sanitizer")
 
-# Let RocksDB aware of jemalloc existence.
-# Do not enable it if sanitizer presents.
-if is_opt_mode and default_allocator.startswith("jemalloc") and sanitizer == "":
+# Do not enable jemalloc if sanitizer presents. RocksDB will further detect
+# whether the binary is linked with jemalloc at runtime.
+if sanitizer == "":
     rocksdb_compiler_flags.append("-DROCKSDB_JEMALLOC")
     rocksdb_external_deps.append(("jemalloc", None, "headers"))
 
@@ -124,7 +122,6 @@ cpp_library(
         "db/merge_helper.cc",
         "db/merge_operator.cc",
         "db/range_del_aggregator.cc",
-        "db/range_del_aggregator_v2.cc",
         "db/range_tombstone_fragmenter.cc",
         "db/repair.cc",
         "db/snapshot_impl.cc",
@@ -143,6 +140,7 @@ cpp_library(
         "env/env_chroot.cc",
         "env/env_encryption.cc",
         "env/env_hdfs.cc",
+        "env/env_openssl.cc",
         "env/env_posix.cc",
         "env/io_posix.cc",
         "env/mock_env.cc",
@@ -230,6 +228,7 @@ cpp_library(
         "util/filter_policy.cc",
         "util/hash.cc",
         "util/jemalloc_nodump_allocator.cc",
+        "util/library_loader.cc",
         "util/log_buffer.cc",
         "util/murmurhash.cc",
         "util/random.cc",
@@ -705,6 +704,11 @@ ROCKS_TESTS = [
         "env/env_basic_test.cc",
         "serial",
     ],
+    [
+        "env_openssl_test",
+        "env/env_openssl_test.cc",
+        "serial",
+    ],
     [
         "env_test",
         "env/env_test.cc",
@@ -815,6 +819,11 @@ ROCKS_TESTS = [
         "tools/ldb_cmd_test.cc",
         "serial",
     ],
+    [
+        "library_loader_test",
+        "util/library_loader_test.cc",
+        "serial",
+    ],
     [
         "listener_test",
         "db/listener_test.cc",
@@ -935,11 +944,6 @@ ROCKS_TESTS = [
         "db/range_del_aggregator_test.cc",
         "serial",
     ],
-    [
-        "range_del_aggregator_v2_test",
-        "db/range_del_aggregator_v2_test.cc",
-        "serial",
-    ],
     [
         "range_tombstone_fragmenter_test",
         "db/range_tombstone_fragmenter_test.cc",

WORKSPACE

@@ -0,0 +1,36 @@
+workspace(name = "rocksdb")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "com_google_googletest",
+    urls = ["https://github.com/google/googletest/archive/release-1.8.1.zip"],
+    strip_prefix = "googletest-release-1.8.1",
+    sha256 = "927827c183d01734cc5cfef85e0ff3f5a92ffe6188e0d18e909c5efebf28a0c7",
+)
+
+http_archive(
+    name = "com_google_snappy",
+    urls = ["https://github.com/stardog-union/snappy/archive/add_bazel.zip"],
+    strip_prefix = "snappy-add_bazel",
+)
+
+new_http_archive(
+    name = "org_lz4",
+    urls = ["https://github.com/lz4/lz4/archive/v1.8.2.zip"],
+    strip_prefix = "lz4-1.8.2",
+    build_file = "third_party/lz4/BUILD.external",
+    sha256 = "6df2bc7b830d4a23ca6f0a19a772fc0a61100f98baa843f9bbf873a80b6840d5",
+)
+
+http_archive(
+    name = "toolchain",
+    urls = [
+        # The file: URL is useful for testing the build, but is not generally necessary since Bazel handles caching
+        # external dependencies.
+        # TODO(james): Remove this URL when the Bazel build is stable.
+        # "file:///home/james/git/toolchain-master.tgz",
+        "https://github.com/stardog-union/toolchain/archive/master.zip",
+    ],
+    strip_prefix = "toolchain-master",
+    sha256 = "d0740cacb99833911baba82041bb4429f9d3182522fe0fd4c131335ac8343891",
+)

buckifier/targets_cfg.py

@@ -71,13 +71,11 @@
 if is_opt_mode:
     rocksdb_compiler_flags.append("-DNDEBUG")
 
-default_allocator = read_config("fbcode", "default_allocator")
-
 sanitizer = read_config("fbcode", "sanitizer")
 
-# Let RocksDB aware of jemalloc existence.
-# Do not enable it if sanitizer presents.
-if is_opt_mode and default_allocator.startswith("jemalloc") and sanitizer == "":
+# Do not enable jemalloc if sanitizer presents. RocksDB will further detect
+# whether the binary is linked with jemalloc at runtime.
+if sanitizer == "":
     rocksdb_compiler_flags.append("-DROCKSDB_JEMALLOC")
     rocksdb_external_deps.append(("jemalloc", None, "headers"))
 """

build_tools/build_detect_platform

@@ -485,6 +485,18 @@ EOF
             PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS -faligned-new -DHAVE_ALIGNED_NEW"
         fi
     fi
+
+        if ! test $ROCKSDB_DISABLE_OPENSSL; then
+        # Test whether openssl is available
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+          #include <openssl/evp.h>
+          int main() {}
+EOF
+        if [ "$?" = 0 ]; then
+            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_OPENSSL_AES_CTR"
+            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ldl"
+        fi
+    fi
 fi
 
 # TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning.
@@ -612,6 +624,7 @@ echo "JEMALLOC_LIB=$JEMALLOC_LIB" >> "$OUTPUT"
 echo "ROCKSDB_MAJOR=$ROCKSDB_MAJOR" >> "$OUTPUT"
 echo "ROCKSDB_MINOR=$ROCKSDB_MINOR" >> "$OUTPUT"
 echo "ROCKSDB_PATCH=$ROCKSDB_PATCH" >> "$OUTPUT"
+echo "ROCKSDB_OPENSSL_AES_CTR=$ROCKSDB_OPENSSL_AES_CTR" >> "$OUTPUT"
 echo "CLANG_SCAN_BUILD=$CLANG_SCAN_BUILD" >> "$OUTPUT"
 echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT"
 echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"