Merge pull request #2096 from Expensify/main

Update expensify_prod branch

BedrockCommandQueue.cpp

@@ -15,7 +15,7 @@ BedrockCommandQueue::BedrockCommandQueue() :
 BedrockCommandQueue::BedrockCommandQueue(
     function<void(unique_ptr<BedrockCommand>& item)> startFunction,
     function<void(unique_ptr<BedrockCommand>& item)> endFunction
-) : SScheduledPriorityQueue<unique_ptr<BedrockCommand>>(startFunction, endFunction)
+) : SScheduledPriorityQueue<unique_ptr<BedrockCommand>>(move(startFunction), move(endFunction))
 { }
 
 list<string> BedrockCommandQueue::getRequestMethodLines() {

BedrockServer.cpp

@@ -1814,6 +1814,7 @@ bool BedrockServer::_isControlCommand(const unique_ptr<BedrockCommand>& command)
         SIEquals(command->request.methodLine, "BlockWrites")            ||
         SIEquals(command->request.methodLine, "UnblockWrites")          ||
         SIEquals(command->request.methodLine, "SetMaxSocketThreads")    ||
+        SIEquals(command->request.methodLine, "OpenDBHandles")    ||
         SIEquals(command->request.methodLine, "CRASH_COMMAND")
         ) {
         return true;
@@ -1974,6 +1975,63 @@ void BedrockServer::_control(unique_ptr<BedrockCommand>& command) {
         } else {
             response.methodLine = "401 Don't Use Zero";
         }
+    } else if (SIEquals(command->request.methodLine, "OpenDBHandles")) {
+        // Note that calling this with more handles than the pool is configured to open will effectively lock up the server.
+        // Calling it with fewer than that should eventually work but may have big performance impacts if this number
+        // is very close to the maximum allowed.
+        // Whether or not this creates a bunch of handles or recycles existing handles is indeterminate.
+        size_t numberOfHandles = command->request.calcU64("NumberOfHandles");
+        size_t maxThreads = 16;
+        if (command->request.isSet("maxThreads")) {
+            maxThreads = command->request.calcU64("maxThreads");
+        }
+
+        if (numberOfHandles < 0 || maxThreads < 1 || maxThreads > 1000) {
+            response.methodLine = "405 Pick saner values.";
+        }
+
+        auto dbPoolCopy = atomic_load(&_dbPool);
+        if (dbPoolCopy && numberOfHandles) {
+            SINFO("Reserving " << numberOfHandles << " DB handles.");
+            vector<size_t> indicies(numberOfHandles);
+            vector<size_t> timings(numberOfHandles);
+            list<thread> threads;
+
+            // Grab the indices of the handles we'll load.
+            for (size_t index = 0; index < numberOfHandles; index++) {
+                indicies[index] = dbPoolCopy->getIndex(false);
+            }
+
+            SINFO("Reserved " << numberOfHandles << " DB handles, initializing.");
+            atomic<size_t> index(0);
+            for (size_t t = 0; t < maxThreads; t++) {
+                threads.emplace_back([&](){
+                    while (true) {
+                        size_t current_index = index.fetch_add(1);
+                        if (current_index >= numberOfHandles) {
+                            return;
+                        }
+                        uint64_t startTime = STimeNow();
+                        dbPoolCopy->initializeIndex(current_index);
+                        uint64_t endTime = STimeNow();
+
+                        timings[current_index] = endTime - startTime;
+                    }
+                });
+            }
+
+            // Wait for them all to initialize.
+            for(thread& t : threads) {
+                t.join();
+            }
+
+            SINFO("Initialized " << numberOfHandles << " DB handles, cleaning up.");
+            for (size_t i = 0; i < numberOfHandles; i++) {
+                dbPoolCopy->returnToPool(indicies[i]);
+            }
+            SINFO("Returned " << numberOfHandles << " DB handles.");
+            command->response.content = SComposeList(timings, "\n");
+        }
     }
 }
 

plugins/Cache.cpp

@@ -82,7 +82,7 @@ pair<string, bool> BedrockPlugin_Cache::LRUMap::popLRU() {
     return make_pair(nameCopy, true);
 }
 
-int64_t BedrockPlugin_Cache::initCacheSize(string cacheString) {
+int64_t BedrockPlugin_Cache::initCacheSize(const string& cacheString) {
     // Check the configuration
     const string& maxCache = SToUpper(cacheString);
     int64_t maxCacheSize = SToInt64(maxCache);

plugins/Cache.h

@@ -46,7 +46,7 @@ class BedrockPlugin_Cache : public BedrockPlugin {
         map<string, Entry*> _lruMap;
     };
 
-    static int64_t initCacheSize(string cacheString);
+    static int64_t initCacheSize(const string& cacheString);
 
     // Constants
     const int64_t _maxCacheSize;

test/lib/BedrockTester.cpp

@@ -19,7 +19,7 @@ mutex BedrockTester::_testersMutex;
 set<BedrockTester*> BedrockTester::_testers;
 const bool BedrockTester::ENABLE_HCTREE{false};
 
-string BedrockTester::getTempFileName(string prefix) {
+string BedrockTester::getTempFileName(const string& prefix) {
     string templateStr = "/tmp/" + prefix + "bedrocktest_XXXXXX.db";
     char buffer[100];
     memset(buffer, 0, 100);
@@ -154,7 +154,7 @@ BedrockTester::~BedrockTester() {
     _testers.erase(this);
 }
 
-void BedrockTester::updateArgs(const map<string, string> args) {
+void BedrockTester::updateArgs(const map<string, string>& args) {
     for (auto& row : args) {
         _args[row.first] = row.second;
     }
@@ -289,7 +289,7 @@ void BedrockTester::stopServer(int signal) {
     }
 }
 
-string BedrockTester::executeWaitVerifyContent(SData request, const string& expectedResult, bool control, uint64_t retryTimeoutUS) {
+string BedrockTester::executeWaitVerifyContent(const SData& request, const string& expectedResult, bool control, uint64_t retryTimeoutUS) {
     uint64_t start = STimeNow();
     vector<SData> results;
     do {
@@ -316,7 +316,7 @@ string BedrockTester::executeWaitVerifyContent(SData request, const string& expe
     return results[0].content;
 }
 
-STable BedrockTester::executeWaitVerifyContentTable(SData request, const string& expectedResult) {
+STable BedrockTester::executeWaitVerifyContentTable(const SData& request, const string& expectedResult) {
     string result = executeWaitVerifyContent(request, expectedResult);
     return SParseJSONObject(result);
 }

test/lib/BedrockTester.h

@@ -54,11 +54,11 @@ class BedrockTester {
 
     // Generate a temporary filename with an optional prefix. Used particularly to create new DB files for each server,
     // but can generally be used for any temporary file required.
-    static string getTempFileName(string prefix = "");
+    static string getTempFileName(const string& prefix = "");
 
     // Change the arguments for a server. Only takes effect when the server next starts. This can change or add args,
     // but not remove args. Any args specified here are added or replaced into the existing set.
-    void updateArgs(const map<string, string> args);
+    void updateArgs(const map<string, string>& args);
 
     string getArg(const string& arg) const;
 
@@ -70,12 +70,12 @@ class BedrockTester {
     // Sends a single request, returning the response content.
     // If the response method line doesn't begin with the expected result, throws.
     // Convenience wrapper around executeWaitMultipleData.
-    virtual string executeWaitVerifyContent(SData request, const string& expectedResult = "200 OK", bool control = false, uint64_t retryTimeoutUS = 0);
+    virtual string executeWaitVerifyContent(const SData& request, const string& expectedResult = "200 OK", bool control = false, uint64_t retryTimeoutUS = 0);
 
     // Sends a single request, returning the response content as a STable.
     // If the response method line doesn't begin with the expected result, throws.
     // Convenience wrapper around executeWaitMultipleData.
-    virtual STable executeWaitVerifyContentTable(SData request, const string& expectedResult = "200 OK");
+    virtual STable executeWaitVerifyContentTable(const SData& request, const string& expectedResult = "200 OK");
 
     // Read from the DB file, without going through the bedrock server. Two interfaces are provided to maintain
     // compatibility with the `SQLite` class.

test/tests/jobs/GetJobTest.cpp

@@ -5,7 +5,7 @@
 #include <libstuff/SQResult.h>
 #include <test/lib/BedrockTester.h>
 
-bool isBetweenSecondsInclusive(uint64_t startTimestamp, uint64_t endTimestamp, string timestampString) {
+bool isBetweenSecondsInclusive(uint64_t startTimestamp, uint64_t endTimestamp, const string& timestampString) {
     uint64_t testTime = startTimestamp;
     while (true) {
         string testTimeString = SComposeTime("%Y-%m-%d %H:%M:%S", testTime);