tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Available memory logging, utils/sysinfo/memory cleanup, tests/concurrent cleanup.

Browse Source 
Summary:
Added a warning to the log. Every 3 seconds (let's not make that configurable). Can be turned off.

I still don't like this. We are raising another thread and reading a file to do monitoring. We're developing a DB, not a sys monitor. Serious admins do that themselves. But, here it is.

UPDATE:
Cleaned up `utils/sysinfo/memory`. Removed all unused functions. Removed the faulty memory check in `tests/concurrent`.

Reviewers: buda, mferencevic, mislav.bradac

Reviewed By: mislav.bradac

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D819
This commit is contained in:
florijan 2017-09-22 14:24:53 +02:00
parent 4e4fdd8029
commit b2f3dc1916
16 changed files with 436 additions and 512 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -19,6 +19,7 @@
* Use \u to specify 4 digit codepoint and \U for 8 digit
* Keywords appearing in header (named expressions) keep original case.
* Our Bolt protocol implementation is now completely compatible with the protocol version 1 specification. (https://boltprotocol.org/v1/)
* Added a log warning when running out of memory and the `memory_warning_threshold` flag
## v0.7.0

5
docs/user_technical/installation.md
View File

@ -72,8 +72,9 @@ parameters:
--snapshot-on-exit | bool | false | Make a snapshot when closing Memgraph.
--snapshot-recover-on-startup | bool | false | Recover the database on startup using the last<br/>stored snapshot.
--query-execution-time-sec | integer | 180 | Maximum allowed query execution time. <br/>Queries exceeding this limit will be aborted. Value of -1 means no limit.
[^1]: Maximum number of concurrent executions on the current CPU.
--memory-warning-threshold | integer | 1024 | Memory warning threshold, in MB. If Memgraph detects there is less available RAM available it will log a warning. Set to 0 to disable.
[^1]: Maximum number of concurrent executions on the current CPU.
To find more about how to execute queries on Memgraph please proceed to
[Quick Start](quick-start.md).

17
src/memgraph_bolt.cpp
View File

@ -12,8 +12,10 @@
#include "io/network/socket.hpp"
#include "utils/flag_validation.hpp"
#include "utils/scheduler.hpp"
#include "utils/signals/handler.hpp"
#include "utils/stacktrace.hpp"
#include "utils/sysinfo/memory.hpp"
#include "utils/terminate_handler.hpp"
namespace fs = std::experimental::filesystem;
@ -34,6 +36,10 @@ DEFINE_VALIDATED_int32(num_workers,
"Number of workers", FLAG_IN_RANGE(1, INT32_MAX));
DEFINE_string(log_file, "memgraph.log",
"Path to where the log should be stored.");
DEFINE_uint64(
memory_warning_threshold, 1024,
"Memory warning treshold, in MB. If Memgraph detects there is less available "
"RAM available it will log a warning. Set to 0 to disable.");
// Load flags in this order, the last one has the highest priority:
// 1) /etc/memgraph/config
@ -143,6 +149,17 @@ int main(int argc, char **argv) {
SignalHandler::register_handler(Signal::Interupt,
[&server]() { server.Shutdown(); });
// Start memory warning logger.
Scheduler mem_log_scheduler;
if (FLAGS_memory_warning_threshold > 0) {
mem_log_scheduler.Run(std::chrono::seconds(3), [] {
auto free_ram_mb = utils::AvailableMem() / 1024;
if (free_ram_mb < FLAGS_memory_warning_threshold)
LOG(WARNING) << "Running out of available RAM, only " << free_ram_mb
<< " MB left.";
});
}
// Start worker threads.
server.Start(FLAGS_num_workers);

73
src/utils/sysinfo/memory.hpp
View File

@ -1,60 +1,27 @@
#pragma mark
#include <fstream>
#include <iostream>
#include <limits>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include "sys/sysinfo.h"
#include "sys/types.h"
auto total_virtual_memory() {
struct sysinfo mem_info;
sysinfo(&mem_info);
long long total_virtual_memory = mem_info.totalram;
total_virtual_memory += mem_info.totalswap;
total_virtual_memory *= mem_info.mem_unit;
return total_virtual_memory;
}
auto used_virtual_memory() {
struct sysinfo mem_info;
sysinfo(&mem_info);
long long virtual_memory_used = mem_info.totalram - mem_info.freeram;
virtual_memory_used += mem_info.totalswap - mem_info.freeswap;
virtual_memory_used *= mem_info.mem_unit;
return virtual_memory_used;
}
// TODO: OS dependent
namespace utils {
/**
* parses memory line from /proc/self/status
* Gets the amount of available RAM in kilobytes. If the information is
* unavalable zero is returned.
*/
auto parse_vm_size(char *line) {
// This assumes that a digit will be found and the line ends in " Kb".
auto i = std::strlen(line);
const char *p = line;
while (*p < '0' || *p > '9') p++;
line[i - 3] = '\0';
return std::atoll(p);
}
/**
* returns VmSize in kB
*/
auto vm_size() {
std::FILE *file = std::fopen("/proc/self/status", "r");
auto result = -1LL;
char line[128];
while (fgets(line, 128, file) != NULL) {
if (strncmp(line, "VmSize:", 7) == 0) {
result = parse_vm_size(line);
break;
inline auto AvailableMem() {
std::string token;
std::ifstream meminfo("/proc/meminfo");
while (meminfo >> token) {
if (token == "MemAvailable:") {
unsigned long mem;
if (meminfo >> mem) {
return mem;
} else {
return 0UL;
}
}
meminfo.ignore(std::numeric_limits<std::streamsize>::max(), '\n');
}
fclose(file);
return result;
return 0UL;
}
} // namespace utils

32
tests/concurrent/common.hpp
View File

@ -12,7 +12,6 @@
#include "data_structures/concurrent/concurrent_set.hpp"
#include "data_structures/concurrent/skiplist.hpp"
#include "utils/assert.hpp"
#include "utils/sysinfo/memory.hpp"
// NOTE: this file is highly coupled to data_structures
// TODO: REFACTOR
@ -195,34 +194,3 @@ auto insert_try(typename S::Accessor &acc, long long &downcount,
}
};
}
// Performs memory check to determine if memory usage before calling given
// function
// is aproximately equal to memory usage after function. Memory usage is thread
// senstive so no_threads spawned in function is necessary.
void memory_check(size_t no_threads, std::function<void()> f) {
DLOG(INFO) << fmt::format("Number of threads: {}", no_threads);
// TODO: replace vm_size with something more appropriate
// the past implementation was teribble wrong
// to that ASAP
// OR
// use custom allocation wrapper
// OR
// user Boost.Test
auto start = vm_size();
DLOG(INFO) << fmt::format("Memory check (used memory at the beginning): {}",
start);
f();
auto end = vm_size();
DLOG(INFO) << fmt::format("Memory check (used memory at the end): {}", end);
long long delta = end - start;
DLOG(INFO) << fmt::format("Delta: {}", delta);
// TODO: do memory check somehow
// the past implementation was wrong
permanent_assert(true, "Memory leak");
}

94
tests/concurrent/conncurent_list.cpp
View File

@ -14,65 +14,63 @@ constexpr size_t no_insert_for_one_delete = 1;
// no_find_per_change and no_insert_for_one_delete.
int main(int argc, char **argv) {
google::InitGoogleLogging(argv[0]);
memory_check(THREADS_NO, [] {
ConcurrentList<std::pair<int, int>> list;
permanent_assert(list.size() == 0, "The list isn't empty");
ConcurrentList<std::pair<int, int>> list;
permanent_assert(list.size() == 0, "The list isn't empty");
auto futures = run<std::pair<long long, long long>>(
THREADS_NO, [&](auto index) mutable {
auto rand = rand_gen(key_range);
auto rand_change = rand_gen_bool(no_find_per_change);
auto rand_delete = rand_gen_bool(no_insert_for_one_delete);
long long sum = 0;
long long count = 0;
auto futures =
run<std::pair<long long, long long>>(THREADS_NO, [&](auto index) mutable {
auto rand = rand_gen(key_range);
auto rand_change = rand_gen_bool(no_find_per_change);
auto rand_delete = rand_gen_bool(no_insert_for_one_delete);
long long sum = 0;
long long count = 0;
for (int i = 0; i < op_per_thread; i++) {
auto num = rand();
auto data = num % max_number;
if (rand_change()) {
if (rand_delete()) {
for (auto it = list.begin(); it != list.end(); it++) {
if (it->first == num) {
if (it.remove()) {
sum -= data;
count--;
}
break;
for (int i = 0; i < op_per_thread; i++) {
auto num = rand();
auto data = num % max_number;
if (rand_change()) {
if (rand_delete()) {
for (auto it = list.begin(); it != list.end(); it++) {
if (it->first == num) {
if (it.remove()) {
sum -= data;
count--;
}
}
} else {
list.begin().push(std::make_pair(num, data));
sum += data;
count++;
}
} else {
for (auto &v : list) {
if (v.first == num) {
permanent_assert(v.second == data, "Data is invalid");
break;
}
}
} else {
list.begin().push(std::make_pair(num, data));
sum += data;
count++;
}
} else {
for (auto &v : list) {
if (v.first == num) {
permanent_assert(v.second == data, "Data is invalid");
break;
}
}
}
}
return std::pair<long long, long long>(sum, count);
});
return std::pair<long long, long long>(sum, count);
});
auto it = list.begin();
long long sums = 0;
long long counters = 0;
for (auto &data : collect(futures)) {
sums += data.second.first;
counters += data.second.second;
}
auto it = list.begin();
long long sums = 0;
long long counters = 0;
for (auto &data : collect(futures)) {
sums += data.second.first;
counters += data.second.second;
}
for (auto &e : list) {
sums -= e.second;
}
for (auto &e : list) {
sums -= e.second;
}
permanent_assert(sums == 0, "Same values aren't present");
check_size_list<ConcurrentList<std::pair<int, int>>>(list, counters);
permanent_assert(sums == 0, "Same values aren't present");
check_size_list<ConcurrentList<std::pair<int, int>>>(list, counters);
std::this_thread::sleep_for(1s);
});
std::this_thread::sleep_for(1s);
}

43
tests/concurrent/sl_insert.cpp
View File

@ -11,32 +11,29 @@ constexpr size_t key_range = elems_per_thread * THREADS_NO * 2;
// Test checks for missing data and changed/overwriten data.
int main(int argc, char **argv) {
google::InitGoogleLogging(argv[0]);
map_t skiplist;
memory_check(THREADS_NO, [] {
map_t skiplist;
auto futures =
run<std::vector<size_t>>(THREADS_NO, skiplist, [](auto acc, auto index) {
auto rand = rand_gen(key_range);
long long downcount = elems_per_thread;
std::vector<size_t> owned;
auto inserter =
insert_try<size_t, size_t, map_t>(acc, downcount, owned);
auto futures = run<std::vector<size_t>>(
THREADS_NO, skiplist, [](auto acc, auto index) {
auto rand = rand_gen(key_range);
long long downcount = elems_per_thread;
std::vector<size_t> owned;
auto inserter =
insert_try<size_t, size_t, map_t>(acc, downcount, owned);
do {
inserter(rand(), index);
} while (downcount > 0);
do {
inserter(rand(), index);
} while (downcount > 0);
check_present_same<map_t>(acc, index, owned);
return owned;
});
check_present_same<map_t>(acc, index, owned);
return owned;
});
auto accessor = skiplist.access();
for (auto &owned : collect(futures)) {
check_present_same<map_t>(accessor, owned);
}
auto accessor = skiplist.access();
for (auto &owned : collect(futures)) {
check_present_same<map_t>(accessor, owned);
}
check_size<map_t>(accessor, THREADS_NO * elems_per_thread);
check_order<map_t>(accessor);
});
check_size<map_t>(accessor, THREADS_NO * elems_per_thread);
check_order<map_t>(accessor);
}

41
tests/concurrent/sl_insert_competetive.cpp
View File

@ -11,33 +11,30 @@ constexpr size_t elems_per_thread = 100000;
// Test checks for missing data and changed/overwriten data.
int main(int argc, char **argv) {
google::InitGoogleLogging(argv[0]);
map_t skiplist;
memory_check(THREADS_NO, [] {
map_t skiplist;
auto futures = run<std::vector<size_t>>(
THREADS_NO, skiplist, [](auto acc, auto index) {
long long downcount = elems_per_thread;
std::vector<size_t> owned;
auto inserter =
insert_try<size_t, size_t, map_t>(acc, downcount, owned);
auto futures =
run<std::vector<size_t>>(THREADS_NO, skiplist, [](auto acc, auto index) {
long long downcount = elems_per_thread;
std::vector<size_t> owned;
auto inserter =
insert_try<size_t, size_t, map_t>(acc, downcount, owned);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wfor-loop-analysis"
for (int i = 0; downcount > 0; i++) {
inserter(i, index);
}
for (int i = 0; downcount > 0; i++) {
inserter(i, index);
}
#pragma GCC diagnostic pop
check_present_same<map_t>(acc, index, owned);
return owned;
});
check_present_same<map_t>(acc, index, owned);
return owned;
});
auto accessor = skiplist.access();
for (auto &owned : collect(futures)) {
check_present_same<map_t>(accessor, owned);
}
auto accessor = skiplist.access();
for (auto &owned : collect(futures)) {
check_present_same<map_t>(accessor, owned);
}
check_size<map_t>(accessor, THREADS_NO * elems_per_thread);
check_order<map_t>(accessor);
});
check_size<map_t>(accessor, THREADS_NO * elems_per_thread);
check_order<map_t>(accessor);
}

118
tests/concurrent/sl_map.cpp
View File

@ -7,71 +7,69 @@ constexpr size_t elems_per_thread = 1e5;
int main(int, char **argv) {
google::InitGoogleLogging(argv[0]);
memory_check(THREADS_NO, [&] {
std::vector<std::thread> threads;
map_t skiplist;
std::vector<std::thread> threads;
map_t skiplist;
// put THREADS_NO * elems_per_thread items to the skiplist
for (size_t thread_i = 0; thread_i < THREADS_NO; ++thread_i) {
threads.emplace_back(
[&skiplist](size_t start, size_t end) {
auto accessor = skiplist.access();
for (size_t elem_i = start; elem_i < end; ++elem_i) {
accessor.insert(elem_i, elem_i);
}
},
thread_i * elems_per_thread,
thread_i * elems_per_thread + elems_per_thread);
}
// wait all threads
for (auto &thread : threads) {
thread.join();
}
// put THREADS_NO * elems_per_thread items to the skiplist
for (size_t thread_i = 0; thread_i < THREADS_NO; ++thread_i) {
threads.emplace_back(
[&skiplist](size_t start, size_t end) {
auto accessor = skiplist.access();
for (size_t elem_i = start; elem_i < end; ++elem_i) {
accessor.insert(elem_i, elem_i);
}
},
thread_i * elems_per_thread,
thread_i * elems_per_thread + elems_per_thread);
}
// wait all threads
for (auto &thread : threads) {
thread.join();
}
// get skiplist size
{
auto accessor = skiplist.access();
permanent_assert(accessor.size() == THREADS_NO * elems_per_thread,
"all elements in skiplist");
}
// get skiplist size
{
auto accessor = skiplist.access();
permanent_assert(accessor.size() == THREADS_NO * elems_per_thread,
"all elements in skiplist");
}
for (size_t thread_i = 0; thread_i < THREADS_NO; ++thread_i) {
threads[thread_i] = std::thread(
[&skiplist](size_t start, size_t end) {
auto accessor = skiplist.access();
for (size_t elem_i = start; elem_i < end; ++elem_i) {
permanent_assert(accessor.remove(elem_i) == true, "");
}
},
thread_i * elems_per_thread,
thread_i * elems_per_thread + elems_per_thread);
}
// // wait all threads
for (auto &thread : threads) {
thread.join();
}
for (size_t thread_i = 0; thread_i < THREADS_NO; ++thread_i) {
threads[thread_i] = std::thread(
[&skiplist](size_t start, size_t end) {
auto accessor = skiplist.access();
for (size_t elem_i = start; elem_i < end; ++elem_i) {
permanent_assert(accessor.remove(elem_i) == true, "");
}
},
thread_i * elems_per_thread,
thread_i * elems_per_thread + elems_per_thread);
}
// // wait all threads
for (auto &thread : threads) {
thread.join();
}
// check size
{
auto accessor = skiplist.access();
permanent_assert(accessor.size() == 0, "Size should be 0, but size is "
<< accessor.size());
}
// check size
{
auto accessor = skiplist.access();
permanent_assert(accessor.size() == 0, "Size should be 0, but size is "
<< accessor.size());
}
// check count
{
size_t iterator_counter = 0;
auto accessor = skiplist.access();
for (auto elem : accessor) {
++iterator_counter;
cout << elem.first << " ";
}
permanent_assert(iterator_counter == 0, "deleted elements");
// check count
{
size_t iterator_counter = 0;
auto accessor = skiplist.access();
for (auto elem : accessor) {
++iterator_counter;
cout << elem.first << " ";
}
permanent_assert(iterator_counter == 0, "deleted elements");
}
{
auto accessor = skiplist.access();
check_order<map_t>(accessor);
}
});
{
auto accessor = skiplist.access();
check_order<map_t>(accessor);
}
}

23
tests/concurrent/sl_memory.cpp
View File

@ -9,19 +9,16 @@ constexpr size_t elements = 2e6;
*/
int main(int argc, char **argv) {
google::InitGoogleLogging(argv[0]);
map_t skiplist;
memory_check(THREADS_NO, [] {
map_t skiplist;
auto futures = run<size_t>(THREADS_NO, skiplist, [](auto acc, auto index) {
for (size_t i = 0; i < elements; i++) {
acc.insert(i, index);
}
return index;
});
collect(futures);
auto accessor = skiplist.access();
check_size<map_t>(accessor, elements);
auto futures = run<size_t>(THREADS_NO, skiplist, [](auto acc, auto index) {
for (size_t i = 0; i < elements; i++) {
acc.insert(i, index);
}
return index;
});
collect(futures);
auto accessor = skiplist.access();
check_size<map_t>(accessor, elements);
}

112
tests/concurrent/sl_memory_leak.cpp
View File

@ -4,7 +4,7 @@ constexpr size_t THREADS_NO = std::min(max_no_threads, 1);
constexpr size_t elems_per_thread = 16e5;
// TODO: Memory leak at 1,600,000 elements (Kruno wrote this here but
// the memory_check method had invalid implementation)
// the previous (now deleted) memory_check method had invalid implementation)
// 1. implement valid memory_check
// 2. analyse this code
// 3. fix the memory leak
@ -12,66 +12,64 @@ constexpr size_t elems_per_thread = 16e5;
int main(int, char **argv) {
google::InitGoogleLogging(argv[0]);
memory_check(THREADS_NO, [&] {
std::vector<std::thread> threads;
map_t skiplist;
std::vector<std::thread> threads;
map_t skiplist;
// put THREADS_NO * elems_per_thread items to the skiplist
for (size_t thread_i = 0; thread_i < THREADS_NO; ++thread_i) {
threads.emplace_back(
[&skiplist](size_t start, size_t end) {
auto accessor = skiplist.access();
for (size_t elem_i = start; elem_i < end; ++elem_i) {
accessor.insert(elem_i, elem_i);
}
},
thread_i * elems_per_thread,
thread_i * elems_per_thread + elems_per_thread);
}
// wait all threads
for (auto &thread : threads) {
thread.join();
}
// put THREADS_NO * elems_per_thread items to the skiplist
for (size_t thread_i = 0; thread_i < THREADS_NO; ++thread_i) {
threads.emplace_back(
[&skiplist](size_t start, size_t end) {
auto accessor = skiplist.access();
for (size_t elem_i = start; elem_i < end; ++elem_i) {
accessor.insert(elem_i, elem_i);
}
},
thread_i * elems_per_thread,
thread_i * elems_per_thread + elems_per_thread);
}
// wait all threads
for (auto &thread : threads) {
thread.join();
}
// get skiplist size
{
auto accessor = skiplist.access();
permanent_assert(accessor.size() == THREADS_NO * elems_per_thread,
"all elements in skiplist");
}
// get skiplist size
{
auto accessor = skiplist.access();
permanent_assert(accessor.size() == THREADS_NO * elems_per_thread,
"all elements in skiplist");
}
for (size_t thread_i = 0; thread_i < THREADS_NO; ++thread_i) {
threads[thread_i] = std::thread(
[&skiplist](size_t start, size_t end) {
auto accessor = skiplist.access();
for (size_t elem_i = start; elem_i < end; ++elem_i) {
permanent_assert(accessor.remove(elem_i) == true, "");
}
},
thread_i * elems_per_thread,
thread_i * elems_per_thread + elems_per_thread);
}
// // wait all threads
for (auto &thread : threads) {
thread.join();
}
for (size_t thread_i = 0; thread_i < THREADS_NO; ++thread_i) {
threads[thread_i] = std::thread(
[&skiplist](size_t start, size_t end) {
auto accessor = skiplist.access();
for (size_t elem_i = start; elem_i < end; ++elem_i) {
permanent_assert(accessor.remove(elem_i) == true, "");
}
},
thread_i * elems_per_thread,
thread_i * elems_per_thread + elems_per_thread);
}
// // wait all threads
for (auto &thread : threads) {
thread.join();
}
// check size
{
auto accessor = skiplist.access();
permanent_assert(accessor.size() == 0, "Size should be 0, but size is "
<< accessor.size());
}
// check size
{
auto accessor = skiplist.access();
permanent_assert(accessor.size() == 0, "Size should be 0, but size is "
<< accessor.size());
}
// check count
{
size_t iterator_counter = 0;
auto accessor = skiplist.access();
for (auto elem : accessor) {
++iterator_counter;
cout << elem.first << " ";
}
permanent_assert(iterator_counter == 0, "deleted elements");
// check count
{
size_t iterator_counter = 0;
auto accessor = skiplist.access();
for (auto elem : accessor) {
++iterator_counter;
cout << elem.first << " ";
}
});
permanent_assert(iterator_counter == 0, "deleted elements");
}
}

81
tests/concurrent/sl_remove_competetive.cpp
View File

@ -14,54 +14,51 @@ constexpr size_t no_insert_for_one_delete = 2;
// Calls of remove method are interleaved with insert calls.
int main(int argc, char **argv) {
google::InitGoogleLogging(argv[0]);
map_t skiplist;
memory_check(THREADS_NO, [] {
map_t skiplist;
auto futures = run<std::pair<long long, long long>>(
THREADS_NO, skiplist, [](auto acc, auto index) {
auto rand_op = rand_gen_bool(no_insert_for_one_delete);
long long downcount = op_per_thread;
long long sum = 0;
long long count = 0;
auto futures = run<std::pair<long long, long long>>(
THREADS_NO, skiplist, [](auto acc, auto index) {
auto rand_op = rand_gen_bool(no_insert_for_one_delete);
long long downcount = op_per_thread;
long long sum = 0;
long long count = 0;
for (int i = 0; downcount > 0; i++) {
auto data = i % max_number;
if (rand_op()) {
auto t = i;
while (t > 0) {
if (acc.remove(t)) {
sum -= t % max_number;
downcount--;
count--;
break;
}
t--;
}
} else {
if (acc.insert(i, data).second) {
sum += data;
count++;
for (int i = 0; downcount > 0; i++) {
auto data = i % max_number;
if (rand_op()) {
auto t = i;
while (t > 0) {
if (acc.remove(t)) {
sum -= t % max_number;
downcount--;
count--;
break;
}
t--;
}
} else {
if (acc.insert(i, data).second) {
sum += data;
count++;
downcount--;
}
}
return std::pair<long long, long long>(sum, count);
});
}
return std::pair<long long, long long>(sum, count);
});
auto accessor = skiplist.access();
long long sums = 0;
long long counters = 0;
for (auto &data : collect(futures)) {
sums += data.second.first;
counters += data.second.second;
}
auto accessor = skiplist.access();
long long sums = 0;
long long counters = 0;
for (auto &data : collect(futures)) {
sums += data.second.first;
counters += data.second.second;
}
for (auto &e : accessor) {
sums -= e.second;
}
permanent_assert(sums == 0, "Aproximetly Same values are present");
check_size<map_t>(accessor, counters);
check_order<map_t>(accessor);
});
for (auto &e : accessor) {
sums -= e.second;
}
permanent_assert(sums == 0, "Aproximetly Same values are present");
check_size<map_t>(accessor, counters);
check_order<map_t>(accessor);
}

63
tests/concurrent/sl_remove_disjoint.cpp
View File

@ -12,41 +12,38 @@ constexpr size_t no_insert_for_one_delete = 1;
// Calls of remove method are interleaved with insert calls.
int main(int argc, char **argv) {
google::InitGoogleLogging(argv[0]);
map_t skiplist;
memory_check(THREADS_NO, [] {
map_t skiplist;
auto futures =
run<std::vector<size_t>>(THREADS_NO, skiplist, [](auto acc, auto index) {
auto rand = rand_gen(key_range);
auto rand_op = rand_gen_bool(no_insert_for_one_delete);
long long downcount = op_per_thread;
std::vector<size_t> owned;
auto inserter =
insert_try<size_t, size_t, map_t>(acc, downcount, owned);
auto futures = run<std::vector<size_t>>(
THREADS_NO, skiplist, [](auto acc, auto index) {
auto rand = rand_gen(key_range);
auto rand_op = rand_gen_bool(no_insert_for_one_delete);
long long downcount = op_per_thread;
std::vector<size_t> owned;
auto inserter =
insert_try<size_t, size_t, map_t>(acc, downcount, owned);
do {
if (owned.size() != 0 && rand_op()) {
auto rem = rand() % owned.size();
permanent_assert(acc.remove(owned[rem]), "Owned data removed");
owned.erase(owned.begin() + rem);
downcount--;
} else {
inserter(rand(), index);
}
} while (downcount > 0);
do {
if (owned.size() != 0 && rand_op()) {
auto rem = rand() % owned.size();
permanent_assert(acc.remove(owned[rem]), "Owned data removed");
owned.erase(owned.begin() + rem);
downcount--;
} else {
inserter(rand(), index);
}
} while (downcount > 0);
check_present_same<map_t>(acc, index, owned);
return owned;
});
check_present_same<map_t>(acc, index, owned);
return owned;
});
auto accessor = skiplist.access();
size_t count = 0;
for (auto &owned : collect(futures)) {
check_present_same<map_t>(accessor, owned);
count += owned.second.size();
}
check_size<map_t>(accessor, count);
check_order<map_t>(accessor);
});
auto accessor = skiplist.access();
size_t count = 0;
for (auto &owned : collect(futures)) {
check_present_same<map_t>(accessor, owned);
count += owned.second.size();
}
check_size<map_t>(accessor, count);
check_order<map_t>(accessor);
}

81
tests/concurrent/sl_remove_joint.cpp
View File

@ -14,52 +14,49 @@ constexpr size_t no_insert_for_one_delete = 2;
// Calls of remove method are interleaved with insert calls.
int main(int argc, char **argv) {
google::InitGoogleLogging(argv[0]);
map_t skiplist;
memory_check(THREADS_NO, [] {
map_t skiplist;
auto futures = run<std::pair<long long, long long>>(
THREADS_NO, skiplist, [](auto acc, auto index) {
auto rand = rand_gen(key_range);
auto rand_op = rand_gen_bool(no_insert_for_one_delete);
long long downcount = op_per_thread;
long long sum = 0;
long long count = 0;
auto futures = run<std::pair<long long, long long>>(
THREADS_NO, skiplist, [](auto acc, auto index) {
auto rand = rand_gen(key_range);
auto rand_op = rand_gen_bool(no_insert_for_one_delete);
long long downcount = op_per_thread;
long long sum = 0;
long long count = 0;
do {
auto num = rand();
auto data = num % max_number;
if (rand_op()) {
if (acc.remove(num)) {
sum -= data;
downcount--;
count--;
}
} else {
if (acc.insert(num, data).second) {
sum += data;
downcount--;
count++;
}
do {
auto num = rand();
auto data = num % max_number;
if (rand_op()) {
if (acc.remove(num)) {
sum -= data;
downcount--;
count--;
}
} while (downcount > 0);
} else {
if (acc.insert(num, data).second) {
sum += data;
downcount--;
count++;
}
}
} while (downcount > 0);
return std::pair<long long, long long>(sum, count);
});
return std::pair<long long, long long>(sum, count);
});
auto accessor = skiplist.access();
long long sums = 0;
long long counters = 0;
for (auto &data : collect(futures)) {
sums += data.second.first;
counters += data.second.second;
}
auto accessor = skiplist.access();
long long sums = 0;
long long counters = 0;
for (auto &data : collect(futures)) {
sums += data.second.first;
counters += data.second.second;
}
for (auto &e : accessor) {
sums -= e.second;
}
permanent_assert(sums == 0, "Aproximetly Same values are present");
check_size<map_t>(accessor, counters);
check_order<map_t>(accessor);
});
for (auto &e : accessor) {
sums -= e.second;
}
permanent_assert(sums == 0, "Aproximetly Same values are present");
check_size<map_t>(accessor, counters);
check_order<map_t>(accessor);
}

81
tests/concurrent/sl_set.cpp
View File

@ -12,55 +12,52 @@ constexpr size_t no_insert_for_one_delete = 2;
// Calls of remove method are interleaved with insert calls.
int main(int argc, char **argv) {
google::InitGoogleLogging(argv[0]);
ConcurrentSet<std::string> skiplist;
memory_check(THREADS_NO, [] {
ConcurrentSet<std::string> skiplist;
auto futures =
run<std::vector<long>>(THREADS_NO, skiplist, [](auto acc, auto index) {
auto rand = rand_gen(key_range);
auto rand_op = rand_gen_bool(no_insert_for_one_delete);
long long downcount = op_per_thread;
std::vector<long> set(key_range);
auto futures =
run<std::vector<long>>(THREADS_NO, skiplist, [](auto acc, auto index) {
auto rand = rand_gen(key_range);
auto rand_op = rand_gen_bool(no_insert_for_one_delete);
long long downcount = op_per_thread;
std::vector<long> set(key_range);
do {
int num = rand();
std::string num_str = std::to_string(num);
if (rand_op()) {
if (acc.remove(num_str)) {
downcount--;
set[num]--;
}
} else {
std::string num_str = std::to_string(num);
if (acc.insert(num_str).second) {
downcount--;
set[num]++;
}
do {
int num = rand();
std::string num_str = std::to_string(num);
if (rand_op()) {
if (acc.remove(num_str)) {
downcount--;
set[num]--;
}
} while (downcount > 0);
} else {
std::string num_str = std::to_string(num);
if (acc.insert(num_str).second) {
downcount--;
set[num]++;
}
}
} while (downcount > 0);
return set;
});
return set;
});
long set[key_range] = {0};
for (auto &data : collect(futures)) {
for (int i = 0; i < key_range; i++) {
set[i] += data.second[i];
}
}
auto accessor = skiplist.access();
long set[key_range] = {0};
for (auto &data : collect(futures)) {
for (int i = 0; i < key_range; i++) {
permanent_assert(set[i] == 0 || set[i] == 1 ||
(set[i] == 1) ^ accessor.contains(std::to_string(i)),
"Set doesn't hold it's guarantees.");
set[i] += data.second[i];
}
}
for (auto &e : accessor) {
set[std::stoi(e)]--;
}
auto accessor = skiplist.access();
for (int i = 0; i < key_range; i++) {
permanent_assert(set[i] == 0 || set[i] == 1 ||
(set[i] == 1) ^ accessor.contains(std::to_string(i)),
"Set doesn't hold it's guarantees.");
}
check_zero(key_range, set, "Set");
});
for (auto &e : accessor) {
set[std::stoi(e)]--;
}
check_zero(key_range, set, "Set");
}

83
tests/concurrent/sl_simulation.cpp
View File

@ -16,56 +16,53 @@ constexpr size_t no_insert_for_one_delete = 1;
// no_find_per_change and no_insert_for_one_delete.
int main(int argc, char **argv) {
google::InitGoogleLogging(argv[0]);
map_t skiplist;
memory_check(THREADS_NO, [] {
map_t skiplist;
auto futures = run<std::pair<long long, long long>>(
THREADS_NO, skiplist, [](auto acc, auto index) {
auto rand = rand_gen(key_range);
auto rand_change = rand_gen_bool(no_find_per_change);
auto rand_delete = rand_gen_bool(no_insert_for_one_delete);
long long sum = 0;
long long count = 0;
auto futures = run<std::pair<long long, long long>>(
THREADS_NO, skiplist, [](auto acc, auto index) {
auto rand = rand_gen(key_range);
auto rand_change = rand_gen_bool(no_find_per_change);
auto rand_delete = rand_gen_bool(no_insert_for_one_delete);
long long sum = 0;
long long count = 0;
for (int i = 0; i < op_per_thread; i++) {
auto num = rand();
auto data = num % max_number;
if (rand_change()) {
if (rand_delete()) {
if (acc.remove(num)) {
sum -= data;
count--;
}
} else {
if (acc.insert(num, data).second) {
sum += data;
count++;
}
for (int i = 0; i < op_per_thread; i++) {
auto num = rand();
auto data = num % max_number;
if (rand_change()) {
if (rand_delete()) {
if (acc.remove(num)) {
sum -= data;
count--;
}
} else {
auto value = acc.find(num);
permanent_assert(value == acc.end() || value->second == data,
"Data is invalid");
if (acc.insert(num, data).second) {
sum += data;
count++;
}
}
} else {
auto value = acc.find(num);
permanent_assert(value == acc.end() || value->second == data,
"Data is invalid");
}
}
return std::pair<long long, long long>(sum, count);
});
return std::pair<long long, long long>(sum, count);
});
auto accessor = skiplist.access();
long long sums = 0;
long long counters = 0;
for (auto &data : collect(futures)) {
sums += data.second.first;
counters += data.second.second;
}
auto accessor = skiplist.access();
long long sums = 0;
long long counters = 0;
for (auto &data : collect(futures)) {
sums += data.second.first;
counters += data.second.second;
}
for (auto &e : accessor) {
sums -= e.second;
}
permanent_assert(sums == 0, "Same values aren't present");
check_size<map_t>(accessor, counters);
check_order<map_t>(accessor);
});
for (auto &e : accessor) {
sums -= e.second;
}
permanent_assert(sums == 0, "Same values aren't present");
check_size<map_t>(accessor, counters);
check_order<map_t>(accessor);
}