From 499ad3ba152a3db2951c770057ab49b5c96be0d2 Mon Sep 17 00:00:00 2001 From: Matej Ferencevic Date: Wed, 10 Jan 2018 13:53:42 +0100 Subject: [PATCH] Add initial version of Apollo config files Summary: Improve Apollo config files Add name to apollo_build Remove old generate script from build Add build_release symlink to release build Rename 'args' to 'arguments' Add run definition for cppcheck Host doxygen documentation Reviewers: teon.banek Reviewed By: teon.banek Subscribers: pullbot Differential Revision: https://phabricator.memgraph.io/D1095 --- Doxyfile | 4 +- apollo_archives.py | 39 +++++++++ apollo_archives.yaml | 21 +++++ apollo_build.yaml | 84 ++++++++++++++++++++ tests/apollo_runs.py | 72 +++++++++++++++++ tests/macro_benchmark/apollo_runs.yaml | 61 ++++++++++++++ tests/public_benchmark/ldbc/apollo_runs.yaml | 15 ++++ tests/qa/apollo_runs.yaml | 8 ++ tests/stress/apollo_runs.yaml | 13 +++ tools/apollo/apollo_runs.yaml | 10 +++ tools/apollo/build_debug | 26 ------ tools/apollo/build_diff | 39 --------- tools/apollo/build_release | 29 ------- tools/apollo/cppcheck | 7 +- tools/tests/apollo_runs.py | 39 +++++++++ 15 files changed, 367 insertions(+), 100 deletions(-) create mode 100755 apollo_archives.py create mode 100644 apollo_archives.yaml create mode 100644 apollo_build.yaml create mode 100755 tests/apollo_runs.py create mode 100644 tests/macro_benchmark/apollo_runs.yaml create mode 100644 tests/public_benchmark/ldbc/apollo_runs.yaml create mode 100644 tests/qa/apollo_runs.yaml create mode 100644 tests/stress/apollo_runs.yaml create mode 100644 tools/apollo/apollo_runs.yaml create mode 100755 tools/tests/apollo_runs.py diff --git a/Doxyfile b/Doxyfile index e79509b78..7010709fe 100644 --- a/Doxyfile +++ b/Doxyfile @@ -705,7 +705,7 @@ CITE_BIB_FILES = # messages are off. # The default value is: NO. -QUIET = NO +QUIET = YES # The WARNINGS tag can be used to turn on/off the warning messages that are # generated to standard error (stderr) by doxygen. If WARNINGS is set to YES @@ -721,7 +721,7 @@ WARNINGS = YES # will automatically be disabled. # The default value is: YES. -WARN_IF_UNDOCUMENTED = YES +WARN_IF_UNDOCUMENTED = NO # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for # potential errors in the documentation, such as not documenting some parameters diff --git a/apollo_archives.py b/apollo_archives.py new file mode 100755 index 000000000..692f34df0 --- /dev/null +++ b/apollo_archives.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +import json +import os +import re +import subprocess +import sys + +# paths +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +BUILD_OUTPUT_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "build", "output")) + +# helpers +def run_cmd(cmd, cwd): + return subprocess.run(cmd, cwd=cwd, check=True, + stdout=subprocess.PIPE).stdout.decode("utf-8") + +# check project +if re.search(r"release", os.environ.get("PROJECT", "")) is None: + print(json.dumps([])) + sys.exit(0) + +# generate archive +deb_name = run_cmd(["find", ".", "-maxdepth", "1", "-type", "f", + "-name", "memgraph*.deb"], BUILD_OUTPUT_DIR).split("\n")[0][2:] +arch = run_cmd(["dpkg", "--print-architecture"], BUILD_OUTPUT_DIR).split("\n")[0] +version = deb_name.split("-")[1] +# generate Debian package file name as expected by Debian Policy +standard_deb_name = "memgraph_{}-1_{}.deb".format(version, arch) + +archive_path = os.path.relpath(os.path.join(BUILD_OUTPUT_DIR, + deb_name), SCRIPT_DIR) + +archives = [{ + "name": "Release (deb package)", + "archive": archive_path, + "filename": standard_deb_name, +}] + +print(json.dumps(archives, indent=4, sort_keys=True)) diff --git a/apollo_archives.yaml b/apollo_archives.yaml new file mode 100644 index 000000000..57e611ddc --- /dev/null +++ b/apollo_archives.yaml @@ -0,0 +1,21 @@ +- name: Binary + archive: + - build/memgraph + - config + filename: binary.tar.gz + +- name: Doxygen documentation + cd: docs/doxygen/html + archive: + - . + filename: documentation.tar.gz + host: true + +- name: Release (tarball) + project: release + archive: build/output/memgraph*.tar.gz + +# - name: Release (user docs) +# project: release +# archive: docs/user_technical/docs.tar.gz +# filename: release_user_docs.tar.gz diff --git a/apollo_build.yaml b/apollo_build.yaml new file mode 100644 index 000000000..301b7d91d --- /dev/null +++ b/apollo_build.yaml @@ -0,0 +1,84 @@ +- name: Diff build + project: ^mg-master-diff$ + commands: | + cd .. + cp -r memgraph parent + cd memgraph + + TIMEOUT=600 ./init + doxygen Doxyfile + + cd build + cmake -DTOOLS=ON -DTEST_COVERAGE=ON .. + TIMEOUT=1000 make -j$THREADS + + cd .. + mkdir build_release + + cd build_release + cmake -DCMAKE_BUILD_TYPE=release .. + TIMEOUT=1000 make -j$THREADS memgraph memgraph__macro_benchmark memgraph__stress + + cd ../../parent + + git checkout HEAD~1 + TIMEOUT=600 ./init + + mkdir build_release + cd build_release + cmake -DCMAKE_BUILD_TYPE=release .. + TIMEOUT=1000 make -j$THREADS memgraph memgraph__macro_benchmark + + # Install tools, because they may be needed to run some benchmarks and tests. + cd ../../memgraph/tools + TIMEOUT=300 ./setup + + +- name: Debug build + project: ^mg-master-debug$ + commands: | + TIMEOUT=600 ./init + doxygen Doxyfile + + cd build + cmake -DTOOLS=ON .. + TIMEOUT=1000 make -j$THREADS + + cd .. + mkdir build_release + cd build_release + + cmake -DCMAKE_BUILD_TYPE=release .. + TIMEOUT=1000 make -j$THREADS memgraph memgraph__macro_benchmark memgraph__stress + + # Install tools, because they may be needed to run some benchmarks and tests. + cd ../tools + TIMEOUT=300 ./setup + + +# release build is the default one +- name: Release build + commands: | + TIMEOUT=600 ./init + doxygen Doxyfile + cd build + cmake -DTOOLS=ON -DCMAKE_BUILD_TYPE=Release -DUSE_READLINE=OFF .. + TIMEOUT=1000 make -j$THREADS + + # Create a binary package (which can then be used for Docker image). + mkdir output + cd output + cpack -D CPACK_SET_DESTDIR=ON -G TGZ --config ../CPackConfig.cmake + cpack -G DEB --config ../CPackConfig.cmake + + # Create user technical documentation for community site + cd ../../docs/user_technical + #./bundle_community + + # Install tools, because they may be needed to run some benchmarks and tests. + cd ../../tools + TIMEOUT=300 ./setup + + # Link build to build_release + cd .. + ln -s build build_release diff --git a/tests/apollo_runs.py b/tests/apollo_runs.py new file mode 100755 index 000000000..3374f4a59 --- /dev/null +++ b/tests/apollo_runs.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +import json +import os +import re +import subprocess + +# paths +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +WORKSPACE_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "..")) +TESTS_DIR_REL = os.path.join("..", "build", "tests") +TESTS_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, TESTS_DIR_REL)) + +# generation mode +mode = "release" +if os.environ.get("PROJECT", "") == "mg-master-diff": + mode = "diff" + +# ctest tests +ctest_output = subprocess.run(["ctest", "-N"], cwd=TESTS_DIR, check=True, + stdout=subprocess.PIPE).stdout.decode("utf-8") +tests = [] + +# test ordering: first unit, then concurrent, then everything else +CTEST_ORDER = {"unit": 0, "concurrent": 1} +CTEST_DELIMITER = "__" +for row in ctest_output.split("\n"): + # Filter rows only containing tests. + if not re.match("^\s*Test\s+#", row): continue + test_name = row.split(":")[1].strip() + name = test_name.replace("memgraph" + CTEST_DELIMITER, "") + path = os.path.join(TESTS_DIR_REL, name.replace(CTEST_DELIMITER, "/", 1)) + order = CTEST_ORDER.get(name.split(CTEST_DELIMITER)[0], len(CTEST_ORDER)) + tests.append((order, name, path)) + +tests.sort() + +runs = [] +for test in tests: + order, name, path = test + dirname, basename = os.path.split(path) + cmakedir = os.path.join("CMakeFiles", + "memgraph" + CTEST_DELIMITER + name + ".dir") + files = [basename, cmakedir] + + # extra files for specific tests + if name == "unit__fswatcher": + files.append(os.path.join("..", "data")) + + # skip benchmark tests on diffs + if name.startswith("benchmark") and mode == "diff": + continue + + # larger timeout for benchmark tests + prefix = "" + if name.startswith("benchmark"): + prefix = "TIMEOUT=600 " + + outfile_paths = [] + if name.startswith("unit"): + cmakedir_abs = os.path.join(TESTS_DIR, "unit", cmakedir) + cmakedir_rel = os.path.relpath(cmakedir_abs, WORKSPACE_DIR) + outfile_paths.append("\./" + cmakedir_rel.replace(".", "\\.") + ".+") + + runs.append({ + "name": name, + "cd": dirname, + "commands": prefix + "./" + basename, + "infiles": files, + "outfile_paths": outfile_paths, + }) + +print(json.dumps(runs, indent=4, sort_keys=True)) diff --git a/tests/macro_benchmark/apollo_runs.yaml b/tests/macro_benchmark/apollo_runs.yaml new file mode 100644 index 000000000..633bbbcfb --- /dev/null +++ b/tests/macro_benchmark/apollo_runs.yaml @@ -0,0 +1,61 @@ +- name: macro_benchmark__query_suite + supervisor: ./harness + arguments: &QUERY_SUITE_ARGS > + QuerySuite MemgraphRunner + --groups aggregation 1000_create unwind_create dense_expand match + --no-strict --database-cpu-ids 1 --client-cpu-ids 2 + infiles: &MACRO_BENCHMARK_INFILES + - . # current directory + - ../../build_release/memgraph # memgraph release binary + - ../../config # directory with config files + - ../../build_release/tests/macro_benchmark # macro benchmark client binaries + - ../../libs/postgresql/lib # postgresql libs dir (for client binaries) + outfile_paths: &MACRO_BENCHMARK_OUTFILE_PATHS + - \./memgraph/tests/macro_benchmark/\.harness_summary + +- name: macro_benchmark__query_parallel_suite + supervisor: ./harness + arguments: &QUERY_PARALLEL_SUITE_ARGS > + QueryParallelSuite MemgraphRunner + --groups aggregation_parallel create_parallel bfs_parallel + --database-cpu-ids 1 2 3 4 5 6 7 8 9 + --client-cpu-ids 10 11 12 13 14 15 16 17 18 19 + --num-database-workers 9 --num-clients-workers 30 + --no-strict + infiles: *MACRO_BENCHMARK_INFILES + outfile_paths: *MACRO_BENCHMARK_OUTFILE_PATHS + slave_group: &QUERY_PARALLEL_SUITE_SLAVE_GROUP remote_20c140g + +- name: macro_benchmark_parent__query_suite + project: ^mg-master-diff$ # regex to match only 'mg-master-diff' + cd: ../../../parent/tests/macro_benchmark # all files are relative to the parent build + supervisor: ./harness + arguments: *QUERY_SUITE_ARGS + infiles: *MACRO_BENCHMARK_INFILES + outfile_paths: &MACRO_BENCHMARK_PARENT_OUTFILE_PATHS + - \./parent/tests/macro_benchmark/\.harness_summary + link_to_run: macro_benchmark__query_suite + +- name: macro_benchmark_parent__query_parallel_suite + project: ^mg-master-diff$ # regex to match only 'mg-master-diff' + cd: ../../../parent/tests/macro_benchmark # all files are relative to the parent build + supervisor: ./harness + arguments: *QUERY_PARALLEL_SUITE_ARGS + infiles: *MACRO_BENCHMARK_INFILES + outfile_paths: *MACRO_BENCHMARK_PARENT_OUTFILE_PATHS + link_to_run: macro_benchmark__query_parallel_suite + slave_group: *QUERY_PARALLEL_SUITE_SLAVE_GROUP + +- name: macro_benchmark_summary + project: ^mg-master-diff$ # regex to match only 'mg-master-diff' + type: data process + cd: ../../.. + commands: > + ./memgraph/tools/apollo/macro_benchmark_summary + --current macro_benchmark__query_suite/memgraph/tests/macro_benchmark/.harness_summary macro_benchmark__query_parallel_suite/memgraph/tests/macro_benchmark/.harness_summary + --previous macro_benchmark_parent__query_suite/parent/tests/macro_benchmark/.harness_summary macro_benchmark_parent__query_parallel_suite/parent/tests/macro_benchmark/.harness_summary + --output .harness_summary + infiles: + - memgraph/tools/apollo/macro_benchmark_summary + outfile_paths: + - \./.harness_summary diff --git a/tests/public_benchmark/ldbc/apollo_runs.yaml b/tests/public_benchmark/ldbc/apollo_runs.yaml new file mode 100644 index 000000000..1f7cd3d3b --- /dev/null +++ b/tests/public_benchmark/ldbc/apollo_runs.yaml @@ -0,0 +1,15 @@ +- name: public_benchmark__ldbc + project: release + commands: . continuous_integration + infiles: + - . # current directory + - ../../../build_release/memgraph # memgraph release binary + - ../../../config # directory with config files + - ../../../libs/neo4j # neo4j directory + - ../../../tools/mg_import_csv # memgraph csv import tool + - ../../../tools/plot_ldbc_latency # ldbc plot generation tool + outfile_paths: # TODO: maybe this should also accept relative paths? + - \./memgraph/tests/public_benchmark/ldbc/results/.+ + - \./memgraph/tests/public_benchmark/ldbc/plots/.+ + slave_group: remote_20c140g + enable_network: true diff --git a/tests/qa/apollo_runs.yaml b/tests/qa/apollo_runs.yaml new file mode 100644 index 000000000..f818fafb4 --- /dev/null +++ b/tests/qa/apollo_runs.yaml @@ -0,0 +1,8 @@ +- name: quality_assurance + commands: TIMEOUT=300 ./continuous_integration + infiles: + - . # current directory + - ../../build/memgraph # memgraph debug binary + - ../../config # directory with config files + outfile_paths: + - \./memgraph/tests/qa/\.quality_assurance_status diff --git a/tests/stress/apollo_runs.yaml b/tests/stress/apollo_runs.yaml new file mode 100644 index 000000000..91001b01e --- /dev/null +++ b/tests/stress/apollo_runs.yaml @@ -0,0 +1,13 @@ +- name: stress + commands: TIMEOUT=600 ./continuous_integration + infiles: &STRESS_INFILES + - . # current directory + - ../../build_release/memgraph # memgraph release binary + - ../../config # directory with config files + - ../../build_release/tests/stress # stress client binaries + +- name: stress_large + project: release + commands: TIMEOUT=43200 ./continuous_integration --large-dataset + infiles: *STRESS_INFILES + slave_group: remote_16c56g diff --git a/tools/apollo/apollo_runs.yaml b/tools/apollo/apollo_runs.yaml new file mode 100644 index 000000000..c1bf1a497 --- /dev/null +++ b/tools/apollo/apollo_runs.yaml @@ -0,0 +1,10 @@ +- name: cppcheck + commands: TIMEOUT=2400 ./cppcheck + infiles: + - cppcheck # cppcheck script + - ../../src # src source dir + - ../../tests # tests source dir + - ../../poc # poc source dir + - ../../.git # git directory (used to find out changed files in commit) + outfile_paths: + - \./memgraph/tools/apollo/\.cppcheck_errors diff --git a/tools/apollo/build_debug b/tools/apollo/build_debug index ad18e248a..8b1378917 100644 --- a/tools/apollo/build_debug +++ b/tools/apollo/build_debug @@ -1,27 +1 @@ -# WARNING: do not run this script without defining THREADS! -# If THREADS isn't defined then this script will call 'make -j'. -# From the manpage: "If the -j option is given without an argument, make will not limit the number of jobs that can run simultaneously." -# That means that the whole build will be started simultaneously and IT WILL CRASH YOUR COMPUTER! -cd ../.. - -TIMEOUT=600 ./init -bash -c "doxygen Doxyfile >/dev/null 2>/dev/null" - -cd build -cmake -DTOOLS=ON .. -TIMEOUT=1000 make -j$THREADS - -cd .. -mkdir build_release -cd build_release - -cmake -DCMAKE_BUILD_TYPE=release .. -TIMEOUT=1000 make -j$THREADS memgraph memgraph__macro_benchmark memgraph__stress - -# Install tools, because they may be needed to run some benchmarks and tests. -cd ../tools -TIMEOUT=300 ./setup - -cd apollo -TIMEOUT=300 ./generate debug diff --git a/tools/apollo/build_diff b/tools/apollo/build_diff index 67376d8b7..8b1378917 100644 --- a/tools/apollo/build_diff +++ b/tools/apollo/build_diff @@ -1,40 +1 @@ -# WARNING: do not run this script without defining THREADS! -# If THREADS isn't defined then this script will call 'make -j'. -# From the manpage: "If the -j option is given without an argument, make will not limit the number of jobs that can run simultaneously." -# That means that the whole build will be started simultaneously and IT WILL CRASH YOUR COMPUTER! -cd ../../.. - -cp -r memgraph parent - -cd memgraph - -TIMEOUT=600 ./init -bash -c "doxygen Doxyfile >/dev/null 2>/dev/null" - -cd build -cmake -DTOOLS=ON -DTEST_COVERAGE=ON .. -TIMEOUT=1000 make -j$THREADS - -cd .. -mkdir build_release - -cd build_release -cmake -DCMAKE_BUILD_TYPE=release .. -TIMEOUT=1000 make -j$THREADS memgraph memgraph__macro_benchmark memgraph__stress - -cd ../../parent - -git checkout HEAD~1 -TIMEOUT=600 ./init - -cd build -cmake -DCMAKE_BUILD_TYPE=release .. -TIMEOUT=1000 make -j$THREADS memgraph memgraph__macro_benchmark - -# Install tools, because they may be needed to run some benchmarks and tests. -cd ../../memgraph/tools -TIMEOUT=300 ./setup - -cd apollo -TIMEOUT=300 ./generate diff diff --git a/tools/apollo/build_release b/tools/apollo/build_release index e65a06942..8b1378917 100644 --- a/tools/apollo/build_release +++ b/tools/apollo/build_release @@ -1,30 +1 @@ -# WARNING: do not run this script without defining THREADS! -# If THREADS isn't defined then this script will call 'make -j'. -# From the manpage: "If the -j option is given without an argument, make will not limit the number of jobs that can run simultaneously." -# That means that the whole build will be started simultaneously and IT WILL CRASH YOUR COMPUTER! -cd ../.. - -TIMEOUT=600 ./init -bash -c "doxygen Doxyfile >/dev/null 2>/dev/null" - -cd build -cmake -DTOOLS=ON -DCMAKE_BUILD_TYPE=Release -DUSE_READLINE=OFF .. -TIMEOUT=1000 make -j$THREADS - -# Create a binary package (which can then be used for Docker image). -mkdir output -cd output -cpack -D CPACK_SET_DESTDIR=ON -G TGZ --config ../CPackConfig.cmake -cpack -G DEB --config ../CPackConfig.cmake - -# Create user technical documentation for community site -cd ../../docs/user_technical -./bundle_community - -# Install tools, because they may be needed to run some benchmarks and tests. -cd ../../tools -TIMEOUT=300 ./setup - -cd apollo -TIMEOUT=300 ./generate release diff --git a/tools/apollo/cppcheck b/tools/apollo/cppcheck index 96208fbe1..3fe374c79 100755 --- a/tools/apollo/cppcheck +++ b/tools/apollo/cppcheck @@ -6,10 +6,9 @@ cd "$DIR/../../" tmpfile="$DIR/.cppcheck_errors.tmp" errfile="$DIR/.cppcheck_errors" -if [ "$1" == "" ]; then - mode=diff -else - mode=$1 +mode=diff +if echo "$PROJECT" | grep "release" >/dev/null 2>/dev/null; then + mode=release fi if [ "$mode" == diff ]; then diff --git a/tools/tests/apollo_runs.py b/tools/tests/apollo_runs.py new file mode 100755 index 000000000..6b43e3e37 --- /dev/null +++ b/tools/tests/apollo_runs.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +import json +import os +import re +import subprocess + +# paths +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +TESTS_DIR_REL = os.path.join("..", "..", "build", "tools", "tests") +TESTS_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, TESTS_DIR_REL)) + +# ctest tests +ctest_output = subprocess.run(["ctest", "-N"], cwd=TESTS_DIR, check=True, + stdout=subprocess.PIPE).stdout.decode("utf-8") + +runs = [] + +# test ordering: first unit, then concurrent, then everything else +for row in ctest_output.split("\n"): + # Filter rows only containing tests. + if not re.match("^\s*Test\s+#", row): continue + name = row.split(":")[1].strip() + path = os.path.join(TESTS_DIR_REL, name) + + dirname, basename = os.path.split(path) + files = [basename, "CTestTestfile.cmake"] + + # extra files for specific tests + if name == "test_mg_import_csv": + files.extend(["csv", "mg_recovery_check", "../src/mg_import_csv"]) + + runs.append({ + "name": "tools__" + name, + "cd": dirname, + "commands": "ctest --output-on-failure -R \"^{}$\"".format(name), + "infiles": files, + }) + +print(json.dumps(runs, indent=4, sort_keys=True))