Add community to Apollo build

Reviewers: teon.banek Reviewed By: teon.banek Subscribers: pullbot Differential Revision: https://phabricator.memgraph.io/D2671
2020-02-14 11:04:14 +01:00 · 2020-02-14 11:04:14 +01:00 · dfc546183d
commit dfc546183d
parent 5953f07be3
7 changed files with 165 additions and 84 deletions
--- a/apollo_archives.py
+++ b/apollo_archives.py
@ -3,17 +3,27 @@ import json
 import os

 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-BUILD_OUTPUT_DIR = os.path.join("build_release", "output")
+
+
+def find_packages(build_output_dir):
+    ret = []
+    output_dir = os.path.join(SCRIPT_DIR, build_output_dir)
+    if os.path.exists(output_dir):
+        for fname in os.listdir(output_dir):
+            if fname.startswith("memgraph") and fname.endswith(".deb"):
+                path = os.path.join(build_output_dir, fname)
+                ret.append({
+                    "name": "Release " + fname.split("_")[1] +
+                            " (deb package)",
+                    "archive": path,
+                })
+    return ret
+

 archives = []
-output_dir = os.path.join(SCRIPT_DIR, BUILD_OUTPUT_DIR)
-if os.path.exists(output_dir):
-    for fname in os.listdir(output_dir):
-        if fname.startswith("memgraph") and fname.endswith(".deb"):
-            path = os.path.join(BUILD_OUTPUT_DIR, fname)
-            archives = [{
-                "name": "Release " + fname.split("_")[1] + " (deb package)",
-                "archive": path,
-            }]
+# Find enterprise package(s).
+archives += find_packages(os.path.join("build_release", "output"))
+# Find community package(s).
+archives += find_packages(os.path.join("build_community", "output"))

 print(json.dumps(archives, indent=4, sort_keys=True))
--- a/apollo_archives.yaml
+++ b/apollo_archives.yaml
@ -1,11 +1,12 @@
 - name: Binaries
  archive:
    - build_debug/memgraph
-    #- build_debug/memgraph_ha
+    - build_debug/config/memgraph.conf
    - build_release/memgraph
-    #- build_release/memgraph_ha
+    - build_release/config/memgraph.conf
    - build_release/tools/src/mg_client
-    - config
+    - build_community/memgraph
+    - build_community/config/memgraph.conf
  filename: binaries.tar.gz

 - name: Doxygen documentation
--- a/apollo_build.yaml
+++ b/apollo_build.yaml
@ -36,6 +36,13 @@
    cd build_release
    cmake -DCMAKE_BUILD_TYPE=release ..
    TIMEOUT=1200 make -j$THREADS
+
+    # Build community binaries.
+    cd ..
+    mkdir build_community
+    cd build_community
+    cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
+    TIMEOUT=1200 make -j$THREADS
    cd ..

    # Checkout to parent commit and initialize.
@ -88,3 +95,16 @@
    mkdir output
    cd output
    cpack -G DEB --config ../CPackConfig.cmake
+    cd ..
+
+    # Build community binaries.
+    cd ..
+    mkdir build_community
+    cd build_community
+    cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
+    TIMEOUT=1200 make -j$THREADS
+
+    # Create Debian package.
+    mkdir output
+    cd output
+    cpack -G DEB --config ../CPackConfig.cmake
--- a/tests/apollo_runs.py
+++ b/tests/apollo_runs.py
@ -10,72 +10,107 @@ WORKSPACE_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", ".."))
 TESTS_DIR_REL = os.path.join("..", "build_debug", "tests")
 TESTS_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, TESTS_DIR_REL))

+# test ordering: first unit, then concurrent, then everything else
+CTEST_ORDER = {"unit": 0, "concurrent": 1}
+CTEST_DELIMITER = "__"
+
+
+def get_runs(build_dir, include=None, exclude=None, outfile=None,
+             name_prefix=""):
+    tests_dir = os.path.join(build_dir, "tests")
+    tests_dir_abs = os.path.normpath(os.path.join(SCRIPT_DIR, tests_dir))
+    ctest_output = subprocess.run(
+        ["ctest", "-N"], cwd=tests_dir_abs, check=True,
+        stdout=subprocess.PIPE).stdout.decode("utf-8")
+    tests = []
+
+    for row in ctest_output.split("\n"):
+        # Filter rows only containing tests.
+        if not re.match("^\s*Test\s+#", row):
+            continue
+        if not row.count("memgraph"):
+            continue
+        test_name = row.split(":")[1].strip()
+        name = test_name.replace("memgraph" + CTEST_DELIMITER, "")
+        path = os.path.join(
+            tests_dir, name.replace(CTEST_DELIMITER, "/", 1))
+        order = CTEST_ORDER.get(
+            name.split(CTEST_DELIMITER)[0], len(CTEST_ORDER))
+        tests.append((order, name, path))
+
+    tests.sort()
+
+    runs = []
+    for test in tests:
+        order, name, path = test
+        dirname, basename = os.path.split(path)
+        files = [basename]
+
+        # check whether the test should be included
+        if include is not None:
+            should_include = False
+            for inc in include:
+                if name.startswith(inc):
+                    should_include = True
+                    break
+            if not should_include:
+                continue
+
+        # check whether the test should be excluded
+        if exclude is not None:
+            should_exclude = False
+            for exc in exclude:
+                if name.startswith(exc):
+                    should_exclude = True
+                    break
+            if should_exclude:
+                continue
+
+        # larger timeout for benchmark and concurrent tests
+        prefix = ""
+        if name.startswith("benchmark") or name.startswith("concurrent"):
+            prefix = "TIMEOUT=600 "
+
+        # larger timeout for storage_v2_durability unit test
+        if name.endswith("storage_v2_durability"):
+            prefix = "TIMEOUT=300 "
+
+        # get output files
+        outfile_paths = []
+        if outfile:
+            curdir_abs = os.path.normpath(os.path.join(SCRIPT_DIR, dirname))
+            curdir_rel = os.path.relpath(curdir_abs, WORKSPACE_DIR)
+            outfile_paths.append("\./" + curdir_rel.replace(".", "\\.") + "/" +
+                                 outfile.replace(".", "\\."))
+
+        runs.append({
+            "name": name_prefix + name,
+            "cd": dirname,
+            "commands": prefix + "./" + basename,
+            "infiles": files,
+            "outfile_paths": outfile_paths,
+        })
+    return runs
+
+
 # generation mode
 mode = "release"
 if os.environ.get("PROJECT", "") == "mg-master-diff":
    mode = "diff"

-# ctest tests
-ctest_output = subprocess.run(["ctest", "-N"], cwd=TESTS_DIR, check=True,
-        stdout=subprocess.PIPE).stdout.decode("utf-8")
-tests = []
+# get unit tests
+runs = get_runs("../build_coverage", include=["unit"],
+                outfile="default.profraw")

-# test ordering: first unit, then concurrent, then everything else
-CTEST_ORDER = {"unit": 0, "concurrent": 1}
-CTEST_DELIMITER = "__"
-for row in ctest_output.split("\n"):
-    # Filter rows only containing tests.
-    if not re.match("^\s*Test\s+#", row): continue
-    if not row.count("memgraph"): continue
-    test_name = row.split(":")[1].strip()
-    name = test_name.replace("memgraph" + CTEST_DELIMITER, "")
-    path = os.path.join(TESTS_DIR_REL, name.replace(CTEST_DELIMITER, "/", 1))
-    order = CTEST_ORDER.get(name.split(CTEST_DELIMITER)[0], len(CTEST_ORDER))
-    tests.append((order, name, path))
+# get all other tests except unit and benchmark
+runs += get_runs("../build_debug", exclude=["unit", "benchmark"])

-tests.sort()
+# get benchmark tests
+if mode != "diff":
+    runs += get_runs("../build_release", include=["benchmark"])

-runs = []
-for test in tests:
-    order, name, path = test
-    dirname, basename = os.path.split(path)
-    files = [basename]
-
-    # skip benchmark tests on diffs
-    if name.startswith("benchmark") and mode == "diff":
-        continue
-
-    # larger timeout for benchmark and concurrent tests
-    prefix = ""
-    if name.startswith("benchmark") or name.startswith("concurrent"):
-        prefix = "TIMEOUT=600 "
-
-    # larger timeout for storage_v2_durability unit test
-    if name.endswith("storage_v2_durability"):
-        prefix = "TIMEOUT=300 "
-
-    outfile_paths = []
-    if name.startswith("unit"):
-        dirname = dirname.replace("/build_debug/", "/build_coverage/")
-        curdir_abs = os.path.normpath(os.path.join(SCRIPT_DIR, dirname))
-        curdir_rel = os.path.relpath(curdir_abs, WORKSPACE_DIR)
-        outfile_paths.append("\./" + curdir_rel.replace(".", "\\.") + "/default\\.profraw")
-    elif name.startswith("benchmark"):
-        dirname = dirname.replace("/build_debug/", "/build_release/")
-
-    runs.append({
-        "name": name,
-        "cd": dirname,
-        "commands": prefix + "./" + basename,
-        "infiles": files,
-        "outfile_paths": outfile_paths,
-    })
-
-runs.append({
-    "name": "test_lcp",
-    "cd": os.path.join(TESTS_DIR_REL, "unit"),
-    "commands": "./test_lcp",
-    "infiles": ["test_lcp"],
-})
+# get community tests
+runs += get_runs("../build_community", include=["unit"],
+                 name_prefix="community__")

 print(json.dumps(runs, indent=4, sort_keys=True))
--- a/tests/apollo_runs.yaml
+++ b/tests/apollo_runs.yaml
@ -0,0 +1,5 @@
+- name: test_lcp
+  cd: ../build_debug/tests/unit
+  commands: ./test_lcp
+  infiles:
+    - test_lcp
--- a/tests/stress/apollo_runs.yaml
+++ b/tests/stress/apollo_runs.yaml
@ -3,7 +3,6 @@
  infiles: &STRESS_INFILES
    - . # current directory
    - ../../build_release/memgraph # memgraph release binary
-    - ../../config # directory with config files
    - ../../build_release/tests/stress # stress client binaries

 - name: stress_ssl
@ -15,6 +14,18 @@
  commands: TIMEOUT=43200 ./continuous_integration --large-dataset
  infiles: *STRESS_INFILES

+- name: community__stress
+  commands: TIMEOUT=600 ./continuous_integration
+  infiles: &COMMUNITY_STRESS_INFILES
+    - . # current directory
+    - ../../build_community/memgraph # memgraph community binary
+    - ../../build_community/tests/stress # stress client binaries
+
+- name: community__stress_large
+  project: release
+  commands: TIMEOUT=43200 ./continuous_integration --large-dataset
+  infiles: *COMMUNITY_STRESS_INFILES
+
 #- name: stress_ha_normal_operation
 #  commands: TIMEOUT=200 ./continuous_integration_ha
 #  infiles: &STRESS_HA_INFILES
@ -32,7 +43,6 @@
  infiles: &DURABILITY_INFILES
    - . # current directory
    - ../../build_release/memgraph # memgraph release binary
-    - ../../config # directory with config files

 - name: durability_large
  project: release
--- a/tests/stress/continuous_integration
+++ b/tests/stress/continuous_integration
@ -70,7 +70,6 @@ LARGE_DATASET = [
 # paths
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 BASE_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", ".."))
-BUILD_DIR = os.path.join(BASE_DIR, "build")
 MEASUREMENTS_FILE = os.path.join(SCRIPT_DIR, ".apollo_measurements")
 KEY_FILE = os.path.join(SCRIPT_DIR, ".key.pem")
 CERT_FILE = os.path.join(SCRIPT_DIR, ".cert.pem")
@ -87,6 +86,14 @@ else:
    THREADS = multiprocessing.cpu_count()


+def get_build_dir():
+    if os.path.exists(os.path.join(BASE_DIR, "build_release")):
+        return os.path.join(BASE_DIR, "build_release")
+    if os.path.exists(os.path.join(BASE_DIR, "build_community")):
+        return os.path.join(BASE_DIR, "build_community")
+    return os.path.join(BASE_DIR, "build")
+
+
 def wait_for_server(port, delay=0.1):
    cmd = ["nc", "-z", "-w", "1", "127.0.0.1", str(port)]
    while subprocess.call(cmd) != 0:
@ -104,10 +111,7 @@ def run_test(args, test, options, timeout):
        binary = [args.python, "-u", os.path.join(SCRIPT_DIR, test),
                "--logging", logging]
    elif test.endswith(".cpp"):
-        exe = os.path.join(BUILD_DIR, "tests", "stress", test[:-4])
-        if not os.path.exists(exe):
-            exe = os.path.join(BASE_DIR, "build_release", "tests", "stress",
-                    test[:-4])
+        exe = os.path.join(get_build_dir(), "tests", "stress", test[:-4])
        binary = [exe]
    else:
        raise Exception("Test '{}' binary not supported!".format(test))
@ -129,7 +133,7 @@ def run_test(args, test, options, timeout):

 # parse arguments
 parser = argparse.ArgumentParser(description = "Run stress tests on Memgraph.")
-parser.add_argument("--memgraph", default = os.path.join(BUILD_DIR,
+parser.add_argument("--memgraph", default = os.path.join(get_build_dir(),
        "memgraph"))
 parser.add_argument("--log-file", default = "")
 parser.add_argument("--data-directory", default = "")
@ -143,10 +147,6 @@ parser.add_argument("--verbose", action = "store_const",
        const = True, default = False)
 args = parser.parse_args()

-# find memgraph binary
-if not os.path.exists(args.memgraph):
-    args.memgraph = os.path.join(BASE_DIR, "build_release", "memgraph")
-
 # generate temporary SSL certs
 if args.use_ssl:
    # https://unix.stackexchange.com/questions/104171/create-ssl-certificate-non-interactively