From ffbe5b449da86ba39e66ba86cdff1711ff2da5de Mon Sep 17 00:00:00 2001 From: Matej Ferencevic Date: Tue, 11 Feb 2020 16:39:54 +0100 Subject: [PATCH] Implement version names for each Memgraph version Summary: With this diff, each build of Memgraph has a version that uniquely identifies it. The given version uniquely identifies both official release builds and development builds. Enterprise/community builds are also differentiated in the version. Also, support for custom suffixes is added to support custom builds for customers. Reviewers: teon.banek Reviewed By: teon.banek Subscribers: pullbot Differential Revision: https://phabricator.memgraph.io/D2662 --- CMakeLists.txt | 98 +++++++++++++-- apollo_archives.py | 42 ++----- query_modules/CMakeLists.txt | 2 +- release/CMakeLists.txt | 9 +- release/README.md | 28 +++++ release/get_version.py | 228 +++++++++++++++++++++++++++++++++++ src/CMakeLists.txt | 6 +- tools/CMakeLists.txt | 2 +- tools/src/CMakeLists.txt | 2 +- 9 files changed, 366 insertions(+), 51 deletions(-) create mode 100755 release/get_version.py diff --git a/CMakeLists.txt b/CMakeLists.txt index d6996dc98..2d47ec8bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,16 +35,97 @@ else() message(FATAL_ERROR "Couldn't find clang and/or clang++!") endif() -# Get current commit hash. -execute_process( - OUTPUT_VARIABLE COMMIT_HASH - COMMAND git rev-parse --short HEAD -) -string(STRIP ${COMMIT_HASH} COMMIT_HASH) - # ----------------------------------------------------------------------------- -project(memgraph VERSION 0.50.0) +project(memgraph) + +# For more information about how to release a new version of Memgraph, see +# `release/README.md`. + +# Option that is used to specify which version of Memgraph should be built. The +# default is `ON` which causes the build system to build Memgraph Enterprise. +# Memgraph Community is built if explicitly set to `OFF`. +option(MG_ENTERPRISE "Build Memgraph Enterprise Edition" ON) + +# Set the current version here to override the automatic version detection. The +# version must be specified as `X.Y.Z`. Primarily used when building new patch +# versions. +set(MEMGRAPH_OVERRIDE_VERSION "") + +# Custom suffix that this version should have. The suffix can be any arbitrary +# string. Primarily used when building a version for a specific customer. +set(MEMGRAPH_OVERRIDE_VERSION_SUFFIX "") + +# Variables used to generate the versions. +if (MG_ENTERPRISE) + set(get_version_enterprise "--enterprise") +else() + set(get_version_enterprise "") +endif() +set(get_version_script "${CMAKE_SOURCE_DIR}/release/get_version.py") + +# Get version that should be used in the binary. +execute_process( + OUTPUT_VARIABLE MEMGRAPH_VERSION + COMMAND "${get_version_script}" ${get_version_enterprise} + "${MEMGRAPH_OVERRIDE_VERSION}" + "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}" +) + +# Get version that should be used in the DEB package. +execute_process( + OUTPUT_VARIABLE MEMGRAPH_VERSION_DEB + COMMAND "${get_version_script}" ${get_version_enterprise} + --variant deb + "${MEMGRAPH_OVERRIDE_VERSION}" + "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}" +) + +# Get version that should be used in the RPM package. +execute_process( + OUTPUT_VARIABLE MEMGRAPH_VERSION_RPM + COMMAND "${get_version_script}" ${get_version_enterprise} + --variant rpm + "${MEMGRAPH_OVERRIDE_VERSION}" + "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}" +) + +# We want the above variables to be updated each time something is committed to +# the repository. That is why we include a dependency on the current git HEAD +# to trigger a new CMake run when the git repository state changes. This is a +# hack, as CMake doesn't have a mechanism to regenerate variables when +# something changes (only files can be regenerated). +# https://cmake.org/pipermail/cmake/2018-October/068389.html +# +# The hack in the above link is nearly correct but it has a fatal flaw. The +# `CMAKE_CONFIGURE_DEPENDS` isn't a `GLOBAL` property, it is instead a +# `DIRECTORY` property and as such must be set in the `DIRECTORY` scope. +# https://cmake.org/cmake/help/v3.14/manual/cmake-properties.7.html +# +# Unlike the above mentioned hack, we don't use the `.git/index` file. That +# file changes on every `git add` (even on `git status`) so it triggers +# unnecessary recalculations of the release version. The release version only +# changes on every `git commit` or `git checkout`. That is why we watch the +# following files for changes: +# - `.git/HEAD` -> changes each time a `git checkout` is issued +# - `.git/refs/heads/...` -> the value in `.git/HEAD` is a branch name (when +# you are on a branch) and you have to monitor the file of the specific +# branch to detect when a `git commit` was issued +# More details about the contents of the `.git` directory and the specific +# files used can be seen here: +# https://git-scm.com/book/en/v2/Git-Internals-Git-References +set(git_directory "${CMAKE_SOURCE_DIR}/.git") +if (EXISTS "${git_directory}") + set_property(DIRECTORY APPEND PROPERTY + CMAKE_CONFIGURE_DEPENDS "${git_directory}/HEAD") + file(STRINGS "${git_directory}/HEAD" git_head_data) + if (git_head_data MATCHES "^ref: ") + string(SUBSTRING "${git_head_data}" 5 -1 git_head_ref) + set_property(DIRECTORY APPEND PROPERTY + CMAKE_CONFIGURE_DEPENDS "${git_directory}/${git_head_ref}") + endif() +endif() + # ----------------------------------------------------------------------------- # setup CMake module path, defines path for include() and find_package() @@ -134,7 +215,6 @@ set(libs_dir ${CMAKE_SOURCE_DIR}/libs) add_subdirectory(libs EXCLUDE_FROM_ALL) # Optional subproject configuration ------------------------------------------- -option(MG_ENTERPRISE "Build Memgraph Enterprise Edition" ON) option(TEST_COVERAGE "Generate coverage reports from running memgraph" OFF) option(TOOLS "Build tools binaries" ON) option(QUERY_MODULES "Build query modules containing custom procedures" ON) diff --git a/apollo_archives.py b/apollo_archives.py index fc945c24a..9f77dd718 100755 --- a/apollo_archives.py +++ b/apollo_archives.py @@ -1,39 +1,19 @@ #!/usr/bin/env python3 import json import os -import re -import subprocess -import sys -# paths SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -BUILD_OUTPUT_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "build_release", "output")) +BUILD_OUTPUT_DIR = os.path.join("build_release", "output") -# helpers -def run_cmd(cmd, cwd): - return subprocess.run(cmd, cwd=cwd, check=True, - stdout=subprocess.PIPE).stdout.decode("utf-8") - -# check project -if re.search(r"release", os.environ.get("PROJECT", "")) is None: - print(json.dumps([])) - sys.exit(0) - -# generate archive -deb_name = run_cmd(["find", ".", "-maxdepth", "1", "-type", "f", - "-name", "memgraph*.deb"], BUILD_OUTPUT_DIR).split("\n")[0][2:] -arch = run_cmd(["dpkg", "--print-architecture"], BUILD_OUTPUT_DIR).split("\n")[0] -version = deb_name.split("-")[1] -# generate Debian package file name as expected by Debian Policy -standard_deb_name = "memgraph_{}-1_{}.deb".format(version, arch) - -archive_path = os.path.relpath(os.path.join(BUILD_OUTPUT_DIR, - deb_name), SCRIPT_DIR) - -archives = [{ - "name": "Release (deb package)", - "archive": archive_path, - "filename": standard_deb_name, -}] +archives = [] +output_dir = os.path.join(SCRIPT_DIR, BUILD_OUTPUT_DIR) +if os.path.exists(output_dir): + for fname in os.listdir(output_dir): + if fname.startswith("memgraph") and fname.endswith(".deb"): + path = os.path.join(BUILD_OUTPUT_DIR, fname) + archives = [{ + "name": "Release " + fname.split("_")[1] + " (deb package)", + "archive": path, + }] print(json.dumps(archives, indent=4, sort_keys=True)) diff --git a/query_modules/CMakeLists.txt b/query_modules/CMakeLists.txt index b8b169be3..48f0ab60f 100644 --- a/query_modules/CMakeLists.txt +++ b/query_modules/CMakeLists.txt @@ -2,7 +2,7 @@ # You should use the top level CMake configuration with -DQUERY_MODULES=ON # These modules are meant to be shipped with Memgraph installation. -project(memgraph_query_modules VERSION ${memgraph_VERSION}) +project(memgraph_query_modules) disallow_in_source_build() diff --git a/release/CMakeLists.txt b/release/CMakeLists.txt index a7534cc44..334a65723 100644 --- a/release/CMakeLists.txt +++ b/release/CMakeLists.txt @@ -28,11 +28,6 @@ set(CPACK_PACKAGE_NAME memgraph) set(CPACK_PACKAGE_VENDOR "Memgraph Ltd.") set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "High performance, in-memory, transactional graph database") -set(CPACK_PACKAGE_VERSION_MAJOR ${memgraph_VERSION_MAJOR}) -set(CPACK_PACKAGE_VERSION_MINOR ${memgraph_VERSION_MINOR}) -set(CPACK_PACKAGE_VERSION_PATCH ${memgraph_VERSION_PATCH}) -set(CPACK_PACKAGE_VERSION_TWEAK ${memgraph_VERSION_TWEAK}) -set(CPACK_PACKAGE_FILE_NAME ${CPACK_PACKAGE_NAME}-${memgraph_VERSION}-${COMMIT_HASH}${CPACK_SYSTEM_NAME}) # DEB specific # Instead of using "name " format, we use "email (name)" to prevent @@ -40,6 +35,8 @@ set(CPACK_PACKAGE_FILE_NAME ${CPACK_PACKAGE_NAME}-${memgraph_VERSION}-${COMMIT_H set(CPACK_DEBIAN_PACKAGE_MAINTAINER "tech@memgraph.com (Memgraph Ltd.)") set(CPACK_DEBIAN_PACKAGE_SECTION non-free/database) set(CPACK_DEBIAN_PACKAGE_HOMEPAGE https://memgraph.com) +set(CPACK_DEBIAN_PACKAGE_VERSION "${MEMGRAPH_VERSION_DEB}") +set(CPACK_DEBIAN_FILE_NAME "memgraph_${MEMGRAPH_VERSION_DEB}_amd64.deb") if (MG_ENTERPRISE) set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/debian/enterprise/conffiles;" @@ -66,6 +63,8 @@ set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0)") # RPM specific set(CPACK_RPM_PACKAGE_URL https://memgraph.com) +set(CPACK_RPM_PACKAGE_VERSION "${MEMGRAPH_VERSION_RPM}") +set(CPACK_RPM_FILE_NAME "memgraph_${MEMGRAPH_VERSION_RPM}.x86_64.rpm") set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION /var /var/lib /var/log /etc/logrotate.d /lib /lib/systemd /lib/systemd/system /lib/systemd/system/memgraph.service) diff --git a/release/README.md b/release/README.md index 1f7c7ff4b..63fe8aa99 100644 --- a/release/README.md +++ b/release/README.md @@ -13,3 +13,31 @@ packages. * RPM package * Docker image * ArchLinux package + +## Release process + +While releasing an official version of Memgraph, there are two possible +scenarios: + * First release in new major.minor series + * Patch release in existing major.minor series + +To release a new major.minor release of Memgraph you should execute the +following steps: + 1. Land all diffs that must be in the new release + 2. Document all changes in `CHANGELOG.md` and land them + 3. From the `master` branch, create a branch named `release/X.Y` and push it + to `origin` + 4. Create the release packages triggering a `mg-master-release-branch-test` + using branch `release/X.Y` on Apollo + 5. Enjoy + +To release a new patch release in an existing major.minor series you should +execute the following steps: + 1. Checkout to the `release/X.Y` branch + 2. Cherry-pick all landed commits that should be included in the patch version + 3. Document all changes in `CHANGELOG.md` and commit them + 4. Edit the root `CMakeLists.txt` and set `MEMGRAPH_OVERRIDE_VERSION` to + `X.Y.patch` and commit the change + 5. Create the release packages triggering a `mg-master-release-branch-test` + using branch `release/X.Y` on Apollo + 6. Enjoy diff --git a/release/get_version.py b/release/get_version.py new file mode 100755 index 000000000..4073e9a10 --- /dev/null +++ b/release/get_version.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +import argparse +import re +import subprocess +import sys + + +# This script is used to determine the current version of Memgraph. The script +# determines the current version using `git` automatically. The user can also +# manually specify a version override and then the supplied version will be +# used instead of the version determined by `git`. All versions (automatically +# detected and manually specified) can have a custom version suffix added to +# them. +# +# The current version can be one of either: +# - release version +# - development version +# +# The release version is either associated with a `release/X.Y` branch +# (automatic detection) or is manually specified. When the version is +# automatically detected from the branch a `.0` is appended to the version. +# Example 1: +# - branch: release/0.50 +# - version: 0.50.0 +# Example 2: +# - manually supplied version: 0.50.1 +# - version: 0.50.1 +# +# The development version is always determined using `git` in the following +# way: +# - release version - nearest (older) `release/X.Y` branch version +# - distance from the release branch - Z commits +# - the current commit short hash +# Example: +# - release version: 0.50.0 (nearest older branch `release/0.50`) +# - distance from the release branch: 42 (commits) +# - current commit short hash: 7e1eef94 +# +# The script then uses the collected information to generate the versions that +# will be used in the binary, DEB package and RPM package. All of the versions +# have different naming conventions that have to be met and they differ among +# each other. +# +# The binary version is determined using the following two templates: +# Release version: +# -[-] +# Development version: +# +~-[-] +# Examples: +# Release version: +# 0.50.1-community +# 0.50.1-enterprise +# 0.50.1-enterprise-veryimportantcustomer +# Development version (master, 12 commits after release/0.50): +# 0.50.0+12~7e1eef94-community +# 0.50.0+12~7e1eef94-enterprise +# 0.50.0+12~7e1eef94-enterprise-veryimportantcustomer +# +# The DEB package version is determined using the following two templates: +# Release version: +# -[-]-1 +# Development version (master, 12 commits after release/0.50): +# +~-[-]-1 +# Examples: +# Release version: +# 0.50.1-community-1 +# 0.50.1-enterprise-1 +# 0.50.1-enterprise-veryimportantcustomer-1 +# Development version (master, 12 commits after release/0.50): +# 0.50.0+12~7e1eef94-community-1 +# 0.50.0+12~7e1eef94-enterprise-1 +# 0.50.0+12~7e1eef94-enterprise-veryimportantcustomer-1 +# For more documentation about the DEB package naming conventions see: +# https://www.debian.org/doc/debian-policy/ch-controlfields.html#version +# +# The RPM package version is determined using the following two templates: +# Release version: +# -1.[.] +# Development version: +# -0...[.] +# Examples: +# Release version: +# 0.50.1-1.community +# 0.50.1-1.enterprise +# 0.50.1-1.enterprise.veryimportantcustomer +# Development version: +# 0.50.0-0.12.7e1eef94.community +# 0.50.0-0.12.7e1eef94.enterprise +# 0.50.0-0.12.7e1eef94.enterprise.veryimportantcustomer +# For more documentation about the RPM package naming conventions see: +# https://docs.fedoraproject.org/en-US/packaging-guidelines/Versioning/ +# https://fedoraproject.org/wiki/Package_Versioning_Examples + + +def get_output(*cmd, multiple=False): + ret = subprocess.run(cmd, stdout=subprocess.PIPE, check=True) + if multiple: + return list(map(lambda x: x.strip(), + ret.stdout.decode("utf-8").strip().split("\n"))) + return ret.stdout.decode("utf-8").strip() + + +def format_version(variant, version, offering, distance=None, shorthash=None, + suffix=None): + if not distance: + # This is a release version. + if variant == "deb": + # -[-]-1 + ret = "{}-{}".format(version, offering) + if suffix: + ret += "-" + suffix + ret += "-1" + return ret + elif variant == "rpm": + # -1.[.] + ret = "{}-1.{}".format(version, offering) + if suffix: + ret += "." + suffix + return ret + else: + # -[-] + ret = "{}-{}".format(version, offering) + if suffix: + ret += "-" + suffix + return ret + else: + # This is a development version. + if variant == "deb": + # +~-[-]-1 + ret = "{}+{}~{}-{}".format(version, distance, shorthash, offering) + if suffix: + ret += "-" + suffix + ret += "-1" + return ret + elif variant == "rpm": + # -0...[.] + ret = "{}-0.{}.{}.{}".format( + version, distance, shorthash, offering) + if suffix: + ret += "." + suffix + return ret + else: + # +~-[-] + ret = "{}+{}~{}-{}".format(version, distance, shorthash, offering) + if suffix: + ret += "-" + suffix + return ret + + +# Parse arguments. +parser = argparse.ArgumentParser( + description="Get the current version of Memgraph.") +parser.add_argument( + "--enterprise", action="store_true", + help="set the current offering to enterprise (default 'community')") +parser.add_argument( + "version", help="manual version override, if supplied the version isn't " + "determined using git") +parser.add_argument( + "suffix", help="custom suffix for the current version being built") +parser.add_argument( + "--variant", choices=("binary", "deb", "rpm"), default="binary", + help="which variant of the version string should be generated") +args = parser.parse_args() + +offering = "enterprise" if args.enterprise else "community" + +# Check whether the version was manually supplied. +if args.version: + if not re.match(r"^[0-9]+\.[0-9]+\.[0-9]+$", args.version): + raise Exception("Invalid version supplied '{}'!".format(args.version)) + print(format_version(args.variant, args.version, offering, + suffix=args.suffix), end="") + sys.exit(0) + +# Get current commit hashes. +current_hash = get_output("git", "rev-parse", "HEAD") +current_hash_short = get_output("git", "rev-parse", "--short", "HEAD") + +# We want to find branches that exist on some remote and that are named +# `release/[0-9]+\.[0-9]+`. +branch_regex = re.compile(r"^remotes/[a-zA-Z0-9]+/release/([0-9]+\.[0-9]+)$") + +# Find all existing versions. +versions = [] +branches = get_output("git", "branch", "--all", multiple=True) +for branch in branches: + match = branch_regex.match(branch) + if match is not None: + version = tuple(map(int, match.group(1).split("."))) + master_branch_merge = get_output("git", "merge-base", "master", branch) + versions.append((version, branch, master_branch_merge)) +versions.sort(reverse=True) + +# Check which existing version branch is closest to the current commit. We are +# only interested in branches that were branched out before this commit was +# created. +current_version = None +for version in versions: + version_tuple, branch, master_branch_merge = version + current_branch_merge = get_output( + "git", "merge-base", current_hash, branch) + master_current_merge = get_output( + "git", "merge-base", current_hash, "master") + # The first check checks whether this commit is a child of `master` and + # the version branch was created before us. + # The second check checks whether this commit is a child of the version + # branch. + if master_branch_merge == current_branch_merge or \ + master_branch_merge == master_current_merge: + current_version = version + break + +# Determine current version. +if current_version is None: + raise Exception("You are attempting to determine the version for a very " + "old version of Memgraph!") +version, branch, master_branch_merge = current_version +distance = int(get_output("git", "rev-list", "--count", "--first-parent", + master_branch_merge + ".." + current_hash)) +version_str = ".".join(map(str, version)) + ".0" +if distance == 0: + print(format_version(version_str, suffix=args.suffix), end="") +else: + print(format_version(args.variant, version_str, offering, + distance=distance, shorthash=current_hash_short, + suffix=args.suffix), + end="") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 068b65b11..bf6d58cd5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -126,7 +126,7 @@ endif() string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type) # Generate a version.hpp file -set(VERSION_STRING ${memgraph_VERSION}) +set(VERSION_STRING ${MEMGRAPH_VERSION}) configure_file(version.hpp.in version.hpp @ONLY) include_directories(${CMAKE_CURRENT_BINARY_DIR}) @@ -161,7 +161,7 @@ target_link_libraries(memgraph ${MG_SINGLE_NODE_V2_LIBS}) target_link_libraries(memgraph "-Wl,--dynamic-list=${CMAKE_SOURCE_DIR}/include/mg_procedure.syms") set_target_properties(memgraph PROPERTIES # Set the executable output name to include version information. - OUTPUT_NAME "memgraph-${memgraph_VERSION}-${COMMIT_HASH}_${CMAKE_BUILD_TYPE}" + OUTPUT_NAME "memgraph-${MEMGRAPH_VERSION}_${CMAKE_BUILD_TYPE}" # Output the executable in main binary dir. RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) # Create symlink to the built executable. @@ -226,7 +226,7 @@ install(CODE "file(MAKE_DIRECTORY \$ENV{DESTDIR}/var/log/memgraph #target_link_libraries(memgraph_ha mg-single-node-ha mg-kvstore telemetry_lib) #set_target_properties(memgraph_ha PROPERTIES # # Set the executable output name to include version information. -# OUTPUT_NAME "memgraph_ha-${memgraph_VERSION}-${COMMIT_HASH}_${CMAKE_BUILD_TYPE}" +# OUTPUT_NAME "memgraph_ha-${MEMGRAPH_VERSION}_${CMAKE_BUILD_TYPE}" # # Output the executable in main binary dir. # RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) ## Create symlink to the built executable. diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index b3bbb10c7..5b4930f05 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,7 +1,7 @@ # MemGraph Tools CMake configuration # You should use the top level CMake configuration with -DTOOLS=ON option set. -project(memgraph_tools VERSION ${memgraph_VERSION}) +project(memgraph_tools) disallow_in_source_build() diff --git a/tools/src/CMakeLists.txt b/tools/src/CMakeLists.txt index a16c96a9e..6afa2d4f2 100644 --- a/tools/src/CMakeLists.txt +++ b/tools/src/CMakeLists.txt @@ -1,5 +1,5 @@ # Generate a version.hpp file -set(VERSION_STRING ${memgraph_VERSION}) +set(VERSION_STRING ${MEMGRAPH_VERSION}) configure_file(../../src/version.hpp.in version.hpp @ONLY) include_directories(${CMAKE_CURRENT_BINARY_DIR})