memgraph/release/package/run2.sh
2024-02-26 00:26:00 +01:00

524 lines
21 KiB
Bash
Executable File

#!/bin/bash
set -Eeuo pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROJECT_ROOT="$SCRIPT_DIR/../.."
HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output"
MGBUILD_HOME_DIR="/home/mg"
MGBUILD_ROOT_DIR="$MGBUILD_HOME_DIR/memgraph"
DEFAULT_TOOLCHAIN="v5"
SUPPORTED_TOOLCHAINS=(
v4 v5
)
DEFAULT_OS="all"
SUPPORTED_OS=(
all
amzn-2
centos-7 centos-9
debian-10 debian-11 debian-11-arm debian-12 debian-12-arm
fedora-36 fedora-38 fedora-39
rocky-9
ubuntu-18.04 ubuntu-20.04 ubuntu-22.04 ubuntu-22.04-arm
)
DEFAULT_BUILD_TYPE="Release"
SUPPORTED_BUILD_TYPES=(
Debug
Release
RelWithDebInfo
)
DEFAULT_ARCH="amd"
SUPPORTED_ARCHS=(
amd
arm
)
SUPPORTED_TESTS=(
unit stress-plain stress-ssl drivers integration durability
gql-behave cppcheck-and-clang-format leftover-CTest
macro-benchmark mgbbench upload-to-bench-graph
)
DEFAULT_THREADS=$(nproc)
DEFAULT_ENTERPRISE_LICENSE=""
DEFAULT_ORGANIZATION_NAME="memgraph"
# TODO(deda):
# * create toolchain + OS combinations
# * add option for selecting server for push and pull
print_help () {
echo -e "\nUsage: ./run.sh [OPTIONS] COMMAND"
echo -e "\nInteract with mgbuild containers"
echo -e "\nCommands:"
echo -e " build Build mgbuild image"
echo -e " run Run mgbuild container"
echo -e " stop [OPTIONS] Stop mgbuild container"
echo -e " pull Pull mgbuild image from dockerhub"
echo -e " push OPTIONS Push mgbuild image to dockerhub"
echo -e " build-memgraph [OPTIONS] Build memgraph inside mgbuild container"
echo -e " package-memgraph Build memgraph and create deb package"
echo -e " test-memgraph TEST Run a specific test on memgraph"
echo -e " copy OPTIONS Copy an artifact from mgbuild container"
echo -e "\nGlobal options:"
echo -e " --arch string Specify target architecture (\"${SUPPORTED_ARCHS[*]}\") (default \"$DEFAULT_ARCH\")"
echo -e " --enterprise-license string Specify the enterprise license (default \"\")"
echo -e " --build-type string Specify build type (\"${SUPPORTED_BUILD_TYPES[*]}\") (default \"$DEFAULT_BUILD_TYPE\")"
echo -e " --organization-name string Specify the organization name (default \"memgraph\")"
echo -e " --os string Specify operating system (\"${SUPPORTED_OS[*]}\") (default \"$DEFAULT_OS\")"
echo -e " --threads int Specify the number of threads a command will use (default \"\$(nproc)\")"
echo -e " --toolchain string Specify toolchain version (\"${SUPPORTED_TOOLCHAINS[*]}\") (default \"$DEFAULT_TOOLCHAIN\")"
echo -e "\nSupported tests:"
echo -e " \"${SUPPORTED_TESTS[*]}\""
echo -e "\nbuild-memgraph options:"
echo -e " --community Build community version"
echo -e " --init-only Only run init script"
echo -e " --for-docker Add flag -DMG_TELEMETRY_ID_OVERRIDE=DOCKER to cmake"
echo -e " --for-platform Add flag -DMG_TELEMETRY_ID_OVERRIDE=DOCKER-PLATFORM to cmake"
echo -e "\nstop options:"
echo -e " --remove Remove the stopped mgbuild container"
echo -e "\ncopy options:"
echo -e " --binary Remove the stopped mgbuild container"
echo -e " --package Remove the stopped mgbuild container"
echo -e "\npush options:"
echo -e " -p, --password string Specify password for docker login"
echo -e " -u, --username string Specify username for docker login"
# echo -e "\ncopy options:"
# echo -e " --binary Copy built memgraph binary"
# echo -e " --build-output Copy entire build/output folder"
# echo -e " --package Copy packaged memgraph"
}
check_support() {
local is_supported=false
case "$1" in
arch)
for e in "${SUPPORTED_ARCHS[@]}"; do
if [[ "$e" == "$2" ]]; then
is_supported=true
break
fi
done
if [[ "$is_supported" == false ]]; then
echo -e "Architecture $2 isn't supported, choose from ${SUPPORTED_ARCHS[*]}"
exit 1
fi
;;
build_type)
for e in "${SUPPORTED_BUILD_TYPES[@]}"; do
if [[ "$e" == "$2" ]]; then
is_supported=true
break
fi
done
if [[ "$is_supported" == false ]]; then
echo -e "Build type $2 isn't supported, choose from ${SUPPORTED_BUILD_TYPES[*]}"
exit 1
fi
;;
os)
for e in "${SUPPORTED_OS[@]}"; do
if [[ "$e" == "$2" ]]; then
is_supported=true
break
fi
done
if [[ "$is_supported" == false ]]; then
echo -e "OS $2 isn't supported, choose from ${SUPPORTED_OS[*]}"
exit 1
fi
;;
toolchain)
for e in "${SUPPORTED_TOOLCHAINS[@]}"; do
if [[ "$e" == "$2" ]]; then
is_supported=true
break
fi
done
if [[ "$is_supported" == false ]]; then
echo -e "Toolchain version $2 isn't supported, choose from ${SUPPORTED_TOOLCHAINS[*]}"
exit 1
fi
;;
*)
echo -e "This function can only check arch, build_type, os and toolchain version"
exit 1
;;
esac
}
##################################################
######## BUILD, COPY AND PACKAGE MEMGRAPH ########
##################################################
build_memgraph () {
local build_container="mgbuild_${toolchain_version}_${os}"
local ACTIVATE_TOOLCHAIN="source /opt/toolchain-${toolchain_version}/activate"
local container_build_dir="$MGBUILD_ROOT_DIR/build"
local container_output_dir="$container_build_dir/output"
local arm_flag=""
if [[ "$arch" == "arm" ]] || [[ "$os" =~ "-arm" ]]; then
arm_flag="-DMG_ARCH="ARM64""
fi
local build_type_flag="-DCMAKE_BUILD_TYPE=$build_type"
local telemetry_id_override_flag=""
local community_flag=""
local coverage_flag=""
local asan_flag=""
local ubsan_flag=""
local init_only=false
local for_docker=false
local for_platform=false
while [[ "$#" -gt 0 ]]; do
case "$1" in
--community)
community_flag="-DMG_ENTERPRISE=OFF"
shift 1
;;
--init-only)
init_only=true
shift 1
;;
--for-docker)
for_docker=true
if [[ "$for_platform" == "true" ]]; then
echo "Error: Cannot combine --for-docker and --for-platform flags"
exit 1
fi
telemetry_id_override_flag=" -DMG_TELEMETRY_ID_OVERRIDE=DOCKER "
shift 1
;;
--for-platform)
for_platform=true
if [[ "$for_docker" == "true" ]]; then
echo "Error: Cannot combine --for-docker and --for-platform flags"
exit 1
fi
telemetry_id_override_flag=" -DMG_TELEMETRY_ID_OVERRIDE=DOCKER-PLATFORM "
shift 1
;;
--coverage)
coverage_flag="-DTEST_COVERAGE=ON"
shift 1
;;
--asan)
asan_flag="-DASAN=ON"
shift 1
;;
--ubsan)
ubsan_flag="-DUBSAN=ON"
shift 1
;;
*)
echo "Error: Unknown flag '$1'"
exit 1
;;
esac
done
echo "Building Memgraph for $os on $build_container..."
echo "Copying project files..."
# If master is not the current branch, fetch it, because the get_version
# script depends on it. If we are on master, the fetch command is going to
# fail so that's why there is the explicit check.
# Required here because Docker build container can't access remote.
cd "$PROJECT_ROOT"
if [[ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]]; then
git fetch origin master:master
fi
# Ensure we have a clean build directory
docker exec -u mg "$build_container" bash -c "rm -rf $MGBUILD_ROOT_DIR && mkdir -p $MGBUILD_ROOT_DIR"
# TODO(gitbuda): Revisit copying the whole repo -> makese sense under CI.
docker cp "$PROJECT_ROOT/." "$build_container:$MGBUILD_ROOT_DIR/"
# Change ownership of copied files so the mg user inside container can access them
docker exec -u root $build_container bash -c "chown -R mg:mg $MGBUILD_ROOT_DIR"
# TODO(gitbuda): TOOLCHAIN_RUN_DEPS should be installed during the Docker
# image build phase, but that is not easy at this point because the
# environment/os/{os}.sh does not come within the toolchain package. When
# migrating to the next version of toolchain do that, and remove the
# TOOLCHAIN_RUN_DEPS installation from here.
# TODO(gitbuda): On the other side, having this here allows updating deps
# wihout reruning the build containers.
# TODO(gitbuda+deda): Make a decision on this, (deda thinks we should move this to the Dockerfiles to save on time)
echo "Installing dependencies using '/memgraph/environment/os/$os.sh' script..."
docker exec -u root "$build_container" bash -c "$MGBUILD_ROOT_DIR/environment/os/$os.sh check TOOLCHAIN_RUN_DEPS || /environment/os/$os.sh install TOOLCHAIN_RUN_DEPS"
docker exec -u root "$build_container" bash -c "$MGBUILD_ROOT_DIR/environment/os/$os.sh check MEMGRAPH_BUILD_DEPS || /environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
echo "Building targeted package..."
# Fix issue with git marking directory as not safe
docker exec -u mg "$build_container" bash -c "cd $MGBUILD_ROOT_DIR && git config --global --add safe.directory '*'"
docker exec -u mg "$build_container" bash -c "cd $MGBUILD_ROOT_DIR && $ACTIVATE_TOOLCHAIN && ./init --ci"
if [[ "$init_only" == "true" ]]; then
return
fi
docker exec -u mg "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
# Fix cmake failing locally if remote is clone via ssh
docker exec -u mg "$build_container" bash -c "cd $MGBUILD_ROOT_DIR && git remote set-url origin https://github.com/memgraph/memgraph.git"
# Define cmake command
local cmake_cmd="cmake $build_type_flag $arm_flag $community_flag $telemetry_id_override_flag $coverage_flag $asan_flag $ubsan_flag .."
docker exec -u mg "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && $cmake_cmd"
# ' is used instead of " because we need to run make within the allowed
# container resources.
# shellcheck disable=SC2016
docker exec -u mg "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN "'&& make -j$threads'
docker exec -u mg "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN "'&& make -j$threads -B mgconsole'
}
package_memgraph() {
local ACTIVATE_TOOLCHAIN="source /opt/toolchain-${toolchain_version}/activate"
local build_container="mgbuild_${toolchain_version}_${os}"
local container_output_dir="$MGBUILD_ROOT_DIR/build/output"
local package_command=""
if [[ "$os" =~ ^"centos".* ]] || [[ "$os" =~ ^"fedora".* ]] || [[ "$os" =~ ^"amzn".* ]]; then
docker exec -u root "$build_container" bash -c "yum -y update"
package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint --file='../../release/rpm/rpmlintrc' memgraph*.rpm "
fi
if [[ "$os" =~ ^"debian".* ]]; then
docker exec -u root "$build_container" bash -c "apt --allow-releaseinfo-change -y update"
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
fi
if [[ "$os" =~ ^"ubuntu".* ]]; then
docker exec -u root "$build_container" bash -c "apt update"
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
fi
docker exec -u mg "$build_container" bash -c "mkdir -p $container_output_dir && cd $container_output_dir && $ACTIVATE_TOOLCHAIN && $package_command"
}
copy_memgraph() {
local container_output_dir="$MGBUILD_ROOT_DIR/build/output"
local build_container="mgbuild_${toolchain_version}_${os}"
echo "Copying targeted package to host..."
local last_package_name=$(docker exec -u mg "$build_container" bash -c "cd $container_output_dir && ls -t memgraph* | head -1")
# The operating system folder is introduced because multiple different
# packages could be preserved during the same build "session".
mkdir -p "$HOST_OUTPUT_DIR/$os"
local package_host_destination="$HOST_OUTPUT_DIR/$os/$last_package_name"
docker cp "$build_container:$container_output_dir/$last_package_name" "$package_host_destination"
echo "Package saved to $package_host_destination."
}
##################################################
##################### TESTS ######################
##################################################
test_memgraph() {
local ACTIVATE_TOOLCHAIN="source /opt/toolchain-${toolchain_version}/activate"
local ACTIVATE_VENV="./setup.sh /opt/toolchain-${toolchain_version}/activate"
local EXPORT_LICENSE="export MEMGRAPH_ENTERPRISE_LICENSE=$enterprise_license"
local EXPORT_ORG_NAME="export MEMGRAPH_ORGANIZATION_NAME=$organization_name"
local BUILD_DIR="$MGBUILD_ROOT_DIR/build"
local build_container="mgbuild_${toolchain_version}_${os}"
echo "Running $1 test on $build_container..."
case "$1" in
unit)
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $BUILD_DIR && $ACTIVATE_TOOLCHAIN "'&& ctest -R memgraph__unit --output-on-failure -j$threads'
;;
unit-coverage)
local setup_lsan_ubsan="export LSAN_OPTIONS=suppressions=$BUILD_DIR/../tools/lsan.supp && export UBSAN_OPTIONS=halt_on_error=1"
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $BUILD_DIR && $ACTIVATE_TOOLCHAIN && $setup_lsan_ubsan "'&& ctest -R memgraph__unit --output-on-failure -j2'
;;
leftover-CTest)
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $BUILD_DIR && $ACTIVATE_TOOLCHAIN "'&& ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure'
;;
drivers)
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR "'&& ./tests/drivers/run.sh'
;;
integration)
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR "'&& tests/integration/run.sh'
;;
cppcheck-and-clang-format)
local test_output_path="$MGBUILD_ROOT_DIR/tools/github/cppcheck_and_clang_format.txt"
local test_output_host_dest="$PROJECT_ROOT/tools/github/cppcheck_and_clang_format.txt"
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tools/github && $ACTIVATE_TOOLCHAIN "'&& ./cppcheck_and_clang_format diff'
docker cp $build_container:$test_output_path $test_output_host_dest
;;
stress-plain)
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests/stress && source ve3/bin/activate "'&& ./continuous_integration'
;;
stress-ssl)
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests/stress && source ve3/bin/activate "'&& ./continuous_integration --use-ssl'
;;
durability)
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests/stress && source ve3/bin/activate "'&& python3 durability --num-steps 5'
;;
gql-behave)
local test_output_dir="$MGBUILD_ROOT_DIR/tests/gql_behave"
local test_output_host_dest="$PROJECT_ROOT/tests/gql_behave"
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests && $ACTIVATE_VENV && cd $MGBUILD_ROOT_DIR/tests/gql_behave "'&& ./continuous_integration'
docker cp $build_container:$test_output_dir/gql_behave_status.csv $test_output_host_dest/gql_behave_status.csv
docker cp $build_container:$test_output_dir/gql_behave_status.html $test_output_host_dest/gql_behave_status.html
;;
macro-benchmark)
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && export USER=mg && export LANG=$(echo $LANG) && cd $MGBUILD_ROOT_DIR/tests/macro_benchmark "'&& ./harness QuerySuite MemgraphRunner --groups aggregation 1000_create unwind_create dense_expand match --no-strict'
;;
mgbench)
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests/mgbbench "'&& ./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*'
;;
upload-to-bench-graph)
shift 1
local setup_ve3_env="virtualenv -p python3 ve3 && source ve3/bin/activate && pip install -r requirements.txt"
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tools/bench-graph-client && $setup_ve3_env "'&& ./main.py $@'
;;
code-analysis)
shift 1
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests/code_analysis "'&& ./python_code_analysis.sh $@'
;;
code-coverage)
local test_output_path="$MGBUILD_ROOT_DIR/tools/github/generated/code_coverage.tar.gz"
local test_output_host_dest="$PROJECT_ROOT/tools/github/generated/code_coverage.tar.gz"
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && $ACTIVATE_TOOLCHAIN && cd $MGBUILD_ROOT_DIR/tools/github && ./coverage_convert && cd $MGBUILD_ROOT_DIR/tools/github/generated && tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu"
docker cp $build_container:$test_output_path $test_output_host_dest
;;
clang-tidy)
shift 1
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && export THREADS=$threads && $ACTIVATE_TOOLCHAIN && cd $MGBUILD_ROOT_DIR/tests/code_analysis "'&& ./clang_tidy.sh $@'
;;
*)
echo "Error: Unknown test '$1'"
exit 1
;;
esac
}
##################################################
################### PARSE ARGS ###################
##################################################
if [ "$#" -eq 0 ] || [ "$1" == "-h" ] || [ "$1" == "--help" ]; then
print_help
exit 0
fi
arch=$DEFAULT_ARCH
build_type=$DEFAULT_BUILD_TYPE
enterprise_license=$DEFAULT_ENTERPRISE_LICENSE
organization_name=$DEFAULT_ORGANIZATION_NAME
os=$DEFAULT_OS
threads=$DEFAULT_THREADS
toolchain_version=$DEFAULT_TOOLCHAIN
while [[ $# -gt 0 ]]; do
case "$1" in
--arch)
arch=$2
check_support arch $arch
shift 2
;;
--build-type)
build_type=$2
check_support build_type $build_type
shift 2
;;
--enterprise-license)
enterprise_license=$2
shift 2
;;
--organization-name)
organization_name=$2
shift 2
;;
--os)
os=$2
check_support os $os
shift 2
;;
--threads)
threads=$2
shift 2
;;
--toolchain)
toolchain=$2
check_support toolchain $toolchain
shift 2
;;
*)
if [[ "$1" =~ ^--.* ]]; then
echo -e "Error: Unknown option '$1'"
exit 1
else
command=$1
shift 1
break
fi
;;
esac
done
docker_compose_cmd="docker-compose"
if ! which "docker-compose" >/dev/null; then
docker_compose_cmd="docker compose"
fi
##################################################
################# PARSE COMMAND ##################
##################################################
case $command in
build)
cd $SCRIPT_DIR
if [[ "$os" == "all" ]]; then
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml build
else
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml build mgbuild_${toolchain_version}_${os}
fi
;;
run)
cd $SCRIPT_DIR
if [[ "$os" == "all" ]]; then
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml up -d
else
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml up -d mgbuild_${toolchain_version}_${os}
fi
;;
stop)
cd $SCRIPT_DIR
remove=false
if [[ "$#" -gt 0 ]]; then
if [[ "$1" == "--remove" ]]; then
remove=true
else
echo "Error: Unknown flag '$1'"
exit 1
fi
fi
if [[ "$os" == "all" ]]; then
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml down
else
docker stop mgbuild_${toolchain_version}_${os}
if [[ "$remove" == "true" ]]; then
docker rm mgbuild_${toolchain_version}_${os}
fi
fi
;;
pull)
docker pull memgraph/mgbuild_${toolchain_version}_${os}
;;
push)
docker login $@
docker push memgraph/mgbuild_${toolchain_version}_${os}
;;
build-memgraph)
build_memgraph $@
;;
package-memgraph)
package_memgraph
;;
test-memgraph)
test_memgraph $@
;;
copy)
copy_memgraph $@
;;
*)
echo "Error: Unknown command '$command'"
exit 1
;;
esac