tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Build for ARM64 (#340)

Browse Source
This commit is contained in:
Antonio Andelic 2022-02-03 13:03:35 +01:00 committed by GitHub
parent 661e5185d8
commit 265b203b00
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 613 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -228,6 +228,8 @@ endif()
message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
# -----------------------------------------------------------------------------
set(MG_ARCH "x86_64" CACHE STRING "Host architecture to build Memgraph on. Supported values are x86_64 (default), ARM64.")
# setup external dependencies -------------------------------------------------
# threading

139
environment/os/centos-9.sh Executable file
View File

@ -0,0 +1,139 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel xz-devel python3-devel texinfo # for gdb
libcurl-devel # for cmake
curl # snappy
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig
file
openssl-devel
gmp-devel
gperf
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
zlib # zlib library used for all builds
expat xz-libs python3 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconf-pkg-config # build system
curl wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
python3-devel # for query modules
openssl-devel
libseccomp-devel
python3 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
#
# IMPORTANT: python3-yaml does NOT exist on CentOS
# Install it manually using `pip3 install PyYAML`
#
PyYAML # Package name here does not correspond to the yum package!
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {
echo "$1"
}
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == "PyYAML" ]; then
if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == "python3-virtualenv" ]; then
continue
fi
if [ "$pkg" == sbcl ]; then
if ! sbcl --version &> /dev/null; then
missing="$pkg $missing"
fi
continue
fi
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
yum update -y
yum install -y wget git python3 python3-pip
for pkg in $1; do
if [ "$pkg" == sbcl ]; then
if ! sbcl --version &> /dev/null; then
curl -s https://altushost-swe.dl.sourceforge.net/project/sbcl/sbcl/1.4.2/sbcl-1.4.2-arm64-linux-binary.tar.bz2 -o /tmp/sbcl-arm64.tar.bz2
tar xvjf /tmp/sbcl-arm64.tar.bz2 -C /tmp
pushd /tmp/sbcl-1.4.2-arm64-linux
INSTALL_ROOT=/usr/local sh install.sh
popd
fi
continue
fi
if [ "$pkg" == PyYAML ]; then
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user PyYAML
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
fi
continue
fi
if [ "$pkg" == python3-virtualenv ]; then
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user virtualenv
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user virtualenv"
fi
continue
fi
yum install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

2
environment/toolchain/folly.diff Normal file
View File

@ -0,0 +1,2 @@
31d30
< add_subdirectory(logging/example)

291
environment/toolchain/v4.sh
View File

@ -10,6 +10,18 @@ cd "$DIR"
source "$DIR/../util.sh"
DISTRO="$(operating_system)"
for_arm=false
if [[ "$#" -eq 1 ]]; then
if [[ "$1" == "--for-arm" ]]; then
for_arm=true
else
echo "Invalid argument received. Use '--for-arm' if you want to build the toolchain for ARM based CPU."
exit 1
fi
fi
os="$1"
# toolchain version
TOOLCHAIN_VERSION=4
@ -21,7 +33,7 @@ case "$DISTRO" in
GDB_VERSION=8.3
;;
*)
GDB_VERSION=11.1
GDB_VERSION=11.2
;;
esac
CMAKE_VERSION=3.22.1
@ -169,36 +181,78 @@ if [ ! -f $PREFIX/bin/gcc ]; then
pushd gcc-$GCC_VERSION
./contrib/download_prerequisites
mkdir build && pushd build
# influenced by: https://buildd.debian.org/status/fetch.php?pkg=gcc-8&arch=amd64&ver=8.3.0-6&stamp=1554588545
../configure -v \
--build=x86_64-linux-gnu \
--host=x86_64-linux-gnu \
--target=x86_64-linux-gnu \
--prefix=$PREFIX \
--disable-multilib \
--with-system-zlib \
--enable-checking=release \
--enable-languages=c,c++,fortran \
--enable-gold=yes \
--enable-ld=yes \
--enable-lto \
--enable-bootstrap \
--disable-vtable-verify \
--disable-werror \
--without-included-gettext \
--enable-threads=posix \
--enable-nls \
--enable-clocale=gnu \
--enable-libstdcxx-debug \
--enable-libstdcxx-time=yes \
--enable-gnu-unique-object \
--enable-libmpx \
--enable-plugin \
--enable-default-pie \
--with-target-system-zlib \
--with-tune=generic \
--without-cuda-driver
#--program-suffix=$( printf "$GCC_VERSION" | cut -d '.' -f 1,2 ) \
# influenced by: https://buildd.debian.org/status/fetch.php?pkg=gcc-11&arch=arm64&ver=11.2.0-14&stamp=1642052446&raw=0
if [[ "$for_arm" = true ]]; then
../configure -v \
--prefix=$PREFIX \
--disable-multilib \
--with-system-zlib \
--enable-languages=c,c++,fortran \
--enable-gold=yes \
--enable-ld=yes \
--disable-vtable-verify \
--enable-libmpx \
--without-cuda-driver \
--enable-shared \
--enable-linker-build-id \
--without-included-gettext \
--enable-threads=posix \
--enable-nls \
--enable-bootstrap \
--enable-clocale=gnu \
--enable-libstdcxx-debug \
--enable-libstdcxx-time=yes \
--with-default-libstdcxx-abi=new \
--enable-gnu-unique-object \
--disable-libquadmath \
--disable-libquadmath-support \
--enable-plugin \
--enable-default-pie \
--with-system-zlib \
--enable-libphobos-checking=release \
--with-target-system-zlib=auto \
--enable-objc-gc=auto \
--enable-multiarch \
--enable-fix-cortex-a53-843419 \
--disable-werror \
--enable-checking=release \
--build=aarch64-linux-gnu \
--host=aarch64-linux-gnu \
--target=aarch64-linux-gnu \
--with-build-config=bootstrap-lto-lean \
--enable-link-serialization=4
else
# influenced by: https://buildd.debian.org/status/fetch.php?pkg=gcc-8&arch=amd64&ver=8.3.0-6&stamp=1554588545
../configure -v \
--build=x86_64-linux-gnu \
--host=x86_64-linux-gnu \
--target=x86_64-linux-gnu \
--prefix=$PREFIX \
--disable-multilib \
--with-system-zlib \
--enable-checking=release \
--enable-languages=c,c++,fortran \
--enable-gold=yes \
--enable-ld=yes \
--enable-lto \
--enable-bootstrap \
--disable-vtable-verify \
--disable-werror \
--without-included-gettext \
--enable-threads=posix \
--enable-nls \
--enable-clocale=gnu \
--enable-libstdcxx-debug \
--enable-libstdcxx-time=yes \
--enable-gnu-unique-object \
--enable-libmpx \
--enable-plugin \
--enable-default-pie \
--with-target-system-zlib \
--with-tune=generic \
--without-cuda-driver
#--program-suffix=$( printf "$GCC_VERSION" | cut -d '.' -f 1,2 ) \
fi
make -j$CPUS
# make -k check # run test suite
make install
@ -217,28 +271,56 @@ if [ ! -f $PREFIX/bin/ld.gold ]; then
tar -xvf ../archives/binutils-$BINUTILS_VERSION.tar.gz
pushd binutils-$BINUTILS_VERSION
mkdir build && pushd build
# influenced by: https://buildd.debian.org/status/fetch.php?pkg=binutils&arch=amd64&ver=2.32-7&stamp=1553247092
env \
CC=gcc \
CXX=g++ \
CFLAGS="-g -O2" \
CXXFLAGS="-g -O2" \
LDFLAGS="" \
../configure \
--build=x86_64-linux-gnu \
--host=x86_64-linux-gnu \
--prefix=$PREFIX \
--enable-ld=default \
--enable-gold \
--enable-lto \
--enable-plugins \
--enable-shared \
--enable-threads \
--with-system-zlib \
--enable-deterministic-archives \
--disable-compressed-debug-sections \
--enable-new-dtags \
--disable-werror
if [[ "$for_arm" = true ]]; then
# influenced by: https://buildd.debian.org/status/fetch.php?pkg=binutils&arch=arm64&ver=2.37.90.20220130-2&stamp=1643576183&raw=0
env \
CC=gcc \
CXX=g++ \
CFLAGS="-g -O2" \
CXXFLAGS="-g -O2" \
LDFLAGS="" \
../configure \
--build=aarch64-linux-gnu \
--host=aarch64-linux-gnu \
--prefix=$PREFIX \
--enable-ld=default \
--enable-gold \
--enable-lto \
--enable-pgo-build=lto \
--enable-plugins \
--enable-shared \
--enable-threads \
--with-system-zlib \
--enable-deterministic-archives \
--disable-compressed-debug-sections \
--disable-x86-used-note \
--enable-obsolete \
--enable-new-dtags \
--disable-werror
else
# influenced by: https://buildd.debian.org/status/fetch.php?pkg=binutils&arch=amd64&ver=2.32-7&stamp=1553247092
env \
CC=gcc \
CXX=g++ \
CFLAGS="-g -O2" \
CXXFLAGS="-g -O2" \
LDFLAGS="" \
../configure \
--build=x86_64-linux-gnu \
--host=x86_64-linux-gnu \
--prefix=$PREFIX \
--enable-ld=default \
--enable-gold \
--enable-lto \
--enable-plugins \
--enable-shared \
--enable-threads \
--with-system-zlib \
--enable-deterministic-archives \
--disable-compressed-debug-sections \
--enable-new-dtags \
--disable-werror
fi
make -j$CPUS
# make -k check # run test suite
make install
@ -253,34 +335,64 @@ if [ ! -f $PREFIX/bin/gdb ]; then
tar -xvf ../archives/gdb-$GDB_VERSION.tar.gz
pushd gdb-$GDB_VERSION
mkdir build && pushd build
# https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=amd64&ver=8.2.1-2&stamp=1550831554&raw=0
env \
CC=gcc \
CXX=g++ \
CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \
LDFLAGS="-Wl,-z,relro" \
PYTHON="" \
../configure \
--build=x86_64-linux-gnu \
--host=x86_64-linux-gnu \
--prefix=$PREFIX \
--disable-maintainer-mode \
--disable-dependency-tracking \
--disable-silent-rules \
--disable-gdbtk \
--disable-shared \
--without-guile \
--with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \
--with-system-readline \
--with-expat \
--with-system-zlib \
--with-lzma \
--with-babeltrace \
--with-intel-pt \
--enable-tui \
--with-python=python3
if [[ "$for_arm" = true ]]; then
# https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=arm64&ver=10.1-2&stamp=1614889767&raw=0
env \
CC=gcc \
CXX=g++ \
CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \
LDFLAGS="-Wl,-z,relro" \
PYTHON="" \
../configure \
--build=aarch64-linux-gnu \
--host=aarch64-linux-gnu \
--prefix=$PREFIX \
--disable-maintainer-mode \
--disable-dependency-tracking \
--disable-silent-rules \
--disable-gdbtk \
--disable-shared \
--without-guile \
--with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \
--with-system-readline \
--with-expat \
--with-system-zlib \
--with-lzma \
--without-babeltrace \
--enable-tui \
--with-python=python3
else
# https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=amd64&ver=8.2.1-2&stamp=1550831554&raw=0
env \
CC=gcc \
CXX=g++ \
CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \
LDFLAGS="-Wl,-z,relro" \
PYTHON="" \
../configure \
--build=x86_64-linux-gnu \
--host=x86_64-linux-gnu \
--prefix=$PREFIX \
--disable-maintainer-mode \
--disable-dependency-tracking \
--disable-silent-rules \
--disable-gdbtk \
--disable-shared \
--without-guile \
--with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \
--with-system-readline \
--with-expat \
--with-system-zlib \
--with-lzma \
--with-babeltrace \
--with-intel-pt \
--enable-tui \
--with-python=python3
fi
make -j$CPUS
make install
popd && popd
@ -424,8 +536,10 @@ if [ ! -f $PREFIX/bin/clang ]; then
-DLLVM_BINUTILS_INCDIR=$PREFIX/include/ \
-DLLVM_USE_PERF=yes
make -j$CPUS
make -j$CPUS check-clang # run clang test suite
make -j$CPUS check-lld # run lld test suite
if [[ "$for_arm" = false ]]; then
make -j$CPUS check-clang # run clang test suite
make -j$CPUS check-lld # run lld test suite
fi
make install
popd && popd
fi
@ -540,12 +654,12 @@ BZIP2_SHA256=a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
BZIP2_VERSION=1.0.6
DOUBLE_CONVERSION_SHA256=8a79e87d02ce1333c9d6c5e47f452596442a343d8c3e9b234e8a62fce1b1d49c
DOUBLE_CONVERSION_VERSION=3.1.6
FBLIBS_VERSION=2021.12.13.00
FIZZ_SHA256=1f14665ea7434b7d0770985a2f64d688c5ddbeeaa85441ae3b38ccc7741d781c
FBLIBS_VERSION=2022.01.31.00
FIZZ_SHA256=32a60e78d41ea2682ce7e5d741b964f0ea83642656e42d4fea90c0936d6d0c7d
FLEX_VERSION=2.6.4
FMT_SHA256=b06ca3130158c625848f3fb7418f235155a4d389b2abc3a6245fb01cb0eb1e01
FMT_VERSION=8.0.1
FOLLY_SHA256=87f87f5c6bf101ef15322c7351039747fb73640504d3d6de1fb719428fb0a5bc
FOLLY_SHA256=7b8d5dd2eb51757858247af0ad27af2e3e93823f84033a628722b01e06cd68a9
GFLAGS_COMMIT_HASH=b37ceb03a0e56c9f15ce80409438a555f8a67b7c
GLOG_SHA256=eede71f28371bf39aa69b45de23b329d37214016e2055269b3b5e7cfd40b59f5
GLOG_VERSION=0.5.0
@ -556,13 +670,13 @@ LIBSODIUM_VERSION=1.0.18
LIBUNWIND_VERSION=1.6.2
LZ4_SHA256=33af5936ac06536805f9745e0b6d61da606a1f8b4cc5c04dd3cbaca3b9b4fc43
LZ4_VERSION=1.8.3
PROXYGEN_SHA256=301627955e23de21d466358ae736ba1302871a6dad6c7e1f8b99a04b5b728da3
PROXYGEN_SHA256=5360a8ccdfb2f5a6c7b3eed331ec7ab0e2c792d579c6fff499c85c516c11fe14
SNAPPY_SHA256=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7
SNAPPY_VERSION=1.1.9
XZ_VERSION=5.2.5 # for LZMA
ZLIB_VERSION=1.2.11
ZSTD_VERSION=1.5.0
WANGLE_SHA256=a8019f4efc4446b8e4769df757df34b14ad6e4937d3242fc3f853a9cc4e45e9c
WANGLE_SHA256=1002e9c32b6f4837f6a760016e3b3e22f3509880ef3eaad191c80dc92655f23f
pushd archives
@ -957,6 +1071,7 @@ if [ ! -d $PREFIX/include/folly ]; then
mkdir folly-$FBLIBS_VERSION
tar -xzf ../archives/folly-$FBLIBS_VERSION.tar.gz -C folly-$FBLIBS_VERSION
pushd folly-$FBLIBS_VERSION
patch folly/CMakeLists.txt ../../folly.diff
# build is used by facebook builder
mkdir _build
pushd _build

70
init
View File

@ -9,7 +9,8 @@ function print_help () {
echo "Usage: $0 [OPTION]"
echo -e "Check for missing packages and setup the project.\n"
echo "Optional arguments:"
echo -e " -h\t\t\t\t\tdisplay this help and exit"
echo -e " -h\tdisplay this help and exit"
echo -e " --without-libs-setup\tskip the step for setting up libs"
echo -e " --wsl-quicklisp-proxy \"host:port\"\tquicklist HTTP proxy (this flag + HTTP proxy are required on WSL)"
}
@ -32,34 +33,35 @@ function setup_virtualenv () {
}
wsl_quicklisp_proxy=""
if [[ $# -gt 2 ]]; then
setup_libs=true
if [[ $# -eq 1 && "$1" == "-h" ]]; then
print_help
exit 1
elif [[ $# -eq 2 ]]; then
case "$1" in
--wsl-quicklisp-proxy)
shift
wsl_quicklisp_proxy=":proxy \"http://$1/\""
shift
;;
*)
# unknown option
print_help
exit 1
;;
esac
elif [[ $# -eq 1 ]]; then
case "$1" in
-h)
print_help
exit 0
;;
*)
# unknown option
print_help
exit 1
;;
esac
exit 0
else
while(($#)); do
case "$1" in
--wsl-quicklisp-proxy)
shift
if [[ $# -eq 0 ]]; then
echo "Missing proxy URL"
print_help
exit 1
fi
wsl_quicklisp_proxy=":proxy \"http://$1/\""
shift
;;
--without-libs-setup)
shift
setup_libs=false
;;
*)
# unknown option
echo "Invalid argument provided: $1"
print_help
exit 1
;;
esac
done
fi
DISTRO=$(operating_system)
@ -95,11 +97,13 @@ echo \
(ql:quickload '(:lcp :lcp/test) :silent t)
" | sbcl --script
# Setup libs (download).
cd libs
./cleanup.sh
./setup.sh
cd ..
if [[ "$setup_libs" == "true" ]]; then
# Setup libs (download).
cd libs
./cleanup.sh
./setup.sh
cd ..
fi
# setup gql_behave dependencies
setup_virtualenv tests/gql_behave

10
libs/CMakeLists.txt
View File

@ -245,3 +245,13 @@ import_external_library(pulsar STATIC
-DUSE_LOG4CXX=OFF
BUILD_COMMAND $(MAKE) pulsarStaticWithDeps)
add_dependencies(pulsar-proj protobuf)
if (${MG_ARCH} STREQUAL "ARM64")
set(MG_LIBRDTSC_CMAKE_ARGS -DLIBRDTSC_ARCH_x86=OFF -DLIBRDTSC_ARCH_ARM64=ON)
endif()
import_external_library(librdtsc STATIC
${CMAKE_CURRENT_SOURCE_DIR}/librdtsc/lib/librdtsc.a
${CMAKE_CURRENT_SOURCE_DIR}/librdtsc/include
CMAKE_ARGS ${MG_LIBRDTSC_CMAKE_ARGS}
BUILD_COMMAND $(MAKE) rdtsc)

29
libs/librdtsc.patch Normal file
View File

@ -0,0 +1,29 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ee9b58c..31359a9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -48,7 +48,7 @@ option(LIBRDTSC_USE_PMU "Enables PMU usage on ARM platforms" OFF)
# | Library Build and Install Properties |
# +--------------------------------------------------------+
-add_library(rdtsc SHARED
+add_library(rdtsc
src/cycles.c
src/common_timer.c
src/timer.c
@@ -72,15 +72,6 @@ target_include_directories(rdtsc
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
)
-# Install directory changes depending on build mode
-if (CMAKE_BUILD_TYPE MATCHES "^[Dd]ebug")
- # During debug, the library will be installed into a local directory
- set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/_install CACHE PATH "" FORCE)
-else ()
- # This will install in /usr/lib and /usr/include
- set(CMAKE_INSTALL_PREFIX /usr CACHE PATH "" FORCE)
-endif ()
-
# Specifying what to export when installing (GNUInstallDirs required)
install(TARGETS rdtsc
EXPORT librstsc-config

9
libs/setup.sh
View File

@ -127,6 +127,7 @@ declare -A primary_urls=(
["protobuf"]="http://$local_cache_host/git/protobuf.git"
["boost"]="http://$local_cache_host/file/boost_1_77_0.tar.gz"
["pulsar"]="http://$local_cache_host/git/pulsar.git"
["librdtsc"]="http://$local_cache_host/git/librdtsc.git"
)
# The goal of secondary urls is to have links to the "source of truth" of
@ -157,6 +158,7 @@ declare -A secondary_urls=(
["protobuf"]="https://github.com/protocolbuffers/protobuf.git"
["boost"]="https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.gz"
["pulsar"]="https://github.com/apache/pulsar.git"
["librdtsc"]="https://github.com/gabrieleara/librdtsc.git"
)
# antlr
@ -241,3 +243,10 @@ repo_clone_try_double "${primary_urls[pulsar]}" "${secondary_urls[pulsar]}" "pul
pushd pulsar
git apply ../pulsar.patch
popd
#librdtsc
librdtsc_tag="v0.3"
repo_clone_try_double "${primary_urls[librdtsc]}" "${secondary_urls[librdtsc]}" "librdtsc" "$librdtsc_tag" true
pushd librdtsc
git apply ../librdtsc.patch
popd

11
release/CMakeLists.txt
View File

@ -36,9 +36,18 @@ set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0)")
# RPM specific
set(MG_ARCH_EXTENSION "noarch")
if (${MG_ARCH} STREQUAL "x86_64")
set(MG_ARCH_EXTENSION "x86_64")
elseif (${MG_ARCH} STREQUAL "ARM64")
set(MG_ARCH_EXTENSION "aarch64")
endif()
set(CPACK_RPM_PACKAGE_URL https://memgraph.com)
set(CPACK_RPM_PACKAGE_VERSION "${MEMGRAPH_VERSION_RPM}")
set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.x86_64.rpm")
set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.${MG_ARCH_EXTENSION}.rpm")
set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
/var /var/lib /var/log /etc/logrotate.d
/lib /lib/systemd /lib/systemd/system /lib/systemd/system/memgraph.service)

19
release/arm64/build_env.dockerfile Normal file
View File

@ -0,0 +1,19 @@
FROM dokken/centos-stream-9
ARG env_folder
ARG toolchain_version
COPY ${env_folder} /env_folder
RUN yum update && yum install -y curl git
RUN /${env_folder}/os/centos-9.sh install MEMGRAPH_BUILD_DEPS
RUN /${env_folder}/os/centos-9.sh install TOOLCHAIN_RUN_DEPS
RUN rm -rf /env_folder
RUN yum clean all
RUN curl https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/${toolchain_version}/${toolchain_version}-binaries-centos-9-arm64.tar.gz -o /tmp/toolchain.tar.gz \
&& tar xvzf /tmp/toolchain.tar.gz -C /opt \
&& rm /tmp/toolchain.tar.gz

5
release/arm64/build_env.sh Executable file
View File

@ -0,0 +1,5 @@
#!/bin/bash
cp -r ../../environment env_folder
docker build -f build_env.dockerfile --build-arg env_folder=env_folder --build-arg toolchain_version=toolchain-v4 -t mg_build_env .
rm -rf env_folder

6
release/docker/memgraph.dockerfile → release/docker/memgraph_deb.dockerfile
View File

@ -1,7 +1,7 @@
FROM debian:bullseye
# NOTE: If you change the base distro update release/package as well.
ARG deb_release
ARG release
RUN apt-get update && apt-get install -y \
openssl libcurl4 libssl1.1 libseccomp2 python3 libpython3.9 python3-pip \
@ -10,10 +10,10 @@ RUN apt-get update && apt-get install -y \
RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3
COPY ${deb_release} /
COPY ${release} /
# Install memgraph package
RUN dpkg -i ${deb_release}
RUN dpkg -i ${release}
# Memgraph listens for Bolt Protocol on this port by default.
EXPOSE 7687

31
release/docker/memgraph_rpm.dockerfile Normal file
View File

@ -0,0 +1,31 @@
FROM dokken/centos-stream-9
# NOTE: If you change the base distro update release/package as well.
ARG release
RUN yum update && yum install -y \
openssl libcurl libseccomp python3 python3-pip \
--nobest --allowerasing \
&& rm -rf /tmp/* \
&& yum clean all
RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3
COPY ${release} /
# Install memgraph package
RUN rpm -i ${release}
# Memgraph listens for Bolt Protocol on this port by default.
EXPOSE 7687
# Snapshots and logging volumes
VOLUME /var/log/memgraph
VOLUME /var/lib/memgraph
# Configuration volume
VOLUME /etc/memgraph
USER memgraph
WORKDIR /usr/lib/memgraph
ENTRYPOINT ["/usr/lib/memgraph/memgraph"]
CMD [""]

46
release/docker/package_deb_docker
View File

@ -1,46 +0,0 @@
#!/bin/bash -e
# Build and package Docker image of Memgraph.
function print_help () {
echo "Usage: $0 [--latest] MEMGRAPH_PACKAGE.deb"
echo "Optional arguments:"
echo -e "\t-h|--help\t\tPrint help."
echo -e "\t--latest\t\tTag image as latest version."
}
working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
latest_image=""
tag_latest=""
if [[ $# -eq 2 && "$1" == "--latest" ]]; then
latest_image="memgraph:latest"
tag_latest="-t memgraph:latest"
shift
elif [[ $# -ne 1 || "$1" == "-h" || "$1" == "--help" ]]; then
print_help
exit 1
fi
deb_path="$1"
if [[ ! -f "$deb_path" ]]; then
echo "File '$deb_path' does not exist!"
exit 1
fi
# Copy the .deb to working directory.
cp "$deb_path" "${working_dir}/"
cd ${working_dir}
# Extract version and offering from deb name.
deb_name=`echo $(basename "$deb_path") | sed 's/.deb$//'`
version=`echo ${deb_name} | cut -d '_' -f 2 | rev | cut -d '-' -f 2- | rev | tr '+~' '__'`
dockerfile_path="${working_dir}/memgraph.dockerfile"
image_name="memgraph:${version}"
package_name="memgraph-${version}-docker.tar.gz"
# Build docker image.
docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} --build-arg deb_release=${deb_name}.deb .
docker save ${image_name} ${latest_image} > ${package_name}
rm "${deb_name}.deb"
echo "Built Docker image at '${working_dir}/${package_name}'"

61
release/docker/package_docker Executable file
View File

@ -0,0 +1,61 @@
#!/bin/bash -e
# Build and package Docker image of Memgraph.
function print_help () {
echo "Usage: $0 [--latest] MEMGRAPH_PACKAGE.(deb|rpm)"
echo "Optional arguments:"
echo -e "\t-h|--help\t\tPrint help."
echo -e "\t--latest\t\tTag image as latest version."
}
working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
latest_image=""
tag_latest=""
if [[ $# -eq 2 && "$1" == "--latest" ]]; then
latest_image="memgraph:latest"
tag_latest="-t memgraph:latest"
shift
elif [[ $# -ne 1 || "$1" == "-h" || "$1" == "--help" ]]; then
print_help
exit 1
fi
package_path="$1"
if [[ ! -f "$package_path" ]]; then
echo "File '$package_path' does not exist!"
exit 1
fi
# Copy the .deb to working directory.
cp "$package_path" "${working_dir}/"
cd ${working_dir}
extension="${package_path##*.}"
if [[ "$extension" == "deb" ]]; then
# Extract version and offering from deb name.
package_name=`echo $(basename "$package_path") | sed 's/.deb$//'`
version=`echo ${package_name} | cut -d '_' -f 2 | rev | cut -d '-' -f 2- | rev | tr '+~' '__'`
dockerfile_path="${working_dir}/memgraph_deb.dockerfile"
elif [[ "$extension" == "rpm" ]]; then
# Extract version and offering from deb name.
package_name=`echo $(basename "$package_path") | sed 's/.rpm$//'`
version=`echo ${package_name} | cut -d '-' -f 2 | rev | cut -d '-' -f 2- | rev`
version=${version%_1}
dockerfile_path="${working_dir}/memgraph_rpm.dockerfile"
else
echo "Invalid file sent as the package"
print_help
exit 1
fi
image_name="memgraph:${version}"
image_package_name="memgraph-${version}-docker.tar.gz"
# Build docker image.
docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} --build-arg release=${package_name}.${extension} .
docker save ${image_name} ${latest_image} > ${image_package_name}
rm "${package_name}.${extension}"
echo "Built Docker image at '${working_dir}/${image_package_name}'"

2
release/package/run.sh
View File

@ -106,7 +106,7 @@ case "$1" in
last_package_name=$(cd "$HOST_OUTPUT_DIR/$based_on_os" && ls -t memgraph* | head -1)
docker_build_folder="$PROJECT_ROOT/release/docker"
cd "$docker_build_folder"
./package_deb_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name"
./package_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name"
# shellcheck disable=SC2012
docker_image_name=$(cd "$docker_build_folder" && ls -t memgraph* | head -1)
docker_host_folder="$HOST_OUTPUT_DIR/docker"

3
src/storage/v2/property_store.cpp
View File

@ -903,9 +903,6 @@ uint64_t ToPowerOf8(uint64_t size) {
// of the two sets of data is currently active. Because the first byte of the
// buffer is used to distinguish which of the two sets of data is used, we can
// only use the leftover 15 bytes for raw data storage.
#ifndef __x86_64__
#error The PropertyStore only supports x86_64
#endif
const uint8_t kUseLocalBuffer = 0x01;

1
src/storage/v2/property_store.hpp
View File

@ -19,6 +19,7 @@
namespace storage {
class PropertyStore {
static_assert(std::endian::native == std::endian::little, "PropertyStore supports only architectures using little-endian.");
public:
PropertyStore();

3
src/utils/CMakeLists.txt
View File

@ -13,6 +13,7 @@ set(utils_src_files
temporal.cpp
thread.cpp
thread_pool.cpp
tsc.cpp
uuid.cpp)
find_package(Boost REQUIRED)
@ -22,7 +23,7 @@ find_package(Threads REQUIRED)
add_library(mg-utils STATIC ${utils_src_files})
target_link_libraries(mg-utils PUBLIC Boost::headers fmt::fmt spdlog::spdlog)
target_link_libraries(mg-utils PRIVATE stdc++fs Threads::Threads gflags uuid rt)
target_link_libraries(mg-utils PRIVATE librdtsc stdc++fs Threads::Threads gflags uuid rt)
set(settings_src_files
settings.cpp)

41
src/utils/tsc.cpp Normal file
View File

@ -0,0 +1,41 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <functional>
extern "C" {
#include <librdtsc/rdtsc.h>
}
#include "utils/tsc.hpp"
namespace utils {
uint64_t ReadTSC() { return rdtsc(); }
std::optional<double> GetTSCFrequency() {
// init is only needed for fetching frequency
static auto result = std::invoke([] { return rdtsc_init(); });
return result == 0 ? std::optional{rdtsc_get_tsc_hz()} : std::nullopt;
}
TSCTimer::TSCTimer(std::optional<double> frequency) : frequency_(frequency) {
if (!frequency_) return;
start_value_ = utils::ReadTSC();
}
double TSCTimer::Elapsed() const {
if (!frequency_) return 0.0;
auto current_value = utils::ReadTSC();
auto delta = current_value - start_value_;
return static_cast<double>(delta) / *frequency_;
}
} // namespace utils

111
src/utils/tsc.hpp
View File

@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@ -11,124 +11,29 @@
#pragma once
#include <sys/prctl.h>
#include <x86intrin.h>
#include <chrono>
#include <cstdint>
#include <optional>
#include <thread>
#include "utils/timer.hpp"
namespace utils {
// TSC stands for Time-Stamp Counter
#ifndef __x86_64__
#error The TSC library only supports x86_64
#endif
uint64_t ReadTSC();
/// This function reads the CPUs internal Time-Stamp Counter. This counter is
/// used to get a precise timestamp. It differs from the usual POSIX
/// `clock_gettime` in that (without additional computing) it can be only used
/// for relative time measurements. The Linux kernel uses the TSC internally to
/// implement `clock_gettime` when the clock source is set to TSC.
///
/// The TSC is implemented as a register in the CPU that increments its value
/// with a constant rate. The behavior of the TSC initially was to increment the
/// counter on each instruction. This was then changed to the current behavior
/// (>= Pentium 4) that increments the TSC with a constant rate so that CPU
/// frequency scaling doesn't affect the measurements.
///
/// The TSC is guaranteed that it won't overflow in 10 years and because it is
/// mostly implemented as a 64bit register it doesn't overflow even in 190 years
/// (see Intel manual).
///
/// One issue of the TSC is that it is unique for each logical CPU. That means
/// that if a context switch happens between two `ReadTSC` calls it could mean
/// that the counters could have a very large offset. Thankfully, the counters
/// are reset to 0 at each CPU reset and the Linux kernel synchronizes all of
/// the counters. The synchronization can be seen here (`tsc_verify_tsc_adjust`
/// and `tsc_store_and_check_tsc_adjust`):
/// https://github.com/torvalds/linux/blob/master/arch/x86/kernel/tsc_sync.c
/// https://github.com/torvalds/linux/blob/master/arch/x86/kernel/tsc.c
/// https://github.com/torvalds/linux/blob/master/arch/x86/include/asm/tsc.h
///
/// All claims here were taken from sections 17.17 and 2.8.6 of the Intel 64 and
/// IA-32 Architectures Software Developer's Manual.
///
/// Here we use the RDTSCP instruction instead of the RDTSC instruction because
/// the RDTSCP instruction forces serialization of code execution in the CPU.
/// Intel recommends the usage of the RDTSCP instruction for precision timing:
/// https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/ia-32-ia-64-benchmark-code-execution-paper.pdf
///
/// Inline assembly MUST NOT be used because the compiler won't be aware that
/// certain registers are being overwritten by the RDTSCP instruction. That is
/// why we use the builtin command.
/// https://stackoverflow.com/questions/13772567/get-cpu-cycle-count/51907627#51907627
///
/// Comparison of hardware time sources can be seen here:
/// https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_MRG/2/html/Realtime_Reference_Guide/chap-Timestamping.html#example-Hardware_Clock_Cost_Comparison
inline unsigned long long ReadTSC() {
unsigned int cpuid;
return __rdtscp(&cpuid);
}
/// The TSC can be disabled using a flag in the CPU. This function checks for
/// the availability of the TSC. If you call `ReadTSC` when the TSC isn't
/// available it will cause a segmentation fault.
/// https://blog.cr0.org/2009/05/time-stamp-counter-disabling-oddities.html
inline bool CheckAvailableTSC() {
int ret;
if (prctl(PR_GET_TSC, &ret) != 0) return false;
return ret == PR_TSC_ENABLE;
}
/// This function calculates the frequency at which the TSC counter increments
/// its value. The frequency is already metered by the Linux kernel, but the
/// value isn't reliably exposed anywhere for us to read it.
/// https://stackoverflow.com/questions/51919219/determine-tsc-frequency-on-linux
/// https://stackoverflow.com/questions/35123379/getting-tsc-rate-in-x86-kernel
/// Because of that, we determine the value ourselves. We read the value two
/// times with a delay between the two reads. The duration of the delay is
/// determined using system calls that themselves internally use the already
/// calibrated TSC. Because of that we get a very accurate value of the TSC
/// frequency.
inline std::optional<double> GetTSCFrequency() {
if (!CheckAvailableTSC()) return std::nullopt;
utils::Timer timer;
auto start_value = utils::ReadTSC();
std::this_thread::sleep_for(std::chrono::milliseconds(1));
auto duration = timer.Elapsed().count();
auto stop_value = utils::ReadTSC();
auto delta = stop_value - start_value;
return static_cast<double>(delta) / duration;
}
std::optional<double> GetTSCFrequency();
/// Class that is used to measure elapsed time using the TSC directly. It has
/// almost zero overhead and is appropriate for use in performance critical
/// paths.
class TSCTimer {
public:
TSCTimer() {}
explicit TSCTimer(std::optional<double> frequency) : frequency_(frequency) {
if (!frequency_) return;
start_value_ = ReadTSC();
}
double Elapsed() const {
if (!frequency_) return 0.0;
auto current_value = ReadTSC();
auto delta = current_value - start_value_;
return static_cast<double>(delta) / *frequency_;
}
TSCTimer() = default;
explicit TSCTimer(std::optional<double> frequency);
double Elapsed() const;
private:
std::optional<double> frequency_;
unsigned long long start_value_{0};
uint64_t start_value_{0};
};
} // namespace utils