diff --git a/CMakeLists.txt b/CMakeLists.txt index 7fee1faa7..0eedf4404 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -228,6 +228,8 @@ endif() message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}") # ----------------------------------------------------------------------------- +set(MG_ARCH "x86_64" CACHE STRING "Host architecture to build Memgraph on. Supported values are x86_64 (default), ARM64.") + # setup external dependencies ------------------------------------------------- # threading diff --git a/environment/os/centos-9.sh b/environment/os/centos-9.sh new file mode 100755 index 000000000..3061b3f50 --- /dev/null +++ b/environment/os/centos-9.sh @@ -0,0 +1,139 @@ +#!/bin/bash + +set -Eeuo pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source "$DIR/../util.sh" + +TOOLCHAIN_BUILD_DEPS=( + coreutils gcc gcc-c++ make # generic build tools + wget # used for archive download + gnupg2 # used for archive signature verification + tar gzip bzip2 xz unzip # used for archive unpacking + zlib-devel # zlib library used for all builds + expat-devel xz-devel python3-devel texinfo # for gdb + libcurl-devel # for cmake + curl # snappy + readline-devel # for cmake and llvm + libffi-devel libxml2-devel # for llvm + libedit-devel pcre-devel automake bison # for swig + file + openssl-devel + gmp-devel + gperf +) + +TOOLCHAIN_RUN_DEPS=( + make # generic build tools + tar gzip bzip2 xz # used for archive unpacking + zlib # zlib library used for all builds + expat xz-libs python3 # for gdb + readline # for cmake and llvm + libffi libxml2 # for llvm + openssl-devel +) + +MEMGRAPH_BUILD_DEPS=( + git # source code control + make pkgconf-pkg-config # build system + curl wget # for downloading libs + libuuid-devel java-11-openjdk # required by antlr + readline-devel # for memgraph console + python3-devel # for query modules + openssl-devel + libseccomp-devel + python3 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests + # + # IMPORTANT: python3-yaml does NOT exist on CentOS + # Install it manually using `pip3 install PyYAML` + # + PyYAML # Package name here does not correspond to the yum package! + libcurl-devel # mg-requests + rpm-build rpmlint # for RPM package building + doxygen graphviz # source documentation generators + which nodejs golang zip unzip java-11-openjdk-devel # for driver tests + sbcl # for custom Lisp C++ preprocessing + autoconf # for jemalloc code generation + libtool # for protobuf code generation +) + +list() { + echo "$1" +} + +check() { + local missing="" + for pkg in $1; do + if [ "$pkg" == "PyYAML" ]; then + if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then + missing="$pkg $missing" + fi + continue + fi + if [ "$pkg" == "python3-virtualenv" ]; then + continue + fi + if [ "$pkg" == sbcl ]; then + if ! sbcl --version &> /dev/null; then + missing="$pkg $missing" + fi + continue + fi + if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then + missing="$pkg $missing" + fi + done + if [ "$missing" != "" ]; then + echo "MISSING PACKAGES: $missing" + exit 1 + fi +} + +install() { + cd "$DIR" + if [ "$EUID" -ne 0 ]; then + echo "Please run as root." + exit 1 + fi + # If GitHub Actions runner is installed, append LANG to the environment. + # Python related tests doesn't work the LANG export. + if [ -d "/home/gh/actions-runner" ]; then + echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env + else + echo "NOTE: export LANG=en_US.utf8" + fi + yum update -y + yum install -y wget git python3 python3-pip + for pkg in $1; do + if [ "$pkg" == sbcl ]; then + if ! sbcl --version &> /dev/null; then + curl -s https://altushost-swe.dl.sourceforge.net/project/sbcl/sbcl/1.4.2/sbcl-1.4.2-arm64-linux-binary.tar.bz2 -o /tmp/sbcl-arm64.tar.bz2 + tar xvjf /tmp/sbcl-arm64.tar.bz2 -C /tmp + pushd /tmp/sbcl-1.4.2-arm64-linux + INSTALL_ROOT=/usr/local sh install.sh + popd + fi + continue + fi + if [ "$pkg" == PyYAML ]; then + if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker). + pip3 install --user PyYAML + else # Running using sudo. + sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML" + fi + continue + fi + if [ "$pkg" == python3-virtualenv ]; then + if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker). + pip3 install --user virtualenv + else # Running using sudo. + sudo -H -u "$SUDO_USER" bash -c "pip3 install --user virtualenv" + fi + continue + fi + yum install -y "$pkg" + done +} + +deps=$2"[*]" +"$1" "${!deps}" diff --git a/environment/toolchain/folly.diff b/environment/toolchain/folly.diff new file mode 100644 index 000000000..45d3a567e --- /dev/null +++ b/environment/toolchain/folly.diff @@ -0,0 +1,2 @@ +31d30 +< add_subdirectory(logging/example) diff --git a/environment/toolchain/v4.sh b/environment/toolchain/v4.sh index 63f39f76d..9802599ea 100755 --- a/environment/toolchain/v4.sh +++ b/environment/toolchain/v4.sh @@ -10,6 +10,18 @@ cd "$DIR" source "$DIR/../util.sh" DISTRO="$(operating_system)" +for_arm=false +if [[ "$#" -eq 1 ]]; then + if [[ "$1" == "--for-arm" ]]; then + for_arm=true + else + echo "Invalid argument received. Use '--for-arm' if you want to build the toolchain for ARM based CPU." + exit 1 + fi +fi + +os="$1" + # toolchain version TOOLCHAIN_VERSION=4 @@ -21,7 +33,7 @@ case "$DISTRO" in GDB_VERSION=8.3 ;; *) - GDB_VERSION=11.1 + GDB_VERSION=11.2 ;; esac CMAKE_VERSION=3.22.1 @@ -169,36 +181,78 @@ if [ ! -f $PREFIX/bin/gcc ]; then pushd gcc-$GCC_VERSION ./contrib/download_prerequisites mkdir build && pushd build - # influenced by: https://buildd.debian.org/status/fetch.php?pkg=gcc-8&arch=amd64&ver=8.3.0-6&stamp=1554588545 - ../configure -v \ - --build=x86_64-linux-gnu \ - --host=x86_64-linux-gnu \ - --target=x86_64-linux-gnu \ - --prefix=$PREFIX \ - --disable-multilib \ - --with-system-zlib \ - --enable-checking=release \ - --enable-languages=c,c++,fortran \ - --enable-gold=yes \ - --enable-ld=yes \ - --enable-lto \ - --enable-bootstrap \ - --disable-vtable-verify \ - --disable-werror \ - --without-included-gettext \ - --enable-threads=posix \ - --enable-nls \ - --enable-clocale=gnu \ - --enable-libstdcxx-debug \ - --enable-libstdcxx-time=yes \ - --enable-gnu-unique-object \ - --enable-libmpx \ - --enable-plugin \ - --enable-default-pie \ - --with-target-system-zlib \ - --with-tune=generic \ - --without-cuda-driver - #--program-suffix=$( printf "$GCC_VERSION" | cut -d '.' -f 1,2 ) \ + # influenced by: https://buildd.debian.org/status/fetch.php?pkg=gcc-11&arch=arm64&ver=11.2.0-14&stamp=1642052446&raw=0 + if [[ "$for_arm" = true ]]; then + ../configure -v \ + --prefix=$PREFIX \ + --disable-multilib \ + --with-system-zlib \ + --enable-languages=c,c++,fortran \ + --enable-gold=yes \ + --enable-ld=yes \ + --disable-vtable-verify \ + --enable-libmpx \ + --without-cuda-driver \ + --enable-shared \ + --enable-linker-build-id \ + --without-included-gettext \ + --enable-threads=posix \ + --enable-nls \ + --enable-bootstrap \ + --enable-clocale=gnu \ + --enable-libstdcxx-debug \ + --enable-libstdcxx-time=yes \ + --with-default-libstdcxx-abi=new \ + --enable-gnu-unique-object \ + --disable-libquadmath \ + --disable-libquadmath-support \ + --enable-plugin \ + --enable-default-pie \ + --with-system-zlib \ + --enable-libphobos-checking=release \ + --with-target-system-zlib=auto \ + --enable-objc-gc=auto \ + --enable-multiarch \ + --enable-fix-cortex-a53-843419 \ + --disable-werror \ + --enable-checking=release \ + --build=aarch64-linux-gnu \ + --host=aarch64-linux-gnu \ + --target=aarch64-linux-gnu \ + --with-build-config=bootstrap-lto-lean \ + --enable-link-serialization=4 + else + # influenced by: https://buildd.debian.org/status/fetch.php?pkg=gcc-8&arch=amd64&ver=8.3.0-6&stamp=1554588545 + ../configure -v \ + --build=x86_64-linux-gnu \ + --host=x86_64-linux-gnu \ + --target=x86_64-linux-gnu \ + --prefix=$PREFIX \ + --disable-multilib \ + --with-system-zlib \ + --enable-checking=release \ + --enable-languages=c,c++,fortran \ + --enable-gold=yes \ + --enable-ld=yes \ + --enable-lto \ + --enable-bootstrap \ + --disable-vtable-verify \ + --disable-werror \ + --without-included-gettext \ + --enable-threads=posix \ + --enable-nls \ + --enable-clocale=gnu \ + --enable-libstdcxx-debug \ + --enable-libstdcxx-time=yes \ + --enable-gnu-unique-object \ + --enable-libmpx \ + --enable-plugin \ + --enable-default-pie \ + --with-target-system-zlib \ + --with-tune=generic \ + --without-cuda-driver + #--program-suffix=$( printf "$GCC_VERSION" | cut -d '.' -f 1,2 ) \ + fi make -j$CPUS # make -k check # run test suite make install @@ -217,28 +271,56 @@ if [ ! -f $PREFIX/bin/ld.gold ]; then tar -xvf ../archives/binutils-$BINUTILS_VERSION.tar.gz pushd binutils-$BINUTILS_VERSION mkdir build && pushd build - # influenced by: https://buildd.debian.org/status/fetch.php?pkg=binutils&arch=amd64&ver=2.32-7&stamp=1553247092 - env \ - CC=gcc \ - CXX=g++ \ - CFLAGS="-g -O2" \ - CXXFLAGS="-g -O2" \ - LDFLAGS="" \ - ../configure \ - --build=x86_64-linux-gnu \ - --host=x86_64-linux-gnu \ - --prefix=$PREFIX \ - --enable-ld=default \ - --enable-gold \ - --enable-lto \ - --enable-plugins \ - --enable-shared \ - --enable-threads \ - --with-system-zlib \ - --enable-deterministic-archives \ - --disable-compressed-debug-sections \ - --enable-new-dtags \ - --disable-werror + if [[ "$for_arm" = true ]]; then + # influenced by: https://buildd.debian.org/status/fetch.php?pkg=binutils&arch=arm64&ver=2.37.90.20220130-2&stamp=1643576183&raw=0 + env \ + CC=gcc \ + CXX=g++ \ + CFLAGS="-g -O2" \ + CXXFLAGS="-g -O2" \ + LDFLAGS="" \ + ../configure \ + --build=aarch64-linux-gnu \ + --host=aarch64-linux-gnu \ + --prefix=$PREFIX \ + --enable-ld=default \ + --enable-gold \ + --enable-lto \ + --enable-pgo-build=lto \ + --enable-plugins \ + --enable-shared \ + --enable-threads \ + --with-system-zlib \ + --enable-deterministic-archives \ + --disable-compressed-debug-sections \ + --disable-x86-used-note \ + --enable-obsolete \ + --enable-new-dtags \ + --disable-werror + else + # influenced by: https://buildd.debian.org/status/fetch.php?pkg=binutils&arch=amd64&ver=2.32-7&stamp=1553247092 + env \ + CC=gcc \ + CXX=g++ \ + CFLAGS="-g -O2" \ + CXXFLAGS="-g -O2" \ + LDFLAGS="" \ + ../configure \ + --build=x86_64-linux-gnu \ + --host=x86_64-linux-gnu \ + --prefix=$PREFIX \ + --enable-ld=default \ + --enable-gold \ + --enable-lto \ + --enable-plugins \ + --enable-shared \ + --enable-threads \ + --with-system-zlib \ + --enable-deterministic-archives \ + --disable-compressed-debug-sections \ + --enable-new-dtags \ + --disable-werror + fi make -j$CPUS # make -k check # run test suite make install @@ -253,34 +335,64 @@ if [ ! -f $PREFIX/bin/gdb ]; then tar -xvf ../archives/gdb-$GDB_VERSION.tar.gz pushd gdb-$GDB_VERSION mkdir build && pushd build - # https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=amd64&ver=8.2.1-2&stamp=1550831554&raw=0 - env \ - CC=gcc \ - CXX=g++ \ - CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \ - CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \ - CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \ - LDFLAGS="-Wl,-z,relro" \ - PYTHON="" \ - ../configure \ - --build=x86_64-linux-gnu \ - --host=x86_64-linux-gnu \ - --prefix=$PREFIX \ - --disable-maintainer-mode \ - --disable-dependency-tracking \ - --disable-silent-rules \ - --disable-gdbtk \ - --disable-shared \ - --without-guile \ - --with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \ - --with-system-readline \ - --with-expat \ - --with-system-zlib \ - --with-lzma \ - --with-babeltrace \ - --with-intel-pt \ - --enable-tui \ - --with-python=python3 + if [[ "$for_arm" = true ]]; then + # https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=arm64&ver=10.1-2&stamp=1614889767&raw=0 + env \ + CC=gcc \ + CXX=g++ \ + CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \ + CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \ + CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \ + LDFLAGS="-Wl,-z,relro" \ + PYTHON="" \ + ../configure \ + --build=aarch64-linux-gnu \ + --host=aarch64-linux-gnu \ + --prefix=$PREFIX \ + --disable-maintainer-mode \ + --disable-dependency-tracking \ + --disable-silent-rules \ + --disable-gdbtk \ + --disable-shared \ + --without-guile \ + --with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \ + --with-system-readline \ + --with-expat \ + --with-system-zlib \ + --with-lzma \ + --without-babeltrace \ + --enable-tui \ + --with-python=python3 + else + # https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=amd64&ver=8.2.1-2&stamp=1550831554&raw=0 + env \ + CC=gcc \ + CXX=g++ \ + CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \ + CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \ + CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \ + LDFLAGS="-Wl,-z,relro" \ + PYTHON="" \ + ../configure \ + --build=x86_64-linux-gnu \ + --host=x86_64-linux-gnu \ + --prefix=$PREFIX \ + --disable-maintainer-mode \ + --disable-dependency-tracking \ + --disable-silent-rules \ + --disable-gdbtk \ + --disable-shared \ + --without-guile \ + --with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \ + --with-system-readline \ + --with-expat \ + --with-system-zlib \ + --with-lzma \ + --with-babeltrace \ + --with-intel-pt \ + --enable-tui \ + --with-python=python3 + fi make -j$CPUS make install popd && popd @@ -424,8 +536,10 @@ if [ ! -f $PREFIX/bin/clang ]; then -DLLVM_BINUTILS_INCDIR=$PREFIX/include/ \ -DLLVM_USE_PERF=yes make -j$CPUS - make -j$CPUS check-clang # run clang test suite - make -j$CPUS check-lld # run lld test suite + if [[ "$for_arm" = false ]]; then + make -j$CPUS check-clang # run clang test suite + make -j$CPUS check-lld # run lld test suite + fi make install popd && popd fi @@ -540,12 +654,12 @@ BZIP2_SHA256=a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd BZIP2_VERSION=1.0.6 DOUBLE_CONVERSION_SHA256=8a79e87d02ce1333c9d6c5e47f452596442a343d8c3e9b234e8a62fce1b1d49c DOUBLE_CONVERSION_VERSION=3.1.6 -FBLIBS_VERSION=2021.12.13.00 -FIZZ_SHA256=1f14665ea7434b7d0770985a2f64d688c5ddbeeaa85441ae3b38ccc7741d781c +FBLIBS_VERSION=2022.01.31.00 +FIZZ_SHA256=32a60e78d41ea2682ce7e5d741b964f0ea83642656e42d4fea90c0936d6d0c7d FLEX_VERSION=2.6.4 FMT_SHA256=b06ca3130158c625848f3fb7418f235155a4d389b2abc3a6245fb01cb0eb1e01 FMT_VERSION=8.0.1 -FOLLY_SHA256=87f87f5c6bf101ef15322c7351039747fb73640504d3d6de1fb719428fb0a5bc +FOLLY_SHA256=7b8d5dd2eb51757858247af0ad27af2e3e93823f84033a628722b01e06cd68a9 GFLAGS_COMMIT_HASH=b37ceb03a0e56c9f15ce80409438a555f8a67b7c GLOG_SHA256=eede71f28371bf39aa69b45de23b329d37214016e2055269b3b5e7cfd40b59f5 GLOG_VERSION=0.5.0 @@ -556,13 +670,13 @@ LIBSODIUM_VERSION=1.0.18 LIBUNWIND_VERSION=1.6.2 LZ4_SHA256=33af5936ac06536805f9745e0b6d61da606a1f8b4cc5c04dd3cbaca3b9b4fc43 LZ4_VERSION=1.8.3 -PROXYGEN_SHA256=301627955e23de21d466358ae736ba1302871a6dad6c7e1f8b99a04b5b728da3 +PROXYGEN_SHA256=5360a8ccdfb2f5a6c7b3eed331ec7ab0e2c792d579c6fff499c85c516c11fe14 SNAPPY_SHA256=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7 SNAPPY_VERSION=1.1.9 XZ_VERSION=5.2.5 # for LZMA ZLIB_VERSION=1.2.11 ZSTD_VERSION=1.5.0 -WANGLE_SHA256=a8019f4efc4446b8e4769df757df34b14ad6e4937d3242fc3f853a9cc4e45e9c +WANGLE_SHA256=1002e9c32b6f4837f6a760016e3b3e22f3509880ef3eaad191c80dc92655f23f pushd archives @@ -957,6 +1071,7 @@ if [ ! -d $PREFIX/include/folly ]; then mkdir folly-$FBLIBS_VERSION tar -xzf ../archives/folly-$FBLIBS_VERSION.tar.gz -C folly-$FBLIBS_VERSION pushd folly-$FBLIBS_VERSION + patch folly/CMakeLists.txt ../../folly.diff # build is used by facebook builder mkdir _build pushd _build diff --git a/init b/init index 4fedf8544..6165b5028 100755 --- a/init +++ b/init @@ -9,7 +9,8 @@ function print_help () { echo "Usage: $0 [OPTION]" echo -e "Check for missing packages and setup the project.\n" echo "Optional arguments:" - echo -e " -h\t\t\t\t\tdisplay this help and exit" + echo -e " -h\tdisplay this help and exit" + echo -e " --without-libs-setup\tskip the step for setting up libs" echo -e " --wsl-quicklisp-proxy \"host:port\"\tquicklist HTTP proxy (this flag + HTTP proxy are required on WSL)" } @@ -32,34 +33,35 @@ function setup_virtualenv () { } wsl_quicklisp_proxy="" -if [[ $# -gt 2 ]]; then +setup_libs=true +if [[ $# -eq 1 && "$1" == "-h" ]]; then print_help - exit 1 -elif [[ $# -eq 2 ]]; then - case "$1" in - --wsl-quicklisp-proxy) - shift - wsl_quicklisp_proxy=":proxy \"http://$1/\"" - shift - ;; - *) - # unknown option - print_help - exit 1 - ;; - esac -elif [[ $# -eq 1 ]]; then - case "$1" in - -h) - print_help - exit 0 - ;; - *) - # unknown option - print_help - exit 1 - ;; - esac + exit 0 +else + while(($#)); do + case "$1" in + --wsl-quicklisp-proxy) + shift + if [[ $# -eq 0 ]]; then + echo "Missing proxy URL" + print_help + exit 1 + fi + wsl_quicklisp_proxy=":proxy \"http://$1/\"" + shift + ;; + --without-libs-setup) + shift + setup_libs=false + ;; + *) + # unknown option + echo "Invalid argument provided: $1" + print_help + exit 1 + ;; + esac + done fi DISTRO=$(operating_system) @@ -95,11 +97,13 @@ echo \ (ql:quickload '(:lcp :lcp/test) :silent t) " | sbcl --script -# Setup libs (download). -cd libs -./cleanup.sh -./setup.sh -cd .. +if [[ "$setup_libs" == "true" ]]; then + # Setup libs (download). + cd libs + ./cleanup.sh + ./setup.sh + cd .. +fi # setup gql_behave dependencies setup_virtualenv tests/gql_behave diff --git a/libs/CMakeLists.txt b/libs/CMakeLists.txt index 849810373..10f30dfe6 100644 --- a/libs/CMakeLists.txt +++ b/libs/CMakeLists.txt @@ -245,3 +245,13 @@ import_external_library(pulsar STATIC -DUSE_LOG4CXX=OFF BUILD_COMMAND $(MAKE) pulsarStaticWithDeps) add_dependencies(pulsar-proj protobuf) + +if (${MG_ARCH} STREQUAL "ARM64") + set(MG_LIBRDTSC_CMAKE_ARGS -DLIBRDTSC_ARCH_x86=OFF -DLIBRDTSC_ARCH_ARM64=ON) +endif() + +import_external_library(librdtsc STATIC + ${CMAKE_CURRENT_SOURCE_DIR}/librdtsc/lib/librdtsc.a + ${CMAKE_CURRENT_SOURCE_DIR}/librdtsc/include + CMAKE_ARGS ${MG_LIBRDTSC_CMAKE_ARGS} + BUILD_COMMAND $(MAKE) rdtsc) diff --git a/libs/librdtsc.patch b/libs/librdtsc.patch new file mode 100644 index 000000000..70a98c94a --- /dev/null +++ b/libs/librdtsc.patch @@ -0,0 +1,29 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index ee9b58c..31359a9 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -48,7 +48,7 @@ option(LIBRDTSC_USE_PMU "Enables PMU usage on ARM platforms" OFF) + # | Library Build and Install Properties | + # +--------------------------------------------------------+ + +-add_library(rdtsc SHARED ++add_library(rdtsc + src/cycles.c + src/common_timer.c + src/timer.c +@@ -72,15 +72,6 @@ target_include_directories(rdtsc + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include + ) + +-# Install directory changes depending on build mode +-if (CMAKE_BUILD_TYPE MATCHES "^[Dd]ebug") +- # During debug, the library will be installed into a local directory +- set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/_install CACHE PATH "" FORCE) +-else () +- # This will install in /usr/lib and /usr/include +- set(CMAKE_INSTALL_PREFIX /usr CACHE PATH "" FORCE) +-endif () +- + # Specifying what to export when installing (GNUInstallDirs required) + install(TARGETS rdtsc + EXPORT librstsc-config diff --git a/libs/setup.sh b/libs/setup.sh index bfcc23dee..7c7160626 100755 --- a/libs/setup.sh +++ b/libs/setup.sh @@ -127,6 +127,7 @@ declare -A primary_urls=( ["protobuf"]="http://$local_cache_host/git/protobuf.git" ["boost"]="http://$local_cache_host/file/boost_1_77_0.tar.gz" ["pulsar"]="http://$local_cache_host/git/pulsar.git" + ["librdtsc"]="http://$local_cache_host/git/librdtsc.git" ) # The goal of secondary urls is to have links to the "source of truth" of @@ -157,6 +158,7 @@ declare -A secondary_urls=( ["protobuf"]="https://github.com/protocolbuffers/protobuf.git" ["boost"]="https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.gz" ["pulsar"]="https://github.com/apache/pulsar.git" + ["librdtsc"]="https://github.com/gabrieleara/librdtsc.git" ) # antlr @@ -241,3 +243,10 @@ repo_clone_try_double "${primary_urls[pulsar]}" "${secondary_urls[pulsar]}" "pul pushd pulsar git apply ../pulsar.patch popd + +#librdtsc +librdtsc_tag="v0.3" +repo_clone_try_double "${primary_urls[librdtsc]}" "${secondary_urls[librdtsc]}" "librdtsc" "$librdtsc_tag" true +pushd librdtsc +git apply ../librdtsc.patch +popd diff --git a/release/CMakeLists.txt b/release/CMakeLists.txt index 3b741b2e1..e5f215d8e 100644 --- a/release/CMakeLists.txt +++ b/release/CMakeLists.txt @@ -36,9 +36,18 @@ set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY} set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0)") # RPM specific + +set(MG_ARCH_EXTENSION "noarch") + +if (${MG_ARCH} STREQUAL "x86_64") + set(MG_ARCH_EXTENSION "x86_64") +elseif (${MG_ARCH} STREQUAL "ARM64") + set(MG_ARCH_EXTENSION "aarch64") +endif() + set(CPACK_RPM_PACKAGE_URL https://memgraph.com) set(CPACK_RPM_PACKAGE_VERSION "${MEMGRAPH_VERSION_RPM}") -set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.x86_64.rpm") +set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.${MG_ARCH_EXTENSION}.rpm") set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION /var /var/lib /var/log /etc/logrotate.d /lib /lib/systemd /lib/systemd/system /lib/systemd/system/memgraph.service) diff --git a/release/arm64/build_env.dockerfile b/release/arm64/build_env.dockerfile new file mode 100644 index 000000000..1e2510094 --- /dev/null +++ b/release/arm64/build_env.dockerfile @@ -0,0 +1,19 @@ +FROM dokken/centos-stream-9 + +ARG env_folder +ARG toolchain_version + +COPY ${env_folder} /env_folder + +RUN yum update && yum install -y curl git + +RUN /${env_folder}/os/centos-9.sh install MEMGRAPH_BUILD_DEPS +RUN /${env_folder}/os/centos-9.sh install TOOLCHAIN_RUN_DEPS + +RUN rm -rf /env_folder + +RUN yum clean all + +RUN curl https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/${toolchain_version}/${toolchain_version}-binaries-centos-9-arm64.tar.gz -o /tmp/toolchain.tar.gz \ + && tar xvzf /tmp/toolchain.tar.gz -C /opt \ + && rm /tmp/toolchain.tar.gz diff --git a/release/arm64/build_env.sh b/release/arm64/build_env.sh new file mode 100755 index 000000000..3a2f221df --- /dev/null +++ b/release/arm64/build_env.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cp -r ../../environment env_folder +docker build -f build_env.dockerfile --build-arg env_folder=env_folder --build-arg toolchain_version=toolchain-v4 -t mg_build_env . +rm -rf env_folder diff --git a/release/docker/memgraph.dockerfile b/release/docker/memgraph_deb.dockerfile similarity index 91% rename from release/docker/memgraph.dockerfile rename to release/docker/memgraph_deb.dockerfile index 6aa8ad0ed..3f1bafd37 100644 --- a/release/docker/memgraph.dockerfile +++ b/release/docker/memgraph_deb.dockerfile @@ -1,7 +1,7 @@ FROM debian:bullseye # NOTE: If you change the base distro update release/package as well. -ARG deb_release +ARG release RUN apt-get update && apt-get install -y \ openssl libcurl4 libssl1.1 libseccomp2 python3 libpython3.9 python3-pip \ @@ -10,10 +10,10 @@ RUN apt-get update && apt-get install -y \ RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3 -COPY ${deb_release} / +COPY ${release} / # Install memgraph package -RUN dpkg -i ${deb_release} +RUN dpkg -i ${release} # Memgraph listens for Bolt Protocol on this port by default. EXPOSE 7687 diff --git a/release/docker/memgraph_rpm.dockerfile b/release/docker/memgraph_rpm.dockerfile new file mode 100644 index 000000000..5ce0e3765 --- /dev/null +++ b/release/docker/memgraph_rpm.dockerfile @@ -0,0 +1,31 @@ +FROM dokken/centos-stream-9 +# NOTE: If you change the base distro update release/package as well. + +ARG release + +RUN yum update && yum install -y \ + openssl libcurl libseccomp python3 python3-pip \ + --nobest --allowerasing \ + && rm -rf /tmp/* \ + && yum clean all + +RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3 + +COPY ${release} / + +# Install memgraph package +RUN rpm -i ${release} + +# Memgraph listens for Bolt Protocol on this port by default. +EXPOSE 7687 +# Snapshots and logging volumes +VOLUME /var/log/memgraph +VOLUME /var/lib/memgraph +# Configuration volume +VOLUME /etc/memgraph + +USER memgraph +WORKDIR /usr/lib/memgraph + +ENTRYPOINT ["/usr/lib/memgraph/memgraph"] +CMD [""] diff --git a/release/docker/package_deb_docker b/release/docker/package_deb_docker deleted file mode 100755 index 9e8fdb9ec..000000000 --- a/release/docker/package_deb_docker +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -e - -# Build and package Docker image of Memgraph. - -function print_help () { - echo "Usage: $0 [--latest] MEMGRAPH_PACKAGE.deb" - echo "Optional arguments:" - echo -e "\t-h|--help\t\tPrint help." - echo -e "\t--latest\t\tTag image as latest version." -} - -working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -latest_image="" -tag_latest="" -if [[ $# -eq 2 && "$1" == "--latest" ]]; then - latest_image="memgraph:latest" - tag_latest="-t memgraph:latest" - shift -elif [[ $# -ne 1 || "$1" == "-h" || "$1" == "--help" ]]; then - print_help - exit 1 -fi - -deb_path="$1" -if [[ ! -f "$deb_path" ]]; then - echo "File '$deb_path' does not exist!" - exit 1 -fi - -# Copy the .deb to working directory. -cp "$deb_path" "${working_dir}/" -cd ${working_dir} - -# Extract version and offering from deb name. -deb_name=`echo $(basename "$deb_path") | sed 's/.deb$//'` -version=`echo ${deb_name} | cut -d '_' -f 2 | rev | cut -d '-' -f 2- | rev | tr '+~' '__'` -dockerfile_path="${working_dir}/memgraph.dockerfile" -image_name="memgraph:${version}" -package_name="memgraph-${version}-docker.tar.gz" - -# Build docker image. -docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} --build-arg deb_release=${deb_name}.deb . -docker save ${image_name} ${latest_image} > ${package_name} -rm "${deb_name}.deb" -echo "Built Docker image at '${working_dir}/${package_name}'" diff --git a/release/docker/package_docker b/release/docker/package_docker new file mode 100755 index 000000000..d5109267b --- /dev/null +++ b/release/docker/package_docker @@ -0,0 +1,61 @@ +#!/bin/bash -e + +# Build and package Docker image of Memgraph. + +function print_help () { + echo "Usage: $0 [--latest] MEMGRAPH_PACKAGE.(deb|rpm)" + echo "Optional arguments:" + echo -e "\t-h|--help\t\tPrint help." + echo -e "\t--latest\t\tTag image as latest version." +} + +working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +latest_image="" +tag_latest="" +if [[ $# -eq 2 && "$1" == "--latest" ]]; then + latest_image="memgraph:latest" + tag_latest="-t memgraph:latest" + shift +elif [[ $# -ne 1 || "$1" == "-h" || "$1" == "--help" ]]; then + print_help + exit 1 +fi + +package_path="$1" +if [[ ! -f "$package_path" ]]; then + echo "File '$package_path' does not exist!" + exit 1 +fi + +# Copy the .deb to working directory. +cp "$package_path" "${working_dir}/" +cd ${working_dir} + +extension="${package_path##*.}" + +if [[ "$extension" == "deb" ]]; then + # Extract version and offering from deb name. + package_name=`echo $(basename "$package_path") | sed 's/.deb$//'` + version=`echo ${package_name} | cut -d '_' -f 2 | rev | cut -d '-' -f 2- | rev | tr '+~' '__'` + dockerfile_path="${working_dir}/memgraph_deb.dockerfile" +elif [[ "$extension" == "rpm" ]]; then + # Extract version and offering from deb name. + package_name=`echo $(basename "$package_path") | sed 's/.rpm$//'` + version=`echo ${package_name} | cut -d '-' -f 2 | rev | cut -d '-' -f 2- | rev` + version=${version%_1} + dockerfile_path="${working_dir}/memgraph_rpm.dockerfile" +else + echo "Invalid file sent as the package" + print_help + exit 1 +fi + +image_name="memgraph:${version}" +image_package_name="memgraph-${version}-docker.tar.gz" + +# Build docker image. +docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} --build-arg release=${package_name}.${extension} . +docker save ${image_name} ${latest_image} > ${image_package_name} +rm "${package_name}.${extension}" +echo "Built Docker image at '${working_dir}/${image_package_name}'" diff --git a/release/package/run.sh b/release/package/run.sh index 46ac31fcb..156f63015 100755 --- a/release/package/run.sh +++ b/release/package/run.sh @@ -106,7 +106,7 @@ case "$1" in last_package_name=$(cd "$HOST_OUTPUT_DIR/$based_on_os" && ls -t memgraph* | head -1) docker_build_folder="$PROJECT_ROOT/release/docker" cd "$docker_build_folder" - ./package_deb_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name" + ./package_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name" # shellcheck disable=SC2012 docker_image_name=$(cd "$docker_build_folder" && ls -t memgraph* | head -1) docker_host_folder="$HOST_OUTPUT_DIR/docker" diff --git a/src/storage/v2/property_store.cpp b/src/storage/v2/property_store.cpp index 19f404f59..ba59ee083 100644 --- a/src/storage/v2/property_store.cpp +++ b/src/storage/v2/property_store.cpp @@ -903,9 +903,6 @@ uint64_t ToPowerOf8(uint64_t size) { // of the two sets of data is currently active. Because the first byte of the // buffer is used to distinguish which of the two sets of data is used, we can // only use the leftover 15 bytes for raw data storage. -#ifndef __x86_64__ -#error The PropertyStore only supports x86_64 -#endif const uint8_t kUseLocalBuffer = 0x01; diff --git a/src/storage/v2/property_store.hpp b/src/storage/v2/property_store.hpp index 4de098fd1..1db2397c9 100644 --- a/src/storage/v2/property_store.hpp +++ b/src/storage/v2/property_store.hpp @@ -19,6 +19,7 @@ namespace storage { class PropertyStore { + static_assert(std::endian::native == std::endian::little, "PropertyStore supports only architectures using little-endian."); public: PropertyStore(); diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index caf59c597..1af78b426 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -13,6 +13,7 @@ set(utils_src_files temporal.cpp thread.cpp thread_pool.cpp + tsc.cpp uuid.cpp) find_package(Boost REQUIRED) @@ -22,7 +23,7 @@ find_package(Threads REQUIRED) add_library(mg-utils STATIC ${utils_src_files}) target_link_libraries(mg-utils PUBLIC Boost::headers fmt::fmt spdlog::spdlog) -target_link_libraries(mg-utils PRIVATE stdc++fs Threads::Threads gflags uuid rt) +target_link_libraries(mg-utils PRIVATE librdtsc stdc++fs Threads::Threads gflags uuid rt) set(settings_src_files settings.cpp) diff --git a/src/utils/tsc.cpp b/src/utils/tsc.cpp new file mode 100644 index 000000000..f87494fde --- /dev/null +++ b/src/utils/tsc.cpp @@ -0,0 +1,41 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include + +extern "C" { +#include +} + +#include "utils/tsc.hpp" + +namespace utils { +uint64_t ReadTSC() { return rdtsc(); } + +std::optional GetTSCFrequency() { + // init is only needed for fetching frequency + static auto result = std::invoke([] { return rdtsc_init(); }); + return result == 0 ? std::optional{rdtsc_get_tsc_hz()} : std::nullopt; +} + +TSCTimer::TSCTimer(std::optional frequency) : frequency_(frequency) { + if (!frequency_) return; + start_value_ = utils::ReadTSC(); +} + +double TSCTimer::Elapsed() const { + if (!frequency_) return 0.0; + auto current_value = utils::ReadTSC(); + auto delta = current_value - start_value_; + return static_cast(delta) / *frequency_; +} + +} // namespace utils diff --git a/src/utils/tsc.hpp b/src/utils/tsc.hpp index a5a9f2172..c30cb2099 100644 --- a/src/utils/tsc.hpp +++ b/src/utils/tsc.hpp @@ -1,4 +1,4 @@ -// Copyright 2021 Memgraph Ltd. +// Copyright 2022 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -11,124 +11,29 @@ #pragma once -#include -#include - -#include +#include #include -#include - -#include "utils/timer.hpp" namespace utils { // TSC stands for Time-Stamp Counter -#ifndef __x86_64__ -#error The TSC library only supports x86_64 -#endif +uint64_t ReadTSC(); -/// This function reads the CPUs internal Time-Stamp Counter. This counter is -/// used to get a precise timestamp. It differs from the usual POSIX -/// `clock_gettime` in that (without additional computing) it can be only used -/// for relative time measurements. The Linux kernel uses the TSC internally to -/// implement `clock_gettime` when the clock source is set to TSC. -/// -/// The TSC is implemented as a register in the CPU that increments its value -/// with a constant rate. The behavior of the TSC initially was to increment the -/// counter on each instruction. This was then changed to the current behavior -/// (>= Pentium 4) that increments the TSC with a constant rate so that CPU -/// frequency scaling doesn't affect the measurements. -/// -/// The TSC is guaranteed that it won't overflow in 10 years and because it is -/// mostly implemented as a 64bit register it doesn't overflow even in 190 years -/// (see Intel manual). -/// -/// One issue of the TSC is that it is unique for each logical CPU. That means -/// that if a context switch happens between two `ReadTSC` calls it could mean -/// that the counters could have a very large offset. Thankfully, the counters -/// are reset to 0 at each CPU reset and the Linux kernel synchronizes all of -/// the counters. The synchronization can be seen here (`tsc_verify_tsc_adjust` -/// and `tsc_store_and_check_tsc_adjust`): -/// https://github.com/torvalds/linux/blob/master/arch/x86/kernel/tsc_sync.c -/// https://github.com/torvalds/linux/blob/master/arch/x86/kernel/tsc.c -/// https://github.com/torvalds/linux/blob/master/arch/x86/include/asm/tsc.h -/// -/// All claims here were taken from sections 17.17 and 2.8.6 of the Intel 64 and -/// IA-32 Architectures Software Developer's Manual. -/// -/// Here we use the RDTSCP instruction instead of the RDTSC instruction because -/// the RDTSCP instruction forces serialization of code execution in the CPU. -/// Intel recommends the usage of the RDTSCP instruction for precision timing: -/// https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/ia-32-ia-64-benchmark-code-execution-paper.pdf -/// -/// Inline assembly MUST NOT be used because the compiler won't be aware that -/// certain registers are being overwritten by the RDTSCP instruction. That is -/// why we use the builtin command. -/// https://stackoverflow.com/questions/13772567/get-cpu-cycle-count/51907627#51907627 -/// -/// Comparison of hardware time sources can be seen here: -/// https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_MRG/2/html/Realtime_Reference_Guide/chap-Timestamping.html#example-Hardware_Clock_Cost_Comparison -inline unsigned long long ReadTSC() { - unsigned int cpuid; - return __rdtscp(&cpuid); -} - -/// The TSC can be disabled using a flag in the CPU. This function checks for -/// the availability of the TSC. If you call `ReadTSC` when the TSC isn't -/// available it will cause a segmentation fault. -/// https://blog.cr0.org/2009/05/time-stamp-counter-disabling-oddities.html -inline bool CheckAvailableTSC() { - int ret; - if (prctl(PR_GET_TSC, &ret) != 0) return false; - return ret == PR_TSC_ENABLE; -} - -/// This function calculates the frequency at which the TSC counter increments -/// its value. The frequency is already metered by the Linux kernel, but the -/// value isn't reliably exposed anywhere for us to read it. -/// https://stackoverflow.com/questions/51919219/determine-tsc-frequency-on-linux -/// https://stackoverflow.com/questions/35123379/getting-tsc-rate-in-x86-kernel -/// Because of that, we determine the value ourselves. We read the value two -/// times with a delay between the two reads. The duration of the delay is -/// determined using system calls that themselves internally use the already -/// calibrated TSC. Because of that we get a very accurate value of the TSC -/// frequency. -inline std::optional GetTSCFrequency() { - if (!CheckAvailableTSC()) return std::nullopt; - - utils::Timer timer; - auto start_value = utils::ReadTSC(); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - auto duration = timer.Elapsed().count(); - auto stop_value = utils::ReadTSC(); - - auto delta = stop_value - start_value; - return static_cast(delta) / duration; -} +std::optional GetTSCFrequency(); /// Class that is used to measure elapsed time using the TSC directly. It has /// almost zero overhead and is appropriate for use in performance critical /// paths. class TSCTimer { public: - TSCTimer() {} - - explicit TSCTimer(std::optional frequency) : frequency_(frequency) { - if (!frequency_) return; - start_value_ = ReadTSC(); - } - - double Elapsed() const { - if (!frequency_) return 0.0; - auto current_value = ReadTSC(); - auto delta = current_value - start_value_; - return static_cast(delta) / *frequency_; - } + TSCTimer() = default; + explicit TSCTimer(std::optional frequency); + double Elapsed() const; private: std::optional frequency_; - unsigned long long start_value_{0}; + uint64_t start_value_{0}; }; } // namespace utils