Build for ARM64 (#340)

CMakeLists.txt

@@ -228,6 +228,8 @@ endif()
 message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
 # -----------------------------------------------------------------------------
 
+set(MG_ARCH "x86_64" CACHE STRING "Host architecture to build Memgraph on. Supported values are x86_64 (default), ARM64.")
+
 # setup external dependencies -------------------------------------------------
 
 # threading

environment/os/centos-9.sh

@@ -0,0 +1,139 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "$DIR/../util.sh"
+
+TOOLCHAIN_BUILD_DEPS=(
+    coreutils gcc gcc-c++ make # generic build tools
+    wget # used for archive download
+    gnupg2 # used for archive signature verification
+    tar gzip bzip2 xz unzip # used for archive unpacking
+    zlib-devel # zlib library used for all builds
+    expat-devel xz-devel python3-devel texinfo # for gdb
+    libcurl-devel # for cmake
+    curl # snappy
+    readline-devel # for cmake and llvm
+    libffi-devel libxml2-devel # for llvm
+    libedit-devel pcre-devel automake bison # for swig
+    file
+    openssl-devel
+    gmp-devel
+    gperf
+)
+
+TOOLCHAIN_RUN_DEPS=(
+    make # generic build tools
+    tar gzip bzip2 xz # used for archive unpacking
+    zlib # zlib library used for all builds
+    expat xz-libs python3 # for gdb
+    readline # for cmake and llvm
+    libffi libxml2 # for llvm
+    openssl-devel
+)
+
+MEMGRAPH_BUILD_DEPS=(
+    git # source code control
+    make pkgconf-pkg-config # build system
+    curl wget # for downloading libs
+    libuuid-devel java-11-openjdk # required by antlr
+    readline-devel # for memgraph console
+    python3-devel # for query modules
+    openssl-devel
+    libseccomp-devel
+    python3 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
+    #
+    # IMPORTANT: python3-yaml does NOT exist on CentOS
+    # Install it manually using `pip3 install PyYAML`
+    #
+    PyYAML # Package name here does not correspond to the yum package!
+    libcurl-devel # mg-requests
+    rpm-build rpmlint # for RPM package building
+    doxygen graphviz # source documentation generators
+    which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
+    sbcl # for custom Lisp C++ preprocessing
+    autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
+)
+
+list() {
+    echo "$1"
+}
+
+check() {
+    local missing=""
+    for pkg in $1; do
+        if [ "$pkg" == "PyYAML" ]; then
+            if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
+                missing="$pkg $missing"
+            fi
+            continue
+        fi
+        if [ "$pkg" == "python3-virtualenv" ]; then
+            continue
+        fi
+        if [ "$pkg" == sbcl ]; then
+            if ! sbcl --version &> /dev/null; then
+	        missing="$pkg $missing"
+            fi
+            continue
+        fi
+        if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
+            missing="$pkg $missing"
+        fi
+    done
+    if [ "$missing" != "" ]; then
+        echo "MISSING PACKAGES: $missing"
+        exit 1
+    fi
+}
+
+install() {
+    cd "$DIR"
+    if [ "$EUID" -ne 0 ]; then
+        echo "Please run as root."
+        exit 1
+    fi
+    # If GitHub Actions runner is installed, append LANG to the environment.
+    # Python related tests doesn't work the LANG export.
+    if [ -d "/home/gh/actions-runner" ]; then
+        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
+    else
+        echo "NOTE: export LANG=en_US.utf8"
+    fi
+    yum update -y
+    yum install -y wget git python3 python3-pip
+    for pkg in $1; do
+        if [ "$pkg" == sbcl ]; then
+            if ! sbcl --version &> /dev/null; then
+	        curl -s https://altushost-swe.dl.sourceforge.net/project/sbcl/sbcl/1.4.2/sbcl-1.4.2-arm64-linux-binary.tar.bz2 -o /tmp/sbcl-arm64.tar.bz2
+		tar xvjf /tmp/sbcl-arm64.tar.bz2 -C /tmp
+		pushd /tmp/sbcl-1.4.2-arm64-linux
+		INSTALL_ROOT=/usr/local sh install.sh
+		popd
+            fi
+            continue
+        fi
+        if [ "$pkg" == PyYAML ]; then
+            if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
+                pip3 install --user PyYAML
+            else # Running using sudo.
+                sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
+            fi
+            continue
+        fi
+        if [ "$pkg" == python3-virtualenv ]; then
+            if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
+                pip3 install --user virtualenv
+            else # Running using sudo.
+                sudo -H -u "$SUDO_USER" bash -c "pip3 install --user virtualenv"
+            fi
+            continue
+        fi
+        yum install -y "$pkg"
+    done
+}
+
+deps=$2"[*]"
+"$1" "${!deps}"

environment/toolchain/folly.diff

@@ -0,0 +1,2 @@
+31d30
+< add_subdirectory(logging/example)

environment/toolchain/v4.sh

@@ -10,6 +10,18 @@ cd "$DIR"
 source "$DIR/../util.sh"
 DISTRO="$(operating_system)"
 
+for_arm=false
+if [[ "$#" -eq 1 ]]; then
+    if [[ "$1" == "--for-arm" ]]; then
+        for_arm=true
+    else
+        echo "Invalid argument received. Use '--for-arm' if you want to build the toolchain for ARM based CPU."
+        exit 1
+   fi
+fi
+
+os="$1"
+
 # toolchain version
 TOOLCHAIN_VERSION=4
 
@@ -21,7 +33,7 @@ case "$DISTRO" in
         GDB_VERSION=8.3
     ;;
     *)
-        GDB_VERSION=11.1
+        GDB_VERSION=11.2
     ;;
 esac
 CMAKE_VERSION=3.22.1
@@ -169,36 +181,78 @@ if [ ! -f $PREFIX/bin/gcc ]; then
     pushd gcc-$GCC_VERSION
     ./contrib/download_prerequisites
     mkdir build && pushd build
-    # influenced by: https://buildd.debian.org/status/fetch.php?pkg=gcc-8&arch=amd64&ver=8.3.0-6&stamp=1554588545
+    # influenced by: https://buildd.debian.org/status/fetch.php?pkg=gcc-11&arch=arm64&ver=11.2.0-14&stamp=1642052446&raw=0
-    ../configure -v \
+    if [[ "$for_arm" = true ]]; then
-        --build=x86_64-linux-gnu \
+        ../configure -v \
-        --host=x86_64-linux-gnu \
+            --prefix=$PREFIX \
-        --target=x86_64-linux-gnu \
+            --disable-multilib \
-        --prefix=$PREFIX \
+            --with-system-zlib \
-        --disable-multilib \
+            --enable-languages=c,c++,fortran \
-        --with-system-zlib \
+            --enable-gold=yes \
-        --enable-checking=release \
+            --enable-ld=yes \
-        --enable-languages=c,c++,fortran \
+            --disable-vtable-verify \
-        --enable-gold=yes \
+            --enable-libmpx \
-        --enable-ld=yes \
+            --without-cuda-driver \
-        --enable-lto \
+            --enable-shared \
-        --enable-bootstrap \
+            --enable-linker-build-id \
-        --disable-vtable-verify \
+            --without-included-gettext \
-        --disable-werror \
+            --enable-threads=posix \
-        --without-included-gettext \
+            --enable-nls \
-        --enable-threads=posix \
+            --enable-bootstrap \
-        --enable-nls \
+            --enable-clocale=gnu \
-        --enable-clocale=gnu \
+            --enable-libstdcxx-debug \
-        --enable-libstdcxx-debug \
+            --enable-libstdcxx-time=yes \
-        --enable-libstdcxx-time=yes \
+            --with-default-libstdcxx-abi=new \
-        --enable-gnu-unique-object \
+            --enable-gnu-unique-object \
-        --enable-libmpx \
+            --disable-libquadmath \
-        --enable-plugin \
+            --disable-libquadmath-support \
-        --enable-default-pie \
+            --enable-plugin \
-        --with-target-system-zlib \
+            --enable-default-pie \
-        --with-tune=generic \
+            --with-system-zlib \
-        --without-cuda-driver
+            --enable-libphobos-checking=release \
-        #--program-suffix=$( printf "$GCC_VERSION" | cut -d '.' -f 1,2 ) \
+            --with-target-system-zlib=auto \
+            --enable-objc-gc=auto \
+            --enable-multiarch \
+            --enable-fix-cortex-a53-843419 \
+            --disable-werror \
+            --enable-checking=release \
+            --build=aarch64-linux-gnu \
+            --host=aarch64-linux-gnu \
+            --target=aarch64-linux-gnu \
+            --with-build-config=bootstrap-lto-lean \
+            --enable-link-serialization=4
+    else
+        # influenced by: https://buildd.debian.org/status/fetch.php?pkg=gcc-8&arch=amd64&ver=8.3.0-6&stamp=1554588545
+        ../configure -v \
+            --build=x86_64-linux-gnu \
+            --host=x86_64-linux-gnu \
+            --target=x86_64-linux-gnu \
+            --prefix=$PREFIX \
+            --disable-multilib \
+            --with-system-zlib \
+            --enable-checking=release \
+            --enable-languages=c,c++,fortran \
+            --enable-gold=yes \
+            --enable-ld=yes \
+            --enable-lto \
+            --enable-bootstrap \
+            --disable-vtable-verify \
+            --disable-werror \
+            --without-included-gettext \
+            --enable-threads=posix \
+            --enable-nls \
+            --enable-clocale=gnu \
+            --enable-libstdcxx-debug \
+            --enable-libstdcxx-time=yes \
+            --enable-gnu-unique-object \
+            --enable-libmpx \
+            --enable-plugin \
+            --enable-default-pie \
+            --with-target-system-zlib \
+            --with-tune=generic \
+            --without-cuda-driver
+            #--program-suffix=$( printf "$GCC_VERSION" | cut -d '.' -f 1,2 ) \
+    fi
     make -j$CPUS
     # make -k check # run test suite
     make install
@@ -217,28 +271,56 @@ if [ ! -f $PREFIX/bin/ld.gold ]; then
     tar -xvf ../archives/binutils-$BINUTILS_VERSION.tar.gz
     pushd binutils-$BINUTILS_VERSION
     mkdir build && pushd build
-    # influenced by: https://buildd.debian.org/status/fetch.php?pkg=binutils&arch=amd64&ver=2.32-7&stamp=1553247092
+    if [[ "$for_arm" = true ]]; then
-    env \
+        # influenced by: https://buildd.debian.org/status/fetch.php?pkg=binutils&arch=arm64&ver=2.37.90.20220130-2&stamp=1643576183&raw=0
-        CC=gcc \
+        env \
-        CXX=g++ \
+            CC=gcc \
-        CFLAGS="-g -O2" \
+            CXX=g++ \
-        CXXFLAGS="-g -O2" \
+            CFLAGS="-g -O2" \
-        LDFLAGS="" \
+            CXXFLAGS="-g -O2" \
-        ../configure \
+            LDFLAGS="" \
-            --build=x86_64-linux-gnu \
+            ../configure \
-            --host=x86_64-linux-gnu \
+                --build=aarch64-linux-gnu \
-            --prefix=$PREFIX \
+                --host=aarch64-linux-gnu \
-            --enable-ld=default \
+                --prefix=$PREFIX \
-            --enable-gold \
+                --enable-ld=default \
-            --enable-lto \
+                --enable-gold \
-            --enable-plugins \
+                --enable-lto \
-            --enable-shared \
+                --enable-pgo-build=lto \
-            --enable-threads \
+                --enable-plugins \
-            --with-system-zlib \
+                --enable-shared \
-            --enable-deterministic-archives \
+                --enable-threads \
-            --disable-compressed-debug-sections \
+                --with-system-zlib \
-            --enable-new-dtags \
+                --enable-deterministic-archives \
-            --disable-werror
+                --disable-compressed-debug-sections \
+                --disable-x86-used-note \
+                --enable-obsolete \
+                --enable-new-dtags \
+                --disable-werror
+    else
+        # influenced by: https://buildd.debian.org/status/fetch.php?pkg=binutils&arch=amd64&ver=2.32-7&stamp=1553247092
+        env \
+            CC=gcc \
+            CXX=g++ \
+            CFLAGS="-g -O2" \
+            CXXFLAGS="-g -O2" \
+            LDFLAGS="" \
+            ../configure \
+                --build=x86_64-linux-gnu \
+                --host=x86_64-linux-gnu \
+                --prefix=$PREFIX \
+                --enable-ld=default \
+                --enable-gold \
+                --enable-lto \
+                --enable-plugins \
+                --enable-shared \
+                --enable-threads \
+                --with-system-zlib \
+                --enable-deterministic-archives \
+                --disable-compressed-debug-sections \
+                --enable-new-dtags \
+                --disable-werror
+    fi
     make -j$CPUS
     # make -k check # run test suite
     make install
@@ -253,34 +335,64 @@ if [ ! -f $PREFIX/bin/gdb ]; then
     tar -xvf ../archives/gdb-$GDB_VERSION.tar.gz
     pushd gdb-$GDB_VERSION
     mkdir build && pushd build
-    # https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=amd64&ver=8.2.1-2&stamp=1550831554&raw=0
+    if [[ "$for_arm" = true ]]; then
-    env \
+        # https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=arm64&ver=10.1-2&stamp=1614889767&raw=0
-        CC=gcc \
+        env \
-        CXX=g++ \
+            CC=gcc \
-        CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
+            CXX=g++ \
-        CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
+            CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
-        CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \
+            CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
-        LDFLAGS="-Wl,-z,relro" \
+            CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \
-        PYTHON="" \
+            LDFLAGS="-Wl,-z,relro" \
-        ../configure \
+            PYTHON="" \
-            --build=x86_64-linux-gnu \
+            ../configure \
-            --host=x86_64-linux-gnu \
+                --build=aarch64-linux-gnu \
-            --prefix=$PREFIX \
+                --host=aarch64-linux-gnu \
-            --disable-maintainer-mode \
+                --prefix=$PREFIX \
-            --disable-dependency-tracking \
+                --disable-maintainer-mode \
-            --disable-silent-rules \
+                --disable-dependency-tracking \
-            --disable-gdbtk \
+                --disable-silent-rules \
-            --disable-shared \
+                --disable-gdbtk \
-            --without-guile \
+                --disable-shared \
-            --with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \
+                --without-guile \
-            --with-system-readline \
+                --with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \
-            --with-expat \
+                --with-system-readline \
-            --with-system-zlib \
+                --with-expat \
-            --with-lzma \
+                --with-system-zlib \
-            --with-babeltrace \
+                --with-lzma \
-            --with-intel-pt \
+                --without-babeltrace \
-            --enable-tui \
+                --enable-tui \
-            --with-python=python3
+                --with-python=python3
+    else
+        # https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=amd64&ver=8.2.1-2&stamp=1550831554&raw=0
+        env \
+            CC=gcc \
+            CXX=g++ \
+            CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
+            CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
+            CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \
+            LDFLAGS="-Wl,-z,relro" \
+            PYTHON="" \
+            ../configure \
+                --build=x86_64-linux-gnu \
+                --host=x86_64-linux-gnu \
+                --prefix=$PREFIX \
+                --disable-maintainer-mode \
+                --disable-dependency-tracking \
+                --disable-silent-rules \
+                --disable-gdbtk \
+                --disable-shared \
+                --without-guile \
+                --with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \
+                --with-system-readline \
+                --with-expat \
+                --with-system-zlib \
+                --with-lzma \
+                --with-babeltrace \
+                --with-intel-pt \
+                --enable-tui \
+                --with-python=python3
+    fi
     make -j$CPUS
     make install
     popd && popd
@@ -424,8 +536,10 @@ if [ ! -f $PREFIX/bin/clang ]; then
         -DLLVM_BINUTILS_INCDIR=$PREFIX/include/ \
         -DLLVM_USE_PERF=yes
     make -j$CPUS
-    make -j$CPUS check-clang # run clang test suite
+    if [[ "$for_arm" = false ]]; then
-    make -j$CPUS check-lld # run lld test suite
+        make -j$CPUS check-clang # run clang test suite
+        make -j$CPUS check-lld # run lld test suite
+    fi
     make install
     popd && popd
 fi
@@ -540,12 +654,12 @@ BZIP2_SHA256=a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
 BZIP2_VERSION=1.0.6
 DOUBLE_CONVERSION_SHA256=8a79e87d02ce1333c9d6c5e47f452596442a343d8c3e9b234e8a62fce1b1d49c
 DOUBLE_CONVERSION_VERSION=3.1.6
-FBLIBS_VERSION=2021.12.13.00
+FBLIBS_VERSION=2022.01.31.00
-FIZZ_SHA256=1f14665ea7434b7d0770985a2f64d688c5ddbeeaa85441ae3b38ccc7741d781c
+FIZZ_SHA256=32a60e78d41ea2682ce7e5d741b964f0ea83642656e42d4fea90c0936d6d0c7d
 FLEX_VERSION=2.6.4
 FMT_SHA256=b06ca3130158c625848f3fb7418f235155a4d389b2abc3a6245fb01cb0eb1e01
 FMT_VERSION=8.0.1
-FOLLY_SHA256=87f87f5c6bf101ef15322c7351039747fb73640504d3d6de1fb719428fb0a5bc
+FOLLY_SHA256=7b8d5dd2eb51757858247af0ad27af2e3e93823f84033a628722b01e06cd68a9
 GFLAGS_COMMIT_HASH=b37ceb03a0e56c9f15ce80409438a555f8a67b7c
 GLOG_SHA256=eede71f28371bf39aa69b45de23b329d37214016e2055269b3b5e7cfd40b59f5
 GLOG_VERSION=0.5.0
@@ -556,13 +670,13 @@ LIBSODIUM_VERSION=1.0.18
 LIBUNWIND_VERSION=1.6.2
 LZ4_SHA256=33af5936ac06536805f9745e0b6d61da606a1f8b4cc5c04dd3cbaca3b9b4fc43
 LZ4_VERSION=1.8.3
-PROXYGEN_SHA256=301627955e23de21d466358ae736ba1302871a6dad6c7e1f8b99a04b5b728da3
+PROXYGEN_SHA256=5360a8ccdfb2f5a6c7b3eed331ec7ab0e2c792d579c6fff499c85c516c11fe14
 SNAPPY_SHA256=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7
 SNAPPY_VERSION=1.1.9
 XZ_VERSION=5.2.5 # for LZMA
 ZLIB_VERSION=1.2.11
 ZSTD_VERSION=1.5.0
-WANGLE_SHA256=a8019f4efc4446b8e4769df757df34b14ad6e4937d3242fc3f853a9cc4e45e9c
+WANGLE_SHA256=1002e9c32b6f4837f6a760016e3b3e22f3509880ef3eaad191c80dc92655f23f
 
 pushd archives
 
@@ -957,6 +1071,7 @@ if [ ! -d $PREFIX/include/folly ]; then
     mkdir folly-$FBLIBS_VERSION
     tar -xzf ../archives/folly-$FBLIBS_VERSION.tar.gz -C folly-$FBLIBS_VERSION
     pushd folly-$FBLIBS_VERSION
+    patch folly/CMakeLists.txt ../../folly.diff
     # build is used by facebook builder
     mkdir _build
     pushd _build

init

@@ -9,7 +9,8 @@ function print_help () {
     echo "Usage: $0 [OPTION]"
     echo -e "Check for missing packages and setup the project.\n"
     echo "Optional arguments:"
-    echo -e "  -h\t\t\t\t\tdisplay this help and exit"
+    echo -e "  -h\tdisplay this help and exit"
+    echo -e "  --without-libs-setup\tskip the step for setting up libs"
     echo -e "  --wsl-quicklisp-proxy \"host:port\"\tquicklist HTTP proxy (this flag + HTTP proxy are required on WSL)"
 }
 
@@ -32,34 +33,35 @@ function setup_virtualenv () {
 }
 
 wsl_quicklisp_proxy=""
-if [[ $# -gt 2 ]]; then
+setup_libs=true
+if [[ $# -eq 1 && "$1" == "-h" ]]; then
     print_help
-    exit 1
+    exit 0
-elif  [[ $# -eq 2 ]]; then
+else
-    case "$1" in
+    while(($#)); do
-      --wsl-quicklisp-proxy)
+        case "$1" in
-          shift
+          --wsl-quicklisp-proxy)
-          wsl_quicklisp_proxy=":proxy \"http://$1/\""
+              shift
-          shift
+              if [[ $# -eq 0 ]]; then
-      ;;
+                  echo "Missing proxy URL"
-      *)
+                  print_help
-          # unknown option
+                  exit 1
-          print_help
+              fi
-          exit 1
+              wsl_quicklisp_proxy=":proxy \"http://$1/\""
-      ;;
+              shift
-    esac
+          ;;
-elif  [[ $# -eq 1 ]]; then
+          --without-libs-setup)
-    case "$1" in
+              shift
-      -h)
+              setup_libs=false
-          print_help
+          ;;
-          exit 0
+          *)
-      ;;
+              # unknown option
-      *)
+              echo "Invalid argument provided: $1"
-          # unknown option
+              print_help
-          print_help
+              exit 1
-          exit 1
+          ;;
-      ;;
+        esac
-  esac
+    done
 fi
 
 DISTRO=$(operating_system)
@@ -95,11 +97,13 @@ echo \
   (ql:quickload '(:lcp :lcp/test) :silent t)
   " | sbcl --script
 
-# Setup libs (download).
+if [[ "$setup_libs" == "true" ]]; then
-cd libs
+    # Setup libs (download).
-./cleanup.sh
+    cd libs
-./setup.sh
+    ./cleanup.sh
-cd ..
+    ./setup.sh
+    cd ..
+fi
 
 # setup gql_behave dependencies
 setup_virtualenv tests/gql_behave

libs/CMakeLists.txt

@@ -245,3 +245,13 @@ import_external_library(pulsar STATIC
                             -DUSE_LOG4CXX=OFF
  BUILD_COMMAND $(MAKE) pulsarStaticWithDeps)
 add_dependencies(pulsar-proj protobuf)
+
+if (${MG_ARCH} STREQUAL "ARM64")
+  set(MG_LIBRDTSC_CMAKE_ARGS -DLIBRDTSC_ARCH_x86=OFF -DLIBRDTSC_ARCH_ARM64=ON)
+endif()
+
+import_external_library(librdtsc STATIC
+  ${CMAKE_CURRENT_SOURCE_DIR}/librdtsc/lib/librdtsc.a
+  ${CMAKE_CURRENT_SOURCE_DIR}/librdtsc/include
+  CMAKE_ARGS ${MG_LIBRDTSC_CMAKE_ARGS}
+  BUILD_COMMAND $(MAKE) rdtsc)

libs/librdtsc.patch

@@ -0,0 +1,29 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index ee9b58c..31359a9 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -48,7 +48,7 @@ option(LIBRDTSC_USE_PMU     "Enables PMU usage on ARM platforms"        OFF)
+ # |          Library Build and Install Properties          |
+ # +--------------------------------------------------------+
+ 
+-add_library(rdtsc SHARED
++add_library(rdtsc
+     src/cycles.c
+     src/common_timer.c
+     src/timer.c
+@@ -72,15 +72,6 @@ target_include_directories(rdtsc
+   PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
+ )
+ 
+-# Install directory changes depending on build mode
+-if (CMAKE_BUILD_TYPE MATCHES "^[Dd]ebug")
+-    # During debug, the library will be installed into a local directory
+-    set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/_install CACHE PATH "" FORCE)
+-else ()
+-    # This will install in /usr/lib and /usr/include
+-    set(CMAKE_INSTALL_PREFIX /usr CACHE PATH "" FORCE)
+-endif ()
+-
+ # Specifying what to export when installing (GNUInstallDirs required)
+ install(TARGETS rdtsc
+         EXPORT librstsc-config

libs/setup.sh

@@ -127,6 +127,7 @@ declare -A primary_urls=(
   ["protobuf"]="http://$local_cache_host/git/protobuf.git"
   ["boost"]="http://$local_cache_host/file/boost_1_77_0.tar.gz"
   ["pulsar"]="http://$local_cache_host/git/pulsar.git"
+  ["librdtsc"]="http://$local_cache_host/git/librdtsc.git"
 )
 
 # The goal of secondary urls is to have links to the "source of truth" of
@@ -157,6 +158,7 @@ declare -A secondary_urls=(
   ["protobuf"]="https://github.com/protocolbuffers/protobuf.git"
   ["boost"]="https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.gz"
   ["pulsar"]="https://github.com/apache/pulsar.git"
+  ["librdtsc"]="https://github.com/gabrieleara/librdtsc.git"
 )
 
 # antlr
@@ -241,3 +243,10 @@ repo_clone_try_double "${primary_urls[pulsar]}" "${secondary_urls[pulsar]}" "pul
 pushd pulsar
 git apply ../pulsar.patch
 popd
+
+#librdtsc
+librdtsc_tag="v0.3"
+repo_clone_try_double "${primary_urls[librdtsc]}" "${secondary_urls[librdtsc]}" "librdtsc" "$librdtsc_tag" true
+pushd librdtsc
+git apply ../librdtsc.patch
+popd

release/CMakeLists.txt

@@ -36,9 +36,18 @@ set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
 set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0)")
 
 # RPM specific
+
+set(MG_ARCH_EXTENSION "noarch")
+
+if (${MG_ARCH} STREQUAL "x86_64")
+    set(MG_ARCH_EXTENSION "x86_64")
+elseif (${MG_ARCH} STREQUAL "ARM64")
+    set(MG_ARCH_EXTENSION  "aarch64")
+endif()
+
 set(CPACK_RPM_PACKAGE_URL https://memgraph.com)
 set(CPACK_RPM_PACKAGE_VERSION "${MEMGRAPH_VERSION_RPM}")
-set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.x86_64.rpm")
+set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.${MG_ARCH_EXTENSION}.rpm")
 set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
   /var /var/lib /var/log /etc/logrotate.d
   /lib /lib/systemd /lib/systemd/system /lib/systemd/system/memgraph.service)

release/arm64/build_env.dockerfile

@@ -0,0 +1,19 @@
+FROM dokken/centos-stream-9
+
+ARG env_folder
+ARG toolchain_version
+
+COPY ${env_folder} /env_folder
+
+RUN yum update && yum install -y curl git
+
+RUN /${env_folder}/os/centos-9.sh install MEMGRAPH_BUILD_DEPS
+RUN /${env_folder}/os/centos-9.sh install TOOLCHAIN_RUN_DEPS
+
+RUN rm -rf /env_folder
+
+RUN yum clean all
+
+RUN curl https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/${toolchain_version}/${toolchain_version}-binaries-centos-9-arm64.tar.gz -o /tmp/toolchain.tar.gz \
+  && tar xvzf /tmp/toolchain.tar.gz -C /opt \
+  && rm /tmp/toolchain.tar.gz

release/arm64/build_env.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cp -r ../../environment env_folder
+docker build -f build_env.dockerfile --build-arg env_folder=env_folder --build-arg toolchain_version=toolchain-v4 -t mg_build_env .
+rm -rf env_folder

release/docker/memgraph_deb.dockerfile

@@ -1,7 +1,7 @@
 FROM debian:bullseye
 # NOTE: If you change the base distro update release/package as well.
 
-ARG deb_release
+ARG release
 
 RUN apt-get update && apt-get install -y \
     openssl libcurl4 libssl1.1 libseccomp2 python3 libpython3.9 python3-pip \
@@ -10,10 +10,10 @@ RUN apt-get update && apt-get install -y \
 
 RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3
 
-COPY ${deb_release} /
+COPY ${release} /
 
 # Install memgraph package
-RUN dpkg -i ${deb_release}
+RUN dpkg -i ${release}
 
 # Memgraph listens for Bolt Protocol on this port by default.
 EXPOSE 7687

release/docker/memgraph_rpm.dockerfile

@@ -0,0 +1,31 @@
+FROM dokken/centos-stream-9
+# NOTE: If you change the base distro update release/package as well.
+
+ARG release
+
+RUN yum update && yum install -y \
+    openssl libcurl libseccomp python3 python3-pip \
+    --nobest --allowerasing \
+  && rm -rf /tmp/* \
+  && yum clean all
+
+RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3
+
+COPY ${release} /
+
+# Install memgraph package
+RUN rpm -i ${release}
+
+# Memgraph listens for Bolt Protocol on this port by default.
+EXPOSE 7687
+# Snapshots and logging volumes
+VOLUME /var/log/memgraph
+VOLUME /var/lib/memgraph
+# Configuration volume
+VOLUME /etc/memgraph
+
+USER memgraph
+WORKDIR /usr/lib/memgraph
+
+ENTRYPOINT ["/usr/lib/memgraph/memgraph"]
+CMD [""]

release/docker/package_deb_docker

@@ -1,46 +0,0 @@
-#!/bin/bash -e
-
-# Build and package Docker image of Memgraph.
-
-function print_help () {
-    echo "Usage: $0 [--latest] MEMGRAPH_PACKAGE.deb"
-    echo "Optional arguments:"
-    echo -e "\t-h|--help\t\tPrint help."
-    echo -e "\t--latest\t\tTag image as latest version."
-}
-
-working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-latest_image=""
-tag_latest=""
-if [[ $# -eq 2 && "$1" == "--latest" ]]; then
-  latest_image="memgraph:latest"
-  tag_latest="-t memgraph:latest"
-  shift
-elif [[ $# -ne 1 || "$1" == "-h" || "$1" == "--help" ]]; then
-  print_help
-  exit 1
-fi
-
-deb_path="$1"
-if [[ ! -f "$deb_path" ]]; then
-  echo "File '$deb_path' does not exist!"
-  exit 1
-fi
-
-# Copy the .deb to working directory.
-cp "$deb_path" "${working_dir}/"
-cd ${working_dir}
-
-# Extract version and offering from deb name.
-deb_name=`echo $(basename "$deb_path") | sed 's/.deb$//'`
-version=`echo ${deb_name} | cut -d '_' -f 2 | rev | cut -d '-' -f 2- | rev | tr '+~' '__'`
-dockerfile_path="${working_dir}/memgraph.dockerfile"
-image_name="memgraph:${version}"
-package_name="memgraph-${version}-docker.tar.gz"
-
-# Build docker image.
-docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} --build-arg deb_release=${deb_name}.deb .
-docker save ${image_name} ${latest_image} > ${package_name}
-rm "${deb_name}.deb"
-echo "Built Docker image at '${working_dir}/${package_name}'"

release/docker/package_docker

@@ -0,0 +1,61 @@
+#!/bin/bash -e
+
+# Build and package Docker image of Memgraph.
+
+function print_help () {
+    echo "Usage: $0 [--latest] MEMGRAPH_PACKAGE.(deb|rpm)"
+    echo "Optional arguments:"
+    echo -e "\t-h|--help\t\tPrint help."
+    echo -e "\t--latest\t\tTag image as latest version."
+}
+
+working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+latest_image=""
+tag_latest=""
+if [[ $# -eq 2 && "$1" == "--latest" ]]; then
+  latest_image="memgraph:latest"
+  tag_latest="-t memgraph:latest"
+  shift
+elif [[ $# -ne 1 || "$1" == "-h" || "$1" == "--help" ]]; then
+  print_help
+  exit 1
+fi
+
+package_path="$1"
+if [[ ! -f "$package_path" ]]; then
+  echo "File '$package_path' does not exist!"
+  exit 1
+fi
+
+# Copy the .deb to working directory.
+cp "$package_path" "${working_dir}/"
+cd ${working_dir}
+
+extension="${package_path##*.}"
+
+if [[ "$extension" == "deb" ]]; then
+  # Extract version and offering from deb name.
+  package_name=`echo $(basename "$package_path") | sed 's/.deb$//'`
+  version=`echo ${package_name} | cut -d '_' -f 2 | rev | cut -d '-' -f 2- | rev | tr '+~' '__'`
+  dockerfile_path="${working_dir}/memgraph_deb.dockerfile"
+elif [[ "$extension" == "rpm" ]]; then
+  # Extract version and offering from deb name.
+  package_name=`echo $(basename "$package_path") | sed 's/.rpm$//'`
+  version=`echo ${package_name} | cut -d '-' -f 2 | rev | cut -d '-' -f 2- | rev`
+  version=${version%_1}
+  dockerfile_path="${working_dir}/memgraph_rpm.dockerfile"
+else
+  echo "Invalid file sent as the package"
+  print_help
+  exit 1
+fi
+
+image_name="memgraph:${version}"
+image_package_name="memgraph-${version}-docker.tar.gz"
+
+# Build docker image.
+docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} --build-arg release=${package_name}.${extension} .
+docker save ${image_name} ${latest_image} > ${image_package_name}
+rm "${package_name}.${extension}"
+echo "Built Docker image at '${working_dir}/${image_package_name}'"

release/package/run.sh

@@ -106,7 +106,7 @@ case "$1" in
         last_package_name=$(cd "$HOST_OUTPUT_DIR/$based_on_os" && ls -t memgraph* | head -1)
         docker_build_folder="$PROJECT_ROOT/release/docker"
         cd "$docker_build_folder"
-        ./package_deb_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name"
+        ./package_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name"
         # shellcheck disable=SC2012
         docker_image_name=$(cd "$docker_build_folder" && ls -t memgraph* | head -1)
         docker_host_folder="$HOST_OUTPUT_DIR/docker"

src/storage/v2/property_store.cpp

@@ -903,9 +903,6 @@ uint64_t ToPowerOf8(uint64_t size) {
 // of the two sets of data is currently active. Because the first byte of the
 // buffer is used to distinguish which of the two sets of data is used, we can
 // only use the leftover 15 bytes for raw data storage.
-#ifndef __x86_64__
-#error The PropertyStore only supports x86_64
-#endif
 
 const uint8_t kUseLocalBuffer = 0x01;
 

src/storage/v2/property_store.hpp

@@ -19,6 +19,7 @@
 namespace storage {
 
 class PropertyStore {
+  static_assert(std::endian::native == std::endian::little, "PropertyStore supports only architectures using little-endian.");
  public:
   PropertyStore();
 

src/utils/CMakeLists.txt

@@ -13,6 +13,7 @@ set(utils_src_files
     temporal.cpp
     thread.cpp
     thread_pool.cpp
+    tsc.cpp
     uuid.cpp)
 
 find_package(Boost REQUIRED)
@@ -22,7 +23,7 @@ find_package(Threads REQUIRED)
 
 add_library(mg-utils STATIC ${utils_src_files})
 target_link_libraries(mg-utils PUBLIC Boost::headers fmt::fmt spdlog::spdlog)
-target_link_libraries(mg-utils PRIVATE stdc++fs Threads::Threads gflags uuid rt)
+target_link_libraries(mg-utils PRIVATE librdtsc stdc++fs Threads::Threads gflags uuid rt)
 
 set(settings_src_files
     settings.cpp)

src/utils/tsc.cpp

@@ -0,0 +1,41 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include <functional>
+
+extern "C" {
+#include <librdtsc/rdtsc.h>
+}
+
+#include "utils/tsc.hpp"
+
+namespace utils {
+uint64_t ReadTSC() { return rdtsc(); }
+
+std::optional<double> GetTSCFrequency() {
+  // init is only needed for fetching frequency
+  static auto result = std::invoke([] { return rdtsc_init(); });
+  return result == 0 ? std::optional{rdtsc_get_tsc_hz()} : std::nullopt;
+}
+
+TSCTimer::TSCTimer(std::optional<double> frequency) : frequency_(frequency) {
+  if (!frequency_) return;
+  start_value_ = utils::ReadTSC();
+}
+
+double TSCTimer::Elapsed() const {
+  if (!frequency_) return 0.0;
+  auto current_value = utils::ReadTSC();
+  auto delta = current_value - start_value_;
+  return static_cast<double>(delta) / *frequency_;
+}
+
+}  // namespace utils

src/utils/tsc.hpp

@@ -1,4 +1,4 @@
-// Copyright 2021 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,124 +11,29 @@
 
 #pragma once
 
-#include <sys/prctl.h>
+#include <cstdint>
-#include <x86intrin.h>
-
-#include <chrono>
 #include <optional>
-#include <thread>
-
-#include "utils/timer.hpp"
 
 namespace utils {
 
 // TSC stands for Time-Stamp Counter
 
-#ifndef __x86_64__
+uint64_t ReadTSC();
-#error The TSC library only supports x86_64
-#endif
 
-/// This function reads the CPUs internal Time-Stamp Counter. This counter is
+std::optional<double> GetTSCFrequency();
-/// used to get a precise timestamp. It differs from the usual POSIX
-/// `clock_gettime` in that (without additional computing) it can be only used
-/// for relative time measurements. The Linux kernel uses the TSC internally to
-/// implement `clock_gettime` when the clock source is set to TSC.
-///
-/// The TSC is implemented as a register in the CPU that increments its value
-/// with a constant rate. The behavior of the TSC initially was to increment the
-/// counter on each instruction. This was then changed to the current behavior
-/// (>= Pentium 4) that increments the TSC with a constant rate so that CPU
-/// frequency scaling doesn't affect the measurements.
-///
-/// The TSC is guaranteed that it won't overflow in 10 years and because it is
-/// mostly implemented as a 64bit register it doesn't overflow even in 190 years
-/// (see Intel manual).
-///
-/// One issue of the TSC is that it is unique for each logical CPU. That means
-/// that if a context switch happens between two `ReadTSC` calls it could mean
-/// that the counters could have a very large offset. Thankfully, the counters
-/// are reset to 0 at each CPU reset and the Linux kernel synchronizes all of
-/// the counters. The synchronization can be seen here (`tsc_verify_tsc_adjust`
-/// and `tsc_store_and_check_tsc_adjust`):
-/// https://github.com/torvalds/linux/blob/master/arch/x86/kernel/tsc_sync.c
-/// https://github.com/torvalds/linux/blob/master/arch/x86/kernel/tsc.c
-/// https://github.com/torvalds/linux/blob/master/arch/x86/include/asm/tsc.h
-///
-/// All claims here were taken from sections 17.17 and 2.8.6 of the Intel 64 and
-/// IA-32 Architectures Software Developer's Manual.
-///
-/// Here we use the RDTSCP instruction instead of the RDTSC instruction because
-/// the RDTSCP instruction forces serialization of code execution in the CPU.
-/// Intel recommends the usage of the RDTSCP instruction for precision timing:
-/// https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/ia-32-ia-64-benchmark-code-execution-paper.pdf
-///
-/// Inline assembly MUST NOT be used because the compiler won't be aware that
-/// certain registers are being overwritten by the RDTSCP instruction. That is
-/// why we use the builtin command.
-/// https://stackoverflow.com/questions/13772567/get-cpu-cycle-count/51907627#51907627
-///
-/// Comparison of hardware time sources can be seen here:
-/// https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_MRG/2/html/Realtime_Reference_Guide/chap-Timestamping.html#example-Hardware_Clock_Cost_Comparison
-inline unsigned long long ReadTSC() {
-  unsigned int cpuid;
-  return __rdtscp(&cpuid);
-}
-
-/// The TSC can be disabled using a flag in the CPU. This function checks for
-/// the availability of the TSC. If you call `ReadTSC` when the TSC isn't
-/// available it will cause a segmentation fault.
-/// https://blog.cr0.org/2009/05/time-stamp-counter-disabling-oddities.html
-inline bool CheckAvailableTSC() {
-  int ret;
-  if (prctl(PR_GET_TSC, &ret) != 0) return false;
-  return ret == PR_TSC_ENABLE;
-}
-
-/// This function calculates the frequency at which the TSC counter increments
-/// its value. The frequency is already metered by the Linux kernel, but the
-/// value isn't reliably exposed anywhere for us to read it.
-/// https://stackoverflow.com/questions/51919219/determine-tsc-frequency-on-linux
-/// https://stackoverflow.com/questions/35123379/getting-tsc-rate-in-x86-kernel
-/// Because of that, we determine the value ourselves. We read the value two
-/// times with a delay between the two reads. The duration of the delay is
-/// determined using system calls that themselves internally use the already
-/// calibrated TSC. Because of that we get a very accurate value of the TSC
-/// frequency.
-inline std::optional<double> GetTSCFrequency() {
-  if (!CheckAvailableTSC()) return std::nullopt;
-
-  utils::Timer timer;
-  auto start_value = utils::ReadTSC();
-  std::this_thread::sleep_for(std::chrono::milliseconds(1));
-  auto duration = timer.Elapsed().count();
-  auto stop_value = utils::ReadTSC();
-
-  auto delta = stop_value - start_value;
-  return static_cast<double>(delta) / duration;
-}
 
 /// Class that is used to measure elapsed time using the TSC directly. It has
 /// almost zero overhead and is appropriate for use in performance critical
 /// paths.
 class TSCTimer {
  public:
-  TSCTimer() {}
+  TSCTimer() = default;
-
+  explicit TSCTimer(std::optional<double> frequency);
-  explicit TSCTimer(std::optional<double> frequency) : frequency_(frequency) {
+  double Elapsed() const;
-    if (!frequency_) return;
-    start_value_ = ReadTSC();
-  }
-
-  double Elapsed() const {
-    if (!frequency_) return 0.0;
-    auto current_value = ReadTSC();
-    auto delta = current_value - start_value_;
-    return static_cast<double>(delta) / *frequency_;
-  }
 
  private:
   std::optional<double> frequency_;
-  unsigned long long start_value_{0};
+  uint64_t start_value_{0};
 };
 
 }  // namespace utils