From cbe15e7f446e548c63dda694277206b966408d30 Mon Sep 17 00:00:00 2001 From: Jure Bajic Date: Thu, 23 Jun 2022 14:02:21 +0200 Subject: [PATCH] Support Ubuntu 22.04 (#411) * Fix toolchain url * Add 22.04 os script * Update centos-9 stream * Update zlib * Upgrade dotnet version * Add libipt * Update toolchain extension * Add release container * Remove CentOS 8 * Enable arm packaging * Add centos 9 container * Ignore missing python virtualenv * Add virtualenvwrapper * Rearrange order * Install virtualenv without user * Run virtualenv via python * Add logrotate dependency * Bump up checkout action version --- .github/workflows/package_all.yaml | 76 ++++++++--- environment/os/centos-8.sh | 161 ----------------------- environment/os/centos-9.sh | 62 ++++++--- environment/os/debian-11-arm.sh | 9 -- environment/os/debian-11.sh | 2 +- environment/os/ubuntu-22.04.sh | 93 +++++++++++++ environment/toolchain/v4.sh | 19 ++- init | 4 +- release/CMakeLists.txt | 2 +- release/docker/memgraph_rpm.dockerfile | 31 ----- release/package/centos-7/Dockerfile | 6 +- release/package/centos-8/Dockerfile | 14 -- release/package/centos-9/Dockerfile | 14 ++ release/package/debian-10/Dockerfile | 6 +- release/package/debian-11-arm/Dockerfile | 17 +++ release/package/debian-11/Dockerfile | 6 +- release/package/docker-compose.yml | 10 +- release/package/run.sh | 10 +- release/package/ubuntu-18.04/Dockerfile | 6 +- release/package/ubuntu-20.04/Dockerfile | 6 +- release/package/ubuntu-22.04/Dockerfile | 17 +++ 21 files changed, 285 insertions(+), 286 deletions(-) delete mode 100755 environment/os/centos-8.sh create mode 100755 environment/os/ubuntu-22.04.sh delete mode 100644 release/docker/memgraph_rpm.dockerfile delete mode 100644 release/package/centos-8/Dockerfile create mode 100644 release/package/centos-9/Dockerfile create mode 100644 release/package/debian-11-arm/Dockerfile create mode 100644 release/package/ubuntu-22.04/Dockerfile diff --git a/.github/workflows/package_all.yaml b/.github/workflows/package_all.yaml index 4c256bad7..b04ec3d02 100644 --- a/.github/workflows/package_all.yaml +++ b/.github/workflows/package_all.yaml @@ -6,11 +6,11 @@ on: workflow_dispatch jobs: centos-7: - runs-on: [self-hosted, DockerMgBuild] + runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: - name: "Set up repository" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 # Required because of release/get_version.py - name: "Build package" @@ -22,29 +22,29 @@ jobs: name: centos-7 path: build/output/centos-7/memgraph*.rpm - centos-8: - runs-on: [self-hosted, DockerMgBuild] + centos-9: + runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: - name: "Set up repository" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 # Required because of release/get_version.py - name: "Build package" run: | - ./release/package/run.sh package centos-8 + ./release/package/run.sh package centos-9 - name: "Upload package" uses: actions/upload-artifact@v2 with: - name: centos-8 - path: build/output/centos-8/memgraph*.rpm + name: centos-9 + path: build/output/centos-9/memgraph*.rpm debian-10: - runs-on: [self-hosted, DockerMgBuild] + runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: - name: "Set up repository" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 # Required because of release/get_version.py - name: "Build package" @@ -57,11 +57,11 @@ jobs: path: build/output/debian-10/memgraph*.deb debian-11: - runs-on: [self-hosted, DockerMgBuild] + runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: - name: "Set up repository" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 # Required because of release/get_version.py - name: "Build package" @@ -74,11 +74,11 @@ jobs: path: build/output/debian-11/memgraph*.deb docker: - runs-on: [self-hosted, DockerMgBuild] + runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: - name: "Set up repository" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 # Required because of release/get_version.py - name: "Build package" @@ -93,11 +93,11 @@ jobs: path: build/output/docker/memgraph*.tar.gz ubuntu-1804: - runs-on: [self-hosted, DockerMgBuild] + runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: - name: "Set up repository" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 # Required because of release/get_version.py - name: "Build package" @@ -110,11 +110,11 @@ jobs: path: build/output/ubuntu-18.04/memgraph*.deb ubuntu-2004: - runs-on: [self-hosted, DockerMgBuild] + runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: - name: "Set up repository" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 # Required because of release/get_version.py - name: "Build package" @@ -126,12 +126,29 @@ jobs: name: ubuntu-2004 path: build/output/ubuntu-20.04/memgraph*.deb - debian-11-platform: - runs-on: [self-hosted, DockerMgBuild] + ubuntu-2204: + runs-on: [self-hosted, DockerMgBuild, X64] timeout-minutes: 60 steps: - name: "Set up repository" - uses: actions/checkout@v2 + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + - name: "Build package" + run: | + ./release/package/run.sh package ubuntu-22.04 + - name: "Upload package" + uses: actions/upload-artifact@v2 + with: + name: ubuntu-2204 + path: build/output/ubuntu-22.04/memgraph*.deb + + debian-11-platform: + runs-on: [self-hosted, DockerMgBuild, X64] + timeout-minutes: 60 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 with: fetch-depth: 0 # Required because of release/get_version.py - name: "Build package" @@ -142,3 +159,20 @@ jobs: with: name: debian-11-platform path: build/output/debian-11/memgraph*.deb + + debian-11-arm: + runs-on: [self-hosted, DockerMgBuild, ARM64, strange] + timeout-minutes: 60 + steps: + - name: "Set up repository" + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Required because of release/get_version.py + - name: "Build package" + run: | + ./release/package/run.sh package debian-11-arm + - name: "Upload package" + uses: actions/upload-artifact@v2 + with: + name: debian-11 + path: build/output/debian-11/memgraph*.deb diff --git a/environment/os/centos-8.sh b/environment/os/centos-8.sh deleted file mode 100755 index 50c4f8f3a..000000000 --- a/environment/os/centos-8.sh +++ /dev/null @@ -1,161 +0,0 @@ -#!/bin/bash - -set -Eeuo pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "$DIR/../util.sh" - -TOOLCHAIN_BUILD_DEPS=( - coreutils-common gcc gcc-c++ make # generic build tools - wget # used for archive download - gnupg2 # used for archive signature verification - tar gzip bzip2 xz unzip # used for archive unpacking - zlib-devel # zlib library used for all builds - expat-devel libipt libipt-devel libbabeltrace-devel xz-devel python36-devel texinfo # for gdb - libcurl-devel # for cmake - curl # snappy - readline-devel # for cmake and llvm - libffi-devel libxml2-devel # for llvm - libedit-devel pcre-devel automake bison # for swig - file - openssl-devel - gmp-devel - gperf - patch -) - -TOOLCHAIN_RUN_DEPS=( - make # generic build tools - tar gzip bzip2 xz # used for archive unpacking - zlib # zlib library used for all builds - expat libipt libbabeltrace xz-libs python36 # for gdb - readline # for cmake and llvm - libffi libxml2 # for llvm - openssl-devel -) - -MEMGRAPH_BUILD_DEPS=( - git # source code control - make pkgconf-pkg-config # build system - curl wget # for downloading libs - libuuid-devel java-11-openjdk # required by antlr - readline-devel # for memgraph console - python36-devel # for query modules - openssl-devel - libseccomp-devel - python36 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests - # - # IMPORTANT: python3-yaml does NOT exist on CentOS - # Install it manually using `pip3 install PyYAML` - # - PyYAML # Package name here does not correspond to the yum package! - libcurl-devel # mg-requests - rpm-build rpmlint # for RPM package building - doxygen graphviz # source documentation generators - which mono-complete dotnet-sdk-3.1 nodejs golang zip unzip java-11-openjdk-devel # for driver tests - sbcl # for custom Lisp C++ preprocessing - autoconf # for jemalloc code generation - libtool # for protobuf code generation -) - -list() { - echo "$1" -} - -check() { - local missing="" - for pkg in $1; do - if [ "$pkg" == "PyYAML" ]; then - if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then - missing="$pkg $missing" - fi - continue - fi - if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then - missing="$pkg $missing" - fi - done - if [ "$missing" != "" ]; then - echo "MISSING PACKAGES: $missing" - exit 1 - fi -} - -install() { - cd "$DIR" - if [ "$EUID" -ne 0 ]; then - echo "Please run as root." - exit 1 - fi - # If GitHub Actions runner is installed, append LANG to the environment. - # Python related tests doesn't work the LANG export. - if [ -d "/home/gh/actions-runner" ]; then - echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env - else - echo "NOTE: export LANG=en_US.utf8" - fi - dnf install -y epel-release - dnf install -y 'dnf-command(config-manager)' - dnf config-manager --set-enabled powertools # Required to install texinfo. - dnf update -y - dnf install -y wget git python36 python3-pip - for pkg in $1; do - if [ "$pkg" == libipt ]; then - if ! dnf list installed libipt >/dev/null 2>/dev/null; then - dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm - fi - continue - fi - if [ "$pkg" == libipt-devel ]; then - if ! yum list installed libipt-devel >/dev/null 2>/dev/null; then - dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm - fi - continue - fi - # Install GDB dependencies not present in the standard repos. - # https://bugs.centos.org/view.php?id=17068 - # https://centos.pkgs.org - # Since 2020, there is Babeltrace2 (https://babeltrace.org). Not used - # within GDB yet (an assumption). - # http://mirror.centos.org/centos/8/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm not working - if [ "$pkg" == libbabeltrace-devel ]; then - if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then - dnf install -y https://rpmfind.net/linux/centos/8-stream/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm - fi - continue - fi - if [ "$pkg" == sbcl ]; then - if ! dnf list installed cl-asdf >/dev/null 2>/dev/null; then - dnf install -y https://pkgs.dyn.su/el8/base/x86_64/cl-asdf-20101028-18.el8.noarch.rpm - fi - if ! dnf list installed common-lisp-controller >/dev/null 2>/dev/null; then - dnf install -y https://pkgs.dyn.su/el8/base/x86_64/common-lisp-controller-7.4-20.el8.noarch.rpm - fi - if ! dnf list installed sbcl >/dev/null 2>/dev/null; then - dnf install -y https://pkgs.dyn.su/el8/base/x86_64/sbcl-2.0.1-4.el8.x86_64.rpm - fi - continue - fi - if [ "$pkg" == dotnet-sdk-3.1 ]; then - if ! dnf list installed dotnet-sdk-3.1 >/dev/null 2>/dev/null; then - wget -nv https://packages.microsoft.com/config/centos/8/packages-microsoft-prod.rpm -O packages-microsoft-prod.rpm - rpm -Uvh https://packages.microsoft.com/config/centos/8/packages-microsoft-prod.rpm - dnf update -y - dnf install -y dotnet-sdk-3.1 - fi - continue - fi - if [ "$pkg" == PyYAML ]; then - if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker). - pip3 install --user PyYAML - else # Running using sudo. - sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML" - fi - continue - fi - dnf install -y "$pkg" - done -} - -deps=$2"[*]" -"$1" "${!deps}" diff --git a/environment/os/centos-9.sh b/environment/os/centos-9.sh index 3061b3f50..ab0843a37 100755 --- a/environment/os/centos-9.sh +++ b/environment/os/centos-9.sh @@ -6,14 +6,12 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source "$DIR/../util.sh" TOOLCHAIN_BUILD_DEPS=( - coreutils gcc gcc-c++ make # generic build tools + coreutils-common gcc gcc-c++ make # generic build tools wget # used for archive download gnupg2 # used for archive signature verification tar gzip bzip2 xz unzip # used for archive unpacking zlib-devel # zlib library used for all builds - expat-devel xz-devel python3-devel texinfo # for gdb - libcurl-devel # for cmake - curl # snappy + expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb readline-devel # for cmake and llvm libffi-devel libxml2-devel # for llvm libedit-devel pcre-devel automake bison # for swig @@ -21,6 +19,9 @@ TOOLCHAIN_BUILD_DEPS=( openssl-devel gmp-devel gperf + diffutils + libipt libipt-devel # intel + patch ) TOOLCHAIN_RUN_DEPS=( @@ -31,18 +32,19 @@ TOOLCHAIN_RUN_DEPS=( readline # for cmake and llvm libffi libxml2 # for llvm openssl-devel + perl # for openssl ) MEMGRAPH_BUILD_DEPS=( git # source code control make pkgconf-pkg-config # build system - curl wget # for downloading libs + wget # for downloading libs libuuid-devel java-11-openjdk # required by antlr readline-devel # for memgraph console python3-devel # for query modules openssl-devel libseccomp-devel - python3 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests + python3 python3-pip python3-virtualenv nmap-ncat # for qa, macro_benchmark and stress tests # # IMPORTANT: python3-yaml does NOT exist on CentOS # Install it manually using `pip3 install PyYAML` @@ -73,12 +75,6 @@ check() { if [ "$pkg" == "python3-virtualenv" ]; then continue fi - if [ "$pkg" == sbcl ]; then - if ! sbcl --version &> /dev/null; then - missing="$pkg $missing" - fi - continue - fi if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then missing="$pkg $missing" fi @@ -105,13 +101,37 @@ install() { yum update -y yum install -y wget git python3 python3-pip for pkg in $1; do + # Since there is no support for libipt-devel for CentOS 9 we install + # Fedoras version of same libs, they are the same version but released + # for different OS + # TODO Update when libipt-devel releases for CentOS 9 + if [ "$pkg" == libipt ]; then + if ! dnf list installed libipt >/dev/null 2>/dev/null; then + dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm + fi + continue + fi + if [ "$pkg" == libipt-devel ]; then + if ! dnf list installed libipt-devel >/dev/null 2>/dev/null; then + dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm + fi + continue + fi + if [ "$pkg" == libbabeltrace-devel ]; then + if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then + dnf install -y http://mirror.stream.centos.org/9-stream/CRB/x86_64/os/Packages/libbabeltrace-devel-1.5.8-10.el9.x86_64.rpm + fi + continue + fi if [ "$pkg" == sbcl ]; then - if ! sbcl --version &> /dev/null; then - curl -s https://altushost-swe.dl.sourceforge.net/project/sbcl/sbcl/1.4.2/sbcl-1.4.2-arm64-linux-binary.tar.bz2 -o /tmp/sbcl-arm64.tar.bz2 - tar xvjf /tmp/sbcl-arm64.tar.bz2 -C /tmp - pushd /tmp/sbcl-1.4.2-arm64-linux - INSTALL_ROOT=/usr/local sh install.sh - popd + if ! dnf list installed cl-asdf >/dev/null 2>/dev/null; then + dnf install -y https://pkgs.dyn.su/el8/base/x86_64/cl-asdf-20101028-18.el8.noarch.rpm + fi + if ! dnf list installed common-lisp-controller >/dev/null 2>/dev/null; then + dnf install -y https://pkgs.dyn.su/el8/base/x86_64/common-lisp-controller-7.4-20.el8.noarch.rpm + fi + if ! dnf list installed sbcl >/dev/null 2>/dev/null; then + dnf install -y https://pkgs.dyn.su/el8/base/x86_64/sbcl-2.0.1-4.el8.x86_64.rpm fi continue fi @@ -125,9 +145,11 @@ install() { fi if [ "$pkg" == python3-virtualenv ]; then if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker). - pip3 install --user virtualenv + pip3 install virtualenv + pip3 install virtualenvwrapper else # Running using sudo. - sudo -H -u "$SUDO_USER" bash -c "pip3 install --user virtualenv" + sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenv" + sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenvwrapper" fi continue fi diff --git a/environment/os/debian-11-arm.sh b/environment/os/debian-11-arm.sh index c73b91487..8d5ad5644 100755 --- a/environment/os/debian-11-arm.sh +++ b/environment/os/debian-11-arm.sh @@ -87,15 +87,6 @@ EOF fi apt install -y wget for pkg in $1; do - if [ "$pkg" == dotnet-sdk-3.1 ]; then - if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then - wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb - dpkg -i packages-microsoft-prod.deb - apt-get update - apt-get install -y apt-transport-https dotnet-sdk-3.1 - fi - continue - fi apt install -y "$pkg" done } diff --git a/environment/os/debian-11.sh b/environment/os/debian-11.sh index 1e361faf0..b6e639f3d 100755 --- a/environment/os/debian-11.sh +++ b/environment/os/debian-11.sh @@ -87,7 +87,7 @@ EOF fi apt install -y wget for pkg in $1; do - if [ "$pkg" == dotnet-sdk-3.1 ]; then + if [ "$pkg" == dotnet-sdk-3.1 ]; then if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb dpkg -i packages-microsoft-prod.deb diff --git a/environment/os/ubuntu-22.04.sh b/environment/os/ubuntu-22.04.sh new file mode 100755 index 000000000..ce9e389ab --- /dev/null +++ b/environment/os/ubuntu-22.04.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +set -Eeuo pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source "$DIR/../util.sh" + +TOOLCHAIN_BUILD_DEPS=( + coreutils gcc g++ build-essential make # generic build tools + wget # used for archive download + gnupg # used for archive signature verification + tar gzip bzip2 xz-utils unzip # used for archive unpacking + zlib1g-dev # zlib library used for all builds + libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb + libcurl4-openssl-dev # for cmake + libreadline-dev # for cmake and llvm + libffi-dev libxml2-dev # for llvm + curl # snappy + file + git # for thrift + libgmp-dev # for gdb + gperf # for proxygen + libssl-dev + libedit-dev libpcre3-dev automake bison # for swig +) + +TOOLCHAIN_RUN_DEPS=( + make # generic build tools + tar gzip bzip2 xz-utils # used for archive unpacking + zlib1g # zlib library used for all builds + libexpat1 libipt2 libbabeltrace1 liblzma5 python3 # for gdb + libcurl4 # for cmake + libreadline8 # for cmake and llvm + libffi7 libxml2 # for llvm + libssl-dev # for libevent +) + +MEMGRAPH_BUILD_DEPS=( + git # source code control + make pkg-config # build system + curl wget # for downloading libs + uuid-dev default-jre-headless # required by antlr + libreadline-dev # for memgraph console + libpython3-dev python3-dev # for query modules + libssl-dev + libseccomp-dev + netcat # tests are using nc to wait for memgraph + python3 python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests + python3-yaml # for the configuration generator + libcurl4-openssl-dev # mg-requests + sbcl # for custom Lisp C++ preprocessing + doxygen graphviz # source documentation generators + mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests + dotnet-sdk-6.0 golang nodejs npm + autoconf # for jemalloc code generation + libtool # for protobuf code generation +) + +list() { + echo "$1" +} + +check() { + check_all_dpkg "$1" +} + +install() { + cd "$DIR" + apt update + # If GitHub Actions runner is installed, append LANG to the environment. + # Python related tests doesn't work the LANG export. + if [ -d "/home/gh/actions-runner" ]; then + echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env + else + echo "NOTE: export LANG=en_US.utf8" + fi + apt install -y wget + for pkg in $1; do + if [ "$pkg" == dotnet-sdk-6.0 ]; then + if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then + wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb + dpkg -i packages-microsoft-prod.deb + apt-get update + apt-get install -y apt-transport-https dotnet-sdk-6.0 + fi + continue + fi + apt install -y "$pkg" + done +} + +deps=$2"[*]" +"$1" "${!deps}" diff --git a/environment/toolchain/v4.sh b/environment/toolchain/v4.sh index 164e1d4be..34ce2c052 100755 --- a/environment/toolchain/v4.sh +++ b/environment/toolchain/v4.sh @@ -675,7 +675,7 @@ PROXYGEN_SHA256=5360a8ccdfb2f5a6c7b3eed331ec7ab0e2c792d579c6fff499c85c516c11fe14 SNAPPY_SHA256=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7 SNAPPY_VERSION=1.1.9 XZ_VERSION=5.2.5 # for LZMA -ZLIB_VERSION=1.2.11 +ZLIB_VERSION=1.2.12 ZSTD_VERSION=1.5.0 WANGLE_SHA256=1002e9c32b6f4837f6a760016e3b3e22f3509880ef3eaad191c80dc92655f23f @@ -1178,12 +1178,21 @@ popd # create toolchain archive if [ ! -f $NAME-binaries-$DISTRO.tar.gz ]; then - DISTRO_FULL_NAME=$DISTRO - if [ "$for_arm" = true ]; then - DISTRO_FULL_NAME="$DISTRO_FULL_NAME-aarch64" + DISTRO_FULL_NAME=${DISTRO} + if [[ "${DISTRO}" == centos* ]]; then + if [[ "$for_arm" = "true" ]]; then + DISTRO_FULL_NAME="$DISTRO_FULL_NAME-aarch64" + else + DISTRO_FULL_NAME="$DISTRO_FULL_NAME-x86_64" + fi else - DISTRO_FULL_NAME="$DISTRO_FULL_NAME-x86_64" + if [[ "$for_arm" = "true" ]]; then + DISTRO_FULL_NAME="$DISTRO_FULL_NAME-arm64" + else + DISTRO_FULL_NAME="$DISTRO_FULL_NAME-amd64" + fi fi + tar --owner=root --group=root -cpvzf $NAME-binaries-$DISTRO_FULL_NAME.tar.gz -C /opt $NAME fi diff --git a/init b/init index b74f019bb..53bec7045 100755 --- a/init +++ b/init @@ -24,7 +24,7 @@ function setup_virtualenv () { fi # create new virtualenv - virtualenv -p python3 ve3 || exit 1 + python3 -m virtualenv -p python3 ve3 || exit 1 source ve3/bin/activate pip --timeout 1000 install -r requirements.txt || exit 1 deactivate @@ -66,7 +66,7 @@ fi DISTRO=$(operating_system) ARCHITECTURE=$(architecture) -if [ "${ARCHITECTURE}" = "arm64" ]; then +if [ "${ARCHITECTURE}" = "arm64" ] || [ "${ARCHITECTURE}" = "aarch64" ]; then OS_SCRIPT=$DIR/environment/os/$DISTRO-arm.sh else OS_SCRIPT=$DIR/environment/os/$DISTRO.sh diff --git a/release/CMakeLists.txt b/release/CMakeLists.txt index 0fb250db8..2715d39c3 100644 --- a/release/CMakeLists.txt +++ b/release/CMakeLists.txt @@ -67,7 +67,7 @@ It aims to deliver developers the speed, simplicity and scale required to build the next generation of applications driver by real-time connected data.") # Add `openssl` package to dependencies list. Used to generate SSL certificates. # We also depend on `python3` because we embed it in Memgraph. -set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, libstdc >= 6") +set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, libstdc >= 6, logrotate") # All variables must be set before including. include(CPack) diff --git a/release/docker/memgraph_rpm.dockerfile b/release/docker/memgraph_rpm.dockerfile deleted file mode 100644 index 5ce0e3765..000000000 --- a/release/docker/memgraph_rpm.dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM dokken/centos-stream-9 -# NOTE: If you change the base distro update release/package as well. - -ARG release - -RUN yum update && yum install -y \ - openssl libcurl libseccomp python3 python3-pip \ - --nobest --allowerasing \ - && rm -rf /tmp/* \ - && yum clean all - -RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3 - -COPY ${release} / - -# Install memgraph package -RUN rpm -i ${release} - -# Memgraph listens for Bolt Protocol on this port by default. -EXPOSE 7687 -# Snapshots and logging volumes -VOLUME /var/log/memgraph -VOLUME /var/lib/memgraph -# Configuration volume -VOLUME /etc/memgraph - -USER memgraph -WORKDIR /usr/lib/memgraph - -ENTRYPOINT ["/usr/lib/memgraph/memgraph"] -CMD [""] diff --git a/release/package/centos-7/Dockerfile b/release/package/centos-7/Dockerfile index 9d7922a5b..11778a179 100644 --- a/release/package/centos-7/Dockerfile +++ b/release/package/centos-7/Dockerfile @@ -7,8 +7,8 @@ RUN yum -y update \ # Do NOT be smart here and clean the cache because the container is used in the # stateful context. -RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-7.tar.gz \ - -O ${TOOLCHAIN_VERSION}-binaries-centos-7.tar.gz \ - && tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-7.tar.gz -C /opt +RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \ + -O ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \ + && tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz -C /opt ENTRYPOINT ["sleep", "infinity"] diff --git a/release/package/centos-8/Dockerfile b/release/package/centos-8/Dockerfile deleted file mode 100644 index e9a1b556d..000000000 --- a/release/package/centos-8/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM centos:8 - -ARG TOOLCHAIN_VERSION - -RUN dnf -y update \ - && dnf install -y wget git -# Do NOT be smart here and clean the cache because the container is used in the -# stateful context. - -RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-8.tar.gz \ - -O ${TOOLCHAIN_VERSION}-binaries-centos-8.tar.gz \ - && tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-8.tar.gz -C /opt - -ENTRYPOINT ["sleep", "infinity"] diff --git a/release/package/centos-9/Dockerfile b/release/package/centos-9/Dockerfile new file mode 100644 index 000000000..87c8f8b02 --- /dev/null +++ b/release/package/centos-9/Dockerfile @@ -0,0 +1,14 @@ +FROM quay.io/centos/centos:stream9 + +ARG TOOLCHAIN_VERSION + +RUN yum -y update \ + && yum install -y wget git +# Do NOT be smart here and clean the cache because the container is used in the +# stateful context. + +RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \ + -O ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \ + && tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz -C /opt + +ENTRYPOINT ["sleep", "infinity"] diff --git a/release/package/debian-10/Dockerfile b/release/package/debian-10/Dockerfile index f59616439..a1519f04b 100644 --- a/release/package/debian-10/Dockerfile +++ b/release/package/debian-10/Dockerfile @@ -10,8 +10,8 @@ RUN apt update && apt install -y \ # Do NOT be smart here and clean the cache because the container is used in the # stateful context. -RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-10.tar.gz \ - -O ${TOOLCHAIN_VERSION}-binaries-debian-10.tar.gz \ - && tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-10.tar.gz -C /opt +RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \ + -O ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \ + && tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz -C /opt ENTRYPOINT ["sleep", "infinity"] diff --git a/release/package/debian-11-arm/Dockerfile b/release/package/debian-11-arm/Dockerfile new file mode 100644 index 000000000..243a527ad --- /dev/null +++ b/release/package/debian-11-arm/Dockerfile @@ -0,0 +1,17 @@ +FROM debian:11 + +ARG TOOLCHAIN_VERSION + +# Stops tzdata interactive configuration. +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt update && apt install -y \ + ca-certificates wget git +# Do NOT be smart here and clean the cache because the container is used in the +# stateful context. + +RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \ + -O ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \ + && tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz -C /opt + +ENTRYPOINT ["sleep", "infinity"] diff --git a/release/package/debian-11/Dockerfile b/release/package/debian-11/Dockerfile index 876768a68..f28c32e89 100644 --- a/release/package/debian-11/Dockerfile +++ b/release/package/debian-11/Dockerfile @@ -10,8 +10,8 @@ RUN apt update && apt install -y \ # Do NOT be smart here and clean the cache because the container is used in the # stateful context. -RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11.tar.gz \ - -O ${TOOLCHAIN_VERSION}-binaries-debian-11.tar.gz \ - && tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11.tar.gz -C /opt +RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \ + -O ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \ + && tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz -C /opt ENTRYPOINT ["sleep", "infinity"] diff --git a/release/package/docker-compose.yml b/release/package/docker-compose.yml index 2d8f01069..2f073267e 100644 --- a/release/package/docker-compose.yml +++ b/release/package/docker-compose.yml @@ -4,10 +4,10 @@ services: build: context: centos-7 container_name: "mgbuild_centos-7" - mgbuild_centos-8: + mgbuild_centos-9: build: - context: centos-8 - container_name: "mgbuild_centos-8" + context: centos-9 + container_name: "mgbuild_centos-9" mgbuild_debian-10: build: context: debian-10 @@ -24,3 +24,7 @@ services: build: context: ubuntu-20.04 container_name: "mgbuild_ubuntu-20.04" + mgbuild_ubuntu-22.04: + build: + context: ubuntu-22.04 + container_name: "mgbuild_ubuntu-22.04" diff --git a/release/package/run.sh b/release/package/run.sh index 156f63015..885a63970 100755 --- a/release/package/run.sh +++ b/release/package/run.sh @@ -3,7 +3,7 @@ set -Eeuo pipefail SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -SUPPORTED_OS=(centos-7 centos-8 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04) +SUPPORTED_OS=(centos-7 centos-9 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04 ubuntu-22.04 debian-11-arm) PROJECT_ROOT="$SCRIPT_DIR/../.." TOOLCHAIN_VERSION="toolchain-v4" ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate" @@ -67,14 +67,18 @@ make_package () { # environment/os/{os}.sh does not come within the toolchain package. When # migrating to the next version of toolchain do that, and remove the # TOOLCHAIN_RUN_DEPS installation from here. - echo "Installing dependencies..." + echo "Installing dependencies using '/memgraph/environment/os/$os.sh' script..." docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS" docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS" echo "Building targeted package..." docker exec "$build_container" bash -c "cd /memgraph && $ACTIVATE_TOOLCHAIN && ./init" docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*" - docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .." + if [[ "$os" == "debian-11-arm" ]]; then + docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release -DMG_ARCH="ARM64" $telemetry_id_override_flag .." + else + docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .." + fi # ' is used instead of " because we need to run make within the allowed # container resources. # shellcheck disable=SC2016 diff --git a/release/package/ubuntu-18.04/Dockerfile b/release/package/ubuntu-18.04/Dockerfile index 6222de609..c2b1d8d50 100644 --- a/release/package/ubuntu-18.04/Dockerfile +++ b/release/package/ubuntu-18.04/Dockerfile @@ -10,8 +10,8 @@ RUN apt update && apt install -y \ # Do NOT be smart here and clean the cache because the container is used in the # stateful context. -RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04.tar.gz \ - -O ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04.tar.gz \ - && tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04.tar.gz -C /opt +RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \ + -O ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \ + && tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz -C /opt ENTRYPOINT ["sleep", "infinity"] diff --git a/release/package/ubuntu-20.04/Dockerfile b/release/package/ubuntu-20.04/Dockerfile index 148876e2b..ffcd67384 100644 --- a/release/package/ubuntu-20.04/Dockerfile +++ b/release/package/ubuntu-20.04/Dockerfile @@ -10,8 +10,8 @@ RUN apt update && apt install -y \ # Do NOT be smart here and clean the cache because the container is used in the # stateful context. -RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04.tar.gz \ - -O ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04.tar.gz \ - && tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04.tar.gz -C /opt +RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \ + -O ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \ + && tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz -C /opt ENTRYPOINT ["sleep", "infinity"] diff --git a/release/package/ubuntu-22.04/Dockerfile b/release/package/ubuntu-22.04/Dockerfile new file mode 100644 index 000000000..86b4648fc --- /dev/null +++ b/release/package/ubuntu-22.04/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:22.04 + +ARG TOOLCHAIN_VERSION + +# Stops tzdata interactive configuration. +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt update && apt install -y \ + ca-certificates wget git +# Do NOT be smart here and clean the cache because the container is used in the +# stateful context. + +RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \ + -O ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \ + && tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz -C /opt + +ENTRYPOINT ["sleep", "infinity"]