Add docker release action (#356)

* Add docker release action * Add debian-11 arm script * Remove test prefix * Update package_docker script * Update release script * Unify architecture extension
2022-04-07 15:23:18 +02:00 · 2022-04-07 15:23:18 +02:00 · 1b619f51b2
commit 1b619f51b2
parent 537855a0b2
7 changed files with 191 additions and 16 deletions
--- a/.github/workflows/release_docker.yaml
+++ b/.github/workflows/release_docker.yaml
@ -0,0 +1,49 @@
+name: Publish Docker images
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Memgraph binary version to publish on Dockerhub."
+        required: true
+
+jobs:
+  docker_publish:
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_ORGANIZATION_NAME: memgraph
+      DOCKER_REPOSITORY_NAME: memgraph
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Download memgraph binary
+        run: |
+          cd release/docker
+          curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11/memgraph_${{ github.event.inputs.version }}-1_amd64.deb > memgraph-amd64.deb
+          curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11-aarch64/memgraph_${{ github.event.inputs.version }}-1_arm64.deb > memgraph-arm64.deb
+
+      - name: Build & push docker images
+        run: |
+          cd release/docker
+          docker buildx build \
+          --build-arg BINARY_NAME="memgraph-" \
+          --build-arg EXTENSION="deb" \
+          --platform linux/amd64,linux/arm64 \
+          --tag $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} \
+          --tag $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:latest \
+          --file memgraph_deb.dockerfile \
+          --push .
--- a/environment/os/debian-11-arm.sh
+++ b/environment/os/debian-11-arm.sh
@ -0,0 +1,104 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "$DIR/../util.sh"
+
+TOOLCHAIN_BUILD_DEPS=(
+    coreutils gcc g++ build-essential make # generic build tools
+    wget # used for archive download
+    gnupg # used for archive signature verification
+    tar gzip bzip2 xz-utils unzip # used for archive unpacking
+    zlib1g-dev # zlib library used for all builds
+    libexpat1-dev liblzma-dev python3-dev texinfo # for gdb
+    libcurl4-openssl-dev # for cmake
+    libreadline-dev # for cmake and llvm
+    libffi-dev libxml2-dev # for llvm
+    libedit-dev libpcre3-dev automake bison # for swig
+    curl # snappy
+    file # for libunwind
+    libssl-dev # for libevent
+    libgmp-dev
+    gperf # for proxygen
+    git # for fbthrift
+)
+
+TOOLCHAIN_RUN_DEPS=(
+    make # generic build tools
+    tar gzip bzip2 xz-utils # used for archive unpacking
+    zlib1g # zlib library used for all builds
+    libexpat1 liblzma5 python3 # for gdb
+    libcurl4 # for cmake
+    file # for CPack
+    libreadline8 # for cmake and llvm
+    libffi7 libxml2 # for llvm
+    libssl-dev # for libevent
+)
+
+MEMGRAPH_BUILD_DEPS=(
+    git # source code control
+    make pkg-config # build system
+    curl wget # for downloading libs
+    uuid-dev default-jre-headless # required by antlr
+    libreadline-dev # for memgraph console
+    libpython3-dev python3-dev # for query modules
+    libssl-dev
+    libseccomp-dev
+    netcat # tests are using nc to wait for memgraph
+    python3 virtualenv python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
+    python3-yaml # for the configuration generator
+    libcurl4-openssl-dev # mg-requests
+    sbcl # for custom Lisp C++ preprocessing
+    doxygen graphviz # source documentation generators
+    mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
+    golang nodejs npm
+    autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
+)
+
+list() {
+    echo "$1"
+}
+
+check() {
+    check_all_dpkg "$1"
+}
+
+install() {
+    cat >/etc/apt/sources.list <<EOF
+deb http://deb.debian.org/debian bullseye main
+deb-src http://deb.debian.org/debian bullseye main
+
+deb http://deb.debian.org/debian-security/ bullseye-security main
+deb-src http://deb.debian.org/debian-security/ bullseye-security main
+
+deb http://deb.debian.org/debian bullseye-updates main
+deb-src http://deb.debian.org/debian bullseye-updates main
+EOF
+    cd "$DIR"
+    apt update
+    # If GitHub Actions runner is installed, append LANG to the environment.
+    # Python related tests doesn't work the LANG export.
+    if [ -d "/home/gh/actions-runner" ]; then
+        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
+    else
+        echo "NOTE: export LANG=en_US.utf8"
+    fi
+    apt install -y wget
+    for pkg in $1; do
+        if [ "$pkg" == dotnet-sdk-3.1 ]; then
+            if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
+                wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
+                dpkg -i packages-microsoft-prod.deb
+                apt-get update
+                apt-get install -y apt-transport-https dotnet-sdk-3.1
+            fi
+            continue
+        fi
+        apt install -y "$pkg"
+    done
+}
+
+deps=$2"[*]"
+"$1" "${!deps}"
--- a/environment/util.sh
+++ b/environment/util.sh
@ -5,6 +5,10 @@ operating_system() {
    sort | cut -d '=' -f 2- | sed 's/"//g' | paste -s -d '-'
 }

+architecture() {
+    uname -m
+}
+
 check_all_yum() {
    local missing=""
    for pkg in $1; do
--- a/10
+++ b/10
@ -65,8 +65,14 @@ else
 fi

 DISTRO=$(operating_system)
-echo "ALL BUILD PACKAGES: $($DIR/environment/os/$DISTRO.sh list MEMGRAPH_BUILD_DEPS)"
-$DIR/environment/os/$DISTRO.sh check MEMGRAPH_BUILD_DEPS
+ARCHITECTURE=$(architecture)
+if [ "${ARCHITECTURE}" = "arm64" ]; then
+  OS_SCRIPT=$DIR/environment/os/$DISTRO-arm.sh
+else
+  OS_SCRIPT=$DIR/environment/os/$DISTRO.sh
+fi
+echo "ALL BUILD PACKAGES: $($OS_SCRIPT list MEMGRAPH_BUILD_DEPS)"
+$OS_SCRIPT check MEMGRAPH_BUILD_DEPS
 echo "All packages are in-place..."

 # create a default build directory
--- a/release/CMakeLists.txt
+++ b/release/CMakeLists.txt
@ -10,6 +10,14 @@ set(CPACK_PACKAGE_VENDOR "Memgraph Ltd.")
 set(CPACK_PACKAGE_DESCRIPTION_SUMMARY
    "High performance, in-memory, transactional graph database")

+# Setting arhitecture extension for deb packages
+set(MG_ARCH_EXTENSION_DEB "all")
+if (${MG_ARCH} STREQUAL "x86_64")
+    set(MG_ARCH_EXTENSION_DEB "amd64")
+elseif (${MG_ARCH} STREQUAL "ARM64")
+    set(MG_ARCH_EXTENSION_DEB  "arm64")
+endif()
+
 # DEB specific
 # Instead of using "name <email>" format, we use "email (name)" to prevent
 # errors due to full stop, '.' at the end of "Ltd". (See: RFC 822)
@ -17,7 +25,7 @@ set(CPACK_DEBIAN_PACKAGE_MAINTAINER "tech@memgraph.com (Memgraph Ltd.)")
 set(CPACK_DEBIAN_PACKAGE_SECTION non-free/database)
 set(CPACK_DEBIAN_PACKAGE_HOMEPAGE https://memgraph.com)
 set(CPACK_DEBIAN_PACKAGE_VERSION "${MEMGRAPH_VERSION_DEB}")
-set(CPACK_DEBIAN_FILE_NAME "memgraph_${MEMGRAPH_VERSION_DEB}_amd64.deb")
+set(CPACK_DEBIAN_FILE_NAME "memgraph_${MEMGRAPH_VERSION_DEB}_${MG_ARCH_EXTENSION_DEB}.deb")
 set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA
    "${CMAKE_CURRENT_SOURCE_DIR}/debian/conffiles;"
    "${CMAKE_CURRENT_SOURCE_DIR}/debian/copyright;"
@ -35,19 +43,18 @@ set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
 # We also depend on `python3` because we embed it in Memgraph.
 set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0)")

-# RPM specific
-
-set(MG_ARCH_EXTENSION "noarch")
-
+# Setting arhitecture extension for rpm packages
+set(MG_ARCH_EXTENSION_RPM "noarch")
 if (${MG_ARCH} STREQUAL "x86_64")
-    set(MG_ARCH_EXTENSION "x86_64")
+    set(MG_ARCH_EXTENSION_RPM "x86_64")
 elseif (${MG_ARCH} STREQUAL "ARM64")
-    set(MG_ARCH_EXTENSION  "aarch64")
+    set(MG_ARCH_EXTENSION_RPM  "aarch64")
 endif()

+# RPM specific
 set(CPACK_RPM_PACKAGE_URL https://memgraph.com)
 set(CPACK_RPM_PACKAGE_VERSION "${MEMGRAPH_VERSION_RPM}")
-set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.${MG_ARCH_EXTENSION}.rpm")
+set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.${MG_ARCH_EXTENSION_RPM}.rpm")
 set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
  /var /var/lib /var/log /etc/logrotate.d
  /lib /lib/systemd /lib/systemd/system /lib/systemd/system/memgraph.service)
--- a/release/docker/memgraph_deb.dockerfile
+++ b/release/docker/memgraph_deb.dockerfile
@ -1,19 +1,21 @@
 FROM debian:bullseye
 # NOTE: If you change the base distro update release/package as well.

-ARG release
+ARG BINARY_NAME
+ARG EXTENSION
+ARG TARGETARCH

 RUN apt-get update && apt-get install -y \
-    openssl libcurl4 libssl1.1 libseccomp2 python3 libpython3.9 python3-pip \
-    --no-install-recommends \
+  openssl libcurl4 libssl1.1 libseccomp2 python3 libpython3.9 python3-pip \
+  --no-install-recommends \
  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

 RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3

-COPY ${release} /
+COPY "${BINARY_NAME}${TARGETARCH}.${EXTENSION}" /

 # Install memgraph package
-RUN dpkg -i ${release}
+RUN dpkg -i "${BINARY_NAME}${TARGETARCH}.deb"

 # Memgraph listens for Bolt Protocol on this port by default.
 EXPOSE 7687
--- a/release/docker/package_docker
+++ b/release/docker/package_docker
@ -55,7 +55,10 @@ image_name="memgraph:${version}"
 image_package_name="memgraph-${version}-docker.tar.gz"

 # Build docker image.
-docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} --build-arg release=${package_name}.${extension} .
+docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} \
+--build-arg BINARY_NAME=${package_name} \
+--build-arg EXTENSION=${extension} \
+--build-arg TARGETARCH="" .
 docker save ${image_name} ${latest_image} | gzip > ${image_package_name}
 rm "${package_name}.${extension}"
 echo "Built Docker image at '${working_dir}/${image_package_name}'"