Commit docker builds WIP

This commit is contained in:
Deda 2024-03-15 14:49:34 +01:00
parent a664c810bc
commit 1865a80722
6 changed files with 282 additions and 41 deletions

View File

@ -26,10 +26,9 @@ on:
- debian-11
- debian-11-arm
- debian-11-platform
- debian-11-docker
- debian-11-arm-docker
- debian-12
- debian-12-arm
- docker
- fedora-36
- fedora-38
- fedora-39
@ -121,20 +120,6 @@ jobs:
artifact_name: "debian-11-platform"
secrets: inherit
Debian11-docker:
if: ${{ github.event.inputs.target_os == 'debian-11-docker' || github.event.inputs.target_os == 'all' }}
uses: ./.github/workflows/reusable_package.yaml
with:
os: "debian-11"
toolchain: ${{ github.event.inputs.toolchain }}
arch: "amd"
runner_arch_label: "X64"
build_type: ${{ github.event.inputs.build_type }}
timeout_minutes: 60
additional_build_args: "--for-docker"
artifact_name: "debian-11-docker"
secrets: inherit
Debian11-arm:
if: ${{ github.event.inputs.target_os == 'debian-11-arm' || github.event.inputs.target_os == 'all' }}
uses: ./.github/workflows/reusable_package.yaml
@ -147,20 +132,6 @@ jobs:
timeout_minutes: 120
secrets: inherit
# Debian11-arm-docker:
# if: ${{ github.event.inputs.target_os == 'debian-11-arm-docker' || github.event.inputs.target_os == 'all' }}
# uses: ./.github/workflows/reusable_package.yaml
# with:
# os: "debian-11-arm"
# toolchain: ${{ github.event.inputs.toolchain }}
# arch: "arm"
# runner_arch_label: "ARM64"
# build_type: ${{ github.event.inputs.build_type }}
# timeout_minutes: 120
# additional_build_args: "--for-docker"
# artifact_name: "debian-11-arm-docker"
# secrets: inherit
Debian12:
if: ${{ (github.event.inputs.target_os == 'debian-12' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v5' }}
uses: ./.github/workflows/reusable_package.yaml
@ -185,6 +156,34 @@ jobs:
timeout_minutes: 120
secrets: inherit
Docker-v4:
if: ${{ (github.event.inputs.target_os == 'docker' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v4' }}
uses: ./.github/workflows/reusable_package.yaml
with:
os: "debian-11"
toolchain: ${{ github.event.inputs.toolchain }}
arch: "amd"
runner_arch_label: "X64"
build_type: ${{ github.event.inputs.build_type }}
timeout_minutes: 60
additional_build_args: "--for-docker"
artifact_name: "debian-11-docker"
secrets: inherit
Docker-v5:
if: ${{ (github.event.inputs.target_os == 'docker' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v5' }}
uses: ./.github/workflows/reusable_package.yaml
with:
os: "debian-12"
toolchain: ${{ github.event.inputs.toolchain }}
arch: "amd"
runner_arch_label: "X64"
build_type: ${{ github.event.inputs.build_type }}
timeout_minutes: 60
additional_build_args: "--for-docker"
artifact_name: "debian-11-docker"
secrets: inherit
Fedora36:
if: ${{ (github.event.inputs.target_os == 'fedora-36' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v4' }}
uses: ./.github/workflows/reusable_package.yaml

View File

@ -0,0 +1,162 @@
name: Reusable docker package make and upload
on:
workflow_call:
inputs:
os:
type: string
description: "Target os. Default value is debian-11."
default: 'debian-11'
arch:
type: string
description: "Target architecture. Default value is amd."
default: 'amd'
runner_arch_label:
type: string
description: "Runner architecture label. Amd is built on X64, arm on ARM64. Default value is X64."
default: 'X64'
toolchain:
type: string
description: "Toolchain version. Default value is v5."
default: 'v5'
build_type:
type: string
description: "Memgraph Build type. Default value is Release."
default: 'Release'
artifact_name:
type: string
description: "Unique artifact name, pass it in to override the default. Default value is os name."
default: ''
additional_build_args:
type: string
description: "Additional build flags (--for-platform, --for-docker ...). Default value is empty."
default: ''
timeout_minutes:
type: number
description: "Timeout in minutes for the job. Default value is 60."
default: 60
push_to_s3:
type: boolean
description: "Should the final package be pushed to an S3 bucket. Default value is false"
default: false
s3_bucket:
type: string
description: "Name of the target S3 bucket."
s3_region:
type: string
description: "Region for the target S3 bucket."
s3_dest_dir:
type: string
description: "Target dir path in S3 bucket."
env:
ARTIFACT_NAME: ${{ inputs.artifact_name || inputs.os }}
jobs:
package:
runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"]
timeout-minutes: ${{ inputs.timeout_minutes }}
steps:
- name: "Set up repository"
uses: actions/checkout@v4
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Spin up mgbuild container"
run: |
./release/package/mgbuild.sh \
--toolchain ${{ inputs.toolchain }} \
--os ${{ inputs.os }} \
--arch ${{ inputs.arch }} \
run
- name: "Build Memgraph binaries"
run: |
./release/package/mgbuild.sh \
--toolchain ${{ inputs.toolchain }} \
--os ${{ inputs.os }} \
--arch ${{ inputs.arch }} \
--build-type ${{ inputs.build_type }} \
build-memgraph ${{ inputs.additional_build_args }}
- name: "Make package"
run: |
./release/package/mgbuild.sh \
--toolchain ${{ inputs.toolchain }} \
--os ${{ inputs.os }} \
--arch ${{ inputs.arch }} \
--build-type ${{ inputs.build_type }} \
package-memgraph
- name: "Copy package"
run: |
./release/package/mgbuild.sh \
--toolchain ${{ inputs.toolchain }} \
--os ${{ inputs.os }} \
--arch ${{ inputs.arch }} \
copy --package
- name: "Create docker package"
run: |
./release/package/mgbuild.sh \
--toolchain ${{ inputs.toolchain }} \
--os ${{ inputs.os }} \
--arch ${{ inputs.arch }} \
docker
- name: "Upload docker package"
uses: actions/upload-artifact@v4
with:
name: ${{ env.ARTIFACT_NAME }}
path: "build/output/${{ inputs.os }}/memgraph*"
- name: "Stop mgbuild container"
if: always()
run: |
./release/package/mgbuild.sh \
--toolchain ${{ inputs.toolchain }} \
--os ${{ inputs.os }} \
--arch ${{ inputs.arch }} \
stop --remove
upload-to-s3:
if: ${{ inputs.push_to_s3 == 'true' }}
needs: [package]
runs-on: ubuntu-latest
steps:
- name: Download artifacts
uses: actions/download-artifact@v4
with:
name: ${{ env.ARTIFACT_NAME }}
path: "build/output/release"
- name: Upload to S3
uses: jakejarvis/s3-sync-action@v0.5.1
env:
AWS_S3_BUCKET: ${{ inputs.s3_bucket }}
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ inputs.s3_region }}
SOURCE_DIR: "build/output/release"
DEST_DIR: ${{ inputs.s3_dest_dir }}
# MAKE DOCKER WORK
# docker:
# if: ${{ inputs.target_os == 'docker' || inputs.target_os == 'all' }}
# runs-on: [self-hosted, DockerMgBuild, X64]
# timeout-minutes: 60
# steps:
# - name: "Set up repository"
# uses: actions/checkout@v4
# with:
# fetch-depth: 0 # Required because of release/get_version.py
# - name: "Build package"
# run: |
# cd release/package
# ./run.sh package debian-11 ${{ inputs.build_type }} --for-docker
# ./run.sh docker
# - name: "Upload package"
# uses: actions/upload-artifact@v4
# with:
# name: docker
# path: build/output/docker/memgraph*.tar.gz

View File

@ -13,17 +13,35 @@ working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
latest_image=""
tag_latest=""
if [[ $# -eq 2 && "$1" == "--latest" ]]; then
latest_image="memgraph:latest"
tag_latest="-t memgraph:latest"
shift
elif [[ $# -ne 1 || "$1" == "-h" || "$1" == "--help" ]]; then
print_help
exit 1
fi
toolchain_version="v5"
package_path=""
while [[ $# -gt 0 ]]; do
case "$1" in
--latest)
latest_image="memgraph:latest"
tag_latest="-t memgraph:latest"
shift
;;
--package-path)
package_path=$2
shift 2
;;
--toolchain)
toolchain_version=$2
shift 2
;;
-h|--help)
print_help
exit 0
;;
*)
echo -e "Error: Unknown option '$1'"
exit 1
;;
esac
done
package_path="$1"
if [[ ! -f "$package_path" ]]; then
if [[ "$package_path" == "" || ! -f "$package_path" ]]; then
echo "File '$package_path' does not exist!"
exit 1
fi

View File

@ -1,4 +1,4 @@
FROM debian:bullseye
FROM debian:11
# NOTE: If you change the base distro update release/package as well.
ARG BINARY_NAME

View File

@ -0,0 +1,32 @@
FROM debian:12
# NOTE: If you change the base distro update release/package as well.
ARG BINARY_NAME
ARG EXTENSION
ARG TARGETARCH
RUN apt-get update && apt-get install -y \
openssl libcurl4 libssl1.1 libseccomp2 python3 libpython3.9 python3-pip \
--no-install-recommends \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3
COPY "${BINARY_NAME}${TARGETARCH}.${EXTENSION}" /
# Install memgraph package
RUN dpkg -i "${BINARY_NAME}${TARGETARCH}.deb"
# Memgraph listens for Bolt Protocol on this port by default.
EXPOSE 7687
# Snapshots and logging volumes
VOLUME /var/log/memgraph
VOLUME /var/lib/memgraph
# Configuration volume
VOLUME /etc/memgraph
USER memgraph
WORKDIR /usr/lib/memgraph
ENTRYPOINT ["/usr/lib/memgraph/memgraph"]
CMD [""]

View File

@ -354,6 +354,33 @@ package_memgraph() {
docker exec -u mg "$build_container" bash -c "mkdir -p $container_output_dir && cd $container_output_dir && $ACTIVATE_TOOLCHAIN && $package_command"
}
package_docker() {
if [[ "$toolchain_version" == "v4" ]]; then
if [[ "$os" != "debian-11" || "$os" != "debian-11-arm" ]]; then
echo -e "Error: When passing '--toolchain v4' the 'docker' command accepts only '--os debian-11' and '--os debian-11-arm'"
exit 1
fi
else
if [[ "$os" != "debian-12" || "$os" != "debian-12-arm" ]]; then
echo -e "Error: When passing '--toolchain v5' the 'docker' command accepts only '--os debian-12' and '--os debian-12-arm'"
exit 1
fi
fi
local package_dir="$PROJECT_ROOT/build/output/$os"
# shellcheck disable=SC2012
local last_package_name=$(cd $package_dir && ls -t memgraph* | head -1)
local docker_build_folder="$PROJECT_ROOT/release/docker"
cd "$docker_build_folder"
./package_docker --latest "$package_dir/$last_package_name" $toolchain_version
# shellcheck disable=SC2012
local docker_image_name=$(cd "$docker_build_folder" && ls -t memgraph* | head -1)
local docker_host_folder="$PROJECT_ROOT/build/output/docker-${toolchain_version}"
local docker_host_image_path="$docker_host_folder/$docker_image_name"
mkdir -p "$docker_host_folder"
cp "$docker_build_folder/$docker_image_name" "$docker_host_image_path"
echo "Docker images saved to $docker_host_image_path."
}
copy_memgraph() {
local build_container="mgbuild_${toolchain_version}_${os}"
case "$1" in
@ -660,6 +687,9 @@ case $command in
copy)
copy_memgraph $@
;;
docker)
package_docker
;;
*)
echo "Error: Unknown command '$command'"
exit 1