diff --git a/.github/workflows/package_memgraph.yaml b/.github/workflows/package_memgraph.yaml index 73d05b509..585e35b09 100644 --- a/.github/workflows/package_memgraph.yaml +++ b/.github/workflows/package_memgraph.yaml @@ -26,10 +26,9 @@ on: - debian-11 - debian-11-arm - debian-11-platform - - debian-11-docker - - debian-11-arm-docker - debian-12 - debian-12-arm + - docker - fedora-36 - fedora-38 - fedora-39 @@ -121,20 +120,6 @@ jobs: artifact_name: "debian-11-platform" secrets: inherit - Debian11-docker: - if: ${{ github.event.inputs.target_os == 'debian-11-docker' || github.event.inputs.target_os == 'all' }} - uses: ./.github/workflows/reusable_package.yaml - with: - os: "debian-11" - toolchain: ${{ github.event.inputs.toolchain }} - arch: "amd" - runner_arch_label: "X64" - build_type: ${{ github.event.inputs.build_type }} - timeout_minutes: 60 - additional_build_args: "--for-docker" - artifact_name: "debian-11-docker" - secrets: inherit - Debian11-arm: if: ${{ github.event.inputs.target_os == 'debian-11-arm' || github.event.inputs.target_os == 'all' }} uses: ./.github/workflows/reusable_package.yaml @@ -147,20 +132,6 @@ jobs: timeout_minutes: 120 secrets: inherit - # Debian11-arm-docker: - # if: ${{ github.event.inputs.target_os == 'debian-11-arm-docker' || github.event.inputs.target_os == 'all' }} - # uses: ./.github/workflows/reusable_package.yaml - # with: - # os: "debian-11-arm" - # toolchain: ${{ github.event.inputs.toolchain }} - # arch: "arm" - # runner_arch_label: "ARM64" - # build_type: ${{ github.event.inputs.build_type }} - # timeout_minutes: 120 - # additional_build_args: "--for-docker" - # artifact_name: "debian-11-arm-docker" - # secrets: inherit - Debian12: if: ${{ (github.event.inputs.target_os == 'debian-12' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v5' }} uses: ./.github/workflows/reusable_package.yaml @@ -185,6 +156,34 @@ jobs: timeout_minutes: 120 secrets: inherit + Docker-v4: + if: ${{ (github.event.inputs.target_os == 'docker' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v4' }} + uses: ./.github/workflows/reusable_package.yaml + with: + os: "debian-11" + toolchain: ${{ github.event.inputs.toolchain }} + arch: "amd" + runner_arch_label: "X64" + build_type: ${{ github.event.inputs.build_type }} + timeout_minutes: 60 + additional_build_args: "--for-docker" + artifact_name: "debian-11-docker" + secrets: inherit + + Docker-v5: + if: ${{ (github.event.inputs.target_os == 'docker' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v5' }} + uses: ./.github/workflows/reusable_package.yaml + with: + os: "debian-12" + toolchain: ${{ github.event.inputs.toolchain }} + arch: "amd" + runner_arch_label: "X64" + build_type: ${{ github.event.inputs.build_type }} + timeout_minutes: 60 + additional_build_args: "--for-docker" + artifact_name: "debian-11-docker" + secrets: inherit + Fedora36: if: ${{ (github.event.inputs.target_os == 'fedora-36' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v4' }} uses: ./.github/workflows/reusable_package.yaml diff --git a/.github/workflows/reusable_docker_package.yaml b/.github/workflows/reusable_docker_package.yaml new file mode 100644 index 000000000..ac038f646 --- /dev/null +++ b/.github/workflows/reusable_docker_package.yaml @@ -0,0 +1,162 @@ +name: Reusable docker package make and upload + +on: + workflow_call: + inputs: + os: + type: string + description: "Target os. Default value is debian-11." + default: 'debian-11' + arch: + type: string + description: "Target architecture. Default value is amd." + default: 'amd' + runner_arch_label: + type: string + description: "Runner architecture label. Amd is built on X64, arm on ARM64. Default value is X64." + default: 'X64' + toolchain: + type: string + description: "Toolchain version. Default value is v5." + default: 'v5' + build_type: + type: string + description: "Memgraph Build type. Default value is Release." + default: 'Release' + artifact_name: + type: string + description: "Unique artifact name, pass it in to override the default. Default value is os name." + default: '' + additional_build_args: + type: string + description: "Additional build flags (--for-platform, --for-docker ...). Default value is empty." + default: '' + timeout_minutes: + type: number + description: "Timeout in minutes for the job. Default value is 60." + default: 60 + push_to_s3: + type: boolean + description: "Should the final package be pushed to an S3 bucket. Default value is false" + default: false + s3_bucket: + type: string + description: "Name of the target S3 bucket." + s3_region: + type: string + description: "Region for the target S3 bucket." + s3_dest_dir: + type: string + description: "Target dir path in S3 bucket." + +env: + ARTIFACT_NAME: ${{ inputs.artifact_name || inputs.os }} + +jobs: + package: + runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"] + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: "Set up repository" + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Required because of release/get_version.py + + - name: "Spin up mgbuild container" + run: | + ./release/package/mgbuild.sh \ + --toolchain ${{ inputs.toolchain }} \ + --os ${{ inputs.os }} \ + --arch ${{ inputs.arch }} \ + run + + - name: "Build Memgraph binaries" + run: | + ./release/package/mgbuild.sh \ + --toolchain ${{ inputs.toolchain }} \ + --os ${{ inputs.os }} \ + --arch ${{ inputs.arch }} \ + --build-type ${{ inputs.build_type }} \ + build-memgraph ${{ inputs.additional_build_args }} + + - name: "Make package" + run: | + ./release/package/mgbuild.sh \ + --toolchain ${{ inputs.toolchain }} \ + --os ${{ inputs.os }} \ + --arch ${{ inputs.arch }} \ + --build-type ${{ inputs.build_type }} \ + package-memgraph + + - name: "Copy package" + run: | + ./release/package/mgbuild.sh \ + --toolchain ${{ inputs.toolchain }} \ + --os ${{ inputs.os }} \ + --arch ${{ inputs.arch }} \ + copy --package + + - name: "Create docker package" + run: | + ./release/package/mgbuild.sh \ + --toolchain ${{ inputs.toolchain }} \ + --os ${{ inputs.os }} \ + --arch ${{ inputs.arch }} \ + docker + + - name: "Upload docker package" + uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }} + path: "build/output/${{ inputs.os }}/memgraph*" + + - name: "Stop mgbuild container" + if: always() + run: | + ./release/package/mgbuild.sh \ + --toolchain ${{ inputs.toolchain }} \ + --os ${{ inputs.os }} \ + --arch ${{ inputs.arch }} \ + stop --remove + + upload-to-s3: + if: ${{ inputs.push_to_s3 == 'true' }} + needs: [package] + runs-on: ubuntu-latest + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }} + path: "build/output/release" + + - name: Upload to S3 + uses: jakejarvis/s3-sync-action@v0.5.1 + env: + AWS_S3_BUCKET: ${{ inputs.s3_bucket }} + AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ inputs.s3_region }} + SOURCE_DIR: "build/output/release" + DEST_DIR: ${{ inputs.s3_dest_dir }} + + # MAKE DOCKER WORK + # docker: + # if: ${{ inputs.target_os == 'docker' || inputs.target_os == 'all' }} + # runs-on: [self-hosted, DockerMgBuild, X64] + # timeout-minutes: 60 + # steps: + # - name: "Set up repository" + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 # Required because of release/get_version.py + # - name: "Build package" + # run: | + # cd release/package + # ./run.sh package debian-11 ${{ inputs.build_type }} --for-docker + # ./run.sh docker + # - name: "Upload package" + # uses: actions/upload-artifact@v4 + # with: + # name: docker + # path: build/output/docker/memgraph*.tar.gz diff --git a/release/docker/package_docker b/release/docker/package_docker index b12c4ca61..1cdadcf0e 100755 --- a/release/docker/package_docker +++ b/release/docker/package_docker @@ -13,17 +13,35 @@ working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" latest_image="" tag_latest="" -if [[ $# -eq 2 && "$1" == "--latest" ]]; then - latest_image="memgraph:latest" - tag_latest="-t memgraph:latest" - shift -elif [[ $# -ne 1 || "$1" == "-h" || "$1" == "--help" ]]; then - print_help - exit 1 -fi +toolchain_version="v5" +package_path="" +while [[ $# -gt 0 ]]; do + case "$1" in + --latest) + latest_image="memgraph:latest" + tag_latest="-t memgraph:latest" + shift + ;; + --package-path) + package_path=$2 + shift 2 + ;; + --toolchain) + toolchain_version=$2 + shift 2 + ;; + -h|--help) + print_help + exit 0 + ;; + *) + echo -e "Error: Unknown option '$1'" + exit 1 + ;; + esac +done -package_path="$1" -if [[ ! -f "$package_path" ]]; then +if [[ "$package_path" == "" || ! -f "$package_path" ]]; then echo "File '$package_path' does not exist!" exit 1 fi diff --git a/release/docker/memgraph_deb.dockerfile b/release/docker/v4_deb.dockerfile similarity index 97% rename from release/docker/memgraph_deb.dockerfile rename to release/docker/v4_deb.dockerfile index 06417c9a6..87e25e94f 100644 --- a/release/docker/memgraph_deb.dockerfile +++ b/release/docker/v4_deb.dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye +FROM debian:11 # NOTE: If you change the base distro update release/package as well. ARG BINARY_NAME diff --git a/release/docker/v5_deb.dockerfile b/release/docker/v5_deb.dockerfile new file mode 100644 index 000000000..d320622d1 --- /dev/null +++ b/release/docker/v5_deb.dockerfile @@ -0,0 +1,32 @@ +FROM debian:12 +# NOTE: If you change the base distro update release/package as well. + +ARG BINARY_NAME +ARG EXTENSION +ARG TARGETARCH + +RUN apt-get update && apt-get install -y \ + openssl libcurl4 libssl1.1 libseccomp2 python3 libpython3.9 python3-pip \ + --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3 + +COPY "${BINARY_NAME}${TARGETARCH}.${EXTENSION}" / + +# Install memgraph package +RUN dpkg -i "${BINARY_NAME}${TARGETARCH}.deb" + +# Memgraph listens for Bolt Protocol on this port by default. +EXPOSE 7687 +# Snapshots and logging volumes +VOLUME /var/log/memgraph +VOLUME /var/lib/memgraph +# Configuration volume +VOLUME /etc/memgraph + +USER memgraph +WORKDIR /usr/lib/memgraph + +ENTRYPOINT ["/usr/lib/memgraph/memgraph"] +CMD [""] diff --git a/release/package/mgbuild.sh b/release/package/mgbuild.sh index da550b4b8..7827e3f55 100755 --- a/release/package/mgbuild.sh +++ b/release/package/mgbuild.sh @@ -354,6 +354,33 @@ package_memgraph() { docker exec -u mg "$build_container" bash -c "mkdir -p $container_output_dir && cd $container_output_dir && $ACTIVATE_TOOLCHAIN && $package_command" } +package_docker() { + if [[ "$toolchain_version" == "v4" ]]; then + if [[ "$os" != "debian-11" || "$os" != "debian-11-arm" ]]; then + echo -e "Error: When passing '--toolchain v4' the 'docker' command accepts only '--os debian-11' and '--os debian-11-arm'" + exit 1 + fi + else + if [[ "$os" != "debian-12" || "$os" != "debian-12-arm" ]]; then + echo -e "Error: When passing '--toolchain v5' the 'docker' command accepts only '--os debian-12' and '--os debian-12-arm'" + exit 1 + fi + fi + local package_dir="$PROJECT_ROOT/build/output/$os" + # shellcheck disable=SC2012 + local last_package_name=$(cd $package_dir && ls -t memgraph* | head -1) + local docker_build_folder="$PROJECT_ROOT/release/docker" + cd "$docker_build_folder" + ./package_docker --latest "$package_dir/$last_package_name" $toolchain_version + # shellcheck disable=SC2012 + local docker_image_name=$(cd "$docker_build_folder" && ls -t memgraph* | head -1) + local docker_host_folder="$PROJECT_ROOT/build/output/docker-${toolchain_version}" + local docker_host_image_path="$docker_host_folder/$docker_image_name" + mkdir -p "$docker_host_folder" + cp "$docker_build_folder/$docker_image_name" "$docker_host_image_path" + echo "Docker images saved to $docker_host_image_path." +} + copy_memgraph() { local build_container="mgbuild_${toolchain_version}_${os}" case "$1" in @@ -660,6 +687,9 @@ case $command in copy) copy_memgraph $@ ;; + docker) + package_docker + ;; *) echo "Error: Unknown command '$command'" exit 1