Add maintainer scripts for DEB package

Summary:
Add postinst script for DEB package

The script creates a 'memgraph' group and sets permission on installed
'/var/*/memgraph' directories. Only the group is created, while
'memgraph' user is not. It seems more sane only to require group
membership for using memgraph.

Add conffiles for DEB package

This allows for `dpkg` to detect changes in configuration files and
present them to the user. Therefore, we don't need to care whether the
configuration merges are handled correctly nor if we accidentally
overwrite them.

Add postrm script for DEB packaging

The script is only used so that `dpkg --purge` removes '/var/*/memgraph'
directories, even if they contain something.

Add email, longer description and license file to DEB packaging, as well
as a systemd service.

Provide a logrotate configuration and support it in memgraph.

Use DEB package for Docker installation

This way, the whole installation process and testing should go through
DEB.

Generate release archives in Apollo with standard names

Reviewers: buda, mferencevic

Reviewed By: mferencevic

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D989

CMakeLists.txt

@@ -286,12 +286,21 @@ endif()
 # we cannot use the recommended `install(TARGETS ...)`.
 install(PROGRAMS ${CMAKE_BINARY_DIR}/${MEMGRAPH_BUILD_NAME}
         DESTINATION bin RENAME memgraph)
-# Install the config file.
+# Install the config file (must use absolute path).
 install(FILES ${CMAKE_SOURCE_DIR}/config/community.conf
         DESTINATION /etc/memgraph RENAME memgraph.conf)
-# Create empty directories for default location of snapshots and logs.
+# Install logrotate configuration (must use absolute path).
+install(FILES ${CMAKE_SOURCE_DIR}/release/logrotate.conf
+        DESTINATION /etc/logrotate.d RENAME memgraph)
+# Create empty directories for default location of lib and log.
 install(CODE "file(MAKE_DIRECTORY \$ENV{DESTDIR}/var/log/memgraph
-                   \$ENV{DESTDIR}/var/lib/memgraph/snapshots)")
+                   \$ENV{DESTDIR}/var/lib/memgraph)")
+# Install the license file.
+install(FILES ${CMAKE_SOURCE_DIR}/release/LICENSE.md
+        DESTINATION share/doc/memgraph RENAME copyright)
+# Install systemd service (must use absolute path).
+install(FILES ${CMAKE_SOURCE_DIR}/release/memgraph.service
+        DESTINATION /lib/systemd/system)
 
 # ---- Setup CPack --------
 # General setup
@@ -304,14 +313,23 @@ set(CPACK_PACKAGE_VERSION_MINOR ${memgraph_VERSION_MINOR})
 set(CPACK_PACKAGE_VERSION_PATCH ${memgraph_VERSION_PATCH})
 set(CPACK_PACKAGE_VERSION_TWEAK ${memgraph_VERSION_TWEAK})
 set(CPACK_PACKAGE_FILE_NAME ${CPACK_PACKAGE_NAME}-${memgraph_VERSION}-${COMMIT_HASH}${CPACK_SYSTEM_NAME})
-# TODO: Longer description, readme and license files.
 # DEB specific
-set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Memgraph Ltd.")
-set(CPACK_DEBIAN_PACKAGE_SECTION database)
+# Instead of using "name <email>" format, we use "email (name)" to prevent
+# errors due to full stop, '.' at the end of "Ltd". (See: RFC 822)
+set(CPACK_DEBIAN_PACKAGE_MAINTAINER "tech@memgraph.com (Memgraph Ltd.)")
+set(CPACK_DEBIAN_PACKAGE_SECTION non-free/database)
 set(CPACK_DEBIAN_PACKAGE_HOMEPAGE https://memgraph.com)
-# We may need conffiles, postinst, postrm or prerm scripts.
-# set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA ...)
+set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA
+    "${CMAKE_CURRENT_SOURCE_DIR}/cmake/debian/conffiles;"
+    "${CMAKE_CURRENT_SOURCE_DIR}/cmake/debian/copyright;"
+    "${CMAKE_CURRENT_SOURCE_DIR}/cmake/debian/postrm;"
+    "${CMAKE_CURRENT_SOURCE_DIR}/cmake/debian/postinst;")
 set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
+# Description formatting is important, summary must be followed with a newline and 1 space.
+set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
+ Contains Memgraph, the graph database. It aims to deliver developers the
+ speed, simplicity and scale required to build the next generation of
+ applications driver by real-time connected data.")
 # All variables must be set before including.
 include(CPack)
 # ---- End Setup CPack ----

cmake/debian/conffiles

@@ -0,0 +1,2 @@
+/etc/memgraph/memgraph.conf
+/etc/logrotate.d/memgraph

cmake/debian/copyright

@@ -0,0 +1 @@
+../../release/LICENSE.md

cmake/debian/postinst

@@ -0,0 +1,63 @@
+#!/bin/sh
+# postinst script for memgraph
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+#        * <postinst> `configure' <most-recently-configured-version>
+#        * <old-postinst> `abort-upgrade' <new version>
+#        * <conflictor's-postinst> `abort-remove' `in-favour' <package>
+#          <new-version>
+#        * <postinst> `abort-remove'
+#        * <deconfigured's-postinst> `abort-deconfigure' `in-favour'
+#          <failed-install-package> <version> `removing'
+#          <conflicting-package> <version>
+# for details, see https://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+case "$1" in
+    configure)
+        # Add the 'memgraph' user and group and set permissions on
+        # 'var/*/memgraph' directories.
+        adduser --quiet --system --group --no-create-home --shell /bin/bash memgraph || exit 1
+        echo "Don't forget to switch to the 'memgraph' user to use Memgraph" || exit 1
+        chown memgraph:memgraph /var/lib/memgraph || exit 1
+        chmod 750 /var/lib/memgraph || exit 1
+        chown memgraph:adm /var/log/memgraph || exit 1
+        chmod 750 /var/log/memgraph || exit 1
+    ;;
+
+    abort-upgrade|abort-remove|abort-deconfigure)
+    ;;
+
+    *)
+        echo "postinst called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+# Always setup the systemd memgraph.service. The following was autogenerated
+# by dh_systemd_enable and dh_systemd_start, so it should behave as expected.
+
+# This will only remove masks created by d-s-h on package removal.
+deb-systemd-helper unmask memgraph.service >/dev/null || true
+
+# was-enabled defaults to true, so new installations run enable.
+if deb-systemd-helper --quiet was-enabled memgraph.service; then
+    # Enables the unit on first installation, creates new
+    # symlinks on upgrades if the unit file has changed.
+    deb-systemd-helper enable memgraph.service >/dev/null || true
+else
+    # Update the statefile to add new symlinks (if any), which need to be
+    # cleaned up on purge. Also remove old symlinks.
+    deb-systemd-helper update-state memgraph.service >/dev/null || true
+fi
+
+if [ -d /run/systemd/system ]; then
+    systemctl --system daemon-reload >/dev/null || true
+    deb-systemd-invoke start memgraph.service >/dev/null || true
+fi
+
+exit 0

cmake/debian/postrm

@@ -0,0 +1,64 @@
+#!/bin/sh
+# postrm script for memgraph
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+#        * <postrm> `remove'
+#        * <postrm> `purge'
+#        * <old-postrm> `upgrade' <new-version>
+#        * <new-postrm> `failed-upgrade' <old-version>
+#        * <new-postrm> `abort-install'
+#        * <new-postrm> `abort-install' <old-version>
+#        * <new-postrm> `abort-upgrade' <old-version>
+#        * <disappearer's-postrm> `disappear' <overwriter>
+#          <overwriter-version>
+# for details, see https://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+var_files="/var/lib/memgraph /var/log/memgraph"
+
+case "$1" in
+    purge)
+        # Remove 'var/*/memgraph' directories, even if they contain something.
+        for var_file in $var_files; do
+            rm -rf $var_file
+        done
+        # Don't remove the 'memgraph' user, since we cannot be sure whether it
+        # existed before.
+    ;;
+
+    remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
+        # Default behaviour does what we expect, removes untouched installed
+        # files but keeps configuration.
+    ;;
+
+    *)
+        echo "postrm called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+# Remove and purge systemd memgraph.service. The following was autogenerated
+# by dh_systemd_enable and dh_systemd_start.
+
+if [ -d /run/systemd/system ]; then
+    systemctl --system daemon-reload >/dev/null || true
+fi
+
+if [ "$1" = "remove" ]; then
+    if [ -x "/usr/bin/deb-systemd-helper" ]; then
+        deb-systemd-helper mask memgraph.service >/dev/null
+    fi
+fi
+
+if [ "$1" = "purge" ]; then
+    if [ -x "/usr/bin/deb-systemd-helper" ]; then
+        deb-systemd-helper purge memgraph.service >/dev/null
+        deb-systemd-helper unmask memgraph.service >/dev/null
+    fi
+fi
+
+exit 0

cmake/debian/prerm

@@ -0,0 +1,32 @@
+#!/bin/sh
+# prerm script for memgraph
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+#        * <prerm> `remove'
+#        * <old-prerm> `upgrade' <new-version>
+#        * <new-prerm> `failed-upgrade' <old-version>
+#        * <conflictor's-prerm> `remove' `in-favour' <package> <new-version>
+#        * <deconfigured's-prerm> `deconfigure' `in-favour'
+#          <package-being-installed> <version> `removing'
+#          <conflicting-package> <version>
+# for details, see https://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+case "$1" in
+    remove|upgrade|deconfigure|failed-upgrade)
+        if [ -d /run/systemd/system ]; then
+            deb-systemd-invoke stop memgraph.service >/dev/null
+        fi
+    ;;
+
+    *)
+        echo "prerm called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+exit 0

libs/CMakeLists.txt

@@ -156,7 +156,8 @@ import_external_library(glog STATIC
   CMAKE_ARGS -Dgflags_DIR=${CMAKE_CURRENT_SOURCE_DIR}/gflags/lib/cmake/gflags
              -DGLOG_NO_FILENAMES=${GLOG_DISABLE_OPTIONS}
              -DGLOG_NO_STACKTRACE=${GLOG_DISABLE_OPTIONS}
-             -DGLOG_NO_BUFFER_SETTINGS=${GLOG_DISABLE_OPTIONS})
+             -DGLOG_NO_BUFFER_SETTINGS=${GLOG_DISABLE_OPTIONS}
+             -DGLOG_NO_TIME_PID_FILENAME=${GLOG_DISABLE_OPTIONS})
 
 # Setup cppitertools
 import_header_library(cppitertools ${CMAKE_CURRENT_SOURCE_DIR})

libs/setup.sh

@@ -68,7 +68,7 @@ cd ..
 # google logging
 # git clone https://github.com/memgraph/glog.git
 git clone git://deps.memgraph.io/glog.git
-glog_tag="ae0ac046e8320245dc42af6671274a7e35ab09e0" # custom version (v0.3.5+)
+glog_tag="042a21657e79784226babab8b942f7bd0949635f" # custom version (v0.3.5+)
 cd glog
 git checkout ${glog_tag}
 cd ..

release/LICENSE.md

@@ -0,0 +1,62 @@
+# User License Agreement
+
+1. Description
+
+THIS LICENSE AGREEMENT GOVERNS LICENSEE’S USE OF THE MEMGRAPH COMMUNITY
+RELEASE AND DOCUMENTATION.
+
+2. License Grant
+
+The Software and Documentation are provided to Licensee at no charge and are
+licensed, not sold to Licensee. No ownership of any part of the Software and
+Documentation is hereby transferred to Licensee. Subject to (i) the terms and
+conditions of this License Agreement, (ii) any additional license restrictions
+and parameters contained on Licensor’s quotation, website, or order form
+(“Order Form”), Licensor hereby grants Licensee a personal, non-assignable,
+non-transferable and non-exclusive license to install, access and use the
+Software (in object code form only) and Documentation for Licensee’s internal
+business purposes only. All rights relating to the Software and Documentation
+that are not expressly licensed in this License Agreement, whether now
+existing or which may hereafter come into existence are reserved for Licensor.
+Licensee shall not remove, obscure, or alter any proprietary rights notices
+(including without limitation copyright and trademark notices), which may be
+affixed to or contained within the Software or Documentation.
+
+3. Restrictions
+
+Licensee will not, directly or indirectly, (a) copy the Software or
+Documentation in any manner or for any purpose; (b) install, access or use any
+component of the Software or Documentation for any purpose not expressly
+granted in Section 2 above; (c) resell, distribute, publicly display or
+publicly perform the Software or Documentation or any component thereof, by
+transfer, lease, loan or any other means, or make it available for use by
+others in any time-sharing, service bureau or similar arrangement; (d)
+disassemble, decrypt, extract, reverse engineer or reverse compile the
+Software, or otherwise attempt to discover the source code, confidential
+algorithms or techniques incorporated in the Software; (e) export the Software
+or Documentation in violation of any applicable laws or regulations; (f)
+modify, translate, adapt, or create derivative works from the Software or
+Documentation; (g) circumvent, disable or otherwise interfere with
+security-related features of the Software or Documentation; (h)
+reverse-engineer, disassemble, attempt to derive the source code; (i) use the
+Software or Documentation for any illegal purpose, in any manner that is
+inconsistent with the terms of this License Agreement, or to engage in illegal
+activity; (j) remove or alter any trademark, logo, copyright or other
+proprietary notices, legends, symbols or labels on, or embedded in, the
+Software or Documentation; or (k) provide access to the Software or
+Documentation to third parties.
+
+4. Warranty Disclaimer
+
+THE MEMGRAPH COMMUNITY RELEASE AND DOCUMENTATION ARE PROVIDED “AS IS” FOR
+DEVELOPMENT, TESTING AND EVALUATION PURPOSES ONLY. IT IS NOT LICENSED FOR
+PRODUCTION USE AND LICENSOR MAKES NO AND DISCLAIMS ALL WARRANTIES, EXPRESS OR
+IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NONINFRINGEMENT OF
+THIRD PARTIES’ INTELLECTUAL PROPERTY RIGHTS OR OTHER PROPRIETARY RIGHTS.
+NEITHER THIS LICENSE AGREEMENT NOR ANY DOCUMENTATION FURNISHED UNDER IT IS
+INTENDED TO EXPRESS OR IMPLY ANY WARRANTY THAT THE OPERATION OF THE SOFTWARE
+WILL BE UNINTERRUPTED, TIMELY, OR ERROR-FREE.
+
+BY DOWNLOADING AND/OR ACCESSING THIS EARLY ACCESS SOFTWARE, YOU AGREE TO SUCH
+TERMS.

release/community.dockerfile

@@ -1,17 +1,11 @@
 FROM debian:stretch
-# FROM debian:stretch     # 104MB
-# FROM ubuntu 16.04       # 130MB
-# FROM phusion/baseimage  # 220MB
 
-ARG build_name
+ARG deb_release
 
-COPY ${build_name} /
+COPY ${deb_release} /
 
-# Setup memgraph user and group
-RUN groupadd -r memgraph
-RUN useradd -lrm -g memgraph memgraph
-RUN chown -R memgraph:memgraph /var/log/memgraph
-RUN chown -R memgraph:memgraph /var/lib/memgraph
+# Install memgraph package
+RUN dpkg -i ${deb_release}
 
 # Memgraph listens for Bolt Protocol on this port by default.
 EXPOSE 7687

release/logrotate.conf

@@ -0,0 +1,13 @@
+# logrotate configuration for Memgraph logs
+# see "man logrotate" for details
+
+/var/log/memgraph/memgraph.log {
+    # rotate log files weekly
+    weekly
+    # keep 5 weeks worth of backlog
+    rotate 5
+    # send SIGUSR1 to notify memgraph to recreate logfile
+    postrotate
+        /usr/bin/killall -s SIGUSR1 memgraph
+    endscript
+}

release/memgraph.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Memgraph: High performance, in-memory, transactional graph database
+
+[Service]
+User=memgraph
+Group=memgraph
+ExecStart=/usr/bin/memgraph
+# Uncomment this if Memgraph needs more time to write the snapshot on exit.
+#TimeoutStopSec=5min
+
+[Install]
+WantedBy=multi-user.target

release/package_docker

@@ -3,7 +3,7 @@
 # Build and Package (docker image) Memgraph
 
 function print_help () {
-    echo "Usage: $0 [--latest] BINARY_PACKAGE.tar.gz"
+    echo "Usage: $0 [--latest] MEMGPRAH_PACKAGE.deb"
     echo "Optional arguments:"
     echo -e "  -h|--help           Print help."
     echo -e "  --latest            Tag image as latest version."
@@ -28,20 +28,17 @@ if [[ ! -f "$1" ]]; then
   exit 1
 fi
 
-tar_release=`realpath "$1"`
-package_name=`echo $(basename $1) | sed 's/.tar.gz//'`
+# Copy the .deb to working directory.
+cp "$1" "${working_dir}/"
 
 cd ${working_dir}
-
-# Unpack the release package.
-echo "Unpacking '$1' to '${working_dir}/${package_name}'"
-tar xf ${tar_release}
-version=`echo ${package_name} | sed 's/.*-\(.*\)-.*/\1/'`
+# Extract version from deb name
+deb_name=`echo $(basename $1) | sed 's/.deb//'`
+version=`echo ${deb_name} | sed 's/.*[-_]\(.*\)-.*/\1/'`
 image_name="memgraph:${version}"
+package_name="memgraph-${version}-docker.tar.gz"
 # Build docker image.
-docker build -t ${image_name} ${tag_latest} -f ${working_dir}/community.dockerfile --build-arg build_name=${package_name} .
-docker save ${image_name} ${latest_image} > ${package_name}-docker.tar.gz
-# Remove unpacked package.
-echo "Removing '${working_dir}/${package_name}'"
-rm -rf ${package_name}
-echo "Built Docker imate at '${working_dir}/${package_name}-docker.tar.gz'"
+docker build -t ${image_name} ${tag_latest} -f ${working_dir}/community.dockerfile --build-arg deb_release=${deb_name}.deb .
+docker save ${image_name} ${latest_image} > ${package_name}
+rm -rf "${deb_name}.deb"
+echo "Built Docker image at '${working_dir}/${package_name}"

src/memgraph_bolt.cpp

@@ -34,8 +34,9 @@ DEFINE_VALIDATED_int32(num_workers,
                        std::max(std::thread::hardware_concurrency(), 1U),
                        "Number of workers", FLAG_IN_RANGE(1, INT32_MAX));
 DEFINE_string(log_file, "", "Path to where the log should be stored.");
-DEFINE_string(log_link_basename, "",
-              "Basename used for symlink creation to the last log file.");
+DEFINE_HIDDEN_string(
+    log_link_basename, "",
+    "Basename used for symlink creation to the last log file.");
 DEFINE_uint64(memory_warning_threshold, 1024,
               "Memory warning treshold, in MB. If Memgraph detects there is "
               "less available RAM available it will log a warning. Set to 0 to "
@@ -104,6 +105,12 @@ int main(int argc, char **argv) {
                                        block_shutdown_signals))
       << "Unable to register SIGINT handler!";
 
+  // Setup SIGUSR1 to be used for reopening log files, when e.g. logrotate
+  // rotates our logs.
+  CHECK(SignalHandler::RegisterHandler(Signal::User1, []() {
+    google::CloseLogDestination(google::INFO);
+  })) << "Unable to register SIGUSR1 handler!";
+
   // Start memory warning logger.
   Scheduler mem_log_scheduler;
   if (FLAGS_memory_warning_threshold > 0) {

src/utils/signals/handler.hpp

@@ -17,6 +17,7 @@ enum class Signal : int {
   Quit = SIGQUIT,
   Abort = SIGABRT,
   BusError = SIGBUS,
+  User1 = SIGUSR1,
 };
 
 class SignalHandler {

tools/apollo/generate

@@ -89,14 +89,18 @@ if mode == "release":
     build_output_dir = os.path.join(BUILD_DIR, "output")
     deb_name = run_cmd(["find", ".", "-maxdepth", "1", "-type", "f",
             "-name", "memgraph*.deb"], build_output_dir).split("\n")[0][2:]
+    arch = run_cmd(["dpkg", "--print-architecture"], build_output_dir).split("\n")[0]
+    version = binary_name.split("-")[1]
+    # Generate Debian package file name as expected by Debian Policy.
+    standard_deb_name = "memgraph_{}-1_{}.deb".format(version, arch)
     tarball_name = run_cmd(["find", ".", "-maxdepth", "1", "-type", "f",
             "-name", "memgraph*.tar.gz"], build_output_dir).split("\n")[0][2:]
     shutil.copyfile(os.path.join(build_output_dir, deb_name),
-            os.path.join(OUTPUT_DIR, "release.deb"))
+            os.path.join(OUTPUT_DIR, standard_deb_name))
     shutil.copyfile(os.path.join(build_output_dir, tarball_name),
-            os.path.join(OUTPUT_DIR, "release.tar.gz"))
-    ARCHIVES.append(generate_archive("Release (deb package)", "release_deb", "release.deb"))
-    ARCHIVES.append(generate_archive("Release (tarball)", "release_tar", "release.tar.gz"))
+            os.path.join(OUTPUT_DIR, tarball_name))
+    ARCHIVES.append(generate_archive("Release (deb package)", standard_deb_name, standard_deb_name))
+    ARCHIVES.append(generate_archive("Release (tarball)", tarball_name, tarball_name))
 
 # store user documentation to archive
 if mode == "release":