From 5a2086b18ffbf815460790bdb4b40bb9e2b01615 Mon Sep 17 00:00:00 2001 From: Matej Ferencevic Date: Thu, 24 Aug 2017 17:53:16 +0200 Subject: [PATCH] Libraries are now downloaded from our local cache server. Summary: Improved libs cleanup script. The root init script now downloads everything. Removed obsolete init scripts. Reviewers: buda, mislav.bradac Reviewed By: buda Subscribers: pullbot Differential Revision: https://phabricator.memgraph.io/D710 --- init | 56 +++++++++++++++++++++---------- libs/cleanup.sh | 8 +++-- libs/setup.sh | 33 ++++++++++++------ tests/macro_benchmark/init | 8 ----- tests/qa/init | 63 ----------------------------------- tests/stress/init | 11 ------ tests/stress/requirements.txt | 5 --- 7 files changed, 67 insertions(+), 117 deletions(-) delete mode 100755 tests/macro_benchmark/init delete mode 100755 tests/qa/init delete mode 100755 tests/stress/init diff --git a/init b/init index fd33f5ba5..3d99e356e 100755 --- a/init +++ b/init @@ -1,4 +1,22 @@ -#!/bin/bash +#!/bin/bash -e + +required_pkgs=(git arcanist # source code control + cmake clang-3.8 llvm-3.8 pkg-config # build system + curl wget # for downloading libs + uuid-dev default-jre-headless # required by antlr + libreadline-dev # for memgraph console + python3 python-virtualenv python3-pip # for qa, macro_benchmark and stress tests + ) + +optional_pkgs=(clang-format # source code formatting + doxygen graphviz # source documentation generators + php-cli # for user technical documentation generators + ) + +use_sudo=0 + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$DIR" function print_help () { echo "Usage: $0 [OPTION]" @@ -8,16 +26,24 @@ function print_help () { echo -e " -h\tdisplay this help and exit" } -required_pkgs=(git cmake clang-3.8 llvm-3.8 doxygen - wget # for downloading libs, e.g. antlr - uuid-dev default-jre-headless # required by antlr - ) +function setup_virtualenv () { + pushd $1 > /dev/null + echo "Setting up virtualenv for: $1" -optional_pkgs=(clang-format-3.8 # source code formatting - libreadline-dev # for Memgraph CLI - ) + # remove old virtualenv + if [ -d ve3 ]; then + rm -rf ve3 + fi -use_sudo=0 + # create new virtualenv + virtualenv -p python3 ve3 || exit 1 + source ve3/bin/activate + pip install -i http://deps.memgraph.io:3141/root/pypi \ + --trusted-host deps.memgraph.io -r requirements.txt || exit 1 + deactivate + + popd > /dev/null +} if [[ $# -gt 1 ]]; then print_help @@ -82,18 +108,12 @@ cd libs cd .. # setup qa dependencies -cd tests/qa -./init -cd ../.. +setup_virtualenv tests/qa # setup macro benchmark -cd tests/macro_benchmark -./init -cd ../.. +setup_virtualenv tests/macro_benchmark # setup stress dependencies -cd tests/stress -./init -cd ../.. +setup_virtualenv tests/stress echo "Done installing dependencies for Memgraph" diff --git a/libs/cleanup.sh b/libs/cleanup.sh index e5be018d8..b4d1d7506 100755 --- a/libs/cleanup.sh +++ b/libs/cleanup.sh @@ -1,10 +1,14 @@ #!/bin/bash +# go to script directory +working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd ${working_dir} + # remove antlr parser generator -rm antlr-4.6-complete.jar +rm *.jar # remove lib directories -for folder in ./* ; do +for folder in * ; do if [ -d "$folder" ]; then rm -rf $folder fi diff --git a/libs/setup.sh b/libs/setup.sh index 1e78cbad7..e19b78716 100755 --- a/libs/setup.sh +++ b/libs/setup.sh @@ -5,10 +5,15 @@ working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd ${working_dir} +# Cleanup existing dependencies. +./cleanup.sh + # antlr antlr_generator_filename="antlr-4.6-complete.jar" -wget -O ${antlr_generator_filename} http://www.antlr.org/download/${antlr_generator_filename} -git clone https://github.com/antlr/antlr4.git +#wget -O ${antlr_generator_filename} http://www.antlr.org/download/${antlr_generator_filename} +wget -O ${antlr_generator_filename} http://deps.memgraph.io/${antlr_generator_filename} +#git clone https://github.com/antlr/antlr4.git +git clone git://deps.memgraph.io/antlr4.git antlr4_tag="aacd2a2c95816d8dc1c05814051d631bfec4cf3e" # v4.6 cd antlr4 git checkout ${antlr4_tag} @@ -18,14 +23,16 @@ cd .. # Use our fork that uses experimental/optional instead of unique_ptr in # DerefHolder. Once we move memgraph to c++17 we can use cpp17 branch from # original repo. -git clone https://github.com/memgraph/cppitertools.git +#git clone https://github.com/memgraph/cppitertools.git +git clone git://deps.memgraph.io/cppitertools.git cd cppitertools cppitertools_tag="4231e0bc6fba2737b2a7a8a1576cf06186b0de6a" # experimental_optional 17 Aug 2017 git checkout ${cppitertools_tag} cd .. # fmt -git clone https://github.com/fmtlib/fmt.git +#git clone https://github.com/fmtlib/fmt.git +git clone git://deps.memgraph.io/fmt.git fmt_tag="7fa8f8fa48b0903deab5bb42e6760477173ac485" # v3.0.1 # Commit which fixes an issue when compiling with C++14 and higher. fmt_cxx14_fix="b9aaa507fc49680d037fd84c043f747a395bce04" @@ -35,42 +42,48 @@ git cherry-pick ${fmt_cxx14_fix} cd .. # rapidcheck -git clone https://github.com/emil-e/rapidcheck.git +#git clone https://github.com/emil-e/rapidcheck.git +git clone git://deps.memgraph.io/rapidcheck.git rapidcheck_tag="853e14f0f4313a9eb3c71e24848373e7b843dfd1" # Jun 23, 2017 cd rapidcheck git checkout ${rapidcheck_tag} cd .. # google benchmark -git clone https://github.com/google/benchmark.git +#git clone https://github.com/google/benchmark.git +git clone git://deps.memgraph.io/benchmark.git benchmark_tag="4f8bfeae470950ef005327973f15b0044eceaceb" # v1.1.0 cd benchmark git checkout ${benchmark_tag} cd .. # google test -git clone https://github.com/google/googletest.git +#git clone https://github.com/google/googletest.git +git clone git://deps.memgraph.io/googletest.git googletest_tag="ec44c6c1675c25b9827aacd08c02433cccde7780" # v1.8.0 cd googletest git checkout ${googletest_tag} cd .. # google logging -git clone https://github.com/google/glog.git +#git clone https://github.com/google/glog.git +git clone git://deps.memgraph.io/glog.git glog_tag="a6a166db069520dbbd653c97c2e5b12e08a8bb26" # v0.3.5 cd glog git checkout ${glog_tag} cd .. # lcov-to-coberatura-xml -git clone https://github.com/eriwen/lcov-to-cobertura-xml.git +#git clone https://github.com/eriwen/lcov-to-cobertura-xml.git +git clone git://deps.memgraph.io/lcov-to-cobertura-xml.git lcov_to_xml_tag="59584761cb5da4687693faec05bf3e2b74e9dde9" # Dec 6, 2016 cd lcov-to-cobertura-xml git checkout ${lcov_to_xml_tag} cd .. # google flags -git clone https://github.com/gflags/gflags.git +#git clone https://github.com/gflags/gflags.git +git clone git://deps.memgraph.io/gflags.git gflags_tag="652651b421ca5ac7b722a34a301fb656deca5198" # May 6, 2017 cd gflags git checkout ${gflags_tag} diff --git a/tests/macro_benchmark/init b/tests/macro_benchmark/init deleted file mode 100755 index 1fa3a7f2e..000000000 --- a/tests/macro_benchmark/init +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -if [ ! -d ve3 ]; then - virtualenv -p python3 ve3 || exit 1 - source ve3/bin/activate - pip install -r requirements.txt || exit 1 - deactivate -fi diff --git a/tests/qa/init b/tests/qa/init deleted file mode 100755 index 8c11ad342..000000000 --- a/tests/qa/init +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -function print_usage_and_exit { - echo "./init [-s]" - echo -e "Prepare the environment for Memgraph QA.\n" - echo "Optional arguments:" - echo -e " -s\tuse sudo apt-get for installing required packages" - exit 1 -} - -# read arguments -use_sudo=false -while [[ $# -gt 0 ]]; do - case $1 in - -s) - use_sudo=true - shift - ;; - *) - # unknown option - print_usage_and_exit - ;; - esac -done - -# exit if any subcommand returns a non-zero status -set -e - -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd ${script_dir} - -required_pkgs=(python-virtualenv python3-pip) # required by 'run' script - -# install all dependencies on debian based operating systems -required_missing=false -for pkg in ${required_pkgs[@]}; do - if dpkg -s $pkg 2>/dev/null >/dev/null; then - echo "Found $pkg" - elif [[ $use_sudo = true ]]; then - echo "Installing $pkg" - if [[ ! `sudo apt-get -y install $pkg` ]]; then - echo "Didn't install $pkg [required]" - required_missing=true - fi - else - echo "Missing $pkg [required]" - required_missing=true - fi -done - -if [[ $required_missing = true ]]; then - echo "Missing required packages. EXITING!" - echo "Please, install required packages and rerun $0 again." - exit 2 -fi - -# setup ve -if [ ! -d "ve3" ]; then - virtualenv -p python3 ve3 -fi -source ve3/bin/activate -pip3 install --upgrade pip -pip3 install -r requirements.txt diff --git a/tests/stress/init b/tests/stress/init deleted file mode 100755 index 25dd71ed6..000000000 --- a/tests/stress/init +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd "$DIR" - -# setup virtual environment -if [ ! -d "ve3" ]; then - virtualenv -p python3 ve3 - source ve3/bin/activate - pip3 install -r requirements.txt -fi diff --git a/tests/stress/requirements.txt b/tests/stress/requirements.txt index b2064e22f..904fd1411 100644 --- a/tests/stress/requirements.txt +++ b/tests/stress/requirements.txt @@ -1,6 +1 @@ -appdirs==1.4.3 neo4j-driver==1.3.0 -packaging==16.8 -pkg-resources==0.0.0 -pyparsing==2.2.0 -six==1.10.0