tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Migrate deps to S3 (#3)

Browse Source
This commit is contained in:
Matej Ferencevic 2020-09-22 19:52:26 +02:00 committed by GitHub
parent 4c0fc11f69
commit c12e4a49b1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 26 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/dev/quick-start.md
View File

@ -52,7 +52,7 @@ After installing the packages, issue the following commands:
### Compiling ### Compiling
Memgraph is compiled using our own custom toolchain that can be obtained from Memgraph is compiled using our own custom toolchain that can be obtained from
[Toolchain repository](https://deps.memgraph.io/toolchain). You should read [Toolchain repository](https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain). You should read
the `README.txt` file in the repository and install the apropriate toolchain the `README.txt` file in the repository and install the apropriate toolchain
for your distribution. After you have installed the toolchain you should read for your distribution. After you have installed the toolchain you should read
the instructions for the toolchain in the toolchain install directory the instructions for the toolchain in the toolchain install directory

5
init
View File

@ -65,10 +65,7 @@ function setup_virtualenv () {
# create new virtualenv # create new virtualenv
virtualenv -p python3 ve3 || exit 1 virtualenv -p python3 ve3 || exit 1
source ve3/bin/activate source ve3/bin/activate
# we need to increase the timeout for pip because our local cache server pip --timeout 1000 install -r requirements.txt || exit 1
# can sometimes be stupid, see: https://github.com/devpi/devpi/issues/208
pip --timeout 1000 install -i http://deps.memgraph.io:3141/root/pypi \
--trusted-host deps.memgraph.io -r requirements.txt || exit 1
deactivate deactivate
popd > /dev/null popd > /dev/null

43
libs/setup.sh
View File

@ -46,10 +46,9 @@ clone () {
# antlr # antlr
antlr_generator_filename="antlr-4.6-complete.jar" antlr_generator_filename="antlr-4.6-complete.jar"
# wget -O ${antlr_generator_filename} http://www.antlr.org/download/${antlr_generator_filename} # wget -O ${antlr_generator_filename} http://www.antlr.org/download/${antlr_generator_filename}
wget -nv -O ${antlr_generator_filename} http://deps.memgraph.io/${antlr_generator_filename} wget -nv -O ${antlr_generator_filename} https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${antlr_generator_filename}
# git clone https://github.com/antlr/antlr4.git
antlr4_tag="aacd2a2c95816d8dc1c05814051d631bfec4cf3e" # v4.6 antlr4_tag="aacd2a2c95816d8dc1c05814051d631bfec4cf3e" # v4.6
clone git://deps.memgraph.io/antlr4.git antlr4 $antlr4_tag clone https://github.com/antlr/antlr4.git antlr4 $antlr4_tag
# fix missing include # fix missing include
sed -i 's/^#pragma once/#pragma once\n#include <functional>/' antlr4/runtime/Cpp/runtime/src/support/CPPUtils.h sed -i 's/^#pragma once/#pragma once\n#include <functional>/' antlr4/runtime/Cpp/runtime/src/support/CPPUtils.h
# remove shared library from install dependencies # remove shared library from install dependencies
@ -59,49 +58,41 @@ sed -i 's/install(TARGETS antlr4_shared/install(TARGETS antlr4_shared OPTIONAL/'
# Use our fork that uses experimental/optional instead of unique_ptr in # Use our fork that uses experimental/optional instead of unique_ptr in
# DerefHolder. Once we move memgraph to c++17 we can use cpp17 branch from # DerefHolder. Once we move memgraph to c++17 we can use cpp17 branch from
# original repo. # original repo.
# git clone https://github.com/memgraph/cppitertools.git
cppitertools_tag="4231e0bc6fba2737b2a7a8a1576cf06186b0de6a" # experimental_optional 17 Aug 2017 cppitertools_tag="4231e0bc6fba2737b2a7a8a1576cf06186b0de6a" # experimental_optional 17 Aug 2017
clone git://deps.memgraph.io/cppitertools.git cppitertools $cppitertools_tag clone https://github.com/memgraph/cppitertools.git cppitertools $cppitertools_tag
# fmt # fmt
# git clone https://github.com/fmtlib/fmt.git
fmt_tag="7fa8f8fa48b0903deab5bb42e6760477173ac485" # v3.0.1 fmt_tag="7fa8f8fa48b0903deab5bb42e6760477173ac485" # v3.0.1
# Commit which fixes an issue when compiling with C++14 and higher. # Commit which fixes an issue when compiling with C++14 and higher.
fmt_cxx14_fix="b9aaa507fc49680d037fd84c043f747a395bce04" fmt_cxx14_fix="b9aaa507fc49680d037fd84c043f747a395bce04"
clone git://deps.memgraph.io/fmt.git fmt $fmt_tag $fmt_cxx14_fix clone https://github.com/fmtlib/fmt.git fmt $fmt_tag $fmt_cxx14_fix
# rapidcheck # rapidcheck
# git clone https://github.com/emil-e/rapidcheck.git
rapidcheck_tag="853e14f0f4313a9eb3c71e24848373e7b843dfd1" # Jun 23, 2017 rapidcheck_tag="853e14f0f4313a9eb3c71e24848373e7b843dfd1" # Jun 23, 2017
clone git://deps.memgraph.io/rapidcheck.git rapidcheck $rapidcheck_tag clone https://github.com/emil-e/rapidcheck.git rapidcheck $rapidcheck_tag
# google benchmark # google benchmark
# git clone https://github.com/google/benchmark.git
benchmark_tag="4f8bfeae470950ef005327973f15b0044eceaceb" # v1.1.0 benchmark_tag="4f8bfeae470950ef005327973f15b0044eceaceb" # v1.1.0
clone git://deps.memgraph.io/benchmark.git benchmark $benchmark_tag clone https://github.com/google/benchmark.git benchmark $benchmark_tag
# google test # google test
# git clone https://github.com/google/googletest.git
googletest_tag="ec44c6c1675c25b9827aacd08c02433cccde7780" # v1.8.0 googletest_tag="ec44c6c1675c25b9827aacd08c02433cccde7780" # v1.8.0
clone git://deps.memgraph.io/googletest.git googletest $googletest_tag clone https://github.com/google/googletest.git googletest $googletest_tag
# google logging # google logging
# git clone https://github.com/memgraph/glog.git
glog_tag="042a21657e79784226babab8b942f7bd0949635f" # custom version (v0.3.5+) glog_tag="042a21657e79784226babab8b942f7bd0949635f" # custom version (v0.3.5+)
clone git://deps.memgraph.io/glog.git glog $glog_tag clone https://github.com/memgraph/glog.git glog $glog_tag
# google flags # google flags
# git clone https://github.com/memgraph/gflags.git
gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017) gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)
clone git://deps.memgraph.io/gflags.git gflags $gflags_tag clone https://github.com/memgraph/gflags.git gflags $gflags_tag
# libbcrypt # libbcrypt
# git clone https://github.com/rg3/libbcrypt
libbcrypt_tag="8aa32ad94ebe06b76853b0767c910c9fbf7ccef4" # custom version (Dec 16, 2016) libbcrypt_tag="8aa32ad94ebe06b76853b0767c910c9fbf7ccef4" # custom version (Dec 16, 2016)
clone git://deps.memgraph.io/libbcrypt.git libbcrypt $libbcrypt_tag clone https://github.com/rg3/libbcrypt libbcrypt $libbcrypt_tag
# neo4j # neo4j
wget -nv http://deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz -O neo4j.tar.gz wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz -O neo4j.tar.gz
tar -xzf neo4j.tar.gz tar -xzf neo4j.tar.gz
rm -rf neo4j rm -rf neo4j
mv neo4j-community-3.2.3 neo4j mv neo4j-community-3.2.3 neo4j
@ -112,20 +103,19 @@ rm neo4j.tar.gz
# We use head on Sep 1, 2017 instead of last release since it was long time ago. # We use head on Sep 1, 2017 instead of last release since it was long time ago.
mkdir -p json mkdir -p json
cd json cd json
# wget "https://raw.githubusercontent.com/nlohmann/json/91e003285312167ad8365f387438ea371b465a7e/src/json.hpp" wget "https://raw.githubusercontent.com/nlohmann/json/91e003285312167ad8365f387438ea371b465a7e/src/json.hpp"
wget -nv http://deps.memgraph.io/json.hpp
cd .. cd ..
bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580" # v1.0.6 (May 26, 2011) bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580" # v1.0.6 (May 26, 2011)
clone git://deps.memgraph.io/bzip2.git bzip2 $bzip2_tag clone https://github.com/VFR-maniac/bzip2 bzip2 $bzip2_tag
zlib_tag="cacf7f1d4e3d44d871b605da3b647f07d718623f" # v1.2.11. zlib_tag="cacf7f1d4e3d44d871b605da3b647f07d718623f" # v1.2.11.
clone git://deps.memgraph.io/zlib.git zlib $zlib_tag clone https://github.com/madler/zlib.git zlib $zlib_tag
# remove shared library from install dependencies # remove shared library from install dependencies
sed -i 's/install(TARGETS zlib zlibstatic/install(TARGETS zlibstatic/g' zlib/CMakeLists.txt sed -i 's/install(TARGETS zlib zlibstatic/install(TARGETS zlibstatic/g' zlib/CMakeLists.txt
rocksdb_tag="641fae60f63619ed5d0c9d9e4c4ea5a0ffa3e253" # v5.18.3 Feb 11, 2019 rocksdb_tag="641fae60f63619ed5d0c9d9e4c4ea5a0ffa3e253" # v5.18.3 Feb 11, 2019
clone git://deps.memgraph.io/rocksdb.git rocksdb $rocksdb_tag clone https://github.com/facebook/rocksdb.git rocksdb $rocksdb_tag
# fix compilation flags to work with clang 8 # fix compilation flags to work with clang 8
sed -i 's/-Wshadow/-Wno-defaulted-function-deleted/' rocksdb/CMakeLists.txt sed -i 's/-Wshadow/-Wno-defaulted-function-deleted/' rocksdb/CMakeLists.txt
# remove shared library from install dependencies # remove shared library from install dependencies
@ -133,6 +123,5 @@ sed -i 's/TARGETS ${ROCKSDB_SHARED_LIB}/TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL/'
# mgclient # mgclient
mgclient_tag="fe94b3631385ef5dbe40a3d8458860dbcc33e6ea" # May 27, 2019 mgclient_tag="fe94b3631385ef5dbe40a3d8458860dbcc33e6ea" # May 27, 2019
# git clone https://github.com/memgraph/mgclient.git clone https://github.com/memgraph/mgclient.git mgclient $mgclient_tag
clone git://deps.memgraph.io/mgclient.git mgclient $mgclient_tag
sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt

2
tests/drivers/csharp/run.sh
View File

@ -16,7 +16,7 @@ if [ ! -f $DRIVER ]; then
driver_dir=$( mktemp -d driver.XXXXXXXX ) || exit 1 driver_dir=$( mktemp -d driver.XXXXXXXX ) || exit 1
cd $driver_dir || exit 1 cd $driver_dir || exit 1
# Driver downloaded from: https://www.nuget.org/packages/Neo4j.Driver/1.5.3 # Driver downloaded from: https://www.nuget.org/packages/Neo4j.Driver/1.5.3
wget -nv http://deps.memgraph.io/drivers/csharp/neo4j.driver.1.5.3.nupkg || exit 1 wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/drivers/csharp/neo4j.driver.1.5.3.nupkg || exit 1
unzip -q neo4j.driver.1.5.3.nupkg || exit 1 unzip -q neo4j.driver.1.5.3.nupkg || exit 1
cp lib/net452/Neo4j.Driver.dll .. || exit 1 cp lib/net452/Neo4j.Driver.dll .. || exit 1
cd .. || exit 1 cd .. || exit 1

2
tests/drivers/java/run.sh
View File

@ -14,7 +14,7 @@ DRIVER=neo4j-java-driver.jar
if [ ! -f $DRIVER ]; then if [ ! -f $DRIVER ]; then
# Driver downloaded from: http://central.maven.org/maven2/org/neo4j/driver/neo4j-java-driver/1.5.2/neo4j-java-driver-1.5.2.jar # Driver downloaded from: http://central.maven.org/maven2/org/neo4j/driver/neo4j-java-driver/1.5.2/neo4j-java-driver-1.5.2.jar
wget -nv http://deps.memgraph.io/drivers/java/neo4j-java-driver-1.5.2.jar -O $DRIVER || exit 1 wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/drivers/java/neo4j-java-driver-1.5.2.jar -O $DRIVER || exit 1
fi fi
javac -classpath .:$DRIVER Basic.java javac -classpath .:$DRIVER Basic.java

2
tests/drivers/javascript/run.sh
View File

@ -10,7 +10,7 @@ fi
if [ ! -d node_modules ]; then if [ ! -d node_modules ]; then
# Driver generated with: `npm install neo4j-driver` # Driver generated with: `npm install neo4j-driver`
wget -nv http://deps.memgraph.io/drivers/javascript/neo4j-javascript-driver-1.6.1.tar.gz -O driver.tar.gz || exit 1 wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/drivers/javascript/neo4j-javascript-driver-1.6.1.tar.gz -O driver.tar.gz || exit 1
tar -xzf driver.tar.gz || exit 1 tar -xzf driver.tar.gz || exit 1
rm driver.tar.gz || exit 1 rm driver.tar.gz || exit 1
fi fi

2
tests/drivers/python/run.sh
View File

@ -12,7 +12,7 @@ fi
# setup virtual environment # setup virtual environment
if [ ! -d "ve3" ]; then if [ ! -d "ve3" ]; then
# Driver downloaded from: https://pypi.org/project/neo4j-driver/1.5.3/ # Driver downloaded from: https://pypi.org/project/neo4j-driver/1.5.3/
wget -nv http://deps.memgraph.io/drivers/python/neo4j-driver-1.5.3.tar.gz -O neo4j-driver.tar.gz || exit 1 wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/drivers/python/neo4j-driver-1.5.3.tar.gz -O neo4j-driver.tar.gz || exit 1
tar -xzf neo4j-driver.tar.gz || exit 1 tar -xzf neo4j-driver.tar.gz || exit 1
mv neo4j-driver-1.5.3 neo4j-driver || exit 1 mv neo4j-driver-1.5.3 neo4j-driver || exit 1
virtualenv -p python3 ve3 || exit 1 virtualenv -p python3 ve3 || exit 1

2
tests/integration/ldap/prepare.sh
View File

@ -17,7 +17,7 @@ if [ -d "$name" ]; then
fi fi
echo_info "Downloading and unpacking OpenLDAP" echo_info "Downloading and unpacking OpenLDAP"
wget -nv -O $name.tgz http://deps.memgraph.io/$name.tgz wget -nv -O $name.tgz https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/$name.tgz
tar -xf $name.tgz tar -xf $name.tgz
rm $name.tgz rm $name.tgz

2
tests/integration/ssl/runner.sh
View File

@ -16,7 +16,7 @@ mkdir -p $tmpdir
cd $tmpdir cd $tmpdir
easyrsa="EasyRSA-3.0.4" easyrsa="EasyRSA-3.0.4"
wget -nv http://deps.memgraph.io/$easyrsa.tgz wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/$easyrsa.tgz
tar -xf $easyrsa.tgz tar -xf $easyrsa.tgz
mv $easyrsa ca1 mv $easyrsa ca1

2
tests/macro_benchmark/groups/bfs_pokec/download_dataset
View File

@ -3,4 +3,4 @@
working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd ${working_dir} cd ${working_dir}
wget -nv -O pokec_small.setup.cypher http://deps.memgraph.io/pokec_small.setup.cypher wget -nv -O pokec_small.setup.cypher https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/pokec_small.setup.cypher

2
tests/macro_benchmark/groups/pokec/download_dataset
View File

@ -3,4 +3,4 @@
working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd ${working_dir} cd ${working_dir}
wget -nv -O pokec_small.setup.cypher http://deps.memgraph.io/pokec_small.setup.cypher wget -nv -O pokec_small.setup.cypher https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/pokec_small.setup.cypher