From c12e4a49b16fbe08df5e1936822e356a8568507b Mon Sep 17 00:00:00 2001
From: Matej Ferencevic <matej.ferencevic@memgraph.io>
Date: Tue, 22 Sep 2020 19:52:26 +0200
Subject: [PATCH] Migrate deps to S3 (#3)

---
 docs/dev/quick-start.md                       |  2 +-
 init                                          |  5 +--
 libs/setup.sh                                 | 43 +++++++------------
 tests/drivers/csharp/run.sh                   |  2 +-
 tests/drivers/java/run.sh                     |  2 +-
 tests/drivers/javascript/run.sh               |  2 +-
 tests/drivers/python/run.sh                   |  2 +-
 tests/integration/ldap/prepare.sh             |  2 +-
 tests/integration/ssl/runner.sh               |  2 +-
 .../groups/bfs_pokec/download_dataset         |  2 +-
 .../groups/pokec/download_dataset             |  2 +-
 11 files changed, 26 insertions(+), 40 deletions(-)

diff --git a/docs/dev/quick-start.md b/docs/dev/quick-start.md
index f0ebc0603..c7fc541ab 100644
--- a/docs/dev/quick-start.md
+++ b/docs/dev/quick-start.md
@@ -52,7 +52,7 @@ After installing the packages, issue the following commands:
 ### Compiling
 
 Memgraph is compiled using our own custom toolchain that can be obtained from
-[Toolchain repository](https://deps.memgraph.io/toolchain).  You should read
+[Toolchain repository](https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain).  You should read
 the `README.txt` file in the repository and install the apropriate toolchain
 for your distribution.  After you have installed the toolchain you should read
 the instructions for the toolchain in the toolchain install directory
diff --git a/init b/init
index 795874fdd..0ebce1b19 100755
--- a/init
+++ b/init
@@ -65,10 +65,7 @@ function setup_virtualenv () {
     # create new virtualenv
     virtualenv -p python3 ve3 || exit 1
     source ve3/bin/activate
-    # we need to increase the timeout for pip because our local cache server
-    # can sometimes be stupid, see: https://github.com/devpi/devpi/issues/208
-    pip --timeout 1000 install -i http://deps.memgraph.io:3141/root/pypi \
-                --trusted-host deps.memgraph.io -r requirements.txt || exit 1
+    pip --timeout 1000 install -r requirements.txt || exit 1
     deactivate
 
     popd > /dev/null
diff --git a/libs/setup.sh b/libs/setup.sh
index 4c9f83cb1..8f57ca897 100755
--- a/libs/setup.sh
+++ b/libs/setup.sh
@@ -46,10 +46,9 @@ clone () {
 # antlr
 antlr_generator_filename="antlr-4.6-complete.jar"
 # wget -O ${antlr_generator_filename} http://www.antlr.org/download/${antlr_generator_filename}
-wget -nv -O ${antlr_generator_filename} http://deps.memgraph.io/${antlr_generator_filename}
-# git clone https://github.com/antlr/antlr4.git
+wget -nv -O ${antlr_generator_filename} https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${antlr_generator_filename}
 antlr4_tag="aacd2a2c95816d8dc1c05814051d631bfec4cf3e" # v4.6
-clone git://deps.memgraph.io/antlr4.git antlr4 $antlr4_tag
+clone https://github.com/antlr/antlr4.git antlr4 $antlr4_tag
 # fix missing include
 sed -i 's/^#pragma once/#pragma once\n#include <functional>/' antlr4/runtime/Cpp/runtime/src/support/CPPUtils.h
 # remove shared library from install dependencies
@@ -59,49 +58,41 @@ sed -i 's/install(TARGETS antlr4_shared/install(TARGETS antlr4_shared OPTIONAL/'
 # Use our fork that uses experimental/optional instead of unique_ptr in
 # DerefHolder. Once we move memgraph to c++17 we can use cpp17 branch from
 # original repo.
-# git clone https://github.com/memgraph/cppitertools.git
 cppitertools_tag="4231e0bc6fba2737b2a7a8a1576cf06186b0de6a" # experimental_optional 17 Aug 2017
-clone git://deps.memgraph.io/cppitertools.git cppitertools $cppitertools_tag
+clone https://github.com/memgraph/cppitertools.git cppitertools $cppitertools_tag
 
 # fmt
-# git clone https://github.com/fmtlib/fmt.git
 fmt_tag="7fa8f8fa48b0903deab5bb42e6760477173ac485" # v3.0.1
 # Commit which fixes an issue when compiling with C++14 and higher.
 fmt_cxx14_fix="b9aaa507fc49680d037fd84c043f747a395bce04"
-clone git://deps.memgraph.io/fmt.git fmt $fmt_tag $fmt_cxx14_fix
+clone https://github.com/fmtlib/fmt.git fmt $fmt_tag $fmt_cxx14_fix
 
 # rapidcheck
-# git clone https://github.com/emil-e/rapidcheck.git
 rapidcheck_tag="853e14f0f4313a9eb3c71e24848373e7b843dfd1" # Jun 23, 2017
-clone git://deps.memgraph.io/rapidcheck.git rapidcheck $rapidcheck_tag
+clone https://github.com/emil-e/rapidcheck.git rapidcheck $rapidcheck_tag
 
 # google benchmark
-# git clone https://github.com/google/benchmark.git
 benchmark_tag="4f8bfeae470950ef005327973f15b0044eceaceb" # v1.1.0
-clone git://deps.memgraph.io/benchmark.git benchmark $benchmark_tag
+clone https://github.com/google/benchmark.git benchmark $benchmark_tag
 
 # google test
-# git clone https://github.com/google/googletest.git
 googletest_tag="ec44c6c1675c25b9827aacd08c02433cccde7780" # v1.8.0
-clone git://deps.memgraph.io/googletest.git googletest $googletest_tag
+clone https://github.com/google/googletest.git googletest $googletest_tag
 
 # google logging
-# git clone https://github.com/memgraph/glog.git
 glog_tag="042a21657e79784226babab8b942f7bd0949635f" # custom version (v0.3.5+)
-clone git://deps.memgraph.io/glog.git glog $glog_tag
+clone https://github.com/memgraph/glog.git glog $glog_tag
 
 # google flags
-# git clone https://github.com/memgraph/gflags.git
 gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)
-clone git://deps.memgraph.io/gflags.git gflags $gflags_tag
+clone https://github.com/memgraph/gflags.git gflags $gflags_tag
 
 # libbcrypt
-# git clone https://github.com/rg3/libbcrypt
 libbcrypt_tag="8aa32ad94ebe06b76853b0767c910c9fbf7ccef4" # custom version (Dec 16, 2016)
-clone git://deps.memgraph.io/libbcrypt.git libbcrypt $libbcrypt_tag
+clone https://github.com/rg3/libbcrypt libbcrypt $libbcrypt_tag
 
 # neo4j
-wget -nv http://deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz -O neo4j.tar.gz
+wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz -O neo4j.tar.gz
 tar -xzf neo4j.tar.gz
 rm -rf neo4j
 mv neo4j-community-3.2.3 neo4j
@@ -112,20 +103,19 @@ rm neo4j.tar.gz
 # We use head on Sep 1, 2017 instead of last release since it was long time ago.
 mkdir -p json
 cd json
-# wget "https://raw.githubusercontent.com/nlohmann/json/91e003285312167ad8365f387438ea371b465a7e/src/json.hpp"
-wget -nv http://deps.memgraph.io/json.hpp
+wget "https://raw.githubusercontent.com/nlohmann/json/91e003285312167ad8365f387438ea371b465a7e/src/json.hpp"
 cd ..
 
 bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580"  # v1.0.6 (May 26, 2011)
-clone git://deps.memgraph.io/bzip2.git bzip2 $bzip2_tag
+clone https://github.com/VFR-maniac/bzip2 bzip2 $bzip2_tag
 
 zlib_tag="cacf7f1d4e3d44d871b605da3b647f07d718623f" # v1.2.11.
-clone git://deps.memgraph.io/zlib.git zlib $zlib_tag
+clone https://github.com/madler/zlib.git zlib $zlib_tag
 # remove shared library from install dependencies
 sed -i 's/install(TARGETS zlib zlibstatic/install(TARGETS zlibstatic/g' zlib/CMakeLists.txt
 
 rocksdb_tag="641fae60f63619ed5d0c9d9e4c4ea5a0ffa3e253" # v5.18.3 Feb 11, 2019
-clone git://deps.memgraph.io/rocksdb.git rocksdb $rocksdb_tag
+clone https://github.com/facebook/rocksdb.git rocksdb $rocksdb_tag
 # fix compilation flags to work with clang 8
 sed -i 's/-Wshadow/-Wno-defaulted-function-deleted/' rocksdb/CMakeLists.txt
 # remove shared library from install dependencies
@@ -133,6 +123,5 @@ sed -i 's/TARGETS ${ROCKSDB_SHARED_LIB}/TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL/'
 
 # mgclient
 mgclient_tag="fe94b3631385ef5dbe40a3d8458860dbcc33e6ea" # May 27, 2019
-# git clone https://github.com/memgraph/mgclient.git
-clone git://deps.memgraph.io/mgclient.git mgclient $mgclient_tag
+clone https://github.com/memgraph/mgclient.git mgclient $mgclient_tag
 sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
diff --git a/tests/drivers/csharp/run.sh b/tests/drivers/csharp/run.sh
index b89df9d11..a3dfb6fcd 100755
--- a/tests/drivers/csharp/run.sh
+++ b/tests/drivers/csharp/run.sh
@@ -16,7 +16,7 @@ if [ ! -f $DRIVER ]; then
     driver_dir=$( mktemp -d driver.XXXXXXXX ) || exit 1
     cd $driver_dir || exit 1
     # Driver downloaded from: https://www.nuget.org/packages/Neo4j.Driver/1.5.3
-    wget -nv http://deps.memgraph.io/drivers/csharp/neo4j.driver.1.5.3.nupkg || exit 1
+    wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/drivers/csharp/neo4j.driver.1.5.3.nupkg || exit 1
     unzip -q neo4j.driver.1.5.3.nupkg || exit 1
     cp lib/net452/Neo4j.Driver.dll .. || exit 1
     cd .. || exit 1
diff --git a/tests/drivers/java/run.sh b/tests/drivers/java/run.sh
index 5a5555243..83e9c2087 100755
--- a/tests/drivers/java/run.sh
+++ b/tests/drivers/java/run.sh
@@ -14,7 +14,7 @@ DRIVER=neo4j-java-driver.jar
 
 if [ ! -f $DRIVER ]; then
     # Driver downloaded from: http://central.maven.org/maven2/org/neo4j/driver/neo4j-java-driver/1.5.2/neo4j-java-driver-1.5.2.jar
-    wget -nv http://deps.memgraph.io/drivers/java/neo4j-java-driver-1.5.2.jar -O $DRIVER || exit 1
+    wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/drivers/java/neo4j-java-driver-1.5.2.jar -O $DRIVER || exit 1
 fi
 
 javac -classpath .:$DRIVER Basic.java
diff --git a/tests/drivers/javascript/run.sh b/tests/drivers/javascript/run.sh
index 4528f4e4f..293257b30 100755
--- a/tests/drivers/javascript/run.sh
+++ b/tests/drivers/javascript/run.sh
@@ -10,7 +10,7 @@ fi
 
 if [ ! -d node_modules ]; then
     # Driver generated with: `npm install neo4j-driver`
-    wget -nv http://deps.memgraph.io/drivers/javascript/neo4j-javascript-driver-1.6.1.tar.gz -O driver.tar.gz || exit 1
+    wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/drivers/javascript/neo4j-javascript-driver-1.6.1.tar.gz -O driver.tar.gz || exit 1
     tar -xzf driver.tar.gz || exit 1
     rm driver.tar.gz || exit 1
 fi
diff --git a/tests/drivers/python/run.sh b/tests/drivers/python/run.sh
index d23e81b69..1b5f959d3 100755
--- a/tests/drivers/python/run.sh
+++ b/tests/drivers/python/run.sh
@@ -12,7 +12,7 @@ fi
 # setup virtual environment
 if [ ! -d "ve3" ]; then
     # Driver downloaded from: https://pypi.org/project/neo4j-driver/1.5.3/
-    wget -nv http://deps.memgraph.io/drivers/python/neo4j-driver-1.5.3.tar.gz -O neo4j-driver.tar.gz || exit 1
+    wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/drivers/python/neo4j-driver-1.5.3.tar.gz -O neo4j-driver.tar.gz || exit 1
     tar -xzf neo4j-driver.tar.gz || exit 1
     mv neo4j-driver-1.5.3 neo4j-driver || exit 1
     virtualenv -p python3 ve3 || exit 1
diff --git a/tests/integration/ldap/prepare.sh b/tests/integration/ldap/prepare.sh
index 4a7704812..c6b50415b 100755
--- a/tests/integration/ldap/prepare.sh
+++ b/tests/integration/ldap/prepare.sh
@@ -17,7 +17,7 @@ if [ -d "$name" ]; then
 fi
 
 echo_info "Downloading and unpacking OpenLDAP"
-wget -nv -O $name.tgz http://deps.memgraph.io/$name.tgz
+wget -nv -O $name.tgz https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/$name.tgz
 tar -xf $name.tgz
 rm $name.tgz
 
diff --git a/tests/integration/ssl/runner.sh b/tests/integration/ssl/runner.sh
index 8eb9e7b74..70c6c0dd4 100755
--- a/tests/integration/ssl/runner.sh
+++ b/tests/integration/ssl/runner.sh
@@ -16,7 +16,7 @@ mkdir -p $tmpdir
 cd $tmpdir
 
 easyrsa="EasyRSA-3.0.4"
-wget -nv http://deps.memgraph.io/$easyrsa.tgz
+wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/$easyrsa.tgz
 
 tar -xf $easyrsa.tgz
 mv $easyrsa ca1
diff --git a/tests/macro_benchmark/groups/bfs_pokec/download_dataset b/tests/macro_benchmark/groups/bfs_pokec/download_dataset
index cd315c5f1..b2aa61bbd 100755
--- a/tests/macro_benchmark/groups/bfs_pokec/download_dataset
+++ b/tests/macro_benchmark/groups/bfs_pokec/download_dataset
@@ -3,4 +3,4 @@
 working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd ${working_dir}
 
-wget -nv -O pokec_small.setup.cypher http://deps.memgraph.io/pokec_small.setup.cypher
+wget -nv -O pokec_small.setup.cypher https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/pokec_small.setup.cypher
diff --git a/tests/macro_benchmark/groups/pokec/download_dataset b/tests/macro_benchmark/groups/pokec/download_dataset
index cd315c5f1..b2aa61bbd 100755
--- a/tests/macro_benchmark/groups/pokec/download_dataset
+++ b/tests/macro_benchmark/groups/pokec/download_dataset
@@ -3,4 +3,4 @@
 working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd ${working_dir}
 
-wget -nv -O pokec_small.setup.cypher http://deps.memgraph.io/pokec_small.setup.cypher
+wget -nv -O pokec_small.setup.cypher https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/pokec_small.setup.cypher