[WIP] Re-install python libraries rather than upgrading

deploy/test-server/install-pn-test-server

@@ -114,8 +114,9 @@ su pn -c '
    echo staging > deploy-branch
    mkdir -p physionet-build python-env deploy
 
-   virtualenv --no-download -ppython3 /physionet/python-env/physionet
-   . /physionet/python-env/physionet/bin/activate
+   virtualenv --no-download -ppython3 /physionet/python-env/initial
+   ln -s initial /physionet/python-env/physionet
+   . /physionet/python-env/initial/bin/activate
 
    git clone --bare "$REPOSITORY" physionet-build.git
    cd /physionet/physionet-build.git

deploy/update

@@ -4,7 +4,8 @@ set -e
 
 install_dir=/physionet/physionet-build
 working_dir=/physionet/physionet-build.new
-venv_dir=/physionet/python-env/physionet
+venv_prefix=/physionet/python-env
+venv_dir=$venv_prefix/physionet
 log_file=/data/log/pn/update.log
 branch=$(cat /physionet/deploy-branch) # production or staging
 
@@ -76,18 +77,28 @@ mkdir -p $working_dir
 git archive "$newrev" | tar -x -C $working_dir
 ln -s $install_dir/.env $working_dir/.env
 
-. $venv_dir/bin/activate
-
-# Install new dependencies from 'requirements.txt' into $venv_dir
+# Install new dependencies from 'requirements.txt' into $new_venv_dir
 if [ -n "$no_pip" ]; then
     echo "- SKIPPING requirements due to --push-option=no-pip"
+    new_venv_dir=$venv_dir
 else
-    echo "* Installing new requirements..."
-    pip3 install -r $working_dir/requirements.txt \
-         --quiet --require-hashes --log $log_file
-    echo >> $log_file
+    hash=$(md5sum "$working_dir/requirements.txt" | cut -d' ' -f1)
+    new_venv_dir=$venv_prefix/$hash
+    if [ -d "$new_venv_dir" ] && cmp -s "$working_dir/requirements.txt" \
+                                      "$new_venv_dir/installed.txt"; then
+        echo "- Requirements previously installed ($hash)."
+    else
+        echo "* Installing new requirements ($hash)..."
+        virtualenv -ppython3 --quiet --clear --no-download "$new_venv_dir"
+        pip_log=$new_venv_dir/pip-install.log
+        "$new_venv_dir/bin/pip3" install -r "$working_dir/requirements.txt" \
+                                 --quiet --require-hashes --log $pip_log
+        cp "$working_dir/requirements.txt" "$new_venv_dir/installed.txt"
+    fi
 fi
 
+. $new_venv_dir/bin/activate
+
 # Run 'getmigrationtargets' to check whether migrations make sense
 # (e.g. broken dependencies, ghost migrations)
 echo "* Checking migrations..."
@@ -98,4 +109,9 @@ echo "* Checking migrations..."
     ./manage.py getmigrationtargets > /dev/null
 )
 
+# Update $venv_dir to point to $new_venv_dir
+if [ "$venv_dir" != "$new_venv_dir" ]; then
+    ln -s -f -T "$new_venv_dir" "$venv_dir"
+fi
+
 echo "$(date '+%F %T %z'): $branch: update finished" >> $log_file

test-upgrade.sh

@@ -266,12 +266,14 @@ export PATH=$venvdir/bin:$PATH
 
     msg_testing "Installing new requirements"
     if ! cmp -s "$olddir/requirements.txt" "$topdir/requirements.txt"; then
-        cachefile=$venvcachedir/$old_reqs_hash-$new_reqs_hash.tar.gz
+        prereq_cmd rm -rf "$venvdir"
+        cachefile=$venvcachedir/$new_reqs_hash.tar.gz
         if [ -n "$venvcachedir" ] && [ -f "$cachefile" ]; then
-            prereq_cmd rm -rf "$venvdir"
             prereq_cmd mkdir "$venvdir"
             prereq_cmd tar -xzf "$cachefile" -C "$venvdir"
         else
+            prereq_cmd virtualenv --quiet --quiet \
+                       --no-download -ppython3 "$venvdir"
             prereq_cmd pip3 install --require-hashes \
                        -r "$topdir/requirements.txt"
             if [ -n "$venvcachedir" ]; then