Quelle  cached_download.sh   Sprache: Shell

#!/bin/bash
# this library works like a wrapper around wget, to allow downloads to be cached
# so that if later the same url is retrieved, the entry from the cache will be
# returned.

pushd "$(dirname "$0")" &>/dev/null || exit
cache_dir="$(pwd)/cache"
popd &>/dev/null || exit
retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3"

# Deletes all files in the cache directory
# We don't support folders or .dot(hidden) files
# By not deleting the cache directory, it allows us to use Docker tmpfs mounts,
# which are the only workaround to poor mount r/w performance on MacOS
# Reference: https://forums.docker.com/t/file-access-in-mounted-volumes-extremely-slow-cpu-bound/8076/288
clear_cache () {
    rm -rf "${cache_dir:?}/*"
}

# download method - you pass a filename to save the file under, and the url to call
cached_download () {
    local output_file="${1}"
    local url="${2}"

    if grep -Fx "${url}" "${cache_dir}/urls.list" >/dev/null; then
        echo "Retrieving '${url}' from cache..."
        local line_number
        line_number="$(grep -Fnx "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')"
        cp "${cache_dir}/obj_$(printf "%05d\n" "${line_number}").cache" "${output_file}"
    else
        echo "Downloading '${url}' and placing in cache..."
        rm -f "${output_file}"
        $retry wget -O "${output_file}" --progress=dot:giga --server-response "${url}" 2>&1
        local exit_code=$?
        if [ "${exit_code}" == 0 ]; then
            echo "${url}" >> "${cache_dir}/urls.list"
            local line_number
            line_number="$(grep -Fnx "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')"
            cp "${output_file}" "${cache_dir}/obj_$(printf "%05d\n" "${line_number}").cache"
        else
            return "${exit_code}"
        fi
    fi
}