Select Git revision
settings_activity.dart
-
Benoît Harrault authoredBenoît Harrault authored
01_download_images.sh 2.80 KiB
#!/usr/bin/env bash
command -v convert >/dev/null 2>&1 || { echo >&2 "I require convert (imagemagick) but it's not installed. Aborting."; exit 1; }
command -v jq >/dev/null 2>&1 || { echo >&2 "I require jq (json parser) but it's not installed. Aborting."; exit 1; }
CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
BASE_DIR="$(dirname "${CURRENT_DIR}")"
IMAGES_CACHE_FOLDER="${CURRENT_DIR}/cache/download"
INPUT_WORDS_LIST="${BASE_DIR}/assets/files/words-fr.json"
# Images variants
KEYWORD_VARIANTS=",image,picture,drawing,black and white,painting,icon"
COUNT_BY_WEYWORD_VARIANT=3
KEYWORD_VARIANTS_LIST="$(echo "${KEYWORD_VARIANTS}" | tr "," "\n")"
# QWANT parameters
LANG="fr_FR"
TYPE="images"
OFFSET=0
BASE_URL="https://api.qwant.com/api/search/${TYPE}"
WORDS_LIST="$(cat "${INPUT_WORDS_LIST}" | grep -E '\"[A-Z\-]+\",$' | cut -d'"' -f2 | sort | uniq | sort -R)"
while read -r KEYWORD; do
if [[ -n "${KEYWORD}" ]]; then
echo "KEYWORD: ${KEYWORD}"
while read -r VARIANT; do
echo " VARIANT: ${VARIANT}"
QUERY_STRING="${KEYWORD}"
if [[ ! -z "${VARIANT}" ]]; then
QUERY_STRING="${QUERY_STRING} ${VARIANT}"
fi
# Get QWANT API query from keyword
QUERY="$(echo "${QUERY_STRING}" | tr "A-Z" "a-z" | sed 's| |%20|g')"
QUERY_URL="${BASE_URL}?count=${COUNT_BY_WEYWORD_VARIANT}&q=${QUERY}&t=${TYPE}&safesearch=1&offset=${OFFSET}&locale=${LANG}&uiv=4"
# Get QWANT thumbnails urls from keyword
URL_LIST="$(curl "${QUERY_URL}" \
--silent \
-H 'User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0' \
--compressed \
| jq | grep "media_preview" | cut -d'"' -f4 | sed 's|^//||g')"
while read -r URL; do
if [[ -n "${URL}" ]]; then
HASH="$(echo "${URL}" | md5sum | awk '{print $1}')"
OUTPUT_FILE="${IMAGES_CACHE_FOLDER}/${KEYWORD}/${HASH}.png"
echo " OUTPUT_FILE: $(echo "${OUTPUT_FILE}" | sed "s|^${IMAGES_CACHE_FOLDER}/||g")"
TMP_IMAGE_FILE="${OUTPUT_FILE}.tmp.png"
if [[ -f "${TMP_IMAGE_FILE}" ]]; then
rm -f "${TMP_IMAGE_FILE}"
fi
if [[ -f "${OUTPUT_FILE}" ]]; then
echo " - Already downloaded"
else
mkdir -p "$(dirname ${OUTPUT_FILE})"
echo " + Downloading..."
wget --quiet --timeout=10 "${URL}" -O "${TMP_IMAGE_FILE}"
if [[ -f "${TMP_IMAGE_FILE}" ]]; then
echo " + Converting..."
convert "${TMP_IMAGE_FILE}" "${OUTPUT_FILE}"
fi
if [[ -f "${TMP_IMAGE_FILE}" ]]; then
rm -f "${TMP_IMAGE_FILE}"
fi
fi
fi
done < <(echo "${URL_LIST}")
done < <(echo "${KEYWORD_VARIANTS_LIST}")
fi
done < <(echo "${WORDS_LIST}")