Merge branch '30-improve-download-check-images-process' into 'master'

Resolve "Improve download/check images process"

Closes #30

See merge request !29

scripts/get_images.sh → scripts/01_download_images.sh

 #!/usr/bin/env bash
 
-command -v optipng >/dev/null 2>&1 || { echo >&2 "I require optipng but it's not installed. Aborting."; exit 1; }
 command -v convert >/dev/null 2>&1 || { echo >&2 "I require convert (imagemagick) but it's not installed. Aborting."; exit 1; }
 command -v jq >/dev/null 2>&1 || { echo >&2 "I require jq (json parser) but it's not installed. Aborting."; exit 1; }
 
 CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 BASE_DIR="$(dirname "${CURRENT_DIR}")"
 
-IMAGES_CACHE_FOLDER="${CURRENT_DIR}/cache"
+IMAGES_CACHE_FOLDER="${CURRENT_DIR}/cache/download"
 INPUT_WORDS_LIST="${BASE_DIR}/assets/files/words-fr.json"
 
-COUNT_BY_WEYWORD_VARIANT=3
-
+# Images variants
 KEYWORD_VARIANTS=",image,picture,drawing,black and white,painting,icon"
+COUNT_BY_WEYWORD_VARIANT=3
 KEYWORD_VARIANTS_LIST="$(echo "${KEYWORD_VARIANTS}" | tr "," "\n")"
 
-RESIZE_OPTION="128x128"
-CROP_PARAMETERS="-auto-orient -trim +repage -gravity center -background white -extent ${RESIZE_OPTION}^"
-CONVERT_OPTIONS="-alpha off +dither -colors 256 -depth 4"
-
-OPTIPNG_OPTIONS="-preserve -quiet -o7"
-
+# QWANT parameters
 LANG="fr_FR"
 TYPE="images"
 OFFSET=0
@@ -71,9 +65,7 @@ while read -r KEYWORD; do
             wget --quiet --timeout=10 "${URL}" -O "${TMP_IMAGE_FILE}"
             if [[ -f "${TMP_IMAGE_FILE}" ]]; then
               echo "   + Converting..."
-              convert "${TMP_IMAGE_FILE}" -resize "${RESIZE_OPTION}^" ${CROP_PARAMETERS} ${CONVERT_OPTIONS} "${OUTPUT_FILE}"
-              echo "   + Optimizing..."
-              optipng ${OPTIPNG_OPTIONS} "${OUTPUT_FILE}"
+              convert "${TMP_IMAGE_FILE}" "${OUTPUT_FILE}"
             fi
 
             if [[ -f "${TMP_IMAGE_FILE}" ]]; then
@@ -85,6 +77,3 @@ while read -r KEYWORD; do
     done < <(echo "${KEYWORD_VARIANTS_LIST}")
   fi
 done < <(echo "${WORDS_LIST}")
-
-# Build assets list
-"${CURRENT_DIR}/build_assets_list.sh"

scripts/02_prepare_images.sh

+#!/usr/bin/env bash
+
+CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+IMAGES_CACHE_FOLDER="${CURRENT_DIR}/cache"
+
+IMAGES_DOWNLOAD_FOLDER="${CURRENT_DIR}/cache/download"
+IMAGES_SELECTED_FOLDER="${CURRENT_DIR}/cache/download"
+
+# delete empty/temp files
+find "${IMAGES_CACHE_FOLDER}" -type f -name "*.png" -empty -exec rm {} \;
+find "${IMAGES_CACHE_FOLDER}" -type f -name "*.tmp*" -exec rm {} \;
+find "${IMAGES_CACHE_FOLDER}" -type d -empty -exec rm -rf {} \;
+
+# copy missing files from "download" to "selected"
+rsync --ignore-existing -raz --progress "${IMAGES_DOWNLOAD_FOLDER}/*" "${IMAGES_SELECTED_FOLDER}"
+
+echo "done: copy files from download to selected."
+echo "please remove unwanted files from selected folders."

scripts/03_optimize_images.sh

+#!/usr/bin/env bash
+
+command -v optipng >/dev/null 2>&1 || { echo >&2 "I require optipng but it's not installed. Aborting."; exit 1; }
+command -v convert >/dev/null 2>&1 || { echo >&2 "I require convert (imagemagick) but it's not installed. Aborting."; exit 1; }
+
+CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+IMAGES_CACHE_FOLDER="${CURRENT_DIR}/cache"
+
+IMAGES_SELECTED_FOLDER="${CURRENT_DIR}/cache/download"
+IMAGES_OPTIMIZED_FOLDER="${CURRENT_DIR}/cache/optimized"
+
+RESIZE_OPTION="128x128"
+CROP_PARAMETERS="-auto-orient -trim +repage -gravity center -background white -extent ${RESIZE_OPTION}^"
+CONVERT_OPTIONS="-alpha off +dither -colors 256 -depth 4"
+
+OPTIPNG_OPTIONS="-preserve -quiet -o7"
+
+# delete empty/temp files
+find "${IMAGES_CACHE_FOLDER}" -type f -name "*.png" -empty -exec rm {} \;
+find "${IMAGES_CACHE_FOLDER}" -type f -name "*.tmp*" -exec rm {} \;
+find "${IMAGES_CACHE_FOLDER}" -type d -empty -exec rm -rf {} \;
+
+IMAGES="$(find "${IMAGES_SELECTED_FOLDER}" -type f -name "*.png" | sort)"
+
+while read -r INPUT_FILE; do
+  if [[ -n "${INPUT_FILE}" ]]; then
+    OUTPUT_FILE="$(echo "${INPUT_FILE}" | sed "s|^${IMAGES_SELECTED_FOLDER}/|${IMAGES_OPTIMIZED_FOLDER}/|g")"
+    echo "  OUTPUT_FILE: ${OUTPUT_FILE}"
+
+    if [[ -f "${OUTPUT_FILE}" ]]; then
+      echo "   - Already optimized"
+    else
+      mkdir -p "$(dirname ${OUTPUT_FILE})"
+
+      echo "   + Converting..."
+      convert "${INPUT_FILE}" -resize "${RESIZE_OPTION}^" ${CROP_PARAMETERS} ${CONVERT_OPTIONS} "${OUTPUT_FILE}"
+      echo "   + Optimizing..."
+      optipng ${OPTIPNG_OPTIONS} "${OUTPUT_FILE}"
+    fi
+  fi
+done < <(echo "${IMAGES}")
+
+echo "done."

scripts/build_assets_list.sh → scripts/04_build_assets_list.sh

@@ -11,16 +11,19 @@ OUTPUT_ASSETS_FILE="${ASSETS_BASE_FOLDER}/assets_images.json"
 touch "${OUTPUT_ASSETS_FILE}"
 
 IMAGES_CACHE_FOLDER="${CURRENT_DIR}/cache"
+IMAGES_OPTIMIZED_FOLDER="${IMAGES_CACHE_FOLDER}/optimized"
 
-# delete empty/temp files
-find "${IMAGES_CACHE_FOLDER}" -type f -name "*.png" -empty -exec rm {} \;
-find "${IMAGES_CACHE_FOLDER}" -type f -name "*.tmp*" -exec rm {} \;
-find "${IMAGES_CACHE_FOLDER}" -type d -empty -exec rm -rf {} \;
+echo "Cleaning empty/temp files..."
+find "${IMAGES_OPTIMIZED_FOLDER}" -type f -name "*.png" -empty -exec rm {} \;
+find "${IMAGES_OPTIMIZED_FOLDER}" -type f -name "*.tmp*" -exec rm {} \;
+find "${IMAGES_OPTIMIZED_FOLDER}" -type d -empty -exec rm -rf {} \;
+
+echo "Cleaning existing assets..."
 find "${IMAGES_ASSETS_FOLDER}" -type f -name "*.png" -exec rm {} \;
 
 echo "Building assets json file..."
 
-FILES="$(find "${IMAGES_CACHE_FOLDER}" -type f -name "*.png" | sed "s|^${IMAGES_CACHE_FOLDER}/||g" | sort)"
+FILES="$(find "${IMAGES_OPTIMIZED_FOLDER}" -type f -name "*.png" | sed "s|^${IMAGES_OPTIMIZED_FOLDER}/||g" | sort)"
 KEYWORDS="$(echo "${FILES}" | cut -d'/' -f1 | sort | uniq)"
 
 OUTPUT_ASSETS_FILE_TMP="${OUTPUT_ASSETS_FILE}.tmp"
@@ -29,6 +32,7 @@ echo "  \"images\": {" >> "${OUTPUT_ASSETS_FILE_TMP}"
 FIRST_KEYWORD=1
 while read -r KEYWORD; do
   if [[ -n "${KEYWORD}" ]]; then
+    echo "- ${KEYWORD}"
     if [[ ${FIRST_KEYWORD} -eq 0 ]]; then
       echo "    ," >> "${OUTPUT_ASSETS_FILE_TMP}"
     fi
@@ -41,7 +45,7 @@ while read -r KEYWORD; do
       fi
       echo "      \"$(echo "${FILE}" | sed "s|${KEYWORD}/||g")\"" >> "${OUTPUT_ASSETS_FILE_TMP}"
       FIRST_FILE=0
-      cp -f "${IMAGES_CACHE_FOLDER}/${FILE}" "${IMAGES_ASSETS_FOLDER}"
+      cp -f "${IMAGES_OPTIMIZED_FOLDER}/${FILE}" "${IMAGES_ASSETS_FOLDER}"
     done < <(echo "${FILES_KEYWORD}")
     echo "    ]" >> "${OUTPUT_ASSETS_FILE_TMP}"
     FIRST_KEYWORD=0