From 30957938b226d297583de69464f98593a47f4971 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Beno=C3=AEt=20Harrault?= <benoit@harrault.fr>
Date: Tue, 4 May 2021 22:21:51 +0200
Subject: [PATCH] Improve download/optimize images assets
---
.../{get_images.sh => 01_download_images.sh} | 21 +++------
scripts/02_prepare_images.sh | 19 ++++++++
scripts/03_optimize_images.sh | 44 +++++++++++++++++++
...assets_list.sh => 04_build_assets_list.sh} | 16 ++++---
scripts/cache/download/.gitkeep | 0
scripts/cache/optimized/.gitkeep | 0
scripts/cache/selected/.gitkeep | 0
7 files changed, 78 insertions(+), 22 deletions(-)
rename scripts/{get_images.sh => 01_download_images.sh} (80%)
create mode 100755 scripts/02_prepare_images.sh
create mode 100755 scripts/03_optimize_images.sh
rename scripts/{build_assets_list.sh => 04_build_assets_list.sh} (75%)
create mode 100644 scripts/cache/download/.gitkeep
create mode 100644 scripts/cache/optimized/.gitkeep
create mode 100644 scripts/cache/selected/.gitkeep
diff --git a/scripts/get_images.sh b/scripts/01_download_images.sh
similarity index 80%
rename from scripts/get_images.sh
rename to scripts/01_download_images.sh
index 5649b5d..09af291 100755
--- a/scripts/get_images.sh
+++ b/scripts/01_download_images.sh
@@ -1,26 +1,20 @@
#!/usr/bin/env bash
-command -v optipng >/dev/null 2>&1 || { echo >&2 "I require optipng but it's not installed. Aborting."; exit 1; }
command -v convert >/dev/null 2>&1 || { echo >&2 "I require convert (imagemagick) but it's not installed. Aborting."; exit 1; }
command -v jq >/dev/null 2>&1 || { echo >&2 "I require jq (json parser) but it's not installed. Aborting."; exit 1; }
CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
BASE_DIR="$(dirname "${CURRENT_DIR}")"
-IMAGES_CACHE_FOLDER="${CURRENT_DIR}/cache"
+IMAGES_CACHE_FOLDER="${CURRENT_DIR}/cache/download"
INPUT_WORDS_LIST="${BASE_DIR}/assets/files/words-fr.json"
-COUNT_BY_WEYWORD_VARIANT=3
-
+# Images variants
KEYWORD_VARIANTS=",image,picture,drawing,black and white,painting,icon"
+COUNT_BY_WEYWORD_VARIANT=3
KEYWORD_VARIANTS_LIST="$(echo "${KEYWORD_VARIANTS}" | tr "," "\n")"
-RESIZE_OPTION="128x128"
-CROP_PARAMETERS="-auto-orient -trim +repage -gravity center -background white -extent ${RESIZE_OPTION}^"
-CONVERT_OPTIONS="-alpha off +dither -colors 256 -depth 4"
-
-OPTIPNG_OPTIONS="-preserve -quiet -o7"
-
+# QWANT parameters
LANG="fr_FR"
TYPE="images"
OFFSET=0
@@ -71,9 +65,7 @@ while read -r KEYWORD; do
wget --quiet --timeout=10 "${URL}" -O "${TMP_IMAGE_FILE}"
if [[ -f "${TMP_IMAGE_FILE}" ]]; then
echo " + Converting..."
- convert "${TMP_IMAGE_FILE}" -resize "${RESIZE_OPTION}^" ${CROP_PARAMETERS} ${CONVERT_OPTIONS} "${OUTPUT_FILE}"
- echo " + Optimizing..."
- optipng ${OPTIPNG_OPTIONS} "${OUTPUT_FILE}"
+ convert "${TMP_IMAGE_FILE}" "${OUTPUT_FILE}"
fi
if [[ -f "${TMP_IMAGE_FILE}" ]]; then
@@ -85,6 +77,3 @@ while read -r KEYWORD; do
done < <(echo "${KEYWORD_VARIANTS_LIST}")
fi
done < <(echo "${WORDS_LIST}")
-
-# Build assets list
-"${CURRENT_DIR}/build_assets_list.sh"
diff --git a/scripts/02_prepare_images.sh b/scripts/02_prepare_images.sh
new file mode 100755
index 0000000..caaa392
--- /dev/null
+++ b/scripts/02_prepare_images.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+IMAGES_CACHE_FOLDER="${CURRENT_DIR}/cache"
+
+IMAGES_DOWNLOAD_FOLDER="${CURRENT_DIR}/cache/download"
+IMAGES_SELECTED_FOLDER="${CURRENT_DIR}/cache/download"
+
+# delete empty/temp files
+find "${IMAGES_CACHE_FOLDER}" -type f -name "*.png" -empty -exec rm {} \;
+find "${IMAGES_CACHE_FOLDER}" -type f -name "*.tmp*" -exec rm {} \;
+find "${IMAGES_CACHE_FOLDER}" -type d -empty -exec rm -rf {} \;
+
+# copy missing files from "download" to "selected"
+rsync --ignore-existing -raz --progress "${IMAGES_DOWNLOAD_FOLDER}/*" "${IMAGES_SELECTED_FOLDER}"
+
+echo "done: copy files from download to selected."
+echo "please remove unwanted files from selected folders."
diff --git a/scripts/03_optimize_images.sh b/scripts/03_optimize_images.sh
new file mode 100755
index 0000000..419bf99
--- /dev/null
+++ b/scripts/03_optimize_images.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+command -v optipng >/dev/null 2>&1 || { echo >&2 "I require optipng but it's not installed. Aborting."; exit 1; }
+command -v convert >/dev/null 2>&1 || { echo >&2 "I require convert (imagemagick) but it's not installed. Aborting."; exit 1; }
+
+CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+IMAGES_CACHE_FOLDER="${CURRENT_DIR}/cache"
+
+IMAGES_SELECTED_FOLDER="${CURRENT_DIR}/cache/download"
+IMAGES_OPTIMIZED_FOLDER="${CURRENT_DIR}/cache/optimized"
+
+RESIZE_OPTION="128x128"
+CROP_PARAMETERS="-auto-orient -trim +repage -gravity center -background white -extent ${RESIZE_OPTION}^"
+CONVERT_OPTIONS="-alpha off +dither -colors 256 -depth 4"
+
+OPTIPNG_OPTIONS="-preserve -quiet -o7"
+
+# delete empty/temp files
+find "${IMAGES_CACHE_FOLDER}" -type f -name "*.png" -empty -exec rm {} \;
+find "${IMAGES_CACHE_FOLDER}" -type f -name "*.tmp*" -exec rm {} \;
+find "${IMAGES_CACHE_FOLDER}" -type d -empty -exec rm -rf {} \;
+
+IMAGES="$(find "${IMAGES_SELECTED_FOLDER}" -type f -name "*.png" | sort)"
+
+while read -r INPUT_FILE; do
+ if [[ -n "${INPUT_FILE}" ]]; then
+ OUTPUT_FILE="$(echo "${INPUT_FILE}" | sed "s|^${IMAGES_SELECTED_FOLDER}/|${IMAGES_OPTIMIZED_FOLDER}/|g")"
+ echo " OUTPUT_FILE: ${OUTPUT_FILE}"
+
+ if [[ -f "${OUTPUT_FILE}" ]]; then
+ echo " - Already optimized"
+ else
+ mkdir -p "$(dirname ${OUTPUT_FILE})"
+
+ echo " + Converting..."
+ convert "${INPUT_FILE}" -resize "${RESIZE_OPTION}^" ${CROP_PARAMETERS} ${CONVERT_OPTIONS} "${OUTPUT_FILE}"
+ echo " + Optimizing..."
+ optipng ${OPTIPNG_OPTIONS} "${OUTPUT_FILE}"
+ fi
+ fi
+done < <(echo "${IMAGES}")
+
+echo "done."
diff --git a/scripts/build_assets_list.sh b/scripts/04_build_assets_list.sh
similarity index 75%
rename from scripts/build_assets_list.sh
rename to scripts/04_build_assets_list.sh
index 82527ca..1ec330e 100755
--- a/scripts/build_assets_list.sh
+++ b/scripts/04_build_assets_list.sh
@@ -11,16 +11,19 @@ OUTPUT_ASSETS_FILE="${ASSETS_BASE_FOLDER}/assets_images.json"
touch "${OUTPUT_ASSETS_FILE}"
IMAGES_CACHE_FOLDER="${CURRENT_DIR}/cache"
+IMAGES_OPTIMIZED_FOLDER="${IMAGES_CACHE_FOLDER}/optimized"
-# delete empty/temp files
-find "${IMAGES_CACHE_FOLDER}" -type f -name "*.png" -empty -exec rm {} \;
-find "${IMAGES_CACHE_FOLDER}" -type f -name "*.tmp*" -exec rm {} \;
-find "${IMAGES_CACHE_FOLDER}" -type d -empty -exec rm -rf {} \;
+echo "Cleaning empty/temp files..."
+find "${IMAGES_OPTIMIZED_FOLDER}" -type f -name "*.png" -empty -exec rm {} \;
+find "${IMAGES_OPTIMIZED_FOLDER}" -type f -name "*.tmp*" -exec rm {} \;
+find "${IMAGES_OPTIMIZED_FOLDER}" -type d -empty -exec rm -rf {} \;
+
+echo "Cleaning existing assets..."
find "${IMAGES_ASSETS_FOLDER}" -type f -name "*.png" -exec rm {} \;
echo "Building assets json file..."
-FILES="$(find "${IMAGES_CACHE_FOLDER}" -type f -name "*.png" | sed "s|^${IMAGES_CACHE_FOLDER}/||g" | sort)"
+FILES="$(find "${IMAGES_OPTIMIZED_FOLDER}" -type f -name "*.png" | sed "s|^${IMAGES_OPTIMIZED_FOLDER}/||g" | sort)"
KEYWORDS="$(echo "${FILES}" | cut -d'/' -f1 | sort | uniq)"
OUTPUT_ASSETS_FILE_TMP="${OUTPUT_ASSETS_FILE}.tmp"
@@ -29,6 +32,7 @@ echo " \"images\": {" >> "${OUTPUT_ASSETS_FILE_TMP}"
FIRST_KEYWORD=1
while read -r KEYWORD; do
if [[ -n "${KEYWORD}" ]]; then
+ echo "- ${KEYWORD}"
if [[ ${FIRST_KEYWORD} -eq 0 ]]; then
echo " ," >> "${OUTPUT_ASSETS_FILE_TMP}"
fi
@@ -41,7 +45,7 @@ while read -r KEYWORD; do
fi
echo " \"$(echo "${FILE}" | sed "s|${KEYWORD}/||g")\"" >> "${OUTPUT_ASSETS_FILE_TMP}"
FIRST_FILE=0
- cp -f "${IMAGES_CACHE_FOLDER}/${FILE}" "${IMAGES_ASSETS_FOLDER}"
+ cp -f "${IMAGES_OPTIMIZED_FOLDER}/${FILE}" "${IMAGES_ASSETS_FOLDER}"
done < <(echo "${FILES_KEYWORD}")
echo " ]" >> "${OUTPUT_ASSETS_FILE_TMP}"
FIRST_KEYWORD=0
diff --git a/scripts/cache/download/.gitkeep b/scripts/cache/download/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/scripts/cache/optimized/.gitkeep b/scripts/cache/optimized/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/scripts/cache/selected/.gitkeep b/scripts/cache/selected/.gitkeep
new file mode 100644
index 0000000..e69de29
--
GitLab