visual_regression.sh 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255
  1. #!/bin/bash
  2. # This script runs a visual regression test on all the images
  3. # generated from OSMD samples (npm run generate:current and npm run generate:blessed)
  4. #
  5. # inspired by and adapted from Vexflow's visual regression tests.
  6. #
  7. # Prerequisites: ImageMagick
  8. #
  9. # On OSX: $ brew install imagemagick
  10. # On Linux: $ apt-get install imagemagick
  11. #
  12. # Usage:
  13. #
  14. #
  15. # First generate the known good or previous state PNG images you want to compare to, e.g. the develop branch or last release:
  16. # (Server has to be running for this: npm start)
  17. #
  18. # npm run generate:blessed
  19. #
  20. # Make changes in OSMD, then generate your new images:
  21. #
  22. # npm run generate:current
  23. #
  24. # Run the regression tests against the blessed images in tests/blessed.
  25. #
  26. # npm run test:visual
  27. # # npm will navigate to the base folder automatically
  28. #
  29. # # or: (this should be done from the main OSMD folder)
  30. # # sh test/Util/visual_regression.sh
  31. #
  32. # Check visual_regression/diff/results.txt for results. This file is sorted
  33. # by PHASH difference (most different files on top.) The composite diff
  34. # images for failed tests (i.e., PHASH > 1.0) are stored in visual_regression/diff.
  35. #
  36. # If you are satisfied with the differences, copy *.png from visual_regression/current
  37. # into visual_regression/blessed, and submit your change.
  38. # PNG viewer on OSX. Switch this to whatever your system uses.
  39. # VIEWER=open
  40. # Show images over this PHASH threshold. This is probably too low, but
  41. # a good first pass.
  42. THRESHOLD=0.01
  43. # Directories. You might want to change BASE, if you're running from a
  44. # different working directory. (only necessary if you're running the script manually, i.e. not from npm run test:visual)
  45. if [ "$1" == "" ]
  46. then
  47. BASEFOLDER="./visual_regression"
  48. else
  49. BASEFOLDER=$1
  50. fi
  51. BLESSED=$BASEFOLDER/blessed
  52. CURRENT=$BASEFOLDER/current
  53. DIFF=$BASEFOLDER/diff
  54. # All results are stored here.
  55. RESULTS=$DIFF/results.txt
  56. WARNINGS=$DIFF/warnings.txt
  57. # If no prefix is provided, test all images.
  58. if [ "$2" == "" ]
  59. then
  60. files=*.png
  61. else
  62. files=$2*.png
  63. fi
  64. # some sanity checks: check if some png images are in the right folder and warn if not. doesn't make sure there are actual, correct png images though.
  65. folderWarningStringMsg="Also, please run this npm script from the base OSMD folder. (npm run test:visual should do this automatically)
  66. Exiting without running visual regression tests.\n"
  67. # check if current directory exists / started from base OSMD folder
  68. if [ ! -e "$CURRENT" ]
  69. then
  70. printf "Warning: directory ${CURRENT} missing. Please run npm run generate:current (and if necessary npm run generate:blessed) first.
  71. ${folderWarningStringMsg}"
  72. exit 1
  73. fi
  74. # check if blessed directory exists / started from base OSMD folder
  75. if [ ! -e "$BLESSED" ]
  76. then
  77. printf "Warning: directory ${BLESSED} missing. Please run npm run generate:blessed first (or otherwise get the blessed images).
  78. ${folderWarningStringMsg}"
  79. exit 1
  80. fi
  81. # note: ls returns errors if the directory doesn't exist
  82. totalCurrentImages=`ls -l $CURRENT/*.png | wc -l | sed 's/[[:space:]]//g'`
  83. totalBlessedImages=`ls -l $BLESSED/*.png | wc -l | sed 's/[[:space:]]//g'`
  84. # check if there are some current images
  85. if [ "$totalCurrentImages" -lt 1 ]
  86. then
  87. printf "Warning: Found no pngs in ${CURRENT}. Please run npm run generate (and if necessary npm run blessed) first.
  88. ${folderWarningStringMsg}"
  89. exit 1
  90. fi
  91. # check if there are some blessed images
  92. if [ "$totalBlessedImages" -lt 1 ]
  93. then
  94. printf "Warning: Found no pngs in ${BLESSED}. Please run npm run blessed first, ideally on the repository's develop state.
  95. ${folderWarningStringMsg}"
  96. exit 1
  97. fi
  98. # check that #currentImages == #blessedImages (will continue anyways)
  99. if [ ! "$totalCurrentImages" -eq "$totalBlessedImages" ]
  100. then
  101. printf "Warning: Number of current images (${totalCurrentImages}) is not the same as blessed images (${totalBlessedImages}). Continuing anyways.\n"
  102. else
  103. printf "Found ${totalCurrentImages} current and ${totalBlessedImages} blessed png files (not tested if valid). Continuing.\n"
  104. fi
  105. mkdir -p $DIFF
  106. if [ -e "$RESULTS" ]
  107. then
  108. rm $DIFF/*
  109. fi
  110. touch $RESULTS
  111. touch $RESULTS.pass
  112. touch $RESULTS.fail
  113. touch $WARNINGS
  114. # Number of simultaneous jobs
  115. nproc=$(sysctl -n hw.physicalcpu 2> /dev/null || nproc)
  116. if [ -n "$NPROC" ]; then
  117. nproc=$NPROC
  118. fi
  119. total=`ls -l $BLESSED/$files | wc -l | sed 's/[[:space:]]//g'`
  120. echo "Running $total tests with threshold $THRESHOLD (nproc=$nproc)..."
  121. function ProgressBar {
  122. let _progress=(${1}*100/${2}*100)/100
  123. let _done=(${_progress}*4)/10
  124. let _left=40-$_done
  125. _fill=$(printf "%${_done}s")
  126. _empty=$(printf "%${_left}s")
  127. printf "\rProgress : [${_fill// /#}${_empty// /-}] ${_progress}%%"
  128. }
  129. function diff_image() {
  130. local image=$1
  131. local name=`basename $image .png`
  132. local blessed=$BLESSED/$name.png
  133. local current=$CURRENT/$name.png
  134. local diff=$current-temp
  135. if [ ! -e "$current" ]
  136. then
  137. echo "Warning: $name.png missing in $CURRENT." >$diff.warn
  138. return
  139. fi
  140. if [ ! -e "$blessed" ]
  141. then
  142. return
  143. fi
  144. cp $blessed $diff-a.png
  145. cp $current $diff-b.png
  146. # Calculate the difference metric and store the composite diff image.
  147. local hash=`compare -metric PHASH -highlight-color '#ff000050' $diff-b.png $diff-a.png $diff-diff.png 2>&1`
  148. local isGT=`echo "$hash > $THRESHOLD" | bc -l`
  149. if [ "$isGT" == "1" ]
  150. then
  151. # Add the result to results.text
  152. echo $name $hash >$diff.fail
  153. # Threshold exceeded, save the diff and the original, current
  154. cp $diff-diff.png $DIFF/$name.png
  155. cp $diff-a.png $DIFF/$name'_'Blessed.png
  156. cp $diff-b.png $DIFF/$name'_'Current.png
  157. echo
  158. echo "Test: $name"
  159. echo " PHASH value exceeds threshold: $hash > $THRESHOLD"
  160. echo " Image diff stored in $DIFF/$name.png"
  161. # $VIEWER "$diff-diff.png" "$diff-a.png" "$diff-b.png"
  162. # echo 'Hit return to process next image...'
  163. # read
  164. else
  165. echo $name $hash >$diff.pass
  166. fi
  167. rm -f $diff-a.png $diff-b.png $diff-diff.png
  168. }
  169. function wait_jobs () {
  170. local n=$1
  171. while [[ "$(jobs -r | wc -l)" -ge "$n" ]] ; do
  172. # echo ===================================== && jobs -lr
  173. # wait the oldest job.
  174. local pid_to_wait=`jobs -rp | head -1`
  175. # echo wait $pid_to_wait
  176. wait $pid_to_wait &> /dev/null
  177. done
  178. }
  179. count=0
  180. for image in $CURRENT/$files
  181. do
  182. count=$((count + 1))
  183. ProgressBar ${count} ${total}
  184. wait_jobs $nproc
  185. diff_image $image &
  186. done
  187. wait
  188. cat $CURRENT/*.warn 1>$WARNINGS 2>/dev/null
  189. rm -f $CURRENT/*.warn
  190. ## Check for files newly built that are not yet blessed.
  191. for image in $CURRENT/$files
  192. do
  193. name=`basename $image .png`
  194. blessed=$BLESSED/$name.png
  195. current=$CURRENT/$name.png
  196. if [ ! -e "$blessed" ]
  197. then
  198. echo " Warning: $name.png missing in $BLESSED." >>$WARNINGS
  199. fi
  200. done
  201. num_warnings=`cat $WARNINGS | wc -l`
  202. cat $CURRENT/*.fail 1>$RESULTS.fail 2>/dev/null
  203. num_fails=`cat $RESULTS.fail | wc -l`
  204. rm -f $CURRENT/*.fail
  205. # Sort results by PHASH
  206. sort -r -n -k 2 $RESULTS.fail >$RESULTS
  207. sort -r -n -k 2 $CURRENT/*.pass 1>>$RESULTS 2>/dev/null
  208. rm -f $CURRENT/*.pass $RESULTS.fail $RESULTS.pass
  209. echo
  210. echo Results stored in $DIFF/results.txt
  211. echo All images with a difference over threshold, $THRESHOLD, are
  212. echo available in $DIFF, sorted by perceptual hash.
  213. echo
  214. if [ "$num_warnings" -gt 0 ]
  215. then
  216. echo
  217. echo "You have $num_warnings warning(s):"
  218. cat $WARNINGS
  219. fi
  220. if [ "$num_fails" -gt 0 ]
  221. then
  222. echo "You have $num_fails fail(s):"
  223. head -n $num_fails $RESULTS
  224. else
  225. echo "Success - All diffs under threshold!"
  226. fi