Add script to calculate percentage of translation (#1826)

* add script to calculate percentage of translation

* test translation change

* change translation

* test

* change translation

* Calculate percentages of each translation file

* test

* Calculate percentages of each translation file

* change translation

* test

* test

* Calculate percentages of each translation file

* test

* Calculate percentages of each translation file

* fix workflow

* test

* test again

* Calculate percentages of each translation file

* Calculate percentages of each translation file

* test

* refactor

* change build logic

* fix types, move English first

* docs added

* test translation file

* test

* test

* test

* test

* test

* test

* test

* test

* test

* test

* test

* test

* Calculate percentages of each translation file

* let this be the final test please

* Calculate percentages of each translation file

* test

* test

* Test

* Calculate percentages of each translation file

* test

* Calculate percentages of each translation file

* test

* Calculate percentages of each translation file

* test

* Auto commit: Calculate translation coverage

* test

* test

* test

* test

* Auto commit: Calculate translation coverage

* test

* only on master

* test

* test

* Auto commit: Calculate translation coverage

* switch to master branch

Co-authored-by: i18n automation <runner@fv-az76.2iswp1o5zimezclxzdlwqia2gf.cx.internal.cloudapp.net>
Co-authored-by: i18n automation <runner@fv-az129.idlktykl4ure3gqe2lnji05orb.cx.internal.cloudapp.net>
Co-authored-by: i18n automation <runner@fv-az76.pjgcdo5npjpenpqz2nk0ztqvxd.cx.internal.cloudapp.net>
Co-authored-by: i18n automation <runner@fv-az33.senarqq4ucbulg04aytwntvgah.cx.internal.cloudapp.net>
Co-authored-by: i18n automation <runner@fv-az51.icvemaqob4xunfekbtdiz2tu2c.cx.internal.cloudapp.net>
Co-authored-by: i18n automation <runner@fv-az78.gikxu4m3dpiulftj3bftpuu3ee.cx.internal.cloudapp.net>
Co-authored-by: i18n automation <runner@fv-az121.cqdewbghluceforu5pkvpnveec.cx.internal.cloudapp.net>
Co-authored-by: i18n automation <runner@fv-az139.jsbds1i2htye3fh1bzwbe4ugmf.cx.internal.cloudapp.net>
Co-authored-by: i18n automation <runner@fv-az50.0bg2cysi0dkefjvuua0a0kbd1h.cx.internal.cloudapp.net>
Co-authored-by: i18n automation <runner@fv-az51.nhi3in4tbx4ehjtltcwuwbwsua.cx.internal.cloudapp.net>

.eslintignore

@@ -3,3 +3,4 @@ build/
 package-lock.json
 .vscode/
 firebase/
+src/locales/percentages.json

.github/workflows/locales-coverage.yml

@@ -0,0 +1,36 @@
+name: Build locales percentages
+
+on:
+  push:
+    branches:
+      - "master"
+    paths:
+      - "src/locales/**.json"
+      - "!src/locales/percentages.json"
+
+jobs:
+  locales:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          token: ${{ secrets.PUSH_TRANSLATIONS_COVERAGE_PAT }}
+
+      - name: Setup Node.js 12.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+
+      - name: Create report file
+        run: |
+          npm run locales-coverage
+          FILE_CHANGED=$(git diff src/locales/percentages.json)
+          if [ ! -z "${FILE_CHANGED}" ]; then
+            git config --global user.name 'Kostas Bariotis'
+            git config --global user.email 'konmpar@gmail.com'
+            git add src/locales/percentages.json
+            git commit -am "Auto commit: Calculate translation coverage"
+            git pull origin master --rebase
+            git push
+          fi

.prettierignore

@@ -0,0 +1 @@
+src/locales/percentages.json

README.md

@@ -80,6 +80,8 @@ Pull requests are welcome. For major changes, please [open an issue](https://git
 
 To translate Excalidraw into other languages, please visit [our Crowdin page](https://crowdin.com/project/excalidraw). To add a new language, [open an issue](https://github.com/excalidraw/excalidraw/issues/new) so we can get things set up on our end first.
 
+Translations will be available on the app if they exceed a certain threshold of completion (currently 85%).
+
 ## Excalidraw is built using these awesome tools
 
 - [React](https://reactjs.org)

package.json

@@ -72,24 +72,25 @@
   },
   "private": true,
   "scripts": {
-    "build": "npm run build:app && npm run build:zip",
     "build-node": "node ./scripts/build-node.js",
-    "build:app": "REACT_APP_INCLUDE_GTAG=true REACT_APP_GIT_SHA=$NOW_GITHUB_COMMIT_SHA react-scripts build",
     "build:app:docker": "REACT_APP_INCLUDE_GTAG=false REACT_APP_DISABLE_SENTRY=true react-scripts build",
+    "build:app": "REACT_APP_INCLUDE_GTAG=true REACT_APP_GIT_SHA=$NOW_GITHUB_COMMIT_SHA react-scripts build",
     "build:zip": "node ./scripts/build-version.js",
+    "build": "npm run build:app && npm run build:zip",
     "eject": "react-scripts eject",
-    "fix": "npm run fix:other && npm run fix:code",
     "fix:code": "npm run test:code -- --fix",
     "fix:other": "npm run prettier -- --write",
+    "fix": "npm run fix:other && npm run fix:code",
+    "locales-coverage": "node scripts/build-locales-coverage.js",
     "prettier": "prettier \"**/*.{css,scss,json,md,html,yml}\" --ignore-path=.eslintignore",
     "start": "react-scripts start",
-    "test": "npm run test:app",
     "test:all": "npm run test:typecheck && npm run test:code && npm run test:other && npm run test:app -- --watchAll=false",
-    "test:update": "npm run test:app -- --updateSnapshot --watchAll=false",
     "test:app": "react-scripts test --env=jsdom-fourteen --passWithNoTests",
     "test:code": "eslint --max-warnings=0 --ignore-path .gitignore --ext .js,.ts,.tsx .",
     "test:debug": "react-scripts --inspect-brk test --runInBand --no-cache",
     "test:other": "npm run prettier -- --list-different",
-    "test:typecheck": "tsc"
+    "test:typecheck": "tsc",
+    "test:update": "npm run test:app -- --updateSnapshot --watchAll=false",
+    "test": "npm run test:app"
   }
 }

scripts/build-locales-coverage.js

@@ -0,0 +1,32 @@
+const { readdirSync, writeFileSync } = require("fs");
+const files = readdirSync(`${__dirname}/../src/locales`);
+
+const flatten = (object) =>
+  Object.keys(object).reduce(
+    (initial, current) => ({ ...initial, ...object[current] }),
+    {},
+  );
+
+const locales = files.filter(
+  (file) => file !== "README.md" && file !== "percentages.json",
+);
+
+const percentages = {};
+
+for (let index = 0; index < locales.length; index++) {
+  const currentLocale = locales[index];
+  const data = flatten(require(`${__dirname}/../src/locales/${currentLocale}`));
+
+  const allKeys = Object.keys(data);
+  const translatedKeys = allKeys.filter((item) => data[item] !== "");
+
+  const percentage = (100 * translatedKeys.length) / allKeys.length;
+
+  percentages[currentLocale.replace(".json", "")] = parseInt(percentage);
+}
+
+writeFileSync(
+  `${__dirname}/../src/locales/percentages.json`,
+  JSON.stringify(percentages),
+  "utf8",
+);

src/i18n.ts

@@ -1,9 +1,18 @@
 import LanguageDetector from "i18next-browser-languagedetector";
 
 import fallbackLanguageData from "./locales/en.json";
+import percentages from "./locales/percentages.json";
 
-export const languages = [
-  { lng: "en", label: "English", data: "en.json" },
+const COMPLETION_THRESHOLD_TO_EXCEED = 85;
+
+interface Language {
+  lng: string;
+  label: string;
+  data: string;
+  rtl?: boolean;
+}
+
+const allLanguages: Language[] = [
   { lng: "bg-BG", label: "Български", data: "bg-BG.json" },
   { lng: "de-DE", label: "Deutsch", data: "de-DE.json" },
   { lng: "es-ES", label: "Español", data: "es-ES.json" },
@@ -30,6 +39,18 @@ export const languages = [
   { lng: "he-IL", label: "עברית", data: "he-IL.json", rtl: true },
 ];
 
+export const languages: Language[] = [
+  { lng: "en", label: "English", data: "en.json" },
+]
+  .concat(
+    allLanguages.sort((left, right) => (left.label > right.label ? 1 : -1)),
+  )
+  .filter(
+    (lang) =>
+      (percentages as Record<string, number>)[lang.lng] >
+      COMPLETION_THRESHOLD_TO_EXCEED,
+  );
+
 let currentLanguage = languages[0];
 let currentLanguageData = {};
 const fallbackLanguage = languages[0];

src/locales/README.md

@@ -1,5 +1,14 @@
+## How to contribute
+
 Please do not contribute changes directly to these files, as we manage them with Crowdin. Instead:
 
 - to request a new translation, [open an issue](https://github.com/excalidraw/excalidraw/issues/new/choose).
 - to update existing translations, [edit them on Crowdin](https://crowdin.com/translate/excalidraw/10)
   and we should have them included in the app soon!
+
+## Completion of translation
+
+[percentages.json](./percentages.json) holds a percentage of completion for each language. We generate these
+automatically [on build time](./../../.github/workflows/locales-coverage.yml) when a new translation PR appears.
+
+We only make a language available on the app if it exceeds a certain threshold of completion.

src/locales/percentages.json

@@ -0,0 +1 @@
+{"ar-SA":57,"bg-BG":81,"ca-ES":92,"de-DE":99,"el-GR":98,"en":100,"es-ES":97,"fa-IR":99,"fi-FI":100,"fr-FR":99,"he-IL":94,"hi-IN":99,"hu-HU":58,"id-ID":59,"it-IT":97,"ja-JP":77,"ko-KR":72,"nb-NO":99,"nl-NL":85,"pl-PL":99,"pt-PT":100,"ru-RU":81,"sq-AL":42,"tr-TR":98,"uk-UA":99,"zh-CN":99,"zh-TW":99}