浏览代码

Fraction.ts: round GCD, refactor delta to scientific notation (#880)

polish for PR #880
sschmid 4 年之前
父节点
当前提交
62a7b57d95
共有 1 个文件被更改,包括 2 次插入2 次删除
  1. 2 2
      src/Common/DataObjects/Fraction.ts

+ 2 - 2
src/Common/DataObjects/Fraction.ts

@@ -64,7 +64,7 @@ export class Fraction {
       return 1;
       return 1;
     }
     }
 
 
-    while (Math.abs(b) > 0.0000001) {
+    while (Math.abs(b) > 1e-8) { // essentially b > 0, accounts for floating point inaccuracies (0.0000...)
       if (a > b) {
       if (a > b) {
         a -= b;
         a -= b;
       } else {
       } else {
@@ -72,7 +72,7 @@ export class Fraction {
       }
       }
     }
     }
 
 
-    return a;
+    return Math.round(a); // prevent returning 4.000001 or something, though it doesn't happen for our samples
   }
   }
 
 
   /**
   /**