Let's use the direction cosines (dot product of unit vectors), not

the arbitrary-magnitude dot product, to classify regions (inside,
outside, coincident) of surfaces against each other.

That lets me always perturb the point for the normals (inside and
outside the edge) by just a chord tolerance, and nothing bad
happens as that distance varies over a few orders of magnitude.

[git-p4: depot-paths = "//depot/solvespace/": change = 1996]

dsc.h

@@ -67,6 +67,7 @@ public:
     Vector Minus(Vector b);
     Vector Negated(void);
     Vector Cross(Vector b);
+    double DirectionCosineWith(Vector b);
     double Dot(Vector b);
     Vector Normal(int which);
     Vector RotatedAbout(Vector orig, Vector axis, double theta);

srf/boolean.cpp

@@ -322,10 +322,6 @@ void SSurface::EdgeNormalsWithinSurface(Point2d auv, Point2d buv,
     Point2d enuv;
     enuv.x = enxyz.Dot(tu) / tu.MagSquared();
     enuv.y = enxyz.Dot(tv) / tv.MagSquared();
-    // Don't let the magnitude get too tiny at small chord tolerances; we
-    // will otherwise have numerical problems subtracting nearly-equal
-    // numbers.
-    enuv = enuv.WithMagnitude(0.01);
 
     // Compute the inner and outer normals of this edge (within the srf),
     // in xyz space. These are not necessarily antiparallel, if the

srf/raycast.cpp

@@ -7,8 +7,9 @@
 //-----------------------------------------------------------------------------
 #include "solvespace.h"
 
-// Dot product tolerance for perpendicular.
-const double SShell::DOTP_TOL = 1e-3;
+// Dot product tolerance for perpendicular; this is on the direction cosine,
+// so it's about 0.001 degrees.
+const double SShell::DOTP_TOL = 1e-5;
 
 extern int FLAG;
 
@@ -394,7 +395,7 @@ void SShell::AllPointsIntersecting(Vector a, Vector b,
 int SShell::ClassifyRegion(Vector edge_n, Vector inter_surf_n,
                            Vector edge_surf_n)
 {
-    double dot = inter_surf_n.Dot(edge_n);
+    double dot = inter_surf_n.DirectionCosineWith(edge_n);
     if(fabs(dot) < DOTP_TOL) {
         // The edge's surface and the edge-on-face surface
         // are coincident. Test the edge's surface normal
@@ -466,7 +467,7 @@ bool SShell::ClassifyEdge(int *indir, int *outdir,
         // TODO, make this use the appropriate curved normals
         double dotp[2];
         for(int i = 0; i < 2; i++) {
-            dotp[i] = edge_n_out.Dot(inter_surf_n[i]);
+            dotp[i] = edge_n_out.DirectionCosineWith(inter_surf_n[i]);
         }
 
         if(fabs(dotp[1]) < DOTP_TOL) {

util.cpp

@@ -362,6 +362,12 @@ double Vector::Dot(Vector b) {
     return (x*b.x + y*b.y + z*b.z);
 }
 
+double Vector::DirectionCosineWith(Vector b) {
+    Vector a = this->WithMagnitude(1);
+    b = b.WithMagnitude(1);
+    return a.Dot(b);
+}
+
 Vector Vector::Normal(int which) {
     Vector n;