FLT_EPSILON for a nth root finder with SSE/AVX

Question

I'm trying to convert a function that finds the nth root in C for a double value from the following link http://rosettacode.org/wiki/Nth_root#C to find the nth root for 8 floats at once using AVX.

Part of that code uses DBL_EPSILON * 10. However, when I convert this to use float with AVX I have to use FLT_EPSILON*1000 or the code hangs and does not converge. When I print out FLT_EPSILON I see it is order 1E-7. But this link, http://www.cplusplus.com/reference/cfloat/ , says it should be 1E-5. When I print out DBL_EPSILON it's 1E-16 but the link says it should only be 1E-9. What's going on?

Here is the code so far (not fully optimized).

#include <stdio.h>
#include <float.h>
#include <immintrin.h>                 // AVX

inline double abs_(double x) { 
    return x >= 0 ? x : -x; 
}

double pow_(double x, int e)
{
    double ret = 1;
    for (ret = 1; e; x *= x, e >>= 1) {
        if ((e & 1)) ret *= x;
    }
    return ret;
}

double root(double a, int n)
{
    double d, x = 1;
    x = a/n;
    if (!a) return 0;
    //if (n < 1 || (a < 0 && !(n&1))) return 0./0.; /* NaN */

    int cnt = 0;
    do {
        cnt++;  
        d = (a / pow_(x, n - 1) - x) / n;
        x+= d;
    } while (abs_(d) >= abs_(x) * (DBL_EPSILON * 10));
    printf("%d\n", cnt);    

    return x;
}


__m256 pow_avx(__m256 x, int e) {
    __m256 ret = _mm256_set1_ps(1.0f);
    for (; e; x = _mm256_mul_ps(x,x), e >>= 1) {
        if ((e & 1)) ret = _mm256_mul_ps(x,ret);
    }
    return ret;
}

inline __m256 abs_avx (__m256 x) { 
    return _mm256_max_ps(_mm256_sub_ps(_mm256_setzero_ps(), x), x);
    //return x >= 0 ? x : -x; 
}

int get_mask(const __m256 d, const __m256 x) {
    __m256 ad = abs_avx(d);
    __m256 ax = abs_avx(x);
    __m256i mask = _mm256_castps_si256(_mm256_cmp_ps(ad, ax, _CMP_GT_OQ));
    return _mm_movemask_epi8(_mm256_castsi256_si128(mask)) + _mm_movemask_epi8(_mm256_extractf128_si256(mask,1));
}

__m256 root_avx(__m256 a, int n) {
    printf("%e\n", FLT_EPSILON);
    printf("%e\n", DBL_EPSILON);
    printf("%e\n", FLT_EPSILON*1000.0f);
    __m256 d;
    __m256 x = _mm256_set1_ps(1.0f);
    //if (!a) return 0;
    //if (n < 1 || (a < 0 && !(n&1))) return 0./0.; /* NaN */
    __m256 in = _mm256_set1_ps(1.0f/n);
    __m256 xtmp;
    do {
        d = _mm256_rcp_ps(pow_avx(x, n - 1));
        d = _mm256_sub_ps(_mm256_mul_ps(a,d),x);
        d = _mm256_mul_ps(d,in);
        //d = (a / pow_avx(x, n - 1) - x) / n;
        x = _mm256_add_ps(x, d); //x+= d;
        xtmp =_mm256_mul_ps(x, _mm256_set1_ps(FLT_EPSILON*100.0f));
    //} while (abs_(d) >= abs_(x) * (DBL_EPSILON * 10));
    } while (get_mask(d, xtmp)); 
    return x;
}

int main()
{
    __m256 d = _mm256_set1_ps(16.0f);
    __m256 out = root_avx(d, 4);
    float result[8];
    int i;
    _mm256_storeu_ps(result, out);

    for(i=0; i<8; i++) {
        printf("%f\n", result[i]);
    } printf("\n");

    //double x = 16;
    //printf("root(%g, 15) = %g\n", x, root(x, 4));

    //double x = pow_(-3.14159, 15);
    //printf("root(%g, 15) = %g\n", x, root(x, 15));
    return 0;
}

Answer 1

_mm256_rcp_ps, which maps to the rcpps instruction, performs only an approximate reciprocal. The Intel 64 and IA-32 Architectures Software Developer’s Manual says its relative error may be up to 1.5•2^-12. This is insufficient to cause the root finder to converge with accuracy 100*FLT_EPSILON.

You could use an exact division, such as:

d = pow_avx(x, n-1);
d = _mm256_sub_ps(_mm256_div_ps(a, d), x);

or add some refinement steps for the reciprocal estimate.

Incidentally, if your compiler supports using regular C operators with SIMD objects, consider using the regular C operators instead:

d = pow_avx(x, n-1);
d = a/d - x;

Answer 2

1e-5 is simply the maximum value the C standard allows an implementation to use for FLT_EPSILON. In practice, you'll be using IEEE-754 single-precision, which has an epsilon of 2^-23, which is approximately 1e-7.