diff --git a/configure.ac b/configure.ac
index a7287d057a..ed4ddfbf60 100644
--- a/configure.ac
+++ b/configure.ac
@@ -510,8 +510,6 @@ case $host_cpu in
   arm*)
     build_cpu_neon="yes"
     AC_MSG_CHECKING([whether to use NEON instructions])
-    CFLAGS_save="${CFLAGS}"
-    CFLAGS="${CFLAGS} -mfpu=neon"
     AC_TRY_COMPILE([#include <arm_neon.h>],
        [asm volatile ("vqadd.u8 d0, d1, d0\n")],
        [
@@ -523,7 +521,6 @@ case $host_cpu in
         AC_MSG_RESULT([no])
         build_cpu_neon="no"
        ])
-    CFLAGS="${CFLAGS_save}"
     ;;
 esac