lua: fix integer overflow in LNUM patch

Safely detect integer overflow in try_addint() and try_subint().
Old code relied on undefined behavior, and recent versions of GCC on x86
optimized away the if-statements.
This caused integer overflow in Lua code instead of falling back to
floating-point numbers.

Signed-off-by: Adam Bailey <aebailey@gmail.com>
(cherry picked from commit 3a2e7c30d3)
This commit is contained in:
Adam Bailey 2023-07-03 20:16:14 -05:00 committed by Hauke Mehrtens
parent 4a3c66a401
commit a5b03a34c3
2 changed files with 16 additions and 16 deletions

View File

@ -1600,18 +1600,18 @@
+ * (and doing them). + * (and doing them).
+ */ + */
+int try_addint( lua_Integer *r, lua_Integer ib, lua_Integer ic ) { +int try_addint( lua_Integer *r, lua_Integer ib, lua_Integer ic ) {
+ lua_Integer v= ib+ic; /* may overflow */ + /* Signed int overflow is undefined behavior, so catch it without causing it. */
+ if (ib>0 && ic>0) { if (v < 0) return 0; /*overflow, use floats*/ } + if (ic>0) { if (ib > LUA_INTEGER_MAX - ic) return 0; /*overflow, use floats*/ }
+ else if (ib<0 && ic<0) { if (v >= 0) return 0; } + else { if (ib < LUA_INTEGER_MIN - ic) return 0; }
+ *r= v; + *r = ib + ic;
+ return 1; + return 1;
+} +}
+ +
+int try_subint( lua_Integer *r, lua_Integer ib, lua_Integer ic ) { +int try_subint( lua_Integer *r, lua_Integer ib, lua_Integer ic ) {
+ lua_Integer v= ib-ic; /* may overflow */ + /* Signed int overflow is undefined behavior, so catch it without causing it. */
+ if (ib>=0 && ic<0) { if (v < 0) return 0; /*overflow, use floats*/ } + if (ic>0) { if (ib < LUA_INTEGER_MIN + ic) return 0; /*overflow, use floats*/ }
+ else if (ib<0 && ic>0) { if (v >= 0) return 0; } + else { if (ib > LUA_INTEGER_MAX + ic) return 0; }
+ *r= v; + *r = ib - ic;
+ return 1; + return 1;
+} +}
+ +

View File

@ -1589,18 +1589,18 @@
+ * (and doing them). + * (and doing them).
+ */ + */
+int try_addint( lua_Integer *r, lua_Integer ib, lua_Integer ic ) { +int try_addint( lua_Integer *r, lua_Integer ib, lua_Integer ic ) {
+ lua_Integer v= ib+ic; /* may overflow */ + /* Signed int overflow is undefined behavior, so catch it without causing it. */
+ if (ib>0 && ic>0) { if (v < 0) return 0; /*overflow, use floats*/ } + if (ic>0) { if (ib > LUA_INTEGER_MAX - ic) return 0; /*overflow, use floats*/ }
+ else if (ib<0 && ic<0) { if (v >= 0) return 0; } + else { if (ib < LUA_INTEGER_MIN - ic) return 0; }
+ *r= v; + *r = ib + ic;
+ return 1; + return 1;
+} +}
+ +
+int try_subint( lua_Integer *r, lua_Integer ib, lua_Integer ic ) { +int try_subint( lua_Integer *r, lua_Integer ib, lua_Integer ic ) {
+ lua_Integer v= ib-ic; /* may overflow */ + /* Signed int overflow is undefined behavior, so catch it without causing it. */
+ if (ib>=0 && ic<0) { if (v < 0) return 0; /*overflow, use floats*/ } + if (ic>0) { if (ib < LUA_INTEGER_MIN + ic) return 0; /*overflow, use floats*/ }
+ else if (ib<0 && ic>0) { if (v >= 0) return 0; } + else { if (ib > LUA_INTEGER_MAX + ic) return 0; }
+ *r= v; + *r = ib - ic;
+ return 1; + return 1;
+} +}
+ +