Copy udivmodti4 from compiler-builtins
Division with remainder on u128 is badly optimized by LLVM. Copying it into our crate allows for inlining and proper optimization.
This commit is contained in:
parent
e9069ce1f1
commit
1d85a0c5f9
31
src/lib.rs
31
src/lib.rs
|
@ -10,6 +10,9 @@
|
||||||
|
|
||||||
#![cfg_attr(feature = "i128", feature(i128_type, i128))]
|
#![cfg_attr(feature = "i128", feature(i128_type, i128))]
|
||||||
|
|
||||||
|
#[cfg(feature = "i128")]
|
||||||
|
mod udiv128;
|
||||||
|
|
||||||
use std::{io, mem, ptr, slice};
|
use std::{io, mem, ptr, slice};
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
|
@ -37,7 +40,19 @@ const MAX_LEN: usize = 40; // i128::MIN (including minus sign)
|
||||||
// Adaptation of the original implementation at
|
// Adaptation of the original implementation at
|
||||||
// https://github.com/rust-lang/rust/blob/b8214dc6c6fc20d0a660fb5700dca9ebf51ebe89/src/libcore/fmt/num.rs#L188-L266
|
// https://github.com/rust-lang/rust/blob/b8214dc6c6fc20d0a660fb5700dca9ebf51ebe89/src/libcore/fmt/num.rs#L188-L266
|
||||||
macro_rules! impl_Integer {
|
macro_rules! impl_Integer {
|
||||||
($($t:ident),* as $conv_fn:ident) => ($(
|
($($t:ident),* as $conv_fn:ident) =>
|
||||||
|
(impl_Integer!(
|
||||||
|
$($t),* as $conv_fn,
|
||||||
|
(|n:$conv_fn, d:$conv_fn, rem:Option<&mut $conv_fn>| {
|
||||||
|
match rem {
|
||||||
|
Some(rem) => *rem = n % d,
|
||||||
|
_ => {},
|
||||||
|
}
|
||||||
|
n / d
|
||||||
|
})
|
||||||
|
););
|
||||||
|
|
||||||
|
($($t:ident),* as $conv_fn:ident, $divmod:expr) => ($(
|
||||||
impl Integer for $t {
|
impl Integer for $t {
|
||||||
fn write<W: io::Write>(self, mut wr: W) -> io::Result<usize> {
|
fn write<W: io::Write>(self, mut wr: W) -> io::Result<usize> {
|
||||||
let mut buf = unsafe { mem::uninitialized() };
|
let mut buf = unsafe { mem::uninitialized() };
|
||||||
|
@ -65,11 +80,13 @@ macro_rules! impl_Integer {
|
||||||
// eagerly decode 4 characters at a time
|
// eagerly decode 4 characters at a time
|
||||||
if <$t>::max_value() as u64 >= 10000 {
|
if <$t>::max_value() as u64 >= 10000 {
|
||||||
while n >= 10000 {
|
while n >= 10000 {
|
||||||
let rem = (n % 10000) as isize;
|
let mut rem = 0;
|
||||||
n /= 10000;
|
// division with remainder on u128 is badly optimized by LLVM.
|
||||||
|
// see “udiv128.rs” for more info.
|
||||||
|
n = $divmod(n, 10000, Some(&mut rem));
|
||||||
|
|
||||||
let d1 = (rem / 100) << 1;
|
let d1 = (rem as isize / 100) << 1;
|
||||||
let d2 = (rem % 100) << 1;
|
let d2 = (rem as isize % 100) << 1;
|
||||||
curr -= 4;
|
curr -= 4;
|
||||||
ptr::copy_nonoverlapping(lut_ptr.offset(d1), buf_ptr.offset(curr), 2);
|
ptr::copy_nonoverlapping(lut_ptr.offset(d1), buf_ptr.offset(curr), 2);
|
||||||
ptr::copy_nonoverlapping(lut_ptr.offset(d2), buf_ptr.offset(curr + 2), 2);
|
ptr::copy_nonoverlapping(lut_ptr.offset(d2), buf_ptr.offset(curr + 2), 2);
|
||||||
|
@ -117,5 +134,5 @@ impl_Integer!(isize, usize as u16);
|
||||||
impl_Integer!(isize, usize as u32);
|
impl_Integer!(isize, usize as u32);
|
||||||
#[cfg(target_pointer_width = "64")]
|
#[cfg(target_pointer_width = "64")]
|
||||||
impl_Integer!(isize, usize as u64);
|
impl_Integer!(isize, usize as u64);
|
||||||
#[cfg(feature = "i128")]
|
#[cfg(all(feature = "i128"))]
|
||||||
impl_Integer!(i128, u128 as u128);
|
impl_Integer!(i128, u128 as u128, udiv128::udivmodti4);
|
||||||
|
|
|
@ -0,0 +1,225 @@
|
||||||
|
// Copyright 2009-2016 compiler-builtins Developers
|
||||||
|
//
|
||||||
|
// The compiler-builtins crate is dual licensed under both the University of
|
||||||
|
// Illinois "BSD-Like" license and the MIT license. As a user of this code you may
|
||||||
|
// choose to use it under either license. As a contributor, you agree to allow
|
||||||
|
// your code to be used under both.
|
||||||
|
//
|
||||||
|
// Full text of the relevant licenses is found here:
|
||||||
|
// https://github.com/rust-lang-nursery/compiler-builtins/blob/master/LICENSE.TXT
|
||||||
|
//
|
||||||
|
//
|
||||||
|
//
|
||||||
|
// The following code is based on Rust’s [compiler-builtins crate]
|
||||||
|
// (https://github.com/rust-lang-nursery/compiler-builtins) which
|
||||||
|
// provides runtime functions for the Rust programs. The Rust
|
||||||
|
// compiler will automatically link your programs against this crate.
|
||||||
|
//
|
||||||
|
// We copied the implementation of '__udivmodti4()' which is an intrinsic
|
||||||
|
// implementing division with remainder for architectures without 128-bit integer support.
|
||||||
|
// We have done this two reasons, to work around [bad optimization by LLVM]
|
||||||
|
// (https://github.com/rust-lang/rust/issues/44545) and to allow function
|
||||||
|
// inlining which doesn’t happen with the intrinsic.
|
||||||
|
|
||||||
|
const BITS: u32 = 128;
|
||||||
|
const BITS_HALF: u32 = 64;
|
||||||
|
|
||||||
|
trait LargeInt {
|
||||||
|
fn low(self) -> u64;
|
||||||
|
fn high(self) -> u64;
|
||||||
|
fn from_parts(low: u64, high: u64) -> Self;
|
||||||
|
}
|
||||||
|
|
||||||
|
trait Int {
|
||||||
|
fn aborting_div(self, other: Self) -> Self;
|
||||||
|
fn aborting_rem(self, other: Self) -> Self;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LargeInt for u128 {
|
||||||
|
fn low(self) -> u64 {
|
||||||
|
self as u64
|
||||||
|
}
|
||||||
|
|
||||||
|
fn high(self) -> u64 {
|
||||||
|
(self >> 64) as u64
|
||||||
|
}
|
||||||
|
|
||||||
|
fn from_parts(low: u64, high: u64) -> u128 {
|
||||||
|
low as u128 | ((high as u128) << 64)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Int for u64 {
|
||||||
|
fn aborting_div(self, other: u64) -> u64 {
|
||||||
|
<u64>::checked_div(self, other).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn aborting_rem(self, other: u64) -> u64 {
|
||||||
|
<u64>::checked_rem(self, other).unwrap()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn udivmodti4(n: u128, d: u128, rem: Option<&mut u128>) -> u128 {
|
||||||
|
// NOTE X is unknown, K != 0
|
||||||
|
if n.high() == 0 {
|
||||||
|
if d.high() == 0 {
|
||||||
|
// 0 X
|
||||||
|
// ---
|
||||||
|
// 0 X
|
||||||
|
|
||||||
|
if let Some(rem) = rem {
|
||||||
|
*rem = <u128>::from(n.low().aborting_rem(d.low()));
|
||||||
|
}
|
||||||
|
return <u128>::from(n.low().aborting_div(d.low()))
|
||||||
|
} else {
|
||||||
|
// 0 X
|
||||||
|
// ---
|
||||||
|
// K X
|
||||||
|
if let Some(rem) = rem {
|
||||||
|
*rem = n;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut sr;
|
||||||
|
let mut q;
|
||||||
|
let mut r;
|
||||||
|
|
||||||
|
if d.low() == 0 {
|
||||||
|
if d.high() == 0 {
|
||||||
|
// K X
|
||||||
|
// ---
|
||||||
|
// 0 0
|
||||||
|
// NOTE This should be unreachable in safe Rust because the program will panic before
|
||||||
|
// this intrinsic is called
|
||||||
|
unreachable!();
|
||||||
|
}
|
||||||
|
|
||||||
|
if n.low() == 0 {
|
||||||
|
// K 0
|
||||||
|
// ---
|
||||||
|
// K 0
|
||||||
|
if let Some(rem) = rem {
|
||||||
|
*rem = <u128>::from_parts(0, n.high().aborting_rem(d.high()));
|
||||||
|
}
|
||||||
|
return <u128>::from(n.high().aborting_div(d.high()))
|
||||||
|
}
|
||||||
|
|
||||||
|
// K K
|
||||||
|
// ---
|
||||||
|
// K 0
|
||||||
|
|
||||||
|
if d.high().is_power_of_two() {
|
||||||
|
if let Some(rem) = rem {
|
||||||
|
*rem = <u128>::from_parts(n.low(), n.high() & (d.high() - 1));
|
||||||
|
}
|
||||||
|
return <u128>::from(n.high() >> d.high().trailing_zeros());
|
||||||
|
}
|
||||||
|
|
||||||
|
sr = d.high().leading_zeros().wrapping_sub(n.high().leading_zeros());
|
||||||
|
|
||||||
|
// D > N
|
||||||
|
if sr > BITS_HALF - 2 {
|
||||||
|
if let Some(rem) = rem {
|
||||||
|
*rem = n;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
sr += 1;
|
||||||
|
|
||||||
|
// 1 <= sr <= BITS_HALF - 1
|
||||||
|
q = n << (BITS - sr);
|
||||||
|
r = n >> sr;
|
||||||
|
} else if d.high() == 0 {
|
||||||
|
// K X
|
||||||
|
// ---
|
||||||
|
// 0 K
|
||||||
|
if d.low().is_power_of_two() {
|
||||||
|
if let Some(rem) = rem {
|
||||||
|
*rem = <u128>::from(n.low() & (d.low() - 1));
|
||||||
|
}
|
||||||
|
|
||||||
|
if d.low() == 1 {
|
||||||
|
return n;
|
||||||
|
} else {
|
||||||
|
let sr = d.low().trailing_zeros();
|
||||||
|
return n >> sr;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
sr = 1 + BITS_HALF + d.low().leading_zeros() - n.high().leading_zeros();
|
||||||
|
|
||||||
|
// 2 <= sr <= u64::BITS - 1
|
||||||
|
q = n << (BITS - sr);
|
||||||
|
r = n >> sr;
|
||||||
|
} else {
|
||||||
|
// K X
|
||||||
|
// ---
|
||||||
|
// K K
|
||||||
|
sr = d.high().leading_zeros().wrapping_sub(n.high().leading_zeros());
|
||||||
|
|
||||||
|
// D > N
|
||||||
|
if sr > BITS_HALF - 1 {
|
||||||
|
if let Some(rem) = rem {
|
||||||
|
*rem = n;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
sr += 1;
|
||||||
|
|
||||||
|
// 1 <= sr <= BITS_HALF
|
||||||
|
q = n << (BITS - sr);
|
||||||
|
r = n >> sr;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Not a special case
|
||||||
|
// q and r are initialized with
|
||||||
|
// q = n << (u64::BITS - sr)
|
||||||
|
// r = n >> sr
|
||||||
|
// 1 <= sr <= u64::BITS - 1
|
||||||
|
let mut carry = 0;
|
||||||
|
|
||||||
|
// Don't use a range because they may generate references to memcpy in unoptimized code
|
||||||
|
let mut i = 0;
|
||||||
|
while i < sr {
|
||||||
|
i += 1;
|
||||||
|
|
||||||
|
// r:q = ((r:q) << 1) | carry
|
||||||
|
r = (r << 1) | (q >> (BITS - 1));
|
||||||
|
q = (q << 1) | carry as u128;
|
||||||
|
|
||||||
|
// carry = 0
|
||||||
|
// if r >= d {
|
||||||
|
// r -= d;
|
||||||
|
// carry = 1;
|
||||||
|
// }
|
||||||
|
let s = (d.wrapping_sub(r).wrapping_sub(1)) as i128 >> (BITS - 1);
|
||||||
|
carry = (s & 1) as u64;
|
||||||
|
r -= d & s as u128;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(rem) = rem {
|
||||||
|
*rem = r;
|
||||||
|
}
|
||||||
|
(q << 1) | carry as u128
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[test]
|
||||||
|
fn test_udivmodti4() {
|
||||||
|
let primes = [
|
||||||
|
3, 7, 31, 73, 127, 179, 233, 283, 353,
|
||||||
|
419, 467, 547, 607, 661, 739, 811, 877, 947,
|
||||||
|
];
|
||||||
|
|
||||||
|
for (i, d) in (0..128).cycle().zip(primes.iter().cycle()).take(1_000) {
|
||||||
|
let n = 1u128 << i;
|
||||||
|
let mut rem = 0;
|
||||||
|
let q = udivmodti4(n, *d, Some(&mut rem));
|
||||||
|
assert_eq!(q, n / d);
|
||||||
|
assert_eq!(rem, n % d);
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue