More compiler hints

sailfish/src/runtime/buffer.rs

@@ -137,31 +137,32 @@ impl Buffer {
     }
 
     #[cfg_attr(feature = "perf-inline", inline)]
+    #[cold]
     fn reserve_internal(&mut self, size: usize) {
         unsafe {
             let new_capacity = std::cmp::max(self.capacity * 2, self.len + size);
-            self.data = self.realloc(new_capacity);
+            self.data = safe_realloc(self.data, self.capacity, new_capacity);
             self.capacity = new_capacity;
         }
     }
+}
 
-    #[cfg_attr(feature = "perf-inline", inline)]
-    unsafe fn realloc(&self, cap: usize) -> *mut u8 {
-        let data = if unlikely!(self.capacity == 0) {
-            let new_layout = Layout::from_size_align_unchecked(cap, 1);
-            alloc(new_layout)
-        } else {
-            debug_assert!(cap <= std::usize::MAX / 2, "capacity is too large");
-            let old_layout = Layout::from_size_align_unchecked(self.capacity, 1);
-            realloc(self.data, old_layout, cap)
-        };
+#[cold]
+unsafe fn safe_realloc(ptr: *mut u8, capacity: usize, new_capacity: usize) -> *mut u8 {
+    assert!(new_capacity <= std::usize::MAX / 2, "capacity is too large");
+    let data = if unlikely!(capacity == 0) {
+        let new_layout = Layout::from_size_align_unchecked(new_capacity, 1);
+        alloc(new_layout)
+    } else {
+        let old_layout = Layout::from_size_align_unchecked(new_capacity, 1);
+        realloc(ptr, old_layout, new_capacity)
+    };
 
-        if data.is_null() {
-            handle_alloc_error(Layout::from_size_align_unchecked(cap, 1));
-        }
-
-        data
+    if data.is_null() {
+        handle_alloc_error(Layout::from_size_align_unchecked(new_capacity, 1));
     }
+
+    data
 }
 
 impl Clone for Buffer {

sailfish/src/runtime/escape/mod.rs

@@ -58,7 +58,7 @@ fn escape(feed: &str, buf: &mut Buffer) {
 #[deprecated(since = "0.1.2", note = "This function does not anything any more")]
 pub fn register_escape_fn(_fun: fn(&str, &mut Buffer)) {}
 
-#[inline]
+#[cfg_attr(feature = "perf-inline", inline)]
 pub(crate) fn escape_to_buf(feed: &str, buf: &mut Buffer) {
     unsafe {
         if feed.len() < 16 {

sailfish/src/runtime/escape/naive.rs

@@ -100,8 +100,9 @@ pub(super) unsafe fn escape_small(feed: &str, mut buf: *mut u8) -> usize {
 }
 
 /// memcpy implementation based on glibc (https://github.molgen.mpg.de/git-mirror/glibc/blob/master/sysdeps/x86_64/multiarch/memcpy-avx-unaligned.S)
-#[inline]
+#[cfg_attr(feature = "perf-inline", inline)]
 unsafe fn memcpy_small(src: *const u8, dst: *mut u8, len: usize) {
+    debug_assert!(len <= 16);
     if len >= 8 {
         let tmp = ptr::read_unaligned(src as *const u64);
         ptr::write_unaligned(dst as *mut u64, tmp);

sailfish/src/runtime/render.rs

@@ -148,7 +148,7 @@ macro_rules! render_int {
     ($($int:ty),*) => {
         $(
             impl Render for $int {
-                #[inline]
+                #[cfg_attr(feature = "perf-inline", inline)]
                 fn render(&self, b: &mut Buffer) -> Result<(), RenderError> {
                     use super::integer::Integer;
 
@@ -175,7 +175,7 @@ macro_rules! render_int {
 render_int!(u8, u16, u32, u64, u128, i8, i16, i32, i64, i128, usize, isize);
 
 impl Render for f32 {
-    #[inline]
+    #[cfg_attr(feature = "perf-inline", inline)]
     fn render(&self, b: &mut Buffer) -> Result<(), RenderError> {
         if self.is_finite() {
             unsafe {
@@ -203,7 +203,7 @@ impl Render for f32 {
 }
 
 impl Render for f64 {
-    #[inline]
+    #[cfg_attr(feature = "perf-inline", inline)]
     fn render(&self, b: &mut Buffer) -> Result<(), RenderError> {
         if self.is_finite() {
             unsafe {