2020-06-07 04:58:52 -04:00
|
|
|
#![allow(clippy::cast_ptr_alignment)]
|
|
|
|
|
2020-06-04 16:39:33 -04:00
|
|
|
#[cfg(target_arch = "x86")]
|
|
|
|
use std::arch::x86::*;
|
|
|
|
#[cfg(target_arch = "x86_64")]
|
|
|
|
use std::arch::x86_64::*;
|
|
|
|
use std::slice;
|
|
|
|
|
2020-06-11 06:42:22 -04:00
|
|
|
use super::super::Buffer;
|
2020-06-04 16:39:33 -04:00
|
|
|
use super::{ESCAPED, ESCAPED_LEN, ESCAPE_LUT};
|
2020-12-20 05:33:13 -05:00
|
|
|
use super::naive::push_escaped_str;
|
2020-06-04 16:39:33 -04:00
|
|
|
|
|
|
|
const VECTOR_BYTES: usize = std::mem::size_of::<__m256i>();
|
|
|
|
|
|
|
|
#[target_feature(enable = "avx2")]
|
2020-06-18 04:23:50 -04:00
|
|
|
pub unsafe fn escape(feed: &str, buffer: &mut Buffer) {
|
2020-07-10 12:18:05 -04:00
|
|
|
debug_assert!(feed.len() >= 16);
|
|
|
|
|
2020-06-18 04:23:50 -04:00
|
|
|
let len = feed.len();
|
2020-07-04 09:32:33 -04:00
|
|
|
if len < VECTOR_BYTES {
|
2020-07-10 12:18:05 -04:00
|
|
|
escape_small(feed, buffer);
|
2020-06-04 16:39:33 -04:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2020-06-18 04:23:50 -04:00
|
|
|
let mut start_ptr = feed.as_ptr();
|
2020-12-19 02:50:28 -05:00
|
|
|
let mut ptr = start_ptr;
|
2020-12-19 22:16:45 -05:00
|
|
|
let end_ptr = feed[len..].as_ptr();
|
2020-06-09 21:56:25 -04:00
|
|
|
|
2020-06-14 23:06:14 -04:00
|
|
|
let v_independent1 = _mm256_set1_epi8(5);
|
2020-06-04 16:39:33 -04:00
|
|
|
let v_independent2 = _mm256_set1_epi8(2);
|
2020-06-14 23:06:14 -04:00
|
|
|
let v_key1 = _mm256_set1_epi8(0x27);
|
2020-06-04 16:39:33 -04:00
|
|
|
let v_key2 = _mm256_set1_epi8(0x3e);
|
|
|
|
|
2020-07-10 12:18:05 -04:00
|
|
|
let maskgen = |x: __m256i| -> u32 {
|
2020-06-04 16:39:33 -04:00
|
|
|
_mm256_movemask_epi8(_mm256_or_si256(
|
|
|
|
_mm256_cmpeq_epi8(_mm256_or_si256(x, v_independent1), v_key1),
|
|
|
|
_mm256_cmpeq_epi8(_mm256_or_si256(x, v_independent2), v_key2),
|
2020-07-10 12:18:05 -04:00
|
|
|
)) as u32
|
2020-06-04 16:39:33 -04:00
|
|
|
};
|
|
|
|
|
2020-12-19 02:50:28 -05:00
|
|
|
while ptr <= end_ptr.sub(VECTOR_BYTES) {
|
2020-06-04 16:39:33 -04:00
|
|
|
let mut mask = maskgen(_mm256_loadu_si256(ptr as *const __m256i));
|
|
|
|
while mask != 0 {
|
|
|
|
let trailing_zeros = mask.trailing_zeros() as usize;
|
2020-12-20 05:33:13 -05:00
|
|
|
mask ^= 1 << trailing_zeros;
|
2020-06-04 16:39:33 -04:00
|
|
|
let ptr2 = ptr.add(trailing_zeros);
|
|
|
|
let c = ESCAPE_LUT[*ptr2 as usize] as usize;
|
2020-06-14 23:06:14 -04:00
|
|
|
if c < ESCAPED_LEN {
|
|
|
|
if start_ptr < ptr2 {
|
|
|
|
let slc = slice::from_raw_parts(
|
|
|
|
start_ptr,
|
|
|
|
ptr2 as usize - start_ptr as usize,
|
|
|
|
);
|
|
|
|
buffer.push_str(std::str::from_utf8_unchecked(slc));
|
|
|
|
}
|
2020-12-20 05:33:13 -05:00
|
|
|
push_escaped_str(*ESCAPED.get_unchecked(c), buffer);
|
2020-06-14 23:06:14 -04:00
|
|
|
start_ptr = ptr2.add(1);
|
2020-06-04 16:39:33 -04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-12-19 02:50:28 -05:00
|
|
|
ptr = ptr.add(VECTOR_BYTES);
|
2020-06-04 16:39:33 -04:00
|
|
|
}
|
|
|
|
|
2020-12-19 02:50:28 -05:00
|
|
|
debug_assert!(ptr.add(VECTOR_BYTES) > end_ptr);
|
2020-07-10 12:18:05 -04:00
|
|
|
|
|
|
|
if ptr < end_ptr {
|
|
|
|
debug_assert!((end_ptr as usize - ptr as usize) < VECTOR_BYTES);
|
|
|
|
let backs = VECTOR_BYTES - (end_ptr as usize - ptr as usize);
|
|
|
|
|
|
|
|
let mut mask =
|
|
|
|
maskgen(_mm256_loadu_si256(ptr.sub(backs) as *const __m256i)) >> backs;
|
|
|
|
while mask != 0 {
|
|
|
|
let trailing_zeros = mask.trailing_zeros() as usize;
|
2020-12-20 05:33:13 -05:00
|
|
|
mask ^= 1 << trailing_zeros;
|
2020-07-10 12:18:05 -04:00
|
|
|
let ptr2 = ptr.add(trailing_zeros);
|
|
|
|
let c = ESCAPE_LUT[*ptr2 as usize] as usize;
|
|
|
|
if c < ESCAPED_LEN {
|
|
|
|
if start_ptr < ptr2 {
|
|
|
|
let slc = slice::from_raw_parts(
|
|
|
|
start_ptr,
|
|
|
|
ptr2 as usize - start_ptr as usize,
|
|
|
|
);
|
|
|
|
buffer.push_str(std::str::from_utf8_unchecked(slc));
|
|
|
|
}
|
2020-12-20 05:33:13 -05:00
|
|
|
push_escaped_str(*ESCAPED.get_unchecked(c), buffer);
|
2020-07-10 12:18:05 -04:00
|
|
|
start_ptr = ptr2.add(1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if end_ptr > start_ptr {
|
|
|
|
let slc = slice::from_raw_parts(start_ptr, end_ptr as usize - start_ptr as usize);
|
|
|
|
buffer.push_str(std::str::from_utf8_unchecked(slc));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[inline]
|
|
|
|
#[target_feature(enable = "avx2")]
|
|
|
|
unsafe fn escape_small(feed: &str, buffer: &mut Buffer) {
|
|
|
|
debug_assert!(feed.len() >= 16);
|
|
|
|
debug_assert!(feed.len() < VECTOR_BYTES);
|
|
|
|
|
|
|
|
let len = feed.len();
|
|
|
|
let mut start_ptr = feed.as_ptr();
|
|
|
|
let mut ptr = start_ptr;
|
|
|
|
let end_ptr = start_ptr.add(len);
|
|
|
|
|
|
|
|
let v_independent1 = _mm_set1_epi8(5);
|
|
|
|
let v_independent2 = _mm_set1_epi8(2);
|
|
|
|
let v_key1 = _mm_set1_epi8(0x27);
|
|
|
|
let v_key2 = _mm_set1_epi8(0x3e);
|
|
|
|
|
|
|
|
let maskgen = |x: __m128i| -> u32 {
|
|
|
|
_mm_movemask_epi8(_mm_or_si128(
|
|
|
|
_mm_cmpeq_epi8(_mm_or_si128(x, v_independent1), v_key1),
|
|
|
|
_mm_cmpeq_epi8(_mm_or_si128(x, v_independent2), v_key2),
|
|
|
|
)) as u32
|
|
|
|
};
|
|
|
|
|
|
|
|
let mut mask = maskgen(_mm_loadu_si128(ptr as *const __m128i));
|
|
|
|
while mask != 0 {
|
|
|
|
let trailing_zeros = mask.trailing_zeros() as usize;
|
2020-12-20 05:33:13 -05:00
|
|
|
mask ^= 1 << trailing_zeros;
|
2020-07-10 12:18:05 -04:00
|
|
|
let ptr2 = ptr.add(trailing_zeros);
|
|
|
|
let c = ESCAPE_LUT[*ptr2 as usize] as usize;
|
|
|
|
if c < ESCAPED_LEN {
|
|
|
|
if start_ptr < ptr2 {
|
|
|
|
let slc =
|
|
|
|
slice::from_raw_parts(start_ptr, ptr2 as usize - start_ptr as usize);
|
|
|
|
buffer.push_str(std::str::from_utf8_unchecked(slc));
|
|
|
|
}
|
2020-12-20 05:33:13 -05:00
|
|
|
push_escaped_str(*ESCAPED.get_unchecked(c), buffer);
|
2020-07-10 12:18:05 -04:00
|
|
|
start_ptr = ptr2.add(1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if len != 16 {
|
|
|
|
ptr = ptr.add(16);
|
|
|
|
let read_ptr = end_ptr.sub(16);
|
|
|
|
let backs = 32 - len;
|
|
|
|
let mut mask = maskgen(_mm_loadu_si128(read_ptr as *const __m128i)) >> backs;
|
|
|
|
|
|
|
|
while mask != 0 {
|
|
|
|
let trailing_zeros = mask.trailing_zeros() as usize;
|
2020-12-20 05:33:13 -05:00
|
|
|
mask ^= 1 << trailing_zeros;
|
2020-07-10 12:18:05 -04:00
|
|
|
let ptr2 = ptr.add(trailing_zeros);
|
|
|
|
let c = ESCAPE_LUT[*ptr2 as usize] as usize;
|
|
|
|
if c < ESCAPED_LEN {
|
|
|
|
if start_ptr < ptr2 {
|
|
|
|
let slc = slice::from_raw_parts(
|
|
|
|
start_ptr,
|
|
|
|
ptr2 as usize - start_ptr as usize,
|
|
|
|
);
|
|
|
|
buffer.push_str(std::str::from_utf8_unchecked(slc));
|
|
|
|
}
|
2020-12-20 05:33:13 -05:00
|
|
|
push_escaped_str(*ESCAPED.get_unchecked(c), buffer);
|
2020-07-10 12:18:05 -04:00
|
|
|
start_ptr = ptr2.add(1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if end_ptr > start_ptr {
|
|
|
|
let slc = slice::from_raw_parts(start_ptr, end_ptr as usize - start_ptr as usize);
|
|
|
|
buffer.push_str(std::str::from_utf8_unchecked(slc));
|
|
|
|
}
|
2020-06-04 16:39:33 -04:00
|
|
|
}
|