Improve performance through inlining and direct arrays

Cargo.toml

@@ -22,7 +22,6 @@ phf = { version = "0.8.0", features = ["macros"] }
 structopt = "0.3.5"
 
 [build-dependencies]
-fastrie = "0.0.6"
 serde = { version = "1.0.104", features = ["derive"] }
 serde_json = "1.0.44"
 

bench/bench.sh

@@ -4,7 +4,7 @@ set -e
 shopt -s nullglob
 
 for i in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
-  echo performance | sudo tee /sys/devices/system/cpu/cpu"$i"/cpufreq/scaling_governor
+  echo performance | sudo dd status=none of="$i"
 done
 
 node sizes.js

bench/hyperbuild-bench/Cargo.toml

@@ -10,3 +10,6 @@ hyperbuild = { path = "../.." }
 structopt = "0.3.5"
 serde = { version = "1.0.104", features = ["derive"] }
 serde_json = "1.0.44"
+
+[profile.release]
+debug = true

bench/profile.sh

@@ -4,7 +4,7 @@ set -e
 shopt -s nullglob
 
 for i in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
-  echo performance | sudo tee /sys/devices/system/cpu/cpu"$i"/cpufreq/scaling_governor
+  echo performance | sudo dd status=none of="$i"
 done
 
 rm -f perf.data

build.rs

@@ -2,9 +2,9 @@ use std::collections::HashMap;
 use std::env;
 use std::fs::File;
 use std::io::Write;
+use std::ops::{Index, IndexMut};
 use std::path::Path;
 
-use fastrie::{FastrieBuild, FastrieBuilderNode};
 use serde::{Deserialize, Serialize};
 
 fn create_byte_string_literal(bytes: &[u8]) -> String {
@@ -47,44 +47,61 @@ fn camel_case(n: &Vec<String>) -> String {
     )).collect::<Vec<String>>().join("")
 }
 
+pub struct TwoDimensionalArray {
+    data: Vec<usize>,
+    cols: usize,
+}
+
+impl TwoDimensionalArray {
+    pub fn new(rows: usize, cols: usize) -> TwoDimensionalArray {
+        TwoDimensionalArray {
+            data: vec![0usize; rows * cols],
+            cols,
+        }
+    }
+
+    pub fn prebuilt(data: Vec<usize>, cols: usize) -> TwoDimensionalArray {
+        TwoDimensionalArray { data, cols }
+    }
+}
+
+type TwoDimensionalArrayIndex = (usize, usize);
+
+impl Index<TwoDimensionalArrayIndex> for TwoDimensionalArray {
+    type Output = usize;
+
+    fn index(&self, (row, col): TwoDimensionalArrayIndex) -> &Self::Output {
+        &self.data[row * self.cols + col]
+    }
+}
+
+impl IndexMut<TwoDimensionalArrayIndex> for TwoDimensionalArray {
+    fn index_mut(&mut self, (row, col): TwoDimensionalArrayIndex) -> &mut Self::Output {
+        &mut self.data[row * self.cols + col]
+    }
+}
+
 fn build_pattern(pattern: String) -> String {
     assert!(pattern.is_ascii());
     let seq = pattern.as_bytes();
-    let mut max_prefix_len = 0usize;
-    let mut table = vec![0usize; seq.len()];
+    let dfa = &mut TwoDimensionalArray::new(256, seq.len());
 
-    let mut i = 1;
-    while i < seq.len() {
-        if seq[i] == seq[max_prefix_len] {
-            max_prefix_len += 1;
-            table[i] = max_prefix_len;
-            i += 1;
-        } else {
-            if max_prefix_len != 0 {
-                max_prefix_len = table[max_prefix_len - 1];
-            } else {
-                table[i] = 0;
-                i += 1;
-            };
+    dfa[(seq[0] as usize, 0)] = 1;
+    let mut x = 0;
+    let mut j = 1;
+    while j < seq.len() {
+        for c in 0..256 {
+            dfa[(c, j)] = dfa[(c, x)];
         };
+        dfa[(seq[j] as usize, j)] = j + 1;
+        x = dfa[(seq[j] as usize, x)];
+        j += 1;
     };
 
-    format!("crate::pattern::SinglePattern {{ seq: {}, table: &[{}] }}",
-        create_byte_string_literal(pattern.as_bytes()),
-        table.iter().map(|v| v.to_string()).collect::<Vec<String>>().join(", "))
-}
-
-fn generate_fastrie_code(var_name: &str, value_type: &str, built: &FastrieBuild<String>) -> String {
-    format!(r"
-        pub static {var_name}: &fastrie::Fastrie<{value_type}> = &fastrie::Fastrie::<{value_type}>::from_prebuilt(
-            &[{values}],
-            &[{data}],
-        );
-    ",
-        var_name = var_name,
-        value_type = value_type,
-        values = built.values.join(", "),
-        data = built.data.iter().map(|v| v.to_string()).collect::<Vec<String>>().join(", "),
+    format!(
+        "crate::pattern::SinglePattern::prebuilt(&[{}], {})",
+        dfa.data.iter().map(|v| v.to_string()).collect::<Vec<String>>().join(", "),
+        seq.len(),
     )
 }
 
@@ -173,12 +190,90 @@ struct Entity {
     characters: String,
 }
 
+pub struct TrieBuilderNode {
+    value: Option<String>,
+    children: Vec<Option<TrieBuilderNode>>,
+}
+
+struct TrieBuilderGenerationContext<'t, 'v, 'o> {
+    trie_name: &'t str,
+    value_type: &'v str,
+    next_id: usize,
+    out: &'o mut String,
+}
+
+impl<'t, 'v, 'o> TrieBuilderGenerationContext<'t, 'v, 'o> {
+    pub fn id(&mut self) -> usize {
+        let next = self.next_id;
+        self.next_id += 1;
+        next
+    }
+}
+
+impl TrieBuilderNode {
+    pub fn new() -> TrieBuilderNode {
+        let mut children = Vec::new();
+        for _ in 0..256 {
+            children.push(None);
+        };
+        TrieBuilderNode { value: None, children }
+    }
+
+    pub fn add(&mut self, seq: &[u8], value: String) -> () {
+        let mut current = self;
+        for c in seq.iter() {
+            current = current.children[*c as usize].get_or_insert_with(|| TrieBuilderNode::new());
+        };
+        current.value.replace(value);
+    }
+
+    fn _generated_node_var_name(&self, trie_name: &str, node_id: usize) -> String {
+        format!("{trie_name}_NODE_{node_id}", trie_name = trie_name, node_id = node_id)
+    }
+
+    fn _generate(&self, ctx: &mut TrieBuilderGenerationContext) -> usize {
+        let children = self.children.iter().map(|c| match c {
+            None => "None".to_string(),
+            Some(c) => {
+                let child_id = c._generate(ctx);
+                format!("Some({})", self._generated_node_var_name(ctx.trie_name, child_id))
+            }
+        }).collect::<Vec<String>>().join(", ");
+        let id = ctx.id();
+        let code = format!(
+            "static {var_name}: &'static crate::pattern::TrieNode<{value_type}> = &crate::pattern::TrieNode {{\n\tvalue: {value},\n\tchildren: [{children}],\n}};\n\n",
+            var_name = self._generated_node_var_name(ctx.trie_name, id),
+            value_type = ctx.value_type,
+            value = self.value.as_ref().map_or("None".to_string(), |v| format!("Some({})", v)),
+            children = children,
+        );
+        ctx.out.push_str(code.as_str());
+        id
+    }
+
+    pub fn generate(&self, trie_name: &str, value_type: &str) -> String {
+        let mut out = String::new();
+        let mut ctx = TrieBuilderGenerationContext {
+            trie_name,
+            value_type,
+            next_id: 0,
+            out: &mut out,
+        };
+        let root_id = self._generate(&mut ctx);
+        // Make root node public and use proper name.
+        ctx.out.replace(
+            format!("static {}", self._generated_node_var_name(trie_name, root_id)).as_str(),
+            format!("pub static {}", trie_name).as_str()
+        )
+    }
+}
+
 fn generate_entities() {
     // Read named entities map from JSON file.
     let entities: HashMap<String, Entity> = read_json("entities");
 
     // Add entities to trie builder.
-    let mut trie_builder: FastrieBuilderNode<String> = FastrieBuilderNode::new();
+    let mut trie_builder: TrieBuilderNode = TrieBuilderNode::new();
     for (rep, entity) in entities {
         let val = if rep.as_bytes().len() < entity.characters.as_bytes().len() {
             // Since we're minifying in place, we need to guarantee we'll never write something longer than source.
@@ -191,10 +286,9 @@ fn generate_entities() {
         trie_builder.add(&(rep.as_bytes())[1..], val);
     };
     // Write trie code to output Rust file.
-    write_rs("entities", generate_fastrie_code(
+    write_rs("entities", trie_builder.generate(
         "ENTITY_REFERENCES",
         "&'static [u8]",
-        &trie_builder.prebuild(),
     ));
 }
 
@@ -208,33 +302,8 @@ fn generate_patterns() {
     };
 }
 
-#[derive(Serialize, Deserialize)]
-struct Trie {
-    value_type: String,
-    values: HashMap<String, String>,
-}
-
-fn generate_tries() {
-    let tries: HashMap<String, Trie> = read_json("value_tries");
-
-    for (name, trie) in tries {
-        let mut trie_builder = FastrieBuilderNode::new();
-        for (seq, value_code) in trie.values {
-            trie_builder.add(seq.as_bytes(), value_code);
-        };
-        let var_name = snake_case(&name_words(name.as_str()));
-        let trie_code = generate_fastrie_code(
-            var_name.as_str(),
-            trie.value_type.as_str(),
-            &trie_builder.prebuild(),
-        );
-        write_rs(format!("trie_{}", var_name).as_str(), trie_code);
-    };
-}
-
 fn main() {
     generate_attr_map();
     generate_entities();
     generate_patterns();
-    generate_tries();
 }

gen/match_tries.json

@@ -1 +0,0 @@
-{}

gen/value_tries.json

@@ -1 +0,0 @@
-{}

src/pattern.rs

@@ -1,35 +1,62 @@
 pub struct SinglePattern {
-    pub seq: &'static [u8],
-    pub table: &'static [usize],
+    dfa: &'static [usize],
+    length: usize,
 }
 
 impl SinglePattern {
-    pub fn len(&self) -> usize {
-        self.table.len()
+    pub const fn prebuilt(dfa: &'static [usize], length: usize) -> SinglePattern {
+        SinglePattern {
+            dfa, length
+        }
     }
 
+    pub fn len(&self) -> usize {
+        self.length
+    }
+
+    #[inline(always)]
     pub fn match_against(&self, haystack: &[u8]) -> Option<usize> {
-        let mut hay_idx = 0usize;
-        let mut pat_idx = 0usize;
-        while hay_idx < haystack.len() {
-            if self.seq[pat_idx] == haystack[hay_idx] {
-                pat_idx += 1;
-                hay_idx += 1;
-            };
-
-            if pat_idx == self.seq.len() {
-                return Some(hay_idx - pat_idx);
-            };
-
-            if hay_idx < haystack.len() && self.seq[pat_idx] != haystack[hay_idx] {
-                if pat_idx != 0 {
-                    pat_idx = self.table[pat_idx - 1];
-                } else {
-                    hay_idx += 1;
-                };
-            };
+        let mut i = 0;
+        let mut j = 0;
+        while i < haystack.len() && j < self.length {
+            j = self.dfa[haystack[i] as usize * self.length + j];
+            i += 1;
         };
-
-        None
+        if j == self.length {
+            Some(i - self.length)
+        } else {
+            None
+        }
     }
 }
+
+// Can't use pub const fn constructor due to Copy trait, so allow directly creating struct publicly for now.
+pub struct TrieNode<V: 'static + Copy> {
+    pub value: Option<V>,
+    pub children: [Option<&'static TrieNode<V>>; 256],
+}
+
+pub struct TrieNodeMatch<V: 'static + Copy> {
+    pub end: usize,
+    pub value: V,
+}
+
+impl<V: 'static + Copy> TrieNode<V> {
+    #[inline(always)]
+    pub fn longest_matching_prefix(&self, text: &[u8]) -> Option<TrieNodeMatch<V>> {
+        let mut node: &TrieNode<V> = self;
+        let mut value: Option<TrieNodeMatch<V>> = None;
+        for (i, &c) in text.iter().enumerate() {
+            match node.children[c as usize] {
+                Some(child) => node = child,
+                None => break,
+            };
+            match node.value {
+                Some(v) => value = Some(TrieNodeMatch { end: i, value: v }),
+                None => {}
+            };
+        };
+        value
+    }
+}
+

src/proc/mod.rs

@@ -2,10 +2,8 @@ use core::fmt;
 use std::fmt::{Debug, Formatter};
 use std::ops::{Index, IndexMut};
 
-use fastrie::Fastrie;
-
 use crate::err::{ErrorType, ProcessingResult};
-use crate::pattern::SinglePattern;
+use crate::pattern::{SinglePattern, TrieNode};
 use crate::proc::MatchAction::*;
 use crate::proc::MatchMode::*;
 use crate::proc::range::ProcessorRange;
@@ -95,6 +93,7 @@ impl<'d> Processor<'d> {
 
     /// Move next `amount` characters to output.
     /// Panics. Does not check bounds for performance (e.g. already checked).
+    #[inline(always)]
     fn _shift(&mut self, amount: usize) -> () {
         // Optimisation: Don't shift if already there (but still update offsets).
         if self.read_next != self.write_next {
@@ -104,6 +103,7 @@ impl<'d> Processor<'d> {
         self.write_next += amount;
     }
 
+    #[inline(always)]
     fn _replace(&mut self, start: usize, end: usize, data: &[u8]) -> usize {
         debug_assert!(start <= end);
         let added = data.len() - (end - start);
@@ -116,15 +116,18 @@ impl<'d> Processor<'d> {
         added
     }
 
+    #[inline(always)]
     fn _insert(&mut self, at: usize, data: &[u8]) -> usize {
         self._replace(at, at, data)
     }
 
     // Matching.
+    #[inline(always)]
     fn _one<C: FnOnce(u8) -> bool>(&mut self, cond: C) -> usize {
         self._maybe_read_offset(0).filter(|n| cond(*n)).is_some() as usize
     }
 
+    #[inline(always)]
     fn _many<C: Fn(u8) -> bool>(&mut self, cond: C) -> usize {
         let mut count = 0usize;
         while self._maybe_read_offset(count).filter(|c| cond(*c)).is_some() {
@@ -133,7 +136,7 @@ impl<'d> Processor<'d> {
         count
     }
 
-    // Make expectation explicit, even for Maybe.
+    #[inline(always)]
     pub fn m(&mut self, mode: MatchMode, action: MatchAction) -> ProcessorRange {
         let count = match mode {
             IsChar(c) => self._one(|n| n == c),
@@ -167,7 +170,8 @@ impl<'d> Processor<'d> {
         ProcessorRange { start, end: start + count }
     }
 
-    pub fn m_trie<V: 'static + Copy>(&mut self, trie: &Fastrie<V>, action: MatchAction) -> Option<V> {
+    #[inline(always)]
+    pub fn m_trie<V: 'static + Copy>(&mut self, trie: &TrieNode<V>, action: MatchAction) -> Option<V> {
         trie.longest_matching_prefix(&self.code[self.read_next..]).map(|m| {
             let count = m.end + 1;
             match action {
@@ -175,7 +179,7 @@ impl<'d> Processor<'d> {
                 Keep => self._shift(count),
                 MatchOnly => {}
             };
-            *m.value
+            m.value
         })
     }
 

src/unit/tag.rs

@@ -60,12 +60,14 @@ impl MaybeClosingTag {
         MaybeClosingTag(None)
     }
 
+    #[inline(always)]
     pub fn write(&mut self, proc: &mut Processor) -> () {
         proc.write_slice(b"</");
         proc.write_range(self.0.take().unwrap());
         proc.write(b'>');
     }
 
+    #[inline(always)]
     pub fn write_if_exists(&mut self, proc: &mut Processor) -> bool {
         self.0.take().filter(|tag| {
             proc.write_slice(b"</");
@@ -75,10 +77,12 @@ impl MaybeClosingTag {
         }).is_some()
     }
 
+    #[inline(always)]
     pub fn exists(&self) -> bool {
         self.0.is_some()
     }
 
+    #[inline(always)]
     pub fn exists_and<F: FnOnce(ProcessorRange) -> bool>(&self, pred: F) -> bool {
         match self.0 {
             Some(range) => pred(range),
@@ -86,6 +90,7 @@ impl MaybeClosingTag {
         }
     }
 
+    #[inline(always)]
     pub fn replace(&mut self, tag: MaybeClosingTag) -> () {
         self.0 = tag.0;
     }