Merge pull request #28 from projectsyn/feat/parallel-inventory

Parallelize inventory rendering

Author: simu

Cargo.toml

@@ -16,6 +16,7 @@ chrono = "0.4.30"
 indexmap = "2.0.0"
 nom = "7.1.3"
 pyo3 = { version = "0.19.2", features = ["chrono"] }
+rayon = "1.7.0"
 serde = { version = "1.0.188", features = ["derive"] }
 serde_json = "1.0.106"
 serde_yaml = "0.9.25"

src/inventory.rs

@@ -2,6 +2,7 @@ use anyhow::{anyhow, Result};
 use chrono::Local;
 use pyo3::prelude::*;
 use pyo3::types::PyDict;
+use rayon::prelude::*;
 use std::collections::HashMap;
 
 use super::{NodeInfo, Reclass};
@@ -28,8 +29,8 @@ impl Inventory {
         // Render all nodes
         let infos: Vec<_> = r
             .nodes
-            .keys()
-            .map(|name| (name, { r.render_node(name) }))
+            .par_iter()
+            .map(|(name, _)| (name, { r.render_node(name) }))
             .collect();
 
         // Generate `Inventory` from the rendered nodes

src/lib.rs

@@ -16,6 +16,8 @@ pub mod types;
 use anyhow::{anyhow, Result};
 use pyo3::exceptions::PyValueError;
 use pyo3::prelude::*;
+use pyo3::types::PyType;
+use rayon::ThreadPoolBuilder;
 use std::collections::HashMap;
 use std::path::{Component, Path, PathBuf, MAIN_SEPARATOR};
 use walkdir::WalkDir;
@@ -193,6 +195,19 @@ impl Reclass {
         Inventory::render(self)
             .map_err(|e| PyValueError::new_err(format!("Error while rendering inventory: {e}")))
     }
+
+    /// Configures the number of threads to use when rendering the full inventory. Calling the
+    /// method with `count=0` will configure the thread pool to have one thread per logical core of
+    /// the system.
+    ///
+    /// Note that this method should only be called once and will print a diagnostic message if
+    /// called again.
+    #[classmethod]
+    pub fn set_thread_count(_cls: &PyType, count: usize) {
+        if let Err(e) = ThreadPoolBuilder::new().num_threads(count).build_global() {
+            eprintln!("While initializing global thread pool: {e}");
+        }
+    }
 }
 
 impl Default for Reclass {