Lua scripting speedups

CHANGELOG.md

@@ -12,6 +12,7 @@
     - Features
       - Added conditional restriction support with `parse-conditional-restrictions=true|false` to osrm-extract. This option saves conditional turn restrictions to the .restrictions file for parsing by contract later. Added `parse-conditionals-from-now=utc time stamp` and `--time-zone-file=/path/to/file`  to osrm-contract
       - Command-line tools (osrm-extract, osrm-contract, osrm-routed, etc) now return error codes and legible error messages for common problem scenarios, rather than ugly C++ crashes
+      - Speed up pre-processing by only running the Lua `node_function` for nodes that have tags.  Cuts OSM file parsing time in half.
     - Files
       - .osrm.nodes file was renamed to .nbg_nodes and .ebg_nodes was added
     - Guidance

include/extractor/profile_properties.hpp

@@ -22,7 +22,7 @@ struct ProfileProperties
         : traffic_signal_penalty(0), u_turn_penalty(0),
           max_speed_for_map_matching(DEFAULT_MAX_SPEED), continue_straight_at_waypoint(true),
           use_turn_restrictions(false), left_hand_driving(false), fallback_to_duration(true),
-          weight_name{"duration"}
+          weight_name{"duration"}, call_tagless_node_function(true)
     {
         BOOST_ASSERT(weight_name[MAX_WEIGHT_NAME_LENGTH] == '\0');
     }
@@ -88,6 +88,8 @@ struct ProfileProperties
     char weight_name[MAX_WEIGHT_NAME_LENGTH + 1];
     unsigned weight_precision = 1;
     bool force_split_edges = false;
+
+    bool call_tagless_node_function = true;
 };
 }
 }

include/extractor/scripting_environment_lua.hpp

@@ -31,6 +31,11 @@ struct LuaScriptingContext final
     bool has_way_function;
     bool has_segment_function;
 
+    sol::function turn_function;
+    sol::function way_function;
+    sol::function node_function;
+    sol::function segment_function;
+
     int api_version;
 };
 

profiles/bicycle.lua

@@ -15,6 +15,11 @@ properties.continue_straight_at_waypoint = false
 properties.weight_name                   = 'duration'
 --properties.weight_name                   = 'cyclability'
 
+-- Set to true if you need to call the node_function for every node.
+-- Generally can be left as false to avoid unnecessary Lua calls
+-- (which slow down pre-processing).
+properties.call_tagless_node_function      = false
+
 
 local default_speed = 15
 local walking_speed = 6

profiles/car.lua

@@ -20,6 +20,12 @@ properties.weight_name                     = 'routability'
 -- For shortest distance without penalties for accessibility
 --properties.weight_name                     = 'distance'
 
+-- Set to true if you need to call the node_function for every node.
+-- Generally can be left as false to avoid unnecessary Lua calls
+-- (which slow down pre-processing).
+properties.call_tagless_node_function      = false
+
+
 local profile = {
   default_mode      = mode.driving,
   default_speed     = 10,

profiles/foot.lua

@@ -14,6 +14,11 @@ properties.continue_straight_at_waypoint = false
 properties.weight_name                   = 'duration'
 --properties.weight_name                   = 'routability'
 
+-- Set to true if you need to call the node_function for every node.
+-- Generally can be left as false to avoid unnecessary Lua calls
+-- (which slow down pre-processing).
+properties.call_tagless_node_function      = false
+
 local walking_speed = 5
 
 local profile = {

profiles/rasterbot.lua

@@ -3,6 +3,11 @@ api_version = 1
 
 properties.force_split_edges = true
 
+-- Set to true if you need to call the node_function for every node.
+-- Generally can be left as false to avoid unnecessary Lua calls
+-- (which slow down pre-processing).
+properties.call_tagless_node_function      = false
+
 -- Minimalist node_ and way_functions in order to test source_ and segment_functions
 
 function node_function (node, result)

profiles/rasterbotinterp.lua

@@ -1,6 +1,11 @@
 api_version = 1
 -- Rasterbot profile
 
+-- Set to true if you need to call the node_function for every node.
+-- Generally can be left as false to avoid unnecessary Lua calls
+-- (which slow down pre-processing).
+properties.call_tagless_node_function      = false
+
 -- Minimalist node_ and way_functions in order to test source_ and segment_functions
 
 function node_function (node, result)

profiles/testbot.lua

@@ -22,6 +22,11 @@ properties.use_turn_restrictions         = true
 properties.max_speed_for_map_matching    = 30/3.6 --km -> m/s
 properties.weight_name                   = 'duration'
 
+-- Set to true if you need to call the node_function for every node.
+-- Generally can be left as false to avoid unnecessary Lua calls
+-- (which slow down pre-processing).
+properties.call_tagless_node_function      = false
+
 local uturn_penalty                      = 20
 local traffic_light_penalty              = 7     -- seconds
 

src/extractor/scripting_environment_lua.cpp

@@ -249,7 +249,9 @@ void Sol2ScriptingEnvironment::InitContext(LuaScriptingContext &context)
         "max_turn_weight",
         sol::property(&ProfileProperties::GetMaxTurnWeight),
         "force_split_edges",
-        &ProfileProperties::force_split_edges);
+        &ProfileProperties::force_split_edges,
+        "call_tagless_node_function",
+        &ProfileProperties::call_tagless_node_function);
 
     context.state.new_usertype<std::vector<std::string>>(
         "vector",
@@ -427,15 +429,16 @@ void Sol2ScriptingEnvironment::InitContext(LuaScriptingContext &context)
 
     context.state.script_file(file_name);
 
-    sol::function turn_function = context.state["turn_function"];
-    sol::function node_function = context.state["node_function"];
-    sol::function way_function = context.state["way_function"];
-    sol::function segment_function = context.state["segment_function"];
+    // cache references to functions for faster execution
+    context.turn_function = context.state["turn_function"];
+    context.node_function = context.state["node_function"];
+    context.way_function = context.state["way_function"];
+    context.segment_function = context.state["segment_function"];
 
-    context.has_turn_penalty_function = turn_function.valid();
-    context.has_node_function = node_function.valid();
-    context.has_way_function = way_function.valid();
-    context.has_segment_function = segment_function.valid();
+    context.has_turn_penalty_function = context.turn_function.valid();
+    context.has_node_function = context.node_function.valid();
+    context.has_way_function = context.way_function.valid();
+    context.has_segment_function = context.segment_function.valid();
 
     // Check profile API version
     auto maybe_version = context.state.get<sol::optional<int>>("api_version");
@@ -509,7 +512,9 @@ void Sol2ScriptingEnvironment::ProcessElements(
                 {
                 case osmium::item_type::node:
                     result_node.clear();
-                    if (local_context.has_node_function)
+                    if (local_context.has_node_function &&
+                        (!static_cast<const osmium::Node &>(*entity).tags().empty() ||
+                         local_context.properties.call_tagless_node_function))
                     {
                         local_context.ProcessNode(static_cast<const osmium::Node &>(*entity),
                                                   result_node);
@@ -590,13 +595,12 @@ void Sol2ScriptingEnvironment::ProcessTurn(ExtractionTurn &turn)
 {
     auto &context = GetSol2Context();
 
-    sol::function turn_function = context.state["turn_function"];
     switch (context.api_version)
     {
     case 1:
         if (context.has_turn_penalty_function)
         {
-            turn_function(turn);
+            context.turn_function(turn);
 
             // Turn weight falls back to the duration value in deciseconds
             // or uses the extracted unit-less weight value
@@ -611,7 +615,7 @@ void Sol2ScriptingEnvironment::ProcessTurn(ExtractionTurn &turn)
             if (turn.turn_type != guidance::TurnType::NoTurn)
             {
                 // Get turn duration and convert deci-seconds to seconds
-                turn.duration = static_cast<double>(turn_function(turn.angle)) / 10.;
+                turn.duration = static_cast<double>(context.turn_function(turn.angle)) / 10.;
                 BOOST_ASSERT(turn.weight == 0);
 
                 // add U-turn penalty
@@ -642,14 +646,14 @@ void Sol2ScriptingEnvironment::ProcessSegment(ExtractionSegment &segment)
 
     if (context.has_segment_function)
     {
-        sol::function segment_function = context.state["segment_function"];
         switch (context.api_version)
         {
         case 1:
-            segment_function(segment);
+            context.segment_function(segment);
             break;
         case 0:
-            segment_function(segment.source, segment.target, segment.distance, segment.duration);
+            context.segment_function(
+                segment.source, segment.target, segment.distance, segment.duration);
             segment.weight = segment.duration; // back-compatibility fallback to duration
             break;
         }
@@ -660,17 +664,13 @@ void LuaScriptingContext::ProcessNode(const osmium::Node &node, ExtractionNode &
 {
     BOOST_ASSERT(state.lua_state() != nullptr);
 
-    sol::function node_function = state["node_function"];
-
     node_function(node, result);
 }
 
 void LuaScriptingContext::ProcessWay(const osmium::Way &way, ExtractionWay &result)
 {
     BOOST_ASSERT(state.lua_state() != nullptr);
 
-    sol::function way_function = state["way_function"];
-
     way_function(way, result);
 }
 }