Ray Cluster features

.github/workflows/run_local_tests.yml

@@ -47,13 +47,13 @@ jobs:
         if: matrix.python-version != '3.9'
         run: |
           echo "$(brew --prefix)/opt/make/libexec/gnubin" >> $GITHUB_PATH
-          python -m pip install -vvv .[dev,ml]
+          python -m pip install -vvv .[dev,ml,ray]
 
       - name: Install SmartSim
         if: matrix.python-version == '3.9'
         run: |
           echo "$(brew --prefix)/opt/make/libexec/gnubin" >> $GITHUB_PATH
-          python -m pip install -vvv .[dev]
+          python -m pip install -vvv .[dev,ray]
 
       - name: Install ML Runtimes with Smart (with pt and tf)
         if: contains(matrix.os, 'macos')

.gitignore

@@ -4,6 +4,7 @@ __pycache__
 .pytest_cache/
 .coverage*
 htmlcov
+smartsim.egg-info
 
 # Dependencies
 third-party

Makefile

@@ -103,23 +103,23 @@ cov:
 # help: test                           - Build and run all tests
 .PHONY: test
 test:
-	@cd ./tests/; python -m pytest --ignore=full_wlm/
+	@cd ./tests/; python -m pytest --ignore=full_wlm/ --ignore=with_ray/full_wlm/
 
 # help: test-verbose                   - Build and run all tests [verbosely]
 .PHONY: test-verbose
 test-verbose:
-	@cd ./tests/; python -m pytest -vv --ignore=full_wlm/
+	@cd ./tests/; python -m pytest -vv --ignore=full_wlm/ --ignore=with_ray/full_wlm/
 
 # help: test-cov                       - run python tests with coverage
 .PHONY: test-cov
 test-cov:
-	@cd ./tests/; python -m pytest --cov=../smartsim -vv --cov-config=${COV_FILE} --ignore=full_wlm/
+	@cd ./tests/; python -m pytest --cov=../smartsim -vv --cov-config=${COV_FILE} --ignore=full_wlm/ --ignore=with_ray/full_wlm/
 
 
 # help: test-full                      - run all WLM tests with Python coverage (full test suite)
 # help:                                  WARNING: do not run test-full on shared systems.
 .PHONY: test-full
 test-full:
-	@cd ./tests/; python -m pytest --cov=../smartsim -vv --cov-config=${COV_FILE}
+	@cd ./tests/; python -m pytest --cov=../smartsim -vv --cov-config=${COV_FILE} --ignore=with_ray/full_wlm/
 
 

README.md

@@ -81,6 +81,9 @@ independently.
     - [Local Launch](#local-launch)
     - [Interactive Launch](#interactive-launch)
     - [Batch Launch](#batch-launch)
+  - [Ray](#ray)
+    - [Ray on Slurm](#ray-on-slurm)
+    - [Ray on PBS](#ray-on-pbs)
 - [SmartRedis](#smartredis)
   - [Tensors](#tensors)
   - [Datasets](#datasets)
@@ -288,6 +291,7 @@ python hello_ensemble_pbs.py
 
 # Infrastructure Library Applications
  - Orchestrator - In-memory data store and Machine Learning Inference (Redis + RedisAI)
+ - Ray - Distributed Reinforcement Learning (RL), Hyperparameter Optimization (HPO)
 
 ## Redis + RedisAI
 
@@ -415,6 +419,80 @@ exp.stop(db_cluster)
 python run_db_pbs_batch.py
 ```
 
+-----
+## Ray
+
+Ray is a distributed computation framework that supports a number of applications
+ - RLlib - Distributed Reinforcement Learning (RL)
+ - RaySGD - Distributed Training
+ - Ray Tune - Hyperparameter Optimization (HPO)
+ - Ray Serve - ML/DL inference
+As well as other integrations with frameworks like Modin, Mars, Dask, and Spark.
+
+Historically, Ray has not been well supported on HPC systems. A few examples exist,
+but none are well maintained. Because SmartSim already has launchers for HPC systems,
+launching Ray through SmartSim is a relatively simple task.
+
+### Ray on Slurm
+
+Below is an example of how to launch a Ray cluster on a Slurm system and connect to it.
+In this example, we set `batch=True`, which means that the cluster will be started
+requesting an allocation through Slurm. If this code is run within a sufficiently large
+interactive allocation, setting `batch=False` will spin the Ray cluster on the
+allocated nodes.
+
+```Python
+import ray
+
+from smartsim import Experiment
+from smartsim.ext.ray import RayCluster
+
+exp = Experiment("ray-cluster", launcher='slurm')
+# 3 workers + 1 head node = 4 node-cluster
+cluster = RayCluster(name="ray-cluster", run_args={},
+                     ray_args={"num-cpus": 24},
+                     launcher=launcher, workers=3, batch=True)
+
+exp.generate(cluster, overwrite=True)
+exp.start(cluster, block=False, summary=True)
+
+# Connect to the Ray cluster
+ray.util.connect(cluster.head_model.address+":10001")
+
+# <run Ray tune, RLlib, HPO...>
+```
+
+
+### Ray on PBS
+
+Below is an example of how to launch a Ray cluster on a PBS system and connect to it.
+In this example, we set `batch=True`, which means that the cluster will be started
+requesting an allocation through Slurm. If this code is run within a sufficiently large
+interactive allocation, setting `batch=False` will spin the Ray cluster on the
+allocated nodes.
+
+```Python
+import ray
+
+from smartsim import Experiment
+from smartsim.ext.ray import RayCluster
+
+exp = Experiment("ray-cluster", launcher='pbs')
+# 3 workers + 1 head node = 4 node-cluster
+cluster = RayCluster(name="ray-cluster", run_args={},
+                     ray_args={"num-cpus": 24},
+                     launcher=launcher, workers=3, batch=True)
+
+exp.generate(cluster, overwrite=True)
+exp.start(cluster, block=False, summary=True)
+
+# Connect to the ray cluster
+ray.util.connect(cluster.head_model.address+":10001")
+
+# <run Ray tune, RLlib, HPO...>
+```
+
+
 ------
 # SmartRedis
 

doc/api/smartsim_api.rst

@@ -490,3 +490,21 @@ Slurm
 
 .. automodule:: smartsim.launcher.slurm.slurm
     :members:
+
+
+Ray
+===
+
+.. currentmodule:: smartsim.exp.ray
+
+.. _ray_api:
+
+``RayCluster`` is used to launch a Ray cluster 
+ and can be launched as a batch or in an interactive allocation.
+
+.. autoclass:: RayCluster
+    :show-inheritance:
+    :members:
+    :inherited-members:
+    :undoc-members:
+    :exclude-members: batch set_path type

doc/conf.py

@@ -41,7 +41,8 @@
     'sphinxfortran.fortran_domain',
     'sphinxfortran.fortran_autodoc',
     'breathe',
-    'nbsphinx'
+    'nbsphinx',
+    'myst_parser'
 ]
 
 

doc/index.rst

@@ -14,6 +14,8 @@
    :caption: Tutorials
 
    tutorials/01_getting_started/01_getting_started
+   tutorlals/03_online_analysis/03_online_analysis
+   tutorials/05_starting_ray/05_starting_ray
    tutorials/using_clients
    tutorials/lattice_boltz_analysis
    tutorials/inference

doc/installation.rst

@@ -429,6 +429,7 @@ Note that SmartSim and SmartRedis will be downloaded to the working directory
 from which these instructions are executed.
 
 .. code-block:: bash
+
   # setup Python and build environment
   module load open-ce
   conda create -p /ccs/home/$USER/.conda/envs/smartsim --clone open-ce-1.2.0-py38-0
@@ -471,6 +472,7 @@ When executing SmartSim, if you want to use the PyTorch backend in the orchestra
 you will need to add the PyTorch library path to the environment with:
 
 .. code-block:: bash
+
   export LD_LIBRARY_PATH=/ccs/home/$USER/.conda/envs/smartsim/lib/python3.8/site-packages/torch/lib/:$LD_LIBRARY_PATH
 
 

setup.cfg

@@ -66,6 +66,8 @@ doc=
     sphinx-fortran==1.1.1
     nbsphinx>=0.8.2
 
+ray=
+    ray==1.6
 
 [options.packages.find]
 exclude =

smart

@@ -245,7 +245,7 @@ def clean(install_path, _all=False):
         rai_path = lib_path.joinpath("redisai.so")
         if rai_path.is_file():
             rai_path.unlink()
-            print("Succesfully removed existing RedisAI installation")
+            print("Successfully removed existing RedisAI installation")
 
         backend_path = lib_path.joinpath("backends")
         if backend_path.is_dir():

smartsim/control/controller.py

@@ -247,9 +247,11 @@ def init_launcher(self, launcher):
             elif launcher == "pbs":
                 self._launcher = PBSLauncher()
                 self._jobs.set_launcher(self._launcher)
+            # Init Cobalt launcher
             elif launcher == "cobalt":
                 self._launcher = CobaltLauncher()
                 self._jobs.set_launcher(self._launcher)
+            # Init LSF launcher
             elif launcher == "lsf":
                 self._launcher = LSFLauncher()
                 self._jobs.set_launcher(self._launcher)
@@ -275,6 +277,9 @@ def _launch(self, manifest):
                 raise SmartSimError(msg)
             self._launch_orchestrator(orchestrator)
 
+        for rc in manifest.ray_clusters:
+            self._launch_ray_cluster(rc)
+
         # create all steps prior to launch
         steps = []
 
@@ -345,6 +350,31 @@ def _launch_orchestrator(self, orchestrator):
         self._save_orchestrator(orchestrator)
         logger.debug(f"Orchestrator launched on nodes: {orchestrator.hosts}")
 
+    def _launch_ray_cluster(self, ray_cluster):
+        """Launch a Ray Cluster instance
+        This function will launch the Ray Cluster instance and
+        if on WLM, find the nodes where it was launched and
+        set them in the JobManager
+
+        :param orchestrator: ray cluster to launch
+        :type orchestrator: RayCluster
+        """
+        # if the Ray cluster was launched as a batch workload
+        ray_cluster._update_workers()
+
+        if ray_cluster.batch:
+            ray_batch_step = self._create_batch_job_step(ray_cluster)
+            self._launch_step(ray_batch_step, ray_cluster)
+
+        else:
+            ray_steps = [
+                (self._create_job_step(ray_node), ray_node) for ray_node in ray_cluster
+            ]
+            for ray_step in ray_steps:
+                self._launch_step(*ray_step)
+
+        logger.info("Ray cluster launched.")
+
     def _launch_step(self, job_step, entity):
         """Use the launcher to launch a job stop
 
@@ -498,7 +528,7 @@ def _orchestrator_launch_wait(self, orchestrator):
                 if not self._jobs.actively_monitoring:
                     self._jobs.check_jobs()
 
-                # _jobs.get_status aquires JM lock for main thread, no need for locking
+                # _jobs.get_status acquires JM lock for main thread, no need for locking
                 statuses = self.get_entity_list_status(orchestrator)
                 if all([stat == STATUS_RUNNING for stat in statuses]):
                     ready = True

smartsim/control/manifest.py

@@ -28,11 +28,12 @@
 from ..entity import EntityList, SmartSimEntity
 from ..error import SmartSimError
 from ..error.errors import SmartSimError
+from ..exp.ray import RayCluster
 
 # List of types derived from EntityList which require specific behavior
 # A corresponding property needs to exist (like db for Orchestrator),
 # otherwise they will not be accessible
-entity_list_exception_types = [Orchestrator]
+entity_list_exception_types = [Orchestrator, RayCluster]
 
 
 class Manifest:
@@ -83,6 +84,14 @@ def ensembles(self):
 
         return _ensembles
 
+    @property
+    def ray_clusters(self):
+        _ray_cluster = []
+        for deployable in self._deployables:
+            if isinstance(deployable, RayCluster):
+                _ray_cluster.append(deployable)
+        return _ray_cluster
+
     def _check_names(self, deployables):
         used = []
         for deployable in deployables:

smartsim/entity/entity.py

@@ -24,8 +24,6 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import os.path as osp
-
 
 class SmartSimEntity:
     def __init__(self, name, path, run_settings):

smartsim/entity/entityList.py

@@ -43,8 +43,7 @@ def batch(self):
         try:
             if self.batch_settings:
                 return True
-            else:
-                return False
+            return False
         # local orchestrator cannot launch with batches
         except AttributeError:
             return False

smartsim/exp/ray/__init__.py

@@ -0,0 +1 @@
+from .raycluster import RayCluster, parse_ray_head_node_address