From 59af59752ceaa87537dc5b9d49ea4a542090777d Mon Sep 17 00:00:00 2001 From: Sam Partee Date: Fri, 21 Jan 2022 13:29:49 -0800 Subject: [PATCH 1/8] Start doc build in docker work --- Makefile | 10 ++++++++++ doc/conf.py | 14 ++++++++++---- doc/requirements-doc.txt | 3 ++- docker-compose.yml | 11 +++++++++++ docker/docs/dev/Dockerfile | 23 +++++++++++++++++++++++ docker/docs/prod/Dockerfile | 26 ++++++++++++++++++++++++++ setup.py | 8 ++++---- 7 files changed, 86 insertions(+), 9 deletions(-) create mode 100644 docker-compose.yml create mode 100644 docker/docs/dev/Dockerfile create mode 100644 docker/docs/prod/Dockerfile diff --git a/Makefile b/Makefile index 915915f82..7455469c1 100644 --- a/Makefile +++ b/Makefile @@ -90,6 +90,16 @@ check-lint: docs: @cd doc; make html +# help: docks - generate project documentation with docker +.PHONY: docks +docks: + @if [ -d "docs" ]; then rm -rf docs; fi + @mkdir -p docs/develop + @docker compose build docs-dev + @docker create -ti --name devdocs smartsim-docs:dev-latest + @docker cp devdocs:/usr/local/src/SmartSim/doc/_build/html/ ./docs/develop/ + @docker container rm devdocs + # help: cov - generate html coverage report for Python client .PHONY: cov cov: diff --git a/doc/conf.py b/doc/conf.py index cbd8831c3..c1693b2ee 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -17,11 +17,17 @@ # -- Project information ----------------------------------------------------- project = 'SmartSim' -copyright = '2021, Hewlett Packard Enterprise' -author = 'HPE AI & Advanced Productivity' +copyright = '2022, Hewlett Packard Enterprise' +author = 'Cray Labs' + +try: + import smartsim + version = smartsim.__version__ +except ImportError: + version = "0.3.2" # The full version, including alpha/beta/rc tags -release = '0.3.2' +release = version # -- General configuration --------------------------------------------------- @@ -68,7 +74,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = "sphinx_rtd_theme" +html_theme = "sphinx_book_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt index 355fa7ee6..b165e3615 100644 --- a/doc/requirements-doc.txt +++ b/doc/requirements-doc.txt @@ -1,6 +1,7 @@ sphinx==3.1.1 breathe==4.27.0 -sphinx_rtd_theme>=0.5.0 +#sphinx_rtd_theme>=0.5.0 +sphinx-book-theme sphinx-fortran==1.1.1 nbsphinx>=0.8.2 Jinja2==2.11 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..15d78ad77 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ + + +version: '3' + +services: + docs-dev: + image: smartsim-docs:dev-latest + build: + context: . + dockerfile: ./docker/docs/dev/Dockerfile + diff --git a/docker/docs/dev/Dockerfile b/docker/docs/dev/Dockerfile new file mode 100644 index 000000000..119666f5a --- /dev/null +++ b/docker/docs/dev/Dockerfile @@ -0,0 +1,23 @@ +FROM ubuntu:20.04 + +LABEL maintainer="Cray Labs" + +ARG DEBIAN_FRONTEND="noninteractive" +ENV TZ=US/Seattle + +RUN apt-get update \ + && apt-get install --no-install-recommends -y build-essential \ + git gcc make \ + python3-pip python3 python3-dev cmake git-lfs doxygen + +RUN ln -s /usr/bin/python3 /usr/bin/python + +COPY . /usr/local/src/SmartSim/ +WORKDIR /usr/local/src/SmartSim/ +RUN NO_CHECKS=1 SMARTSIM_SUFFIX=dev python -m pip install .[doc] + +RUN git clone https://github.com/CrayLabs/SmartRedis.git --branch develop --depth=1 smartredis \ + && cd smartredis \ + && python -m pip install . + +RUN make docs diff --git a/docker/docs/prod/Dockerfile b/docker/docs/prod/Dockerfile new file mode 100644 index 000000000..9bf884252 --- /dev/null +++ b/docker/docs/prod/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:20.04 + +LABEL maintainer="Sam Partee" + +ARG DEBIAN_FRONTEND="noninteractive" +ENV TZ=US/Seattle +RUN apt-get update \ + && apt-get install --no-install-recommends -y build-essential \ + git gcc make \ + python3-pip python3 python3-dev cmake git-lfs doxygen + +RUN ln -s /usr/bin/python3 /usr/bin/python + +RUN mkdir /data +WORKDIR /data +RUN git clone https://github.com/CrayLabs/SmartSim.git --branch develop --depth=1 \ + && cd SmartSim \ + && NO_CHECKS=1 python -m pip install .[doc] + + +WORKDIR /data/SmartSim/ +RUN git clone https://github.com/CrayLabs/SmartRedis.git --branch develop --depth=1 smartredis \ + && cd smartredis \ + && python -m pip install . + +RUN make docs diff --git a/setup.py b/setup.py index e10b02294..bd17ccd42 100644 --- a/setup.py +++ b/setup.py @@ -156,11 +156,11 @@ def has_ext_modules(_placeholder): # see smartsim/_core/_install/buildenv.py for more details "ml": versions.ml_extras_required(), "doc": [ - "sphinx==3.1.1", - "breathe==4.27.0", - "sphinx_rtd_theme>=0.5.0", + "sphinx==4.4.0", + "breathe==4.31.0", + "sphinx-book-theme==0.2.0", "sphinx-fortran==1.1.1", - "nbsphinx>=0.8.2", + "nbsphinx>=0.8.2" ], "ray": "ray>=1.6" } From d72252d11fc49297f19dbe2ee89b2a0726e4519e Mon Sep 17 00:00:00 2001 From: Sam Partee Date: Mon, 24 Jan 2022 17:26:20 -0800 Subject: [PATCH 2/8] Add workflow to commit develop docs This new GH action should consistently update the documentation with the latest code on develop. --- .github/workflows/build_docs.yml | 39 ++++++++++++++++++++++++++++++++ Makefile | 2 +- docker/docs/prod/Dockerfile | 26 --------------------- 3 files changed, 40 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/build_docs.yml delete mode 100644 docker/docs/prod/Dockerfile diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml new file mode 100644 index 000000000..f865bbbe1 --- /dev/null +++ b/.github/workflows/build_docs.yml @@ -0,0 +1,39 @@ +name: deploy_dev_docs + +on: + pull_request: + types: [closed] + +jobs: + build_docs: + name: build_dev_docs + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + + - name: build documentation with docker + run: make docks + + - uses: actions/checkout@v2 + with: + ref: doc + path: doc-branch + + - name: Move over release docs and stubs + run: | + rm -rf doc-branch/docs/docs/ + cp -r doc-branch/docs/* ./docs/ + rm -rf doc-branch + + - name: Commit files and push + run: | + git config --local user.email "spartee@hpe.com" + git config --local user.name "Sam Partee" + git checkout -b doc + git add . + git commit -m "Update develop documentation" + git push -f origin doc + diff --git a/Makefile b/Makefile index 7455469c1..898eb784d 100644 --- a/Makefile +++ b/Makefile @@ -94,7 +94,7 @@ docs: .PHONY: docks docks: @if [ -d "docs" ]; then rm -rf docs; fi - @mkdir -p docs/develop + @mkdir -p docs @docker compose build docs-dev @docker create -ti --name devdocs smartsim-docs:dev-latest @docker cp devdocs:/usr/local/src/SmartSim/doc/_build/html/ ./docs/develop/ diff --git a/docker/docs/prod/Dockerfile b/docker/docs/prod/Dockerfile deleted file mode 100644 index 9bf884252..000000000 --- a/docker/docs/prod/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM ubuntu:20.04 - -LABEL maintainer="Sam Partee" - -ARG DEBIAN_FRONTEND="noninteractive" -ENV TZ=US/Seattle -RUN apt-get update \ - && apt-get install --no-install-recommends -y build-essential \ - git gcc make \ - python3-pip python3 python3-dev cmake git-lfs doxygen - -RUN ln -s /usr/bin/python3 /usr/bin/python - -RUN mkdir /data -WORKDIR /data -RUN git clone https://github.com/CrayLabs/SmartSim.git --branch develop --depth=1 \ - && cd SmartSim \ - && NO_CHECKS=1 python -m pip install .[doc] - - -WORKDIR /data/SmartSim/ -RUN git clone https://github.com/CrayLabs/SmartRedis.git --branch develop --depth=1 smartredis \ - && cd smartredis \ - && python -m pip install . - -RUN make docs From 7148c0fcf4235ffd28231909c7a168b7ef7eebcc Mon Sep 17 00:00:00 2001 From: Sam Partee Date: Fri, 28 Jan 2022 11:30:24 -0800 Subject: [PATCH 3/8] Fix ray notebook --- .github/workflows/build_docs.yml | 4 +- Makefile | 2 - docker/docs/dev/Dockerfile | 12 +++++- smartsim/_core/_install/buildenv.py | 40 +++++++++++++++---- .../01_getting_started.ipynb | 2 +- 5 files changed, 46 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml index f865bbbe1..5c7ec9138 100644 --- a/.github/workflows/build_docs.yml +++ b/.github/workflows/build_docs.yml @@ -24,8 +24,10 @@ jobs: - name: Move over release docs and stubs run: | - rm -rf doc-branch/docs/docs/ + rm -rf doc-branch/docs/develop/ cp -r doc-branch/docs/* ./docs/ + touch ./docs/.nojekyll + touch ./docs/develop/.nojekyll rm -rf doc-branch - name: Commit files and push diff --git a/Makefile b/Makefile index 898eb784d..9f331c95f 100644 --- a/Makefile +++ b/Makefile @@ -136,5 +136,3 @@ test-cov: .PHONY: test-full test-full: @python -m pytest --cov=./smartsim -vv --cov-config=${COV_FILE} - - diff --git a/docker/docs/dev/Dockerfile b/docker/docs/dev/Dockerfile index 119666f5a..b86bb1963 100644 --- a/docker/docs/dev/Dockerfile +++ b/docker/docs/dev/Dockerfile @@ -8,7 +8,8 @@ ENV TZ=US/Seattle RUN apt-get update \ && apt-get install --no-install-recommends -y build-essential \ git gcc make \ - python3-pip python3 python3-dev cmake git-lfs doxygen + python3-pip python3 python3-dev cmake pandoc doxygen \ + && rm -rf /var/lib/apt/lists/* RUN ln -s /usr/bin/python3 /usr/bin/python @@ -18,6 +19,13 @@ RUN NO_CHECKS=1 SMARTSIM_SUFFIX=dev python -m pip install .[doc] RUN git clone https://github.com/CrayLabs/SmartRedis.git --branch develop --depth=1 smartredis \ && cd smartredis \ - && python -m pip install . + && python -m pip install . \ + && rm -rf ~/.cache/pip +# address needs of nbsphinx +# see https://nbsphinx.readthedocs.io/en/0.8.1/installation.html#Jupyter-Kernel +RUN python -m pip install ipython ipykernel + +RUN cd doc/tutorials/ && \ + ln -s ../../tutorials/* . RUN make docs diff --git a/smartsim/_core/_install/buildenv.py b/smartsim/_core/_install/buildenv.py index 164c763c5..04e2526d9 100644 --- a/smartsim/_core/_install/buildenv.py +++ b/smartsim/_core/_install/buildenv.py @@ -382,19 +382,43 @@ def is_windows(self): def is_macos(self): return self.PLATFORM == "darwin" - @property - def site_packages_path(self): - """Find location of user pip packages for this py env""" - site_path = Path(site.getsitepackages()[0]).resolve() - return site_path - @property def torch_cmake_path(self): """Find the path to the cmake directory within a pip installed pytorch package""" - site_path = self.site_packages_path - torch_path = site_path.joinpath("torch/share/cmake/Torch/").resolve() + def _torch_import_path(): + """Find through importing torch""" + try: + import torch as t + torch_paths = [Path(p) for p in t.__path__] + for _path in torch_paths: + torch_path = _path / "share/cmake/Torch" + if torch_path.is_dir(): + return torch_path + return None + except ModuleNotFoundError: + return None + + def _torch_site_path(): + """find torch through site packages""" + site_paths = [Path(p) for p in site.getsitepackages()] + + # check user site (~/.local/lib) + if Path(site.USER_SITE).is_dir(): + site_paths.append(Path(site.USER_SITE)) + + for _path in site_paths: + torch_path = _path / "torch/share/cmake/Torch" + if torch_path.is_dir(): + return torch_path + return None + + torch_path = _torch_import_path() + if not torch_path: + torch_path = _torch_site_path() + if not torch_path: + raise SetupError("Could not located torch cmake path") return str(torch_path) @staticmethod diff --git a/tutorials/01_getting_started/01_getting_started.ipynb b/tutorials/01_getting_started/01_getting_started.ipynb index f804ef140..cf197817e 100644 --- a/tutorials/01_getting_started/01_getting_started.ipynb +++ b/tutorials/01_getting_started/01_getting_started.ipynb @@ -1154,7 +1154,7 @@ } ], "metadata": { - "language_info": { +"language_info": { "codemirror_mode": { "name": "ipython", "version": 3 From 0feb014005cda42822f8f757faa5d223642d5964 Mon Sep 17 00:00:00 2001 From: Sam Partee Date: Tue, 25 Jan 2022 21:36:22 -0800 Subject: [PATCH 4/8] tide up docs --- doc/api/smartsim_api.rst | 265 +++++++++++++++++++-------------------- 1 file changed, 131 insertions(+), 134 deletions(-) diff --git a/doc/api/smartsim_api.rst b/doc/api/smartsim_api.rst index f146b6f6c..73f7a93bc 100644 --- a/doc/api/smartsim_api.rst +++ b/doc/api/smartsim_api.rst @@ -18,6 +18,8 @@ Experiment Experiment.stop Experiment.create_ensemble Experiment.create_model + Experiment.create_run_settings + Experiment.create_batch_settings Experiment.generate Experiment.poll Experiment.finished @@ -49,17 +51,17 @@ Types of Settings: RunSettings SrunSettings - SbatchSettings AprunSettings - QsubBatchSettings - CobaltBatchSettings MpirunSettings JsrunSettings + SbatchSettings + QsubBatchSettings + CobaltBatchSettings BsubBatchSettings -Local ------ +RunSettings +----------- .. _rs-api: @@ -80,8 +82,6 @@ launches that utilize a parallel launch binary like :undoc-members: :members: ------------------------------------------- - SrunSettings ------------ @@ -95,11 +95,13 @@ steps to a batch. .. autosummary:: - SrunSettings.set_cpus_per_task - SrunSettings.set_hostlist SrunSettings.set_nodes SrunSettings.set_tasks SrunSettings.set_tasks_per_node + SrunSettings.set_walltime + SrunSettings.set_hostlist + SrunSettings.set_excluded_hosts + SrunSettings.set_cpus_per_task SrunSettings.add_exe_args SrunSettings.format_run_args SrunSettings.format_env_vars @@ -111,32 +113,6 @@ steps to a batch. :members: -SbatchSettings --------------- - -.. _sbatch_api: - -``SbatchSettings`` are used for launching batches onto Slurm -WLM systems. - - -.. autosummary:: - - SbatchSettings.set_account - SbatchSettings.set_batch_command - SbatchSettings.set_nodes - SbatchSettings.set_hostlist - SbatchSettings.set_partition - SbatchSettings.set_walltime - SbatchSettings.format_batch_args - -.. autoclass:: SbatchSettings - :inherited-members: - :undoc-members: - :members: - ------------------------------------------- - AprunSettings ------------- @@ -169,6 +145,100 @@ and within batch launches (e.g., ``QsubBatchSettings``) :members: + + +JsrunSettings +------------- + +.. _jsrun_api: + +``JsrunSettings`` can be used on any system that supports the +IBM LSF launcher. + +``JsrunSettings`` can be used in interactive session (on allocation) +and within batch launches (i.e. ``BsubBatchSettings``) + + +.. autosummary:: + + JsrunSettings.set_num_rs + JsrunSettings.set_cpus_per_rs + JsrunSettings.set_gpus_per_rs + JsrunSettings.set_rs_per_host + JsrunSettings.set_tasks + JsrunSettings.set_tasks_per_rs + JsrunSettings.set_binding + JsrunSettings.make_mpmd + JsrunSettings.set_mpmd_preamble + JsrunSettings.update_env + JsrunSettings.set_erf_sets + JsrunSettings.format_env_vars + JsrunSettings.format_run_args + + +.. autoclass:: JsrunSettings + :inherited-members: + :undoc-members: + :members: + + +MpirunSettings +-------------- + +.. _openmpi_api: + +``MpirunSettings`` are for launching with OpenMPI. ``MpirunSettings`` are +supported on Slurm, PBSpro, and Cobalt. + + +.. autosummary:: + + MpirunSettings.set_cpus_per_task + MpirunSettings.set_hostlist + MpirunSettings.set_tasks + MpirunSettings.set_task_map + MpirunSettings.make_mpmd + MpirunSettings.add_exe_args + MpirunSettings.format_run_args + MpirunSettings.format_env_vars + MpirunSettings.update_env + +.. autoclass:: MpirunSettings + :inherited-members: + :undoc-members: + :members: + + +------------------------------------------ + + + +SbatchSettings +-------------- + +.. _sbatch_api: + +``SbatchSettings`` are used for launching batches onto Slurm +WLM systems. + + +.. autosummary:: + + SbatchSettings.set_account + SbatchSettings.set_batch_command + SbatchSettings.set_nodes + SbatchSettings.set_hostlist + SbatchSettings.set_partition + SbatchSettings.set_queue + SbatchSettings.set_walltime + SbatchSettings.format_batch_args + +.. autoclass:: SbatchSettings + :inherited-members: + :undoc-members: + :members: + + QsubBatchSettings ----------------- @@ -196,8 +266,6 @@ be launched as a batch on PBSPro systems. :members: ------------------------------------------- - CobaltBatchSettings ------------------- @@ -223,42 +291,6 @@ that of the ``QsubBatchSettings`` for PBSPro. :undoc-members: :members: ------------------------------------------- - - -JsrunSettings -------------- - -.. _jsrun_api: - -``JsrunSettings`` can be used on any system that supports the -IBM LSF launcher. - -``JsrunSettings`` can be used in interactive session (on allocation) -and within batch launches (i.e. ``BsubBatchSettings``) - - -.. autosummary:: - - JsrunSettings.set_num_rs - JsrunSettings.set_cpus_per_rs - JsrunSettings.set_gpus_per_rs - JsrunSettings.set_rs_per_host - JsrunSettings.set_tasks - JsrunSettings.set_tasks_per_rs - JsrunSettings.set_binding - JsrunSettings.make_mpmd - JsrunSettings.set_mpmd_preamble - JsrunSettings.update_env - JsrunSettings.set_erf_sets - JsrunSettings.format_env_vars - JsrunSettings.format_run_args - - -.. autoclass:: JsrunSettings - :inherited-members: - :undoc-members: - :members: BsubBatchSettings @@ -287,38 +319,6 @@ be launched as a batch on LSF systems. :undoc-members: :members: ------------------------------------------- - - -The following are ``RunSettings`` types that are supported on multiple -launchers - -MpirunSettings --------------- - -.. _openmpi_api: - -``MpirunSettings`` are for launching with OpenMPI. ``MpirunSettings`` are -supported on Slurm, PBSpro, and Cobalt. - - -.. autosummary:: - - MpirunSettings.set_cpus_per_task - MpirunSettings.set_hostlist - MpirunSettings.set_tasks - MpirunSettings.set_task_map - MpirunSettings.make_mpmd - MpirunSettings.add_exe_args - MpirunSettings.format_run_args - MpirunSettings.format_env_vars - MpirunSettings.update_env - -.. autoclass:: MpirunSettings - :inherited-members: - :undoc-members: - :members: - Orchestrator ============ @@ -411,44 +411,45 @@ in an interactive allocation. :undoc-members: :exclude-members: create_cluster -Entity -====== -Ensemble --------- +Model +===== -.. currentmodule:: smartsim.entity.ensemble + +.. currentmodule:: smartsim.entity.model .. autosummary:: - Ensemble.__init__ - Ensemble.add_model - Ensemble.attach_generator_files - Ensemble.register_incoming_entity - Ensemble.enable_key_prefixing - Ensemble.query_key_prefixing + Model.__init__ + Model.attach_generator_files + Model.params_to_args + Model.register_incoming_entity + Model.enable_key_prefixing + Model.disable_key_prefixing + Model.query_key_prefixing -.. autoclass:: Ensemble +.. autoclass:: Model :members: :show-inheritance: :inherited-members: -Model ------ +Ensemble +======== -.. currentmodule:: smartsim.entity.model +.. currentmodule:: smartsim.entity.ensemble .. autosummary:: - Model.__init__ - Model.attach_generator_files - Model.register_incoming_entity - Model.enable_key_prefixing - Model.disable_key_prefixing - Model.query_key_prefixing + Ensemble.__init__ + Ensemble.models + Ensemble.add_model + Ensemble.attach_generator_files + Ensemble.register_incoming_entity + Ensemble.enable_key_prefixing + Ensemble.query_key_prefixing -.. autoclass:: Model +.. autoclass:: Ensemble :members: :show-inheritance: :inherited-members: @@ -476,19 +477,15 @@ Slurm .. _slurm_module_api: -.. note:: - This module is importable through smartsim e.g., from smartsim import slurm - - -.. currentmodule:: smartsim.launcher.slurm +.. currentmodule:: smartsim.slurm .. autosummary:: slurm.get_allocation slurm.release_allocation -.. automodule:: smartsim.launcher.slurm.slurm +.. automodule:: smartsim.slurm :members: @@ -499,7 +496,7 @@ Ray .. _ray_api: -``RayCluster`` is used to launch a Ray cluster +``RayCluster`` is used to launch a Ray cluster and can be launched as a batch or in an interactive allocation. .. autoclass:: RayCluster From a7dc1c49b2a52fbb9767742f92e6711022232f69 Mon Sep 17 00:00:00 2001 From: Sam Partee Date: Tue, 25 Jan 2022 12:00:55 -0800 Subject: [PATCH 5/8] Tutorials container --- docker-compose.yml | 7 +++++++ docker/dev/Dockerfile | 47 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 docker/dev/Dockerfile diff --git a/docker-compose.yml b/docker-compose.yml index 15d78ad77..827c66810 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,3 +9,10 @@ services: context: . dockerfile: ./docker/docs/dev/Dockerfile + tutorials: + image: smartsim-tutorials:dev-latest + build: + context: . + dockerfile: ./docker/dev/Dockerfile + ports: + - "8888:8888" \ No newline at end of file diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile new file mode 100644 index 000000000..cb507479a --- /dev/null +++ b/docker/dev/Dockerfile @@ -0,0 +1,47 @@ +FROM ubuntu:20.04 + +LABEL maintainer="Cray Labs" +ARG DEBIAN_FRONTEND="noninteractive" +ENV TZ=US/Seattle + +#TODO +# - add openmpi +# - add tutorials from other sections +# - add permissions to the craylabs user + +RUN useradd --create-home --shell /bin/bash craylabs + +RUN apt-get update \ + && apt-get install --no-install-recommends -y build-essential \ + git gcc make git-lfs wget \ + python3-pip python3 python3-dev cmake \ + && rm -rf /var/lib/apt/lists/* \ + && ln -s /usr/bin/python3 /usr/bin/python + +WORKDIR /home/craylabs +USER craylabs + +RUN git clone https://github.com/CrayLabs/SmartRedis.git --branch develop --depth=1 smartredis \ + && cd smartredis \ + && python -m pip install . + + +COPY . /home/craylabs/SmartSim +RUN cd SmartSim && NO_CHECKS=1 SMARTSIM_SUFFIX=dev python -m pip install . + +# TODO set SmartRedis version when thats put in +#RUN git clone https://github.com/CrayLabs/SmartSim.git --branch develop --depth=1 \ +# && cd SmartSim \ +# && python -m pip install .[ml,ray] + +# Jupyter +RUN python -m pip install ipython ipykernel \ + jupyter jupyterlab \ + && rm -rf ~/.cache/pip + +RUN echo "export PATH=/home/craylabs/.local/bin:$PATH" >> /home/craylabs/.bashrc && \ + export PATH=/home/craylabs/.local/bin:$PATH && \ + smart build --device cpu -v --no_tf + +SHELL ["bash", "-c"] +CMD ["/home/craylabs/.local/bin/jupyter", "lab", "--port", "8888", "--no-browser", "--ip=0.0.0.0"] \ No newline at end of file From 82f4913fc94fa0c1d5ad05f3c194124a805c6db1 Mon Sep 17 00:00:00 2001 From: Sam Partee Date: Wed, 26 Jan 2022 16:37:58 -0800 Subject: [PATCH 6/8] Fix torch patch and add tutorials docker tutorials dockerfile added to the composition. Currently uses develop, but a prod dockerfile will eventually be added as well --- docker/dev/Dockerfile | 14 +++++++++----- smartsim/_core/_install/builder.py | 4 +++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index cb507479a..65a69c4c8 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -9,25 +9,29 @@ ENV TZ=US/Seattle # - add tutorials from other sections # - add permissions to the craylabs user -RUN useradd --create-home --shell /bin/bash craylabs +RUN useradd --system --create-home --shell /bin/bash -g root -G sudo craylabs RUN apt-get update \ && apt-get install --no-install-recommends -y build-essential \ - git gcc make git-lfs wget \ + git gcc make git-lfs wget libopenmpi-dev openmpi-bin \ python3-pip python3 python3-dev cmake \ && rm -rf /var/lib/apt/lists/* \ && ln -s /usr/bin/python3 /usr/bin/python WORKDIR /home/craylabs -USER craylabs - RUN git clone https://github.com/CrayLabs/SmartRedis.git --branch develop --depth=1 smartredis \ + && chown -R craylabs:root smartredis \ && cd smartredis \ && python -m pip install . COPY . /home/craylabs/SmartSim -RUN cd SmartSim && NO_CHECKS=1 SMARTSIM_SUFFIX=dev python -m pip install . +RUN chown craylabs:root -R SmartSim +USER craylabs + +RUN cd SmartSim && \ + SMARTSIM_SUFFIX=dev python -m pip install . + # TODO set SmartRedis version when thats put in #RUN git clone https://github.com/CrayLabs/SmartSim.git --branch develop --depth=1 \ diff --git a/smartsim/_core/_install/builder.py b/smartsim/_core/_install/builder.py index 0c7ee550e..8cf783fdd 100644 --- a/smartsim/_core/_install/builder.py +++ b/smartsim/_core/_install/builder.py @@ -351,7 +351,9 @@ def _move_torch_libs(self): # retrieve torch shared libraries and copy to the # smartsim/_core/lib/backends/redisai_torch/lib dir - pip_torch_path = Path(site.getsitepackages()[0]) / "torch" + # self.torch_dir should be /path/to/torch/share/cmake/Torch + # so we take the great grandparent here + pip_torch_path = Path(self.torch_dir).parent.parent.parent pip_torch_lib_path = pip_torch_path / "lib" self.copy_dir(pip_torch_lib_path, ss_rai_torch_lib_path, set_exe=True) From 8920f4c59cc5b6f9478754d601964e26cc16be17 Mon Sep 17 00:00:00 2001 From: Sam Partee Date: Fri, 28 Jan 2022 11:25:21 -0800 Subject: [PATCH 7/8] Address review comments --- doc/conf.py | 2 +- doc/requirements-doc.txt | 10 ++++------ smartsim/_core/_install/buildenv.py | 2 +- smartsim/_core/_install/builder.py | 1 - 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index c1693b2ee..14d4e2952 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -17,7 +17,7 @@ # -- Project information ----------------------------------------------------- project = 'SmartSim' -copyright = '2022, Hewlett Packard Enterprise' +copyright = '2021-2022, Hewlett Packard Enterprise' author = 'Cray Labs' try: diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt index b165e3615..fee41447a 100644 --- a/doc/requirements-doc.txt +++ b/doc/requirements-doc.txt @@ -1,7 +1,5 @@ -sphinx==3.1.1 -breathe==4.27.0 -#sphinx_rtd_theme>=0.5.0 -sphinx-book-theme +sphinx==4.4.0 +breathe==4.31.0 +sphinx-book-theme==0.2.0 sphinx-fortran==1.1.1 -nbsphinx>=0.8.2 -Jinja2==2.11 \ No newline at end of file +nbsphinx>=0.8.2 \ No newline at end of file diff --git a/smartsim/_core/_install/buildenv.py b/smartsim/_core/_install/buildenv.py index 04e2526d9..920298474 100644 --- a/smartsim/_core/_install/buildenv.py +++ b/smartsim/_core/_install/buildenv.py @@ -418,7 +418,7 @@ def _torch_site_path(): if not torch_path: torch_path = _torch_site_path() if not torch_path: - raise SetupError("Could not located torch cmake path") + raise SetupError("Could not locate torch cmake path") return str(torch_path) @staticmethod diff --git a/smartsim/_core/_install/builder.py b/smartsim/_core/_install/builder.py index 8cf783fdd..757b191a1 100644 --- a/smartsim/_core/_install/builder.py +++ b/smartsim/_core/_install/builder.py @@ -1,7 +1,6 @@ import os import re import shutil -import site import stat import subprocess import sys From 72d34d6bdd6ca2244747f46f6d4638a055b13d1e Mon Sep 17 00:00:00 2001 From: Sam Partee Date: Fri, 28 Jan 2022 12:02:44 -0800 Subject: [PATCH 8/8] Add makefile stub --- Makefile | 10 +++++++++- docker/dev/Dockerfile | 6 ------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 9f331c95f..671a88124 100644 --- a/Makefile +++ b/Makefile @@ -90,7 +90,7 @@ check-lint: docs: @cd doc; make html -# help: docks - generate project documentation with docker +# help: docks - generate project documentation with docker .PHONY: docks docks: @if [ -d "docs" ]; then rm -rf docs; fi @@ -106,6 +106,14 @@ cov: @coverage html @echo if data was present, coverage report is in ./htmlcov/index.html + +# help: tutorials-dev - Build and start a docker container to run the tutorials +.PHONY: tutorials-dev +tutorials-dev: + @docker compose build tutorials + @docker run -p 8888:8888 smartsim-tutorials:dev-latest + + # help: # help: Test # help: ------- diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 65a69c4c8..8c237f8e6 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -4,11 +4,6 @@ LABEL maintainer="Cray Labs" ARG DEBIAN_FRONTEND="noninteractive" ENV TZ=US/Seattle -#TODO -# - add openmpi -# - add tutorials from other sections -# - add permissions to the craylabs user - RUN useradd --system --create-home --shell /bin/bash -g root -G sudo craylabs RUN apt-get update \ @@ -32,7 +27,6 @@ USER craylabs RUN cd SmartSim && \ SMARTSIM_SUFFIX=dev python -m pip install . - # TODO set SmartRedis version when thats put in #RUN git clone https://github.com/CrayLabs/SmartSim.git --branch develop --depth=1 \ # && cd SmartSim \