Update, Apply, and Automate Python Linting

.github/workflows/run_tests.yml

@@ -118,6 +118,16 @@ jobs:
         if: (matrix.py_v == '3.10')
         run: smart build --device cpu -v
 
+      - name: Run mypy
+        # TF 2.6.2 has a dep conflict with new mypy versions
+        if: (matrix.rai != '1.2.5')
+        run: |
+          python -m pip install .[mypy]
+          make check-mypy
+
+      - name: Run Pylint
+        run: make check-lint
+
       - name: Run Pytest
         run: |
           echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV
@@ -128,10 +138,3 @@ jobs:
         with:
           fail_ci_if_error: true
           files: ./coverage.xml
-
-      - name: Run mypy
-        # TF 2.6.2 has a dep conflict with new mypy versions
-        if: (matrix.rai != '1.2.5')
-        run: |
-          python -m pip install .[mypy]
-          make check-mypy

.pylintrc

@@ -1,3 +1,29 @@
+# BSD 2-Clause License
+#
+# Copyright (c) 2021-2023 Hewlett Packard Enterprise
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 [MASTER]
 
 # Use multiple processes to speed up Pylint.
@@ -10,20 +36,17 @@ persistent=yes
 # user-friendly hints instead of false-positive error messages.
 suggestion-mode=yes
 
+# Ignore problematic extensions
+extension-pkg-whitelist=pydantic,
+
+# Dirs where we do not care about style
+ignore-paths=smartsim/ml/torch,
+             smartsim/ml/tf
+
 
 [MESSAGES CONTROL]
 
 disable=logging-fstring-interpolation,
-        print-statement,
-        parameter-unpacking,
-        unpacking-in-except,
-        old-raise-syntax,
-        backtick,
-        long-suffix,
-        old-ne-operator,
-        old-octal-literal,
-        import-star-module-level,
-        non-ascii-bytes-literal,
         raw-checker-failed,
         bad-inline-option,
         locally-disabled,
@@ -32,69 +55,6 @@ disable=logging-fstring-interpolation,
         useless-suppression,
         deprecated-pragma,
         use-symbolic-message-instead,
-        apply-builtin,
-        basestring-builtin,
-        buffer-builtin,
-        cmp-builtin,
-        coerce-builtin,
-        execfile-builtin,
-        file-builtin,
-        long-builtin,
-        raw_input-builtin,
-        reduce-builtin,
-        standarderror-builtin,
-        unicode-builtin,
-        xrange-builtin,
-        coerce-method,
-        delslice-method,
-        getslice-method,
-        setslice-method,
-        no-absolute-import,
-        old-division,
-        dict-iter-method,
-        dict-view-method,
-        next-method-called,
-        metaclass-assignment,
-        indexing-exception,
-        raising-string,
-        reload-builtin,
-        oct-method,
-        hex-method,
-        nonzero-method,
-        cmp-method,
-        input-builtin,
-        round-builtin,
-        intern-builtin,
-        unichr-builtin,
-        map-builtin-not-iterating,
-        zip-builtin-not-iterating,
-        range-builtin-not-iterating,
-        filter-builtin-not-iterating,
-        using-cmp-argument,
-        eq-without-hash,
-        div-method,
-        idiv-method,
-        rdiv-method,
-        exception-message-attribute,
-        invalid-str-codec,
-        sys-max-int,
-        bad-python3-import,
-        deprecated-string-function,
-        deprecated-str-translate-call,
-        deprecated-itertools-function,
-        deprecated-types-field,
-        next-method-defined,
-        dict-items-not-iterating,
-        dict-keys-not-iterating,
-        dict-values-not-iterating,
-        deprecated-operator-function,
-        deprecated-urllib-function,
-        xreadlines-attribute,
-        deprecated-sys-function,
-        exception-escape,
-        comprehension-escape
-        bad-continuation,
-        invalid-name,
         too-many-instance-attributes,
         too-many-arguments,
         unused-argument,
@@ -104,18 +64,27 @@ disable=logging-fstring-interpolation,
         missing-function-docstring,
         too-many-branches,
         too-many-nested-blocks,
-        no-self-use,
         no-else-break,
         broad-except,
-        pointless-string-statement
+        pointless-string-statement,
+        too-few-public-methods,
+        fixme,
+        broad-exception-raised,
+        duplicate-code, # TODO: remove this after fixing duplicate code
 
 enable=useless-object-inheritance,
        unused-variable,
        unused-import,
-       undefined-variable
+       unused-argument,
+       undefined-variable,
        not-callable,
        arguments-differ,
-       redefined-outer-name
+       redefined-outer-name,
+       bare-except,
+
+load-plugins=pylint.extensions.no_self_use,
+             pylint.extensions.eq_without_hash,
+             pylint.extensions.broad_try_clause,
 
 [REPORTS]
 
@@ -139,7 +108,7 @@ logging-format-style=new
 
 # Logging modules to check that the string format arguments are in logging
 # function parameter format.
-logging-modules=logging
+logging-modules=logging,
 
 
 [VARIABLES]
@@ -154,11 +123,11 @@ allow-global-unused-variables=yes
 # List of strings which can identify a callback function by name. A callback
 # name must start or end with one of those strings.
 callbacks=cb_,
-          _cb
+          _cb,
 
 # A regular expression matching the name of dummy variables (i.e. expected to
 # not be used).
-dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)
 
 # Argument names that match this expression will be ignored. Default to name
 # with leading underscore.
@@ -169,7 +138,10 @@ init-import=no
 
 # List of qualified module names which can have objects that can redefine
 # builtins.
-redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
+redefining-builtins-modules=six.moves,
+                            past.builtins,
+                            future.builtins,
+                            builtins,io
 
 
 [FORMAT]
@@ -188,18 +160,11 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=160
+max-line-length=88
 
 # Maximum number of lines in a module.
 max-module-lines=1000
 
-# List of optional constructs for which whitespace checking is disabled. `dict-
-# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
-# `trailing-comma` allows a space between comma and closing bracket: (a, ).
-# `empty-line` allows space-only lines.
-no-space-check=trailing-comma,
-               dict-separator
-
 # Allow the body of a class to be on the same line as the declaration if body
 # contains single statement.
 single-line-class-stmt=no
@@ -214,15 +179,14 @@ single-line-if-stmt=no
 # Naming style matching correct argument names.
 argument-naming-style=snake_case
 
-# Regular expression matching correct argument names. Overrides argument-
-# naming-style.
-#argument-rgx=
+# Regular expression matching correct argument names. Overrides argument-naming-style.
+# Same as `argument-naming-style=snake_case` but allow for two letter args names
+# argument-rgx=([^\W\dA-Z][^\WA-Z]{1,}|_[^\WA-Z]*|__[^\WA-Z\d_][^\WA-Z]+__)$
 
 # Naming style matching correct attribute names.
 attr-naming-style=snake_case
 
-# Regular expression matching correct attribute names. Overrides attr-naming-
-# style.
+# Regular expression matching correct attribute names. Overrides attr-naming-style.
 #attr-rgx=
 
 # Bad variable names which should always be refused, separated by a comma.
@@ -231,7 +195,7 @@ bad-names=foo,
           baz,
           toto,
           tutu,
-          tata
+          tata,
 
 # Naming style matching correct class attribute names.
 class-attribute-naming-style=any
@@ -250,9 +214,14 @@ function-naming-style=snake_case
 good-names=i,
            j,
            k,
+           v,
+           x,
+           e,
            ex,
-           Run,
-           _
+           db,
+           ar,
+           _,
+           fn,
 
 # Include a hint for the correct naming format with invalid-name.
 include-naming-hint=no
@@ -264,7 +233,8 @@ inlinevar-naming-style=any
 method-naming-style=snake_case
 
 # Naming style matching correct module names.
-module-naming-style=snake_case
+# usually snake_case
+module-naming-style=any
 
 # List of decorators that produce properties, such as abc.abstractproperty. Add
 # to this list to register other decorators that produce valid properties.
@@ -274,6 +244,9 @@ property-classes=abc.abstractproperty
 # Naming style matching correct variable names.
 variable-naming-style=snake_case
 
+# Same as `variable-naming-style=snake_case` but allow for two letter vars
+# variable-rgx=([^\W\dA-Z][^\WA-Z]{1,}|_[^\WA-Z]*|__[^\WA-Z\d_][^\WA-Z]+__)$
+
 
 [STRING]
 
@@ -329,18 +302,42 @@ preferred-modules=
 defining-attr-methods=__init__,
                       __new__,
                       setUp,
-                      __post_init__
+                      __post_init__,
 
 # List of member names, which should be excluded from the protected access
 # warning.
 exclude-protected=_asdict,
                   _fields,
                   _replace,
                   _source,
-                  _make
+                  _make,
 
 # List of valid names for the first argument in a class method.
 valid-classmethod-first-arg=cls
 
 # List of valid names for the first argument in a metaclass class method.
-valid-metaclass-classmethod-first-arg=cls
+valid-metaclass-classmethod-first-arg=mcs
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=9
+
+# Maximum number of locals for function / method body
+max-locals=19
+
+# Maximum number of return / yield for function / method body
+max-returns=11
+
+# Maximum number of branch for function / method body
+max-branches=20
+
+# Maximum number of statements in function / method body
+max-statements=50
+
+# Maximum number of statements in a try-block
+max-try-statements=7
+
+# Maximum level of inheritance, bumping up to account for db mixins
+max-parents=25

Makefile

@@ -183,3 +183,8 @@ test-cov:
 .PHONY: test-full
 test-full:
 	@python -m pytest --cov=./smartsim -vv --cov-config=${COV_FILE}
+
+# help: test-wlm                   - Run the wlm-specific tests
+.PHONY: test-wlm
+test-wlm:
+	@python -m pytest -vv tests/full_wlm/ tests/on_wlm

doc/changelog.rst

@@ -11,6 +11,22 @@ Jump to :ref:`SmartRedis Changelog <changelog>`
 SmartSim
 ========
 
+
+Development branch
+------------------
+
+To be released at some future point in time
+
+Description
+
+- Update linting support and apply to existing errors
+
+Detailed Notes
+
+- Update pylint dependency, update .pylintrc, mitigate non-breaking issues, suppress api breaks (PR311_)
+
+.. _PR311: https://github.com/CrayLabs/SmartSim/pull/311
+
 0.5.0
 ------------------