Fix test requirements and simplified tox config (#295)

fix test requirements and simplified tox config

Co-authored-by: R Max Espinoza <me@rmax.dev>

Author: rmax

.dockerignore

@@ -40,4 +40,7 @@ nosetests.xml
 .pydevproject
 
 # JetBrains PyCharm IDE
-/.idea/
+/.idea/
+
+.venv
+.tags

.github/workflows/builds.yml

@@ -25,5 +25,5 @@ jobs:
       env: 
         TOXENV: build
       run: |
-        pip install -U tox
+        pip install -r requirements-tests.txt
         tox

.github/workflows/checks.yml

@@ -25,5 +25,5 @@ jobs:
       env: 
         TOXENV: ${{ matrix.env }}
       run: |
-        pip install -U tox
+        pip install -r requirements-tests.txt
         tox

.github/workflows/tests.yml

@@ -12,17 +12,25 @@ jobs:
       matrix:
         python-version: ["3.11.3"]
 
+    services:
+      redis:
+        image: redis
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    container: python:${{ matrix.python-version }}
+
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
     - name: Run pytest
-      env: 
+      env:
+        REDIS_HOST: redis
         TOXENV: pytest
+        TOX_TESTENV_PASSENV: REDIS_HOST
       run: |
-        pip install -U tox
+        pip install -r requirements-tests.txt
         tox

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.11-slim
+
+# Set working directory
+WORKDIR /app
+
+# Install tox and dependencies (replace 'your-requirements.txt' with your actual file)
+COPY requirements.txt .
+COPY requirements-tests.txt .
+RUN pip install -r requirements.txt -r requirements-tests.txt
+
+# Copy your project code
+COPY . .
+
+# Run Tox tests
+CMD ["tox"]
+

VERSION

@@ -1 +1 @@
-0.7.3
+0.7.3

docker-compose.yaml

@@ -0,0 +1,20 @@
+version: '3.8'
+
+services:
+  python:
+    build: .
+    command: tox -e security,flake8,pytest
+    environment:
+      REDIS_HOST: redis  # Use service name for hostname within docker network
+      REDIS_PORT: 6379
+      TOX_TESTENV_PASSENV: "REDIS_HOST REDIS_PORT"
+    volumes:
+      - ./:/app  # Mount your project directory into the container
+    depends_on:
+      - redis
+
+  redis:
+    image: redis:6.2-alpine
+    ports:
+      - "6379:6379"  # Map Redis port to host port
+

requirements-tests.txt

@@ -1,6 +1,6 @@
 # This packages are required to run all the tests.
 flake8
 mock
-pytest
+pytest>=6.0,<7
 pytest-cov
-tox
+tox>=3.0,<4

tests/test_scrapy_redis.py

@@ -15,7 +15,7 @@
 
 
 # allow test settings from environment
-REDIS_HOST = os.environ.get('REDIST_HOST', 'localhost')
+REDIS_HOST = os.environ.get('REDIS_HOST', 'localhost')
 REDIS_PORT = int(os.environ.get('REDIS_PORT', 6379))
 
 

tests/test_spiders.py

@@ -1,5 +1,6 @@
 import contextlib
 import mock
+import os
 import pytest
 
 from scrapy import signals
@@ -12,6 +13,10 @@
 )
 
 
+REDIS_HOST = os.environ.get('REDIS_HOST', 'localhost')
+REDIS_PORT = int(os.environ.get('REDIS_PORT', 6379))
+
+
 @contextlib.contextmanager
 def flushall(server):
     try:
@@ -29,7 +34,10 @@ class MyCrawlSpider(RedisCrawlSpider):
 
 
 def get_crawler(**kwargs):
-    return mock.Mock(settings=Settings(), **kwargs)
+    return mock.Mock(settings=Settings({
+        "REDIS_HOST": REDIS_HOST,
+        "REDIS_PORT": REDIS_PORT,
+    }), **kwargs)
 
 
 class TestRedisMixin_setup_redis(object):
@@ -124,6 +132,8 @@ def test_consume_urls_from_redis(start_urls_as_zset, start_urls_as_set, spider_c
     redis_key = 'start:urls'
     crawler = get_crawler()
     crawler.settings.setdict({
+        'REDIS_HOST': REDIS_HOST,
+        'REDIS_PORT': REDIS_PORT,
         'REDIS_START_URLS_KEY': redis_key,
         'REDIS_START_URLS_AS_ZSET': start_urls_as_zset,
         'REDIS_START_URLS_AS_SET': start_urls_as_set,

tox.ini

@@ -1,12 +1,14 @@
 [tox]
-envlist = security,flake8,py{38,39,310,311}-scrapy{26,27,28,29}-redis{42,43,44,45}
-minversion = 1.7.0
+envlist = 
+    security
+    flake8
+    py{38,39,310,311}-scrapy{26,27,28,29}-redis{42,43,44,45,46,50}
+minversion = 3.0.0
 
 [base]
 deps = 
-    scrapy>=2.6
-    redis>=4.2
-    six>=1.5.2
+    -r requirements-tests.txt
+    -r requirements.txt
 
 [testenv]
 basepython =
@@ -16,26 +18,26 @@ basepython =
     py311: python3.11
 deps = 
     {[base]deps}
-    mock
-    pytest
-    pytest-cov
+    scrapy26: scrapy~=2.6.0
+    scrapy27: scrapy~=2.7.0
+    scrapy28: scrapy~=2.8.0
+    scrapy29: scrapy~=2.9.0
+    scrapy210: scrapy~=2.10.0
+    scrapy211: scrapy~=2.11.0
+    redis42: redis~=4.2.0
+    redis43: redis~=4.3.0
+    redis44: redis~=4.4.0
+    redis45: redis~=4.5.0
+    redis46: redis~=4.6.0
+    redis50: redis~=5.0.0
 commands = 
-    scrapy26: pip install scrapy==2.6.3
-    scrapy27: pip install scrapy==2.7.1
-    scrapy28: pip install scrapy==2.8.0
-    scrapy29: pip install scrapy==2.9.0
-    redis42: pip install redis==4.2.0
-    redis43: pip install redis==4.3.6
-    redis44: pip install redis==4.4.4
-    redis45: pip install redis==4.5.5
-    pip install .
     python -m pytest # --cov-report term --cov=scrapy_redis
 
 [testenv:flake8]
-basepython = python3.11
+basepython =
+    python3.11
 deps =
     {[base]deps}
-    flake8 # https://github.com/tholo/pytest-flake8/issues/81
 commands =
     flake8 --ignore=W503,E265,E731 docs/ tests/
 
@@ -46,31 +48,17 @@ deps =
 commands =
     bandit -r -c .bandit.yml src/ tests/
 
-[testenv:pylint]
-basepython = python3.11
-deps =
-    {[base]deps}
-    pylint==2.12.2
-commands =
-    pylint setup.py docs/ src/ tests/
-
 [testenv:pytest]
 basepython = python3.11
 deps = 
     {[testenv]deps}
-    scrapy==2.6.1
-    redis==4.2.2
-allowlist_externals = sudo
 commands =
-    sudo apt-get update
-    sudo apt-get install -y redis
-    sudo systemctl start redis-server
-    pip install .
     python -m pytest --cov-report term --cov=scrapy_redis
 
 [testenv:build]
 basepython=python3.11
 deps = 
     {[base]deps}
-commands = 
-    pip install .
+    build
+commands =
+    python -m build