add a basic set of functional tests

.dockerignore

@@ -20,6 +20,13 @@ README.rst
 *.py[cod]
 **/__pycache__/
 
+# test and coverage output
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+coverage.xml
+
 # celery
 celerybeat-schedule
 celerybeat.pid

.github/workflows/ci.yml

@@ -9,6 +9,19 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
+    services:
+      postgresql:
+        image: postgres:16
+        env:
+          POSTGRES_USER: test
+          POSTGRES_PASSWORD: test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U test"
+          --health-interval 1s
+          --health-timeout 5s
+          --health-retries 30
 
     steps:
     - uses: actions/checkout@v4
@@ -21,7 +34,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements-dev.txt
+        pip install -r requirements.txt -r requirements-dev.txt
 
     - name: Check formatting with black
       run: |
@@ -34,6 +47,9 @@ jobs:
     - name: Run basic syntax check
       run: |
         python -m py_compile pypistats/*.py pypistats/**/*.py
+
+    - name: Run functional tests
+      run: python -m pytest -q
 
     - name: Success
       run: |

Makefile

@@ -8,6 +8,13 @@ check-fmt:
 	docker-compose run --rm web isort . --check-only
 	docker-compose run --rm web black . --check
 
+# run functional tests in the development image against PostgreSQL
+.PHONY: test
+test:
+	docker compose up -d --wait postgresql
+	docker compose run --build --rm --no-deps -T --entrypoint python web -m pytest -q \
+		--postgresql-host=postgresql --postgresql-port=5432 --postgresql-user=admin --postgresql-password=root
+
 # launch the application in docker-compose
 .PHONY: pypistats
 pypistats:

README.rst

@@ -31,3 +31,15 @@ Development
 
 2. Run ``make pypistats`` to launch a complete development environment using docker-compose.
 
+Functional tests
+----------------
+
+After setting up ``.env`` as above, run:
+
+.. code-block:: bash
+
+   make test
+
+This runs the tests in Docker Compose using isolated PostgreSQL databases and
+reports code coverage in the terminal and at ``htmlcov/index.html``.
+PostgreSQL remains running afterward.

docker-compose.yml

@@ -85,6 +85,11 @@ services:
       - POSTGRES_USER=admin
       - POSTGRES_PASSWORD=root
       - POSTGRES_DB=pypistats
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
+      interval: 2s
+      timeout: 5s
+      retries: 15
     ports:
       - "5433:5432"
     volumes:

pypistats/config.py

@@ -68,6 +68,7 @@ class TestConfig(Config):
     ENV = "dev"
     TESTING = True
     WTF_CSRF_ENABLED = False  # Allows form testing
+    RATELIMIT_ENABLED = False
 
 
 configs = {"development": DevConfig, "local": LocalConfig, "production": ProdConfig, "test": TestConfig}

pypistats/tasks/backfill.py

@@ -144,7 +144,8 @@ def backfill_sequential(
             # as it will use wrong date calculations
             result = etl(date_str, purge=False, use_sqlite=True, update_recent=False)
             results[date_str] = result
-            last_successful_date = date_str
+            if result["downloads"]["success"]:
+                last_successful_date = date_str
 
             # Add delay between days
             if current < end and delay_seconds > 0:

pyproject.toml

@@ -40,3 +40,9 @@ line_length = 120
 [build-system]
 requires = ["setuptools>=61.0", "wheel"]
 build-backend = "setuptools.build_meta"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "--cov=pypistats --cov-report=term-missing --cov-report=html"
+postgresql_user = "test"
+postgresql_password = "test"

requirements-dev.in

@@ -3,6 +3,12 @@
 # To install: pip install -r requirements-dev.txt
 
 # Development tools only - production deps are in requirements.txt
+-c requirements.txt
 black
 isort>=5.3
 pip-tools
+pytest
+pytest-cov
+pytest-flask
+pytest-postgresql
+psycopg[binary]

tests/conftest.py

@@ -0,0 +1,154 @@
+"""Application fixtures; pytest plugins own the client and database lifecycle."""
+
+import datetime
+import uuid
+from copy import deepcopy
+from unittest.mock import Mock
+from unittest.mock import create_autospec
+
+import pytest
+from flask_migrate import upgrade
+from pytest_postgresql import factories
+from sqlalchemy.engine import URL
+
+postgresql_server = factories.postgresql_noproc(dbname="pypistats_test_" + uuid.uuid4().hex)
+postgresql = factories.postgresql("postgresql_server")
+
+
+@pytest.fixture
+def app(postgresql, postgresql_server, monkeypatch):
+    database_url = URL.create(
+        "postgresql",
+        username=postgresql.info.user,
+        password=postgresql_server.password,
+        host=postgresql.info.host,
+        port=postgresql.info.port,
+        database=postgresql.info.dbname,
+    ).render_as_string(hide_password=False)
+    for key, value in {
+        "DATABASE_URL": database_url,
+        "REDIS_URL": "memory://",
+        "ENV": "test",
+        "BASIC_AUTH_USER": "test",
+        "BASIC_AUTH_PASSWORD": "test-password",
+        "PYPISTATS_SECRET": "functional-tests",
+    }.items():
+        monkeypatch.setenv(key, value)
+
+    from pypistats.extensions import db
+    from pypistats.run import app
+
+    # Reject an app imported with another database before touching its schema.
+    assert app.config["SQLALCHEMY_DATABASE_URI"] == database_url
+    with app.app_context():
+        upgrade()
+    yield app
+    with app.app_context():
+        db.session.remove()
+        db.engine.dispose()
+
+
+@pytest.fixture
+def date():
+    return datetime.date.today() - datetime.timedelta(days=1)
+
+
+@pytest.fixture(autouse=True)
+def sample_data(app, date):
+    from pypistats.extensions import db
+    from pypistats.models.download import RecentDownloadCount
+    from pypistats.views.general import MODELS
+
+    for package, counts in {
+        "sample-package": [(date - datetime.timedelta(days=1), 20), (date, 10)],
+        "sample-other": [(date, 40)],
+        "__all__": [(date - datetime.timedelta(days=1), 20), (date, 50)],
+    }.items():
+        for day, downloads in counts:
+            for model, category in zip(MODELS, ["without_mirrors", "3", "3.13", "Linux"]):
+                db.session.add(model(package=package, date=day, category=category, downloads=downloads))
+            db.session.add(MODELS[0](package=package, date=day, category="with_mirrors", downloads=downloads * 2))
+        for period in ["day", "week", "month"]:
+            downloads = counts[-1][1] if period == "day" else sum(count for _, count in counts)
+            db.session.add(RecentDownloadCount(package=package, category=period, downloads=downloads))
+    db.session.commit()
+
+
+@pytest.fixture(autouse=True)
+def pypi_request(app, monkeypatch):
+    metadata = {
+        "info": {
+            "package_url": "https://pypi.org/project/sample-package/",
+            "home_page": None,
+            "project_urls": {"Homepage": "https://example.org/"},
+            "author": None,
+            "author_email": '"Doe, Jane" <jane@example.org>',
+            "license": "MIT",
+            "summary": "A sample <package> & its tools",
+            "version": "1.2.3",
+            "requires_dist": ["requests>=2", 'pytest; extra == "test"'],
+        }
+    }
+    get = Mock(return_value=Mock(json=lambda: deepcopy(metadata)))
+    monkeypatch.setattr("pypistats.views.general.requests.get", get)
+    monkeypatch.setattr(
+        "requests.sessions.Session.request", Mock(side_effect=AssertionError("Unexpected external HTTP request"))
+    )
+    return get
+
+
+@pytest.fixture
+def pypi(app, tmp_path, monkeypatch):
+    from pypistats.tasks import pypi
+
+    monkeypatch.setattr(pypi.tempfile, "gettempdir", lambda: str(tmp_path))
+    monkeypatch.setattr(pypi, "get_google_credentials", lambda: (None, "test-project"))
+    monkeypatch.setattr(pypi, "BATCH_SIZE", 3)
+    return pypi
+
+
+@pytest.fixture
+def bigquery(pypi, monkeypatch):
+    client = create_autospec(pypi.bigquery.Client, spec_set=True)
+    job = create_autospec(pypi.bigquery.QueryJob, instance=True, spec_set=True)
+    client.return_value.query.return_value = job
+    monkeypatch.setattr(pypi.bigquery, "Client", client)
+    return job.result
+
+
+@pytest.fixture
+def update(pypi, bigquery, date):
+    def run(downloads=80, minor="3.12", system="Windows"):
+        # Interleaved categories include full and partial batches. Invalid records
+        # must not stop valid records from reaching the API or inflate __all__.
+        rows = [
+            ("sample-package", "overall", "without_mirrors", downloads),
+            ("sample-package", "overall", "with_mirrors", downloads + 20),
+            ("sample-package", "python_major", "3", downloads - 7),
+            ("sample-package", "python_minor", "3.13", downloads - 12),
+            ("sample-package", "system", "Linux", downloads - 11),
+            ("sample-other", "overall", "without_mirrors", 20),
+            ("sample-other", "overall", "with_mirrors", 40),
+            ("sample-other", "python_major", "3", 20),
+            ("sample-other", "python_minor", "3.13", 20),
+            ("sample-other", "system", "Linux", 20),
+            ("sample-package", "python_major", None, 7),
+            ("sample-package", "python_minor", minor, 5),
+            ("sample-package", "python_minor", None, 7),
+            ("sample-package", "system", system, 8),
+            ("sample-package", "system", "other", 3),
+            ("sample-package", "python_major", "", 1),
+            ("sample-package", "python_minor", ".", 1),
+            ("sample-package", "python_minor", "3.100", 1),
+            ("x" * 129, "overall", "without_mirrors", 1),
+        ]
+        columns = {"package": 0, "category_label": 1, "category": 2, "downloads": 3}
+        bigquery.return_value = iter(pypi.bigquery.Row(values, columns) for values in rows)
+        result = pypi.etl.apply(args=(str(date),), throw=True).get()
+        assert result["downloads"]["success"]
+        assert result["downloads"]["rows_processed"] == len(rows)
+        assert result["downloads"]["batches_processed"] > 0
+        assert all(result["recent"][period] for period in ["day", "week", "month"])
+        return result
+
+    return run

tests/test_backfill.py

@@ -0,0 +1,89 @@
+"""Backfill range selection and database effects."""
+
+import datetime
+from unittest.mock import Mock
+
+import pytest
+
+
+@pytest.fixture
+def backfill(pypi, monkeypatch):
+    from pypistats.tasks import backfill
+
+    # Progress reporting needs a Celery result backend; it does not affect imports.
+    monkeypatch.setattr(backfill.backfill_sequential, "update_state", Mock())
+    return backfill
+
+
+def daily_rows(pypi, downloads, major="3"):
+    columns = {"package": 0, "category_label": 1, "category": 2, "downloads": 3}
+    return iter(
+        pypi.bigquery.Row(("sample-package", table, category, count), columns)
+        for table, category, count in [
+            ("overall", "without_mirrors", downloads),
+            ("overall", "with_mirrors", downloads + 20),
+            ("python_major", major, downloads),
+            ("python_minor", "3.13", downloads),
+            ("system", "Linux", downloads),
+        ]
+    )
+
+
+def test_date_and_month_ranges_include_boundaries(backfill):
+    assert backfill.get_date_ranges("2024-02-28", "2024-03-03", chunk_days=2) == [
+        ("2024-02-28", "2024-02-29"),
+        ("2024-03-01", "2024-03-02"),
+        ("2024-03-03", "2024-03-03"),
+    ]
+    assert backfill.get_month_ranges("2023-12", "2024-02") == [
+        ("2023-12-01", "2023-12-31"),
+        ("2024-01-01", "2024-01-31"),
+        ("2024-02-01", "2024-02-29"),
+    ]
+
+
+def test_backfill_fills_missing_days_skips_existing_and_preserves_history(
+    backfill, pypi, bigquery, postgresql, client, date
+):
+    start = date - datetime.timedelta(days=3)
+    old = date - datetime.timedelta(days=500)
+    postgresql.execute("INSERT INTO overall VALUES (%s, %s, %s, %s)", (old, "sample-package", "without_mirrors", 7))
+    postgresql.commit()
+    bigquery.side_effect = [daily_rows(pypi, 30), daily_rows(pypi, 40)]
+
+    result = backfill.backfill_sequential.apply(
+        args=(str(start), str(date)), kwargs={"delay_seconds": 0, "skip_existing": True}, throw=True
+    ).get()
+    for day in (start, start + datetime.timedelta(days=1)):
+        assert result[str(day)]["downloads"]["success"]
+    for day in (date - datetime.timedelta(days=1), date):
+        assert result[str(day)]["skipped"]
+    assert bigquery.call_count == 2
+    response = client.get("/api/packages/sample-package/overall?mirrors=false")
+    assert response.status_code == 200
+    assert [(row["date"], row["downloads"]) for row in response.json["data"]] == [
+        (str(old), 7),
+        (str(start), 30),
+        (str(start + datetime.timedelta(days=1)), 40),
+        (str(date - datetime.timedelta(days=1)), 20),
+        (str(date), 10),
+    ]
+    assert result["recent_stats_updated"]["day"]
+    assert client.get("/api/packages/sample-package/recent").json["data"]["last_day"] == 40
+
+
+def test_backfill_recent_uses_last_successful_import(backfill, pypi, bigquery, client, date):
+    start = date - datetime.timedelta(days=3)
+    end = start + datetime.timedelta(days=1)
+    # The second day's PostgreSQL transfer fails after staging overall.
+    bigquery.side_effect = [daily_rows(pypi, 30), daily_rows(pypi, 900, major="too-long")]
+
+    result = backfill.backfill_sequential.apply(
+        args=(str(start), str(end)), kwargs={"delay_seconds": 0}, throw=True
+    ).get()
+    assert result[str(start)]["downloads"]["success"]
+    assert not result[str(end)]["downloads"]["success"]
+    assert result["recent_stats_updated"]["day"]
+    assert client.get("/api/packages/sample-package/recent").json["data"]["last_day"] == 30
+    history = client.get("/api/packages/sample-package/overall?mirrors=false").json["data"]
+    assert str(end) not in {row["date"] for row in history}