bulk upgrade python dependencies (#137)

* bulk upgrade python dependencies

tested nightly update job in full locally

* remove pins

* remove greenlet dep, no longer needed

Author: ewdurbin

CLAUDE.md

@@ -129,7 +129,7 @@ pypistats/
 - Flask & extensions (SQLAlchemy, Migrate, Login, WTF, Limiter, HTTPAuth)
 - Google Cloud BigQuery client
 - Celery & Redis
-- PostgreSQL (psycopg2)
+- PostgreSQL (psycopg 3)
 - Requests
 - Gunicorn (production server)
 
@@ -179,4 +179,4 @@ pypistats/
 - All timestamps are in UTC
 - Package statistics exclude known mirror downloads by default
 - Maximum lookback period is 180 days to manage database size
-- Uses BigQuery's public PyPI dataset (bigquery-public-data.pypi.file_downloads)
+- Uses BigQuery's public PyPI dataset (bigquery-public-data.pypi.file_downloads)

Dockerfile

@@ -17,7 +17,7 @@ RUN set -eux; \
     echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache;
 
 # Install system dependencies with cache mount
-# libpq-dev is required for psycopg2-binary compilation
+# libpq-dev supports building PostgreSQL client extensions if needed
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     --mount=type=cache,target=/var/lib/apt,sharing=locked \
     set -x \
@@ -93,4 +93,4 @@ COPY . /app/
 RUN python -c "from pypistats.application import create_app; create_app()" || true
 
 # Set the entrypoint script
-ENTRYPOINT ["./docker-entrypoint.sh"]
+ENTRYPOINT ["./docker-entrypoint.sh"]

Makefile

@@ -33,9 +33,10 @@ setup:
 	pip install -r requirements-dev.txt
 
 # update requirements files
+.PHONY: update-deps
 update-deps:
-	pip-compile --generate-hashes --resolver=backtracking requirements.in -o requirements.txt
-	pip-compile --allow-unsafe --generate-hashes --resolver=backtracking requirements-dev.in -o requirements-dev.txt
+	pip-compile --upgrade --generate-hashes --resolver=backtracking requirements.in -o requirements.txt
+	pip-compile --upgrade --allow-unsafe --generate-hashes --resolver=backtracking requirements-dev.in -o requirements-dev.txt
 
 
 # port forward flower

pypistats/config.py

@@ -8,7 +8,10 @@
 
 def get_db_uri():
     """Get the database URI."""
-    return os.environ.get("DATABASE_URL")
+    uri = os.environ.get("DATABASE_URL")
+    if uri and uri.startswith("postgresql://"):
+        return uri.replace("postgresql://", "postgresql+psycopg://", 1)
+    return uri
 
 
 class Config:

pypistats/tasks/pypi.py

@@ -7,9 +7,9 @@
 import time
 from contextlib import contextmanager
 
-import psycopg2
+import psycopg
 from google.cloud import bigquery
-from psycopg2.extras import execute_values
+from psycopg import sql
 
 from pypistats.extensions import celery
 
@@ -157,6 +157,16 @@ def process_batch_to_sqlite(cursor, table, rows):
         return False
 
 
+def copy_rows(cursor, table, rows, columns=("date", "package", "category", "downloads")):
+    """Bulk-load rows within the caller's transaction using PostgreSQL COPY."""
+    query = sql.SQL("COPY {} ({}) FROM STDIN").format(
+        sql.Identifier(table), sql.SQL(", ").join(map(sql.Identifier, columns))
+    )
+    with cursor.copy(query) as copy:
+        for row in rows:
+            copy.write_row(row)
+
+
 def transfer_sqlite_to_postgres(sqlite_cursor, date):
     """Transfer all data from SQLite to PostgreSQL in a single atomic transaction."""
     pg_conn, pg_cursor = get_connection_cursor()
@@ -208,12 +218,7 @@ def transfer_sqlite_to_postgres(sqlite_cursor, date):
                         break
 
                     # Insert this chunk into PostgreSQL
-                    insert_query = f"""
-                        INSERT INTO {table} (date, package, category, downloads)
-                        VALUES %s
-                    """
-                    # Smaller page_size to reduce memory usage when building SQL
-                    execute_values(pg_cursor, insert_query, chunk, page_size=1000)
+                    copy_rows(pg_cursor, table, chunk)
 
                     chunks_transferred += 1
                     offset += TRANSFER_CHUNK_SIZE
@@ -228,7 +233,7 @@ def transfer_sqlite_to_postgres(sqlite_cursor, date):
 
         return True
 
-    except psycopg2.Error as e:
+    except psycopg.Error as e:
         print(f"Error during PostgreSQL transfer: {e}")
         pg_conn.rollback()
         return False
@@ -444,15 +449,11 @@ def update_table(connection, cursor, table, rows, date):
         print(delete_query)
         cursor.execute(delete_query)
 
-    insert_query = f"""INSERT INTO {table} (date, package, category, downloads)
-            VALUES %s"""
-
     try:
-        print(insert_query)
-        execute_values(cursor, insert_query, rows)
+        copy_rows(cursor, table, rows)
         connection.commit()
         return True
-    except psycopg2.IntegrityError as e:
+    except psycopg.IntegrityError as e:
         connection.rollback()
         return False
 
@@ -478,18 +479,15 @@ def update_all_package_stats(date=None):
 
         delete_query = f"""DELETE FROM {table}
                 WHERE date = %s and package = '__all__'"""
-        insert_query = f"""INSERT INTO {table} (date, package, category, downloads)
-                VALUES %s"""
         try:
             print(delete_query)
             cursor.execute(delete_query, (date,))
-            print(insert_query)
             if values:  # Only insert if there are values
-                execute_values(cursor, insert_query, values)
+                copy_rows(cursor, table, values)
                 print(f"Inserted {len(values)} __all__ records into {table}")
             connection.commit()
             success[table] = True
-        except psycopg2.IntegrityError as e:
+        except psycopg.IntegrityError as e:
             print(f"Error updating __all__ for {table}: {e}")
             connection.rollback()
             success[table] = False
@@ -534,16 +532,13 @@ def update_recent_stats(date=None):
 
         delete_query = f"""DELETE FROM {recent_table}
                 WHERE category = '{period}'"""
-        insert_query = f"""INSERT INTO {recent_table}
-               (package, category, downloads) VALUES %s"""
         try:
             print(delete_query)
             cursor.execute(delete_query)
-            print(insert_query)
-            execute_values(cursor, insert_query, values)
+            copy_rows(cursor, recent_table, values, columns=("package", "category", "downloads"))
             connection.commit()
             success[period] = True
-        except psycopg2.IntegrityError as e:
+        except psycopg.IntegrityError as e:
             connection.rollback()
             success[period] = False
 
@@ -554,7 +549,7 @@ def update_recent_stats(date=None):
 
 def get_connection_cursor():
     """Get a db connection cursor."""
-    connection = psycopg2.connect(os.environ["DATABASE_URL"])
+    connection = psycopg.connect(os.environ["DATABASE_URL"])
     cursor = connection.cursor()
     return connection, cursor
 
@@ -582,7 +577,7 @@ def purge_old_data(date=None):
             cursor.execute(delete_query)
             connection.commit()
             success[table] = True
-        except psycopg2.IntegrityError as e:
+        except psycopg.IntegrityError as e:
             connection.rollback()
             success[table] = False
 
@@ -594,7 +589,7 @@ def purge_old_data(date=None):
 def vacuum_analyze():
     """Vacuum and analyze the db."""
     connection, cursor = get_connection_cursor()
-    connection.set_isolation_level(0)
+    connection.autocommit = True
 
     results = {}
     start = time.time()

pypistats/views/general.py

@@ -148,7 +148,7 @@ def package_page(package):
     except ValueError:
         lookback = 180
 
-    start_date = str(datetime.date.today() - datetime.timedelta(lookback))
+    start_date = datetime.date.today() - datetime.timedelta(lookback)
 
     recent_downloads = RecentDownloadCount.query.filter_by(package=package).all()
 

requirements-dev.in

@@ -1,14 +1,13 @@
 # Development dependencies for pypistats.org
-# To update: pip-compile requirements-dev.in
+# To update all dependencies and hashes: make update-deps
 # To install: pip install -r requirements-dev.txt
 
 # Development tools only - production deps are in requirements.txt
 -c requirements.txt
 black
-isort>=5.3
+isort
 pip-tools
 pytest
 pytest-cov
 pytest-flask
 pytest-postgresql
-psycopg[binary]