Address #74 review: export coverage, CSV safety, and backfill memory

citation/export_data.py

@@ -46,6 +46,16 @@
     "year_published",
 ]
 
+# Spreadsheet applications evaluate cells starting with these characters as formulas:
+# https://owasp.org/www-community/attacks/CSV_Injection
+FORMULA_PREFIXES = ("=", "+", "-", "@", "\t", "\r")
+
+
+def neutralize_formula(value):
+    if isinstance(value, str) and value.startswith(FORMULA_PREFIXES):
+        return "'" + value
+    return value
+
 
 # Streaming CSV follows Django's documented pseudo-buffer pattern:
 # https://docs.djangoproject.com/en/5.2/howto/outputting-csv/
@@ -182,12 +192,12 @@ def get_publications(self):
                     order_by=("creators__family_name", "creators__given_name"),
                 )
             )
-        return publications
+        return publications.order_by("pk")
 
     def rows(self):
-        yield self.get_header()
+        yield [neutralize_formula(cell) for cell in self.get_header()]
         for publication in self.get_publications():
-            yield self.get_row(publication)
+            yield [neutralize_formula(cell) for cell in self.get_row(publication)]
 
     def write_all(self, file):
         writer = csv.writer(file, delimiter=",")
@@ -338,7 +348,7 @@ def get_platforms(publications):
         .rename(columns={"name": "raw_name"})
     )
     platform_df["name"] = "Platform (" + platform_df["raw_name"].map(str) + ")"
-    platform_df["name"][50:] = "Platform Other"
+    platform_df.loc[platform_df.index[50:], "name"] = "Platform Other"
 
     return publication_platform_df, platform_df
 
@@ -366,7 +376,7 @@ def get_sponsors(publications):
     )
     sponsor_df["name"] = sponsor_df["raw_name"]
     sponsor_df["name"] = "Sponsor (" + sponsor_df["name"].map(str) + ")"
-    sponsor_df["name"][50:] = "Sponsor Other"
+    sponsor_df.loc[sponsor_df.index[50:], "name"] = "Sponsor Other"
 
     return publication_sponsor_df, sponsor_df
 
@@ -468,7 +478,7 @@ def get_publication_row(publication):
                         Value(" "),
                         F("creators__family_name"),
                     ),
-                    ordering=("creators__family_name", "creators__given_name"),
+                    order_by=("creators__family_name", "creators__given_name"),
                 )
             )
         ),
@@ -495,7 +505,7 @@ def get_publication_row(publication):
         .join(platform_dummies)
         .join(sponsor_dummies)
     )
-    criteria = (df.dtypes == np.float) & pd.Series(
+    criteria = (df.dtypes == np.float64) & pd.Series(
         df.columns != "year_published", df.columns
     )
     df.loc[:, criteria] = df.loc[:, criteria].fillna(0.0)

citation/migrations/0038_backfill_publication_year_published.py

@@ -2,19 +2,28 @@
 
 from django.db import migrations
 
+BATCH_SIZE = 1000
 YEAR_PUBLISHED_REGEX = re.compile(r"(?<!\d)\d{4}(?!\d)")
 
 
 def backfill_year_published(apps, schema_editor):
     Publication = apps.get_model("citation", "Publication")
-    updates = []
-    for pub in Publication.objects.exclude(date_published_text="").iterator():
+    publications = (
+        Publication.objects.exclude(date_published_text="")
+        .only("pk", "date_published_text")
+        .iterator(chunk_size=BATCH_SIZE)
+    )
+    batch = []
+    for pub in publications:
         match = YEAR_PUBLISHED_REGEX.search(pub.date_published_text)
-        year = int(match.group(0)) if match else None
-        if year is not None:
-            pub.year_published = year
-            updates.append(pub)
-    Publication.objects.bulk_update(updates, ["year_published"], batch_size=1000)
+        if match is None:
+            continue
+        pub.year_published = int(match.group(0))
+        batch.append(pub)
+        if len(batch) >= BATCH_SIZE:
+            Publication.objects.bulk_update(batch, ["year_published"])
+            batch = []
+    Publication.objects.bulk_update(batch, ["year_published"])
 
 
 def noop(apps, schema_editor):