From 85ed6aff68da99ca518b3a4fb0108858a25257a3 Mon Sep 17 00:00:00 2001 From: Anton Suharev Date: Sat, 26 Sep 2026 17:23:14 -0700 Subject: [PATCH 1/8] test(export): cover filtering, categorical encoding, and quoting Every case runs through both write_all and stream and asserts that they agree. --- tests/test_export_data.py | 141 +++++++++++++++++++++++++++++++------- 1 file changed, 117 insertions(+), 24 deletions(-) diff --git a/tests/test_export_data.py b/tests/test_export_data.py index be84b3e..988cd4f 100644 --- a/tests/test_export_data.py +++ b/tests/test_export_data.py @@ -5,7 +5,16 @@ from django.test import TestCase from citation.export_data import PublicationCSVExporter -from citation.models import Author, Container, Publication, PublicationAuthors +from citation.models import ( + Author, + Container, + Platform, + Publication, + PublicationAuthors, + PublicationPlatforms, + PublicationSponsors, + Sponsor, +) class PublicationCSVExporterTests(TestCase): @@ -17,17 +26,12 @@ class PublicationCSVExporterTests(TestCase): EXPECTED_AUTHOR_NAMES = "Grace Alpha; Ada Zephyr" def setUp(self): - user = User.objects.create_user(username="csv-export-user") - container = Container.objects.create( + self.user = User.objects.create_user(username="csv-export-user") + self.container = Container.objects.create( name=self.CONTAINER_NAME, issn=self.CONTAINER_ISSN, ) - publication = Publication.objects.create( - title=self.PUBLICATION_TITLE, - date_published_text="2019", - container=container, - added_by=user, - ) + self.publication = self.create_publication(self.PUBLICATION_TITLE) for given_name, family_name in self.AUTHORS: author = Author.objects.create( given_name=given_name, @@ -35,11 +39,30 @@ def setUp(self): type=Author.INDIVIDUAL, ) PublicationAuthors.objects.create( - publication=publication, + publication=self.publication, author=author, role=PublicationAuthors.RoleChoices.AUTHOR, ) + def create_publication(self, title, **kwargs): + return Publication.objects.create( + title=title, + date_published_text="2019", + container=self.container, + added_by=self.user, + **kwargs, + ) + + def export_rows(self, exporter): + output = StringIO() + exporter.write_all(output) + + written = list(csv.reader(StringIO(output.getvalue()))) + streamed = list(csv.reader(StringIO("".join(exporter.stream())))) + + self.assertEqual(written, streamed) + return written + def test_rejects_nested_many_to_many_attribute_paths(self): self.assertTrue(PublicationCSVExporter.attribute_exists("platforms")) self.assertFalse(PublicationCSVExporter.attribute_exists("platforms__name")) @@ -47,6 +70,16 @@ def test_rejects_nested_many_to_many_attribute_paths(self): with self.assertRaisesRegex(AttributeError, "platforms__name"): PublicationCSVExporter(attributes=["platforms__name"]) + def test_rejects_unsupported_many_to_many_attributes(self): + for name in ("creators", "tags"): + with ( + self.subTest(name=name), + self.assertRaisesRegex( + AttributeError, f"Unsupported many-to-many attribute: {name}" + ), + ): + PublicationCSVExporter(attributes=[name]) + def test_write_and_stream_keep_each_publication_in_one_csv_row(self): attributes = [ "id", @@ -55,18 +88,78 @@ def test_write_and_stream_keep_each_publication_in_one_csv_row(self): "container__issn", "container__name", ] - exporter = PublicationCSVExporter(attributes=attributes) - output = StringIO() - exporter.write_all(output) - written_rows = list(csv.reader(StringIO(output.getvalue()))) - streamed_rows = list(csv.reader(StringIO("".join(exporter.stream())))) - - self.assertEqual(written_rows, streamed_rows) - self.assertEqual(written_rows[0], attributes) - self.assertEqual(len(written_rows), 2) - self.assertTrue(all(len(row) == len(attributes) for row in written_rows)) - self.assertEqual(written_rows[1][1], self.PUBLICATION_TITLE) - self.assertEqual(written_rows[1][2], self.EXPECTED_AUTHOR_NAMES) - self.assertEqual(written_rows[1][3], self.CONTAINER_ISSN) - self.assertEqual(written_rows[1][4], self.CONTAINER_NAME) + rows = self.export_rows(PublicationCSVExporter(attributes=attributes)) + + self.assertEqual(rows[0], attributes) + self.assertEqual(len(rows), 2) + self.assertTrue(all(len(row) == len(attributes) for row in rows)) + self.assertEqual( + rows[1][1:], + [ + self.PUBLICATION_TITLE, + self.EXPECTED_AUTHOR_NAMES, + self.CONTAINER_ISSN, + self.CONTAINER_NAME, + ], + ) + + def test_excludes_non_primary_publications(self): + self.create_publication("Secondary citation", is_primary=False) + + rows = self.export_rows(PublicationCSVExporter(attributes=["title"])) + + self.assertEqual(rows[1:], [[self.PUBLICATION_TITLE]]) + + def test_encodes_categorical_many_to_many_levels_as_columns(self): + netlogo = Platform.objects.create(name="NetLogo") + repast = Platform.objects.create(name="Repast") + Platform.objects.create(name="MASON") + for platform in (repast, netlogo): + PublicationPlatforms.objects.create( + publication=self.publication, platform=platform + ) + PublicationSponsors.objects.create( + publication=self.publication, sponsor=Sponsor.objects.create(name="NSF") + ) + + rows = self.export_rows( + PublicationCSVExporter(attributes=["title", "platforms", "sponsors"]) + ) + + self.assertEqual( + rows[0], + ["title", "platforms", "MASON", "NetLogo", "Repast", "sponsors", "NSF"], + ) + self.assertEqual( + rows[1], + [ + self.PUBLICATION_TITLE, + "NetLogo; Repast", + "False", + "True", + "True", + "NSF", + "True", + ], + ) + + def test_publication_without_authors_exports_an_empty_cell(self): + PublicationAuthors.objects.filter(publication=self.publication).delete() + + rows = self.export_rows( + PublicationCSVExporter(attributes=["title", "author_names"]) + ) + + self.assertEqual(rows[1], [self.PUBLICATION_TITLE, ""]) + + def test_quotes_embedded_newlines_and_quotes(self): + abstract = 'First line\nSecond "quoted" line,\r\nthird line' + self.publication.abstract = abstract + self.publication.save() + + rows = self.export_rows( + PublicationCSVExporter(attributes=["title", "abstract"]) + ) + + self.assertEqual(rows[1], [self.PUBLICATION_TITLE, abstract]) From 2901036041b5bc5d47e377aa51ad7f90eda2ca8f Mon Sep 17 00:00:00 2001 From: Anton Suharev Date: Sat, 26 Sep 2026 17:24:05 -0700 Subject: [PATCH 2/8] fix(export): order CSV rows by publication id Without an explicit order, rows followed PostgreSQL's physical order, which changes after updates. --- citation/export_data.py | 2 +- tests/test_export_data.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/citation/export_data.py b/citation/export_data.py index ef9e198..eb3c277 100644 --- a/citation/export_data.py +++ b/citation/export_data.py @@ -182,7 +182,7 @@ def get_publications(self): order_by=("creators__family_name", "creators__given_name"), ) ) - return publications + return publications.order_by("pk") def rows(self): yield self.get_header() diff --git a/tests/test_export_data.py b/tests/test_export_data.py index 988cd4f..ff7b4c6 100644 --- a/tests/test_export_data.py +++ b/tests/test_export_data.py @@ -144,6 +144,40 @@ def test_encodes_categorical_many_to_many_levels_as_columns(self): ], ) + def test_exports_every_primary_publication_in_id_order(self): + second = self.create_publication("Second") + third = self.create_publication("Third") + PublicationPlatforms.objects.create( + publication=second, platform=Platform.objects.create(name="NetLogo") + ) + # the update moves the first row to the end of the table's physical order + self.publication.abstract = "Updated abstract" + self.publication.save() + + rows = self.export_rows( + PublicationCSVExporter( + attributes=["id", "title", "author_names", "platforms"] + ) + ) + + self.assertEqual( + rows[0], ["id", "title", "author_names", "platforms", "NetLogo"] + ) + self.assertEqual( + rows[1:], + [ + [ + str(self.publication.pk), + self.PUBLICATION_TITLE, + self.EXPECTED_AUTHOR_NAMES, + "", + "False", + ], + [str(second.pk), "Second", "", "NetLogo", "True"], + [str(third.pk), "Third", "", "", "False"], + ], + ) + def test_publication_without_authors_exports_an_empty_cell(self): PublicationAuthors.objects.filter(publication=self.publication).delete() From 96da7f4f21f7b293fa6bca2e5d9ca7d805bc1f80 Mon Sep 17 00:00:00 2001 From: Anton Suharev Date: Sat, 26 Sep 2026 17:25:07 -0700 Subject: [PATCH 3/8] fix(export): neutralize spreadsheet formulas in CSV cells Prefix a single quote to text cells that start with =, +, -, @, tab, or carriage return, following the OWASP CSV injection guidance. --- citation/export_data.py | 14 ++++++++++++-- tests/test_export_data.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/citation/export_data.py b/citation/export_data.py index eb3c277..b327ed2 100644 --- a/citation/export_data.py +++ b/citation/export_data.py @@ -46,6 +46,16 @@ "year_published", ] +# Spreadsheet applications evaluate cells starting with these characters as formulas: +# https://owasp.org/www-community/attacks/CSV_Injection +FORMULA_PREFIXES = ("=", "+", "-", "@", "\t", "\r") + + +def neutralize_formula(value): + if isinstance(value, str) and value.startswith(FORMULA_PREFIXES): + return "'" + value + return value + # Streaming CSV follows Django's documented pseudo-buffer pattern: # https://docs.djangoproject.com/en/5.2/howto/outputting-csv/ @@ -185,9 +195,9 @@ def get_publications(self): return publications.order_by("pk") def rows(self): - yield self.get_header() + yield [neutralize_formula(cell) for cell in self.get_header()] for publication in self.get_publications(): - yield self.get_row(publication) + yield [neutralize_formula(cell) for cell in self.get_row(publication)] def write_all(self, file): writer = csv.writer(file, delimiter=",") diff --git a/tests/test_export_data.py b/tests/test_export_data.py index ff7b4c6..3e621aa 100644 --- a/tests/test_export_data.py +++ b/tests/test_export_data.py @@ -178,6 +178,38 @@ def test_exports_every_primary_publication_in_id_order(self): ], ) + def test_neutralizes_spreadsheet_formulas(self): + for payload in ( + "=cmd|' /C calc'!A1", + "+1+1", + "-2+3", + "@SUM(A1:A2)", + "\tTab", + "\rCR", + ): + with self.subTest(payload=payload): + self.publication.title = payload + self.publication.save() + + rows = self.export_rows(PublicationCSVExporter(attributes=["title"])) + + self.assertEqual(rows[1], ["'" + payload]) + + def test_neutralizes_formulas_only_at_the_start_of_a_cell(self): + formula = '=HYPERLINK("https://example.com")' + PublicationPlatforms.objects.create( + publication=self.publication, platform=Platform.objects.create(name=formula) + ) + self.publication.title = "Model = 1 - 2" + self.publication.save() + + rows = self.export_rows( + PublicationCSVExporter(attributes=["title", "platforms"]) + ) + + self.assertEqual(rows[0], ["title", "platforms", "'" + formula]) + self.assertEqual(rows[1], ["Model = 1 - 2", "'" + formula, "True"]) + def test_publication_without_authors_exports_an_empty_cell(self): PublicationAuthors.objects.filter(publication=self.publication).delete() From 7d6766c5d4f68387cb109889244792c53a2f4e64 Mon Sep 17 00:00:00 2001 From: Anton Suharev Date: Sat, 26 Sep 2026 17:26:03 -0700 Subject: [PATCH 4/8] test(graphviz): cover missing years and multi-author filtering Compare sorted primary key lists instead of sets so duplicate rows fail the assertions. --- tests/test_graphviz_data.py | 91 +++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 39 deletions(-) diff --git a/tests/test_graphviz_data.py b/tests/test_graphviz_data.py index 9958d2c..5f80338 100644 --- a/tests/test_graphviz_data.py +++ b/tests/test_graphviz_data.py @@ -16,56 +16,46 @@ ) +def pks(publications): + # a sorted list, unlike a set, exposes duplicate rows + return sorted(publication.pk for publication in publications) + + class GraphDataFilterTests(TestCase): def setUp(self): self.user = User.objects.create_user(username="graph-data-user") - container = Container.objects.create(name="Graph Data Journal") - ada = Author.objects.create( + self.container = Container.objects.create(name="Graph Data Journal") + self.ada = Author.objects.create( given_name="Ada", family_name="Modeler", type=Author.INDIVIDUAL, ) - grace = Author.objects.create( + self.grace = Author.objects.create( given_name="Grace", family_name="Researcher", type=Author.INDIVIDUAL, ) - self.in_range = Publication.objects.create( - title="In range", - date_published_text="2019", - status=Publication.Status.REVIEWED, - container=container, - added_by=self.user, - ) - PublicationAuthors.objects.create( - publication=self.in_range, - author=ada, - role=PublicationAuthors.RoleChoices.AUTHOR, - ) - self.out_of_range = Publication.objects.create( - title="Out of range", - date_published_text="2021", - status=Publication.Status.REVIEWED, - container=container, - added_by=self.user, - ) - PublicationAuthors.objects.create( - publication=self.out_of_range, - author=ada, - role=PublicationAuthors.RoleChoices.AUTHOR, + self.in_range = self.create_publication("In range", "2019", self.ada) + self.out_of_range = self.create_publication("Out of range", "2021", self.ada) + self.in_range_other_author = self.create_publication( + "In range, other author", "2019", self.grace ) - self.in_range_other_author = Publication.objects.create( - title="In range, other author", - date_published_text="2019", + + def create_publication(self, title, date_published_text, *authors): + publication = Publication.objects.create( + title=title, + date_published_text=date_published_text, status=Publication.Status.REVIEWED, - container=container, + container=self.container, added_by=self.user, ) - PublicationAuthors.objects.create( - publication=self.in_range_other_author, - author=grace, - role=PublicationAuthors.RoleChoices.AUTHOR, - ) + for author in authors: + PublicationAuthors.objects.create( + publication=publication, + author=author, + role=PublicationAuthors.RoleChoices.AUTHOR, + ) + return publication def test_filters_by_iso_year_without_mutating_input(self): criteria = { @@ -84,8 +74,8 @@ def test_filters_by_iso_year_without_mutating_input(self): ) self.assertEqual( - {publication.pk for publication in publications}, - {self.in_range.pk, self.in_range_other_author.pk}, + pks(publications), + sorted([self.in_range.pk, self.in_range_other_author.pk]), ) self.assertEqual(criteria, original_criteria) self.assertEqual(len(distribution), 1) @@ -103,11 +93,34 @@ def test_filters_by_author_without_mutating_input(self): publications = filtered_publications(criteria) self.assertEqual( - {publication.pk for publication in publications}, - {self.in_range.pk, self.out_of_range.pk}, + pks(publications), sorted([self.in_range.pk, self.out_of_range.pk]) ) self.assertEqual(criteria, original_criteria) + def test_filters_multi_author_publication_once_per_author(self): + shared = self.create_publication("Shared", "2020", self.ada, self.grace) + expected = { + "Ada Modeler": [self.in_range.pk, self.out_of_range.pk, shared.pk], + "Grace Researcher": [self.in_range_other_author.pk, shared.pk], + } + + for author_name, expected_pks in expected.items(): + with self.subTest(author_name=author_name): + publications = filtered_publications( + {"is_primary": True, "authors__name__exact": author_name} + ) + + self.assertEqual(pks(publications), sorted(expected_pks)) + + def test_require_year_excludes_publications_without_year(self): + no_year = self.create_publication("Unknown year", "", self.ada) + criteria = {"is_primary": True, "status": Publication.Status.REVIEWED} + + self.assertIn(no_year.pk, pks(filtered_publications(criteria))) + self.assertNotIn( + no_year.pk, pks(filtered_publications(criteria, require_year=True)) + ) + def test_counts_current_archive_categories_for_matching_publications(self): category = CodeArchiveUrlCategory.objects.create( category="Archive", From 2ba2f1482e678aa5059c4da2af08590313d5207b Mon Sep 17 00:00:00 2001 From: Anton Suharev Date: Sat, 26 Sep 2026 17:28:35 -0700 Subject: [PATCH 5/8] fix(export): restore the multi-file research export export() crashed on np.float under NumPy 2, and pandas 3 silently ignored the chained assignments that group platforms and sponsors beyond the top 50. Also move ArrayAgg to order_by, since ordering is deprecated in Django 5.2. --- citation/export_data.py | 8 ++-- tests/test_export_data.py | 99 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 102 insertions(+), 5 deletions(-) diff --git a/citation/export_data.py b/citation/export_data.py index b327ed2..6eaf727 100644 --- a/citation/export_data.py +++ b/citation/export_data.py @@ -348,7 +348,7 @@ def get_platforms(publications): .rename(columns={"name": "raw_name"}) ) platform_df["name"] = "Platform (" + platform_df["raw_name"].map(str) + ")" - platform_df["name"][50:] = "Platform Other" + platform_df.loc[platform_df.index[50:], "name"] = "Platform Other" return publication_platform_df, platform_df @@ -376,7 +376,7 @@ def get_sponsors(publications): ) sponsor_df["name"] = sponsor_df["raw_name"] sponsor_df["name"] = "Sponsor (" + sponsor_df["name"].map(str) + ")" - sponsor_df["name"][50:] = "Sponsor Other" + sponsor_df.loc[sponsor_df.index[50:], "name"] = "Sponsor Other" return publication_sponsor_df, sponsor_df @@ -478,7 +478,7 @@ def get_publication_row(publication): Value(" "), F("creators__family_name"), ), - ordering=("creators__family_name", "creators__given_name"), + order_by=("creators__family_name", "creators__given_name"), ) ) ), @@ -505,7 +505,7 @@ def get_publication_row(publication): .join(platform_dummies) .join(sponsor_dummies) ) - criteria = (df.dtypes == np.float) & pd.Series( + criteria = (df.dtypes == np.float64) & pd.Series( df.columns != "year_published", df.columns ) df.loc[:, criteria] = df.loc[:, criteria].fillna(0.0) diff --git a/tests/test_export_data.py b/tests/test_export_data.py index 3e621aa..74393a1 100644 --- a/tests/test_export_data.py +++ b/tests/test_export_data.py @@ -1,16 +1,24 @@ import csv from io import StringIO +from pathlib import Path +from tempfile import TemporaryDirectory +import pandas as pd from django.contrib.auth.models import User from django.test import TestCase -from citation.export_data import PublicationCSVExporter +from citation.export_data import PublicationCSVExporter, export from citation.models import ( Author, + CodeArchiveUrl, + CodeArchiveUrlCategory, Container, + ModelDocumentation, Platform, Publication, PublicationAuthors, + PublicationCitations, + PublicationModelDocumentations, PublicationPlatforms, PublicationSponsors, Sponsor, @@ -229,3 +237,92 @@ def test_quotes_embedded_newlines_and_quotes(self): ) self.assertEqual(rows[1], [self.PUBLICATION_TITLE, abstract]) + + +class ResearchExportTests(TestCase): + # more than 50 platforms and sponsors exercise the "Other" bucket + CATEGORY_COUNT = 51 + + def setUp(self): + user = User.objects.create_user(username="research-export-user") + container = Container.objects.create(name="Research Journal") + self.publication = Publication.objects.create( + title="Exported publication", + date_published_text="2019", + status=Publication.Status.REVIEWED, + container=container, + added_by=user, + ) + cited = Publication.objects.create( + title="Cited publication", + date_published_text="2018", + status=Publication.Status.REVIEWED, + container=container, + added_by=user, + ) + PublicationCitations.objects.create( + publication=self.publication, citation=cited + ) + PublicationAuthors.objects.create( + publication=self.publication, + author=Author.objects.create( + given_name="Ada", family_name="Lovelace", type=Author.INDIVIDUAL + ), + role=PublicationAuthors.RoleChoices.AUTHOR, + ) + CodeArchiveUrl.objects.create( + publication=self.publication, + category=CodeArchiveUrlCategory.objects.create( + category="Archive", subcategory="Research export test" + ), + status=CodeArchiveUrl.STATUS.available, + creator=user, + url="https://example.com/model", + ) + PublicationModelDocumentations.objects.create( + publication=self.publication, + model_documentation=ModelDocumentation.objects.create(name="ODD"), + ) + for index in range(self.CATEGORY_COUNT): + PublicationPlatforms.objects.create( + publication=self.publication, + platform=Platform.objects.create(name=f"Platform {index:02d}"), + ) + PublicationSponsors.objects.create( + publication=self.publication, + sponsor=Sponsor.objects.create(name=f"Sponsor {index:02d}"), + ) + + def test_export_writes_every_table(self): + with TemporaryDirectory() as directory: + export(directory) + + self.assertEqual( + sorted(path.name for path in Path(directory).iterdir()), + [ + "author.csv", + "codearchiveurl.csv", + "modeldocumentation.csv", + "platform.csv", + "publication.csv", + "publication_author.csv", + "publication_modeldocumentation.csv", + "publication_network.csv", + "publication_platform.csv", + "publication_sponsor.csv", + "sponsor.csv", + ], + ) + publications = pd.read_csv( + Path(directory, "publication.csv"), index_col="id" + ) + + exported = publications.loc[self.publication.pk] + self.assertEqual(exported["code_archival_status"], "ARCHIVED") + self.assertEqual(exported["year_published"], 2019) + self.assertEqual(exported["Documentation (ODD)"], 1) + self.assertEqual(exported["Platform Other"], 1) + self.assertEqual(exported["Sponsor Other"], 1) + self.assertEqual( + sum(column.startswith("Platform (") for column in publications.columns), 50 + ) From 49d0179299a42e9a71405b5e5270def30c8106ee Mon Sep 17 00:00:00 2001 From: Anton Suharev Date: Sat, 26 Sep 2026 17:33:36 -0700 Subject: [PATCH 6/8] perf(migrations): stream the year_published backfill The backfill kept every updated publication in memory before writing. On the production dump it now peaks at 100 MiB instead of 1045 MiB with identical results. 0038 has not run in production yet. --- ...038_backfill_publication_year_published.py | 23 +++++--- tests/test_migrations.py | 56 +++++++++++++++++++ 2 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 tests/test_migrations.py diff --git a/citation/migrations/0038_backfill_publication_year_published.py b/citation/migrations/0038_backfill_publication_year_published.py index 7ae60f2..32b89f6 100644 --- a/citation/migrations/0038_backfill_publication_year_published.py +++ b/citation/migrations/0038_backfill_publication_year_published.py @@ -2,19 +2,28 @@ from django.db import migrations +BATCH_SIZE = 1000 YEAR_PUBLISHED_REGEX = re.compile(r"(?= BATCH_SIZE: + Publication.objects.bulk_update(batch, ["year_published"]) + batch = [] + Publication.objects.bulk_update(batch, ["year_published"]) def noop(apps, schema_editor): diff --git a/tests/test_migrations.py b/tests/test_migrations.py new file mode 100644 index 0000000..0f6d5fb --- /dev/null +++ b/tests/test_migrations.py @@ -0,0 +1,56 @@ +from importlib import import_module +from unittest import mock + +from django.db import connection +from django.db.migrations.executor import MigrationExecutor +from django.test import TransactionTestCase + +backfill_migration = import_module( + "citation.migrations.0038_backfill_publication_year_published" +) + +BEFORE_BACKFILL = [("citation", "0037_publication_year_published")] +AFTER_BACKFILL = [("citation", "0038_backfill_publication_year_published")] + + +class BackfillYearPublishedMigrationTests(TransactionTestCase): + def migrate(self, targets): + executor = MigrationExecutor(connection) + executor.migrate(targets) + return executor.loader.project_state(targets).apps + + def tearDown(self): + self.migrate(AFTER_BACKFILL) + + def test_backfills_years_across_batches(self): + apps = self.migrate(BEFORE_BACKFILL) + user = apps.get_model("auth", "User").objects.create(username="backfill-user") + container = apps.get_model("citation", "Container").objects.create( + name="Backfill Journal" + ) + Publication = apps.get_model("citation", "Publication") + expected = { + "2019": 2019, + "JUN 2014": 2014, + "SEP-OCT 2007": 2007, + "": None, + "n.d.": None, + } + for index, date_published_text in enumerate(expected): + # historical models bypass Publication.save(), so year_published stays empty + Publication.objects.create( + title=f"Publication {index}", + date_published_text=date_published_text, + added_by_id=user.pk, + container_id=container.pk, + ) + + with mock.patch.object(backfill_migration, "BATCH_SIZE", 2): + apps = self.migrate(AFTER_BACKFILL) + + years = dict( + apps.get_model("citation", "Publication").objects.values_list( + "date_published_text", "year_published" + ) + ) + self.assertEqual(years, expected) From 905ea093ebb1b932495f1cfd70263424fdbc573a Mon Sep 17 00:00:00 2001 From: Anton Suharev Date: Sat, 26 Sep 2026 18:32:28 -0700 Subject: [PATCH 7/8] fix(export): neutralize line feeds and full-width formula prefixes The current OWASP CSV injection guidance also lists line feed and the full-width =, +, -, and @ used in some locales. --- citation/export_data.py | 17 +++++++++++++++-- tests/test_export_data.py | 2 ++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/citation/export_data.py b/citation/export_data.py index 6eaf727..f19c4f9 100644 --- a/citation/export_data.py +++ b/citation/export_data.py @@ -46,9 +46,22 @@ "year_published", ] -# Spreadsheet applications evaluate cells starting with these characters as formulas: +# Spreadsheet applications evaluate cells starting with these characters as formulas, +# including full-width variants in some locales: # https://owasp.org/www-community/attacks/CSV_Injection -FORMULA_PREFIXES = ("=", "+", "-", "@", "\t", "\r") +FORMULA_PREFIXES = ( + "=", + "+", + "-", + "@", + "\t", + "\r", + "\n", + "＝", + "＋", + "－", + "＠", +) def neutralize_formula(value): diff --git a/tests/test_export_data.py b/tests/test_export_data.py index 74393a1..199efeb 100644 --- a/tests/test_export_data.py +++ b/tests/test_export_data.py @@ -194,6 +194,8 @@ def test_neutralizes_spreadsheet_formulas(self): "@SUM(A1:A2)", "\tTab", "\rCR", + "\n=cmd|' /C calc'!A1", + "＝SUM(A1:A2)", ): with self.subTest(payload=payload): self.publication.title = payload From 60da0e518f80d3b18ccb2c2f61062f03d0b905ca Mon Sep 17 00:00:00 2001 From: Anton Suharev Date: Sat, 26 Sep 2026 18:34:19 -0700 Subject: [PATCH 8/8] fix(export): neutralize formulas in the multi-file export export() wrote database values straight to pandas to_csv(), bypassing the curator exporter's sanitizer. --- citation/export_data.py | 33 +++++++++++++++++++-------------- tests/test_export_data.py | 16 ++++++++++++++++ 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/citation/export_data.py b/citation/export_data.py index f19c4f9..ca53ba2 100644 --- a/citation/export_data.py +++ b/citation/export_data.py @@ -70,6 +70,10 @@ def neutralize_formula(value): return value +def write_csv(df, path): + df.map(neutralize_formula).to_csv(path) + + # Streaming CSV follows Django's documented pseudo-buffer pattern: # https://docs.djangoproject.com/en/5.2/howto/outputting-csv/ @@ -533,35 +537,36 @@ def remove_recoded(df): publications = get_queryset() publication_author_df, author_df = get_authors(publications) - publication_author_df.to_csv(path.joinpath("publication_author.csv")) - author_df.to_csv(path.joinpath("author.csv")) + write_csv(publication_author_df, path.joinpath("publication_author.csv")) + write_csv(author_df, path.joinpath("author.csv")) codearchiveurl_df = get_code_archive_urls(publications) - codearchiveurl_df.to_csv(path.joinpath("codearchiveurl.csv")) + write_csv(codearchiveurl_df, path.joinpath("codearchiveurl.csv")) publication_modeldocumentation_df, modeldocumentation_df = get_model_documentation( publications ) - publication_modeldocumentation_df.to_csv( - path.joinpath("publication_modeldocumentation.csv") + write_csv( + publication_modeldocumentation_df, + path.joinpath("publication_modeldocumentation.csv"), ) - remove_recoded(modeldocumentation_df).to_csv( - path.joinpath("modeldocumentation.csv") + write_csv( + remove_recoded(modeldocumentation_df), path.joinpath("modeldocumentation.csv") ) modeldocumentation_dummies_df = create_publication_modeldocumentation_dummies( publication_modeldocumentation_df, modeldocumentation_df ) publication_platform_df, platform_df = get_platforms(publications) - publication_platform_df.to_csv(path.joinpath("publication_platform.csv")) - remove_recoded(platform_df).to_csv(path.joinpath("platform.csv")) + write_csv(publication_platform_df, path.joinpath("publication_platform.csv")) + write_csv(remove_recoded(platform_df), path.joinpath("platform.csv")) platform_dummies_df = create_publication_platform_dummies( publication_platform_df, platform_df ) publication_sponsor_df, sponsor_df = get_sponsors(publications) - publication_sponsor_df.to_csv(path.joinpath("publication_sponsor.csv")) - remove_recoded(sponsor_df).to_csv(path.joinpath("sponsor.csv")) + write_csv(publication_sponsor_df, path.joinpath("publication_sponsor.csv")) + write_csv(remove_recoded(sponsor_df), path.joinpath("sponsor.csv")) sponsor_dummies_df = create_publication_sponsor_dummies( publication_sponsor_df, sponsor_df ) @@ -573,8 +578,8 @@ def remove_recoded(df): sponsor_dummies=sponsor_dummies_df, codearchiveurls=codearchiveurl_df, ) - publication_df.to_csv(path.joinpath("publication.csv")) + write_csv(publication_df, path.joinpath("publication.csv")) - get_publication_network(publications).to_csv( - path.joinpath("publication_network.csv") + write_csv( + get_publication_network(publications), path.joinpath("publication_network.csv") ) diff --git a/tests/test_export_data.py b/tests/test_export_data.py index 199efeb..ce9deb9 100644 --- a/tests/test_export_data.py +++ b/tests/test_export_data.py @@ -328,3 +328,19 @@ def test_export_writes_every_table(self): self.assertEqual( sum(column.startswith("Platform (") for column in publications.columns), 50 ) + + def test_export_neutralizes_spreadsheet_formulas(self): + formula = '=HYPERLINK("https://example.com")' + self.publication.title = formula + self.publication.save() + Platform.objects.filter(name="Platform 00").update(name=formula) + + with TemporaryDirectory() as directory: + export(directory) + publications = pd.read_csv( + Path(directory, "publication.csv"), index_col="id" + ) + platforms = pd.read_csv(Path(directory, "platform.csv")) + + self.assertEqual(publications.loc[self.publication.pk, "title"], "'" + formula) + self.assertIn("'" + formula, platforms["name"].to_list())