Release v4.26.0

.circleci/build-and-test/jobs.yml

@@ -59,6 +59,19 @@
       - docker-set-registry-owner
       - docker-compose-check
       - docker-compose-up-backend
+      - run:
+          name: Configure accelerated stale-submission E2E timeout
+          command: |
+            cd tdrs-backend
+            STALE_PARSE_TIMEOUT_SECONDS=60 \
+              STALE_PARSE_CHECK_INTERVAL_SECONDS=5 \
+              GO_PARSER_SHADOW_MODE=false \
+              docker-compose up -d --force-recreate web celery
+            docker-compose exec -T celery sh -c '
+              test "$STALE_PARSE_TIMEOUT_SECONDS" = 60 &&
+              test "$STALE_PARSE_CHECK_INTERVAL_SECONDS" = 5 &&
+              test "$GO_PARSER_SHADOW_MODE" = false
+            '
       - docker-compose-up-frontend
       - install-nodejs-machine
       - install-yarn-packages:
@@ -88,9 +101,122 @@
           command: |
             cd tdrs-backend
             docker-compose exec web python manage.py loaddata cypress/users cypress/data_files cypress/regions cypress/profile_editing_regions cypress/profile_editing_users cypress/feature_flags
+      - run:
+          name: Verify stale-submission lifecycle processing
+          command: |
+            cd tdrs-backend
+            # Celery can initially start before a fresh database has the
+            # django-celery-beat tables. Remove the schedule created by that
+            # initial default-configured beat, then restart it after migrations
+            # so the accelerated interval is persisted before Cypress runs.
+            docker-compose stop celery
+            docker-compose exec -T web python manage.py shell -c "
+            from django_celery_beat.models import PeriodicTask
+
+            PeriodicTask.objects.filter(name='Mark Stale Data Files Stuck').delete()
+            "
+            celery_start_time="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+            docker-compose start celery
+            for lifecycle_probe_attempt in $(seq 1 60); do
+              celery_logs="$(docker-compose logs --since "$celery_start_time" --no-color celery 2>/dev/null)"
+              if [[ "$celery_logs" == *"DatabaseScheduler: Schedule changed"* ]]; then
+                break
+              fi
+              if [ "$lifecycle_probe_attempt" -eq 60 ]; then
+                docker-compose logs --no-color --tail=300 celery
+                exit 1
+              fi
+              sleep 2
+            done
+            docker-compose exec -T celery python manage.py shell -c "
+            from django.conf import settings
+            from django_celery_beat.models import IntervalSchedule, PeriodicTask
+            from tdpservice.data_files.tasks import mark_stale_files_stuck
+
+            interval, _ = IntervalSchedule.objects.get_or_create(
+                every=5,
+                period=IntervalSchedule.SECONDS,
+            )
+            schedule = PeriodicTask.objects.get(name='Mark Stale Data Files Stuck')
+            schedule.interval = interval
+            schedule.crontab = None
+            schedule.solar = None
+            schedule.clocked = None
+            schedule.enabled = True
+            schedule.args = '[60]'
+            schedule.save()
+            result = mark_stale_files_stuck.delay(60)
+            marked_count = result.get(timeout=15)
+            assert isinstance(marked_count, int)
+            print(
+                'Lifecycle smoke task completed; '
+                f'marked {marked_count} file(s); '
+                f'Django timeout setting={settings.STALE_PARSE_TIMEOUT_SECONDS!r}.'
+            )
+            "
       - run:
           name: Run Cypress e2e tests
-          command: cd tdrs-frontend; yarn test:e2e-ci
+          command: |
+            cd tdrs-frontend
+            export CYPRESS_ADMIN_URL=http://localhost:3000/admin
+            export CYPRESS_STUCK_TIMEOUT_TEST=true
+
+            # Exercise lifecycle timeouts and reparsing before the upload-heavy
+            # suite puts sustained load on the single parser and LocalStack.
+            yarn test:e2e-ci --spec cypress/e2e/data-files/submission_lifecycle.feature
+
+            # The timeout scenario revokes a deliberately long parser task.
+            # Reset the disposable worker so that task cannot delay later specs.
+            cd ../tdrs-backend
+            docker-compose restart celery
+            cd ../tdrs-frontend
+
+            remaining_specs="$(find cypress/e2e -type f -name '*.feature' ! -path '*/submission_lifecycle.feature' | sort | paste -sd, -)"
+            test -n "$remaining_specs"
+            yarn test:e2e-ci --spec "$remaining_specs"
+      - run:
+          name: Show backend logs after an E2E failure
+          when: on_fail
+          command: |
+            cd tdrs-backend
+            docker-compose logs --no-color --tail=500 web celery localstack
+            docker-compose exec -T web python manage.py shell -c "
+            from django.conf import settings
+            from django_celery_beat.models import PeriodicTask
+            from django.db.models import Q
+            from tdpservice.data_files.models import DataFile
+
+            schedule = PeriodicTask.objects.get(name='Mark Stale Data Files Stuck')
+            print(
+                'Stale lifecycle diagnostics:',
+                {
+                    'django_timeout': settings.STALE_PARSE_TIMEOUT_SECONDS,
+                    'schedule_args': schedule.args,
+                    'schedule_enabled': schedule.enabled,
+                    'schedule_last_run_at': schedule.last_run_at,
+                    'schedule_total_run_count': schedule.total_run_count,
+                },
+            )
+            files = DataFile.objects.filter(
+                Q(original_filename='long_running_tanf_active_case.txt')
+                | Q(original_filename__startswith='accepted_tanf_reparse_')
+            )
+            for data_file in files:
+                summary = getattr(data_file, 'summary', None)
+                print(
+                    'Lifecycle DataFile:',
+                    {
+                        'id': data_file.id,
+                        'filename': data_file.original_filename,
+                        'state': data_file.state,
+                        'state_changed_at': data_file.state_changed_at,
+                        'parse_token': data_file.current_parse_token,
+                        'summary_status': getattr(summary, 'status', None),
+                    },
+                )
+            "
+            docker-compose exec -T celery sh -c \
+              "find /tmp -maxdepth 1 -type f -name '*.log' -size -5M -print -exec tail -n 200 {} \\;"
       - store_artifacts:
           path: tdrs-frontend/cypress/screenshots/
       - store_artifacts:

.codex/config.toml

@@ -1,22 +1,19 @@
 # Codex configuration for the TANF-app repository.
 # Reference: https://developers.openai.com/codex/config-advanced
 
-model                  = "gpt-5.5"
+model                  = "gpt-5.6"
 model_provider         = "openai"
 
-# Sandbox: allow writing within the workspace but not outside it.
-sandbox_mode         = "workspace-write"
-approval_policy      = "on-request"
-default_permissions  = "tdp-workspace"
+approval_policy = "on-request"
+default_permissions = "tdp-workspace"
+
+[permissions.tdp-workspace]
+extends = ":workspace"
 
-# ---------------------------------------------------------------------------
-# Filesystem permissions
-# ---------------------------------------------------------------------------
 [permissions.tdp-workspace.filesystem]
 glob_scan_max_depth = 12
 
 [permissions.tdp-workspace.filesystem.":workspace_roots"]
-"." = "write"
 "**/.env" = "deny"
 "**/.env.*" = "deny"
 "**/*.tfvars" = "deny"

.gitconfig

@@ -16,11 +16,10 @@
 	allowed = ./tdrs-backend/plg/mimir/manifest.yml:*
 	allowed = ./tdrs-backend/plg/tempo/manifest.yml:*
 	allowed = ./tdrs-backend/plg/deploy.sh:*
-	allowed = .*tdrs-backend/docker-compose.yml:[0-9]+:      - LOGIN_GOV_JWT_KEY=[$][{]JWT_KEY[}]
-	allowed = .*tdrs-backend/docker-compose.yml:[0-9]+:      - AWS_SECRET_ACCESS_KEY=test
-	allowed = ./tdrs-backend/docker-compose.yml:106
-	allowed = ./tdrs-backend/docker-compose.yml:362
-	allowed = ./tdrs-backend/docker-compose.yml:364
+	allowed = ./tdrs-backend/docker-compose.yml:[0-9]+:      - LOGIN_GOV_JWT_KEY=[$][{]JWT_KEY[}]
+	allowed = .*tdrs-backend/docker-compose.yml:[0-9]+:      - LOGIN_GOV_JWT_KEY=[$][{]JWT_KEY:-[}]
+	allowed = ./tdrs-backend/docker-compose.yml:[0-9]+:      - JWT_KEY=[$][{]JWT_KEY[}]
+	allowed = ./tdrs-backend/docker-compose.yml:[0-9]+:      - AWS_SECRET_ACCESS_KEY=test
 	allowed = ./tdrs-services/parser/cloudgov_start.sh:69
 	patterns = (A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}
 	patterns = (\"|')?(AWS|aws|Aws)?_?(SECRET|secret|Secret)?_?(ACCESS|access|Access)?_?(KEY|key|Key)(\"|')?\\s*(:|=>|=)\\s*(\"|')?[A-Za-z0-9/\\+=]{40}(\"|')?

.github/ISSUE_TEMPLATE/bug-template.md

@@ -3,7 +3,7 @@ name: Bug Report template
 about: Template for bug reporting
 title: ''
 labels: bug, dev
-assignees: kennymcnett, reitermb, victoriaatraft, elipe17
+assignees: 
 
 ---
 

.github/ISSUE_TEMPLATE/design-deliverable-issue-template.md

@@ -2,7 +2,7 @@
 name: Design Deliverable issue template
 about: Template for dev-ready design issues
 title: ''
-labels: Design Deliverable, Research & Design
+labels: Design Deliverable
 assignees: ''
 
 ---

.github/ISSUE_TEMPLATE/research-facilitation-issue-template.md

@@ -2,7 +2,7 @@
 name: Research facilitation issue template
 about: Template for research facilitation issues (1 per research session)
 title: ''
-labels: Research & Design, Research Facilitation
+labels: UX Research
 assignees: ''
 
 ---

.github/ISSUE_TEMPLATE/research-planning-issue-template.md

@@ -2,7 +2,7 @@
 name: Research planning issue template
 about: Template for research planning issues
 title: ''
-labels: Research & Design, Research Planning
+labels: UX Research
 assignees: ''
 
 ---

.github/ISSUE_TEMPLATE/research-synthesis-issue-template.md

@@ -2,7 +2,7 @@
 name: Research synthesis issue template
 about: Template for research synthesis issues
 title: ''
-labels: Research & Design, Research Synthesis
+labels: UX Research
 assignees: ''
 
 ---

.github/ISSUE_TEMPLATE/story-template.md

@@ -0,0 +1,28 @@
+---
+name: Story template
+about: Template for story with dev / design tickets as sub issues
+title: ''
+labels: 
+task: Story
+assignees: ''
+
+---
+
+### Background
+
+Briefly describe the problem or need this story is addressing.
+
+### User Story
+
+As a [type of user], I want to [goal/action], so that I can [benefit/outcome].
+
+### Why It Matters
+
+Briefly explain the user or business value.
+
+### Acceptance Criteria
+* [ ] [Expected outcome 1]
+* [ ] [Expected outcome 2]
+* [ ] [Expected outcome 3]
+* [ ] UX recommendations or approved designs are implemented, if applicable
+* [ ] Testing Checklist has been run and all tests pass

.gitignore

@@ -40,7 +40,6 @@ tdrs-backend/plg/grafana_views/user_views/*
 /mediafiles/*
 __pycache__/
 /.vscode/
-/.codex/
 venv
 *.sqlite3
 *.sqlite3-journal

Taskfile.yml

@@ -140,9 +140,11 @@ tasks:
         queue="go-parser-integration-$(date +%s)-$$"
         post_parse_queue="${queue}-post-parse"
         container="tdrs-backend-go-parser-integration-$(date +%s)-$$"
+        post_parse_container="tdrs-backend-post-parse-integration-$(date +%s)-$$"
 
         cleanup() {
           docker rm -f "${container}" >/dev/null 2>&1 || true
+          docker rm -f "${post_parse_container}" >/dev/null 2>&1 || true
           docker compose -f docker-compose.yml exec -T redis-server \
             redis-cli DEL "${queue}" "${post_parse_queue}" >/dev/null 2>&1 || true
           docker compose -f docker-compose.yml exec -T postgres \
@@ -168,6 +170,15 @@ tasks:
           -e GO_PARSER_POST_PARSE_QUEUE="${post_parse_queue}" \
           go-parser
 
+        docker compose -f docker-compose.yml run --rm --detach \
+          --name "${post_parse_container}" \
+          -e DB_NAME="${db_name}" \
+          -e DJANGO_SETTINGS_MODULE=tdpservice.settings.go_parser_integration \
+          -e DJANGO_CONFIGURATION=GoParserIntegration \
+          -e GO_PARSER_SHADOW_MODE=false \
+          celery \
+          bash -c "./wait_for_services.sh && exec celery -A tdpservice.settings worker --loglevel=INFO --pool=solo --without-gossip --without-mingle --without-heartbeat -Q '${post_parse_queue}' -n 'post-parse@%h'"
+
         docker compose -f docker-compose.yml exec web sh -c "DB_NAME=${db_name} CELERY_GO_PARSER_QUEUE=${queue} DJANGO_SETTINGS_MODULE=tdpservice.settings.go_parser_integration DJANGO_CONFIGURATION=GoParserIntegration pytest -c pytest-go-parser.ini --basetemp=/tmp/tdp-pytest-go-integration tdpservice/parsers/test/integration {{.PYTEST_ARGS}}"
 
   backend-pytest-go-integration-active:
@@ -442,12 +453,13 @@ tasks:
     desc: Run Cypress E2E tests locally (Cypress on host, app in docker)
     dir: tdrs-frontend
     cmds:
+      - STALE_PARSE_TIMEOUT_SECONDS=60 STALE_PARSE_CHECK_INTERVAL_SECONDS=5 GO_PARSER_SHADOW_MODE=false CLAMAV_NEEDED=True docker compose -f ../tdrs-backend/docker-compose.yml up -d --force-recreate web celery
       - docker compose -f docker-compose.local.yml up -d --build tdp-frontend
       - |
         if command -v corepack >/dev/null 2>&1; then
-          corepack prepare yarn@4.6.0 --activate && env -u ELECTRON_RUN_AS_NODE yarn test:e2e
+          corepack prepare yarn@4.6.0 --activate && CYPRESS_STUCK_TIMEOUT_TEST=true env -u ELECTRON_RUN_AS_NODE yarn test:e2e
         else
-          env -u ELECTRON_RUN_AS_NODE npx --yes @yarnpkg/cli-dist@4.6.0 test:e2e
+          CYPRESS_STUCK_TIMEOUT_TEST=true env -u ELECTRON_RUN_AS_NODE npx --yes @yarnpkg/cli-dist@4.6.0 test:e2e
         fi
 
   frontend-e2e-ci-local:
@@ -457,14 +469,15 @@ tasks:
       CYPRESS_TOKEN: '{{.CYPRESS_TOKEN | default "local-cypress-token"}}'
     cmds:
       - task: backend-up
+      - STALE_PARSE_TIMEOUT_SECONDS=60 STALE_PARSE_CHECK_INTERVAL_SECONDS=5 GO_PARSER_SHADOW_MODE=false CLAMAV_NEEDED=True docker compose -f ../tdrs-backend/docker-compose.yml up -d --force-recreate web celery
       - task: e2e-env-var-setup
       - docker compose -f docker-compose.local.yml up -d --build tdp-frontend
       - env -u ELECTRON_RUN_AS_NODE npx cypress verify || env -u ELECTRON_RUN_AS_NODE npx cypress install --force
       - |
         if command -v corepack >/dev/null 2>&1; then
-          CYPRESS_TOKEN={{.CYPRESS_TOKEN}} env -u ELECTRON_RUN_AS_NODE yarn test:e2e-ci
+          CYPRESS_TOKEN={{.CYPRESS_TOKEN}} CYPRESS_STUCK_TIMEOUT_TEST=true env -u ELECTRON_RUN_AS_NODE yarn test:e2e-ci
         else
-          CYPRESS_TOKEN={{.CYPRESS_TOKEN}} env -u ELECTRON_RUN_AS_NODE npx --yes @yarnpkg/cli-dist@4.6.0 test:e2e-ci
+          CYPRESS_TOKEN={{.CYPRESS_TOKEN}} CYPRESS_STUCK_TIMEOUT_TEST=true env -u ELECTRON_RUN_AS_NODE npx --yes @yarnpkg/cli-dist@4.6.0 test:e2e-ci
         fi
 
   frontend-e2e-cmd: