Modernize Dagster config and launch scripts

.env.example

@@ -0,0 +1,23 @@
+# Copy this file to .env and customize values for your local environment.
+# The .env file is ignored by git.
+
+# Required workspace paths
+PUDL_INPUT=/absolute/path/to/pudl-input
+PUDL_OUTPUT=/absolute/path/to/pudl-output
+DAGSTER_HOME=/absolute/path/to/dagster-home
+
+# Logging controls (read by pudl.logging_helpers.configure_root_logger)
+PUDL_LOGLEVEL=INFO
+PUDL_COLOR_LOGS=true
+
+# Optional: write logs to a file in addition to console output.
+# Leave unset for console-only logging.
+# PUDL_LOGFILE=/absolute/path/to/pudl/logs/pudl.log
+
+# Optional: don't try and use intersphinx to link to external documentation
+# during the docs build -- it can be flaky and isn't required for most docs edits.
+# PUDL_DOCS_DISABLE_INTERSPHINX=1
+
+# Optional: don't remove generated rst files after the docs build. Can be helpful
+# when debugging formatting errors.
+# PUDL_DOCS_KEEP_GENERATED_FILES=1

.github/.gitignore

@@ -0,0 +1 @@
+skills/**

.github/workflows/build-deploy-ferceqr.yml

@@ -11,6 +11,8 @@ env:
   GCS_LOGS_BUCKET: gs://builds.catalyst.coop/ferceqr_logs
   S3_OUTPUT_BUCKET: s3://pudl.catalyst.coop/ferceqr
   BATCH_JOB_JSON: batch_job.json
+  BUILD_ID: ""
+  BATCH_JOB_ID: ""
 
 jobs:
   build_and_deploy_eqr:

.github/workflows/build-deploy-pudl.yml

@@ -139,7 +139,7 @@ jobs:
             --container-env OMP_NUM_THREADS=4 \
             --container-env PUDL_BOT_PAT=${{ secrets.PUDL_BOT_PAT }} \
             --container-env PUDL_GCS_OUTPUT=${{ env.PUDL_GCS_OUTPUT }} \
-            --container-env PUDL_SETTINGS_YML="/home/ubuntu/pudl/src/pudl/package_data/settings/etl_full.yml" \
+            --container-env DG_NIGHTLY_CONFIG="src/pudl/package_data/settings/dg_nightly.yml" \
             --container-env SLACK_TOKEN=${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} \
             --container-env ZENODO_SANDBOX_TOKEN_PUBLISH=${{ secrets.ZENODO_SANDBOX_TOKEN_PUBLISH }} \
             --container-env ZENODO_TARGET_ENV=${{ (startsWith(github.ref_name, 'v20') && 'production') || 'sandbox' }} \

.github/workflows/com-dev-notify.yml

@@ -7,17 +7,23 @@ on:
     types: [created]
 
 env:
-  username: ${{ github.event.issue.user.login }}
-  url: ${{ github.event.issue.html_url }}
+  username: ""
+  url: ""
   org: catalyst-cooperative
 
 jobs:
   com-dev-notify:
     name: Notify Catalyst of community activity
     runs-on: ubuntu-latest
     steps:
+      - name: Get username if an issue was opened
+        if: ${{ github.event_name == 'issues' }}
+        run: |
+          echo "username=${{ github.event.issue.user.login }}" >> "${GITHUB_ENV}"
+          echo "url=${{ github.event.issue.html_url }}" >> "${GITHUB_ENV}"
+
       - name: Get username if a discussion was created
-        if: ${{ (github.event_name == 'discussion') }}
+        if: ${{ github.event_name == 'discussion' }}
         run: |
           echo "username=${{ github.event.discussion.user.login }}" >> "${GITHUB_ENV}"
           echo "url=${{ github.event.discussion.html_url }}" >> "${GITHUB_ENV}"
@@ -36,13 +42,11 @@ jobs:
         uses: slackapi/slack-github-action@v3
 
         with:
-          # Slack channel id, channel name, or user id to post message.
-          # See also: https://api.slack.com/methods/chat.postMessage#channels
-          # You can pass in multiple channels to post to by providing a comma-delimited list of channel IDs.
-          channel-id: "community-dev"
-          # For posting a markdown message
+          method: chat.postMessage
+          token: ${{ secrets.COMMUNITY_DEV_SLACK_BOT_TOKEN }}
           payload: |
             {
+              "channel": "community-dev",
               "blocks": [
                 {
                   "type": "section",
@@ -53,5 +57,3 @@ jobs:
                 }
               ]
             }
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.COMMUNITY_DEV_SLACK_BOT_TOKEN }}

.github/workflows/pytest.yml

@@ -20,8 +20,8 @@ jobs:
     permissions:
       pull-requests: read
     outputs:
-      # 2025-07-17: because merge_group is an Object and run_code_checks is not a conditional, need to explicitly check for null-ness instead of relying on truthiness.
-      run_code_checks: ${{ github.event_name =='workflow_dispatch' || (steps.filter.outputs.code == 'true' && (github.event.merge_group != null)) }}
+      # Run code checks for manual dispatches and merge queue runs with code changes.
+      run_code_checks: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'merge_group' && steps.filter.outputs.code == 'true') }}
     steps:
       - uses: actions/checkout@v6
         with:
@@ -47,6 +47,7 @@ jobs:
               - '!.github/workflows/bot-auto-merge.yml'
               - '!.github/workflows/build-deploy-docs.yml'
               - '!.github/workflows/build-deploy-pudl.yml'
+              - '!.github/workflows/deploy-pudl.yml'
               - '!.github/workflows/com-dev-notify.yml'
               - '!.github/workflows/docker-build-test.yml'
               - '!.github/workflows/q-update-issue-scheduler.yml'
@@ -77,7 +78,7 @@ jobs:
         run: |
           echo "event name (${{ github.event_name }}) is workflow dispatch: ${{ github.event_name == 'workflow_dispatch' }}"
           echo "found code changes: ${{ steps.filter.outputs.code }}"
-          echo "merge_group ${{ github.event.merge_group }} is not null: ${{ github.event.merge_group != null }}"
+          echo "event name (${{ github.event_name }}) is merge_group: ${{ github.event_name == 'merge_group' }}"
 
   ci-docs:
     permissions:

.github/workflows/update-lockfiles.yml

@@ -10,6 +10,9 @@ on:
 # - workflow_dispatch: Whatever branch it was run against.
 # - schedule: Always runs on main
 
+env:
+  TODAY: ""
+
 jobs:
   update-lockfiles:
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -72,6 +72,7 @@ repos:
             |.*\.bib
             |.*\.csv
             |.*\.html
+            |.*\.json
             |src/pudl/package_data/ferc1/.*_categories\.yaml
           )$ | migrations/ | devtools/ | test/ | notebooks/ | src/pudl/metadata/codes.py | src/pudl/transform/params/ferc1.py
         args: [] # Make this read, not write
@@ -131,6 +132,15 @@ repos:
         always_run: false
         entry: pixi run jupyter nbconvert --clear-output
 
+      - id: pixi-lock-update
+        name: pixi-lock-update
+        stages: [pre-commit]
+        language: system
+        verbose: false
+        pass_filenames: false
+        always_run: true
+        entry: bash -c 'pixi install --quiet && git add pixi.lock'
+
       - id: unit-tests
         name: unit-tests
         stages: [pre-commit]
@@ -150,5 +160,5 @@ ci:
   autoupdate_branch: main
   autoupdate_commit_msg: "[pre-commit.ci] pre-commit autoupdate"
   autoupdate_schedule: weekly
-  skip: [unit-tests, nb-output-clear, shellcheck]
+  skip: [pixi-lock-update, unit-tests, nb-output-clear, shellcheck]
   submodules: false