Integrate last 4 RUS tables

dbt/models/rus12/core_rus12__yearly_depreciation_changes/schema.yml

@@ -0,0 +1,78 @@
+version: 2
+sources:
+  - name: pudl
+    tables:
+      - name: core_rus12__yearly_depreciation_changes
+        data_tests:
+          - expect_columns_not_all_null
+          - check_row_counts_per_partition:
+              arguments:
+                table_name: core_rus12__yearly_depreciation_changes
+                partition_expr: "EXTRACT(YEAR FROM report_date)"
+          - dbt_utils.expression_is_true:
+              arguments:
+                expression: accruals + retirements_less_net_salvage + adjustments_and_transfers <= ending_balance
+                config:
+                  error_if: ">18"
+                  where: depreciation_and_amortization_item != 'retirement_work_in_progress'
+              description: "Check that annual changes don't exceed ending balance."
+          - subcomponents_sum_to_total:
+              arguments:
+                group_by_columns: [borrower_id_rus, report_date]
+                categorical_column: depreciation_and_amortization_group
+                value_column: accruals
+                total_label: "total_depreciation"
+              row_condition: depreciation_and_amortization_group='electric_plant_in_service'
+              description: "Check the totals for all of the depreciation_and_amortization_group's that are just the sum of the components."
+          - subcomponents_sum_to_total:
+              arguments:
+                group_by_columns: [borrower_id_rus, report_date]
+                categorical_column: depreciation_and_amortization_group
+                value_column: retirements_less_net_salvage
+                total_label: "total_depreciation"
+              row_condition: depreciation_and_amortization_group='electric_plant_in_service'
+              description: "Check the totals for all of the depreciation_and_amortization_group's that are just the sum of the components."
+          - subcomponents_sum_to_total:
+              arguments:
+                group_by_columns: [borrower_id_rus, report_date]
+                categorical_column: depreciation_and_amortization_group
+                value_column: adjustments_and_transfers
+                total_label: "total_depreciation"
+              row_condition: depreciation_and_amortization_group='electric_plant_in_service'
+              description: "Check the totals for all of the depreciation_and_amortization_group's that are just the sum of the components."
+          - subcomponents_sum_to_total:
+              arguments:
+                group_by_columns: [borrower_id_rus, report_date]
+                categorical_column: depreciation_and_amortization_group
+                value_column: ending_balance
+                total_label: "total_depreciation"
+              row_condition: depreciation_and_amortization_group='electric_plant_in_service'
+              description: "Check the totals for all of the depreciation_and_amortization_group's that are just the sum of the components."
+        columns:
+          - name: report_date
+          - name: borrower_id_rus
+          - name: depreciation_and_amortization_group
+          - name: depreciation_and_amortization_item
+          - name: composite_depreciation_rate
+            data_tests:
+              - dbt_expectations.expect_column_values_to_be_between:
+                  arguments:
+                    min_value: 0.0
+                    max_value: 100.0
+                  row_condition: depreciation_and_amortization_item !='retirement_work_in_progress'
+                  config:
+                    error_if: ">1"
+                  description: "Check that most of the composite_depreciation_rate's within 0-100."
+          - name: accruals
+          - name: retirements_less_net_salvage
+          - name: adjustments_and_transfers
+          - name: ending_balance
+            data_tests:
+              - dbt_expectations.expect_column_values_to_be_between:
+                  arguments:
+                    min_value: 0.0
+                  row_condition: depreciation_and_amortization_item !='retirement_work_in_progress'
+                  config:
+                    error_if: ">34"
+                  description: "Check that most of the ending_balance's are positive - excluding the retirement_work_in_progress."
+          - name: is_total

dbt/models/rus12/core_rus12__yearly_depreciation_misc/schema.yml

@@ -0,0 +1,23 @@
+version: 2
+sources:
+  - name: pudl
+    tables:
+      - name: core_rus12__yearly_depreciation_misc
+        data_tests:
+          - expect_columns_not_all_null
+          - check_row_counts_per_partition:
+              arguments:
+                table_name: core_rus12__yearly_depreciation_misc
+                partition_expr: "EXTRACT(YEAR FROM report_date)"
+        columns:
+          - name: report_date
+          - name: borrower_id_rus
+          - name: depreciation_and_amortization_item
+          - name: ending_balance
+            data_tests:
+              - dbt_expectations.expect_column_values_to_be_between:
+                  arguments:
+                    min_value: 0.0
+                  config:
+                    error_if: ">126"
+                  description: "Check that most of the ending_balance's are positive. About 4% of the table is negative, which seems reasonable."

dbt/models/rus12/core_rus12__yearly_non_utility_plant_changes/schema.yml

@@ -0,0 +1,43 @@
+version: 2
+sources:
+  - name: pudl
+    tables:
+      - name: core_rus12__yearly_non_utility_plant_changes
+        data_tests:
+          - expect_columns_not_all_null
+          - check_row_counts_per_partition:
+              arguments:
+                table_name: core_rus12__yearly_non_utility_plant_changes
+                partition_expr: "EXTRACT(YEAR FROM report_date)"
+          - dbt_utils.expression_is_true:
+              arguments:
+                expression: retirements + additions + adjustments_and_transfers <= ending_balance
+              description: "Check that annual changes don't exceed ending balance."
+              config:
+                error_if: ">2"
+        columns:
+          - name: report_date
+          - name: borrower_id_rus
+          - name: non_utility_plant_item
+          - name: additions
+            data_tests:
+              - dbt_expectations.expect_column_values_to_be_between:
+                  arguments:
+                    min_value: 0.0
+                  config:
+                    error_if: ">1"
+                  description: "Check that most of the additions are positive."
+          - name: retirements
+            data_tests:
+              - dbt_expectations.expect_column_values_to_be_between:
+                  arguments:
+                    min_value: 0.0
+                  config:
+                    error_if: ">1"
+                  description: "Check that most of the retirements are positive."
+          - name: adjustments_and_transfers
+          - name: ending_balance
+            data_tests:
+              - dbt_expectations.expect_column_values_to_be_between:
+                  arguments:
+                    min_value: 0.0

dbt/models/rus12/core_rus12__yearly_utility_plant_changes/schema.yml

@@ -0,0 +1,81 @@
+version: 2
+sources:
+  - name: pudl
+    tables:
+      - name: core_rus12__yearly_utility_plant_changes
+        data_tests:
+          - expect_columns_not_all_null
+          - check_row_counts_per_partition:
+              arguments:
+                table_name: core_rus12__yearly_utility_plant_changes
+                partition_expr: "EXTRACT(YEAR FROM report_date)"
+          - subcomponents_sum_to_total:
+              arguments:
+                group_by_columns: [borrower_id_rus, report_date]
+                categorical_column: utility_plant_group
+                value_column: retirements
+                total_label: "total"
+              row_condition: utility_plant_group NOT IN ('electric_plant_in_service', 'utility_plant_in_service', 'total_utility_plant')
+              description: "Check the totals for all of the utility_plant_group's that are just the sum of the components."
+          - subcomponents_sum_to_total:
+              arguments:
+                group_by_columns: [borrower_id_rus, report_date]
+                categorical_column: utility_plant_group
+                value_column: additions
+                total_label: "total"
+              row_condition: utility_plant_group NOT IN ('electric_plant_in_service', 'utility_plant_in_service', 'total_utility_plant')
+              description: "Check the totals for all of the utility_plant_group's that are just the sum of the components."
+          - subcomponents_sum_to_total:
+              arguments:
+                group_by_columns: [borrower_id_rus, report_date]
+                categorical_column: utility_plant_group
+                value_column: adjustments_and_transfers
+                total_label: "total"
+              row_condition: utility_plant_group NOT IN ('electric_plant_in_service', 'utility_plant_in_service', 'total_utility_plant')
+              description: "Check the totals for all of the utility_plant_group's that are just the sum of the components."
+          - subcomponents_sum_to_total:
+              arguments:
+                group_by_columns: [borrower_id_rus, report_date]
+                categorical_column: utility_plant_group
+                value_column: ending_balance
+                total_label: "total"
+              row_condition: utility_plant_group NOT IN ('electric_plant_in_service', 'utility_plant_in_service', 'total_utility_plant')
+              description: "Check the totals for all of the utility_plant_group's that are just the sum of the components."
+          - dbt_utils.expression_is_true:
+              arguments:
+                expression: retirements + additions + adjustments_and_transfers <= ending_balance
+              description: "Check that annual changes don't exceed ending balance."
+              config:
+                error_if: ">72"
+        columns:
+          - name: report_date
+          - name: borrower_id_rus
+          - name: utility_plant_group
+          - name: utility_plant_item
+          - name: retirements
+            data_tests:
+              - dbt_expectations.expect_column_values_to_be_between:
+                  arguments:
+                    min_value: 0.0
+                  config:
+                    error_if: ">21"
+                  description: "Check that most of the retirements are positive."
+          - name: additions
+            data_tests:
+              - dbt_expectations.expect_column_values_to_be_between:
+                  arguments:
+                    min_value: 0.0
+                  row_condition: utility_plant_item!='construction_work_in_progress'
+                  config:
+                    error_if: ">199"
+                  description: "Check that most of the additions are positive - expect construction_work_in_progress. This is about 1% of the table."
+          - name: adjustments_and_transfers
+          - name: ending_balance
+            data_tests:
+              - dbt_expectations.expect_column_values_to_be_between:
+                  arguments:
+                    min_value: 0.0
+                  config:
+                    error_if: ">19"
+                  description: "Check that most of the ending balances are positive."
+          - name: is_total

dbt/models/rus12/out_rus12__yearly_depreciation_changes/schema.yml

@@ -0,0 +1,80 @@
+version: 2
+sources:
+  - name: pudl
+    tables:
+      - name: out_rus12__yearly_depreciation_changes
+        data_tests:
+          - expect_columns_not_all_null
+          - check_row_counts_per_partition:
+              arguments:
+                table_name: out_rus12__yearly_depreciation_changes
+                partition_expr: "EXTRACT(YEAR FROM report_date)"
+          - dbt_utils.expression_is_true:
+              arguments:
+                expression: accruals + retirements_less_net_salvage + adjustments_and_transfers <= ending_balance
+                config:
+                  error_if: ">18"
+                  where: depreciation_and_amortization_item != 'retirement_work_in_progress'
+              description: "Check that annual changes don't exceed ending balance."
+          - subcomponents_sum_to_total:
+              arguments:
+                group_by_columns: [borrower_id_rus, report_date]
+                categorical_column: depreciation_and_amortization_group
+                value_column: accruals
+                total_label: "total_depreciation"
+              row_condition: depreciation_and_amortization_group='electric_plant_in_service'
+              description: "Check the totals for all of the depreciation_and_amortization_group's that are just the sum of the components."
+          - subcomponents_sum_to_total:
+              arguments:
+                group_by_columns: [borrower_id_rus, report_date]
+                categorical_column: depreciation_and_amortization_group
+                value_column: retirements_less_net_salvage
+                total_label: "total_depreciation"
+              row_condition: depreciation_and_amortization_group='electric_plant_in_service'
+              description: "Check the totals for all of the depreciation_and_amortization_group's that are just the sum of the components."
+          - subcomponents_sum_to_total:
+              arguments:
+                group_by_columns: [borrower_id_rus, report_date]
+                categorical_column: depreciation_and_amortization_group
+                value_column: adjustments_and_transfers
+                total_label: "total_depreciation"
+              row_condition: depreciation_and_amortization_group='electric_plant_in_service'
+              description: "Check the totals for all of the depreciation_and_amortization_group's that are just the sum of the components."
+          - subcomponents_sum_to_total:
+              arguments:
+                group_by_columns: [borrower_id_rus, report_date]
+                categorical_column: depreciation_and_amortization_group
+                value_column: ending_balance
+                total_label: "total_depreciation"
+              row_condition: depreciation_and_amortization_group='electric_plant_in_service'
+              description: "Check the totals for all of the depreciation_and_amortization_group's that are just the sum of the components."
+        columns:
+          - name: report_date
+          - name: borrower_id_rus
+          - name: borrower_name_rus
+          - name: state
+          - name: depreciation_and_amortization_group
+          - name: depreciation_and_amortization_item
+          - name: composite_depreciation_rate
+            data_tests:
+              - dbt_expectations.expect_column_values_to_be_between:
+                  arguments:
+                    min_value: 0.0
+                    max_value: 100.0
+                  row_condition: depreciation_and_amortization_item !='retirement_work_in_progress'
+                  config:
+                    error_if: ">1"
+                  description: "Check that most of the composite_depreciation_rate's within 0-100."
+          - name: accruals
+          - name: retirements_less_net_salvage
+          - name: adjustments_and_transfers
+          - name: ending_balance
+            data_tests:
+              - dbt_expectations.expect_column_values_to_be_between:
+                  arguments:
+                    min_value: 0.0
+                  row_condition: depreciation_and_amortization_item !='retirement_work_in_progress'
+                  config:
+                    error_if: ">34"
+                  description: "Check that most of the ending_balance's are positive - excluding the retirement_work_in_progress."
+          - name: is_total

dbt/models/rus12/out_rus12__yearly_depreciation_misc/schema.yml

@@ -0,0 +1,25 @@
+version: 2
+sources:
+  - name: pudl
+    tables:
+      - name: out_rus12__yearly_depreciation_misc
+        data_tests:
+          - expect_columns_not_all_null
+          - check_row_counts_per_partition:
+              arguments:
+                table_name: out_rus12__yearly_depreciation_misc
+                partition_expr: "EXTRACT(YEAR FROM report_date)"
+        columns:
+          - name: report_date
+          - name: borrower_id_rus
+          - name: borrower_name_rus
+          - name: state
+          - name: depreciation_and_amortization_item
+          - name: ending_balance
+            data_tests:
+              - dbt_expectations.expect_column_values_to_be_between:
+                  arguments:
+                    min_value: 0.0
+                  config:
+                    error_if: ">126"
+                  description: "Check that most of the ending_balance's are positive. About 4% of the table is negative, which seems reasonable."

dbt/models/rus12/out_rus12__yearly_non_utility_plant_changes/schema.yml

@@ -0,0 +1,45 @@
+version: 2
+sources:
+  - name: pudl
+    tables:
+      - name: out_rus12__yearly_non_utility_plant_changes
+        data_tests:
+          - expect_columns_not_all_null
+          - check_row_counts_per_partition:
+              arguments:
+                table_name: out_rus12__yearly_non_utility_plant_changes
+                partition_expr: "EXTRACT(YEAR FROM report_date)"
+          - dbt_utils.expression_is_true:
+              arguments:
+                expression: retirements + additions + adjustments_and_transfers <= ending_balance
+              description: "Check that annual changes don't exceed ending balance."
+              config:
+                error_if: ">2"
+        columns:
+          - name: report_date
+          - name: borrower_id_rus
+          - name: borrower_name_rus
+          - name: state
+          - name: non_utility_plant_item
+          - name: additions
+            data_tests:
+              - dbt_expectations.expect_column_values_to_be_between:
+                  arguments:
+                    min_value: 0.0
+                  config:
+                    error_if: ">1"
+                  description: "Check that most of the additions are positive."
+          - name: retirements
+            data_tests:
+              - dbt_expectations.expect_column_values_to_be_between:
+                  arguments:
+                    min_value: 0.0
+                  config:
+                    error_if: ">1"
+                  description: "Check that most of the retirements are positive."
+          - name: adjustments_and_transfers
+          - name: ending_balance
+            data_tests:
+              - dbt_expectations.expect_column_values_to_be_between:
+                  arguments:
+                    min_value: 0.0