Skip to content

Add missing eia923 file mappings discovered in audit #4317

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
krivard wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Add missing eia923 file mappings discovered in audit #4317

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8bbee01
Add missing eia923 file mappings; correct off-by-one error in page map
krivard Jun 10, 2025
5b0184d
Add raw assets for mapped-but-unextracted columns
krivard Jun 10, 2025
cab91cd
Nope, not column mapped, leave em unextracted
krivard Jun 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/pudl/extract/eia923.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def __init__(self, *args, **kwargs):
# Pages not supported by the metadata:
# puerto_rico, github issue #457
# energy_storage, github issue #458
# oil_stocks, coal_stocks, petcoke_stocks
# oil stocks, coal stocks, petcoke stocks: file- and pagemapped but not columnmapped

def process_raw(self, df, page, **partition):
"""Drops reserved columns."""
Expand Down
4 changes: 2 additions & 2 deletions src/pudl/package_data/eia923/file_map.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
page,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022,2023,2024,2025
boiler_fuel,-1,-1,-1,-1,-1,-1,-1,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2024_21FEB2025.xlsx,EIA923_Schedules_2_3_4_5_M_02_2025_22APR2025.xlsx
coal_stocks,-1,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,-1,-1,-1,-1,-1
coal_stocks,-1,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Final.xlsx,-1,-1
energy_storage,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2024_21FEB2025.xlsx,EIA923_Schedules_2_3_4_5_M_02_2025_22APR2025.xlsx
fuel_receipts_costs,-1,-1,-1,-1,-1,-1,-1,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2024_21FEB2025.xlsx,EIA923_Schedules_2_3_4_5_M_02_2025_22APR2025.xlsx
generation_fuel,f906920y2001.xls,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2024_21FEB2025.xlsx,EIA923_Schedules_2_3_4_5_M_02_2025_22APR2025.xlsx
generator,-1,-1,-1,-1,-1,-1,-1,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2024_21FEB2025.xlsx,EIA923_Schedules_2_3_4_5_M_02_2025_22APR2025.xlsx
oil_stocks,-1,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2024_21FEB2025.xlsx,EIA923_Schedules_2_3_4_5_M_02_2025_22APR2025.xlsx
petcoke_stocks,-1,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,-1,-1,-1,-1,-1
petcoke_stocks,-1,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,-1,EIA923_Schedules_2_3_4_5_M_12_2023_Final.xlsx,-1,-1
plant_frame,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2024_21FEB2025.xlsx,EIA923_Schedules_2_3_4_5_M_02_2025_22APR2025.xlsx
puerto_rico,-1,-1,-1,-1,-1,-1,-1,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2024_21FEB2025.xlsx,EIA923_Schedules_2_3_4_5_M_02_2025_22APR2025.xlsx
stocks,f906920y2001.xls,f906920y2002.xls,f906920_2003.xls,f906920_2004.xls,f906920_2005.xls,f906920_2006.xls,f906920_2007.xls,eia923December2008.xls,EIA923 SCHEDULES 2_3_4_5 M Final 2009 REVISED 05252011.XLS,EIA923 SCHEDULES 2_3_4_5 Final 2010.xls,EIA923_Schedules_2_3_4_5_2011_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2012_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_2013_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2014_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2015_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2016_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2017_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2018_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2019_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2020_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2021_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2022_Final_Revision.xlsx,EIA923_Schedules_2_3_4_5_M_12_2023_Final.xlsx,EIA923_Schedules_2_3_4_5_M_12_2024_21FEB2025.xlsx,EIA923_Schedules_2_3_4_5_M_02_2025_22APR2025.xlsx
Expand Down
2 changes: 1 addition & 1 deletion src/pudl/package_data/eia923/page_map.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ puerto_rico,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1
energy_storage,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,2,2,2,2,2,2,2,2,2
stocks,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,3,3,3,3,3,3,3,3,3
oil_stocks,-1,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,4,4,4,4,4,4,4,4,4
coal_stocks,-1,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5,5,5,-1,-1
coal_stocks,-1,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,5,5,5,5,5,5,5,-1,-1
petcoke_stocks,-1,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,6,6,6,6,6,-1,6,-1,-1
boiler_fuel,-1,-1,-1,-1,-1,-1,-1,5,5,5,5,5,5,6,6,6,7,7,7,7,7,6,7,5,5
generator,-1,-1,-1,-1,-1,-1,-1,6,6,6,6,6,6,7,7,7,8,8,8,8,8,7,8,6,6
Expand Down