Skip to content

BIDS MRI import integration test (PR 3-bis) #1392

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MaximeBICMTL wants to merge 6 commits into
base: bids_staging_branch
Choose a base branch
from
Open

BIDS MRI import integration test (PR 3-bis) #1392

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
98e24b0
factorize copy bids file
MaximeBICMTL Feb 19, 2026
358e7e8
fix sessionless and scans file path
MaximeBICMTL Mar 19, 2026
e4db817
do not use useless intermediary variable
MaximeBICMTL Mar 19, 2026
816096d
fix scans.tsv path
MaximeBICMTL Mar 20, 2026
27089d3
typed_mri_bids_import
MaximeBICMTL Mar 4, 2026
8696d87
bids mri import integration test
MaximeBICMTL Mar 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 55 additions & 108 deletions python/lib/eeg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,12 @@
from lib.db.models.session import DbSession
from lib.db.queries.physio_file import try_get_physio_file_with_path
from lib.env import Env
from lib.import_bids_dataset.copy_files import copy_scans_tsv_file_to_loris_bids_dir
from lib.import_bids_dataset.copy_files import (
copy_loris_bids_file,
copy_scans_tsv_file_to_loris_bids_dir,
get_loris_bids_file_path,
)
from lib.import_bids_dataset.env import BidsImportEnv
from lib.import_bids_dataset.file_type import get_check_bids_imaging_file_type_from_extension
from lib.import_bids_dataset.physio import (
get_check_bids_physio_file_hash,
Expand All @@ -40,9 +45,8 @@ class Eeg:
into the database by calling the lib.physiological class.
"""

def __init__(self, env: Env, bids_layout, bids_info: BidsDataTypeInfo, session: DbSession, db,
data_dir, loris_bids_eeg_rel_dir,
loris_bids_root_dir, dataset_tag_dict, dataset_type):
def __init__(self, env: Env, import_env: BidsImportEnv, bids_layout, bids_info: BidsDataTypeInfo,
session: DbSession, db, dataset_tag_dict, dataset_type):
"""
Constructor method for the Eeg class.

Expand All @@ -52,12 +56,7 @@ def __init__(self, env: Env, bids_layout, bids_info: BidsDataTypeInfo, session:
:param session : The LORIS session the EEG datasets are linked to
:param db : Database class object
:type db : object
:param data_dir : LORIS data directory path (usually /data/PROJECT/data)
:type data_dir : str
:param loris_bids_eeg_rel_dir: LORIS BIDS EEG relative dir path to data_dir
:type loris_bids_eeg_rel_dir: str
:param loris_bids_root_dir : LORIS BIDS root directory path
:type loris_bids_root_dir : str
:param info : The BIDS import pipeline information
:param dataset_tag_dict : Dict of dataset-inherited HED tags
:type dataset_tag_dict : dict
:param dataset_type : raw | derivative. Type of the dataset
Expand All @@ -71,9 +70,8 @@ def __init__(self, env: Env, bids_layout, bids_info: BidsDataTypeInfo, session:

# load the LORIS BIDS import root directory where the eeg files will
# be copied
self.loris_bids_eeg_rel_dir = loris_bids_eeg_rel_dir
self.loris_bids_root_dir = loris_bids_root_dir
self.data_dir = data_dir
self.info = import_env
self.data_dir = self.info.data_dir_path

# load bids subject, visit and modality
self.bids_info = bids_info
Expand Down Expand Up @@ -294,14 +292,9 @@ def fetch_and_insert_eeg_files(self, derivatives=False, detect=True):
if sidecar_json is not None:
eeg_file_data = sidecar_json.data

sidecar_json_path = os.path.relpath(sidecar_json.path, self.data_dir)
if self.loris_bids_root_dir:
# copy the JSON file to the LORIS BIDS import directory
sidecar_json_path = self.copy_file_to_loris_bids_dir(
sidecar_json.path, derivatives
)
sidecar_json_path = self.copy_file_to_loris_bids_dir(sidecar_json.path, derivatives)
eeg_file_data['eegjson_file'] = str(sidecar_json_path)

eeg_file_data['eegjson_file'] = sidecar_json_path
json_blake2 = compute_file_blake2b_hash(sidecar_json.path)
eeg_file_data['physiological_json_file_blake2b_hash'] = json_blake2

Expand All @@ -325,12 +318,12 @@ def fetch_and_insert_eeg_files(self, derivatives=False, detect=True):
print(f"ERROR: {error}")
sys.exit(lib.exitcode.PROGRAM_EXECUTION_FAILURE)

if self.loris_bids_root_dir:
if self.info.loris_bids_path:
# copy the scans.tsv file to the LORIS BIDS import directory
scans_path = copy_scans_tsv_file_to_loris_bids_dir(
self.scans_file,
self.bids_info.subject,
self.loris_bids_root_dir,
self.session,
self.info.data_dir_path / self.info.loris_bids_path,
self.data_dir,
)

Expand All @@ -342,13 +335,7 @@ def fetch_and_insert_eeg_files(self, derivatives=False, detect=True):
# eeg_file_data dictionary
fdt_file_path = None
if file_type.name == 'set' and fdt_file:
fdt_file_path = os.path.relpath(fdt_file, self.data_dir)
if self.loris_bids_root_dir:
# copy the fdt file to the LORIS BIDS import directory
fdt_file_path = self.copy_file_to_loris_bids_dir(
fdt_file.path, derivatives
)

fdt_file_path = self.copy_file_to_loris_bids_dir(fdt_file.path, derivatives)
eeg_file_data['fdt_file'] = fdt_file_path
fdt_blake2 = compute_file_blake2b_hash(fdt_file.path)
eeg_file_data['physiological_fdt_file_blake2b_hash'] = fdt_blake2
Expand All @@ -367,18 +354,17 @@ def fetch_and_insert_eeg_files(self, derivatives=False, detect=True):
# grep the modality ID from physiological_modality table
modality = get_check_bids_physio_modality(self.env, self.bids_info.data_type)

if self.loris_bids_root_dir:
# copy the eeg_file to the LORIS BIDS import directory
eeg_path = self.copy_file_to_loris_bids_dir(
eeg_file.path, derivatives
)
# copy the eeg_file to the LORIS BIDS import directory
eeg_path = self.copy_file_to_loris_bids_dir(
eeg_file.path, derivatives
)

# insert the file along with its information into
# physiological_file and physiological_parameter_file tables
physio_file = insert_physio_file(
self.env,
self.session,
Path(eeg_path),
eeg_path,
file_type,
modality,
output_type,
Expand All @@ -388,7 +374,7 @@ def fetch_and_insert_eeg_files(self, derivatives=False, detect=True):
insert_physio_file_parameters(self.env, physio_file, eeg_file_data)
self.env.db.commit()

if self.loris_bids_root_dir:
if self.info.loris_bids_path:
# If we copy the file in assembly_bids and
# if the EEG file was a set file, then update the filename for the .set
# and .fdt files in the .set file so it can find the proper file for
Expand Down Expand Up @@ -457,12 +443,10 @@ def fetch_and_insert_electrode_file(
)
if not result:
electrode_data = utilities.read_tsv_file(electrode_file.path)
electrode_path = os.path.relpath(electrode_file.path, self.data_dir)
if self.loris_bids_root_dir:
# copy the electrode file to the LORIS BIDS import directory
electrode_path = self.copy_file_to_loris_bids_dir(
electrode_file.path, derivatives
)
# copy the electrode file to the LORIS BIDS import directory
electrode_path = self.copy_file_to_loris_bids_dir(
electrode_file.path, derivatives
)
# get the blake2b hash of the electrode file
blake2 = compute_file_blake2b_hash(electrode_file.path)

Expand Down Expand Up @@ -497,12 +481,10 @@ def fetch_and_insert_electrode_file(
electrode_ids
)
else:
electrode_metadata_path = os.path.relpath(coordsystem_metadata_file, self.data_dir)
if self.loris_bids_root_dir:
# copy the electrode metadata file to the LORIS BIDS import directory
electrode_metadata_path = self.copy_file_to_loris_bids_dir(
coordsystem_metadata_file.path, derivatives
)
# copy the electrode metadata file to the LORIS BIDS import directory
electrode_metadata_path = self.copy_file_to_loris_bids_dir(
coordsystem_metadata_file.path, derivatives
)
# load json data
with open(coordsystem_metadata_file.path) as metadata_file:
electrode_metadata = json.load(metadata_file)
Expand Down Expand Up @@ -562,12 +544,10 @@ def fetch_and_insert_channel_file(
if physiological_file.channels != []:
return physiological_file.channels[0].file_path

channel_path = os.path.relpath(channels_file.path, self.data_dir)
if self.loris_bids_root_dir:
# copy the channel file to the LORIS BIDS import directory
channel_path = self.copy_file_to_loris_bids_dir(
channels_file.path, derivatives
)
# copy the channel file to the LORIS BIDS import directory
channel_path = self.copy_file_to_loris_bids_dir(
channels_file.path, derivatives
)
# get the blake2b hash of the channel file
blake2 = compute_file_blake2b_hash(channels_file.path)
# insert the channel data in the database
Expand Down Expand Up @@ -637,19 +617,16 @@ def fetch_and_insert_event_files(
full_search = False,
subject=self.bids_info.subject,
)
inheritance = False

if not event_metadata_file:
message = "WARNING: no events metadata files (events.json) associated " \
f"with physiological file ID {physiological_file.id}"
print(message)
else:
event_metadata_path = os.path.relpath(event_metadata_file.path, self.data_dir)
if self.loris_bids_root_dir:
# copy the event file to the LORIS BIDS import directory
event_metadata_path = self.copy_file_to_loris_bids_dir(
event_metadata_file.path, derivatives, inheritance
)
# copy the event file to the LORIS BIDS import directory
event_metadata_path = self.copy_file_to_loris_bids_dir(
event_metadata_file.path, derivatives
)
# load json data
with open(event_metadata_file.path) as metadata_file:
event_metadata = json.load(metadata_file)
Expand All @@ -658,7 +635,7 @@ def fetch_and_insert_event_files(
# insert event metadata in the database
_, file_tag_dict = physiological.insert_event_metadata(
event_metadata=event_metadata,
event_metadata_file=event_metadata_path,
event_metadata_file=str(event_metadata_path),
physiological_file=physiological_file,
project_id=self.session.project.id,
blake2=blake2,
Expand All @@ -668,19 +645,17 @@ def fetch_and_insert_event_files(
event_paths.extend([event_metadata_path])

# get events.tsv file and insert
event_path = os.path.relpath(events_data_file.path, self.data_dir)
if self.loris_bids_root_dir:
# copy the event file to the LORIS BIDS import directory
event_path = self.copy_file_to_loris_bids_dir(
events_data_file.path, derivatives
)
# copy the event file to the LORIS BIDS import directory
event_path = self.copy_file_to_loris_bids_dir(
events_data_file.path, derivatives
)
# get the blake2b hash of the task events file
blake2 = compute_file_blake2b_hash(events_data_file.path)

# insert event data in the database
physiological.insert_event_file(
events_file=events_data_file,
event_file=event_path,
event_file=str(event_path),
physiological_file=physiological_file,
project_id=self.session.project.id,
blake2=blake2,
Expand All @@ -693,7 +668,7 @@ def fetch_and_insert_event_files(

return event_paths

def copy_file_to_loris_bids_dir(self, file, derivatives=False, inheritance=False):
def copy_file_to_loris_bids_dir(self, file, derivatives=False):
"""
Wrapper around the utilities.copy_file function that copies the file
to the LORIS BIDS import directory and returns the relative path of the
Expand All @@ -706,47 +681,19 @@ def copy_file_to_loris_bids_dir(self, file, derivatives=False, inheritance=False
:type derivatives: boolean

:return: relative path to the copied file
:rtype: str
:rtype: Path
"""

# Handle derivatives differently
# Data path structure is unpredictable, so keep the same relative path
if derivatives:
copy_file = os.path.relpath(file, self.bids_layout.root)
copy_file = os.path.join(self.loris_bids_root_dir, copy_file)
else :
# determine the path of the copied file
copy_file = ""
if not inheritance:
copy_file = self.loris_bids_eeg_rel_dir
if self.bids_info.session is not None:
copy_file = os.path.join(copy_file, os.path.basename(file))
else:
# make sure the ses- is included in the new filename if using
# default visit label from the LORIS config
copy_file = os.path.join(
copy_file,
os.path.basename(file).replace(
f'sub-{self.bids_info.subject}',
f'sub-{self.bids_info.subject}_ses-{self.session.visit_label}'
)
)

copy_file = os.path.join(self.loris_bids_root_dir, copy_file)

# create the directory if it does not exist
lib.utilities.create_dir(
os.path.dirname(copy_file),
self.env.verbose
loris_file_path = get_loris_bids_file_path(
self.info,
self.session,
self.bids_info.data_type,
Path(file),
derivatives,
)

# copy the file
utilities.copy_file(file, copy_file, self.env.verbose)

# determine the relative path and return it
relative_path = os.path.relpath(copy_file, self.data_dir)

return relative_path
copy_loris_bids_file(self.info, Path(file), loris_file_path)
return loris_file_path

def create_and_insert_archive(self, files_to_archive: list[str], archive_rel_name: str, eeg_file: DbPhysioFile):
"""
Expand Down
17 changes: 17 additions & 0 deletions python/lib/imaging_lib/scan_type.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
from lib.db.models.mri_scan_type import DbMriScanType
from lib.env import Env


def create_mri_scan_type(env: Env, name: str) -> DbMriScanType:
"""
Create an MRI scan type in the database.
"""

scan_type = DbMriScanType(
name = name,
)

env.db.add(scan_type)
env.db.flush()

return scan_type
43 changes: 43 additions & 0 deletions python/lib/import_bids_dataset/acquisitions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
from collections.abc import Callable
from typing import TypeVar

from loris_bids_reader.info import BidsAcquisitionInfo

from lib.env import Env
from lib.import_bids_dataset.env import BidsImportEnv
from lib.logging import log, log_error

T = TypeVar('T')


def import_bids_acquisitions(
env: Env,
import_env: BidsImportEnv,
acquisitions: list[tuple[T, BidsAcquisitionInfo]],
importer: Callable[[T, BidsAcquisitionInfo], None]
):
"""
Run an import function on a list of BIDS acquisitions, logging the overall import progress,
and catching the eventual exceptions raised during each import.
"""

for acquisition, bids_info in acquisitions:
log(
env,
f"Importing {bids_info.data_type} acquisition '{bids_info.name}'...",
)

try:
importer(acquisition, bids_info)
log(env, f"Successfully imported acquisition '{bids_info.name}'.")
import_env.imported_acquisitions_count += 1
except Exception as exception:
log_error(
env,
(
f"Error while importing acquisition '{bids_info.name}'. Error message:\n"
f"{exception}\n"
"Skipping."
)
)
import_env.failed_acquisitions_count += 1
Loading
Loading