diff --git a/python/lib/eeg.py b/python/lib/eeg.py index 2f32cc80f..8e88d48bb 100644 --- a/python/lib/eeg.py +++ b/python/lib/eeg.py @@ -20,7 +20,12 @@ from lib.db.models.session import DbSession from lib.db.queries.physio_file import try_get_physio_file_with_path from lib.env import Env -from lib.import_bids_dataset.copy_files import copy_scans_tsv_file_to_loris_bids_dir +from lib.import_bids_dataset.copy_files import ( + copy_loris_bids_file, + copy_scans_tsv_file_to_loris_bids_dir, + get_loris_bids_file_path, +) +from lib.import_bids_dataset.env import BidsImportEnv from lib.import_bids_dataset.file_type import get_check_bids_imaging_file_type_from_extension from lib.import_bids_dataset.physio import ( get_check_bids_physio_file_hash, @@ -40,9 +45,8 @@ class Eeg: into the database by calling the lib.physiological class. """ - def __init__(self, env: Env, bids_layout, bids_info: BidsDataTypeInfo, session: DbSession, db, - data_dir, loris_bids_eeg_rel_dir, - loris_bids_root_dir, dataset_tag_dict, dataset_type): + def __init__(self, env: Env, import_env: BidsImportEnv, bids_layout, bids_info: BidsDataTypeInfo, + session: DbSession, db, dataset_tag_dict, dataset_type): """ Constructor method for the Eeg class. @@ -52,12 +56,7 @@ def __init__(self, env: Env, bids_layout, bids_info: BidsDataTypeInfo, session: :param session : The LORIS session the EEG datasets are linked to :param db : Database class object :type db : object - :param data_dir : LORIS data directory path (usually /data/PROJECT/data) - :type data_dir : str - :param loris_bids_eeg_rel_dir: LORIS BIDS EEG relative dir path to data_dir - :type loris_bids_eeg_rel_dir: str - :param loris_bids_root_dir : LORIS BIDS root directory path - :type loris_bids_root_dir : str + :param info : The BIDS import pipeline information :param dataset_tag_dict : Dict of dataset-inherited HED tags :type dataset_tag_dict : dict :param dataset_type : raw | derivative. Type of the dataset @@ -71,9 +70,8 @@ def __init__(self, env: Env, bids_layout, bids_info: BidsDataTypeInfo, session: # load the LORIS BIDS import root directory where the eeg files will # be copied - self.loris_bids_eeg_rel_dir = loris_bids_eeg_rel_dir - self.loris_bids_root_dir = loris_bids_root_dir - self.data_dir = data_dir + self.info = import_env + self.data_dir = self.info.data_dir_path # load bids subject, visit and modality self.bids_info = bids_info @@ -294,14 +292,9 @@ def fetch_and_insert_eeg_files(self, derivatives=False, detect=True): if sidecar_json is not None: eeg_file_data = sidecar_json.data - sidecar_json_path = os.path.relpath(sidecar_json.path, self.data_dir) - if self.loris_bids_root_dir: - # copy the JSON file to the LORIS BIDS import directory - sidecar_json_path = self.copy_file_to_loris_bids_dir( - sidecar_json.path, derivatives - ) + sidecar_json_path = self.copy_file_to_loris_bids_dir(sidecar_json.path, derivatives) + eeg_file_data['eegjson_file'] = str(sidecar_json_path) - eeg_file_data['eegjson_file'] = sidecar_json_path json_blake2 = compute_file_blake2b_hash(sidecar_json.path) eeg_file_data['physiological_json_file_blake2b_hash'] = json_blake2 @@ -325,12 +318,12 @@ def fetch_and_insert_eeg_files(self, derivatives=False, detect=True): print(f"ERROR: {error}") sys.exit(lib.exitcode.PROGRAM_EXECUTION_FAILURE) - if self.loris_bids_root_dir: + if self.info.loris_bids_path: # copy the scans.tsv file to the LORIS BIDS import directory scans_path = copy_scans_tsv_file_to_loris_bids_dir( self.scans_file, - self.bids_info.subject, - self.loris_bids_root_dir, + self.session, + self.info.data_dir_path / self.info.loris_bids_path, self.data_dir, ) @@ -342,13 +335,7 @@ def fetch_and_insert_eeg_files(self, derivatives=False, detect=True): # eeg_file_data dictionary fdt_file_path = None if file_type.name == 'set' and fdt_file: - fdt_file_path = os.path.relpath(fdt_file, self.data_dir) - if self.loris_bids_root_dir: - # copy the fdt file to the LORIS BIDS import directory - fdt_file_path = self.copy_file_to_loris_bids_dir( - fdt_file.path, derivatives - ) - + fdt_file_path = self.copy_file_to_loris_bids_dir(fdt_file.path, derivatives) eeg_file_data['fdt_file'] = fdt_file_path fdt_blake2 = compute_file_blake2b_hash(fdt_file.path) eeg_file_data['physiological_fdt_file_blake2b_hash'] = fdt_blake2 @@ -367,18 +354,17 @@ def fetch_and_insert_eeg_files(self, derivatives=False, detect=True): # grep the modality ID from physiological_modality table modality = get_check_bids_physio_modality(self.env, self.bids_info.data_type) - if self.loris_bids_root_dir: - # copy the eeg_file to the LORIS BIDS import directory - eeg_path = self.copy_file_to_loris_bids_dir( - eeg_file.path, derivatives - ) + # copy the eeg_file to the LORIS BIDS import directory + eeg_path = self.copy_file_to_loris_bids_dir( + eeg_file.path, derivatives + ) # insert the file along with its information into # physiological_file and physiological_parameter_file tables physio_file = insert_physio_file( self.env, self.session, - Path(eeg_path), + eeg_path, file_type, modality, output_type, @@ -388,7 +374,7 @@ def fetch_and_insert_eeg_files(self, derivatives=False, detect=True): insert_physio_file_parameters(self.env, physio_file, eeg_file_data) self.env.db.commit() - if self.loris_bids_root_dir: + if self.info.loris_bids_path: # If we copy the file in assembly_bids and # if the EEG file was a set file, then update the filename for the .set # and .fdt files in the .set file so it can find the proper file for @@ -457,12 +443,10 @@ def fetch_and_insert_electrode_file( ) if not result: electrode_data = utilities.read_tsv_file(electrode_file.path) - electrode_path = os.path.relpath(electrode_file.path, self.data_dir) - if self.loris_bids_root_dir: - # copy the electrode file to the LORIS BIDS import directory - electrode_path = self.copy_file_to_loris_bids_dir( - electrode_file.path, derivatives - ) + # copy the electrode file to the LORIS BIDS import directory + electrode_path = self.copy_file_to_loris_bids_dir( + electrode_file.path, derivatives + ) # get the blake2b hash of the electrode file blake2 = compute_file_blake2b_hash(electrode_file.path) @@ -497,12 +481,10 @@ def fetch_and_insert_electrode_file( electrode_ids ) else: - electrode_metadata_path = os.path.relpath(coordsystem_metadata_file, self.data_dir) - if self.loris_bids_root_dir: - # copy the electrode metadata file to the LORIS BIDS import directory - electrode_metadata_path = self.copy_file_to_loris_bids_dir( - coordsystem_metadata_file.path, derivatives - ) + # copy the electrode metadata file to the LORIS BIDS import directory + electrode_metadata_path = self.copy_file_to_loris_bids_dir( + coordsystem_metadata_file.path, derivatives + ) # load json data with open(coordsystem_metadata_file.path) as metadata_file: electrode_metadata = json.load(metadata_file) @@ -562,12 +544,10 @@ def fetch_and_insert_channel_file( if physiological_file.channels != []: return physiological_file.channels[0].file_path - channel_path = os.path.relpath(channels_file.path, self.data_dir) - if self.loris_bids_root_dir: - # copy the channel file to the LORIS BIDS import directory - channel_path = self.copy_file_to_loris_bids_dir( - channels_file.path, derivatives - ) + # copy the channel file to the LORIS BIDS import directory + channel_path = self.copy_file_to_loris_bids_dir( + channels_file.path, derivatives + ) # get the blake2b hash of the channel file blake2 = compute_file_blake2b_hash(channels_file.path) # insert the channel data in the database @@ -637,19 +617,16 @@ def fetch_and_insert_event_files( full_search = False, subject=self.bids_info.subject, ) - inheritance = False if not event_metadata_file: message = "WARNING: no events metadata files (events.json) associated " \ f"with physiological file ID {physiological_file.id}" print(message) else: - event_metadata_path = os.path.relpath(event_metadata_file.path, self.data_dir) - if self.loris_bids_root_dir: - # copy the event file to the LORIS BIDS import directory - event_metadata_path = self.copy_file_to_loris_bids_dir( - event_metadata_file.path, derivatives, inheritance - ) + # copy the event file to the LORIS BIDS import directory + event_metadata_path = self.copy_file_to_loris_bids_dir( + event_metadata_file.path, derivatives + ) # load json data with open(event_metadata_file.path) as metadata_file: event_metadata = json.load(metadata_file) @@ -658,7 +635,7 @@ def fetch_and_insert_event_files( # insert event metadata in the database _, file_tag_dict = physiological.insert_event_metadata( event_metadata=event_metadata, - event_metadata_file=event_metadata_path, + event_metadata_file=str(event_metadata_path), physiological_file=physiological_file, project_id=self.session.project.id, blake2=blake2, @@ -668,19 +645,17 @@ def fetch_and_insert_event_files( event_paths.extend([event_metadata_path]) # get events.tsv file and insert - event_path = os.path.relpath(events_data_file.path, self.data_dir) - if self.loris_bids_root_dir: - # copy the event file to the LORIS BIDS import directory - event_path = self.copy_file_to_loris_bids_dir( - events_data_file.path, derivatives - ) + # copy the event file to the LORIS BIDS import directory + event_path = self.copy_file_to_loris_bids_dir( + events_data_file.path, derivatives + ) # get the blake2b hash of the task events file blake2 = compute_file_blake2b_hash(events_data_file.path) # insert event data in the database physiological.insert_event_file( events_file=events_data_file, - event_file=event_path, + event_file=str(event_path), physiological_file=physiological_file, project_id=self.session.project.id, blake2=blake2, @@ -693,7 +668,7 @@ def fetch_and_insert_event_files( return event_paths - def copy_file_to_loris_bids_dir(self, file, derivatives=False, inheritance=False): + def copy_file_to_loris_bids_dir(self, file, derivatives=False): """ Wrapper around the utilities.copy_file function that copies the file to the LORIS BIDS import directory and returns the relative path of the @@ -706,47 +681,19 @@ def copy_file_to_loris_bids_dir(self, file, derivatives=False, inheritance=False :type derivatives: boolean :return: relative path to the copied file - :rtype: str + :rtype: Path """ - # Handle derivatives differently - # Data path structure is unpredictable, so keep the same relative path - if derivatives: - copy_file = os.path.relpath(file, self.bids_layout.root) - copy_file = os.path.join(self.loris_bids_root_dir, copy_file) - else : - # determine the path of the copied file - copy_file = "" - if not inheritance: - copy_file = self.loris_bids_eeg_rel_dir - if self.bids_info.session is not None: - copy_file = os.path.join(copy_file, os.path.basename(file)) - else: - # make sure the ses- is included in the new filename if using - # default visit label from the LORIS config - copy_file = os.path.join( - copy_file, - os.path.basename(file).replace( - f'sub-{self.bids_info.subject}', - f'sub-{self.bids_info.subject}_ses-{self.session.visit_label}' - ) - ) - - copy_file = os.path.join(self.loris_bids_root_dir, copy_file) - - # create the directory if it does not exist - lib.utilities.create_dir( - os.path.dirname(copy_file), - self.env.verbose + loris_file_path = get_loris_bids_file_path( + self.info, + self.session, + self.bids_info.data_type, + Path(file), + derivatives, ) - # copy the file - utilities.copy_file(file, copy_file, self.env.verbose) - - # determine the relative path and return it - relative_path = os.path.relpath(copy_file, self.data_dir) - - return relative_path + copy_loris_bids_file(self.info, Path(file), loris_file_path) + return loris_file_path def create_and_insert_archive(self, files_to_archive: list[str], archive_rel_name: str, eeg_file: DbPhysioFile): """ diff --git a/python/lib/imaging_lib/scan_type.py b/python/lib/imaging_lib/scan_type.py new file mode 100644 index 000000000..dfa552600 --- /dev/null +++ b/python/lib/imaging_lib/scan_type.py @@ -0,0 +1,17 @@ +from lib.db.models.mri_scan_type import DbMriScanType +from lib.env import Env + + +def create_mri_scan_type(env: Env, name: str) -> DbMriScanType: + """ + Create an MRI scan type in the database. + """ + + scan_type = DbMriScanType( + name = name, + ) + + env.db.add(scan_type) + env.db.flush() + + return scan_type diff --git a/python/lib/import_bids_dataset/acquisitions.py b/python/lib/import_bids_dataset/acquisitions.py new file mode 100644 index 000000000..8d81828da --- /dev/null +++ b/python/lib/import_bids_dataset/acquisitions.py @@ -0,0 +1,43 @@ +from collections.abc import Callable +from typing import TypeVar + +from loris_bids_reader.info import BidsAcquisitionInfo + +from lib.env import Env +from lib.import_bids_dataset.env import BidsImportEnv +from lib.logging import log, log_error + +T = TypeVar('T') + + +def import_bids_acquisitions( + env: Env, + import_env: BidsImportEnv, + acquisitions: list[tuple[T, BidsAcquisitionInfo]], + importer: Callable[[T, BidsAcquisitionInfo], None] +): + """ + Run an import function on a list of BIDS acquisitions, logging the overall import progress, + and catching the eventual exceptions raised during each import. + """ + + for acquisition, bids_info in acquisitions: + log( + env, + f"Importing {bids_info.data_type} acquisition '{bids_info.name}'...", + ) + + try: + importer(acquisition, bids_info) + log(env, f"Successfully imported acquisition '{bids_info.name}'.") + import_env.imported_acquisitions_count += 1 + except Exception as exception: + log_error( + env, + ( + f"Error while importing acquisition '{bids_info.name}'. Error message:\n" + f"{exception}\n" + "Skipping." + ) + ) + import_env.failed_acquisitions_count += 1 diff --git a/python/lib/import_bids_dataset/copy_files.py b/python/lib/import_bids_dataset/copy_files.py index 4df03ddd3..ae68ef9c8 100644 --- a/python/lib/import_bids_dataset/copy_files.py +++ b/python/lib/import_bids_dataset/copy_files.py @@ -1,23 +1,102 @@ - import os +import re +import shutil +from pathlib import Path from loris_bids_reader.files.scans import BidsScansTsvFile import lib.utilities +from lib.db.models.session import DbSession +from lib.import_bids_dataset.env import BidsImportEnv + + +def get_loris_bids_file_path( + import_env: BidsImportEnv, + session: DbSession, + data_type: str, + file_path: Path, + derivative: bool = False, +) -> Path: + """ + Get the path of a BIDS file in LORIS, relative to the LORIS data directory. + """ + + # In the import is run in no-copy mode, simply return the original file path. + if import_env.loris_bids_path is None: + return file_path.relative_to(import_env.data_dir_path) + + # If the file is a derivative, the path is unpredictable, so return a copy of that path in the + # LORIS BIDS dataset. + if derivative: + return import_env.loris_bids_path / file_path.relative_to(import_env.source_bids_path) + + # Otherwise, normalize the subject and session directrory names using the LORIS session + # information. + loris_file_name = get_loris_bids_file_name(file_path.name, session) + + return ( + import_env.loris_bids_path + / f'sub-{session.candidate.psc_id}' + / f'ses-{session.visit_label}' + / data_type + / loris_file_name + ) + + +def get_loris_bids_file_name(file_name: str, session: DbSession) -> str: + """ + Get the name of a BIDS file in LORIS, replacing or adding the BIDS subject and session labels + with the LORIS PSCID and visit label. + """ + + # Remove the subject and session entities if they are present. + file_name = re.sub(r'sub-[a-zA-Z0-9]+_?', '', file_name) + file_name = re.sub(r'ses-[a-zA-Z0-9]+_?', '', file_name) + + # Add the LORIS subject and session information back in the correct order. + return f'sub-{session.candidate.psc_id}_ses-{session.visit_label}_{file_name}' + + +def copy_loris_bids_file(import_env: BidsImportEnv, file_path: Path, loris_file_path: Path): + """ + Copy a BIDS file to the LORIS data directory, unless the no-copy mode is enabled. + """ + + # Do not copy the file in no-copy mode. + if import_env.loris_bids_path is None: + return + + full_loris_file_path = import_env.data_dir_path / loris_file_path + + if full_loris_file_path.exists(): + raise Exception(f"File '{loris_file_path}' already exists in LORIS.") + + full_loris_file_path.parent.mkdir(parents=True, exist_ok=True) + if file_path.is_file(): + shutil.copyfile(file_path, full_loris_file_path) + elif file_path.is_dir(): + shutil.copytree(file_path, full_loris_file_path) +# TODO: This function is ugly and should be replaced. def copy_scans_tsv_file_to_loris_bids_dir( scans_file: BidsScansTsvFile, - bids_sub_id: str, - loris_bids_root_dir: str, - data_dir: str, + session: DbSession, + loris_bids_root_dir: Path, + data_dir: Path, ) -> str: """ Copy the scans.tsv file to the LORIS BIDS directory for the subject. """ original_file_path = scans_file.path - final_file_path = os.path.join(loris_bids_root_dir, f'sub-{bids_sub_id}', scans_file.path.name) + loris_file_name = get_loris_bids_file_name(scans_file.path.name, session) + final_file_path = ( + loris_bids_root_dir + / f'sub-{session.candidate.psc_id}' + / f'ses-{session.visit_label}' + / loris_file_name + ) # copy the scans.tsv file to the new directory if os.path.exists(final_file_path): diff --git a/python/lib/import_bids_dataset/env.py b/python/lib/import_bids_dataset/env.py new file mode 100644 index 000000000..2a849fcaf --- /dev/null +++ b/python/lib/import_bids_dataset/env.py @@ -0,0 +1,39 @@ +from dataclasses import dataclass +from pathlib import Path + + +@dataclass +class BidsImportEnv: + """ + Information about a specific BIDS import pipeline run. + """ + + data_dir_path: Path + """ + The LORIS data directory path. + """ + + source_bids_path: Path + """ + The source BIDS directory path. + """ + + loris_bids_path: Path | None + """ + The LORIS BIDS directory path for this import, relative to the LORIS data directory. + """ + + imported_acquisitions_count: int = 0 + """ + The number of succesfully imported BIDS acquisitions. + """ + + ignored_acquisitions_count: int = 0 + """ + The number of ignored BIDS acquisition imports. + """ + + failed_acquisitions_count: int = 0 + """ + The number of failed BIDS acquisition imports. + """ diff --git a/python/lib/import_bids_dataset/mri.py b/python/lib/import_bids_dataset/mri.py new file mode 100644 index 000000000..0ce87e19c --- /dev/null +++ b/python/lib/import_bids_dataset/mri.py @@ -0,0 +1,218 @@ +from pathlib import Path +from typing import Any + +from loris_bids_reader.info import BidsAcquisitionInfo +from loris_bids_reader.mri.acquisition import MriAcquisition +from loris_bids_reader.mri.reader import BidsMriDataTypeReader +from loris_utils.crypto import compute_file_blake2b_hash +from loris_utils.error import group_errors_tuple + +from lib.db.models.mri_scan_type import DbMriScanType +from lib.db.models.session import DbSession +from lib.db.queries.file import try_get_file_with_hash, try_get_file_with_path +from lib.db.queries.mri_scan_type import try_get_mri_scan_type_with_name +from lib.env import Env +from lib.imaging_lib.file import register_mri_file +from lib.imaging_lib.file_parameter import register_mri_file_parameter, register_mri_file_parameters +from lib.imaging_lib.nifti import add_nifti_spatial_file_parameters +from lib.imaging_lib.nifti_pic import create_nifti_preview_picture +from lib.imaging_lib.scan_type import create_mri_scan_type +from lib.import_bids_dataset.acquisitions import import_bids_acquisitions +from lib.import_bids_dataset.copy_files import copy_loris_bids_file, get_loris_bids_file_path +from lib.import_bids_dataset.env import BidsImportEnv +from lib.import_bids_dataset.file_type import get_check_bids_imaging_file_type_from_extension +from lib.import_bids_dataset.mri_sidecar import add_bids_mri_sidecar_file_parameters +from lib.import_bids_dataset.scans import add_bids_scans_file_parameters +from lib.logging import log + +KNOWN_SUFFIXES_PER_MRI_DATA_TYPE = { + 'anat': [ + 'T1w', 'T2w', 'T1rho', 'T1map', 'T2map', 'T2star', 'FLAIR', 'FLASH', 'PD', 'PDmap', 'PDT2', + 'inplaneT1', 'inplaneT2', 'angio', + ], + 'func': [ + 'bold', 'cbv', 'phase', + ], + 'dwi': [ + 'dwi', 'sbref', + ], + 'fmap': [ + 'phasediff', 'magnitude1', 'magnitude2', 'phase1', 'phase2', 'fieldmap', 'epi', + ], +} + + +def import_bids_mri_data_type( + env: Env, + import_env: BidsImportEnv, + session: DbSession, + data_type: BidsMriDataTypeReader, +): + """ + Import the MRI acquisitions found in a BIDS MRI data type directory. + """ + + import_bids_acquisitions( + env, + import_env, + data_type.acquisitions, + lambda acquisition, bids_info: import_bids_mri_acquisition( + env, + import_env, + session, + acquisition, + bids_info, + ), + ) + + +def import_bids_mri_acquisition( + env: Env, + import_env: BidsImportEnv, + session: DbSession, + acquisition: MriAcquisition, + bids_info: BidsAcquisitionInfo, +): + """ + Import a BIDS NIfTI file and its associated files in LORIS. + """ + + # The files to copy to LORIS, with the source path on the left and the LORIS path on the right. + files_to_copy: list[tuple[Path, Path]] = [] + + loris_file_path = get_loris_bids_file_path(import_env, session, bids_info.data_type, acquisition.nifti_path) + files_to_copy.append((acquisition.nifti_path, loris_file_path)) + + # Check whether the file is already registered in LORIS. + + loris_file = try_get_file_with_path(env.db, loris_file_path) + if loris_file is not None: + import_env.ignored_acquisitions_count += 1 + log(env, f"File '{loris_file_path}' is already registered in LORIS. Skipping.") + return + + # Get information about the file. + + file_type, file_hash, scan_type = group_errors_tuple( + f"Error while checking database information for MRI acquisition '{bids_info.name}'.", + lambda: get_check_bids_imaging_file_type_from_extension(env, acquisition.nifti_path), + lambda: get_check_bids_nifti_file_hash(env, acquisition), + lambda: get_check_bids_nifti_mri_scan_type(env, bids_info), + ) + + # Get the auxiliary files. + + # The auxiliary files to the NIfTI file and its sidecar, with the file type on the left and the + # file path on the right. + aux_file_paths: list[tuple[str, Path]] = [] + + if acquisition.bval_path is not None: + aux_file_paths.append(('bval', acquisition.bval_path)) + + if acquisition.bvec_path is not None: + aux_file_paths.append(('bvec', acquisition.bvec_path)) + + if acquisition.physio_path is not None: + aux_file_paths.append(('physio', acquisition.physio_path)) + + if acquisition.events_path is not None: + aux_file_paths.append(('events', acquisition.events_path)) + + # Get the file parameters. + + file_parameters: dict[str, Any] = {} + + if acquisition.sidecar_file is not None: + add_bids_mri_sidecar_file_parameters(env, acquisition.sidecar_file, file_parameters) + json_loris_path = get_loris_bids_file_path( + import_env, + session, + bids_info.data_type, + acquisition.sidecar_file.path, + ) + + files_to_copy.append((acquisition.sidecar_file.path, json_loris_path)) + file_parameters['bids_json_file'] = json_loris_path + file_parameters['bids_json_file_blake2b_hash'] = compute_file_blake2b_hash(acquisition.sidecar_file.path) + + add_nifti_spatial_file_parameters(acquisition.nifti_path, file_parameters) + file_parameters['file_blake2b_hash'] = file_hash + + if bids_info.scans_file is not None and bids_info.scan_row is not None: + add_bids_scans_file_parameters(bids_info.scans_file, bids_info.scan_row, file_parameters) + + for aux_file_type, aux_file_path in aux_file_paths: + aux_file_hash = compute_file_blake2b_hash(aux_file_path) + aux_file_loris_path = get_loris_bids_file_path(import_env, session, bids_info.data_type, aux_file_path) + files_to_copy.append((aux_file_path, aux_file_loris_path)) + file_parameters[f'bids_{aux_file_type}'] = str(aux_file_loris_path) + file_parameters[f'bids_{aux_file_type}_blake2b_hash'] = aux_file_hash + + # Copy the files on the file system. + for copied_file_path, loris_copied_file_path in files_to_copy: + copy_loris_bids_file(import_env, copied_file_path, loris_copied_file_path) + + # Register the file and its parameters in the database. + + file = register_mri_file( + env, + loris_file_path, + file_type, + session, + scan_type, + None, + None, + file_parameters.get('SeriesInstanceUID'), + file_parameters.get('EchoTime'), + file_parameters.get('EchoNumber'), + file_parameters.get('PhaseEncodingDirection'), + bids_info.scan_row.get_acquisition_time() if bids_info.scan_row is not None else None, + False, + ) + + register_mri_file_parameters(env, file, file_parameters) + + env.db.commit() + + # Create and register the file picture. + + pic_rel_path = create_nifti_preview_picture(env, file) + + register_mri_file_parameter(env, file, 'check_pic_filename', str(pic_rel_path)) + + env.db.commit() + + +def get_check_bids_nifti_file_hash(env: Env, acquisition: MriAcquisition) -> str: + """ + Compute the BLAKE2b hash of a NIfTI file and raise an exception if that hash is already + registered in the database. + """ + + file_hash = compute_file_blake2b_hash(acquisition.nifti_path) + + file = try_get_file_with_hash(env.db, file_hash) + if file is not None: + raise Exception(f"File with hash '{file_hash}' already present in the database.") + + return file_hash + + +def get_check_bids_nifti_mri_scan_type(env: Env, bids_info: BidsAcquisitionInfo) -> DbMriScanType: + """ + Get the MRI scan type corresponding to a BIDS MRI acquisition using its BIDS suffix. Create the + MRI scan type in the database the suffix is a standard BIDS suffix and the scan type does not + already exist in the database, or raise an exception if no known scan type is found. + """ + + if bids_info.suffix is None: + raise Exception("No BIDS suffix found in the NIfTI file name, cannot infer the file data type.") + + mri_scan_type = try_get_mri_scan_type_with_name(env.db, bids_info.suffix) + if mri_scan_type is not None: + return mri_scan_type + + if bids_info.suffix not in KNOWN_SUFFIXES_PER_MRI_DATA_TYPE[bids_info.data_type]: + raise Exception(f"Found unknown MRI file suffix '{bids_info.suffix}'.") + + return create_mri_scan_type(env, bids_info.suffix) diff --git a/python/lib/import_bids_dataset/scans.py b/python/lib/import_bids_dataset/scans.py new file mode 100644 index 000000000..c7bef44c5 --- /dev/null +++ b/python/lib/import_bids_dataset/scans.py @@ -0,0 +1,20 @@ +from typing import Any + +from loris_bids_reader.files.scans import BidsScansTsvFile, BidsScanTsvRow +from loris_utils.crypto import compute_file_blake2b_hash + + +def add_bids_scans_file_parameters( + scans_file: BidsScansTsvFile, + scan_row: BidsScanTsvRow, + file_parameters: dict[str, Any], +): + """ + Read a BIDS `scans.tsv` file and row, and add its information to the LORIS file parameters + dictionary. + """ + + file_parameters['scan_acquisition_time'] = scan_row.get_acquisition_time() + file_parameters['age_at_scan'] = scan_row.get_age_at_scan() + file_parameters['scans_tsv_file'] = scans_file.path + file_parameters['scans_tsv_file_bake2hash'] = compute_file_blake2b_hash(scans_file.path) diff --git a/python/lib/mri.py b/python/lib/mri.py deleted file mode 100644 index 9017e37d9..000000000 --- a/python/lib/mri.py +++ /dev/null @@ -1,399 +0,0 @@ -"""Deals with MRI BIDS datasets and register them into the database.""" - -import getpass -import os -import re -import sys -from pathlib import Path - -from loris_bids_reader.files.scans import BidsScansTsvFile -from loris_bids_reader.mri.sidecar import BidsMriSidecarJsonFile -from loris_utils.crypto import compute_file_blake2b_hash - -import lib.exitcode -import lib.utilities as utilities -from lib.db.models.session import DbSession -from lib.env import Env -from lib.imaging import Imaging -from lib.import_bids_dataset.copy_files import copy_scans_tsv_file_to_loris_bids_dir -from lib.import_bids_dataset.file_type import get_check_bids_imaging_file_type_from_extension - - -class Mri: - """ - This class reads the BIDS MRI data structure and registers the MRI datasets into the - database by calling lib.imaging class. - - :Example: - - from lib.mri import Mri - from lib.database import Database - - # database connection - db = Database(config_file.mysql, verbose) - db.connect() - - # grep config settings from the Config module - config_obj = Config(db, verbose) - default_bids_vl = config_obj.get_config('default_bids_vl') - data_dir = config_obj.get_config('dataDirBasepath') - - # create the LORIS_BIDS directory in data_dir based on Name and BIDS version - loris_bids_root_dir = create_loris_bids_directory( - bids_reader, data_dir, verbose - ) - for row in bids_reader.cand_session_modalities_list: - for modality in row['modalities']: - if modality in ['anat', 'dwi', 'fmap', 'func']: - bids_session = row['bids_ses_id'] - visit_label = bids_session if bids_session else default_bids_vl - loris_bids_mri_rel_dir = "sub-" + row['bids_sub_id'] + "/" + \ - "ses-" + visit_label + "/mri/" - lib.utilities.create_dir( - loris_bids_root_dir + loris_bids_mri_rel_dir, verbose - ) - Mri( - env = env, - bids_layout = bids_layout, - session = session, - bids_sub_id = row['bids_sub_id'], - bids_ses_id = row['bids_ses_id'], - bids_modality = modality, - db = db, - verbose = verbose, - data_dir = data_dir, - default_visit_label = default_bids_vl, - loris_bids_eeg_rel_dir = loris_bids_mri_rel_dir, - loris_bids_root_dir = loris_bids_root_dir - ) - - # disconnect from the database - db.disconnect() - """ - - def __init__(self, env: Env, bids_layout, session: DbSession, bids_sub_id, bids_ses_id, bids_modality, db, - verbose, data_dir, default_visit_label, - loris_bids_mri_rel_dir, loris_bids_root_dir): - - # enumerate the different suffixes supported by BIDS per modality type - self.possible_suffix_per_modality = { - 'anat' : [ - 'T1w', 'T2w', 'T1rho', 'T1map', 'T2map', 'T2star', 'FLAIR', - 'FLASH', 'PD', 'PDmap', 'PDT2', 'inplaneT1', 'inplaneT2', 'angio' - ], - 'func' : [ - 'bold', 'cbv', 'phase' - ], - 'dwi' : [ - 'dwi', 'sbref' - ], - 'fmap' : [ - 'phasediff', 'magnitude1', 'magnitude2', 'phase1', 'phase2', - 'fieldmap', 'epi' - ] - } - - self.env = env - - # load bids objects - self.bids_layout = bids_layout - - # load the LORIS BIDS import root directory where the files will be copied - self.loris_bids_mri_rel_dir = loris_bids_mri_rel_dir - self.loris_bids_root_dir = loris_bids_root_dir - self.data_dir = data_dir - - # load BIDS subject, visit and modality - self.bids_sub_id = bids_sub_id - self.bids_ses_id = bids_ses_id - self.bids_modality = bids_modality - - # load database handler object and verbose bool - self.db = db - self.verbose = verbose - - # find corresponding CandID and SessionID in LORIS - self.session = session - self.default_vl = default_visit_label - - # grep all the NIfTI files for the modality - self.nifti_files = self.grep_nifti_files() - - # check if a tsv with acquisition dates or age is available for the subject - self.scans_file = None - if self.bids_layout.get(suffix='scans', subject=self.session.candidate.psc_id, return_type='filename'): - scans_file_path = self.bids_layout.get(suffix='scans', subject=self.session.candidate.psc_id, - return_type='filename', extension='tsv')[0] - self.scans_file = BidsScansTsvFile(Path(scans_file_path)) - - # loop through NIfTI files and register them in the DB - for nifti_file in self.nifti_files: - self.register_raw_file(nifti_file) - - def grep_nifti_files(self): - """ - Returns the list of NIfTI files found for the modality. - - :return: list of NIfTI files found for the modality - :rtype: list - """ - - # grep all the possible suffixes for the modality - modality_possible_suffix = self.possible_suffix_per_modality[self.bids_modality] - - # loop through the possible suffixes and grep the NIfTI files - nii_files_list = [] - for suffix in modality_possible_suffix: - nii_files_list.extend(self.grep_bids_files(suffix, 'nii.gz')) - - # return the list of found NIfTI files - return nii_files_list - - def grep_bids_files(self, bids_type, extension): - """ - Greps the BIDS files and their layout information from the BIDSLayout - and return that list. - - :param bids_type: the BIDS type to use to grep files (T1w, T2w, bold, dwi...) - :type bids_type: str - :param extension: extension of the file to look for (nii.gz, json...) - :type extension: str - - :return: list of files from the BIDS layout - :rtype: list - """ - - if self.bids_ses_id: - return self.bids_layout.get( - subject = self.bids_sub_id, - session = self.bids_ses_id, - datatype = self.bids_modality, - extension = extension, - suffix = bids_type - ) - else: - return self.bids_layout.get( - subject = self.bids_sub_id, - datatype = self.bids_modality, - extension = extension, - suffix = bids_type - ) - - def register_raw_file(self, nifti_file): - """ - Registers raw MRI files and related files into the files and parameter_file tables. - - :param nifti_file: NIfTI file object - :type nifti_file: pybids NIfTI file object - """ - - # insert the NIfTI file - self.fetch_and_insert_nifti_file(nifti_file) - - def fetch_and_insert_nifti_file(self, nifti_file, derivatives=None): - """ - Gather NIfTI file information to insert into the files and parameter_file tables. - Once all the information has been gathered, it will call imaging.insert_imaging_file - that will perform the insertion into the files and parameter_file tables. - - :param nifti_file : NIfTI file object - :type nifti_file : pybids NIfTI file object - :param derivatives: whether the file to be registered is a derivative file - :type derivatives: bool - - :return: dictionary with the inserted file_id and file_path - :rtype: dict - """ - - # load the Imaging object that will be used to insert the imaging data into the database - imaging = Imaging(self.db, self.verbose) - - # load the list of associated files with the NIfTI file - associated_files = nifti_file.get_associations() - - # load the entity information from the NIfTI file - entities = nifti_file.get_entities() - scan_type = entities['suffix'] - - # loop through the associated files to grep JSON, bval, bvec... - sidecar_json = None - other_assoc_files = {} - for assoc_file in associated_files: - file_info = assoc_file.get_entities() - if re.search(r'json$', file_info['extension']): - sidecar_json = BidsMriSidecarJsonFile(Path(assoc_file.path)) - elif re.search(r'bvec$', file_info['extension']): - other_assoc_files['bvec_file'] = assoc_file.path - elif re.search(r'bval$', file_info['extension']): - other_assoc_files['bval_file'] = assoc_file.path - elif re.search(r'tsv$', file_info['extension']) and file_info['suffix'] == 'events': - other_assoc_files['task_file'] = assoc_file.path - elif re.search(r'tsv$', file_info['extension']) and file_info['suffix'] == 'physio': - other_assoc_files['physio_file'] = assoc_file.path - - # read the json file if it exists - file_parameters = {} - if sidecar_json is not None: - file_parameters = imaging.map_bids_param_to_loris_param(sidecar_json.data) - # copy the JSON file to the LORIS BIDS import directory - json_path = self.copy_file_to_loris_bids_dir(sidecar_json.path) - file_parameters['bids_json_file'] = json_path - json_blake2 = compute_file_blake2b_hash(sidecar_json.path) - file_parameters['bids_json_file_blake2b_hash'] = json_blake2 - - # grep the file type from the ImagingFileTypes table - file_type = get_check_bids_imaging_file_type_from_extension(self.env, Path(nifti_file.filename)) - - # determine the output type - output_type = 'derivatives' if derivatives else 'native' - if not derivatives: - coordinate_space = 'native' - - # get the acquisition date of the MRI or the age at the time of acquisition - if self.scans_file is not None: - scan_info = self.scans_file.get_row(Path(nifti_file.path)) - if scan_info is not None: - try: - file_parameters['scan_acquisition_time'] = scan_info.get_acquisition_time() - file_parameters['age_at_scan'] = scan_info.get_age_at_scan() - except Exception as error: - print(f"ERROR: {error}") - sys.exit(lib.exitcode.PROGRAM_EXECUTION_FAILURE) - - # copy the scans.tsv file to the LORIS BIDS import directory - scans_path = copy_scans_tsv_file_to_loris_bids_dir( - self.scans_file, - self.bids_sub_id, - self.loris_bids_root_dir, - self.data_dir, - ) - - file_parameters['scans_tsv_file'] = scans_path - scans_blake2 = compute_file_blake2b_hash(self.scans_file.path) - file_parameters['scans_tsv_file_bake2hash'] = scans_blake2 - - # grep voxel step from the NIfTI file header - step_parameters = imaging.get_nifti_image_step_parameters(nifti_file.path) - file_parameters['xstep'] = step_parameters[0] - file_parameters['ystep'] = step_parameters[1] - file_parameters['zstep'] = step_parameters[2] - - # grep the time length from the NIfTI file header - is_4d_dataset = False - length_parameters = imaging.get_nifti_image_length_parameters(nifti_file.path) - if len(length_parameters) == 4: - file_parameters['time'] = length_parameters[3] - is_4d_dataset = True - - # add all other associated files to the file_parameters so they get inserted - # in parameter_file - for type in other_assoc_files: - original_file_path = other_assoc_files[type] - copied_path = self.copy_file_to_loris_bids_dir(original_file_path) - file_param_name = 'bids_' + type - file_parameters[file_param_name] = copied_path - file_blake2 = compute_file_blake2b_hash(original_file_path) - hash_param_name = file_param_name + '_blake2b_hash' - file_parameters[hash_param_name] = file_blake2 - - # append the blake2b to the MRI file parameters dictionary - blake2 = compute_file_blake2b_hash(nifti_file.path) - file_parameters['file_blake2b_hash'] = blake2 - - # check that the file is not already inserted before inserting it - result = imaging.grep_file_info_from_hash(blake2) - file_id = result['FileID'] if result else None - file_path = result['File'] if result else None - if not file_id: - # grep the scan type ID from the mri_scan_type table (if it is not already in - # the table, it will add a row to the mri_scan_type table) - scan_type_id = self.db.grep_id_from_lookup_table( - id_field_name = 'MriScanTypeID', - table_name = 'mri_scan_type', - where_field_name = 'MriScanTypeName', - where_value = scan_type, - insert_if_not_found = True - ) - - # copy the NIfTI file to the LORIS BIDS import directory - file_path = self.copy_file_to_loris_bids_dir(nifti_file.path) - - # insert the file along with its information into files and parameter_file tables - echo_time = file_parameters['EchoTime'] if 'EchoTime' in file_parameters.keys() else None - echo_nb = file_parameters['EchoNumber'] if 'EchoNumber' in file_parameters.keys() else None - phase_enc_dir = file_parameters['PhaseEncodingDirection'] \ - if 'PhaseEncodingDirection' in file_parameters.keys() else None - file_info = { - 'FileType' : file_type.name, - 'File' : file_path, - 'SessionID' : self.session.id, - 'InsertedByUserID': getpass.getuser(), - 'CoordinateSpace' : coordinate_space, - 'OutputType' : output_type, - 'EchoTime' : echo_time, - 'PhaseEncodingDirection': phase_enc_dir, - 'EchoNumber' : echo_nb, - 'SourceFileID' : None, - 'MriScanTypeID' : scan_type_id - } - file_id = imaging.insert_imaging_file(file_info, file_parameters) - - # create the pic associated with the file - pic_rel_path = imaging.create_imaging_pic( - { - 'cand_id' : self.session.candidate.cand_id, - 'data_dir_path': self.data_dir, - 'file_rel_path': file_path, - 'is_4D_dataset': is_4d_dataset, - 'file_id' : file_id - } - ) - if os.path.exists(os.path.join(self.data_dir, 'pic/', pic_rel_path)): - imaging.insert_parameter_file(file_id, 'check_pic_filename', pic_rel_path) - - return {'file_id': file_id, 'file_path': file_path} - - def copy_file_to_loris_bids_dir(self, file, derivatives_path=None): - """ - Wrapper around the utilities.copy_file function that copies the file - to the LORIS BIDS import directory and returns the relative path of the - file (without the data_dir part). - - :param file: full path to the original file - :type file: str - :param derivatives_path: path to the derivative folder - :type derivatives_path: str - - :return: relative path to the copied file - :rtype: str - """ - - # determine the path of the copied file - copy_file = self.loris_bids_mri_rel_dir - if self.bids_ses_id: - copy_file += os.path.basename(file) - else: - # make sure the ses- is included in the new filename if using - # default visit label from the LORIS config - copy_file += str.replace( - os.path.basename(file), - "sub-" + self.bids_sub_id, - "sub-" + self.bids_sub_id + "_ses-" + self.default_vl - ) - if derivatives_path: - # create derivative subject/vl/modality directory - lib.utilities.create_dir( - derivatives_path + self.loris_bids_mri_rel_dir, - self.verbose - ) - copy_file = derivatives_path + copy_file - else: - copy_file = self.loris_bids_root_dir + copy_file - - # copy the file - utilities.copy_file(file, copy_file, self.verbose) - - # determine the relative path and return it - relative_path = copy_file.replace(self.data_dir, "") - - return relative_path diff --git a/python/loris_bids_reader/src/loris_bids_reader/info.py b/python/loris_bids_reader/src/loris_bids_reader/info.py index c84a4a369..e106ce6aa 100644 --- a/python/loris_bids_reader/src/loris_bids_reader/info.py +++ b/python/loris_bids_reader/src/loris_bids_reader/info.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from loris_bids_reader.files.participants import BidsParticipantTsvRow +from loris_bids_reader.files.scans import BidsScansTsvFile, BidsScanTsvRow @dataclass @@ -31,6 +32,11 @@ class BidsSessionInfo(BidsSubjectInfo): The BIDS session label. """ + scans_file: BidsScansTsvFile | None + """ + The BIDS `scans.tsv` file of this session, if any. + """ + @dataclass class BidsDataTypeInfo(BidsSessionInfo): @@ -42,3 +48,25 @@ class BidsDataTypeInfo(BidsSessionInfo): """ The BIDS data type name. """ + + +@dataclass +class BidsAcquisitionInfo(BidsDataTypeInfo): + """ + Information about a BIDS acquisition. + """ + + name: str + """ + The name of this acquisition (usually the file name without the extension). + """ + + suffix: str | None + """ + The BIDS suffix of this acquisition, if any. + """ + + scan_row: BidsScanTsvRow | None + """ + The BIDS `scans.tsv` row of this acquisition, if any. + """ diff --git a/python/loris_bids_reader/src/loris_bids_reader/mri/acquisition.py b/python/loris_bids_reader/src/loris_bids_reader/mri/acquisition.py new file mode 100644 index 000000000..a9c071397 --- /dev/null +++ b/python/loris_bids_reader/src/loris_bids_reader/mri/acquisition.py @@ -0,0 +1,41 @@ +from dataclasses import dataclass +from pathlib import Path + +from loris_bids_reader.mri.sidecar import BidsMriSidecarJsonFile + + +@dataclass +class MriAcquisition: + """ + An MRI acquisition and its related files. + """ + + nifti_path: Path + """ + The main NIfTI file path. + """ + + sidecar_file: BidsMriSidecarJsonFile | None + """ + The related JSON sidecar file path, if it exists. + """ + + bval_path: Path | None + """ + The related bval file path, if it exists. + """ + + bvec_path: Path | None + """ + The related bvec file path, if it exists. + """ + + physio_path: Path | None + """ + The related physio file path, if it exists. + """ + + events_path: Path | None + """ + The related events file path, if it exists. + """ diff --git a/python/loris_bids_reader/src/loris_bids_reader/mri/reader.py b/python/loris_bids_reader/src/loris_bids_reader/mri/reader.py new file mode 100644 index 000000000..0cef6077e --- /dev/null +++ b/python/loris_bids_reader/src/loris_bids_reader/mri/reader.py @@ -0,0 +1,81 @@ + +from dataclasses import dataclass +from functools import cached_property +from pathlib import Path + +from bids.layout import BIDSFile +from loris_utils.path import remove_path_extension + +from loris_bids_reader.info import BidsAcquisitionInfo +from loris_bids_reader.mri.acquisition import MriAcquisition +from loris_bids_reader.mri.sidecar import BidsMriSidecarJsonFile +from loris_bids_reader.reader import BidsDataTypeReader +from loris_bids_reader.utils import find_pybids_file_path, get_pybids_file_path + + +@dataclass +class BidsMriDataTypeReader(BidsDataTypeReader): + @cached_property + def acquisitions(self) -> list[tuple[MriAcquisition, BidsAcquisitionInfo]]: + pybids_layout = self.session.subject.dataset.layout + pybids_files: list[BIDSFile] = pybids_layout.get( # type: ignore + subject = self.session.subject.label, + session = self.session.label, + datatype = self.name, + extension = ['.nii', '.nii.gz'], + ) + + acquisitions: list[tuple[MriAcquisition, BidsAcquisitionInfo]] = [] + for pybids_file in pybids_files: + nifti_path = get_pybids_file_path(pybids_file) + + # Get all associated files + associations: list[BIDSFile] = pybids_file.get_associations() # type: ignore + + # Find associated files using predicates + sidecar_path = find_pybids_file_path(associations, lambda file: file.entities.get('extension') == '.json') + + pybids_bval_path = pybids_layout.get_nearest(pybids_file, extension='.bval') # type: ignore + bval_path = Path(pybids_bval_path) if pybids_bval_path is not None else None # type: ignore + + pybids_bvec_path = pybids_layout.get_nearest(pybids_file, extension='.bvec') # type: ignore + bvec_path = Path(pybids_bvec_path) if pybids_bvec_path is not None else None # type: ignore + + events_path = find_pybids_file_path( + associations, + lambda file: file.entities.get('suffix') == 'events' and file.entities.get('extension') == '.tsv', + ) + + physio_path = find_pybids_file_path( + associations, + lambda file: file.entities.get('suffix') in ['physio', 'stim'] + and file.entities.get('extension') in ['.tsv.gz', '.tsv'], + ) + + sidecar_file = BidsMriSidecarJsonFile(sidecar_path) if sidecar_path is not None else None + scan_row = self.session.scans_file.get_row(nifti_path) if self.session.scans_file is not None else None + acquisition_name = remove_path_extension(nifti_path).name + + bids_info = BidsAcquisitionInfo( + subject = self.session.subject.label, + participant_row = self.session.subject.participant_row, + session = self.session.label, + scans_file = self.session.scans_file, + data_type = self.name, + scan_row = scan_row, + name = acquisition_name, + suffix = pybids_file.entities.get('suffix'), + ) + + acquisition = MriAcquisition( + nifti_path = nifti_path, + sidecar_file = sidecar_file, + bval_path = bval_path, + bvec_path = bvec_path, + physio_path = physio_path, + events_path = events_path, + ) + + acquisitions.append((acquisition, bids_info)) + + return acquisitions diff --git a/python/loris_bids_reader/src/loris_bids_reader/reader.py b/python/loris_bids_reader/src/loris_bids_reader/reader.py index eeb15b402..7eaa8612e 100644 --- a/python/loris_bids_reader/src/loris_bids_reader/reader.py +++ b/python/loris_bids_reader/src/loris_bids_reader/reader.py @@ -1,14 +1,21 @@ import re +from collections.abc import Sequence from dataclasses import dataclass from functools import cached_property from pathlib import Path +from typing import TYPE_CHECKING from bids import BIDSLayout, BIDSLayoutIndexer from loris_bids_reader.files.dataset_description import BidsDatasetDescriptionJsonFile from loris_bids_reader.files.participants import BidsParticipantsTsvFile, BidsParticipantTsvRow +from loris_bids_reader.files.scans import BidsScansTsvFile from loris_bids_reader.info import BidsDataTypeInfo, BidsSessionInfo, BidsSubjectInfo +# Circular imports +if TYPE_CHECKING: + from loris_bids_reader.mri.reader import BidsMriDataTypeReader + PYBIDS_IGNORE = ['.git', 'code/', 'log/', 'sourcedata/'] PYBIDS_FORCE_INDEX = [re.compile(r"_annotations\.(tsv|json)$")] @@ -208,9 +215,42 @@ class BidsSessionReader: """ @cached_property - def data_types(self) -> list['BidsDataTypeReader']: + def scans_file(self) -> BidsScansTsvFile | None: + scans_paths: list[str] = self.subject.dataset.layout.get( # type: ignore + subject=self.subject.label, + session=self.label, + suffix='scans', + return_type='filename', + ) + + if scans_paths == []: + return None + + return BidsScansTsvFile(Path(scans_paths[0])) + + @cached_property + def mri_data_types(self) -> list['BidsMriDataTypeReader']: """ - Get the data type directory readers of this session. + Get the MRI data type directory readers of this session. + """ + + from loris_bids_reader.mri.reader import BidsMriDataTypeReader + + return [ + BidsMriDataTypeReader( + session=self, + name=data_type, # type: ignore + ) for data_type in self.subject.dataset.layout.get_datatypes( # type: ignore + subject=self.subject.label, + session=self.label, + datatype=['anat', 'dwi', 'fmap', 'func'], + ) + ] + + @cached_property + def eeg_data_types(self) -> list['BidsDataTypeReader']: + """ + Get the EEG data type directory readers of this session. """ return [ @@ -220,9 +260,18 @@ def data_types(self) -> list['BidsDataTypeReader']: ) for data_type in self.subject.dataset.layout.get_datatypes( # type: ignore subject=self.subject.label, session=self.label, + datatype=['eeg', 'ieeg'], ) ] + @cached_property + def data_types(self) -> Sequence['BidsDataTypeReader']: + """ + Get all the data type directory readers of this session. + """ + + return self.eeg_data_types + self.mri_data_types + @cached_property def info(self) -> BidsSessionInfo: """ @@ -233,6 +282,7 @@ def info(self) -> BidsSessionInfo: subject = self.subject.label, participant_row = self.subject.participant_row, session = self.label, + scans_file = self.scans_file, ) @@ -262,5 +312,6 @@ def info(self) -> BidsDataTypeInfo: subject = self.session.subject.label, participant_row = self.session.subject.participant_row, session = self.session.label, + scans_file = self.session.scans_file, data_type = self.name, ) diff --git a/python/loris_bids_reader/src/loris_bids_reader/utils.py b/python/loris_bids_reader/src/loris_bids_reader/utils.py new file mode 100644 index 000000000..9d633f697 --- /dev/null +++ b/python/loris_bids_reader/src/loris_bids_reader/utils.py @@ -0,0 +1,41 @@ +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from bids import BIDSLayout +from bids.layout import BIDSFile +from loris_utils.iter import find + + +def try_get_pybids_value(layout: BIDSLayout, **args: Any) -> Any | None: + """ + Get zero or one PyBIDS value using the provided arguments, or raise an exception if multiple + values are found. + """ + + match layout.get(args): # type: ignore + case []: + return None + case [value]: # type: ignore + return value # type: ignore + case values: # type: ignore + raise Exception(f"Expected one or zero PyBIDS value but found {len(values)}.") # type: ignore + + +def get_pybids_file_path(file: BIDSFile) -> Path: + """ + Get the path of a PyBIDS file. + """ + + # The PyBIDS file class does not use the standard path object nor supports type checking. + return Path(file.path) # type: ignore + + +def find_pybids_file_path(files: list[BIDSFile], predicate: Callable[[BIDSFile], bool]) -> Path | None: + """ + Find the path of a file in a list of PyBIDS files using a predicate, or return `None` if no + file matches the predicate. + """ + + file = find(files, predicate) + return get_pybids_file_path(file) if file is not None else None diff --git a/python/scripts/bids_import.py b/python/scripts/bids_import.py index 840cf3e79..b2083b3af 100755 --- a/python/scripts/bids_import.py +++ b/python/scripts/bids_import.py @@ -10,6 +10,7 @@ from pathlib import Path from loris_bids_reader.files.participants import BidsParticipantsTsvFile +from loris_bids_reader.mri.reader import BidsMriDataTypeReader from loris_bids_reader.reader import BidsDatasetReader from loris_utils.crypto import compute_file_blake2b_hash @@ -27,8 +28,9 @@ from lib.env import Env from lib.import_bids_dataset.check_sessions import check_or_create_bids_sessions from lib.import_bids_dataset.check_subjects import check_or_create_bids_subjects +from lib.import_bids_dataset.env import BidsImportEnv +from lib.import_bids_dataset.mri import import_bids_mri_data_type from lib.make_env import make_env -from lib.mri import Mri def main(): @@ -291,6 +293,12 @@ def read_and_insert_bids( hed_union=hed_union ) + import_env = BidsImportEnv( + data_dir_path = Path(data_dir), + source_bids_path = Path(bids_dir), + loris_bids_path = Path(loris_bids_root_dir).relative_to(data_dir) if loris_bids_root_dir is not None else None, + ) + # read list of modalities per session / candidate and register data for data_type_reader in bids_reader.data_types: bids_info = data_type_reader.info @@ -310,35 +318,20 @@ def read_and_insert_bids( session = try_get_session_with_cand_id_visit_label(env.db, candidate.cand_id, visit_label) - match bids_info.data_type: - case 'eeg' | 'ieeg': + match (data_type_reader, bids_info.data_type): + case (_, 'eeg' | 'ieeg'): Eeg( env, - bids_layout = bids_reader.layout, - session = session, - bids_info = bids_info, - db = db, - data_dir = data_dir, - loris_bids_eeg_rel_dir = loris_bids_data_type_rel_dir, - loris_bids_root_dir = loris_bids_root_dir, - dataset_tag_dict = dataset_tag_dict, - dataset_type = type - ) - case 'anat' | 'dwi' | 'fmap' | 'func': - Mri( - env, - bids_layout = bids_reader.layout, - session = session, - bids_sub_id = bids_info.subject, - bids_ses_id = bids_info.session, - bids_modality = bids_info.data_type, - db = db, - verbose = verbose, - data_dir = data_dir, - default_visit_label = default_bids_vl, - loris_bids_mri_rel_dir = loris_bids_data_type_rel_dir, - loris_bids_root_dir = loris_bids_root_dir + import_env, + bids_layout = bids_reader.layout, + session = session, + bids_info = bids_info, + db = db, + dataset_tag_dict = dataset_tag_dict, + dataset_type = type ) + case (BidsMriDataTypeReader(), _): + import_bids_mri_data_type(env, import_env, session, data_type_reader) case _: print(f"Data type {bids_info.data_type} is not supported. Skipping.") diff --git a/python/tests/integration/scripts/test_import_bids_dataset.py b/python/tests/integration/scripts/test_import_bids_dataset.py index 7400a7ffb..0d49bba2d 100644 --- a/python/tests/integration/scripts/test_import_bids_dataset.py +++ b/python/tests/integration/scripts/test_import_bids_dataset.py @@ -116,3 +116,53 @@ def test_import_eeg_bids_dataset(): } } }) + + +def test_import_mri_bids_dataset(): + db = get_integration_database_session() + + process = run_integration_script([ + 'bids_import.py', + '--createcandidate', '--createsession', + '--directory', '/data/loris/incoming/DCC090_587630_V2', + ]) + + # Check the return code. + assert process.returncode == 0 + + # Check that the candidate and sessions are present in the database. + candidate = try_get_candidate_with_psc_id(db, 'DCC090') + assert candidate is not None + session = try_get_session_with_cand_id_visit_label(db, candidate.cand_id, 'V2') + assert session is not None + + # Check that the files has been inserted in the database. + assert len(session.files) == 5 + + # Check that the BIDS files have been copied. + assert_files_exist('/data/loris/bids_imports', { + 'DCC090_587630_V2_BIDSVersion_1.8.0': { + 'dataset_description.json': None, + 'participants.tsv': None, + 'sub-DCC090': { + 'ses-V2': { + 'anat': { + 'sub-DCC090_ses-V2_acq-spc3p2_run-4_T2w.json': None, + 'sub-DCC090_ses-V2_acq-spc3p2_run-4_T2w.nii.gz': None, + 'sub-DCC090_ses-V2_acq-tfl3p2_run-3_T1w.json': None, + 'sub-DCC090_ses-V2_acq-tfl3p2_run-3_T1w.nii.gz': None, + 'sub-DCC090_ses-V2_acq-tse2p2_run-2_echo-1_T2w.json': None, + 'sub-DCC090_ses-V2_acq-tse2p2_run-2_echo-1_T2w.nii.gz': None, + 'sub-DCC090_ses-V2_acq-tse2p2_run-2_echo-2_T2w.json': None, + 'sub-DCC090_ses-V2_acq-tse2p2_run-2_echo-2_T2w.nii.gz': None, + }, + 'dwi': { + 'sub-DCC090_ses-V2_acq-epb0_dir-AP_run-5_dwi.bval': None, + 'sub-DCC090_ses-V2_acq-epb0_dir-AP_run-5_dwi.bvec': None, + 'sub-DCC090_ses-V2_acq-epb0_dir-AP_run-5_dwi.json': None, + 'sub-DCC090_ses-V2_acq-epb0_dir-AP_run-5_dwi.nii.gz': None, + }, + } + } + } + })