Aivo import my dev version

aivo/aivo_get_info.m

@@ -91,6 +91,9 @@
     warning('Could not find a value for every requested subject. Please make sure the image_ids are not misspelled');
 end
 
+if(numeric)
+    value = cell2mat(value);
+end
 value = value(sort_idx);
 
 end

aivo/aivo_import.m

@@ -1,49 +1,140 @@
-function aivo_import(spreadsheet)
+function T = aivo_import(spreadsheet)
 
-T = readtable(spreadsheet);
-N = size(T,1);
+T = readtable(spreadsheet,'DateTimeType','text');
 
-%% Extract birthdate and gender from the patient_id
+%% Exclude columns that consists of NaNs
 
-birthday = cell(N,1);
-gender = cell(N,1);
+column_names = T.Properties.VariableNames;
 
-for i = 1:N
-    pid = T.patient_id{i};
-    birthday{i} = aivo_extract_birthday(pid);
-    gender{i} = aivo_extract_gender(pid);
+for i = 1:size(T,2)
+    field = column_names{i};
+    val = T.(field);
+    if(~iscell(val) && ~isdatetime(val) && all(isnan(val)))
+        T.(field) = [];
+    end
 end
 
-T.gender = gender;
-T.birthday = birthday;
-T.age = (datenum(T.study_date) - datenum(datetime(T.birthday,'format','yyyy-MM-dd')))/365;
-T.study_date = char(T.study_date);
-T.pet = ones(N,1);
+column_names = T.Properties.VariableNames;
 
-%%
+num_studies = size(T,1);
+num_columns = size(T,2);
 
-fields = T.Properties.VariableNames;
-pet_fields = {'image_id' 'ac' 'study_code' 'study_date' 'project' 'group_name' 'description' 'scanner' 'tracer' 'frames' 'start_time' 'mri' 'plasma' 'dynamic' 'dc' 'weight' 'height' 'dose' 'notes' 'type' 'source' 'injection_time' 'gender' 'age'};
-patient_fields = {'patient_id' 'study_date' 'gender' 'birthday' 'age' 'pet'};
-pet_idx = ismember(fields,pet_fields);
-patient_idx = ismember(fields,patient_fields);
+%% First modify the table so that it better satisfies the requirements
 
-T_pet = T(:,pet_idx);
-T_patient = T(:,patient_idx);
+for i = 1:num_columns
+    current_column = column_names{i};
+    column_values = T.(current_column);
+    if(ismember(current_column,{'dc' 'weight' 'height' 'dose' 'start_time' 'glucose' 'hct'}))
+        if(iscell(column_values))
+            column_values = str2double(column_values);
+        end
+    end
+    for j = 1:num_studies
+        switch current_column
+            case {'patient_id' 'ac_number' 'study_date' 'project' 'group_name' 'description' 'scanner' 'tracer' 'mri_code' 'notes' 'injection_time'}
+                    val = column_values{j};
+                    if(~isempty(val))
+                        updated_val = aivo_import_check(val,current_column,j);
+                        column_values{j} = updated_val;
+                    end
+            case {'dc' 'weight' 'height' 'dose' 'start_time' 'glucose' 'hct'}
+                val = column_values(j);
+                if(~isempty(val))
+                    updated_val = aivo_import_check(val,current_column,j);
+                    column_values(j) = updated_val;
+                end
+        end
+    end
+    T.(current_column) = column_values;
+end
 
-conn = aivo_connect();
+%% Insert new studies to the study_table table
+% Inserting new rows to study_table will automatically trigger insertion of
+% rows also to patient, lab, study and magia tables
+
+if(ismember('mri_code',column_names))
+    ac = [T.ac_number;T.mri_code];
+    image_id = ac;
+    pet = [ones(num_studies,1);zeros(num_studies,1)];
+    mri = [zeros(num_studies,1);ones(num_studies,1)];
+else
+    ac = T.ac_number;
+    image_id = ac;
+    pet = ones(num_studies,1);
+    mri = zeros(num_studies,1);
+end
+t = table(image_id,ac,pet,mri);
+
+image_id = T.ac_number;
+empty_idx = cellfun(@isempty,image_id);
+project = T.project;
+group_name = T.group_name;
+description = T.description;
 
-insert(conn,'megabase.aivo.pet',pet_fields,T_pet);
-N = size(T,1);
 
-image_id_idx = strcmp(fields,'image_id');
+project_T = table(image_id,project,group_name,description);
+
+image_id = T.ac_number;
+
+N = size(T,1);
 
 for i = 1:N
-    id = table2array(T(i,image_id_idx));
-    whereclause = sprintf('WHERE patient.image_id = ''%s''',id{1});
-    update(conn,'megabase.aivo.patient',patient_fields,T_patient,whereclause);
+
+    disp(['Processing subject ' num2str(i) ' of ' num2str(N)])
+
+    conn = aivo_connect;
+    refresh(conn);
+    id = T.ac_number(i);
+    found = aivo_check_id(conn,'study_code',id);
+    if found
+        whereclause = sprintf('WHERE image_id = ''%s''',id{1});
+        update(conn,'megabase.aivo2.study_code',t.Properties.VariableNames,t(i,:),whereclause);
+    else
+        insert(conn,'megabase.aivo2.study_code',t.Properties.VariableNames,t(i,:));
+    end
+    found = aivo_check_id(conn,'project',id);
+    if found
+        whereclause = sprintf('WHERE image_id = ''%s''',id{1});
+        update(conn,'megabase.aivo2.project',{'image_id' 'project' 'group_name' 'description'},project_T(i,:),whereclause);
+    else
+        insert(conn,'megabase.aivo2.project',{'image_id' 'project' 'group_name' 'description'},project_T(i,:));
+    end
+
+close(conn); 
+end
+
+
+%% Update the contents of the other tables
+
+remaining_columns = {'patient_id' 'study_date' 'scanner' 'tracer' 'mri_code' 'injection_time' 'dc' 'weight' 'height' 'dose' 'scan_start_time'};% 'glucose' 'hct'};
+
+for i = 1:length(remaining_columns) 
+
+    field = remaining_columns{i};
+
+    disp(['Writing: ' field ', ' num2str(i) ' of ' num2str(length(remaining_columns)) ' columns'])
+
+    aivo_set_info(T.ac_number,field,T.(field));
 end
 
-close(conn);
+end
+
+function refresh(conn)
+
+refresh_cmd = 'REFRESH MATERIALIZED VIEW aivo2.materia';
+curs = exec(conn,refresh_cmd);
+close(curs);
 
-end
+end
+
+function found = aivo_check_id(conn,table,id)
+
+check_cmd = ['do $$ BEGIN IF (SELECT image_id FROM aivo2.' table ' WHERE image_id = ''' id{1} ''') IS NULL THEN RAISE Exception ''ERROR''; END IF; END $$'];
+curs = exec(conn,check_cmd);
+if ~strcmp(curs.Message(1:5),'ERROR')
+    found = 1;
+else
+    found = 0;
+end
+
+end

aivo/aivo_import_check.m

@@ -0,0 +1,61 @@
+function updated_val = aivo_import_check(val,column,row)
+
+switch column
+    case 'patient_id'
+        d = str2double(val(1:2));
+        mo = str2double(val(3:4));
+        if(d <= 0 || d > 31 || mo <= 0 || mo > 12 || length(val) ~= 11)
+            error('Invalid patient_id on row %.0f',row);
+        else
+            updated_val = val;
+        end
+    case 'ac_number'
+        if(length(val) ~= 7)
+            error('Invalid ac_number on row %.0f. The AC number is supposed to have 7 characters.',row);
+        else
+            updated_val = lower(val);
+        end
+    case 'study_date'
+        try
+            val = char(datetime(val,'format','yyyy-MM-dd'));
+            h = strsplit(val,'-');
+            y = str2double(h{1});
+            mo = str2double(h{2});
+            d = str2double(h{3});
+            if(y < 1989 || d > 31 || mo <= 0 || mo > 12)
+                error('Invalid study_date on row %.0f',row);
+            else
+                updated_val = lower(val);
+            end
+        catch
+
+
+        end
+    case 'tracer'
+        if(~strcmp(val(1),'['))
+            error('Invalid tracer name on row %.0f. Please use the following notation: [18f]fdg',row);
+        else
+            updated_val = lower(val);
+        end
+    case 'mri_code'
+        if(length(val) ~= 7)
+            error('Invalid mri_code on row %.0f. The AC number is supposed to have 7 characters.',row);
+        else
+            updated_val = lower(val);
+        end
+    case 'injection_time'
+        idx = regexp(val,' ');
+        val(idx) = [];
+        updated_val = val;
+    case 'group_name'
+        idx = regexp(val,' ');
+        val(idx) = '_';
+        updated_val = lower(val);
+    case {'project' 'description' 'scanner'}
+        idx = regexp(val,' ');
+        val(idx) = '-';
+        updated_val = lower(val);
+    otherwise
+        updated_val = val;
+
+end

aivo/aivo_read_magia_specs.m

@@ -1,71 +1,58 @@
 function specs = aivo_read_magia_specs(image_id)
 
-%%% specs.study
-% mandatory
-specs.study.tracer = char(aivo_get_info(image_id,'tracer'));
-if(strcmp(specs.study.tracer,'null'))
-    error('%s: The tracer has not been specified. Please use aivo_set_info to specify the tracer.',image_id);
-end
-specs.study.frames = parse_frames_string(char(aivo_get_info(image_id,'frames')));
-if(strcmp(specs.study.frames,'null'))
-    error('%s: The frames have not been specified. Please use aivo_set_info to specify the frames',image_id);
-end
-% optional
-specs.study.weight = aivo_get_info(image_id,'weight');
-specs.study.dose = aivo_get_info(image_id,'dose');
-specs.study.scanner = char(aivo_get_info(image_id,'scanner'));
-specs.study.mri_code = char(aivo_get_info(image_id,'mri_code'));
-specs.study.glucose = aivo_get_info(image_id,'glucose');
-
-
-%%% specs.magia
-% mandatory
-specs.magia.model = char(aivo_get_info(image_id,'model'));
-if(strcmp(specs.magia.model,'null'))
-    error('%s: The model has not been specified. Please use aivo_set_info to specify the model.',image_id);
-end
-specs.magia.input_type = char(aivo_get_info(image_id,'input_type'));
-if(strcmp(specs.magia.input_type,'null'))
-    error('%s: The field ''input_type'' has not been specified. Please use aivo_set_info to specify the ''input_type'' either as ''ref'', ''plasma'', ''plasma&blood'', or ''sca_ref''. ',image_id);
-end
-specs.magia.roi_type = char(aivo_get_info(image_id,'roi_type'));
-if(strcmp(specs.magia.roi_type,'null'))
-    error('%s: The field ''magia.roi_type'' has not been specified. Please use aivo_set_info to specify the field either as ''freesurfer'' or ''atlas''.',image_id);
-end
-specs.magia.dc = aivo_get_info(image_id,'dc');
-if(strcmp(specs.magia.dc,'null'))
-    error('%s: The field ''magia.dc'' has not been specified. Please use aivo_set_info to specify the field either as 1 (decay-corrected to injection time) or 0 (not decay-corrected to injection time).',image_id);
-end
-specs.magia.rc = aivo_get_info(image_id,'rc');
-
-specs.magia.fwhm_pre = aivo_get_info(image_id,'fwhm_pre');
-specs.magia.fwhm_post = aivo_get_info(image_id,'fwhm_post');
-specs.magia.fwhm_roi = aivo_get_info(image_id,'fwhm_roi');
-specs.magia.cpi = aivo_get_info(image_id,'cpi');
-if(strcmp(specs.magia.cpi,'null'))
-    error('%s: The field ''magia.cpi'' has not been specified. Please use aivo_set_info to specify the field either as 1 (calculate parametric images) or 0 (do not calculate parametric images).',image_id);
-end
-
-specs.magia.cut_time = aivo_get_info(image_id,'cut_time');
-specs.magia.gu = aivo_get_info(image_id,'gu');
-
-% optional
-
-if(specs.magia.cpi)
-    specs.magia.norm_method = char(aivo_get_info(image_id,'norm_method'));
-    if(strcmp(specs.magia.norm_method,'pet'))
-        specs.magia.template = char(aivo_get_info(image_id,'template'));
+if(iscell(image_id))
+    N = length(image_id);
+    specs = cell(N,1);
+    conn = aivo_connect();
+    for i = 1:N
+        specs{i} = aivo_read_magia_specs(image_id{i});
     end
+else
+    conn = aivo_connect();
+
+    q = sprintf('SELECT * FROM megabase.aivo2.magia WHERE image_id = ''%s''',image_id);
+    curs = exec(conn,q);
+    curs = fetch(curs);
+    value = curs.Data;
+    close(curs);
+
+    magia_cols = columns(conn,'megabase','aivo2','magia');
+    [magia_cols,idx] = setdiff(magia_cols,'image_id');
+    value = value(idx);
+
+    n_cols = length(magia_cols);
+
+    for i = 1:n_cols
+        col = magia_cols{i};
+        specs.magia.(col) = value{i};
+    end
+
+    q = sprintf('SELECT tracer,frames,dose,scanner,mri_code FROM megabase.aivo2.study WHERE image_id = ''%s''',image_id);
+    curs = exec(conn,q);
+    curs = fetch(curs);
+    value = curs.Data;
+    close(curs);
+
+    study_cols = {'tracer' 'frames' 'dose' 'scanner' 'mri_code'};
+    [study_cols,idx] = setdiff(study_cols,'image_id');
+    value = value(idx);
+
+    n_cols = length(study_cols);
+
+    for i = 1:n_cols
+        col = study_cols{i};
+        if(strcmp(col,'frames'))
+            specs.study.(col) = parse_frames_string(value{i});
+        else
+            specs.study.(col) = value{i};
+        end
+    end
+
+    specs.study.glucose = aivo_get_info(image_id,'glucose');
+    specs.study.weight = aivo_get_info(image_id,'weight');
+
 end
 
-if(strcmp(specs.magia.roi_type,'freesurfer'))
-    specs.magia.roi_set = char(aivo_get_info(image_id,'roi_set'));
-elseif(strcmp(specs.magia.roi_type,'atlas'))
-    specs.magia.mni_roi_mask_dir = char(aivo_get_info(image_id,'mni_roi_mask_dir'));
-end
-
-if(strcmp(specs.magia.input_type,'sca_ref'))
-    specs.magia.classfile = char(aivo_get_info(image_id,'classfile'));
-end
+close(conn);
 
 end