From 610052f8701e547d41e3935eacb5268795acbac7 Mon Sep 17 00:00:00 2001 From: adambasha0 Date: Tue, 15 Oct 2024 07:42:13 +0000 Subject: [PATCH 1/6] enhance import samples feature and spec tests and additional samples attributes (flash point and refractive index) --- app/api/helpers/report_helpers.rb | 21 ++- .../components/contextActions/ModalExport.js | 6 +- lib/export/export_excel.rb | 6 + lib/export/export_sdf.rb | 41 +++-- lib/export/export_table.rb | 18 ++- lib/import/import_samples.rb | 146 ++++++++++++++++-- lib/import/import_sdf.rb | 15 +- spec/api/chemotion/sample_api_spec.rb | 6 +- spec/lib/export/export_excel_spec.rb | 30 +++- spec/lib/import/import_samples_spec.rb | 101 ++++++++++++ spec/lib/import/import_sdf_spec.rb | 73 ++++++++- 11 files changed, 411 insertions(+), 52 deletions(-) diff --git a/app/api/helpers/report_helpers.rb b/app/api/helpers/report_helpers.rb index 1e24dc2a2a..0c275f1d5d 100644 --- a/app/api/helpers/report_helpers.rb +++ b/app/api/helpers/report_helpers.rb @@ -556,7 +556,10 @@ def build_sql_reaction_sample(columns, c_id, ids, checkedAll = false) # deleted_at: ['wp.deleted_at', nil, 10], molecule_name: ['mn."name"', '"molecule name"', 1], molarity_value: ['s."molarity_value"', '"molarity_value"', 0], + molarity_unit: ['s."molarity_unit"', '"molarity_unit"', 0], dry_solvent: ['s."dry_solvent"', '"dry_solvent"', 0], + flash_point: ['s."flash_point"', '"flash point"', 0], + refractive_index: ['s."refractive_index"', '"refractive index"', 0], }, sample_id: { external_label: ['s.external_label', '"sample external label"', 0], @@ -634,14 +637,18 @@ def build_sql_reaction_sample(columns, c_id, ids, checkedAll = false) }.freeze def custom_column_query(table, col, selection, user_id, attrs) - if col == 'user_labels' - selection << "labels_by_user_sample(#{user_id}, s_id) as user_labels" - elsif col == 'literature' - selection << "literatures_by_element('Sample', s_id) as literatures" - elsif col == 'cas' - selection << "s.xref->>'cas' as cas" + column_map = { + 'user_labels' => "labels_by_user_sample(#{user_id}, s_id) as user_labels", + 'literature' => "literatures_by_element('Sample', s_id) as literatures", + 'cas' => "s.xref->>'cas' as cas", + 'refractive_index' => "s.xref->>'refractive_index' as refractive_index", + 'flash_point' => "s.xref->>'flash_point' as flash_point", + } + + if column_map[col] + selection << column_map[col] elsif (s = attrs[table][col.to_sym]) - selection << ("#{s[1] && s[0]} as #{s[1] || s[0]}") + selection << "#{s[1] && s[0]} as #{s[1] || s[0]}" end end diff --git a/app/packs/src/components/contextActions/ModalExport.js b/app/packs/src/components/contextActions/ModalExport.js index 8f48c83f7a..f5b98210fd 100644 --- a/app/packs/src/components/contextActions/ModalExport.js +++ b/app/packs/src/components/contextActions/ModalExport.js @@ -53,16 +53,18 @@ export default class ModalExport extends React.Component { { value: ['molarity_value', 'molarity_unit'], text: 'molarity', checked: false }, { value: 'density', text: 'density', checked: false }, { value: 'molfile', text: 'molfile', checked: false }, - // {value: "purity", text: "purity", checked: false}, + { value: 'purity', text: 'purity', checked: false }, { value: "solvent", text: "solvent", checked: false }, { value: "location", text: "location", checked: false }, { value: "is_top_secret", text: "is top secret?", checked: false }, - { value: "dry_solvent", text: "dry solvent", checked: false }, + { value: 'anhydrous', text: 'dry solvent', checked: false }, // {value: "ancestry", text: "ancestry", checked: false}, { value: 'imported_readout', text: 'imported readout', checked: false }, // {value: "identifier", text: "identifier", checked: false}, { value: 'melting_point', text: 'melting point', checked: false }, { value: 'boiling_point', text: 'boiling point', checked: false }, + { value: 'refractive_index', text: 'refractive index', checked: false }, + { value: 'flash_point', text: 'flash point', checked: false }, { value: 'created_at', text: 'created at', checked: true }, { value: 'updated_at', text: 'updated at', checked: false }, { value: 'user_labels', text: 'user labels', checked: false }, diff --git a/lib/export/export_excel.rb b/lib/export/export_excel.rb index 49f23536b4..92f0fb7b67 100644 --- a/lib/export/export_excel.rb +++ b/lib/export/export_excel.rb @@ -127,6 +127,12 @@ def filter_with_permission_and_detail_level(sample) string.split(',').join(' - ') elsif column == 'solvent' extract_label_from_solvent_column(sample[column]) || '' + elsif column == 'refractive index' + sample['refractive_index'] + elsif column == 'flash point' + flash_point_format(sample['flash_point']) + elsif column == 'molarity' + "#{sample['molarity_value']} #{sample['molarity_unit']}" else sample[column] end diff --git a/lib/export/export_sdf.rb b/lib/export/export_sdf.rb index dd759c8307..3f14c126a7 100644 --- a/lib/export/export_sdf.rb +++ b/lib/export/export_sdf.rb @@ -34,6 +34,26 @@ def read private + def concatenate_data(sample, data, headers = @headers) + headers.each do |column| + next unless column + + raw_value = case column + when 'molarity' + "#{sample['molarity_value']} #{sample['molarity_unit']}" + when 'flash point' + sample['flash_point'] + when 'refractive index' + sample['refractive_index'] + else + sample[column] + end + column_data = format_field(column, raw_value) + data.concat(column_data) + end + data + end + def filter_with_permission_and_detail_level(sample) if sample['shared_sync'] == 'f' || sample['shared_sync'] == false data = validate_molfile(sample['molfile']) @@ -44,10 +64,7 @@ def filter_with_permission_and_detail_level(sample) end data = data.rstrip data += "\n" - @headers.each do |column| - column_data = format_field(column, sample[column]) - data.concat(column_data) - end + data = concatenate_data(sample, data) else # return no data if molfile not allowed return nil if sample['dl_s'].zero? @@ -62,12 +79,8 @@ def filter_with_permission_and_detail_level(sample) # NB: as of now , only dl 0 and 10 are implemented dl = 10 if dl.positive? headers = instance_variable_get("headers#{sample['dl_s']}#{dl}") - headers.each do |column| - next unless column + data = concatenate_data(sample, data, headers) - column_data = format_field(column, sample[column]) - data.concat(column_data) - end end data.concat("\$\$\$\$\n") end @@ -78,14 +91,24 @@ def extract_reference_values(raw_value) string.split(',').join(' - ') end + def flash_point_format(value) + return if value.blank? + + flash_point = JSON.parse(value) + "#{flash_point['value']} #{flash_point['unit']}" + end + def format_field(column, raw_value) field = column.gsub(/\s+/, '_').upcase reference_values = ['melting pt', 'boiling pt'] + flash_point = ['flash point', 'flash_point'] sample_column = if reference_values.include?(column) extract_reference_values(raw_value) elsif column == 'solvent' extract_label_from_solvent_column(raw_value) || '' + elsif flash_point.include?(column) + flash_point_format(raw_value) else raw_value end diff --git a/lib/export/export_table.rb b/lib/export/export_table.rb index ef37a7827a..859308b66b 100644 --- a/lib/export/export_table.rb +++ b/lib/export/export_table.rb @@ -110,11 +110,25 @@ def generate_headers_reaction } end + def flash_point_format(value) + return if value.blank? + + flash_point = JSON.parse(value) + "#{flash_point['value']} #{flash_point['unit']}" + end + def format_headers(headers) headers.map! do |header| - header.tr('_', ' ') + header = header.tr('_', ' ') + if header.scan('molarity value').first == 'molarity value' + 'molarity' + elsif header == 'molarity unit' + nil + else + header + end end - headers + headers.compact end def generate_headers_sample diff --git a/lib/import/import_samples.rb b/lib/import/import_samples.rb index 8c2c6e6b21..7f43fb6845 100644 --- a/lib/import/import_samples.rb +++ b/lib/import/import_samples.rb @@ -8,6 +8,10 @@ class ImportSamples attr_reader :xlsx, :sheet, :header, :mandatory_check, :rows, :unprocessable, :processed, :file_path, :collection_id, :current_user_id, :file_name + MOLARITY_UNIT = %r{m/L|mol/L|M}i.freeze + DENSITY_UNIT = %r{g/mL}i.freeze + FLASH_POINT_UNIT = /°C|F|K/i.freeze + def initialize(file_path, collection_id, user_id, file_name, import_type) @rows = [] @unprocessable = [] @@ -209,6 +213,17 @@ def format_to_interval_syntax(row_field) "[#{lower_bound}, #{upper_bound}]" end + def format_molarity_value(value, type) + return if value.empty? + + if type == 'value' + value.to_f + else + molarity_unit_exists = value.match?(MOLARITY_UNIT) + molarity_unit_exists ? 'M' : nil + end + end + def assign_molecule_name_id(sample, value) split_names = value.split(';') molecule_name_id = MoleculeName.find_by(name: split_names[0]).id @@ -217,32 +232,134 @@ def assign_molecule_name_id(sample, value) def handle_sample_fields(sample, db_column, value) case db_column - when 'cas' - sample['xref']['cas'] = value + when 'cas', 'refractive_index' + handle_xref_fields(sample, db_column, value) when 'mn.name' assign_molecule_name_id(sample, value) + when 'flash_point' + handle_flash_point(sample, value) + when 'density' + handle_density(sample, value) + when 'molarity' + handle_molarity(sample, value) else - sample[db_column] = value || '' + handle_default_fields(sample, db_column, value) end end - def process_fields(sample, db_column, field, row, molecule) + def handle_xref_fields(sample, db_column, value) + return sample if value.nil? + + sample['xref'][db_column] ||= {} + sample['xref'][db_column] = value + end + + def handle_flash_point(sample, value) + return sample if value[:unit].nil? || value[:value].nil? + + sample['xref']['flash_point'] ||= {} + sample['xref']['flash_point']['value'] = value[:value] + sample['xref']['flash_point']['unit'] = value[:unit] + end + + def handle_density(sample, value) + return sample if value[:unit].nil? || value[:value].nil? + + sample['density'] = value[:value] if value[:unit] == 'g/mL' + sample + end + + def handle_molarity(sample, value) + return sample if value[:unit].nil? || value[:value].nil? + + sample['molarity_value'] = value[:value] + sample['molarity_unit'] = value[:unit] + end + + def handle_default_fields(sample, db_column, value) + sample[db_column] = value || '' + end + + # rubocop:disable Style/StringLiterals + def process_fields(sample, map_column, field, row, molecule) + array = ["\"cas\"", "\"purity\"", "\"density\""] + conditions = map_column.nil? || array.include?(map_column[1]) + db_column = conditions ? field : map_column[0].sub('s.', '').delete!('"') molecule.create_molecule_name_by_user(row[field], current_user_id) if field == 'molecule name' process_sample_fields(sample, db_column, field, row) end + # rubocop:enable Style/StringLiterals + + def process_value(value, db_column) + fields_with_units = %w[density molarity flash_point].freeze + comparison_values = %w[melting_point boiling_point].freeze + if comparison_values.include?(db_column) + format_to_interval_syntax(value) + elsif fields_with_units.include?(db_column) + to_value_unit_format(value, db_column) + else + value + end + end + + def clean_value(value) + string = value&.gsub(/\s+/, ' ') + string&.strip + end + + def extract_numerical_value(value) + cleaned_value = clean_value(value) + numerical_match = cleaned_value.scan(/\b\d+(?:\.\d+)?\b/).first if cleaned_value + numerical_match&.to_f + end + + def unit_regex_pattern(db_column) + units = { + 'density' => DENSITY_UNIT, + 'molarity' => MOLARITY_UNIT, + 'flash_point' => FLASH_POINT_UNIT, + } + + if db_column == 'flash_point' + # Create a regex pattern for matching the unit as a standalone word + /(? properties['STEREO_ABS'], 'rel' => properties['STEREO_REL']) sample.target_amount_value = properties['TARGET_AMOUNT'] unless properties['TARGET_AMOUNT'].blank? diff --git a/spec/api/chemotion/sample_api_spec.rb b/spec/api/chemotion/sample_api_spec.rb index 94007584e3..0ec0376a62 100644 --- a/spec/api/chemotion/sample_api_spec.rb +++ b/spec/api/chemotion/sample_api_spec.rb @@ -209,7 +209,7 @@ short_label: 'C9H12ClNO2', target_amount: '10 g /  g', real_amount: '15mg/mg', - density: '30', + density: '30 g/mL', decoupled: 'f', molarity: '900', melting_point: '900.0', @@ -249,7 +249,7 @@ expect(sample['location']).to eq 'location' expect(sample['external_label']).to eq 'external_label' expect(sample['name']).to eq 'name' - expect(sample['molarity_value']).to eq 900 + expect(sample['molarity_value']).to eq 0.0 expect(sample['boiling_point']).to eq 900.0..1500.0 expect(sample['melting_point']).to eq 900.0...Float::INFINITY @@ -319,7 +319,7 @@ expect(sample['location']).to eq 'location' expect(sample['external_label']).to eq 'external_label' expect(sample['name']).to eq 'name' - expect(sample['molarity_value']).to eq 900 + expect(sample['molarity_value']).to eq 0.0 expect(sample['boiling_point']).to eq 1000.0...Float::INFINITY expect(sample['melting_point']).to eq 900.0...Float::INFINITY diff --git a/spec/lib/export/export_excel_spec.rb b/spec/lib/export/export_excel_spec.rb index 2a465a46ab..e4e8937789 100644 --- a/spec/lib/export/export_excel_spec.rb +++ b/spec/lib/export/export_excel_spec.rb @@ -12,34 +12,48 @@ describe '.filter_with_permission_and_detail_level' do let(:formated_value) { exporter.filter_with_permission_and_detail_level(sample_json) } + let(:flash_point) { '{"value": 25, "unit": "°C"}' } + let(:melting_point) { '[13.0]' } + let(:refractive_index) { '1' } + let(:molarity) { '2.45 M' } before do sample_json['shared_sync'] = 'f' sample_json['melting pt'] = melting_point - exporter.instance_eval('@headers =["melting pt"]', __FILE__, __LINE__) + sample_json['flash_point'] = flash_point + sample_json['refractive_index'] = refractive_index + sample_json['molarity_value'] = '2.45' + sample_json['molarity_unit'] = 'M' + exporter.instance_eval('@headers =["melting pt", "flash point", "refractive index", "molarity"]', __FILE__, __LINE__) end context 'with integer melting point' do - let(:melting_point) { '[13.0,]' } - it 'db range string formatted correctly' do - expect(formated_value).to eq ['13.0'] + expect(formated_value).to eq ['13.0', '25 °C', '1', '2.45 M'] end end - context 'with float melting point' do + context 'with float sample attributes' do let(:melting_point) { '[13.123,)' } it 'db range string formatted correctly' do - expect(formated_value).to eq ['13.123'] + expect(formated_value).to eq ['13.123', '25 °C', '1', '2.45 M'] end end - context 'with integer/float range melting point' do + context 'with float melting point in a range and other sample attributes' do let(:melting_point) { '[1.0,100.01]' } it 'db range string formatted correctly' do - expect(formated_value).to eq ['1.0 - 100.01'] + expect(formated_value).to eq ['1.0 - 100.01', '25 °C', '1', '2.45 M'] + end + end + + context 'with flash point in Fahrenheit' do + let(:flash_point) { '{"value": 25, "unit": "°F"}' } + + it 'db range string formatted correctly' do + expect(formated_value).to eq ['13.0', '25 °F', '1', '2.45 M'] end end end diff --git a/spec/lib/import/import_samples_spec.rb b/spec/lib/import/import_samples_spec.rb index 08125c4862..1b38471868 100644 --- a/spec/lib/import/import_samples_spec.rb +++ b/spec/lib/import/import_samples_spec.rb @@ -8,6 +8,7 @@ let(:file_path) { 'spec/fixtures/import/sample_import_template.xlsx' } let(:file_name) { File.basename(file_path) } let(:importer) { Import::ImportSamples.new(file_path, collection_id, user_id, file_name, 'sample') } + let(:sample) { create(:sample) } before do stub_rest_request('BYHVGQHIAFURIL-UHFFFAOYSA-N') @@ -102,4 +103,104 @@ def stub_rest_request(identifier) ) .to_return(status: 200, body: '', headers: {}) end + + describe '#format_molarity_value' do + it 'returns nil if value is empty' do + expect(importer.format_molarity_value('', 'value')).to be_nil + end + + it 'returns the value as a float if type is value' do + expect(importer.format_molarity_value('1.5', 'value')).to eq(1.5) + end + + it 'returns M if the value contains m/L or M' do + expect(importer.format_molarity_value('0.5 m/L', 'unit')).to eq('M') + expect(importer.format_molarity_value('0.5 M', 'unit')).to eq('M') + end + + it 'returns nil if the value does not contain a valid molarity unit' do + expect(importer.format_molarity_value('0.5', 'unit')).to be_nil + end + end + + describe '#to_value_unit_format' do + context 'when db_column is density' do + it 'extracts the numerical value and unit when value contains valid density unit' do + result = importer.to_value_unit_format('1.25 g/ml', 'density') + expect(result).to eq({ value: 1.25, unit: 'g/ml' }) + end + + it 'returns nil for both value and unit when value does not contain a valid unit' do + result = importer.to_value_unit_format('1.25 kg/L', 'density') + expect(result).to eq({ value: nil, unit: nil }) + end + end + + context 'when db_column is molarity' do + it 'extracts the numerical value and unit when value contains valid molarity unit' do + result = importer.to_value_unit_format('0.5 M', 'molarity') + expect(result).to eq({ value: 0.5, unit: 'M' }) + end + + it 'returns expected value and unit if molarity unit is valid' do + result = importer.to_value_unit_format('0.7 mol/L', 'molarity') + expect(result).to eq({ value: 0.7, unit: 'M' }) + end + end + + context 'when db_column is flash_point' do + it 'extracts the numerical value and unit when value contains valid flash point unit' do + result = importer.to_value_unit_format('100 °C', 'flash_point') + expect(result).to eq({ value: 100, unit: '°C' }) + end + + it 'returns nil for both value and unit when value does not contain a valid flash point unit' do + result = importer.to_value_unit_format('100 Koma', 'flash_point') + expect(result).to eq({ value: nil, unit: nil }) + end + end + + context 'when value is nil or empty' do + it 'returns nil for both value and unit when value is nil' do + result = importer.to_value_unit_format(nil, 'density') + expect(result).to eq({ value: nil, unit: nil }) + end + + it 'returns nil for both value and unit when value is an empty string' do + result = importer.to_value_unit_format('', 'density') + expect(result).to eq({ value: nil, unit: nil }) + end + end + + context 'when value does not contain a numerical value' do + it 'returns nil for both value and unit when no numerical value is found' do + result = importer.to_value_unit_format('abc g/ml', 'density') + expect(result).to eq({ value: nil, unit: nil }) + end + end + end + + describe '#handle_sample_fields' do + it 'update sample flash point attribute with expected values' do + importer.handle_sample_fields(sample, 'flash_point', { value: 15, unit: '°C' }) + expect(sample['xref']['flash_point']['value']).to eq(15) + expect(sample['xref']['flash_point']['unit']).to eq('°C') + end + + it 'update sample molarity attribute with expected values' do + importer.handle_sample_fields(sample, 'molarity', { value: 1.24, unit: 'M' }) + expect(sample['molarity_value']).to eq(1.24) + expect(sample['molarity_unit']).to eq('M') + end + + it 'update sample default attribute with expected values' do + importer.handle_sample_fields(sample, 'refractive_index', 0.85) + expect(sample['xref']['refractive_index']).to eq(0.85) + end + + it 'update sample density attribute with expected values' do + importer.handle_sample_fields(sample, 'density', { value: 2.24, unit: 'g/mL' }) + expect(sample['density']).to eq(2.24) + end + end end diff --git a/spec/lib/import/import_sdf_spec.rb b/spec/lib/import/import_sdf_spec.rb index d7ca113f73..c27c1b8a87 100644 --- a/spec/lib/import/import_sdf_spec.rb +++ b/spec/lib/import/import_sdf_spec.rb @@ -2,7 +2,74 @@ require 'rails_helper' -RSpec.describe 'ImportSdf' do - let(:two_compound_sdf) { File.read(Rails.root.join('spec/fixtures/two_compound_sdf')) } - let(:sdf_import) { Import::ImportSdf.new(data: :two_compound_sdf) } +RSpec.describe Import::ImportSdf do + let(:mock_user) { create(:user) } + let(:mock_collection) { create(:collection) } + let(:molecule) { create(:molecule, inchikey: 'DTHMTBUWTGVEFG-DDWIOCJRSA-N', is_partial: false) } + let(:sdf_import) do + described_class.new( + collection_id: mock_collection.id, + current_user_id: mock_user.id, + mapped_keys: { + description: %w[ + MOLECULE_NAME + SAFETY_R_S + SMILES_STEREO + ], + short_label: 'EMP_FORMULA_SHORT', + target_amount: 'AMOUNT', + real_amount: 'REAL_AMOUNT', + decoupled: 'MOLECULE-LESS', + molarity: 'MOLARITY', + melting_point: 'melting_point', + boiling_point: 'boiling_point', + location: 'location', + external_label: 'external_label', + name: 'name', + }, + rows: [{ + 'inchikey' => 'DTHMTBUWTGVEFG-DDWIOCJRSA-N', + 'molfile' => Rails.root.join('spec/fixtures/mf_with_data_01.sdf').read, + 'description' => "MOLECULE_NAME\n(R)-Methyl-2-amino-2-phenylacetate hydrochloride ?96%; (R)-(?)-2-Phenylglycine methyl ester hydrochloride\n\nSAFETY_R_S\nH: 319; P: 305+351+338\n\nSMILES_STEREO\n[Cl-].COC(=O)[C@H](N)c1ccccc1.[H+]\n", + 'short_label' => 'C9H12ClNO2', + 'target_amount' => '10 g /  g', + 'real_amount' => '15mg/mg', + 'decoupled' => 'f', + 'molarity' => '700 M', + 'melting_point' => '50.0', + 'boiling_point' => '150.0-160.0', + 'location' => 'location', + 'external_label' => 'external_label', + 'name' => 'name', + 'flash_point' => { 'value' => '94.0', 'unit' => '°C' }.to_json, + 'refractive_index' => '1.0', + }], + ) + end + + before do + allow(Chemotion::OpenBabelService).to receive(:molecule_info_from_molfile).and_return( + { inchikey: 'DTHMTBUWTGVEFG-DDWIOCJRSA-N', + is_partial: false, + molfile_version: 'V2000', + } + ) + allow(Molecule).to receive(:find_by).with( + inchikey: 'DTHMTBUWTGVEFG-DDWIOCJRSA-N', is_partial: false, + ).and_return(molecule) + end + + describe '#initialize' do + it 'initializes with correct attributes' do + expect(sdf_import.collection_id).to eq(mock_collection.id) + expect(sdf_import.current_user_id).to eq(mock_user.id) + end + end + + describe '#create_samples' do + it 'creates samples from valid raw_data' do + expect { sdf_import.create_samples }.to change(Sample, :count).by(1) + expect(sdf_import.message.scan('Import successful!').size).to eq(1) + end + end end From a59becf149f189f8d0d8f4ba90de15126f6d78ad Mon Sep 17 00:00:00 2001 From: adambasha0 Date: Wed, 16 Oct 2024 11:13:21 +0000 Subject: [PATCH 2/6] refactor: fix for rubocop warning messages --- spec/lib/export/export_excel_spec.rb | 3 ++- spec/lib/import/import_sdf_spec.rb | 9 +-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/spec/lib/export/export_excel_spec.rb b/spec/lib/export/export_excel_spec.rb index e4e8937789..6e7e65f932 100644 --- a/spec/lib/export/export_excel_spec.rb +++ b/spec/lib/export/export_excel_spec.rb @@ -16,6 +16,7 @@ let(:melting_point) { '[13.0]' } let(:refractive_index) { '1' } let(:molarity) { '2.45 M' } + let(:headers) { '@headers =["melting pt", "flash point", "refractive index", "molarity"]' } before do sample_json['shared_sync'] = 'f' @@ -24,7 +25,7 @@ sample_json['refractive_index'] = refractive_index sample_json['molarity_value'] = '2.45' sample_json['molarity_unit'] = 'M' - exporter.instance_eval('@headers =["melting pt", "flash point", "refractive index", "molarity"]', __FILE__, __LINE__) + exporter.instance_eval(headers, __FILE__, __LINE__) end context 'with integer melting point' do diff --git a/spec/lib/import/import_sdf_spec.rb b/spec/lib/import/import_sdf_spec.rb index c27c1b8a87..e3167c7eb7 100644 --- a/spec/lib/import/import_sdf_spec.rb +++ b/spec/lib/import/import_sdf_spec.rb @@ -11,11 +11,6 @@ collection_id: mock_collection.id, current_user_id: mock_user.id, mapped_keys: { - description: %w[ - MOLECULE_NAME - SAFETY_R_S - SMILES_STEREO - ], short_label: 'EMP_FORMULA_SHORT', target_amount: 'AMOUNT', real_amount: 'REAL_AMOUNT', @@ -30,7 +25,6 @@ rows: [{ 'inchikey' => 'DTHMTBUWTGVEFG-DDWIOCJRSA-N', 'molfile' => Rails.root.join('spec/fixtures/mf_with_data_01.sdf').read, - 'description' => "MOLECULE_NAME\n(R)-Methyl-2-amino-2-phenylacetate hydrochloride ?96%; (R)-(?)-2-Phenylglycine methyl ester hydrochloride\n\nSAFETY_R_S\nH: 319; P: 305+351+338\n\nSMILES_STEREO\n[Cl-].COC(=O)[C@H](N)c1ccccc1.[H+]\n", 'short_label' => 'C9H12ClNO2', 'target_amount' => '10 g /  g', 'real_amount' => '15mg/mg', @@ -51,8 +45,7 @@ allow(Chemotion::OpenBabelService).to receive(:molecule_info_from_molfile).and_return( { inchikey: 'DTHMTBUWTGVEFG-DDWIOCJRSA-N', is_partial: false, - molfile_version: 'V2000', - } + molfile_version: 'V2000' }, ) allow(Molecule).to receive(:find_by).with( inchikey: 'DTHMTBUWTGVEFG-DDWIOCJRSA-N', is_partial: false, From dc380ee8f2759619c6b086c77865b986569b3d51 Mon Sep 17 00:00:00 2001 From: adambasha0 Date: Tue, 29 Oct 2024 22:48:22 +0000 Subject: [PATCH 3/6] merge: resolve conflict --- .../components/contextActions/ModalExport.js | 9 +-- lib/export/export_chemicals.rb | 21 +++---- lib/import/import_chemicals.rb | 59 +++++++++++++------ spec/lib/export/export_chemicals_spec.rb | 12 ++-- spec/lib/import/import_chemicals_spec.rb | 10 +++- 5 files changed, 71 insertions(+), 40 deletions(-) diff --git a/app/packs/src/components/contextActions/ModalExport.js b/app/packs/src/components/contextActions/ModalExport.js index f5b98210fd..3cf02a02f8 100644 --- a/app/packs/src/components/contextActions/ModalExport.js +++ b/app/packs/src/components/contextActions/ModalExport.js @@ -54,9 +54,9 @@ export default class ModalExport extends React.Component { { value: 'density', text: 'density', checked: false }, { value: 'molfile', text: 'molfile', checked: false }, { value: 'purity', text: 'purity', checked: false }, - { value: "solvent", text: "solvent", checked: false }, - { value: "location", text: "location", checked: false }, - { value: "is_top_secret", text: "is top secret?", checked: false }, + { value: 'solvent', text: 'solvent', checked: false }, + { value: 'location', text: 'location', checked: false }, + { value: 'is_top_secret', text: 'is top secret?', checked: false }, { value: 'anhydrous', text: 'dry solvent', checked: false }, // {value: "ancestry", text: "ancestry", checked: false}, { value: 'imported_readout', text: 'imported readout', checked: false }, @@ -106,6 +106,7 @@ export default class ModalExport extends React.Component { { value: 'vendor', text: 'vendor', checked: false }, { value: 'order_number', text: 'order number', checked: false }, { value: 'amount', text: 'amount', checked: false }, + { value: 'volume', text: 'volume', checked: false }, { value: 'price', text: 'price', checked: false }, { value: 'person', text: 'person', checked: false }, { value: 'required_date', text: 'required date', checked: false }, @@ -114,7 +115,7 @@ export default class ModalExport extends React.Component { { value: ['safety_sheet_link_merck', 'safety_sheet_link_thermofischer'], text: 'safety sheet link', - checked: false + checked: false, }, { value: ['product_link_merck', 'product_link_thermofischer'], text: 'product link', checked: false }, { value: 'pictograms', text: 'pictograms', checked: false }, diff --git a/lib/export/export_chemicals.rb b/lib/export/export_chemicals.rb index e854b40bec..9296268ba3 100644 --- a/lib/export/export_chemicals.rb +++ b/lib/export/export_chemicals.rb @@ -3,7 +3,7 @@ module Export class ExportChemicals CHEMICAL_FIELDS = %w[ - chemical_sample_id cas status vendor order_number amount price person required_date + chemical_sample_id cas status vendor order_number amount volume price person required_date ordered_date required_by pictograms h_statements p_statements safety_sheet_link_merck safety_sheet_link_thermofischer product_link_merck product_link_thermofischer host_building host_room host_cabinet host_group owner borrowed_by current_building @@ -20,6 +20,7 @@ class ExportChemicals person: ['c."chemical_data"->0->\'person\'', '"person"', nil], price: ['c."chemical_data"->0->\'price\'', '"price"', nil], amount: ['c."chemical_data"->0->\'amount\'', '"amount"', nil], + volume: ['c."chemical_data"->0->\'volume\'', '"volume"', nil], order_number: ['c."chemical_data"->0->\'order_number\'', '"order_number"', nil], required_date: ['c."chemical_data"->0->\'required_date\'', '"required_date"', nil], required_by: ['c."chemical_data"->0->\'required_by\'', '"required_by"', nil], @@ -67,7 +68,7 @@ def self.format_chemical_results(result) end def self.construct_column_name(column_name, index, columns_index) - format_chemical_column = ['p statements', 'h statements', 'amount', 'safety sheet link thermofischer', + format_chemical_column = ['p statements', 'h statements', 'amount', 'volume', 'safety sheet link thermofischer', 'safety sheet link merck', 'product link thermofischer', 'product link merck'].freeze if column_name.is_a?(String) && CHEMICAL_FIELDS.include?(column_name) column_name = column_name.tr('_', ' ') @@ -84,12 +85,12 @@ def self.construct_column_name_hash(columns_index, column_name, index) columns_index['p_statements'] = index when 'h statements' columns_index['h_statements'] = index - when 'amount' - columns_index['amount'] = index when 'safety sheet link merck', 'safety sheet link thermofischer' columns_index['safety_sheet_link'].push(index) when 'product link merck', 'product link thermofischer' columns_index['product_link'].push(index) + else + columns_index[column_name] = index end end @@ -110,8 +111,8 @@ def self.format_row(row, columns_index, indexes_to_delete) case index when columns_index['p_statements'], columns_index['h_statements'] value = format_p_and_h_statements(value) - when columns_index['amount'] - value = format_chemical_amount(value) + when columns_index['amount'], columns_index['volume'] + value = format_chemical_amount_or_volume(value) when columns_index['safety_sheet_link'][0] value = format_link(value, row, columns_index['safety_sheet_link'][1], indexes_to_delete) when columns_index['product_link'][0] @@ -126,10 +127,10 @@ def self.format_p_and_h_statements(value) keys.join('-') end - def self.format_chemical_amount(value) - amount_value_unit = JSON.parse(value).values - sorted = amount_value_unit.sort_by { |element| [element.is_a?(Integer) || element.is_a?(Float) ? 0 : 1, element] } - sorted.join + def self.format_chemical_amount_or_volume(value) + value_unit = JSON.parse(value).values + sorted = value_unit.sort_by { |element| [element.is_a?(Integer) || element.is_a?(Float) ? 0 : 1, element] } + sorted.join(' ') end def self.format_link(value, row, next_index, indexes_to_delete) diff --git a/lib/import/import_chemicals.rb b/lib/import/import_chemicals.rb index ed88f15754..ac2bd1f66e 100644 --- a/lib/import/import_chemicals.rb +++ b/lib/import/import_chemicals.rb @@ -4,6 +4,7 @@ module Import class ImportChemicals SAFETY_PHRASES = %w[pictograms h_statements p_statements].freeze AMOUNT = %w[amount].freeze + VOLUME = %w[volume].freeze SAFETY_SHEET = %w[safety_sheet_link_merck product_link_merck].freeze KEYS_TO_EXCLUDE = SAFETY_SHEET + %w[cas].freeze SIGMA_ALDRICH_PATTERN = /(sigmaaldrich|merck)/.freeze @@ -14,13 +15,14 @@ class ImportChemicals THERMOFISCHER_PATTERN => 'Alfa', }.freeze CHEMICAL_FIELDS = [ - 'cas', 'status', 'vendor', 'order number', 'amount', 'price', 'person', 'required date', 'ordered date', + 'cas', 'status', 'vendor', 'order number', 'amount', 'volume', 'price', 'person', 'required date', 'ordered date', 'required by', 'pictograms', 'h statements', 'p statements', 'safety sheet link', 'product link', 'host building', 'host room', 'host cabinet', 'host group', 'owner', 'borrowed by', 'current building', 'current room', 'current cabinet', 'current group', 'disposal info', 'important notes' ].freeze GHS_VALUES = %w[GHS01 GHS02 GHS03 GHS04 GHS05 GHS06 GHS07 GHS08 GHS09].freeze AMOUNT_UNITS = %w[g mg μg].freeze + VOLUME_UNITS = %w[ml l μl].freeze def self.build_chemical(row, header) chemical = Chemical.new @@ -46,28 +48,33 @@ def self.skip_import?(value, column_header) value.blank? || column_header.nil? end - def self.process_column(chemical, column_header, value) - map_column = CHEMICAL_FIELDS.find { |e| e == column_header.downcase.rstrip } - key = to_snake_case(column_header) - format_value = value.strip + def self.build_chemical_data(map_column, chemical, key, formated_value) + if map_column.present? && should_process_key(key) - chemical['chemical_data'][0][key] = format_value + chemical['chemical_data'][0][key] = formated_value elsif SAFETY_SHEET.include?(key) - set_safety_sheet(chemical, key, format_value) + set_safety_sheet(chemical, key, formated_value) elsif SAFETY_PHRASES.include?(key) - set_safety_phrases(chemical, key, format_value) - elsif AMOUNT.include?(key) - set_amount(chemical, format_value) + set_safety_phrases(chemical, key, formated_value) + elsif AMOUNT.include?(key) || VOLUME.include?(key) + set_chemical_amount_or_volume(chemical, key, formated_value) end end + def self.process_column(chemical, column_header, value) + map_column = CHEMICAL_FIELDS.find { |e| e == column_header.downcase.rstrip } + key = to_snake_case(column_header) + formated_value = value.strip + build_chemical_data(map_column, chemical, key, formated_value) + end + def self.to_snake_case(column_header) key = column_header.downcase.rstrip.gsub(/\s+/, '_') key == 'owner' ? 'host_owner' : key end def self.should_process_key(key) - KEYS_TO_EXCLUDE.exclude?(key) && (AMOUNT + SAFETY_PHRASES).exclude?(key) + KEYS_TO_EXCLUDE.exclude?(key) && (AMOUNT + VOLUME + SAFETY_PHRASES).exclude?(key) end def self.set_safety_sheet(chemical, key, value) @@ -159,14 +166,30 @@ def self.set_safety_phrases(chemical, key, value) assign_phrases(key, values, phrases) end - def self.set_amount(chemical, value) - chemical['chemical_data'][0]['amount'] = {} if chemical['chemical_data'][0]['amount'].nil? - quantity = value.to_f - unit = value.gsub(/\d+(\.\d+)?/, '') - return chemical unless AMOUNT_UNITS.include?(unit) + def self.extract_quantity(value) + value.to_f + end + + def self.extract_unit(value) + value.gsub(/[0-9.]+/, '').strip + end + + def self.format_value_unit(chemical, key, value, unit) + chemical['chemical_data'][0][key] ||= {} + chemical['chemical_data'][0][key]['value'] = value + chemical['chemical_data'][0][key]['unit'] = unit + chemical + end + + def self.set_chemical_amount_or_volume(chemical, key, value) + quantity = extract_quantity(value) + unit = extract_unit(value) + + return chemical if key == 'amount' && AMOUNT_UNITS.exclude?(unit) + + return chemical if key == 'volume' && VOLUME_UNITS.exclude?(unit) - chemical['chemical_data'][0]['amount']['value'] = quantity - chemical['chemical_data'][0]['amount']['unit'] = unit + format_value_unit(chemical, key, quantity, unit) end end end diff --git a/spec/lib/export/export_chemicals_spec.rb b/spec/lib/export/export_chemicals_spec.rb index 4d8c102eff..866fc7a99c 100644 --- a/spec/lib/export/export_chemicals_spec.rb +++ b/spec/lib/export/export_chemicals_spec.rb @@ -5,11 +5,11 @@ # rubocop: disable Style/OpenStructUse describe Export::ExportChemicals do - describe '.format_chemical_amount' do + describe '.format_chemical_amount_or_volume' do it 'formats chemical amount correctly' do input_value = '{"value": 50, "unit": "mg"}' - formatted_amount = described_class.format_chemical_amount(input_value) - expect(formatted_amount).to eq('50mg') + formatted_amount = described_class.format_chemical_amount_or_volume(input_value) + expect(formatted_amount).to eq('50 mg') end end @@ -129,9 +129,9 @@ formatted_row = described_class.format_row(row, columns_index, indexes_to_delete) expect(indexes_to_delete).to eq([5, 7]) - expect(formatted_row).to eq(%w[value1 key1-key2 key3-key4 300g - safety_link_value-safety_link_value2 safety_link_value2 - product_link_value-product_next_value product_next_value]) + expect(formatted_row).to eq(['value1', 'key1-key2', 'key3-key4', '300 g', + 'safety_link_value-safety_link_value2', 'safety_link_value2', + 'product_link_value-product_next_value', 'product_next_value']) end describe '.construct_column_name' do diff --git a/spec/lib/import/import_chemicals_spec.rb b/spec/lib/import/import_chemicals_spec.rb index 15a25507c7..98abf61b08 100644 --- a/spec/lib/import/import_chemicals_spec.rb +++ b/spec/lib/import/import_chemicals_spec.rb @@ -43,14 +43,20 @@ end end - describe '.sets amount of chemical' do + describe '.sets amount and volume of chemical' do let(:chemical) { { 'chemical_data' => [{}] } } it 'add amount value and unit' do - described_class.set_amount(chemical, '10mg') + described_class.set_chemical_amount_or_volume(chemical, 'amount', '10 mg') expect(chemical['chemical_data'][0]['amount']['value']).to eq(10.0) expect(chemical['chemical_data'][0]['amount']['unit']).to eq('mg') end + + it 'add volume value and unit' do + described_class.set_chemical_amount_or_volume(chemical, 'volume', '6.4 mg') + expect(chemical['chemical_data'][0]['volume']['value']).to eq(6.4) + expect(chemical['chemical_data'][0]['volume']['unit']).to eq('ml') + end end describe '.to_snake_case' do From 3fe1fca01cfc366aceef6ff399cb09c29e319590 Mon Sep 17 00:00:00 2001 From: adambasha0 Date: Wed, 16 Oct 2024 15:15:07 +0000 Subject: [PATCH 4/6] fix: refactor spec test for import chemicals module --- spec/lib/import/import_chemicals_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/import/import_chemicals_spec.rb b/spec/lib/import/import_chemicals_spec.rb index 98abf61b08..0406a8378f 100644 --- a/spec/lib/import/import_chemicals_spec.rb +++ b/spec/lib/import/import_chemicals_spec.rb @@ -53,7 +53,7 @@ end it 'add volume value and unit' do - described_class.set_chemical_amount_or_volume(chemical, 'volume', '6.4 mg') + described_class.set_chemical_amount_or_volume(chemical, 'volume', '6.4 ml') expect(chemical['chemical_data'][0]['volume']['value']).to eq(6.4) expect(chemical['chemical_data'][0]['volume']['unit']).to eq('ml') end From 4b540520869ecdc7e85206832299769be33f8a9a Mon Sep 17 00:00:00 2001 From: adambasha0 Date: Tue, 5 Nov 2024 16:46:25 +0000 Subject: [PATCH 5/6] refactor: allow parcing of flash point and add unit to density on sample export --- lib/export/export_excel.rb | 2 ++ lib/export/export_sdf.rb | 9 ++------- lib/export/export_table.rb | 7 +++++++ lib/import/import_samples.rb | 23 ++++++++++++++++------- 4 files changed, 27 insertions(+), 14 deletions(-) diff --git a/lib/export/export_excel.rb b/lib/export/export_excel.rb index 92f0fb7b67..a452450715 100644 --- a/lib/export/export_excel.rb +++ b/lib/export/export_excel.rb @@ -133,6 +133,8 @@ def filter_with_permission_and_detail_level(sample) flash_point_format(sample['flash_point']) elsif column == 'molarity' "#{sample['molarity_value']} #{sample['molarity_unit']}" + elsif column == 'density' + "#{sample['density']} g/ml" else sample[column] end diff --git a/lib/export/export_sdf.rb b/lib/export/export_sdf.rb index 3f14c126a7..b1fe7e4bd0 100644 --- a/lib/export/export_sdf.rb +++ b/lib/export/export_sdf.rb @@ -45,6 +45,8 @@ def concatenate_data(sample, data, headers = @headers) sample['flash_point'] when 'refractive index' sample['refractive_index'] + when 'density' + "#{sample['density']} g/mL" else sample[column] end @@ -91,13 +93,6 @@ def extract_reference_values(raw_value) string.split(',').join(' - ') end - def flash_point_format(value) - return if value.blank? - - flash_point = JSON.parse(value) - "#{flash_point['value']} #{flash_point['unit']}" - end - def format_field(column, raw_value) field = column.gsub(/\s+/, '_').upcase reference_values = ['melting pt', 'boiling pt'] diff --git a/lib/export/export_table.rb b/lib/export/export_table.rb index 859308b66b..991958f7e6 100644 --- a/lib/export/export_table.rb +++ b/lib/export/export_table.rb @@ -113,8 +113,15 @@ def generate_headers_reaction def flash_point_format(value) return if value.blank? + # Add quotes around unquoted keys & values + value = value.gsub(/(\w+):/, '"\1":') + value = value.gsub(/:\s*([^",{}\s]+)/, ':"\1"') + flash_point = JSON.parse(value) "#{flash_point['value']} #{flash_point['unit']}" + rescue JSON::ParserError => e + Rails.logger.warn("Failed to parse flash_point JSON: #{e.message}") + nil end def format_headers(headers) diff --git a/lib/import/import_samples.rb b/lib/import/import_samples.rb index 7f43fb6845..44f15777f4 100644 --- a/lib/import/import_samples.rb +++ b/lib/import/import_samples.rb @@ -9,7 +9,7 @@ class ImportSamples :processed, :file_path, :collection_id, :current_user_id, :file_name MOLARITY_UNIT = %r{m/L|mol/L|M}i.freeze - DENSITY_UNIT = %r{g/mL}i.freeze + DENSITY_UNIT = %r{g/mL|g/ml}i.freeze FLASH_POINT_UNIT = /°C|F|K/i.freeze def initialize(file_path, collection_id, user_id, file_name, import_type) @@ -265,8 +265,7 @@ def handle_flash_point(sample, value) def handle_density(sample, value) return sample if value[:unit].nil? || value[:value].nil? - sample['density'] = value[:value] if value[:unit] == 'g/mL' - sample + sample['density'] = value[:value] if value[:unit].match?(DENSITY_UNIT) end def handle_molarity(sample, value) @@ -282,9 +281,9 @@ def handle_default_fields(sample, db_column, value) # rubocop:disable Style/StringLiterals def process_fields(sample, map_column, field, row, molecule) - array = ["\"cas\"", "\"purity\"", "\"density\""] + array = ["\"cas\""] conditions = map_column.nil? || array.include?(map_column[1]) - db_column = conditions ? field : map_column[0].sub('s.', '').delete!('"') + db_column = conditions ? field : (map_column[0].sub('s.', '').delete!('"') || map_column[0].sub('s.', '')) molecule.create_molecule_name_by_user(row[field], current_user_id) if field == 'molecule name' process_sample_fields(sample, db_column, field, row) end @@ -351,10 +350,20 @@ def to_value_unit_format(value, db_column) end def process_sample_fields(sample, db_column, field, row) - additional_columns = %w[cas mn.name refractive_index molarity flash_point].freeze + additional_columns = %w[ + cas + mn.name + molarity + refractive_index + flash_point + density + location + melting_point + purity + ].freeze return unless included_fields.include?(db_column) || additional_columns.include?(db_column) - excluded_column = %w[description solvent location].freeze + excluded_column = %w[description solvent].freeze val = row[field] value = process_value(val, db_column) handle_sample_fields(sample, db_column, value) unless value.nil? From 17dda99c156e8781aa4c60c20b5be4a2ba9040a3 Mon Sep 17 00:00:00 2001 From: adambasha0 Date: Wed, 6 Nov 2024 09:22:34 +0000 Subject: [PATCH 6/6] fix: fix rubocop warning --- lib/import/import_chemicals.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/import/import_chemicals.rb b/lib/import/import_chemicals.rb index ac2bd1f66e..7fc4054d55 100644 --- a/lib/import/import_chemicals.rb +++ b/lib/import/import_chemicals.rb @@ -49,7 +49,6 @@ def self.skip_import?(value, column_header) end def self.build_chemical_data(map_column, chemical, key, formated_value) - if map_column.present? && should_process_key(key) chemical['chemical_data'][0][key] = formated_value elsif SAFETY_SHEET.include?(key)