Merge pull request #2716 from AllenInstitute/psb-172/update_dependenc…

…y_compatibility

Update Dependency Compatibility

.github/workflows/github-actions-ci.yml

@@ -44,7 +44,7 @@ jobs:
     strategy:
       matrix:
         os: ["macos-latest", "windows-latest", "ubuntu-latest"]
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
       fail-fast: false
     defaults:
       run:

allensdk/brain_observatory/behavior/behavior_project_cache/tables/util/prior_exposure_processing.py

@@ -157,7 +157,6 @@ def __get_prior_exposure_count(
     index = df.index
     df = df.sort_values("date_of_acquisition")
     df = df[df["session_type"].notnull()]
-
     # reindex "to" to df
     to = to.loc[df.index]
 
@@ -171,12 +170,13 @@ def __get_prior_exposure_count(
         counts = df.groupby(["mouse_id", to]).cumcount()
     elif agg_method == "cumsum":
         df["to"] = to
-
+        df_index_name = df.index.name
         def cumsum(x):
             return x.cumsum().shift(fill_value=0).astype("int64")
 
         counts = df.groupby(["mouse_id"])["to"].apply(cumsum)
         counts.name = None
+        counts.index = counts.index.get_level_values(df_index_name)
     else:
         raise ValueError(f"agg method {agg_method} not supported")
 

allensdk/brain_observatory/behavior/data_objects/cell_specimens/cell_specimens.py

@@ -820,7 +820,7 @@ def _postprocess(
             curr_roi.width = table_row["width"]
             curr_roi.height = table_row["height"]
             curr_roi.mask = np.array(table_row["roi_mask"])
-            roi_mask_list.append(curr_roi.get_mask_plane().astype(np.bool))
+            roi_mask_list.append(curr_roi.get_mask_plane().astype(bool))
 
         cell_specimen_table["roi_mask"] = roi_mask_list
         cell_specimen_table = cell_specimen_table[

allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py

@@ -361,7 +361,7 @@ def from_stimulus_file(
             stim_pres_df = stim_pres_df[
                 stim_pres_df["image_name"].isin(limit_to_images)
             ]
-            stim_pres_df.index = pd.Int64Index(
+            stim_pres_df.index = pd.Index(
                 range(stim_pres_df.shape[0]), name=stim_pres_df.index.name
             )
 

allensdk/brain_observatory/behavior/data_objects/stimuli/stimulus_templates.py

@@ -76,7 +76,7 @@ def _exclude_unseen_pixels(self, arr: np.ndarray):
         """After warping, some pixels are not visible on the screen.
         This sets those pixels to nan to make downstream analysis easier."""
         mask = self._monitor.get_mask()
-        arr = arr.astype(np.float)
+        arr = arr.astype('float')
         arr *= mask
         arr[mask == 0] = np.nan
         return arr

allensdk/brain_observatory/behavior/dprime.py

@@ -9,13 +9,13 @@
 
 def get_go_responses(hit=None, miss=None, aborted=None):
     assert len(hit) == len(miss) == len(aborted)
-    not_aborted = np.logical_not(np.array(aborted, dtype=np.bool))
-    hit = np.array(hit, dtype=np.bool)[not_aborted]
-    miss = np.array(miss, dtype=np.bool)[not_aborted]
+    not_aborted = np.logical_not(np.array(aborted, dtype=bool))
+    hit = np.array(hit, dtype=bool)[not_aborted]
+    miss = np.array(miss, dtype=bool)[not_aborted]
 
     # Go responses are nan when catch (aborted are masked out); 0 for miss, 1 for hit
     # This allows pd.Series.rolling to ignore non-go trial data
-    go_responses = np.empty_like(hit, dtype=np.float)
+    go_responses = np.empty_like(hit, dtype='float')
     go_responses.fill(float('nan'))
     go_responses[hit] = 1
     go_responses[miss] = 0
@@ -38,13 +38,13 @@ def get_trial_count_corrected_hit_rate(hit=None, miss=None, aborted=None, slidin
 
 def get_catch_responses(correct_reject=None, false_alarm=None, aborted=None):
     assert len(correct_reject) == len(false_alarm) == len(aborted)
-    not_aborted = np.logical_not(np.array(aborted, dtype=np.bool))
-    correct_reject = np.array(correct_reject, dtype=np.bool)[not_aborted]
-    false_alarm = np.array(false_alarm, dtype=np.bool)[not_aborted]
+    not_aborted = np.logical_not(np.array(aborted, dtype=bool))
+    correct_reject = np.array(correct_reject, dtype=bool)[not_aborted]
+    false_alarm = np.array(false_alarm, dtype=bool)[not_aborted]
 
     # Catch responses are nan when go (aborted are masked out); 0 for correct-rejection, 1 for false-alarm
     # This allows pd.Series.rolling to ignore non-catch trial data
-    catch_responses = np.empty_like(correct_reject, dtype=np.float)
+    catch_responses = np.empty_like(correct_reject, dtype='float')
     catch_responses.fill(float('nan'))
     catch_responses[false_alarm] = 1
     catch_responses[correct_reject] = 0

allensdk/brain_observatory/behavior/swdb/summary_figures.py

@@ -381,7 +381,7 @@ def plot_max_proj_and_roi_masks(session, save_dir=None):
     ax[2].set_title(str(session.metadata['ophys_experiment_id']))
 
     tmp = session.segmentation_mask_image.data.copy()
-    mask = np.empty(session.segmentation_mask_image.data.shape, dtype=np.float)
+    mask = np.empty(session.segmentation_mask_image.data.shape, dtype='float')
     mask[:] = np.nan
     mask[tmp > 0] = 1
     cax = ax[2].imshow(mask, cmap='hsv', alpha=0.4, vmin=0, vmax=1)

allensdk/brain_observatory/chisquare_categorical.py

@@ -84,12 +84,12 @@ def stim_table_to_categories(stim_table,
 
     category = 0
     sweep_categories = -1*np.ones((num_sweeps,))
-    curr_combination = np.zeros((num_params,),dtype=np.int)
-    options_per_column = np.array(options_per_column).astype(np.int)
+    curr_combination = np.zeros((num_params,),dtype='int')
+    options_per_column = np.array(options_per_column).astype('int')
     all_tried = False
     while not all_tried:
 
-        matches_combination = np.ones((num_sweeps,),dtype=np.bool)
+        matches_combination = np.ones((num_sweeps,),dtype=bool)
         for i_col,column in enumerate(columns):
             param = unique_params[i_col][curr_combination[i_col]]
             matches_param = stim_table[column].values == param
@@ -140,7 +140,7 @@ def make_category_dummy(sweep_categories):
     categories = np.unique(sweep_categories)
     num_categories = len(categories)
 
-    sweep_category_mat = np.zeros((num_sweeps,num_categories),dtype=np.bool)
+    sweep_category_mat = np.zeros((num_sweeps,num_categories),dtype=bool)
     for i_cat,category in enumerate(categories):
         category_idx = np.argwhere(sweep_categories==category)[:,0]
         sweep_category_mat[category_idx,i_cat] = True

allensdk/brain_observatory/ecephys/ecephys_project_api/ecephys_project_warehouse_api.py

@@ -288,7 +288,7 @@ def get_unit_analysis_metrics(self, unit_ids=None, ecephys_session_ids=None, ses
         columns = set(output.columns.values.tolist())
         if "p_value_rf" in columns and "on_screen_rf" in columns:
 
-            pv_is_bool = np.issubdtype(output["p_value_rf"].values[0], np.bool)
+            pv_is_bool = np.issubdtype(output["p_value_rf"].values[0], bool)
             on_screen_is_float = np.issubdtype(output["on_screen_rf"].values[0].dtype, np.floating)
 
             # this is not a good test, but it avoids the case where we fix

allensdk/brain_observatory/ecephys/stimulus_table/naming_utilities.py

@@ -154,7 +154,7 @@ def replace(match_obj):
 
     movie_rows = table[stim_colname].str.contains(movie_re, na=False)
     table.loc[movie_rows, stim_colname] = \
-        table.loc[movie_rows, stim_colname].str.replace(numeral_re, replace)
+        table.loc[movie_rows, stim_colname].str.replace(numeral_re, replace, regex=True)
 
     return table
 

allensdk/brain_observatory/receptive_field_analysis/chisquarerf.py

@@ -146,9 +146,9 @@ def get_peak_significance(chi_square_grid_NLL,
         best_p[n] = p_value_corrected_per_pixel[y,x]
         if best_p[n] < alpha:
             significant_cells[n] = True
-            best_exclusion_region_list.append(disc_masks[y, x, :,:].astype(np.bool))
+            best_exclusion_region_list.append(disc_masks[y, x, :,:].astype(bool))
         else:
-            best_exclusion_region_list.append(np.zeros((disc_masks.shape[0], disc_masks.shape[1]), dtype=np.bool))
+            best_exclusion_region_list.append(np.zeros((disc_masks.shape[0], disc_masks.shape[1]), dtype=bool))
 
     return significant_cells, best_p, corrected_p_value_array_list, best_exclusion_region_list
 

allensdk/brain_observatory/receptive_field_analysis/eventdetection.py

@@ -128,7 +128,7 @@ def detect_events(data, cell_index, stimulus, debug_plots=False):
 
 
     assert len(var_dict) == len(stimulus_table)
-    b = np.zeros(len(stimulus_table), dtype=np.bool)
+    b = np.zeros(len(stimulus_table), dtype=bool)
     for yi in yes_set:
         b[yi] = True
 

allensdk/brain_observatory/receptive_field_analysis/postprocessing.py

@@ -106,7 +106,7 @@ def run_postprocessing(data, rf):
     cell_index = rf['attrs']['cell_index']
     locally_sparse_noise_template = data.get_stimulus_template(stimulus)
 
-    event_array = np.zeros((rf['event_vector']['data'].shape[0], 1), dtype=np.bool)
+    event_array = np.zeros((rf['event_vector']['data'].shape[0], 1), dtype=bool)
     event_array[:,0] = rf['event_vector']['data']
 
     chi_squared_grid = chi_square_binary(event_array, locally_sparse_noise_template)

allensdk/brain_observatory/receptive_field_analysis/receptive_field.py

@@ -93,13 +93,13 @@ def compute_receptive_field(data, cell_index, stimulus, **kwargs):
 
     fdr_corrected_pvalues_on = fdr_corrected_pvalues[
                                :number_of_pixels].reshape(s1, s2)
-    _fdr_mask_on = np.zeros_like(pvalues_on, dtype=np.bool)
+    _fdr_mask_on = np.zeros_like(pvalues_on, dtype=bool)
     _fdr_mask_on[fdr_corrected_pvalues_on < alpha] = True
     components_on, number_of_components_on = get_components(_fdr_mask_on)
 
     fdr_corrected_pvalues_off = fdr_corrected_pvalues[
                                 number_of_pixels:].reshape(s1, s2)
-    _fdr_mask_off = np.zeros_like(pvalues_off, dtype=np.bool)
+    _fdr_mask_off = np.zeros_like(pvalues_off, dtype=bool)
     _fdr_mask_off[fdr_corrected_pvalues_off < alpha] = True
     components_off, number_of_components_off = get_components(_fdr_mask_off)
 

allensdk/brain_observatory/receptive_field_analysis/utilities.py

@@ -130,7 +130,7 @@ def get_shuffle_matrix(data, event_vector, A, number_of_shuffles=5000, response_
         size = number_of_events + int(np.round(response_detection_error_std_dev*number_of_events*np.random.randn()))
         shuffled_event_inds = np.random.choice(evr, size=size, replace=False)
 
-        b_tmp = np.zeros(len(event_vector), dtype=np.bool)
+        b_tmp = np.zeros(len(event_vector), dtype=bool)
         b_tmp[shuffled_event_inds] = True
         shuffle_data[:, ii] = A[:,b_tmp].sum(axis=1)/float(size)
 
@@ -270,7 +270,7 @@ def get_components(receptive_field_data):
         return_array = np.zeros((len(component_list), receptive_field_data.shape[0], receptive_field_data.shape[1]))
 
     for ii, component in enumerate(component_list):
-        curr_component_mask = np.zeros_like(receptive_field_data, dtype=np.bool).flatten()
+        curr_component_mask = np.zeros_like(receptive_field_data, dtype=bool).flatten()
         curr_component_mask[component] = True
         return_array[ii,:,:] = curr_component_mask.reshape(receptive_field_data.shape)
 

allensdk/brain_observatory/stimulus_info.py

@@ -518,9 +518,6 @@ def lsn_image_to_screen(self, img, stimulus_type, origin='lower', background_col
 
     def natural_scene_image_to_screen(self, img, origin='lower', translation=(0,0)):
 
-        # assert img.dtype == np.float32
-        # img = img.astype(np.uint8)
-
         full_image = np.full((self.n_pixels_r, self.n_pixels_c), 127, dtype=np.uint8)
         mr, mc = natural_scene_coordinate_to_monitor_coordinate((0, 0), (self.n_pixels_r, self.n_pixels_c))
         Mr, Mc = natural_scene_coordinate_to_monitor_coordinate((img.shape[0], img.shape[1]), (self.n_pixels_r, self.n_pixels_c))
@@ -756,7 +753,7 @@ def warp_stimulus_coords(vertices,
     distance = float(distance)
     mon_res_x, mon_res_y = float(mon_res[0]), float(mon_res[1])
 
-    vertices = vertices.astype(np.float)
+    vertices = vertices.astype('float')
 
     # from pixels (-1920/2 -> 1920/2) to stimulus space (-0.5->0.5)
     vertices[:, 0] = vertices[:, 0] / mon_res_x

allensdk/core/json_utilities.py

@@ -164,7 +164,7 @@ def json_handler(obj):
         return float(obj)
     elif isinstance(obj, np.integer):
         return int(obj)
-    elif (isinstance(obj, np.bool) or
+    elif (isinstance(obj, bool) or
           isinstance(obj, np.bool_)):
         return bool(obj)
     elif hasattr(obj, 'isoformat'):

allensdk/core/mouse_connectivity_cache.py

@@ -572,7 +572,11 @@ def rank_structures(
                 :, output_keys
             ]
 
-            records = this_experiment_unionizes.to_dict("record")
+            this_experiment_unionizes = unionizes[unionizes['experiment_id'] == eid]
+            this_experiment_unionizes = this_experiment_unionizes.sort_values(by=rank_on, ascending=False)
+            this_experiment_unionizes = this_experiment_unionizes.loc[:, output_keys]
+
+            records = this_experiment_unionizes.to_dict('records')
             if len(records) > n:
                 records = records[:n]
             results.append(records)

allensdk/internal/mouse_connectivity/interval_unionize/data_utilities.py

@@ -93,11 +93,11 @@ def get_projection_data(projection_density_path, projection_energy_path,
         logging.info('getting aav exclusion fraction')
         aav_exclusion_fraction = read(aav_exclusion_fraction_path)
         aav_exclusion_fraction[aav_exclusion_fraction > 0] = 1
-        aav_exclusion_fraction = aav_exclusion_fraction.astype(np.bool_, order='C')
+        aav_exclusion_fraction = aav_exclusion_fraction.astype(bool, order='C')
 
     except (IOError, OSError, RuntimeError):
         logging.info('skipping aav exclusion fraction')
-        aav_exclusion_fraction = np.zeros(projection_density.shape, dtype=np.bool_, order='C')
+        aav_exclusion_fraction = np.zeros(projection_density.shape, dtype=bool, order='C')
 
     return {'projection_density': projection_density, 
             'projection_energy': projection_energy, 

allensdk/test/brain_observatory/ecephys/stimulus_analysis/test_stimulus_analysis.py

@@ -185,7 +185,7 @@ def test_conditionwise_statistics(ecephys_api):
         ["spike_count", "stimulus_presentation_count", "spike_mean", "spike_std", "spike_sem"]
     )
     obtained = stim_analysis.conditionwise_statistics.loc[(0, 1)]
-    pd.testing.assert_series_equal(expected, obtained[expected.index], check_less_precise=5, check_names=False)
+    pd.testing.assert_series_equal(expected, obtained[expected.index], check_dtype=True, check_names=False, rtol=1e-5, atol=1e-8)
 
 
 def test_presentationwise_spike_times(ecephys_api):

allensdk/test/brain_observatory/ecephys/test_write_nwb.py

@@ -563,7 +563,7 @@ def test_add_raw_running_data_to_nwbfile(
 def test_read_stimulus_table(tmpdir_factory, presentations,
                              column_renames_map, columns_to_drop, expected):
     expected = expected.set_index(
-        pd.Int64Index(range(expected.shape[0]),
+        pd.Index(range(expected.shape[0]),
                       name='stimulus_presentations_id'))
     dirname = str(tmpdir_factory.mktemp("ecephys_nwb_test"))
     stim_table_path = os.path.join(dirname, "stim_table.csv")

requirements.txt

@@ -1,33 +1,33 @@
 psycopg2-binary
 hdmf<=3.4.7
 h5py
-matplotlib>=1.4.3
-numpy
-pandas>=1.1.5
-jinja2>=3.0.0
-scipy>=1.4.0,<2.0.0
-six>=1.9.0,<2.0.0
-pynrrd>=0.2.1,<1.0.0
-future >= 0.14.3,<1.0.0
-requests<3.0.0
-requests-toolbelt<1.0.0
-simplejson>=3.10.0,<4.0.0
-scikit-image>=0.14.0
-scikit-build<1.0.0
+matplotlib
+numpy<1.24
+pandas==1.5.3
+jinja2
+scipy<1.11
+six
+pynrrd
+future
+requests
+requests-toolbelt
+simplejson
+scikit-image
+scikit-build
 statsmodels
-simpleitk>=2.0.2,<3.0.0
-argschema>=3.0.1,<4.0.0
-glymur==0.8.19
-xarray
-pynwb
+simpleitk
+argschema
+glymur
+xarray<2023.2.0
+pynwb<=2.3.3
 tables
-seaborn<1.0.0
-aiohttp==3.7.4
+seaborn
+aiohttp
 nest_asyncio
-tqdm>=4.27
-ndx-events<=0.2.0
-boto3==1.17.21
+tqdm
+ndx-events
+boto3
 semver
-cachetools>=4.2.1,<5.0.0
+cachetools
 sqlalchemy
 python-dateutil

setup.cfg

@@ -4,15 +4,6 @@ author = David Feng
 author-email = [email protected]
 summary = Core libraries for the allensdk.
 description-file = README.md
-classifier = 
-	Development Status :: 3 - Alpha',
-	Intended Audience :: Science/Research',
-	Natural Language :: English',
-	Operating System :: OS Independent',
-	Programming Language :: Python
-	Programming Language :: Python :: 2
-	Programming Language :: Python :: 2.7
-	Topic :: Scientific/Engineering :: Bio-Informatics
 
 [build_sphinx]
 source-dir = .

setup.py

@@ -86,9 +86,13 @@ def prepend_find_packages(*roots):
         "License :: Other/Proprietary License", # Allen Institute License
         "Natural Language :: English",
         "Operating System :: OS Independent",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 3",
+        'Programming Language :: Python :: 3 :: Only',
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "Topic :: Scientific/Engineering :: Bio-Informatics",
     ],
 )