From a1f470becfe97ee6d6354862acff97824264caf4 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Fri, 22 Mar 2024 11:35:14 +0100 Subject: [PATCH] Make compatible with python 3.12 --- bluepyopt/ephys/create_acc.py | 571 +++++++++++------- bluepyopt/ephys/simulators.py | 165 ++--- bluepyopt/tests/test_ephys/test_simulators.py | 60 +- versioneer.py | 288 +++++---- 4 files changed, 644 insertions(+), 440 deletions(-) diff --git a/bluepyopt/ephys/create_acc.py b/bluepyopt/ephys/create_acc.py index 4d176014..b0974d1e 100644 --- a/bluepyopt/ephys/create_acc.py +++ b/bluepyopt/ephys/create_acc.py @@ -1,4 +1,4 @@ -'''create JSON/ACC files for Arbor from a set of BluePyOpt.ephys parameters''' +"""create JSON/ACC files for Arbor from a set of BluePyOpt.ephys parameters""" # pylint: disable=R0914 @@ -14,14 +14,18 @@ from bluepyopt.ephys.acc import arbor from bluepyopt.ephys.morphologies import ArbFileMorphology -from bluepyopt.ephys.create_hoc import \ - Location, RangeExpr, PointExpr, \ - _get_template_params, format_float +from bluepyopt.ephys.create_hoc import ( + Location, + RangeExpr, + PointExpr, + _get_template_params, + format_float, +) logger = logging.getLogger(__name__) # Inhomogeneous expression for scaled parameter in Arbor -RangeIExpr = namedtuple('RangeIExpr', 'name, value, scale') +RangeIExpr = namedtuple("RangeIExpr", "name, value, scale") class ArbVar: @@ -39,25 +43,32 @@ def __init__(self, name, conv=None): self.conv = conv def __repr__(self): - return 'ArbVar(%s, %s)' % (self.name, self.conv) + return "ArbVar(%s, %s)" % (self.name, self.conv) class Nrn2ArbParamAdapter: """Converts a Neuron parameter to Arbor format (name and value)""" _mapping = dict( - v_init=ArbVar(name='membrane-potential'), - celsius=ArbVar(name='temperature-kelvin', - conv=lambda celsius: celsius + 273.15), - Ra=ArbVar(name='axial-resistivity'), - cm=ArbVar(name='membrane-capacitance', - conv=lambda cm: cm / 100.), # NEURON: uF/cm^2, Arbor: F/m^2 - **{species + loc[0]: - ArbVar(name='ion-%sternal-concentration \"%s\"' % (loc, species)) - for species in ['na', 'k', 'ca'] for loc in ['in', 'ex']}, - **{'e' + species: - ArbVar(name='ion-reversal-potential \"%s\"' % species) - for species in ['na', 'k', 'ca']} + v_init=ArbVar(name="membrane-potential"), + celsius=ArbVar( + name="temperature-kelvin", conv=lambda celsius: celsius + 273.15 + ), + Ra=ArbVar(name="axial-resistivity"), + cm=ArbVar( + name="membrane-capacitance", conv=lambda cm: cm / 100.0 + ), # NEURON: uF/cm^2, Arbor: F/m^2 + **{ + species + loc[0]: ArbVar( + name='ion-%sternal-concentration "%s"' % (loc, species) + ) + for species in ["na", "k", "ca"] + for loc in ["in", "ex"] + }, + **{ + "e" + species: ArbVar(name='ion-reversal-potential "%s"' % species) + for species in ["na", "k", "ca"] + }, ) @classmethod @@ -77,13 +88,19 @@ def _param_value(cls, param): param (): A Neuron parameter with a value in Neuron units """ - if param.name in cls._mapping and \ - cls._mapping[param.name].conv is not None: + if ( + param.name in cls._mapping + and cls._mapping[param.name].conv is not None + ): return format_float( - cls._mapping[param.name].conv(float(param.value))) + cls._mapping[param.name].conv(float(param.value)) + ) else: - return param.value if isinstance(param.value, str) else \ - format_float(param.value) + return ( + param.value + if isinstance(param.value, str) + else format_float(param.value) + ) @classmethod def _conv_param(cls, param, name): @@ -95,20 +112,26 @@ def _conv_param(cls, param, name): """ if isinstance(param, Location): - return Location(name=cls._param_name(name), - value=cls._param_value(param)) + return Location( + name=cls._param_name(name), value=cls._param_value(param) + ) elif isinstance(param, RangeExpr): - return RangeExpr(location=param.location, - name=cls._param_name(name), - value=cls._param_value(param), - value_scaler=param.value_scaler) + return RangeExpr( + location=param.location, + name=cls._param_name(name), + value=cls._param_value(param), + value_scaler=param.value_scaler, + ) elif isinstance(param, PointExpr): - return PointExpr(name=cls._param_name(name), - point_loc=param.point_loc, - value=cls._param_value(param)) + return PointExpr( + name=cls._param_name(name), + point_loc=param.point_loc, + value=cls._param_value(param), + ) else: raise CreateAccException( - 'Unsupported parameter expression type %s.' % type(param)) + "Unsupported parameter expression type %s." % type(param) + ) @classmethod def format(cls, param, mechs): @@ -123,12 +146,16 @@ def format(cls, param, mechs): parameter in Arbor format """ if not isinstance(param, PointExpr): - mech_matches = [i for i, mech in enumerate(mechs) - if param.name.endswith("_" + mech)] + mech_matches = [ + i + for i, mech in enumerate(mechs) + if param.name.endswith("_" + mech) + ] else: param_pprocesses = [loc.pprocess_mech for loc in param.point_loc] - mech_matches = [i for i, mech in enumerate(mechs) - if mech in param_pprocesses] + mech_matches = [ + i for i, mech in enumerate(mechs) if mech in param_pprocesses + ] if len(mech_matches) == 0: return None, cls._conv_param(param, name=param.name) @@ -136,15 +163,17 @@ def format(cls, param, mechs): elif len(mech_matches) == 1: mech = mechs[mech_matches[0]] if not isinstance(param, PointExpr): - name = param.name[:-(len(mech) + 1)] + name = param.name[: -(len(mech) + 1)] else: name = param.name return mech, cls._conv_param(param, name=name) else: - raise CreateAccException("Parameter name %s matches" % param.name + - " multiple mechanisms %s" % - [repr(mechs[i]) for i in mech_matches]) + raise CreateAccException( + "Parameter name %s matches" % param.name + + " multiple mechanisms %s" + % [repr(mechs[i]) for i in mech_matches] + ) class Nrn2ArbMechGrouper: @@ -159,14 +188,21 @@ def _is_global_property(loc, param): param (): A parameter in Arbor format (name and units) """ - return loc == ArbFileMorphology.region_labels['all'] and ( - param.name in ['membrane-potential', - 'temperature-kelvin', - 'axial-resistivity', - 'membrane-capacitance'] or - param.name.split(' ')[0] in ['ion-internal-concentration', - 'ion-external-concentration', - 'ion-reversal-potential']) + return loc == ArbFileMorphology.region_labels["all"] and ( + param.name + in [ + "membrane-potential", + "temperature-kelvin", + "axial-resistivity", + "membrane-capacitance", + ] + or param.name.split(" ")[0] + in [ + "ion-internal-concentration", + "ion-external-concentration", + "ion-reversal-potential", + ] + ) @classmethod def _separate_global_properties(cls, loc, mechs): @@ -187,7 +223,6 @@ def _separate_global_properties(cls, loc, mechs): global_properties = [] for mech, params in mechs.items(): - if mech is None: local_properties = [] for param in params: @@ -213,8 +248,9 @@ def _format_params_and_group_by_mech(params, channels): Mapping of Arbor mechanism name to list of parameters in Arbor format """ - mech_params = [Nrn2ArbParamAdapter.format( - param, channels) for param in params] + mech_params = [ + Nrn2ArbParamAdapter.format(param, channels) for param in params + ] mechs = {mech: [] for mech, _ in mech_params} for mech in channels: if mech not in mechs: @@ -236,9 +272,11 @@ def process_global(cls, params): (mechanism name is None for non-mechanism parameters). """ return cls._format_params_and_group_by_mech( - [Location(name=name, value=value) - for name, value in params.items()], - [] # no default mechanisms + [ + Location(name=name, value=value) + for name, value in params.items() + ], + [], # no default mechanisms ) @classmethod @@ -261,16 +299,19 @@ def process_local(cls, params, channels): global_properties = dict() for loc, loc_params in params: mechs = cls._format_params_and_group_by_mech( - loc_params, channels[loc]) + loc_params, channels[loc] + ) # move Arbor global properties to global_params mechs, global_props = cls._separate_global_properties(loc, mechs) if global_props.keys() != {None}: raise CreateAccException( - 'Support for Arbor default mechanisms not implemented.') + "Support for Arbor default mechanisms not implemented." + ) # iterate over global_props items if above exception triggers - global_properties[None] = \ + global_properties[None] = ( global_properties.get(None, []) + global_props[None] + ) local_mechs[loc] = mechs return local_mechs, global_properties @@ -289,8 +330,11 @@ def _arb_filter_point_proc_locs(pprocess_mechs): for mech, point_exprs in mechs.items(): result[loc][mech.name] = dict( mech=mech.suffix, - params=[Location(point_expr.name, point_expr.value) - for point_expr in point_exprs]) + params=[ + Location(point_expr.name, point_expr.value) + for point_expr in point_exprs + ], + ) return result @@ -300,18 +344,21 @@ def _arb_append_scaled_mechs(mechs, scaled_mechs): for mech, scaled_params in scaled_mechs.items(): if mech is None and len(scaled_params) > 0: raise CreateAccException( - 'Non-mechanism parameters cannot have inhomogeneous' - ' expressions in Arbor %s' % scaled_params) - mechs[mech] = mechs.get(mech, []) + \ - [RangeIExpr( + "Non-mechanism parameters cannot have inhomogeneous" + " expressions in Arbor %s" % scaled_params + ) + mechs[mech] = mechs.get(mech, []) + [ + RangeIExpr( name=p.name, value=p.value, - scale=p.value_scaler.acc_scale_iexpr(p.value)) - for p in scaled_params] + scale=p.value_scaler.acc_scale_iexpr(p.value), + ) + for p in scaled_params + ] # An mechanism's NMODL GLOBAL and RANGE variables in Arbor -MechMetaData = namedtuple('MechMetaData', 'globals, ranges') +MechMetaData = namedtuple("MechMetaData", "globals, ranges") class ArbNmodlMechFormatter: @@ -338,37 +385,42 @@ def _load_catalogue_meta(cat_dir): """ # used to generate arbor_mechanisms.json on NMODL from arbor/mechanisms - nmodl_pattern = '^\s*%s\s+((?:\w+\,\s*)*?\w+)\s*?$' # NOQA - suffix_pattern = nmodl_pattern % 'SUFFIX' - globals_pattern = nmodl_pattern % 'GLOBAL' - ranges_pattern = nmodl_pattern % 'RANGE' + nmodl_pattern = r"^\s*%s\s+((?:\w+\,\s*)*?\w+)\s*?$" # NOQA + suffix_pattern = nmodl_pattern % "SUFFIX" + globals_pattern = nmodl_pattern % "GLOBAL" + ranges_pattern = nmodl_pattern % "RANGE" def process_nmodl(nmodl_str): """Extract global and range params from Arbor-conforming NMODL""" try: - nrn = re.search(r'NEURON\s+{([^}]+)}', nmodl_str, - flags=re.MULTILINE).group(1) - suffix_ = re.search(suffix_pattern, nrn, - flags=re.MULTILINE) + nrn = re.search( + r"NEURON\s+{([^}]+)}", nmodl_str, flags=re.MULTILINE + ).group(1) + suffix_ = re.search(suffix_pattern, nrn, flags=re.MULTILINE) suffix_ = suffix_ if suffix_ is None else suffix_.group(1) - globals_ = re.search(globals_pattern, nrn, - flags=re.MULTILINE) - globals_ = globals_ if globals_ is None \ - else re.findall(r'\w+', globals_.group(1)) - ranges_ = re.search(ranges_pattern, nrn, - flags=re.MULTILINE) - ranges_ = ranges_ if ranges_ is None \ - else re.findall(r'\w+', ranges_.group(1)) + globals_ = re.search(globals_pattern, nrn, flags=re.MULTILINE) + globals_ = ( + globals_ + if globals_ is None + else re.findall(r"\w+", globals_.group(1)) + ) + ranges_ = re.search(ranges_pattern, nrn, flags=re.MULTILINE) + ranges_ = ( + ranges_ + if ranges_ is None + else re.findall(r"\w+", ranges_.group(1)) + ) except Exception as e: raise CreateAccException( - 'NMODL-inspection for %s failed.' % nmodl_file) from e + "NMODL-inspection for %s failed." % nmodl_file + ) from e # skipping suffix_ return MechMetaData(globals=globals_, ranges=ranges_) mechs = dict() cat_dir = pathlib.Path(cat_dir) - for nmodl_file in cat_dir.glob('*.mod'): + for nmodl_file in cat_dir.glob("*.mod"): with open(cat_dir.joinpath(nmodl_file)) as f: mechs[nmodl_file.stem] = process_nmodl(f.read()) @@ -393,18 +445,23 @@ def _load_mech_catalogue_meta(cls, ext_catalogues): if ext_catalogues is not None: for cat, cat_nmodl in ext_catalogues.items(): arb_cats[cat] = cls._load_catalogue_meta( - pathlib.Path(cat_nmodl).resolve()) + pathlib.Path(cat_nmodl).resolve() + ) - builtin_catalogues = pathlib.Path(__file__).parent.joinpath( - 'static/arbor_mechanisms.json').resolve() + builtin_catalogues = ( + pathlib.Path(__file__) + .parent.joinpath("static/arbor_mechanisms.json") + .resolve() + ) with open(builtin_catalogues) as f: builtin_arb_cats = json.load(f) - for cat in ['BBP', 'default', 'allen']: + for cat in ["BBP", "default", "allen"]: if cat not in arb_cats: arb_cats[cat] = { mech: MechMetaData(**meta) - for mech, meta in builtin_arb_cats[cat].items()} + for mech, meta in builtin_arb_cats[cat].items() + } return arb_cats @@ -415,7 +472,7 @@ def _mech_name(name): Args: name (): A Neuron mechanism name """ - if name in ['Exp2Syn', 'ExpSyn']: + if name in ["Exp2Syn", "ExpSyn"]: return name.lower() else: return name @@ -441,51 +498,64 @@ def _translate_mech(cls, mech_name, mech_params, arb_cats): for cat in arb_cats: # in order of precedence if arb_mech_name in arb_cats[cat]: arb_mech = arb_cats[cat][arb_mech_name] - mech_name = cat + '::' + arb_mech_name + mech_name = cat + "::" + arb_mech_name break if arb_mech is None: # not Arbor built-in mech, no qualifier added if mech_name is not None: logger.warn( - 'create_acc: Could not find Arbor mech for %s (%s).' - % (mech_name, mech_params)) + "create_acc: Could not find Arbor mech for %s (%s)." + % (mech_name, mech_params) + ) return (mech_name, mech_params) else: if arb_mech.globals is None: # only Arbor range params for param in mech_params: if param.name not in arb_mech.ranges: raise CreateAccException( - '%s not a GLOBAL or RANGE parameter of %s' % - (param.name, mech_name)) + "%s not a GLOBAL or RANGE parameter of %s" + % (param.name, mech_name) + ) return (mech_name, mech_params) else: for param in mech_params: - if param.name not in arb_mech.globals and \ - param.name not in arb_mech.ranges: + if ( + param.name not in arb_mech.globals + and param.name not in arb_mech.ranges + ): raise CreateAccException( - '%s not a GLOBAL or RANGE parameter of %s' % - (param.name, mech_name)) + "%s not a GLOBAL or RANGE parameter of %s" + % (param.name, mech_name) + ) mech_name_suffix = [] remaining_mech_params = [] for mech_param in mech_params: if mech_param.name in arb_mech.globals: - mech_name_suffix.append(mech_param.name + '=' + - mech_param.value) + mech_name_suffix.append( + mech_param.name + "=" + mech_param.value + ) if isinstance(mech_param, RangeIExpr): remaining_mech_params.append( - RangeIExpr(name=mech_param.name, - value=None, - scale=mech_param.scale)) + RangeIExpr( + name=mech_param.name, + value=None, + scale=mech_param.scale, + ) + ) else: remaining_mech_params.append(mech_param) if len(mech_name_suffix) > 0: - mech_name += '/' + ','.join(mech_name_suffix) + mech_name += "/" + ",".join(mech_name_suffix) return (mech_name, remaining_mech_params) def translate_density(self, mechs): """Translate all density mechanisms in a specific region""" - return dict([self._translate_mech(mech, params, self.cats) - for mech, params in mechs.items()]) + return dict( + [ + self._translate_mech(mech, params, self.cats) + for mech, params in mechs.items() + ] + ) def translate_points(self, mechs): """Translate all point mechanisms for a specific locset""" @@ -493,9 +563,9 @@ def translate_points(self, mechs): for synapse_name, mech_desc in mechs.items(): mech, params = self._translate_mech( - mech_desc['mech'], mech_desc['params'], self.cats) - result[synapse_name] = dict(mech=mech, - params=params) + mech_desc["mech"], mech_desc["params"], self.cats + ) + result[synapse_name] = dict(mech=mech, params=params) return result @@ -527,13 +597,16 @@ def _arb_populate_label_dict(local_mechs, local_scaled_mechs, pprocess_mechs): acc_labels = ChainMap(local_mechs, local_scaled_mechs, pprocess_mechs) for acc_label in acc_labels: - if acc_label.name in label_dict and \ - acc_label != label_dict[acc_label.name]: + if ( + acc_label.name in label_dict + and acc_label != label_dict[acc_label.name] + ): raise CreateAccException( - 'Label %s already exists in' % acc_label.name + - ' label_dict with different s-expression: ' - ' %s != %s.' % (label_dict[acc_label.name].loc, - acc_label.loc)) + "Label %s already exists in" + % acc_label.name + + " label_dict with different s-expression: " + " %s != %s." % (label_dict[acc_label.name].loc, acc_label.loc) + ) elif acc_label.name not in label_dict: label_dict[acc_label.name] = acc_label @@ -542,10 +615,11 @@ def _arb_populate_label_dict(local_mechs, local_scaled_mechs, pprocess_mechs): def _read_templates(template_dir, template_filename): """Expand Jinja2 template filepath with glob and - return dict of target filename -> parsed template""" + return dict of target filename -> parsed template""" if template_dir is None: - template_dir = \ - pathlib.Path(__file__).parent.joinpath('templates').resolve() + template_dir = ( + pathlib.Path(__file__).parent.joinpath("templates").resolve() + ) template_paths = pathlib.Path(template_dir).glob(template_filename) @@ -554,17 +628,18 @@ def _read_templates(template_dir, template_filename): with open(template_path) as template_file: template = template_file.read() name = template_path.name - if name.endswith('.jinja2'): + if name.endswith(".jinja2"): name = name[:-7] - if name.endswith('_template'): + if name.endswith("_template"): name = name[:-9] - if '_' in name: - name = '.'.join(name.rsplit('_', 1)) + if "_" in name: + name = ".".join(name.rsplit("_", 1)) templates[name] = jinja2.Template(template) if templates == {}: raise FileNotFoundError( - f'No templates found for JSON/ACC-export in {template_dir}') + f"No templates found for JSON/ACC-export in {template_dir}" + ) return templates @@ -574,20 +649,22 @@ def _arb_loc_desc(location, param_or_mech): return location.acc_label() -def create_acc(mechs, - parameters, - morphology=None, - morphology_dir=None, - ext_catalogues=None, - ignored_globals=(), - replace_axon=None, - create_mod_morph=False, - template_name='CCell', - template_filename='acc/*_template.jinja2', - disable_banner=None, - template_dir=None, - custom_jinja_params=None): - '''return a dict with strings containing the rendered JSON/ACC templates +def create_acc( + mechs, + parameters, + morphology=None, + morphology_dir=None, + ext_catalogues=None, + ignored_globals=(), + replace_axon=None, + create_mod_morph=False, + template_name="CCell", + template_filename="acc/*_template.jinja2", + disable_banner=None, + template_dir=None, + custom_jinja_params=None, +): + """return a dict with strings containing the rendered JSON/ACC templates Args: mechs (): All the mechs for the decor template @@ -604,39 +681,45 @@ def create_acc(mechs, template_dir (str): dir name of the jinja2 templates custom_jinja_params (dict): dict of additional jinja2 params in case of a custom template - ''' + """ if custom_jinja_params is None: custom_jinja_params = {} - if pathlib.Path(morphology).suffix.lower() not in ['.swc', '.asc']: - raise CreateAccException("Morphology file %s not supported in Arbor " - " (only supported types are .swc and .asc)." - % morphology) + if pathlib.Path(morphology).suffix.lower() not in [".swc", ".asc"]: + raise CreateAccException( + "Morphology file %s not supported in Arbor " + " (only supported types are .swc and .asc)." % morphology + ) if replace_axon is not None: - if not hasattr(arbor.segment_tree, 'tag_roots'): - raise NotImplementedError("Need a newer version of Arbor" - " for axon replacement.") - logger.debug("Obtain axon replacement by applying " - "ArbFileMorphology.replace_axon after loading " - "morphology in Arbor.") - replace_axon_path = \ - pathlib.Path(morphology).stem + '_axon_replacement.acc' + if not hasattr(arbor.segment_tree, "tag_roots"): + raise NotImplementedError( + "Need a newer version of Arbor" " for axon replacement." + ) + logger.debug( + "Obtain axon replacement by applying " + "ArbFileMorphology.replace_axon after loading " + "morphology in Arbor." + ) + replace_axon_path = ( + pathlib.Path(morphology).stem + "_axon_replacement.acc" + ) replace_axon_acc = io.StringIO() arbor.write_component(replace_axon, replace_axon_acc) replace_axon_acc.seek(0) if create_mod_morph: - modified_morphology_path = \ - pathlib.Path(morphology).stem + '_modified.acc' + modified_morphology_path = ( + pathlib.Path(morphology).stem + "_modified.acc" + ) modified_morpho = ArbFileMorphology.load( pathlib.Path(morphology_dir).joinpath(morphology), - replace_axon_acc) + replace_axon_acc, + ) replace_axon_acc.seek(0) modified_morphology_acc = io.StringIO() - arbor.write_component( - modified_morpho, modified_morphology_acc) + arbor.write_component(modified_morpho, modified_morphology_acc) modified_morphology_acc.seek(0) modified_morphology_acc = modified_morphology_acc.read() else: @@ -652,47 +735,49 @@ def create_acc(mechs, default_location_order = list(ArbFileMorphology.region_labels.values()) - template_params = _get_template_params(mechs, - parameters, - ignored_globals, - disable_banner, - default_location_order, - _arb_loc_desc) + template_params = _get_template_params( + mechs, + parameters, + ignored_globals, + disable_banner, + default_location_order, + _arb_loc_desc, + ) filenames = { - name: template_name + (name if name.startswith('.') else "_" + name) - for name in templates.keys()} + name: template_name + (name if name.startswith(".") else "_" + name) + for name in templates.keys() + } # postprocess template parameters for Arbor - channels = template_params['channels'] - point_channels = template_params['point_channels'] - banner = template_params['banner'] + channels = template_params["channels"] + point_channels = template_params["point_channels"] + banner = template_params["banner"] # global_mechs refer to default density mechs/params in Arbor # [mech -> param] (params under mech == None) - global_mechs = \ - Nrn2ArbMechGrouper.process_global( - template_params['global_params']) + global_mechs = Nrn2ArbMechGrouper.process_global( + template_params["global_params"] + ) # local_mechs refer to locally painted density mechs/params in Arbor # [label -> mech -> param.name/.value] (params under mech == None) - local_mechs, additional_global_mechs = \ - Nrn2ArbMechGrouper.process_local( - template_params['section_params'], channels) + local_mechs, additional_global_mechs = Nrn2ArbMechGrouper.process_local( + template_params["section_params"], channels + ) for mech, params in additional_global_mechs.items(): - global_mechs[mech] = \ - global_mechs.get(mech, []) + params + global_mechs[mech] = global_mechs.get(mech, []) + params # scaled_mechs refer to iexpr params of scaled density mechs in Arbor # [label -> mech -> param.location/.name/.value/.value_scaler] range_params = {loc: [] for loc in default_location_order} - for param in template_params['range_params']: + for param in template_params["range_params"]: range_params[param.location].append(param) range_params = list(range_params.items()) - local_scaled_mechs, global_scaled_mechs = \ - Nrn2ArbMechGrouper.process_local( - range_params, channels) + local_scaled_mechs, global_scaled_mechs = Nrn2ArbMechGrouper.process_local( + range_params, channels + ) # join each mech's constant params with inhomogeneous ones on mechanisms _arb_append_scaled_mechs(global_mechs, global_scaled_mechs) @@ -701,12 +786,13 @@ def create_acc(mechs, # pprocess_mechs refer to locally placed mechs/params in Arbor # [label -> mech -> param.name/.value] - pprocess_mechs, global_pprocess_mechs = \ - Nrn2ArbMechGrouper.process_local( - template_params['pprocess_params'], point_channels) + pprocess_mechs, global_pprocess_mechs = Nrn2ArbMechGrouper.process_local( + template_params["pprocess_params"], point_channels + ) if any(len(params) > 0 for params in global_pprocess_mechs.values()): - raise CreateAccException('Point process mechanisms cannot be' - ' placed globally in Arbor.') + raise CreateAccException( + "Point process mechanisms cannot be" " placed globally in Arbor." + ) # Evaluate synapse locations # (no new labels introduced, but locations explicitly defined) @@ -720,36 +806,43 @@ def create_acc(mechs, global_mechs = nmodl_formatter.translate_density(global_mechs) local_mechs = { loc: nmodl_formatter.translate_density(mechs) - for loc, mechs in local_mechs.items()} + for loc, mechs in local_mechs.items() + } pprocess_mechs = { loc: nmodl_formatter.translate_points(mechs) - for loc, mechs in pprocess_mechs.items()} + for loc, mechs in pprocess_mechs.items() + } # get iexpr parameters of scaled density mechs global_scaled_mechs = _arb_project_scaled_mechs(global_mechs) - local_scaled_mechs = {loc: _arb_project_scaled_mechs(mechs) - for loc, mechs in local_mechs.items()} + local_scaled_mechs = { + loc: _arb_project_scaled_mechs(mechs) + for loc, mechs in local_mechs.items() + } # populate label dict - label_dict = _arb_populate_label_dict(local_mechs, - local_scaled_mechs, - pprocess_mechs) - - ret = {filenames[name]: - template.render(template_name=template_name, - banner=banner, - morphology=morphology, - replace_axon=replace_axon_path, - modified_morphology=modified_morphology_path, - filenames=filenames, - label_dict=label_dict, - global_mechs=global_mechs, - global_scaled_mechs=global_scaled_mechs, - local_mechs=local_mechs, - local_scaled_mechs=local_scaled_mechs, - pprocess_mechs=pprocess_mechs, - **custom_jinja_params) - for name, template in templates.items()} + label_dict = _arb_populate_label_dict( + local_mechs, local_scaled_mechs, pprocess_mechs + ) + + ret = { + filenames[name]: template.render( + template_name=template_name, + banner=banner, + morphology=morphology, + replace_axon=replace_axon_path, + modified_morphology=modified_morphology_path, + filenames=filenames, + label_dict=label_dict, + global_mechs=global_mechs, + global_scaled_mechs=global_scaled_mechs, + local_mechs=local_mechs, + local_scaled_mechs=local_scaled_mechs, + pprocess_mechs=pprocess_mechs, + **custom_jinja_params, + ) + for name, template in templates.items() + } if replace_axon is not None: ret[replace_axon_path] = replace_axon_acc @@ -759,12 +852,16 @@ def create_acc(mechs, return ret -def write_acc(output_dir, cell, parameters, - template_filename='acc/*_template.jinja2', - ext_catalogues=None, - create_mod_morph=False, - sim=None): - '''Output mixed JSON/ACC format for Arbor cable cell to files +def write_acc( + output_dir, + cell, + parameters, + template_filename="acc/*_template.jinja2", + ext_catalogues=None, + create_mod_morph=False, + sim=None, +): + """Output mixed JSON/ACC format for Arbor cable cell to files Args: output_dir (str): Output directory. If not exists, will be created @@ -777,18 +874,24 @@ def write_acc(output_dir, cell, parameters, create_mod_morph (str): Output ACC with axon replacement sim (): Neuron simulator instance (only used used with axon replacement if morphology has not yet been instantiated) - ''' - output = cell.create_acc(parameters, template=template_filename, - ext_catalogues=ext_catalogues, - create_mod_morph=create_mod_morph, - sim=sim) - - cell_json = [comp_rendered - for comp, comp_rendered in output.items() - if pathlib.Path(comp).suffix == '.json'] + """ + output = cell.create_acc( + parameters, + template=template_filename, + ext_catalogues=ext_catalogues, + create_mod_morph=create_mod_morph, + sim=sim, + ) + + cell_json = [ + comp_rendered + for comp, comp_rendered in output.items() + if pathlib.Path(comp).suffix == ".json" + ] if len(cell_json) != 1: raise CreateAccException( - 'JSON file from create_acc is non-unique: %s' % cell_json) + "JSON file from create_acc is non-unique: %s" % cell_json + ) cell_json = json.loads(cell_json[0]) @@ -799,10 +902,10 @@ def write_acc(output_dir, cell, parameters, comp_filename = output_dir.joinpath(comp) if comp_filename.exists(): raise CreateAccException("%s already exists!" % comp_filename) - with open(output_dir.joinpath(comp), 'w') as f: + with open(output_dir.joinpath(comp), "w") as f: f.write(comp_rendered) - morpho_filename = output_dir.joinpath(cell_json['morphology']['original']) + morpho_filename = output_dir.joinpath(cell_json["morphology"]["original"]) if morpho_filename.exists(): raise CreateAccException("%s already exists!" % morpho_filename) shutil.copy2(cell.morphology.morphology_path, morpho_filename) @@ -810,12 +913,12 @@ def write_acc(output_dir, cell, parameters, # Read the mixed JSON/ACC-output, to be moved to Arbor in future release def read_acc(cell_json_filename): - '''Return constituents to build an Arbor cable cell from create_acc-export + """Return constituents to build an Arbor cable cell from create_acc-export Args: cell_json_filename (str): The path to the JSON file containing meta-information on morphology, label-dict and decor of exported cell - ''' + """ with open(cell_json_filename) as cell_json_file: cell_json = json.load(cell_json_file) @@ -823,22 +926,24 @@ def read_acc(cell_json_filename): cell_json_dir = pathlib.Path(cell_json_filename).parent morpho_filename = cell_json_dir.joinpath( - cell_json['morphology']['original']) - replace_axon = cell_json['morphology'].get('replace_axon', None) + cell_json["morphology"]["original"] + ) + replace_axon = cell_json["morphology"].get("replace_axon", None) if replace_axon is not None: replace_axon = cell_json_dir.joinpath(replace_axon) morpho = ArbFileMorphology.load(morpho_filename, replace_axon) decor = arbor.load_component( - cell_json_dir.joinpath(cell_json['decor'])).component + cell_json_dir.joinpath(cell_json["decor"]) + ).component labels = arbor.load_component( - cell_json_dir.joinpath(cell_json['label_dict'])).component + cell_json_dir.joinpath(cell_json["label_dict"]) + ).component return cell_json, morpho, decor, labels class CreateAccException(Exception): - """Exceptions generated by create_acc module""" def __init__(self, message): diff --git a/bluepyopt/ephys/simulators.py b/bluepyopt/ephys/simulators.py index 71790bc8..de546633 100644 --- a/bluepyopt/ephys/simulators.py +++ b/bluepyopt/ephys/simulators.py @@ -2,26 +2,30 @@ # pylint: disable=W0511 -import os -import logging -import imp import ctypes +import importlib.util +import logging +import os +import pathlib import platform import warnings -import pathlib from bluepyopt.ephys.acc import arbor - logger = logging.getLogger(__name__) class NrnSimulator(object): - """Neuron simulator""" - def __init__(self, dt=None, cvode_active=True, cvode_minstep=None, - random123_globalindex=None, mechanisms_directory=None): + def __init__( + self, + dt=None, + cvode_active=True, + cvode_minstep=None, + random123_globalindex=None, + mechanisms_directory=None, + ): """Constructor Args: @@ -41,7 +45,7 @@ def __init__(self, dt=None, cvode_active=True, cvode_minstep=None, # hoc.so does not exist on NEURON Windows or MacOS # although \\hoc.pyd can work here, it gives an error for # nrn_nobanner_ line - self.disable_banner = platform.system() not in ['Windows', 'Darwin'] + self.disable_banner = platform.system() not in ["Windows", "Darwin"] self.banner_disabled = False self.mechanisms_directory = mechanisms_directory @@ -77,18 +81,21 @@ def cvode_minstep(self, value): def _nrn_disable_banner(): """Disable Neuron banner""" - nrnpy_path = os.path.join(imp.find_module('neuron')[1]) - import glob - hoc_so_list = \ - glob.glob(os.path.join(nrnpy_path, 'hoc*.so')) + nrnpy_path = pathlib.Path( + importlib.util.find_spec("neuron").origin + ).parent + + hoc_so_list = list(nrnpy_path.glob("hoc*.so")) if len(hoc_so_list) != 1: - warnings.warn('Unable to find Neuron hoc shared library in %s, ' - 'not disabling banner' % nrnpy_path) + warnings.warn( + "Unable to find Neuron hoc shared library in %s, " + "not disabling banner" % nrnpy_path + ) else: hoc_so = hoc_so_list[0] - nrndll = ctypes.cdll[hoc_so] - ctypes.c_int.in_dll(nrndll, 'nrn_nobanner_').value = 1 + nrndll = ctypes.cdll[str(hoc_so)] + ctypes.c_int.in_dll(nrndll, "nrn_nobanner_").value = 1 # pylint: disable=R0201 @property @@ -110,24 +117,26 @@ def neuron(self): def initialize(self): """Initialize simulator: Set Neuron variables""" - self.neuron.h.load_file('stdrun.hoc') + self.neuron.h.load_file("stdrun.hoc") self.neuron.h.dt = self.dt self.neuron.h.cvode_active(1 if self.cvode_active else 0) def run( - self, - tstop=None, - dt=None, - cvode_active=None, - random123_globalindex=None): + self, + tstop=None, + dt=None, + cvode_active=None, + random123_globalindex=None, + ): """Run protocol""" self.neuron.h.tstop = tstop if cvode_active and dt is not None: raise ValueError( - 'NrnSimulator: Impossible to combine the dt argument when ' - 'cvode_active is True in the NrnSimulator run method') + "NrnSimulator: Impossible to combine the dt argument when " + "cvode_active is True in the NrnSimulator run method" + ) if cvode_active is None: cvode_active = self.cvode_active @@ -135,11 +144,12 @@ def run( if not cvode_active and dt is None: # use dt of simulator if self.neuron.h.dt != self.dt: raise Exception( - 'NrnSimulator: Some process has changed the ' - 'time step dt of Neuron since the creation of this ' - 'NrnSimulator object. Not sure this is intended:\n' - 'current dt: %.6g\n' - 'init dt: %.6g' % (self.neuron.h.dt, self.dt)) + "NrnSimulator: Some process has changed the " + "time step dt of Neuron since the creation of this " + "NrnSimulator object. Not sure this is intended:\n" + "current dt: %.6g\n" + "init dt: %.6g" % (self.neuron.h.dt, self.dt) + ) dt = self.dt self.neuron.h.cvode_active(1 if cvode_active else 0) @@ -148,14 +158,13 @@ def run( self.cvode_minstep = self.cvode_minstep_value if cvode_active: - logger.debug('Running Neuron simulator %.6g ms, with cvode', tstop) + logger.debug("Running Neuron simulator %.6g ms, with cvode", tstop) else: self.neuron.h.dt = dt self.neuron.h.steps_per_ms = 1.0 / dt logger.debug( - 'Running Neuron simulator %.6g ms, with dt=%r', - tstop, - dt) + "Running Neuron simulator %.6g ms, with dt=%r", tstop, dt + ) if random123_globalindex is None: random123_globalindex = self.random123_globalindex @@ -167,16 +176,15 @@ def run( try: self.neuron.h.run() except Exception as e: - raise NrnSimulatorException('Neuron simulator error', e) + raise NrnSimulatorException("Neuron simulator error", e) if self.cvode_minstep_value is not None: self.cvode_minstep = save_minstep - logger.debug('Neuron simulation finished') + logger.debug("Neuron simulation finished") class NrnSimulatorException(Exception): - """All exception generated by Neuron simulator""" def __init__(self, message, original): @@ -187,11 +195,16 @@ def __init__(self, message, original): class LFPySimulator(NrnSimulator): - """LFPy simulator""" - def __init__(self, dt=None, cvode_active=True, cvode_minstep=None, - random123_globalindex=None, mechanisms_directory=None): + def __init__( + self, + dt=None, + cvode_active=True, + cvode_minstep=None, + random123_globalindex=None, + mechanisms_directory=None, + ): """Constructor Args: @@ -213,17 +226,18 @@ def __init__(self, dt=None, cvode_active=True, cvode_minstep=None, cvode_active=cvode_active, cvode_minstep=cvode_minstep, random123_globalindex=random123_globalindex, - mechanisms_directory=mechanisms_directory + mechanisms_directory=mechanisms_directory, ) def run( - self, - lfpy_cell, - lfpy_electrode, - tstop=None, - dt=None, - cvode_active=None, - random123_globalindex=None): + self, + lfpy_cell, + lfpy_electrode, + tstop=None, + dt=None, + cvode_active=None, + random123_globalindex=None, + ): """Run protocol""" _ = self.neuron @@ -235,8 +249,9 @@ def run( if cvode_active and dt is not None: raise ValueError( - 'NrnSimulator: Impossible to combine the dt argument when ' - 'cvode_active is True in the NrnSimulator run method') + "NrnSimulator: Impossible to combine the dt argument when " + "cvode_active is True in the NrnSimulator run method" + ) if cvode_active is None: cvode_active = self.cvode_active @@ -276,7 +291,6 @@ def run( class LFPySimulatorException(Exception): - """All exception generated by LFPy simulator""" def __init__(self, message, original): @@ -287,7 +301,6 @@ def __init__(self, message, original): class ArbSimulator(object): - """Arbor simulator""" def __init__(self, dt=None, ext_catalogues=None): @@ -303,14 +316,15 @@ def __init__(self, dt=None, ext_catalogues=None): self.ext_catalogues = ext_catalogues if ext_catalogues is not None: for cat, cat_path in ext_catalogues.items(): - cat_lib = '%s-catalogue.so' % cat + cat_lib = "%s-catalogue.so" % cat cat_path = pathlib.Path(cat_path).resolve() if not os.path.exists(cat_path / cat_lib): raise ArbSimulatorException( - 'Cannot find %s at %s - first build' % (cat_lib, - cat_path) + - ' mechanism catalogue with modcc:' + - ' arbor-build-catalogue %s %s' % (cat, cat_path)) + "Cannot find %s at %s - first build" + % (cat_lib, cat_path) + + " mechanism catalogue with modcc:" + + " arbor-build-catalogue %s %s" % (cat, cat_path) + ) # TODO: add parameters for discretization def initialize(self): @@ -318,9 +332,9 @@ def initialize(self): pass def instantiate(self, morph, decor, labels): - cable_cell = arbor.cable_cell(morphology=morph, - decor=decor, - labels=labels) + cable_cell = arbor.cable_cell( + morphology=morph, decor=decor, labels=labels + ) arb_cell_model = arbor.single_cell_model(cable_cell) @@ -330,35 +344,31 @@ def instantiate(self, morph, decor, labels): # User-supplied catalogues take precedence if self.ext_catalogues is not None: for cat, cat_path in self.ext_catalogues.items(): - cat_lib = '%s-catalogue.so' % cat + cat_lib = "%s-catalogue.so" % cat cat_path = pathlib.Path(cat_path).resolve() arb_cell_model.properties.catalogue.extend( - arbor.load_catalogue(cat_path / cat_lib), - cat + "::") + arbor.load_catalogue(cat_path / cat_lib), cat + "::" + ) # Built-in catalogues are always added (could be made optional) - if self.ext_catalogues is None or \ - 'default' not in self.ext_catalogues: + if self.ext_catalogues is None or "default" not in self.ext_catalogues: arb_cell_model.properties.catalogue.extend( - arbor.default_catalogue(), "default::") + arbor.default_catalogue(), "default::" + ) - if self.ext_catalogues is None or \ - 'BBP' not in self.ext_catalogues: + if self.ext_catalogues is None or "BBP" not in self.ext_catalogues: arb_cell_model.properties.catalogue.extend( - arbor.bbp_catalogue(), "BBP::") + arbor.bbp_catalogue(), "BBP::" + ) - if self.ext_catalogues is None or \ - 'allen' not in self.ext_catalogues: + if self.ext_catalogues is None or "allen" not in self.ext_catalogues: arb_cell_model.properties.catalogue.extend( - arbor.allen_catalogue(), "allen::") + arbor.allen_catalogue(), "allen::" + ) return arb_cell_model - def run(self, - arb_cell_model, - tstop=None, - dt=None): - + def run(self, arb_cell_model, tstop=None, dt=None): dt = dt if dt is not None else self.dt if dt is not None: @@ -368,7 +378,6 @@ def run(self, class ArbSimulatorException(Exception): - """All exception generated by Arbor simulator""" def __init__(self, message): diff --git a/bluepyopt/tests/test_ephys/test_simulators.py b/bluepyopt/tests/test_ephys/test_simulators.py index f35fd006..4e4203f5 100644 --- a/bluepyopt/tests/test_ephys/test_simulators.py +++ b/bluepyopt/tests/test_ephys/test_simulators.py @@ -22,13 +22,12 @@ # pylint:disable=W0612 import os +import pathlib import types - -import pytest -import numpy - from unittest import mock + import numpy +import pytest import bluepyopt.ephys as ephys import bluepyopt.ephys.examples as examples @@ -46,7 +45,7 @@ def test_nrnsimulator_init(): def test_nrnsimulator_init_windows(): """ephys.simulators: test if NrnSimulator constructor works on Windows""" - with mock.patch('platform.system', mock.MagicMock(return_value="Windows")): + with mock.patch("platform.system", mock.MagicMock(return_value="Windows")): neuron_sim = ephys.simulators.NrnSimulator() assert isinstance(neuron_sim, ephys.simulators.NrnSimulator) assert not neuron_sim.disable_banner @@ -69,9 +68,9 @@ def test_nrnsimulator_cvode_minstep(): # Check default minstep before and after run neuron_sim = ephys.simulators.NrnSimulator(cvode_minstep=0.01) - assert neuron_sim.cvode.minstep() == 0. + assert neuron_sim.cvode.minstep() == 0.0 neuron_sim.run(tstop=10) - assert neuron_sim.cvode.minstep() == 0. + assert neuron_sim.cvode.minstep() == 0.0 # Check with that minstep is set back to the original value after run neuron_sim = ephys.simulators.NrnSimulator(cvode_minstep=0.0) @@ -82,20 +81,26 @@ def test_nrnsimulator_cvode_minstep(): # Check that the minstep is effective cvode_minstep = 0.012 - params = {'gnabar_hh': 0.10299326453483033, - 'gkbar_hh': 0.027124836082684685} + params = { + "gnabar_hh": 0.10299326453483033, + "gkbar_hh": 0.027124836082684685, + } simplecell = examples.simplecell.SimpleCell() evaluator = simplecell.cell_evaluator evaluator.cell_model.unfreeze(params.keys()) evaluator.sim = ephys.simulators.NrnSimulator(cvode_minstep=cvode_minstep) responses = evaluator.run_protocols( - protocols=evaluator.fitness_protocols.values(), - param_values=params) + protocols=evaluator.fitness_protocols.values(), param_values=params + ) ton = list(evaluator.fitness_protocols.values())[0].stimuli[0].step_delay - toff = ton + list(evaluator.fitness_protocols.values())[0].stimuli[ - 0].step_duration - t_series = numpy.array(responses['Step1.soma.v']['time']) - t_series = t_series[((ton + 1.) < t_series) & (t_series < (toff - 1.))] + toff = ( + ton + + list(evaluator.fitness_protocols.values())[0] + .stimuli[0] + .step_duration + ) + t_series = numpy.array(responses["Step1.soma.v"]["time"]) + t_series = t_series[((ton + 1.0) < t_series) & (t_series < (toff - 1.0))] min_dt = numpy.min(numpy.ediff1d(t_series)) assert (min_dt >= cvode_minstep) == 1 evaluator.cell_model.freeze(params) @@ -105,6 +110,7 @@ def test_nrnsimulator_cvode_minstep(): def test_neuron_import(): """ephys.simulators: test if bluepyopt.neuron import was successful""" from bluepyopt import ephys # NOQA + neuron_sim = ephys.simulators.NrnSimulator() assert isinstance(neuron_sim.neuron, types.ModuleType) @@ -114,6 +120,7 @@ def test_nrnsim_run_dt_exception(): """ephys.simulators: test if run return exception when dt was changed""" from bluepyopt import ephys # NOQA + neuron_sim = ephys.simulators.NrnSimulator() neuron_sim.neuron.h.dt = 1.0 pytest.raises(Exception, neuron_sim.run, 10, cvode_active=False) @@ -124,18 +131,20 @@ def test_nrnsim_run_cvodeactive_dt_exception(): """ephys.simulators: test if run return exception cvode and dt both used""" from bluepyopt import ephys # NOQA + neuron_sim = ephys.simulators.NrnSimulator() neuron_sim.neuron.h.dt = 1.0 pytest.raises(ValueError, neuron_sim.run, 10, dt=0.1, cvode_active=True) @pytest.mark.unit -@mock.patch('glob.glob') +@mock.patch.object(pathlib.Path, "glob") def test_disable_banner_exception(mock_glob): """ephys.simulators: test if disable_banner raises exception""" mock_glob.return_value = [] import warnings + with warnings.catch_warnings(record=True) as warnings_record: ephys.simulators.NrnSimulator._nrn_disable_banner() assert len(warnings_record) == 1 @@ -154,7 +163,7 @@ def test_lfpysimulator_init(): def test_lfpyimulator_init_windows(): """ephys.simulators: test if LFPySimulator constructor works on Windows""" - with mock.patch('platform.system', mock.MagicMock(return_value="Windows")): + with mock.patch("platform.system", mock.MagicMock(return_value="Windows")): empty_cell = ephys.models.LFPyCellModel(name="empty_cell") neuron_sim = ephys.simulators.LFPySimulator() assert isinstance(neuron_sim, ephys.simulators.LFPySimulator) @@ -171,6 +180,7 @@ def test_lfpyimulator_init_windows(): def test__lfpysimulator_neuron_import(): """ephys.simulators: test neuron import from LFPySimulator""" from bluepyopt import ephys # NOQA + empty_cell = ephys.models.LFPyCellModel(name="empty_cell") neuron_sim = ephys.simulators.LFPySimulator() assert isinstance(neuron_sim.neuron, types.ModuleType) @@ -181,10 +191,11 @@ def test_lfpysim_run_cvodeactive_dt_exception(): """ephys.simulators: test if LFPySimulator run returns exception""" from bluepyopt import ephys # NOQA + TESTDATA_DIR = os.path.join( - os.path.dirname(os.path.abspath(__file__)), 'testdata' + os.path.dirname(os.path.abspath(__file__)), "testdata" ) - simple_morphology_path = os.path.join(TESTDATA_DIR, 'simple.swc') + simple_morphology_path = os.path.join(TESTDATA_DIR, "simple.swc") test_morph = ephys.morphologies.NrnFileMorphology(simple_morphology_path) lfpy_cell = ephys.models.LFPyCellModel( @@ -196,9 +207,9 @@ def test_lfpysim_run_cvodeactive_dt_exception(): with pytest.raises( ValueError, match=( - 'NrnSimulator: ' - 'Impossible to combine the dt argument when ' - 'cvode_active is True in the NrnSimulator run method' + "NrnSimulator: " + "Impossible to combine the dt argument when " + "cvode_active is True in the NrnSimulator run method" ), ): neuron_sim.run( @@ -206,19 +217,20 @@ def test_lfpysim_run_cvodeactive_dt_exception(): dt=0.1, cvode_active=True, lfpy_cell=lfpy_cell.lfpy_cell, - lfpy_electrode=lfpy_cell.lfpy_electrode + lfpy_electrode=lfpy_cell.lfpy_electrode, ) lfpy_cell.destroy(sim=neuron_sim) @pytest.mark.unit -@mock.patch('glob.glob') +@mock.patch.object(pathlib.Path, "glob") def test_lfpysimulator_disable_banner_exception(mock_glob): """ephys.simulators: test LFPySimulator disable_banner raises exception""" mock_glob.return_value = [] import warnings + with warnings.catch_warnings(record=True) as warnings_record: ephys.simulators.LFPySimulator._nrn_disable_banner() assert len(warnings_record) == 1 diff --git a/versioneer.py b/versioneer.py index f11c1fee..3dd263e9 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,4 +1,3 @@ - # Version: 0.18 """The Versioneer - like a rocketeer, but for versions. @@ -277,6 +276,7 @@ """ from __future__ import print_function + try: import configparser except ImportError: @@ -308,11 +308,13 @@ def get_root(): setup_py = os.path.join(root, "setup.py") versioneer_py = os.path.join(root, "versioneer.py") if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ("Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND').") + err = ( + "Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND')." + ) raise VersioneerBadRootError(err) try: # Certain runtime workflows (setup.py install/develop in a setuptools @@ -325,8 +327,10 @@ def get_root(): me_dir = os.path.normcase(os.path.splitext(me)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir: - print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py)) + print( + "Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py) + ) except NameError: pass return root @@ -339,15 +343,16 @@ def get_config_from_root(root): # configparser.NoOptionError (if it lacks "VCS="). See the docstring at # the top of versioneer.py for instructions on writing your setup.cfg . setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() + parser = configparser.ConfigParser() with open(setup_cfg, "r") as f: - parser.readfp(f) + parser.read_file(f) VCS = parser.get("versioneer", "VCS") # mandatory def get(parser, name): if parser.has_option("versioneer", name): return parser.get("versioneer", name) return None + cfg = VersioneerConfig() cfg.VCS = VCS cfg.style = get(parser, "style") or "" @@ -372,17 +377,20 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs, method): # decorator """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f + return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command( + commands, args, cwd=None, verbose=False, hide_stderr=False, env=None +): """Call the given command(s).""" assert isinstance(commands, list) p = None @@ -390,10 +398,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + p = subprocess.Popen( + [c] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -418,7 +429,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, return stdout, p.returncode -LONG_VERSION_PY['git'] = ''' +LONG_VERSION_PY["git"] = ''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -1011,7 +1022,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1020,7 +1031,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = set([r for r in refs if re.search(r"\d", r)]) if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1028,19 +1039,26 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") @@ -1055,8 +1073,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + out, rc = run_command( + GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True + ) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1064,10 +1083,19 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = run_command( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + "%s*" % tag_prefix, + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -1090,17 +1118,18 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = ( + "unable to parse git-describe output: '%s'" % describe_out + ) return pieces # tag @@ -1109,10 +1138,12 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -1123,13 +1154,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command( + GITS, ["rev-list", "HEAD", "--count"], cwd=root + ) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ + 0 + ].strip() pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -1185,16 +1218,22 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): for i in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } else: rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -1223,11 +1262,17 @@ def versions_from_file(filename): contents = f.read() except EnvironmentError: raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, + re.M | re.S, + ) if not mo: - mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\r\n(.*)''' # END VERSION_JSON", + contents, + re.M | re.S, + ) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) @@ -1236,8 +1281,9 @@ def versions_from_file(filename): def write_to_version_file(filename, versions): """Write the given version number to the given _version.py file.""" os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, - indent=1, separators=(",", ": ")) + contents = json.dumps( + versions, sort_keys=True, indent=1, separators=(",", ": ") + ) with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) @@ -1269,8 +1315,7 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1291,6 +1336,7 @@ def render_pep440_pre(pieces): rendered = "0.post.dev%d" % pieces["distance"] return rendered + def render_pep440_minor(pieces): # TAG[.DISTANCE] . No -dirty @@ -1307,7 +1353,6 @@ def render_pep440_minor(pieces): return rendered - def render_pep440_post(pieces): """TAG[.postDISTANCE[.dev0]+gHEX] . @@ -1400,11 +1445,13 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -1426,9 +1473,13 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } class VersioneerBadRootError(Exception): @@ -1451,8 +1502,9 @@ def get_versions(verbose=False): handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS verbose = verbose or cfg.verbose - assert cfg.versionfile_source is not None, \ - "please set versioneer.versionfile_source" + assert ( + cfg.versionfile_source is not None + ), "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" versionfile_abs = os.path.join(root, cfg.versionfile_source) @@ -1506,9 +1558,13 @@ def get_versions(verbose=False): if verbose: print("unable to compute version") - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } def get_version(): @@ -1557,6 +1613,7 @@ def run(self): print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version # we override "build_py" in both distutils and setuptools @@ -1589,10 +1646,12 @@ def run(self): # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) + target_versionfile = os.path.join( + self.build_lib, cfg.versionfile_build + ) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py if "cx_Freeze" in sys.modules: # cx_freeze enabled? @@ -1617,17 +1676,21 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["build_exe"] = cmd_build_exe del cmds["build_py"] - if 'py2exe' in sys.modules: # py2exe enabled? + if "py2exe" in sys.modules: # py2exe enabled? try: from py2exe.distutils_buildexe import py2exe as _py2exe # py3 except ImportError: @@ -1646,13 +1709,17 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["py2exe"] = cmd_py2exe # we override different "sdist" commands for both environments @@ -1679,8 +1746,10 @@ def make_release_tree(self, base_dir, files): # updated value target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, - self._versioneer_generated_versions) + write_to_version_file( + target_versionfile, self._versioneer_generated_versions + ) + cmds["sdist"] = cmd_sdist return cmds @@ -1735,11 +1804,15 @@ def do_setup(): root = get_root() try: cfg = get_config_from_root(root) - except (EnvironmentError, configparser.NoSectionError, - configparser.NoOptionError) as e: + except ( + EnvironmentError, + configparser.NoSectionError, + configparser.NoOptionError, + ) as e: if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", - file=sys.stderr) + print( + "Adding sample versioneer config to setup.cfg", file=sys.stderr + ) with open(os.path.join(root, "setup.cfg"), "a") as f: f.write(SAMPLE_CONFIG) print(CONFIG_ERROR, file=sys.stderr) @@ -1748,15 +1821,18 @@ def do_setup(): print(" creating %s" % cfg.versionfile_source) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), - "__init__.py") + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") if os.path.exists(ipy): try: with open(ipy, "r") as f: @@ -1798,8 +1874,10 @@ def do_setup(): else: print(" 'versioneer.py' already in MANIFEST.in") if cfg.versionfile_source not in simple_includes: - print(" appending versionfile_source ('%s') to MANIFEST.in" % - cfg.versionfile_source) + print( + " appending versionfile_source ('%s') to MANIFEST.in" + % cfg.versionfile_source + ) with open(manifest_in, "a") as f: f.write("include %s\n" % cfg.versionfile_source) else: