From 35150e9bb1d6796fb6d017ecec43d90c2677b2fa Mon Sep 17 00:00:00 2001 From: ehennestad Date: Sat, 23 Nov 2024 20:14:07 +0100 Subject: [PATCH] Add functionality for installing extensions --- +matnwb/+extension/dispExtensionInfo.m | 15 +++ +matnwb/+extension/installAll.m | 6 + +matnwb/+extension/installExtension.m | 126 ++++++++++++++++++ +matnwb/+extension/listExtensions.m | 53 ++++++++ .gitignore | 11 ++ nwbInstallExtension.m | 79 +++++++++++ .../nwbInstallExtension.txt | 35 +++++ .../matnwb_createNwbInstallExtension.m | 28 ++++ 8 files changed, 353 insertions(+) create mode 100644 +matnwb/+extension/dispExtensionInfo.m create mode 100644 +matnwb/+extension/installAll.m create mode 100644 +matnwb/+extension/installExtension.m create mode 100644 +matnwb/+extension/listExtensions.m create mode 100644 nwbInstallExtension.m create mode 100644 resources/function_templates/nwbInstallExtension.txt create mode 100644 tools/maintenance/matnwb_createNwbInstallExtension.m diff --git a/+matnwb/+extension/dispExtensionInfo.m b/+matnwb/+extension/dispExtensionInfo.m new file mode 100644 index 00000000..337c3523 --- /dev/null +++ b/+matnwb/+extension/dispExtensionInfo.m @@ -0,0 +1,15 @@ +function dispExtensionInfo(extensionName) + arguments + extensionName (1,1) string + end + + T = matnwb.extension.listExtensions(); + isMatch = T.name == extensionName; + extensionList = join( compose(" %s", [T.name]), newline ); + assert( ... + any(isMatch), ... + 'NWB:DisplayExtensionMetadata:ExtensionNotFound', ... + 'Extension "%s" was not found in the extension catalog:\n%s', extensionName, extensionList) + metadata = table2struct(T(isMatch, :)); + disp(metadata) +end diff --git a/+matnwb/+extension/installAll.m b/+matnwb/+extension/installAll.m new file mode 100644 index 00000000..a4fff669 --- /dev/null +++ b/+matnwb/+extension/installAll.m @@ -0,0 +1,6 @@ +function installAll() + T = matnwb.extension.listExtensions(); + for i = 1:height(T) + matnwb.extension.installExtension( T.name(i) ) + end +end diff --git a/+matnwb/+extension/installExtension.m b/+matnwb/+extension/installExtension.m new file mode 100644 index 00000000..48f30044 --- /dev/null +++ b/+matnwb/+extension/installExtension.m @@ -0,0 +1,126 @@ +function installExtension(extensionName) + arguments + extensionName (1,1) string + end + + repoTargetFolder = fullfile(userpath, "NWB-Extension-Source"); + if ~isfolder(repoTargetFolder); mkdir(repoTargetFolder); end + + T = matnwb.extension.listExtensions(); + isMatch = T.name == extensionName; + + extensionList = join( compose(" %s", [T.name]), newline ); + assert( ... + any(isMatch), ... + 'NWB:InstallExtension:ExtensionNotFound', ... + 'Extension "%s" was not found in the extension catalog:\n', extensionList) + + defaultBranchNames = ["main", "master"]; + + wasDownloaded = false; + for i = 1:2 + try + repositoryUrl = T{isMatch, 'src'}; + if endsWith(repositoryUrl, '/') + repositoryUrl = extractBefore(repositoryUrl, strlength(repositoryUrl)); + end + if contains(repositoryUrl, 'github.com') + downloadUrl = sprintf( '%s/archive/refs/heads/%s.zip', repositoryUrl, defaultBranchNames(i) ); + + elseif contains(repositoryUrl, 'gitlab.com') + repoPathSegments = strsplit(repositoryUrl, '/'); + repoName = repoPathSegments{end}; + downloadUrl = sprintf( '%s/-/archive/%s/%s-%s.zip', ... + repositoryUrl, defaultBranchNames(i), repoName, defaultBranchNames(i)); + else + error('NWB:InstallExtension:UnknownRepository', ... + 'Extension "%s" is located in an unsupported repository / source location. Please create an issue on matnwb''s github page', extensionName) + end + repoTargetFolder = downloadZippedRepo(downloadUrl, repoTargetFolder, true, true); + wasDownloaded = true; + break + catch ME + if strcmp(ME.identifier, 'MATLAB:webservices:HTTP404StatusCodeError') + continue + else + rethrow(ME) + end + end + end + if ~wasDownloaded + error('NWB:InstallExtension:DownloadFailed', ... + 'Failed to download spec for extension "%s"', extensionName) + end + L = dir(fullfile(repoTargetFolder, 'spec', '*namespace.yaml')); + assert(... + ~isempty(L), ... + 'NWB:InstallExtension:NamespaceNotFound', ... + 'No namespace file was found for extension "%s"', extension.Identifier ... + ) + assert(... + numel(L)==1, ... + 'NWB:InstallExtension:MultipleNamespacesFound', ... + 'More than one namespace file was found for extension "%s"', extension.Identifier ... + ) + generateExtension( fullfile(L.folder, L.name) ); +end + +function repoFolder = downloadZippedRepo(githubUrl, targetFolder, updateFlag, throwErrorIfFails) +%downloadZippedRepo Download zipped repo + + if nargin < 3; updateFlag = false; end + if nargin < 4; throwErrorIfFails = false; end + + if isa(updateFlag, 'char') && strcmp(updateFlag, 'update') + updateFlag = true; + end + + % Create a temporary path for storing the downloaded file. + [~, ~, fileType] = fileparts(githubUrl); + tempFilepath = [tempname, fileType]; + + % Download the file containing the addon toolbox + try + tempFilepath = websave(tempFilepath, githubUrl); + fileCleanupObj = onCleanup( @(fname) delete(tempFilepath) ); + catch ME + if throwErrorIfFails + rethrow(ME) + end + end + + unzippedFiles = unzip(tempFilepath, tempdir); + unzippedFolder = unzippedFiles{1}; + if endsWith(unzippedFolder, filesep) + unzippedFolder = unzippedFolder(1:end-1); + end + + [~, repoFolderName] = fileparts(unzippedFolder); + targetFolder = fullfile(targetFolder, repoFolderName); + + if updateFlag && isfolder(targetFolder) + + % Delete current version + if isfolder(targetFolder) + if contains(path, fullfile(targetFolder, filesep)) + pathList = strsplit(path, pathsep); + pathList_ = pathList(startsWith(pathList, fullfile(targetFolder, filesep))); + rmpath(strjoin(pathList_, pathsep)) + end + try + rmdir(targetFolder, 's') + catch + warning('Could not remove old installation... Please report') + end + end + else + % pass + end + + movefile(unzippedFolder, targetFolder); + + % Delete the temp zip file + clear fileCleanupObj + + repoFolder = targetFolder; +end diff --git a/+matnwb/+extension/listExtensions.m b/+matnwb/+extension/listExtensions.m new file mode 100644 index 00000000..e8ae59c1 --- /dev/null +++ b/+matnwb/+extension/listExtensions.m @@ -0,0 +1,53 @@ +function extensionTable = listExtensions(options) + arguments + options.Refresh (1,1) logical = false + end + + persistent extensionRecords + + if isempty(extensionRecords) || options.Refresh + catalogUrl = "https://raw.githubusercontent.com/nwb-extensions/nwb-extensions.github.io/refs/heads/main/data/records.json"; + extensionRecords = jsondecode(webread(catalogUrl)); + extensionRecords = consolidateStruct(extensionRecords); + + extensionRecords = struct2table(extensionRecords); + + fieldsKeep = ["name", "version", "last_updated", "src", "license", "maintainers", "readme"]; + extensionRecords = extensionRecords(:, fieldsKeep); + + for name = fieldsKeep + if ischar(extensionRecords.(name){1}) + extensionRecords.(name) = string(extensionRecords.(name)); + end + end + end + extensionTable = extensionRecords; +end + +function structArray = consolidateStruct(S) + % Get all field names of S + mainFields = fieldnames(S); + + % Initialize an empty struct array + structArray = struct(); + + % Iterate over each field of S + for i = 1:numel(mainFields) + subStruct = S.(mainFields{i}); % Extract sub-struct + + % Add all fields of the sub-struct to the struct array + fields = fieldnames(subStruct); + for j = 1:numel(fields) + structArray(i).(fields{j}) = subStruct.(fields{j}); + end + end + + % Ensure consistency by filling missing fields with [] + allFields = unique([fieldnames(structArray)]); + for i = 1:numel(structArray) + missingFields = setdiff(allFields, fieldnames(structArray(i))); + for j = 1:numel(missingFields) + structArray(i).(missingFields{j}) = []; + end + end +end diff --git a/.gitignore b/.gitignore index 988501fa..c4dc1454 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,14 @@ workspace/ *.swp .DS_Store +tests/env.mat + +# Ignore everything in the +types/ folder ++types/* + +# Explicitly include these subdirectories +!+types/+core/ +!+types/+hdmf_common/ +!+types/+hdmf_experimental/ +!+types/+untyped/ +!+types/+util/ + diff --git a/nwbInstallExtension.m b/nwbInstallExtension.m new file mode 100644 index 00000000..1c7d8516 --- /dev/null +++ b/nwbInstallExtension.m @@ -0,0 +1,79 @@ +function nwbInstallExtension(extensionName) +% nwbInstallExtension - Installs a specified NWB extension. +% +% nwbInstallExtension(extensionName) installs a Neurodata Without Borders +% (NWB) extension to extend the functionality of the core NWB schemas. +% extensionName is a scalar string or a string array, containing the name +% of one or more extensions from the Neurodata Extesions Catalog +% +% Usage: +% nwbInstallExtension(extensionName) +% +% Valid Extension Names: +% - "ndx-miniscope" +% - "ndx-simulation-output" +% - "ndx-ecog" +% - "ndx-fret" +% - "ndx-icephys-meta" +% - "ndx-events" +% - "ndx-nirs" +% - "ndx-hierarchical-behavioral-data" +% - "ndx-sound" +% - "ndx-extract" +% - "ndx-photometry" +% - "ndx-acquisition-module" +% - "ndx-odor-metadata" +% - "ndx-whisk" +% - "ndx-ecg" +% - "ndx-franklab-novela" +% - "ndx-photostim" +% - "ndx-multichannel-volume" +% - "ndx-depth-moseq" +% - "ndx-probeinterface" +% - "ndx-dbs" +% - "ndx-hed" +% - "ndx-ophys-devices" +% +% Example: +% % Install the "ndx-miniscope" extension +% nwbInstallExtension("ndx-miniscope") +% +% See also: +% matnwb.extension.listExtensions, matnwb.extension.installExtension + + arguments + extensionName (1,:) string {mustBeMember(extensionName, [... + "ndx-miniscope", ... + "ndx-simulation-output", ... + "ndx-ecog", ... + "ndx-fret", ... + "ndx-icephys-meta", ... + "ndx-events", ... + "ndx-nirs", ... + "ndx-hierarchical-behavioral-data", ... + "ndx-sound", ... + "ndx-extract", ... + "ndx-photometry", ... + "ndx-acquisition-module", ... + "ndx-odor-metadata", ... + "ndx-whisk", ... + "ndx-ecg", ... + "ndx-franklab-novela", ... + "ndx-photostim", ... + "ndx-multichannel-volume", ... + "ndx-depth-moseq", ... + "ndx-probeinterface", ... + "ndx-dbs", ... + "ndx-hed", ... + "ndx-ophys-devices" ... + ] ... + )} = [] + end + if isempty(extensionName) + T = matnwb.extension.listExtensions(); + extensionList = join( compose(" %s", [T.name]), newline ); + error("Please specify the name of an extension. Available extensions:\n\n%s\n", extensionList) + else + matnwb.extension.installExtension(extensionName) + end +end diff --git a/resources/function_templates/nwbInstallExtension.txt b/resources/function_templates/nwbInstallExtension.txt new file mode 100644 index 00000000..2a7f22a8 --- /dev/null +++ b/resources/function_templates/nwbInstallExtension.txt @@ -0,0 +1,35 @@ +function nwbInstallExtension(extensionName) +% nwbInstallExtension - Installs a specified NWB extension. +% +% nwbInstallExtension(extensionName) installs a Neurodata Without Borders +% (NWB) extension to extend the functionality of the core NWB schemas. +% extensionName is a scalar string or a string array, containing the name +% of one or more extensions from the Neurodata Extesions Catalog +% +% Usage: +% nwbInstallExtension(extensionName) +% +% Valid Extension Names: +{{extensionNamesDoc}} +% +% Example: +% % Install the "ndx-miniscope" extension +% nwbInstallExtension("ndx-miniscope") +% +% See also: +% matnwb.extension.listExtensions, matnwb.extension.installExtension + + arguments + extensionName (1,:) string {mustBeMember(extensionName, [... +{{extensionNames}} ... + ] ... + )} = [] + end + if isempty(extensionName) + T = matnwb.extension.listExtensions(); + extensionList = join( compose(" %s", [T.name]), newline ); + error("Please specify the name of an extension. Available extensions:\n\n%s\n", extensionList) + else + matnwb.extension.installExtension(extensionName) + end +end diff --git a/tools/maintenance/matnwb_createNwbInstallExtension.m b/tools/maintenance/matnwb_createNwbInstallExtension.m new file mode 100644 index 00000000..f9af731d --- /dev/null +++ b/tools/maintenance/matnwb_createNwbInstallExtension.m @@ -0,0 +1,28 @@ +function matnwb_createNwbInstallExtension() +% matnwb_createNwbInstallExtension - Create nwbInstallExtension from template +% +% This function can be run to update the list of available extension +% names in the function's arguments block based on the neurodata +% extensions catalog + + matnwbRootDir = misc.getMatnwbDir(); + fcnTemplate = fileread(fullfile(matnwbRootDir, ... + 'resources', 'function_templates', 'nwbInstallExtension.txt')); + + extensionTable = matnwb.extension.listExtensions(); + extensionNames = extensionTable.name; + + indentStr = repmat(' ', 1, 12); + extensionNamesStr = compose("%s""%s""", indentStr, extensionNames); + extensionNamesStr = strjoin(extensionNamesStr, ", ..." + newline); + fcnStr = replace(fcnTemplate, "{{extensionNames}}", extensionNamesStr); + + extensionNamesStr = compose("%% - ""%s""", extensionNames); + extensionNamesStr = strjoin(extensionNamesStr, newline); + fcnStr = replace(fcnStr, "{{extensionNamesDoc}}", extensionNamesStr); + + + fid = fopen(fullfile(matnwbRootDir, 'nwbInstallExtension.m'), "wt"); + fwrite(fid, fcnStr); + fclose(fid); +end