From 8d5e5133ad803b72a8275f025fe641c7699e9ad3 Mon Sep 17 00:00:00 2001 From: Nenad Amodaj Date: Tue, 17 Dec 2024 16:31:49 -0800 Subject: [PATCH] device adapter bug fixes and update to the latest API changes --- DeviceAdapters/go2scope/AcqZarrStorage.cpp | 10 + DeviceAdapters/go2scope/AcqZarrStorage.h | 4 +- DeviceAdapters/go2scope/CFileUtil.cpp | 50 ++- DeviceAdapters/go2scope/G2SBigTiffDataset.cpp | 185 +++++++- DeviceAdapters/go2scope/G2SBigTiffDataset.h | 43 +- DeviceAdapters/go2scope/G2SBigTiffStorage.cpp | 422 ++++++++++++------ DeviceAdapters/go2scope/G2SBigTiffStorage.h | 10 +- DeviceAdapters/go2scope/G2SFileUtil.h | 5 +- DeviceAdapters/go2scope/G2SStorage.h | 62 +-- 9 files changed, 594 insertions(+), 197 deletions(-) diff --git a/DeviceAdapters/go2scope/AcqZarrStorage.cpp b/DeviceAdapters/go2scope/AcqZarrStorage.cpp index 9ac7de14b..31a9a1e5e 100644 --- a/DeviceAdapters/go2scope/AcqZarrStorage.cpp +++ b/DeviceAdapters/go2scope/AcqZarrStorage.cpp @@ -506,6 +506,11 @@ int AcqZarrStorage::GetCoordinate(const char* handle, int dimension, int coordin return DEVICE_NOT_YET_IMPLEMENTED; } +int AcqZarrStorage::GetImageCount(const char* handle, int& imgcount) +{ + return DEVICE_NOT_YET_IMPLEMENTED; +} + bool AcqZarrStorage::IsOpen(const char* handle) { if (streamHandle.compare(handle) != 0) @@ -515,6 +520,11 @@ bool AcqZarrStorage::IsOpen(const char* handle) return true; } +bool AcqZarrStorage::IsReadOnly(const char* handle) +{ + return false; +} + int AcqZarrStorage::GetPath(const char* handle, char* path, int maxPathLength) { return 0; diff --git a/DeviceAdapters/go2scope/AcqZarrStorage.h b/DeviceAdapters/go2scope/AcqZarrStorage.h index 036624842..cd054e5a5 100644 --- a/DeviceAdapters/go2scope/AcqZarrStorage.h +++ b/DeviceAdapters/go2scope/AcqZarrStorage.h @@ -65,8 +65,10 @@ class AcqZarrStorage : public CStorageBase int GetNumberOfDimensions(const char* handle, int& numDimensions); int GetDimension(const char* handle, int dimension, char* name, int nameLength, char* meaning, int meaningLength); int GetCoordinate(const char* handle, int dimension, int coordinate, char* name, int nameLength); + int GetImageCount(const char* handle, int& imgcnt); bool IsOpen(const char* handle); - int GetPath(const char* handle, char* path, int maxPathLength); + bool IsReadOnly(const char* handle); + int GetPath(const char* handle, char* path, int maxPathLength); // action interface diff --git a/DeviceAdapters/go2scope/CFileUtil.cpp b/DeviceAdapters/go2scope/CFileUtil.cpp index c9bc0b10f..c69e4e279 100644 --- a/DeviceAdapters/go2scope/CFileUtil.cpp +++ b/DeviceAdapters/go2scope/CFileUtil.cpp @@ -62,4 +62,52 @@ std::uint64_t readInt(const unsigned char* buff, std::uint8_t len) noexcept ret |= xval; } return ret; -} \ No newline at end of file +} + +/** + * Split CSV line into tokens + * @param line CSV line + * @return Tokens list + */ +std::vector splitLineCSV(const std::string& line) noexcept +{ + std::vector ret; + if(line.empty()) + return ret; + + std::string curr = ""; + bool qopen = false; + int qcnt = 0; + for(char c : line) + { + bool endswithQ = curr.size() >= 1 && curr[curr.size() - 1] == '\"'; + bool endswithS = curr.size() >= 1 && curr[curr.size() - 1] == ' '; + bool endswithEQ = curr.size() >= 2 && curr[curr.size() - 1] == '\"' && curr[curr.size() - 1] == '\\'; + if(c == ',' && (!qopen || (qcnt % 2 == 0 && (endswithQ || endswithS) && !endswithEQ))) + { + if(curr.size() >= 2 && curr[0] == '\"' && curr[curr.size() - 1] == '\"') + curr = curr.substr(1, curr.size() - 2); + ret.push_back(curr); + curr = ""; + qcnt = 0; + qopen = false; + } + else if(c == '"') + { + if(qcnt == 0) + qopen = true; + qcnt++; + //if(qcnt > 1 && qcnt % 2 == 1) + curr += "\""; + } + else + curr += c; + } + if(!curr.empty()) + { + if(curr.size() >= 2 && curr[0] == '\"' && curr[curr.size() - 1] == '\"') + curr = curr.substr(1, curr.size() - 2); + ret.push_back(curr); + } + return ret; +} diff --git a/DeviceAdapters/go2scope/G2SBigTiffDataset.cpp b/DeviceAdapters/go2scope/G2SBigTiffDataset.cpp index f0d0c6e0a..fc9f81b59 100644 --- a/DeviceAdapters/go2scope/G2SBigTiffDataset.cpp +++ b/DeviceAdapters/go2scope/G2SBigTiffDataset.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include "G2SBigTiffDataset.h" #ifdef _WIN32 #include @@ -40,6 +41,7 @@ #define G2SFOLDER_EXT ".g2s" #define G2SFILE_EXT ".g2s.tif" +#define G2SAXISINFO_FILE "axisinfo.txt" /** * Class constructor @@ -83,6 +85,7 @@ void G2SBigTiffDataset::create(const std::string& path, bool dio, bool fbig, std directIo = dio; writemode = true; chunksize = chunksz; + bigTiff = fbig; // Extract dataset name std::filesystem::path basepath = std::filesystem::u8path(path); @@ -109,7 +112,7 @@ void G2SBigTiffDataset::create(const std::string& path, bool dio, bool fbig, std std::filesystem::create_directories(fp.parent_path(), ec); if(ec.value() != 0) throw std::runtime_error("Unable to create a file stream. Directory tree creation failed"); - activechunk = std::make_shared(fp.u8string(), directIo); + activechunk = std::make_shared(fp.u8string(), directIo, bigTiff); if(!activechunk) throw std::runtime_error("Unable to create a file stream. Data chunk allocation failed"); activechunk->open(true); @@ -199,6 +202,8 @@ void G2SBigTiffDataset::load(const std::string& path, bool dio) activechunk->open(false); activechunk->parse(datasetuid, shape, chunksize, metadata, bitdepth); imgcounter += activechunk->getImageCount(); + resetAxisInfo(); + parseAxisInfo(); // Validate dataset parameters if(activechunk->getChunkIndex() != 0) @@ -241,6 +246,7 @@ void G2SBigTiffDataset::close() noexcept { if(writemode && datachunks.size() == 1 && datachunks[0]->isOpen()) datachunks[0]->appendMetadata(metadata); + writeAxisInfo(); for(const auto& fx : datachunks) fx->close(); imgcounter = 0; @@ -250,6 +256,7 @@ void G2SBigTiffDataset::close() noexcept shape.clear(); datachunks.clear(); activechunk.reset(); + axisinfo.clear(); } /** @@ -277,6 +284,11 @@ void G2SBigTiffDataset::setShape(const std::vector& dims) return; } shape = dims; + + // Resize axis descriptors vector + resetAxisInfo(); + + // Write shape info if(activechunk) activechunk->writeShapeInfo(shape, chunksize); } @@ -306,6 +318,11 @@ void G2SBigTiffDataset::setShape(std::initializer_list dims) return; } shape = dims; + + // Resize axis descriptors vector + resetAxisInfo(); + + // Write shape info if(activechunk) activechunk->writeShapeInfo(shape, chunksize); } @@ -389,6 +406,41 @@ void G2SBigTiffDataset::setUID(const std::string& val) activechunk->writeDatasetUid(datasetuid); } +/** + * Configure axis info + * If axis index is invalid this method will have no effect + * @param dim Axis index + * @param name Axis name + * @param desc Axis description + */ +void G2SBigTiffDataset::configureAxis(int dim, const std::string& name, const std::string& desc) noexcept +{ + if(!writemode) + return; + if(dim < 0 || (std::size_t)dim >= axisinfo.size()) + return; + axisinfo[dim].Name = name; + axisinfo[dim].Description = desc; +} + +/** + * Configure axis coordinate info + * If axis / coordinate index is invalid this method will have no effect + * @param dim Axis index + * @param coord Axis coordinate index + * @param desc Coordinate description + */ +void G2SBigTiffDataset::configureCoordinate(int dim, int coord, const std::string& desc) noexcept +{ + if(!writemode) + return; + if(dim < 0 || (std::size_t)dim >= axisinfo.size() - 2) + return; + if(coord < 0 || (std::size_t)coord >= axisinfo[dim].Coordinates.size()) + return; + axisinfo[dim].Coordinates[coord] = desc; +} + /** * Get dataset metadata * If metadata is specified value will be returned from cache, otherwise it will be read from a file stream @@ -512,6 +564,27 @@ std::vector G2SBigTiffDataset::getImage(const std::vectorgetImage(); } +/** + * Check if image for the specified coordinates is already set + * @param coordinates Coordinates list + * @param numCoordinates Coordinates count + * @return Does image at the specified coordinates exists + */ +bool G2SBigTiffDataset::isCoordinateSet(int coordinates[], int numCoordinates) const noexcept +{ + std::uint32_t imgind = 0; + for(int i = 0; i < numCoordinates; i++) + { + if(i >= shape.size() - 2) + break; + std::uint32_t sum = 1; + for(int j = i + 1; j < shape.size() - 2; j++) + sum *= shape[j]; + imgind += sum * coordinates[i]; + } + return imgind < imgcounter; +} + /** * Change active data chunk * This method is used only for reading data @@ -709,3 +782,113 @@ void G2SBigTiffDataset::calcImageIndex(const std::vector& coord, } imgind = ind; } + +/** + * Reset axis info structure + */ +void G2SBigTiffDataset::resetAxisInfo() noexcept +{ + axisinfo.clear(); + axisinfo.resize(shape.size()); + for(std::size_t i = 0; i < shape.size() - 2; i++) + axisinfo[i].setSize((std::size_t)shape[i]); +} + +/** + * Parse axis info + * Axis info is expected to be stored in a file: 'axisinfo.txt' + * @throws std::runtime_error + */ +void G2SBigTiffDataset::parseAxisInfo() +{ + auto fpath = std::filesystem::u8path(dspath) / G2SAXISINFO_FILE; + if(!std::filesystem::exists(fpath) || axisinfo.empty()) + return; + + // Load file content + std::fstream fs(fpath.u8string(), std::ios::in); + if(!fs.is_open()) + throw std::runtime_error("Unable to load axis info. Opening axis info file failed"); + + int ind = 0; + std::string line = ""; + while(std::getline(fs, line)) + { + if(line.empty()) + continue; + if((std::size_t)ind >= axisinfo.size()) + throw std::runtime_error("Unable to load axis info. Invalid axis info data"); + std::vector tokens = splitLineCSV(line); + if(tokens.size() < 3) + throw std::runtime_error("Unable to load axis info. Corrupted axis info, axis: " + std::to_string(ind)); + std::uint32_t axisdim = 0; + try + { + axisdim = std::stoul(tokens[2]); + } + catch(std::exception& e) + { + throw std::runtime_error("Unable to load axis info. " + std::string(e.what()) + ", axis: " + std::to_string(ind)); + } + if(axisinfo[ind].Coordinates.size() != axisdim || tokens.size() != (std::size_t)(3 + axisdim)) + throw std::runtime_error("Unable to load axis info. Axis size missmatch, axis: " + std::to_string(ind)); + axisinfo[ind].Name = tokens[0]; + axisinfo[ind].Description = tokens[1]; + for(std::uint32_t i = 0; i < axisdim; i++) + axisinfo[ind].Coordinates[i] = tokens[3 + i]; + ind++; + } +} + +/** + * Write axis info + * Axis info will be stored in a separate file: 'axisinfo.txt' + * If no axis info is defined file won't be created + * Axis info will be stored in plain text, CSV-like format + */ +void G2SBigTiffDataset::writeAxisInfo() const noexcept +{ + // Check if axis info is set, and that we are in WRITE mode + if(axisinfo.empty() || !writemode) + return; + + bool hasinfo = false; + for(const auto& dinf : axisinfo) + { + if(!dinf.Name.empty() || !dinf.Description.empty()) + { + hasinfo = true; + break; + } + for(const auto& cinf : dinf.Coordinates) + { + if(!cinf.empty()) + { + hasinfo = true; + break; + } + } + if(hasinfo) + break; + } + + // If axis info is empty, but the file exists -> delete it before exiting + auto fpath = std::filesystem::u8path(dspath) / G2SAXISINFO_FILE; + if(!hasinfo) + { + return; + } + + // Write data to a file + std::fstream fs(fpath.u8string(), std::ios::out | std::ios::trunc); + if(!fs.is_open()) + return; + for(const auto& dinf : axisinfo) + { + fs << "\"" << dinf.Name << "\",\"" << dinf.Description << "\"," << dinf.Coordinates.size(); + for(const auto& cinf : dinf.Coordinates) + fs << ",\"" << cinf << "\""; + fs << std::endl; + } + fs.close(); +} diff --git a/DeviceAdapters/go2scope/G2SBigTiffDataset.h b/DeviceAdapters/go2scope/G2SBigTiffDataset.h index e85d489b6..b3eb3d71e 100644 --- a/DeviceAdapters/go2scope/G2SBigTiffDataset.h +++ b/DeviceAdapters/go2scope/G2SBigTiffDataset.h @@ -52,6 +52,40 @@ class G2SBigTiffDataset G2SBigTiffDataset(const G2SBigTiffDataset& src) noexcept = default; ~G2SBigTiffDataset() noexcept { close(); } +public: + //============================================================================================================================ + // Internal data types + //============================================================================================================================ + /** + * Dataset dimension descriptor + * @author Miloš Jovanović + * @version 1.0 + */ + struct G2SDimensionInfo + { + /** + * Default initializer + * @param vname Axis name + * @param ndim Axis size + */ + G2SDimensionInfo(int ndim = 0) noexcept : Name(""), Description(""), Coordinates(ndim) { } + + /** + * Set dimensions size + * @param sz Number of axis coordinates + */ + void setSize(std::size_t sz) noexcept { Coordinates.resize(sz); } + /** + * Get dimension size + * @return Number of axis coordinates + */ + std::size_t getSize() const noexcept { return Coordinates.size(); } + + std::string Name; ///< Axis name + std::string Description; ///< Axis description + std::vector Coordinates; ///< Axis coordinates + }; + public: //============================================================================================================================ // Public interface @@ -76,6 +110,9 @@ class G2SBigTiffDataset std::string getMetadata() const noexcept; void setUID(const std::string& val); std::string getUID() const noexcept { return datasetuid; } + void configureAxis(int dim, const std::string& name, const std::string& desc) noexcept; + void configureCoordinate(int dim, int coord, const std::string& desc) noexcept; + const G2SDimensionInfo& getAxisInfo(std::uint32_t dim) const { if(dim >= axisinfo.size()) throw std::runtime_error("Unable to obtain axis info. Invalid axis index"); return axisinfo[dim]; } std::string getImageMetadata(const std::vector& coord = {}); void addImage(const std::vector& buff, const std::string& meta = "") { addImage(&buff[0], buff.size(), meta); } void addImage(const unsigned char* buff, std::size_t len, const std::string& meta = ""); @@ -88,9 +125,9 @@ class G2SBigTiffDataset bool isBigTIFF() const noexcept { return bigTiff; } bool isInWriteMode() const noexcept { return writemode; } bool isInReadMode() const noexcept { return !writemode; } + bool isCoordinateSet(int coordinates[], int numCoordinates) const noexcept; bool isOpen() const noexcept { return !datachunks.empty() && activechunk; } - private: //============================================================================================================================ // Internal methods @@ -101,6 +138,9 @@ class G2SBigTiffDataset void advanceImage(); std::uint32_t getChunkImageCount() const noexcept; void calcImageIndex(const std::vector& coord, std::uint32_t& chunkind, std::uint32_t& imgind) const; + void resetAxisInfo() noexcept; + void parseAxisInfo(); + void writeAxisInfo() const noexcept; private: //============================================================================================================================ @@ -113,6 +153,7 @@ class G2SBigTiffDataset std::vector datachunks; ///< Data chunks / File stream descriptors G2SFileStreamHandle activechunk; ///< Active data chunk std::vector metadata; ///< Dataset metdata (cache) + std::vector axisinfo; ///< Dataset axis descriptors std::uint32_t imgcounter; ///< Image counter std::uint32_t flushcnt; ///< Image flush cycles std::uint32_t chunksize; ///< Chunk size diff --git a/DeviceAdapters/go2scope/G2SBigTiffStorage.cpp b/DeviceAdapters/go2scope/G2SBigTiffStorage.cpp index 48d894b61..07176df25 100644 --- a/DeviceAdapters/go2scope/G2SBigTiffStorage.cpp +++ b/DeviceAdapters/go2scope/G2SBigTiffStorage.cpp @@ -51,6 +51,12 @@ G2SBigTiffStorage::G2SBigTiffStorage() : initialized(false) SetErrorText(ERR_TIFF_INVALID_PIXEL_TYPE, "Invalid or unsupported pixel type."); SetErrorText(ERR_TIFF_OPEN_FAILED, "Failed opening TIF file."); SetErrorText(ERR_TIFF_HANDLE_INVALID, "Dataset handle is not valid."); + SetErrorText(ERR_TIFF_STRING_TOO_LONG, "Requested string is too long for the provided buffer."); + SetErrorText(ERR_TIFF_INVALID_COORDINATE, "Dataset coordinates are valid."); + SetErrorText(ERR_TIFF_DATASET_CLOSED, "Operation unavailable - dataset is closed."); + SetErrorText(ERR_TIFF_DATASET_READONLY, "Operation unavailable - dataset is read-only."); + SetErrorText(ERR_TIFF_DELETE_FAILED, "File / folder delete failed."); + SetErrorText(ERR_TIFF_ALLOCATION_FAILED, "Dataset memory allocation failed."); // create pre-initialization properties // ------------------------------------ @@ -140,15 +146,11 @@ int G2SBigTiffStorage::Shutdown() */ int G2SBigTiffStorage::Create(const char* path, const char* name, int numberOfDimensions, const int shape[], MM::StorageDataType pixType, const char* meta, char* handle) { - if (path == nullptr) + if(path == nullptr) return ERR_TIFF_INVALID_PATH; - - if (numberOfDimensions < 3) + if(numberOfDimensions < 3) return ERR_TIFF_INVALID_DIMENSIONS; - - if (!(pixType == MM::StorageDataType::StorageDataType_GRAY16 || - pixType == MM::StorageDataType::StorageDataType_GRAY8 || - pixType == MM::StorageDataType::StorageDataType_RGB32)) + if(!(pixType == MM::StorageDataType::StorageDataType_GRAY16 || pixType == MM::StorageDataType::StorageDataType_GRAY8 || pixType == MM::StorageDataType::StorageDataType_RGB32)) return ERR_TIFF_INVALID_PIXEL_TYPE; // Check cache size limits @@ -206,7 +208,7 @@ int G2SBigTiffStorage::Create(const char* path, const char* name, int numberOfDi return ERR_TIFF_OPEN_FAILED; } - G2SStorageEntry sdesc(fhandle->getPath(), numberOfDimensions, shape, meta); + G2SStorageEntry sdesc(fhandle->getPath(), numberOfDimensions); sdesc.FileHandle = fhandle; // Set dataset UUID / shape / metadata @@ -229,12 +231,14 @@ int G2SBigTiffStorage::Create(const char* path, const char* name, int numberOfDi if(it.first == cache.end()) { delete fhandle; + LogMessage("Adding BigTIFF dataset to cache failed. Path: " + dsName.u8string() + ", GUID: " + guid); return ERR_TIFF_CACHE_INSERT; } if(!it.second) { // Dataset already exists delete fhandle; + LogMessage("Adding BigTIFF dataset to cache failed. Path: " + dsName.u8string() + ", GUID: " + guid); return ERR_TIFF_CACHE_INSERT; } @@ -277,7 +281,7 @@ int G2SBigTiffStorage::Load(const char* path, char* handle) } } if(!fnd) - return DEVICE_INVALID_INPUT_PARAM; + return ERR_TIFF_INVALID_PATH; } else if(std::filesystem::is_regular_file(actpath)) { @@ -308,7 +312,10 @@ int G2SBigTiffStorage::Load(const char* path, char* handle) // Use existing object descriptor fhandle = (G2SBigTiffDataset*)cit->second.FileHandle; if(fhandle == nullptr) - return ERR_TIFF_OPEN_FAILED; + { + LogMessage("Loading BigTIFF dataset failed (" + actpath.u8string() + "). Dataset allocation failed"); + return ERR_TIFF_ALLOCATION_FAILED; + } try { @@ -323,6 +330,7 @@ int G2SBigTiffStorage::Load(const char* path, char* handle) catch(std::exception& e) { delete fhandle; + LogMessage("Loading BigTIFF dataset failed (" + actpath.u8string() + "). " + std::string(e.what())); return ERR_TIFF_OPEN_FAILED; } @@ -331,20 +339,21 @@ int G2SBigTiffStorage::Load(const char* path, char* handle) if(guid.size() > MM::MaxStrLength) { delete fhandle; - return DEVICE_INVALID_PROPERTY_LIMTS; + return ERR_TIFF_STRING_TOO_LONG; } // Append dataset storage descriptor to cache if(cit == cache.end()) { // Create dataset storage descriptor - G2SStorageEntry sdesc(std::filesystem::absolute(actpath).u8string(), (int)fhandle->getDimension(), reinterpret_cast(&fhandle->getShape()[0]), fhandle->getMetadata().empty() ? nullptr : fhandle->getMetadata().c_str()); + G2SStorageEntry sdesc(std::filesystem::absolute(actpath).u8string(), (int)fhandle->getDimension()); sdesc.FileHandle = fhandle; auto it = cache.insert(std::make_pair(guid, sdesc)); if(it.first == cache.end()) { delete fhandle; + LogMessage("Loading BigTIFF dataset failed (" + actpath.u8string() + "). Dataset cache is full"); return DEVICE_OUT_OF_MEMORY; } } @@ -369,7 +378,7 @@ int G2SBigTiffStorage::GetShape(const char* handle, int shape[]) // Obtain dataset descriptor from cache auto it = cache.find(handle); if(it == cache.end()) - return DEVICE_INVALID_INPUT_PARAM; + return ERR_TIFF_HANDLE_INVALID; auto fs = reinterpret_cast(it->second.FileHandle); for(std::size_t i = 0; i < fs->getDimension(); i++) @@ -391,7 +400,7 @@ int G2SBigTiffStorage::GetDataType(const char* handle, MM::StorageDataType& pixe // Obtain dataset descriptor from cache auto it = cache.find(handle); if(it == cache.end()) - return DEVICE_INVALID_INPUT_PARAM; + return ERR_TIFF_HANDLE_INVALID; // Get pixel format if(!it->second.isOpen()) @@ -431,7 +440,7 @@ int G2SBigTiffStorage::Close(const char* handle) { auto it = cache.find(handle); if(it == cache.end()) - return DEVICE_INVALID_INPUT_PARAM; + return ERR_TIFF_HANDLE_INVALID; if(it->second.isOpen()) { auto fs = reinterpret_cast(it->second.FileHandle); @@ -455,12 +464,12 @@ int G2SBigTiffStorage::Delete(char* handle) return DEVICE_INVALID_INPUT_PARAM; auto it = cache.find(handle); if(it == cache.end()) - return DEVICE_INVALID_INPUT_PARAM; + return ERR_TIFF_HANDLE_INVALID; // Check if the file exists auto fp = std::filesystem::u8path(it->second.Path); if(!std::filesystem::exists(fp)) - return DEVICE_NO_PROPERTY_DATA; + return ERR_TIFF_INVALID_PATH; // Close the file handle if(it->second.isOpen()) @@ -473,7 +482,7 @@ int G2SBigTiffStorage::Delete(char* handle) // Delete the file if(!std::filesystem::remove(fp)) - return DEVICE_ERR; + return ERR_TIFF_DELETE_FAILED; // Discard the cache entry cache.erase(it); @@ -498,9 +507,9 @@ int G2SBigTiffStorage::List(const char* path, char** listOfDatasets, int maxItem return DEVICE_INVALID_INPUT_PARAM; auto dp = std::filesystem::u8path(path); if(!std::filesystem::exists(dp) || !std::filesystem::is_directory(dp)) - return DEVICE_INVALID_INPUT_PARAM; + return ERR_TIFF_INVALID_PATH; auto allfnd = scanDir(path, listOfDatasets, maxItems, maxItemLength, 0); - return allfnd ? DEVICE_OK : DEVICE_SEQUENCE_TOO_LARGE; + return allfnd ? DEVICE_OK : ERR_TIFF_STRING_TOO_LONG; } /** @@ -521,27 +530,52 @@ int G2SBigTiffStorage::AddImage(const char* handle, int sizeInBytes, unsigned ch // Obtain dataset descriptor from cache auto it = cache.find(handle); - if (it == cache.end()) - return DEVICE_INVALID_INPUT_PARAM; + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; - // Validate image dimensions + // Validate image dimensions / coordinates auto fs = reinterpret_cast(it->second.FileHandle); + if(fs->isInReadMode()) + return ERR_TIFF_DATASET_READONLY; if(!validateCoordinates(fs, coordinates, numCoordinates)) - return DEVICE_INVALID_INPUT_PARAM; + return ERR_TIFF_INVALID_COORDINATE; + if(fs->isCoordinateSet(coordinates, numCoordinates)) + return ERR_TIFF_INVALID_COORDINATE; // Add image fs->addImage(pixels, sizeInBytes, imageMeta); - - // Add image metadata to the dataset cache - auto ikey = getImageKey(coordinates, numCoordinates); - it->second.ImageMetadata.insert(std::make_pair(ikey, std::string(imageMeta))); return DEVICE_OK; } +/** + * Append image / write image to file + * Image metadata will be stored in cache + * @param handle Entry GUID + * @param pixels Pixel data buffer + * @param sizeInBytes pixel array size + * @param imageMeta Image metadata + * @return Status code + */ int G2SBigTiffStorage::AppendImage(const char* handle, int sizeInBytes, unsigned char* pixels, const char* imageMeta) { - // TODO: implement append - return DEVICE_NOT_YET_IMPLEMENTED; + if(handle == nullptr || pixels == nullptr || sizeInBytes <= 0) + return DEVICE_INVALID_INPUT_PARAM; + + // Obtain dataset descriptor from cache + auto it = cache.find(handle); + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + + // Append image + auto fs = reinterpret_cast(it->second.FileHandle); + if(fs->isInReadMode()) + return ERR_TIFF_DATASET_READONLY; + fs->addImage(pixels, sizeInBytes, imageMeta); + return DEVICE_OK; } /** @@ -561,11 +595,15 @@ int G2SBigTiffStorage::GetSummaryMeta(const char* handle, char* meta, int bufSiz // Obtain dataset descriptor from cache auto it = cache.find(handle); if(it == cache.end()) - return DEVICE_INVALID_INPUT_PARAM; + return ERR_TIFF_HANDLE_INVALID; + + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; // Copy metadata string - strncpy(meta, it->second.Metadata.c_str(), bufSize); - return it->second.Metadata.size() > (std::size_t)bufSize ? DEVICE_SEQUENCE_TOO_LARGE : DEVICE_OK; + auto fs = reinterpret_cast(it->second.FileHandle); + strncpy(meta, fs->getMetadata().c_str(), bufSize); + return fs->getMetadata().size() > (std::size_t)bufSize ? ERR_TIFF_STRING_TOO_LONG : DEVICE_OK; } /** @@ -587,40 +625,28 @@ int G2SBigTiffStorage::GetImageMeta(const char* handle, int coordinates[], int n // Obtain dataset descriptor from cache auto it = cache.find(handle); if(it == cache.end()) - return DEVICE_INVALID_INPUT_PARAM; + return ERR_TIFF_HANDLE_INVALID; auto fs = reinterpret_cast(it->second.FileHandle); if(!validateCoordinates(fs, coordinates, numCoordinates)) - return DEVICE_INVALID_INPUT_PARAM; + return ERR_TIFF_INVALID_COORDINATE; - // Check the dataset cache first - auto ikey = getImageKey(coordinates, numCoordinates); - auto iit = it->second.ImageMetadata.find(ikey); - if(iit == it->second.ImageMetadata.end()) - { - // Obtain metadata from the file stream - if(!it->second.isOpen()) - return ERR_TIFF_STREAM_UNAVAILABLE; - - // Copy coordinates without including the width and height - std::vector coords(fs->getDimension() - 2); - for(int i = 0; i < coords.size(); i++) - { - if(i >= numCoordinates) - break; - coords[i] = coordinates[i]; - } + // Obtain metadata from the file stream + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; - auto fmeta = fs->getImageMetadata(coords); - if(!fmeta.empty()) - strncpy(meta, fmeta.c_str(), bufSize); - } - else + // Copy coordinates without including the width and height + std::vector coords(fs->getDimension() - 2); + for(int i = 0; i < coords.size(); i++) { - // Copy metadata from cache - if(!iit->second.empty()) - strncpy(meta, iit->second.c_str(), bufSize); + if(i >= numCoordinates) + break; + coords[i] = coordinates[i]; } + + auto fmeta = fs->getImageMetadata(coords); + if(!fmeta.empty()) + strncpy(meta, fmeta.c_str(), bufSize); return DEVICE_OK; } @@ -635,32 +661,39 @@ int G2SBigTiffStorage::GetImageMeta(const char* handle, int coordinates[], int n */ const unsigned char* G2SBigTiffStorage::GetImage(const char* handle, int coordinates[], int numCoordinates) { - if(handle == nullptr || numCoordinates <= 0) - return nullptr; + try { + if (handle == nullptr || numCoordinates <= 0) + return nullptr; - // Obtain dataset descriptor from cache - auto it = cache.find(handle); - if(it == cache.end()) - return nullptr; - - auto fs = reinterpret_cast(it->second.FileHandle); - if(!validateCoordinates(fs, coordinates, numCoordinates)) - return nullptr; - - if(!it->second.isOpen()) - return nullptr; + // Obtain dataset descriptor from cache + auto it = cache.find(handle); + if (it == cache.end()) + return nullptr; - // Copy coordinates without including the width and height - std::vector coords(fs->getDimension() - 2); - for(int i = 0; i < coords.size(); i++) + auto fs = reinterpret_cast(it->second.FileHandle); + if (!validateCoordinates(fs, coordinates, numCoordinates)) + return nullptr; + + if (!it->second.isOpen()) + return nullptr; + + // Copy coordinates without including the width and height + std::vector coords(fs->getDimension() - 2); + for (int i = 0; i < coords.size(); i++) + { + if (i >= numCoordinates) + break; + coords[i] = coordinates[i]; + } + + it->second.ImageData = fs->getImage(coords); + return &it->second.ImageData[0]; + } + catch (std::runtime_error& e) { - if(i >= numCoordinates) - break; - coords[i] = coordinates[i]; + LogMessage("GetImage error: " +std::string(e.what())); + return nullptr; } - - it->second.ImageData = fs->getImage(coords); - return &it->second.ImageData[0]; } /** @@ -677,11 +710,16 @@ int G2SBigTiffStorage::ConfigureDimension(const char* handle, int dimension, con return DEVICE_INVALID_INPUT_PARAM; auto it = cache.find(handle); if(it == cache.end()) - return DEVICE_INVALID_INPUT_PARAM; - if((std::size_t)dimension >= it->second.getDimSize()) - return DEVICE_INVALID_INPUT_PARAM; - it->second.Dimensions[dimension].Name = std::string(name); - it->second.Dimensions[dimension].Metadata = std::string(meaning); + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + auto fs = reinterpret_cast(it->second.FileHandle); + if(fs->isInReadMode()) + return ERR_TIFF_DATASET_READONLY; + + if((std::size_t)dimension >= fs->getDimension()) + return ERR_TIFF_INVALID_DIMENSIONS; + fs->configureAxis(dimension, std::string(name), std::string(meaning)); return DEVICE_OK; } @@ -699,12 +737,18 @@ int G2SBigTiffStorage::ConfigureCoordinate(const char* handle, int dimension, in return DEVICE_INVALID_INPUT_PARAM; auto it = cache.find(handle); if(it == cache.end()) - return DEVICE_INVALID_INPUT_PARAM; - if((std::size_t)dimension >= it->second.getDimSize()) - return DEVICE_INVALID_INPUT_PARAM; - if((std::size_t)coordinate >= it->second.Dimensions[dimension].getSize()) - return DEVICE_INVALID_INPUT_PARAM; - it->second.Dimensions[dimension].Coordinates[coordinate] = std::string(name); + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + auto fs = reinterpret_cast(it->second.FileHandle); + if(fs->isInReadMode()) + return ERR_TIFF_DATASET_READONLY; + + if(dimension < 0 || (std::size_t)dimension >= fs->getDimension()) + return ERR_TIFF_INVALID_DIMENSIONS; + if(coordinate < 0 || (std::size_t)coordinate >= fs->getShape()[dimension]) + return ERR_TIFF_INVALID_COORDINATE; + fs->configureCoordinate(dimension, coordinate, std::string(name)); return DEVICE_OK; } @@ -720,8 +764,12 @@ int G2SBigTiffStorage::GetNumberOfDimensions(const char* handle, int& numDimensi return DEVICE_INVALID_INPUT_PARAM; auto it = cache.find(handle); if(it == cache.end()) - return DEVICE_INVALID_INPUT_PARAM; - numDimensions = (int)it->second.getDimSize(); + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + + auto fs = reinterpret_cast(it->second.FileHandle); + numDimensions = (int)fs->getDimension(); return DEVICE_OK; } @@ -736,15 +784,21 @@ int G2SBigTiffStorage::GetDimension(const char* handle, int dimension, char* nam return DEVICE_INVALID_INPUT_PARAM; auto it = cache.find(handle); if(it == cache.end()) - return DEVICE_INVALID_INPUT_PARAM; - if((std::size_t)dimension >= it->second.getDimSize()) - return DEVICE_INVALID_INPUT_PARAM; - if(it->second.Dimensions[dimension].Name.size() > (std::size_t)nameLength) - return DEVICE_INVALID_PROPERTY_LIMTS; - if(it->second.Dimensions[dimension].Metadata.size() > (std::size_t)meaningLength) - return DEVICE_INVALID_PROPERTY_LIMTS; - strncpy(name, it->second.Dimensions[dimension].Name.c_str(), nameLength); - strncpy(meaning, it->second.Dimensions[dimension].Metadata.c_str(), meaningLength); + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + auto fs = reinterpret_cast(it->second.FileHandle); + + if(dimension < 0 || (std::size_t)dimension >= fs->getDimension()) + return ERR_TIFF_INVALID_DIMENSIONS; + + const auto& axinf = fs->getAxisInfo((std::uint32_t)dimension); + if(axinf.Name.size() > (std::size_t)nameLength) + return ERR_TIFF_STRING_TOO_LONG; + if(axinf.Description.size() > (std::size_t)meaningLength) + return ERR_TIFF_STRING_TOO_LONG; + strncpy(name, axinf.Name.c_str(), nameLength); + strncpy(meaning, axinf.Description.c_str(), meaningLength); return DEVICE_OK; } @@ -759,39 +813,157 @@ int G2SBigTiffStorage::GetCoordinate(const char* handle, int dimension, int coor return DEVICE_INVALID_INPUT_PARAM; auto it = cache.find(handle); if(it == cache.end()) - return DEVICE_INVALID_INPUT_PARAM; - if((std::size_t)dimension >= it->second.getDimSize()) - return DEVICE_INVALID_INPUT_PARAM; - if((std::size_t)coordinate >= it->second.Dimensions[dimension].getSize()) - return DEVICE_INVALID_INPUT_PARAM; - if(it->second.Dimensions[dimension].Coordinates[coordinate].size() > (std::size_t)nameLength) - return DEVICE_INVALID_PROPERTY_LIMTS; - strncpy(name, it->second.Dimensions[dimension].Coordinates[coordinate].c_str(), nameLength); + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + auto fs = reinterpret_cast(it->second.FileHandle); + + if(dimension < 0 || (std::size_t)dimension >= fs->getDimension()) + return ERR_TIFF_INVALID_DIMENSIONS; + if(coordinate < 0 || (std::size_t)coordinate >= fs->getShape()[dimension]) + return ERR_TIFF_INVALID_COORDINATE; + + const auto& axinf = fs->getAxisInfo((std::uint32_t)dimension); + if((std::size_t)coordinate >= axinf.Coordinates.size()) + return ERR_TIFF_INVALID_COORDINATE; + if(axinf.Coordinates[coordinate].size() > (std::size_t)nameLength) + return ERR_TIFF_STRING_TOO_LONG; + strncpy(name, axinf.Coordinates[coordinate].c_str(), nameLength); return DEVICE_OK; } /** - * Returns true if images can be added to the dataset + * Get number of available images + * @param handle Entry GUID + * @param imgcount Image count [out] + * @return Status code */ -bool G2SBigTiffStorage::IsOpen(const char* handle) +int G2SBigTiffStorage::GetImageCount(const char* handle, int& imgcount) { - // TODO: is this the correct way to check if the dataset is accepting images? + if(handle == nullptr) + return DEVICE_INVALID_INPUT_PARAM; auto it = cache.find(handle); - if (it == cache.end()) + if(it == cache.end()) + return ERR_TIFF_HANDLE_INVALID; + if(!it->second.isOpen()) + return ERR_TIFF_DATASET_CLOSED; + auto fs = reinterpret_cast(it->second.FileHandle); + imgcount = (int)fs->getImageCount(); + return DEVICE_OK; +} + +/** + * Check if dataset is open + * If the dataset doesn't exist, or the GUID is invalid this method will return false + * @param handle Entry GUID + * @return true if dataset is open + */ +bool G2SBigTiffStorage::IsOpen(const char* handle) +{ + if(handle == nullptr) return false; + auto it = cache.find(handle); + if(it == cache.end()) + return false; + return it->second.isOpen(); +} - return true; +/** + * Check if dataset is read-only + * If the dataset doesn't exist, or the GUID is invalid this method will return true + * @param handle Entry GUID + * @return true if images can't be added to the dataset + */ +bool G2SBigTiffStorage::IsReadOnly(const char* handle) +{ + if(handle == nullptr) + return true; + auto it = cache.find(handle); + if(it == cache.end()) + return true; + if(!it->second.isOpen()) + return true; + auto fs = reinterpret_cast(it->second.FileHandle); + return fs->isInReadMode(); } +/** + * Get dataset path + * @param handle Entry GUID + * @param path Dataset path [out] + * @param maxPathLength Max path length + * @return Status code + */ int G2SBigTiffStorage::GetPath(const char* handle, char* path, int maxPathLength) { + if(handle == nullptr || maxPathLength <= 0) + return DEVICE_INVALID_INPUT_PARAM; auto it = cache.find(handle); - if (it == cache.end()) + if(it == cache.end()) return ERR_TIFF_HANDLE_INVALID; + if(it->second.Path.size() > (std::size_t)maxPathLength) + return ERR_TIFF_STRING_TOO_LONG; strncpy(path, it->second.Path.c_str(), it->second.Path.size()); return DEVICE_OK; } +/** + * Check if there is a valid dataset on the selected path + * @param path Dataset path + * @return Path is a valid dataset + */ +bool G2SBigTiffStorage::CanLoad(const char* path) +{ + if(path == nullptr) + return false; + std::filesystem::path xpath = std::filesystem::u8path(path); + if(!std::filesystem::exists(xpath)) + return false; + + if(std::filesystem::is_directory(xpath)) + { + // If directory is selected check if it's not empty and if the name ends with .g2s + auto dname = xpath.filename().u8string(); + if(dname.find(".g2s") != dname.size() - 4) + return false; + + // Check for valid files + int validfiles = 0; + for(const auto& entry : std::filesystem::directory_iterator(xpath)) + { + // Skip auto folder paths + auto fname = entry.path().filename().u8string(); + if(fname == "." || fname == "..") + continue; + + // Skip folders + if(std::filesystem::is_directory(entry)) + continue; + + // Skip unsupported file formats + auto fext = entry.path().extension().u8string(); + if(fext.size() == 0) + continue; + if(fext[0] == '.') + fext = fext.substr(1); + std::transform(fext.begin(), fext.end(), fext.begin(), [](char c) { return (char)tolower(c); }); + if(fext != "tiff" && fext != "tif" && fext != "g2s.tiff" && fext != "g2s.tif") + continue; + + // We found a supported file type -> Increment the counter + validfiles++; + } + return validfiles > 0; + } + else + { + // If file is selected check file extension + auto fext = xpath.extension().u8string(); + std::transform(fext.begin(), fext.end(), fext.begin(), [](char c) { return (char)tolower(c); }); + return fext == "tiff" || fext == "tif" || fext == "g2s.tiff" || fext == "g2s.tif"; + } +} + /** * Discard closed dataset storage descriptors from cache * By default storage descriptors are preserved even after the dataset is closed @@ -885,20 +1057,6 @@ bool G2SBigTiffStorage::validateCoordinates(const G2SBigTiffDataset* fs, int coo return true; } -/** - * Calculate image key from the specified image coordinates - * @param coordinates Image coordinates - * @param numCoordinates Coordinate count - * @return Image key (for image cache indices) - */ -std::string G2SBigTiffStorage::getImageKey(int coordinates[], int numCoordinates) noexcept -{ - std::stringstream ss; - for(int i = 0; i < numCoordinates; i++) - ss << (i == 0 ? "" : ".") << coordinates[i]; - return ss.str(); -} - /** * Get direct I/O property * @return Is direct I/O enabled diff --git a/DeviceAdapters/go2scope/G2SBigTiffStorage.h b/DeviceAdapters/go2scope/G2SBigTiffStorage.h index ec7189379..ff5dc5311 100644 --- a/DeviceAdapters/go2scope/G2SBigTiffStorage.h +++ b/DeviceAdapters/go2scope/G2SBigTiffStorage.h @@ -72,14 +72,11 @@ class G2SBigTiffStorage : public CStorageBase int GetNumberOfDimensions(const char* handle, int& numDimensions); int GetDimension(const char* handle, int dimension, char* name, int nameLength, char* meaning, int meaningLength); int GetCoordinate(const char* handle, int dimension, int coordinate, char* name, int nameLength); + int GetImageCount(const char* handle, int& imgcnt); bool IsOpen(const char* handle); + bool IsReadOnly(const char* handle); int GetPath(const char* handle, char* path, int maxPathLength); - - -public: - //========================================================================================================================= - // Public interface - Action interface - //========================================================================================================================= + bool CanLoad(const char* path); protected: //========================================================================================================================= @@ -88,7 +85,6 @@ class G2SBigTiffStorage : public CStorageBase void cacheReduce() noexcept; bool scanDir(const std::string& path, char** listOfDatasets, int maxItems, int maxItemLength, int cpos) noexcept; bool validateCoordinates(const G2SBigTiffDataset* fs, int coordinates[], int numCoordinates) noexcept; - std::string getImageKey(int coordinates[], int numCoordinates) noexcept; bool getDirectIO() const noexcept; int getFlushCycle() const noexcept; int getChunkSize() const noexcept; diff --git a/DeviceAdapters/go2scope/G2SFileUtil.h b/DeviceAdapters/go2scope/G2SFileUtil.h index 0b07140bc..b17c4b7ac 100644 --- a/DeviceAdapters/go2scope/G2SFileUtil.h +++ b/DeviceAdapters/go2scope/G2SFileUtil.h @@ -26,11 +26,13 @@ /////////////////////////////////////////////////////////////////////////////// #pragma once #include +#include +#include //=============================================================================================================================== // Driver version //=============================================================================================================================== -#define G2STIFF_VERSION "1.0.1" +#define G2STIFF_VERSION "1.0.2" //=============================================================================================================================== // Literals @@ -47,3 +49,4 @@ //=============================================================================================================================== void writeInt(unsigned char* buff, std::uint8_t len, std::uint64_t val) noexcept; std::uint64_t readInt(const unsigned char* buff, std::uint8_t len) noexcept; +std::vector splitLineCSV(const std::string& line) noexcept; diff --git a/DeviceAdapters/go2scope/G2SStorage.h b/DeviceAdapters/go2scope/G2SStorage.h index 8b2fb28d2..b76a360c2 100644 --- a/DeviceAdapters/go2scope/G2SStorage.h +++ b/DeviceAdapters/go2scope/G2SStorage.h @@ -54,6 +54,12 @@ #define ERR_TIFF_OPEN_FAILED 140506 #define ERR_TIFF_CACHE_INSERT 140507 #define ERR_TIFF_HANDLE_INVALID 140508 +#define ERR_TIFF_STRING_TOO_LONG 140509 +#define ERR_TIFF_INVALID_COORDINATE 140510 +#define ERR_TIFF_DATASET_CLOSED 140511 +#define ERR_TIFF_DATASET_READONLY 140512 +#define ERR_TIFF_DELETE_FAILED 140513 +#define ERR_TIFF_ALLOCATION_FAILED 140514 ////////////////////////////////////////////////////////////////////////////// // Cache configuration @@ -63,40 +69,9 @@ #define CACHE_HARD_LIMIT 0 static const char* g_Go2Scope = "Go2Scope"; -static const char* g_MMV1Storage = "MMV1Storage"; static const char* g_AcqZarrStorage = "AcquireZarrStorage"; static const char* g_BigTiffStorage = "G2SBigTiffStorage"; -/** - * Dataset dimension descriptor - * @author Miloš Jovanović - * @version 1.0 - */ -struct G2SDimensionInfo -{ - /** - * Default initializer - * @param vname Axis name - * @param ndim Axis size - */ - G2SDimensionInfo(int ndim = 0) noexcept : Name(""), Coordinates(ndim) { } - - /** - * Set dimensions size - * @param sz Number of axis coordinates - */ - void setSize(std::size_t sz) noexcept { Coordinates.resize(sz); } - /** - * Get dimension size - * @return Number of axis coordinates - */ - std::size_t getSize() const noexcept { return Coordinates.size(); } - - std::string Name; ///< Axis name - std::string Metadata; ///< Axis metadata - std::vector Coordinates; ///< Axis coordinates -}; - /** * Storage entry descriptor * @author Miloš Jovanović @@ -109,39 +84,20 @@ struct G2SStorageEntry * @param vpath Absoulute path on disk * @param ndim Number of dimensions * @param shape Axis sizes - * @param vmeta Dataset metadata */ - G2SStorageEntry(const std::string& vpath, int ndim, const int* shape = nullptr, const char* vmeta = nullptr) noexcept : Path(vpath), Dimensions(ndim) - { - if(shape != nullptr) - { - for(std::size_t i = 0; i < Dimensions.size(); i++) - Dimensions[i].setSize((std::size_t)shape[i]); - } - if(vmeta != nullptr) - Metadata = std::string(vmeta); - FileHandle = nullptr; - } + G2SStorageEntry(const std::string& vpath, int ndim) noexcept : Path(vpath), FileHandle(nullptr) { } /** * Close the descriptor */ - void close() noexcept { FileHandle = nullptr; Metadata.clear(); ImageMetadata.clear(); ImageData.clear(); } + void close() noexcept { FileHandle = nullptr; ImageData.clear(); } /** * Check if file handle is open * @return Is file handle open */ bool isOpen() noexcept { return FileHandle != nullptr; } - /** - * Get number of dimensions - * @return Number of dataset dimensions - */ - std::size_t getDimSize() const noexcept { return Dimensions.size(); } std::string Path; ///< Absoulute path on disk - std::string Metadata; ///< Dataset metadata - std::vector Dimensions; ///< Dataset dimensions vector - std::map ImageMetadata; ///< Per-image metadata - std::vector ImageData; ///< Current image cache + std::vector ImageData; ///< Current image data void* FileHandle; ///< File handle };