Merge branch 'disney'

* disney: (52 commits)
  Add Ctrl-W shortcut to close editor window
  Expose additional arg in partio::write()
  partedit: display attr size 9 as 3x3 matrix
  make test
  Code review updates
  Add testSpecs for mentor run
  partjson: Convert between bgeo and json (both ways)
  partedit: Make main widget a splitter
  partedit: Add support for viewing indexed strings
  partedit: Add support for fixed attributes
  Expose functions to python
  Toward FXSOFT-1129: Add Partio::merge() function
  partedit: Row indices should be 0-based to match particle indices
  Partio::print() needs a little love
  Fix Partio::print() function
  Add .workonrc.py
  Fix Makefile to not put collection include before partio
  partedit fixes
  partedit: Resize newly added particles/attributes
  Install partedit as a module for reuse
  ...

Signed-off-by: David Aguilar <[email protected]>

.workonrc.products

@@ -0,0 +1 @@
+partio

.workonrc.py

@@ -0,0 +1,14 @@
+#!/usr/bin/python
+
+import os, dshell
+
+from pathfinder import build
+_opts = build.Options(absolute=True)
+
+shell = dshell.create()
+shell.export('PARTIO', os.getcwd())
+
+# Add to paths
+test_dir = build.test('partio', opts=_opts)
+shell.prepend('PATH', test_dir)
+shell.prepend('PYTHONPATH', test_dir)

CMakeLists.txt

@@ -39,6 +39,7 @@ PROJECT( partio LANGUAGES CXX VERSION 0.1.1 )
 
 # PARTIO_SE_ENABLED enables SeExpr support
 option(PARTIO_SE_ENABLED "PARTIO_SE_ENABLED" false)
+option(GTEST_ENABLED "Enable GTest for tests" false)
 
 # Enable C++11
 SET(CMAKE_CXX_STANDARD 11)
@@ -67,6 +68,9 @@ IF(NOT CMAKE_BUILD_TYPE)
       FORCE)
 ENDIF(NOT CMAKE_BUILD_TYPE)
 
+include(CTest)
+enable_testing()
+
 ## Set install location
 IF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
     EXECUTE_PROCESS(COMMAND sh -c "echo `uname`-`uname -r | cut -d'-' -f1`-`uname -m`" OUTPUT_VARIABLE VARIANT_DIRECTORY OUTPUT_STRIP_TRAILING_WHITESPACE)
@@ -103,6 +107,12 @@ if (${PARTIO_SE_ENABLED})
     find_library(SEEXPR_LIB SeExpr HINTS ${SEEXPR_LINK_PATH})
 endif ()
 
+if (${GTEST_ENABLED})
+    set($GEST_LOCATION "/usr" CACHE STRING "gtest installation prefix")
+    set(GTEST_INCLUDE_PATH ${GTEST_BASE}/include)
+    set(GTEST_LIBDIR ${CMAKE_INSTALL_LIBDIR})
+    set(GTEST_LINK_PATH ${GTEST_LOCATION}/${GTEST_LIBDIR} CACHE STRING "gtest library directory")
+endif()
 
 ## Traverse subdirectories
 ADD_SUBDIRECTORY (src/lib)

Makefile

@@ -3,6 +3,7 @@ SH ?= sh
 uname_S := $(shell $(SH) -c 'uname -s || echo kernel')
 uname_R := $(shell $(SH) -c 'uname -r | cut -d- -f1 || echo release')
 uname_M := $(shell $(SH) -c 'uname -m || echo machine')
+lib ?= lib64
 FLAVOR ?= optimize
 platformdir ?= $(uname_S)-$(uname_R)-$(uname_M)-$(FLAVOR)
 builddir ?= $(CURDIR)/build/$(platformdir)
@@ -28,17 +29,31 @@ ifdef RP_SeExpr
     CMAKE_FLAGS += -DPARTIO_SE_ENABLED=1
 endif
 
+# gtest location: RP_gtest=<path>
+ifdef RP_gtest
+    CMAKE_FLAGS += -DGTEST_LOCATION=$(RP_gtest)
+    CMAKE_FLAGS += -DGTEST_ENABLED=1
+endif
+
 # Extra cmake flags: CMAKE_EXTRA_FLAGS=<flags>
 ifdef CMAKE_EXTRA_FLAGS
     CMAKE_FLAGS += $(CMAKE_EXTRA_FLAGS)
 endif
 
+ifdef RP_zlib
+    CMAKE_FLAGS += -DZLIB_INCLUDE_DIR=$(RP_zlib)/include
+    CMAKE_FLAGS += -DZLIB_LIBRARY_RELEASE=$(RP_zlib)/$(lib)/libz.so
+endif
+
 # The default target in this Makefile is...
 all::
 
 install: all
 	$(MAKE) -C $(builddir) DESTDIR=$(DESTDIR) install
 
+test: all
+	$(MAKE) -C $(builddir) DESTDIR=$(DESTDIR) test
+
 $(builddir)/stamp:
 	mkdir -p $(builddir)
 	cd $(builddir) && cmake $(CMAKE_FLAGS) ../..

UPDATE

@@ -0,0 +1,6 @@
+1.  fetch github-wdas/master
+2.  reset master from github-wdas master
+3.  push master to origin/master (origin/master should be identical to github-wdas/master at this point)
+4.  merge master into disney
+5.  push disney to origin/disney (do this without creating code review branch:
+    git push origin disney:disney)

src/lib/CMakeLists.txt

@@ -59,7 +59,9 @@ if (SEEXPR_BASE)
         target_include_directories(partioSe
             PUBLIC
                 $<INSTALL_INTERFACE:${SEEXPR_INCLUDE_PATH}>
-                $<BUILD_INTERFACE:${SEEXPR_INCLUDE_DIR}>)
+                $<BUILD_INTERFACE:${SEEXPR_INCLUDE_DIR}>
+            PRIVATE
+                ${SEEXPR_INCLUDE_PATH})
         target_link_libraries(partioSe PUBLIC partio ${SEEXPR_LIB})
         install(TARGETS partioSe DESTINATION ${CMAKE_INSTALL_LIBDIR})
     endif ()

src/lib/Partio.h

@@ -318,5 +318,18 @@ void endCachedAccess(ParticlesData* particles);
 void print(const ParticlesData* particles);
 
 ParticlesDataMutable* computeClustering(ParticlesDataMutable* particles, const int numNeighbors,const double radiusSearch,const double radiusInside,const int connections,const double density);
+
+//! Merges one particle set into another
+/*!
+  Given a ParticleSetMutable, merges it with a second ParticleSet,
+  copying particles and attributes that align with the base particle
+  set. If an identifier is provided, that will be used as a key
+  to replace the particle in the base set with the particle in the second
+  set with the same identifier attribute value. If the identifier is not
+  provided or the particle's attribute value is not found in the base set,
+  a new particle is added. If used, the identifier must be a single INT.
+*/
+void merge(ParticlesDataMutable& base, const ParticlesData& delta, const std::string& identifier=std::string());
+
 }
 #endif

src/lib/core/Particle.cpp

@@ -43,6 +43,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
 #include <string>
 #include <cstring>
 #include <cassert>
+#include <vector>
+#include <unordered_map>
+#include <memory>
+
 namespace Partio{
 
 std::string
@@ -161,18 +165,27 @@ printAttr(const ParticlesData* p,const ParticleAttribute& attr,const int particl
     for(int k=0;k<attr.count;k++) std::cout<<" "<<data[k];
 }
 
+std::vector<ParticleAttribute>
+getAttrs(const ParticlesData& particles)
+{
+    std::vector<ParticleAttribute> attrs(particles.numAttributes());
+    for(int i=0;i<particles.numAttributes();i++){
+        particles.attributeInfo(i,attrs[i]);
+    }
+    return attrs;
+}
+
 void
 print(const ParticlesData* particles)
 {
     std::cout<<"Particle count "<<particles->numParticles()<<std::endl;
     std::cout<<"Attribute count "<<particles->numAttributes()<<std::endl;
 
-    std::vector<ParticleAttribute> attrs;
-    for(int i=0;i<particles->numAttributes();i++){
-        ParticleAttribute attr;
-        particles->attributeInfo(i,attr);
-        attrs.push_back(attr);
-        std::cout<<"attribute "<<attr.name<<" "<<int(attr.type)<<" "<<attr.count<<std::endl;
+    std::vector<ParticleAttribute> attrs = getAttrs(*particles);
+    for (const ParticleAttribute& attr : attrs) {
+        std::cout << "attribute " << attr.name
+                  << " type=" << TypeName(attr.type)
+                  << " count=" << attr.count << std::endl;
     }
 
     int numToPrint=std::min(10,particles->numParticles());
@@ -183,22 +196,35 @@ print(const ParticlesData* particles)
     for(size_t k=0;k<attrs.size();k++) accessors.push_back(ParticleAccessor(attrs[k]));
     for(size_t k=0;k<attrs.size();k++) it.addAccessor(accessors[k]);
 
-    for(int i=0;i<numToPrint && it != end;i++){
+    for(int i=0;i<numToPrint && it != end;i++, it++){
         std::cout<<i<<": ";
         for(unsigned int k=0;k<attrs.size();k++){
+            std::cout<<attrs[k].name<<"=";
+            if (attrs[k].count > 1) std::cout<<"(";
             switch(attrs[k].type){
             case NONE:break;
             case FLOAT:
             case VECTOR:
-                for(int c=0;c<attrs[k].count;c++) std::cout<<accessors[k].raw<float>(it)[c];
+                for(int c=0;c<attrs[k].count;c++) {
+                    if (c) std::cout << ",";
+                    std::cout<<accessors[k].raw<float>(it)[c];
+                }
                 break;
             case INT:
-                for(int c=0;c<attrs[k].count;c++) std::cout<<accessors[k].raw<int>(it)[c];
+                for(int c=0;c<attrs[k].count;c++) {
+                    if (c) std::cout << ",";
+                    std::cout<<accessors[k].raw<int>(it)[c]<<",";
+                }
                 break;
             case INDEXEDSTR:
-                for(int c=0;c<attrs[k].count;c++) std::cout<<accessors[k].raw<int>(it)[c];
+                for(int c=0;c<attrs[k].count;c++) {
+                    if (c) std::cout << ",";
+                    std::cout<<accessors[k].raw<int>(it)[c]<<",";
+                }
                 break;
             }
+            if (attrs[k].count > 1) std::cout<<")";
+            std::cout<<"\t";
         }
         std::cout<<std::endl;
     }
@@ -404,4 +430,145 @@ computeClustering(ParticlesDataMutable* particles, const int numNeighbors,const
     return cluster;;
 }
 
+template<typename T>
+struct AttributePair {
+    T base;
+    T delta;
+};
+
+void merge(ParticlesDataMutable& base, const ParticlesData& delta, const std::string& identifier)
+{
+    // Build a map from the identifier value to the particle index
+    // and locate the identifier attribute in the base.
+    // This assumes unique identifiers per particle.
+    std::unordered_map<int,int> idToParticleIndex;
+    ParticleAttribute baseIdAttr;
+    bool baseHasIdentifier = base.attributeInfo(identifier.c_str(), baseIdAttr);
+    if (baseHasIdentifier) {
+        if (baseIdAttr.type == INT) {
+            for (int i=0; i<base.numParticles(); i++) {
+                idToParticleIndex[base.data<int>(baseIdAttr,i)[0]] = i;
+            }
+        } else {
+            baseHasIdentifier = false;
+        }
+    }
+
+    // Locate the identifier attribute in the delta
+    ParticleAttribute deltaIdAttr;
+    bool deltaHasIdentifier = delta.attributeInfo(identifier.c_str(), deltaIdAttr);
+    if (deltaHasIdentifier) {
+        deltaHasIdentifier &= deltaIdAttr.type == INT;
+    }
+    bool hasIdentifier = baseHasIdentifier && deltaHasIdentifier;
+
+    // Identify the attributes to be copied (base present in delta)
+    std::vector<AttributePair<ParticleAttribute>> attrs;
+    std::vector<size_t> indexStrAttrs; /* indexes into attrs */
+    for (int i=0; i<base.numAttributes(); ++i) {
+        ParticleAttribute baseAttr, deltaAttr;
+        base.attributeInfo(i, baseAttr);
+        if (delta.attributeInfo(baseAttr.name.c_str(), deltaAttr)) {
+            if (baseAttr.type == INDEXEDSTR) {
+                indexStrAttrs.push_back(attrs.size());
+            }
+            attrs.emplace_back(AttributePair<ParticleAttribute>({std::move(baseAttr), std::move(deltaAttr)}));
+        }
+    }
+
+    // Identify fixed attributes to override
+    for (int i=0; i<base.numFixedAttributes(); ++i) {
+        FixedAttribute baseAttr, deltaAttr;
+        base.fixedAttributeInfo(i, baseAttr);
+        if (delta.fixedAttributeInfo(baseAttr.name.c_str(), deltaAttr)) {
+            size_t size = Partio::TypeSize(baseAttr.type) * baseAttr.count;
+            const void *src = delta.fixedData<void>(deltaAttr);
+            void *dst = base.fixedDataWrite<void>(baseAttr);
+            std::memcpy(dst, src, size);
+        }
+    }
+
+    // Identify fixed attributes to extend
+    for (int i=0; i<delta.numFixedAttributes(); ++i) {
+        FixedAttribute baseAttr, deltaAttr;
+        delta.fixedAttributeInfo(i, deltaAttr);
+        if (!base.fixedAttributeInfo(deltaAttr.name.c_str(), baseAttr)) {
+            baseAttr = base.addFixedAttribute(deltaAttr.name.c_str(), deltaAttr.type, deltaAttr.count);
+            size_t size = Partio::TypeSize(deltaAttr.type) * deltaAttr.count;
+            const void *src = delta.fixedData<void>(deltaAttr);
+            void *dst = base.fixedDataWrite<void>(baseAttr);
+            std::memcpy(dst, src, size);
+        }
+    }
+
+    // Merge the indexed strings. If the delta holds new strings for the same attribute,
+    // we have to re-index it and extend the base string list with the new strings.
+    // If the string exists in both, we still have to map the delta index to the base index.
+    std::unordered_map</*attr_name*/std::string,
+                       std::unordered_map</*delta_index*/int, /*base_index*/int> > stringToString;
+    for (size_t index : indexStrAttrs) {
+        const AttributePair<ParticleAttribute>& attr = attrs[index];
+
+        /* For each string in the delta, add to base if missing. And re-index. */
+        const std::vector<std::string>& baseStrs = base.indexedStrs(attr.base);
+        // Map source indices name->index for faster searching
+        std::unordered_map<std::string, size_t> indexInBase;
+        for (size_t i=0; i<baseStrs.size(); ++i) {
+            indexInBase[baseStrs[i]] = i;
+        }
+
+        // Loop through delta strs and reindex
+        const std::vector<std::string>& deltaStrs = delta.indexedStrs(attr.delta);
+        for (size_t i=0; i<deltaStrs.size(); ++i) {
+            const std::string& deltaStr = deltaStrs[i];
+
+            auto it = indexInBase.find(deltaStr);
+            if (it != indexInBase.end()) {
+                stringToString[attr.base.name][i] = it->second;
+            } else {
+                int newIndex = base.registerIndexedStr(attr.base, deltaStr.c_str());
+                stringToString[attr.base.name][i] = newIndex;
+            }
+        }
+    }
+
+
+    // Loop through the delta particles and incorporate into the base
+    for (int i=0; i<delta.numParticles(); ++i) {
+
+        // Grab index into base particle set - either existing or new
+        int index(-1);
+        if (hasIdentifier) {
+            int idValue = *(delta.data<int>(deltaIdAttr, i));
+            auto it = idToParticleIndex.find(idValue);
+            if (it != idToParticleIndex.end()) {
+                index = it->second;
+            }
+        }
+        if (index == -1) {
+            index = base.addParticle();
+        }
+
+        // Copy the attributes to the new/overridden particle
+        for (const AttributePair<ParticleAttribute>& attr : attrs) {
+            size_t size = Partio::TypeSize(attr.base.type) * attr.base.count;
+            void *dst = base.dataWrite<void>(attr.base, index);
+            const void* src;
+            std::unique_ptr<int> newIndices;
+            if (attr.base.type == INDEXEDSTR) {
+                newIndices.reset(new int[attr.base.count]);
+                const int* indices = delta.data<int>(attr.delta, i);
+                for (int j=0; j<attr.delta.count; ++j) {
+                    newIndices.get()[j] = stringToString[attr.base.name][indices[j]];
+                }
+                src = (void*)(newIndices.get());
+            }
+            else {
+                src = delta.data<void>(attr.delta, i);
+            }
+            std::memcpy(dst, src, size);
+        }
+    }
+}
+
 }

src/py/example/listAttr.py

@@ -34,7 +34,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
 
 import sys
-import papi
+import partio
 
 if __name__=="__main__":
     filename=None
@@ -45,13 +45,13 @@
         sys.exit(1)
 
     # read particle name
-    p=papi.read(filename)
+    p=partio.read(filename)
 
     for i in range(p.numAttributes()):
         attr=p.attributeInfo(i)
         typeStr="NONE"
-        if attr.type==papi.VECTOR: typeStr="VECTOR"
-        if attr.type==papi.FLOAT: typeStr="FLOAT"
-        if attr.type==papi.INT: typeStr="INT"
+        if attr.type==partio.VECTOR: typeStr="VECTOR"
+        if attr.type==partio.FLOAT: typeStr="FLOAT"
+        if attr.type==partio.INT: typeStr="INT"
         print "%10s[%d] %-30s "%(typeStr,attr.count,attr.name)