From 3d1d9707bb8fca2dbbb30b2fbf74587245214594 Mon Sep 17 00:00:00 2001 From: datomo Date: Sat, 30 Sep 2023 00:46:46 +0200 Subject: [PATCH] we push now, enabled restore, still an error --- .../org/polypheny/db/catalog/Catalog.java | 5 ++ .../db/catalog/catalogs/LogicalCatalog.java | 1 + .../polypheny/db/catalog/impl/Persister.java | 83 +++++++++++++++++++ .../db/catalog/impl/PolyCatalog.java | 54 ++++++++++-- .../impl/allocation/PolyAllocDocCatalog.java | 11 +++ .../allocation/PolyAllocGraphCatalog.java | 11 +++ .../impl/allocation/PolyAllocRelCatalog.java | 11 +++ .../catalog/impl/logical/DocumentCatalog.java | 16 ++-- .../db/catalog/impl/logical/GraphCatalog.java | 24 ++++-- .../impl/logical/RelationalCatalog.java | 59 ++++++++----- .../java/org/polypheny/db/PolyphenyDb.java | 27 ++++-- .../transaction/TransactionManagerImpl.java | 7 +- .../polypheny/db/adapter/file/source/Qfs.java | 66 +++++++++------ .../java/org/polypheny/db/webui/Crud.java | 1 - .../org/polypheny/db/webui/auth/AuthCrud.java | 17 ++-- .../db/webui/auth/PartnerConnection.java | 62 ++++++++++++++ .../db/webui/auth/PartnerStatus.java | 40 --------- .../polypheny/db/webui/models/Namespace.java | 11 ++- 18 files changed, 383 insertions(+), 123 deletions(-) create mode 100644 core/src/main/java/org/polypheny/db/catalog/impl/Persister.java create mode 100644 webui/src/main/java/org/polypheny/db/webui/auth/PartnerConnection.java delete mode 100644 webui/src/main/java/org/polypheny/db/webui/auth/PartnerStatus.java diff --git a/core/src/main/java/org/polypheny/db/catalog/Catalog.java b/core/src/main/java/org/polypheny/db/catalog/Catalog.java index 972d70886f..39e6500d98 100644 --- a/core/src/main/java/org/polypheny/db/catalog/Catalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/Catalog.java @@ -93,6 +93,8 @@ public static Catalog getInstance() { public abstract void updateSnapshot(); + public abstract void change(); + public abstract void commit(); public abstract void rollback(); @@ -275,6 +277,9 @@ public static Snapshot snapshot() { public abstract void removeAdapterTemplate( long templateId ); + public abstract PropertyChangeListener getChangeListener(); + + public abstract void restore(); } diff --git a/core/src/main/java/org/polypheny/db/catalog/catalogs/LogicalCatalog.java b/core/src/main/java/org/polypheny/db/catalog/catalogs/LogicalCatalog.java index 26b074bd71..678da7a315 100644 --- a/core/src/main/java/org/polypheny/db/catalog/catalogs/LogicalCatalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/catalogs/LogicalCatalog.java @@ -29,6 +29,7 @@ GraphCatalog.class }) public interface LogicalCatalog { + LogicalCatalog withLogicalNamespace( LogicalNamespace namespace ); LogicalNamespace getLogicalNamespace(); diff --git a/core/src/main/java/org/polypheny/db/catalog/impl/Persister.java b/core/src/main/java/org/polypheny/db/catalog/impl/Persister.java new file mode 100644 index 0000000000..efd3dd0ff5 --- /dev/null +++ b/core/src/main/java/org/polypheny/db/catalog/impl/Persister.java @@ -0,0 +1,83 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.catalog.impl; + +import com.drew.lang.Charsets; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; +import org.polypheny.db.catalog.exceptions.GenericRuntimeException; +import org.polypheny.db.util.PolyphenyHomeDirManager; + +public class Persister { + + ExecutorService service = Executors.newSingleThreadExecutor(); + + + private final File backup; + + + public Persister() { + this.backup = initBackupFile(); + } + + + private static File initBackupFile() { + if ( !PolyphenyHomeDirManager.getInstance().checkIfExists( "catalog" ) ) { + PolyphenyHomeDirManager.getInstance().registerNewFolder( "catalog" ); + } + File folder = PolyphenyHomeDirManager.getInstance().getFileIfExists( "catalog" ); + if ( !folder.isDirectory() ) { + throw new GenericRuntimeException( "There is an error with the catalog folder in the .polypheny folder." ); + } + return PolyphenyHomeDirManager.getInstance().registerNewFile( "catalog/catalog.poly" ); + } + + + public synchronized void write( String data ) { + service.execute( () -> { + try { + BufferedWriter writer = new BufferedWriter( new FileWriter( backup, Charsets.ISO_8859_1 ) ); + writer.write( data ); + writer.flush(); + writer.close(); + } catch ( IOException e ) { + throw new GenericRuntimeException( e ); + } + } ); + } + + + public synchronized String read() { + String data; + try { + BufferedReader reader = new BufferedReader( new FileReader( backup, Charsets.ISO_8859_1 ) ); + data = reader.lines().collect( Collectors.joining() ); + reader.close(); + } catch ( IOException e ) { + throw new GenericRuntimeException( e ); + } + return data; + } + +} diff --git a/core/src/main/java/org/polypheny/db/catalog/impl/PolyCatalog.java b/core/src/main/java/org/polypheny/db/catalog/impl/PolyCatalog.java index 7e14bc83ba..43d254501e 100644 --- a/core/src/main/java/org/polypheny/db/catalog/impl/PolyCatalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/impl/PolyCatalog.java @@ -20,11 +20,13 @@ import io.activej.serializer.BinarySerializer; import io.activej.serializer.annotations.Deserialize; import io.activej.serializer.annotations.Serialize; +import java.beans.PropertyChangeListener; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.polypheny.db.adapter.AbstractAdapterSetting; @@ -98,15 +100,22 @@ public class PolyCatalog extends Catalog implements PolySerializable { public final Map storeCatalogs; private final IdBuilder idBuilder = IdBuilder.getInstance(); + private final Persister persister; @Getter private Snapshot snapshot; private String backup; + private final AtomicBoolean dirty = new AtomicBoolean( false ); + + @Getter + PropertyChangeListener changeListener = evt -> { + dirty.set( true ); + }; + public PolyCatalog() { - this( - new HashMap<>(), + this( new HashMap<>(), new HashMap<>(), new HashMap<>(), new HashMap<>(), @@ -129,13 +138,17 @@ public PolyCatalog( this.interfaces = new ConcurrentHashMap<>( interfaces ); this.adapterTemplates = new ConcurrentHashMap<>(); this.storeCatalogs = new ConcurrentHashMap<>(); - updateSnapshot(); + + this.persister = new Persister(); + + } @Override public void init() { //new DefaultInserter(); + updateSnapshot(); } @@ -167,7 +180,9 @@ public void updateSnapshot() { // update with newly generated physical entities this.snapshot = SnapshotBuilder.createSnapshot( idBuilder.getNewSnapshotId(), this, logicalCatalogs, allocationCatalogs ); + this.listeners.firePropertyChange( "snapshot", null, this.snapshot ); + this.dirty.set( false ); } @@ -187,17 +202,25 @@ private void addNamespaceIfNecessary( AllocationEntity entity ) { } - private void change() { + @Override + public void change() { // empty for now - updateSnapshot(); + this.dirty.set( true ); } - public void commit() { + public synchronized void commit() { + if ( !this.dirty.get() ) { + log.debug( "Nothing changed" ); + return; + } + log.debug( "commit" ); this.backup = serialize(); updateSnapshot(); + persister.write( backup ); + this.dirty.set( false ); } @@ -374,6 +397,7 @@ public void deleteNamespace( long id ) { public long addAdapter( String uniqueName, String clazz, AdapterType type, Map settings, DeployMode mode ) { long id = idBuilder.getNewAdapterId(); adapters.put( id, new CatalogAdapter( id, uniqueName, clazz, type, mode, settings ) ); + change(); return id; } @@ -384,12 +408,14 @@ public void updateAdapterSettings( long adapterId, Map newSettin return; } adapters.put( adapterId, adapters.get( adapterId ).toBuilder().settings( ImmutableMap.copyOf( newSettings ) ).build() ); + change(); } @Override public void deleteAdapter( long id ) { adapters.remove( id ); + change(); } @@ -399,7 +425,7 @@ public long addQueryInterface( String uniqueName, String clazz, Map> clazz, String adapterName, String description, List modes, List settings, Function4, Adapter> deployer ) { long id = idBuilder.getNewAdapterTemplateId(); adapterTemplates.put( id, new AdapterTemplate( id, clazz, adapterName, settings, modes, description, deployer ) ); + change(); return id; } @@ -421,6 +449,18 @@ public long addAdapterTemplate( Class> clazz, String adapte @Override public void removeAdapterTemplate( long templateId ) { adapterTemplates.remove( templateId ); + change(); + } + + + @Override + public void restore() { + this.backup = persister.read(); + if ( this.backup == null ) { + log.warn( "No file found to restore" ); + return; + } + rollback(); } diff --git a/core/src/main/java/org/polypheny/db/catalog/impl/allocation/PolyAllocDocCatalog.java b/core/src/main/java/org/polypheny/db/catalog/impl/allocation/PolyAllocDocCatalog.java index cd4f26d49e..25ea3eb048 100644 --- a/core/src/main/java/org/polypheny/db/catalog/impl/allocation/PolyAllocDocCatalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/impl/allocation/PolyAllocDocCatalog.java @@ -19,11 +19,13 @@ import io.activej.serializer.BinarySerializer; import io.activej.serializer.annotations.Deserialize; import io.activej.serializer.annotations.Serialize; +import java.beans.PropertyChangeSupport; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import lombok.Getter; import lombok.Value; +import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.IdBuilder; import org.polypheny.db.catalog.catalogs.AllocationDocumentCatalog; import org.polypheny.db.catalog.entity.allocation.AllocationCollection; @@ -72,6 +74,15 @@ public PolyAllocDocCatalog( this.collections = new ConcurrentHashMap<>( collections ); this.placements = new ConcurrentHashMap<>( placements ); this.partitions = new ConcurrentHashMap<>( partitions ); + listeners.addPropertyChangeListener( Catalog.getInstance().getChangeListener() ); + } + + + PropertyChangeSupport listeners = new PropertyChangeSupport( this ); + + + public void change() { + listeners.firePropertyChange( "change", null, null ); } diff --git a/core/src/main/java/org/polypheny/db/catalog/impl/allocation/PolyAllocGraphCatalog.java b/core/src/main/java/org/polypheny/db/catalog/impl/allocation/PolyAllocGraphCatalog.java index b738b42ec5..44492c6cd1 100644 --- a/core/src/main/java/org/polypheny/db/catalog/impl/allocation/PolyAllocGraphCatalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/impl/allocation/PolyAllocGraphCatalog.java @@ -19,11 +19,13 @@ import io.activej.serializer.BinarySerializer; import io.activej.serializer.annotations.Deserialize; import io.activej.serializer.annotations.Serialize; +import java.beans.PropertyChangeSupport; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import lombok.Getter; import lombok.Value; +import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.IdBuilder; import org.polypheny.db.catalog.catalogs.AllocationGraphCatalog; import org.polypheny.db.catalog.entity.allocation.AllocationGraph; @@ -74,6 +76,15 @@ public PolyAllocGraphCatalog( this.graphs = new ConcurrentHashMap<>( graphs ); this.placements = new ConcurrentHashMap<>( placements ); this.partitions = new ConcurrentHashMap<>( partitions ); + listeners.addPropertyChangeListener( Catalog.getInstance().getChangeListener() ); + } + + + PropertyChangeSupport listeners = new PropertyChangeSupport( this ); + + + public void change() { + listeners.firePropertyChange( "change", null, null ); } diff --git a/core/src/main/java/org/polypheny/db/catalog/impl/allocation/PolyAllocRelCatalog.java b/core/src/main/java/org/polypheny/db/catalog/impl/allocation/PolyAllocRelCatalog.java index 8ab568783d..79ad60d763 100644 --- a/core/src/main/java/org/polypheny/db/catalog/impl/allocation/PolyAllocRelCatalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/impl/allocation/PolyAllocRelCatalog.java @@ -19,6 +19,7 @@ import io.activej.serializer.BinarySerializer; import io.activej.serializer.annotations.Deserialize; import io.activej.serializer.annotations.Serialize; +import java.beans.PropertyChangeSupport; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -28,6 +29,7 @@ import lombok.Value; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.NotImplementedException; +import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.IdBuilder; import org.polypheny.db.catalog.catalogs.AllocationRelationalCatalog; import org.polypheny.db.catalog.entity.allocation.AllocationColumn; @@ -109,6 +111,15 @@ public PolyAllocRelCatalog( this.partitions = new ConcurrentHashMap<>( partitions ); this.properties = new ConcurrentHashMap<>( properties ); this.placements = new ConcurrentHashMap<>( placements ); + listeners.addPropertyChangeListener( Catalog.getInstance().getChangeListener() ); + } + + + PropertyChangeSupport listeners = new PropertyChangeSupport( this ); + + + public void change() { + listeners.firePropertyChange( "change", null, null ); } diff --git a/core/src/main/java/org/polypheny/db/catalog/impl/logical/DocumentCatalog.java b/core/src/main/java/org/polypheny/db/catalog/impl/logical/DocumentCatalog.java index 3096ab4fc8..55c59eede8 100644 --- a/core/src/main/java/org/polypheny/db/catalog/impl/logical/DocumentCatalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/impl/logical/DocumentCatalog.java @@ -19,13 +19,13 @@ import io.activej.serializer.BinarySerializer; import io.activej.serializer.annotations.Deserialize; import io.activej.serializer.annotations.Serialize; +import java.beans.PropertyChangeSupport; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import lombok.Builder; import lombok.Getter; import lombok.Value; -import lombok.experimental.NonFinal; import lombok.experimental.SuperBuilder; +import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.IdBuilder; import org.polypheny.db.catalog.catalogs.LogicalCatalog; import org.polypheny.db.catalog.catalogs.LogicalDocumentCatalog; @@ -61,14 +61,17 @@ public DocumentCatalog( this.logicalNamespace = logicalNamespace; this.collections = new ConcurrentHashMap<>( collections ); + listeners.addPropertyChangeListener( Catalog.getInstance().getChangeListener() ); } - @NonFinal - @Builder.Default - boolean openChanges = false; + PropertyChangeSupport listeners = new PropertyChangeSupport( this ); + public void change() { + listeners.firePropertyChange( "change", null, null ); + } + @Override public PolySerializable copy() { return PolySerializable.deserialize( serialize(), DocumentCatalog.class ); @@ -80,6 +83,7 @@ public LogicalCollection addCollection( String name, EntityType entity, boolean long id = idBuilder.getNewLogicalId(); LogicalCollection collection = new LogicalCollection( id, name, logicalNamespace.id, entity, modifiable ); collections.put( id, collection ); + change(); return collection; } @@ -87,6 +91,7 @@ public LogicalCollection addCollection( String name, EntityType entity, boolean @Override public void deleteCollection( long id ) { collections.remove( id ); + change(); } @@ -94,6 +99,7 @@ public void deleteCollection( long id ) { public void renameCollection( LogicalCollection collection, String newName ) { LogicalCollection newCollection = collection.toBuilder().name( newName ).build(); collections.put( newCollection.id, newCollection ); + change(); } diff --git a/core/src/main/java/org/polypheny/db/catalog/impl/logical/GraphCatalog.java b/core/src/main/java/org/polypheny/db/catalog/impl/logical/GraphCatalog.java index 2d564bfb80..367fc7ab16 100644 --- a/core/src/main/java/org/polypheny/db/catalog/impl/logical/GraphCatalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/impl/logical/GraphCatalog.java @@ -19,13 +19,13 @@ import io.activej.serializer.BinarySerializer; import io.activej.serializer.annotations.Deserialize; import io.activej.serializer.annotations.Serialize; +import java.beans.PropertyChangeSupport; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import lombok.Builder; import lombok.Getter; import lombok.Value; -import lombok.experimental.NonFinal; import lombok.experimental.SuperBuilder; +import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.IdBuilder; import org.polypheny.db.catalog.catalogs.LogicalCatalog; import org.polypheny.db.catalog.catalogs.LogicalGraphCatalog; @@ -49,11 +49,6 @@ public class GraphCatalog implements PolySerializable, LogicalGraphCatalog { public ConcurrentHashMap graphs; - @NonFinal - @Builder.Default - boolean openChanges = false; - - public GraphCatalog( LogicalNamespace logicalNamespace ) { this( logicalNamespace, new ConcurrentHashMap<>() ); } @@ -67,6 +62,15 @@ public GraphCatalog( // already add only graph addGraph( logicalNamespace.id, logicalNamespace.name, true ); + listeners.addPropertyChangeListener( Catalog.getInstance().getChangeListener() ); + } + + + PropertyChangeSupport listeners = new PropertyChangeSupport( this ); + + + public void change() { + listeners.firePropertyChange( "change", null, null ); } @@ -84,13 +88,13 @@ public LogicalCatalog withLogicalNamespace( LogicalNamespace namespace ) { @Override public void addGraphAlias( long graphId, String alias, boolean ifNotExists ) { - + change(); } @Override public void removeGraphAlias( long graphId, String alias, boolean ifExists ) { - + change(); } @@ -98,6 +102,7 @@ public void removeGraphAlias( long graphId, String alias, boolean ifExists ) { public LogicalGraph addGraph( long id, String name, boolean modifiable ) { LogicalGraph graph = new LogicalGraph( id, name, modifiable, logicalNamespace.caseSensitive ); graphs.put( id, graph ); + change(); return graph; } @@ -105,6 +110,7 @@ public LogicalGraph addGraph( long id, String name, boolean modifiable ) { @Override public void deleteGraph( long id ) { graphs.remove( id ); + change(); } } diff --git a/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java b/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java index fc5adb14f7..28bc7a53d9 100644 --- a/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java @@ -29,10 +29,8 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; -import lombok.Builder; import lombok.Getter; import lombok.Value; -import lombok.experimental.NonFinal; import lombok.experimental.SuperBuilder; import org.polypheny.db.algebra.AlgCollation; import org.polypheny.db.algebra.AlgNode; @@ -106,12 +104,6 @@ public class RelationalCatalog implements PolySerializable, LogicalRelationalCat public Map constraints; - @NonFinal - @Builder.Default - boolean openChanges = false; - - PropertyChangeSupport listeners = new PropertyChangeSupport( this ); - List tablesFlaggedForDeletion = new ArrayList<>(); @@ -130,19 +122,24 @@ public RelationalCatalog( this.keys = new ConcurrentHashMap<>( keys ); this.constraints = new ConcurrentHashMap<>( constraints ); this.nodes = new ConcurrentHashMap<>(); + listeners.addPropertyChangeListener( Catalog.getInstance().getChangeListener() ); } - public RelationalCatalog( LogicalNamespace namespace ) { - this( namespace, new HashMap<>(), new HashMap<>(), new HashMap<>(), new HashMap<>(), new HashMap<>() ); - } + PropertyChangeSupport listeners = new PropertyChangeSupport( this ); public void change() { - openChanges = true; + listeners.firePropertyChange( "change", null, null ); } + public RelationalCatalog( LogicalNamespace namespace ) { + this( namespace, new HashMap<>(), new HashMap<>(), new HashMap<>(), new HashMap<>(), new HashMap<>() ); + } + + + @Override public PolySerializable copy() { return PolySerializable.deserialize( serialize(), RelationalCatalog.class ); @@ -160,6 +157,7 @@ public LogicalTable addTable( String name, EntityType entityType, boolean modifi long id = idBuilder.getNewLogicalId(); LogicalTable table = new LogicalTable( id, name, logicalNamespace.id, entityType, null, modifiable ); tables.put( id, table ); + change(); return table; } @@ -172,7 +170,7 @@ public LogicalView addView( String name, long namespaceId, boolean modifiable, A tables.put( id, view ); nodes.put( id, definition ); - + change(); return view; } @@ -195,7 +193,7 @@ public LogicalMaterializedView addMaterializedView( final String name, long name tables.put( id, materializedViewTable ); nodes.put( id, definition ); - + change(); return materializedViewTable; } @@ -203,6 +201,7 @@ public LogicalMaterializedView addMaterializedView( final String name, long name @Override public void renameTable( long tableId, String name ) { tables.put( tableId, tables.get( tableId ).toBuilder().name( name ).build() ); + change(); } @@ -212,6 +211,7 @@ public void deleteTable( long tableId ) { columns.remove( columnId ); } tables.remove( tableId ); + change(); } @@ -220,6 +220,7 @@ public void setPrimaryKey( long tableId, Long keyId ) { tables.put( tableId, tables.get( tableId ).toBuilder().primaryKey( keyId ).build() ); keys.put( keyId, new LogicalPrimaryKey( keys.get( keyId ) ) ); + change(); } @@ -245,6 +246,7 @@ public LogicalIndex addIndex( long tableId, List columnIds, boolean unique indexes.put( id, index ); } listeners.firePropertyChange( "index", null, keyId ); + change(); return index; } @@ -265,7 +267,7 @@ private long addKey( long tableId, List columnIds, EnforcementTime enforce synchronized ( this ) { keys.put( id, key ); } - listeners.firePropertyChange( "key", null, key ); + change(); return id; } @@ -273,18 +275,21 @@ private long addKey( long tableId, List columnIds, EnforcementTime enforce @Override public void setIndexPhysicalName( long indexId, String physicalName ) { indexes.put( indexId, indexes.get( indexId ).toBuilder().physicalName( physicalName ).build() ); + change(); } @Override public void deleteIndex( long indexId ) { indexes.remove( indexId ); + change(); } @Override public void deleteKey( long id ) { keys.remove( id ); + change(); } @@ -293,6 +298,7 @@ public LogicalColumn addColumn( String name, long tableId, int position, PolyTyp long id = idBuilder.getNewFieldId(); LogicalColumn column = new LogicalColumn( id, name, tableId, logicalNamespace.id, position, type, collectionsType, length, scale, dimension, cardinality, nullable, collation, null ); columns.put( id, column ); + change(); return column; } @@ -300,12 +306,14 @@ public LogicalColumn addColumn( String name, long tableId, int position, PolyTyp @Override public void renameColumn( long columnId, String name ) { columns.put( columnId, columns.get( columnId ).toBuilder().name( name ).build() ); + change(); } @Override public void setColumnPosition( long columnId, int position ) { columns.put( columnId, columns.get( columnId ).toBuilder().position( position ).build() ); + change(); } @@ -316,24 +324,28 @@ public void setColumnType( long columnId, PolyType type, PolyType collectionsTyp } columns.put( columnId, columns.get( columnId ).toBuilder().type( type ).length( length ).scale( scale ).dimension( dimension ).cardinality( cardinality ).build() ); + change(); } @Override public void setNullable( long columnId, boolean nullable ) { columns.put( columnId, columns.get( columnId ).toBuilder().nullable( nullable ).build() ); + change(); } @Override public void setCollation( long columnId, Collation collation ) { columns.put( columnId, columns.get( columnId ).toBuilder().collation( collation ).build() ); + change(); } @Override public void deleteColumn( long columnId ) { columns.remove( columnId ); + change(); } @@ -341,6 +353,7 @@ public void deleteColumn( long columnId ) { public LogicalColumn setDefaultValue( long columnId, PolyType type, String defaultValue ) { LogicalColumn column = columns.get( columnId ).toBuilder().defaultValue( new CatalogDefaultValue( columnId, type, defaultValue, "defaultValue" ) ).build(); columns.put( columnId, column ); + change(); return column; } @@ -348,6 +361,7 @@ public LogicalColumn setDefaultValue( long columnId, PolyType type, String defau @Override public void deleteDefaultValue( long columnId ) { columns.put( columnId, columns.get( columnId ).toBuilder().defaultValue( null ).build() ); + change(); } @@ -375,6 +389,7 @@ public void addPrimaryKey( long tableId, List columnIds ) { } long keyId = getOrAddKey( tableId, columnIds, EnforcementTime.ON_QUERY ); setPrimaryKey( tableId, keyId ); + change(); } @@ -412,7 +427,7 @@ private void deleteKeyIfNoLongerUsed( Long keyId ) { synchronized ( this ) { keys.remove( keyId ); } - listeners.firePropertyChange( "key", key, null ); + change(); } @@ -476,10 +491,9 @@ public void addForeignKey( long tableId, List columnIds, long referencesTa onDelete ); synchronized ( this ) { keys.put( keyId, key ); + change(); } return; - - } } @@ -492,12 +506,13 @@ public void addUniqueConstraint( long tableId, String constraintName, List List logicalConstraints = constraints.values().stream() .filter( c -> c.keyId == keyId && c.type == ConstraintType.UNIQUE ) .collect( Collectors.toList() ); - if ( logicalConstraints.size() > 0 ) { + if ( !logicalConstraints.isEmpty() ) { throw new GenericRuntimeException( "There is already a unique constraint!" ); } long id = idBuilder.getNewConstraintId(); synchronized ( this ) { constraints.put( id, new LogicalConstraint( id, keyId, ConstraintType.UNIQUE, constraintName, Objects.requireNonNull( keys.get( keyId ) ) ) ); + change(); } } @@ -519,6 +534,7 @@ public void deletePrimaryKey( long tableId ) { setPrimaryKey( tableId, null ); deleteKeyIfNoLongerUsed( table.primaryKey ); } + change(); } @@ -529,6 +545,7 @@ public void deleteForeignKey( long foreignKeyId ) { //keys.remove( logicalForeignKey.id ); deleteKeyIfNoLongerUsed( logicalForeignKey.id ); } + change(); } @@ -545,7 +562,7 @@ public void deleteConstraint( long constraintId ) { constraints.remove( logicalConstraint.id ); } deleteKeyIfNoLongerUsed( logicalConstraint.keyId ); - + change(); } @@ -560,7 +577,7 @@ public void updateMaterializedViewRefreshTime( long materializedViewId ) { synchronized ( this ) { tables.put( materializedViewId, old.toBuilder().materializedCriteria( materializedCriteria ).build() ); } - + change(); } diff --git a/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java b/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java index 9f2e8fb34f..f94ab4dd76 100644 --- a/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java +++ b/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java @@ -470,7 +470,7 @@ public void join( final long millis ) throws InterruptedException { Catalog.getInstance().updateSnapshot(); - restoreDefaults( authenticator, catalog ); + restore( authenticator, catalog ); // Add tracker, which rechecks constraints after enabling ConstraintTracker tracker = new ConstraintTracker( transactionManager ); @@ -525,16 +525,24 @@ private Catalog startCatalog() { } - private void restoreDefaults( Authenticator authenticator, Catalog catalog ) { + private void restore( Authenticator authenticator, Catalog catalog ) { Catalog.defaultStore = AdapterTemplate.fromString( defaultStoreName, AdapterType.STORE ); Catalog.defaultSource = AdapterTemplate.fromString( defaultSourceName, AdapterType.SOURCE ); PolyPluginManager.startUp( transactionManager, authenticator ); - catalog.updateSnapshot(); - DefaultInserter.restoreData( DdlManager.getInstance() ); - DefaultInserter.restoreInterfacesIfNecessary(); + + if ( !resetCatalog ) { + Catalog.getInstance().restore(); + } + restoreDefaults( catalog ); + QueryInterfaceManager.getInstance().restoreInterfaces( catalog.getSnapshot() ); // AdapterManager.getInstance().restoreAdapters(); + commitRestore(); + } + + + private void commitRestore() { Transaction trx = null; try { trx = transactionManager.startTransaction( @@ -542,8 +550,6 @@ private void restoreDefaults( Authenticator authenticator, Catalog catalog ) { Catalog.defaultNamespaceId, false, "Catalog Startup" ); - Catalog.getInstance().restoreColumnAllocations( trx ); - Catalog.getInstance().restoreViews( trx ); trx.commit(); } catch ( TransactionException e ) { try { @@ -556,4 +562,11 @@ private void restoreDefaults( Authenticator authenticator, Catalog catalog ) { } + private static void restoreDefaults( Catalog catalog ) { + catalog.updateSnapshot(); + DefaultInserter.restoreData( DdlManager.getInstance() ); + DefaultInserter.restoreInterfacesIfNecessary(); + } + + } diff --git a/dbms/src/main/java/org/polypheny/db/transaction/TransactionManagerImpl.java b/dbms/src/main/java/org/polypheny/db/transaction/TransactionManagerImpl.java index 957c228cd3..6913310497 100644 --- a/dbms/src/main/java/org/polypheny/db/transaction/TransactionManagerImpl.java +++ b/dbms/src/main/java/org/polypheny/db/transaction/TransactionManagerImpl.java @@ -90,7 +90,12 @@ private Transaction startTransaction( CatalogUser user, LogicalNamespace default @Override public Transaction startTransaction( long userId, long defaultNamespaceId, boolean analyze, String origin ) { - return startTransaction( Catalog.snapshot().getUser( userId ).orElseThrow(), Catalog.snapshot().getNamespace( defaultNamespaceId ).orElseThrow(), analyze, origin, MultimediaFlavor.DEFAULT ); + return startTransaction( + Catalog.snapshot().getUser( userId ).orElseThrow(), + Catalog.snapshot().getNamespace( defaultNamespaceId ).orElseThrow(), + analyze, + origin, + MultimediaFlavor.DEFAULT ); } diff --git a/plugins/file-adapter/src/main/java/org/polypheny/db/adapter/file/source/Qfs.java b/plugins/file-adapter/src/main/java/org/polypheny/db/adapter/file/source/Qfs.java index 4ef6f5ec56..48765ff8aa 100644 --- a/plugins/file-adapter/src/main/java/org/polypheny/db/adapter/file/source/Qfs.java +++ b/plugins/file-adapter/src/main/java/org/polypheny/db/adapter/file/source/Qfs.java @@ -29,10 +29,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.StringJoiner; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.NotImplementedException; +import org.jetbrains.annotations.NotNull; import org.polypheny.db.adapter.DataSource; import org.polypheny.db.adapter.DeployMode; import org.polypheny.db.adapter.annotations.AdapterProperties; @@ -208,16 +210,7 @@ protected void validateSettings( Map newSettings, boolean initia int numberOfWhitelistEntries = 0; File whitelistFolder = PolyphenyHomeDirManager.getInstance().registerNewFolder( "config" ); File whitelist = new File( whitelistFolder, "whitelist.config" ); - String path = whitelist.getAbsolutePath(); - if ( !whitelist.exists() ) { - try ( FileWriter fw = new FileWriter( whitelist ); PrintWriter pw = new PrintWriter( fw ) ) { - pw.println( "# A list of allowed directories for the Query File System (QFS) data source adapter" ); - pw.println( "# The list must be non-empty. A QFS directory will only be accepted if it is listed here or is a subdirectory of a directory listed here." ); - } catch ( IOException e ) { - throw new RuntimeException( "Could not write QFS whitelist file " + path, e ); - } - throw new RuntimeException( "The QFS whitelist did not exist. A new one was generated. Make sure to add at least one entry to the whitelist before deploying a QFS data source. The whitelist is located in " + path ); - } + String path = getString( whitelist ); try ( FileInputStream fis = new FileInputStream( whitelist ); BufferedReader br = new BufferedReader( new InputStreamReader( fis ) ) ) { String line; while ( (line = br.readLine()) != null ) { @@ -238,14 +231,30 @@ protected void validateSettings( Map newSettings, boolean initia } } } catch ( IOException e ) { - throw new RuntimeException( "Could not read QFS whitelist. A whitelist must be present and contain at least one entry. It must be located in " + path, e ); + throw new GenericRuntimeException( "Could not read QFS whitelist. A whitelist must be present and contain at least one entry. It must be located in " + path, e ); } if ( numberOfWhitelistEntries == 0 ) { - throw new RuntimeException( "The QFS whitelist must contain at least one entry. The file can be edited in " + path ); + throw new GenericRuntimeException( "The QFS whitelist must contain at least one entry. The file can be edited in " + path ); } if ( !allowed ) { - throw new RuntimeException( "The selected path (" + newSettings.get( "rootDir" ) + ") is not allowed. It must be a subdirectory of one of the following paths:\n" + allowedPaths.toString() ); + throw new GenericRuntimeException( "The selected path (" + newSettings.get( "rootDir" ) + ") is not allowed. It must be a subdirectory of one of the following paths:\n" + allowedPaths.toString() ); + } + } + + + @NotNull + private static String getString( File whitelist ) { + String path = whitelist.getAbsolutePath(); + if ( !whitelist.exists() ) { + try ( FileWriter fw = new FileWriter( whitelist ); PrintWriter pw = new PrintWriter( fw ) ) { + pw.println( "# A list of allowed directories for the Query File System (QFS) data source adapter" ); + pw.println( "# The list must be non-empty. A QFS directory will only be accepted if it is listed here or is a subdirectory of a directory listed here." ); + } catch ( IOException e ) { + throw new GenericRuntimeException( "Could not write QFS whitelist file " + path, e ); + } + throw new GenericRuntimeException( "The QFS whitelist did not exist. A new one was generated. Make sure to add at least one entry to the whitelist before deploying a QFS data source. The whitelist is located in " + path ); } + return path; } @@ -341,21 +350,28 @@ protected void registerInformationPage( String uniqueName ) { im.addGroup( group ); informationGroups.add( group ); - InformationTable table = new InformationTable( - group, - Arrays.asList( "Position", "Column Name", "Type", "Nullable", "Primary" ) ); - for ( ExportedColumn exportedColumn : entry.getValue() ) { - table.addRow( - exportedColumn.physicalPosition, - exportedColumn.name, - exportedColumn.getDisplayType(), - exportedColumn.nullable ? "✔" : "", - exportedColumn.primary ? "✔" : "" - ); - } + InformationTable table = getInformationTable( entry, group ); im.registerInformation( table ); informationElements.add( table ); } } + + @NotNull + private static InformationTable getInformationTable( Entry> entry, InformationGroup group ) { + InformationTable table = new InformationTable( + group, + Arrays.asList( "Position", "Column Name", "Type", "Nullable", "Primary" ) ); + for ( ExportedColumn exportedColumn : entry.getValue() ) { + table.addRow( + exportedColumn.physicalPosition, + exportedColumn.name, + exportedColumn.getDisplayType(), + exportedColumn.nullable ? "✔" : "", + exportedColumn.primary ? "✔" : "" + ); + } + return table; + } + } diff --git a/webui/src/main/java/org/polypheny/db/webui/Crud.java b/webui/src/main/java/org/polypheny/db/webui/Crud.java index ad5949e4b2..a982bc131d 100644 --- a/webui/src/main/java/org/polypheny/db/webui/Crud.java +++ b/webui/src/main/java/org/polypheny/db/webui/Crud.java @@ -3684,7 +3684,6 @@ public void getAvailablePlugins( Context ctx ) { @Override public void propertyChange( PropertyChangeEvent evt ) { - log.warn( "changed" ); authCrud.broadcast( SnapshotModel.from( Catalog.snapshot() ) ); } diff --git a/webui/src/main/java/org/polypheny/db/webui/auth/AuthCrud.java b/webui/src/main/java/org/polypheny/db/webui/auth/AuthCrud.java index b186943c6e..4632c7f9b1 100644 --- a/webui/src/main/java/org/polypheny/db/webui/auth/AuthCrud.java +++ b/webui/src/main/java/org/polypheny/db/webui/auth/AuthCrud.java @@ -30,7 +30,7 @@ public class AuthCrud { private final ObjectMapper mapper = new ObjectMapper(); private final Crud crud; - private final ConcurrentHashMap partners = new ConcurrentHashMap<>(); + private final ConcurrentHashMap partners = new ConcurrentHashMap<>(); public AuthCrud( Crud crud ) { @@ -40,13 +40,19 @@ public AuthCrud( Crud crud ) { public void register( RegisterRequest registerRequest, WsMessageContext context ) { String id = registerRequest.source; - if ( id != null && partners.containsKey( UUID.fromString( id ) ) ) { + if ( id != null ) { log.warn( "Partner " + id + " already registered" ); + if ( partners.containsKey( UUID.fromString( id ) ) ) { + partners.get( UUID.fromString( id ) ).addContext( context ); + } else { + partners.put( UUID.fromString( id ), new PartnerConnection( UUID.fromString( id ), context ) ); + } + context.send( new RegisterRequest( id ) ); return; } - PartnerStatus status = new PartnerStatus( context ); + PartnerConnection status = new PartnerConnection( context ); log.warn( "New partner with id " + status.id + " registered" ); partners.put( status.id, status ); context.send( new RegisterRequest( status.id.toString() ) ); @@ -64,10 +70,9 @@ public void deregister( Context context ) { public void broadcast( E msg ) { - for ( PartnerStatus status : partners.values() ) { - status.getContext().send( msg ); + for ( PartnerConnection connection : partners.values() ) { + connection.broadcast( msg ); } - //partners.values().forEach( p -> HttpServer.getInstance().getWebSocketHandler(). ); } } diff --git a/webui/src/main/java/org/polypheny/db/webui/auth/PartnerConnection.java b/webui/src/main/java/org/polypheny/db/webui/auth/PartnerConnection.java new file mode 100644 index 0000000000..fedd62927a --- /dev/null +++ b/webui/src/main/java/org/polypheny/db/webui/auth/PartnerConnection.java @@ -0,0 +1,62 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.webui.auth; + +import io.javalin.websocket.WsMessageContext; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import lombok.Getter; + +@Getter +public class PartnerConnection { + + public final UUID id; + public final List contexts; + + + public PartnerConnection( UUID id, WsMessageContext... contexts ) { + this.id = id; + this.contexts = new ArrayList<>( List.of( contexts ) ); + } + + + public PartnerConnection( WsMessageContext... contexts ) { + this( UUID.randomUUID(), contexts ); + } + + + public void broadcast( E msg ) { + List invalid = new ArrayList<>(); + for ( WsMessageContext context : contexts ) { + if ( !context.session.isOpen() ) { + invalid.add( context ); + continue; + } + context.send( msg ); + } + invalid.forEach( contexts::remove ); + } + + + public void addContext( WsMessageContext context ) { + if ( !contexts.contains( context ) ) { + contexts.add( context ); + } + } + +} diff --git a/webui/src/main/java/org/polypheny/db/webui/auth/PartnerStatus.java b/webui/src/main/java/org/polypheny/db/webui/auth/PartnerStatus.java deleted file mode 100644 index 4639c8b3d3..0000000000 --- a/webui/src/main/java/org/polypheny/db/webui/auth/PartnerStatus.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2019-2023 The Polypheny Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.polypheny.db.webui.auth; - -import io.javalin.websocket.WsMessageContext; -import java.util.UUID; -import lombok.Getter; - -@Getter -public class PartnerStatus { - - public final UUID id; - private final WsMessageContext context; - - - public PartnerStatus( UUID id, WsMessageContext context ) { - this.id = id; - this.context = context; - } - - - public PartnerStatus( WsMessageContext context ) { - this( UUID.randomUUID(), context ); - } - -} diff --git a/webui/src/main/java/org/polypheny/db/webui/models/Namespace.java b/webui/src/main/java/org/polypheny/db/webui/models/Namespace.java index 1b2e1cb753..8a4e2e463d 100644 --- a/webui/src/main/java/org/polypheny/db/webui/models/Namespace.java +++ b/webui/src/main/java/org/polypheny/db/webui/models/Namespace.java @@ -17,6 +17,8 @@ package org.polypheny.db.webui.models; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import javax.annotation.Nullable; import lombok.Getter; import org.polypheny.db.catalog.logistic.NamespaceType; @@ -28,16 +30,23 @@ @Getter public class Namespace { + @JsonSerialize private String name; + @JsonSerialize private NamespaceType type; + @JsonSerialize private final String store; // fields for creation + @JsonSerialize private boolean create; + @JsonSerialize private String authorization; // fields for deletion + @JsonSerialize private boolean drop; + @JsonSerialize private boolean cascade; @@ -47,7 +56,7 @@ public class Namespace { * @param name name of the schema * @param type type of the schema, e.g. relational */ - public Namespace( final String name, final NamespaceType type, @Nullable final String store ) { + public Namespace( final @JsonProperty("name") String name, @JsonProperty("type") final NamespaceType type, @JsonProperty("store") @Nullable final String store ) { this.name = name; this.type = type;