Merge branch 'unite-reachability-stores' into HEAD

domain/consensus/factory.go

@@ -497,12 +497,14 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas
 	}
 
 	if isOldReachabilityInitialized {
+		log.Infof("Migrating the database to the new reachability structure")
 		stagingArea := model.NewStagingArea()
 		dbTx, err := dbManager.Begin()
 		if err != nil {
 			return nil, err
 		}
 
+		log.Infof("Deleting the new reachability store (in case a previous migration failed)")
 		err = newReachabilityDataStore.Delete(dbTx)
 		if err != nil {
 			return nil, err
@@ -513,11 +515,13 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas
 			return nil, err
 		}
 
+		log.Infof("Initializing new reachability store")
 		err = newReachabilityManager.Init(stagingArea)
 		if err != nil {
 			return nil, err
 		}
 
+		log.Infof("Committing changes")
 		err = staging.CommitAllChanges(dbManager, stagingArea)
 		if err != nil {
 			return nil, err
@@ -528,23 +532,34 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas
 			return nil, err
 		}
 
+		// Because we use reachabilityDataStores[0] for the migration indication, this is the only store we
+		// need to delete in an atomic way. For the rest of the stores we don't need database transactions,
+		// so we can delete them directly, hence saving memory.
+		log.Infof("Deleting the old level 0 reachability store")
 		dbTx, err = dbManager.Begin()
 		if err != nil {
 			return nil, err
 		}
 
-		for _, store := range reachabilityDataStores {
-			err = store.Delete(dbTx)
-			if err != nil {
-				return nil, err
-			}
+		err = reachabilityDataStores[0].Delete(dbTx)
+		if err != nil {
+			return nil, err
 		}
 
 		err = dbTx.Commit()
 		if err != nil {
 			return nil, err
 		}
 
+		for i, store := range reachabilityDataStores[1:] {
+			log.Infof("Deleting the old level %d reachability store", i+1)
+			err = store.Delete(dbManager)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		log.Infof("Restarting the consensus")
 		return f.NewConsensus(config, db, dbPrefix)
 	}
 

domain/consensus/processes/pruningproofmanager/pruningproofmanager.go

@@ -622,16 +622,19 @@ func (ppm *pruningProofManager) dagProcesses(
 }
 
 func (ppm *pruningProofManager) RebuildReachability(targetReachabilityDataStore model.ReachabilityDataStore) error {
+	log.Infof("Building the pruning proof for RebuildReachability")
 	pruningPointProof, err := ppm.buildPruningPointProof(model.NewStagingArea())
 	if err != nil {
 		return err
 	}
 
+	log.Infof("Populating the reachability store with data from the pruning point proof")
 	err = ppm.populateProofReachabilityAndHeaders(pruningPointProof, targetReachabilityDataStore)
 	if err != nil {
 		return err
 	}
 
+	log.Infof("Rebuilding reachability with data from pruning point antipast")
 	pruningPointAndItsAnticone, err := ppm.pruningManager.PruningPointAndItsAnticone()
 	if err != nil {
 		return err
@@ -687,6 +690,20 @@ func (ppm *pruningProofManager) RebuildReachability(targetReachabilityDataStore
 		}
 
 		if !hasReachabilityData {
+			ghostdagData, err := ppm.ghostdagDataStores[0].Get(ppm.databaseContext, stagingArea, current, false)
+			if err != nil {
+				return err
+			}
+
+			ghostdagDataWithoutPrunedBlocks, changed, err := ppm.ghostdagDataWithoutPrunedBlocks(stagingArea, targetReachabilityDataStore, ghostdagData)
+			if err != nil {
+				return err
+			}
+
+			if changed {
+				ppm.ghostdagDataStores[0].Stage(stagingArea, current, ghostdagDataWithoutPrunedBlocks, false)
+				ppm.ghostdagDataStores[0].Stage(stagingArea, current, ghostdagData, true)
+			}
 			err = targetReachabilityManager.AddBlock(stagingArea, current)
 			if err != nil {
 				return err
@@ -733,6 +750,62 @@ func (ppm *pruningProofManager) RebuildReachability(targetReachabilityDataStore
 	return staging.CommitAllChanges(ppm.databaseContext, stagingArea)
 }
 
+func (ppm *pruningProofManager) ghostdagDataWithoutPrunedBlocks(stagingArea *model.StagingArea, targetReachabilityDataStore model.ReachabilityDataStore,
+	data *externalapi.BlockGHOSTDAGData) (*externalapi.BlockGHOSTDAGData, bool, error) {
+
+	changed := false
+	mergeSetBlues := make([]*externalapi.DomainHash, 0, len(data.MergeSetBlues()))
+	for _, blockHash := range data.MergeSetBlues() {
+		hasReachabilityData, err := targetReachabilityDataStore.HasReachabilityData(ppm.databaseContext, stagingArea, blockHash)
+		if err != nil {
+			return nil, false, err
+		}
+		if !hasReachabilityData {
+			changed = true
+			if data.SelectedParent().Equal(blockHash) {
+				mergeSetBlues = append(mergeSetBlues, model.VirtualGenesisBlockHash)
+			}
+			continue
+		}
+
+		mergeSetBlues = append(mergeSetBlues, blockHash)
+	}
+
+	mergeSetReds := make([]*externalapi.DomainHash, 0, len(data.MergeSetReds()))
+	for _, blockHash := range data.MergeSetReds() {
+		hasReachabilityData, err := targetReachabilityDataStore.HasReachabilityData(ppm.databaseContext, stagingArea, blockHash)
+		if err != nil {
+			return nil, false, err
+		}
+		if !hasReachabilityData {
+			changed = true
+			continue
+		}
+
+		mergeSetReds = append(mergeSetReds, blockHash)
+	}
+
+	selectedParent := data.SelectedParent()
+	hasReachabilityData, err := targetReachabilityDataStore.HasReachabilityData(ppm.databaseContext, stagingArea, data.SelectedParent())
+	if err != nil {
+		return nil, false, err
+	}
+
+	if !hasReachabilityData {
+		changed = true
+		selectedParent = model.VirtualGenesisBlockHash
+	}
+
+	return externalapi.NewBlockGHOSTDAGData(
+		data.BlueScore(),
+		data.BlueWork(),
+		selectedParent,
+		mergeSetBlues,
+		mergeSetReds,
+		data.BluesAnticoneSizes(),
+	), changed, nil
+}
+
 func (ppm *pruningProofManager) populateProofReachabilityAndHeaders(pruningPointProof *externalapi.PruningPointProof,
 	targetReachabilityDataStore model.ReachabilityDataStore) error {
 	// We build a DAG of all multi-level relations between blocks in the proof. We make a upHeap of all blocks, so we can iterate