Skip to content

Commit

Permalink
mercy k-mer simple test through
Browse files Browse the repository at this point in the history
  • Loading branch information
@rhinempi
rhinempi committed Oct 16, 2024
1 parent 767e22f commit 96e697b
Show file tree
Hide file tree
Showing 4 changed files with 74 additions and 63 deletions.
24 changes: 12 additions & 12 deletions src/main/java/uni/bielefeld/cmg/reflexiv/pipeline/Pipelines.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -686,7 +686,7 @@ public void reflexivDSDynamicAssemblyStepsPipe() throws IOException{
if (checkOutputFile(param.outputPath + "/Assembly_intermediate/01Iteration" + param.startIteration + "_" + param.endIteration)){
info.readMessage("Removing: " + param.inputKmerPath.substring(0,param.inputKmerPath.length()-6));
info.screenDump();
cleanDiskStorage(param.inputKmerPath.substring(0,param.inputKmerPath.length()-6));
// cleanDiskStorage(param.inputKmerPath.substring(0,param.inputKmerPath.length()-6));
}else{
info.readMessage("Failed " + param.startIteration + " -> " + param.endIteration + " iterations : ");
info.screenDump();
Expand Down Expand Up @@ -1084,7 +1084,7 @@ public void reflexivDSDynamicReductionPipe() throws IOException {
info.readMessage("Removing : Count_" + param.kmerSize1);
info.screenDump();

cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize1);
// cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize1);
} else {
info.readMessage("Failed k-mer sorting : " + param.kmerSize1 + " failed:");
info.screenDump();
Expand All @@ -1098,7 +1098,7 @@ public void reflexivDSDynamicReductionPipe() throws IOException {
info.readMessage("Removing : Count_" + param.kmerSize1);
info.screenDump();

cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize1);
// cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize1);
}

if (!checkOutputFile(param.outputPath + "/Count_" + param.kmerSize2 + "_sorted")) {
Expand Down Expand Up @@ -1156,7 +1156,7 @@ public void reflexivDSDynamicReductionPipe() throws IOException {
info.readMessage("Removing : Count_" + param.kmerSize2);
info.screenDump();

cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize2);
// cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize2);
} else {
info.readMessage("Failed k-mer sorting : " + param.kmerSize2 + " failed:");
info.screenDump();
Expand All @@ -1170,7 +1170,7 @@ public void reflexivDSDynamicReductionPipe() throws IOException {
info.readMessage("Removing : Count_" + param.kmerSize2);
info.screenDump();

cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize2);
// cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize2);
}

param.inputKmerPath1 = param.outputPath + "/Count_" + param.kmerSize1 + "_sorted/part*.csv.gz";
Expand All @@ -1187,7 +1187,7 @@ public void reflexivDSDynamicReductionPipe() throws IOException {
info.readMessage("Removing: Count_" + param.kmerSize1 + "_sorted");
info.screenDump();

cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize1 + "_sorted");
// cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize1 + "_sorted");
} else {
info.readMessage("Failed k-mer reduction : " + param.kmerSize2 + " vs " + param.kmerSize1 + " failed:");
info.screenDump();
Expand All @@ -1202,7 +1202,7 @@ public void reflexivDSDynamicReductionPipe() throws IOException {
// if (param.kmerSize2 <100) {
info.readMessage("Removing: Count_" + param.kmerSize2 + "_sorted");
info.screenDump();
cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize2 + "_sorted");
// cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize2 + "_sorted");
/* }else {
info.readMessage("Rename last k-mer sorted to k-mer reduced");
info.screenDump();
Expand All @@ -1216,9 +1216,9 @@ public void reflexivDSDynamicReductionPipe() throws IOException {
info.readMessage("Removing: Count_" + param.kmerSize1 + "_sorted, Count_" + param.kmerSize1 + ", and Count_" + param.kmerSize2);
info.screenDump();

cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize1 + "_sorted");
cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize1);
cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize2);
// cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize1 + "_sorted");
// cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize1);
// cleanDiskStorage(param.outputPath + "/Count_" + param.kmerSize2);
}
}

Expand Down Expand Up @@ -1279,7 +1279,7 @@ public void reflexivDSDynamicReductionPipe() throws IOException {
info.readMessage("Removing : Count_" + param.kmerListInt[param.kmerListInt.length - 1]);
info.screenDump();

cleanDiskStorage(param.outputPath + "/Count_" + param.kmerListInt[param.kmerListInt.length - 1]);
// cleanDiskStorage(param.outputPath + "/Count_" + param.kmerListInt[param.kmerListInt.length - 1]);
} else {
info.readMessage("Failed k-mer sorting : " + param.kmerListInt[param.kmerListInt.length - 1] + " failed:");
info.screenDump();
Expand All @@ -1292,7 +1292,7 @@ public void reflexivDSDynamicReductionPipe() throws IOException {
info.readMessage("Removing : Count_" + param.kmerListInt[param.kmerListInt.length-1] );
info.screenDump();

cleanDiskStorage(param.outputPath + "/Count_" + param.kmerListInt[param.kmerListInt.length-1] );
// cleanDiskStorage(param.outputPath + "/Count_" + param.kmerListInt[param.kmerListInt.length-1] );

info.readMessage("This is the last k-mer reduction round");
info.screenDump();
Expand Down
10 changes: 8 additions & 2 deletions src/main/java/uni/bielefeld/cmg/reflexiv/pipeline/ReflexivDSDynamicKmerFirstFour.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,12 @@ public void assemblyFromKmer() {
iterations++;
ReflexivSubKmerDS = ReflexivSubKmerDS.sort("k-1");
ReflexivSubKmerDS = ReflexivSubKmerDS.mapPartitions(DSKmerExtention, ReflexivSubKmerEncoderCompressed);

// long kmerNumber = ReflexivSubKmerDS.count();
// System.out.println("kmer count after extension " + iterations + " is " + kmerNumber);
}

// ReflexivSubKmerDS.persist(StorageLevel.MEMORY_AND_DISK());
// ReflexivSubKmerDS.persist(StorageLevel.MEMORY_AND_DISK());

DSBinarySubKmerWithShortExtensionToString SubKmerToString = new DSBinarySubKmerWithShortExtensionToString();
ReflexivLongSubKmerStringDS = ReflexivSubKmerDS.mapPartitions(SubKmerToString, ReflexivLongKmerStringEncoder);
Expand All @@ -211,7 +214,7 @@ public void assemblyFromKmer() {
ReflexivLongSubKmerStringDS.write().
mode(SaveMode.Overwrite).
format("csv").
option("compression", "lz4").
option("compression", "gzip").
save(param.outputPath + "/Assembly_intermediate/00firstFour");


Expand Down Expand Up @@ -1750,6 +1753,9 @@ public Iterator<Row> call(Iterator<Row> sIterator) throws Exception {
lineMarker++;
// return reflexivKmerConcatList.iterator();
}

// System.out.println("tempReflexivkmer size: " + tmpReflexivKmerExtendList.size());
// System.out.println("reflexivKmerConcatList size: " + reflexivKmerConcatList.size());
} // while loop
tmpKmerRandomizer();

Expand Down
Loading

0 comments on commit 96e697b

Please sign in to comment.