Merge remote-tracking branch 'origin/dev' into update_doc

.github/workflows/main.yml

@@ -67,11 +67,11 @@ jobs:
         ppanggolin spot -p stepbystep/pangenome.h5 --spot_graph --overlapping_match 2 --set_size 3 --exact_match_size 1
         ppanggolin draw -p stepbystep/pangenome.h5 --draw_spots -o stepbystep -f
         ppanggolin module -p stepbystep/pangenome.h5 --transitive 4 --size 3 --jaccard 0.86 --dup_margin 0.05
-        ppanggolin write -p stepbystep/pangenome.h5 --output stepbystep -f --soft_core 0.9 --dup_margin 0.06 --gexf --light_gexf --csv --Rtab --projection --stats --partitions --compress --json --regions --spots --borders --families_tsv --cpu 1
+        ppanggolin write -p stepbystep/pangenome.h5 --output stepbystep -f --soft_core 0.9 --dup_margin 0.06 --gexf --light_gexf --csv --Rtab --projection --stats --partitions --compress --json --regions --spots --borders --families_tsv --cpu 1 --fasta organisms.fasta.list --gff --proksee
         ppanggolin fasta -p stepbystep/pangenome.h5 --output stepbystep -f --prot_families all --gene_families shell --regions all --fasta organisms.fasta.list
         ppanggolin draw -p stepbystep/pangenome.h5 --draw_spots --spots all -o stepbystep -f
         ppanggolin metrics -p stepbystep/pangenome.h5 --genome_fluidity --info_modules --no_print_info -f --log metrics.log
-        cd -
+        cd - 
     - name: gbff parsing and MSA computing
       shell: bash -l {0}
       run: |
@@ -100,14 +100,18 @@ jobs:
       shell: bash -l {0}
       run: |
         cd testingDataset
-        ppanggolin align --pangenome mybasicpangenome/pangenome.h5 --sequences some_chlam_proteins.fasta --output test_align --draw_related --getinfo
+        ppanggolin align --pangenome mybasicpangenome/pangenome.h5 --sequences some_chlam_proteins.fasta \
+                         --output test_align --draw_related --getinfo --fast
         cd -
     - name: testing context command
       shell: bash -l {0}
       run: |
         cd testingDataset
-        ppanggolin context --pangenome myannopang/pangenome.h5 --sequences some_chlam_proteins.fasta --output test_context
-        ppanggolin context --pangenome readclusterpang/pangenome.h5 --family some_chlam_families.txt --output test_context -f
+        ppanggolin context --pangenome myannopang/pangenome.h5 --sequences some_chlam_proteins.fasta --output test_context --fast
+
+        # test from gene family ids. Test here with one family of module 1. The context should find all families of module 1
+        echo AP288_RS05055 > one_family_of_module_1.txt 
+        ppanggolin context --pangenome myannopang/pangenome.h5 --family one_family_of_module_1.txt  --output test_context_from_id
         cd -
     - name: testing metadata command
       shell: bash -l {0}
@@ -124,3 +128,16 @@ jobs:
         cd testingDataset
         ppanggolin utils --default_config panrgp -o panrgp_default_config.yaml
         ppanggolin panrgp  --anno organisms.gbff.list --cluster clusters.tsv -o test_config --config panrgp_default_config.yaml
+        cd -
+    - name: testing projection cmd
+      shell: bash -l {0}
+      run: |
+        cd testingDataset
+        head organisms.gbff.list | sed 's/^/input_org_/g' > organisms.gbff.head.list
+        ppanggolin projection --pangenome stepbystep/pangenome.h5  -o projection_from_list_of_gbff --anno organisms.gbff.head.list --gff --proksee
+
+
+        ppanggolin projection --pangenome mybasicpangenome/pangenome.h5  -o projection_from_single_fasta \
+                              --organism_name chlam_A --fasta FASTA/GCF_002776845.1_ASM277684v1_genomic.fna.gz \
+                              --spot_graph --graph_formats graphml --fast --keep_tmp -f --add_sequences --gff --proksee
+

VERSION

@@ -1 +1 @@
-1.2.189
+1.2.194

docs/index.md

@@ -163,6 +163,7 @@ user/Regions-of-Genome-Plasticity
 user/Conserved-modules
 user/Align
 user/Genomic-context
+user/projection
 user/metadata
 user/Outputs
 ```

docs/user/Flat/orgStat.md

@@ -19,7 +19,7 @@ This file is made of 15 columns described in the following table
 | nb_cloud_genes         | The number of genes whose family is cloud in that genome                                                                                                                                                                                                                                                                                                                                                                                                                                |
 | nb_exact_core_genes    | The number of genes whose family is exact core in that genome                                                                                                                                                                                                                                                                                                                                                                                                                           |
 | nb_soft_core_genes     | The number of genes whose family is soft core in that genome                                                                                                                                                                                                                                                                                                                                                                                                                            |
-| completeness           | This is an indicator of the proportion of single copy markers in the persistent that are present in the genome. While it is expected to be relatively close to 100 when working with isolates, it may be particularly interesting when working with very fragmented genomes as this provide a *de novo* estimation of the completess based on the expectation that single copy markers within the persistent should be mostly present in all individuals of the studied taxonomic group |
+| completeness           | This is an indicator of the proportion of single copy markers in the persistent that are present in the genome. While it is expected to be relatively close to 100 when working with isolates, it may be particularly interesting when working with very fragmented genomes as this provide a *de novo* estimation of the completeness based on the expectation that single copy markers within the persistent should be mostly present in all individuals of the studied taxonomic group |
 | nb_single_copy_markers | This indicates the number of present single copy markers in the genomes. They are computed using the parameter duplication_margin indicated at the beginning of the file. They correspond to all of the persistent gene families that are not present in more than one copy in 5% (or more) of the genomes by default.                                                                                                                                                                  |
 
 It can be generated using the 'write' subcommand as such : 

docs/user/Genomic-context.md

@@ -30,21 +30,25 @@ In this case, you can give a pangenome without gene families representatives seq
 
 In case of you are using families ID, you will only have as output the `gene_context.tsv` file. In the other case, you use sequences, you will have another output file to report the alignment between sequences and pangenome families (see detail in align subcommand).
 
-There are 4 columns in `gene_context.tsv`. 
+There are 6 columns in `gene_context.tsv`. 
 
-1. **geneContext ID**: identifier of the found context. It is incrementally generated, beginning with 1
+1. **geneContext ID**: Identifier of the found context. It is incrementally generated, beginning with 1
 2. **Gene family name**: Identifier of the gene family, from the pangenome, correspond to the found context
 3. **Sequence ID**: Identifier of the searched sequence in the pangenome
 4. **Nb Genomes**: Number of genomes where the genomic context is found
 5. **Partition**: Partition of the gene family corresponding to the found context
+6. **Target family**: Whether the family is a target family, meaning it matches an input sequence, or a family provided as input.
 
 In **sequence Id**, it is possible to find a NA value. This case, correspond to another gene family found in the context.
 
 ## Detailed options
-| option name      | Description                                                                                                                                                                                                       |
-|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| --no_defrag      | Do not use the defragmentation step, to align sequences with MMseqs2                                                                                                                                              |
-| --identity       | Minimum identity percentage threshold                                                                                                                                                                             |
-| --coverage       | Minimum coverage percentage threshold                                                                                                                                                                             |
-| -t, --transitive | Size of the transitive closure used to build the graph. This indicates the number of non-related genes allowed in-between two related genes. Increasing it will improve precision but lower sensitivity a little. |
-| -s, --jaccard    | Minimum jaccard similarity used to filter edges between gene families. Increasing it will improve precision but lower sensitivity a lot.                                                                          |
+
+| option name | Description |
+|-----------------------------|---------------------------------------------------------------------------|
+| --fast | Use representative sequences of gene families for input gene alignment. This option is recommended for faster processing but may be less sensitive. By default, all pangenome genes are used for alignment. This argument makes sense only when --sequence is provided. (default: False) |
+| --no_defrag | Do not use the defragmentation step, to align sequences with MMseqs2 (default: False) |
+| --identity | Minimum identity percentage threshold (default: 0.8)|
+| --coverage | Minimum coverage percentage threshold (default: 0.8)|
+| -t, --transitive | Size of the transitive closure used to build the graph. This indicates the number of non-related genes allowed in-between two related genes. Increasing it will improve precision but lower sensitivity a little. (default: 4) |
+| -s, --jaccard | Minimum jaccard similarity used to filter edges between gene families. Increasing it will improve precision but lower sensitivity a lot. (default: 0.85) |
+| -w, --window_size | Number of neighboring genes that are considered on each side of a gene of interest when searching for conserved genomic contexts. (default: 5) |

docs/user/projection.md

@@ -0,0 +1,64 @@
+# Projection
+The ppanggolin projection command allows you to annotate external genomes using an existing pangenome. This process eliminates the need to recompute all components, streamlining the annotation process. Input genomes are expected to belong to the same species.
+
+Genes within the input genome are aligned with genes in the pangenome to determine their gene families and partitions. Genes that do not align with any existing gene in the pangenome are considered specific to the input genome and are assigned to the "Cloud" partition. Based on the alignment and partition assignment, Regions of Plasticity (RGPs) within the input genome are predicted. Each RGP that is not located on a contig border is assigned to a spot of insertion. Finally, conserved modules of the pangenome found in the input genome are reported in the output files.
+
+## Input files:
+
+This command supports two input modes depending on whether you want to project a single genome or multiple genomes at once:
+
+Multiple Files in One TSV:
+- **Options**: `--fasta` or `--anno`
+- **Description**: You can provide a tab-separated file listing organism names alongside their respective FASTA genomic sequences or annotation filepaths, with one line per organism. This mode is suitable when you want to annotate multiple genomes in a single operation. The format of this file is identical to the format used in the annotate and workflow commands; for more details, refer here.
+
+Single File:
+- **Options**: `--organism_name` with `--fasta` or `--anno` and `--circular_contigs` (optional)
+- **Description**: When annotating a single genome, you can directly provide a single FASTA genomic sequence file or an annotation file in GFF/GBFF format. Additionally, specify the name of the organism using the `--organism_name` option. You can also indicate circular contigs using the `--circular_contigs` option when necessary.
+
+
+## Output files:
+
+The Output directory contains `summary_projection.tsv` giving an overview of the projection. one line per organism.
+
+
+| Column                               | Description|
+|--------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Organism name                        | This column contains name or identifier of the organisms being analyzed.|
+| Pangenome file                       | The path to the pangenome file (pangenome.h5) used for the analysis.|
+| Contigs                              | The number of contigs in the projected genome.|
+| Genes                                | The total number of genes identified in the input genome.|
+| Families                             | The total number of gene families to which genes in the genome of the input organism are assigned.|
+| Persistent genes                     | The number of genes in the "Persistent" partition.|
+| Persistent families                  | The number of gene families in the "Persistent" partition.|
+| Shell genes                          | The number of genes in the "Shell" partition.|
+| Shell families                       | The number of gene families in the "Shell" partition.|
+| Cloud genes                          | The number of genes in the "Cloud" partition.|
+| Cloud families                       | The number of gene families in the "Cloud" parition.|
+| Cloud specific families              | The number of gene families that are specific to the input organism. These families are unique to the input organism and do not have homologs in any other genomes within the pangenome and have been assigned to the "Cloud" partition.|
+| completeness           | This indicates the proportion of single copy markers from the persistent partition that are present in the genome. While it is expected to be relatively close to 100 when working with isolates, it may be particularly interesting when working with very fragmented genomes as this provide a *de novo* estimation of the completeness based on the expectation that single copy markers within the persistent should be mostly present in all individuals of the studied taxonomic group. |
+| RGPs (Regions of Genomic Plasticity) | The number of Regions of Genomic Plasticity (RGPs) predicted within the input genome.|
+| Spots                                | The total number of spots of insertion associated with RGPs in the input genome.|
+| New spots                            | The number of new insertion spots that have been identified in the input genome. These spots represent novel genomic regions compared to other genomes in the pangenome.|
+| Modules                              | The number of modules that have been projected onto the input genome.|
+
+
+Additionally, within the Output directory, there is a subdirectory for each input genome, named after the input genome itself. Each of these subdirectories contains several files:
+
+For Gene Family and Partition of Input Genes:
+
+- `cds_sequences.fasta`: This file contains the sequences of coding regions (CDS) from the input genome.
+- `gene_to_gene_family.tsv`: It provides the mapping of genes to gene families of the pangenome. its format follows [this output](Outputs.md#gene-families-and-genes)
+- `sequences_partition_projection.tsv`: This file maps the input genes to its partition (Persistent, Shell or Cloud).
+- `specific_genes.tsv`: This file list the gene of the input genomes that do not align to any gene of the pangenome. These genes are assigned to Cloud parititon. 
+
+For RGPs and Spots:
+
+- `plastic_regions.tsv`: This file contains information about Regions of Genomic Plasticity (RGPs) within the input genome. Its format follows [this output](Outputs.md#plastic-regions).
+- `input_organism_rgp_to_spot.tsv`: It provides information about the association between RGPs and insertion spots in the input genome. Its format follows [this ouput](Outputs.md#spots).
+
+Optionally, you can produce a graph of the RGPs using the `--spot_graph` option. This graph is similar as the one produce by the `ppanggolin spot` command.
+
+For Modules:
+
+- `modules_in_input_organism.tsv`: This file lists the modules that have been found in the input genome. Its format follows [this ouput](Outputs.md#modules-in-organisms).
+