diff --git a/src/lib.rs b/src/lib.rs index 6607391..fe5d8d7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -220,6 +220,31 @@ impl Client { .await } + /// Tokenize a prompt for a specific model. + /// + /// ```no_run + /// use aleph_alpha_client::{Client, Error, How, TaskTokenization}; + /// + /// async fn tokenize() -> Result<(), Error> { + /// let client = Client::new(AA_API_TOKEN)?; + /// + /// // Name of the model for which we want to tokenize text. + /// let model = "luminous-base"; + /// + /// // Text prompt to be tokenized. + /// let prompt = "An apple a day"; + /// + /// let task = TaskTokenization { + /// prompt, + /// tokens: true, // return text-tokens + /// token_ids: true, // return numeric token-ids + /// }; + /// let respones = client.tokenize(&task, model, &How::default()).await?; + /// + /// dbg!(&respones); + /// Ok(()) + /// } + /// ``` pub async fn tokenize( &self, task: &TaskTokenization<'_>, @@ -231,6 +256,29 @@ impl Client { .await } + /// Detokenize a list of token ids into a string. + /// + /// ```no_run + /// use aleph_alpha_client::{Client, Error, How, TaskDetokenization}; + /// + /// async fn detokenize() -> Result<(), Error> { + /// let client = Client::new(AA_API_TOKEN)?; + /// + /// // Specify the name of the model whose tokenizer was used to generate the input token ids. + /// let model = "luminous-base"; + /// + /// // Token ids to convert into text. + /// let token_ids: Vec = vec![556, 48741, 247, 2983]; + /// + /// let task = TaskDetokenization { + /// token_ids: &token_ids, + /// }; + /// let respones = client.detokenize(&task, model, &How::default()).await?; + /// + /// dbg!(&respones); + /// Ok(()) + /// } + /// ``` pub async fn detokenize( &self, task: &TaskDetokenization<'_>,