Add basic CI (#156)

* Add basic CI

* Add liburing installation step to CI workflow

* Run `npm install` as part of ci/check

* Add `@types/node` package

* Add `submodules: 'recursive'` to CI

* Skip test if test data is not available

* Install `cargo-about` in CI

.github/workflows/ci.yaml

@@ -0,0 +1,48 @@
+# Based on: https://github.com/clap-rs/clap/blob/master/.github/workflows/ci.yml
+
+name: CI
+
+permissions:
+ contents: read
+
+on:
+ pull_request:
+ push:
+
+env:
+ RUST_BACKTRACE: 1
+ CARGO_TERM_COLOR: always
+ CLICOLOR: 1
+
+jobs:
+ test:
+ name: Test
+ strategy:
+ matrix:
+ build: [linux]
+ include:
+ - build: linux
+ os: ubuntu-latest
+ rust: "stable"
+ runs-on: ${{ matrix.os }}
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+ with:
+ submodules: 'recursive'
+ - name: Install Rust
+ uses: dtolnay/rust-toolchain@stable
+ with:
+ toolchain: ${{ matrix.rust }}
+ - uses: Swatinem/rust-cache@v2
+ - name: Install liburing
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y liburing-dev
+ - uses: taiki-e/install-action@v2
+ with:
+ tool: cargo-about
+ - name: Run CI
+ run: ./scripts/ci/all
+ - name: Test
+ run: cargo test

crates/core/src/ranking/models/cross_encoder.rs

@@ -165,12 +165,12 @@ mod tests {
 
  #[test]
  fn sanity_check() {
- if !Path::new("../../data/cross_encoder").exists() {
+ let data_path = Path::new("../../data/cross_encoder");
+ if !data_path.exists() {
+ // Skip the test if the test data is not available
  return;
  }
-
- let model = CrossEncoderModel::open("../../data/cross_encoder")
- .expect("Failed to find cross-encoder model");
+ let model = CrossEncoderModel::open(data_path).expect("Failed to find cross-encoder model");
 
  let s = model.run(
  "how many people live in paris",

crates/core/src/summarizer.rs

@@ -545,12 +545,12 @@ mod tests {
 
  #[test]
  fn test_dual_encoder() {
- if !Path::new("../../data/summarizer/dual_encoder").exists() {
+ let data_path = Path::new("../../data/summarizer/dual_encoder");
+ if !data_path.exists() {
+ // Skip the test if the test data is not available
  return;
  }
-
- let model =
- DualEncoder::open("../../data/summarizer/dual_encoder").expect("Failed to load model");
+ let model = DualEncoder::open(data_path).expect("Failed to load model");
  let query = "What is the capital of France?";
  let pos = "The capital of France is Paris.";
  let neg = "The best baguette in Paris can be found at Boulangerie Pichard.";

crates/core/src/warc.rs

@@ -725,16 +725,15 @@ mod tests {
 
  #[test]
  fn internet_archive_parse() {
- if !Path::new("../../data/internet_archive.warc.gz").exists() {
+ let data_path = Path::new("../../data/internet_archive.warc.gz");
+ if !data_path.exists() {
+ // Skip the test if the test data is not available
  return;
  }
 
  let mut records = 0;
 
- for record in WarcFile::open("../../data/internet_archive.warc.gz")
- .unwrap()
- .records()
- {
+ for record in WarcFile::open(data_path).unwrap().records() {
  records += 1;
  if let Err(err) = record {
  panic!("Error: {:?}", err);

crates/core/src/widgets/thesaurus.rs

@@ -504,7 +504,12 @@ mod tests {
 
  #[test]
  fn build_dict() {
- let dict = Dictionary::build("../../data/english-wordnet-2022-subset.ttl").unwrap();
+ let data_path = Path::new("../../data/english-wordnet-2022-subset.ttl");
+ if !data_path.exists() {
+ // Skip the test if the test data is not available
+ return;
+ }
+ let dict = Dictionary::build(data_path).unwrap();
 
  let infos = dict.get(Lemma("barely".to_string()));
 

crates/zimba/src/lib.rs

@@ -636,11 +636,13 @@ mod tests {
 
  #[test]
  fn it_works() {
- if !Path::new("../../data/test.zim").exists() {
+ let data_path = Path::new("../../data/test.zim");
+ if !data_path.exists() {
+ // Skip test if data file is not present
  return;
  }
 
- let zim = ZimFile::open("../../data/test.zim").unwrap();
+ let zim = ZimFile::open(data_path).unwrap();
 
  assert_eq!(zim.header.magic, 72173914);
  assert_eq!(zim.header.major_version, 5);

crates/zimba/src/wiki.rs

@@ -256,11 +256,18 @@ impl<'a> Iterator for ImageIterator<'a> {
 
 #[cfg(test)]
 mod tests {
+ use std::path::Path;
+
  use super::*;
 
  #[test]
  fn test_article_iterator() {
- let zim = ZimFile::open("../../data/test.zim").unwrap();
+ let data_path = Path::new("../../data/test.zim");
+ if !data_path.exists() {
+ // Skip the test if the test data is not available
+ return;
+ }
+ let zim = ZimFile::open(data_path).unwrap();
  let mut iter = ArticleIterator::new(&zim).unwrap();
 
  let article = iter.next().unwrap();
@@ -271,7 +278,12 @@ mod tests {
 
  #[test]
  fn test_image_iterator() {
- let zim = ZimFile::open("../../data/test.zim").unwrap();
+ let data_path = Path::new("../../data/test.zim");
+ if !data_path.exists() {
+ // Skip the test if the test data is not available
+ return;
+ }
+ let zim = ZimFile::open(data_path).unwrap();
  let mut iter = ImageIterator::new(&zim).unwrap();
 
  let image = iter.next().unwrap();

frontend/package.json

@@ -28,6 +28,7 @@
  "@tailwindcss/line-clamp": "^0.4.4",
  "@tailwindcss/typography": "^0.5.10",
  "@types/file-saver": "^2.0.7",
+ "@types/node": "^20.11.19",
  "@typescript-eslint/eslint-plugin": "^6.14.0",
  "@typescript-eslint/parser": "^6.14.0",
  "autoprefixer": "^10.4.16",
@@ -60,4 +61,4 @@
  "tailwind-merge": "^2.1.0",
  "ts-pattern": "^5.0.6"
  }
-}
+}

scripts/ci/check

@@ -3,4 +3,4 @@ set -e
 
 cargo check
 cargo check --no-default-features
-cd frontend && npm run check
+cd frontend && npm install && npm run check