This repository contains the evaluation code we used in CALAMITA. It is a fork of the lm-eval-harness. We forked at the main branch commit: b2bf7bc4a601c643343757c92c1a51eb69caf1d7.
@inproceedings{calamita2024,
title={{CALAMITA – Challenge the Abilities of LAnguage Models in ITAlian: Overview}},
author={Attanasio, Giuseppe and Basile, Pierpaolo and Borazio, Federico and Croce, Danilo and Francis, Maria and Gili, Jacopo and Musacchio, Elio and Nissim, Malvina and Patti, Viviana and Rinaldi, Matteo and Scalena, Daniel},
booktitle={Proceedings of the 10th Italian Conference on Computational Linguistics (CLiC-it 2024)},
year={2024}
}