@inproceedings{czehmann:26064:sign-lang:lrec, author = {Czehmann, Vera and Yazdani, Shakib and Hamidullah, Yasser and Nunnari, Fabrizio and Avramidis, Eleftherios}, title = {"A Sacred Bird Called the Phoenix". Auditing the most-used Parallel Corpus for {German} {Sign} {Language} Recognition and Translation}, pages = {80--92}, editor = {Efthimiou, Eleni and Fotinea, Stavroula-Evita and Hanke, Thomas and Hochgesang, Julie A. and Mesch, Johanna and Schulder, Marc}, booktitle = {Proceedings of the {LREC2026} 12th Workshop on the Representation and Processing of Sign Languages: Language in Motion}, maintitle = {15th International Conference on Language Resources and Evaluation ({LREC} 2026)}, publisher = {{ELRA Language Resources Association (ELRA)}}, address = {Palma, Mallorca, Spain}, day = {16}, month = may, year = {2026}, isbn = {978-2-493814-82-1}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/26064.html}, doi = {10.63317/3xftynuw7i5c}, abstract = {This paper presents an empirical audit of the widely used RWTH-PHOENIX-2014T corpus, examining its suitability as a benchmark for sign language recognition and translation. Through human annotation of the training set and extensive sign-to-text back translation of the test set, we provide detailed statistics that indicate substantial quality issues, including information loss and lexical errors. Automatic scores comparing human sign-to-text back translations to the original speech transcribed references are remarkably low, suggesting strong translationese effects and substantial paraphrasing, revealing limitations of lexical metrics in adequately scoring translation quality. Replacing the original speech-transcribed references with human sign-to-text back translations while scoring existing sign language translation systems reveals the lack of robustness of system evaluation with lexical metrics against this test set. Our findings highlight risks associated with relying on this corpus for model evaluation and call for more rigorous, linguistically grounded evaluation practices in sign language technology research. The back-translated test set and error annotations are made publicly available.} } @inproceedings{deshpande:70008:sltat:lrec, author = {Deshpande, Neha and Nunnari, Fabrizio and Avramidis, Eleftherios}, title = {Fine-tuning of Convolutional Neural Networks for the Recognition of Facial Expressions in Sign Language Video Samples}, pages = {29--38}, editor = {Efthimiou, Eleni and Fotinea, Stavroula-Evita and Hanke, Thomas and McDonald, John C. and Shterionov, Dimitar and Wolfe, Rosalee}, booktitle = {Proceedings of the 7th International Workshop on Sign Language Translation and Avatar Technology: The Junction of the Visual and the Textual: Challenges and Perspectives}, maintitle = {13th International Conference on Language Resources and Evaluation ({LREC} 2022)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Marseille, France}, day = {24}, month = jun, year = {2022}, isbn = {979-10-95546-82-5}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/2022.sltat-1.5.html}, doi = {10.63317/2mskh4gjnncy}, abstract = {In this paper, we investigate the capability of convolutional neural networks to recognize in sign language video frames the six basic Ekman facial expressions for 'fear', 'disgust', 'surprise', 'sadness', 'happiness', 'anger' along with the 'neutral' class. Given the limited amount of annotated facial expression data for the sign language domain, we started from a model pre-trained on general-purpose facial expression datasets and we applied various machine learning techniques such as fine-tuning, data augmentation, class balancing, as well as image preprocessing to reach a better accuracy. The models were evaluated using K-fold cross-validation to get more accurate conclusions. It is experimentally demonstrated that fine-tuning a pre-trained model along with data augmentation by horizontally flipping images and image normalization, helps in providing the best accuracy on the sign language dataset. The best setting achieves satisfactory classification accuracy, comparable to state-of-the-art systems in generic facial expression recognition. Experiments were performed using different combinations of the above-mentioned techniques based on two different architectures, namely MobileNet and EfficientNet, and is deemed that both architectures seem equally suitable for the purpose of fine-tuning, whereas class balancing is discouraged.} } @inproceedings{yazdani-etal-2026-critical:lrec, author = {Yazdani, Shakib and Hamidullah, Yasser and Espa{\~n}a-Bonet, Cristina and Avramidis, Eleftherios and van Genabith, Josef}, title = {A Critical Study of Automatic Evaluation in Sign Language Translation}, pages = {9535--9548}, editor = {Piperidis, Stelios and Bel, N{\'u}ria and van den Heuvel, Henk and Ide, Nancy and Krek, Simon and Toral, Antonio}, booktitle = {15th International Conference on Language Resources and Evaluation ({LREC} 2026)}, publisher = {{ELRA Language Resources Association (ELRA)}}, address = {Palma, Mallorca, Spain}, day = {11--16}, month = may, year = {2026}, isbn = {978-2-493814-49-4}, language = {english}, url = {https://lrec.elra.info/lrec2026-main-749}, doi = {10.63317/4n2sooe4fb2i}, abstract = {Automatic evaluation metrics are crucial for advancing sign language translation (SLT). Current SLT evaluation metrics, such as BLEU and ROUGE, are only text-based, and it remains unclear to what extent text-based metrics can reliably capture the quality of SLT outputs. To address this gap, we investigate the limitations of text-based SLT evaluation metrics by analyzing six metrics, including BLEU, chrF, and ROUGE, as well as BLEURT on the one hand, and large language model (LLM)-based evaluators such as G-Eval and GEMBA zero-shot direct assessment on the other hand. Specifically, we assess the consistency and robustness of these metrics under three controlled conditions: paraphrasing, hallucinations in model outputs, and variations in sentence length. Our analysis highlights the limitations of lexical overlap metrics and demonstrates that while LLM-based evaluators better capture semantic equivalence often missed by conventional metrics, they can also exhibit bias toward LLM-paraphrased translations. Moreover, although all metrics are able to detect hallucinations, BLEU tends to be overly sensitive, whereas BLEURT and LLM-based evaluators are comparatively lenient toward subtle cases. This motivates the need for multimodal evaluation frameworks that extend beyond text-based metrics to enable a more holistic assessment of SLT outputs.} } @inproceedings{nunnari-etal-2024-fabeln:lrec, author = {Nunnari, Fabrizio and Avramidis, Eleftherios and Espa{\~n}a-Bonet, Cristina and Gonz{\'a}lez, Marco and Hennes, Anna and Gebhard, Patrick}, title = {{DGS-Fabeln-1}: A Multi-Angle Parallel Corpus of Fairy Tales between {German} {Sign} {Language} and {German} Text}, pages = {4847--4857}, editor = {Calzolari, Nicoletta and Kan, Min-Yen and Hoste, Veronique and Lenci, Alessandro and Sakti, Sakriani and Xue, Nianwen}, booktitle = {2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation ({LREC-COLING} 2024)}, publisher = {{ELRA Language Resources Association (ELRA) and the International Committee on Computational Linguistics (ICCL)}}, address = {Torino, Italy}, day = {20--25}, month = may, year = {2024}, isbn = {978-2-493814-10-4}, language = {english}, url = {https://aclanthology.org/2024.lrec-main.434}, doi = {10.63317/46w4kw3gmw3v}, abstract = {We present the acquisition process and the data of DGS-Fabeln-1, a parallel corpus of German text and videos containing German fairy tales interpreted into the German Sign Language (DGS) by a native DGS signer. The corpus contains 573 segments of videos with a total duration of 1 hour and 32 minutes, corresponding with 1428 written sentences. It is the first corpus of semi-naturally expressed DGS that has been filmed from 7 angles, and one of the few sign language (SL) corpora globally which have been filmed from more than 3 angles and where the listener has been simultaneously filmed. The corpus aims at aiding research at SL linguistics, SL machine translation and affective computing, and is freely available for research purposes at the following address: https://doi.org/10.5281/zenodo.10822097.} }